From ba836e1e75565aeb1db683b12d0d0e4956c772fc Mon Sep 17 00:00:00 2001 From: Alexander Miroshnichenko Date: Thu, 19 Nov 2026 10:33:29 +0300 Subject: [PATCH] perf(kyojin): split large prefix writes under disk pressure Bead: nix-overlay-3c1 --- .beads/issues.jsonl | 1 + packages/kyojin/package.nix | 1 + packages/kyojin/prefix-split-segments.patch | 209 ++++++++++++++++++++ 3 files changed, 211 insertions(+) create mode 100644 packages/kyojin/prefix-split-segments.patch diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 30698cc..f47a6ca 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -5,6 +5,7 @@ {"id":"nix-overlay-2p0","title":"Verify kyojin 1.8.26 on remote builder or CI","description":"Run nix build .#kyojin .#kyojin.tests.cpu .#kyojin.tests.jit on a remote builder or CI only. This workstation lacks resources; user explicitly forbids local builds. Local evaluation previously reported 1.8.26; build was interrupted, success unconfirmed. Downloaded/built dependencies were removed (232 paths, 21.7 GiB).","status":"open","priority":2,"issue_type":"task","created_at":"2026-11-17T18:18:48.554328412Z","created_by":"minder","updated_at":"2026-11-17T18:18:48.554328412Z","source_repo":"nix-overlay","compaction_level":0,"original_size":0} {"id":"nix-overlay-2q7","title":"Add opt-in direct writes for large Kyojin prefix segments","description":"Retain the standard torch checkpoint and ordinary reads, small records and storage admission. Let the Flash-Next agent write large KV segments with aligned O_DIRECT to reduce page-cache pressure on bcachefs. Preserve CRC, mmap restore, unsupported-filesystem fallback and error handling. Verify native launcher, fs behavior, repeated agent timing and CPU tests.","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-18T22:28:42.144819734Z","created_by":"minder","updated_at":"2026-11-18T22:34:07.087715703Z","closed_at":"2026-11-18T22:34:07.085648146Z","close_reason":"Verified on native 128K save and fresh-process restore, strict CPU checks and repeated real agent continuations.","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"comments":[{"id":22,"issue_id":"nix-overlay-2q7","author":"minder","text":"Scope packages/kyojin. Opt-in environment for KV segments >=64 MiB only, 2 MiB staging, ordinary torch ZIP and mmap loader. Validate source-identical candidate in a real private engine.","created_at":"2026-11-18T22:28:52Z"},{"id":23,"issue_id":"nix-overlay-2q7","author":"minder","text":"ms-s1-max, bcachefs. Strict 22 filesystem/queue tests passed. Packaged CPU suite: 547 passed, 189 skipped, 349 subtests passed. Both local overlay and independent source tree build /nix/store/7ahzb4yyqj6wgqxpvcn13cpzs71y4qx6-kyojin-1.8.26. Native process opened a 3.76 GiB segment with O_DIRECT, reported 4 KiB peak residency, and served three correct receipts from 131072 cached tokens without restaging. Fresh native process using the on-disk segment and deepest checkpoint restored all 131072 tokens and returned emerald-88, 212 fresh tokens, TTFT 3.455 s, total 4.09 s. ABBA 128K/500-token JSON powers-of-two snapshots: buffered TTFT 3.861/2.772 s and total 19.907/39.741 s, direct 1.496/1.465 and 13.194/13.086 s. Identical answer and reasoning hashes. Direct serialization itself 19.4/28.7 s vs buffered 6.7/25.9 s. Archive: /DATA/LLMs/benchmark_results/kyojin-checkpoint-io-20261119.","created_at":"2026-11-18T22:34:06Z"}]} {"id":"nix-overlay-2yv","title":"strata: ship hipBLASLt tuning table calibrated for therock ROCm 10.1","description":"Upstream gfx1151-hipblaslt-100401.txt was calibrated on TheRock 7.14.1; therock 10.1.0rc3 reports the same hipBLASLt 100401 but its solution ids differ (HCD_EXACT: solution -1), so the table is a no-op and prefill runs ~418 tok/s. Recalibrated with tune_hipblaslt on ms-s1-max (gfx1151, ROCm 10.1): 17.5K cold prompt UD-Q5_K_XL 418 -> ~720 tok/s. Ship it in $out/bin/tuning/ so the engine finds it by itself.","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-16T15:49:56.904695144Z","created_by":"minder","updated_at":"2026-11-16T17:22:22.934929696Z","closed_at":"2026-11-16T17:22:22.934303630Z","close_reason":"table calibrated, shipped and measured","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"comments":[{"id":8,"issue_id":"nix-overlay-2yv","author":"minder","text":"Built on ms-s1-max (nix build --store ssh-ng): /nix/store/vv7j7ffdhcpi804pzhximszz959impgm-strata-0.2.13 ships bin/tuning/gfx1151-hipblaslt-100401.txt (rows identical to the tune_hipblaslt output). UD-Q5_K_XL 17.5K cold prompt: 418 -> 700-725 tok/s with the table; HCD_EXACT solution -1 warning gone.","created_at":"2026-11-16T17:22:22Z"}]} +{"id":"nix-overlay-3c1","title":"Split oversized paged KV saves when two segments can respect the disk floor","description":"Kyojin on ms-s1-max: a single cold 200K prompt misses the SSD cache when a monolithic 5.87 GiB segment needs twice its bytes above the 46 GiB floor. Try an opt-in two-segment fallback with total cap admission, per-write floor rechecks, independent indexes and retry. Validate immediate and restarted real tool turns before enabling.","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-19T06:57:35.016699443Z","created_by":"minder","updated_at":"2026-11-19T07:33:29.292246978Z","closed_at":"2026-11-19T07:33:29.291538581Z","close_reason":"Optional bounded segments and contiguous copies verified on native long-context inference, failed-write retry, scattered pages, cap and floor boundaries.","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"comments":[{"id":24,"issue_id":"nix-overlay-3c1","author":"minder","text":"Work scope packages/kyojin new patch and package patch list. Keep normal single-file path if it fits; test refusal and partial failures. No external push.","created_at":"2026-11-19T06:57:35Z"},{"id":25,"issue_id":"nix-overlay-3c1","author":"minder","text":"Original two-part writer passed 27 CPU tests and one full native 200K restart, but a small free-space change prevented the second 400-page file in a repeat. Final candidate bounds split segments at 256 pages, uses contiguous copies only when the physical rows allow, and preserves the per-write gate. Hardware and full-suite results recorded in /DATA/LLMs/benchmark_results/kyojin-first-200k-20261119.","created_at":"2026-11-19T07:28:29Z"},{"id":26,"issue_id":"nix-overlay-3c1","author":"minder","text":"Final packaged writer stored 256+256+256+32 pages from one cold 204901-token initial HTTP request. Reload used exactly 204800 cached tokens and 187 new, answered opal-91, first token 5.162 s (control 152.599 s). Native staging 889 ms. Exact direct segment fd, minimum disk free 48.178 GiB at a 46 GiB floor. Strict suite 29 passed, original build red in 3 cases. Nix CPU suite 554 passed, 189 skipped, 349 subtests passed. Native and CPU build paths in bounded/package.path and bounded/tests.out.","created_at":"2026-11-19T07:33:29Z"}]} {"id":"nix-overlay-3sa","title":"pdfcraft: add PdfCraft 0.18.0 (build from source)","description":"Add packages/pdfcraft building the storytold/pdfcraft Rust workspace from source (no pre-compiled binaries): desktop app + pdfcraft-cli, desktop entry, icons, metainfo, OCR models. Plus a Home Manager module.","status":"closed","priority":2,"issue_type":"feature","assignee":"minder","created_at":"2026-11-03T08:31:41.141131190Z","created_by":"minder","updated_at":"2026-11-03T10:48:31.465220459Z","closed_at":"2026-11-03T10:48:31.464736388Z","close_reason":"nix build .#packages.x86_64-linux.pdfcraft succeeds; pdfcraft/pdfcraft-cli 0.18.0 run, pdfcraft-cli render+info+extract work on a test PDF, mcp initialize answers, ocr_status reports available:true, desktop-file-validate clean, home-manager eval picks the package up from programs.pdfcraft.enable","source_repo":"nix-overlay","compaction_level":0,"original_size":0} {"id":"nix-overlay-481","title":"strata: build the MMQ prompt path (-DSTRATA_MMQ=ON)","description":"0.2.14+ leaves llama.cpp's MMQ prompt kernels out by default; then STRATA_MMQ_KQ is inert and UD-Q5_K_XL layer 2 (Q6_K) runs on Strata's int8 kernel. Measured on ms-s1-max, 0.2.15 + section 5: with MMQ and STRATA_MMQ_KQ=1 the first-token logits are bit-identical to 0.2.13 + section 5 over 15 prompts, speed equal (~1180 tok/s at 16K); without MMQ an agent prompt reached KL 0.199 against 0.2.13. Binary 39 -> 54 MB.","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-16T18:39:38.376087754Z","created_by":"minder","updated_at":"2026-11-16T18:42:33.442718186Z","closed_at":"2026-11-16T18:42:33.442193397Z","close_reason":"MMQ built, measured, committed","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"comments":[{"id":10,"issue_id":"nix-overlay-481","author":"minder","text":"Built on ms-s1-max: /nix/store/9yx7raabzwkn31wmd12v7r2n9k60cqsn-strata-0.2.15 (engine 0.1.61, 54 MB, bin/tuning present); its strata binary equals the measured strata-mmq test build byte for byte apart from the store hash. Test build results: 0.2.15 + section 5 + STRATA_MMQ_KQ=1 logits bit-identical to 0.2.13 + section 5 over 15 prompts, 16K ~1180 tok/s.","created_at":"2026-11-16T18:42:33Z"}]} {"id":"nix-overlay-4ou","title":"Vendor TheRock ROCm SDK recipe and update to 10.1","description":"Own the gfx1151 TheRock SDK recipe locally, update official SDK to 10.1.0, review nixpkgs PR 579527 and issue 563157, verify dependent builds and hardware checks, document and merge main.","notes":"Local SDK recipe implemented. Official archive hash sha256-UN65kwNbwvalq6Xj3o1HbYxRKi7sLL/VqladceC4szc=. Manifest rocm_version is 10.1.0 and internal rocm_package_version is 10.1.0rc3. The private patchelf matches TheRock commit d0f70eea5397606c486857e0a105e53ec123904a, includes PHDR fix #544, and passed 56 tests with 2 skipped. Stock patchelf 0.19.2 fails when extending librocsparse RPATH. LLVM 24 requires patching auxiliary ELF tools including llvm-offload-binary. SDK builds; HIP vector add and rocBLAS GPU matmul pass on gfx1151. Eight core libraries dlopen successfully. Crucible build passed, and the Strixite GPU check passed all 9 CPU-reference comparisons. Follow-up nix-overlay-9i8 tracks removal of the private patchelf. Source and migration context are documented in docs/therock-rocm.md.\nStrata rebuild completed successfully. Upstream follow-up reviewed TheRock issues/PRs #6712, #4568, #2787, #7049 and #9515, plus current TheRock Dockerfile. Current upstream still pins patchelf d0f70ee. Fresh patchelf 0.19.3 passed 69 upstream tests with 6 skipped, but reproduces the same librocsparse RPATH alignment failure as 0.19.2; the TheRock pin passes the identical rewrite. Maintenance notes updated with this evidence.\nStrixite, hipfire-kernels (349 gfx1151 modules), and hipfire rebuilds completed successfully. Packaged hipfire CLI help runs successfully with a clean runtime environment. No model inference was tested; model weights are supplied separately.\nFinal x86_64-linux flake evaluation passed with allowUnfree, and nixfmt plus git diff --check passed. Reviewed local recipe, SDK wrappers, test coverage and consumer wiring before merge.","status":"closed","priority":2,"issue_type":"feature","assignee":"minder","created_at":"2026-11-12T19:03:42.805500168Z","created_by":"minder","updated_at":"2026-11-12T20:40:38.904418019Z","closed_at":"2026-11-12T20:40:38.903746036Z","close_reason":"Local ROCm 10.1 SDK recipe, upstream investigation, consumer rebuilds and gfx1151 GPU checks completed; maintenance follow-up tracked separately.","source_repo":"nix-overlay","compaction_level":0,"original_size":0} diff --git a/packages/kyojin/package.nix b/packages/kyojin/package.nix index dea4c7f..14ae6f8 100644 --- a/packages/kyojin/package.nix +++ b/packages/kyojin/package.nix @@ -11,6 +11,7 @@ let ./prefix-writer-release.patch ./prefix-inflight-pages.patch ./prefix-direct-write.patch + ./prefix-split-segments.patch ./expert-direct-read-eof.patch ]; engine = exllamav3.overrideAttrs (old: { diff --git a/packages/kyojin/prefix-split-segments.patch b/packages/kyojin/prefix-split-segments.patch new file mode 100644 index 0000000..a09afb0 --- /dev/null +++ b/packages/kyojin/prefix-split-segments.patch @@ -0,0 +1,209 @@ +--- a/exllamav3/util/disk_store.py ++++ b/exllamav3/util/disk_store.py +@@ -11,6 +11,7 @@ + EXL3_PREFIX_DISK_GB byte cap of the whole root, all models and builds together (default 30) + EXL3_PREFIX_DISK_FLOOR_GB free space that must remain on the volume after a write (default 60) + EXL3_PREFIX_DISK_DIRECT write KV segments of at least 64 MiB with Linux O_DIRECT (default off) ++ EXL3_PREFIX_DISK_SPLIT allow smaller KV segments when one file would exceed the free-space budget (default off) + EXL3_PREFIX_DISK_MB_MIN write budget per minute (default 8000: the QLC write cliff starts near 10 GB/min) + """ + from __future__ import annotations +@@ -52,6 +53,7 @@ + self.cap = int(float(e("EXL3_PREFIX_DISK_GB", "30") if cap_gb is None else cap_gb) * GIB) + self.floor = int(float(e("EXL3_PREFIX_DISK_FLOOR_GB", "60") if floor_gb is None else floor_gb) * GIB) + self.direct_segments = e("EXL3_PREFIX_DISK_DIRECT", "0") == "1" ++ self.split_segments = e("EXL3_PREFIX_DISK_SPLIT", "0") == "1" + self.rate = float(e("EXL3_PREFIX_DISK_MB_MIN", "8000") if mb_per_min is None else mb_per_min) * (1 << 20) / 60.0 + self.stats = {"written": 0, "bytes": 0, "skipped_floor": 0, "skipped_cap": 0, "evicted": 0, "errors": 0, "hits": 0} + self._q: queue.Queue = queue.Queue() +@@ -151,17 +153,29 @@ + + # ---- async writes + +- def can_stage(self, min_bytes: int) -> bool: +- """Check the writer's cap and free-space policy before allocating a segment's host copy.""" +- if min_bytes > self.cap: ++ def stage_width(self, page_bytes: list[int]) -> int: ++ """Choose the number of pages per KV segment, or reject the save before making host copies.""" ++ total = sum(page_bytes) ++ if total > self.cap: + self.stats["skipped_cap"] += 1 +- return False +- # The writer leaves another payload's worth of headroom above the floor +- # while the serialized temporary already occupies space on the volume. +- if shutil.disk_usage(self.dir).free - 2 * min_bytes < self.floor: +- self.stats["skipped_floor"] += 1 +- return False +- return True ++ return 0 ++ headroom = shutil.disk_usage(self.dir).free - self.floor ++ # The writer checks free space again while its temporary file already occupies disk space. ++ if headroom >= 2 * total: ++ return len(page_bytes) ++ if self.split_segments and len(page_bytes) > 1: ++ # A 256-page segment ends at a 64K-token recurrent checkpoint on a fresh chain. ++ # Bounding the segment also leaves room for recurrent states queued by prefill. ++ width = min(256, (len(page_bytes) + 1) // 2) ++ largest = max(sum(page_bytes[i:i + width]) for i in range(0, len(page_bytes), width)) ++ if headroom >= total + largest: ++ return width ++ self.stats["skipped_floor"] += 1 ++ return 0 ++ ++ def can_stage(self, min_bytes: int) -> bool: ++ """Recheck a single serialized payload against the current space budget.""" ++ return bool(self.stage_width([min_bytes])) + + def put(self, name: str, payload, idx, *, min_bytes: int = 0) -> threading.Event | None: + """Queue immutable host copies. For byte-budgeted segments, signal when the write attempt ends.""" +--- a/exllamav3/generator/prefix_disk.py ++++ b/exllamav3/generator/prefix_disk.py +@@ -195,26 +195,43 @@ + new = [i for i in range(keep_depth) if hashes[i] not in staged and not on_disk(hashes[i])] + wrote = 0 + if new: +- src = [pages[i] for i in new] + tensors = sl._tensors() +- # index_select materializes one row per missing page in each cache tensor. +- min_bytes = len(src) * sum(t[0].nbytes for t in tensors) + sum(p.sequence.nbytes for p in src) +- if not self.disk.can_stage(min_bytes): ++ tensor_bytes = sum(t[0].nbytes for t in tensors) ++ page_bytes = [tensor_bytes + pages[i].sequence.nbytes for i in new] ++ width = self.disk.stage_width(page_bytes) ++ if width == 0: + return {"saved": 0} + torch.cuda.synchronize() +- tens = [] +- for t in tensors: +- idx = torch.tensor([p.page_index for p in src], dtype=torch.long, device=t.device) +- tens.append(t.index_select(0, idx).cpu()) +- hs = [hashes[i] for i in new] +- name = f"s_{_hx(hs[0])[:20]}_{len(hs)}" +- done = self.disk.put(name, {"hashes": hs, "sequences": [p.sequence.cpu().clone() for p in src], "tensors": tens}, +- {"ns": self.ns, "k": "seg", "hashes": hs}, min_bytes=min_bytes) +- if done is not None: +- self._segment_writes.append((done, hs)) +- for pos, h in enumerate(hs): +- self.pages[h] = (name, pos) +- wrote += len(hs) ++ for start in range(0, len(new), width): ++ selected = new[start:start + width] ++ # A newly allocated chain often occupies consecutive physical pages. Store them ++ # in physical order and let the segment index map hashes to their actual rows. ++ first_hash = hashes[selected[0]] ++ first_row = None ++ if width < len(new): ++ physical = sorted(selected, key=lambda i: pages[i].page_index) ++ base = pages[physical[0]].page_index ++ if all(pages[i].page_index == base + j for j, i in enumerate(physical)): ++ selected = physical ++ first_row = base ++ src = [pages[i] for i in selected] ++ hs = [hashes[i] for i in selected] ++ if first_row is not None: ++ # Make an independent copy on both GPU and CPU-backed caches. ++ tens = [t.narrow(0, first_row, len(src)).to("cpu", copy=True) for t in tensors] ++ else: ++ tens = [] ++ for t in tensors: ++ idx = torch.tensor([p.page_index for p in src], dtype=torch.long, device=t.device) ++ tens.append(t.index_select(0, idx).cpu()) ++ name = f"s_{_hx(first_hash)[:20]}_{len(hs)}" ++ done = self.disk.put(name, {"hashes": hs, "sequences": [p.sequence.cpu().clone() for p in src], "tensors": tens}, ++ {"ns": self.ns, "k": "seg", "hashes": hs}, min_bytes=sum(page_bytes[start:start + width])) ++ if done is not None: ++ self._segment_writes.append((done, hs)) ++ for pos, h in enumerate(hs): ++ self.pages[h] = (name, pos) ++ wrote += len(hs) + # 2. stashes (deepest + 2K, 4K, 8K ...), the newest deepest supersedes the older non-pinned ones of this chain + for d in sorted(want): + a, s = stash_at[d] +--- a/tests/test_disk_store_cpu.py ++++ b/tests/test_disk_store_cpu.py +@@ -298,3 +298,85 @@ + assert writer.call_count == expected + finally: + store.close() ++ ++ ++@pytest.mark.parametrize("enabled,cap_pages,headroom_pages,segments,fragmented", [ ++ (False, 20, 8, [], False), ++ (True, 4, 20, [], False), ++ (True, 20, 7, [], False), ++ (True, 20, 8, [3, 2], False), ++ (True, 20, 8, [3, 2], True), ++ (True, 20, 10, [5], False), ++]) ++def test_split_prefix_admission_and_retry(tmp_path, monkeypatch, enabled, cap_pages, headroom_pages, segments, fragmented): ++ from exllamav3.constants import PAGE_SIZE ++ from exllamav3.generator.prefix_disk import PagedPrefixDisk ++ ++ monkeypatch.setenv("EXL3_PREFIX_DISK_SPLIT", str(int(enabled))) ++ ids = torch.arange(5 * PAGE_SIZE).view(1, -1) ++ pool = torch.arange(5 * 32768, dtype=torch.int32).view(5, -1) ++ hashes = PagedPrefixDisk.chain(ids) ++ indices = [4, 2, 0, 3, 1] if fragmented else [4, 3, 2, 1, 0] ++ pages = [SimpleNamespace(page_index=p, sequence=ids[:, i*PAGE_SIZE:(i+1)*PAGE_SIZE]) for i, p in enumerate(indices)] ++ rc = Mock() ++ rc.get.side_effect = lambda h: {"position": (hashes.index(h)+1) * PAGE_SIZE} ++ slots = SimpleNamespace(generator=SimpleNamespace(recurrent_cache=rc), _tensors=lambda: [pool], ++ _resident_chain=lambda _: (hashes, pages), _find_tail=lambda *_: None) ++ pd = PagedPrefixDisk(slots, str(tmp_path)) ++ page_bytes = pool[0].nbytes + pages[0].sequence.nbytes ++ pd.disk.cap = cap_pages * page_bytes ++ pd.disk.floor = 1 << 20 ++ pd.disk.rate = 1e12 ++ monkeypatch.setattr(ds.shutil, "disk_usage", lambda _: SimpleNamespace(free=pd.disk.floor + headroom_pages * page_bytes)) ++ sync = Mock() ++ monkeypatch.setattr(torch.cuda, "synchronize", sync) ++ serialize = torch.save ++ fail_once = segments == [3, 2] ++ def save(obj, path, *args, **kwargs): ++ nonlocal fail_once ++ if fail_once and isinstance(obj, dict) and set(obj.get("hashes", ())) == set(hashes[3:]): ++ fail_once = False ++ raise OSError("second segment interrupted") ++ return serialize(obj, path, *args, **kwargs) ++ monkeypatch.setattr(torch, "save", save) ++ try: ++ assert pd.save(ids)["saved"] == sum(segments) ++ pd.disk.flush() ++ if not segments: ++ sync.assert_not_called() ++ assert not pd.disk.entries() ++ return ++ if len(segments) == 2: ++ assert pd.disk.stats["errors"] == 1 ++ assert pd.save(ids)["saved"] == 2 ++ pd.disk.flush() ++ pd.load_index() ++ names = [pd.pages[h][0] for h in hashes] ++ assert [names.count(name) for name in dict.fromkeys(names)] == segments ++ for h, page in zip(hashes, pages): ++ name, index = pd.pages[h] ++ payload = pd.disk.read(name) ++ assert torch.equal(payload["sequences"][index], page.sequence) ++ assert torch.equal(payload["tensors"][0][index], pool[page.page_index]) ++ assert pd.save(ids)["saved"] == 0 ++ finally: ++ pd.disk.close() ++ ++ ++def test_split_prefix_at_64k_checkpoint(tmp_path, monkeypatch): ++ monkeypatch.setenv("EXL3_PREFIX_DISK_SPLIT", "1") ++ store = ds.DiskStore(str(tmp_path), "ns") ++ try: ++ store.cap = 900 ++ store.floor = 200 ++ # 800 pages fit together. With one temporary committed, only a bounded ++ # segment fits into the remaining headroom while recurrent writes drain. ++ free = SimpleNamespace(free=store.floor + 800 + 256) ++ monkeypatch.setattr(ds.shutil, "disk_usage", lambda _: free) ++ assert store.stage_width([1] * 800) == 256 ++ free.free -= 1 ++ assert store.stage_width([1] * 800) == 0 ++ free.free = store.floor + 1600 ++ assert store.stage_width([1] * 800) == 800 ++ finally: ++ store.close()