From 42c2a416fed4276e395659d522fcfb2e6be5123b Mon Sep 17 00:00:00 2001 From: Alexander Miroshnichenko Date: Thu, 19 Nov 2026 01:34:07 +0300 Subject: [PATCH] perf(kyojin): support direct writes for large prefix segments Bead: nix-overlay-2q7 --- .beads/issues.jsonl | 1 + packages/kyojin/package.nix | 1 + packages/kyojin/prefix-direct-write.patch | 171 ++++++++++++++++++++++ 3 files changed, 173 insertions(+) create mode 100644 packages/kyojin/prefix-direct-write.patch diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index dd248ab..30698cc 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -3,6 +3,7 @@ {"id":"nix-overlay-2fk","title":"marmel: bump d6627d7 (1.0.0) -> 3f238ff (main)","description":"Update packages/marmel to the latest upstream main commit 3f238ff217ee2e1033c937c840bd2d33ec57abc4 (feat: add per-role reasoning_effort configuration, #12). Verify src/hash, cargoHash, and whether the Landlock ro_paths fix landed (see nix-overlay-5ml) so the cargoTestFlags skips can be dropped or kept.","status":"closed","priority":2,"issue_type":"task","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-10-16T09:30:54Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-10-16T10:25:09Z","closed_at":"2026-10-16T10:25:09Z","close_reason":"Closed","source_repo":".","compaction_level":0,"original_size":0} {"id":"nix-overlay-2gf","title":"Update all packages to latest versions","description":"## Why\\nUser request: update every package in the overlay to its latest upstream version.\\n\\n## What\\nBump version + src hash (and cargoHash for Rust) in each packages//package.nix, update README table if needed, verify builds.\\n\\n## Acceptance\\n- Every package builds with nix build .#\\n- README reflects new versions\\n- Changes committed and pushed","notes":"All packages updated and built successfully. Commit 1f3f0f468bdd on local main. PUSH BLOCKED: git.millerson.name:22002 and DoltHub unreachable from this network (SSH connect timeout). Re-run git push + bd dolt push when network is back, then close.","status":"closed","priority":2,"issue_type":"task","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-08-07T07:22:50Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-08-07T13:17:34Z","closed_at":"2026-08-07T13:17:34Z","close_reason":"All packages updated to latest versions, verified building, pushed to origin","source_repo":".","compaction_level":0,"original_size":0} {"id":"nix-overlay-2p0","title":"Verify kyojin 1.8.26 on remote builder or CI","description":"Run nix build .#kyojin .#kyojin.tests.cpu .#kyojin.tests.jit on a remote builder or CI only. This workstation lacks resources; user explicitly forbids local builds. Local evaluation previously reported 1.8.26; build was interrupted, success unconfirmed. Downloaded/built dependencies were removed (232 paths, 21.7 GiB).","status":"open","priority":2,"issue_type":"task","created_at":"2026-11-17T18:18:48.554328412Z","created_by":"minder","updated_at":"2026-11-17T18:18:48.554328412Z","source_repo":"nix-overlay","compaction_level":0,"original_size":0} +{"id":"nix-overlay-2q7","title":"Add opt-in direct writes for large Kyojin prefix segments","description":"Retain the standard torch checkpoint and ordinary reads, small records and storage admission. Let the Flash-Next agent write large KV segments with aligned O_DIRECT to reduce page-cache pressure on bcachefs. Preserve CRC, mmap restore, unsupported-filesystem fallback and error handling. Verify native launcher, fs behavior, repeated agent timing and CPU tests.","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-18T22:28:42.144819734Z","created_by":"minder","updated_at":"2026-11-18T22:34:07.087715703Z","closed_at":"2026-11-18T22:34:07.085648146Z","close_reason":"Verified on native 128K save and fresh-process restore, strict CPU checks and repeated real agent continuations.","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"comments":[{"id":22,"issue_id":"nix-overlay-2q7","author":"minder","text":"Scope packages/kyojin. Opt-in environment for KV segments >=64 MiB only, 2 MiB staging, ordinary torch ZIP and mmap loader. Validate source-identical candidate in a real private engine.","created_at":"2026-11-18T22:28:52Z"},{"id":23,"issue_id":"nix-overlay-2q7","author":"minder","text":"ms-s1-max, bcachefs. Strict 22 filesystem/queue tests passed. Packaged CPU suite: 547 passed, 189 skipped, 349 subtests passed. Both local overlay and independent source tree build /nix/store/7ahzb4yyqj6wgqxpvcn13cpzs71y4qx6-kyojin-1.8.26. Native process opened a 3.76 GiB segment with O_DIRECT, reported 4 KiB peak residency, and served three correct receipts from 131072 cached tokens without restaging. Fresh native process using the on-disk segment and deepest checkpoint restored all 131072 tokens and returned emerald-88, 212 fresh tokens, TTFT 3.455 s, total 4.09 s. ABBA 128K/500-token JSON powers-of-two snapshots: buffered TTFT 3.861/2.772 s and total 19.907/39.741 s, direct 1.496/1.465 and 13.194/13.086 s. Identical answer and reasoning hashes. Direct serialization itself 19.4/28.7 s vs buffered 6.7/25.9 s. Archive: /DATA/LLMs/benchmark_results/kyojin-checkpoint-io-20261119.","created_at":"2026-11-18T22:34:06Z"}]} {"id":"nix-overlay-2yv","title":"strata: ship hipBLASLt tuning table calibrated for therock ROCm 10.1","description":"Upstream gfx1151-hipblaslt-100401.txt was calibrated on TheRock 7.14.1; therock 10.1.0rc3 reports the same hipBLASLt 100401 but its solution ids differ (HCD_EXACT: solution -1), so the table is a no-op and prefill runs ~418 tok/s. Recalibrated with tune_hipblaslt on ms-s1-max (gfx1151, ROCm 10.1): 17.5K cold prompt UD-Q5_K_XL 418 -> ~720 tok/s. Ship it in $out/bin/tuning/ so the engine finds it by itself.","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-16T15:49:56.904695144Z","created_by":"minder","updated_at":"2026-11-16T17:22:22.934929696Z","closed_at":"2026-11-16T17:22:22.934303630Z","close_reason":"table calibrated, shipped and measured","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"comments":[{"id":8,"issue_id":"nix-overlay-2yv","author":"minder","text":"Built on ms-s1-max (nix build --store ssh-ng): /nix/store/vv7j7ffdhcpi804pzhximszz959impgm-strata-0.2.13 ships bin/tuning/gfx1151-hipblaslt-100401.txt (rows identical to the tune_hipblaslt output). UD-Q5_K_XL 17.5K cold prompt: 418 -> 700-725 tok/s with the table; HCD_EXACT solution -1 warning gone.","created_at":"2026-11-16T17:22:22Z"}]} {"id":"nix-overlay-3sa","title":"pdfcraft: add PdfCraft 0.18.0 (build from source)","description":"Add packages/pdfcraft building the storytold/pdfcraft Rust workspace from source (no pre-compiled binaries): desktop app + pdfcraft-cli, desktop entry, icons, metainfo, OCR models. Plus a Home Manager module.","status":"closed","priority":2,"issue_type":"feature","assignee":"minder","created_at":"2026-11-03T08:31:41.141131190Z","created_by":"minder","updated_at":"2026-11-03T10:48:31.465220459Z","closed_at":"2026-11-03T10:48:31.464736388Z","close_reason":"nix build .#packages.x86_64-linux.pdfcraft succeeds; pdfcraft/pdfcraft-cli 0.18.0 run, pdfcraft-cli render+info+extract work on a test PDF, mcp initialize answers, ocr_status reports available:true, desktop-file-validate clean, home-manager eval picks the package up from programs.pdfcraft.enable","source_repo":"nix-overlay","compaction_level":0,"original_size":0} {"id":"nix-overlay-481","title":"strata: build the MMQ prompt path (-DSTRATA_MMQ=ON)","description":"0.2.14+ leaves llama.cpp's MMQ prompt kernels out by default; then STRATA_MMQ_KQ is inert and UD-Q5_K_XL layer 2 (Q6_K) runs on Strata's int8 kernel. Measured on ms-s1-max, 0.2.15 + section 5: with MMQ and STRATA_MMQ_KQ=1 the first-token logits are bit-identical to 0.2.13 + section 5 over 15 prompts, speed equal (~1180 tok/s at 16K); without MMQ an agent prompt reached KL 0.199 against 0.2.13. Binary 39 -> 54 MB.","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-16T18:39:38.376087754Z","created_by":"minder","updated_at":"2026-11-16T18:42:33.442718186Z","closed_at":"2026-11-16T18:42:33.442193397Z","close_reason":"MMQ built, measured, committed","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"comments":[{"id":10,"issue_id":"nix-overlay-481","author":"minder","text":"Built on ms-s1-max: /nix/store/9yx7raabzwkn31wmd12v7r2n9k60cqsn-strata-0.2.15 (engine 0.1.61, 54 MB, bin/tuning present); its strata binary equals the measured strata-mmq test build byte for byte apart from the store hash. Test build results: 0.2.15 + section 5 + STRATA_MMQ_KQ=1 logits bit-identical to 0.2.13 + section 5 over 15 prompts, 16K ~1180 tok/s.","created_at":"2026-11-16T18:42:33Z"}]} diff --git a/packages/kyojin/package.nix b/packages/kyojin/package.nix index 6fc6029..dea4c7f 100644 --- a/packages/kyojin/package.nix +++ b/packages/kyojin/package.nix @@ -10,6 +10,7 @@ let ./prefix-disk-admission.patch ./prefix-writer-release.patch ./prefix-inflight-pages.patch + ./prefix-direct-write.patch ./expert-direct-read-eof.patch ]; engine = exllamav3.overrideAttrs (old: { diff --git a/packages/kyojin/prefix-direct-write.patch b/packages/kyojin/prefix-direct-write.patch new file mode 100644 index 0000000..66b2709 --- /dev/null +++ b/packages/kyojin/prefix-direct-write.patch @@ -0,0 +1,171 @@ +--- /dev/null ++++ b/exllamav3/util/direct_save.py +@@ -0,0 +1,76 @@ ++"""Serialize a large CPU checkpoint through an aligned, bounded O_DIRECT buffer.""" ++import errno ++import mmap ++import os ++ ++import torch ++ ++ ++class _DirectWriter: ++ def __init__(self, fd, buffer): ++ self.fd = fd ++ self.buffer = buffer ++ self.used = 0 ++ self.size = 0 ++ self.error = None ++ ++ def _send(self, length): ++ with memoryview(self.buffer) as block: ++ offset = 0 ++ while offset < length: ++ with block[offset:length] as chunk: ++ written = os.write(self.fd, chunk) ++ if written <= 0 or written % 4096: ++ raise OSError(errno.EIO, "invalid short direct write") ++ offset += written ++ ++ def write(self, data): ++ data = memoryview(data).cast("B") ++ length = len(data) ++ if self.error is None: ++ try: ++ while data: ++ n = min(len(data), len(self.buffer) - self.used) ++ self.buffer[self.used:self.used + n] = data[:n] ++ self.used += n ++ data = data[n:] ++ if self.used == len(self.buffer): ++ self._send(self.used) ++ self.used = 0 ++ self.size += length ++ except OSError as exc: ++ # Let PyTorch finish its ZIP callback before propagating the original errno. ++ self.error = OSError(exc.errno, exc.strerror) ++ return length ++ ++ def flush(self): ++ # torch.save flushes after its ZIP footer. finish() writes the padded tail. ++ pass ++ ++ def finish(self): ++ if self.error is not None: ++ raise self.error ++ if self.used: ++ padded = (self.used + 4095) & ~4095 ++ self.buffer[self.used:padded] = bytes(padded - self.used) ++ self._send(padded) ++ os.ftruncate(self.fd, self.size) ++ ++ ++def save_segment(payload, tmp): ++ """Write torch's normal ZIP payload, falling back on filesystems without compatible direct I/O.""" ++ if hasattr(os, "O_DIRECT"): ++ try: ++ fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_DIRECT, 0o666) ++ try: ++ with mmap.mmap(-1, 2 << 20) as buffer: ++ writer = _DirectWriter(fd, buffer) ++ torch.save(payload, writer) ++ writer.finish() ++ return ++ finally: ++ os.close(fd) ++ except OSError as exc: ++ if exc.errno not in (errno.EINVAL, errno.EOPNOTSUPP, errno.ENOSYS): ++ raise ++ torch.save(payload, tmp) +--- a/exllamav3/util/disk_store.py ++++ b/exllamav3/util/disk_store.py +@@ -10,6 +10,7 @@ + EXL3_PREFIX_DISK_DIR root directory; empty = off + EXL3_PREFIX_DISK_GB byte cap of the whole root, all models and builds together (default 30) + EXL3_PREFIX_DISK_FLOOR_GB free space that must remain on the volume after a write (default 60) ++ EXL3_PREFIX_DISK_DIRECT write KV segments of at least 64 MiB with Linux O_DIRECT (default off) + EXL3_PREFIX_DISK_MB_MIN write budget per minute (default 8000: the QLC write cliff starts near 10 GB/min) + """ + from __future__ import annotations +@@ -50,6 +51,7 @@ + e = os.environ.get + self.cap = int(float(e("EXL3_PREFIX_DISK_GB", "30") if cap_gb is None else cap_gb) * GIB) + self.floor = int(float(e("EXL3_PREFIX_DISK_FLOOR_GB", "60") if floor_gb is None else floor_gb) * GIB) ++ self.direct_segments = e("EXL3_PREFIX_DISK_DIRECT", "0") == "1" + self.rate = float(e("EXL3_PREFIX_DISK_MB_MIN", "8000") if mb_per_min is None else mb_per_min) * (1 << 20) / 60.0 + self.stats = {"written": 0, "bytes": 0, "skipped_floor": 0, "skipped_cap": 0, "evicted": 0, "errors": 0, "hits": 0} + self._q: queue.Queue = queue.Queue() +@@ -195,7 +197,11 @@ + try: + if not self.can_stage(min_bytes): + continue +- torch.save(payload, tmp) ++ if self.direct_segments and min_bytes >= 64 << 20: ++ from .direct_save import save_segment ++ save_segment(payload, tmp) ++ else: ++ torch.save(payload, tmp) + n = os.path.getsize(tmp) + free = shutil.disk_usage(self.dir).free + if free - n < self.floor: # live check: other writers share the disk +--- a/tests/test_disk_store_cpu.py ++++ b/tests/test_disk_store_cpu.py +@@ -243,3 +243,58 @@ + finally: + resume.set() + pd.disk.close() ++ ++ ++@pytest.mark.parametrize("case", ["supported", "unsupported", "misaligned", "no_space"]) ++def test_direct_snapshot_round_trip_and_write_errors(tmp_path, monkeypatch, case): ++ import errno ++ from exllamav3.util import direct_save ++ ++ if not hasattr(os, "O_DIRECT"): ++ pytest.skip("Linux direct I/O") ++ path = tmp_path / "segment.pt.tmp" ++ open_file = os.open ++ def open_direct(name, flags, mode=0o777): ++ assert flags & os.O_DIRECT ++ if case == "unsupported": ++ raise OSError(errno.EOPNOTSUPP, "direct I/O unavailable") ++ # Build machines may use a filesystem without direct I/O. ++ return open_file(name, flags & ~os.O_DIRECT, mode) ++ monkeypatch.setattr(os, "open", open_direct) ++ original = os.write ++ writes = [] ++ def check_write(fd, data): ++ writes.append(len(data)) ++ if len(writes) == 2 and case != "supported": ++ error = errno.EINVAL if case == "misaligned" else errno.ENOSPC ++ raise OSError(error, os.strerror(error)) ++ return original(fd, data) ++ monkeypatch.setattr(os, "write", check_write) ++ tensor = torch.arange(700013, dtype=torch.int64) ++ if case == "no_space": ++ with pytest.raises(OSError) as exc: ++ direct_save.save_segment({"x": tensor}, path) ++ assert exc.value.errno == errno.ENOSPC ++ else: ++ direct_save.save_segment({"x": tensor}, path) ++ restored = torch.load(path, mmap=True, weights_only=False)["x"] ++ assert torch.equal(tensor, restored) ++ assert not writes if case == "unsupported" else len(writes) >= 2 ++ assert all(n % 4096 == 0 for n in writes) ++ ++ ++@pytest.mark.parametrize("enabled,estimate,expected", [(False, 64 << 20, 0), (True, 63 << 20, 0), (True, 64 << 20, 1)]) ++def test_direct_write_only_for_large_enabled_segments(tmp_path, monkeypatch, enabled, estimate, expected): ++ from exllamav3.util import direct_save ++ ++ monkeypatch.setenv("EXL3_PREFIX_DISK_DIRECT", str(int(enabled))) ++ writer = Mock(side_effect=torch.save) ++ monkeypatch.setattr(direct_save, "save_segment", writer) ++ store = mk(tmp_path, cap_gb=1) ++ try: ++ store.put("seg", torch.arange(32), {}, min_bytes=estimate) ++ store.flush() ++ assert torch.equal(torch.arange(32), store.read("seg")) ++ assert writer.call_count == expected ++ finally: ++ store.close()