diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index f47a6ca..91adcc2 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -6,6 +6,7 @@ {"id":"nix-overlay-2q7","title":"Add opt-in direct writes for large Kyojin prefix segments","description":"Retain the standard torch checkpoint and ordinary reads, small records and storage admission. Let the Flash-Next agent write large KV segments with aligned O_DIRECT to reduce page-cache pressure on bcachefs. Preserve CRC, mmap restore, unsupported-filesystem fallback and error handling. Verify native launcher, fs behavior, repeated agent timing and CPU tests.","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-18T22:28:42.144819734Z","created_by":"minder","updated_at":"2026-11-18T22:34:07.087715703Z","closed_at":"2026-11-18T22:34:07.085648146Z","close_reason":"Verified on native 128K save and fresh-process restore, strict CPU checks and repeated real agent continuations.","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"comments":[{"id":22,"issue_id":"nix-overlay-2q7","author":"minder","text":"Scope packages/kyojin. Opt-in environment for KV segments >=64 MiB only, 2 MiB staging, ordinary torch ZIP and mmap loader. Validate source-identical candidate in a real private engine.","created_at":"2026-11-18T22:28:52Z"},{"id":23,"issue_id":"nix-overlay-2q7","author":"minder","text":"ms-s1-max, bcachefs. Strict 22 filesystem/queue tests passed. Packaged CPU suite: 547 passed, 189 skipped, 349 subtests passed. Both local overlay and independent source tree build /nix/store/7ahzb4yyqj6wgqxpvcn13cpzs71y4qx6-kyojin-1.8.26. Native process opened a 3.76 GiB segment with O_DIRECT, reported 4 KiB peak residency, and served three correct receipts from 131072 cached tokens without restaging. Fresh native process using the on-disk segment and deepest checkpoint restored all 131072 tokens and returned emerald-88, 212 fresh tokens, TTFT 3.455 s, total 4.09 s. ABBA 128K/500-token JSON powers-of-two snapshots: buffered TTFT 3.861/2.772 s and total 19.907/39.741 s, direct 1.496/1.465 and 13.194/13.086 s. Identical answer and reasoning hashes. Direct serialization itself 19.4/28.7 s vs buffered 6.7/25.9 s. Archive: /DATA/LLMs/benchmark_results/kyojin-checkpoint-io-20261119.","created_at":"2026-11-18T22:34:06Z"}]} {"id":"nix-overlay-2yv","title":"strata: ship hipBLASLt tuning table calibrated for therock ROCm 10.1","description":"Upstream gfx1151-hipblaslt-100401.txt was calibrated on TheRock 7.14.1; therock 10.1.0rc3 reports the same hipBLASLt 100401 but its solution ids differ (HCD_EXACT: solution -1), so the table is a no-op and prefill runs ~418 tok/s. Recalibrated with tune_hipblaslt on ms-s1-max (gfx1151, ROCm 10.1): 17.5K cold prompt UD-Q5_K_XL 418 -> ~720 tok/s. Ship it in $out/bin/tuning/ so the engine finds it by itself.","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-16T15:49:56.904695144Z","created_by":"minder","updated_at":"2026-11-16T17:22:22.934929696Z","closed_at":"2026-11-16T17:22:22.934303630Z","close_reason":"table calibrated, shipped and measured","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"comments":[{"id":8,"issue_id":"nix-overlay-2yv","author":"minder","text":"Built on ms-s1-max (nix build --store ssh-ng): /nix/store/vv7j7ffdhcpi804pzhximszz959impgm-strata-0.2.13 ships bin/tuning/gfx1151-hipblaslt-100401.txt (rows identical to the tune_hipblaslt output). UD-Q5_K_XL 17.5K cold prompt: 418 -> 700-725 tok/s with the table; HCD_EXACT solution -1 warning gone.","created_at":"2026-11-16T17:22:22Z"}]} {"id":"nix-overlay-3c1","title":"Split oversized paged KV saves when two segments can respect the disk floor","description":"Kyojin on ms-s1-max: a single cold 200K prompt misses the SSD cache when a monolithic 5.87 GiB segment needs twice its bytes above the 46 GiB floor. Try an opt-in two-segment fallback with total cap admission, per-write floor rechecks, independent indexes and retry. Validate immediate and restarted real tool turns before enabling.","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-19T06:57:35.016699443Z","created_by":"minder","updated_at":"2026-11-19T07:33:29.292246978Z","closed_at":"2026-11-19T07:33:29.291538581Z","close_reason":"Optional bounded segments and contiguous copies verified on native long-context inference, failed-write retry, scattered pages, cap and floor boundaries.","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"comments":[{"id":24,"issue_id":"nix-overlay-3c1","author":"minder","text":"Work scope packages/kyojin new patch and package patch list. Keep normal single-file path if it fits; test refusal and partial failures. No external push.","created_at":"2026-11-19T06:57:35Z"},{"id":25,"issue_id":"nix-overlay-3c1","author":"minder","text":"Original two-part writer passed 27 CPU tests and one full native 200K restart, but a small free-space change prevented the second 400-page file in a repeat. Final candidate bounds split segments at 256 pages, uses contiguous copies only when the physical rows allow, and preserves the per-write gate. Hardware and full-suite results recorded in /DATA/LLMs/benchmark_results/kyojin-first-200k-20261119.","created_at":"2026-11-19T07:28:29Z"},{"id":26,"issue_id":"nix-overlay-3c1","author":"minder","text":"Final packaged writer stored 256+256+256+32 pages from one cold 204901-token initial HTTP request. Reload used exactly 204800 cached tokens and 187 new, answered opal-91, first token 5.162 s (control 152.599 s). Native staging 889 ms. Exact direct segment fd, minimum disk free 48.178 GiB at a 46 GiB floor. Strict suite 29 passed, original build red in 3 cases. Nix CPU suite 554 passed, 189 skipped, 349 subtests passed. Native and CPU build paths in bounded/package.path and bounded/tests.out.","created_at":"2026-11-19T07:33:29Z"}]} +{"id":"nix-overlay-3k7","title":"Bound cold prefill deferral beside a resident agent reply","description":"Opt-in Kyojin generator optimization on ms-s1-max. Give an older highly cached reply a short fixed opportunity to finish while a second fresh long prefill arrives. Preserve cold throughput, output budget, expiry and existing scheduler behavior. Compare native HTTP timings and exact tool outputs.","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-19T07:49:09.998243985Z","created_by":"minder","updated_at":"2026-11-19T08:02:32.035285104Z","closed_at":"2026-11-19T08:02:32.031438908Z","close_reason":"Opt-in bounded scheduling verified with real warm agent replies, long output and deadline/bypass regressions.","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"comments":[{"id":27,"issue_id":"nix-overlay-3k7","author":"minder","text":"File scope: packages/kyojin/package.nix, new warm-reply-grace.patch and tracker export. Private server and isolated CPU regression.","created_at":"2026-11-19T07:49:46Z"},{"id":28,"issue_id":"nix-overlay-3k7","author":"minder","text":"Native 200K resident/33378 fresh with 2 s grace, 4 HTTP runs: last content around 1.3 s, completion 3.896–4.010 s vs 24.222–24.578 s stock; no forced decode limit. The 257-token tool receipt was exact, the independent request started before its completion. Red old build (7 failed), green strict policy 7 passed; full CPU Nix build 561 passed, 189 skipped, 349 subtests. nixfmt and staged diff checks pass. Host policy and 128K/200K matrix tracked in nixos-config-7hss.","created_at":"2026-11-19T08:02:31Z"}]} {"id":"nix-overlay-3sa","title":"pdfcraft: add PdfCraft 0.18.0 (build from source)","description":"Add packages/pdfcraft building the storytold/pdfcraft Rust workspace from source (no pre-compiled binaries): desktop app + pdfcraft-cli, desktop entry, icons, metainfo, OCR models. Plus a Home Manager module.","status":"closed","priority":2,"issue_type":"feature","assignee":"minder","created_at":"2026-11-03T08:31:41.141131190Z","created_by":"minder","updated_at":"2026-11-03T10:48:31.465220459Z","closed_at":"2026-11-03T10:48:31.464736388Z","close_reason":"nix build .#packages.x86_64-linux.pdfcraft succeeds; pdfcraft/pdfcraft-cli 0.18.0 run, pdfcraft-cli render+info+extract work on a test PDF, mcp initialize answers, ocr_status reports available:true, desktop-file-validate clean, home-manager eval picks the package up from programs.pdfcraft.enable","source_repo":"nix-overlay","compaction_level":0,"original_size":0} {"id":"nix-overlay-481","title":"strata: build the MMQ prompt path (-DSTRATA_MMQ=ON)","description":"0.2.14+ leaves llama.cpp's MMQ prompt kernels out by default; then STRATA_MMQ_KQ is inert and UD-Q5_K_XL layer 2 (Q6_K) runs on Strata's int8 kernel. Measured on ms-s1-max, 0.2.15 + section 5: with MMQ and STRATA_MMQ_KQ=1 the first-token logits are bit-identical to 0.2.13 + section 5 over 15 prompts, speed equal (~1180 tok/s at 16K); without MMQ an agent prompt reached KL 0.199 against 0.2.13. Binary 39 -> 54 MB.","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-16T18:39:38.376087754Z","created_by":"minder","updated_at":"2026-11-16T18:42:33.442718186Z","closed_at":"2026-11-16T18:42:33.442193397Z","close_reason":"MMQ built, measured, committed","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"comments":[{"id":10,"issue_id":"nix-overlay-481","author":"minder","text":"Built on ms-s1-max: /nix/store/9yx7raabzwkn31wmd12v7r2n9k60cqsn-strata-0.2.15 (engine 0.1.61, 54 MB, bin/tuning present); its strata binary equals the measured strata-mmq test build byte for byte apart from the store hash. Test build results: 0.2.15 + section 5 + STRATA_MMQ_KQ=1 logits bit-identical to 0.2.13 + section 5 over 15 prompts, 16K ~1180 tok/s.","created_at":"2026-11-16T18:42:33Z"}]} {"id":"nix-overlay-4ou","title":"Vendor TheRock ROCm SDK recipe and update to 10.1","description":"Own the gfx1151 TheRock SDK recipe locally, update official SDK to 10.1.0, review nixpkgs PR 579527 and issue 563157, verify dependent builds and hardware checks, document and merge main.","notes":"Local SDK recipe implemented. Official archive hash sha256-UN65kwNbwvalq6Xj3o1HbYxRKi7sLL/VqladceC4szc=. Manifest rocm_version is 10.1.0 and internal rocm_package_version is 10.1.0rc3. The private patchelf matches TheRock commit d0f70eea5397606c486857e0a105e53ec123904a, includes PHDR fix #544, and passed 56 tests with 2 skipped. Stock patchelf 0.19.2 fails when extending librocsparse RPATH. LLVM 24 requires patching auxiliary ELF tools including llvm-offload-binary. SDK builds; HIP vector add and rocBLAS GPU matmul pass on gfx1151. Eight core libraries dlopen successfully. Crucible build passed, and the Strixite GPU check passed all 9 CPU-reference comparisons. Follow-up nix-overlay-9i8 tracks removal of the private patchelf. Source and migration context are documented in docs/therock-rocm.md.\nStrata rebuild completed successfully. Upstream follow-up reviewed TheRock issues/PRs #6712, #4568, #2787, #7049 and #9515, plus current TheRock Dockerfile. Current upstream still pins patchelf d0f70ee. Fresh patchelf 0.19.3 passed 69 upstream tests with 6 skipped, but reproduces the same librocsparse RPATH alignment failure as 0.19.2; the TheRock pin passes the identical rewrite. Maintenance notes updated with this evidence.\nStrixite, hipfire-kernels (349 gfx1151 modules), and hipfire rebuilds completed successfully. Packaged hipfire CLI help runs successfully with a clean runtime environment. No model inference was tested; model weights are supplied separately.\nFinal x86_64-linux flake evaluation passed with allowUnfree, and nixfmt plus git diff --check passed. Reviewed local recipe, SDK wrappers, test coverage and consumer wiring before merge.","status":"closed","priority":2,"issue_type":"feature","assignee":"minder","created_at":"2026-11-12T19:03:42.805500168Z","created_by":"minder","updated_at":"2026-11-12T20:40:38.904418019Z","closed_at":"2026-11-12T20:40:38.903746036Z","close_reason":"Local ROCm 10.1 SDK recipe, upstream investigation, consumer rebuilds and gfx1151 GPU checks completed; maintenance follow-up tracked separately.","source_repo":"nix-overlay","compaction_level":0,"original_size":0} diff --git a/packages/kyojin/package.nix b/packages/kyojin/package.nix index 14ae6f8..43a16ea 100644 --- a/packages/kyojin/package.nix +++ b/packages/kyojin/package.nix @@ -12,6 +12,7 @@ let ./prefix-inflight-pages.patch ./prefix-direct-write.patch ./prefix-split-segments.patch + ./warm-reply-grace.patch ./expert-direct-read-eof.patch ]; engine = exllamav3.overrideAttrs (old: { diff --git a/packages/kyojin/warm-reply-grace.patch b/packages/kyojin/warm-reply-grace.patch new file mode 100644 index 0000000..7dd2218 --- /dev/null +++ b/packages/kyojin/warm-reply-grace.patch @@ -0,0 +1,105 @@ +--- a/exllamav3/generator/generator.py ++++ b/exllamav3/generator/generator.py +@@ -46,6 +46,9 @@ + # once the results are handed out; every iterate / cancel / clear_queue runs it first, so later work sees the same cache state in + # the same order (exact by construction). Pays where the stash is a blocking copy (HIP .cpu(), CUDA EXL3_TAIL_PIN=0). + _REPLY_FIRST = os.environ.get("EXL3_REPLY_FIRST", "0") == "1" ++# With full prefill chunks, give a recently started, highly cached reply a bounded head start. ++_WARM_REPLY_GRACE_S = max(0, int(os.environ.get("EXL3_WARM_REPLY_GRACE_MS", "0"))) / 1000 ++ + _LAG1_DEBUG = os.environ.get("EXL3_LAG1_DEBUG", "0") == "1" + _LAG1 = os.environ.get("EXL3_LAG1", "0") == "1" # lag-1 decode pipeline (see Generator._gen_lag1) + from . import spec_sample +@@ -836,6 +839,23 @@ + for job in self.active_jobs: + job._cil_hold = False + ++ def _park_for_warm_reply(self, grace: float) -> tuple: ++ older, incoming = sorted(self.active_jobs, key=lambda j: j.serial_number) ++ if incoming.time_first_prefill is not None or older.new_tokens >= 64: ++ return () ++ cached = older.cached_pages * PAGE_SIZE + older.cached_tokens ++ if cached < 32768 or len(older.sequences[0].input_ids) - cached > 512: ++ return () ++ # The ordinary planner also discounts recurrent tail hits before the first chunk. ++ # A long but cached follow-up should run immediately. ++ if next(v.remaining for v in self._cil_views() if v.key == id(incoming)) < 32768 - 1: ++ return () ++ now = time.monotonic() ++ deadline = getattr(incoming, "_warm_reply_deadline", None) ++ if deadline is None: ++ deadline = incoming._warm_reply_deadline = now + grace ++ return (incoming,) if now < deadline else () ++ + def _prefill_round(self, parked, results: list): + """One prefill chunk for each active job that is not parked. The first token of a job is sampled in the generation round + after this one, so jobs that finish their prefill here all wait for the longest prefill of the step: four sub-agents of one +@@ -949,7 +969,12 @@ + + # Perform one round of prefill + _t0 = _ttrace_begin() +- parked = self._cil_parked() if chunk_il.chunk_il_enabled() else () ++ if chunk_il.chunk_il_enabled(): ++ parked = self._cil_parked() ++ elif _WARM_REPLY_GRACE_S and len(self.active_jobs) == 2: ++ parked = self._park_for_warm_reply(_WARM_REPLY_GRACE_S) ++ else: ++ parked = () + self._prefill_round(parked, results) + _ttrace_end("prefill total", _t0) + +--- /dev/null ++++ b/tests/test_warm_reply_grace_cpu.py +@@ -0,0 +1,52 @@ ++from types import SimpleNamespace ++ ++import pytest ++ ++from exllamav3.constants import PAGE_SIZE ++from exllamav3.generator import generator as gen ++ ++ ++def jobs(reused=204800, old_fresh=187, new_length=32768, generated=0, peek=0): ++ def job(order, cached, length, done): ++ seq = SimpleNamespace(input_ids=range(length), sequence_ids=range(length), kv_position=cached) ++ return SimpleNamespace(serial_number=order, cached_pages=cached // PAGE_SIZE, ++ cached_tokens=cached % PAGE_SIZE, sequences=[seq], new_tokens=generated, ++ time_first_prefill=None, is_prefill_done=lambda: done, ++ _tail_peek=lambda _: peek) ++ old = job(1, reused, reused + old_fresh, True) ++ incoming = job(2, 0, new_length, False) ++ g = object.__new__(gen.Generator) ++ g.max_chunk_size = 4096 ++ g.active_jobs = [incoming, old] ++ return g, old, incoming ++ ++ ++def test_one_deadline_per_cold_prompt(monkeypatch): ++ now = [1.0] ++ monkeypatch.setattr(gen.time, "monotonic", lambda: now[0]) ++ g, old, new = jobs() ++ old.is_prefill_done = lambda: False ++ assert g._park_for_warm_reply(2) == (new,) ++ old.is_prefill_done = lambda: True ++ now[0] = 2.99 ++ assert g._park_for_warm_reply(2) == (new,) ++ now[0] = 3.0 ++ assert g._park_for_warm_reply(2) == () ++ now[0] = 4.0 ++ assert g._park_for_warm_reply(2) == () ++ ++ ++@pytest.mark.parametrize("params", [ ++ {"reused": 0}, {"old_fresh": 513}, {"new_length": 32767}, {"generated": 64}, ++ {"peek": 32000}, ++]) ++def test_bypass_ineligible_requests(params): ++ g, _, new = jobs(**params) ++ assert g._park_for_warm_reply(2) == () ++ assert not hasattr(new, "_warm_reply_deadline") ++ ++ ++def test_prefill_already_started(): ++ g, _, new = jobs() ++ new.time_first_prefill = 1.0 ++ assert g._park_for_warm_reply(2) == ()