diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 8716e7b..dd248ab 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -14,6 +14,7 @@ {"id":"nix-overlay-5p5","title":"Update versions of all packages in repo","description":"Run nix-update for every package in packages/ to bump to latest upstream versions, then verify builds.","status":"in_progress","priority":2,"issue_type":"task","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-10-06T11:18:07Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-10-06T11:18:16Z","source_repo":".","compaction_level":0,"original_size":0} {"id":"nix-overlay-67e","title":"Add traycer package","description":"Add new package for traycer from https://github.com/traycerai/traycer. Steps: research repo build system, create packages/traycer/, stage with git add, test nix build, update README.","status":"closed","priority":2,"issue_type":"task","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-08-07T08:59:25Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-08-07T09:22:07Z","closed_at":"2026-08-07T09:22:07Z","close_reason":"Traycer package added, built, pushed","source_repo":".","compaction_level":0,"original_size":0} {"id":"nix-overlay-6pp","title":"Migrate issue tracking from bd (beads) to br (beads_rust)","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-10-21T15:46:41.935573511Z","created_by":"minder","updated_at":"2026-10-21T15:46:47.717419622Z","closed_at":"2026-10-21T15:46:47.716915059Z","close_reason":"Migrated AGENTS.md, docs/agents/*, .serena memories, .claude hooks, and .beads config/ignore from bd to br; removed bd git hooks; br verified in sync with .beads/issues.jsonl","source_repo":".","compaction_level":0,"original_size":0} +{"id":"nix-overlay-7p6","title":"Avoid staging in-flight KV pages twice on consecutive agent turns","description":"A 128K Qwen agent saved 512 pages; before the async disk index committed, a follow-up copied 513 pages instead of its single new page and stalled the driver for 8.483s. Track in-flight page segments only for save admission. Restore must still require committed files; failed, evicted or skipped writes must remain retryable. Verify concurrent branches, CPU regression and real short tool turns.","status":"closed","priority":2,"issue_type":"bug","assignee":"minder","created_at":"2026-11-18T22:11:32.447609925Z","created_by":"minder","updated_at":"2026-11-18T22:28:42.054861243Z","closed_at":"2026-11-18T22:28:42.053904304Z","close_reason":"In-flight pages now suppress duplicate staging while disk reads still require a committed index; retries and full-context restoration verified.","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"comments":[{"id":20,"issue_id":"nix-overlay-7p6","author":"minder","text":"Scope packages/kyojin only. Verify admission while the writer is blocked before index publication, retry after an async failure, and disk restoration only from committed segments.","created_at":"2026-11-18T22:11:42Z"},{"id":21,"issue_id":"nix-overlay-7p6","author":"minder","text":"Published writer under a slow 128K segment write staged 513 pages a second time; driver log 8.483 s. Dedicated CPU test with overlapping writes and a branched two-page history failed twice on the old wheel and passed both variants on r70wp920, including injected write failure, retry, committed eviction, readback and restore refusal before index publication. Package and CPU check built (540 passed, 189 skipped, 349 subtests). Normal buffered 128K tool run returned all three correct receipts and copied [512,0,0,0] pages; control happened to avoid a duplicate in that run too. Restart recovered all 131072 tokens and returned opal-91 (5.355 s TTFT). Evidence: /DATA/LLMs/benchmark_results/kyojin-checkpoint-io-20261119.","created_at":"2026-11-18T22:28:41Z"}]} {"id":"nix-overlay-858","title":"Correct Crucible compiler provenance documentation","description":"Review of 24195451db1a0cd8c9633a500bcfd4df9398d011...2f3f3f3250face8f341d62dc7558938806522207 found README.md:160-161 and packages/crucible-strix-halo/package.nix:32 incorrectly say upstream requires ROCm 7.2.4 for source builds. In pinned v0.37.0, tools/release_engine.py restricts compiler series for official release packaging; ordinary engine builds do not enforce it, and tools/hipcc_checked.sh refers to both 7.2.4 and 10.1. Describe official binary build provenance and tested runtime compatibility accurately. Keep binary packaging unchanged unless source compilation is separately requested and verified.","notes":"User requested fixes for the review findings. Scope also includes sharing the duplicated Strata/Strixite host compiler flags while preserving evaluated derivations and existing overrides.\nCorrected README and package comment to describe the official Crucible binary built with ROCm 7.2.4 and upstream-tested with ROCm 10.1. Extracted Strata/Strixite host flags into _lib/therock-host-flags.nix, preserving the explicit Strata glibc override and Strixite stdenv libc. Before/after drvPaths match exactly for Strata, Strixite and Crucible; nix build succeeds for all three with existing outputs. nixfmt and git diff --check pass. Reviewed the final diff; no behavior change.","status":"closed","priority":3,"issue_type":"bug","assignee":"minder","created_at":"2026-11-12T20:52:04.370312466Z","created_by":"minder","updated_at":"2026-11-12T20:55:05.309065392Z","closed_at":"2026-11-12T20:55:05.308129263Z","close_reason":"Both review findings addressed: compiler provenance documentation corrected and shared host flags extracted; identical derivations and all three package builds verified.","source_repo":"nix-overlay","compaction_level":0,"original_size":0} {"id":"nix-overlay-8wl","title":"marmel: bump to 1.0.0 (pin d6627d7c)","description":"Update packages/marmel to version 1.0.0, pinning upstream commit d6627d7cf3bf509d1e6db0d9d78736116e92e82a (the 1.0.0 release commit). Update src hash and cargoHash, verify build.","status":"closed","priority":2,"issue_type":"task","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-10-06T15:44:55Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-10-16T13:34:44Z","closed_at":"2026-10-16T13:34:44Z","close_reason":"Closed","source_repo":".","compaction_level":0,"original_size":0} {"id":"nix-overlay-8y9","title":"Package opencode plugin as open-code-review passthru","description":"Expose plugins/open-code-review/opencode from the open-code-review source as passthru.opencode-plugin: single TS plugin file + node_modules built from the repo lockfile, installed under share/open-code-review/opencode","status":"closed","priority":2,"issue_type":"feature","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-09-26T07:41:49Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-09-26T08:35:15Z","closed_at":"2026-09-26T08:35:15Z","close_reason":"Closed","source_repo":".","compaction_level":0,"original_size":0} diff --git a/packages/kyojin/package.nix b/packages/kyojin/package.nix index 4ae4200..6fc6029 100644 --- a/packages/kyojin/package.nix +++ b/packages/kyojin/package.nix @@ -9,6 +9,7 @@ let enginePatches = [ ./prefix-disk-admission.patch ./prefix-writer-release.patch + ./prefix-inflight-pages.patch ./expert-direct-read-eof.patch ]; engine = exllamav3.overrideAttrs (old: { diff --git a/packages/kyojin/prefix-inflight-pages.patch b/packages/kyojin/prefix-inflight-pages.patch new file mode 100644 index 0000000..10696df --- /dev/null +++ b/packages/kyojin/prefix-inflight-pages.patch @@ -0,0 +1,184 @@ +--- a/exllamav3/generator/prefix_disk.py ++++ b/exllamav3/generator/prefix_disk.py +@@ -23,6 +23,7 @@ + + import hashlib + import os ++import threading + import time + + import torch +@@ -90,6 +91,7 @@ + self.pages: dict[bytes, tuple[str, int]] = {} # page hash -> (segment, position in segment) + self.stashes: dict[bytes, tuple[str, int]] = {} # anchor hash -> (entry, depth in pages) + self.tails: dict[str, dict] = {} # entry -> idx ++ self._segment_writes: list[tuple[threading.Event, list[bytes]]] = [] + self.pending: list[torch.Tensor] = [] + self._staging = ([None, None], [None, None]) # pinned host buffers + their last device-copy events + self.stats = {"saves": 0, "save_ms": 0.0, "restores": 0, "restore_ms": 0.0, "restored_tokens": 0} +@@ -130,6 +132,7 @@ + def clear(self) -> None: + """Forget everything (cache_salt change): the entries were computed under another namespace.""" + self.disk.flush() ++ self._segment_writes.clear() + for name in self.disk.entries(): + self.disk.remove(name) + self.pages, self.stashes, self.tails = {}, {}, {} +@@ -185,7 +188,11 @@ + if ent[0] not in ok: + ok[ent[0]] = self._have(ent[0]) + return ok[ent[0]] +- new = [i for i in range(keep_depth) if not on_disk(hashes[i])] ++ # A follow-up can finish before the previous snapshot publishes its index. ++ # Save can reuse these exact host copies. Restore requires a published index. ++ self._segment_writes = [(done, hs) for done, hs in self._segment_writes if not done.is_set()] ++ staged = {h for _, hs in self._segment_writes for h in hs} ++ new = [i for i in range(keep_depth) if hashes[i] not in staged and not on_disk(hashes[i])] + wrote = 0 + if new: + src = [pages[i] for i in new] +@@ -201,8 +208,10 @@ + tens.append(t.index_select(0, idx).cpu()) + hs = [hashes[i] for i in new] + name = f"s_{_hx(hs[0])[:20]}_{len(hs)}" +- self.disk.put(name, {"hashes": hs, "sequences": [p.sequence.cpu().clone() for p in src], "tensors": tens}, +- {"ns": self.ns, "k": "seg", "hashes": hs}, min_bytes=min_bytes) ++ done = self.disk.put(name, {"hashes": hs, "sequences": [p.sequence.cpu().clone() for p in src], "tensors": tens}, ++ {"ns": self.ns, "k": "seg", "hashes": hs}, min_bytes=min_bytes) ++ if done is not None: ++ self._segment_writes.append((done, hs)) + for pos, h in enumerate(hs): + self.pages[h] = (name, pos) + wrote += len(hs) +--- a/exllamav3/util/disk_store.py ++++ b/exllamav3/util/disk_store.py +@@ -161,10 +161,12 @@ + return False + return True + +- def put(self, name: str, payload, idx, *, min_bytes: int = 0) -> None: +- """Queue an entry: payload and idx are torch.save-able and must not change afterwards (host copies).""" ++ def put(self, name: str, payload, idx, *, min_bytes: int = 0) -> threading.Event | None: ++ """Queue immutable host copies. For byte-budgeted segments, signal when the write attempt ends.""" + if not self._closed: +- self._q.put((name, payload, idx, min_bytes)) ++ finished = threading.Event() if min_bytes else None ++ self._q.put((name, payload, idx, min_bytes, finished)) ++ return finished + + def flush(self) -> None: + self._q.join() +@@ -180,11 +182,12 @@ + def _run(self) -> None: + window_start, window_bytes = time.monotonic(), 0 + while True: ++ finished = None + item = self._q.get() + try: + if item is None: + return +- name, payload, idx, min_bytes = item ++ name, payload, idx, min_bytes, finished = item + if os.path.exists(self._p(name, ".idx")): + self.touch(name) + continue +@@ -215,6 +218,9 @@ + spent = time.monotonic() - window_start + ahead = window_bytes / self.rate - spent # stay under the per-minute write budget + if ahead > 0: ++ if finished is not None: ++ finished.set() ++ finished = None + item = payload = idx = None # Release host tensors before write-rate throttling. + time.sleep(min(ahead, 120.0)) + if spent > 60: +@@ -228,4 +234,6 @@ + pass + finally: + item = payload = idx = None # Skipped and failed writes can also precede an idle queue. ++ if finished is not None: ++ finished.set() + self._q.task_done() +--- a/tests/test_disk_store_cpu.py ++++ b/tests/test_disk_store_cpu.py +@@ -145,8 +145,9 @@ + try: + assert s.can_stage(n) + free.return_value = SimpleNamespace(free=s.floor + n) +- s.put("seg", payload, {}, min_bytes=n) ++ finished = s.put("seg", payload, {}, min_bytes=n) + s.flush() ++ assert finished.is_set() + save.assert_not_called() + assert not s.entries() and s.stats["skipped_floor"] == 1 + finally: +@@ -165,9 +166,10 @@ + tensor = torch.arange(32) + ref = weakref.ref(tensor) + try: +- store.put("seg", {"x": tensor}, {}) ++ finished = store.put("seg", {"x": tensor}, {}, min_bytes=tensor.nbytes) + del tensor + assert sleeping.wait(5) ++ assert finished.is_set() + assert ref() is None + assert torch.equal(store.read("seg")["x"], torch.arange(32)) + finally: +@@ -187,3 +189,57 @@ + assert ref() is None + finally: + store.close() ++ ++ ++@pytest.mark.parametrize("fail_first", [False, True]) ++def test_consecutive_saves_share_only_inflight_pages(tmp_path, monkeypatch, fail_first): ++ from exllamav3.constants import PAGE_SIZE ++ from exllamav3.generator.prefix_disk import PagedPrefixDisk ++ ++ ids = torch.arange(2 * PAGE_SIZE).view(1, -1) ++ pool = torch.arange(6 * 128).view(6, -1) ++ pages = [SimpleNamespace(page_index=i, sequence=ids[:, n * PAGE_SIZE:(n + 1) * PAGE_SIZE]) ++ for n, i in enumerate((4, 2))] ++ hashes = PagedPrefixDisk.chain(ids) ++ rc = Mock() ++ rc.get.side_effect = lambda h: {"position": (1 + hashes.index(h)) * PAGE_SIZE} ++ def chain(tokens): ++ count = tokens.numel() // PAGE_SIZE ++ return hashes[:count], pages[:count] ++ pages_on_device = SimpleNamespace(referenced_pages={}, unreferenced_pages={}) ++ generator = SimpleNamespace(recurrent_cache=rc, pagetable=pages_on_device, tail_ckpts=[]) ++ slots = SimpleNamespace(generator=generator, _tensors=lambda: [pool], ++ _resident_chain=chain, _find_tail=lambda *_: None) ++ pd = PagedPrefixDisk(slots, str(tmp_path)) ++ pd.disk.floor = 0 ++ pd.disk.rate = 1e12 ++ monkeypatch.setattr(torch.cuda, "synchronize", lambda: None) ++ started, resume = Event(), Event() ++ serialize = torch.save ++ def hold_first(obj, path, *args, **kwargs): ++ if isinstance(obj, dict) and obj.get("hashes") == hashes[:1] and not started.is_set(): ++ started.set() ++ assert resume.wait(10) ++ if fail_first: ++ raise OSError("injected write failure") ++ return serialize(obj, path, *args, **kwargs) ++ monkeypatch.setattr(torch, "save", hold_first) ++ try: ++ assert pd.save(ids[:, :PAGE_SIZE])["saved"] == 1 ++ assert started.wait(5) ++ assert not pd._have(pd.pages[hashes[0]][0]) ++ assert pd.restore(ids) is None ++ assert pd.save(ids)["saved"] == 1 ++ resume.set() ++ pd.disk.flush() ++ assert pd.save(ids)["saved"] == int(fail_first) ++ pd.disk.flush() ++ for h, page in zip(hashes, pages): ++ name, pos = pd.pages[h] ++ assert torch.equal(pd.disk.read(name)["tensors"][0][pos], pool[page.page_index]) ++ # A committed segment that is later evicted must be staged again. ++ pd.disk.remove(pd.pages[hashes[0]][0]) ++ assert pd.save(ids)["saved"] == 1 ++ finally: ++ resume.set() ++ pd.disk.close()