From 7843d79e7610e2e211052f96cd92197014a5451b Mon Sep 17 00:00:00 2001 From: Alexander Miroshnichenko Date: Thu, 19 Nov 2026 00:41:57 +0300 Subject: [PATCH] fix(kyojin): release prefix buffers before the writer waits Bead: nix-overlay-9hu --- .beads/issues.jsonl | 1 + packages/kyojin/package.nix | 1 + packages/kyojin/prefix-writer-release.patch | 67 +++++++++++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 packages/kyojin/prefix-writer-release.patch diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index fce3fff..8716e7b 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -17,6 +17,7 @@ {"id":"nix-overlay-858","title":"Correct Crucible compiler provenance documentation","description":"Review of 24195451db1a0cd8c9633a500bcfd4df9398d011...2f3f3f3250face8f341d62dc7558938806522207 found README.md:160-161 and packages/crucible-strix-halo/package.nix:32 incorrectly say upstream requires ROCm 7.2.4 for source builds. In pinned v0.37.0, tools/release_engine.py restricts compiler series for official release packaging; ordinary engine builds do not enforce it, and tools/hipcc_checked.sh refers to both 7.2.4 and 10.1. Describe official binary build provenance and tested runtime compatibility accurately. Keep binary packaging unchanged unless source compilation is separately requested and verified.","notes":"User requested fixes for the review findings. Scope also includes sharing the duplicated Strata/Strixite host compiler flags while preserving evaluated derivations and existing overrides.\nCorrected README and package comment to describe the official Crucible binary built with ROCm 7.2.4 and upstream-tested with ROCm 10.1. Extracted Strata/Strixite host flags into _lib/therock-host-flags.nix, preserving the explicit Strata glibc override and Strixite stdenv libc. Before/after drvPaths match exactly for Strata, Strixite and Crucible; nix build succeeds for all three with existing outputs. nixfmt and git diff --check pass. Reviewed the final diff; no behavior change.","status":"closed","priority":3,"issue_type":"bug","assignee":"minder","created_at":"2026-11-12T20:52:04.370312466Z","created_by":"minder","updated_at":"2026-11-12T20:55:05.309065392Z","closed_at":"2026-11-12T20:55:05.308129263Z","close_reason":"Both review findings addressed: compiler provenance documentation corrected and shared host flags extracted; identical derivations and all three package builds verified.","source_repo":"nix-overlay","compaction_level":0,"original_size":0} {"id":"nix-overlay-8wl","title":"marmel: bump to 1.0.0 (pin d6627d7c)","description":"Update packages/marmel to version 1.0.0, pinning upstream commit d6627d7cf3bf509d1e6db0d9d78736116e92e82a (the 1.0.0 release commit). Update src hash and cargoHash, verify build.","status":"closed","priority":2,"issue_type":"task","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-10-06T15:44:55Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-10-16T13:34:44Z","closed_at":"2026-10-16T13:34:44Z","close_reason":"Closed","source_repo":".","compaction_level":0,"original_size":0} {"id":"nix-overlay-8y9","title":"Package opencode plugin as open-code-review passthru","description":"Expose plugins/open-code-review/opencode from the open-code-review source as passthru.opencode-plugin: single TS plugin file + node_modules built from the repo lockfile, installed under share/open-code-review/opencode","status":"closed","priority":2,"issue_type":"feature","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-09-26T07:41:49Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-09-26T08:35:15Z","closed_at":"2026-09-26T08:35:15Z","close_reason":"Closed","source_repo":".","compaction_level":0,"original_size":0} +{"id":"nix-overlay-9hu","title":"Release serialized Kyojin prefix buffers before the disk writer waits","description":"On ms-s1-max the 128K prefix writer stages about 3.8 GiB in host tensors. Test whether its queue tuple and local payload keep tensors alive through write throttling or an empty queue. If reproduced, release consumed references, preserve cache bytes, readback, eviction, throttling and worker completion. Verify in the packaged engine.","status":"closed","priority":2,"issue_type":"bug","assignee":"minder","created_at":"2026-11-18T21:34:04.937443676Z","created_by":"minder","updated_at":"2026-11-18T21:41:57.921498885Z","closed_at":"2026-11-18T21:41:57.920704512Z","close_reason":"Released consumed buffers before writer waits; verified packaged memory and both regression paths.","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"comments":[{"id":18,"issue_id":"nix-overlay-9hu","author":"minder","text":"Scope packages/kyojin package and engine patch. Reproduce on the published writer first with weak references, including during throttling and while idle after flush.","created_at":"2026-11-18T21:34:10Z"},{"id":19,"issue_id":"nix-overlay-9hu","author":"minder","text":"Published 42kplva6 engine failed both added regression tests, patched xiiyp9p5 passed both with EXL3_TESTS_STRICT=1. Actual torch CPU DiskStore with a 512 MiB tensor in fresh processes: 512.1 MiB anonymous memory above baseline retained during throttling before, 0.2 MiB after; mapped readback checked the tensor and index both times. The skipped-cap path also releases its last queued tensor while idle. nix build --offline --no-link .#kyojin .#kyojin.tests.cpu passed, 538 passed, 189 skipped, 349 subtests passed; wrappers point at the patched engine. An observed slow 128K follow-up coincided with memory pressure; these tests isolate retention after serialization, not its HTTP latency. Evidence in /DATA/LLMs/benchmark_results/kyojin-long-decode-20261119/writer.","created_at":"2026-11-18T21:41:57Z"}]} {"id":"nix-overlay-9i8","title":"therock-rocm: drop private patchelf after SDK RPATH alignment fix","description":"TheRock 10.1.0 gfx1151 SDK (internal stamp 10.1.0rc3) librocsparse.so.1.0 fails both stock patchelf 0.19.2 and freshly tested 0.19.3 when its RPATH is extended: section content is not naturally aligned. The SDK recipe uses TheRock-pinned patchelf d0f70eea5397606c486857e0a105e53ec123904a, which includes PHDR fix #544 and passes the exact same rewrite. TheRock main still pins this commit. Revisit the private override once stock patchelf passes this real SDK input without ELF corruption. Require SDK install checks, dlopen of librocsparse/HSA/HIP and therock-rocm.tests.hip GPU tests before dropping it. See docs/therock-rocm.md.","status":"open","priority":3,"issue_type":"task","created_at":"2026-11-12T19:33:13.832941867Z","created_by":"minder","updated_at":"2026-11-12T20:12:38.441357445Z","source_repo":"nix-overlay","compaction_level":0,"original_size":0} {"id":"nix-overlay-9xs","title":"Update all packages to latest versions","description":"Update every package in packages/ to its latest upstream version, refresh source hashes and cargoHashes, verify builds, update README.","status":"in_progress","priority":2,"issue_type":"task","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-08-07T05:45:16Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-08-07T05:45:22Z","source_repo":".","compaction_level":0,"original_size":0} {"id":"nix-overlay-a9m","title":"Fix haivemind startup after MainScreen mount","description":"Restore deferred engine startup while preserving MainScreen widget lookup fix.","status":"closed","priority":1,"issue_type":"bug","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-08-20T11:20:36Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-08-20T11:21:43Z","closed_at":"2026-08-20T11:21:43Z","close_reason":"Restored engine startup after MainScreen mount and verified package build.","source_repo":".","compaction_level":0,"original_size":0} diff --git a/packages/kyojin/package.nix b/packages/kyojin/package.nix index 1f6711e..4ae4200 100644 --- a/packages/kyojin/package.nix +++ b/packages/kyojin/package.nix @@ -8,6 +8,7 @@ let enginePatches = [ ./prefix-disk-admission.patch + ./prefix-writer-release.patch ./expert-direct-read-eof.patch ]; engine = exllamav3.overrideAttrs (old: { diff --git a/packages/kyojin/prefix-writer-release.patch b/packages/kyojin/prefix-writer-release.patch new file mode 100644 index 0000000..5ccc65c --- /dev/null +++ b/packages/kyojin/prefix-writer-release.patch @@ -0,0 +1,67 @@ +--- a/exllamav3/util/disk_store.py ++++ b/exllamav3/util/disk_store.py +@@ -215,6 +215,7 @@ + spent = time.monotonic() - window_start + ahead = window_bytes / self.rate - spent # stay under the per-minute write budget + if ahead > 0: ++ item = payload = idx = None # Release host tensors before write-rate throttling. + time.sleep(min(ahead, 120.0)) + if spent > 60: + window_start, window_bytes = time.monotonic(), 0 +@@ -226,4 +227,5 @@ + except OSError: + pass + finally: ++ item = payload = idx = None # Skipped and failed writes can also precede an idle queue. + self._q.task_done() +--- a/tests/test_disk_store_cpu.py ++++ b/tests/test_disk_store_cpu.py +@@ -1,6 +1,8 @@ + """exllamav3/util/disk_store.py: atomic async writes, LRU byte cap, free-space floor, torn entries. CPU only.""" + import os + import time ++import weakref ++from threading import Event + from types import SimpleNamespace + from unittest.mock import Mock + +@@ -149,3 +151,39 @@ + assert not s.entries() and s.stats["skipped_floor"] == 1 + finally: + s.close() ++ ++ ++def test_writer_releases_payload_before_throttle(tmp_path, monkeypatch): ++ sleeping, resume = Event(), Event() ++ ++ def wait(_): ++ sleeping.set() ++ resume.wait(5) ++ ++ monkeypatch.setattr(ds, "time", SimpleNamespace(monotonic=time.monotonic, sleep=wait)) ++ store = mk(tmp_path, mb_per_min=0.001) ++ tensor = torch.arange(32) ++ ref = weakref.ref(tensor) ++ try: ++ store.put("seg", {"x": tensor}, {}) ++ del tensor ++ assert sleeping.wait(5) ++ assert ref() is None ++ assert torch.equal(store.read("seg")["x"], torch.arange(32)) ++ finally: ++ resume.set() ++ store.close() ++ ++ ++def test_idle_writer_releases_skipped_payload(tmp_path): ++ store = mk(tmp_path, cap_gb=0) ++ tensor = torch.arange(32) ++ ref = weakref.ref(tensor) ++ try: ++ store.put("seg", {"x": tensor}, {}, min_bytes=tensor.nbytes) ++ del tensor ++ store.flush() ++ assert store.stats["skipped_cap"] == 1 ++ assert ref() is None ++ finally: ++ store.close()