perf(kyojin): finish completed streams before the next prefill
CI / check (push) Has been cancelled

Bead: nix-overlay-8gs
This commit is contained in:
2026-11-19 11:22:29 +03:00
parent 848a6ec9c4
commit 8389c88d04
3 changed files with 107 additions and 0 deletions
+1
View File
@@ -19,6 +19,7 @@
{"id":"nix-overlay-6pp","title":"Migrate issue tracking from bd (beads) to br (beads_rust)","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-10-21T15:46:41.935573511Z","created_by":"minder","updated_at":"2026-10-21T15:46:47.717419622Z","closed_at":"2026-10-21T15:46:47.716915059Z","close_reason":"Migrated AGENTS.md, docs/agents/*, .serena memories, .claude hooks, and .beads config/ignore from bd to br; removed bd git hooks; br verified in sync with .beads/issues.jsonl","source_repo":".","compaction_level":0,"original_size":0} {"id":"nix-overlay-6pp","title":"Migrate issue tracking from bd (beads) to br (beads_rust)","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-10-21T15:46:41.935573511Z","created_by":"minder","updated_at":"2026-10-21T15:46:47.717419622Z","closed_at":"2026-10-21T15:46:47.716915059Z","close_reason":"Migrated AGENTS.md, docs/agents/*, .serena memories, .claude hooks, and .beads config/ignore from bd to br; removed bd git hooks; br verified in sync with .beads/issues.jsonl","source_repo":".","compaction_level":0,"original_size":0}
{"id":"nix-overlay-7p6","title":"Avoid staging in-flight KV pages twice on consecutive agent turns","description":"A 128K Qwen agent saved 512 pages; before the async disk index committed, a follow-up copied 513 pages instead of its single new page and stalled the driver for 8.483s. Track in-flight page segments only for save admission. Restore must still require committed files; failed, evicted or skipped writes must remain retryable. Verify concurrent branches, CPU regression and real short tool turns.","status":"closed","priority":2,"issue_type":"bug","assignee":"minder","created_at":"2026-11-18T22:11:32.447609925Z","created_by":"minder","updated_at":"2026-11-18T22:28:42.054861243Z","closed_at":"2026-11-18T22:28:42.053904304Z","close_reason":"In-flight pages now suppress duplicate staging while disk reads still require a committed index; retries and full-context restoration verified.","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"comments":[{"id":20,"issue_id":"nix-overlay-7p6","author":"minder","text":"Scope packages/kyojin only. Verify admission while the writer is blocked before index publication, retry after an async failure, and disk restoration only from committed segments.","created_at":"2026-11-18T22:11:42Z"},{"id":21,"issue_id":"nix-overlay-7p6","author":"minder","text":"Published writer under a slow 128K segment write staged 513 pages a second time; driver log 8.483 s. Dedicated CPU test with overlapping writes and a branched two-page history failed twice on the old wheel and passed both variants on r70wp920, including injected write failure, retry, committed eviction, readback and restore refusal before index publication. Package and CPU check built (540 passed, 189 skipped, 349 subtests). Normal buffered 128K tool run returned all three correct receipts and copied [512,0,0,0] pages; control happened to avoid a duplicate in that run too. Restart recovered all 131072 tokens and returned opal-91 (5.355 s TTFT). Evidence: /DATA/LLMs/benchmark_results/kyojin-checkpoint-io-20261119.","created_at":"2026-11-18T22:28:41Z"}]} {"id":"nix-overlay-7p6","title":"Avoid staging in-flight KV pages twice on consecutive agent turns","description":"A 128K Qwen agent saved 512 pages; before the async disk index committed, a follow-up copied 513 pages instead of its single new page and stalled the driver for 8.483s. Track in-flight page segments only for save admission. Restore must still require committed files; failed, evicted or skipped writes must remain retryable. Verify concurrent branches, CPU regression and real short tool turns.","status":"closed","priority":2,"issue_type":"bug","assignee":"minder","created_at":"2026-11-18T22:11:32.447609925Z","created_by":"minder","updated_at":"2026-11-18T22:28:42.054861243Z","closed_at":"2026-11-18T22:28:42.053904304Z","close_reason":"In-flight pages now suppress duplicate staging while disk reads still require a committed index; retries and full-context restoration verified.","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"comments":[{"id":20,"issue_id":"nix-overlay-7p6","author":"minder","text":"Scope packages/kyojin only. Verify admission while the writer is blocked before index publication, retry after an async failure, and disk restoration only from committed segments.","created_at":"2026-11-18T22:11:42Z"},{"id":21,"issue_id":"nix-overlay-7p6","author":"minder","text":"Published writer under a slow 128K segment write staged 513 pages a second time; driver log 8.483 s. Dedicated CPU test with overlapping writes and a branched two-page history failed twice on the old wheel and passed both variants on r70wp920, including injected write failure, retry, committed eviction, readback and restore refusal before index publication. Package and CPU check built (540 passed, 189 skipped, 349 subtests). Normal buffered 128K tool run returned all three correct receipts and copied [512,0,0,0] pages; control happened to avoid a duplicate in that run too. Restart recovered all 131072 tokens and returned opal-91 (5.355 s TTFT). Evidence: /DATA/LLMs/benchmark_results/kyojin-checkpoint-io-20261119.","created_at":"2026-11-18T22:28:41Z"}]}
{"id":"nix-overlay-858","title":"Correct Crucible compiler provenance documentation","description":"Review of 24195451db1a0cd8c9633a500bcfd4df9398d011...2f3f3f3250face8f341d62dc7558938806522207 found README.md:160-161 and packages/crucible-strix-halo/package.nix:32 incorrectly say upstream requires ROCm 7.2.4 for source builds. In pinned v0.37.0, tools/release_engine.py restricts compiler series for official release packaging; ordinary engine builds do not enforce it, and tools/hipcc_checked.sh refers to both 7.2.4 and 10.1. Describe official binary build provenance and tested runtime compatibility accurately. Keep binary packaging unchanged unless source compilation is separately requested and verified.","notes":"User requested fixes for the review findings. Scope also includes sharing the duplicated Strata/Strixite host compiler flags while preserving evaluated derivations and existing overrides.\nCorrected README and package comment to describe the official Crucible binary built with ROCm 7.2.4 and upstream-tested with ROCm 10.1. Extracted Strata/Strixite host flags into _lib/therock-host-flags.nix, preserving the explicit Strata glibc override and Strixite stdenv libc. Before/after drvPaths match exactly for Strata, Strixite and Crucible; nix build succeeds for all three with existing outputs. nixfmt and git diff --check pass. Reviewed the final diff; no behavior change.","status":"closed","priority":3,"issue_type":"bug","assignee":"minder","created_at":"2026-11-12T20:52:04.370312466Z","created_by":"minder","updated_at":"2026-11-12T20:55:05.309065392Z","closed_at":"2026-11-12T20:55:05.308129263Z","close_reason":"Both review findings addressed: compiler provenance documentation corrected and shared host flags extracted; identical derivations and all three package builds verified.","source_repo":"nix-overlay","compaction_level":0,"original_size":0} {"id":"nix-overlay-858","title":"Correct Crucible compiler provenance documentation","description":"Review of 24195451db1a0cd8c9633a500bcfd4df9398d011...2f3f3f3250face8f341d62dc7558938806522207 found README.md:160-161 and packages/crucible-strix-halo/package.nix:32 incorrectly say upstream requires ROCm 7.2.4 for source builds. In pinned v0.37.0, tools/release_engine.py restricts compiler series for official release packaging; ordinary engine builds do not enforce it, and tools/hipcc_checked.sh refers to both 7.2.4 and 10.1. Describe official binary build provenance and tested runtime compatibility accurately. Keep binary packaging unchanged unless source compilation is separately requested and verified.","notes":"User requested fixes for the review findings. Scope also includes sharing the duplicated Strata/Strixite host compiler flags while preserving evaluated derivations and existing overrides.\nCorrected README and package comment to describe the official Crucible binary built with ROCm 7.2.4 and upstream-tested with ROCm 10.1. Extracted Strata/Strixite host flags into _lib/therock-host-flags.nix, preserving the explicit Strata glibc override and Strixite stdenv libc. Before/after drvPaths match exactly for Strata, Strixite and Crucible; nix build succeeds for all three with existing outputs. nixfmt and git diff --check pass. Reviewed the final diff; no behavior change.","status":"closed","priority":3,"issue_type":"bug","assignee":"minder","created_at":"2026-11-12T20:52:04.370312466Z","created_by":"minder","updated_at":"2026-11-12T20:55:05.309065392Z","closed_at":"2026-11-12T20:55:05.308129263Z","close_reason":"Both review findings addressed: compiler provenance documentation corrected and shared host flags extracted; identical derivations and all three package builds verified.","source_repo":"nix-overlay","compaction_level":0,"original_size":0}
{"id":"nix-overlay-8gs","title":"Finish a completed Qwen HTTP stream before the next prefill chunk","description":"ms-s1-max: removing an already-finished route waits behind another agent’s GPU prefill chunk after EOS. Retire a proven finished serial on the driver at dispatch while another job is live. Preserve requeue/cancel semantics and the solo reply-first path; verify real HTTP completion and stream content.","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-19T08:02:32.252352176Z","created_by":"minder","updated_at":"2026-11-19T08:22:28.859976897Z","closed_at":"2026-11-19T08:22:28.859707001Z","close_reason":"Completed stream retires at EOS with concurrent prefill. Verified dispatcher lifecycle and two real 200K cached agent turns.","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"comments":[{"id":29,"issue_id":"nix-overlay-8gs","author":"minder","text":"Scope: packages/kyojin/package.nix and new finished-route.patch; real Qwen driver regression and a packaged two-client comparison. Original source fails the finished-route check.","created_at":"2026-11-19T08:03:14Z"},{"id":30,"issue_id":"nix-overlay-8gs","author":"minder","text":"Synthetic driver with a blocking next iterate reproduced the lingering completed route on the old build. FinishedRouteTests passed on the packaged candidate, including same-serial active/pending replacements and solo cleanup. Full patched-source Qwen suite ran 119 tests (one optional test rerun with the real model tokenizer and Transformers). Native 32K/128K/200K verification in progress.","created_at":"2026-11-19T08:18:43Z"},{"id":31,"issue_id":"nix-overlay-8gs","author":"minder","text":"Native final packaged server: a warm 200K tool stream beside a fresh 33378-token prompt completed at 1.709/1.448 s, final frame within 1 ms of last delta (scheduling-only 4.010/3.960 with 2.74/2.66 s after last delta). Both reply and incoming reasoning/function/content hashes identical to same-input controls. Negative cases keep non-EOS, pending/active same-serial replacement and solo routes. Full Qwen suite 119 discovered, one optional template checked separately using real pack; both succeeded. nixfmt and staged diff clean.","created_at":"2026-11-19T08:22:28Z"}]}
{"id":"nix-overlay-8wl","title":"marmel: bump to 1.0.0 (pin d6627d7c)","description":"Update packages/marmel to version 1.0.0, pinning upstream commit d6627d7cf3bf509d1e6db0d9d78736116e92e82a (the 1.0.0 release commit). Update src hash and cargoHash, verify build.","status":"closed","priority":2,"issue_type":"task","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-10-06T15:44:55Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-10-16T13:34:44Z","closed_at":"2026-10-16T13:34:44Z","close_reason":"Closed","source_repo":".","compaction_level":0,"original_size":0} {"id":"nix-overlay-8wl","title":"marmel: bump to 1.0.0 (pin d6627d7c)","description":"Update packages/marmel to version 1.0.0, pinning upstream commit d6627d7cf3bf509d1e6db0d9d78736116e92e82a (the 1.0.0 release commit). Update src hash and cargoHash, verify build.","status":"closed","priority":2,"issue_type":"task","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-10-06T15:44:55Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-10-16T13:34:44Z","closed_at":"2026-10-16T13:34:44Z","close_reason":"Closed","source_repo":".","compaction_level":0,"original_size":0}
{"id":"nix-overlay-8y9","title":"Package opencode plugin as open-code-review passthru","description":"Expose plugins/open-code-review/opencode from the open-code-review source as passthru.opencode-plugin: single TS plugin file + node_modules built from the repo lockfile, installed under share/open-code-review/opencode","status":"closed","priority":2,"issue_type":"feature","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-09-26T07:41:49Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-09-26T08:35:15Z","closed_at":"2026-09-26T08:35:15Z","close_reason":"Closed","source_repo":".","compaction_level":0,"original_size":0} {"id":"nix-overlay-8y9","title":"Package opencode plugin as open-code-review passthru","description":"Expose plugins/open-code-review/opencode from the open-code-review source as passthru.opencode-plugin: single TS plugin file + node_modules built from the repo lockfile, installed under share/open-code-review/opencode","status":"closed","priority":2,"issue_type":"feature","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-09-26T07:41:49Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-09-26T08:35:15Z","closed_at":"2026-09-26T08:35:15Z","close_reason":"Closed","source_repo":".","compaction_level":0,"original_size":0}
{"id":"nix-overlay-9hu","title":"Release serialized Kyojin prefix buffers before the disk writer waits","description":"On ms-s1-max the 128K prefix writer stages about 3.8 GiB in host tensors. Test whether its queue tuple and local payload keep tensors alive through write throttling or an empty queue. If reproduced, release consumed references, preserve cache bytes, readback, eviction, throttling and worker completion. Verify in the packaged engine.","status":"closed","priority":2,"issue_type":"bug","assignee":"minder","created_at":"2026-11-18T21:34:04.937443676Z","created_by":"minder","updated_at":"2026-11-18T21:41:57.921498885Z","closed_at":"2026-11-18T21:41:57.920704512Z","close_reason":"Released consumed buffers before writer waits; verified packaged memory and both regression paths.","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"comments":[{"id":18,"issue_id":"nix-overlay-9hu","author":"minder","text":"Scope packages/kyojin package and engine patch. Reproduce on the published writer first with weak references, including during throttling and while idle after flush.","created_at":"2026-11-18T21:34:10Z"},{"id":19,"issue_id":"nix-overlay-9hu","author":"minder","text":"Published 42kplva6 engine failed both added regression tests, patched xiiyp9p5 passed both with EXL3_TESTS_STRICT=1. Actual torch CPU DiskStore with a 512 MiB tensor in fresh processes: 512.1 MiB anonymous memory above baseline retained during throttling before, 0.2 MiB after; mapped readback checked the tensor and index both times. The skipped-cap path also releases its last queued tensor while idle. nix build --offline --no-link .#kyojin .#kyojin.tests.cpu passed, 538 passed, 189 skipped, 349 subtests passed; wrappers point at the patched engine. An observed slow 128K follow-up coincided with memory pressure; these tests isolate retention after serialization, not its HTTP latency. Evidence in /DATA/LLMs/benchmark_results/kyojin-long-decode-20261119/writer.","created_at":"2026-11-18T21:41:57Z"}]} {"id":"nix-overlay-9hu","title":"Release serialized Kyojin prefix buffers before the disk writer waits","description":"On ms-s1-max the 128K prefix writer stages about 3.8 GiB in host tensors. Test whether its queue tuple and local payload keep tensors alive through write throttling or an empty queue. If reproduced, release consumed references, preserve cache bytes, readback, eviction, throttling and worker completion. Verify in the packaged engine.","status":"closed","priority":2,"issue_type":"bug","assignee":"minder","created_at":"2026-11-18T21:34:04.937443676Z","created_by":"minder","updated_at":"2026-11-18T21:41:57.921498885Z","closed_at":"2026-11-18T21:41:57.920704512Z","close_reason":"Released consumed buffers before writer waits; verified packaged memory and both regression paths.","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"comments":[{"id":18,"issue_id":"nix-overlay-9hu","author":"minder","text":"Scope packages/kyojin package and engine patch. Reproduce on the published writer first with weak references, including during throttling and while idle after flush.","created_at":"2026-11-18T21:34:10Z"},{"id":19,"issue_id":"nix-overlay-9hu","author":"minder","text":"Published 42kplva6 engine failed both added regression tests, patched xiiyp9p5 passed both with EXL3_TESTS_STRICT=1. Actual torch CPU DiskStore with a 512 MiB tensor in fresh processes: 512.1 MiB anonymous memory above baseline retained during throttling before, 0.2 MiB after; mapped readback checked the tensor and index both times. The skipped-cap path also releases its last queued tensor while idle. nix build --offline --no-link .#kyojin .#kyojin.tests.cpu passed, 538 passed, 189 skipped, 349 subtests passed; wrappers point at the patched engine. An observed slow 128K follow-up coincided with memory pressure; these tests isolate retention after serialization, not its HTTP latency. Evidence in /DATA/LLMs/benchmark_results/kyojin-long-decode-20261119/writer.","created_at":"2026-11-18T21:41:57Z"}]}
+105
View File
@@ -0,0 +1,105 @@
--- a/tools/qwen/serve.py
+++ b/tools/qwen/serve.py
@@ -1262,6 +1262,15 @@
for serial, evs in by_serial.items():
route = self._routes.get(serial)
if route is not None: # None: cancelled after the step started
+ if evs[-1].get("eos") and self.generator.num_remaining_jobs():
+ # A requeued replacement keeps its serial. Retire only a finished request,
+ # before the next GPU step can delay its handler's final _drop().
+ live = self.generator.active_jobs + self.generator.pending_jobs
+ if all(job.serial_number != serial for job in live):
+ del self._routes[serial]
+ finished = loop.create_future()
+ finished.set_result(None)
+ route[0]._qserve_finished = finished
route[1].put_nowait(evs)
def _fail_unroutable(self, ev, exc: Exception) -> None:
@@ -1298,6 +1307,10 @@
def _drop(self, job) -> asyncio.Future:
"""Stop routing the job and remove it from the Generator (a no-op when it already finished). Runs on the driver, after
any enqueue posted before it, so a job cancelled while its enqueue is still queued does not stay behind."""
+ finished = getattr(job, "_qserve_finished", None)
+ if finished is not None:
+ return finished
+
def forget():
serial = getattr(job, "serial_number", None)
route = self._routes.get(serial)
--- a/tools/qwen/test_serve.py
+++ b/tools/qwen/test_serve.py
@@ -4,10 +4,12 @@
import asyncio
import base64
+import contextlib
import io
import json
import os
import sys
+import threading
import types
import unittest
import unittest.mock
@@ -1246,5 +1248,60 @@
self.assertIn(b"hel", body)
+class FinishedRouteTests(unittest.IsolatedAsyncioTestCase):
+ async def start_driver(self, mode):
+ engine = object.__new__(serve.QwenEngine)
+ old = types.SimpleNamespace(serial_number=4)
+ other = types.SimpleNamespace(serial_number=5)
+ replacement = types.SimpleNamespace(serial_number=4)
+ generator = types.SimpleNamespace(active_jobs=[old], pending_jobs=[])
+ release = threading.Event()
+
+ def iterate():
+ if any(job is old for job in generator.active_jobs):
+ generator.active_jobs = [] if mode == "solo" else [other]
+ if mode == "active":
+ generator.active_jobs.append(replacement)
+ if mode == "pending":
+ generator.pending_jobs.append(replacement)
+ return [dict(serial=4, eos=mode != "ongoing", text="last piece")]
+ release.wait(2)
+ generator.active_jobs.clear()
+ generator.pending_jobs.clear()
+ return []
+
+ generator.iterate = iterate
+ generator.num_remaining_jobs = lambda: len(generator.active_jobs) + len(generator.pending_jobs)
+ generator.cancel = unittest.mock.Mock()
+ engine.generator = generator
+ engine._ensure_driver()
+
+ async def stop():
+ release.set()
+ engine._driver.cancel()
+ with contextlib.suppress(asyncio.CancelledError):
+ await engine._driver
+
+ self.addAsyncCleanup(stop)
+ queue = asyncio.Queue()
+ engine._routes[4] = (old, queue)
+ events = await asyncio.wait_for(queue.get(), 1)
+ self.assertEqual(events[-1]["text"], "last piece")
+ return engine, old
+
+ async def test_completed_request_retires_before_another_chunk(self):
+ engine, old = await self.start_driver("done")
+ self.assertNotIn(4, engine._routes)
+ self.assertTrue(engine._drop(old).done())
+ self.assertTrue(engine._drop(old).done())
+ engine.generator.cancel.assert_not_called()
+
+ async def test_keep_live_requeued_and_solo_routes(self):
+ for mode in ("ongoing", "active", "pending", "solo"):
+ with self.subTest(mode=mode):
+ engine, old = await self.start_driver(mode)
+ self.assertIs(engine._routes[4][0], old)
+
+
class PrefixDiskConcurrencyTests(unittest.TestCase):
"""The disk cache must work while other requests run (3 users): restore with a busy Generator, save of the request's own prompt."""
+1
View File
@@ -35,6 +35,7 @@ kyojin.overrideAttrs (old: {
patches = [ patches = [
./short-reply-prefill-grace.patch ./short-reply-prefill-grace.patch
./hip-blocking-sync.patch ./hip-blocking-sync.patch
./finished-route.patch
]; ];
}; };
nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ pkgs.makeWrapper ]; nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ pkgs.makeWrapper ];