From 41a50393f9e97c57a5caffd19add146bf42ca526 Mon Sep 17 00:00:00 2001 From: Alexander Miroshnichenko Date: Wed, 18 Nov 2026 21:16:42 +0300 Subject: [PATCH] perf(kyojin): allow blocking HIP waits in the Qwen server Bead: nix-overlay-j2g --- .beads/issues.jsonl | 1 + packages/kyojin/hip-blocking-sync.patch | 23 +++++++++++++++++++++++ packages/kyojin/package.nix | 5 ++++- 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 packages/kyojin/hip-blocking-sync.patch diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index ca5a5e5..fce3fff 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -36,6 +36,7 @@ {"id":"nix-overlay-gkl","title":"Add codex-oauth-proxy package to overlay","description":"Package https://github.com/dvcrn/codex-oauth-proxy in packages/codex-oauth-proxy, verify nix build, document in README.","status":"closed","priority":2,"issue_type":"task","assignee":"generalist_specialist","created_at":"2026-11-11T18:51:43.391027417Z","created_by":"minder","updated_at":"2026-11-11T23:03:27.340922308Z","closed_at":"2026-11-11T23:03:27.340385233Z","close_reason":"codex-oauth-proxy 1.1.1 packaged, builds green with tests, documented in README","source_repo":"nix-overlay","compaction_level":0,"original_size":0} {"id":"nix-overlay-hcf","title":"Update all overlay packages to latest upstream versions","description":"Bump every package in packages/ to its latest upstream release/commit using nix-update, then verify each builds. Working tree already carries uncommitted bumps for 15 packages (awg-tool, ds4, freebuff, freetoken, g3, graphify, hipengine, kubernetes-mcp-server, llama-cpp, marmel, omniroute, open-code-review, radar, skillsmcp, traycer); remaining packages (aionui, container-use, desloppify, haivemind, loop, mcp-gateway, nftables-analyzer, nftablesbuilder, relay-free-llm, shardr, stakpak) still need checking.","status":"closed","priority":2,"issue_type":"task","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-10-06T12:54:12Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-10-06T13:46:41Z","closed_at":"2026-10-06T13:46:41Z","close_reason":"All 26 packages checked: 16 committed in 6013b17aadbc; aionui/container-use/desloppify/loop/mcp-gateway/nftables-analyzer/nftablesbuilder/relay-free-llm/stakpak already current; 3 evaluation/version-string errors from nix-update repaired by hand. omniroute deferred to its own issue (stale npmDepsHash).","source_repo":".","compaction_level":0,"original_size":0} {"id":"nix-overlay-ikz","title":"Add g3 package (dhanji/g3)","description":"Package g3 AI coding agent from https://github.com/dhanji/g3 as rustPlatform.buildRustPackage. Pin commit (no tags), ship g3 + g3-studio binaries, category AI Coding Agents.","status":"closed","priority":2,"issue_type":"feature","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-09-23T10:59:57Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-09-23T11:54:07Z","closed_at":"2026-09-23T11:54:07Z","close_reason":"Closed","source_repo":".","compaction_level":0,"original_size":0} +{"id":"nix-overlay-j2g","title":"Allow the kyojin Qwen server to block the host thread while HIP executes","description":"Expose model-scoped opt-in hipDeviceScheduleBlockingSync in qwen/serve.py before model load, preserving non-scheduling flags and the existing default. On ms-s1-max default prefill and decode consume one CPU core in ROCr signal wait, verified by perf; verify the packaged path with real tool calls, CPU and latency.","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-18T18:03:00.852819489Z","created_by":"minder","updated_at":"2026-11-18T18:16:42.461414484Z","closed_at":"2026-11-18T18:16:42.460020486Z","close_reason":"Opt-in HIP blocking scheduling built and exercised on real Qwen agent requests. Arithmetic, sampler and default scheduling behavior are preserved.","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"labels":["ready-for-agent"],"comments":[{"id":16,"issue_id":"nix-overlay-j2g","author":"minder","text":"Package scope: packages/kyojin/package.nix plus a Qwen serving patch and tracker export. Default spin behavior stays available; re-use the package HIP library loader.","created_at":"2026-11-18T18:03:17Z"},{"id":17,"issue_id":"nix-overlay-j2g","author":"minder","text":"The default Qwen host thread accumulates roughly one CPU-second per second in decode and prefill. perf userspace: InterruptSignal/BusyWaitSignal wait functions account for 76.12% of the decode samples and 92.39% in the 32K-dominated request. With blocking sync, the actual packaged server used 16.3% of one CPU on a cold 32K request, down from 104.9%; repeated 384-token decode about 37% versus 108%. All deterministic digests and tool receipts in the short suite matched. The first integration test caught an early import that latched EXL3 flags before qserve set them; the final patch installs the HIP policy after the existing EXL3 import in QwenEngine. Both stock and opted-in server entry points load normally. nix build --no-link .#kyojin and .#kyojin.passthru.tests.cpu passed, and treefmt --fail-on-change packages/kyojin/package.nix passed. Final tested package /nix/store/77ixf3l3qvws9v0i46asrrf45mhgiq1m-kyojin-1.8.26. Host and further agent A/B tracked in nixos-config-pm9f.","created_at":"2026-11-18T18:16:42Z"}]} {"id":"nix-overlay-j48","title":"Add purpleflow package","description":"Package PurpleFlow (PurpleSchool/purpleflow), an Electron orchestrator for claude/codex, from source. Repo: https://github.com/PurpleSchool/purpleflow","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-13T07:32:22.851076323Z","created_by":"minder","updated_at":"2026-11-13T07:40:08.109358347Z","closed_at":"2026-11-13T07:40:08.108248543Z","close_reason":"Completed: packages/purpleflow builds from source (nix build .#purpleflow -> /nix/store/ig3an708ycb4h9832m2965s60xc5a5ig-purpleflow-0.3.2); Electron app starts under Xvfb with no errors and creates its settings; README updated.","source_repo":"nix-overlay","compaction_level":0,"original_size":0} {"id":"nix-overlay-ka5","title":"Add Strata with ROCm 10 HIP support for gfx1151","description":"Package Niko1221/Strata using the existing TheRock ROCm 10 SDK, compile HIP kernels for gfx1151, document usage, and verify build and CLI.","status":"closed","priority":2,"issue_type":"feature","assignee":"minder","created_at":"2026-11-12T18:13:13.558104256Z","created_by":"minder","updated_at":"2026-11-12T18:28:20.466366213Z","closed_at":"2026-11-12T18:28:20.465072110Z","close_reason":"Completed: Strata 0.2.13 builds with TheRock ROCm 10.0 for gfx1151; installed command checks, formatting, flake evaluation, HTTP health/models/OpenAI/Anthropic mock APIs pass. Upstream HIP intrinsics numerical parity passed on host Radeon 8060S. Engine starts with package runtime libraries (unset inherited LD_LIBRARY_PATH pointing to GCC 15). Full model inference not exercised.","source_repo":"nix-overlay","compaction_level":0,"original_size":0} {"id":"nix-overlay-m5y","title":"Add CodeAF package (Agent-Field/CodeAF)","description":"Add https://github.com/Agent-Field/CodeAF as a new package under packages/codeaf with default.nix + package.nix, staged for blueprint discovery, README table updated, nix build verified.","status":"closed","priority":2,"issue_type":"feature","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-10-06T18:07:38Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-10-06T18:46:34Z","closed_at":"2026-10-06T18:46:34Z","close_reason":"Closed","source_repo":".","compaction_level":0,"original_size":0} diff --git a/packages/kyojin/hip-blocking-sync.patch b/packages/kyojin/hip-blocking-sync.patch new file mode 100644 index 0000000..d262388 --- /dev/null +++ b/packages/kyojin/hip-blocking-sync.patch @@ -0,0 +1,23 @@ +--- a/tools/qwen/serve.py ++++ b/tools/qwen/serve.py +@@ -826,6 +826,20 @@ + # first token 81.3 / 84.1 s instead of 81.2 / 81.5 s + os.environ.setdefault("EXL3_CIL_DPRIO_CHUNK_S", "0.4") + from exllamav3 import Cache, Config, Generator, Job, Model, Tokenizer ++ if torch.version.hip and os.environ.get("KYOJIN_HIP_BLOCKING_SYNC") == "1": ++ import ctypes ++ from exllamav3.util.hip_lib import load_hip_runtime ++ ++ hip = load_hip_runtime() ++ flags = ctypes.c_uint() ++ status = hip.hipGetDeviceFlags(ctypes.byref(flags)) ++ if status: ++ raise RuntimeError(f"hipGetDeviceFlags failed with status {status}") ++ # Replace only hipDeviceScheduleMask (7) with hipDeviceScheduleBlockingSync (4). ++ status = hip.hipSetDeviceFlags(ctypes.c_uint((flags.value & ~7) | 4)) ++ if status: ++ raise RuntimeError(f"hipSetDeviceFlags failed with status {status}") ++ print("qserve: HIP blocking host synchronization enabled", flush=True) + torch.set_grad_enabled(False) + self.torch, self.Generator, self.Job = torch, Generator, Job + self.sessions = sessions diff --git a/packages/kyojin/package.nix b/packages/kyojin/package.nix index 4c7530f..1f6711e 100644 --- a/packages/kyojin/package.nix +++ b/packages/kyojin/package.nix @@ -27,7 +27,10 @@ kyojin.overrideAttrs (old: { # Upstream installs tools directly from $src, bypassing the unpacked tree. src = pkgs.applyPatches { src = old.src; - patches = [ ./short-reply-prefill-grace.patch ]; + patches = [ + ./short-reply-prefill-grace.patch + ./hip-blocking-sync.patch + ]; }; nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ pkgs.makeWrapper ]; meta = (old.meta or { }) // {