perf(kyojin): allow blocking HIP waits in the Qwen server
CI / check (push) Has been cancelled

Bead: nix-overlay-j2g
This commit is contained in:
2026-11-18 21:16:42 +03:00
parent 2b8eddf7d2
commit 41a50393f9
3 changed files with 28 additions and 1 deletions
+1
View File
@@ -36,6 +36,7 @@
{"id":"nix-overlay-gkl","title":"Add codex-oauth-proxy package to overlay","description":"Package https://github.com/dvcrn/codex-oauth-proxy in packages/codex-oauth-proxy, verify nix build, document in README.","status":"closed","priority":2,"issue_type":"task","assignee":"generalist_specialist","created_at":"2026-11-11T18:51:43.391027417Z","created_by":"minder","updated_at":"2026-11-11T23:03:27.340922308Z","closed_at":"2026-11-11T23:03:27.340385233Z","close_reason":"codex-oauth-proxy 1.1.1 packaged, builds green with tests, documented in README","source_repo":"nix-overlay","compaction_level":0,"original_size":0}
{"id":"nix-overlay-hcf","title":"Update all overlay packages to latest upstream versions","description":"Bump every package in packages/ to its latest upstream release/commit using nix-update, then verify each builds. Working tree already carries uncommitted bumps for 15 packages (awg-tool, ds4, freebuff, freetoken, g3, graphify, hipengine, kubernetes-mcp-server, llama-cpp, marmel, omniroute, open-code-review, radar, skillsmcp, traycer); remaining packages (aionui, container-use, desloppify, haivemind, loop, mcp-gateway, nftables-analyzer, nftablesbuilder, relay-free-llm, shardr, stakpak) still need checking.","status":"closed","priority":2,"issue_type":"task","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-10-06T12:54:12Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-10-06T13:46:41Z","closed_at":"2026-10-06T13:46:41Z","close_reason":"All 26 packages checked: 16 committed in 6013b17aadbc; aionui/container-use/desloppify/loop/mcp-gateway/nftables-analyzer/nftablesbuilder/relay-free-llm/stakpak already current; 3 evaluation/version-string errors from nix-update repaired by hand. omniroute deferred to its own issue (stale npmDepsHash).","source_repo":".","compaction_level":0,"original_size":0}
{"id":"nix-overlay-ikz","title":"Add g3 package (dhanji/g3)","description":"Package g3 AI coding agent from https://github.com/dhanji/g3 as rustPlatform.buildRustPackage. Pin commit (no tags), ship g3 + g3-studio binaries, category AI Coding Agents.","status":"closed","priority":2,"issue_type":"feature","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-09-23T10:59:57Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-09-23T11:54:07Z","closed_at":"2026-09-23T11:54:07Z","close_reason":"Closed","source_repo":".","compaction_level":0,"original_size":0}
{"id":"nix-overlay-j2g","title":"Allow the kyojin Qwen server to block the host thread while HIP executes","description":"Expose model-scoped opt-in hipDeviceScheduleBlockingSync in qwen/serve.py before model load, preserving non-scheduling flags and the existing default. On ms-s1-max default prefill and decode consume one CPU core in ROCr signal wait, verified by perf; verify the packaged path with real tool calls, CPU and latency.","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-18T18:03:00.852819489Z","created_by":"minder","updated_at":"2026-11-18T18:16:42.461414484Z","closed_at":"2026-11-18T18:16:42.460020486Z","close_reason":"Opt-in HIP blocking scheduling built and exercised on real Qwen agent requests. Arithmetic, sampler and default scheduling behavior are preserved.","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"labels":["ready-for-agent"],"comments":[{"id":16,"issue_id":"nix-overlay-j2g","author":"minder","text":"Package scope: packages/kyojin/package.nix plus a Qwen serving patch and tracker export. Default spin behavior stays available; re-use the package HIP library loader.","created_at":"2026-11-18T18:03:17Z"},{"id":17,"issue_id":"nix-overlay-j2g","author":"minder","text":"The default Qwen host thread accumulates roughly one CPU-second per second in decode and prefill. perf userspace: InterruptSignal/BusyWaitSignal wait functions account for 76.12% of the decode samples and 92.39% in the 32K-dominated request. With blocking sync, the actual packaged server used 16.3% of one CPU on a cold 32K request, down from 104.9%; repeated 384-token decode about 37% versus 108%. All deterministic digests and tool receipts in the short suite matched. The first integration test caught an early import that latched EXL3 flags before qserve set them; the final patch installs the HIP policy after the existing EXL3 import in QwenEngine. Both stock and opted-in server entry points load normally. nix build --no-link .#kyojin and .#kyojin.passthru.tests.cpu passed, and treefmt --fail-on-change packages/kyojin/package.nix passed. Final tested package /nix/store/77ixf3l3qvws9v0i46asrrf45mhgiq1m-kyojin-1.8.26. Host and further agent A/B tracked in nixos-config-pm9f.","created_at":"2026-11-18T18:16:42Z"}]}
{"id":"nix-overlay-j48","title":"Add purpleflow package","description":"Package PurpleFlow (PurpleSchool/purpleflow), an Electron orchestrator for claude/codex, from source. Repo: https://github.com/PurpleSchool/purpleflow","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-13T07:32:22.851076323Z","created_by":"minder","updated_at":"2026-11-13T07:40:08.109358347Z","closed_at":"2026-11-13T07:40:08.108248543Z","close_reason":"Completed: packages/purpleflow builds from source (nix build .#purpleflow -> /nix/store/ig3an708ycb4h9832m2965s60xc5a5ig-purpleflow-0.3.2); Electron app starts under Xvfb with no errors and creates its settings; README updated.","source_repo":"nix-overlay","compaction_level":0,"original_size":0}
{"id":"nix-overlay-ka5","title":"Add Strata with ROCm 10 HIP support for gfx1151","description":"Package Niko1221/Strata using the existing TheRock ROCm 10 SDK, compile HIP kernels for gfx1151, document usage, and verify build and CLI.","status":"closed","priority":2,"issue_type":"feature","assignee":"minder","created_at":"2026-11-12T18:13:13.558104256Z","created_by":"minder","updated_at":"2026-11-12T18:28:20.466366213Z","closed_at":"2026-11-12T18:28:20.465072110Z","close_reason":"Completed: Strata 0.2.13 builds with TheRock ROCm 10.0 for gfx1151; installed command checks, formatting, flake evaluation, HTTP health/models/OpenAI/Anthropic mock APIs pass. Upstream HIP intrinsics numerical parity passed on host Radeon 8060S. Engine starts with package runtime libraries (unset inherited LD_LIBRARY_PATH pointing to GCC 15). Full model inference not exercised.","source_repo":"nix-overlay","compaction_level":0,"original_size":0}
{"id":"nix-overlay-m5y","title":"Add CodeAF package (Agent-Field/CodeAF)","description":"Add https://github.com/Agent-Field/CodeAF as a new package under packages/codeaf with default.nix + package.nix, staged for blueprint discovery, README table updated, nix build verified.","status":"closed","priority":2,"issue_type":"feature","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-10-06T18:07:38Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-10-06T18:46:34Z","closed_at":"2026-10-06T18:46:34Z","close_reason":"Closed","source_repo":".","compaction_level":0,"original_size":0}
+23
View File
@@ -0,0 +1,23 @@
--- a/tools/qwen/serve.py
+++ b/tools/qwen/serve.py
@@ -826,6 +826,20 @@
# first token 81.3 / 84.1 s instead of 81.2 / 81.5 s
os.environ.setdefault("EXL3_CIL_DPRIO_CHUNK_S", "0.4")
from exllamav3 import Cache, Config, Generator, Job, Model, Tokenizer
+ if torch.version.hip and os.environ.get("KYOJIN_HIP_BLOCKING_SYNC") == "1":
+ import ctypes
+ from exllamav3.util.hip_lib import load_hip_runtime
+
+ hip = load_hip_runtime()
+ flags = ctypes.c_uint()
+ status = hip.hipGetDeviceFlags(ctypes.byref(flags))
+ if status:
+ raise RuntimeError(f"hipGetDeviceFlags failed with status {status}")
+ # Replace only hipDeviceScheduleMask (7) with hipDeviceScheduleBlockingSync (4).
+ status = hip.hipSetDeviceFlags(ctypes.c_uint((flags.value & ~7) | 4))
+ if status:
+ raise RuntimeError(f"hipSetDeviceFlags failed with status {status}")
+ print("qserve: HIP blocking host synchronization enabled", flush=True)
torch.set_grad_enabled(False)
self.torch, self.Generator, self.Job = torch, Generator, Job
self.sessions = sessions
+4 -1
View File
@@ -27,7 +27,10 @@ kyojin.overrideAttrs (old: {
# Upstream installs tools directly from $src, bypassing the unpacked tree.
src = pkgs.applyPatches {
src = old.src;
patches = [ ./short-reply-prefill-grace.patch ];
patches = [
./short-reply-prefill-grace.patch
./hip-blocking-sync.patch
];
};
nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ pkgs.makeWrapper ];
meta = (old.meta or { }) // {