fix(kyojin): load calibrated HC sidecars from the model directory
CI / check (push) Has been cancelled

Bead: nix-overlay-kzh
This commit is contained in:
2026-11-19 14:58:32 +03:00
parent 8389c88d04
commit 303eaa945e
3 changed files with 127 additions and 0 deletions
+1
View File
@@ -45,6 +45,7 @@
{"id":"nix-overlay-j2g","title":"Allow the kyojin Qwen server to block the host thread while HIP executes","description":"Expose model-scoped opt-in hipDeviceScheduleBlockingSync in qwen/serve.py before model load, preserving non-scheduling flags and the existing default. On ms-s1-max default prefill and decode consume one CPU core in ROCr signal wait, verified by perf; verify the packaged path with real tool calls, CPU and latency.","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-18T18:03:00.852819489Z","created_by":"minder","updated_at":"2026-11-18T18:16:42.461414484Z","closed_at":"2026-11-18T18:16:42.460020486Z","close_reason":"Opt-in HIP blocking scheduling built and exercised on real Qwen agent requests. Arithmetic, sampler and default scheduling behavior are preserved.","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"labels":["ready-for-agent"],"comments":[{"id":16,"issue_id":"nix-overlay-j2g","author":"minder","text":"Package scope: packages/kyojin/package.nix plus a Qwen serving patch and tracker export. Default spin behavior stays available; re-use the package HIP library loader.","created_at":"2026-11-18T18:03:17Z"},{"id":17,"issue_id":"nix-overlay-j2g","author":"minder","text":"The default Qwen host thread accumulates roughly one CPU-second per second in decode and prefill. perf userspace: InterruptSignal/BusyWaitSignal wait functions account for 76.12% of the decode samples and 92.39% in the 32K-dominated request. With blocking sync, the actual packaged server used 16.3% of one CPU on a cold 32K request, down from 104.9%; repeated 384-token decode about 37% versus 108%. All deterministic digests and tool receipts in the short suite matched. The first integration test caught an early import that latched EXL3 flags before qserve set them; the final patch installs the HIP policy after the existing EXL3 import in QwenEngine. Both stock and opted-in server entry points load normally. nix build --no-link .#kyojin and .#kyojin.passthru.tests.cpu passed, and treefmt --fail-on-change packages/kyojin/package.nix passed. Final tested package /nix/store/77ixf3l3qvws9v0i46asrrf45mhgiq1m-kyojin-1.8.26. Host and further agent A/B tracked in nixos-config-pm9f.","created_at":"2026-11-18T18:16:42Z"}]} {"id":"nix-overlay-j2g","title":"Allow the kyojin Qwen server to block the host thread while HIP executes","description":"Expose model-scoped opt-in hipDeviceScheduleBlockingSync in qwen/serve.py before model load, preserving non-scheduling flags and the existing default. On ms-s1-max default prefill and decode consume one CPU core in ROCr signal wait, verified by perf; verify the packaged path with real tool calls, CPU and latency.","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-18T18:03:00.852819489Z","created_by":"minder","updated_at":"2026-11-18T18:16:42.461414484Z","closed_at":"2026-11-18T18:16:42.460020486Z","close_reason":"Opt-in HIP blocking scheduling built and exercised on real Qwen agent requests. Arithmetic, sampler and default scheduling behavior are preserved.","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"labels":["ready-for-agent"],"comments":[{"id":16,"issue_id":"nix-overlay-j2g","author":"minder","text":"Package scope: packages/kyojin/package.nix plus a Qwen serving patch and tracker export. Default spin behavior stays available; re-use the package HIP library loader.","created_at":"2026-11-18T18:03:17Z"},{"id":17,"issue_id":"nix-overlay-j2g","author":"minder","text":"The default Qwen host thread accumulates roughly one CPU-second per second in decode and prefill. perf userspace: InterruptSignal/BusyWaitSignal wait functions account for 76.12% of the decode samples and 92.39% in the 32K-dominated request. With blocking sync, the actual packaged server used 16.3% of one CPU on a cold 32K request, down from 104.9%; repeated 384-token decode about 37% versus 108%. All deterministic digests and tool receipts in the short suite matched. The first integration test caught an early import that latched EXL3 flags before qserve set them; the final patch installs the HIP policy after the existing EXL3 import in QwenEngine. Both stock and opted-in server entry points load normally. nix build --no-link .#kyojin and .#kyojin.passthru.tests.cpu passed, and treefmt --fail-on-change packages/kyojin/package.nix passed. Final tested package /nix/store/77ixf3l3qvws9v0i46asrrf45mhgiq1m-kyojin-1.8.26. Host and further agent A/B tracked in nixos-config-pm9f.","created_at":"2026-11-18T18:16:42Z"}]}
{"id":"nix-overlay-j48","title":"Add purpleflow package","description":"Package PurpleFlow (PurpleSchool/purpleflow), an Electron orchestrator for claude/codex, from source. Repo: https://github.com/PurpleSchool/purpleflow","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-13T07:32:22.851076323Z","created_by":"minder","updated_at":"2026-11-13T07:40:08.109358347Z","closed_at":"2026-11-13T07:40:08.108248543Z","close_reason":"Completed: packages/purpleflow builds from source (nix build .#purpleflow -> /nix/store/ig3an708ycb4h9832m2965s60xc5a5ig-purpleflow-0.3.2); Electron app starts under Xvfb with no errors and creates its settings; README updated.","source_repo":"nix-overlay","compaction_level":0,"original_size":0} {"id":"nix-overlay-j48","title":"Add purpleflow package","description":"Package PurpleFlow (PurpleSchool/purpleflow), an Electron orchestrator for claude/codex, from source. Repo: https://github.com/PurpleSchool/purpleflow","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-13T07:32:22.851076323Z","created_by":"minder","updated_at":"2026-11-13T07:40:08.109358347Z","closed_at":"2026-11-13T07:40:08.108248543Z","close_reason":"Completed: packages/purpleflow builds from source (nix build .#purpleflow -> /nix/store/ig3an708ycb4h9832m2965s60xc5a5ig-purpleflow-0.3.2); Electron app starts under Xvfb with no errors and creates its settings; README updated.","source_repo":"nix-overlay","compaction_level":0,"original_size":0}
{"id":"nix-overlay-ka5","title":"Add Strata with ROCm 10 HIP support for gfx1151","description":"Package Niko1221/Strata using the existing TheRock ROCm 10 SDK, compile HIP kernels for gfx1151, document usage, and verify build and CLI.","status":"closed","priority":2,"issue_type":"feature","assignee":"minder","created_at":"2026-11-12T18:13:13.558104256Z","created_by":"minder","updated_at":"2026-11-12T18:28:20.466366213Z","closed_at":"2026-11-12T18:28:20.465072110Z","close_reason":"Completed: Strata 0.2.13 builds with TheRock ROCm 10.0 for gfx1151; installed command checks, formatting, flake evaluation, HTTP health/models/OpenAI/Anthropic mock APIs pass. Upstream HIP intrinsics numerical parity passed on host Radeon 8060S. Engine starts with package runtime libraries (unset inherited LD_LIBRARY_PATH pointing to GCC 15). Full model inference not exercised.","source_repo":"nix-overlay","compaction_level":0,"original_size":0} {"id":"nix-overlay-ka5","title":"Add Strata with ROCm 10 HIP support for gfx1151","description":"Package Niko1221/Strata using the existing TheRock ROCm 10 SDK, compile HIP kernels for gfx1151, document usage, and verify build and CLI.","status":"closed","priority":2,"issue_type":"feature","assignee":"minder","created_at":"2026-11-12T18:13:13.558104256Z","created_by":"minder","updated_at":"2026-11-12T18:28:20.466366213Z","closed_at":"2026-11-12T18:28:20.465072110Z","close_reason":"Completed: Strata 0.2.13 builds with TheRock ROCm 10.0 for gfx1151; installed command checks, formatting, flake evaluation, HTTP health/models/OpenAI/Anthropic mock APIs pass. Upstream HIP intrinsics numerical parity passed on host Radeon 8060S. Engine starts with package runtime libraries (unset inherited LD_LIBRARY_PATH pointing to GCC 15). Full model inference not exercised.","source_repo":"nix-overlay","compaction_level":0,"original_size":0}
{"id":"nix-overlay-kzh","title":"Autoload a calibrated Kyojin HC sidecar from the model directory","description":"ms-s1-max / Qwen: use hc_gs_sidecar.safetensors by convention with a verified model-local checksum. Keep Yamz signature and unknown-file fallback, isolate persisted state by the sidecar selected, and qualify a real load of the 6.05 bpw file without a sidecar environment setting.","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-19T11:50:52.697232601Z","created_by":"minder","updated_at":"2026-11-19T11:58:32.264165296Z","closed_at":"2026-11-19T11:58:32.258339526Z","close_reason":"Canonical model-local HC calibration verified end to end","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"comments":[{"id":32,"issue_id":"nix-overlay-kzh","author":"minder","text":"Scope packages/kyojin/package.nix and one upstream server patch. Install the verified sidecar under the conventional name in the local 6.05 model folder, leaving model-specific env out of the test; run detector tests and a native private service check.","created_at":"2026-11-19T11:51:18Z"},{"id":33,"issue_id":"nix-overlay-kzh","author":"minder","text":"Built .#kyojin and evaluated and built ms-s1-max with a local input override. The old and new packages started privately with the same 6.05 bpw directory, 8192 ctx and one slot, with no external sidecar path or cache tag. Old: 0/97 sidecar sites (published SHA mismatch). New: 96 sidecar, 1 plain, 0 rejected. Separate disk namespaces, same public model ID. 6/6 tool calls and 6/6 structured responses correct. Qwen source suite in the ROCm environment: 120 tests, OK (1 skipped). nixfmt --check passed. Evidence: /DATA/LLMs/benchmark_results/kyojin-hc-calibration-20261119/autoload.json","created_at":"2026-11-19T11:58:32Z"}]}
{"id":"nix-overlay-m5y","title":"Add CodeAF package (Agent-Field/CodeAF)","description":"Add https://github.com/Agent-Field/CodeAF as a new package under packages/codeaf with default.nix + package.nix, staged for blueprint discovery, README table updated, nix build verified.","status":"closed","priority":2,"issue_type":"feature","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-10-06T18:07:38Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-10-06T18:46:34Z","closed_at":"2026-10-06T18:46:34Z","close_reason":"Closed","source_repo":".","compaction_level":0,"original_size":0} {"id":"nix-overlay-m5y","title":"Add CodeAF package (Agent-Field/CodeAF)","description":"Add https://github.com/Agent-Field/CodeAF as a new package under packages/codeaf with default.nix + package.nix, staged for blueprint discovery, README table updated, nix build verified.","status":"closed","priority":2,"issue_type":"feature","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-10-06T18:07:38Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-10-06T18:46:34Z","closed_at":"2026-10-06T18:46:34Z","close_reason":"Closed","source_repo":".","compaction_level":0,"original_size":0}
{"id":"nix-overlay-nn2","title":"marmel: drop session-boot-trust-store.patch once upstream stops panicking on a missing CA store","description":"packages/marmel/session-boot-trust-store.patch forwards SSL_CERT_FILE into\nthe env_clear()d child in tests/test_session_boot.rs. It is a workaround,\nnot the fix: src/llm/client.rs default_http_client() calls unwrap_or_default()\non Result<Client, reqwest::Error>, and reqwest's Client::default() is\nClient::new(), which repeats the same build and .expect()s it, so the fallback\nalways panics. Reproduced independently of Nix on a stock busybox image with no\n/etc/ssl.\n\nUpstream issue filed 2026-10-16, still open:\nhttps://github.com/Na1w/marmel/issues/13\n(\"marmel panics at startup instead of reporting a missing CA trust store\")\n\nBlocked on upstream. Once upstream propagates the error instead of discarding\nit, drop the patch, bump marmel to the fixing commit, and rebuild.\n","status":"closed","priority":3,"issue_type":"task","owner":"alex@millerson.name","created_at":"2026-10-16T11:32:29Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-11-09T07:03:29.680285424Z","closed_at":"2026-11-09T07:03:29.673967963Z","close_reason":"Upstream fixed in 62cb756 (#16, 2026-10-18); session-boot-trust-store.patch was already dropped in adbed5f3c074. Current pin afcc1dc is 31 commits ahead of the fix and the package carries no such patch.","external_ref":"https://github.com/Na1w/marmel/issues/13","source_repo":".","compaction_level":0,"original_size":0} {"id":"nix-overlay-nn2","title":"marmel: drop session-boot-trust-store.patch once upstream stops panicking on a missing CA store","description":"packages/marmel/session-boot-trust-store.patch forwards SSL_CERT_FILE into\nthe env_clear()d child in tests/test_session_boot.rs. It is a workaround,\nnot the fix: src/llm/client.rs default_http_client() calls unwrap_or_default()\non Result<Client, reqwest::Error>, and reqwest's Client::default() is\nClient::new(), which repeats the same build and .expect()s it, so the fallback\nalways panics. Reproduced independently of Nix on a stock busybox image with no\n/etc/ssl.\n\nUpstream issue filed 2026-10-16, still open:\nhttps://github.com/Na1w/marmel/issues/13\n(\"marmel panics at startup instead of reporting a missing CA trust store\")\n\nBlocked on upstream. Once upstream propagates the error instead of discarding\nit, drop the patch, bump marmel to the fixing commit, and rebuild.\n","status":"closed","priority":3,"issue_type":"task","owner":"alex@millerson.name","created_at":"2026-10-16T11:32:29Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-11-09T07:03:29.680285424Z","closed_at":"2026-11-09T07:03:29.673967963Z","close_reason":"Upstream fixed in 62cb756 (#16, 2026-10-18); session-boot-trust-store.patch was already dropped in adbed5f3c074. Current pin afcc1dc is 31 commits ahead of the fix and the package carries no such patch.","external_ref":"https://github.com/Na1w/marmel/issues/13","source_repo":".","compaction_level":0,"original_size":0}
{"id":"nix-overlay-ocf","title":"agnix: ship agnix-lsp and agnix-mcp in the same package","description":"Extend packages/agnix to build -p agnix-cli -p agnix-lsp -p agnix-mcp, as upstream's release job does.","status":"closed","priority":3,"issue_type":"task","assignee":"minder","created_at":"2026-11-03T12:17:08.080999670Z","created_by":"minder","updated_at":"2026-11-03T12:33:21.279454904Z","closed_at":"2026-11-03T12:33:21.278951385Z","close_reason":"packages/agnix now builds -p agnix-cli -p agnix-lsp -p agnix-mcp; nix build .#agnix produces all three binaries; agnix-lsp answers LSP initialize with capabilities, agnix-mcp answers MCP initialize with serverInfo agnix 0.60.3","source_repo":"nix-overlay","compaction_level":0,"original_size":0} {"id":"nix-overlay-ocf","title":"agnix: ship agnix-lsp and agnix-mcp in the same package","description":"Extend packages/agnix to build -p agnix-cli -p agnix-lsp -p agnix-mcp, as upstream's release job does.","status":"closed","priority":3,"issue_type":"task","assignee":"minder","created_at":"2026-11-03T12:17:08.080999670Z","created_by":"minder","updated_at":"2026-11-03T12:33:21.279454904Z","closed_at":"2026-11-03T12:33:21.278951385Z","close_reason":"packages/agnix now builds -p agnix-cli -p agnix-lsp -p agnix-mcp; nix build .#agnix produces all three binaries; agnix-lsp answers LSP initialize with capabilities, agnix-mcp answers MCP initialize with serverInfo agnix 0.60.3","source_repo":"nix-overlay","compaction_level":0,"original_size":0}
+125
View File
@@ -0,0 +1,125 @@
--- a/tools/qwen/serve.py
+++ b/tools/qwen/serve.py
@@ -663,7 +663,7 @@
CUDA_SERVE_ENV = (("EXL3_NC2_ASYNC_STASH", "1"), ("EXL3_STASH_SIDE", "1"), ("EXL3_PF_EMB_UPLOAD", "1"), ("EXL3_CUDA_GRAPH", "1"), ("EXL3_DRAFT_VOCAB", "98304"), ("QWSPEC_THF", "0.3"), ("QWSPEC_THV", "0.1"), ("QWSPEC_THFD", "0.6"), ("QWSPEC_THVD", "0.4"))
HC_GS_SIDECAR = "hc_gs_sidecar.safetensors"
-# The sidecar of the published pack. A file with another hash (wrong pack, damaged download) is not used: plain group rounding, one log line.
+# Default checksum for packs without a local calibration.
HC_GS_SIDECAR_SHA256 = "473e7547adaad190788634556ce6b32614bdb280ccedadbed8789d39b59dcc8f"
@@ -676,22 +676,41 @@
return h.hexdigest()
-def configure_hc_sidecar(model_path: str) -> str:
- """Point EXL3_GR_GS_SIDECAR at <model_path>/hc_gs_sidecar.safetensors when group scales are on, the variable is unset and
- the file exists. Call before importing exllamav3 (the engine reads it at import). Returns the log line."""
+def _configure_hc_sidecar(model_path: str) -> tuple[str, str | None]:
+ """Select the mixer codes before engine import. Keep the digest for saved KV state."""
+ plain = "hc mixer: group scales, plain rounding"
if os.environ.get("EXL3_GR_GS", "0") == "0":
- return "hc mixer: per-row int8 scales (EXL3_GR_GS=0)"
+ return "hc mixer: per-row int8 scales (EXL3_GR_GS=0)", None
env = os.environ.get("EXL3_GR_GS_SIDECAR")
if env:
- return f"hc mixer: group scales, codes from {env} (EXL3_GR_GS_SIDECAR)"
+ try:
+ digest = _sha256_file(env)
+ except OSError:
+ digest = None # The engine already falls back to plain codes for unreadable overrides.
+ return f"hc mixer: group scales, codes from {env} (EXL3_GR_GS_SIDECAR)", digest
+
f = Path(model_path) / HC_GS_SIDECAR
- if f.is_file():
- got = _sha256_file(f)
- if got != HC_GS_SIDECAR_SHA256:
- return f"hc mixer: group scales, plain rounding ({f} has sha256 {got[:12]}, expected {HC_GS_SIDECAR_SHA256[:12]}: sidecar not used)"
- os.environ["EXL3_GR_GS_SIDECAR"] = str(f)
- return f"hc mixer: group scales, codes from {f}"
- return f"hc mixer: group scales, plain rounding (no {HC_GS_SIDECAR} in the model directory)"
+ if not f.is_file():
+ return f"{plain} (no {HC_GS_SIDECAR} in the model directory)", None
+ checksum = f.with_suffix(f.suffix + ".sha256")
+ try:
+ expected = checksum.read_text(encoding="ascii").strip().lower()
+ except FileNotFoundError:
+ expected = HC_GS_SIDECAR_SHA256
+ except (OSError, UnicodeError):
+ return f"{plain} (unreadable {checksum})", None
+ if not re.fullmatch(r"[0-9a-f]{64}", expected):
+ return f"{plain} (invalid SHA-256 in {checksum})", None
+ got = _sha256_file(f)
+ if got != expected:
+ return f"{plain} ({f} has sha256 {got[:12]}, expected {expected[:12]}: sidecar not used)", None
+ os.environ["EXL3_GR_GS_SIDECAR"] = str(f)
+ return f"hc mixer: group scales, codes from {f}", got
+
+
+def configure_hc_sidecar(model_path: str) -> str:
+ """Select mixer codes and return the diagnostic used by the standalone tools."""
+ return _configure_hc_sidecar(model_path)[0]
DRAFT2S_IN_PACK = "draft2s.st" # safetensors layout, but not *.safetensors so the weight loader never indexes it
@@ -774,7 +793,8 @@
continue
os.environ.setdefault(k, v)
check_latched_env()
- print("qserve:", configure_hc_sidecar(model_path), flush=True)
+ hc_message, self.hc_sidecar_digest = _configure_hc_sidecar(model_path)
+ print("qserve:", hc_message, flush=True)
print("qserve:", configure_draft2s(model_path), flush=True)
if on_cuda:
for k, v in CUDA_SERVE_ENV:
@@ -2373,12 +2393,13 @@
env = {k: v for k, v in sorted(os.environ.items()) if k.startswith(("EXL3_", "MPW"))}
print(f"qserve: effective env {env}", flush=True)
from exllamav3.generator.slot_store import SlotStore
+ cache_model_id = args.model_id if not engine.hc_sidecar_digest else f"{args.model_id}:hc:{engine.hc_sidecar_digest}"
engine.slot_store = SlotStore(engine.generator, [c for c in (engine.cache, engine.draft_cache) if c is not None],
- args.slot_save_path, args.model_id)
+ args.slot_save_path, cache_model_id)
from exllamav3.util.disk_store import env_dir
if env_dir():
from exllamav3.generator.prefix_disk import PagedPrefixDisk, weights_identity
- engine.prefix_disk = PagedPrefixDisk(engine.slot_store, model_id=args.model_id, extra=weights_identity(args.model))
+ engine.prefix_disk = PagedPrefixDisk(engine.slot_store, model_id=cache_model_id, extra=weights_identity(args.model))
print(f"qserve: prefix disk {engine.prefix_disk.disk.dir} ({len(engine.prefix_disk.pages)} pages, "
f"{len(engine.prefix_disk.stashes)} checkpoints, cap {engine.prefix_disk.disk.cap >> 30} GiB)", flush=True)
startup_health.stage("warm-up")
--- a/tools/qwen/test_serve.py
+++ b/tools/qwen/test_serve.py
@@ -930,6 +930,31 @@
self.assertEqual(serve._partial_tag("abc", ["</think>"]), 0)
+class HCSidecarTests(unittest.TestCase):
+ def test_discovery_and_integrity(self):
+ import hashlib
+ import tempfile
+ with tempfile.TemporaryDirectory() as folder, unittest.mock.patch.dict(os.environ, {"EXL3_GR_GS": "1"}, clear=True):
+ file = Path(folder) / serve.HC_GS_SIDECAR
+ file.write_bytes(b"calibrated codes")
+ digest = hashlib.sha256(file.read_bytes()).hexdigest()
+ checksum = file.with_suffix(".safetensors.sha256")
+ for value in (None, "bad", "0" * 64):
+ if value is not None:
+ checksum.write_text(value)
+ self.assertIsNone(serve._configure_hc_sidecar(folder)[1])
+ self.assertNotIn("EXL3_GR_GS_SIDECAR", os.environ)
+ checksum.write_text(digest + "\n")
+ self.assertEqual(serve._configure_hc_sidecar(folder)[1], digest)
+ self.assertEqual(os.environ.pop("EXL3_GR_GS_SIDECAR"), str(file))
+ checksum.unlink()
+ with unittest.mock.patch.object(serve, "HC_GS_SIDECAR_SHA256", digest):
+ self.assertIn(str(file), serve.configure_hc_sidecar(folder))
+ self.assertEqual(serve._configure_hc_sidecar(folder)[1], digest) # explicit path
+ os.environ["EXL3_GR_GS"] = "0"
+ self.assertIsNone(serve._configure_hc_sidecar(folder)[1])
+
+
class ServeEnvDefaultsTest(unittest.TestCase):
def test_prefill_set_is_default_and_overridable(self):
d = dict(serve.SERVE_ENV)
+1
View File
@@ -36,6 +36,7 @@ kyojin.overrideAttrs (old: {
./short-reply-prefill-grace.patch ./short-reply-prefill-grace.patch
./hip-blocking-sync.patch ./hip-blocking-sync.patch
./finished-route.patch ./finished-route.patch
./hc-sidecar-discovery.patch
]; ];
}; };
nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ pkgs.makeWrapper ]; nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ pkgs.makeWrapper ];