diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 8b71a26..0338542 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -45,6 +45,7 @@ {"id":"nix-overlay-j2g","title":"Allow the kyojin Qwen server to block the host thread while HIP executes","description":"Expose model-scoped opt-in hipDeviceScheduleBlockingSync in qwen/serve.py before model load, preserving non-scheduling flags and the existing default. On ms-s1-max default prefill and decode consume one CPU core in ROCr signal wait, verified by perf; verify the packaged path with real tool calls, CPU and latency.","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-18T18:03:00.852819489Z","created_by":"minder","updated_at":"2026-11-18T18:16:42.461414484Z","closed_at":"2026-11-18T18:16:42.460020486Z","close_reason":"Opt-in HIP blocking scheduling built and exercised on real Qwen agent requests. Arithmetic, sampler and default scheduling behavior are preserved.","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"labels":["ready-for-agent"],"comments":[{"id":16,"issue_id":"nix-overlay-j2g","author":"minder","text":"Package scope: packages/kyojin/package.nix plus a Qwen serving patch and tracker export. Default spin behavior stays available; re-use the package HIP library loader.","created_at":"2026-11-18T18:03:17Z"},{"id":17,"issue_id":"nix-overlay-j2g","author":"minder","text":"The default Qwen host thread accumulates roughly one CPU-second per second in decode and prefill. perf userspace: InterruptSignal/BusyWaitSignal wait functions account for 76.12% of the decode samples and 92.39% in the 32K-dominated request. With blocking sync, the actual packaged server used 16.3% of one CPU on a cold 32K request, down from 104.9%; repeated 384-token decode about 37% versus 108%. All deterministic digests and tool receipts in the short suite matched. The first integration test caught an early import that latched EXL3 flags before qserve set them; the final patch installs the HIP policy after the existing EXL3 import in QwenEngine. Both stock and opted-in server entry points load normally. nix build --no-link .#kyojin and .#kyojin.passthru.tests.cpu passed, and treefmt --fail-on-change packages/kyojin/package.nix passed. Final tested package /nix/store/77ixf3l3qvws9v0i46asrrf45mhgiq1m-kyojin-1.8.26. Host and further agent A/B tracked in nixos-config-pm9f.","created_at":"2026-11-18T18:16:42Z"}]} {"id":"nix-overlay-j48","title":"Add purpleflow package","description":"Package PurpleFlow (PurpleSchool/purpleflow), an Electron orchestrator for claude/codex, from source. Repo: https://github.com/PurpleSchool/purpleflow","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-13T07:32:22.851076323Z","created_by":"minder","updated_at":"2026-11-13T07:40:08.109358347Z","closed_at":"2026-11-13T07:40:08.108248543Z","close_reason":"Completed: packages/purpleflow builds from source (nix build .#purpleflow -> /nix/store/ig3an708ycb4h9832m2965s60xc5a5ig-purpleflow-0.3.2); Electron app starts under Xvfb with no errors and creates its settings; README updated.","source_repo":"nix-overlay","compaction_level":0,"original_size":0} {"id":"nix-overlay-ka5","title":"Add Strata with ROCm 10 HIP support for gfx1151","description":"Package Niko1221/Strata using the existing TheRock ROCm 10 SDK, compile HIP kernels for gfx1151, document usage, and verify build and CLI.","status":"closed","priority":2,"issue_type":"feature","assignee":"minder","created_at":"2026-11-12T18:13:13.558104256Z","created_by":"minder","updated_at":"2026-11-12T18:28:20.466366213Z","closed_at":"2026-11-12T18:28:20.465072110Z","close_reason":"Completed: Strata 0.2.13 builds with TheRock ROCm 10.0 for gfx1151; installed command checks, formatting, flake evaluation, HTTP health/models/OpenAI/Anthropic mock APIs pass. Upstream HIP intrinsics numerical parity passed on host Radeon 8060S. Engine starts with package runtime libraries (unset inherited LD_LIBRARY_PATH pointing to GCC 15). Full model inference not exercised.","source_repo":"nix-overlay","compaction_level":0,"original_size":0} +{"id":"nix-overlay-kzh","title":"Autoload a calibrated Kyojin HC sidecar from the model directory","description":"ms-s1-max / Qwen: use hc_gs_sidecar.safetensors by convention with a verified model-local checksum. Keep Yamz signature and unknown-file fallback, isolate persisted state by the sidecar selected, and qualify a real load of the 6.05 bpw file without a sidecar environment setting.","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-19T11:50:52.697232601Z","created_by":"minder","updated_at":"2026-11-19T11:58:32.264165296Z","closed_at":"2026-11-19T11:58:32.258339526Z","close_reason":"Canonical model-local HC calibration verified end to end","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"comments":[{"id":32,"issue_id":"nix-overlay-kzh","author":"minder","text":"Scope packages/kyojin/package.nix and one upstream server patch. Install the verified sidecar under the conventional name in the local 6.05 model folder, leaving model-specific env out of the test; run detector tests and a native private service check.","created_at":"2026-11-19T11:51:18Z"},{"id":33,"issue_id":"nix-overlay-kzh","author":"minder","text":"Built .#kyojin and evaluated and built ms-s1-max with a local input override. The old and new packages started privately with the same 6.05 bpw directory, 8192 ctx and one slot, with no external sidecar path or cache tag. Old: 0/97 sidecar sites (published SHA mismatch). New: 96 sidecar, 1 plain, 0 rejected. Separate disk namespaces, same public model ID. 6/6 tool calls and 6/6 structured responses correct. Qwen source suite in the ROCm environment: 120 tests, OK (1 skipped). nixfmt --check passed. Evidence: /DATA/LLMs/benchmark_results/kyojin-hc-calibration-20261119/autoload.json","created_at":"2026-11-19T11:58:32Z"}]} {"id":"nix-overlay-m5y","title":"Add CodeAF package (Agent-Field/CodeAF)","description":"Add https://github.com/Agent-Field/CodeAF as a new package under packages/codeaf with default.nix + package.nix, staged for blueprint discovery, README table updated, nix build verified.","status":"closed","priority":2,"issue_type":"feature","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-10-06T18:07:38Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-10-06T18:46:34Z","closed_at":"2026-10-06T18:46:34Z","close_reason":"Closed","source_repo":".","compaction_level":0,"original_size":0} {"id":"nix-overlay-nn2","title":"marmel: drop session-boot-trust-store.patch once upstream stops panicking on a missing CA store","description":"packages/marmel/session-boot-trust-store.patch forwards SSL_CERT_FILE into\nthe env_clear()d child in tests/test_session_boot.rs. It is a workaround,\nnot the fix: src/llm/client.rs default_http_client() calls unwrap_or_default()\non Result, and reqwest's Client::default() is\nClient::new(), which repeats the same build and .expect()s it, so the fallback\nalways panics. Reproduced independently of Nix on a stock busybox image with no\n/etc/ssl.\n\nUpstream issue filed 2026-10-16, still open:\nhttps://github.com/Na1w/marmel/issues/13\n(\"marmel panics at startup instead of reporting a missing CA trust store\")\n\nBlocked on upstream. Once upstream propagates the error instead of discarding\nit, drop the patch, bump marmel to the fixing commit, and rebuild.\n","status":"closed","priority":3,"issue_type":"task","owner":"alex@millerson.name","created_at":"2026-10-16T11:32:29Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-11-09T07:03:29.680285424Z","closed_at":"2026-11-09T07:03:29.673967963Z","close_reason":"Upstream fixed in 62cb756 (#16, 2026-10-18); session-boot-trust-store.patch was already dropped in adbed5f3c074. Current pin afcc1dc is 31 commits ahead of the fix and the package carries no such patch.","external_ref":"https://github.com/Na1w/marmel/issues/13","source_repo":".","compaction_level":0,"original_size":0} {"id":"nix-overlay-ocf","title":"agnix: ship agnix-lsp and agnix-mcp in the same package","description":"Extend packages/agnix to build -p agnix-cli -p agnix-lsp -p agnix-mcp, as upstream's release job does.","status":"closed","priority":3,"issue_type":"task","assignee":"minder","created_at":"2026-11-03T12:17:08.080999670Z","created_by":"minder","updated_at":"2026-11-03T12:33:21.279454904Z","closed_at":"2026-11-03T12:33:21.278951385Z","close_reason":"packages/agnix now builds -p agnix-cli -p agnix-lsp -p agnix-mcp; nix build .#agnix produces all three binaries; agnix-lsp answers LSP initialize with capabilities, agnix-mcp answers MCP initialize with serverInfo agnix 0.60.3","source_repo":"nix-overlay","compaction_level":0,"original_size":0} diff --git a/packages/kyojin/hc-sidecar-discovery.patch b/packages/kyojin/hc-sidecar-discovery.patch new file mode 100644 index 0000000..f3a3831 --- /dev/null +++ b/packages/kyojin/hc-sidecar-discovery.patch @@ -0,0 +1,125 @@ +--- a/tools/qwen/serve.py ++++ b/tools/qwen/serve.py +@@ -663,7 +663,7 @@ + CUDA_SERVE_ENV = (("EXL3_NC2_ASYNC_STASH", "1"), ("EXL3_STASH_SIDE", "1"), ("EXL3_PF_EMB_UPLOAD", "1"), ("EXL3_CUDA_GRAPH", "1"), ("EXL3_DRAFT_VOCAB", "98304"), ("QWSPEC_THF", "0.3"), ("QWSPEC_THV", "0.1"), ("QWSPEC_THFD", "0.6"), ("QWSPEC_THVD", "0.4")) + + HC_GS_SIDECAR = "hc_gs_sidecar.safetensors" +-# The sidecar of the published pack. A file with another hash (wrong pack, damaged download) is not used: plain group rounding, one log line. ++# Default checksum for packs without a local calibration. + HC_GS_SIDECAR_SHA256 = "473e7547adaad190788634556ce6b32614bdb280ccedadbed8789d39b59dcc8f" + + +@@ -676,22 +676,41 @@ + return h.hexdigest() + + +-def configure_hc_sidecar(model_path: str) -> str: +- """Point EXL3_GR_GS_SIDECAR at /hc_gs_sidecar.safetensors when group scales are on, the variable is unset and +- the file exists. Call before importing exllamav3 (the engine reads it at import). Returns the log line.""" ++def _configure_hc_sidecar(model_path: str) -> tuple[str, str | None]: ++ """Select the mixer codes before engine import. Keep the digest for saved KV state.""" ++ plain = "hc mixer: group scales, plain rounding" + if os.environ.get("EXL3_GR_GS", "0") == "0": +- return "hc mixer: per-row int8 scales (EXL3_GR_GS=0)" ++ return "hc mixer: per-row int8 scales (EXL3_GR_GS=0)", None + env = os.environ.get("EXL3_GR_GS_SIDECAR") + if env: +- return f"hc mixer: group scales, codes from {env} (EXL3_GR_GS_SIDECAR)" ++ try: ++ digest = _sha256_file(env) ++ except OSError: ++ digest = None # The engine already falls back to plain codes for unreadable overrides. ++ return f"hc mixer: group scales, codes from {env} (EXL3_GR_GS_SIDECAR)", digest ++ + f = Path(model_path) / HC_GS_SIDECAR +- if f.is_file(): +- got = _sha256_file(f) +- if got != HC_GS_SIDECAR_SHA256: +- return f"hc mixer: group scales, plain rounding ({f} has sha256 {got[:12]}, expected {HC_GS_SIDECAR_SHA256[:12]}: sidecar not used)" +- os.environ["EXL3_GR_GS_SIDECAR"] = str(f) +- return f"hc mixer: group scales, codes from {f}" +- return f"hc mixer: group scales, plain rounding (no {HC_GS_SIDECAR} in the model directory)" ++ if not f.is_file(): ++ return f"{plain} (no {HC_GS_SIDECAR} in the model directory)", None ++ checksum = f.with_suffix(f.suffix + ".sha256") ++ try: ++ expected = checksum.read_text(encoding="ascii").strip().lower() ++ except FileNotFoundError: ++ expected = HC_GS_SIDECAR_SHA256 ++ except (OSError, UnicodeError): ++ return f"{plain} (unreadable {checksum})", None ++ if not re.fullmatch(r"[0-9a-f]{64}", expected): ++ return f"{plain} (invalid SHA-256 in {checksum})", None ++ got = _sha256_file(f) ++ if got != expected: ++ return f"{plain} ({f} has sha256 {got[:12]}, expected {expected[:12]}: sidecar not used)", None ++ os.environ["EXL3_GR_GS_SIDECAR"] = str(f) ++ return f"hc mixer: group scales, codes from {f}", got ++ ++ ++def configure_hc_sidecar(model_path: str) -> str: ++ """Select mixer codes and return the diagnostic used by the standalone tools.""" ++ return _configure_hc_sidecar(model_path)[0] + + + DRAFT2S_IN_PACK = "draft2s.st" # safetensors layout, but not *.safetensors so the weight loader never indexes it +@@ -774,7 +793,8 @@ + continue + os.environ.setdefault(k, v) + check_latched_env() +- print("qserve:", configure_hc_sidecar(model_path), flush=True) ++ hc_message, self.hc_sidecar_digest = _configure_hc_sidecar(model_path) ++ print("qserve:", hc_message, flush=True) + print("qserve:", configure_draft2s(model_path), flush=True) + if on_cuda: + for k, v in CUDA_SERVE_ENV: +@@ -2373,12 +2393,13 @@ + env = {k: v for k, v in sorted(os.environ.items()) if k.startswith(("EXL3_", "MPW"))} + print(f"qserve: effective env {env}", flush=True) + from exllamav3.generator.slot_store import SlotStore ++ cache_model_id = args.model_id if not engine.hc_sidecar_digest else f"{args.model_id}:hc:{engine.hc_sidecar_digest}" + engine.slot_store = SlotStore(engine.generator, [c for c in (engine.cache, engine.draft_cache) if c is not None], +- args.slot_save_path, args.model_id) ++ args.slot_save_path, cache_model_id) + from exllamav3.util.disk_store import env_dir + if env_dir(): + from exllamav3.generator.prefix_disk import PagedPrefixDisk, weights_identity +- engine.prefix_disk = PagedPrefixDisk(engine.slot_store, model_id=args.model_id, extra=weights_identity(args.model)) ++ engine.prefix_disk = PagedPrefixDisk(engine.slot_store, model_id=cache_model_id, extra=weights_identity(args.model)) + print(f"qserve: prefix disk {engine.prefix_disk.disk.dir} ({len(engine.prefix_disk.pages)} pages, " + f"{len(engine.prefix_disk.stashes)} checkpoints, cap {engine.prefix_disk.disk.cap >> 30} GiB)", flush=True) + startup_health.stage("warm-up") +--- a/tools/qwen/test_serve.py ++++ b/tools/qwen/test_serve.py +@@ -930,6 +930,31 @@ + self.assertEqual(serve._partial_tag("abc", [""]), 0) + + ++class HCSidecarTests(unittest.TestCase): ++ def test_discovery_and_integrity(self): ++ import hashlib ++ import tempfile ++ with tempfile.TemporaryDirectory() as folder, unittest.mock.patch.dict(os.environ, {"EXL3_GR_GS": "1"}, clear=True): ++ file = Path(folder) / serve.HC_GS_SIDECAR ++ file.write_bytes(b"calibrated codes") ++ digest = hashlib.sha256(file.read_bytes()).hexdigest() ++ checksum = file.with_suffix(".safetensors.sha256") ++ for value in (None, "bad", "0" * 64): ++ if value is not None: ++ checksum.write_text(value) ++ self.assertIsNone(serve._configure_hc_sidecar(folder)[1]) ++ self.assertNotIn("EXL3_GR_GS_SIDECAR", os.environ) ++ checksum.write_text(digest + "\n") ++ self.assertEqual(serve._configure_hc_sidecar(folder)[1], digest) ++ self.assertEqual(os.environ.pop("EXL3_GR_GS_SIDECAR"), str(file)) ++ checksum.unlink() ++ with unittest.mock.patch.object(serve, "HC_GS_SIDECAR_SHA256", digest): ++ self.assertIn(str(file), serve.configure_hc_sidecar(folder)) ++ self.assertEqual(serve._configure_hc_sidecar(folder)[1], digest) # explicit path ++ os.environ["EXL3_GR_GS"] = "0" ++ self.assertIsNone(serve._configure_hc_sidecar(folder)[1]) ++ ++ + class ServeEnvDefaultsTest(unittest.TestCase): + def test_prefill_set_is_default_and_overridable(self): + d = dict(serve.SERVE_ENV) diff --git a/packages/kyojin/package.nix b/packages/kyojin/package.nix index 0a2615f..85260c1 100644 --- a/packages/kyojin/package.nix +++ b/packages/kyojin/package.nix @@ -36,6 +36,7 @@ kyojin.overrideAttrs (old: { ./short-reply-prefill-grace.patch ./hip-blocking-sync.patch ./finished-route.patch + ./hc-sidecar-discovery.patch ]; }; nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ pkgs.makeWrapper ];