diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 975a187..77d2df8 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -48,6 +48,7 @@ {"id":"nix-overlay-q00","title":"Add haivemind package","description":"Package https://github.com/dev-boz/haivemind in the Nix overlay, document it, validate the build, and complete repository workflow.","status":"closed","priority":2,"issue_type":"task","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-08-08T16:32:37Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-08-08T16:36:41Z","closed_at":"2026-08-08T16:36:41Z","close_reason":"Closed","source_repo":".","compaction_level":0,"original_size":0} {"id":"nix-overlay-q2m","title":"Add FreeToken package (FlashML-org/FreeToken)","description":"Package https://github.com/FlashML-org/FreeToken into the overlay following the standard packages/<name>/{default.nix,package.nix} pattern. Must build via nix build, README updated, category set.","notes":"Build in progress. Resolved: tvm-ffi 0.1.13.post3 builds (cython 3.3.0 vendored, dontUsePytestCheck due to xdist addopts), flashlib wheel + pythonRemoveDeps, freetoken pythonRelaxDeps gguf/modelscope. Remaining: torch-bin CUDA closure (nccl done; libnvshmem source build ~hours), then freetoken extensions. Note: torch-bin 2.11 = +cu128 PyPI wheel autoPatchelf'd to nixpkgs cuda12.9 libs.","status":"closed","priority":2,"issue_type":"feature","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-08-30T06:40:20Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-08-30T17:51:10Z","closed_at":"2026-08-30T17:51:10Z","close_reason":"freetoken 0.1.2 packaged: builds via nix build .#freetoken, ft CLI verified (ft --version), imports checked, RPATHs correct (cudart 12.9 + torch-bin libs). Docs in README + meta.longDescription.","source_repo":".","compaction_level":0,"original_size":0} {"id":"nix-overlay-qcg","title":"freetoken: bump 0.1.3 -> 0.2.0 once nixpkgs has torch 2.14 / transformers 5.18","description":"FreeToken 0.2.0 requires torch>=2.14,<2.15, torchvision>=0.29,<0.30, transformers>=5.18,<5.19, triton 3.8 and flashlib==0.4.0 (currently packaged: 0.3.0). nixos-unstable ships torch 2.11 and transformers 5.5.4, so the pin in flake.nix (nixpkgs-torch211) and packages/freetoken/package.nix must be reworked once a newer torch lands in nixpkgs.","status":"open","priority":3,"issue_type":"task","created_at":"2026-10-24T09:01:39.154532688Z","created_by":"minder","updated_at":"2026-10-24T09:01:39.154532688Z","source_repo":"nix-overlay","compaction_level":0,"original_size":0} +{"id":"nix-overlay-qfi","title":"Benchmark Qwen speculative depth and bounded prefill deferral during active decode","description":"On ms-s1-max compare draft depth costs, assess whether XDNA can prefill a second Qwen3.8 EXL3 request, and package a bounded scheduler in the overlay only if it improves the mixed workload with output checks.","status":"closed","priority":2,"issue_type":"task","assignee":"codex","created_at":"2026-11-18T11:51:09.731217906Z","created_by":"minder","updated_at":"2026-11-18T12:19:23.830501603Z","closed_at":"2026-11-18T12:19:23.830171714Z","close_reason":"Validated bounded prefill grace on two-slot Qwen3.8 EXL3 at 32K, 128K and 200K, with matching sampled texts for the tested pairs. Checked the NPU graph and memory limits and retained the measured faster stock draft-depth policy.","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"comments":[{"id":11,"issue_id":"nix-overlay-qfi","author":"minder","text":"Current experiment uses an isolated loopback model on ms-s1-max. Candidate source scope if justified: packages/kyojin. Also assess NPU-only prefill and cache/state handoff for a second request.","created_at":"2026-11-18T11:51:31Z"},{"id":12,"issue_id":"nix-overlay-qfi","author":"minder","text":"XDNA feasibility on ms-s1-max: gufo src/core/npu/qwen3_prefill.hpp is a fixed 28-layer dense Qwen3 with width 2048. hipfire docs/npu/npu-trunk.md dispatches only GdnIn/QsaIn; MoE, mixers and decode still require the GPU. This EXL3 pack has 48 fused MoE layers, 512 experts/layer, width 2560, intermediate 640, top-k 10. A separate INT8 copy of all routed gate/up/down weights is 112.5 GiB before other weights or caches. A native packed-weight or streaming prefill backend would need the full hybrid graph and compatible KV/indexer/recurrent/PLE handoff. Isolated single-panel timings and error are archived in /DATA/LLMs/benchmark_results/kyojin-xdna-20261118. No full-NPU throughput or end-to-end quality was measured.","created_at":"2026-11-18T12:11:43Z"},{"id":13,"issue_id":"nix-overlay-qfi","author":"minder","text":"Built the patched wrapper offline with nix build --offline --no-link .#kyojin; nixfmt --check and git diff --check passed.\nNew planner tests: 3 passed (positive, bypass, and fixed deadline subcases). Patched-source Qwen suite: 117 discovered, 116 passed with the packaged Python environment; the one optional chat-template test separately passed with the real model pack and a Transformers environment.\nA preliminary private server with the stock chunk planner and a client-side delay established the 32K hypothesis. The final server comparison used two package outputs with byte-identical serve.py and prefill_grace.py; the only scheduler difference was KYOJIN_PREFILL_GRACE_S=10 or 0. Two live slots, cold KV, identical model/requests/seeds, immediate second HTTP submission after 12 visible prose stream events.\n32768 tokens / scene 1: decode 13.91 -> 31.38 t/s, cold prefill 773.7 -> 1388.4 t/s, second TTFT from arrival 42.99 -> 33.59 s, short-reply event gap p95 526.9 -> 72.2 ms. Text hashes matched (story/long): True/True.\n32768 tokens / scene 0: decode 14.61 -> 31.32 t/s, cold prefill 790.4 -> 1385.3 t/s, second TTFT from arrival 41.98 -> 33.42 s, short-reply event gap p95 516.2 -> 72.0 ms. Text hashes matched (story/long): True/True.\n131072 tokens / scene 0: decode 16.07 -> 32.17 t/s, cold prefill 1166.6 -> 1347.4 t/s, second TTFT from arrival 112.95 -> 106.80 s, short-reply event gap p95 492.2 -> 69.4 ms. Text hashes matched (story/long): True/True.\n204800 tokens / scene 0: decode 15.67 -> 31.94 t/s, cold prefill 1208.1 -> 1321.0 t/s, second TTFT from arrival 170.24 -> 164.96 s, short-reply event gap p95 497.3 -> 69.9 ms. Text hashes matched (story/long): True/True.\nDraft-depth controller comparison on four other 192-token Russian prose prompts: tier 30.49, stock table 29.43, measured-cost table 29.77 t/s (medians). Only one of two 96-token greedy probes matched the tier output. Kept the shipped depth policy.\nReproducibility, raw timings, prompts/seeds and outputs' digest prefixes: /DATA/LLMs/benchmark_results/kyojin-prefill-grace-20261118/benchmark.json. The measured workload is an already-emitting, budget-bounded short reply plus a long, uncached incoming prompt. This is an opt-in scheduler; production service pin and settings were not changed.\n","created_at":"2026-11-18T12:19:23Z"}]} {"id":"nix-overlay-rij","title":"add codex-api-proxy package","description":"Package codex-api-proxy (PyPI, MIT) for the overlay","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-09T19:02:53.916339210Z","created_by":"minder","updated_at":"2026-11-09T19:08:12.028126119Z","closed_at":"2026-11-09T19:08:12.027132991Z","close_reason":"Completed: nix build .#codex-api-proxy passes (60 tests, 2 upstream-credential tests deselected); CLI --help and status run; README updated","source_repo":"nix-overlay","compaction_level":0,"original_size":0} {"id":"nix-overlay-v09","title":"Package workstation LLM engines from upstream flakes","description":"Migration coordinated by nixos-config-npn0. Move hipfire + TheRock kernels/toolchain, gufo NPU + XRT and patch, two llama.cpp forks, and upstream Kyojin v1.8.10 into this overlay. Replace old Kyojin 1.8.5 and its obsolete tests. Use nix-strix-halo as non-flake source so no Thunderbolt dependency or dummy remains. Preserve existing revisions and dedicated engine/gufo/kyojin nixpkgs. Current machine laptop-x13: evaluation only, no compilation and no source downloads. Plan: preserve package derivations where possible; update exports and README; offline evaluation; keep issue open until builds and runtime checks of migrated packages run on ms-s1-max. No push or deployment.","notes":"Partial workstation validation completed on ms-s1-max by nix-overlay-4ou (commit 16166f115b95): therock-rocm 10.1.0, hipfire-kernels and hipfire builds passed; packaged hipfire CLI help runs; SDK HIP vector addition and rocBLAS matmul pass on gfx1151; core runtime libraries load successfully. Strata, Crucible and Strixite also passed their builds, and the Strixite GPU check passed, but those packages do not cover the remaining migration gate. Still unverified here: Gufo/XRT, Kyojin and both llama.cpp fork builds; Kyojin JIT/CPU tests; consumer system build and hipfire LLVM tools check; model-based hipfire kernel matching/quantizers and Gufo GPU+NPU startup. Keep this issue in progress until its recorded acceptance checks are complete.","status":"in_progress","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-12T17:11:06.267689950Z","created_by":"minder","updated_at":"2026-11-12T20:43:28.894701490Z","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"comments":[{"id":4,"issue_id":"nix-overlay-v09","author":"minder","text":"Implementation and offline evaluation complete; keep open for workstation builds. All migrated pins (hipfire, gufo, Kyojin v1.8.10, both llama.cpp forks, nix-strix-halo, XRT and xdna-driver) and the engine nixpkgs pin match nixos-config before migration; all existing overlay pins are unchanged. nix-strix-halo is now flake=false, with direct TheRock SDK and XRT recipes, so Thunderbolt and its dummy are absent from both lock graphs. Kyojin replaces the old v1.8.5 nixpkgs torch/ROCm recipe with upstream v1.8.10 plus the existing CC wrapper. Upstream still labels its derivation 1.5.8; source rev d83cfeb3d6a7d42a4cf2b720efb1eb8a4ff04da7 is the requested v1.8.10. Old check-jit.py was removed with the old package; kyojin.tests.jit and tests.cpu now expose upstream HIP compiler and CPU checks. New package defaults share _lib/llm-packages.nix and defer unsupported platforms without loading their engine inputs. Exported packages: hipfire, hipfire-kernels, therock-rocm, gufo, xrt-amdxdna, kyojin, pwilkin-llama-cpp, strix-llama-cpp. Re-exported hipfire overlay and service module remain opt-in. README updated. Offline drvPath evaluation passed for all 11 packages/tool outputs with IFD disabled, plus integration assertions against both nixos-config consumers and the laptop-x13 toplevel. Gufo, XRT, Kyojin and both llama forks have identical drvPaths before/after; TheRock and hipfire now depend on genuine nixpkgs rdma-core instead of the dummy. Formatter and git diff --check passed. No compilation or source downloads. No push, commit or deployment. Temporary local consumer path must be replaced with the remote URL and a published lock pin after publication.","created_at":"2026-11-12T17:20:29Z"},{"id":5,"issue_id":"nix-overlay-v09","author":"minder","text":"Build gate: run only on ms-s1-max after hostname confirms the workstation. In nix-overlay: nix build --no-link .#hipfire .#hipfire-kernels .#therock-rocm .#gufo .#xrt-amdxdna .#kyojin .#pwilkin-llama-cpp .#strix-llama-cpp .#kyojin.tests.jit .#kyojin.tests.cpu. Then in nixos-config evaluate and build nixosConfigurations.ms-s1-max.config.system.build.toplevel, and run checks.x86_64-linux.hipfire-llvm-tools-check. Confirm Kyojin Triton JIT, hipfire precompiled-kernel matching/quantizers, and Gufo GPU+NPU startup against existing models. Particular risk to check: TheRock real rdma-core dependency replaces the previous dummy, so its and hipfire dependency paths change. Record evidence in this issue and nixos-config-npn0 before closure. Restore and verify the consumer remote overlay URL only after approved publication.","created_at":"2026-11-12T17:20:32Z"},{"id":6,"issue_id":"nix-overlay-v09","author":"minder","text":"Final verification update: normal Git-flake evaluation of all eight migrated package drvPaths passed; new files are indexed so Blueprint can see them. No commit created. Full git diff --check HEAD reports gufo-unsloth-ud3.patch:6 trailing whitespace: a required single-space prefix for an empty unified-diff context line. Patch bytes match the original nixos-config patch exactly, and git apply --check against the pinned Gufo source passes. Preserved the patch and did not suppress the checker. Earlier successful whitespace check covered the unstaged diff only.","created_at":"2026-11-12T17:22:21Z"},{"id":7,"issue_id":"nix-overlay-v09","author":"minder","text":"Astra code-review completed on gpt-6-astra medium: Spec has no findings; two Standards metadata findings fixed. Kyojin restores homepage and MIT license, confirmed from the cached upstream LICENSE. XRT shared producer declares mainProgram=xrt-smi, matching the workstation NPU prep command. Offline Nix assertions passed for all four metadata fields and the shared Gufo/XRT metadata. Kyojin drvPath is unchanged. XRT mainProgram causes nixpkgs to add NIX_MAIN_PROGRAM=xrt-smi to its derivation; independent derivation comparison found only that environment change (plus output paths), so Gufo changes transitively. Formatting passed. Preparing the user-requested migration commit; no compilation, source downloads, push or deployment. Existing whitespace warning is the unchanged unified-diff patch context. Keep this task open for the recorded workstation build/runtime gate. Pre-existing unrelated tracker change nix-overlay-y5r will remain outside this commit.","created_at":"2026-11-12T17:36:38Z"}]} {"id":"nix-overlay-vhq","title":"Note missing Dolt remote in AGENTS.md","description":"## Why\\nAgents running bd dolt push see 'No remote is configured - skipping' and may mistake it for a failure (blocking session close). Document that this is expected: no Dolt remote is set up, beads state is versioned locally in .beads/ and committed via git.\\n\\n## What\\nAdd a short note in the Beads Issue Tracker / Session Completion section of AGENTS.md.\\n\\n## Acceptance\\n- AGENTS.md explains the expected bd dolt push output and that git push is the actual sync mechanism","status":"closed","priority":3,"issue_type":"task","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-08-07T13:18:33Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-08-07T13:19:26Z","closed_at":"2026-08-07T13:19:26Z","close_reason":"AGENTS.md updated and pushed","source_repo":".","compaction_level":0,"original_size":0} diff --git a/packages/kyojin/package.nix b/packages/kyojin/package.nix index a3c7889..16803f7 100644 --- a/packages/kyojin/package.nix +++ b/packages/kyojin/package.nix @@ -5,6 +5,11 @@ cpuCheck, }: kyojin.overrideAttrs (old: { + # Upstream installs tools directly from $src, bypassing the unpacked tree. + src = pkgs.applyPatches { + src = old.src; + patches = [ ./short-reply-prefill-grace.patch ]; + }; nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ pkgs.makeWrapper ]; meta = (old.meta or { }) // { homepage = "https://github.com/Yamz-Labs/kyojin"; diff --git a/packages/kyojin/short-reply-prefill-grace.patch b/packages/kyojin/short-reply-prefill-grace.patch new file mode 100644 index 0000000..df15722 --- /dev/null +++ b/packages/kyojin/short-reply-prefill-grace.patch @@ -0,0 +1,116 @@ +--- a/tools/qwen/serve.py ++++ b/tools/qwen/serve.py +@@ -984,4 +984,11 @@ + thf, thv, maxd, spec_policy.ngram_from_env(), {}, + rule=spec_policy.rule_from_env(thf, thv, maxd), fixed=int(fx) if fx else None) ++ if self.sessions == 2: ++ grace = float(os.environ.get("KYOJIN_PREFILL_GRACE_S") or 0) ++ if not math.isfinite(grace) or grace < 0: ++ raise ValueError("KYOJIN_PREFILL_GRACE_S must be finite and nonnegative") ++ if grace: ++ from prefill_grace import ShortReplyPlanner ++ self.generator._cil = ShortReplyPlanner(self.generator.active_jobs, grace) + self.spec_on = spec + self.generator.pre_start_hook = self._pre_start +--- /dev/null ++++ b/tools/qwen/prefill_grace.py +@@ -0,0 +1,28 @@ ++"""Optionally let a short decode finish before starting a large uncached prefill.""" ++from exllamav3.generator.chunk_il import ChunkPlanner, Plan ++ ++ ++class ShortReplyPlanner(ChunkPlanner): ++ def __init__(self, jobs, grace, **kwargs): ++ super().__init__(**kwargs) ++ self.jobs = jobs ++ self.grace = grace ++ ++ def plan_prefill(self, views): ++ if self.dprio and len(views) == 2: ++ old, new = sorted(views, key=lambda view: view.order) ++ # The final prompt token is decoded. 'remaining' discounts cached pages ++ # and the recurrent tail checkpoint, including on the first iteration. ++ if old.done and not new.done and new.remaining >= 32768 - 1: ++ jobs = {id(job): job for job in self.jobs} ++ reply, prompt = jobs[old.key], jobs[new.key] ++ if (prompt.time_first_prefill is None and reply.new_tokens > 0 ++ and reply.max_new_tokens is not None ++ and 0 < reply.max_new_tokens - reply.new_tokens <= 320): ++ now = self._clock() ++ deadline = getattr(prompt, "_short_reply_deadline", None) ++ if deadline is None: ++ deadline = prompt._short_reply_deadline = now + self.grace ++ if now < deadline: ++ return Plan({new.key}, "shorts", new.key) ++ return super().plan_prefill(views) +--- a/tools/qwen/SERVE.md ++++ b/tools/qwen/SERVE.md +@@ -168,4 +168,11 @@ + save/restore/erase wait until the running requests finish. + ++With `--sessions 2` and decode-priority chunk interleaving, set `KYOJIN_PREFILL_GRACE_S=10` to give ++a running reply up to ten additional seconds to finish before a large prefill starts. The default is ++zero. The running reply must have started emitting tokens and have at most 320 tokens left in its ++generation budget. The new request must still need at least 32,767 prefill rows after prefix reuse ++(the last prompt token is handled by decode). When the reply finishes, the new prompt can run ++immediately. At the deadline the normal interleave policy resumes. ++ + Measured with `--ndt 3`, `-c 262144`, temperature 0, two short prompts sent together (four for `--sessions 4`): + +--- /dev/null ++++ b/tools/qwen/test_prefill_grace.py +@@ -0,0 +1,54 @@ ++import unittest ++from types import SimpleNamespace ++from unittest.mock import patch ++ ++from exllamav3.generator.chunk_il import ChunkPlanner, View ++from prefill_grace import ShortReplyPlanner ++ ++ ++class ShortReplyTests(unittest.TestCase): ++ def setUp(self): ++ self.reply = SimpleNamespace(new_tokens=12, max_new_tokens=332) ++ self.prompt = SimpleNamespace(time_first_prefill=None) ++ self.now = 80 ++ self.planner = ShortReplyPlanner([self.reply, self.prompt], 10, clock=lambda: self.now) ++ self.planner.dprio = True ++ self.old = View(id(self.reply), 1, 0, True, 0, 18) ++ self.new = View(id(self.prompt), 2, 32767, False, 4096, 0) ++ ++ def test_deadline_from_first_eligible_step(self): ++ views = [self.new, self.old] ++ with patch.object(ChunkPlanner, "plan_prefill", autospec=True) as normal: ++ for self.now in (80, 89.99): ++ plan = self.planner.plan_prefill(views) ++ self.assertEqual(plan.parked, {self.new.key}) ++ self.assertEqual(self.planner.plan_decode(views, plan), set()) ++ normal.assert_not_called() ++ for self.now in (90, 91): ++ self.assertIs(self.planner.plan_prefill(views), normal.return_value) ++ self.assertEqual(normal.call_count, 2) ++ ++ def test_only_older_decode_and_large_unstarted_prefill(self): ++ cases = ([self.new], [self.new, self.old._replace(order=3)], ++ [self.old, self.new._replace(remaining=32766)], ++ [self.old, self.new._replace(done=True, remaining=0)], ++ [self.old._replace(done=False), self.new]) ++ with patch.object(ChunkPlanner, "plan_prefill", autospec=True) as normal: ++ for views in cases: ++ with self.subTest(views=views): ++ self.assertIs(self.planner.plan_prefill(views), normal.return_value) ++ self.prompt.time_first_prefill = 75 ++ self.assertIs(self.planner.plan_prefill([self.old, self.new]), normal.return_value) ++ ++ def test_only_bounded_running_reply_in_decode_priority_mode(self): ++ for used, cap, dprio in ((0, 10, True), (12, 333, True), (12, None, True), ++ (12, 12, True), (12, 332, False)): ++ with self.subTest(used=used, cap=cap, dprio=dprio): ++ self.reply.new_tokens, self.reply.max_new_tokens = used, cap ++ self.planner.dprio = dprio ++ with patch.object(ChunkPlanner, "plan_prefill", autospec=True) as normal: ++ self.assertIs(self.planner.plan_prefill([self.old, self.new]), normal.return_value) ++ ++ ++if __name__ == "__main__": ++ unittest.main()