diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 810ea7d..ca5a5e5 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -49,6 +49,7 @@ {"id":"nix-overlay-q2m","title":"Add FreeToken package (FlashML-org/FreeToken)","description":"Package https://github.com/FlashML-org/FreeToken into the overlay following the standard packages/<name>/{default.nix,package.nix} pattern. Must build via nix build, README updated, category set.","notes":"Build in progress. Resolved: tvm-ffi 0.1.13.post3 builds (cython 3.3.0 vendored, dontUsePytestCheck due to xdist addopts), flashlib wheel + pythonRemoveDeps, freetoken pythonRelaxDeps gguf/modelscope. Remaining: torch-bin CUDA closure (nccl done; libnvshmem source build ~hours), then freetoken extensions. Note: torch-bin 2.11 = +cu128 PyPI wheel autoPatchelf'd to nixpkgs cuda12.9 libs.","status":"closed","priority":2,"issue_type":"feature","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-08-30T06:40:20Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-08-30T17:51:10Z","closed_at":"2026-08-30T17:51:10Z","close_reason":"freetoken 0.1.2 packaged: builds via nix build .#freetoken, ft CLI verified (ft --version), imports checked, RPATHs correct (cudart 12.9 + torch-bin libs). Docs in README + meta.longDescription.","source_repo":".","compaction_level":0,"original_size":0} {"id":"nix-overlay-qcg","title":"freetoken: bump 0.1.3 -> 0.2.0 once nixpkgs has torch 2.14 / transformers 5.18","description":"FreeToken 0.2.0 requires torch>=2.14,<2.15, torchvision>=0.29,<0.30, transformers>=5.18,<5.19, triton 3.8 and flashlib==0.4.0 (currently packaged: 0.3.0). nixos-unstable ships torch 2.11 and transformers 5.5.4, so the pin in flake.nix (nixpkgs-torch211) and packages/freetoken/package.nix must be reworked once a newer torch lands in nixpkgs.","status":"open","priority":3,"issue_type":"task","created_at":"2026-10-24T09:01:39.154532688Z","created_by":"minder","updated_at":"2026-10-24T09:01:39.154532688Z","source_repo":"nix-overlay","compaction_level":0,"original_size":0} {"id":"nix-overlay-qfi","title":"Benchmark Qwen speculative depth and bounded prefill deferral during active decode","description":"On ms-s1-max compare draft depth costs, assess whether XDNA can prefill a second Qwen3.8 EXL3 request, and package a bounded scheduler in the overlay only if it improves the mixed workload with output checks.","status":"closed","priority":2,"issue_type":"task","assignee":"codex","created_at":"2026-11-18T11:51:09.731217906Z","created_by":"minder","updated_at":"2026-11-18T12:19:23.830501603Z","closed_at":"2026-11-18T12:19:23.830171714Z","close_reason":"Validated bounded prefill grace on two-slot Qwen3.8 EXL3 at 32K, 128K and 200K, with matching sampled texts for the tested pairs. Checked the NPU graph and memory limits and retained the measured faster stock draft-depth policy.","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"comments":[{"id":11,"issue_id":"nix-overlay-qfi","author":"minder","text":"Current experiment uses an isolated loopback model on ms-s1-max. Candidate source scope if justified: packages/kyojin. Also assess NPU-only prefill and cache/state handoff for a second request.","created_at":"2026-11-18T11:51:31Z"},{"id":12,"issue_id":"nix-overlay-qfi","author":"minder","text":"XDNA feasibility on ms-s1-max: gufo src/core/npu/qwen3_prefill.hpp is a fixed 28-layer dense Qwen3 with width 2048. hipfire docs/npu/npu-trunk.md dispatches only GdnIn/QsaIn; MoE, mixers and decode still require the GPU. This EXL3 pack has 48 fused MoE layers, 512 experts/layer, width 2560, intermediate 640, top-k 10. A separate INT8 copy of all routed gate/up/down weights is 112.5 GiB before other weights or caches. A native packed-weight or streaming prefill backend would need the full hybrid graph and compatible KV/indexer/recurrent/PLE handoff. Isolated single-panel timings and error are archived in /DATA/LLMs/benchmark_results/kyojin-xdna-20261118. No full-NPU throughput or end-to-end quality was measured.","created_at":"2026-11-18T12:11:43Z"},{"id":13,"issue_id":"nix-overlay-qfi","author":"minder","text":"Built the patched wrapper offline with nix build --offline --no-link .#kyojin; nixfmt --check and git diff --check passed.\nNew planner tests: 3 passed (positive, bypass, and fixed deadline subcases). Patched-source Qwen suite: 117 discovered, 116 passed with the packaged Python environment; the one optional chat-template test separately passed with the real model pack and a Transformers environment.\nA preliminary private server with the stock chunk planner and a client-side delay established the 32K hypothesis. The final server comparison used two package outputs with byte-identical serve.py and prefill_grace.py; the only scheduler difference was KYOJIN_PREFILL_GRACE_S=10 or 0. Two live slots, cold KV, identical model/requests/seeds, immediate second HTTP submission after 12 visible prose stream events.\n32768 tokens / scene 1: decode 13.91 -> 31.38 t/s, cold prefill 773.7 -> 1388.4 t/s, second TTFT from arrival 42.99 -> 33.59 s, short-reply event gap p95 526.9 -> 72.2 ms. Text hashes matched (story/long): True/True.\n32768 tokens / scene 0: decode 14.61 -> 31.32 t/s, cold prefill 790.4 -> 1385.3 t/s, second TTFT from arrival 41.98 -> 33.42 s, short-reply event gap p95 516.2 -> 72.0 ms. Text hashes matched (story/long): True/True.\n131072 tokens / scene 0: decode 16.07 -> 32.17 t/s, cold prefill 1166.6 -> 1347.4 t/s, second TTFT from arrival 112.95 -> 106.80 s, short-reply event gap p95 492.2 -> 69.4 ms. Text hashes matched (story/long): True/True.\n204800 tokens / scene 0: decode 15.67 -> 31.94 t/s, cold prefill 1208.1 -> 1321.0 t/s, second TTFT from arrival 170.24 -> 164.96 s, short-reply event gap p95 497.3 -> 69.9 ms. Text hashes matched (story/long): True/True.\nDraft-depth controller comparison on four other 192-token Russian prose prompts: tier 30.49, stock table 29.43, measured-cost table 29.77 t/s (medians). Only one of two 96-token greedy probes matched the tier output. Kept the shipped depth policy.\nReproducibility, raw timings, prompts/seeds and outputs' digest prefixes: /DATA/LLMs/benchmark_results/kyojin-prefill-grace-20261118/benchmark.json. The measured workload is an already-emitting, budget-bounded short reply plus a long, uncached incoming prompt. This is an opt-in scheduler; production service pin and settings were not changed.\n","created_at":"2026-11-18T12:19:23Z"}]} +{"id":"nix-overlay-qy9","title":"Stop direct expert reads at a short final block","description":"Kyojin 1.8.26 test_promote and real-file trace on ms-s1-max show os.preadv retries 3718 bytes at offset 3195258 (exact unaligned EOF) after a 4474-byte final read. Preserve retries for short reads before EOF. Verify the existing expert-store regression and full CPU suite on this host. Discovered during the prefix cache work nix-overlay-tb3.","status":"closed","priority":2,"issue_type":"bug","assignee":"minder","created_at":"2026-11-18T15:32:18.507565731Z","created_by":"minder","updated_at":"2026-11-18T15:34:41.955069008Z","closed_at":"2026-11-18T15:34:41.954806423Z","close_reason":"Verified the direct O_DIRECT reader on bcachefs including a premature short read and unaligned EOF.","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"comments":[{"id":15,"issue_id":"nix-overlay-qy9","author":"minder","text":"Stock and prefix-only builds both failed tests/test_expert_store_protect_cpu.py::test_promote on bcachefs. Traced 4474 bytes of 8192 then EINVAL at offset 3195258 equal to file length. Fixed Nix store wheel also passes a forced aligned short read followed by unaligned EOF (5773/5773 bytes match), and a normal 4096-byte read with no fstat. Full patched CPU suite 536 passed, 189 skipped, 349 subtests passed; HIP JIT smoke, package build, formatting and host registry eval passed.","created_at":"2026-11-18T15:34:41Z"}]} {"id":"nix-overlay-rij","title":"add codex-api-proxy package","description":"Package codex-api-proxy (PyPI, MIT) for the overlay","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-09T19:02:53.916339210Z","created_by":"minder","updated_at":"2026-11-09T19:08:12.028126119Z","closed_at":"2026-11-09T19:08:12.027132991Z","close_reason":"Completed: nix build .#codex-api-proxy passes (60 tests, 2 upstream-credential tests deselected); CLI --help and status run; README updated","source_repo":"nix-overlay","compaction_level":0,"original_size":0} {"id":"nix-overlay-tb3","title":"Skip unretainable Kyojin prefix segments before GPU staging","description":"Implement early size/space admission for paged prefix snapshots on the packaged Kyojin import path. Preserve the writer's final exact checks, LRU and cache format; compare rejected writes, restore and tool correctness on ms-s1-max.","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-18T15:16:58.686700563Z","created_by":"minder","updated_at":"2026-11-18T15:31:54.294238067Z","closed_at":"2026-11-18T15:31:54.293324084Z","close_reason":"Prefix admission fixed and verified on ms-s1-max for rejected and accepted snapshots.","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"comments":[{"id":14,"issue_id":"nix-overlay-tb3","author":"minder","text":"Built kyojin and HIP JIT. Focused DiskStore tests 11/11 (same tests on stock: 3 red). Isolated same-seed 128K A / 147709-token B / A tool return: B .tmp reached 4485646592 bytes on stock, absent with patch; A write_bytes 4447.73 -> 0 MiB, TTFT 12.015 -> 1.836 s (single pair). Accepted 32K cache subsequently restored 32872/32873 tokens, TTFT 1.455 s; all tool validations passed. Host registry resolves the tested binary with a local flake input override. Upstream CPU test_promote also fails on stock: O_DIRECT retry exactly at unaligned EOF; independent of cache admission. Evidence: /DATA/LLMs/benchmark_results/kyojin-prefix-admission-20261118/report.json","created_at":"2026-11-18T15:31:54Z"}]} {"id":"nix-overlay-v09","title":"Package workstation LLM engines from upstream flakes","description":"Migration coordinated by nixos-config-npn0. Move hipfire + TheRock kernels/toolchain, gufo NPU + XRT and patch, two llama.cpp forks, and upstream Kyojin v1.8.10 into this overlay. Replace old Kyojin 1.8.5 and its obsolete tests. Use nix-strix-halo as non-flake source so no Thunderbolt dependency or dummy remains. Preserve existing revisions and dedicated engine/gufo/kyojin nixpkgs. Current machine laptop-x13: evaluation only, no compilation and no source downloads. Plan: preserve package derivations where possible; update exports and README; offline evaluation; keep issue open until builds and runtime checks of migrated packages run on ms-s1-max. No push or deployment.","notes":"Partial workstation validation completed on ms-s1-max by nix-overlay-4ou (commit 16166f115b95): therock-rocm 10.1.0, hipfire-kernels and hipfire builds passed; packaged hipfire CLI help runs; SDK HIP vector addition and rocBLAS matmul pass on gfx1151; core runtime libraries load successfully. Strata, Crucible and Strixite also passed their builds, and the Strixite GPU check passed, but those packages do not cover the remaining migration gate. Still unverified here: Gufo/XRT, Kyojin and both llama.cpp fork builds; Kyojin JIT/CPU tests; consumer system build and hipfire LLVM tools check; model-based hipfire kernel matching/quantizers and Gufo GPU+NPU startup. Keep this issue in progress until its recorded acceptance checks are complete.","status":"in_progress","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-12T17:11:06.267689950Z","created_by":"minder","updated_at":"2026-11-12T20:43:28.894701490Z","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"comments":[{"id":4,"issue_id":"nix-overlay-v09","author":"minder","text":"Implementation and offline evaluation complete; keep open for workstation builds. All migrated pins (hipfire, gufo, Kyojin v1.8.10, both llama.cpp forks, nix-strix-halo, XRT and xdna-driver) and the engine nixpkgs pin match nixos-config before migration; all existing overlay pins are unchanged. nix-strix-halo is now flake=false, with direct TheRock SDK and XRT recipes, so Thunderbolt and its dummy are absent from both lock graphs. Kyojin replaces the old v1.8.5 nixpkgs torch/ROCm recipe with upstream v1.8.10 plus the existing CC wrapper. Upstream still labels its derivation 1.5.8; source rev d83cfeb3d6a7d42a4cf2b720efb1eb8a4ff04da7 is the requested v1.8.10. Old check-jit.py was removed with the old package; kyojin.tests.jit and tests.cpu now expose upstream HIP compiler and CPU checks. New package defaults share _lib/llm-packages.nix and defer unsupported platforms without loading their engine inputs. Exported packages: hipfire, hipfire-kernels, therock-rocm, gufo, xrt-amdxdna, kyojin, pwilkin-llama-cpp, strix-llama-cpp. Re-exported hipfire overlay and service module remain opt-in. README updated. Offline drvPath evaluation passed for all 11 packages/tool outputs with IFD disabled, plus integration assertions against both nixos-config consumers and the laptop-x13 toplevel. Gufo, XRT, Kyojin and both llama forks have identical drvPaths before/after; TheRock and hipfire now depend on genuine nixpkgs rdma-core instead of the dummy. Formatter and git diff --check passed. No compilation or source downloads. No push, commit or deployment. Temporary local consumer path must be replaced with the remote URL and a published lock pin after publication.","created_at":"2026-11-12T17:20:29Z"},{"id":5,"issue_id":"nix-overlay-v09","author":"minder","text":"Build gate: run only on ms-s1-max after hostname confirms the workstation. In nix-overlay: nix build --no-link .#hipfire .#hipfire-kernels .#therock-rocm .#gufo .#xrt-amdxdna .#kyojin .#pwilkin-llama-cpp .#strix-llama-cpp .#kyojin.tests.jit .#kyojin.tests.cpu. Then in nixos-config evaluate and build nixosConfigurations.ms-s1-max.config.system.build.toplevel, and run checks.x86_64-linux.hipfire-llvm-tools-check. Confirm Kyojin Triton JIT, hipfire precompiled-kernel matching/quantizers, and Gufo GPU+NPU startup against existing models. Particular risk to check: TheRock real rdma-core dependency replaces the previous dummy, so its and hipfire dependency paths change. Record evidence in this issue and nixos-config-npn0 before closure. Restore and verify the consumer remote overlay URL only after approved publication.","created_at":"2026-11-12T17:20:32Z"},{"id":6,"issue_id":"nix-overlay-v09","author":"minder","text":"Final verification update: normal Git-flake evaluation of all eight migrated package drvPaths passed; new files are indexed so Blueprint can see them. No commit created. Full git diff --check HEAD reports gufo-unsloth-ud3.patch:6 trailing whitespace: a required single-space prefix for an empty unified-diff context line. Patch bytes match the original nixos-config patch exactly, and git apply --check against the pinned Gufo source passes. Preserved the patch and did not suppress the checker. Earlier successful whitespace check covered the unstaged diff only.","created_at":"2026-11-12T17:22:21Z"},{"id":7,"issue_id":"nix-overlay-v09","author":"minder","text":"Astra code-review completed on gpt-6-astra medium: Spec has no findings; two Standards metadata findings fixed. Kyojin restores homepage and MIT license, confirmed from the cached upstream LICENSE. XRT shared producer declares mainProgram=xrt-smi, matching the workstation NPU prep command. Offline Nix assertions passed for all four metadata fields and the shared Gufo/XRT metadata. Kyojin drvPath is unchanged. XRT mainProgram causes nixpkgs to add NIX_MAIN_PROGRAM=xrt-smi to its derivation; independent derivation comparison found only that environment change (plus output paths), so Gufo changes transitively. Formatting passed. Preparing the user-requested migration commit; no compilation, source downloads, push or deployment. Existing whitespace warning is the unchanged unified-diff patch context. Keep this task open for the recorded workstation build/runtime gate. Pre-existing unrelated tracker change nix-overlay-y5r will remain outside this commit.","created_at":"2026-11-12T17:36:38Z"}]} diff --git a/packages/kyojin/expert-direct-read-eof.patch b/packages/kyojin/expert-direct-read-eof.patch new file mode 100644 index 0000000..681502d --- /dev/null +++ b/packages/kyojin/expert-direct-read-eof.patch @@ -0,0 +1,12 @@ +--- a/exllamav3/modules/expert_store.py ++++ b/exllamav3/modules/expert_store.py +@@ -1182,6 +1182,9 @@ + if r <= 0: + break # EOF inside the last block: the copies only touch real tensor bytes + got += r ++ # An unaligned EOF cannot be used as the offset for another O_DIRECT read. ++ if got < n and off + got == os.fstat(fd).st_size: ++ break + return got + + def _publish(self, lt: LayerTier, item, e: int, s: int): diff --git a/packages/kyojin/package.nix b/packages/kyojin/package.nix index adb4d16..4c7530f 100644 --- a/packages/kyojin/package.nix +++ b/packages/kyojin/package.nix @@ -6,9 +6,12 @@ cpuCheck, }: let - admissionPatch = ./prefix-disk-admission.patch; + enginePatches = [ + ./prefix-disk-admission.patch + ./expert-direct-read-eof.patch + ]; engine = exllamav3.overrideAttrs (old: { - patches = (old.patches or [ ]) ++ [ admissionPatch ]; + patches = (old.patches or [ ]) ++ enginePatches; }); # Both the upstream serve wrappers and the CPU check prepend the wheel to PYTHONPATH. withEngine = @@ -36,7 +39,7 @@ kyojin.overrideAttrs (old: { tests = (old.passthru.tests or { }) // { jit = jitCheck; cpu = cpuCheck.overrideAttrs (check: { - patches = (check.patches or [ ]) ++ [ admissionPatch ]; + patches = (check.patches or [ ]) ++ enginePatches; checkPhase = withEngine check.checkPhase; }); };