diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 6f1447c..b8100cd 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,5 +1,6 @@ -{"_type":"issue","id":"nix-overlay-dv3","title":"Fix Config screen crash on F2","description":"F2 opens ConfigScreen then Textual raises InvalidSelectValueError because _refresh_aggregator_dropdown assigns Select.BLANK, resolved as False, instead of clearing selection.\\n\\nAcceptance: Config screen opens with saved consensus configuration; regression covers no selected aggregator; targeted tests pass.","status":"in_progress","priority":1,"issue_type":"bug","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-08-20T11:34:03Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-08-20T11:34:07Z","started_at":"2026-08-20T11:34:07Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"nix-overlay-dv3","title":"Fix Config screen crash on F2","description":"F2 opens ConfigScreen then Textual raises InvalidSelectValueError because _refresh_aggregator_dropdown assigns Select.BLANK, resolved as False, instead of clearing selection.\\n\\nAcceptance: Config screen opens with saved consensus configuration; regression covers no selected aggregator; targeted tests pass.","status":"closed","priority":1,"issue_type":"bug","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-08-20T11:34:03Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-08-20T11:37:47Z","started_at":"2026-08-20T11:34:07Z","closed_at":"2026-08-20T11:37:47Z","close_reason":"Replaced invalid Select.BLANK handling; built haivemind and verified ConfigScreen headlessly.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"nix-overlay-a9m","title":"Fix haivemind startup after MainScreen mount","description":"Restore deferred engine startup while preserving MainScreen widget lookup fix.","status":"closed","priority":1,"issue_type":"bug","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-08-20T11:20:36Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-08-20T11:21:43Z","started_at":"2026-08-20T11:20:43Z","closed_at":"2026-08-20T11:21:43Z","close_reason":"Restored engine startup after MainScreen mount and verified package build.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"nix-overlay-q2m","title":"Add FreeToken package (FlashML-org/FreeToken)","description":"Package https://github.com/FlashML-org/FreeToken into the overlay following the standard packages/\u0026lt;name\u0026gt;/{default.nix,package.nix} pattern. Must build via nix build, README updated, category set.","notes":"Build in progress. Resolved: tvm-ffi 0.1.13.post3 builds (cython 3.3.0 vendored, dontUsePytestCheck due to xdist addopts), flashlib wheel + pythonRemoveDeps, freetoken pythonRelaxDeps gguf/modelscope. Remaining: torch-bin CUDA closure (nccl done; libnvshmem source build ~hours), then freetoken extensions. Note: torch-bin 2.11 = +cu128 PyPI wheel autoPatchelf'd to nixpkgs cuda12.9 libs.","status":"closed","priority":2,"issue_type":"feature","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-08-30T06:40:20Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-08-30T17:51:10Z","started_at":"2026-08-30T06:40:34Z","closed_at":"2026-08-30T17:51:10Z","close_reason":"freetoken 0.1.2 packaged: builds via nix build .#freetoken, ft CLI verified (ft --version), imports checked, RPATHs correct (cudart 12.9 + torch-bin libs). Docs in README + meta.longDescription.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"nix-overlay-q00","title":"Add haivemind package","description":"Package https://github.com/dev-boz/haivemind in the Nix overlay, document it, validate the build, and complete repository workflow.","status":"closed","priority":2,"issue_type":"task","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-08-08T16:32:37Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-08-08T16:36:41Z","started_at":"2026-08-08T16:33:00Z","closed_at":"2026-08-08T16:36:41Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"nix-overlay-67e","title":"Add traycer package","description":"Add new package for traycer from https://github.com/traycerai/traycer. Steps: research repo build system, create packages/traycer/, stage with git add, test nix build, update README.","status":"closed","priority":2,"issue_type":"task","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-08-07T08:59:25Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-08-07T09:22:07Z","started_at":"2026-08-07T09:00:29Z","closed_at":"2026-08-07T09:22:07Z","close_reason":"Traycer package added, built, pushed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"nix-overlay-2gf","title":"Update all packages to latest versions","description":"## Why\\nUser request: update every package in the overlay to its latest upstream version.\\n\\n## What\\nBump version + src hash (and cargoHash for Rust) in each packages/\u003cname\u003e/package.nix, update README table if needed, verify builds.\\n\\n## Acceptance\\n- Every package builds with nix build .#\u003cname\u003e\\n- README reflects new versions\\n- Changes committed and pushed","notes":"All packages updated and built successfully. Commit 1f3f0f468bdd on local main. PUSH BLOCKED: git.millerson.name:22002 and DoltHub unreachable from this network (SSH connect timeout). Re-run git push + bd dolt push when network is back, then close.","status":"closed","priority":2,"issue_type":"task","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-08-07T07:22:50Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-08-07T13:17:34Z","started_at":"2026-08-07T07:22:55Z","closed_at":"2026-08-07T13:17:34Z","close_reason":"All packages updated to latest versions, verified building, pushed to origin","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/README.md b/README.md index 07ae3de..184e1e0 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ A custom Nix overlay and flake providing additional packages not found in upstre | `desloppify` | Multi-language codebase health scanner and technical debt tracker for AI agents | AI Coding Agents | | `ds4` | DeepSeek 4 Flash and PRO local inference engine for ROCm (Strix Halo) | AI Inference | | `freebuff` | The world's strongest free coding agent | AI Coding Agents | +| `freetoken` | Local MoE-offload LLM inference runtime with OpenAI- and Anthropic-compatible APIs | AI Inference | | `graphify` | Turn any folder of code, docs, papers, images, or videos into a queryable knowledge graph | AI Coding Agents | | `haivemind` | Multi-model AI consensus, aggregation, and fusion runner for popular AI CLIs | AI Coding Agents | | `hipengine` | ROCm-native local LLM inference engine with torch-free runtime for AMD RDNA GPUs | AI Inference | @@ -150,6 +151,7 @@ nix-overlay/ │ ├── ds4/ # DeepSeek V4 Flash/PRO inference engine (ROCm) │ ├── flake-inputs/ # Utility for caching flake inputs │ ├── freebuff/ # Free coding agent (Codebuff) +│ ├── freetoken/ # Local MoE inference runtime (NVIDIA CUDA) │ ├── graphify/ # Knowledge graph generator for code folders │ ├── haivemind/ # Multi-model AI consensus runner │ ├── hipengine/ # ROCm-native LLM inference engine for AMD GPUs diff --git a/packages/freetoken/default.nix b/packages/freetoken/default.nix new file mode 100644 index 0000000..291b4f4 --- /dev/null +++ b/packages/freetoken/default.nix @@ -0,0 +1,5 @@ +{ + pkgs, + ... +}: +pkgs.callPackage ./package.nix { } diff --git a/packages/freetoken/flashlib.nix b/packages/freetoken/flashlib.nix new file mode 100644 index 0000000..a921e0d --- /dev/null +++ b/packages/freetoken/flashlib.nix @@ -0,0 +1,51 @@ +{ + lib, + buildPythonPackage, + fetchurl, + # dependencies + numpy, + numba, + # torch-bin (CUDA wheel build) and triton-bin are passed by the caller; + # the python scope defaults are the source-built torch (CPU-only) and + # triton — using them would duplicate torch/triton in the closure. + torch, + triton, + tqdm, +}: + +buildPythonPackage rec { + pname = "flashlib"; + version = "0.3.0"; + format = "wheel"; + + src = fetchurl { + url = "https://files.pythonhosted.org/packages/e5/b8/4c085892462e521bb9f2d943ff34fa219e3a5a206798f3c96a983538039f/flashlib-0.3.0-py3-none-any.whl"; + hash = "sha256-kDeRHzFf7zyfRFMmFZc2Ory4OGiBOtGyWKsTCLNP0Ro="; + }; + + # freetoken pins flashlib==0.3.0 (not in nixpkgs) and only uses its Triton + # kernels (flashlib.kernels.slot_cache). nvidia-cutlass-dsl — needed only by + # the CuTeDSL GEMM backends in flashlib.linalg, which drag in cuda-python and + # nvdisasm binary wheels — is deliberately not packaged (and removed from + # the wheel metadata so the runtime deps check passes). + pythonRemoveDeps = [ "nvidia-cutlass-dsl" ]; + + dependencies = [ + numpy + numba + torch + triton + tqdm + ]; + + # flashlib is CPU-safe at import time: CuteDSL imports are lazy and hardware + # detection (flashlib._hw) is cuda-optional. + pythonImportsCheck = [ "flashlib" ]; + + meta = { + description = "High-performance ML primitives — Triton and CuteDSL kernels for NVIDIA GPUs"; + homepage = "https://pypi.org/project/flashlib/"; + license = lib.licenses.asl20; + platforms = lib.platforms.linux; + }; +} diff --git a/packages/freetoken/package.nix b/packages/freetoken/package.nix new file mode 100644 index 0000000..dd43dfd --- /dev/null +++ b/packages/freetoken/package.nix @@ -0,0 +1,191 @@ +{ + lib, + pkgs, + fetchFromGitHub, +}: + +let + # FreeToken is an NVIDIA-GPU inference engine: setup.py builds two C++ + # extensions that link the CUDA runtime, and the runtime needs a + # CUDA-enabled torch. CUDA libraries carry the unfree "CUDA EULA" license, + # so this package re-imports the flake's nixpkgs with allowUnfree enabled — + # scoped to freetoken only, leaving the rest of the overlay unchanged. + nixpkgsUnfree = import pkgs.path { + inherit (pkgs.stdenv.hostPlatform) system; + config.allowUnfree = true; + }; + + py = nixpkgsUnfree.python3Packages; + + # torch>=2.11,<2.12: torch-bin 2.11 is this nixpkgs' CUDA build (it links + # the cuda12.9-* libraries). The extensions must link the same libcudart + # instance torch-bin links, so CUDA_HOME below is built from the matching + # cudaPackages.cuda_cudart. + cudart = nixpkgsUnfree.cudaPackages.cuda_cudart; + # cuda_runtime_api.h includes crt/host_defines.h & co., which ship with + # nvcc's header set, not with cudart. + nvccHeaders = nixpkgsUnfree.cudaPackages.cuda_nvcc; + + # setup.py requires CUDA_HOME containing CUDA headers and libcudart. + # Single-output cudart provides ${cudart}/include and ${cudart}/lib; the + # crt/ headers come from nvcc; lib64/ is added for torch's cpp_extension, + # lib/ is picked up by setup.py's cuda-home/lib fallback. + cudaHome = nixpkgsUnfree.runCommand "freetoken-cuda-home" { } '' + mkdir -p $out/include $out/lib64 + ln -s ${cudart}/include/* $out/include/ + ln -s ${nvccHeaders}/include/crt $out/include/crt + ln -s ${cudart}/lib $out/lib + for f in ${cudart}/lib/*.so*; do + ln -s "$f" "$out/lib64/$(basename "$f")" + done + ''; + + # flashlib==0.3.0, pinned by freetoken, is not in nixpkgs. torch and triton + # come from the prebuilt wheel builds (torch-bin / triton-bin), the same + # instances torch-bin propagates — the scope defaults (source-built torch + # and triton) would put two tritons/torches into the closure. + flashlib = py.callPackage ./flashlib.nix { + torch = py.torch-bin; + triton = py.triton-bin; + }; + + # freetoken pins apache-tvm-ffi==0.1.13.post3; nixpkgs has 0.1.10. + # 0.1.13 needs cython>=3.2.8 (nixpkgs: 3.2.4), so build it with a newer + # cython, scoped to this package only. + apache-tvm-ffi = + (py.apache-tvm-ffi.override { + cython = py.cython.overridePythonAttrs (old: rec { + version = "3.3.0"; + src = fetchFromGitHub { + owner = "cython"; + repo = "cython"; + tag = version; + hash = "sha256-gIEqq8DAJF197gH1cMuq5JxI4rV/VHS70vg8hywXDqw="; + }; + }); + }).overridePythonAttrs + (old: { + version = "0.1.13.post3"; + src = fetchFromGitHub { + owner = "apache"; + repo = "tvm-ffi"; + tag = "v0.1.13-post3"; + fetchSubmodules = true; + hash = "sha256-AN7AqBl62T8DqnRt7KRvGjqo/c0SJ66QZrCQQ5yicHw="; + }; + # 0.1.13 runs its suite with pytest-xdist (pyproject addopts = [ "-n" + # "auto" ]) and touches GPUs; the nixpkgs check inputs have neither + # xdist nor a GPU. The imports check still runs (and passes). + dontUsePytestCheck = true; + }); +in +py.buildPythonApplication (finalAttrs: { + pname = "freetoken"; + version = "0.1.2"; + pyproject = true; + + src = fetchFromGitHub { + owner = "FlashML-org"; + repo = "FreeToken"; + tag = "v${finalAttrs.version}"; + hash = "sha256-0MhuubuTjNvtQZxisC2cg1dJeR+A6wZ901H5FRv+l+c="; + }; + + # setup.py imports torch.utils.cpp_extension at build time and compiles two + # C++ extensions (freetoken.kernel._pinned_tensor and _cpu_moe) that link + # libcudart from CUDA_HOME. ninja is required by BuildExtension. + build-system = with py; [ + ninja + setuptools + torch-bin + wheel + ]; + + nativeBuildInputs = [ nixpkgsUnfree.autoPatchelfHook ]; + + # cudart: RPATH target for the built extensions (autoPatchelf finds + # libcudart.so.12 here); libtorch comes from torch-bin via dependencies. + buildInputs = [ cudart ]; + + env.CUDA_HOME = cudaHome; + + dependencies = with py; [ + apache-tvm-ffi + einops + fastapi + flashlib + gguf + huggingface-hub + # freetoken's floor is modelscope>=1.37 (nixpkgs has 1.36.2); the only + # API used is modelscope.snapshot_download (server/args.py), which is + # stable across both. + modelscope + msgpack + numpy + openai + partial-json-parser + prompt-toolkit + pydantic + pyzmq + safetensors + torch-bin + # triton==3.6.0 pin, satisfied with the wheel build (triton-bin) — the + # same instance torch-bin propagates, keeping one triton in the closure. + triton-bin + tqdm + transformers + uvicorn + ]; + + # Tests need an NVIDIA GPU and real model checkpoints (pytest markers + # slow/needs_weights) — impossible in the build sandbox. + doCheck = false; + pythonImportsCheck = [ "freetoken" ]; + + # Wheel metadata constraints that nixpkgs packages do not match: + # - gguf is versioned by upstream git rev in nixpkgs ("8951") rather than + # PyPI semver; the reader API freetoken uses is stable. + # - modelscope floor is 1.37 (nixpkgs: 1.36.2); only the stable + # snapshot_download API is used. + pythonRelaxDeps = [ + "gguf" + "modelscope" + ]; + + passthru = { + category = "AI Inference"; + updateScript = [ + "nix-update" + "--flake" + ".#freetoken" + ]; + }; + + meta = { + description = "Local MoE-offload LLM inference runtime with OpenAI- and Anthropic-compatible APIs"; + longDescription = '' + FreeToken is an edge-native MoE serving engine for running + frontier-scale open-weight models on consumer NVIDIA GPUs (RTX + 30/40/50 series). It treats GPUs, CPUs and host memory as one elastic + inference platform, with semantic-aware KV/expert caching and runtime + VRAM re-allocation between caches. It serves OpenAI- and + Anthropic-compatible APIs. + + Nix packaging notes: + - Built against this nixpkgs' CUDA 12.9 stack (torch-bin 2.11 + + cudaPackages.cuda_cudart) instead of upstream's cu130 wheels; CUDA + 12.9 supports the same GPU range, including RTX 50 (sm_120). + - The optional native accel extras (flashinfer, sglang-kernel) and the + freetoken-kernel-cache companion wheel are not packaged; the runtime + falls back to its pure-Triton kernels where those would be used. + - Requires an NVIDIA GPU at runtime. + ''; + homepage = "https://github.com/FlashML-org/FreeToken"; + changelog = "https://github.com/FlashML-org/FreeToken/releases/tag/v${finalAttrs.version}"; + # Upstream code is Apache-2.0, but this derivation links CUDA libraries + # under the unfree CUDA EULA (same as nixpkgs' torch-bin). + license = lib.licenses.unfree; + platforms = [ "x86_64-linux" ]; + mainProgram = "ft"; + }; +})