feat(packages): add freetoken 0.1.2 NVIDIA MoE inference runtime

Package FreeToken (FlashML-org/FreeToken) — edge-native MoE serving
engine with OpenAI/Anthropic-compatible APIs, serving the `ft` CLI.

- torch-bin 2.11 (CUDA 12.9 wheel build) satisfies the torch>=2.11,<2.12
  pin; extensions link the same cudaPackages.cuda_cudart via a synthetic
  CUDA_HOME (cudart headers + nvcc crt/ headers, no nvcc needed).
- flashlib==0.3.0 vendored from the PyPI wheel (Triton-only; freetoken
  never imports the CuTeDSL GEMM backends, so nvidia-cutlass-dsl is
  omitted via pythonRemoveDeps).
- apache-tvm-ffi overridden to the pinned 0.1.13.post3 with a vendored
  cython 3.3.0 (build needs >=3.2.8, nixpkgs has 3.2.4); pytest skipped
  (upstream runs pytest-xdist with GPU tests).
- Unfree (CUDA EULA) is self-scoped: the package re-imports nixpkgs with
  config.allowUnfree so no flake-level or user config change is needed.
- Optional accel extras (flashinfer/sglang-kernel) and the kernel-cache
  wheel are not packaged; runtime falls back to pure-Triton kernels.
- Verified: nix build .#freetoken, ft --version, python imports check,
  extension RPATHs.

.beads/issues.jsonl

@@ -1,5 +1,6 @@
-{"_type":"issue","id":"nix-overlay-dv3","title":"Fix Config screen crash on F2","description":"F2 opens ConfigScreen then Textual raises InvalidSelectValueError because _refresh_aggregator_dropdown assigns Select.BLANK, resolved as False, instead of clearing selection.\\n\\nAcceptance: Config screen opens with saved consensus configuration; regression covers no selected aggregator; targeted tests pass.","status":"in_progress","priority":1,"issue_type":"bug","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-08-20T11:34:03Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-08-20T11:34:07Z","started_at":"2026-08-20T11:34:07Z","dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"nix-overlay-dv3","title":"Fix Config screen crash on F2","description":"F2 opens ConfigScreen then Textual raises InvalidSelectValueError because _refresh_aggregator_dropdown assigns Select.BLANK, resolved as False, instead of clearing selection.\\n\\nAcceptance: Config screen opens with saved consensus configuration; regression covers no selected aggregator; targeted tests pass.","status":"closed","priority":1,"issue_type":"bug","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-08-20T11:34:03Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-08-20T11:37:47Z","started_at":"2026-08-20T11:34:07Z","closed_at":"2026-08-20T11:37:47Z","close_reason":"Replaced invalid Select.BLANK handling; built haivemind and verified ConfigScreen headlessly.","dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"nix-overlay-a9m","title":"Fix haivemind startup after MainScreen mount","description":"Restore deferred engine startup while preserving MainScreen widget lookup fix.","status":"closed","priority":1,"issue_type":"bug","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-08-20T11:20:36Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-08-20T11:21:43Z","started_at":"2026-08-20T11:20:43Z","closed_at":"2026-08-20T11:21:43Z","close_reason":"Restored engine startup after MainScreen mount and verified package build.","dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"nix-overlay-q2m","title":"Add FreeToken package (FlashML-org/FreeToken)","description":"Package https://github.com/FlashML-org/FreeToken into the overlay following the standard packages/\u0026lt;name\u0026gt;/{default.nix,package.nix} pattern. Must build via nix build, README updated, category set.","notes":"Build in progress. Resolved: tvm-ffi 0.1.13.post3 builds (cython 3.3.0 vendored, dontUsePytestCheck due to xdist addopts), flashlib wheel + pythonRemoveDeps, freetoken pythonRelaxDeps gguf/modelscope. Remaining: torch-bin CUDA closure (nccl done; libnvshmem source build ~hours), then freetoken extensions. Note: torch-bin 2.11 = +cu128 PyPI wheel autoPatchelf'd to nixpkgs cuda12.9 libs.","status":"closed","priority":2,"issue_type":"feature","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-08-30T06:40:20Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-08-30T17:51:10Z","started_at":"2026-08-30T06:40:34Z","closed_at":"2026-08-30T17:51:10Z","close_reason":"freetoken 0.1.2 packaged: builds via nix build .#freetoken, ft CLI verified (ft --version), imports checked, RPATHs correct (cudart 12.9 + torch-bin libs). Docs in README + meta.longDescription.","dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"nix-overlay-q00","title":"Add haivemind package","description":"Package https://github.com/dev-boz/haivemind in the Nix overlay, document it, validate the build, and complete repository workflow.","status":"closed","priority":2,"issue_type":"task","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-08-08T16:32:37Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-08-08T16:36:41Z","started_at":"2026-08-08T16:33:00Z","closed_at":"2026-08-08T16:36:41Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"nix-overlay-67e","title":"Add traycer package","description":"Add new package for traycer from https://github.com/traycerai/traycer. Steps: research repo build system, create packages/traycer/, stage with git add, test nix build, update README.","status":"closed","priority":2,"issue_type":"task","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-08-07T08:59:25Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-08-07T09:22:07Z","started_at":"2026-08-07T09:00:29Z","closed_at":"2026-08-07T09:22:07Z","close_reason":"Traycer package added, built, pushed","dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"nix-overlay-2gf","title":"Update all packages to latest versions","description":"## Why\\nUser request: update every package in the overlay to its latest upstream version.\\n\\n## What\\nBump version + src hash (and cargoHash for Rust) in each packages/\u003cname\u003e/package.nix, update README table if needed, verify builds.\\n\\n## Acceptance\\n- Every package builds with nix build .#\u003cname\u003e\\n- README reflects new versions\\n- Changes committed and pushed","notes":"All packages updated and built successfully. Commit 1f3f0f468bdd on local main. PUSH BLOCKED: git.millerson.name:22002 and DoltHub unreachable from this network (SSH connect timeout). Re-run git push + bd dolt push when network is back, then close.","status":"closed","priority":2,"issue_type":"task","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-08-07T07:22:50Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-08-07T13:17:34Z","started_at":"2026-08-07T07:22:55Z","closed_at":"2026-08-07T13:17:34Z","close_reason":"All packages updated to latest versions, verified building, pushed to origin","dependency_count":0,"dependent_count":0,"comment_count":0}

README.md

@@ -18,6 +18,7 @@ A custom Nix overlay and flake providing additional packages not found in upstre
 | `desloppify` | Multi-language codebase health scanner and technical debt tracker for AI agents | AI Coding Agents |
 | `ds4` | DeepSeek 4 Flash and PRO local inference engine for ROCm (Strix Halo) | AI Inference |
 | `freebuff` | The world's strongest free coding agent | AI Coding Agents |
+| `freetoken` | Local MoE-offload LLM inference runtime with OpenAI- and Anthropic-compatible APIs | AI Inference |
 | `graphify` | Turn any folder of code, docs, papers, images, or videos into a queryable knowledge graph | AI Coding Agents |
 | `haivemind` | Multi-model AI consensus, aggregation, and fusion runner for popular AI CLIs | AI Coding Agents |
 | `hipengine` | ROCm-native local LLM inference engine with torch-free runtime for AMD RDNA GPUs | AI Inference |
@@ -150,6 +151,7 @@ nix-overlay/
 │   ├── ds4/                    # DeepSeek V4 Flash/PRO inference engine (ROCm)
 │   ├── flake-inputs/           # Utility for caching flake inputs
 │   ├── freebuff/               # Free coding agent (Codebuff)
+│   ├── freetoken/             # Local MoE inference runtime (NVIDIA CUDA)
 │   ├── graphify/               # Knowledge graph generator for code folders
 │   ├── haivemind/              # Multi-model AI consensus runner
 │   ├── hipengine/              # ROCm-native LLM inference engine for AMD GPUs

packages/freetoken/default.nix

@@ -0,0 +1,5 @@
+{
+  pkgs,
+  ...
+}:
+pkgs.callPackage ./package.nix { }

packages/freetoken/flashlib.nix

@@ -0,0 +1,51 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchurl,
+  # dependencies
+  numpy,
+  numba,
+  # torch-bin (CUDA wheel build) and triton-bin are passed by the caller;
+  # the python scope defaults are the source-built torch (CPU-only) and
+  # triton — using them would duplicate torch/triton in the closure.
+  torch,
+  triton,
+  tqdm,
+}:
+
+buildPythonPackage rec {
+  pname = "flashlib";
+  version = "0.3.0";
+  format = "wheel";
+
+  src = fetchurl {
+    url = "https://files.pythonhosted.org/packages/e5/b8/4c085892462e521bb9f2d943ff34fa219e3a5a206798f3c96a983538039f/flashlib-0.3.0-py3-none-any.whl";
+    hash = "sha256-kDeRHzFf7zyfRFMmFZc2Ory4OGiBOtGyWKsTCLNP0Ro=";
+  };
+
+  # freetoken pins flashlib==0.3.0 (not in nixpkgs) and only uses its Triton
+  # kernels (flashlib.kernels.slot_cache). nvidia-cutlass-dsl — needed only by
+  # the CuTeDSL GEMM backends in flashlib.linalg, which drag in cuda-python and
+  # nvdisasm binary wheels — is deliberately not packaged (and removed from
+  # the wheel metadata so the runtime deps check passes).
+  pythonRemoveDeps = [ "nvidia-cutlass-dsl" ];
+
+  dependencies = [
+    numpy
+    numba
+    torch
+    triton
+    tqdm
+  ];
+
+  # flashlib is CPU-safe at import time: CuteDSL imports are lazy and hardware
+  # detection (flashlib._hw) is cuda-optional.
+  pythonImportsCheck = [ "flashlib" ];
+
+  meta = {
+    description = "High-performance ML primitives — Triton and CuteDSL kernels for NVIDIA GPUs";
+    homepage = "https://pypi.org/project/flashlib/";
+    license = lib.licenses.asl20;
+    platforms = lib.platforms.linux;
+  };
+}

packages/freetoken/package.nix

@@ -0,0 +1,191 @@
+{
+  lib,
+  pkgs,
+  fetchFromGitHub,
+}:
+
+let
+  # FreeToken is an NVIDIA-GPU inference engine: setup.py builds two C++
+  # extensions that link the CUDA runtime, and the runtime needs a
+  # CUDA-enabled torch. CUDA libraries carry the unfree "CUDA EULA" license,
+  # so this package re-imports the flake's nixpkgs with allowUnfree enabled —
+  # scoped to freetoken only, leaving the rest of the overlay unchanged.
+  nixpkgsUnfree = import pkgs.path {
+    inherit (pkgs.stdenv.hostPlatform) system;
+    config.allowUnfree = true;
+  };
+
+  py = nixpkgsUnfree.python3Packages;
+
+  # torch>=2.11,<2.12: torch-bin 2.11 is this nixpkgs' CUDA build (it links
+  # the cuda12.9-* libraries). The extensions must link the same libcudart
+  # instance torch-bin links, so CUDA_HOME below is built from the matching
+  # cudaPackages.cuda_cudart.
+  cudart = nixpkgsUnfree.cudaPackages.cuda_cudart;
+  # cuda_runtime_api.h includes crt/host_defines.h & co., which ship with
+  # nvcc's header set, not with cudart.
+  nvccHeaders = nixpkgsUnfree.cudaPackages.cuda_nvcc;
+
+  # setup.py requires CUDA_HOME containing CUDA headers and libcudart.
+  # Single-output cudart provides ${cudart}/include and ${cudart}/lib; the
+  # crt/ headers come from nvcc; lib64/ is added for torch's cpp_extension,
+  # lib/ is picked up by setup.py's cuda-home/lib fallback.
+  cudaHome = nixpkgsUnfree.runCommand "freetoken-cuda-home" { } ''
+    mkdir -p $out/include $out/lib64
+    ln -s ${cudart}/include/* $out/include/
+    ln -s ${nvccHeaders}/include/crt $out/include/crt
+    ln -s ${cudart}/lib $out/lib
+    for f in ${cudart}/lib/*.so*; do
+      ln -s "$f" "$out/lib64/$(basename "$f")"
+    done
+  '';
+
+  # flashlib==0.3.0, pinned by freetoken, is not in nixpkgs. torch and triton
+  # come from the prebuilt wheel builds (torch-bin / triton-bin), the same
+  # instances torch-bin propagates — the scope defaults (source-built torch
+  # and triton) would put two tritons/torches into the closure.
+  flashlib = py.callPackage ./flashlib.nix {
+    torch = py.torch-bin;
+    triton = py.triton-bin;
+  };
+
+  # freetoken pins apache-tvm-ffi==0.1.13.post3; nixpkgs has 0.1.10.
+  # 0.1.13 needs cython>=3.2.8 (nixpkgs: 3.2.4), so build it with a newer
+  # cython, scoped to this package only.
+  apache-tvm-ffi =
+    (py.apache-tvm-ffi.override {
+      cython = py.cython.overridePythonAttrs (old: rec {
+        version = "3.3.0";
+        src = fetchFromGitHub {
+          owner = "cython";
+          repo = "cython";
+          tag = version;
+          hash = "sha256-gIEqq8DAJF197gH1cMuq5JxI4rV/VHS70vg8hywXDqw=";
+        };
+      });
+    }).overridePythonAttrs
+      (old: {
+        version = "0.1.13.post3";
+        src = fetchFromGitHub {
+          owner = "apache";
+          repo = "tvm-ffi";
+          tag = "v0.1.13-post3";
+          fetchSubmodules = true;
+          hash = "sha256-AN7AqBl62T8DqnRt7KRvGjqo/c0SJ66QZrCQQ5yicHw=";
+        };
+        # 0.1.13 runs its suite with pytest-xdist (pyproject addopts = [ "-n"
+        # "auto" ]) and touches GPUs; the nixpkgs check inputs have neither
+        # xdist nor a GPU. The imports check still runs (and passes).
+        dontUsePytestCheck = true;
+      });
+in
+py.buildPythonApplication (finalAttrs: {
+  pname = "freetoken";
+  version = "0.1.2";
+  pyproject = true;
+
+  src = fetchFromGitHub {
+    owner = "FlashML-org";
+    repo = "FreeToken";
+    tag = "v${finalAttrs.version}";
+    hash = "sha256-0MhuubuTjNvtQZxisC2cg1dJeR+A6wZ901H5FRv+l+c=";
+  };
+
+  # setup.py imports torch.utils.cpp_extension at build time and compiles two
+  # C++ extensions (freetoken.kernel._pinned_tensor and _cpu_moe) that link
+  # libcudart from CUDA_HOME. ninja is required by BuildExtension.
+  build-system = with py; [
+    ninja
+    setuptools
+    torch-bin
+    wheel
+  ];
+
+  nativeBuildInputs = [ nixpkgsUnfree.autoPatchelfHook ];
+
+  # cudart: RPATH target for the built extensions (autoPatchelf finds
+  # libcudart.so.12 here); libtorch comes from torch-bin via dependencies.
+  buildInputs = [ cudart ];
+
+  env.CUDA_HOME = cudaHome;
+
+  dependencies = with py; [
+    apache-tvm-ffi
+    einops
+    fastapi
+    flashlib
+    gguf
+    huggingface-hub
+    # freetoken's floor is modelscope>=1.37 (nixpkgs has 1.36.2); the only
+    # API used is modelscope.snapshot_download (server/args.py), which is
+    # stable across both.
+    modelscope
+    msgpack
+    numpy
+    openai
+    partial-json-parser
+    prompt-toolkit
+    pydantic
+    pyzmq
+    safetensors
+    torch-bin
+    # triton==3.6.0 pin, satisfied with the wheel build (triton-bin) — the
+    # same instance torch-bin propagates, keeping one triton in the closure.
+    triton-bin
+    tqdm
+    transformers
+    uvicorn
+  ];
+
+  # Tests need an NVIDIA GPU and real model checkpoints (pytest markers
+  # slow/needs_weights) — impossible in the build sandbox.
+  doCheck = false;
+  pythonImportsCheck = [ "freetoken" ];
+
+  # Wheel metadata constraints that nixpkgs packages do not match:
+  # - gguf is versioned by upstream git rev in nixpkgs ("8951") rather than
+  #   PyPI semver; the reader API freetoken uses is stable.
+  # - modelscope floor is 1.37 (nixpkgs: 1.36.2); only the stable
+  #   snapshot_download API is used.
+  pythonRelaxDeps = [
+    "gguf"
+    "modelscope"
+  ];
+
+  passthru = {
+    category = "AI Inference";
+    updateScript = [
+      "nix-update"
+      "--flake"
+      ".#freetoken"
+    ];
+  };
+
+  meta = {
+    description = "Local MoE-offload LLM inference runtime with OpenAI- and Anthropic-compatible APIs";
+    longDescription = ''
+      FreeToken is an edge-native MoE serving engine for running
+      frontier-scale open-weight models on consumer NVIDIA GPUs (RTX
+      30/40/50 series). It treats GPUs, CPUs and host memory as one elastic
+      inference platform, with semantic-aware KV/expert caching and runtime
+      VRAM re-allocation between caches. It serves OpenAI- and
+      Anthropic-compatible APIs.
+
+      Nix packaging notes:
+      - Built against this nixpkgs' CUDA 12.9 stack (torch-bin 2.11 +
+        cudaPackages.cuda_cudart) instead of upstream's cu130 wheels; CUDA
+        12.9 supports the same GPU range, including RTX 50 (sm_120).
+      - The optional native accel extras (flashinfer, sglang-kernel) and the
+        freetoken-kernel-cache companion wheel are not packaged; the runtime
+        falls back to its pure-Triton kernels where those would be used.
+      - Requires an NVIDIA GPU at runtime.
+    '';
+    homepage = "https://github.com/FlashML-org/FreeToken";
+    changelog = "https://github.com/FlashML-org/FreeToken/releases/tag/v${finalAttrs.version}";
+    # Upstream code is Apache-2.0, but this derivation links CUDA libraries
+    # under the unfree CUDA EULA (same as nixpkgs' torch-bin).
+    license = lib.licenses.unfree;
+    platforms = [ "x86_64-linux" ];
+    mainProgram = "ft";
+  };
+})