From 85bd7421de16fbd805f92e02bf2a9dde7d8c3277 Mon Sep 17 00:00:00 2001 From: Alexander Miroshnichenko Date: Thu, 12 Nov 2026 20:36:52 +0300 Subject: [PATCH] feat(llm): package upstream inference engines Centralize engine pins, compiler wrappers and NPU packages. Use upstream Kyojin v1.8.10 and drop the Thunderbolt dependency. Offline evaluation passed; workstation builds remain pending. Bead: nix-overlay-v09 --- .beads/issues.jsonl | 1 + README.md | 47 +-- _lib/llm-packages.nix | 42 +++ flake.lock | 458 ++++++++++++++++++++++++- flake.nix | 33 ++ packages/gufo/default.nix | 5 + packages/gufo/gufo-unsloth-ud3.patch | 58 ++++ packages/gufo/package.nix | 44 +++ packages/hipfire-kernels/default.nix | 5 + packages/hipfire/default.nix | 5 + packages/hipfire/package.nix | 123 +++++++ packages/kyojin/check-jit.py | 46 --- packages/kyojin/default.nix | 18 +- packages/kyojin/package.nix | 248 ++----------- packages/pwilkin-llama-cpp/default.nix | 5 + packages/pwilkin-llama-cpp/package.nix | 6 + packages/strix-llama-cpp/default.nix | 5 + packages/strix-llama-cpp/package.nix | 5 + packages/therock-rocm/default.nix | 5 + packages/therock-rocm/package.nix | 10 + packages/xrt-amdxdna/default.nix | 5 + 21 files changed, 865 insertions(+), 309 deletions(-) create mode 100644 _lib/llm-packages.nix create mode 100644 packages/gufo/default.nix create mode 100644 packages/gufo/gufo-unsloth-ud3.patch create mode 100644 packages/gufo/package.nix create mode 100644 packages/hipfire-kernels/default.nix create mode 100644 packages/hipfire/default.nix create mode 100644 packages/hipfire/package.nix delete mode 100644 packages/kyojin/check-jit.py create mode 100644 packages/pwilkin-llama-cpp/default.nix create mode 100644 packages/pwilkin-llama-cpp/package.nix create mode 100644 packages/strix-llama-cpp/default.nix create mode 100644 packages/strix-llama-cpp/package.nix create mode 100644 packages/therock-rocm/default.nix create mode 100644 packages/therock-rocm/package.nix create mode 100644 packages/xrt-amdxdna/default.nix diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 293c720..00949d0 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -45,3 +45,4 @@ {"id":"nix-overlay-yf4","title":"codex-oauth-proxy: fix gpt-6.1-terra normalization","description":"Requests for gpt-6.1-terra are rewritten to gpt-5.5 by normalizeModel because the model is missing from the hardcoded allowlist. The account is not entitled to gpt-5.5, so /v1/chat/completions and /v1/responses return 400 model_not_found. Add modelGPT61Terra to the exact-match switch, modelAllowedEfforts, and modelDefaultEffort in codex-oauth-proxy 1.1.1 via an overlay patch. Verify with nix build .#codex-oauth-proxy and a unit test for the new model.","status":"closed","priority":2,"issue_type":"bug","assignee":"opencode","created_at":"2026-11-12T12:46:01.000023180Z","created_by":"minder","updated_at":"2026-11-12T12:47:40.200015184Z","closed_at":"2026-11-12T12:47:40.199512016Z","close_reason":"Completed: added gpt-6.1-terra to codex-oauth-proxy 1.1.1 via packages/codex-oauth-proxy/gpt-6.1-terra.patch; nix build .#codex-oauth-proxy passes, including new normalize and reasoning-effort tests for gpt-6.1-terra","source_repo":"nix-overlay","compaction_level":0,"original_size":0} {"_type":"issue","id":"nix-overlay-ylf","title":"Add open-code-review package","description":"Package alibaba/open-code-review (AI code review CLI, npm @alibaba-group/open-code-review) into the overlay","status":"closed","priority":2,"issue_type":"feature","assignee":"Alexander Miroshnichenko","owner":"alex@millerson.name","created_at":"2026-09-26T07:18:05Z","created_by":"Alexander Miroshnichenko","updated_at":"2026-09-26T07:24:49Z","started_at":"2026-09-26T07:18:48Z","closed_at":"2026-09-26T07:24:49Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"id":"nix-overlay-yxj","title":"Add memdoor package","description":"Add https://github.com/guregodevo/memdoor (Go CLI coding/automation agent) to packages/","status":"closed","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-01T18:25:31.524424793Z","created_by":"minder","updated_at":"2026-11-01T22:53:37.479304645Z","closed_at":"2026-11-01T22:53:37.477846036Z","close_reason":"Added packages/memdoor (v1.769.0), builds with upstream Go suite enabled","source_repo":"nix-overlay","compaction_level":0,"original_size":0} +{"id":"nix-overlay-v09","title":"Package workstation LLM engines from upstream flakes","description":"Migration coordinated by nixos-config-npn0. Move hipfire + TheRock kernels/toolchain, gufo NPU + XRT and patch, two llama.cpp forks, and upstream Kyojin v1.8.10 into this overlay. Replace old Kyojin 1.8.5 and its obsolete tests. Use nix-strix-halo as non-flake source so no Thunderbolt dependency or dummy remains. Preserve existing revisions and dedicated engine/gufo/kyojin nixpkgs. Current machine laptop-x13: evaluation only, no compilation and no source downloads. Plan: preserve package derivations where possible; update exports and README; offline evaluation; keep issue open until builds and runtime checks of migrated packages run on ms-s1-max. No push or deployment.","status":"in_progress","priority":2,"issue_type":"task","assignee":"minder","created_at":"2026-11-12T17:11:06.267689950Z","created_by":"minder","updated_at":"2026-11-12T17:36:38.545982558Z","source_repo":"nix-overlay","compaction_level":0,"original_size":0,"comments":[{"id":4,"issue_id":"nix-overlay-v09","author":"minder","text":"Implementation and offline evaluation complete; keep open for workstation builds. All migrated pins (hipfire, gufo, Kyojin v1.8.10, both llama.cpp forks, nix-strix-halo, XRT and xdna-driver) and the engine nixpkgs pin match nixos-config before migration; all existing overlay pins are unchanged. nix-strix-halo is now flake=false, with direct TheRock SDK and XRT recipes, so Thunderbolt and its dummy are absent from both lock graphs. Kyojin replaces the old v1.8.5 nixpkgs torch/ROCm recipe with upstream v1.8.10 plus the existing CC wrapper. Upstream still labels its derivation 1.5.8; source rev d83cfeb3d6a7d42a4cf2b720efb1eb8a4ff04da7 is the requested v1.8.10. Old check-jit.py was removed with the old package; kyojin.tests.jit and tests.cpu now expose upstream HIP compiler and CPU checks. New package defaults share _lib/llm-packages.nix and defer unsupported platforms without loading their engine inputs. Exported packages: hipfire, hipfire-kernels, therock-rocm, gufo, xrt-amdxdna, kyojin, pwilkin-llama-cpp, strix-llama-cpp. Re-exported hipfire overlay and service module remain opt-in. README updated. Offline drvPath evaluation passed for all 11 packages/tool outputs with IFD disabled, plus integration assertions against both nixos-config consumers and the laptop-x13 toplevel. Gufo, XRT, Kyojin and both llama forks have identical drvPaths before/after; TheRock and hipfire now depend on genuine nixpkgs rdma-core instead of the dummy. Formatter and git diff --check passed. No compilation or source downloads. No push, commit or deployment. Temporary local consumer path must be replaced with the remote URL and a published lock pin after publication.","created_at":"2026-11-12T17:20:29Z"},{"id":5,"issue_id":"nix-overlay-v09","author":"minder","text":"Build gate: run only on ms-s1-max after hostname confirms the workstation. In nix-overlay: nix build --no-link .#hipfire .#hipfire-kernels .#therock-rocm .#gufo .#xrt-amdxdna .#kyojin .#pwilkin-llama-cpp .#strix-llama-cpp .#kyojin.tests.jit .#kyojin.tests.cpu. Then in nixos-config evaluate and build nixosConfigurations.ms-s1-max.config.system.build.toplevel, and run checks.x86_64-linux.hipfire-llvm-tools-check. Confirm Kyojin Triton JIT, hipfire precompiled-kernel matching/quantizers, and Gufo GPU+NPU startup against existing models. Particular risk to check: TheRock real rdma-core dependency replaces the previous dummy, so its and hipfire dependency paths change. Record evidence in this issue and nixos-config-npn0 before closure. Restore and verify the consumer remote overlay URL only after approved publication.","created_at":"2026-11-12T17:20:32Z"},{"id":6,"issue_id":"nix-overlay-v09","author":"minder","text":"Final verification update: normal Git-flake evaluation of all eight migrated package drvPaths passed; new files are indexed so Blueprint can see them. No commit created. Full git diff --check HEAD reports gufo-unsloth-ud3.patch:6 trailing whitespace: a required single-space prefix for an empty unified-diff context line. Patch bytes match the original nixos-config patch exactly, and git apply --check against the pinned Gufo source passes. Preserved the patch and did not suppress the checker. Earlier successful whitespace check covered the unstaged diff only.","created_at":"2026-11-12T17:22:21Z"},{"id":7,"issue_id":"nix-overlay-v09","author":"minder","text":"Astra code-review completed on gpt-6-astra medium: Spec has no findings; two Standards metadata findings fixed. Kyojin restores homepage and MIT license, confirmed from the cached upstream LICENSE. XRT shared producer declares mainProgram=xrt-smi, matching the workstation NPU prep command. Offline Nix assertions passed for all four metadata fields and the shared Gufo/XRT metadata. Kyojin drvPath is unchanged. XRT mainProgram causes nixpkgs to add NIX_MAIN_PROGRAM=xrt-smi to its derivation; independent derivation comparison found only that environment change (plus output paths), so Gufo changes transitively. Formatting passed. Preparing the user-requested migration commit; no compilation, source downloads, push or deployment. Existing whitespace warning is the unchanged unified-diff patch context. Keep this task open for the recorded workstation build/runtime gate. Pre-existing unrelated tracker change nix-overlay-y5r will remain outside this commit.","created_at":"2026-11-12T17:36:38Z"}]} diff --git a/README.md b/README.md index 32ae61c..db336e4 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,14 @@ A custom Nix overlay and flake providing additional packages not found in upstre | `shardr` | Decentralized LLM repository: content-addressed model store, BitTorrent-based sync, OpenAI-compatible serving via llama-server | AI Inference | | `skillsmcp` | MCP server that exposes Agent Skills to AI agents via the Model Context Protocol | MCP Servers | | `kubernetes-mcp-server` | Model Context Protocol (MCP) server for Kubernetes and OpenShift | MCP Servers | -| `kyojin` | ExLlamaV3-based inference engine with speculative decoding for 100 GB-class MoE packs on one 128 GB AMD Strix Halo (gfx1151, ROCm 7) | AI Inference | +| `kyojin` | Upstream Kyojin 1.8.10 with AMD ROCm SDK wheels and a host C compiler for Triton JIT | AI Inference | +| `hipfire` | AMD RDNA inference engine with TheRock compiler wrappers and quantization tools | AI Inference | +| `hipfire-kernels` | Precompiled hipfire kernels for gfx1151 | AI Inference | +| `gufo` | Gufo with XDNA2 NPU support and the Unsloth UD 3.1 compatibility patch | AI Inference | +| `xrt-amdxdna` | XRT with the amdxdna plugin, built against gufo's nixpkgs | AI Inference | +| `therock-rocm` | TheRock 10.0 SDK for gfx1151 | AI Inference | +| `pwilkin-llama-cpp` | pwilkin llama.cpp fork with ROCm for gfx1151 | AI Inference | +| `strix-llama-cpp` | halo-box llama.cpp fork with Vulkan | AI Inference | | `loop` | Corporate messenger for your team | Communication | | `radar` | Modern Kubernetes visibility — topology, event timeline, service traffic, resource browsing, Helm management, and GitOps support | Kubernetes | | `omniroute` | Unified AI router with 160+ providers, auto fallback, MCP/A2A, OpenAI-compatible APIs | AI LLM Gateway | @@ -102,29 +109,29 @@ nix run git+https://git.millerson.name/alex/millerson-overlay.nix.git#mcp-gatewa nix profile install git+https://git.millerson.name/alex/millerson-overlay.nix.git#mcp-gateway ``` -### Kyojin JIT Toolchain +### Inference Engines -Kyojin's serving wrappers use the packaged ROCm compiler by default. Supply a -pinned SDK to compile HIP JIT kernels with another compiler without rebuilding -PyTorch or the HIP extension: +The engine packages target AMD Strix Halo (`gfx1151`) on `x86_64-linux`. +`nixpkgs-engine` pins the shared engine toolchain independently of the other +packages. Gufo and Kyojin retain their upstream nixpkgs pins. -```nix -kyojin.override { jitRocmSdk = myJitSdk; } -``` +Kyojin uses the upstream 1.8.10 flake with its AMD ROCm SDK wheels. The serving +wrappers also set `CC` for Triton's runtime C compilation. The upstream derivation +still reports version `1.5.8`; the source pin is the `v1.8.10` tag. +`kyojin.tests.jit` and `kyojin.tests.cpu` expose the upstream HIP compiler smoke +check and CPU tests. The former nixpkgs +PyTorch/ROCm package and its `jitRocmSdk` override have been removed. -The SDK must provide `bin/hipcc` and matching `amdgcn/bitcode`. Its compiler -wrapper must set `HIP_DEVICE_LIB_PATH` to that bitcode **in the compiler -subprocess only**, along with any required `ROCM_PATH` and Nix C++ toolchain -flags. Keep the compiler wrapper's real path inside the SDK prefix so upstream's -compiler provenance check recognizes it. +Hipfire exports `kernels`, `hipcc`, `rocm`, `llvmReadobj` and `llvmObjdump` as +package attributes. Set the model directory with +`hipfire.override { modelDir = "/path/to/models"; }`. +The upstream service module is available as `nixosModules.hipfire`, and its +ROCm overlay as `overlays.hipfire`. Neither is enabled by the default module. -The serving process keeps PyTorch's ROCm device libraries. Setting TheRock's -Clang 23 bitcode globally makes the ROCm 7.2 runtime crash on a basic PyTorch -matrix multiplication. The serving wrappers also disable unsupported expandable -allocator segments. `nix build .#kyojin.tests.jit` checks compilation of five HIP -kernel modules without requiring a GPU; `packages/kyojin/check-jit.py --gpu`, run -with the package's Python environment and serving variables, additionally checks -PyTorch matrix multiplication and module loading through its existing HIP runtime. +Gufo exports its matching `xrt`, `npuDir` and `fabricClock` as package attributes. +The consumer must configure NPU power and fabric clocks before using the NPU. +TheRock and XRT use individual recipes from the pinned `nix-strix-halo` source +tree. Its flake and Thunderbolt dependencies are not loaded. ### nftablesbuilder Runtime Notes diff --git a/_lib/llm-packages.nix b/_lib/llm-packages.nix new file mode 100644 index 0000000..768e61a --- /dev/null +++ b/_lib/llm-packages.nix @@ -0,0 +1,42 @@ +{ inputs, system }: +let + pkgs = import inputs.nixpkgs-engine { + inherit system; + config.allowUnfree = true; + overlays = [ inputs.hipfire.overlays.default ]; + }; + therock-rocm = pkgs.callPackage ../packages/therock-rocm/package.nix { + inherit (inputs) nix-strix-halo; + }; + hipfire = pkgs.callPackage ../packages/hipfire/package.nix { + hipfireSource = inputs.hipfire; + therockRocm = therock-rocm; + }; + gufo = import ../packages/gufo/package.nix { inherit inputs system; }; +in +builtins.mapAttrs + ( + _: package: + if system == "x86_64-linux" then + package + else + { + meta.platforms = [ "x86_64-linux" ]; + } + ) + { + inherit hipfire therock-rocm gufo; + hipfire-kernels = hipfire.kernels; + xrt-amdxdna = gufo.xrt; + kyojin = pkgs.callPackage ../packages/kyojin/package.nix { + kyojin = inputs.kyojin.packages.${system}.kyojin; + jitCheck = inputs.kyojin.checks.${system}.kyojin-hipcc-smoke; + cpuCheck = inputs.kyojin.checks.${system}.kyojin-tests; + }; + pwilkin-llama-cpp = pkgs.callPackage ../packages/pwilkin-llama-cpp/package.nix { + source = inputs.pwilkin-llama-cpp; + }; + strix-llama-cpp = pkgs.callPackage ../packages/strix-llama-cpp/package.nix { + source = inputs.strix-llama-cpp; + }; + } diff --git a/flake.lock b/flake.lock index 9842298..b372937 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,26 @@ { "nodes": { + "audio-cpp": { + "inputs": { + "nixpkgs": [ + "gufo", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1789940897, + "narHash": "sha256-+9tVcd+0TkDxg7JRU7M06Lvm20sXEbuerm62uhm6JKA=", + "owner": "0xShug0", + "repo": "audio.cpp", + "rev": "e3de8e3f3cbfac55ffa58df71426c41550a8598b", + "type": "github" + }, + "original": { + "owner": "0xShug0", + "repo": "audio.cpp", + "type": "github" + } + }, "blueprint": { "inputs": { "nixpkgs": [ @@ -53,6 +74,22 @@ "type": "github" } }, + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1767039857, + "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=", + "owner": "NixOS", + "repo": "flake-compat", + "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "flake-compat", + "type": "github" + } + }, "flake-parts": { "inputs": { "nixpkgs-lib": [ @@ -73,13 +110,170 @@ "type": "github" } }, + "flake-parts_2": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1790898470, + "narHash": "sha256-zTwuwezD0w3hpga6a6P8emidzAZFyWqlNeRBlpWOOl8=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "024633cd702b10285db5cb19b40ad48d2399ba60", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "git-hooks-nix": { + "inputs": { + "flake-compat": "flake-compat", + "nixpkgs": [ + "kyojin", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1790500375, + "narHash": "sha256-XN3sDtn8TU9hAc9xqZ+SqRB/HHm2wjxM6aSWYLJU+oo=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "a0e4241b51206fbcbf52fd322eb5f0cd80f153c4", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, + "gufo": { + "inputs": { + "audio-cpp": "audio-cpp", + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1793664946, + "narHash": "sha256-fsARElSJAbcRu+OBEZXfE+m90W9Cc/696rMaFhGiSNQ=", + "owner": "mikealanni", + "repo": "gufo", + "rev": "6a107cf6aeba7f28db266c2ba100bfb274f65e5a", + "type": "github" + }, + "original": { + "owner": "mikealanni", + "ref": "fork-enhancements", + "repo": "gufo", + "type": "github" + } + }, + "hipfire": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": [ + "nixpkgs-engine" + ], + "rust-overlay": "rust-overlay" + }, + "locked": { + "lastModified": 1793080987, + "narHash": "sha256-g2S6zvHD4BeIpVpVKQDlnE5V7A+5kj9MPR7TARgmIo8=", + "owner": "warpfront", + "repo": "hipfire", + "rev": "75c28fc06486457139e47a4abc2df32dd55470f0", + "type": "github" + }, + "original": { + "owner": "warpfront", + "ref": "v0.4.4", + "repo": "hipfire", + "type": "github" + } + }, + "kyojin": { + "inputs": { + "flake-parts": "flake-parts_2", + "git-hooks-nix": "git-hooks-nix", + "nixpkgs": "nixpkgs_2", + "pyproject-build-systems": "pyproject-build-systems", + "pyproject-nix": "pyproject-nix", + "uv2nix": "uv2nix" + }, + "locked": { + "lastModified": 1793864505, + "narHash": "sha256-EGLX2sHByKsbRx1+p9TZRF2YKE2MkniS7+0o/x+q4mA=", + "owner": "Yamz-Labs", + "repo": "kyojin", + "rev": "d83cfeb3d6a7d42a4cf2b720efb1eb8a4ff04da7", + "type": "github" + }, + "original": { + "owner": "Yamz-Labs", + "ref": "v1.8.10", + "repo": "kyojin", + "type": "github" + } + }, + "nix-strix-halo": { + "flake": false, + "locked": { + "lastModified": 1793862511, + "narHash": "sha256-4JsLvkORL3XdZ1mbWbqi+uTvkzlQRbM3YlH2iUGbZ9M=", + "owner": "hellas-ai", + "repo": "nix-strix-halo", + "rev": "b2969bd90fedc4a1f9d91648bfdf8f21479b8178", + "type": "github" + }, + "original": { + "owner": "hellas-ai", + "repo": "nix-strix-halo", + "type": "github" + } + }, "nixpkgs": { "locked": { - "lastModified": 1789149629, - "narHash": "sha256-H6GwaZzZf+4npqv0tph94w9tZddSjFjmQrVsW0z78uk=", + "lastModified": 1786384358, + "narHash": "sha256-RzPPiWeUtuvymnpuEWsdtzli5w4kjZs49FqEs3/1u+I=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "eaad089433ca2bb662274377d33df3d0e51ef28b", + "rev": "2fcb964de67fcf60b43471c55d5d99e61a9ccb5a", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-engine": { + "locked": { + "lastModified": 1793726008, + "narHash": "sha256-jQp1wKvbwWdLBASTXkSi/VYmDJ5AbBi1WvcMt9HWJC8=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "1d060086d60cb76a40cf37109f979d88a5bd6396", "type": "github" }, "original": { @@ -105,6 +299,21 @@ "type": "github" } }, + "nixpkgs-lib": { + "locked": { + "lastModified": 1790477367, + "narHash": "sha256-oujbxmVKDhci2E4EM3KyAbRUgp5/IsCoiA/LfEF7OOk=", + "owner": "nix-community", + "repo": "nixpkgs.lib", + "rev": "f7cd230690d9fc982d06129f9189b101fddcfa3b", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixpkgs.lib", + "type": "github" + } + }, "nixpkgs-rust195": { "locked": { "lastModified": 1793633738, @@ -137,17 +346,176 @@ "type": "github" } }, + "nixpkgs_2": { + "locked": { + "lastModified": 1791456040, + "narHash": "sha256-H83yz/do+hjOO0uujHyH8RgtIQP2v7Mw+cLwcZK/xKc=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "e7439b6b14ad3cc35d05608ebca9bce01a25f5f8", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_3": { + "locked": { + "lastModified": 1789149629, + "narHash": "sha256-H6GwaZzZf+4npqv0tph94w9tZddSjFjmQrVsW0z78uk=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "eaad089433ca2bb662274377d33df3d0e51ef28b", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "pwilkin-llama-cpp": { + "inputs": { + "flake-parts": [ + "flake-parts" + ], + "nixpkgs": [ + "nixpkgs-engine" + ] + }, + "locked": { + "lastModified": 1789554350, + "narHash": "sha256-xZcqZroS3rUid6w7bVeilixJxUUFDq5S1EeypWCiZBM=", + "owner": "pwilkin", + "repo": "llama.cpp", + "rev": "b0f31f5876ef3856b55f5bb88072cc96e5effafe", + "type": "github" + }, + "original": { + "owner": "pwilkin", + "ref": "strix-halo", + "repo": "llama.cpp", + "type": "github" + } + }, + "pyproject-build-systems": { + "inputs": { + "nixpkgs": [ + "kyojin", + "nixpkgs" + ], + "pyproject-nix": [ + "kyojin", + "pyproject-nix" + ], + "uv2nix": [ + "kyojin", + "uv2nix" + ] + }, + "locked": { + "lastModified": 1791170410, + "narHash": "sha256-pyru9MpuXRiyq0Bd4q0nHpL4T6HCuZT4B+alcgAYE4A=", + "owner": "pyproject-nix", + "repo": "build-system-pkgs", + "rev": "953d35d03a51dbecaf3feb3fb679eb63e90ebeb4", + "type": "github" + }, + "original": { + "owner": "pyproject-nix", + "repo": "build-system-pkgs", + "type": "github" + } + }, + "pyproject-nix": { + "inputs": { + "nixpkgs": [ + "kyojin", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1788394333, + "narHash": "sha256-xYOP0Xym1eVGUaHFx+eJEaycFGoiCbst3fZf0xFHP78=", + "owner": "pyproject-nix", + "repo": "pyproject.nix", + "rev": "6a8a7881d75b6f98967e7b8069f4ead331384301", + "type": "github" + }, + "original": { + "owner": "pyproject-nix", + "repo": "pyproject.nix", + "type": "github" + } + }, "root": { "inputs": { "blueprint": "blueprint", "bun2nix": "bun2nix", "flake-parts": "flake-parts", - "nixpkgs": "nixpkgs", + "gufo": "gufo", + "hipfire": "hipfire", + "kyojin": "kyojin", + "nix-strix-halo": "nix-strix-halo", + "nixpkgs": "nixpkgs_3", + "nixpkgs-engine": "nixpkgs-engine", "nixpkgs-latest": "nixpkgs-latest", "nixpkgs-rust195": "nixpkgs-rust195", "nixpkgs-torch211": "nixpkgs-torch211", - "systems": "systems", - "treefmt-nix": "treefmt-nix" + "pwilkin-llama-cpp": "pwilkin-llama-cpp", + "strix-llama-cpp": "strix-llama-cpp", + "systems": "systems_2", + "treefmt-nix": "treefmt-nix", + "xdna-driver-src": "xdna-driver-src", + "xrt-src": "xrt-src" + } + }, + "rust-overlay": { + "inputs": { + "nixpkgs": [ + "hipfire", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1778123869, + "narHash": "sha256-hV9D8ET33kXjdoMXBT2bwM/j8WQM1SP/dVKZtjQKhAQ=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "592e5dedf04f0eaff1ed0f01ce5db7407d9fc7be", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } + }, + "strix-llama-cpp": { + "inputs": { + "flake-parts": [ + "flake-parts" + ], + "nixpkgs": [ + "nixpkgs-engine" + ] + }, + "locked": { + "lastModified": 1793787912, + "narHash": "sha256-2TlKGpTiwIDoFiWwGEYLcXrEJ5eZR7JeZiUjQMgJ8eY=", + "owner": "halo-box", + "repo": "strix-llama.cpp", + "rev": "ea5285ddfc4eb2e457886c4105c77a0cece5f205", + "type": "github" + }, + "original": { + "owner": "halo-box", + "repo": "strix-llama.cpp", + "type": "github" } }, "systems": { @@ -165,6 +533,21 @@ "type": "github" } }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, "treefmt-nix": { "inputs": { "nixpkgs": [ @@ -184,6 +567,69 @@ "repo": "treefmt-nix", "type": "github" } + }, + "uv2nix": { + "inputs": { + "nixpkgs": [ + "kyojin", + "nixpkgs" + ], + "pyproject-nix": [ + "kyojin", + "pyproject-nix" + ] + }, + "locked": { + "lastModified": 1790341335, + "narHash": "sha256-wFnDm/o/4OebDlpGFyVXP8DLUx0/gR/jUFuGqmTwsQg=", + "owner": "pyproject-nix", + "repo": "uv2nix", + "rev": "a24323e9e6ecbbf305c238845ff6c612d50467c0", + "type": "github" + }, + "original": { + "owner": "pyproject-nix", + "repo": "uv2nix", + "type": "github" + } + }, + "xdna-driver-src": { + "flake": false, + "locked": { + "lastModified": 1784758690, + "narHash": "sha256-YUiM9u9vtISttbThTt9fTtBB/w3d5UatGyVK5mAgWNM=", + "ref": "1.7", + "rev": "4e5aed38f3b74a5a9a2c7a6222eaff1a8be54305", + "revCount": 874, + "submodules": true, + "type": "git", + "url": "https://github.com/amd/xdna-driver" + }, + "original": { + "ref": "1.7", + "submodules": true, + "type": "git", + "url": "https://github.com/amd/xdna-driver" + } + }, + "xrt-src": { + "flake": false, + "locked": { + "lastModified": 1780589208, + "narHash": "sha256-JrqJIGJoQiXTwXjZpqAXVaHx+6i09B1qtqkJzoRPZKw=", + "ref": "XRT-2.21", + "rev": "8661761775a266b11992a3bd6eb08209d88aa845", + "revCount": 8684, + "submodules": true, + "type": "git", + "url": "https://github.com/Xilinx/XRT" + }, + "original": { + "ref": "XRT-2.21", + "submodules": true, + "type": "git", + "url": "https://github.com/Xilinx/XRT" + } } }, "root": "root", diff --git a/flake.nix b/flake.nix index 1c46314..df4c016 100644 --- a/flake.nix +++ b/flake.nix @@ -2,6 +2,36 @@ description = "Various packages for Nix"; inputs = { + + nixpkgs-engine.url = "github:NixOS/nixpkgs/nixos-unstable"; + hipfire = { + url = "github:warpfront/hipfire/v0.4.4"; + inputs.nixpkgs.follows = "nixpkgs-engine"; + }; + nix-strix-halo = { + url = "github:hellas-ai/nix-strix-halo"; + flake = false; + }; + xrt-src = { + url = "git+https://github.com/Xilinx/XRT?ref=XRT-2.21&submodules=1"; + flake = false; + }; + xdna-driver-src = { + url = "git+https://github.com/amd/xdna-driver?ref=1.7&submodules=1"; + flake = false; + }; + pwilkin-llama-cpp = { + url = "github:pwilkin/llama.cpp/strix-halo"; + inputs.nixpkgs.follows = "nixpkgs-engine"; + inputs.flake-parts.follows = "flake-parts"; + }; + strix-llama-cpp = { + url = "github:halo-box/strix-llama.cpp"; + inputs.nixpkgs.follows = "nixpkgs-engine"; + inputs.flake-parts.follows = "flake-parts"; + }; + gufo.url = "github:mikealanni/gufo/fork-enhancements"; + kyojin.url = "github:Yamz-Labs/kyojin/v1.8.10"; nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; # Newer nixpkgs whose toolchains are required by some packages: # rustc >= 1.95 (mcp-gateway >= 3.4.0), go >= 1.26.3 (kubernetes-mcp-server >= 0.0.66). @@ -52,6 +82,7 @@ blueprintOutputs // { overlays = { + hipfire = inputs.hipfire.overlays.default; default = import ./overlays { inherit (blueprintOutputs) packages; }; @@ -60,6 +91,8 @@ }; }; + nixosModules.hipfire = inputs.hipfire.nixosModules.default; + nixosModules.default = { nixpkgs.overlays = [ self.overlays.default ]; }; diff --git a/packages/gufo/default.nix b/packages/gufo/default.nix new file mode 100644 index 0000000..f8de9ac --- /dev/null +++ b/packages/gufo/default.nix @@ -0,0 +1,5 @@ +{ pkgs, inputs, ... }: +(import ../../_lib/llm-packages.nix { + inherit inputs; + system = pkgs.stdenv.hostPlatform.system; +}).gufo diff --git a/packages/gufo/gufo-unsloth-ud3.patch b/packages/gufo/gufo-unsloth-ud3.patch new file mode 100644 index 0000000..c6dcba5 --- /dev/null +++ b/packages/gufo/gufo-unsloth-ud3.patch @@ -0,0 +1,58 @@ +--- a/src/core/gguf_reader.cpp ++++ b/src/core/gguf_reader.cpp +@@ -747,8 +747,11 @@ + tensors_.push_back(std::move(info)); + } + +- // Align data payload base to alignment boundary ++ // Align data payload base to alignment boundary. A tensorless file, such as ++ // a metadata-only first split shard, may end unpadded after its header. + const auto padding = (alignment_ - offset % alignment_) % alignment_; ++ if (tensors_.empty()) ++ return true; + if (offset > size_ || padding > size_ - offset) + return false; + const std::size_t data_base = offset + padding; +--- a/src/models/qwen38_flash_next/weights.cpp ++++ b/src/models/qwen38_flash_next/weights.cpp +@@ -292,7 +292,9 @@ + b.Fail("per_layer_token_embd.weight is missing or too short"); + } else { + w.ple_table = b.Get("per_layer_token_embd.weight", c.ple_head_dim, rows, +- 1, {GgmlType::kIQ4_NL, GgmlType::kBF16}); ++ 1, ++ {GgmlType::kIQ4_NL, GgmlType::kQ8_0, ++ GgmlType::kBF16}); + } + } + w.layers.reserve(c.num_layers); +--- a/src/models/qwen38_flash_next/ngram.cpp ++++ b/src/models/qwen38_flash_next/ngram.cpp +@@ -81,10 +81,14 @@ + int file_descriptor, std::uint64_t file_offset, std::uint64_t rows, + std::uint32_t row_dim, core::GgmlType type, std::string* error_msg) { + std::unique_ptr t(new NgramTable()); +- const std::size_t block = type == core::GgmlType::kIQ4_NL ? 32 : 1; +- const std::size_t block_bytes = type == core::GgmlType::kIQ4_NL ? 18 : 2; +- if ((type != core::GgmlType::kIQ4_NL && type != core::GgmlType::kBF16) || +- row_dim == 0 || row_dim % block != 0) { ++ const bool blocked = ++ type == core::GgmlType::kIQ4_NL || type == core::GgmlType::kQ8_0; ++ const std::size_t block = blocked ? 32 : 1; ++ const std::size_t block_bytes = type == core::GgmlType::kIQ4_NL ? 18 ++ : type == core::GgmlType::kQ8_0 ? 34 ++ : 2; ++ if ((!blocked && type != core::GgmlType::kBF16) || row_dim == 0 || ++ row_dim % block != 0) { + if (error_msg != nullptr) { + *error_msg = "unsupported n-gram table format"; + } +@@ -127,6 +131,8 @@ + void NgramTable::DecodeRow(const std::uint8_t* src, float* dst) const { + if (type_ == core::GgmlType::kIQ4_NL) { + gufo::quant::DequantizeIQ4_NL(src, dst, row_dim_); ++ } else if (type_ == core::GgmlType::kQ8_0) { ++ gufo::quant::DequantizeQ8_0(src, dst, row_dim_); + } else { + for (std::uint32_t i = 0; i < row_dim_; ++i) { + std::uint16_t bits = 0; diff --git a/packages/gufo/package.nix b/packages/gufo/package.nix new file mode 100644 index 0000000..f612bb5 --- /dev/null +++ b/packages/gufo/package.nix @@ -0,0 +1,44 @@ +{ system, inputs }: +let + gufoPkgs = import inputs.gufo.inputs.nixpkgs { + inherit system; + config.allowUnfree = true; + }; + nsh = inputs.nix-strix-halo; + + xrt = + (gufoPkgs.callPackage (nsh + /pkgs/xrt) { + src = inputs.xrt-src; + version = "2.21"; + xdnaSrc = inputs.xdna-driver-src; + xdnaVersion = "1.7"; + }).xdna.overrideAttrs + (old: { + meta = (old.meta or { }) // { + mainProgram = "xrt-smi"; + }; + }); + + package = inputs.gufo.packages.${system}.release.overrideAttrs (old: { + nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ + gufoPkgs.libuuid.dev + xrt + ]; + cmakeFlags = old.cmakeFlags ++ [ + "-DGUFO_ENABLE_NPU=ON" + "-DGUFO_XRT_INCLUDE_DIR=${xrt}/include" + "-DGUFO_XRT_LIB_DIR=${xrt}/lib" + ]; + patches = (old.patches or [ ]) ++ [ ./gufo-unsloth-ud3.patch ]; + }); + + npuDir = "${inputs.gufo}/tools/npu"; + + fabricClock = "${inputs.gufo}/tools/npu/fabric-clock.sh"; +in +package.overrideAttrs (old: { + passthru = (old.passthru or { }) // { + inherit xrt npuDir fabricClock; + category = "AI Inference"; + }; +}) diff --git a/packages/hipfire-kernels/default.nix b/packages/hipfire-kernels/default.nix new file mode 100644 index 0000000..c9d8590 --- /dev/null +++ b/packages/hipfire-kernels/default.nix @@ -0,0 +1,5 @@ +{ pkgs, inputs, ... }: +(import ../../_lib/llm-packages.nix { + inherit inputs; + system = pkgs.stdenv.hostPlatform.system; +}).hipfire-kernels diff --git a/packages/hipfire/default.nix b/packages/hipfire/default.nix new file mode 100644 index 0000000..fb9e18f --- /dev/null +++ b/packages/hipfire/default.nix @@ -0,0 +1,5 @@ +{ pkgs, inputs, ... }: +(import ../../_lib/llm-packages.nix { + inherit inputs; + system = pkgs.stdenv.hostPlatform.system; +}).hipfire diff --git a/packages/hipfire/package.nix b/packages/hipfire/package.nix new file mode 100644 index 0000000..35ce180 --- /dev/null +++ b/packages/hipfire/package.nix @@ -0,0 +1,123 @@ +{ + pkgs, + hipfireSource, + therockRocm, + modelDir ? "/var/lib/hipfire/models", +}: +let + lib = pkgs.lib; + gcc = pkgs.stdenv.cc; + + hipccFixed = pkgs.writeShellScriptBin "hipcc" '' + export ROCM_PATH=${therockRocm} + exec ${therockRocm}/bin/hipcc \ + --gcc-toolchain=${gcc.cc} \ + -idirafter ${gcc.libc.dev}/include \ + -B${gcc.libc}/lib \ + -L${gcc.libc}/lib \ + -L${gcc.cc.lib}/lib \ + -L${gcc.cc}/lib \ + "$@" + ''; + + glibcLoader = "${pkgs.glibc}/lib/ld-linux-x86-64.so.2"; + llvmRunPath = lib.makeSearchPathOutput "lib" "lib" [ + gcc.cc.lib + gcc.cc + ]; + therockLlvmTool = + name: + pkgs.writeShellScriptBin name '' + exec ${glibcLoader} --library-path ${therockRocm}/lib:${llvmRunPath} \ + ${therockRocm}/lib/llvm/bin/${name} "$@" + ''; + llvmReadobj = therockLlvmTool "llvm-readobj"; + llvmObjdump = therockLlvmTool "llvm-objdump"; + + therockDeviceLibs = pkgs.runCommand "therock-device-libs" { } '' + mkdir -p $out/amdgcn + ln -sfn ${therockRocm}/amdgcn/bitcode $out/amdgcn/bitcode + ''; + + hipfireKernelsPkg = + (pkgs.callPackage (hipfireSource + "/nix/kernels.nix") { + src = hipfireSource; + cargoLockFile = hipfireSource + "/Cargo.lock"; + gpuTargets = [ "gfx1151" ]; + rocmPackages = pkgs.rocmPackages // { + clr = therockRocm; + rocm-device-libs = therockDeviceLibs; + }; + }).overrideAttrs + (old: { + nativeBuildInputs = old.nativeBuildInputs ++ [ hipccFixed ]; + env.HIPFIRE_HIPCC = "${hipccFixed}/bin/hipcc"; + }); + + hipfireWithQuantize = pkgs.hipfire.overrideAttrs (_: { + buildPhase = '' + runHook preBuild + cargo build --release -p hipfire-daemon + cargo build --release -p hipfire-cli + cargo build --release -p hipfire-quantize \ + --bin hipfire-quantize --bin dflash_convert --bin draft_to_mq4 + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + + mkdir -p $out/bin + + cp target/release/daemon $out/bin/hipfire-daemon-unwrapped + makeWrapper $out/bin/hipfire-daemon-unwrapped $out/bin/hipfire-daemon \ + --set HIPFIRE_ROCM_PATH ${therockRocm} \ + --set HIPFIRE_HIPCC ${hipccFixed}/bin/hipcc \ + --set ROCM_PATH ${therockRocm} \ + --set RADIOWAVE_READOBJ ${llvmReadobj}/bin/llvm-readobj \ + --set RADIOWAVE_OBJDUMP ${llvmObjdump}/bin/llvm-objdump \ + --prefix PATH : ${ + pkgs.lib.makeBinPath [ + hipccFixed + pkgs.pciutils + ] + } \ + --prefix LD_LIBRARY_PATH : ${therockRocm}/lib + + cp target/release/hipfire $out/bin/hipfire-unwrapped + makeWrapper $out/bin/hipfire-unwrapped $out/bin/hipfire \ + --set HIPFIRE_DAEMON_BIN $out/bin/hipfire-daemon \ + --set HIPFIRE_NORMALIZE_PROMPT "1" \ + --set HIPFIRE_MODELS_DIR "${modelDir}" \ + --set HIP_PATH "${therockRocm}" \ + --set HIPFIRE_HIPCC ${hipccFixed}/bin/hipcc \ + --set ROCM_PATH ${therockRocm} \ + --set RADIOWAVE_READOBJ ${llvmReadobj}/bin/llvm-readobj \ + --set RADIOWAVE_OBJDUMP ${llvmObjdump}/bin/llvm-objdump \ + --prefix PATH : ${ + pkgs.lib.makeBinPath [ + hipccFixed + pkgs.pciutils + ] + } \ + --prefix LD_LIBRARY_PATH : ${therockRocm}/lib + + mkdir -p $out/share/hipfire/target/release + cp target/release/hipfire-quantize $out/share/hipfire/target/release/ + cp target/release/dflash_convert $out/share/hipfire/target/release/ + cp target/release/draft_to_mq4 $out/share/hipfire/target/release/ + + runHook postInstall + ''; + }); + +in +hipfireWithQuantize.overrideAttrs (old: { + passthru = (old.passthru or { }) // { + kernels = hipfireKernelsPkg; + hipcc = hipccFixed; + inherit llvmReadobj llvmObjdump; + rocm = therockRocm; + category = "AI Inference"; + }; +}) diff --git a/packages/kyojin/check-jit.py b/packages/kyojin/check-jit.py deleted file mode 100644 index 1b85500..0000000 --- a/packages/kyojin/check-jit.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Compile Kyojin's JIT kernels; --gpu also loads them through torch's HIP runtime.""" -import ctypes -import importlib -import os -from pathlib import Path -import sys - -from exllamav3.util import hip_compiler -from exllamav3.util.hip_lib import load_hip_runtime - -sdk = Path(os.environ["EXL3_ROCM_SDK"]) -assert Path(hip_compiler.hipcc()) == sdk / "bin/hipcc" -assert Path(os.environ["HIP_DEVICE_LIB_PATH"]).is_dir() -assert (sdk / "amdgcn/bitcode").is_dir() -if os.environ.get("KYOJIN_REQUIRE_SDK") == "1": - assert hip_compiler.warning() is None, hip_compiler.warning() -print(hip_compiler.describe(), flush=True) - -lib = None -if "--gpu" in sys.argv: - import torch - - # Also exercise the runtime bitcode path: merely loading a HIP module - # does not catch incompatible device libraries in CLR/COMGR. - x = torch.ones((32, 32), device="cuda", dtype=torch.float32) - assert torch.equal(x @ x, torch.full_like(x, 32)) - torch.cuda.synchronize() - lib = load_hip_runtime() - lib.hipModuleLoad.argtypes = [ctypes.POINTER(ctypes.c_void_p), ctypes.c_char_p] - lib.hipModuleUnload.argtypes = [ctypes.c_void_p] - -for module, function, args in [ - ("exllamav3.modules.gr_fn.gr_mix_hip", "compile_hsaco", ()), - ("exllamav3.modules.attention_fn.qsa_prefill_hip", "compile_hsaco", ()), - ("exllamav3.modules.ple_fn.ple_hip", "compile_hsaco", ()), - ("exllamav3.vendor.fla.hip.gdn_fused_h_hip", "_compile", ("", "gfx1151")), - ("exllamav3.vendor.fla.hip.kda_fused_h_hip", "_compile", ("", "gfx1151")), -]: - path = Path(getattr(importlib.import_module(module), function)(*args)) - # HIP accepts ELF, offload bundles, and compressed offload bundles (CCOB). - assert path.read_bytes().startswith((b"\x7fELF", b"__CLANG_OFFLOAD_BUNDLE__", b"CCOB")), path - if lib is not None: - handle = ctypes.c_void_p() - assert lib.hipModuleLoad(ctypes.byref(handle), os.fsencode(path)) == 0, path - assert lib.hipModuleUnload(handle) == 0, path - print(f"OK: {module}", flush=True) diff --git a/packages/kyojin/default.nix b/packages/kyojin/default.nix index a2bdcad..5fd0544 100644 --- a/packages/kyojin/default.nix +++ b/packages/kyojin/default.nix @@ -1,13 +1,5 @@ -{ - pkgs, - inputs, - jitRocmSdk ? null, - ... -}: -# The HIP extension has to be compiled against a ROCm build of PyTorch, and the -# only ROCm torch in this flake with a cached binary is the one from the pinned -# nixpkgs-torch211 input (2.11 + ROCm 7.2, python 3.13). Current nixos-unstable -# ships torch 2.13, whose ROCm build fails to configure in nixpkgs and is not -# on cache.nixos.org. -inputs.nixpkgs-torch211.legacyPackages.${pkgs.stdenv.hostPlatform.system}.callPackage ./package.nix - { inherit jitRocmSdk; } +{ pkgs, inputs, ... }: +(import ../../_lib/llm-packages.nix { + inherit inputs; + system = pkgs.stdenv.hostPlatform.system; +}).kyojin diff --git a/packages/kyojin/package.nix b/packages/kyojin/package.nix index ff3781f..a3c7889 100644 --- a/packages/kyojin/package.nix +++ b/packages/kyojin/package.nix @@ -1,230 +1,30 @@ { - lib, - stdenv, pkgs, - python3, - fetchFromGitHub, - makeWrapper, - rocmPackages, - # Override only the serving toolchain; torch and the extension keep their ROCm. - jitRocmSdk ? null, + kyojin, + jitCheck, + cpuCheck, }: - -let - py = python3.pkgs; - - # Kyojin is the Yamz fork of ExLlamaV3: a Python package plus a C++ extension - # that torch's cpp_extension cross-compiles to HIP code objects. Upstream - # only supports AMD Strix Halo, so the kernels are built for gfx1151 alone. - rocmArch = "gfx1151"; - - version = "1.8.5"; - - src = fetchFromGitHub { - owner = "Yamz-Labs"; - repo = "kyojin"; - rev = "v${version}"; - hash = "sha256-OZhzRnQUrkJOrn2lVatvVxD6JSwtJkwfWwpRUQHawac="; +kyojin.overrideAttrs (old: { + nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ pkgs.makeWrapper ]; + meta = (old.meta or { }) // { + homepage = "https://github.com/Yamz-Labs/kyojin"; + license = pkgs.lib.licenses.mit; }; - - # setup.py passes EXL3_ROCM_DEV_INCLUDE as -I to every translation unit. The - # AMD wheels ship one devel tree holding hipsparse/ and thrust/; nixpkgs - # splits them per package, so join the include dirs. torch's public HIP - # headers also pull hipsolver/, and it no longer vendors pybind11. - rocmDevInclude = pkgs.symlinkJoin { - name = "kyojin-rocm-dev-include"; - paths = map lib.getDev ( - [ py.pybind11 ] - ++ [ - rocmPackages.clr - rocmPackages.hipblas - rocmPackages.hipblaslt - rocmPackages.hipcub - rocmPackages.hipfft - rocmPackages.hiprand - rocmPackages.hipsparse - rocmPackages.hipsolver - rocmPackages.rocblas - rocmPackages.rocprim - rocmPackages.rocrand - rocmPackages.rocsolver - rocmPackages.rocsparse - rocmPackages.rocthrust - ] - ); - }; - - # torch's HIP path in cpp_extension expects one ROCM_HOME holding bin/hipcc, - # the headers and the amdgcn bitcode; nixpkgs ships them separately. - rocmToolkit = pkgs.symlinkJoin { - name = "kyojin-rocm-toolkit"; - paths = [ - rocmPackages."rocm-core" - rocmPackages."rocm-runtime" - rocmPackages."rocm-device-libs" - rocmPackages."rocm-comgr" - rocmPackages.clr - (lib.getBin rocmPackages.hipcc) - rocmPackages.rocblas - rocmPackages.hipblas - rocmPackages.hipblas-common - rocmPackages.hipblaslt - rocmPackages.hipsparse - rocmPackages.hipsolver - rocmPackages.hiprand - rocmPackages.rocrand - rocmPackages.hipfft - rocmPackages.rocsparse - rocmPackages.rocsolver - rocmPackages.rocthrust - rocmPackages.rocprim - rocmPackages.hipcub - ]; - # The setup hooks of the joined packages would rewrite build flags; the - # compiler only needs the files. - postBuild = '' - rm -rf $out/nix-support - ''; - }; - - # The library: exllamav3 plus the compiled exllamav3_ext module. - library = py.buildPythonPackage { - inherit src version; - pname = "kyojin"; - - format = "pyproject"; - build-system = with py; [ - setuptools - wheel - ]; - - nativeBuildInputs = [ - py.ninja - rocmPackages."rocm-runtime" - ]; - - buildInputs = [ (lib.getBin rocmPackages.hipcc) ]; - - propagatedBuildInputs = with py; [ - aiohttp - huggingface-hub - jinja2 - llguidance - marisa-trie - ninja - numpy - pillow - pydantic - pyyaml - rich - safetensors - tokenizers - torchWithRocm - typing-extensions - ]; - - env = { - PYTORCH_ROCM_ARCH = rocmArch; - ROCM_PATH = "${rocmToolkit}"; - ROCM_HOME = "${rocmToolkit}"; - HIP_PATH = "${rocmToolkit}"; - HIPCC = "${rocmToolkit}/bin/hipcc"; - HIP_DEVICE_LIB_PATH = "${rocmPackages."rocm-device-libs"}/amdgcn/bitcode"; - # Upstream build scripts look these up to find the ROCm devel tree. - EXL3_ROCM_SDK = "${rocmToolkit}"; - EXL3_ROCM_DEV_INCLUDE = "${rocmDevInclude}/include"; - # Default defines of upstream build.sh (tools/strix_halo/rebuild.sh). - EXL3_HIP_DEFINES = "EXL3_HIP_STG_PAD"; - }; - - # setup.py loads exllamav3/exllamav3_ext/build_config.py by path (and - # pyproject.toml ships the .hip sources, qsa_proof markers, and dense-GEMM - # tuning seed), so no source patches are needed. - - # tests/ needs a real gfx1151 GPU and the published model packs. - doCheck = false; - - meta = with lib; { - description = "Inference engine for 100 GB-class EXL3 MoE models on AMD Strix Halo (gfx1151, ROCm 7)"; - homepage = "https://github.com/Yamz-Labs/kyojin"; - changelog = "https://github.com/Yamz-Labs/kyojin/releases"; - license = licenses.mit; - sourceProvenance = with sourceTypes; [ fromSource ]; - # HIP code objects are gfx1151-only and ROCm is Linux-only. - platforms = platforms.linux; - }; - }; - - python = py.python.withPackages (_: [ library ]); - jitSdk = if jitRocmSdk == null then rocmToolkit else jitRocmSdk; -in -stdenv.mkDerivation { - inherit src version; - pname = "kyojin"; - - # Only the wrappers are installed here: the serve scripts are plain scripts, - # they import their siblings by path, and the module itself is in `library`. - # - # Upstream `tools/strix_halo/env.sh` is sourced before serving, and on a - # distro box that brings a system `cc` (the AMD Triton backend compiles a - # CPython glue module at runtime, and again per kernel launcher) plus a hipcc - # for the JIT HIP kernels (qsa prefill, gdn, kda, ple). Nix gives the wrapper - # neither, so export them here instead: Triton takes the compiler from $CC, - # exllamav3 finds hipcc under $EXL3_ROCM_SDK, and hipclang only finds the - # amdgcn bitcode through $HIP_DEVICE_LIB_PATH (hipcc's own DEVICE_LIB_PATH is - # not enough for a --genco call). The ROCm paths stay out of LD_LIBRARY_PATH - # (upstream Trap 1/5: a second libhsa segfaults torch). - dontConfigure = true; - dontBuild = true; - - nativeBuildInputs = [ makeWrapper ]; - - installPhase = '' - runHook preInstall - - mkdir -p $out/bin - for model in glm mimo qwen; do - makeWrapper ${python}/bin/python $out/bin/kyojin-serve-$model \ - --set PYTORCH_ROCM_ARCH ${rocmArch} \ - --set CC ${stdenv.cc}/bin/cc \ - --set EXL3_ROCM_SDK ${jitSdk} \ - --set HIP_DEVICE_LIB_PATH ${rocmPackages."rocm-device-libs"}/amdgcn/bitcode \ - --set EXL3_EXPANDABLE_SEGMENTS 0 \ - --prefix PATH : ${lib.makeBinPath [ stdenv.cc ]} \ - --add-flags "${src}/tools/$model/serve.py" - done - - runHook postInstall - ''; - - passthru = { - inherit python rocmArch; - pythonPackage = library; - jitRocmSdk = jitSdk; - tests.jit = - pkgs.runCommand "kyojin-jit-check" - { - EXL3_ROCM_SDK = jitSdk; - # CLR/COMGR also reads this variable. Newer bitcode crashes torch - # matmul; a custom hipcc must select its own bitcode in its subprocess. - HIP_DEVICE_LIB_PATH = "${rocmPackages."rocm-device-libs"}/amdgcn/bitcode"; - EXL3_EXPANDABLE_SEGMENTS = "0"; - KYOJIN_REQUIRE_SDK = lib.boolToString (jitRocmSdk != null); - } - '' - export HOME="$TMPDIR" - ${python}/bin/python ${./check-jit.py} - touch "$out" - ''; + passthru = (old.passthru or { }) // { category = "AI Inference"; - updateScript = [ - "nix-update" - "--flake" - ".#kyojin" - ]; + tests = (old.passthru.tests or { }) // { + jit = jitCheck; + cpu = cpuCheck; + }; }; - - meta = library.meta // { - mainProgram = "kyojin-serve-qwen"; - }; -} + postFixup = '' + # Fail the build, not the first request, if upstream renames the wrappers. + ls "$out"/bin/kyojin-serve-* > /dev/null 2>&1 || { + echo "kyojin-cc: no kyojin-serve-* in $out/bin; upstream moved them" >&2 + exit 1 + } + for b in $out/bin/kyojin-serve-*; do + wrapProgram "$b" --set CC ${pkgs.stdenv.cc}/bin/gcc + done + ''; +}) diff --git a/packages/pwilkin-llama-cpp/default.nix b/packages/pwilkin-llama-cpp/default.nix new file mode 100644 index 0000000..b3de5c8 --- /dev/null +++ b/packages/pwilkin-llama-cpp/default.nix @@ -0,0 +1,5 @@ +{ pkgs, inputs, ... }: +(import ../../_lib/llm-packages.nix { + inherit inputs; + system = pkgs.stdenv.hostPlatform.system; +}).pwilkin-llama-cpp diff --git a/packages/pwilkin-llama-cpp/package.nix b/packages/pwilkin-llama-cpp/package.nix new file mode 100644 index 0000000..a69fed7 --- /dev/null +++ b/packages/pwilkin-llama-cpp/package.nix @@ -0,0 +1,6 @@ +{ pkgs, source }: +pkgs.callPackage (source + "/.devops/nix/package.nix") { + llamaVersion = source.shortRev or "0.0.0"; + useRocm = true; + rocmGpuTargets = "gfx1151"; +} diff --git a/packages/strix-llama-cpp/default.nix b/packages/strix-llama-cpp/default.nix new file mode 100644 index 0000000..161f340 --- /dev/null +++ b/packages/strix-llama-cpp/default.nix @@ -0,0 +1,5 @@ +{ pkgs, inputs, ... }: +(import ../../_lib/llm-packages.nix { + inherit inputs; + system = pkgs.stdenv.hostPlatform.system; +}).strix-llama-cpp diff --git a/packages/strix-llama-cpp/package.nix b/packages/strix-llama-cpp/package.nix new file mode 100644 index 0000000..7996296 --- /dev/null +++ b/packages/strix-llama-cpp/package.nix @@ -0,0 +1,5 @@ +{ pkgs, source }: +pkgs.callPackage (source + "/.devops/nix/package.nix") { + llamaVersion = source.shortRev or "0.0.0"; + useVulkan = true; +} diff --git a/packages/therock-rocm/default.nix b/packages/therock-rocm/default.nix new file mode 100644 index 0000000..c5c4c13 --- /dev/null +++ b/packages/therock-rocm/default.nix @@ -0,0 +1,5 @@ +{ pkgs, inputs, ... }: +(import ../../_lib/llm-packages.nix { + inherit inputs; + system = pkgs.stdenv.hostPlatform.system; +}).therock-rocm diff --git a/packages/therock-rocm/package.nix b/packages/therock-rocm/package.nix new file mode 100644 index 0000000..0be5717 --- /dev/null +++ b/packages/therock-rocm/package.nix @@ -0,0 +1,10 @@ +{ pkgs, nix-strix-halo }: +let + source = + (builtins.fromJSON (builtins.readFile (nix-strix-halo + "/pkgs/therock/sources/rocm.json"))) + .linux.gfx1151; +in +pkgs.callPackage (nix-strix-halo + "/pkgs/therock/rocm-sdk") { + target = "gfx1151"; + inherit (source) version url hash; +} diff --git a/packages/xrt-amdxdna/default.nix b/packages/xrt-amdxdna/default.nix new file mode 100644 index 0000000..05e2356 --- /dev/null +++ b/packages/xrt-amdxdna/default.nix @@ -0,0 +1,5 @@ +{ pkgs, inputs, ... }: +(import ../../_lib/llm-packages.nix { + inherit inputs; + system = pkgs.stdenv.hostPlatform.system; +}).xrt-amdxdna