feat(llm): package upstream inference engines
Centralize engine pins, compiler wrappers and NPU packages. Use upstream Kyojin v1.8.10 and drop the Thunderbolt dependency. Offline evaluation passed; workstation builds remain pending. Bead: nix-overlay-v09
This commit is contained in:
File diff suppressed because one or more lines are too long
@@ -37,7 +37,14 @@ A custom Nix overlay and flake providing additional packages not found in upstre
|
|||||||
| `shardr` | Decentralized LLM repository: content-addressed model store, BitTorrent-based sync, OpenAI-compatible serving via llama-server | AI Inference |
|
| `shardr` | Decentralized LLM repository: content-addressed model store, BitTorrent-based sync, OpenAI-compatible serving via llama-server | AI Inference |
|
||||||
| `skillsmcp` | MCP server that exposes Agent Skills to AI agents via the Model Context Protocol | MCP Servers |
|
| `skillsmcp` | MCP server that exposes Agent Skills to AI agents via the Model Context Protocol | MCP Servers |
|
||||||
| `kubernetes-mcp-server` | Model Context Protocol (MCP) server for Kubernetes and OpenShift | MCP Servers |
|
| `kubernetes-mcp-server` | Model Context Protocol (MCP) server for Kubernetes and OpenShift | MCP Servers |
|
||||||
| `kyojin` | ExLlamaV3-based inference engine with speculative decoding for 100 GB-class MoE packs on one 128 GB AMD Strix Halo (gfx1151, ROCm 7) | AI Inference |
|
| `kyojin` | Upstream Kyojin 1.8.10 with AMD ROCm SDK wheels and a host C compiler for Triton JIT | AI Inference |
|
||||||
|
| `hipfire` | AMD RDNA inference engine with TheRock compiler wrappers and quantization tools | AI Inference |
|
||||||
|
| `hipfire-kernels` | Precompiled hipfire kernels for gfx1151 | AI Inference |
|
||||||
|
| `gufo` | Gufo with XDNA2 NPU support and the Unsloth UD 3.1 compatibility patch | AI Inference |
|
||||||
|
| `xrt-amdxdna` | XRT with the amdxdna plugin, built against gufo's nixpkgs | AI Inference |
|
||||||
|
| `therock-rocm` | TheRock 10.0 SDK for gfx1151 | AI Inference |
|
||||||
|
| `pwilkin-llama-cpp` | pwilkin llama.cpp fork with ROCm for gfx1151 | AI Inference |
|
||||||
|
| `strix-llama-cpp` | halo-box llama.cpp fork with Vulkan | AI Inference |
|
||||||
| `loop` | Corporate messenger for your team | Communication |
|
| `loop` | Corporate messenger for your team | Communication |
|
||||||
| `radar` | Modern Kubernetes visibility — topology, event timeline, service traffic, resource browsing, Helm management, and GitOps support | Kubernetes |
|
| `radar` | Modern Kubernetes visibility — topology, event timeline, service traffic, resource browsing, Helm management, and GitOps support | Kubernetes |
|
||||||
| `omniroute` | Unified AI router with 160+ providers, auto fallback, MCP/A2A, OpenAI-compatible APIs | AI LLM Gateway |
|
| `omniroute` | Unified AI router with 160+ providers, auto fallback, MCP/A2A, OpenAI-compatible APIs | AI LLM Gateway |
|
||||||
@@ -102,29 +109,29 @@ nix run git+https://git.millerson.name/alex/millerson-overlay.nix.git#mcp-gatewa
|
|||||||
nix profile install git+https://git.millerson.name/alex/millerson-overlay.nix.git#mcp-gateway
|
nix profile install git+https://git.millerson.name/alex/millerson-overlay.nix.git#mcp-gateway
|
||||||
```
|
```
|
||||||
|
|
||||||
### Kyojin JIT Toolchain
|
### Inference Engines
|
||||||
|
|
||||||
Kyojin's serving wrappers use the packaged ROCm compiler by default. Supply a
|
The engine packages target AMD Strix Halo (`gfx1151`) on `x86_64-linux`.
|
||||||
pinned SDK to compile HIP JIT kernels with another compiler without rebuilding
|
`nixpkgs-engine` pins the shared engine toolchain independently of the other
|
||||||
PyTorch or the HIP extension:
|
packages. Gufo and Kyojin retain their upstream nixpkgs pins.
|
||||||
|
|
||||||
```nix
|
Kyojin uses the upstream 1.8.10 flake with its AMD ROCm SDK wheels. The serving
|
||||||
kyojin.override { jitRocmSdk = myJitSdk; }
|
wrappers also set `CC` for Triton's runtime C compilation. The upstream derivation
|
||||||
```
|
still reports version `1.5.8`; the source pin is the `v1.8.10` tag.
|
||||||
|
`kyojin.tests.jit` and `kyojin.tests.cpu` expose the upstream HIP compiler smoke
|
||||||
|
check and CPU tests. The former nixpkgs
|
||||||
|
PyTorch/ROCm package and its `jitRocmSdk` override have been removed.
|
||||||
|
|
||||||
The SDK must provide `bin/hipcc` and matching `amdgcn/bitcode`. Its compiler
|
Hipfire exports `kernels`, `hipcc`, `rocm`, `llvmReadobj` and `llvmObjdump` as
|
||||||
wrapper must set `HIP_DEVICE_LIB_PATH` to that bitcode **in the compiler
|
package attributes. Set the model directory with
|
||||||
subprocess only**, along with any required `ROCM_PATH` and Nix C++ toolchain
|
`hipfire.override { modelDir = "/path/to/models"; }`.
|
||||||
flags. Keep the compiler wrapper's real path inside the SDK prefix so upstream's
|
The upstream service module is available as `nixosModules.hipfire`, and its
|
||||||
compiler provenance check recognizes it.
|
ROCm overlay as `overlays.hipfire`. Neither is enabled by the default module.
|
||||||
|
|
||||||
The serving process keeps PyTorch's ROCm device libraries. Setting TheRock's
|
Gufo exports its matching `xrt`, `npuDir` and `fabricClock` as package attributes.
|
||||||
Clang 23 bitcode globally makes the ROCm 7.2 runtime crash on a basic PyTorch
|
The consumer must configure NPU power and fabric clocks before using the NPU.
|
||||||
matrix multiplication. The serving wrappers also disable unsupported expandable
|
TheRock and XRT use individual recipes from the pinned `nix-strix-halo` source
|
||||||
allocator segments. `nix build .#kyojin.tests.jit` checks compilation of five HIP
|
tree. Its flake and Thunderbolt dependencies are not loaded.
|
||||||
kernel modules without requiring a GPU; `packages/kyojin/check-jit.py --gpu`, run
|
|
||||||
with the package's Python environment and serving variables, additionally checks
|
|
||||||
PyTorch matrix multiplication and module loading through its existing HIP runtime.
|
|
||||||
|
|
||||||
### nftablesbuilder Runtime Notes
|
### nftablesbuilder Runtime Notes
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,42 @@
|
|||||||
|
{ inputs, system }:
|
||||||
|
let
|
||||||
|
pkgs = import inputs.nixpkgs-engine {
|
||||||
|
inherit system;
|
||||||
|
config.allowUnfree = true;
|
||||||
|
overlays = [ inputs.hipfire.overlays.default ];
|
||||||
|
};
|
||||||
|
therock-rocm = pkgs.callPackage ../packages/therock-rocm/package.nix {
|
||||||
|
inherit (inputs) nix-strix-halo;
|
||||||
|
};
|
||||||
|
hipfire = pkgs.callPackage ../packages/hipfire/package.nix {
|
||||||
|
hipfireSource = inputs.hipfire;
|
||||||
|
therockRocm = therock-rocm;
|
||||||
|
};
|
||||||
|
gufo = import ../packages/gufo/package.nix { inherit inputs system; };
|
||||||
|
in
|
||||||
|
builtins.mapAttrs
|
||||||
|
(
|
||||||
|
_: package:
|
||||||
|
if system == "x86_64-linux" then
|
||||||
|
package
|
||||||
|
else
|
||||||
|
{
|
||||||
|
meta.platforms = [ "x86_64-linux" ];
|
||||||
|
}
|
||||||
|
)
|
||||||
|
{
|
||||||
|
inherit hipfire therock-rocm gufo;
|
||||||
|
hipfire-kernels = hipfire.kernels;
|
||||||
|
xrt-amdxdna = gufo.xrt;
|
||||||
|
kyojin = pkgs.callPackage ../packages/kyojin/package.nix {
|
||||||
|
kyojin = inputs.kyojin.packages.${system}.kyojin;
|
||||||
|
jitCheck = inputs.kyojin.checks.${system}.kyojin-hipcc-smoke;
|
||||||
|
cpuCheck = inputs.kyojin.checks.${system}.kyojin-tests;
|
||||||
|
};
|
||||||
|
pwilkin-llama-cpp = pkgs.callPackage ../packages/pwilkin-llama-cpp/package.nix {
|
||||||
|
source = inputs.pwilkin-llama-cpp;
|
||||||
|
};
|
||||||
|
strix-llama-cpp = pkgs.callPackage ../packages/strix-llama-cpp/package.nix {
|
||||||
|
source = inputs.strix-llama-cpp;
|
||||||
|
};
|
||||||
|
}
|
||||||
Generated
+452
-6
@@ -1,5 +1,26 @@
|
|||||||
{
|
{
|
||||||
"nodes": {
|
"nodes": {
|
||||||
|
"audio-cpp": {
|
||||||
|
"inputs": {
|
||||||
|
"nixpkgs": [
|
||||||
|
"gufo",
|
||||||
|
"nixpkgs"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1789940897,
|
||||||
|
"narHash": "sha256-+9tVcd+0TkDxg7JRU7M06Lvm20sXEbuerm62uhm6JKA=",
|
||||||
|
"owner": "0xShug0",
|
||||||
|
"repo": "audio.cpp",
|
||||||
|
"rev": "e3de8e3f3cbfac55ffa58df71426c41550a8598b",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "0xShug0",
|
||||||
|
"repo": "audio.cpp",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
"blueprint": {
|
"blueprint": {
|
||||||
"inputs": {
|
"inputs": {
|
||||||
"nixpkgs": [
|
"nixpkgs": [
|
||||||
@@ -53,6 +74,22 @@
|
|||||||
"type": "github"
|
"type": "github"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"flake-compat": {
|
||||||
|
"flake": false,
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1767039857,
|
||||||
|
"narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=",
|
||||||
|
"owner": "NixOS",
|
||||||
|
"repo": "flake-compat",
|
||||||
|
"rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "NixOS",
|
||||||
|
"repo": "flake-compat",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
"flake-parts": {
|
"flake-parts": {
|
||||||
"inputs": {
|
"inputs": {
|
||||||
"nixpkgs-lib": [
|
"nixpkgs-lib": [
|
||||||
@@ -73,13 +110,170 @@
|
|||||||
"type": "github"
|
"type": "github"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"flake-parts_2": {
|
||||||
|
"inputs": {
|
||||||
|
"nixpkgs-lib": "nixpkgs-lib"
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1790898470,
|
||||||
|
"narHash": "sha256-zTwuwezD0w3hpga6a6P8emidzAZFyWqlNeRBlpWOOl8=",
|
||||||
|
"owner": "hercules-ci",
|
||||||
|
"repo": "flake-parts",
|
||||||
|
"rev": "024633cd702b10285db5cb19b40ad48d2399ba60",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "hercules-ci",
|
||||||
|
"repo": "flake-parts",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"flake-utils": {
|
||||||
|
"inputs": {
|
||||||
|
"systems": "systems"
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1731533236,
|
||||||
|
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
|
||||||
|
"owner": "numtide",
|
||||||
|
"repo": "flake-utils",
|
||||||
|
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "numtide",
|
||||||
|
"repo": "flake-utils",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"git-hooks-nix": {
|
||||||
|
"inputs": {
|
||||||
|
"flake-compat": "flake-compat",
|
||||||
|
"nixpkgs": [
|
||||||
|
"kyojin",
|
||||||
|
"nixpkgs"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1790500375,
|
||||||
|
"narHash": "sha256-XN3sDtn8TU9hAc9xqZ+SqRB/HHm2wjxM6aSWYLJU+oo=",
|
||||||
|
"owner": "cachix",
|
||||||
|
"repo": "git-hooks.nix",
|
||||||
|
"rev": "a0e4241b51206fbcbf52fd322eb5f0cd80f153c4",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "cachix",
|
||||||
|
"repo": "git-hooks.nix",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"gufo": {
|
||||||
|
"inputs": {
|
||||||
|
"audio-cpp": "audio-cpp",
|
||||||
|
"nixpkgs": "nixpkgs"
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1793664946,
|
||||||
|
"narHash": "sha256-fsARElSJAbcRu+OBEZXfE+m90W9Cc/696rMaFhGiSNQ=",
|
||||||
|
"owner": "mikealanni",
|
||||||
|
"repo": "gufo",
|
||||||
|
"rev": "6a107cf6aeba7f28db266c2ba100bfb274f65e5a",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "mikealanni",
|
||||||
|
"ref": "fork-enhancements",
|
||||||
|
"repo": "gufo",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"hipfire": {
|
||||||
|
"inputs": {
|
||||||
|
"flake-utils": "flake-utils",
|
||||||
|
"nixpkgs": [
|
||||||
|
"nixpkgs-engine"
|
||||||
|
],
|
||||||
|
"rust-overlay": "rust-overlay"
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1793080987,
|
||||||
|
"narHash": "sha256-g2S6zvHD4BeIpVpVKQDlnE5V7A+5kj9MPR7TARgmIo8=",
|
||||||
|
"owner": "warpfront",
|
||||||
|
"repo": "hipfire",
|
||||||
|
"rev": "75c28fc06486457139e47a4abc2df32dd55470f0",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "warpfront",
|
||||||
|
"ref": "v0.4.4",
|
||||||
|
"repo": "hipfire",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"kyojin": {
|
||||||
|
"inputs": {
|
||||||
|
"flake-parts": "flake-parts_2",
|
||||||
|
"git-hooks-nix": "git-hooks-nix",
|
||||||
|
"nixpkgs": "nixpkgs_2",
|
||||||
|
"pyproject-build-systems": "pyproject-build-systems",
|
||||||
|
"pyproject-nix": "pyproject-nix",
|
||||||
|
"uv2nix": "uv2nix"
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1793864505,
|
||||||
|
"narHash": "sha256-EGLX2sHByKsbRx1+p9TZRF2YKE2MkniS7+0o/x+q4mA=",
|
||||||
|
"owner": "Yamz-Labs",
|
||||||
|
"repo": "kyojin",
|
||||||
|
"rev": "d83cfeb3d6a7d42a4cf2b720efb1eb8a4ff04da7",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "Yamz-Labs",
|
||||||
|
"ref": "v1.8.10",
|
||||||
|
"repo": "kyojin",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nix-strix-halo": {
|
||||||
|
"flake": false,
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1793862511,
|
||||||
|
"narHash": "sha256-4JsLvkORL3XdZ1mbWbqi+uTvkzlQRbM3YlH2iUGbZ9M=",
|
||||||
|
"owner": "hellas-ai",
|
||||||
|
"repo": "nix-strix-halo",
|
||||||
|
"rev": "b2969bd90fedc4a1f9d91648bfdf8f21479b8178",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "hellas-ai",
|
||||||
|
"repo": "nix-strix-halo",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
"nixpkgs": {
|
"nixpkgs": {
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1789149629,
|
"lastModified": 1786384358,
|
||||||
"narHash": "sha256-H6GwaZzZf+4npqv0tph94w9tZddSjFjmQrVsW0z78uk=",
|
"narHash": "sha256-RzPPiWeUtuvymnpuEWsdtzli5w4kjZs49FqEs3/1u+I=",
|
||||||
"owner": "NixOS",
|
"owner": "NixOS",
|
||||||
"repo": "nixpkgs",
|
"repo": "nixpkgs",
|
||||||
"rev": "eaad089433ca2bb662274377d33df3d0e51ef28b",
|
"rev": "2fcb964de67fcf60b43471c55d5d99e61a9ccb5a",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "NixOS",
|
||||||
|
"ref": "nixos-unstable",
|
||||||
|
"repo": "nixpkgs",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nixpkgs-engine": {
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1793726008,
|
||||||
|
"narHash": "sha256-jQp1wKvbwWdLBASTXkSi/VYmDJ5AbBi1WvcMt9HWJC8=",
|
||||||
|
"owner": "NixOS",
|
||||||
|
"repo": "nixpkgs",
|
||||||
|
"rev": "1d060086d60cb76a40cf37109f979d88a5bd6396",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
@@ -105,6 +299,21 @@
|
|||||||
"type": "github"
|
"type": "github"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"nixpkgs-lib": {
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1790477367,
|
||||||
|
"narHash": "sha256-oujbxmVKDhci2E4EM3KyAbRUgp5/IsCoiA/LfEF7OOk=",
|
||||||
|
"owner": "nix-community",
|
||||||
|
"repo": "nixpkgs.lib",
|
||||||
|
"rev": "f7cd230690d9fc982d06129f9189b101fddcfa3b",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "nix-community",
|
||||||
|
"repo": "nixpkgs.lib",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
"nixpkgs-rust195": {
|
"nixpkgs-rust195": {
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1793633738,
|
"lastModified": 1793633738,
|
||||||
@@ -137,17 +346,176 @@
|
|||||||
"type": "github"
|
"type": "github"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"nixpkgs_2": {
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1791456040,
|
||||||
|
"narHash": "sha256-H83yz/do+hjOO0uujHyH8RgtIQP2v7Mw+cLwcZK/xKc=",
|
||||||
|
"owner": "NixOS",
|
||||||
|
"repo": "nixpkgs",
|
||||||
|
"rev": "e7439b6b14ad3cc35d05608ebca9bce01a25f5f8",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "NixOS",
|
||||||
|
"ref": "nixos-unstable",
|
||||||
|
"repo": "nixpkgs",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nixpkgs_3": {
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1789149629,
|
||||||
|
"narHash": "sha256-H6GwaZzZf+4npqv0tph94w9tZddSjFjmQrVsW0z78uk=",
|
||||||
|
"owner": "NixOS",
|
||||||
|
"repo": "nixpkgs",
|
||||||
|
"rev": "eaad089433ca2bb662274377d33df3d0e51ef28b",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "NixOS",
|
||||||
|
"ref": "nixos-unstable",
|
||||||
|
"repo": "nixpkgs",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"pwilkin-llama-cpp": {
|
||||||
|
"inputs": {
|
||||||
|
"flake-parts": [
|
||||||
|
"flake-parts"
|
||||||
|
],
|
||||||
|
"nixpkgs": [
|
||||||
|
"nixpkgs-engine"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1789554350,
|
||||||
|
"narHash": "sha256-xZcqZroS3rUid6w7bVeilixJxUUFDq5S1EeypWCiZBM=",
|
||||||
|
"owner": "pwilkin",
|
||||||
|
"repo": "llama.cpp",
|
||||||
|
"rev": "b0f31f5876ef3856b55f5bb88072cc96e5effafe",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "pwilkin",
|
||||||
|
"ref": "strix-halo",
|
||||||
|
"repo": "llama.cpp",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"pyproject-build-systems": {
|
||||||
|
"inputs": {
|
||||||
|
"nixpkgs": [
|
||||||
|
"kyojin",
|
||||||
|
"nixpkgs"
|
||||||
|
],
|
||||||
|
"pyproject-nix": [
|
||||||
|
"kyojin",
|
||||||
|
"pyproject-nix"
|
||||||
|
],
|
||||||
|
"uv2nix": [
|
||||||
|
"kyojin",
|
||||||
|
"uv2nix"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1791170410,
|
||||||
|
"narHash": "sha256-pyru9MpuXRiyq0Bd4q0nHpL4T6HCuZT4B+alcgAYE4A=",
|
||||||
|
"owner": "pyproject-nix",
|
||||||
|
"repo": "build-system-pkgs",
|
||||||
|
"rev": "953d35d03a51dbecaf3feb3fb679eb63e90ebeb4",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "pyproject-nix",
|
||||||
|
"repo": "build-system-pkgs",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"pyproject-nix": {
|
||||||
|
"inputs": {
|
||||||
|
"nixpkgs": [
|
||||||
|
"kyojin",
|
||||||
|
"nixpkgs"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1788394333,
|
||||||
|
"narHash": "sha256-xYOP0Xym1eVGUaHFx+eJEaycFGoiCbst3fZf0xFHP78=",
|
||||||
|
"owner": "pyproject-nix",
|
||||||
|
"repo": "pyproject.nix",
|
||||||
|
"rev": "6a8a7881d75b6f98967e7b8069f4ead331384301",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "pyproject-nix",
|
||||||
|
"repo": "pyproject.nix",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
"root": {
|
"root": {
|
||||||
"inputs": {
|
"inputs": {
|
||||||
"blueprint": "blueprint",
|
"blueprint": "blueprint",
|
||||||
"bun2nix": "bun2nix",
|
"bun2nix": "bun2nix",
|
||||||
"flake-parts": "flake-parts",
|
"flake-parts": "flake-parts",
|
||||||
"nixpkgs": "nixpkgs",
|
"gufo": "gufo",
|
||||||
|
"hipfire": "hipfire",
|
||||||
|
"kyojin": "kyojin",
|
||||||
|
"nix-strix-halo": "nix-strix-halo",
|
||||||
|
"nixpkgs": "nixpkgs_3",
|
||||||
|
"nixpkgs-engine": "nixpkgs-engine",
|
||||||
"nixpkgs-latest": "nixpkgs-latest",
|
"nixpkgs-latest": "nixpkgs-latest",
|
||||||
"nixpkgs-rust195": "nixpkgs-rust195",
|
"nixpkgs-rust195": "nixpkgs-rust195",
|
||||||
"nixpkgs-torch211": "nixpkgs-torch211",
|
"nixpkgs-torch211": "nixpkgs-torch211",
|
||||||
"systems": "systems",
|
"pwilkin-llama-cpp": "pwilkin-llama-cpp",
|
||||||
"treefmt-nix": "treefmt-nix"
|
"strix-llama-cpp": "strix-llama-cpp",
|
||||||
|
"systems": "systems_2",
|
||||||
|
"treefmt-nix": "treefmt-nix",
|
||||||
|
"xdna-driver-src": "xdna-driver-src",
|
||||||
|
"xrt-src": "xrt-src"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"rust-overlay": {
|
||||||
|
"inputs": {
|
||||||
|
"nixpkgs": [
|
||||||
|
"hipfire",
|
||||||
|
"nixpkgs"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1778123869,
|
||||||
|
"narHash": "sha256-hV9D8ET33kXjdoMXBT2bwM/j8WQM1SP/dVKZtjQKhAQ=",
|
||||||
|
"owner": "oxalica",
|
||||||
|
"repo": "rust-overlay",
|
||||||
|
"rev": "592e5dedf04f0eaff1ed0f01ce5db7407d9fc7be",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "oxalica",
|
||||||
|
"repo": "rust-overlay",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"strix-llama-cpp": {
|
||||||
|
"inputs": {
|
||||||
|
"flake-parts": [
|
||||||
|
"flake-parts"
|
||||||
|
],
|
||||||
|
"nixpkgs": [
|
||||||
|
"nixpkgs-engine"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1793787912,
|
||||||
|
"narHash": "sha256-2TlKGpTiwIDoFiWwGEYLcXrEJ5eZR7JeZiUjQMgJ8eY=",
|
||||||
|
"owner": "halo-box",
|
||||||
|
"repo": "strix-llama.cpp",
|
||||||
|
"rev": "ea5285ddfc4eb2e457886c4105c77a0cece5f205",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "halo-box",
|
||||||
|
"repo": "strix-llama.cpp",
|
||||||
|
"type": "github"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"systems": {
|
"systems": {
|
||||||
@@ -165,6 +533,21 @@
|
|||||||
"type": "github"
|
"type": "github"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"systems_2": {
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1681028828,
|
||||||
|
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||||
|
"owner": "nix-systems",
|
||||||
|
"repo": "default",
|
||||||
|
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "nix-systems",
|
||||||
|
"repo": "default",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
"treefmt-nix": {
|
"treefmt-nix": {
|
||||||
"inputs": {
|
"inputs": {
|
||||||
"nixpkgs": [
|
"nixpkgs": [
|
||||||
@@ -184,6 +567,69 @@
|
|||||||
"repo": "treefmt-nix",
|
"repo": "treefmt-nix",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
"uv2nix": {
|
||||||
|
"inputs": {
|
||||||
|
"nixpkgs": [
|
||||||
|
"kyojin",
|
||||||
|
"nixpkgs"
|
||||||
|
],
|
||||||
|
"pyproject-nix": [
|
||||||
|
"kyojin",
|
||||||
|
"pyproject-nix"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1790341335,
|
||||||
|
"narHash": "sha256-wFnDm/o/4OebDlpGFyVXP8DLUx0/gR/jUFuGqmTwsQg=",
|
||||||
|
"owner": "pyproject-nix",
|
||||||
|
"repo": "uv2nix",
|
||||||
|
"rev": "a24323e9e6ecbbf305c238845ff6c612d50467c0",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "pyproject-nix",
|
||||||
|
"repo": "uv2nix",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"xdna-driver-src": {
|
||||||
|
"flake": false,
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1784758690,
|
||||||
|
"narHash": "sha256-YUiM9u9vtISttbThTt9fTtBB/w3d5UatGyVK5mAgWNM=",
|
||||||
|
"ref": "1.7",
|
||||||
|
"rev": "4e5aed38f3b74a5a9a2c7a6222eaff1a8be54305",
|
||||||
|
"revCount": 874,
|
||||||
|
"submodules": true,
|
||||||
|
"type": "git",
|
||||||
|
"url": "https://github.com/amd/xdna-driver"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"ref": "1.7",
|
||||||
|
"submodules": true,
|
||||||
|
"type": "git",
|
||||||
|
"url": "https://github.com/amd/xdna-driver"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"xrt-src": {
|
||||||
|
"flake": false,
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1780589208,
|
||||||
|
"narHash": "sha256-JrqJIGJoQiXTwXjZpqAXVaHx+6i09B1qtqkJzoRPZKw=",
|
||||||
|
"ref": "XRT-2.21",
|
||||||
|
"rev": "8661761775a266b11992a3bd6eb08209d88aa845",
|
||||||
|
"revCount": 8684,
|
||||||
|
"submodules": true,
|
||||||
|
"type": "git",
|
||||||
|
"url": "https://github.com/Xilinx/XRT"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"ref": "XRT-2.21",
|
||||||
|
"submodules": true,
|
||||||
|
"type": "git",
|
||||||
|
"url": "https://github.com/Xilinx/XRT"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"root": "root",
|
"root": "root",
|
||||||
|
|||||||
@@ -2,6 +2,36 @@
|
|||||||
description = "Various packages for Nix";
|
description = "Various packages for Nix";
|
||||||
|
|
||||||
inputs = {
|
inputs = {
|
||||||
|
|
||||||
|
nixpkgs-engine.url = "github:NixOS/nixpkgs/nixos-unstable";
|
||||||
|
hipfire = {
|
||||||
|
url = "github:warpfront/hipfire/v0.4.4";
|
||||||
|
inputs.nixpkgs.follows = "nixpkgs-engine";
|
||||||
|
};
|
||||||
|
nix-strix-halo = {
|
||||||
|
url = "github:hellas-ai/nix-strix-halo";
|
||||||
|
flake = false;
|
||||||
|
};
|
||||||
|
xrt-src = {
|
||||||
|
url = "git+https://github.com/Xilinx/XRT?ref=XRT-2.21&submodules=1";
|
||||||
|
flake = false;
|
||||||
|
};
|
||||||
|
xdna-driver-src = {
|
||||||
|
url = "git+https://github.com/amd/xdna-driver?ref=1.7&submodules=1";
|
||||||
|
flake = false;
|
||||||
|
};
|
||||||
|
pwilkin-llama-cpp = {
|
||||||
|
url = "github:pwilkin/llama.cpp/strix-halo";
|
||||||
|
inputs.nixpkgs.follows = "nixpkgs-engine";
|
||||||
|
inputs.flake-parts.follows = "flake-parts";
|
||||||
|
};
|
||||||
|
strix-llama-cpp = {
|
||||||
|
url = "github:halo-box/strix-llama.cpp";
|
||||||
|
inputs.nixpkgs.follows = "nixpkgs-engine";
|
||||||
|
inputs.flake-parts.follows = "flake-parts";
|
||||||
|
};
|
||||||
|
gufo.url = "github:mikealanni/gufo/fork-enhancements";
|
||||||
|
kyojin.url = "github:Yamz-Labs/kyojin/v1.8.10";
|
||||||
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
|
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
|
||||||
# Newer nixpkgs whose toolchains are required by some packages:
|
# Newer nixpkgs whose toolchains are required by some packages:
|
||||||
# rustc >= 1.95 (mcp-gateway >= 3.4.0), go >= 1.26.3 (kubernetes-mcp-server >= 0.0.66).
|
# rustc >= 1.95 (mcp-gateway >= 3.4.0), go >= 1.26.3 (kubernetes-mcp-server >= 0.0.66).
|
||||||
@@ -52,6 +82,7 @@
|
|||||||
blueprintOutputs
|
blueprintOutputs
|
||||||
// {
|
// {
|
||||||
overlays = {
|
overlays = {
|
||||||
|
hipfire = inputs.hipfire.overlays.default;
|
||||||
default = import ./overlays {
|
default = import ./overlays {
|
||||||
inherit (blueprintOutputs) packages;
|
inherit (blueprintOutputs) packages;
|
||||||
};
|
};
|
||||||
@@ -60,6 +91,8 @@
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
nixosModules.hipfire = inputs.hipfire.nixosModules.default;
|
||||||
|
|
||||||
nixosModules.default = {
|
nixosModules.default = {
|
||||||
nixpkgs.overlays = [ self.overlays.default ];
|
nixpkgs.overlays = [ self.overlays.default ];
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -0,0 +1,5 @@
|
|||||||
|
{ pkgs, inputs, ... }:
|
||||||
|
(import ../../_lib/llm-packages.nix {
|
||||||
|
inherit inputs;
|
||||||
|
system = pkgs.stdenv.hostPlatform.system;
|
||||||
|
}).gufo
|
||||||
@@ -0,0 +1,58 @@
|
|||||||
|
--- a/src/core/gguf_reader.cpp
|
||||||
|
+++ b/src/core/gguf_reader.cpp
|
||||||
|
@@ -747,8 +747,11 @@
|
||||||
|
tensors_.push_back(std::move(info));
|
||||||
|
}
|
||||||
|
|
||||||
|
- // Align data payload base to alignment boundary
|
||||||
|
+ // Align data payload base to alignment boundary. A tensorless file, such as
|
||||||
|
+ // a metadata-only first split shard, may end unpadded after its header.
|
||||||
|
const auto padding = (alignment_ - offset % alignment_) % alignment_;
|
||||||
|
+ if (tensors_.empty())
|
||||||
|
+ return true;
|
||||||
|
if (offset > size_ || padding > size_ - offset)
|
||||||
|
return false;
|
||||||
|
const std::size_t data_base = offset + padding;
|
||||||
|
--- a/src/models/qwen38_flash_next/weights.cpp
|
||||||
|
+++ b/src/models/qwen38_flash_next/weights.cpp
|
||||||
|
@@ -292,7 +292,9 @@
|
||||||
|
b.Fail("per_layer_token_embd.weight is missing or too short");
|
||||||
|
} else {
|
||||||
|
w.ple_table = b.Get("per_layer_token_embd.weight", c.ple_head_dim, rows,
|
||||||
|
- 1, {GgmlType::kIQ4_NL, GgmlType::kBF16});
|
||||||
|
+ 1,
|
||||||
|
+ {GgmlType::kIQ4_NL, GgmlType::kQ8_0,
|
||||||
|
+ GgmlType::kBF16});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
w.layers.reserve(c.num_layers);
|
||||||
|
--- a/src/models/qwen38_flash_next/ngram.cpp
|
||||||
|
+++ b/src/models/qwen38_flash_next/ngram.cpp
|
||||||
|
@@ -81,10 +81,14 @@
|
||||||
|
int file_descriptor, std::uint64_t file_offset, std::uint64_t rows,
|
||||||
|
std::uint32_t row_dim, core::GgmlType type, std::string* error_msg) {
|
||||||
|
std::unique_ptr<NgramTable> t(new NgramTable());
|
||||||
|
- const std::size_t block = type == core::GgmlType::kIQ4_NL ? 32 : 1;
|
||||||
|
- const std::size_t block_bytes = type == core::GgmlType::kIQ4_NL ? 18 : 2;
|
||||||
|
- if ((type != core::GgmlType::kIQ4_NL && type != core::GgmlType::kBF16) ||
|
||||||
|
- row_dim == 0 || row_dim % block != 0) {
|
||||||
|
+ const bool blocked =
|
||||||
|
+ type == core::GgmlType::kIQ4_NL || type == core::GgmlType::kQ8_0;
|
||||||
|
+ const std::size_t block = blocked ? 32 : 1;
|
||||||
|
+ const std::size_t block_bytes = type == core::GgmlType::kIQ4_NL ? 18
|
||||||
|
+ : type == core::GgmlType::kQ8_0 ? 34
|
||||||
|
+ : 2;
|
||||||
|
+ if ((!blocked && type != core::GgmlType::kBF16) || row_dim == 0 ||
|
||||||
|
+ row_dim % block != 0) {
|
||||||
|
if (error_msg != nullptr) {
|
||||||
|
*error_msg = "unsupported n-gram table format";
|
||||||
|
}
|
||||||
|
@@ -127,6 +131,8 @@
|
||||||
|
void NgramTable::DecodeRow(const std::uint8_t* src, float* dst) const {
|
||||||
|
if (type_ == core::GgmlType::kIQ4_NL) {
|
||||||
|
gufo::quant::DequantizeIQ4_NL(src, dst, row_dim_);
|
||||||
|
+ } else if (type_ == core::GgmlType::kQ8_0) {
|
||||||
|
+ gufo::quant::DequantizeQ8_0(src, dst, row_dim_);
|
||||||
|
} else {
|
||||||
|
for (std::uint32_t i = 0; i < row_dim_; ++i) {
|
||||||
|
std::uint16_t bits = 0;
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
{ system, inputs }:
|
||||||
|
let
|
||||||
|
gufoPkgs = import inputs.gufo.inputs.nixpkgs {
|
||||||
|
inherit system;
|
||||||
|
config.allowUnfree = true;
|
||||||
|
};
|
||||||
|
nsh = inputs.nix-strix-halo;
|
||||||
|
|
||||||
|
xrt =
|
||||||
|
(gufoPkgs.callPackage (nsh + /pkgs/xrt) {
|
||||||
|
src = inputs.xrt-src;
|
||||||
|
version = "2.21";
|
||||||
|
xdnaSrc = inputs.xdna-driver-src;
|
||||||
|
xdnaVersion = "1.7";
|
||||||
|
}).xdna.overrideAttrs
|
||||||
|
(old: {
|
||||||
|
meta = (old.meta or { }) // {
|
||||||
|
mainProgram = "xrt-smi";
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
package = inputs.gufo.packages.${system}.release.overrideAttrs (old: {
|
||||||
|
nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [
|
||||||
|
gufoPkgs.libuuid.dev
|
||||||
|
xrt
|
||||||
|
];
|
||||||
|
cmakeFlags = old.cmakeFlags ++ [
|
||||||
|
"-DGUFO_ENABLE_NPU=ON"
|
||||||
|
"-DGUFO_XRT_INCLUDE_DIR=${xrt}/include"
|
||||||
|
"-DGUFO_XRT_LIB_DIR=${xrt}/lib"
|
||||||
|
];
|
||||||
|
patches = (old.patches or [ ]) ++ [ ./gufo-unsloth-ud3.patch ];
|
||||||
|
});
|
||||||
|
|
||||||
|
npuDir = "${inputs.gufo}/tools/npu";
|
||||||
|
|
||||||
|
fabricClock = "${inputs.gufo}/tools/npu/fabric-clock.sh";
|
||||||
|
in
|
||||||
|
package.overrideAttrs (old: {
|
||||||
|
passthru = (old.passthru or { }) // {
|
||||||
|
inherit xrt npuDir fabricClock;
|
||||||
|
category = "AI Inference";
|
||||||
|
};
|
||||||
|
})
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
{ pkgs, inputs, ... }:
|
||||||
|
(import ../../_lib/llm-packages.nix {
|
||||||
|
inherit inputs;
|
||||||
|
system = pkgs.stdenv.hostPlatform.system;
|
||||||
|
}).hipfire-kernels
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
{ pkgs, inputs, ... }:
|
||||||
|
(import ../../_lib/llm-packages.nix {
|
||||||
|
inherit inputs;
|
||||||
|
system = pkgs.stdenv.hostPlatform.system;
|
||||||
|
}).hipfire
|
||||||
@@ -0,0 +1,123 @@
|
|||||||
|
{
|
||||||
|
pkgs,
|
||||||
|
hipfireSource,
|
||||||
|
therockRocm,
|
||||||
|
modelDir ? "/var/lib/hipfire/models",
|
||||||
|
}:
|
||||||
|
let
|
||||||
|
lib = pkgs.lib;
|
||||||
|
gcc = pkgs.stdenv.cc;
|
||||||
|
|
||||||
|
hipccFixed = pkgs.writeShellScriptBin "hipcc" ''
|
||||||
|
export ROCM_PATH=${therockRocm}
|
||||||
|
exec ${therockRocm}/bin/hipcc \
|
||||||
|
--gcc-toolchain=${gcc.cc} \
|
||||||
|
-idirafter ${gcc.libc.dev}/include \
|
||||||
|
-B${gcc.libc}/lib \
|
||||||
|
-L${gcc.libc}/lib \
|
||||||
|
-L${gcc.cc.lib}/lib \
|
||||||
|
-L${gcc.cc}/lib \
|
||||||
|
"$@"
|
||||||
|
'';
|
||||||
|
|
||||||
|
glibcLoader = "${pkgs.glibc}/lib/ld-linux-x86-64.so.2";
|
||||||
|
llvmRunPath = lib.makeSearchPathOutput "lib" "lib" [
|
||||||
|
gcc.cc.lib
|
||||||
|
gcc.cc
|
||||||
|
];
|
||||||
|
therockLlvmTool =
|
||||||
|
name:
|
||||||
|
pkgs.writeShellScriptBin name ''
|
||||||
|
exec ${glibcLoader} --library-path ${therockRocm}/lib:${llvmRunPath} \
|
||||||
|
${therockRocm}/lib/llvm/bin/${name} "$@"
|
||||||
|
'';
|
||||||
|
llvmReadobj = therockLlvmTool "llvm-readobj";
|
||||||
|
llvmObjdump = therockLlvmTool "llvm-objdump";
|
||||||
|
|
||||||
|
therockDeviceLibs = pkgs.runCommand "therock-device-libs" { } ''
|
||||||
|
mkdir -p $out/amdgcn
|
||||||
|
ln -sfn ${therockRocm}/amdgcn/bitcode $out/amdgcn/bitcode
|
||||||
|
'';
|
||||||
|
|
||||||
|
hipfireKernelsPkg =
|
||||||
|
(pkgs.callPackage (hipfireSource + "/nix/kernels.nix") {
|
||||||
|
src = hipfireSource;
|
||||||
|
cargoLockFile = hipfireSource + "/Cargo.lock";
|
||||||
|
gpuTargets = [ "gfx1151" ];
|
||||||
|
rocmPackages = pkgs.rocmPackages // {
|
||||||
|
clr = therockRocm;
|
||||||
|
rocm-device-libs = therockDeviceLibs;
|
||||||
|
};
|
||||||
|
}).overrideAttrs
|
||||||
|
(old: {
|
||||||
|
nativeBuildInputs = old.nativeBuildInputs ++ [ hipccFixed ];
|
||||||
|
env.HIPFIRE_HIPCC = "${hipccFixed}/bin/hipcc";
|
||||||
|
});
|
||||||
|
|
||||||
|
hipfireWithQuantize = pkgs.hipfire.overrideAttrs (_: {
|
||||||
|
buildPhase = ''
|
||||||
|
runHook preBuild
|
||||||
|
cargo build --release -p hipfire-daemon
|
||||||
|
cargo build --release -p hipfire-cli
|
||||||
|
cargo build --release -p hipfire-quantize \
|
||||||
|
--bin hipfire-quantize --bin dflash_convert --bin draft_to_mq4
|
||||||
|
runHook postBuild
|
||||||
|
'';
|
||||||
|
|
||||||
|
installPhase = ''
|
||||||
|
runHook preInstall
|
||||||
|
|
||||||
|
mkdir -p $out/bin
|
||||||
|
|
||||||
|
cp target/release/daemon $out/bin/hipfire-daemon-unwrapped
|
||||||
|
makeWrapper $out/bin/hipfire-daemon-unwrapped $out/bin/hipfire-daemon \
|
||||||
|
--set HIPFIRE_ROCM_PATH ${therockRocm} \
|
||||||
|
--set HIPFIRE_HIPCC ${hipccFixed}/bin/hipcc \
|
||||||
|
--set ROCM_PATH ${therockRocm} \
|
||||||
|
--set RADIOWAVE_READOBJ ${llvmReadobj}/bin/llvm-readobj \
|
||||||
|
--set RADIOWAVE_OBJDUMP ${llvmObjdump}/bin/llvm-objdump \
|
||||||
|
--prefix PATH : ${
|
||||||
|
pkgs.lib.makeBinPath [
|
||||||
|
hipccFixed
|
||||||
|
pkgs.pciutils
|
||||||
|
]
|
||||||
|
} \
|
||||||
|
--prefix LD_LIBRARY_PATH : ${therockRocm}/lib
|
||||||
|
|
||||||
|
cp target/release/hipfire $out/bin/hipfire-unwrapped
|
||||||
|
makeWrapper $out/bin/hipfire-unwrapped $out/bin/hipfire \
|
||||||
|
--set HIPFIRE_DAEMON_BIN $out/bin/hipfire-daemon \
|
||||||
|
--set HIPFIRE_NORMALIZE_PROMPT "1" \
|
||||||
|
--set HIPFIRE_MODELS_DIR "${modelDir}" \
|
||||||
|
--set HIP_PATH "${therockRocm}" \
|
||||||
|
--set HIPFIRE_HIPCC ${hipccFixed}/bin/hipcc \
|
||||||
|
--set ROCM_PATH ${therockRocm} \
|
||||||
|
--set RADIOWAVE_READOBJ ${llvmReadobj}/bin/llvm-readobj \
|
||||||
|
--set RADIOWAVE_OBJDUMP ${llvmObjdump}/bin/llvm-objdump \
|
||||||
|
--prefix PATH : ${
|
||||||
|
pkgs.lib.makeBinPath [
|
||||||
|
hipccFixed
|
||||||
|
pkgs.pciutils
|
||||||
|
]
|
||||||
|
} \
|
||||||
|
--prefix LD_LIBRARY_PATH : ${therockRocm}/lib
|
||||||
|
|
||||||
|
mkdir -p $out/share/hipfire/target/release
|
||||||
|
cp target/release/hipfire-quantize $out/share/hipfire/target/release/
|
||||||
|
cp target/release/dflash_convert $out/share/hipfire/target/release/
|
||||||
|
cp target/release/draft_to_mq4 $out/share/hipfire/target/release/
|
||||||
|
|
||||||
|
runHook postInstall
|
||||||
|
'';
|
||||||
|
});
|
||||||
|
|
||||||
|
in
|
||||||
|
hipfireWithQuantize.overrideAttrs (old: {
|
||||||
|
passthru = (old.passthru or { }) // {
|
||||||
|
kernels = hipfireKernelsPkg;
|
||||||
|
hipcc = hipccFixed;
|
||||||
|
inherit llvmReadobj llvmObjdump;
|
||||||
|
rocm = therockRocm;
|
||||||
|
category = "AI Inference";
|
||||||
|
};
|
||||||
|
})
|
||||||
@@ -1,46 +0,0 @@
|
|||||||
"""Compile Kyojin's JIT kernels; --gpu also loads them through torch's HIP runtime."""
|
|
||||||
import ctypes
|
|
||||||
import importlib
|
|
||||||
import os
|
|
||||||
from pathlib import Path
|
|
||||||
import sys
|
|
||||||
|
|
||||||
from exllamav3.util import hip_compiler
|
|
||||||
from exllamav3.util.hip_lib import load_hip_runtime
|
|
||||||
|
|
||||||
sdk = Path(os.environ["EXL3_ROCM_SDK"])
|
|
||||||
assert Path(hip_compiler.hipcc()) == sdk / "bin/hipcc"
|
|
||||||
assert Path(os.environ["HIP_DEVICE_LIB_PATH"]).is_dir()
|
|
||||||
assert (sdk / "amdgcn/bitcode").is_dir()
|
|
||||||
if os.environ.get("KYOJIN_REQUIRE_SDK") == "1":
|
|
||||||
assert hip_compiler.warning() is None, hip_compiler.warning()
|
|
||||||
print(hip_compiler.describe(), flush=True)
|
|
||||||
|
|
||||||
lib = None
|
|
||||||
if "--gpu" in sys.argv:
|
|
||||||
import torch
|
|
||||||
|
|
||||||
# Also exercise the runtime bitcode path: merely loading a HIP module
|
|
||||||
# does not catch incompatible device libraries in CLR/COMGR.
|
|
||||||
x = torch.ones((32, 32), device="cuda", dtype=torch.float32)
|
|
||||||
assert torch.equal(x @ x, torch.full_like(x, 32))
|
|
||||||
torch.cuda.synchronize()
|
|
||||||
lib = load_hip_runtime()
|
|
||||||
lib.hipModuleLoad.argtypes = [ctypes.POINTER(ctypes.c_void_p), ctypes.c_char_p]
|
|
||||||
lib.hipModuleUnload.argtypes = [ctypes.c_void_p]
|
|
||||||
|
|
||||||
for module, function, args in [
|
|
||||||
("exllamav3.modules.gr_fn.gr_mix_hip", "compile_hsaco", ()),
|
|
||||||
("exllamav3.modules.attention_fn.qsa_prefill_hip", "compile_hsaco", ()),
|
|
||||||
("exllamav3.modules.ple_fn.ple_hip", "compile_hsaco", ()),
|
|
||||||
("exllamav3.vendor.fla.hip.gdn_fused_h_hip", "_compile", ("", "gfx1151")),
|
|
||||||
("exllamav3.vendor.fla.hip.kda_fused_h_hip", "_compile", ("", "gfx1151")),
|
|
||||||
]:
|
|
||||||
path = Path(getattr(importlib.import_module(module), function)(*args))
|
|
||||||
# HIP accepts ELF, offload bundles, and compressed offload bundles (CCOB).
|
|
||||||
assert path.read_bytes().startswith((b"\x7fELF", b"__CLANG_OFFLOAD_BUNDLE__", b"CCOB")), path
|
|
||||||
if lib is not None:
|
|
||||||
handle = ctypes.c_void_p()
|
|
||||||
assert lib.hipModuleLoad(ctypes.byref(handle), os.fsencode(path)) == 0, path
|
|
||||||
assert lib.hipModuleUnload(handle) == 0, path
|
|
||||||
print(f"OK: {module}", flush=True)
|
|
||||||
@@ -1,13 +1,5 @@
|
|||||||
{
|
{ pkgs, inputs, ... }:
|
||||||
pkgs,
|
(import ../../_lib/llm-packages.nix {
|
||||||
inputs,
|
inherit inputs;
|
||||||
jitRocmSdk ? null,
|
system = pkgs.stdenv.hostPlatform.system;
|
||||||
...
|
}).kyojin
|
||||||
}:
|
|
||||||
# The HIP extension has to be compiled against a ROCm build of PyTorch, and the
|
|
||||||
# only ROCm torch in this flake with a cached binary is the one from the pinned
|
|
||||||
# nixpkgs-torch211 input (2.11 + ROCm 7.2, python 3.13). Current nixos-unstable
|
|
||||||
# ships torch 2.13, whose ROCm build fails to configure in nixpkgs and is not
|
|
||||||
# on cache.nixos.org.
|
|
||||||
inputs.nixpkgs-torch211.legacyPackages.${pkgs.stdenv.hostPlatform.system}.callPackage ./package.nix
|
|
||||||
{ inherit jitRocmSdk; }
|
|
||||||
|
|||||||
+22
-222
@@ -1,230 +1,30 @@
|
|||||||
{
|
{
|
||||||
lib,
|
|
||||||
stdenv,
|
|
||||||
pkgs,
|
pkgs,
|
||||||
python3,
|
kyojin,
|
||||||
fetchFromGitHub,
|
jitCheck,
|
||||||
makeWrapper,
|
cpuCheck,
|
||||||
rocmPackages,
|
|
||||||
# Override only the serving toolchain; torch and the extension keep their ROCm.
|
|
||||||
jitRocmSdk ? null,
|
|
||||||
}:
|
}:
|
||||||
|
kyojin.overrideAttrs (old: {
|
||||||
let
|
nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ pkgs.makeWrapper ];
|
||||||
py = python3.pkgs;
|
meta = (old.meta or { }) // {
|
||||||
|
|
||||||
# Kyojin is the Yamz fork of ExLlamaV3: a Python package plus a C++ extension
|
|
||||||
# that torch's cpp_extension cross-compiles to HIP code objects. Upstream
|
|
||||||
# only supports AMD Strix Halo, so the kernels are built for gfx1151 alone.
|
|
||||||
rocmArch = "gfx1151";
|
|
||||||
|
|
||||||
version = "1.8.5";
|
|
||||||
|
|
||||||
src = fetchFromGitHub {
|
|
||||||
owner = "Yamz-Labs";
|
|
||||||
repo = "kyojin";
|
|
||||||
rev = "v${version}";
|
|
||||||
hash = "sha256-OZhzRnQUrkJOrn2lVatvVxD6JSwtJkwfWwpRUQHawac=";
|
|
||||||
};
|
|
||||||
|
|
||||||
# setup.py passes EXL3_ROCM_DEV_INCLUDE as -I to every translation unit. The
|
|
||||||
# AMD wheels ship one devel tree holding hipsparse/ and thrust/; nixpkgs
|
|
||||||
# splits them per package, so join the include dirs. torch's public HIP
|
|
||||||
# headers also pull hipsolver/, and it no longer vendors pybind11.
|
|
||||||
rocmDevInclude = pkgs.symlinkJoin {
|
|
||||||
name = "kyojin-rocm-dev-include";
|
|
||||||
paths = map lib.getDev (
|
|
||||||
[ py.pybind11 ]
|
|
||||||
++ [
|
|
||||||
rocmPackages.clr
|
|
||||||
rocmPackages.hipblas
|
|
||||||
rocmPackages.hipblaslt
|
|
||||||
rocmPackages.hipcub
|
|
||||||
rocmPackages.hipfft
|
|
||||||
rocmPackages.hiprand
|
|
||||||
rocmPackages.hipsparse
|
|
||||||
rocmPackages.hipsolver
|
|
||||||
rocmPackages.rocblas
|
|
||||||
rocmPackages.rocprim
|
|
||||||
rocmPackages.rocrand
|
|
||||||
rocmPackages.rocsolver
|
|
||||||
rocmPackages.rocsparse
|
|
||||||
rocmPackages.rocthrust
|
|
||||||
]
|
|
||||||
);
|
|
||||||
};
|
|
||||||
|
|
||||||
# torch's HIP path in cpp_extension expects one ROCM_HOME holding bin/hipcc,
|
|
||||||
# the headers and the amdgcn bitcode; nixpkgs ships them separately.
|
|
||||||
rocmToolkit = pkgs.symlinkJoin {
|
|
||||||
name = "kyojin-rocm-toolkit";
|
|
||||||
paths = [
|
|
||||||
rocmPackages."rocm-core"
|
|
||||||
rocmPackages."rocm-runtime"
|
|
||||||
rocmPackages."rocm-device-libs"
|
|
||||||
rocmPackages."rocm-comgr"
|
|
||||||
rocmPackages.clr
|
|
||||||
(lib.getBin rocmPackages.hipcc)
|
|
||||||
rocmPackages.rocblas
|
|
||||||
rocmPackages.hipblas
|
|
||||||
rocmPackages.hipblas-common
|
|
||||||
rocmPackages.hipblaslt
|
|
||||||
rocmPackages.hipsparse
|
|
||||||
rocmPackages.hipsolver
|
|
||||||
rocmPackages.hiprand
|
|
||||||
rocmPackages.rocrand
|
|
||||||
rocmPackages.hipfft
|
|
||||||
rocmPackages.rocsparse
|
|
||||||
rocmPackages.rocsolver
|
|
||||||
rocmPackages.rocthrust
|
|
||||||
rocmPackages.rocprim
|
|
||||||
rocmPackages.hipcub
|
|
||||||
];
|
|
||||||
# The setup hooks of the joined packages would rewrite build flags; the
|
|
||||||
# compiler only needs the files.
|
|
||||||
postBuild = ''
|
|
||||||
rm -rf $out/nix-support
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
# The library: exllamav3 plus the compiled exllamav3_ext module.
|
|
||||||
library = py.buildPythonPackage {
|
|
||||||
inherit src version;
|
|
||||||
pname = "kyojin";
|
|
||||||
|
|
||||||
format = "pyproject";
|
|
||||||
build-system = with py; [
|
|
||||||
setuptools
|
|
||||||
wheel
|
|
||||||
];
|
|
||||||
|
|
||||||
nativeBuildInputs = [
|
|
||||||
py.ninja
|
|
||||||
rocmPackages."rocm-runtime"
|
|
||||||
];
|
|
||||||
|
|
||||||
buildInputs = [ (lib.getBin rocmPackages.hipcc) ];
|
|
||||||
|
|
||||||
propagatedBuildInputs = with py; [
|
|
||||||
aiohttp
|
|
||||||
huggingface-hub
|
|
||||||
jinja2
|
|
||||||
llguidance
|
|
||||||
marisa-trie
|
|
||||||
ninja
|
|
||||||
numpy
|
|
||||||
pillow
|
|
||||||
pydantic
|
|
||||||
pyyaml
|
|
||||||
rich
|
|
||||||
safetensors
|
|
||||||
tokenizers
|
|
||||||
torchWithRocm
|
|
||||||
typing-extensions
|
|
||||||
];
|
|
||||||
|
|
||||||
env = {
|
|
||||||
PYTORCH_ROCM_ARCH = rocmArch;
|
|
||||||
ROCM_PATH = "${rocmToolkit}";
|
|
||||||
ROCM_HOME = "${rocmToolkit}";
|
|
||||||
HIP_PATH = "${rocmToolkit}";
|
|
||||||
HIPCC = "${rocmToolkit}/bin/hipcc";
|
|
||||||
HIP_DEVICE_LIB_PATH = "${rocmPackages."rocm-device-libs"}/amdgcn/bitcode";
|
|
||||||
# Upstream build scripts look these up to find the ROCm devel tree.
|
|
||||||
EXL3_ROCM_SDK = "${rocmToolkit}";
|
|
||||||
EXL3_ROCM_DEV_INCLUDE = "${rocmDevInclude}/include";
|
|
||||||
# Default defines of upstream build.sh (tools/strix_halo/rebuild.sh).
|
|
||||||
EXL3_HIP_DEFINES = "EXL3_HIP_STG_PAD";
|
|
||||||
};
|
|
||||||
|
|
||||||
# setup.py loads exllamav3/exllamav3_ext/build_config.py by path (and
|
|
||||||
# pyproject.toml ships the .hip sources, qsa_proof markers, and dense-GEMM
|
|
||||||
# tuning seed), so no source patches are needed.
|
|
||||||
|
|
||||||
# tests/ needs a real gfx1151 GPU and the published model packs.
|
|
||||||
doCheck = false;
|
|
||||||
|
|
||||||
meta = with lib; {
|
|
||||||
description = "Inference engine for 100 GB-class EXL3 MoE models on AMD Strix Halo (gfx1151, ROCm 7)";
|
|
||||||
homepage = "https://github.com/Yamz-Labs/kyojin";
|
homepage = "https://github.com/Yamz-Labs/kyojin";
|
||||||
changelog = "https://github.com/Yamz-Labs/kyojin/releases";
|
license = pkgs.lib.licenses.mit;
|
||||||
license = licenses.mit;
|
|
||||||
sourceProvenance = with sourceTypes; [ fromSource ];
|
|
||||||
# HIP code objects are gfx1151-only and ROCm is Linux-only.
|
|
||||||
platforms = platforms.linux;
|
|
||||||
};
|
};
|
||||||
};
|
passthru = (old.passthru or { }) // {
|
||||||
|
|
||||||
python = py.python.withPackages (_: [ library ]);
|
|
||||||
jitSdk = if jitRocmSdk == null then rocmToolkit else jitRocmSdk;
|
|
||||||
in
|
|
||||||
stdenv.mkDerivation {
|
|
||||||
inherit src version;
|
|
||||||
pname = "kyojin";
|
|
||||||
|
|
||||||
# Only the wrappers are installed here: the serve scripts are plain scripts,
|
|
||||||
# they import their siblings by path, and the module itself is in `library`.
|
|
||||||
#
|
|
||||||
# Upstream `tools/strix_halo/env.sh` is sourced before serving, and on a
|
|
||||||
# distro box that brings a system `cc` (the AMD Triton backend compiles a
|
|
||||||
# CPython glue module at runtime, and again per kernel launcher) plus a hipcc
|
|
||||||
# for the JIT HIP kernels (qsa prefill, gdn, kda, ple). Nix gives the wrapper
|
|
||||||
# neither, so export them here instead: Triton takes the compiler from $CC,
|
|
||||||
# exllamav3 finds hipcc under $EXL3_ROCM_SDK, and hipclang only finds the
|
|
||||||
# amdgcn bitcode through $HIP_DEVICE_LIB_PATH (hipcc's own DEVICE_LIB_PATH is
|
|
||||||
# not enough for a --genco call). The ROCm paths stay out of LD_LIBRARY_PATH
|
|
||||||
# (upstream Trap 1/5: a second libhsa segfaults torch).
|
|
||||||
dontConfigure = true;
|
|
||||||
dontBuild = true;
|
|
||||||
|
|
||||||
nativeBuildInputs = [ makeWrapper ];
|
|
||||||
|
|
||||||
installPhase = ''
|
|
||||||
runHook preInstall
|
|
||||||
|
|
||||||
mkdir -p $out/bin
|
|
||||||
for model in glm mimo qwen; do
|
|
||||||
makeWrapper ${python}/bin/python $out/bin/kyojin-serve-$model \
|
|
||||||
--set PYTORCH_ROCM_ARCH ${rocmArch} \
|
|
||||||
--set CC ${stdenv.cc}/bin/cc \
|
|
||||||
--set EXL3_ROCM_SDK ${jitSdk} \
|
|
||||||
--set HIP_DEVICE_LIB_PATH ${rocmPackages."rocm-device-libs"}/amdgcn/bitcode \
|
|
||||||
--set EXL3_EXPANDABLE_SEGMENTS 0 \
|
|
||||||
--prefix PATH : ${lib.makeBinPath [ stdenv.cc ]} \
|
|
||||||
--add-flags "${src}/tools/$model/serve.py"
|
|
||||||
done
|
|
||||||
|
|
||||||
runHook postInstall
|
|
||||||
'';
|
|
||||||
|
|
||||||
passthru = {
|
|
||||||
inherit python rocmArch;
|
|
||||||
pythonPackage = library;
|
|
||||||
jitRocmSdk = jitSdk;
|
|
||||||
tests.jit =
|
|
||||||
pkgs.runCommand "kyojin-jit-check"
|
|
||||||
{
|
|
||||||
EXL3_ROCM_SDK = jitSdk;
|
|
||||||
# CLR/COMGR also reads this variable. Newer bitcode crashes torch
|
|
||||||
# matmul; a custom hipcc must select its own bitcode in its subprocess.
|
|
||||||
HIP_DEVICE_LIB_PATH = "${rocmPackages."rocm-device-libs"}/amdgcn/bitcode";
|
|
||||||
EXL3_EXPANDABLE_SEGMENTS = "0";
|
|
||||||
KYOJIN_REQUIRE_SDK = lib.boolToString (jitRocmSdk != null);
|
|
||||||
}
|
|
||||||
''
|
|
||||||
export HOME="$TMPDIR"
|
|
||||||
${python}/bin/python ${./check-jit.py}
|
|
||||||
touch "$out"
|
|
||||||
'';
|
|
||||||
category = "AI Inference";
|
category = "AI Inference";
|
||||||
updateScript = [
|
tests = (old.passthru.tests or { }) // {
|
||||||
"nix-update"
|
jit = jitCheck;
|
||||||
"--flake"
|
cpu = cpuCheck;
|
||||||
".#kyojin"
|
|
||||||
];
|
|
||||||
};
|
};
|
||||||
|
|
||||||
meta = library.meta // {
|
|
||||||
mainProgram = "kyojin-serve-qwen";
|
|
||||||
};
|
};
|
||||||
}
|
postFixup = ''
|
||||||
|
# Fail the build, not the first request, if upstream renames the wrappers.
|
||||||
|
ls "$out"/bin/kyojin-serve-* > /dev/null 2>&1 || {
|
||||||
|
echo "kyojin-cc: no kyojin-serve-* in $out/bin; upstream moved them" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
for b in $out/bin/kyojin-serve-*; do
|
||||||
|
wrapProgram "$b" --set CC ${pkgs.stdenv.cc}/bin/gcc
|
||||||
|
done
|
||||||
|
'';
|
||||||
|
})
|
||||||
|
|||||||
@@ -0,0 +1,5 @@
|
|||||||
|
{ pkgs, inputs, ... }:
|
||||||
|
(import ../../_lib/llm-packages.nix {
|
||||||
|
inherit inputs;
|
||||||
|
system = pkgs.stdenv.hostPlatform.system;
|
||||||
|
}).pwilkin-llama-cpp
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
{ pkgs, source }:
|
||||||
|
pkgs.callPackage (source + "/.devops/nix/package.nix") {
|
||||||
|
llamaVersion = source.shortRev or "0.0.0";
|
||||||
|
useRocm = true;
|
||||||
|
rocmGpuTargets = "gfx1151";
|
||||||
|
}
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
{ pkgs, inputs, ... }:
|
||||||
|
(import ../../_lib/llm-packages.nix {
|
||||||
|
inherit inputs;
|
||||||
|
system = pkgs.stdenv.hostPlatform.system;
|
||||||
|
}).strix-llama-cpp
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
{ pkgs, source }:
|
||||||
|
pkgs.callPackage (source + "/.devops/nix/package.nix") {
|
||||||
|
llamaVersion = source.shortRev or "0.0.0";
|
||||||
|
useVulkan = true;
|
||||||
|
}
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
{ pkgs, inputs, ... }:
|
||||||
|
(import ../../_lib/llm-packages.nix {
|
||||||
|
inherit inputs;
|
||||||
|
system = pkgs.stdenv.hostPlatform.system;
|
||||||
|
}).therock-rocm
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
{ pkgs, nix-strix-halo }:
|
||||||
|
let
|
||||||
|
source =
|
||||||
|
(builtins.fromJSON (builtins.readFile (nix-strix-halo + "/pkgs/therock/sources/rocm.json")))
|
||||||
|
.linux.gfx1151;
|
||||||
|
in
|
||||||
|
pkgs.callPackage (nix-strix-halo + "/pkgs/therock/rocm-sdk") {
|
||||||
|
target = "gfx1151";
|
||||||
|
inherit (source) version url hash;
|
||||||
|
}
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
{ pkgs, inputs, ... }:
|
||||||
|
(import ../../_lib/llm-packages.nix {
|
||||||
|
inherit inputs;
|
||||||
|
system = pkgs.stdenv.hostPlatform.system;
|
||||||
|
}).xrt-amdxdna
|
||||||
Reference in New Issue
Block a user