feat(llm): package upstream inference engines

Centralize engine pins, compiler wrappers and NPU packages.
Use upstream Kyojin v1.8.10 and drop the Thunderbolt dependency.

Offline evaluation passed; workstation builds remain pending.

Bead: nix-overlay-v09
This commit is contained in:
2026-11-12 20:36:52 +03:00
parent 47829fe9a6
commit 85bd7421de
21 changed files with 865 additions and 309 deletions
-46
View File
@@ -1,46 +0,0 @@
"""Compile Kyojin's JIT kernels; --gpu also loads them through torch's HIP runtime."""
import ctypes
import importlib
import os
from pathlib import Path
import sys
from exllamav3.util import hip_compiler
from exllamav3.util.hip_lib import load_hip_runtime
sdk = Path(os.environ["EXL3_ROCM_SDK"])
assert Path(hip_compiler.hipcc()) == sdk / "bin/hipcc"
assert Path(os.environ["HIP_DEVICE_LIB_PATH"]).is_dir()
assert (sdk / "amdgcn/bitcode").is_dir()
if os.environ.get("KYOJIN_REQUIRE_SDK") == "1":
assert hip_compiler.warning() is None, hip_compiler.warning()
print(hip_compiler.describe(), flush=True)
lib = None
if "--gpu" in sys.argv:
import torch
# Also exercise the runtime bitcode path: merely loading a HIP module
# does not catch incompatible device libraries in CLR/COMGR.
x = torch.ones((32, 32), device="cuda", dtype=torch.float32)
assert torch.equal(x @ x, torch.full_like(x, 32))
torch.cuda.synchronize()
lib = load_hip_runtime()
lib.hipModuleLoad.argtypes = [ctypes.POINTER(ctypes.c_void_p), ctypes.c_char_p]
lib.hipModuleUnload.argtypes = [ctypes.c_void_p]
for module, function, args in [
("exllamav3.modules.gr_fn.gr_mix_hip", "compile_hsaco", ()),
("exllamav3.modules.attention_fn.qsa_prefill_hip", "compile_hsaco", ()),
("exllamav3.modules.ple_fn.ple_hip", "compile_hsaco", ()),
("exllamav3.vendor.fla.hip.gdn_fused_h_hip", "_compile", ("", "gfx1151")),
("exllamav3.vendor.fla.hip.kda_fused_h_hip", "_compile", ("", "gfx1151")),
]:
path = Path(getattr(importlib.import_module(module), function)(*args))
# HIP accepts ELF, offload bundles, and compressed offload bundles (CCOB).
assert path.read_bytes().startswith((b"\x7fELF", b"__CLANG_OFFLOAD_BUNDLE__", b"CCOB")), path
if lib is not None:
handle = ctypes.c_void_p()
assert lib.hipModuleLoad(ctypes.byref(handle), os.fsencode(path)) == 0, path
assert lib.hipModuleUnload(handle) == 0, path
print(f"OK: {module}", flush=True)
+5 -13
View File
@@ -1,13 +1,5 @@
{
pkgs,
inputs,
jitRocmSdk ? null,
...
}:
# The HIP extension has to be compiled against a ROCm build of PyTorch, and the
# only ROCm torch in this flake with a cached binary is the one from the pinned
# nixpkgs-torch211 input (2.11 + ROCm 7.2, python 3.13). Current nixos-unstable
# ships torch 2.13, whose ROCm build fails to configure in nixpkgs and is not
# on cache.nixos.org.
inputs.nixpkgs-torch211.legacyPackages.${pkgs.stdenv.hostPlatform.system}.callPackage ./package.nix
{ inherit jitRocmSdk; }
{ pkgs, inputs, ... }:
(import ../../_lib/llm-packages.nix {
inherit inputs;
system = pkgs.stdenv.hostPlatform.system;
}).kyojin
+24 -224
View File
@@ -1,230 +1,30 @@
{
lib,
stdenv,
pkgs,
python3,
fetchFromGitHub,
makeWrapper,
rocmPackages,
# Override only the serving toolchain; torch and the extension keep their ROCm.
jitRocmSdk ? null,
kyojin,
jitCheck,
cpuCheck,
}:
let
py = python3.pkgs;
# Kyojin is the Yamz fork of ExLlamaV3: a Python package plus a C++ extension
# that torch's cpp_extension cross-compiles to HIP code objects. Upstream
# only supports AMD Strix Halo, so the kernels are built for gfx1151 alone.
rocmArch = "gfx1151";
version = "1.8.5";
src = fetchFromGitHub {
owner = "Yamz-Labs";
repo = "kyojin";
rev = "v${version}";
hash = "sha256-OZhzRnQUrkJOrn2lVatvVxD6JSwtJkwfWwpRUQHawac=";
kyojin.overrideAttrs (old: {
nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ pkgs.makeWrapper ];
meta = (old.meta or { }) // {
homepage = "https://github.com/Yamz-Labs/kyojin";
license = pkgs.lib.licenses.mit;
};
# setup.py passes EXL3_ROCM_DEV_INCLUDE as -I to every translation unit. The
# AMD wheels ship one devel tree holding hipsparse/ and thrust/; nixpkgs
# splits them per package, so join the include dirs. torch's public HIP
# headers also pull hipsolver/, and it no longer vendors pybind11.
rocmDevInclude = pkgs.symlinkJoin {
name = "kyojin-rocm-dev-include";
paths = map lib.getDev (
[ py.pybind11 ]
++ [
rocmPackages.clr
rocmPackages.hipblas
rocmPackages.hipblaslt
rocmPackages.hipcub
rocmPackages.hipfft
rocmPackages.hiprand
rocmPackages.hipsparse
rocmPackages.hipsolver
rocmPackages.rocblas
rocmPackages.rocprim
rocmPackages.rocrand
rocmPackages.rocsolver
rocmPackages.rocsparse
rocmPackages.rocthrust
]
);
};
# torch's HIP path in cpp_extension expects one ROCM_HOME holding bin/hipcc,
# the headers and the amdgcn bitcode; nixpkgs ships them separately.
rocmToolkit = pkgs.symlinkJoin {
name = "kyojin-rocm-toolkit";
paths = [
rocmPackages."rocm-core"
rocmPackages."rocm-runtime"
rocmPackages."rocm-device-libs"
rocmPackages."rocm-comgr"
rocmPackages.clr
(lib.getBin rocmPackages.hipcc)
rocmPackages.rocblas
rocmPackages.hipblas
rocmPackages.hipblas-common
rocmPackages.hipblaslt
rocmPackages.hipsparse
rocmPackages.hipsolver
rocmPackages.hiprand
rocmPackages.rocrand
rocmPackages.hipfft
rocmPackages.rocsparse
rocmPackages.rocsolver
rocmPackages.rocthrust
rocmPackages.rocprim
rocmPackages.hipcub
];
# The setup hooks of the joined packages would rewrite build flags; the
# compiler only needs the files.
postBuild = ''
rm -rf $out/nix-support
'';
};
# The library: exllamav3 plus the compiled exllamav3_ext module.
library = py.buildPythonPackage {
inherit src version;
pname = "kyojin";
format = "pyproject";
build-system = with py; [
setuptools
wheel
];
nativeBuildInputs = [
py.ninja
rocmPackages."rocm-runtime"
];
buildInputs = [ (lib.getBin rocmPackages.hipcc) ];
propagatedBuildInputs = with py; [
aiohttp
huggingface-hub
jinja2
llguidance
marisa-trie
ninja
numpy
pillow
pydantic
pyyaml
rich
safetensors
tokenizers
torchWithRocm
typing-extensions
];
env = {
PYTORCH_ROCM_ARCH = rocmArch;
ROCM_PATH = "${rocmToolkit}";
ROCM_HOME = "${rocmToolkit}";
HIP_PATH = "${rocmToolkit}";
HIPCC = "${rocmToolkit}/bin/hipcc";
HIP_DEVICE_LIB_PATH = "${rocmPackages."rocm-device-libs"}/amdgcn/bitcode";
# Upstream build scripts look these up to find the ROCm devel tree.
EXL3_ROCM_SDK = "${rocmToolkit}";
EXL3_ROCM_DEV_INCLUDE = "${rocmDevInclude}/include";
# Default defines of upstream build.sh (tools/strix_halo/rebuild.sh).
EXL3_HIP_DEFINES = "EXL3_HIP_STG_PAD";
};
# setup.py loads exllamav3/exllamav3_ext/build_config.py by path (and
# pyproject.toml ships the .hip sources, qsa_proof markers, and dense-GEMM
# tuning seed), so no source patches are needed.
# tests/ needs a real gfx1151 GPU and the published model packs.
doCheck = false;
meta = with lib; {
description = "Inference engine for 100 GB-class EXL3 MoE models on AMD Strix Halo (gfx1151, ROCm 7)";
homepage = "https://github.com/Yamz-Labs/kyojin";
changelog = "https://github.com/Yamz-Labs/kyojin/releases";
license = licenses.mit;
sourceProvenance = with sourceTypes; [ fromSource ];
# HIP code objects are gfx1151-only and ROCm is Linux-only.
platforms = platforms.linux;
};
};
python = py.python.withPackages (_: [ library ]);
jitSdk = if jitRocmSdk == null then rocmToolkit else jitRocmSdk;
in
stdenv.mkDerivation {
inherit src version;
pname = "kyojin";
# Only the wrappers are installed here: the serve scripts are plain scripts,
# they import their siblings by path, and the module itself is in `library`.
#
# Upstream `tools/strix_halo/env.sh` is sourced before serving, and on a
# distro box that brings a system `cc` (the AMD Triton backend compiles a
# CPython glue module at runtime, and again per kernel launcher) plus a hipcc
# for the JIT HIP kernels (qsa prefill, gdn, kda, ple). Nix gives the wrapper
# neither, so export them here instead: Triton takes the compiler from $CC,
# exllamav3 finds hipcc under $EXL3_ROCM_SDK, and hipclang only finds the
# amdgcn bitcode through $HIP_DEVICE_LIB_PATH (hipcc's own DEVICE_LIB_PATH is
# not enough for a --genco call). The ROCm paths stay out of LD_LIBRARY_PATH
# (upstream Trap 1/5: a second libhsa segfaults torch).
dontConfigure = true;
dontBuild = true;
nativeBuildInputs = [ makeWrapper ];
installPhase = ''
runHook preInstall
mkdir -p $out/bin
for model in glm mimo qwen; do
makeWrapper ${python}/bin/python $out/bin/kyojin-serve-$model \
--set PYTORCH_ROCM_ARCH ${rocmArch} \
--set CC ${stdenv.cc}/bin/cc \
--set EXL3_ROCM_SDK ${jitSdk} \
--set HIP_DEVICE_LIB_PATH ${rocmPackages."rocm-device-libs"}/amdgcn/bitcode \
--set EXL3_EXPANDABLE_SEGMENTS 0 \
--prefix PATH : ${lib.makeBinPath [ stdenv.cc ]} \
--add-flags "${src}/tools/$model/serve.py"
done
runHook postInstall
'';
passthru = {
inherit python rocmArch;
pythonPackage = library;
jitRocmSdk = jitSdk;
tests.jit =
pkgs.runCommand "kyojin-jit-check"
{
EXL3_ROCM_SDK = jitSdk;
# CLR/COMGR also reads this variable. Newer bitcode crashes torch
# matmul; a custom hipcc must select its own bitcode in its subprocess.
HIP_DEVICE_LIB_PATH = "${rocmPackages."rocm-device-libs"}/amdgcn/bitcode";
EXL3_EXPANDABLE_SEGMENTS = "0";
KYOJIN_REQUIRE_SDK = lib.boolToString (jitRocmSdk != null);
}
''
export HOME="$TMPDIR"
${python}/bin/python ${./check-jit.py}
touch "$out"
'';
passthru = (old.passthru or { }) // {
category = "AI Inference";
updateScript = [
"nix-update"
"--flake"
".#kyojin"
];
tests = (old.passthru.tests or { }) // {
jit = jitCheck;
cpu = cpuCheck;
};
};
meta = library.meta // {
mainProgram = "kyojin-serve-qwen";
};
}
postFixup = ''
# Fail the build, not the first request, if upstream renames the wrappers.
ls "$out"/bin/kyojin-serve-* > /dev/null 2>&1 || {
echo "kyojin-cc: no kyojin-serve-* in $out/bin; upstream moved them" >&2
exit 1
}
for b in $out/bin/kyojin-serve-*; do
wrapProgram "$b" --set CC ${pkgs.stdenv.cc}/bin/gcc
done
'';
})