feat(llm): package upstream inference engines
Centralize engine pins, compiler wrappers and NPU packages. Use upstream Kyojin v1.8.10 and drop the Thunderbolt dependency. Offline evaluation passed; workstation builds remain pending. Bead: nix-overlay-v09
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
{ pkgs, inputs, ... }:
|
||||
(import ../../_lib/llm-packages.nix {
|
||||
inherit inputs;
|
||||
system = pkgs.stdenv.hostPlatform.system;
|
||||
}).gufo
|
||||
@@ -0,0 +1,58 @@
|
||||
--- a/src/core/gguf_reader.cpp
|
||||
+++ b/src/core/gguf_reader.cpp
|
||||
@@ -747,8 +747,11 @@
|
||||
tensors_.push_back(std::move(info));
|
||||
}
|
||||
|
||||
- // Align data payload base to alignment boundary
|
||||
+ // Align data payload base to alignment boundary. A tensorless file, such as
|
||||
+ // a metadata-only first split shard, may end unpadded after its header.
|
||||
const auto padding = (alignment_ - offset % alignment_) % alignment_;
|
||||
+ if (tensors_.empty())
|
||||
+ return true;
|
||||
if (offset > size_ || padding > size_ - offset)
|
||||
return false;
|
||||
const std::size_t data_base = offset + padding;
|
||||
--- a/src/models/qwen38_flash_next/weights.cpp
|
||||
+++ b/src/models/qwen38_flash_next/weights.cpp
|
||||
@@ -292,7 +292,9 @@
|
||||
b.Fail("per_layer_token_embd.weight is missing or too short");
|
||||
} else {
|
||||
w.ple_table = b.Get("per_layer_token_embd.weight", c.ple_head_dim, rows,
|
||||
- 1, {GgmlType::kIQ4_NL, GgmlType::kBF16});
|
||||
+ 1,
|
||||
+ {GgmlType::kIQ4_NL, GgmlType::kQ8_0,
|
||||
+ GgmlType::kBF16});
|
||||
}
|
||||
}
|
||||
w.layers.reserve(c.num_layers);
|
||||
--- a/src/models/qwen38_flash_next/ngram.cpp
|
||||
+++ b/src/models/qwen38_flash_next/ngram.cpp
|
||||
@@ -81,10 +81,14 @@
|
||||
int file_descriptor, std::uint64_t file_offset, std::uint64_t rows,
|
||||
std::uint32_t row_dim, core::GgmlType type, std::string* error_msg) {
|
||||
std::unique_ptr<NgramTable> t(new NgramTable());
|
||||
- const std::size_t block = type == core::GgmlType::kIQ4_NL ? 32 : 1;
|
||||
- const std::size_t block_bytes = type == core::GgmlType::kIQ4_NL ? 18 : 2;
|
||||
- if ((type != core::GgmlType::kIQ4_NL && type != core::GgmlType::kBF16) ||
|
||||
- row_dim == 0 || row_dim % block != 0) {
|
||||
+ const bool blocked =
|
||||
+ type == core::GgmlType::kIQ4_NL || type == core::GgmlType::kQ8_0;
|
||||
+ const std::size_t block = blocked ? 32 : 1;
|
||||
+ const std::size_t block_bytes = type == core::GgmlType::kIQ4_NL ? 18
|
||||
+ : type == core::GgmlType::kQ8_0 ? 34
|
||||
+ : 2;
|
||||
+ if ((!blocked && type != core::GgmlType::kBF16) || row_dim == 0 ||
|
||||
+ row_dim % block != 0) {
|
||||
if (error_msg != nullptr) {
|
||||
*error_msg = "unsupported n-gram table format";
|
||||
}
|
||||
@@ -127,6 +131,8 @@
|
||||
void NgramTable::DecodeRow(const std::uint8_t* src, float* dst) const {
|
||||
if (type_ == core::GgmlType::kIQ4_NL) {
|
||||
gufo::quant::DequantizeIQ4_NL(src, dst, row_dim_);
|
||||
+ } else if (type_ == core::GgmlType::kQ8_0) {
|
||||
+ gufo::quant::DequantizeQ8_0(src, dst, row_dim_);
|
||||
} else {
|
||||
for (std::uint32_t i = 0; i < row_dim_; ++i) {
|
||||
std::uint16_t bits = 0;
|
||||
@@ -0,0 +1,44 @@
|
||||
{ system, inputs }:
|
||||
let
|
||||
gufoPkgs = import inputs.gufo.inputs.nixpkgs {
|
||||
inherit system;
|
||||
config.allowUnfree = true;
|
||||
};
|
||||
nsh = inputs.nix-strix-halo;
|
||||
|
||||
xrt =
|
||||
(gufoPkgs.callPackage (nsh + /pkgs/xrt) {
|
||||
src = inputs.xrt-src;
|
||||
version = "2.21";
|
||||
xdnaSrc = inputs.xdna-driver-src;
|
||||
xdnaVersion = "1.7";
|
||||
}).xdna.overrideAttrs
|
||||
(old: {
|
||||
meta = (old.meta or { }) // {
|
||||
mainProgram = "xrt-smi";
|
||||
};
|
||||
});
|
||||
|
||||
package = inputs.gufo.packages.${system}.release.overrideAttrs (old: {
|
||||
nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [
|
||||
gufoPkgs.libuuid.dev
|
||||
xrt
|
||||
];
|
||||
cmakeFlags = old.cmakeFlags ++ [
|
||||
"-DGUFO_ENABLE_NPU=ON"
|
||||
"-DGUFO_XRT_INCLUDE_DIR=${xrt}/include"
|
||||
"-DGUFO_XRT_LIB_DIR=${xrt}/lib"
|
||||
];
|
||||
patches = (old.patches or [ ]) ++ [ ./gufo-unsloth-ud3.patch ];
|
||||
});
|
||||
|
||||
npuDir = "${inputs.gufo}/tools/npu";
|
||||
|
||||
fabricClock = "${inputs.gufo}/tools/npu/fabric-clock.sh";
|
||||
in
|
||||
package.overrideAttrs (old: {
|
||||
passthru = (old.passthru or { }) // {
|
||||
inherit xrt npuDir fabricClock;
|
||||
category = "AI Inference";
|
||||
};
|
||||
})
|
||||
@@ -0,0 +1,5 @@
|
||||
{ pkgs, inputs, ... }:
|
||||
(import ../../_lib/llm-packages.nix {
|
||||
inherit inputs;
|
||||
system = pkgs.stdenv.hostPlatform.system;
|
||||
}).hipfire-kernels
|
||||
@@ -0,0 +1,5 @@
|
||||
{ pkgs, inputs, ... }:
|
||||
(import ../../_lib/llm-packages.nix {
|
||||
inherit inputs;
|
||||
system = pkgs.stdenv.hostPlatform.system;
|
||||
}).hipfire
|
||||
@@ -0,0 +1,123 @@
|
||||
{
|
||||
pkgs,
|
||||
hipfireSource,
|
||||
therockRocm,
|
||||
modelDir ? "/var/lib/hipfire/models",
|
||||
}:
|
||||
let
|
||||
lib = pkgs.lib;
|
||||
gcc = pkgs.stdenv.cc;
|
||||
|
||||
hipccFixed = pkgs.writeShellScriptBin "hipcc" ''
|
||||
export ROCM_PATH=${therockRocm}
|
||||
exec ${therockRocm}/bin/hipcc \
|
||||
--gcc-toolchain=${gcc.cc} \
|
||||
-idirafter ${gcc.libc.dev}/include \
|
||||
-B${gcc.libc}/lib \
|
||||
-L${gcc.libc}/lib \
|
||||
-L${gcc.cc.lib}/lib \
|
||||
-L${gcc.cc}/lib \
|
||||
"$@"
|
||||
'';
|
||||
|
||||
glibcLoader = "${pkgs.glibc}/lib/ld-linux-x86-64.so.2";
|
||||
llvmRunPath = lib.makeSearchPathOutput "lib" "lib" [
|
||||
gcc.cc.lib
|
||||
gcc.cc
|
||||
];
|
||||
therockLlvmTool =
|
||||
name:
|
||||
pkgs.writeShellScriptBin name ''
|
||||
exec ${glibcLoader} --library-path ${therockRocm}/lib:${llvmRunPath} \
|
||||
${therockRocm}/lib/llvm/bin/${name} "$@"
|
||||
'';
|
||||
llvmReadobj = therockLlvmTool "llvm-readobj";
|
||||
llvmObjdump = therockLlvmTool "llvm-objdump";
|
||||
|
||||
therockDeviceLibs = pkgs.runCommand "therock-device-libs" { } ''
|
||||
mkdir -p $out/amdgcn
|
||||
ln -sfn ${therockRocm}/amdgcn/bitcode $out/amdgcn/bitcode
|
||||
'';
|
||||
|
||||
hipfireKernelsPkg =
|
||||
(pkgs.callPackage (hipfireSource + "/nix/kernels.nix") {
|
||||
src = hipfireSource;
|
||||
cargoLockFile = hipfireSource + "/Cargo.lock";
|
||||
gpuTargets = [ "gfx1151" ];
|
||||
rocmPackages = pkgs.rocmPackages // {
|
||||
clr = therockRocm;
|
||||
rocm-device-libs = therockDeviceLibs;
|
||||
};
|
||||
}).overrideAttrs
|
||||
(old: {
|
||||
nativeBuildInputs = old.nativeBuildInputs ++ [ hipccFixed ];
|
||||
env.HIPFIRE_HIPCC = "${hipccFixed}/bin/hipcc";
|
||||
});
|
||||
|
||||
hipfireWithQuantize = pkgs.hipfire.overrideAttrs (_: {
|
||||
buildPhase = ''
|
||||
runHook preBuild
|
||||
cargo build --release -p hipfire-daemon
|
||||
cargo build --release -p hipfire-cli
|
||||
cargo build --release -p hipfire-quantize \
|
||||
--bin hipfire-quantize --bin dflash_convert --bin draft_to_mq4
|
||||
runHook postBuild
|
||||
'';
|
||||
|
||||
installPhase = ''
|
||||
runHook preInstall
|
||||
|
||||
mkdir -p $out/bin
|
||||
|
||||
cp target/release/daemon $out/bin/hipfire-daemon-unwrapped
|
||||
makeWrapper $out/bin/hipfire-daemon-unwrapped $out/bin/hipfire-daemon \
|
||||
--set HIPFIRE_ROCM_PATH ${therockRocm} \
|
||||
--set HIPFIRE_HIPCC ${hipccFixed}/bin/hipcc \
|
||||
--set ROCM_PATH ${therockRocm} \
|
||||
--set RADIOWAVE_READOBJ ${llvmReadobj}/bin/llvm-readobj \
|
||||
--set RADIOWAVE_OBJDUMP ${llvmObjdump}/bin/llvm-objdump \
|
||||
--prefix PATH : ${
|
||||
pkgs.lib.makeBinPath [
|
||||
hipccFixed
|
||||
pkgs.pciutils
|
||||
]
|
||||
} \
|
||||
--prefix LD_LIBRARY_PATH : ${therockRocm}/lib
|
||||
|
||||
cp target/release/hipfire $out/bin/hipfire-unwrapped
|
||||
makeWrapper $out/bin/hipfire-unwrapped $out/bin/hipfire \
|
||||
--set HIPFIRE_DAEMON_BIN $out/bin/hipfire-daemon \
|
||||
--set HIPFIRE_NORMALIZE_PROMPT "1" \
|
||||
--set HIPFIRE_MODELS_DIR "${modelDir}" \
|
||||
--set HIP_PATH "${therockRocm}" \
|
||||
--set HIPFIRE_HIPCC ${hipccFixed}/bin/hipcc \
|
||||
--set ROCM_PATH ${therockRocm} \
|
||||
--set RADIOWAVE_READOBJ ${llvmReadobj}/bin/llvm-readobj \
|
||||
--set RADIOWAVE_OBJDUMP ${llvmObjdump}/bin/llvm-objdump \
|
||||
--prefix PATH : ${
|
||||
pkgs.lib.makeBinPath [
|
||||
hipccFixed
|
||||
pkgs.pciutils
|
||||
]
|
||||
} \
|
||||
--prefix LD_LIBRARY_PATH : ${therockRocm}/lib
|
||||
|
||||
mkdir -p $out/share/hipfire/target/release
|
||||
cp target/release/hipfire-quantize $out/share/hipfire/target/release/
|
||||
cp target/release/dflash_convert $out/share/hipfire/target/release/
|
||||
cp target/release/draft_to_mq4 $out/share/hipfire/target/release/
|
||||
|
||||
runHook postInstall
|
||||
'';
|
||||
});
|
||||
|
||||
in
|
||||
hipfireWithQuantize.overrideAttrs (old: {
|
||||
passthru = (old.passthru or { }) // {
|
||||
kernels = hipfireKernelsPkg;
|
||||
hipcc = hipccFixed;
|
||||
inherit llvmReadobj llvmObjdump;
|
||||
rocm = therockRocm;
|
||||
category = "AI Inference";
|
||||
};
|
||||
})
|
||||
@@ -1,46 +0,0 @@
|
||||
"""Compile Kyojin's JIT kernels; --gpu also loads them through torch's HIP runtime."""
|
||||
import ctypes
|
||||
import importlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
from exllamav3.util import hip_compiler
|
||||
from exllamav3.util.hip_lib import load_hip_runtime
|
||||
|
||||
sdk = Path(os.environ["EXL3_ROCM_SDK"])
|
||||
assert Path(hip_compiler.hipcc()) == sdk / "bin/hipcc"
|
||||
assert Path(os.environ["HIP_DEVICE_LIB_PATH"]).is_dir()
|
||||
assert (sdk / "amdgcn/bitcode").is_dir()
|
||||
if os.environ.get("KYOJIN_REQUIRE_SDK") == "1":
|
||||
assert hip_compiler.warning() is None, hip_compiler.warning()
|
||||
print(hip_compiler.describe(), flush=True)
|
||||
|
||||
lib = None
|
||||
if "--gpu" in sys.argv:
|
||||
import torch
|
||||
|
||||
# Also exercise the runtime bitcode path: merely loading a HIP module
|
||||
# does not catch incompatible device libraries in CLR/COMGR.
|
||||
x = torch.ones((32, 32), device="cuda", dtype=torch.float32)
|
||||
assert torch.equal(x @ x, torch.full_like(x, 32))
|
||||
torch.cuda.synchronize()
|
||||
lib = load_hip_runtime()
|
||||
lib.hipModuleLoad.argtypes = [ctypes.POINTER(ctypes.c_void_p), ctypes.c_char_p]
|
||||
lib.hipModuleUnload.argtypes = [ctypes.c_void_p]
|
||||
|
||||
for module, function, args in [
|
||||
("exllamav3.modules.gr_fn.gr_mix_hip", "compile_hsaco", ()),
|
||||
("exllamav3.modules.attention_fn.qsa_prefill_hip", "compile_hsaco", ()),
|
||||
("exllamav3.modules.ple_fn.ple_hip", "compile_hsaco", ()),
|
||||
("exllamav3.vendor.fla.hip.gdn_fused_h_hip", "_compile", ("", "gfx1151")),
|
||||
("exllamav3.vendor.fla.hip.kda_fused_h_hip", "_compile", ("", "gfx1151")),
|
||||
]:
|
||||
path = Path(getattr(importlib.import_module(module), function)(*args))
|
||||
# HIP accepts ELF, offload bundles, and compressed offload bundles (CCOB).
|
||||
assert path.read_bytes().startswith((b"\x7fELF", b"__CLANG_OFFLOAD_BUNDLE__", b"CCOB")), path
|
||||
if lib is not None:
|
||||
handle = ctypes.c_void_p()
|
||||
assert lib.hipModuleLoad(ctypes.byref(handle), os.fsencode(path)) == 0, path
|
||||
assert lib.hipModuleUnload(handle) == 0, path
|
||||
print(f"OK: {module}", flush=True)
|
||||
@@ -1,13 +1,5 @@
|
||||
{
|
||||
pkgs,
|
||||
inputs,
|
||||
jitRocmSdk ? null,
|
||||
...
|
||||
}:
|
||||
# The HIP extension has to be compiled against a ROCm build of PyTorch, and the
|
||||
# only ROCm torch in this flake with a cached binary is the one from the pinned
|
||||
# nixpkgs-torch211 input (2.11 + ROCm 7.2, python 3.13). Current nixos-unstable
|
||||
# ships torch 2.13, whose ROCm build fails to configure in nixpkgs and is not
|
||||
# on cache.nixos.org.
|
||||
inputs.nixpkgs-torch211.legacyPackages.${pkgs.stdenv.hostPlatform.system}.callPackage ./package.nix
|
||||
{ inherit jitRocmSdk; }
|
||||
{ pkgs, inputs, ... }:
|
||||
(import ../../_lib/llm-packages.nix {
|
||||
inherit inputs;
|
||||
system = pkgs.stdenv.hostPlatform.system;
|
||||
}).kyojin
|
||||
|
||||
+24
-224
@@ -1,230 +1,30 @@
|
||||
{
|
||||
lib,
|
||||
stdenv,
|
||||
pkgs,
|
||||
python3,
|
||||
fetchFromGitHub,
|
||||
makeWrapper,
|
||||
rocmPackages,
|
||||
# Override only the serving toolchain; torch and the extension keep their ROCm.
|
||||
jitRocmSdk ? null,
|
||||
kyojin,
|
||||
jitCheck,
|
||||
cpuCheck,
|
||||
}:
|
||||
|
||||
let
|
||||
py = python3.pkgs;
|
||||
|
||||
# Kyojin is the Yamz fork of ExLlamaV3: a Python package plus a C++ extension
|
||||
# that torch's cpp_extension cross-compiles to HIP code objects. Upstream
|
||||
# only supports AMD Strix Halo, so the kernels are built for gfx1151 alone.
|
||||
rocmArch = "gfx1151";
|
||||
|
||||
version = "1.8.5";
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = "Yamz-Labs";
|
||||
repo = "kyojin";
|
||||
rev = "v${version}";
|
||||
hash = "sha256-OZhzRnQUrkJOrn2lVatvVxD6JSwtJkwfWwpRUQHawac=";
|
||||
kyojin.overrideAttrs (old: {
|
||||
nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ pkgs.makeWrapper ];
|
||||
meta = (old.meta or { }) // {
|
||||
homepage = "https://github.com/Yamz-Labs/kyojin";
|
||||
license = pkgs.lib.licenses.mit;
|
||||
};
|
||||
|
||||
# setup.py passes EXL3_ROCM_DEV_INCLUDE as -I to every translation unit. The
|
||||
# AMD wheels ship one devel tree holding hipsparse/ and thrust/; nixpkgs
|
||||
# splits them per package, so join the include dirs. torch's public HIP
|
||||
# headers also pull hipsolver/, and it no longer vendors pybind11.
|
||||
rocmDevInclude = pkgs.symlinkJoin {
|
||||
name = "kyojin-rocm-dev-include";
|
||||
paths = map lib.getDev (
|
||||
[ py.pybind11 ]
|
||||
++ [
|
||||
rocmPackages.clr
|
||||
rocmPackages.hipblas
|
||||
rocmPackages.hipblaslt
|
||||
rocmPackages.hipcub
|
||||
rocmPackages.hipfft
|
||||
rocmPackages.hiprand
|
||||
rocmPackages.hipsparse
|
||||
rocmPackages.hipsolver
|
||||
rocmPackages.rocblas
|
||||
rocmPackages.rocprim
|
||||
rocmPackages.rocrand
|
||||
rocmPackages.rocsolver
|
||||
rocmPackages.rocsparse
|
||||
rocmPackages.rocthrust
|
||||
]
|
||||
);
|
||||
};
|
||||
|
||||
# torch's HIP path in cpp_extension expects one ROCM_HOME holding bin/hipcc,
|
||||
# the headers and the amdgcn bitcode; nixpkgs ships them separately.
|
||||
rocmToolkit = pkgs.symlinkJoin {
|
||||
name = "kyojin-rocm-toolkit";
|
||||
paths = [
|
||||
rocmPackages."rocm-core"
|
||||
rocmPackages."rocm-runtime"
|
||||
rocmPackages."rocm-device-libs"
|
||||
rocmPackages."rocm-comgr"
|
||||
rocmPackages.clr
|
||||
(lib.getBin rocmPackages.hipcc)
|
||||
rocmPackages.rocblas
|
||||
rocmPackages.hipblas
|
||||
rocmPackages.hipblas-common
|
||||
rocmPackages.hipblaslt
|
||||
rocmPackages.hipsparse
|
||||
rocmPackages.hipsolver
|
||||
rocmPackages.hiprand
|
||||
rocmPackages.rocrand
|
||||
rocmPackages.hipfft
|
||||
rocmPackages.rocsparse
|
||||
rocmPackages.rocsolver
|
||||
rocmPackages.rocthrust
|
||||
rocmPackages.rocprim
|
||||
rocmPackages.hipcub
|
||||
];
|
||||
# The setup hooks of the joined packages would rewrite build flags; the
|
||||
# compiler only needs the files.
|
||||
postBuild = ''
|
||||
rm -rf $out/nix-support
|
||||
'';
|
||||
};
|
||||
|
||||
# The library: exllamav3 plus the compiled exllamav3_ext module.
|
||||
library = py.buildPythonPackage {
|
||||
inherit src version;
|
||||
pname = "kyojin";
|
||||
|
||||
format = "pyproject";
|
||||
build-system = with py; [
|
||||
setuptools
|
||||
wheel
|
||||
];
|
||||
|
||||
nativeBuildInputs = [
|
||||
py.ninja
|
||||
rocmPackages."rocm-runtime"
|
||||
];
|
||||
|
||||
buildInputs = [ (lib.getBin rocmPackages.hipcc) ];
|
||||
|
||||
propagatedBuildInputs = with py; [
|
||||
aiohttp
|
||||
huggingface-hub
|
||||
jinja2
|
||||
llguidance
|
||||
marisa-trie
|
||||
ninja
|
||||
numpy
|
||||
pillow
|
||||
pydantic
|
||||
pyyaml
|
||||
rich
|
||||
safetensors
|
||||
tokenizers
|
||||
torchWithRocm
|
||||
typing-extensions
|
||||
];
|
||||
|
||||
env = {
|
||||
PYTORCH_ROCM_ARCH = rocmArch;
|
||||
ROCM_PATH = "${rocmToolkit}";
|
||||
ROCM_HOME = "${rocmToolkit}";
|
||||
HIP_PATH = "${rocmToolkit}";
|
||||
HIPCC = "${rocmToolkit}/bin/hipcc";
|
||||
HIP_DEVICE_LIB_PATH = "${rocmPackages."rocm-device-libs"}/amdgcn/bitcode";
|
||||
# Upstream build scripts look these up to find the ROCm devel tree.
|
||||
EXL3_ROCM_SDK = "${rocmToolkit}";
|
||||
EXL3_ROCM_DEV_INCLUDE = "${rocmDevInclude}/include";
|
||||
# Default defines of upstream build.sh (tools/strix_halo/rebuild.sh).
|
||||
EXL3_HIP_DEFINES = "EXL3_HIP_STG_PAD";
|
||||
};
|
||||
|
||||
# setup.py loads exllamav3/exllamav3_ext/build_config.py by path (and
|
||||
# pyproject.toml ships the .hip sources, qsa_proof markers, and dense-GEMM
|
||||
# tuning seed), so no source patches are needed.
|
||||
|
||||
# tests/ needs a real gfx1151 GPU and the published model packs.
|
||||
doCheck = false;
|
||||
|
||||
meta = with lib; {
|
||||
description = "Inference engine for 100 GB-class EXL3 MoE models on AMD Strix Halo (gfx1151, ROCm 7)";
|
||||
homepage = "https://github.com/Yamz-Labs/kyojin";
|
||||
changelog = "https://github.com/Yamz-Labs/kyojin/releases";
|
||||
license = licenses.mit;
|
||||
sourceProvenance = with sourceTypes; [ fromSource ];
|
||||
# HIP code objects are gfx1151-only and ROCm is Linux-only.
|
||||
platforms = platforms.linux;
|
||||
};
|
||||
};
|
||||
|
||||
python = py.python.withPackages (_: [ library ]);
|
||||
jitSdk = if jitRocmSdk == null then rocmToolkit else jitRocmSdk;
|
||||
in
|
||||
stdenv.mkDerivation {
|
||||
inherit src version;
|
||||
pname = "kyojin";
|
||||
|
||||
# Only the wrappers are installed here: the serve scripts are plain scripts,
|
||||
# they import their siblings by path, and the module itself is in `library`.
|
||||
#
|
||||
# Upstream `tools/strix_halo/env.sh` is sourced before serving, and on a
|
||||
# distro box that brings a system `cc` (the AMD Triton backend compiles a
|
||||
# CPython glue module at runtime, and again per kernel launcher) plus a hipcc
|
||||
# for the JIT HIP kernels (qsa prefill, gdn, kda, ple). Nix gives the wrapper
|
||||
# neither, so export them here instead: Triton takes the compiler from $CC,
|
||||
# exllamav3 finds hipcc under $EXL3_ROCM_SDK, and hipclang only finds the
|
||||
# amdgcn bitcode through $HIP_DEVICE_LIB_PATH (hipcc's own DEVICE_LIB_PATH is
|
||||
# not enough for a --genco call). The ROCm paths stay out of LD_LIBRARY_PATH
|
||||
# (upstream Trap 1/5: a second libhsa segfaults torch).
|
||||
dontConfigure = true;
|
||||
dontBuild = true;
|
||||
|
||||
nativeBuildInputs = [ makeWrapper ];
|
||||
|
||||
installPhase = ''
|
||||
runHook preInstall
|
||||
|
||||
mkdir -p $out/bin
|
||||
for model in glm mimo qwen; do
|
||||
makeWrapper ${python}/bin/python $out/bin/kyojin-serve-$model \
|
||||
--set PYTORCH_ROCM_ARCH ${rocmArch} \
|
||||
--set CC ${stdenv.cc}/bin/cc \
|
||||
--set EXL3_ROCM_SDK ${jitSdk} \
|
||||
--set HIP_DEVICE_LIB_PATH ${rocmPackages."rocm-device-libs"}/amdgcn/bitcode \
|
||||
--set EXL3_EXPANDABLE_SEGMENTS 0 \
|
||||
--prefix PATH : ${lib.makeBinPath [ stdenv.cc ]} \
|
||||
--add-flags "${src}/tools/$model/serve.py"
|
||||
done
|
||||
|
||||
runHook postInstall
|
||||
'';
|
||||
|
||||
passthru = {
|
||||
inherit python rocmArch;
|
||||
pythonPackage = library;
|
||||
jitRocmSdk = jitSdk;
|
||||
tests.jit =
|
||||
pkgs.runCommand "kyojin-jit-check"
|
||||
{
|
||||
EXL3_ROCM_SDK = jitSdk;
|
||||
# CLR/COMGR also reads this variable. Newer bitcode crashes torch
|
||||
# matmul; a custom hipcc must select its own bitcode in its subprocess.
|
||||
HIP_DEVICE_LIB_PATH = "${rocmPackages."rocm-device-libs"}/amdgcn/bitcode";
|
||||
EXL3_EXPANDABLE_SEGMENTS = "0";
|
||||
KYOJIN_REQUIRE_SDK = lib.boolToString (jitRocmSdk != null);
|
||||
}
|
||||
''
|
||||
export HOME="$TMPDIR"
|
||||
${python}/bin/python ${./check-jit.py}
|
||||
touch "$out"
|
||||
'';
|
||||
passthru = (old.passthru or { }) // {
|
||||
category = "AI Inference";
|
||||
updateScript = [
|
||||
"nix-update"
|
||||
"--flake"
|
||||
".#kyojin"
|
||||
];
|
||||
tests = (old.passthru.tests or { }) // {
|
||||
jit = jitCheck;
|
||||
cpu = cpuCheck;
|
||||
};
|
||||
};
|
||||
|
||||
meta = library.meta // {
|
||||
mainProgram = "kyojin-serve-qwen";
|
||||
};
|
||||
}
|
||||
postFixup = ''
|
||||
# Fail the build, not the first request, if upstream renames the wrappers.
|
||||
ls "$out"/bin/kyojin-serve-* > /dev/null 2>&1 || {
|
||||
echo "kyojin-cc: no kyojin-serve-* in $out/bin; upstream moved them" >&2
|
||||
exit 1
|
||||
}
|
||||
for b in $out/bin/kyojin-serve-*; do
|
||||
wrapProgram "$b" --set CC ${pkgs.stdenv.cc}/bin/gcc
|
||||
done
|
||||
'';
|
||||
})
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
{ pkgs, inputs, ... }:
|
||||
(import ../../_lib/llm-packages.nix {
|
||||
inherit inputs;
|
||||
system = pkgs.stdenv.hostPlatform.system;
|
||||
}).pwilkin-llama-cpp
|
||||
@@ -0,0 +1,6 @@
|
||||
{ pkgs, source }:
|
||||
pkgs.callPackage (source + "/.devops/nix/package.nix") {
|
||||
llamaVersion = source.shortRev or "0.0.0";
|
||||
useRocm = true;
|
||||
rocmGpuTargets = "gfx1151";
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
{ pkgs, inputs, ... }:
|
||||
(import ../../_lib/llm-packages.nix {
|
||||
inherit inputs;
|
||||
system = pkgs.stdenv.hostPlatform.system;
|
||||
}).strix-llama-cpp
|
||||
@@ -0,0 +1,5 @@
|
||||
{ pkgs, source }:
|
||||
pkgs.callPackage (source + "/.devops/nix/package.nix") {
|
||||
llamaVersion = source.shortRev or "0.0.0";
|
||||
useVulkan = true;
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
{ pkgs, inputs, ... }:
|
||||
(import ../../_lib/llm-packages.nix {
|
||||
inherit inputs;
|
||||
system = pkgs.stdenv.hostPlatform.system;
|
||||
}).therock-rocm
|
||||
@@ -0,0 +1,10 @@
|
||||
{ pkgs, nix-strix-halo }:
|
||||
let
|
||||
source =
|
||||
(builtins.fromJSON (builtins.readFile (nix-strix-halo + "/pkgs/therock/sources/rocm.json")))
|
||||
.linux.gfx1151;
|
||||
in
|
||||
pkgs.callPackage (nix-strix-halo + "/pkgs/therock/rocm-sdk") {
|
||||
target = "gfx1151";
|
||||
inherit (source) version url hash;
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
{ pkgs, inputs, ... }:
|
||||
(import ../../_lib/llm-packages.nix {
|
||||
inherit inputs;
|
||||
system = pkgs.stdenv.hostPlatform.system;
|
||||
}).xrt-amdxdna
|
||||
Reference in New Issue
Block a user