feat(llm): package upstream inference engines

Centralize engine pins, compiler wrappers and NPU packages.
Use upstream Kyojin v1.8.10 and drop the Thunderbolt dependency.

Offline evaluation passed; workstation builds remain pending.

Bead: nix-overlay-v09
This commit is contained in:
2026-11-12 20:36:52 +03:00
parent 47829fe9a6
commit 85bd7421de
21 changed files with 865 additions and 309 deletions
+5
View File
@@ -0,0 +1,5 @@
{ pkgs, inputs, ... }:
(import ../../_lib/llm-packages.nix {
inherit inputs;
system = pkgs.stdenv.hostPlatform.system;
}).gufo
+58
View File
@@ -0,0 +1,58 @@
--- a/src/core/gguf_reader.cpp
+++ b/src/core/gguf_reader.cpp
@@ -747,8 +747,11 @@
tensors_.push_back(std::move(info));
}
- // Align data payload base to alignment boundary
+ // Align data payload base to alignment boundary. A tensorless file, such as
+ // a metadata-only first split shard, may end unpadded after its header.
const auto padding = (alignment_ - offset % alignment_) % alignment_;
+ if (tensors_.empty())
+ return true;
if (offset > size_ || padding > size_ - offset)
return false;
const std::size_t data_base = offset + padding;
--- a/src/models/qwen38_flash_next/weights.cpp
+++ b/src/models/qwen38_flash_next/weights.cpp
@@ -292,7 +292,9 @@
b.Fail("per_layer_token_embd.weight is missing or too short");
} else {
w.ple_table = b.Get("per_layer_token_embd.weight", c.ple_head_dim, rows,
- 1, {GgmlType::kIQ4_NL, GgmlType::kBF16});
+ 1,
+ {GgmlType::kIQ4_NL, GgmlType::kQ8_0,
+ GgmlType::kBF16});
}
}
w.layers.reserve(c.num_layers);
--- a/src/models/qwen38_flash_next/ngram.cpp
+++ b/src/models/qwen38_flash_next/ngram.cpp
@@ -81,10 +81,14 @@
int file_descriptor, std::uint64_t file_offset, std::uint64_t rows,
std::uint32_t row_dim, core::GgmlType type, std::string* error_msg) {
std::unique_ptr<NgramTable> t(new NgramTable());
- const std::size_t block = type == core::GgmlType::kIQ4_NL ? 32 : 1;
- const std::size_t block_bytes = type == core::GgmlType::kIQ4_NL ? 18 : 2;
- if ((type != core::GgmlType::kIQ4_NL && type != core::GgmlType::kBF16) ||
- row_dim == 0 || row_dim % block != 0) {
+ const bool blocked =
+ type == core::GgmlType::kIQ4_NL || type == core::GgmlType::kQ8_0;
+ const std::size_t block = blocked ? 32 : 1;
+ const std::size_t block_bytes = type == core::GgmlType::kIQ4_NL ? 18
+ : type == core::GgmlType::kQ8_0 ? 34
+ : 2;
+ if ((!blocked && type != core::GgmlType::kBF16) || row_dim == 0 ||
+ row_dim % block != 0) {
if (error_msg != nullptr) {
*error_msg = "unsupported n-gram table format";
}
@@ -127,6 +131,8 @@
void NgramTable::DecodeRow(const std::uint8_t* src, float* dst) const {
if (type_ == core::GgmlType::kIQ4_NL) {
gufo::quant::DequantizeIQ4_NL(src, dst, row_dim_);
+ } else if (type_ == core::GgmlType::kQ8_0) {
+ gufo::quant::DequantizeQ8_0(src, dst, row_dim_);
} else {
for (std::uint32_t i = 0; i < row_dim_; ++i) {
std::uint16_t bits = 0;
+44
View File
@@ -0,0 +1,44 @@
{ system, inputs }:
let
gufoPkgs = import inputs.gufo.inputs.nixpkgs {
inherit system;
config.allowUnfree = true;
};
nsh = inputs.nix-strix-halo;
xrt =
(gufoPkgs.callPackage (nsh + /pkgs/xrt) {
src = inputs.xrt-src;
version = "2.21";
xdnaSrc = inputs.xdna-driver-src;
xdnaVersion = "1.7";
}).xdna.overrideAttrs
(old: {
meta = (old.meta or { }) // {
mainProgram = "xrt-smi";
};
});
package = inputs.gufo.packages.${system}.release.overrideAttrs (old: {
nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [
gufoPkgs.libuuid.dev
xrt
];
cmakeFlags = old.cmakeFlags ++ [
"-DGUFO_ENABLE_NPU=ON"
"-DGUFO_XRT_INCLUDE_DIR=${xrt}/include"
"-DGUFO_XRT_LIB_DIR=${xrt}/lib"
];
patches = (old.patches or [ ]) ++ [ ./gufo-unsloth-ud3.patch ];
});
npuDir = "${inputs.gufo}/tools/npu";
fabricClock = "${inputs.gufo}/tools/npu/fabric-clock.sh";
in
package.overrideAttrs (old: {
passthru = (old.passthru or { }) // {
inherit xrt npuDir fabricClock;
category = "AI Inference";
};
})