feat(llm): package upstream inference engines
Centralize engine pins, compiler wrappers and NPU packages. Use upstream Kyojin v1.8.10 and drop the Thunderbolt dependency. Offline evaluation passed; workstation builds remain pending. Bead: nix-overlay-v09
This commit is contained in:
@@ -0,0 +1,58 @@
|
||||
--- a/src/core/gguf_reader.cpp
|
||||
+++ b/src/core/gguf_reader.cpp
|
||||
@@ -747,8 +747,11 @@
|
||||
tensors_.push_back(std::move(info));
|
||||
}
|
||||
|
||||
- // Align data payload base to alignment boundary
|
||||
+ // Align data payload base to alignment boundary. A tensorless file, such as
|
||||
+ // a metadata-only first split shard, may end unpadded after its header.
|
||||
const auto padding = (alignment_ - offset % alignment_) % alignment_;
|
||||
+ if (tensors_.empty())
|
||||
+ return true;
|
||||
if (offset > size_ || padding > size_ - offset)
|
||||
return false;
|
||||
const std::size_t data_base = offset + padding;
|
||||
--- a/src/models/qwen38_flash_next/weights.cpp
|
||||
+++ b/src/models/qwen38_flash_next/weights.cpp
|
||||
@@ -292,7 +292,9 @@
|
||||
b.Fail("per_layer_token_embd.weight is missing or too short");
|
||||
} else {
|
||||
w.ple_table = b.Get("per_layer_token_embd.weight", c.ple_head_dim, rows,
|
||||
- 1, {GgmlType::kIQ4_NL, GgmlType::kBF16});
|
||||
+ 1,
|
||||
+ {GgmlType::kIQ4_NL, GgmlType::kQ8_0,
|
||||
+ GgmlType::kBF16});
|
||||
}
|
||||
}
|
||||
w.layers.reserve(c.num_layers);
|
||||
--- a/src/models/qwen38_flash_next/ngram.cpp
|
||||
+++ b/src/models/qwen38_flash_next/ngram.cpp
|
||||
@@ -81,10 +81,14 @@
|
||||
int file_descriptor, std::uint64_t file_offset, std::uint64_t rows,
|
||||
std::uint32_t row_dim, core::GgmlType type, std::string* error_msg) {
|
||||
std::unique_ptr<NgramTable> t(new NgramTable());
|
||||
- const std::size_t block = type == core::GgmlType::kIQ4_NL ? 32 : 1;
|
||||
- const std::size_t block_bytes = type == core::GgmlType::kIQ4_NL ? 18 : 2;
|
||||
- if ((type != core::GgmlType::kIQ4_NL && type != core::GgmlType::kBF16) ||
|
||||
- row_dim == 0 || row_dim % block != 0) {
|
||||
+ const bool blocked =
|
||||
+ type == core::GgmlType::kIQ4_NL || type == core::GgmlType::kQ8_0;
|
||||
+ const std::size_t block = blocked ? 32 : 1;
|
||||
+ const std::size_t block_bytes = type == core::GgmlType::kIQ4_NL ? 18
|
||||
+ : type == core::GgmlType::kQ8_0 ? 34
|
||||
+ : 2;
|
||||
+ if ((!blocked && type != core::GgmlType::kBF16) || row_dim == 0 ||
|
||||
+ row_dim % block != 0) {
|
||||
if (error_msg != nullptr) {
|
||||
*error_msg = "unsupported n-gram table format";
|
||||
}
|
||||
@@ -127,6 +131,8 @@
|
||||
void NgramTable::DecodeRow(const std::uint8_t* src, float* dst) const {
|
||||
if (type_ == core::GgmlType::kIQ4_NL) {
|
||||
gufo::quant::DequantizeIQ4_NL(src, dst, row_dim_);
|
||||
+ } else if (type_ == core::GgmlType::kQ8_0) {
|
||||
+ gufo::quant::DequantizeQ8_0(src, dst, row_dim_);
|
||||
} else {
|
||||
for (std::uint32_t i = 0; i < row_dim_; ++i) {
|
||||
std::uint16_t bits = 0;
|
||||
Reference in New Issue
Block a user