7bb05ea685
CI / check (push) Has been cancelled
Strata 0.2.14 leaves llama.cpp's MMQ prompt kernels out by default, so STRATA_MMQ_KQ does nothing and UD-Q5_K_XL's Q6_K layer runs on Strata's int8 kernel. On ms-s1-max, 0.2.15 with the gfx1151 prompt set and STRATA_MMQ_KQ=1 gives first-token logits bit-identical to 0.2.13 over 15 cold prompts at the same speed (~1180 tok/s at 16K); without MMQ an agent prompt reached KL 0.199 against 0.2.13. The engine grows from 39 to 54 MB. Bead: nix-overlay-481
132 lines
3.8 KiB
Nix
132 lines
3.8 KiB
Nix
{
|
|
lib,
|
|
stdenv,
|
|
fetchFromGitHub,
|
|
cmake,
|
|
ninja,
|
|
makeWrapper,
|
|
python3,
|
|
glibc,
|
|
therockRocm,
|
|
}:
|
|
let
|
|
python = python3.withPackages (
|
|
ps: with ps; [
|
|
jinja2
|
|
pyyaml
|
|
numpy
|
|
regex
|
|
requests
|
|
psutil
|
|
pillow
|
|
jsonschema
|
|
referencing
|
|
truststore
|
|
certifi
|
|
tqdm
|
|
]
|
|
);
|
|
hipHostFlags = import ../../_lib/therock-host-flags.nix {
|
|
inherit lib stdenv;
|
|
libc = glibc;
|
|
};
|
|
in
|
|
stdenv.mkDerivation (finalAttrs: {
|
|
pname = "strata";
|
|
version = "0.2.15";
|
|
|
|
src = fetchFromGitHub {
|
|
owner = "Niko1221";
|
|
repo = "Strata";
|
|
tag = "v${finalAttrs.version}";
|
|
hash = "sha256-ehbQHpljUeclywp0ubGcrGG+xQ35SppRoosVWZLJiDY=";
|
|
};
|
|
|
|
nativeBuildInputs = [
|
|
cmake
|
|
ninja
|
|
makeWrapper
|
|
];
|
|
|
|
cmakeFlags = [
|
|
"-DSTRATA_ENABLE_HIP=ON"
|
|
"-DSTRATA_ENABLE_CUDA=OFF"
|
|
"-DSTRATA_PREFILL_MMQ=ON"
|
|
# Off upstream since 0.2.14. With it, STRATA_MMQ_KQ=1 keeps UD-Q5_K_XL's
|
|
# Q6_K layer on MMQ: the same logits as 0.2.13, at the same prompt speed.
|
|
"-DSTRATA_MMQ=ON"
|
|
"-DSTRATA_BUILD_TESTS=OFF"
|
|
"-DSTRATA_PORTABLE=ON"
|
|
"-DCMAKE_HIP_ARCHITECTURES=gfx1151"
|
|
"-DCMAKE_HIP_COMPILER=${therockRocm}/lib/llvm/bin/clang++"
|
|
"-DCMAKE_HIP_COMPILER_ROCM_ROOT=${therockRocm}"
|
|
"-DCMAKE_PREFIX_PATH=${therockRocm};${therockRocm}/lib/rocm_sysdeps;${therockRocm}/lib/llvm"
|
|
"-DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath,${therockRocm}/lib:${therockRocm}/lib/rocm_sysdeps/lib"
|
|
];
|
|
|
|
env = {
|
|
ROCM_PATH = "${therockRocm}";
|
|
HIP_PATH = "${therockRocm}";
|
|
HIP_PLATFORM = "amd";
|
|
};
|
|
|
|
preConfigure = ''
|
|
cmakeFlagsArray+=("-DCMAKE_HIP_FLAGS=--rocm-path=${therockRocm} --rocm-device-lib-path=${therockRocm}/amdgcn/bitcode ${hipHostFlags}")
|
|
export PATH=${therockRocm}/bin:${therockRocm}/lib/llvm/bin:$PATH
|
|
export LD_LIBRARY_PATH=${therockRocm}/lib:${therockRocm}/lib/rocm_sysdeps/lib:${therockRocm}/lib/llvm/lib''${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}
|
|
'';
|
|
|
|
ninjaFlags = [ "strata" ];
|
|
|
|
# Upstream's 100401 table was calibrated on TheRock 7.14.1. therock-rocm 10.1 reports the same
|
|
# hipBLASLt version but other solution ids, so that table is a silent no-op here.
|
|
postPatch = ''
|
|
cp ${./gfx1151-hipblaslt-100401.txt} tools/hip/gfx1151-hipblaslt-100401.txt
|
|
'';
|
|
|
|
# Vision builds from the vendored third_party/strata-llama (LLAMA_DIR default).
|
|
postBuild = ''
|
|
cmake -S ../tools/vision -B ../build-vision -G Ninja \
|
|
-DCMAKE_BUILD_TYPE=Release \
|
|
-DSTRATA_VISION_CUDA=OFF -DSTRATA_VISION_HIP=OFF -DSTRATA_PORTABLE=ON
|
|
cmake --build ../build-vision --target strata-vision -j $NIX_BUILD_CORES
|
|
'';
|
|
|
|
installPhase = ''
|
|
runHook preInstall
|
|
install -Dm755 strata $out/bin/strata
|
|
install -Dm755 ../build-vision/bin/strata-vision $out/bin/strata-vision
|
|
# The engine reads <exe dir>/tuning/ by itself (postPatch puts the 10.1 table in tools/hip).
|
|
install -Dm644 ../tools/hip/gfx1151-hipblaslt-100401.txt $out/bin/tuning/gfx1151-hipblaslt-100401.txt
|
|
mkdir -p $out/share/strata
|
|
cp -r ../serve ../strata_control ../strata_cli ../tools ../data $out/share/strata/
|
|
makeWrapper ${python}/bin/python $out/bin/strata-server \
|
|
--add-flags "-m serve.server" \
|
|
--prefix PYTHONPATH : $out/share/strata \
|
|
--prefix PATH : $out/bin
|
|
runHook postInstall
|
|
'';
|
|
|
|
doInstallCheck = true;
|
|
installCheckPhase = ''
|
|
runHook preInstallCheck
|
|
$out/bin/strata --help > /dev/null
|
|
$out/bin/strata-server --help > /dev/null
|
|
$out/bin/strata-vision --version > /dev/null
|
|
runHook postInstallCheck
|
|
'';
|
|
|
|
passthru = {
|
|
rocm = therockRocm;
|
|
category = "AI Inference";
|
|
};
|
|
|
|
meta = {
|
|
description = "Strata inference engine and API server with ROCm 10 HIP kernels for gfx1151";
|
|
homepage = "https://github.com/Niko1221/Strata";
|
|
license = lib.licenses.mit;
|
|
platforms = [ "x86_64-linux" ];
|
|
mainProgram = "strata-server";
|
|
};
|
|
})
|