d14f6d5e37
Upstream's gfx1151-hipblaslt-100401.txt was calibrated on TheRock 7.14.1. therock-rocm 10.1.0rc3 reports the same hipBLASLt version, so the engine accepted the table, but its solution ids do not exist in that build and every prompt GEMM fell back to the heuristic. The table now comes from tools/hip/tune_hipblaslt run on gfx1151 against therock-rocm 10.1 and is installed in bin/tuning/, where the engine finds it by itself. UD-Q5_K_XL, 17.5K cold prompt on ms-s1-max: 418 -> 700-725 tok/s. Bead: nix-overlay-2yv
135 lines
3.9 KiB
Nix
135 lines
3.9 KiB
Nix
{
|
|
lib,
|
|
stdenv,
|
|
fetchFromGitHub,
|
|
cmake,
|
|
ninja,
|
|
makeWrapper,
|
|
python3,
|
|
glibc,
|
|
therockRocm,
|
|
}:
|
|
let
|
|
# The vision encoder needs the full llama.cpp tree at Strata's pinned ggml revision.
|
|
llama = fetchFromGitHub {
|
|
owner = "ggml-org";
|
|
repo = "llama.cpp";
|
|
rev = "3cf03257f219afbe7334045ff7c6a06ac68c627d";
|
|
hash = "sha256-SRGoXa+4ACBCB3eaG9XFYhMN1i0FyPEy9Rrer+dFGYI=";
|
|
};
|
|
python = python3.withPackages (
|
|
ps: with ps; [
|
|
jinja2
|
|
pyyaml
|
|
numpy
|
|
regex
|
|
requests
|
|
psutil
|
|
pillow
|
|
jsonschema
|
|
referencing
|
|
truststore
|
|
certifi
|
|
tqdm
|
|
]
|
|
);
|
|
hipHostFlags = import ../../_lib/therock-host-flags.nix {
|
|
inherit lib stdenv;
|
|
libc = glibc;
|
|
};
|
|
in
|
|
stdenv.mkDerivation (finalAttrs: {
|
|
pname = "strata";
|
|
version = "0.2.13";
|
|
|
|
src = fetchFromGitHub {
|
|
owner = "Niko1221";
|
|
repo = "Strata";
|
|
tag = "v${finalAttrs.version}";
|
|
hash = "sha256-ygLtDYyrwfRKez46xKnPkO7BtY0Q4uMwxlvh3JyUHbE=";
|
|
};
|
|
|
|
nativeBuildInputs = [
|
|
cmake
|
|
ninja
|
|
makeWrapper
|
|
];
|
|
|
|
cmakeFlags = [
|
|
"-DSTRATA_ENABLE_HIP=ON"
|
|
"-DSTRATA_ENABLE_CUDA=OFF"
|
|
"-DSTRATA_PREFILL_MMQ=ON"
|
|
"-DSTRATA_BUILD_TESTS=OFF"
|
|
"-DSTRATA_PORTABLE=ON"
|
|
"-DCMAKE_HIP_ARCHITECTURES=gfx1151"
|
|
"-DCMAKE_HIP_COMPILER=${therockRocm}/lib/llvm/bin/clang++"
|
|
"-DCMAKE_HIP_COMPILER_ROCM_ROOT=${therockRocm}"
|
|
"-DCMAKE_PREFIX_PATH=${therockRocm};${therockRocm}/lib/rocm_sysdeps;${therockRocm}/lib/llvm"
|
|
"-DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath,${therockRocm}/lib:${therockRocm}/lib/rocm_sysdeps/lib"
|
|
];
|
|
|
|
env = {
|
|
ROCM_PATH = "${therockRocm}";
|
|
HIP_PATH = "${therockRocm}";
|
|
HIP_PLATFORM = "amd";
|
|
};
|
|
|
|
preConfigure = ''
|
|
cmakeFlagsArray+=("-DCMAKE_HIP_FLAGS=--rocm-path=${therockRocm} --rocm-device-lib-path=${therockRocm}/amdgcn/bitcode ${hipHostFlags}")
|
|
export PATH=${therockRocm}/bin:${therockRocm}/lib/llvm/bin:$PATH
|
|
export LD_LIBRARY_PATH=${therockRocm}/lib:${therockRocm}/lib/rocm_sysdeps/lib:${therockRocm}/lib/llvm/lib''${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}
|
|
'';
|
|
|
|
ninjaFlags = [ "strata" ];
|
|
|
|
# Upstream's 100401 table was calibrated on TheRock 7.14.1. therock-rocm 10.1 reports the same
|
|
# hipBLASLt version but other solution ids, so that table is a silent no-op here.
|
|
postPatch = ''
|
|
cp ${./gfx1151-hipblaslt-100401.txt} tools/hip/gfx1151-hipblaslt-100401.txt
|
|
'';
|
|
|
|
postBuild = ''
|
|
cmake -S ../tools/vision -B ../build-vision -G Ninja \
|
|
-DCMAKE_BUILD_TYPE=Release -DLLAMA_DIR=${llama} \
|
|
-DSTRATA_VISION_CUDA=OFF -DSTRATA_VISION_HIP=OFF -DSTRATA_PORTABLE=ON
|
|
cmake --build ../build-vision --target strata-vision -j $NIX_BUILD_CORES
|
|
'';
|
|
|
|
installPhase = ''
|
|
runHook preInstall
|
|
install -Dm755 strata $out/bin/strata
|
|
install -Dm755 ../build-vision/bin/strata-vision $out/bin/strata-vision
|
|
# The engine reads <exe dir>/tuning/ by itself (postPatch puts the 10.1 table in tools/hip).
|
|
install -Dm644 ../tools/hip/gfx1151-hipblaslt-100401.txt $out/bin/tuning/gfx1151-hipblaslt-100401.txt
|
|
mkdir -p $out/share/strata
|
|
cp -r ../serve ../strata_control ../strata_cli ../tools ../data $out/share/strata/
|
|
makeWrapper ${python}/bin/python $out/bin/strata-server \
|
|
--add-flags "-m serve.server" \
|
|
--prefix PYTHONPATH : $out/share/strata \
|
|
--prefix PATH : $out/bin
|
|
runHook postInstall
|
|
'';
|
|
|
|
doInstallCheck = true;
|
|
installCheckPhase = ''
|
|
runHook preInstallCheck
|
|
$out/bin/strata --help > /dev/null
|
|
$out/bin/strata-server --help > /dev/null
|
|
$out/bin/strata-vision --version > /dev/null
|
|
runHook postInstallCheck
|
|
'';
|
|
|
|
passthru = {
|
|
rocm = therockRocm;
|
|
category = "AI Inference";
|
|
};
|
|
|
|
meta = {
|
|
description = "Strata inference engine and API server with ROCm 10 HIP kernels for gfx1151";
|
|
homepage = "https://github.com/Niko1221/Strata";
|
|
license = lib.licenses.mit;
|
|
platforms = [ "x86_64-linux" ];
|
|
mainProgram = "strata-server";
|
|
};
|
|
})
|