afd0c4b84d
CI / check (push) Has been cancelled
Give two-session Qwen serving a bounded, opt-in decode grace for large cold prompts. Returning to normal interleaving at the deadline bounds the wait when the running reply takes longer than expected. Bead: nix-overlay-qfi
36 lines
1002 B
Nix
36 lines
1002 B
Nix
{
|
|
pkgs,
|
|
kyojin,
|
|
jitCheck,
|
|
cpuCheck,
|
|
}:
|
|
kyojin.overrideAttrs (old: {
|
|
# Upstream installs tools directly from $src, bypassing the unpacked tree.
|
|
src = pkgs.applyPatches {
|
|
src = old.src;
|
|
patches = [ ./short-reply-prefill-grace.patch ];
|
|
};
|
|
nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ pkgs.makeWrapper ];
|
|
meta = (old.meta or { }) // {
|
|
homepage = "https://github.com/Yamz-Labs/kyojin";
|
|
license = pkgs.lib.licenses.mit;
|
|
};
|
|
passthru = (old.passthru or { }) // {
|
|
category = "AI Inference";
|
|
tests = (old.passthru.tests or { }) // {
|
|
jit = jitCheck;
|
|
cpu = cpuCheck;
|
|
};
|
|
};
|
|
postFixup = ''
|
|
# Fail the build, not the first request, if upstream renames the wrappers.
|
|
ls "$out"/bin/kyojin-serve-* > /dev/null 2>&1 || {
|
|
echo "kyojin-cc: no kyojin-serve-* in $out/bin; upstream moved them" >&2
|
|
exit 1
|
|
}
|
|
for b in $out/bin/kyojin-serve-*; do
|
|
wrapProgram "$b" --set CC ${pkgs.stdenv.cc}/bin/gcc
|
|
done
|
|
'';
|
|
})
|