Give two-session Qwen serving a bounded, opt-in decode grace for large cold prompts. Returning to normal interleaving at the deadline bounds the wait when the running reply takes longer than expected. Bead: nix-overlay-qfi
This commit is contained in:
@@ -5,6 +5,11 @@
|
||||
cpuCheck,
|
||||
}:
|
||||
kyojin.overrideAttrs (old: {
|
||||
# Upstream installs tools directly from $src, bypassing the unpacked tree.
|
||||
src = pkgs.applyPatches {
|
||||
src = old.src;
|
||||
patches = [ ./short-reply-prefill-grace.patch ];
|
||||
};
|
||||
nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ pkgs.makeWrapper ];
|
||||
meta = (old.meta or { }) // {
|
||||
homepage = "https://github.com/Yamz-Labs/kyojin";
|
||||
|
||||
Reference in New Issue
Block a user