Give two-session Qwen serving a bounded, opt-in decode grace for large cold prompts. Returning to normal interleaving at the deadline bounds the wait when the running reply takes longer than expected. Bead: nix-overlay-qfi
This commit is contained in:
File diff suppressed because one or more lines are too long
@@ -5,6 +5,11 @@
|
|||||||
cpuCheck,
|
cpuCheck,
|
||||||
}:
|
}:
|
||||||
kyojin.overrideAttrs (old: {
|
kyojin.overrideAttrs (old: {
|
||||||
|
# Upstream installs tools directly from $src, bypassing the unpacked tree.
|
||||||
|
src = pkgs.applyPatches {
|
||||||
|
src = old.src;
|
||||||
|
patches = [ ./short-reply-prefill-grace.patch ];
|
||||||
|
};
|
||||||
nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ pkgs.makeWrapper ];
|
nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ pkgs.makeWrapper ];
|
||||||
meta = (old.meta or { }) // {
|
meta = (old.meta or { }) // {
|
||||||
homepage = "https://github.com/Yamz-Labs/kyojin";
|
homepage = "https://github.com/Yamz-Labs/kyojin";
|
||||||
|
|||||||
@@ -0,0 +1,116 @@
|
|||||||
|
--- a/tools/qwen/serve.py
|
||||||
|
+++ b/tools/qwen/serve.py
|
||||||
|
@@ -984,4 +984,11 @@
|
||||||
|
thf, thv, maxd, spec_policy.ngram_from_env(), {},
|
||||||
|
rule=spec_policy.rule_from_env(thf, thv, maxd), fixed=int(fx) if fx else None)
|
||||||
|
+ if self.sessions == 2:
|
||||||
|
+ grace = float(os.environ.get("KYOJIN_PREFILL_GRACE_S") or 0)
|
||||||
|
+ if not math.isfinite(grace) or grace < 0:
|
||||||
|
+ raise ValueError("KYOJIN_PREFILL_GRACE_S must be finite and nonnegative")
|
||||||
|
+ if grace:
|
||||||
|
+ from prefill_grace import ShortReplyPlanner
|
||||||
|
+ self.generator._cil = ShortReplyPlanner(self.generator.active_jobs, grace)
|
||||||
|
self.spec_on = spec
|
||||||
|
self.generator.pre_start_hook = self._pre_start
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/tools/qwen/prefill_grace.py
|
||||||
|
@@ -0,0 +1,28 @@
|
||||||
|
+"""Optionally let a short decode finish before starting a large uncached prefill."""
|
||||||
|
+from exllamav3.generator.chunk_il import ChunkPlanner, Plan
|
||||||
|
+
|
||||||
|
+
|
||||||
|
+class ShortReplyPlanner(ChunkPlanner):
|
||||||
|
+ def __init__(self, jobs, grace, **kwargs):
|
||||||
|
+ super().__init__(**kwargs)
|
||||||
|
+ self.jobs = jobs
|
||||||
|
+ self.grace = grace
|
||||||
|
+
|
||||||
|
+ def plan_prefill(self, views):
|
||||||
|
+ if self.dprio and len(views) == 2:
|
||||||
|
+ old, new = sorted(views, key=lambda view: view.order)
|
||||||
|
+ # The final prompt token is decoded. 'remaining' discounts cached pages
|
||||||
|
+ # and the recurrent tail checkpoint, including on the first iteration.
|
||||||
|
+ if old.done and not new.done and new.remaining >= 32768 - 1:
|
||||||
|
+ jobs = {id(job): job for job in self.jobs}
|
||||||
|
+ reply, prompt = jobs[old.key], jobs[new.key]
|
||||||
|
+ if (prompt.time_first_prefill is None and reply.new_tokens > 0
|
||||||
|
+ and reply.max_new_tokens is not None
|
||||||
|
+ and 0 < reply.max_new_tokens - reply.new_tokens <= 320):
|
||||||
|
+ now = self._clock()
|
||||||
|
+ deadline = getattr(prompt, "_short_reply_deadline", None)
|
||||||
|
+ if deadline is None:
|
||||||
|
+ deadline = prompt._short_reply_deadline = now + self.grace
|
||||||
|
+ if now < deadline:
|
||||||
|
+ return Plan({new.key}, "shorts", new.key)
|
||||||
|
+ return super().plan_prefill(views)
|
||||||
|
--- a/tools/qwen/SERVE.md
|
||||||
|
+++ b/tools/qwen/SERVE.md
|
||||||
|
@@ -168,4 +168,11 @@
|
||||||
|
save/restore/erase wait until the running requests finish.
|
||||||
|
|
||||||
|
+With `--sessions 2` and decode-priority chunk interleaving, set `KYOJIN_PREFILL_GRACE_S=10` to give
|
||||||
|
+a running reply up to ten additional seconds to finish before a large prefill starts. The default is
|
||||||
|
+zero. The running reply must have started emitting tokens and have at most 320 tokens left in its
|
||||||
|
+generation budget. The new request must still need at least 32,767 prefill rows after prefix reuse
|
||||||
|
+(the last prompt token is handled by decode). When the reply finishes, the new prompt can run
|
||||||
|
+immediately. At the deadline the normal interleave policy resumes.
|
||||||
|
+
|
||||||
|
Measured with `--ndt 3`, `-c 262144`, temperature 0, two short prompts sent together (four for `--sessions 4`):
|
||||||
|
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/tools/qwen/test_prefill_grace.py
|
||||||
|
@@ -0,0 +1,54 @@
|
||||||
|
+import unittest
|
||||||
|
+from types import SimpleNamespace
|
||||||
|
+from unittest.mock import patch
|
||||||
|
+
|
||||||
|
+from exllamav3.generator.chunk_il import ChunkPlanner, View
|
||||||
|
+from prefill_grace import ShortReplyPlanner
|
||||||
|
+
|
||||||
|
+
|
||||||
|
+class ShortReplyTests(unittest.TestCase):
|
||||||
|
+ def setUp(self):
|
||||||
|
+ self.reply = SimpleNamespace(new_tokens=12, max_new_tokens=332)
|
||||||
|
+ self.prompt = SimpleNamespace(time_first_prefill=None)
|
||||||
|
+ self.now = 80
|
||||||
|
+ self.planner = ShortReplyPlanner([self.reply, self.prompt], 10, clock=lambda: self.now)
|
||||||
|
+ self.planner.dprio = True
|
||||||
|
+ self.old = View(id(self.reply), 1, 0, True, 0, 18)
|
||||||
|
+ self.new = View(id(self.prompt), 2, 32767, False, 4096, 0)
|
||||||
|
+
|
||||||
|
+ def test_deadline_from_first_eligible_step(self):
|
||||||
|
+ views = [self.new, self.old]
|
||||||
|
+ with patch.object(ChunkPlanner, "plan_prefill", autospec=True) as normal:
|
||||||
|
+ for self.now in (80, 89.99):
|
||||||
|
+ plan = self.planner.plan_prefill(views)
|
||||||
|
+ self.assertEqual(plan.parked, {self.new.key})
|
||||||
|
+ self.assertEqual(self.planner.plan_decode(views, plan), set())
|
||||||
|
+ normal.assert_not_called()
|
||||||
|
+ for self.now in (90, 91):
|
||||||
|
+ self.assertIs(self.planner.plan_prefill(views), normal.return_value)
|
||||||
|
+ self.assertEqual(normal.call_count, 2)
|
||||||
|
+
|
||||||
|
+ def test_only_older_decode_and_large_unstarted_prefill(self):
|
||||||
|
+ cases = ([self.new], [self.new, self.old._replace(order=3)],
|
||||||
|
+ [self.old, self.new._replace(remaining=32766)],
|
||||||
|
+ [self.old, self.new._replace(done=True, remaining=0)],
|
||||||
|
+ [self.old._replace(done=False), self.new])
|
||||||
|
+ with patch.object(ChunkPlanner, "plan_prefill", autospec=True) as normal:
|
||||||
|
+ for views in cases:
|
||||||
|
+ with self.subTest(views=views):
|
||||||
|
+ self.assertIs(self.planner.plan_prefill(views), normal.return_value)
|
||||||
|
+ self.prompt.time_first_prefill = 75
|
||||||
|
+ self.assertIs(self.planner.plan_prefill([self.old, self.new]), normal.return_value)
|
||||||
|
+
|
||||||
|
+ def test_only_bounded_running_reply_in_decode_priority_mode(self):
|
||||||
|
+ for used, cap, dprio in ((0, 10, True), (12, 333, True), (12, None, True),
|
||||||
|
+ (12, 12, True), (12, 332, False)):
|
||||||
|
+ with self.subTest(used=used, cap=cap, dprio=dprio):
|
||||||
|
+ self.reply.new_tokens, self.reply.max_new_tokens = used, cap
|
||||||
|
+ self.planner.dprio = dprio
|
||||||
|
+ with patch.object(ChunkPlanner, "plan_prefill", autospec=True) as normal:
|
||||||
|
+ self.assertIs(self.planner.plan_prefill([self.old, self.new]), normal.return_value)
|
||||||
|
+
|
||||||
|
+
|
||||||
|
+if __name__ == "__main__":
|
||||||
|
+ unittest.main()
|
||||||
Reference in New Issue
Block a user