Files
millerson-overlay.nix/packages/kyojin/short-reply-prefill-grace.patch
T
alex afd0c4b84d
CI / check (push) Has been cancelled
perf(kyojin): Let short replies finish before a long prefill
Give two-session Qwen serving a bounded, opt-in decode grace for large cold prompts. Returning to normal interleaving at the deadline bounds the wait when the running reply takes longer than expected.

Bead: nix-overlay-qfi
2026-11-18 15:19:23 +03:00

117 lines
5.8 KiB
Diff

--- a/tools/qwen/serve.py
+++ b/tools/qwen/serve.py
@@ -984,4 +984,11 @@
thf, thv, maxd, spec_policy.ngram_from_env(), {},
rule=spec_policy.rule_from_env(thf, thv, maxd), fixed=int(fx) if fx else None)
+ if self.sessions == 2:
+ grace = float(os.environ.get("KYOJIN_PREFILL_GRACE_S") or 0)
+ if not math.isfinite(grace) or grace < 0:
+ raise ValueError("KYOJIN_PREFILL_GRACE_S must be finite and nonnegative")
+ if grace:
+ from prefill_grace import ShortReplyPlanner
+ self.generator._cil = ShortReplyPlanner(self.generator.active_jobs, grace)
self.spec_on = spec
self.generator.pre_start_hook = self._pre_start
--- /dev/null
+++ b/tools/qwen/prefill_grace.py
@@ -0,0 +1,28 @@
+"""Optionally let a short decode finish before starting a large uncached prefill."""
+from exllamav3.generator.chunk_il import ChunkPlanner, Plan
+
+
+class ShortReplyPlanner(ChunkPlanner):
+ def __init__(self, jobs, grace, **kwargs):
+ super().__init__(**kwargs)
+ self.jobs = jobs
+ self.grace = grace
+
+ def plan_prefill(self, views):
+ if self.dprio and len(views) == 2:
+ old, new = sorted(views, key=lambda view: view.order)
+ # The final prompt token is decoded. 'remaining' discounts cached pages
+ # and the recurrent tail checkpoint, including on the first iteration.
+ if old.done and not new.done and new.remaining >= 32768 - 1:
+ jobs = {id(job): job for job in self.jobs}
+ reply, prompt = jobs[old.key], jobs[new.key]
+ if (prompt.time_first_prefill is None and reply.new_tokens > 0
+ and reply.max_new_tokens is not None
+ and 0 < reply.max_new_tokens - reply.new_tokens <= 320):
+ now = self._clock()
+ deadline = getattr(prompt, "_short_reply_deadline", None)
+ if deadline is None:
+ deadline = prompt._short_reply_deadline = now + self.grace
+ if now < deadline:
+ return Plan({new.key}, "shorts", new.key)
+ return super().plan_prefill(views)
--- a/tools/qwen/SERVE.md
+++ b/tools/qwen/SERVE.md
@@ -168,4 +168,11 @@
save/restore/erase wait until the running requests finish.
+With `--sessions 2` and decode-priority chunk interleaving, set `KYOJIN_PREFILL_GRACE_S=10` to give
+a running reply up to ten additional seconds to finish before a large prefill starts. The default is
+zero. The running reply must have started emitting tokens and have at most 320 tokens left in its
+generation budget. The new request must still need at least 32,767 prefill rows after prefix reuse
+(the last prompt token is handled by decode). When the reply finishes, the new prompt can run
+immediately. At the deadline the normal interleave policy resumes.
+
Measured with `--ndt 3`, `-c 262144`, temperature 0, two short prompts sent together (four for `--sessions 4`):
--- /dev/null
+++ b/tools/qwen/test_prefill_grace.py
@@ -0,0 +1,54 @@
+import unittest
+from types import SimpleNamespace
+from unittest.mock import patch
+
+from exllamav3.generator.chunk_il import ChunkPlanner, View
+from prefill_grace import ShortReplyPlanner
+
+
+class ShortReplyTests(unittest.TestCase):
+ def setUp(self):
+ self.reply = SimpleNamespace(new_tokens=12, max_new_tokens=332)
+ self.prompt = SimpleNamespace(time_first_prefill=None)
+ self.now = 80
+ self.planner = ShortReplyPlanner([self.reply, self.prompt], 10, clock=lambda: self.now)
+ self.planner.dprio = True
+ self.old = View(id(self.reply), 1, 0, True, 0, 18)
+ self.new = View(id(self.prompt), 2, 32767, False, 4096, 0)
+
+ def test_deadline_from_first_eligible_step(self):
+ views = [self.new, self.old]
+ with patch.object(ChunkPlanner, "plan_prefill", autospec=True) as normal:
+ for self.now in (80, 89.99):
+ plan = self.planner.plan_prefill(views)
+ self.assertEqual(plan.parked, {self.new.key})
+ self.assertEqual(self.planner.plan_decode(views, plan), set())
+ normal.assert_not_called()
+ for self.now in (90, 91):
+ self.assertIs(self.planner.plan_prefill(views), normal.return_value)
+ self.assertEqual(normal.call_count, 2)
+
+ def test_only_older_decode_and_large_unstarted_prefill(self):
+ cases = ([self.new], [self.new, self.old._replace(order=3)],
+ [self.old, self.new._replace(remaining=32766)],
+ [self.old, self.new._replace(done=True, remaining=0)],
+ [self.old._replace(done=False), self.new])
+ with patch.object(ChunkPlanner, "plan_prefill", autospec=True) as normal:
+ for views in cases:
+ with self.subTest(views=views):
+ self.assertIs(self.planner.plan_prefill(views), normal.return_value)
+ self.prompt.time_first_prefill = 75
+ self.assertIs(self.planner.plan_prefill([self.old, self.new]), normal.return_value)
+
+ def test_only_bounded_running_reply_in_decode_priority_mode(self):
+ for used, cap, dprio in ((0, 10, True), (12, 333, True), (12, None, True),
+ (12, 12, True), (12, 332, False)):
+ with self.subTest(used=used, cap=cap, dprio=dprio):
+ self.reply.new_tokens, self.reply.max_new_tokens = used, cap
+ self.planner.dprio = dprio
+ with patch.object(ChunkPlanner, "plan_prefill", autospec=True) as normal:
+ self.assertIs(self.planner.plan_prefill([self.old, self.new]), normal.return_value)
+
+
+if __name__ == "__main__":
+ unittest.main()