--- a/tools/qwen/serve.py +++ b/tools/qwen/serve.py @@ -984,4 +984,11 @@ thf, thv, maxd, spec_policy.ngram_from_env(), {}, rule=spec_policy.rule_from_env(thf, thv, maxd), fixed=int(fx) if fx else None) + if self.sessions == 2: + grace = float(os.environ.get("KYOJIN_PREFILL_GRACE_S") or 0) + if not math.isfinite(grace) or grace < 0: + raise ValueError("KYOJIN_PREFILL_GRACE_S must be finite and nonnegative") + if grace: + from prefill_grace import ShortReplyPlanner + self.generator._cil = ShortReplyPlanner(self.generator.active_jobs, grace) self.spec_on = spec self.generator.pre_start_hook = self._pre_start --- /dev/null +++ b/tools/qwen/prefill_grace.py @@ -0,0 +1,28 @@ +"""Optionally let a short decode finish before starting a large uncached prefill.""" +from exllamav3.generator.chunk_il import ChunkPlanner, Plan + + +class ShortReplyPlanner(ChunkPlanner): + def __init__(self, jobs, grace, **kwargs): + super().__init__(**kwargs) + self.jobs = jobs + self.grace = grace + + def plan_prefill(self, views): + if self.dprio and len(views) == 2: + old, new = sorted(views, key=lambda view: view.order) + # The final prompt token is decoded. 'remaining' discounts cached pages + # and the recurrent tail checkpoint, including on the first iteration. + if old.done and not new.done and new.remaining >= 32768 - 1: + jobs = {id(job): job for job in self.jobs} + reply, prompt = jobs[old.key], jobs[new.key] + if (prompt.time_first_prefill is None and reply.new_tokens > 0 + and reply.max_new_tokens is not None + and 0 < reply.max_new_tokens - reply.new_tokens <= 320): + now = self._clock() + deadline = getattr(prompt, "_short_reply_deadline", None) + if deadline is None: + deadline = prompt._short_reply_deadline = now + self.grace + if now < deadline: + return Plan({new.key}, "shorts", new.key) + return super().plan_prefill(views) --- a/tools/qwen/SERVE.md +++ b/tools/qwen/SERVE.md @@ -168,4 +168,11 @@ save/restore/erase wait until the running requests finish. +With `--sessions 2` and decode-priority chunk interleaving, set `KYOJIN_PREFILL_GRACE_S=10` to give +a running reply up to ten additional seconds to finish before a large prefill starts. The default is +zero. The running reply must have started emitting tokens and have at most 320 tokens left in its +generation budget. The new request must still need at least 32,767 prefill rows after prefix reuse +(the last prompt token is handled by decode). When the reply finishes, the new prompt can run +immediately. At the deadline the normal interleave policy resumes. + Measured with `--ndt 3`, `-c 262144`, temperature 0, two short prompts sent together (four for `--sessions 4`): --- /dev/null +++ b/tools/qwen/test_prefill_grace.py @@ -0,0 +1,54 @@ +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +from exllamav3.generator.chunk_il import ChunkPlanner, View +from prefill_grace import ShortReplyPlanner + + +class ShortReplyTests(unittest.TestCase): + def setUp(self): + self.reply = SimpleNamespace(new_tokens=12, max_new_tokens=332) + self.prompt = SimpleNamespace(time_first_prefill=None) + self.now = 80 + self.planner = ShortReplyPlanner([self.reply, self.prompt], 10, clock=lambda: self.now) + self.planner.dprio = True + self.old = View(id(self.reply), 1, 0, True, 0, 18) + self.new = View(id(self.prompt), 2, 32767, False, 4096, 0) + + def test_deadline_from_first_eligible_step(self): + views = [self.new, self.old] + with patch.object(ChunkPlanner, "plan_prefill", autospec=True) as normal: + for self.now in (80, 89.99): + plan = self.planner.plan_prefill(views) + self.assertEqual(plan.parked, {self.new.key}) + self.assertEqual(self.planner.plan_decode(views, plan), set()) + normal.assert_not_called() + for self.now in (90, 91): + self.assertIs(self.planner.plan_prefill(views), normal.return_value) + self.assertEqual(normal.call_count, 2) + + def test_only_older_decode_and_large_unstarted_prefill(self): + cases = ([self.new], [self.new, self.old._replace(order=3)], + [self.old, self.new._replace(remaining=32766)], + [self.old, self.new._replace(done=True, remaining=0)], + [self.old._replace(done=False), self.new]) + with patch.object(ChunkPlanner, "plan_prefill", autospec=True) as normal: + for views in cases: + with self.subTest(views=views): + self.assertIs(self.planner.plan_prefill(views), normal.return_value) + self.prompt.time_first_prefill = 75 + self.assertIs(self.planner.plan_prefill([self.old, self.new]), normal.return_value) + + def test_only_bounded_running_reply_in_decode_priority_mode(self): + for used, cap, dprio in ((0, 10, True), (12, 333, True), (12, None, True), + (12, 12, True), (12, 332, False)): + with self.subTest(used=used, cap=cap, dprio=dprio): + self.reply.new_tokens, self.reply.max_new_tokens = used, cap + self.planner.dprio = dprio + with patch.object(ChunkPlanner, "plan_prefill", autospec=True) as normal: + self.assertIs(self.planner.plan_prefill([self.old, self.new]), normal.return_value) + + +if __name__ == "__main__": + unittest.main()