Files
millerson-overlay.nix/packages/kyojin/finished-route.patch
T
alex 8389c88d04
CI / check (push) Has been cancelled
perf(kyojin): finish completed streams before the next prefill
Bead: nix-overlay-8gs
2026-11-19 11:22:29 +03:00

106 lines
4.4 KiB
Diff

--- a/tools/qwen/serve.py
+++ b/tools/qwen/serve.py
@@ -1262,6 +1262,15 @@
for serial, evs in by_serial.items():
route = self._routes.get(serial)
if route is not None: # None: cancelled after the step started
+ if evs[-1].get("eos") and self.generator.num_remaining_jobs():
+ # A requeued replacement keeps its serial. Retire only a finished request,
+ # before the next GPU step can delay its handler's final _drop().
+ live = self.generator.active_jobs + self.generator.pending_jobs
+ if all(job.serial_number != serial for job in live):
+ del self._routes[serial]
+ finished = loop.create_future()
+ finished.set_result(None)
+ route[0]._qserve_finished = finished
route[1].put_nowait(evs)
def _fail_unroutable(self, ev, exc: Exception) -> None:
@@ -1298,6 +1307,10 @@
def _drop(self, job) -> asyncio.Future:
"""Stop routing the job and remove it from the Generator (a no-op when it already finished). Runs on the driver, after
any enqueue posted before it, so a job cancelled while its enqueue is still queued does not stay behind."""
+ finished = getattr(job, "_qserve_finished", None)
+ if finished is not None:
+ return finished
+
def forget():
serial = getattr(job, "serial_number", None)
route = self._routes.get(serial)
--- a/tools/qwen/test_serve.py
+++ b/tools/qwen/test_serve.py
@@ -4,10 +4,12 @@
import asyncio
import base64
+import contextlib
import io
import json
import os
import sys
+import threading
import types
import unittest
import unittest.mock
@@ -1246,5 +1248,60 @@
self.assertIn(b"hel", body)
+class FinishedRouteTests(unittest.IsolatedAsyncioTestCase):
+ async def start_driver(self, mode):
+ engine = object.__new__(serve.QwenEngine)
+ old = types.SimpleNamespace(serial_number=4)
+ other = types.SimpleNamespace(serial_number=5)
+ replacement = types.SimpleNamespace(serial_number=4)
+ generator = types.SimpleNamespace(active_jobs=[old], pending_jobs=[])
+ release = threading.Event()
+
+ def iterate():
+ if any(job is old for job in generator.active_jobs):
+ generator.active_jobs = [] if mode == "solo" else [other]
+ if mode == "active":
+ generator.active_jobs.append(replacement)
+ if mode == "pending":
+ generator.pending_jobs.append(replacement)
+ return [dict(serial=4, eos=mode != "ongoing", text="last piece")]
+ release.wait(2)
+ generator.active_jobs.clear()
+ generator.pending_jobs.clear()
+ return []
+
+ generator.iterate = iterate
+ generator.num_remaining_jobs = lambda: len(generator.active_jobs) + len(generator.pending_jobs)
+ generator.cancel = unittest.mock.Mock()
+ engine.generator = generator
+ engine._ensure_driver()
+
+ async def stop():
+ release.set()
+ engine._driver.cancel()
+ with contextlib.suppress(asyncio.CancelledError):
+ await engine._driver
+
+ self.addAsyncCleanup(stop)
+ queue = asyncio.Queue()
+ engine._routes[4] = (old, queue)
+ events = await asyncio.wait_for(queue.get(), 1)
+ self.assertEqual(events[-1]["text"], "last piece")
+ return engine, old
+
+ async def test_completed_request_retires_before_another_chunk(self):
+ engine, old = await self.start_driver("done")
+ self.assertNotIn(4, engine._routes)
+ self.assertTrue(engine._drop(old).done())
+ self.assertTrue(engine._drop(old).done())
+ engine.generator.cancel.assert_not_called()
+
+ async def test_keep_live_requeued_and_solo_routes(self):
+ for mode in ("ongoing", "active", "pending", "solo"):
+ with self.subTest(mode=mode):
+ engine, old = await self.start_driver(mode)
+ self.assertIs(engine._routes[4][0], old)
+
+
class PrefixDiskConcurrencyTests(unittest.TestCase):
"""The disk cache must work while other requests run (3 users): restore with a busy Generator, save of the request's own prompt."""