--- a/tools/qwen/serve.py +++ b/tools/qwen/serve.py @@ -1262,6 +1262,15 @@ for serial, evs in by_serial.items(): route = self._routes.get(serial) if route is not None: # None: cancelled after the step started + if evs[-1].get("eos") and self.generator.num_remaining_jobs(): + # A requeued replacement keeps its serial. Retire only a finished request, + # before the next GPU step can delay its handler's final _drop(). + live = self.generator.active_jobs + self.generator.pending_jobs + if all(job.serial_number != serial for job in live): + del self._routes[serial] + finished = loop.create_future() + finished.set_result(None) + route[0]._qserve_finished = finished route[1].put_nowait(evs) def _fail_unroutable(self, ev, exc: Exception) -> None: @@ -1298,6 +1307,10 @@ def _drop(self, job) -> asyncio.Future: """Stop routing the job and remove it from the Generator (a no-op when it already finished). Runs on the driver, after any enqueue posted before it, so a job cancelled while its enqueue is still queued does not stay behind.""" + finished = getattr(job, "_qserve_finished", None) + if finished is not None: + return finished + def forget(): serial = getattr(job, "serial_number", None) route = self._routes.get(serial) --- a/tools/qwen/test_serve.py +++ b/tools/qwen/test_serve.py @@ -4,10 +4,12 @@ import asyncio import base64 +import contextlib import io import json import os import sys +import threading import types import unittest import unittest.mock @@ -1246,5 +1248,60 @@ self.assertIn(b"hel", body) +class FinishedRouteTests(unittest.IsolatedAsyncioTestCase): + async def start_driver(self, mode): + engine = object.__new__(serve.QwenEngine) + old = types.SimpleNamespace(serial_number=4) + other = types.SimpleNamespace(serial_number=5) + replacement = types.SimpleNamespace(serial_number=4) + generator = types.SimpleNamespace(active_jobs=[old], pending_jobs=[]) + release = threading.Event() + + def iterate(): + if any(job is old for job in generator.active_jobs): + generator.active_jobs = [] if mode == "solo" else [other] + if mode == "active": + generator.active_jobs.append(replacement) + if mode == "pending": + generator.pending_jobs.append(replacement) + return [dict(serial=4, eos=mode != "ongoing", text="last piece")] + release.wait(2) + generator.active_jobs.clear() + generator.pending_jobs.clear() + return [] + + generator.iterate = iterate + generator.num_remaining_jobs = lambda: len(generator.active_jobs) + len(generator.pending_jobs) + generator.cancel = unittest.mock.Mock() + engine.generator = generator + engine._ensure_driver() + + async def stop(): + release.set() + engine._driver.cancel() + with contextlib.suppress(asyncio.CancelledError): + await engine._driver + + self.addAsyncCleanup(stop) + queue = asyncio.Queue() + engine._routes[4] = (old, queue) + events = await asyncio.wait_for(queue.get(), 1) + self.assertEqual(events[-1]["text"], "last piece") + return engine, old + + async def test_completed_request_retires_before_another_chunk(self): + engine, old = await self.start_driver("done") + self.assertNotIn(4, engine._routes) + self.assertTrue(engine._drop(old).done()) + self.assertTrue(engine._drop(old).done()) + engine.generator.cancel.assert_not_called() + + async def test_keep_live_requeued_and_solo_routes(self): + for mode in ("ongoing", "active", "pending", "solo"): + with self.subTest(mode=mode): + engine, old = await self.start_driver(mode) + self.assertIs(engine._routes[4][0], old) + + class PrefixDiskConcurrencyTests(unittest.TestCase): """The disk cache must work while other requests run (3 users): restore with a busy Generator, save of the request's own prompt."""