106 lines
4.4 KiB
Diff
106 lines
4.4 KiB
Diff
--- a/tools/qwen/serve.py
|
|
+++ b/tools/qwen/serve.py
|
|
@@ -1262,6 +1262,15 @@
|
|
for serial, evs in by_serial.items():
|
|
route = self._routes.get(serial)
|
|
if route is not None: # None: cancelled after the step started
|
|
+ if evs[-1].get("eos") and self.generator.num_remaining_jobs():
|
|
+ # A requeued replacement keeps its serial. Retire only a finished request,
|
|
+ # before the next GPU step can delay its handler's final _drop().
|
|
+ live = self.generator.active_jobs + self.generator.pending_jobs
|
|
+ if all(job.serial_number != serial for job in live):
|
|
+ del self._routes[serial]
|
|
+ finished = loop.create_future()
|
|
+ finished.set_result(None)
|
|
+ route[0]._qserve_finished = finished
|
|
route[1].put_nowait(evs)
|
|
|
|
def _fail_unroutable(self, ev, exc: Exception) -> None:
|
|
@@ -1298,6 +1307,10 @@
|
|
def _drop(self, job) -> asyncio.Future:
|
|
"""Stop routing the job and remove it from the Generator (a no-op when it already finished). Runs on the driver, after
|
|
any enqueue posted before it, so a job cancelled while its enqueue is still queued does not stay behind."""
|
|
+ finished = getattr(job, "_qserve_finished", None)
|
|
+ if finished is not None:
|
|
+ return finished
|
|
+
|
|
def forget():
|
|
serial = getattr(job, "serial_number", None)
|
|
route = self._routes.get(serial)
|
|
--- a/tools/qwen/test_serve.py
|
|
+++ b/tools/qwen/test_serve.py
|
|
@@ -4,10 +4,12 @@
|
|
|
|
import asyncio
|
|
import base64
|
|
+import contextlib
|
|
import io
|
|
import json
|
|
import os
|
|
import sys
|
|
+import threading
|
|
import types
|
|
import unittest
|
|
import unittest.mock
|
|
@@ -1246,5 +1248,60 @@
|
|
self.assertIn(b"hel", body)
|
|
|
|
|
|
+class FinishedRouteTests(unittest.IsolatedAsyncioTestCase):
|
|
+ async def start_driver(self, mode):
|
|
+ engine = object.__new__(serve.QwenEngine)
|
|
+ old = types.SimpleNamespace(serial_number=4)
|
|
+ other = types.SimpleNamespace(serial_number=5)
|
|
+ replacement = types.SimpleNamespace(serial_number=4)
|
|
+ generator = types.SimpleNamespace(active_jobs=[old], pending_jobs=[])
|
|
+ release = threading.Event()
|
|
+
|
|
+ def iterate():
|
|
+ if any(job is old for job in generator.active_jobs):
|
|
+ generator.active_jobs = [] if mode == "solo" else [other]
|
|
+ if mode == "active":
|
|
+ generator.active_jobs.append(replacement)
|
|
+ if mode == "pending":
|
|
+ generator.pending_jobs.append(replacement)
|
|
+ return [dict(serial=4, eos=mode != "ongoing", text="last piece")]
|
|
+ release.wait(2)
|
|
+ generator.active_jobs.clear()
|
|
+ generator.pending_jobs.clear()
|
|
+ return []
|
|
+
|
|
+ generator.iterate = iterate
|
|
+ generator.num_remaining_jobs = lambda: len(generator.active_jobs) + len(generator.pending_jobs)
|
|
+ generator.cancel = unittest.mock.Mock()
|
|
+ engine.generator = generator
|
|
+ engine._ensure_driver()
|
|
+
|
|
+ async def stop():
|
|
+ release.set()
|
|
+ engine._driver.cancel()
|
|
+ with contextlib.suppress(asyncio.CancelledError):
|
|
+ await engine._driver
|
|
+
|
|
+ self.addAsyncCleanup(stop)
|
|
+ queue = asyncio.Queue()
|
|
+ engine._routes[4] = (old, queue)
|
|
+ events = await asyncio.wait_for(queue.get(), 1)
|
|
+ self.assertEqual(events[-1]["text"], "last piece")
|
|
+ return engine, old
|
|
+
|
|
+ async def test_completed_request_retires_before_another_chunk(self):
|
|
+ engine, old = await self.start_driver("done")
|
|
+ self.assertNotIn(4, engine._routes)
|
|
+ self.assertTrue(engine._drop(old).done())
|
|
+ self.assertTrue(engine._drop(old).done())
|
|
+ engine.generator.cancel.assert_not_called()
|
|
+
|
|
+ async def test_keep_live_requeued_and_solo_routes(self):
|
|
+ for mode in ("ongoing", "active", "pending", "solo"):
|
|
+ with self.subTest(mode=mode):
|
|
+ engine, old = await self.start_driver(mode)
|
|
+ self.assertIs(engine._routes[4][0], old)
|
|
+
|
|
+
|
|
class PrefixDiskConcurrencyTests(unittest.TestCase):
|
|
"""The disk cache must work while other requests run (3 users): restore with a busy Generator, save of the request's own prompt."""
|