Bead: nix-overlay-8gs
This commit is contained in:
@@ -0,0 +1,105 @@
|
||||
--- a/tools/qwen/serve.py
|
||||
+++ b/tools/qwen/serve.py
|
||||
@@ -1262,6 +1262,15 @@
|
||||
for serial, evs in by_serial.items():
|
||||
route = self._routes.get(serial)
|
||||
if route is not None: # None: cancelled after the step started
|
||||
+ if evs[-1].get("eos") and self.generator.num_remaining_jobs():
|
||||
+ # A requeued replacement keeps its serial. Retire only a finished request,
|
||||
+ # before the next GPU step can delay its handler's final _drop().
|
||||
+ live = self.generator.active_jobs + self.generator.pending_jobs
|
||||
+ if all(job.serial_number != serial for job in live):
|
||||
+ del self._routes[serial]
|
||||
+ finished = loop.create_future()
|
||||
+ finished.set_result(None)
|
||||
+ route[0]._qserve_finished = finished
|
||||
route[1].put_nowait(evs)
|
||||
|
||||
def _fail_unroutable(self, ev, exc: Exception) -> None:
|
||||
@@ -1298,6 +1307,10 @@
|
||||
def _drop(self, job) -> asyncio.Future:
|
||||
"""Stop routing the job and remove it from the Generator (a no-op when it already finished). Runs on the driver, after
|
||||
any enqueue posted before it, so a job cancelled while its enqueue is still queued does not stay behind."""
|
||||
+ finished = getattr(job, "_qserve_finished", None)
|
||||
+ if finished is not None:
|
||||
+ return finished
|
||||
+
|
||||
def forget():
|
||||
serial = getattr(job, "serial_number", None)
|
||||
route = self._routes.get(serial)
|
||||
--- a/tools/qwen/test_serve.py
|
||||
+++ b/tools/qwen/test_serve.py
|
||||
@@ -4,10 +4,12 @@
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
+import contextlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
+import threading
|
||||
import types
|
||||
import unittest
|
||||
import unittest.mock
|
||||
@@ -1246,5 +1248,60 @@
|
||||
self.assertIn(b"hel", body)
|
||||
|
||||
|
||||
+class FinishedRouteTests(unittest.IsolatedAsyncioTestCase):
|
||||
+ async def start_driver(self, mode):
|
||||
+ engine = object.__new__(serve.QwenEngine)
|
||||
+ old = types.SimpleNamespace(serial_number=4)
|
||||
+ other = types.SimpleNamespace(serial_number=5)
|
||||
+ replacement = types.SimpleNamespace(serial_number=4)
|
||||
+ generator = types.SimpleNamespace(active_jobs=[old], pending_jobs=[])
|
||||
+ release = threading.Event()
|
||||
+
|
||||
+ def iterate():
|
||||
+ if any(job is old for job in generator.active_jobs):
|
||||
+ generator.active_jobs = [] if mode == "solo" else [other]
|
||||
+ if mode == "active":
|
||||
+ generator.active_jobs.append(replacement)
|
||||
+ if mode == "pending":
|
||||
+ generator.pending_jobs.append(replacement)
|
||||
+ return [dict(serial=4, eos=mode != "ongoing", text="last piece")]
|
||||
+ release.wait(2)
|
||||
+ generator.active_jobs.clear()
|
||||
+ generator.pending_jobs.clear()
|
||||
+ return []
|
||||
+
|
||||
+ generator.iterate = iterate
|
||||
+ generator.num_remaining_jobs = lambda: len(generator.active_jobs) + len(generator.pending_jobs)
|
||||
+ generator.cancel = unittest.mock.Mock()
|
||||
+ engine.generator = generator
|
||||
+ engine._ensure_driver()
|
||||
+
|
||||
+ async def stop():
|
||||
+ release.set()
|
||||
+ engine._driver.cancel()
|
||||
+ with contextlib.suppress(asyncio.CancelledError):
|
||||
+ await engine._driver
|
||||
+
|
||||
+ self.addAsyncCleanup(stop)
|
||||
+ queue = asyncio.Queue()
|
||||
+ engine._routes[4] = (old, queue)
|
||||
+ events = await asyncio.wait_for(queue.get(), 1)
|
||||
+ self.assertEqual(events[-1]["text"], "last piece")
|
||||
+ return engine, old
|
||||
+
|
||||
+ async def test_completed_request_retires_before_another_chunk(self):
|
||||
+ engine, old = await self.start_driver("done")
|
||||
+ self.assertNotIn(4, engine._routes)
|
||||
+ self.assertTrue(engine._drop(old).done())
|
||||
+ self.assertTrue(engine._drop(old).done())
|
||||
+ engine.generator.cancel.assert_not_called()
|
||||
+
|
||||
+ async def test_keep_live_requeued_and_solo_routes(self):
|
||||
+ for mode in ("ongoing", "active", "pending", "solo"):
|
||||
+ with self.subTest(mode=mode):
|
||||
+ engine, old = await self.start_driver(mode)
|
||||
+ self.assertIs(engine._routes[4][0], old)
|
||||
+
|
||||
+
|
||||
class PrefixDiskConcurrencyTests(unittest.TestCase):
|
||||
"""The disk cache must work while other requests run (3 users): restore with a busy Generator, save of the request's own prompt."""
|
||||
@@ -35,6 +35,7 @@ kyojin.overrideAttrs (old: {
|
||||
patches = [
|
||||
./short-reply-prefill-grace.patch
|
||||
./hip-blocking-sync.patch
|
||||
./finished-route.patch
|
||||
];
|
||||
};
|
||||
nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ pkgs.makeWrapper ];
|
||||
|
||||
Reference in New Issue
Block a user