848a6ec9c4
Bead: nix-overlay-3k7
106 lines
4.5 KiB
Diff
106 lines
4.5 KiB
Diff
--- a/exllamav3/generator/generator.py
|
|
+++ b/exllamav3/generator/generator.py
|
|
@@ -46,6 +46,9 @@
|
|
# once the results are handed out; every iterate / cancel / clear_queue runs it first, so later work sees the same cache state in
|
|
# the same order (exact by construction). Pays where the stash is a blocking copy (HIP .cpu(), CUDA EXL3_TAIL_PIN=0).
|
|
_REPLY_FIRST = os.environ.get("EXL3_REPLY_FIRST", "0") == "1"
|
|
+# With full prefill chunks, give a recently started, highly cached reply a bounded head start.
|
|
+_WARM_REPLY_GRACE_S = max(0, int(os.environ.get("EXL3_WARM_REPLY_GRACE_MS", "0"))) / 1000
|
|
+
|
|
_LAG1_DEBUG = os.environ.get("EXL3_LAG1_DEBUG", "0") == "1"
|
|
_LAG1 = os.environ.get("EXL3_LAG1", "0") == "1" # lag-1 decode pipeline (see Generator._gen_lag1)
|
|
from . import spec_sample
|
|
@@ -836,6 +839,23 @@
|
|
for job in self.active_jobs:
|
|
job._cil_hold = False
|
|
|
|
+ def _park_for_warm_reply(self, grace: float) -> tuple:
|
|
+ older, incoming = sorted(self.active_jobs, key=lambda j: j.serial_number)
|
|
+ if incoming.time_first_prefill is not None or older.new_tokens >= 64:
|
|
+ return ()
|
|
+ cached = older.cached_pages * PAGE_SIZE + older.cached_tokens
|
|
+ if cached < 32768 or len(older.sequences[0].input_ids) - cached > 512:
|
|
+ return ()
|
|
+ # The ordinary planner also discounts recurrent tail hits before the first chunk.
|
|
+ # A long but cached follow-up should run immediately.
|
|
+ if next(v.remaining for v in self._cil_views() if v.key == id(incoming)) < 32768 - 1:
|
|
+ return ()
|
|
+ now = time.monotonic()
|
|
+ deadline = getattr(incoming, "_warm_reply_deadline", None)
|
|
+ if deadline is None:
|
|
+ deadline = incoming._warm_reply_deadline = now + grace
|
|
+ return (incoming,) if now < deadline else ()
|
|
+
|
|
def _prefill_round(self, parked, results: list):
|
|
"""One prefill chunk for each active job that is not parked. The first token of a job is sampled in the generation round
|
|
after this one, so jobs that finish their prefill here all wait for the longest prefill of the step: four sub-agents of one
|
|
@@ -949,7 +969,12 @@
|
|
|
|
# Perform one round of prefill
|
|
_t0 = _ttrace_begin()
|
|
- parked = self._cil_parked() if chunk_il.chunk_il_enabled() else ()
|
|
+ if chunk_il.chunk_il_enabled():
|
|
+ parked = self._cil_parked()
|
|
+ elif _WARM_REPLY_GRACE_S and len(self.active_jobs) == 2:
|
|
+ parked = self._park_for_warm_reply(_WARM_REPLY_GRACE_S)
|
|
+ else:
|
|
+ parked = ()
|
|
self._prefill_round(parked, results)
|
|
_ttrace_end("prefill total", _t0)
|
|
|
|
--- /dev/null
|
|
+++ b/tests/test_warm_reply_grace_cpu.py
|
|
@@ -0,0 +1,52 @@
|
|
+from types import SimpleNamespace
|
|
+
|
|
+import pytest
|
|
+
|
|
+from exllamav3.constants import PAGE_SIZE
|
|
+from exllamav3.generator import generator as gen
|
|
+
|
|
+
|
|
+def jobs(reused=204800, old_fresh=187, new_length=32768, generated=0, peek=0):
|
|
+ def job(order, cached, length, done):
|
|
+ seq = SimpleNamespace(input_ids=range(length), sequence_ids=range(length), kv_position=cached)
|
|
+ return SimpleNamespace(serial_number=order, cached_pages=cached // PAGE_SIZE,
|
|
+ cached_tokens=cached % PAGE_SIZE, sequences=[seq], new_tokens=generated,
|
|
+ time_first_prefill=None, is_prefill_done=lambda: done,
|
|
+ _tail_peek=lambda _: peek)
|
|
+ old = job(1, reused, reused + old_fresh, True)
|
|
+ incoming = job(2, 0, new_length, False)
|
|
+ g = object.__new__(gen.Generator)
|
|
+ g.max_chunk_size = 4096
|
|
+ g.active_jobs = [incoming, old]
|
|
+ return g, old, incoming
|
|
+
|
|
+
|
|
+def test_one_deadline_per_cold_prompt(monkeypatch):
|
|
+ now = [1.0]
|
|
+ monkeypatch.setattr(gen.time, "monotonic", lambda: now[0])
|
|
+ g, old, new = jobs()
|
|
+ old.is_prefill_done = lambda: False
|
|
+ assert g._park_for_warm_reply(2) == (new,)
|
|
+ old.is_prefill_done = lambda: True
|
|
+ now[0] = 2.99
|
|
+ assert g._park_for_warm_reply(2) == (new,)
|
|
+ now[0] = 3.0
|
|
+ assert g._park_for_warm_reply(2) == ()
|
|
+ now[0] = 4.0
|
|
+ assert g._park_for_warm_reply(2) == ()
|
|
+
|
|
+
|
|
+@pytest.mark.parametrize("params", [
|
|
+ {"reused": 0}, {"old_fresh": 513}, {"new_length": 32767}, {"generated": 64},
|
|
+ {"peek": 32000},
|
|
+])
|
|
+def test_bypass_ineligible_requests(params):
|
|
+ g, _, new = jobs(**params)
|
|
+ assert g._park_for_warm_reply(2) == ()
|
|
+ assert not hasattr(new, "_warm_reply_deadline")
|
|
+
|
|
+
|
|
+def test_prefill_already_started():
|
|
+ g, _, new = jobs()
|
|
+ new.time_first_prefill = 1.0
|
|
+ assert g._park_for_warm_reply(2) == ()
|