Files
millerson-overlay.nix/packages/kyojin/warm-reply-grace.patch
T
2026-11-19 11:02:32 +03:00

106 lines
4.5 KiB
Diff

--- a/exllamav3/generator/generator.py
+++ b/exllamav3/generator/generator.py
@@ -46,6 +46,9 @@
# once the results are handed out; every iterate / cancel / clear_queue runs it first, so later work sees the same cache state in
# the same order (exact by construction). Pays where the stash is a blocking copy (HIP .cpu(), CUDA EXL3_TAIL_PIN=0).
_REPLY_FIRST = os.environ.get("EXL3_REPLY_FIRST", "0") == "1"
+# With full prefill chunks, give a recently started, highly cached reply a bounded head start.
+_WARM_REPLY_GRACE_S = max(0, int(os.environ.get("EXL3_WARM_REPLY_GRACE_MS", "0"))) / 1000
+
_LAG1_DEBUG = os.environ.get("EXL3_LAG1_DEBUG", "0") == "1"
_LAG1 = os.environ.get("EXL3_LAG1", "0") == "1" # lag-1 decode pipeline (see Generator._gen_lag1)
from . import spec_sample
@@ -836,6 +839,23 @@
for job in self.active_jobs:
job._cil_hold = False
+ def _park_for_warm_reply(self, grace: float) -> tuple:
+ older, incoming = sorted(self.active_jobs, key=lambda j: j.serial_number)
+ if incoming.time_first_prefill is not None or older.new_tokens >= 64:
+ return ()
+ cached = older.cached_pages * PAGE_SIZE + older.cached_tokens
+ if cached < 32768 or len(older.sequences[0].input_ids) - cached > 512:
+ return ()
+ # The ordinary planner also discounts recurrent tail hits before the first chunk.
+ # A long but cached follow-up should run immediately.
+ if next(v.remaining for v in self._cil_views() if v.key == id(incoming)) < 32768 - 1:
+ return ()
+ now = time.monotonic()
+ deadline = getattr(incoming, "_warm_reply_deadline", None)
+ if deadline is None:
+ deadline = incoming._warm_reply_deadline = now + grace
+ return (incoming,) if now < deadline else ()
+
def _prefill_round(self, parked, results: list):
"""One prefill chunk for each active job that is not parked. The first token of a job is sampled in the generation round
after this one, so jobs that finish their prefill here all wait for the longest prefill of the step: four sub-agents of one
@@ -949,7 +969,12 @@
# Perform one round of prefill
_t0 = _ttrace_begin()
- parked = self._cil_parked() if chunk_il.chunk_il_enabled() else ()
+ if chunk_il.chunk_il_enabled():
+ parked = self._cil_parked()
+ elif _WARM_REPLY_GRACE_S and len(self.active_jobs) == 2:
+ parked = self._park_for_warm_reply(_WARM_REPLY_GRACE_S)
+ else:
+ parked = ()
self._prefill_round(parked, results)
_ttrace_end("prefill total", _t0)
--- /dev/null
+++ b/tests/test_warm_reply_grace_cpu.py
@@ -0,0 +1,52 @@
+from types import SimpleNamespace
+
+import pytest
+
+from exllamav3.constants import PAGE_SIZE
+from exllamav3.generator import generator as gen
+
+
+def jobs(reused=204800, old_fresh=187, new_length=32768, generated=0, peek=0):
+ def job(order, cached, length, done):
+ seq = SimpleNamespace(input_ids=range(length), sequence_ids=range(length), kv_position=cached)
+ return SimpleNamespace(serial_number=order, cached_pages=cached // PAGE_SIZE,
+ cached_tokens=cached % PAGE_SIZE, sequences=[seq], new_tokens=generated,
+ time_first_prefill=None, is_prefill_done=lambda: done,
+ _tail_peek=lambda _: peek)
+ old = job(1, reused, reused + old_fresh, True)
+ incoming = job(2, 0, new_length, False)
+ g = object.__new__(gen.Generator)
+ g.max_chunk_size = 4096
+ g.active_jobs = [incoming, old]
+ return g, old, incoming
+
+
+def test_one_deadline_per_cold_prompt(monkeypatch):
+ now = [1.0]
+ monkeypatch.setattr(gen.time, "monotonic", lambda: now[0])
+ g, old, new = jobs()
+ old.is_prefill_done = lambda: False
+ assert g._park_for_warm_reply(2) == (new,)
+ old.is_prefill_done = lambda: True
+ now[0] = 2.99
+ assert g._park_for_warm_reply(2) == (new,)
+ now[0] = 3.0
+ assert g._park_for_warm_reply(2) == ()
+ now[0] = 4.0
+ assert g._park_for_warm_reply(2) == ()
+
+
+@pytest.mark.parametrize("params", [
+ {"reused": 0}, {"old_fresh": 513}, {"new_length": 32767}, {"generated": 64},
+ {"peek": 32000},
+])
+def test_bypass_ineligible_requests(params):
+ g, _, new = jobs(**params)
+ assert g._park_for_warm_reply(2) == ()
+ assert not hasattr(new, "_warm_reply_deadline")
+
+
+def test_prefill_already_started():
+ g, _, new = jobs()
+ new.time_first_prefill = 1.0
+ assert g._park_for_warm_reply(2) == ()