--- a/exllamav3/generator/generator.py +++ b/exllamav3/generator/generator.py @@ -46,6 +46,9 @@ # once the results are handed out; every iterate / cancel / clear_queue runs it first, so later work sees the same cache state in # the same order (exact by construction). Pays where the stash is a blocking copy (HIP .cpu(), CUDA EXL3_TAIL_PIN=0). _REPLY_FIRST = os.environ.get("EXL3_REPLY_FIRST", "0") == "1" +# With full prefill chunks, give a recently started, highly cached reply a bounded head start. +_WARM_REPLY_GRACE_S = max(0, int(os.environ.get("EXL3_WARM_REPLY_GRACE_MS", "0"))) / 1000 + _LAG1_DEBUG = os.environ.get("EXL3_LAG1_DEBUG", "0") == "1" _LAG1 = os.environ.get("EXL3_LAG1", "0") == "1" # lag-1 decode pipeline (see Generator._gen_lag1) from . import spec_sample @@ -836,6 +839,23 @@ for job in self.active_jobs: job._cil_hold = False + def _park_for_warm_reply(self, grace: float) -> tuple: + older, incoming = sorted(self.active_jobs, key=lambda j: j.serial_number) + if incoming.time_first_prefill is not None or older.new_tokens >= 64: + return () + cached = older.cached_pages * PAGE_SIZE + older.cached_tokens + if cached < 32768 or len(older.sequences[0].input_ids) - cached > 512: + return () + # The ordinary planner also discounts recurrent tail hits before the first chunk. + # A long but cached follow-up should run immediately. + if next(v.remaining for v in self._cil_views() if v.key == id(incoming)) < 32768 - 1: + return () + now = time.monotonic() + deadline = getattr(incoming, "_warm_reply_deadline", None) + if deadline is None: + deadline = incoming._warm_reply_deadline = now + grace + return (incoming,) if now < deadline else () + def _prefill_round(self, parked, results: list): """One prefill chunk for each active job that is not parked. The first token of a job is sampled in the generation round after this one, so jobs that finish their prefill here all wait for the longest prefill of the step: four sub-agents of one @@ -949,7 +969,12 @@ # Perform one round of prefill _t0 = _ttrace_begin() - parked = self._cil_parked() if chunk_il.chunk_il_enabled() else () + if chunk_il.chunk_il_enabled(): + parked = self._cil_parked() + elif _WARM_REPLY_GRACE_S and len(self.active_jobs) == 2: + parked = self._park_for_warm_reply(_WARM_REPLY_GRACE_S) + else: + parked = () self._prefill_round(parked, results) _ttrace_end("prefill total", _t0) --- /dev/null +++ b/tests/test_warm_reply_grace_cpu.py @@ -0,0 +1,52 @@ +from types import SimpleNamespace + +import pytest + +from exllamav3.constants import PAGE_SIZE +from exllamav3.generator import generator as gen + + +def jobs(reused=204800, old_fresh=187, new_length=32768, generated=0, peek=0): + def job(order, cached, length, done): + seq = SimpleNamespace(input_ids=range(length), sequence_ids=range(length), kv_position=cached) + return SimpleNamespace(serial_number=order, cached_pages=cached // PAGE_SIZE, + cached_tokens=cached % PAGE_SIZE, sequences=[seq], new_tokens=generated, + time_first_prefill=None, is_prefill_done=lambda: done, + _tail_peek=lambda _: peek) + old = job(1, reused, reused + old_fresh, True) + incoming = job(2, 0, new_length, False) + g = object.__new__(gen.Generator) + g.max_chunk_size = 4096 + g.active_jobs = [incoming, old] + return g, old, incoming + + +def test_one_deadline_per_cold_prompt(monkeypatch): + now = [1.0] + monkeypatch.setattr(gen.time, "monotonic", lambda: now[0]) + g, old, new = jobs() + old.is_prefill_done = lambda: False + assert g._park_for_warm_reply(2) == (new,) + old.is_prefill_done = lambda: True + now[0] = 2.99 + assert g._park_for_warm_reply(2) == (new,) + now[0] = 3.0 + assert g._park_for_warm_reply(2) == () + now[0] = 4.0 + assert g._park_for_warm_reply(2) == () + + +@pytest.mark.parametrize("params", [ + {"reused": 0}, {"old_fresh": 513}, {"new_length": 32767}, {"generated": 64}, + {"peek": 32000}, +]) +def test_bypass_ineligible_requests(params): + g, _, new = jobs(**params) + assert g._park_for_warm_reply(2) == () + assert not hasattr(new, "_warm_reply_deadline") + + +def test_prefill_already_started(): + g, _, new = jobs() + new.time_first_prefill = 1.0 + assert g._park_for_warm_reply(2) == ()