--- a/exllamav3/generator/prefix_disk.py +++ b/exllamav3/generator/prefix_disk.py @@ -189,15 +189,20 @@ wrote = 0 if new: src = [pages[i] for i in new] + tensors = sl._tensors() + # index_select materializes one row per missing page in each cache tensor. + min_bytes = len(src) * sum(t[0].nbytes for t in tensors) + sum(p.sequence.nbytes for p in src) + if not self.disk.can_stage(min_bytes): + return {"saved": 0} torch.cuda.synchronize() tens = [] - for t in sl._tensors(): + for t in tensors: idx = torch.tensor([p.page_index for p in src], dtype=torch.long, device=t.device) tens.append(t.index_select(0, idx).cpu()) hs = [hashes[i] for i in new] name = f"s_{_hx(hs[0])[:20]}_{len(hs)}" self.disk.put(name, {"hashes": hs, "sequences": [p.sequence.cpu().clone() for p in src], "tensors": tens}, - {"ns": self.ns, "k": "seg", "hashes": hs}) + {"ns": self.ns, "k": "seg", "hashes": hs}, min_bytes=min_bytes) for pos, h in enumerate(hs): self.pages[h] = (name, pos) wrote += len(hs) --- a/exllamav3/util/disk_store.py +++ b/exllamav3/util/disk_store.py @@ -149,10 +149,22 @@ # ---- async writes - def put(self, name: str, payload, idx) -> None: + def can_stage(self, min_bytes: int) -> bool: + """Check the writer's cap and free-space policy before allocating a segment's host copy.""" + if min_bytes > self.cap: + self.stats["skipped_cap"] += 1 + return False + # The writer leaves another payload's worth of headroom above the floor + # while the serialized temporary already occupies space on the volume. + if shutil.disk_usage(self.dir).free - 2 * min_bytes < self.floor: + self.stats["skipped_floor"] += 1 + return False + return True + + def put(self, name: str, payload, idx, *, min_bytes: int = 0) -> None: """Queue an entry: payload and idx are torch.save-able and must not change afterwards (host copies).""" if not self._closed: - self._q.put((name, payload, idx)) + self._q.put((name, payload, idx, min_bytes)) def flush(self) -> None: self._q.join() @@ -172,12 +184,14 @@ try: if item is None: return - name, payload, idx = item + name, payload, idx, min_bytes = item if os.path.exists(self._p(name, ".idx")): self.touch(name) continue tmp = self._p(name, ".pt.tmp") try: + if not self.can_stage(min_bytes): + continue torch.save(payload, tmp) n = os.path.getsize(tmp) free = shutil.disk_usage(self.dir).free --- a/tests/test_disk_store_cpu.py +++ b/tests/test_disk_store_cpu.py @@ -1,6 +1,10 @@ """exllamav3/util/disk_store.py: atomic async writes, LRU byte cap, free-space floor, torn entries. CPU only.""" import os import time +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest import torch @@ -89,3 +93,59 @@ new.flush() time.sleep(0.02) assert old.entries() == ["b"] and new.entries() == ["c", "d"] # the old build's oldest entry went first + + +@pytest.mark.parametrize("limit", ["cap", "floor", "fits"]) +def test_page_admission_before_device_copy(tmp_path, monkeypatch, limit): + from exllamav3.constants import PAGE_SIZE + from exllamav3.generator.prefix_disk import PagedPrefixDisk + + ids = torch.arange(PAGE_SIZE).view(1, -1) + pool = torch.arange(8 * 8192, dtype=torch.int32).view(8, -1) + page = SimpleNamespace(page_index=7, sequence=ids) + rc = Mock() + rc.get.return_value = {"position": PAGE_SIZE} + slots = SimpleNamespace(generator=SimpleNamespace(recurrent_cache=rc), _tensors=lambda: [pool], + _resident_chain=lambda _: ([b"page"], [page]), _find_tail=lambda *_: None) + pd = PagedPrefixDisk(slots, root=str(tmp_path)) + page_bytes = pool[7].nbytes + ids.nbytes + pd.disk.rate = 1e12 + pd.disk.cap = page_bytes - 1 if limit == "cap" else page_bytes * 2 + pd.disk.floor = 1 << 20 + free = pd.disk.floor + (2 * page_bytes - 1 if limit == "floor" else 10 * page_bytes) + monkeypatch.setattr(ds.shutil, "disk_usage", lambda _: SimpleNamespace(free=free)) + sync = Mock() + monkeypatch.setattr(torch.cuda, "synchronize", sync) + try: + saved = pd.save(ids)["saved"] + pd.disk.flush() + if limit == "fits": + assert saved == 1 and sync.called + name, pos = pd.pages[b"page"] + assert torch.equal(pd.disk.read(name)["tensors"][0][pos], pool[7]) + else: + sync.assert_not_called() + assert saved == 0 and not pd.pages and not pd.disk.entries() + assert pd.disk.stats["skipped_" + limit] == 1 + finally: + pd.disk.close() + + +def test_worker_checks_space_before_writing_queued_segment(tmp_path, monkeypatch): + s = mk(tmp_path, cap_gb=1) + s.floor = 4096 + payload = blob(1) + n = payload["x"].nbytes + free = Mock(return_value=SimpleNamespace(free=s.floor + 3 * n)) + monkeypatch.setattr(ds.shutil, "disk_usage", free) + save = Mock(wraps=torch.save) + monkeypatch.setattr(torch, "save", save) + try: + assert s.can_stage(n) + free.return_value = SimpleNamespace(free=s.floor + n) + s.put("seg", payload, {}, min_bytes=n) + s.flush() + save.assert_not_called() + assert not s.entries() and s.stats["skipped_floor"] == 1 + finally: + s.close()