--- a/exllamav3/util/disk_store.py +++ b/exllamav3/util/disk_store.py @@ -11,6 +11,7 @@ EXL3_PREFIX_DISK_GB byte cap of the whole root, all models and builds together (default 30) EXL3_PREFIX_DISK_FLOOR_GB free space that must remain on the volume after a write (default 60) EXL3_PREFIX_DISK_DIRECT write KV segments of at least 64 MiB with Linux O_DIRECT (default off) + EXL3_PREFIX_DISK_SPLIT allow smaller KV segments when one file would exceed the free-space budget (default off) EXL3_PREFIX_DISK_MB_MIN write budget per minute (default 8000: the QLC write cliff starts near 10 GB/min) """ from __future__ import annotations @@ -52,6 +53,7 @@ self.cap = int(float(e("EXL3_PREFIX_DISK_GB", "30") if cap_gb is None else cap_gb) * GIB) self.floor = int(float(e("EXL3_PREFIX_DISK_FLOOR_GB", "60") if floor_gb is None else floor_gb) * GIB) self.direct_segments = e("EXL3_PREFIX_DISK_DIRECT", "0") == "1" + self.split_segments = e("EXL3_PREFIX_DISK_SPLIT", "0") == "1" self.rate = float(e("EXL3_PREFIX_DISK_MB_MIN", "8000") if mb_per_min is None else mb_per_min) * (1 << 20) / 60.0 self.stats = {"written": 0, "bytes": 0, "skipped_floor": 0, "skipped_cap": 0, "evicted": 0, "errors": 0, "hits": 0} self._q: queue.Queue = queue.Queue() @@ -151,17 +153,29 @@ # ---- async writes - def can_stage(self, min_bytes: int) -> bool: - """Check the writer's cap and free-space policy before allocating a segment's host copy.""" - if min_bytes > self.cap: + def stage_width(self, page_bytes: list[int]) -> int: + """Choose the number of pages per KV segment, or reject the save before making host copies.""" + total = sum(page_bytes) + if total > self.cap: self.stats["skipped_cap"] += 1 - return False - # The writer leaves another payload's worth of headroom above the floor - # while the serialized temporary already occupies space on the volume. - if shutil.disk_usage(self.dir).free - 2 * min_bytes < self.floor: - self.stats["skipped_floor"] += 1 - return False - return True + return 0 + headroom = shutil.disk_usage(self.dir).free - self.floor + # The writer checks free space again while its temporary file already occupies disk space. + if headroom >= 2 * total: + return len(page_bytes) + if self.split_segments and len(page_bytes) > 1: + # A 256-page segment ends at a 64K-token recurrent checkpoint on a fresh chain. + # Bounding the segment also leaves room for recurrent states queued by prefill. + width = min(256, (len(page_bytes) + 1) // 2) + largest = max(sum(page_bytes[i:i + width]) for i in range(0, len(page_bytes), width)) + if headroom >= total + largest: + return width + self.stats["skipped_floor"] += 1 + return 0 + + def can_stage(self, min_bytes: int) -> bool: + """Recheck a single serialized payload against the current space budget.""" + return bool(self.stage_width([min_bytes])) def put(self, name: str, payload, idx, *, min_bytes: int = 0) -> threading.Event | None: """Queue immutable host copies. For byte-budgeted segments, signal when the write attempt ends.""" --- a/exllamav3/generator/prefix_disk.py +++ b/exllamav3/generator/prefix_disk.py @@ -195,26 +195,43 @@ new = [i for i in range(keep_depth) if hashes[i] not in staged and not on_disk(hashes[i])] wrote = 0 if new: - src = [pages[i] for i in new] tensors = sl._tensors() - # index_select materializes one row per missing page in each cache tensor. - min_bytes = len(src) * sum(t[0].nbytes for t in tensors) + sum(p.sequence.nbytes for p in src) - if not self.disk.can_stage(min_bytes): + tensor_bytes = sum(t[0].nbytes for t in tensors) + page_bytes = [tensor_bytes + pages[i].sequence.nbytes for i in new] + width = self.disk.stage_width(page_bytes) + if width == 0: return {"saved": 0} torch.cuda.synchronize() - tens = [] - for t in tensors: - idx = torch.tensor([p.page_index for p in src], dtype=torch.long, device=t.device) - tens.append(t.index_select(0, idx).cpu()) - hs = [hashes[i] for i in new] - name = f"s_{_hx(hs[0])[:20]}_{len(hs)}" - done = self.disk.put(name, {"hashes": hs, "sequences": [p.sequence.cpu().clone() for p in src], "tensors": tens}, - {"ns": self.ns, "k": "seg", "hashes": hs}, min_bytes=min_bytes) - if done is not None: - self._segment_writes.append((done, hs)) - for pos, h in enumerate(hs): - self.pages[h] = (name, pos) - wrote += len(hs) + for start in range(0, len(new), width): + selected = new[start:start + width] + # A newly allocated chain often occupies consecutive physical pages. Store them + # in physical order and let the segment index map hashes to their actual rows. + first_hash = hashes[selected[0]] + first_row = None + if width < len(new): + physical = sorted(selected, key=lambda i: pages[i].page_index) + base = pages[physical[0]].page_index + if all(pages[i].page_index == base + j for j, i in enumerate(physical)): + selected = physical + first_row = base + src = [pages[i] for i in selected] + hs = [hashes[i] for i in selected] + if first_row is not None: + # Make an independent copy on both GPU and CPU-backed caches. + tens = [t.narrow(0, first_row, len(src)).to("cpu", copy=True) for t in tensors] + else: + tens = [] + for t in tensors: + idx = torch.tensor([p.page_index for p in src], dtype=torch.long, device=t.device) + tens.append(t.index_select(0, idx).cpu()) + name = f"s_{_hx(first_hash)[:20]}_{len(hs)}" + done = self.disk.put(name, {"hashes": hs, "sequences": [p.sequence.cpu().clone() for p in src], "tensors": tens}, + {"ns": self.ns, "k": "seg", "hashes": hs}, min_bytes=sum(page_bytes[start:start + width])) + if done is not None: + self._segment_writes.append((done, hs)) + for pos, h in enumerate(hs): + self.pages[h] = (name, pos) + wrote += len(hs) # 2. stashes (deepest + 2K, 4K, 8K ...), the newest deepest supersedes the older non-pinned ones of this chain for d in sorted(want): a, s = stash_at[d] --- a/tests/test_disk_store_cpu.py +++ b/tests/test_disk_store_cpu.py @@ -298,3 +298,85 @@ assert writer.call_count == expected finally: store.close() + + +@pytest.mark.parametrize("enabled,cap_pages,headroom_pages,segments,fragmented", [ + (False, 20, 8, [], False), + (True, 4, 20, [], False), + (True, 20, 7, [], False), + (True, 20, 8, [3, 2], False), + (True, 20, 8, [3, 2], True), + (True, 20, 10, [5], False), +]) +def test_split_prefix_admission_and_retry(tmp_path, monkeypatch, enabled, cap_pages, headroom_pages, segments, fragmented): + from exllamav3.constants import PAGE_SIZE + from exllamav3.generator.prefix_disk import PagedPrefixDisk + + monkeypatch.setenv("EXL3_PREFIX_DISK_SPLIT", str(int(enabled))) + ids = torch.arange(5 * PAGE_SIZE).view(1, -1) + pool = torch.arange(5 * 32768, dtype=torch.int32).view(5, -1) + hashes = PagedPrefixDisk.chain(ids) + indices = [4, 2, 0, 3, 1] if fragmented else [4, 3, 2, 1, 0] + pages = [SimpleNamespace(page_index=p, sequence=ids[:, i*PAGE_SIZE:(i+1)*PAGE_SIZE]) for i, p in enumerate(indices)] + rc = Mock() + rc.get.side_effect = lambda h: {"position": (hashes.index(h)+1) * PAGE_SIZE} + slots = SimpleNamespace(generator=SimpleNamespace(recurrent_cache=rc), _tensors=lambda: [pool], + _resident_chain=lambda _: (hashes, pages), _find_tail=lambda *_: None) + pd = PagedPrefixDisk(slots, str(tmp_path)) + page_bytes = pool[0].nbytes + pages[0].sequence.nbytes + pd.disk.cap = cap_pages * page_bytes + pd.disk.floor = 1 << 20 + pd.disk.rate = 1e12 + monkeypatch.setattr(ds.shutil, "disk_usage", lambda _: SimpleNamespace(free=pd.disk.floor + headroom_pages * page_bytes)) + sync = Mock() + monkeypatch.setattr(torch.cuda, "synchronize", sync) + serialize = torch.save + fail_once = segments == [3, 2] + def save(obj, path, *args, **kwargs): + nonlocal fail_once + if fail_once and isinstance(obj, dict) and set(obj.get("hashes", ())) == set(hashes[3:]): + fail_once = False + raise OSError("second segment interrupted") + return serialize(obj, path, *args, **kwargs) + monkeypatch.setattr(torch, "save", save) + try: + assert pd.save(ids)["saved"] == sum(segments) + pd.disk.flush() + if not segments: + sync.assert_not_called() + assert not pd.disk.entries() + return + if len(segments) == 2: + assert pd.disk.stats["errors"] == 1 + assert pd.save(ids)["saved"] == 2 + pd.disk.flush() + pd.load_index() + names = [pd.pages[h][0] for h in hashes] + assert [names.count(name) for name in dict.fromkeys(names)] == segments + for h, page in zip(hashes, pages): + name, index = pd.pages[h] + payload = pd.disk.read(name) + assert torch.equal(payload["sequences"][index], page.sequence) + assert torch.equal(payload["tensors"][0][index], pool[page.page_index]) + assert pd.save(ids)["saved"] == 0 + finally: + pd.disk.close() + + +def test_split_prefix_at_64k_checkpoint(tmp_path, monkeypatch): + monkeypatch.setenv("EXL3_PREFIX_DISK_SPLIT", "1") + store = ds.DiskStore(str(tmp_path), "ns") + try: + store.cap = 900 + store.floor = 200 + # 800 pages fit together. With one temporary committed, only a bounded + # segment fits into the remaining headroom while recurrent writes drain. + free = SimpleNamespace(free=store.floor + 800 + 256) + monkeypatch.setattr(ds.shutil, "disk_usage", lambda _: free) + assert store.stage_width([1] * 800) == 256 + free.free -= 1 + assert store.stage_width([1] * 800) == 0 + free.free = store.floor + 1600 + assert store.stage_width([1] * 800) == 800 + finally: + store.close()