Files
millerson-overlay.nix/packages/kyojin/prefix-split-segments.patch
T
2026-11-19 10:33:29 +03:00

210 lines
11 KiB
Diff

--- a/exllamav3/util/disk_store.py
+++ b/exllamav3/util/disk_store.py
@@ -11,6 +11,7 @@
EXL3_PREFIX_DISK_GB byte cap of the whole root, all models and builds together (default 30)
EXL3_PREFIX_DISK_FLOOR_GB free space that must remain on the volume after a write (default 60)
EXL3_PREFIX_DISK_DIRECT write KV segments of at least 64 MiB with Linux O_DIRECT (default off)
+ EXL3_PREFIX_DISK_SPLIT allow smaller KV segments when one file would exceed the free-space budget (default off)
EXL3_PREFIX_DISK_MB_MIN write budget per minute (default 8000: the QLC write cliff starts near 10 GB/min)
"""
from __future__ import annotations
@@ -52,6 +53,7 @@
self.cap = int(float(e("EXL3_PREFIX_DISK_GB", "30") if cap_gb is None else cap_gb) * GIB)
self.floor = int(float(e("EXL3_PREFIX_DISK_FLOOR_GB", "60") if floor_gb is None else floor_gb) * GIB)
self.direct_segments = e("EXL3_PREFIX_DISK_DIRECT", "0") == "1"
+ self.split_segments = e("EXL3_PREFIX_DISK_SPLIT", "0") == "1"
self.rate = float(e("EXL3_PREFIX_DISK_MB_MIN", "8000") if mb_per_min is None else mb_per_min) * (1 << 20) / 60.0
self.stats = {"written": 0, "bytes": 0, "skipped_floor": 0, "skipped_cap": 0, "evicted": 0, "errors": 0, "hits": 0}
self._q: queue.Queue = queue.Queue()
@@ -151,17 +153,29 @@
# ---- async writes
- def can_stage(self, min_bytes: int) -> bool:
- """Check the writer's cap and free-space policy before allocating a segment's host copy."""
- if min_bytes > self.cap:
+ def stage_width(self, page_bytes: list[int]) -> int:
+ """Choose the number of pages per KV segment, or reject the save before making host copies."""
+ total = sum(page_bytes)
+ if total > self.cap:
self.stats["skipped_cap"] += 1
- return False
- # The writer leaves another payload's worth of headroom above the floor
- # while the serialized temporary already occupies space on the volume.
- if shutil.disk_usage(self.dir).free - 2 * min_bytes < self.floor:
- self.stats["skipped_floor"] += 1
- return False
- return True
+ return 0
+ headroom = shutil.disk_usage(self.dir).free - self.floor
+ # The writer checks free space again while its temporary file already occupies disk space.
+ if headroom >= 2 * total:
+ return len(page_bytes)
+ if self.split_segments and len(page_bytes) > 1:
+ # A 256-page segment ends at a 64K-token recurrent checkpoint on a fresh chain.
+ # Bounding the segment also leaves room for recurrent states queued by prefill.
+ width = min(256, (len(page_bytes) + 1) // 2)
+ largest = max(sum(page_bytes[i:i + width]) for i in range(0, len(page_bytes), width))
+ if headroom >= total + largest:
+ return width
+ self.stats["skipped_floor"] += 1
+ return 0
+
+ def can_stage(self, min_bytes: int) -> bool:
+ """Recheck a single serialized payload against the current space budget."""
+ return bool(self.stage_width([min_bytes]))
def put(self, name: str, payload, idx, *, min_bytes: int = 0) -> threading.Event | None:
"""Queue immutable host copies. For byte-budgeted segments, signal when the write attempt ends."""
--- a/exllamav3/generator/prefix_disk.py
+++ b/exllamav3/generator/prefix_disk.py
@@ -195,26 +195,43 @@
new = [i for i in range(keep_depth) if hashes[i] not in staged and not on_disk(hashes[i])]
wrote = 0
if new:
- src = [pages[i] for i in new]
tensors = sl._tensors()
- # index_select materializes one row per missing page in each cache tensor.
- min_bytes = len(src) * sum(t[0].nbytes for t in tensors) + sum(p.sequence.nbytes for p in src)
- if not self.disk.can_stage(min_bytes):
+ tensor_bytes = sum(t[0].nbytes for t in tensors)
+ page_bytes = [tensor_bytes + pages[i].sequence.nbytes for i in new]
+ width = self.disk.stage_width(page_bytes)
+ if width == 0:
return {"saved": 0}
torch.cuda.synchronize()
- tens = []
- for t in tensors:
- idx = torch.tensor([p.page_index for p in src], dtype=torch.long, device=t.device)
- tens.append(t.index_select(0, idx).cpu())
- hs = [hashes[i] for i in new]
- name = f"s_{_hx(hs[0])[:20]}_{len(hs)}"
- done = self.disk.put(name, {"hashes": hs, "sequences": [p.sequence.cpu().clone() for p in src], "tensors": tens},
- {"ns": self.ns, "k": "seg", "hashes": hs}, min_bytes=min_bytes)
- if done is not None:
- self._segment_writes.append((done, hs))
- for pos, h in enumerate(hs):
- self.pages[h] = (name, pos)
- wrote += len(hs)
+ for start in range(0, len(new), width):
+ selected = new[start:start + width]
+ # A newly allocated chain often occupies consecutive physical pages. Store them
+ # in physical order and let the segment index map hashes to their actual rows.
+ first_hash = hashes[selected[0]]
+ first_row = None
+ if width < len(new):
+ physical = sorted(selected, key=lambda i: pages[i].page_index)
+ base = pages[physical[0]].page_index
+ if all(pages[i].page_index == base + j for j, i in enumerate(physical)):
+ selected = physical
+ first_row = base
+ src = [pages[i] for i in selected]
+ hs = [hashes[i] for i in selected]
+ if first_row is not None:
+ # Make an independent copy on both GPU and CPU-backed caches.
+ tens = [t.narrow(0, first_row, len(src)).to("cpu", copy=True) for t in tensors]
+ else:
+ tens = []
+ for t in tensors:
+ idx = torch.tensor([p.page_index for p in src], dtype=torch.long, device=t.device)
+ tens.append(t.index_select(0, idx).cpu())
+ name = f"s_{_hx(first_hash)[:20]}_{len(hs)}"
+ done = self.disk.put(name, {"hashes": hs, "sequences": [p.sequence.cpu().clone() for p in src], "tensors": tens},
+ {"ns": self.ns, "k": "seg", "hashes": hs}, min_bytes=sum(page_bytes[start:start + width]))
+ if done is not None:
+ self._segment_writes.append((done, hs))
+ for pos, h in enumerate(hs):
+ self.pages[h] = (name, pos)
+ wrote += len(hs)
# 2. stashes (deepest + 2K, 4K, 8K ...), the newest deepest supersedes the older non-pinned ones of this chain
for d in sorted(want):
a, s = stash_at[d]
--- a/tests/test_disk_store_cpu.py
+++ b/tests/test_disk_store_cpu.py
@@ -298,3 +298,85 @@
assert writer.call_count == expected
finally:
store.close()
+
+
+@pytest.mark.parametrize("enabled,cap_pages,headroom_pages,segments,fragmented", [
+ (False, 20, 8, [], False),
+ (True, 4, 20, [], False),
+ (True, 20, 7, [], False),
+ (True, 20, 8, [3, 2], False),
+ (True, 20, 8, [3, 2], True),
+ (True, 20, 10, [5], False),
+])
+def test_split_prefix_admission_and_retry(tmp_path, monkeypatch, enabled, cap_pages, headroom_pages, segments, fragmented):
+ from exllamav3.constants import PAGE_SIZE
+ from exllamav3.generator.prefix_disk import PagedPrefixDisk
+
+ monkeypatch.setenv("EXL3_PREFIX_DISK_SPLIT", str(int(enabled)))
+ ids = torch.arange(5 * PAGE_SIZE).view(1, -1)
+ pool = torch.arange(5 * 32768, dtype=torch.int32).view(5, -1)
+ hashes = PagedPrefixDisk.chain(ids)
+ indices = [4, 2, 0, 3, 1] if fragmented else [4, 3, 2, 1, 0]
+ pages = [SimpleNamespace(page_index=p, sequence=ids[:, i*PAGE_SIZE:(i+1)*PAGE_SIZE]) for i, p in enumerate(indices)]
+ rc = Mock()
+ rc.get.side_effect = lambda h: {"position": (hashes.index(h)+1) * PAGE_SIZE}
+ slots = SimpleNamespace(generator=SimpleNamespace(recurrent_cache=rc), _tensors=lambda: [pool],
+ _resident_chain=lambda _: (hashes, pages), _find_tail=lambda *_: None)
+ pd = PagedPrefixDisk(slots, str(tmp_path))
+ page_bytes = pool[0].nbytes + pages[0].sequence.nbytes
+ pd.disk.cap = cap_pages * page_bytes
+ pd.disk.floor = 1 << 20
+ pd.disk.rate = 1e12
+ monkeypatch.setattr(ds.shutil, "disk_usage", lambda _: SimpleNamespace(free=pd.disk.floor + headroom_pages * page_bytes))
+ sync = Mock()
+ monkeypatch.setattr(torch.cuda, "synchronize", sync)
+ serialize = torch.save
+ fail_once = segments == [3, 2]
+ def save(obj, path, *args, **kwargs):
+ nonlocal fail_once
+ if fail_once and isinstance(obj, dict) and set(obj.get("hashes", ())) == set(hashes[3:]):
+ fail_once = False
+ raise OSError("second segment interrupted")
+ return serialize(obj, path, *args, **kwargs)
+ monkeypatch.setattr(torch, "save", save)
+ try:
+ assert pd.save(ids)["saved"] == sum(segments)
+ pd.disk.flush()
+ if not segments:
+ sync.assert_not_called()
+ assert not pd.disk.entries()
+ return
+ if len(segments) == 2:
+ assert pd.disk.stats["errors"] == 1
+ assert pd.save(ids)["saved"] == 2
+ pd.disk.flush()
+ pd.load_index()
+ names = [pd.pages[h][0] for h in hashes]
+ assert [names.count(name) for name in dict.fromkeys(names)] == segments
+ for h, page in zip(hashes, pages):
+ name, index = pd.pages[h]
+ payload = pd.disk.read(name)
+ assert torch.equal(payload["sequences"][index], page.sequence)
+ assert torch.equal(payload["tensors"][0][index], pool[page.page_index])
+ assert pd.save(ids)["saved"] == 0
+ finally:
+ pd.disk.close()
+
+
+def test_split_prefix_at_64k_checkpoint(tmp_path, monkeypatch):
+ monkeypatch.setenv("EXL3_PREFIX_DISK_SPLIT", "1")
+ store = ds.DiskStore(str(tmp_path), "ns")
+ try:
+ store.cap = 900
+ store.floor = 200
+ # 800 pages fit together. With one temporary committed, only a bounded
+ # segment fits into the remaining headroom while recurrent writes drain.
+ free = SimpleNamespace(free=store.floor + 800 + 256)
+ monkeypatch.setattr(ds.shutil, "disk_usage", lambda _: free)
+ assert store.stage_width([1] * 800) == 256
+ free.free -= 1
+ assert store.stage_width([1] * 800) == 0
+ free.free = store.floor + 1600
+ assert store.stage_width([1] * 800) == 800
+ finally:
+ store.close()