ba836e1e75
Bead: nix-overlay-3c1
210 lines
11 KiB
Diff
210 lines
11 KiB
Diff
--- a/exllamav3/util/disk_store.py
|
|
+++ b/exllamav3/util/disk_store.py
|
|
@@ -11,6 +11,7 @@
|
|
EXL3_PREFIX_DISK_GB byte cap of the whole root, all models and builds together (default 30)
|
|
EXL3_PREFIX_DISK_FLOOR_GB free space that must remain on the volume after a write (default 60)
|
|
EXL3_PREFIX_DISK_DIRECT write KV segments of at least 64 MiB with Linux O_DIRECT (default off)
|
|
+ EXL3_PREFIX_DISK_SPLIT allow smaller KV segments when one file would exceed the free-space budget (default off)
|
|
EXL3_PREFIX_DISK_MB_MIN write budget per minute (default 8000: the QLC write cliff starts near 10 GB/min)
|
|
"""
|
|
from __future__ import annotations
|
|
@@ -52,6 +53,7 @@
|
|
self.cap = int(float(e("EXL3_PREFIX_DISK_GB", "30") if cap_gb is None else cap_gb) * GIB)
|
|
self.floor = int(float(e("EXL3_PREFIX_DISK_FLOOR_GB", "60") if floor_gb is None else floor_gb) * GIB)
|
|
self.direct_segments = e("EXL3_PREFIX_DISK_DIRECT", "0") == "1"
|
|
+ self.split_segments = e("EXL3_PREFIX_DISK_SPLIT", "0") == "1"
|
|
self.rate = float(e("EXL3_PREFIX_DISK_MB_MIN", "8000") if mb_per_min is None else mb_per_min) * (1 << 20) / 60.0
|
|
self.stats = {"written": 0, "bytes": 0, "skipped_floor": 0, "skipped_cap": 0, "evicted": 0, "errors": 0, "hits": 0}
|
|
self._q: queue.Queue = queue.Queue()
|
|
@@ -151,17 +153,29 @@
|
|
|
|
# ---- async writes
|
|
|
|
- def can_stage(self, min_bytes: int) -> bool:
|
|
- """Check the writer's cap and free-space policy before allocating a segment's host copy."""
|
|
- if min_bytes > self.cap:
|
|
+ def stage_width(self, page_bytes: list[int]) -> int:
|
|
+ """Choose the number of pages per KV segment, or reject the save before making host copies."""
|
|
+ total = sum(page_bytes)
|
|
+ if total > self.cap:
|
|
self.stats["skipped_cap"] += 1
|
|
- return False
|
|
- # The writer leaves another payload's worth of headroom above the floor
|
|
- # while the serialized temporary already occupies space on the volume.
|
|
- if shutil.disk_usage(self.dir).free - 2 * min_bytes < self.floor:
|
|
- self.stats["skipped_floor"] += 1
|
|
- return False
|
|
- return True
|
|
+ return 0
|
|
+ headroom = shutil.disk_usage(self.dir).free - self.floor
|
|
+ # The writer checks free space again while its temporary file already occupies disk space.
|
|
+ if headroom >= 2 * total:
|
|
+ return len(page_bytes)
|
|
+ if self.split_segments and len(page_bytes) > 1:
|
|
+ # A 256-page segment ends at a 64K-token recurrent checkpoint on a fresh chain.
|
|
+ # Bounding the segment also leaves room for recurrent states queued by prefill.
|
|
+ width = min(256, (len(page_bytes) + 1) // 2)
|
|
+ largest = max(sum(page_bytes[i:i + width]) for i in range(0, len(page_bytes), width))
|
|
+ if headroom >= total + largest:
|
|
+ return width
|
|
+ self.stats["skipped_floor"] += 1
|
|
+ return 0
|
|
+
|
|
+ def can_stage(self, min_bytes: int) -> bool:
|
|
+ """Recheck a single serialized payload against the current space budget."""
|
|
+ return bool(self.stage_width([min_bytes]))
|
|
|
|
def put(self, name: str, payload, idx, *, min_bytes: int = 0) -> threading.Event | None:
|
|
"""Queue immutable host copies. For byte-budgeted segments, signal when the write attempt ends."""
|
|
--- a/exllamav3/generator/prefix_disk.py
|
|
+++ b/exllamav3/generator/prefix_disk.py
|
|
@@ -195,26 +195,43 @@
|
|
new = [i for i in range(keep_depth) if hashes[i] not in staged and not on_disk(hashes[i])]
|
|
wrote = 0
|
|
if new:
|
|
- src = [pages[i] for i in new]
|
|
tensors = sl._tensors()
|
|
- # index_select materializes one row per missing page in each cache tensor.
|
|
- min_bytes = len(src) * sum(t[0].nbytes for t in tensors) + sum(p.sequence.nbytes for p in src)
|
|
- if not self.disk.can_stage(min_bytes):
|
|
+ tensor_bytes = sum(t[0].nbytes for t in tensors)
|
|
+ page_bytes = [tensor_bytes + pages[i].sequence.nbytes for i in new]
|
|
+ width = self.disk.stage_width(page_bytes)
|
|
+ if width == 0:
|
|
return {"saved": 0}
|
|
torch.cuda.synchronize()
|
|
- tens = []
|
|
- for t in tensors:
|
|
- idx = torch.tensor([p.page_index for p in src], dtype=torch.long, device=t.device)
|
|
- tens.append(t.index_select(0, idx).cpu())
|
|
- hs = [hashes[i] for i in new]
|
|
- name = f"s_{_hx(hs[0])[:20]}_{len(hs)}"
|
|
- done = self.disk.put(name, {"hashes": hs, "sequences": [p.sequence.cpu().clone() for p in src], "tensors": tens},
|
|
- {"ns": self.ns, "k": "seg", "hashes": hs}, min_bytes=min_bytes)
|
|
- if done is not None:
|
|
- self._segment_writes.append((done, hs))
|
|
- for pos, h in enumerate(hs):
|
|
- self.pages[h] = (name, pos)
|
|
- wrote += len(hs)
|
|
+ for start in range(0, len(new), width):
|
|
+ selected = new[start:start + width]
|
|
+ # A newly allocated chain often occupies consecutive physical pages. Store them
|
|
+ # in physical order and let the segment index map hashes to their actual rows.
|
|
+ first_hash = hashes[selected[0]]
|
|
+ first_row = None
|
|
+ if width < len(new):
|
|
+ physical = sorted(selected, key=lambda i: pages[i].page_index)
|
|
+ base = pages[physical[0]].page_index
|
|
+ if all(pages[i].page_index == base + j for j, i in enumerate(physical)):
|
|
+ selected = physical
|
|
+ first_row = base
|
|
+ src = [pages[i] for i in selected]
|
|
+ hs = [hashes[i] for i in selected]
|
|
+ if first_row is not None:
|
|
+ # Make an independent copy on both GPU and CPU-backed caches.
|
|
+ tens = [t.narrow(0, first_row, len(src)).to("cpu", copy=True) for t in tensors]
|
|
+ else:
|
|
+ tens = []
|
|
+ for t in tensors:
|
|
+ idx = torch.tensor([p.page_index for p in src], dtype=torch.long, device=t.device)
|
|
+ tens.append(t.index_select(0, idx).cpu())
|
|
+ name = f"s_{_hx(first_hash)[:20]}_{len(hs)}"
|
|
+ done = self.disk.put(name, {"hashes": hs, "sequences": [p.sequence.cpu().clone() for p in src], "tensors": tens},
|
|
+ {"ns": self.ns, "k": "seg", "hashes": hs}, min_bytes=sum(page_bytes[start:start + width]))
|
|
+ if done is not None:
|
|
+ self._segment_writes.append((done, hs))
|
|
+ for pos, h in enumerate(hs):
|
|
+ self.pages[h] = (name, pos)
|
|
+ wrote += len(hs)
|
|
# 2. stashes (deepest + 2K, 4K, 8K ...), the newest deepest supersedes the older non-pinned ones of this chain
|
|
for d in sorted(want):
|
|
a, s = stash_at[d]
|
|
--- a/tests/test_disk_store_cpu.py
|
|
+++ b/tests/test_disk_store_cpu.py
|
|
@@ -298,3 +298,85 @@
|
|
assert writer.call_count == expected
|
|
finally:
|
|
store.close()
|
|
+
|
|
+
|
|
+@pytest.mark.parametrize("enabled,cap_pages,headroom_pages,segments,fragmented", [
|
|
+ (False, 20, 8, [], False),
|
|
+ (True, 4, 20, [], False),
|
|
+ (True, 20, 7, [], False),
|
|
+ (True, 20, 8, [3, 2], False),
|
|
+ (True, 20, 8, [3, 2], True),
|
|
+ (True, 20, 10, [5], False),
|
|
+])
|
|
+def test_split_prefix_admission_and_retry(tmp_path, monkeypatch, enabled, cap_pages, headroom_pages, segments, fragmented):
|
|
+ from exllamav3.constants import PAGE_SIZE
|
|
+ from exllamav3.generator.prefix_disk import PagedPrefixDisk
|
|
+
|
|
+ monkeypatch.setenv("EXL3_PREFIX_DISK_SPLIT", str(int(enabled)))
|
|
+ ids = torch.arange(5 * PAGE_SIZE).view(1, -1)
|
|
+ pool = torch.arange(5 * 32768, dtype=torch.int32).view(5, -1)
|
|
+ hashes = PagedPrefixDisk.chain(ids)
|
|
+ indices = [4, 2, 0, 3, 1] if fragmented else [4, 3, 2, 1, 0]
|
|
+ pages = [SimpleNamespace(page_index=p, sequence=ids[:, i*PAGE_SIZE:(i+1)*PAGE_SIZE]) for i, p in enumerate(indices)]
|
|
+ rc = Mock()
|
|
+ rc.get.side_effect = lambda h: {"position": (hashes.index(h)+1) * PAGE_SIZE}
|
|
+ slots = SimpleNamespace(generator=SimpleNamespace(recurrent_cache=rc), _tensors=lambda: [pool],
|
|
+ _resident_chain=lambda _: (hashes, pages), _find_tail=lambda *_: None)
|
|
+ pd = PagedPrefixDisk(slots, str(tmp_path))
|
|
+ page_bytes = pool[0].nbytes + pages[0].sequence.nbytes
|
|
+ pd.disk.cap = cap_pages * page_bytes
|
|
+ pd.disk.floor = 1 << 20
|
|
+ pd.disk.rate = 1e12
|
|
+ monkeypatch.setattr(ds.shutil, "disk_usage", lambda _: SimpleNamespace(free=pd.disk.floor + headroom_pages * page_bytes))
|
|
+ sync = Mock()
|
|
+ monkeypatch.setattr(torch.cuda, "synchronize", sync)
|
|
+ serialize = torch.save
|
|
+ fail_once = segments == [3, 2]
|
|
+ def save(obj, path, *args, **kwargs):
|
|
+ nonlocal fail_once
|
|
+ if fail_once and isinstance(obj, dict) and set(obj.get("hashes", ())) == set(hashes[3:]):
|
|
+ fail_once = False
|
|
+ raise OSError("second segment interrupted")
|
|
+ return serialize(obj, path, *args, **kwargs)
|
|
+ monkeypatch.setattr(torch, "save", save)
|
|
+ try:
|
|
+ assert pd.save(ids)["saved"] == sum(segments)
|
|
+ pd.disk.flush()
|
|
+ if not segments:
|
|
+ sync.assert_not_called()
|
|
+ assert not pd.disk.entries()
|
|
+ return
|
|
+ if len(segments) == 2:
|
|
+ assert pd.disk.stats["errors"] == 1
|
|
+ assert pd.save(ids)["saved"] == 2
|
|
+ pd.disk.flush()
|
|
+ pd.load_index()
|
|
+ names = [pd.pages[h][0] for h in hashes]
|
|
+ assert [names.count(name) for name in dict.fromkeys(names)] == segments
|
|
+ for h, page in zip(hashes, pages):
|
|
+ name, index = pd.pages[h]
|
|
+ payload = pd.disk.read(name)
|
|
+ assert torch.equal(payload["sequences"][index], page.sequence)
|
|
+ assert torch.equal(payload["tensors"][0][index], pool[page.page_index])
|
|
+ assert pd.save(ids)["saved"] == 0
|
|
+ finally:
|
|
+ pd.disk.close()
|
|
+
|
|
+
|
|
+def test_split_prefix_at_64k_checkpoint(tmp_path, monkeypatch):
|
|
+ monkeypatch.setenv("EXL3_PREFIX_DISK_SPLIT", "1")
|
|
+ store = ds.DiskStore(str(tmp_path), "ns")
|
|
+ try:
|
|
+ store.cap = 900
|
|
+ store.floor = 200
|
|
+ # 800 pages fit together. With one temporary committed, only a bounded
|
|
+ # segment fits into the remaining headroom while recurrent writes drain.
|
|
+ free = SimpleNamespace(free=store.floor + 800 + 256)
|
|
+ monkeypatch.setattr(ds.shutil, "disk_usage", lambda _: free)
|
|
+ assert store.stage_width([1] * 800) == 256
|
|
+ free.free -= 1
|
|
+ assert store.stage_width([1] * 800) == 0
|
|
+ free.free = store.floor + 1600
|
|
+ assert store.stage_width([1] * 800) == 800
|
|
+ finally:
|
|
+ store.close()
|