635f9f7dfc
Bead: nix-overlay-7p6
185 lines
8.3 KiB
Diff
185 lines
8.3 KiB
Diff
--- a/exllamav3/generator/prefix_disk.py
|
|
+++ b/exllamav3/generator/prefix_disk.py
|
|
@@ -23,6 +23,7 @@
|
|
|
|
import hashlib
|
|
import os
|
|
+import threading
|
|
import time
|
|
|
|
import torch
|
|
@@ -90,6 +91,7 @@
|
|
self.pages: dict[bytes, tuple[str, int]] = {} # page hash -> (segment, position in segment)
|
|
self.stashes: dict[bytes, tuple[str, int]] = {} # anchor hash -> (entry, depth in pages)
|
|
self.tails: dict[str, dict] = {} # entry -> idx
|
|
+ self._segment_writes: list[tuple[threading.Event, list[bytes]]] = []
|
|
self.pending: list[torch.Tensor] = []
|
|
self._staging = ([None, None], [None, None]) # pinned host buffers + their last device-copy events
|
|
self.stats = {"saves": 0, "save_ms": 0.0, "restores": 0, "restore_ms": 0.0, "restored_tokens": 0}
|
|
@@ -130,6 +132,7 @@
|
|
def clear(self) -> None:
|
|
"""Forget everything (cache_salt change): the entries were computed under another namespace."""
|
|
self.disk.flush()
|
|
+ self._segment_writes.clear()
|
|
for name in self.disk.entries():
|
|
self.disk.remove(name)
|
|
self.pages, self.stashes, self.tails = {}, {}, {}
|
|
@@ -185,7 +188,11 @@
|
|
if ent[0] not in ok:
|
|
ok[ent[0]] = self._have(ent[0])
|
|
return ok[ent[0]]
|
|
- new = [i for i in range(keep_depth) if not on_disk(hashes[i])]
|
|
+ # A follow-up can finish before the previous snapshot publishes its index.
|
|
+ # Save can reuse these exact host copies. Restore requires a published index.
|
|
+ self._segment_writes = [(done, hs) for done, hs in self._segment_writes if not done.is_set()]
|
|
+ staged = {h for _, hs in self._segment_writes for h in hs}
|
|
+ new = [i for i in range(keep_depth) if hashes[i] not in staged and not on_disk(hashes[i])]
|
|
wrote = 0
|
|
if new:
|
|
src = [pages[i] for i in new]
|
|
@@ -201,8 +208,10 @@
|
|
tens.append(t.index_select(0, idx).cpu())
|
|
hs = [hashes[i] for i in new]
|
|
name = f"s_{_hx(hs[0])[:20]}_{len(hs)}"
|
|
- self.disk.put(name, {"hashes": hs, "sequences": [p.sequence.cpu().clone() for p in src], "tensors": tens},
|
|
- {"ns": self.ns, "k": "seg", "hashes": hs}, min_bytes=min_bytes)
|
|
+ done = self.disk.put(name, {"hashes": hs, "sequences": [p.sequence.cpu().clone() for p in src], "tensors": tens},
|
|
+ {"ns": self.ns, "k": "seg", "hashes": hs}, min_bytes=min_bytes)
|
|
+ if done is not None:
|
|
+ self._segment_writes.append((done, hs))
|
|
for pos, h in enumerate(hs):
|
|
self.pages[h] = (name, pos)
|
|
wrote += len(hs)
|
|
--- a/exllamav3/util/disk_store.py
|
|
+++ b/exllamav3/util/disk_store.py
|
|
@@ -161,10 +161,12 @@
|
|
return False
|
|
return True
|
|
|
|
- def put(self, name: str, payload, idx, *, min_bytes: int = 0) -> None:
|
|
- """Queue an entry: payload and idx are torch.save-able and must not change afterwards (host copies)."""
|
|
+ def put(self, name: str, payload, idx, *, min_bytes: int = 0) -> threading.Event | None:
|
|
+ """Queue immutable host copies. For byte-budgeted segments, signal when the write attempt ends."""
|
|
if not self._closed:
|
|
- self._q.put((name, payload, idx, min_bytes))
|
|
+ finished = threading.Event() if min_bytes else None
|
|
+ self._q.put((name, payload, idx, min_bytes, finished))
|
|
+ return finished
|
|
|
|
def flush(self) -> None:
|
|
self._q.join()
|
|
@@ -180,11 +182,12 @@
|
|
def _run(self) -> None:
|
|
window_start, window_bytes = time.monotonic(), 0
|
|
while True:
|
|
+ finished = None
|
|
item = self._q.get()
|
|
try:
|
|
if item is None:
|
|
return
|
|
- name, payload, idx, min_bytes = item
|
|
+ name, payload, idx, min_bytes, finished = item
|
|
if os.path.exists(self._p(name, ".idx")):
|
|
self.touch(name)
|
|
continue
|
|
@@ -215,6 +218,9 @@
|
|
spent = time.monotonic() - window_start
|
|
ahead = window_bytes / self.rate - spent # stay under the per-minute write budget
|
|
if ahead > 0:
|
|
+ if finished is not None:
|
|
+ finished.set()
|
|
+ finished = None
|
|
item = payload = idx = None # Release host tensors before write-rate throttling.
|
|
time.sleep(min(ahead, 120.0))
|
|
if spent > 60:
|
|
@@ -228,4 +234,6 @@
|
|
pass
|
|
finally:
|
|
item = payload = idx = None # Skipped and failed writes can also precede an idle queue.
|
|
+ if finished is not None:
|
|
+ finished.set()
|
|
self._q.task_done()
|
|
--- a/tests/test_disk_store_cpu.py
|
|
+++ b/tests/test_disk_store_cpu.py
|
|
@@ -145,8 +145,9 @@
|
|
try:
|
|
assert s.can_stage(n)
|
|
free.return_value = SimpleNamespace(free=s.floor + n)
|
|
- s.put("seg", payload, {}, min_bytes=n)
|
|
+ finished = s.put("seg", payload, {}, min_bytes=n)
|
|
s.flush()
|
|
+ assert finished.is_set()
|
|
save.assert_not_called()
|
|
assert not s.entries() and s.stats["skipped_floor"] == 1
|
|
finally:
|
|
@@ -165,9 +166,10 @@
|
|
tensor = torch.arange(32)
|
|
ref = weakref.ref(tensor)
|
|
try:
|
|
- store.put("seg", {"x": tensor}, {})
|
|
+ finished = store.put("seg", {"x": tensor}, {}, min_bytes=tensor.nbytes)
|
|
del tensor
|
|
assert sleeping.wait(5)
|
|
+ assert finished.is_set()
|
|
assert ref() is None
|
|
assert torch.equal(store.read("seg")["x"], torch.arange(32))
|
|
finally:
|
|
@@ -187,3 +189,57 @@
|
|
assert ref() is None
|
|
finally:
|
|
store.close()
|
|
+
|
|
+
|
|
+@pytest.mark.parametrize("fail_first", [False, True])
|
|
+def test_consecutive_saves_share_only_inflight_pages(tmp_path, monkeypatch, fail_first):
|
|
+ from exllamav3.constants import PAGE_SIZE
|
|
+ from exllamav3.generator.prefix_disk import PagedPrefixDisk
|
|
+
|
|
+ ids = torch.arange(2 * PAGE_SIZE).view(1, -1)
|
|
+ pool = torch.arange(6 * 128).view(6, -1)
|
|
+ pages = [SimpleNamespace(page_index=i, sequence=ids[:, n * PAGE_SIZE:(n + 1) * PAGE_SIZE])
|
|
+ for n, i in enumerate((4, 2))]
|
|
+ hashes = PagedPrefixDisk.chain(ids)
|
|
+ rc = Mock()
|
|
+ rc.get.side_effect = lambda h: {"position": (1 + hashes.index(h)) * PAGE_SIZE}
|
|
+ def chain(tokens):
|
|
+ count = tokens.numel() // PAGE_SIZE
|
|
+ return hashes[:count], pages[:count]
|
|
+ pages_on_device = SimpleNamespace(referenced_pages={}, unreferenced_pages={})
|
|
+ generator = SimpleNamespace(recurrent_cache=rc, pagetable=pages_on_device, tail_ckpts=[])
|
|
+ slots = SimpleNamespace(generator=generator, _tensors=lambda: [pool],
|
|
+ _resident_chain=chain, _find_tail=lambda *_: None)
|
|
+ pd = PagedPrefixDisk(slots, str(tmp_path))
|
|
+ pd.disk.floor = 0
|
|
+ pd.disk.rate = 1e12
|
|
+ monkeypatch.setattr(torch.cuda, "synchronize", lambda: None)
|
|
+ started, resume = Event(), Event()
|
|
+ serialize = torch.save
|
|
+ def hold_first(obj, path, *args, **kwargs):
|
|
+ if isinstance(obj, dict) and obj.get("hashes") == hashes[:1] and not started.is_set():
|
|
+ started.set()
|
|
+ assert resume.wait(10)
|
|
+ if fail_first:
|
|
+ raise OSError("injected write failure")
|
|
+ return serialize(obj, path, *args, **kwargs)
|
|
+ monkeypatch.setattr(torch, "save", hold_first)
|
|
+ try:
|
|
+ assert pd.save(ids[:, :PAGE_SIZE])["saved"] == 1
|
|
+ assert started.wait(5)
|
|
+ assert not pd._have(pd.pages[hashes[0]][0])
|
|
+ assert pd.restore(ids) is None
|
|
+ assert pd.save(ids)["saved"] == 1
|
|
+ resume.set()
|
|
+ pd.disk.flush()
|
|
+ assert pd.save(ids)["saved"] == int(fail_first)
|
|
+ pd.disk.flush()
|
|
+ for h, page in zip(hashes, pages):
|
|
+ name, pos = pd.pages[h]
|
|
+ assert torch.equal(pd.disk.read(name)["tensors"][0][pos], pool[page.page_index])
|
|
+ # A committed segment that is later evicted must be staged again.
|
|
+ pd.disk.remove(pd.pages[hashes[0]][0])
|
|
+ assert pd.save(ids)["saved"] == 1
|
|
+ finally:
|
|
+ resume.set()
|
|
+ pd.disk.close()
|