fix(kyojin): release prefix buffers before the writer waits
Bead: nix-overlay-9hu
This commit is contained in:
@@ -8,6 +8,7 @@
|
||||
let
|
||||
enginePatches = [
|
||||
./prefix-disk-admission.patch
|
||||
./prefix-writer-release.patch
|
||||
./expert-direct-read-eof.patch
|
||||
];
|
||||
engine = exllamav3.overrideAttrs (old: {
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
--- a/exllamav3/util/disk_store.py
|
||||
+++ b/exllamav3/util/disk_store.py
|
||||
@@ -215,6 +215,7 @@
|
||||
spent = time.monotonic() - window_start
|
||||
ahead = window_bytes / self.rate - spent # stay under the per-minute write budget
|
||||
if ahead > 0:
|
||||
+ item = payload = idx = None # Release host tensors before write-rate throttling.
|
||||
time.sleep(min(ahead, 120.0))
|
||||
if spent > 60:
|
||||
window_start, window_bytes = time.monotonic(), 0
|
||||
@@ -226,4 +227,5 @@
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
+ item = payload = idx = None # Skipped and failed writes can also precede an idle queue.
|
||||
self._q.task_done()
|
||||
--- a/tests/test_disk_store_cpu.py
|
||||
+++ b/tests/test_disk_store_cpu.py
|
||||
@@ -1,6 +1,8 @@
|
||||
"""exllamav3/util/disk_store.py: atomic async writes, LRU byte cap, free-space floor, torn entries. CPU only."""
|
||||
import os
|
||||
import time
|
||||
+import weakref
|
||||
+from threading import Event
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
@@ -149,3 +151,39 @@
|
||||
assert not s.entries() and s.stats["skipped_floor"] == 1
|
||||
finally:
|
||||
s.close()
|
||||
+
|
||||
+
|
||||
+def test_writer_releases_payload_before_throttle(tmp_path, monkeypatch):
|
||||
+ sleeping, resume = Event(), Event()
|
||||
+
|
||||
+ def wait(_):
|
||||
+ sleeping.set()
|
||||
+ resume.wait(5)
|
||||
+
|
||||
+ monkeypatch.setattr(ds, "time", SimpleNamespace(monotonic=time.monotonic, sleep=wait))
|
||||
+ store = mk(tmp_path, mb_per_min=0.001)
|
||||
+ tensor = torch.arange(32)
|
||||
+ ref = weakref.ref(tensor)
|
||||
+ try:
|
||||
+ store.put("seg", {"x": tensor}, {})
|
||||
+ del tensor
|
||||
+ assert sleeping.wait(5)
|
||||
+ assert ref() is None
|
||||
+ assert torch.equal(store.read("seg")["x"], torch.arange(32))
|
||||
+ finally:
|
||||
+ resume.set()
|
||||
+ store.close()
|
||||
+
|
||||
+
|
||||
+def test_idle_writer_releases_skipped_payload(tmp_path):
|
||||
+ store = mk(tmp_path, cap_gb=0)
|
||||
+ tensor = torch.arange(32)
|
||||
+ ref = weakref.ref(tensor)
|
||||
+ try:
|
||||
+ store.put("seg", {"x": tensor}, {}, min_bytes=tensor.nbytes)
|
||||
+ del tensor
|
||||
+ store.flush()
|
||||
+ assert store.stats["skipped_cap"] == 1
|
||||
+ assert ref() is None
|
||||
+ finally:
|
||||
+ store.close()
|
||||
Reference in New Issue
Block a user