Files
millerson-overlay.nix/packages/kyojin/prefix-writer-release.patch
T
2026-11-19 00:41:57 +03:00

68 lines
2.3 KiB
Diff

--- a/exllamav3/util/disk_store.py
+++ b/exllamav3/util/disk_store.py
@@ -215,6 +215,7 @@
spent = time.monotonic() - window_start
ahead = window_bytes / self.rate - spent # stay under the per-minute write budget
if ahead > 0:
+ item = payload = idx = None # Release host tensors before write-rate throttling.
time.sleep(min(ahead, 120.0))
if spent > 60:
window_start, window_bytes = time.monotonic(), 0
@@ -226,4 +227,5 @@
except OSError:
pass
finally:
+ item = payload = idx = None # Skipped and failed writes can also precede an idle queue.
self._q.task_done()
--- a/tests/test_disk_store_cpu.py
+++ b/tests/test_disk_store_cpu.py
@@ -1,6 +1,8 @@
"""exllamav3/util/disk_store.py: atomic async writes, LRU byte cap, free-space floor, torn entries. CPU only."""
import os
import time
+import weakref
+from threading import Event
from types import SimpleNamespace
from unittest.mock import Mock
@@ -149,3 +151,39 @@
assert not s.entries() and s.stats["skipped_floor"] == 1
finally:
s.close()
+
+
+def test_writer_releases_payload_before_throttle(tmp_path, monkeypatch):
+ sleeping, resume = Event(), Event()
+
+ def wait(_):
+ sleeping.set()
+ resume.wait(5)
+
+ monkeypatch.setattr(ds, "time", SimpleNamespace(monotonic=time.monotonic, sleep=wait))
+ store = mk(tmp_path, mb_per_min=0.001)
+ tensor = torch.arange(32)
+ ref = weakref.ref(tensor)
+ try:
+ store.put("seg", {"x": tensor}, {})
+ del tensor
+ assert sleeping.wait(5)
+ assert ref() is None
+ assert torch.equal(store.read("seg")["x"], torch.arange(32))
+ finally:
+ resume.set()
+ store.close()
+
+
+def test_idle_writer_releases_skipped_payload(tmp_path):
+ store = mk(tmp_path, cap_gb=0)
+ tensor = torch.arange(32)
+ ref = weakref.ref(tensor)
+ try:
+ store.put("seg", {"x": tensor}, {}, min_bytes=tensor.nbytes)
+ del tensor
+ store.flush()
+ assert store.stats["skipped_cap"] == 1
+ assert ref() is None
+ finally:
+ store.close()