--- a/exllamav3/util/disk_store.py +++ b/exllamav3/util/disk_store.py @@ -215,6 +215,7 @@ spent = time.monotonic() - window_start ahead = window_bytes / self.rate - spent # stay under the per-minute write budget if ahead > 0: + item = payload = idx = None # Release host tensors before write-rate throttling. time.sleep(min(ahead, 120.0)) if spent > 60: window_start, window_bytes = time.monotonic(), 0 @@ -226,4 +227,5 @@ except OSError: pass finally: + item = payload = idx = None # Skipped and failed writes can also precede an idle queue. self._q.task_done() --- a/tests/test_disk_store_cpu.py +++ b/tests/test_disk_store_cpu.py @@ -1,6 +1,8 @@ """exllamav3/util/disk_store.py: atomic async writes, LRU byte cap, free-space floor, torn entries. CPU only.""" import os import time +import weakref +from threading import Event from types import SimpleNamespace from unittest.mock import Mock @@ -149,3 +151,39 @@ assert not s.entries() and s.stats["skipped_floor"] == 1 finally: s.close() + + +def test_writer_releases_payload_before_throttle(tmp_path, monkeypatch): + sleeping, resume = Event(), Event() + + def wait(_): + sleeping.set() + resume.wait(5) + + monkeypatch.setattr(ds, "time", SimpleNamespace(monotonic=time.monotonic, sleep=wait)) + store = mk(tmp_path, mb_per_min=0.001) + tensor = torch.arange(32) + ref = weakref.ref(tensor) + try: + store.put("seg", {"x": tensor}, {}) + del tensor + assert sleeping.wait(5) + assert ref() is None + assert torch.equal(store.read("seg")["x"], torch.arange(32)) + finally: + resume.set() + store.close() + + +def test_idle_writer_releases_skipped_payload(tmp_path): + store = mk(tmp_path, cap_gb=0) + tensor = torch.arange(32) + ref = weakref.ref(tensor) + try: + store.put("seg", {"x": tensor}, {}, min_bytes=tensor.nbytes) + del tensor + store.flush() + assert store.stats["skipped_cap"] == 1 + assert ref() is None + finally: + store.close()