Files
millerson-overlay.nix/packages/kyojin/prefix-direct-write.patch
T
2026-11-19 01:34:07 +03:00

172 lines
7.0 KiB
Diff

--- /dev/null
+++ b/exllamav3/util/direct_save.py
@@ -0,0 +1,76 @@
+"""Serialize a large CPU checkpoint through an aligned, bounded O_DIRECT buffer."""
+import errno
+import mmap
+import os
+
+import torch
+
+
+class _DirectWriter:
+ def __init__(self, fd, buffer):
+ self.fd = fd
+ self.buffer = buffer
+ self.used = 0
+ self.size = 0
+ self.error = None
+
+ def _send(self, length):
+ with memoryview(self.buffer) as block:
+ offset = 0
+ while offset < length:
+ with block[offset:length] as chunk:
+ written = os.write(self.fd, chunk)
+ if written <= 0 or written % 4096:
+ raise OSError(errno.EIO, "invalid short direct write")
+ offset += written
+
+ def write(self, data):
+ data = memoryview(data).cast("B")
+ length = len(data)
+ if self.error is None:
+ try:
+ while data:
+ n = min(len(data), len(self.buffer) - self.used)
+ self.buffer[self.used:self.used + n] = data[:n]
+ self.used += n
+ data = data[n:]
+ if self.used == len(self.buffer):
+ self._send(self.used)
+ self.used = 0
+ self.size += length
+ except OSError as exc:
+ # Let PyTorch finish its ZIP callback before propagating the original errno.
+ self.error = OSError(exc.errno, exc.strerror)
+ return length
+
+ def flush(self):
+ # torch.save flushes after its ZIP footer. finish() writes the padded tail.
+ pass
+
+ def finish(self):
+ if self.error is not None:
+ raise self.error
+ if self.used:
+ padded = (self.used + 4095) & ~4095
+ self.buffer[self.used:padded] = bytes(padded - self.used)
+ self._send(padded)
+ os.ftruncate(self.fd, self.size)
+
+
+def save_segment(payload, tmp):
+ """Write torch's normal ZIP payload, falling back on filesystems without compatible direct I/O."""
+ if hasattr(os, "O_DIRECT"):
+ try:
+ fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_DIRECT, 0o666)
+ try:
+ with mmap.mmap(-1, 2 << 20) as buffer:
+ writer = _DirectWriter(fd, buffer)
+ torch.save(payload, writer)
+ writer.finish()
+ return
+ finally:
+ os.close(fd)
+ except OSError as exc:
+ if exc.errno not in (errno.EINVAL, errno.EOPNOTSUPP, errno.ENOSYS):
+ raise
+ torch.save(payload, tmp)
--- a/exllamav3/util/disk_store.py
+++ b/exllamav3/util/disk_store.py
@@ -10,6 +10,7 @@
EXL3_PREFIX_DISK_DIR root directory; empty = off
EXL3_PREFIX_DISK_GB byte cap of the whole root, all models and builds together (default 30)
EXL3_PREFIX_DISK_FLOOR_GB free space that must remain on the volume after a write (default 60)
+ EXL3_PREFIX_DISK_DIRECT write KV segments of at least 64 MiB with Linux O_DIRECT (default off)
EXL3_PREFIX_DISK_MB_MIN write budget per minute (default 8000: the QLC write cliff starts near 10 GB/min)
"""
from __future__ import annotations
@@ -50,6 +51,7 @@
e = os.environ.get
self.cap = int(float(e("EXL3_PREFIX_DISK_GB", "30") if cap_gb is None else cap_gb) * GIB)
self.floor = int(float(e("EXL3_PREFIX_DISK_FLOOR_GB", "60") if floor_gb is None else floor_gb) * GIB)
+ self.direct_segments = e("EXL3_PREFIX_DISK_DIRECT", "0") == "1"
self.rate = float(e("EXL3_PREFIX_DISK_MB_MIN", "8000") if mb_per_min is None else mb_per_min) * (1 << 20) / 60.0
self.stats = {"written": 0, "bytes": 0, "skipped_floor": 0, "skipped_cap": 0, "evicted": 0, "errors": 0, "hits": 0}
self._q: queue.Queue = queue.Queue()
@@ -195,7 +197,11 @@
try:
if not self.can_stage(min_bytes):
continue
- torch.save(payload, tmp)
+ if self.direct_segments and min_bytes >= 64 << 20:
+ from .direct_save import save_segment
+ save_segment(payload, tmp)
+ else:
+ torch.save(payload, tmp)
n = os.path.getsize(tmp)
free = shutil.disk_usage(self.dir).free
if free - n < self.floor: # live check: other writers share the disk
--- a/tests/test_disk_store_cpu.py
+++ b/tests/test_disk_store_cpu.py
@@ -243,3 +243,58 @@
finally:
resume.set()
pd.disk.close()
+
+
+@pytest.mark.parametrize("case", ["supported", "unsupported", "misaligned", "no_space"])
+def test_direct_snapshot_round_trip_and_write_errors(tmp_path, monkeypatch, case):
+ import errno
+ from exllamav3.util import direct_save
+
+ if not hasattr(os, "O_DIRECT"):
+ pytest.skip("Linux direct I/O")
+ path = tmp_path / "segment.pt.tmp"
+ open_file = os.open
+ def open_direct(name, flags, mode=0o777):
+ assert flags & os.O_DIRECT
+ if case == "unsupported":
+ raise OSError(errno.EOPNOTSUPP, "direct I/O unavailable")
+ # Build machines may use a filesystem without direct I/O.
+ return open_file(name, flags & ~os.O_DIRECT, mode)
+ monkeypatch.setattr(os, "open", open_direct)
+ original = os.write
+ writes = []
+ def check_write(fd, data):
+ writes.append(len(data))
+ if len(writes) == 2 and case != "supported":
+ error = errno.EINVAL if case == "misaligned" else errno.ENOSPC
+ raise OSError(error, os.strerror(error))
+ return original(fd, data)
+ monkeypatch.setattr(os, "write", check_write)
+ tensor = torch.arange(700013, dtype=torch.int64)
+ if case == "no_space":
+ with pytest.raises(OSError) as exc:
+ direct_save.save_segment({"x": tensor}, path)
+ assert exc.value.errno == errno.ENOSPC
+ else:
+ direct_save.save_segment({"x": tensor}, path)
+ restored = torch.load(path, mmap=True, weights_only=False)["x"]
+ assert torch.equal(tensor, restored)
+ assert not writes if case == "unsupported" else len(writes) >= 2
+ assert all(n % 4096 == 0 for n in writes)
+
+
+@pytest.mark.parametrize("enabled,estimate,expected", [(False, 64 << 20, 0), (True, 63 << 20, 0), (True, 64 << 20, 1)])
+def test_direct_write_only_for_large_enabled_segments(tmp_path, monkeypatch, enabled, estimate, expected):
+ from exllamav3.util import direct_save
+
+ monkeypatch.setenv("EXL3_PREFIX_DISK_DIRECT", str(int(enabled)))
+ writer = Mock(side_effect=torch.save)
+ monkeypatch.setattr(direct_save, "save_segment", writer)
+ store = mk(tmp_path, cap_gb=1)
+ try:
+ store.put("seg", torch.arange(32), {}, min_bytes=estimate)
+ store.flush()
+ assert torch.equal(torch.arange(32), store.read("seg"))
+ assert writer.call_count == expected
+ finally:
+ store.close()