42c2a416fe
Bead: nix-overlay-2q7
172 lines
7.0 KiB
Diff
172 lines
7.0 KiB
Diff
--- /dev/null
|
|
+++ b/exllamav3/util/direct_save.py
|
|
@@ -0,0 +1,76 @@
|
|
+"""Serialize a large CPU checkpoint through an aligned, bounded O_DIRECT buffer."""
|
|
+import errno
|
|
+import mmap
|
|
+import os
|
|
+
|
|
+import torch
|
|
+
|
|
+
|
|
+class _DirectWriter:
|
|
+ def __init__(self, fd, buffer):
|
|
+ self.fd = fd
|
|
+ self.buffer = buffer
|
|
+ self.used = 0
|
|
+ self.size = 0
|
|
+ self.error = None
|
|
+
|
|
+ def _send(self, length):
|
|
+ with memoryview(self.buffer) as block:
|
|
+ offset = 0
|
|
+ while offset < length:
|
|
+ with block[offset:length] as chunk:
|
|
+ written = os.write(self.fd, chunk)
|
|
+ if written <= 0 or written % 4096:
|
|
+ raise OSError(errno.EIO, "invalid short direct write")
|
|
+ offset += written
|
|
+
|
|
+ def write(self, data):
|
|
+ data = memoryview(data).cast("B")
|
|
+ length = len(data)
|
|
+ if self.error is None:
|
|
+ try:
|
|
+ while data:
|
|
+ n = min(len(data), len(self.buffer) - self.used)
|
|
+ self.buffer[self.used:self.used + n] = data[:n]
|
|
+ self.used += n
|
|
+ data = data[n:]
|
|
+ if self.used == len(self.buffer):
|
|
+ self._send(self.used)
|
|
+ self.used = 0
|
|
+ self.size += length
|
|
+ except OSError as exc:
|
|
+ # Let PyTorch finish its ZIP callback before propagating the original errno.
|
|
+ self.error = OSError(exc.errno, exc.strerror)
|
|
+ return length
|
|
+
|
|
+ def flush(self):
|
|
+ # torch.save flushes after its ZIP footer. finish() writes the padded tail.
|
|
+ pass
|
|
+
|
|
+ def finish(self):
|
|
+ if self.error is not None:
|
|
+ raise self.error
|
|
+ if self.used:
|
|
+ padded = (self.used + 4095) & ~4095
|
|
+ self.buffer[self.used:padded] = bytes(padded - self.used)
|
|
+ self._send(padded)
|
|
+ os.ftruncate(self.fd, self.size)
|
|
+
|
|
+
|
|
+def save_segment(payload, tmp):
|
|
+ """Write torch's normal ZIP payload, falling back on filesystems without compatible direct I/O."""
|
|
+ if hasattr(os, "O_DIRECT"):
|
|
+ try:
|
|
+ fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_DIRECT, 0o666)
|
|
+ try:
|
|
+ with mmap.mmap(-1, 2 << 20) as buffer:
|
|
+ writer = _DirectWriter(fd, buffer)
|
|
+ torch.save(payload, writer)
|
|
+ writer.finish()
|
|
+ return
|
|
+ finally:
|
|
+ os.close(fd)
|
|
+ except OSError as exc:
|
|
+ if exc.errno not in (errno.EINVAL, errno.EOPNOTSUPP, errno.ENOSYS):
|
|
+ raise
|
|
+ torch.save(payload, tmp)
|
|
--- a/exllamav3/util/disk_store.py
|
|
+++ b/exllamav3/util/disk_store.py
|
|
@@ -10,6 +10,7 @@
|
|
EXL3_PREFIX_DISK_DIR root directory; empty = off
|
|
EXL3_PREFIX_DISK_GB byte cap of the whole root, all models and builds together (default 30)
|
|
EXL3_PREFIX_DISK_FLOOR_GB free space that must remain on the volume after a write (default 60)
|
|
+ EXL3_PREFIX_DISK_DIRECT write KV segments of at least 64 MiB with Linux O_DIRECT (default off)
|
|
EXL3_PREFIX_DISK_MB_MIN write budget per minute (default 8000: the QLC write cliff starts near 10 GB/min)
|
|
"""
|
|
from __future__ import annotations
|
|
@@ -50,6 +51,7 @@
|
|
e = os.environ.get
|
|
self.cap = int(float(e("EXL3_PREFIX_DISK_GB", "30") if cap_gb is None else cap_gb) * GIB)
|
|
self.floor = int(float(e("EXL3_PREFIX_DISK_FLOOR_GB", "60") if floor_gb is None else floor_gb) * GIB)
|
|
+ self.direct_segments = e("EXL3_PREFIX_DISK_DIRECT", "0") == "1"
|
|
self.rate = float(e("EXL3_PREFIX_DISK_MB_MIN", "8000") if mb_per_min is None else mb_per_min) * (1 << 20) / 60.0
|
|
self.stats = {"written": 0, "bytes": 0, "skipped_floor": 0, "skipped_cap": 0, "evicted": 0, "errors": 0, "hits": 0}
|
|
self._q: queue.Queue = queue.Queue()
|
|
@@ -195,7 +197,11 @@
|
|
try:
|
|
if not self.can_stage(min_bytes):
|
|
continue
|
|
- torch.save(payload, tmp)
|
|
+ if self.direct_segments and min_bytes >= 64 << 20:
|
|
+ from .direct_save import save_segment
|
|
+ save_segment(payload, tmp)
|
|
+ else:
|
|
+ torch.save(payload, tmp)
|
|
n = os.path.getsize(tmp)
|
|
free = shutil.disk_usage(self.dir).free
|
|
if free - n < self.floor: # live check: other writers share the disk
|
|
--- a/tests/test_disk_store_cpu.py
|
|
+++ b/tests/test_disk_store_cpu.py
|
|
@@ -243,3 +243,58 @@
|
|
finally:
|
|
resume.set()
|
|
pd.disk.close()
|
|
+
|
|
+
|
|
+@pytest.mark.parametrize("case", ["supported", "unsupported", "misaligned", "no_space"])
|
|
+def test_direct_snapshot_round_trip_and_write_errors(tmp_path, monkeypatch, case):
|
|
+ import errno
|
|
+ from exllamav3.util import direct_save
|
|
+
|
|
+ if not hasattr(os, "O_DIRECT"):
|
|
+ pytest.skip("Linux direct I/O")
|
|
+ path = tmp_path / "segment.pt.tmp"
|
|
+ open_file = os.open
|
|
+ def open_direct(name, flags, mode=0o777):
|
|
+ assert flags & os.O_DIRECT
|
|
+ if case == "unsupported":
|
|
+ raise OSError(errno.EOPNOTSUPP, "direct I/O unavailable")
|
|
+ # Build machines may use a filesystem without direct I/O.
|
|
+ return open_file(name, flags & ~os.O_DIRECT, mode)
|
|
+ monkeypatch.setattr(os, "open", open_direct)
|
|
+ original = os.write
|
|
+ writes = []
|
|
+ def check_write(fd, data):
|
|
+ writes.append(len(data))
|
|
+ if len(writes) == 2 and case != "supported":
|
|
+ error = errno.EINVAL if case == "misaligned" else errno.ENOSPC
|
|
+ raise OSError(error, os.strerror(error))
|
|
+ return original(fd, data)
|
|
+ monkeypatch.setattr(os, "write", check_write)
|
|
+ tensor = torch.arange(700013, dtype=torch.int64)
|
|
+ if case == "no_space":
|
|
+ with pytest.raises(OSError) as exc:
|
|
+ direct_save.save_segment({"x": tensor}, path)
|
|
+ assert exc.value.errno == errno.ENOSPC
|
|
+ else:
|
|
+ direct_save.save_segment({"x": tensor}, path)
|
|
+ restored = torch.load(path, mmap=True, weights_only=False)["x"]
|
|
+ assert torch.equal(tensor, restored)
|
|
+ assert not writes if case == "unsupported" else len(writes) >= 2
|
|
+ assert all(n % 4096 == 0 for n in writes)
|
|
+
|
|
+
|
|
+@pytest.mark.parametrize("enabled,estimate,expected", [(False, 64 << 20, 0), (True, 63 << 20, 0), (True, 64 << 20, 1)])
|
|
+def test_direct_write_only_for_large_enabled_segments(tmp_path, monkeypatch, enabled, estimate, expected):
|
|
+ from exllamav3.util import direct_save
|
|
+
|
|
+ monkeypatch.setenv("EXL3_PREFIX_DISK_DIRECT", str(int(enabled)))
|
|
+ writer = Mock(side_effect=torch.save)
|
|
+ monkeypatch.setattr(direct_save, "save_segment", writer)
|
|
+ store = mk(tmp_path, cap_gb=1)
|
|
+ try:
|
|
+ store.put("seg", torch.arange(32), {}, min_bytes=estimate)
|
|
+ store.flush()
|
|
+ assert torch.equal(torch.arange(32), store.read("seg"))
|
|
+ assert writer.call_count == expected
|
|
+ finally:
|
|
+ store.close()
|