perf(kyojin): support direct writes for large prefix segments
Bead: nix-overlay-2q7
This commit is contained in:
@@ -10,6 +10,7 @@ let
|
||||
./prefix-disk-admission.patch
|
||||
./prefix-writer-release.patch
|
||||
./prefix-inflight-pages.patch
|
||||
./prefix-direct-write.patch
|
||||
./expert-direct-read-eof.patch
|
||||
];
|
||||
engine = exllamav3.overrideAttrs (old: {
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
--- /dev/null
|
||||
+++ b/exllamav3/util/direct_save.py
|
||||
@@ -0,0 +1,76 @@
|
||||
+"""Serialize a large CPU checkpoint through an aligned, bounded O_DIRECT buffer."""
|
||||
+import errno
|
||||
+import mmap
|
||||
+import os
|
||||
+
|
||||
+import torch
|
||||
+
|
||||
+
|
||||
+class _DirectWriter:
|
||||
+ def __init__(self, fd, buffer):
|
||||
+ self.fd = fd
|
||||
+ self.buffer = buffer
|
||||
+ self.used = 0
|
||||
+ self.size = 0
|
||||
+ self.error = None
|
||||
+
|
||||
+ def _send(self, length):
|
||||
+ with memoryview(self.buffer) as block:
|
||||
+ offset = 0
|
||||
+ while offset < length:
|
||||
+ with block[offset:length] as chunk:
|
||||
+ written = os.write(self.fd, chunk)
|
||||
+ if written <= 0 or written % 4096:
|
||||
+ raise OSError(errno.EIO, "invalid short direct write")
|
||||
+ offset += written
|
||||
+
|
||||
+ def write(self, data):
|
||||
+ data = memoryview(data).cast("B")
|
||||
+ length = len(data)
|
||||
+ if self.error is None:
|
||||
+ try:
|
||||
+ while data:
|
||||
+ n = min(len(data), len(self.buffer) - self.used)
|
||||
+ self.buffer[self.used:self.used + n] = data[:n]
|
||||
+ self.used += n
|
||||
+ data = data[n:]
|
||||
+ if self.used == len(self.buffer):
|
||||
+ self._send(self.used)
|
||||
+ self.used = 0
|
||||
+ self.size += length
|
||||
+ except OSError as exc:
|
||||
+ # Let PyTorch finish its ZIP callback before propagating the original errno.
|
||||
+ self.error = OSError(exc.errno, exc.strerror)
|
||||
+ return length
|
||||
+
|
||||
+ def flush(self):
|
||||
+ # torch.save flushes after its ZIP footer. finish() writes the padded tail.
|
||||
+ pass
|
||||
+
|
||||
+ def finish(self):
|
||||
+ if self.error is not None:
|
||||
+ raise self.error
|
||||
+ if self.used:
|
||||
+ padded = (self.used + 4095) & ~4095
|
||||
+ self.buffer[self.used:padded] = bytes(padded - self.used)
|
||||
+ self._send(padded)
|
||||
+ os.ftruncate(self.fd, self.size)
|
||||
+
|
||||
+
|
||||
+def save_segment(payload, tmp):
|
||||
+ """Write torch's normal ZIP payload, falling back on filesystems without compatible direct I/O."""
|
||||
+ if hasattr(os, "O_DIRECT"):
|
||||
+ try:
|
||||
+ fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_DIRECT, 0o666)
|
||||
+ try:
|
||||
+ with mmap.mmap(-1, 2 << 20) as buffer:
|
||||
+ writer = _DirectWriter(fd, buffer)
|
||||
+ torch.save(payload, writer)
|
||||
+ writer.finish()
|
||||
+ return
|
||||
+ finally:
|
||||
+ os.close(fd)
|
||||
+ except OSError as exc:
|
||||
+ if exc.errno not in (errno.EINVAL, errno.EOPNOTSUPP, errno.ENOSYS):
|
||||
+ raise
|
||||
+ torch.save(payload, tmp)
|
||||
--- a/exllamav3/util/disk_store.py
|
||||
+++ b/exllamav3/util/disk_store.py
|
||||
@@ -10,6 +10,7 @@
|
||||
EXL3_PREFIX_DISK_DIR root directory; empty = off
|
||||
EXL3_PREFIX_DISK_GB byte cap of the whole root, all models and builds together (default 30)
|
||||
EXL3_PREFIX_DISK_FLOOR_GB free space that must remain on the volume after a write (default 60)
|
||||
+ EXL3_PREFIX_DISK_DIRECT write KV segments of at least 64 MiB with Linux O_DIRECT (default off)
|
||||
EXL3_PREFIX_DISK_MB_MIN write budget per minute (default 8000: the QLC write cliff starts near 10 GB/min)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
@@ -50,6 +51,7 @@
|
||||
e = os.environ.get
|
||||
self.cap = int(float(e("EXL3_PREFIX_DISK_GB", "30") if cap_gb is None else cap_gb) * GIB)
|
||||
self.floor = int(float(e("EXL3_PREFIX_DISK_FLOOR_GB", "60") if floor_gb is None else floor_gb) * GIB)
|
||||
+ self.direct_segments = e("EXL3_PREFIX_DISK_DIRECT", "0") == "1"
|
||||
self.rate = float(e("EXL3_PREFIX_DISK_MB_MIN", "8000") if mb_per_min is None else mb_per_min) * (1 << 20) / 60.0
|
||||
self.stats = {"written": 0, "bytes": 0, "skipped_floor": 0, "skipped_cap": 0, "evicted": 0, "errors": 0, "hits": 0}
|
||||
self._q: queue.Queue = queue.Queue()
|
||||
@@ -195,7 +197,11 @@
|
||||
try:
|
||||
if not self.can_stage(min_bytes):
|
||||
continue
|
||||
- torch.save(payload, tmp)
|
||||
+ if self.direct_segments and min_bytes >= 64 << 20:
|
||||
+ from .direct_save import save_segment
|
||||
+ save_segment(payload, tmp)
|
||||
+ else:
|
||||
+ torch.save(payload, tmp)
|
||||
n = os.path.getsize(tmp)
|
||||
free = shutil.disk_usage(self.dir).free
|
||||
if free - n < self.floor: # live check: other writers share the disk
|
||||
--- a/tests/test_disk_store_cpu.py
|
||||
+++ b/tests/test_disk_store_cpu.py
|
||||
@@ -243,3 +243,58 @@
|
||||
finally:
|
||||
resume.set()
|
||||
pd.disk.close()
|
||||
+
|
||||
+
|
||||
+@pytest.mark.parametrize("case", ["supported", "unsupported", "misaligned", "no_space"])
|
||||
+def test_direct_snapshot_round_trip_and_write_errors(tmp_path, monkeypatch, case):
|
||||
+ import errno
|
||||
+ from exllamav3.util import direct_save
|
||||
+
|
||||
+ if not hasattr(os, "O_DIRECT"):
|
||||
+ pytest.skip("Linux direct I/O")
|
||||
+ path = tmp_path / "segment.pt.tmp"
|
||||
+ open_file = os.open
|
||||
+ def open_direct(name, flags, mode=0o777):
|
||||
+ assert flags & os.O_DIRECT
|
||||
+ if case == "unsupported":
|
||||
+ raise OSError(errno.EOPNOTSUPP, "direct I/O unavailable")
|
||||
+ # Build machines may use a filesystem without direct I/O.
|
||||
+ return open_file(name, flags & ~os.O_DIRECT, mode)
|
||||
+ monkeypatch.setattr(os, "open", open_direct)
|
||||
+ original = os.write
|
||||
+ writes = []
|
||||
+ def check_write(fd, data):
|
||||
+ writes.append(len(data))
|
||||
+ if len(writes) == 2 and case != "supported":
|
||||
+ error = errno.EINVAL if case == "misaligned" else errno.ENOSPC
|
||||
+ raise OSError(error, os.strerror(error))
|
||||
+ return original(fd, data)
|
||||
+ monkeypatch.setattr(os, "write", check_write)
|
||||
+ tensor = torch.arange(700013, dtype=torch.int64)
|
||||
+ if case == "no_space":
|
||||
+ with pytest.raises(OSError) as exc:
|
||||
+ direct_save.save_segment({"x": tensor}, path)
|
||||
+ assert exc.value.errno == errno.ENOSPC
|
||||
+ else:
|
||||
+ direct_save.save_segment({"x": tensor}, path)
|
||||
+ restored = torch.load(path, mmap=True, weights_only=False)["x"]
|
||||
+ assert torch.equal(tensor, restored)
|
||||
+ assert not writes if case == "unsupported" else len(writes) >= 2
|
||||
+ assert all(n % 4096 == 0 for n in writes)
|
||||
+
|
||||
+
|
||||
+@pytest.mark.parametrize("enabled,estimate,expected", [(False, 64 << 20, 0), (True, 63 << 20, 0), (True, 64 << 20, 1)])
|
||||
+def test_direct_write_only_for_large_enabled_segments(tmp_path, monkeypatch, enabled, estimate, expected):
|
||||
+ from exllamav3.util import direct_save
|
||||
+
|
||||
+ monkeypatch.setenv("EXL3_PREFIX_DISK_DIRECT", str(int(enabled)))
|
||||
+ writer = Mock(side_effect=torch.save)
|
||||
+ monkeypatch.setattr(direct_save, "save_segment", writer)
|
||||
+ store = mk(tmp_path, cap_gb=1)
|
||||
+ try:
|
||||
+ store.put("seg", torch.arange(32), {}, min_bytes=estimate)
|
||||
+ store.flush()
|
||||
+ assert torch.equal(torch.arange(32), store.read("seg"))
|
||||
+ assert writer.call_count == expected
|
||||
+ finally:
|
||||
+ store.close()
|
||||
Reference in New Issue
Block a user