--- /dev/null +++ b/exllamav3/util/direct_save.py @@ -0,0 +1,76 @@ +"""Serialize a large CPU checkpoint through an aligned, bounded O_DIRECT buffer.""" +import errno +import mmap +import os + +import torch + + +class _DirectWriter: + def __init__(self, fd, buffer): + self.fd = fd + self.buffer = buffer + self.used = 0 + self.size = 0 + self.error = None + + def _send(self, length): + with memoryview(self.buffer) as block: + offset = 0 + while offset < length: + with block[offset:length] as chunk: + written = os.write(self.fd, chunk) + if written <= 0 or written % 4096: + raise OSError(errno.EIO, "invalid short direct write") + offset += written + + def write(self, data): + data = memoryview(data).cast("B") + length = len(data) + if self.error is None: + try: + while data: + n = min(len(data), len(self.buffer) - self.used) + self.buffer[self.used:self.used + n] = data[:n] + self.used += n + data = data[n:] + if self.used == len(self.buffer): + self._send(self.used) + self.used = 0 + self.size += length + except OSError as exc: + # Let PyTorch finish its ZIP callback before propagating the original errno. + self.error = OSError(exc.errno, exc.strerror) + return length + + def flush(self): + # torch.save flushes after its ZIP footer. finish() writes the padded tail. + pass + + def finish(self): + if self.error is not None: + raise self.error + if self.used: + padded = (self.used + 4095) & ~4095 + self.buffer[self.used:padded] = bytes(padded - self.used) + self._send(padded) + os.ftruncate(self.fd, self.size) + + +def save_segment(payload, tmp): + """Write torch's normal ZIP payload, falling back on filesystems without compatible direct I/O.""" + if hasattr(os, "O_DIRECT"): + try: + fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_DIRECT, 0o666) + try: + with mmap.mmap(-1, 2 << 20) as buffer: + writer = _DirectWriter(fd, buffer) + torch.save(payload, writer) + writer.finish() + return + finally: + os.close(fd) + except OSError as exc: + if exc.errno not in (errno.EINVAL, errno.EOPNOTSUPP, errno.ENOSYS): + raise + torch.save(payload, tmp) --- a/exllamav3/util/disk_store.py +++ b/exllamav3/util/disk_store.py @@ -10,6 +10,7 @@ EXL3_PREFIX_DISK_DIR root directory; empty = off EXL3_PREFIX_DISK_GB byte cap of the whole root, all models and builds together (default 30) EXL3_PREFIX_DISK_FLOOR_GB free space that must remain on the volume after a write (default 60) + EXL3_PREFIX_DISK_DIRECT write KV segments of at least 64 MiB with Linux O_DIRECT (default off) EXL3_PREFIX_DISK_MB_MIN write budget per minute (default 8000: the QLC write cliff starts near 10 GB/min) """ from __future__ import annotations @@ -50,6 +51,7 @@ e = os.environ.get self.cap = int(float(e("EXL3_PREFIX_DISK_GB", "30") if cap_gb is None else cap_gb) * GIB) self.floor = int(float(e("EXL3_PREFIX_DISK_FLOOR_GB", "60") if floor_gb is None else floor_gb) * GIB) + self.direct_segments = e("EXL3_PREFIX_DISK_DIRECT", "0") == "1" self.rate = float(e("EXL3_PREFIX_DISK_MB_MIN", "8000") if mb_per_min is None else mb_per_min) * (1 << 20) / 60.0 self.stats = {"written": 0, "bytes": 0, "skipped_floor": 0, "skipped_cap": 0, "evicted": 0, "errors": 0, "hits": 0} self._q: queue.Queue = queue.Queue() @@ -195,7 +197,11 @@ try: if not self.can_stage(min_bytes): continue - torch.save(payload, tmp) + if self.direct_segments and min_bytes >= 64 << 20: + from .direct_save import save_segment + save_segment(payload, tmp) + else: + torch.save(payload, tmp) n = os.path.getsize(tmp) free = shutil.disk_usage(self.dir).free if free - n < self.floor: # live check: other writers share the disk --- a/tests/test_disk_store_cpu.py +++ b/tests/test_disk_store_cpu.py @@ -243,3 +243,58 @@ finally: resume.set() pd.disk.close() + + +@pytest.mark.parametrize("case", ["supported", "unsupported", "misaligned", "no_space"]) +def test_direct_snapshot_round_trip_and_write_errors(tmp_path, monkeypatch, case): + import errno + from exllamav3.util import direct_save + + if not hasattr(os, "O_DIRECT"): + pytest.skip("Linux direct I/O") + path = tmp_path / "segment.pt.tmp" + open_file = os.open + def open_direct(name, flags, mode=0o777): + assert flags & os.O_DIRECT + if case == "unsupported": + raise OSError(errno.EOPNOTSUPP, "direct I/O unavailable") + # Build machines may use a filesystem without direct I/O. + return open_file(name, flags & ~os.O_DIRECT, mode) + monkeypatch.setattr(os, "open", open_direct) + original = os.write + writes = [] + def check_write(fd, data): + writes.append(len(data)) + if len(writes) == 2 and case != "supported": + error = errno.EINVAL if case == "misaligned" else errno.ENOSPC + raise OSError(error, os.strerror(error)) + return original(fd, data) + monkeypatch.setattr(os, "write", check_write) + tensor = torch.arange(700013, dtype=torch.int64) + if case == "no_space": + with pytest.raises(OSError) as exc: + direct_save.save_segment({"x": tensor}, path) + assert exc.value.errno == errno.ENOSPC + else: + direct_save.save_segment({"x": tensor}, path) + restored = torch.load(path, mmap=True, weights_only=False)["x"] + assert torch.equal(tensor, restored) + assert not writes if case == "unsupported" else len(writes) >= 2 + assert all(n % 4096 == 0 for n in writes) + + +@pytest.mark.parametrize("enabled,estimate,expected", [(False, 64 << 20, 0), (True, 63 << 20, 0), (True, 64 << 20, 1)]) +def test_direct_write_only_for_large_enabled_segments(tmp_path, monkeypatch, enabled, estimate, expected): + from exllamav3.util import direct_save + + monkeypatch.setenv("EXL3_PREFIX_DISK_DIRECT", str(int(enabled))) + writer = Mock(side_effect=torch.save) + monkeypatch.setattr(direct_save, "save_segment", writer) + store = mk(tmp_path, cap_gb=1) + try: + store.put("seg", torch.arange(32), {}, min_bytes=estimate) + store.flush() + assert torch.equal(torch.arange(32), store.read("seg")) + assert writer.call_count == expected + finally: + store.close()