perf(kyojin): Skip prefix segments that cannot fit on disk
Bead: nix-overlay-tb3
This commit is contained in:
File diff suppressed because one or more lines are too long
@@ -36,7 +36,7 @@ builtins.mapAttrs
|
|||||||
hipfire-kernels = hipfire.kernels;
|
hipfire-kernels = hipfire.kernels;
|
||||||
xrt-amdxdna = gufo.xrt;
|
xrt-amdxdna = gufo.xrt;
|
||||||
kyojin = pkgs.callPackage ../packages/kyojin/package.nix {
|
kyojin = pkgs.callPackage ../packages/kyojin/package.nix {
|
||||||
kyojin = inputs.kyojin.packages.${system}.kyojin;
|
inherit (inputs.kyojin.packages.${system}) kyojin exllamav3;
|
||||||
jitCheck = inputs.kyojin.checks.${system}.kyojin-hipcc-smoke;
|
jitCheck = inputs.kyojin.checks.${system}.kyojin-hipcc-smoke;
|
||||||
cpuCheck = inputs.kyojin.checks.${system}.kyojin-tests;
|
cpuCheck = inputs.kyojin.checks.${system}.kyojin-tests;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,10 +1,26 @@
|
|||||||
{
|
{
|
||||||
pkgs,
|
pkgs,
|
||||||
kyojin,
|
kyojin,
|
||||||
|
exllamav3,
|
||||||
jitCheck,
|
jitCheck,
|
||||||
cpuCheck,
|
cpuCheck,
|
||||||
}:
|
}:
|
||||||
|
let
|
||||||
|
admissionPatch = ./prefix-disk-admission.patch;
|
||||||
|
engine = exllamav3.overrideAttrs (old: {
|
||||||
|
patches = (old.patches or [ ]) ++ [ admissionPatch ];
|
||||||
|
});
|
||||||
|
# Both the upstream serve wrappers and the CPU check prepend the wheel to PYTHONPATH.
|
||||||
|
withEngine =
|
||||||
|
script:
|
||||||
|
let
|
||||||
|
updated = builtins.replaceStrings [ "${exllamav3}/" ] [ "${engine}/" ] script;
|
||||||
|
in
|
||||||
|
assert updated != script;
|
||||||
|
updated;
|
||||||
|
in
|
||||||
kyojin.overrideAttrs (old: {
|
kyojin.overrideAttrs (old: {
|
||||||
|
installPhase = withEngine old.installPhase;
|
||||||
# Upstream installs tools directly from $src, bypassing the unpacked tree.
|
# Upstream installs tools directly from $src, bypassing the unpacked tree.
|
||||||
src = pkgs.applyPatches {
|
src = pkgs.applyPatches {
|
||||||
src = old.src;
|
src = old.src;
|
||||||
@@ -19,7 +35,10 @@ kyojin.overrideAttrs (old: {
|
|||||||
category = "AI Inference";
|
category = "AI Inference";
|
||||||
tests = (old.passthru.tests or { }) // {
|
tests = (old.passthru.tests or { }) // {
|
||||||
jit = jitCheck;
|
jit = jitCheck;
|
||||||
cpu = cpuCheck;
|
cpu = cpuCheck.overrideAttrs (check: {
|
||||||
|
patches = (check.patches or [ ]) ++ [ admissionPatch ];
|
||||||
|
checkPhase = withEngine check.checkPhase;
|
||||||
|
});
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
postFixup = ''
|
postFixup = ''
|
||||||
|
|||||||
@@ -0,0 +1,141 @@
|
|||||||
|
--- a/exllamav3/generator/prefix_disk.py
|
||||||
|
+++ b/exllamav3/generator/prefix_disk.py
|
||||||
|
@@ -189,15 +189,20 @@
|
||||||
|
wrote = 0
|
||||||
|
if new:
|
||||||
|
src = [pages[i] for i in new]
|
||||||
|
+ tensors = sl._tensors()
|
||||||
|
+ # index_select materializes one row per missing page in each cache tensor.
|
||||||
|
+ min_bytes = len(src) * sum(t[0].nbytes for t in tensors) + sum(p.sequence.nbytes for p in src)
|
||||||
|
+ if not self.disk.can_stage(min_bytes):
|
||||||
|
+ return {"saved": 0}
|
||||||
|
torch.cuda.synchronize()
|
||||||
|
tens = []
|
||||||
|
- for t in sl._tensors():
|
||||||
|
+ for t in tensors:
|
||||||
|
idx = torch.tensor([p.page_index for p in src], dtype=torch.long, device=t.device)
|
||||||
|
tens.append(t.index_select(0, idx).cpu())
|
||||||
|
hs = [hashes[i] for i in new]
|
||||||
|
name = f"s_{_hx(hs[0])[:20]}_{len(hs)}"
|
||||||
|
self.disk.put(name, {"hashes": hs, "sequences": [p.sequence.cpu().clone() for p in src], "tensors": tens},
|
||||||
|
- {"ns": self.ns, "k": "seg", "hashes": hs})
|
||||||
|
+ {"ns": self.ns, "k": "seg", "hashes": hs}, min_bytes=min_bytes)
|
||||||
|
for pos, h in enumerate(hs):
|
||||||
|
self.pages[h] = (name, pos)
|
||||||
|
wrote += len(hs)
|
||||||
|
--- a/exllamav3/util/disk_store.py
|
||||||
|
+++ b/exllamav3/util/disk_store.py
|
||||||
|
@@ -149,10 +149,22 @@
|
||||||
|
|
||||||
|
# ---- async writes
|
||||||
|
|
||||||
|
- def put(self, name: str, payload, idx) -> None:
|
||||||
|
+ def can_stage(self, min_bytes: int) -> bool:
|
||||||
|
+ """Check the writer's cap and free-space policy before allocating a segment's host copy."""
|
||||||
|
+ if min_bytes > self.cap:
|
||||||
|
+ self.stats["skipped_cap"] += 1
|
||||||
|
+ return False
|
||||||
|
+ # The writer leaves another payload's worth of headroom above the floor
|
||||||
|
+ # while the serialized temporary already occupies space on the volume.
|
||||||
|
+ if shutil.disk_usage(self.dir).free - 2 * min_bytes < self.floor:
|
||||||
|
+ self.stats["skipped_floor"] += 1
|
||||||
|
+ return False
|
||||||
|
+ return True
|
||||||
|
+
|
||||||
|
+ def put(self, name: str, payload, idx, *, min_bytes: int = 0) -> None:
|
||||||
|
"""Queue an entry: payload and idx are torch.save-able and must not change afterwards (host copies)."""
|
||||||
|
if not self._closed:
|
||||||
|
- self._q.put((name, payload, idx))
|
||||||
|
+ self._q.put((name, payload, idx, min_bytes))
|
||||||
|
|
||||||
|
def flush(self) -> None:
|
||||||
|
self._q.join()
|
||||||
|
@@ -172,12 +184,14 @@
|
||||||
|
try:
|
||||||
|
if item is None:
|
||||||
|
return
|
||||||
|
- name, payload, idx = item
|
||||||
|
+ name, payload, idx, min_bytes = item
|
||||||
|
if os.path.exists(self._p(name, ".idx")):
|
||||||
|
self.touch(name)
|
||||||
|
continue
|
||||||
|
tmp = self._p(name, ".pt.tmp")
|
||||||
|
try:
|
||||||
|
+ if not self.can_stage(min_bytes):
|
||||||
|
+ continue
|
||||||
|
torch.save(payload, tmp)
|
||||||
|
n = os.path.getsize(tmp)
|
||||||
|
free = shutil.disk_usage(self.dir).free
|
||||||
|
--- a/tests/test_disk_store_cpu.py
|
||||||
|
+++ b/tests/test_disk_store_cpu.py
|
||||||
|
@@ -1,6 +1,10 @@
|
||||||
|
"""exllamav3/util/disk_store.py: atomic async writes, LRU byte cap, free-space floor, torn entries. CPU only."""
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
+from types import SimpleNamespace
|
||||||
|
+from unittest.mock import Mock
|
||||||
|
+
|
||||||
|
+import pytest
|
||||||
|
|
||||||
|
import torch
|
||||||
|
|
||||||
|
@@ -89,3 +93,59 @@
|
||||||
|
new.flush()
|
||||||
|
time.sleep(0.02)
|
||||||
|
assert old.entries() == ["b"] and new.entries() == ["c", "d"] # the old build's oldest entry went first
|
||||||
|
+
|
||||||
|
+
|
||||||
|
+@pytest.mark.parametrize("limit", ["cap", "floor", "fits"])
|
||||||
|
+def test_page_admission_before_device_copy(tmp_path, monkeypatch, limit):
|
||||||
|
+ from exllamav3.constants import PAGE_SIZE
|
||||||
|
+ from exllamav3.generator.prefix_disk import PagedPrefixDisk
|
||||||
|
+
|
||||||
|
+ ids = torch.arange(PAGE_SIZE).view(1, -1)
|
||||||
|
+ pool = torch.arange(8 * 8192, dtype=torch.int32).view(8, -1)
|
||||||
|
+ page = SimpleNamespace(page_index=7, sequence=ids)
|
||||||
|
+ rc = Mock()
|
||||||
|
+ rc.get.return_value = {"position": PAGE_SIZE}
|
||||||
|
+ slots = SimpleNamespace(generator=SimpleNamespace(recurrent_cache=rc), _tensors=lambda: [pool],
|
||||||
|
+ _resident_chain=lambda _: ([b"page"], [page]), _find_tail=lambda *_: None)
|
||||||
|
+ pd = PagedPrefixDisk(slots, root=str(tmp_path))
|
||||||
|
+ page_bytes = pool[7].nbytes + ids.nbytes
|
||||||
|
+ pd.disk.rate = 1e12
|
||||||
|
+ pd.disk.cap = page_bytes - 1 if limit == "cap" else page_bytes * 2
|
||||||
|
+ pd.disk.floor = 1 << 20
|
||||||
|
+ free = pd.disk.floor + (2 * page_bytes - 1 if limit == "floor" else 10 * page_bytes)
|
||||||
|
+ monkeypatch.setattr(ds.shutil, "disk_usage", lambda _: SimpleNamespace(free=free))
|
||||||
|
+ sync = Mock()
|
||||||
|
+ monkeypatch.setattr(torch.cuda, "synchronize", sync)
|
||||||
|
+ try:
|
||||||
|
+ saved = pd.save(ids)["saved"]
|
||||||
|
+ pd.disk.flush()
|
||||||
|
+ if limit == "fits":
|
||||||
|
+ assert saved == 1 and sync.called
|
||||||
|
+ name, pos = pd.pages[b"page"]
|
||||||
|
+ assert torch.equal(pd.disk.read(name)["tensors"][0][pos], pool[7])
|
||||||
|
+ else:
|
||||||
|
+ sync.assert_not_called()
|
||||||
|
+ assert saved == 0 and not pd.pages and not pd.disk.entries()
|
||||||
|
+ assert pd.disk.stats["skipped_" + limit] == 1
|
||||||
|
+ finally:
|
||||||
|
+ pd.disk.close()
|
||||||
|
+
|
||||||
|
+
|
||||||
|
+def test_worker_checks_space_before_writing_queued_segment(tmp_path, monkeypatch):
|
||||||
|
+ s = mk(tmp_path, cap_gb=1)
|
||||||
|
+ s.floor = 4096
|
||||||
|
+ payload = blob(1)
|
||||||
|
+ n = payload["x"].nbytes
|
||||||
|
+ free = Mock(return_value=SimpleNamespace(free=s.floor + 3 * n))
|
||||||
|
+ monkeypatch.setattr(ds.shutil, "disk_usage", free)
|
||||||
|
+ save = Mock(wraps=torch.save)
|
||||||
|
+ monkeypatch.setattr(torch, "save", save)
|
||||||
|
+ try:
|
||||||
|
+ assert s.can_stage(n)
|
||||||
|
+ free.return_value = SimpleNamespace(free=s.floor + n)
|
||||||
|
+ s.put("seg", payload, {}, min_bytes=n)
|
||||||
|
+ s.flush()
|
||||||
|
+ save.assert_not_called()
|
||||||
|
+ assert not s.entries() and s.stats["skipped_floor"] == 1
|
||||||
|
+ finally:
|
||||||
|
+ s.close()
|
||||||
Reference in New Issue
Block a user