Files
millerson-overlay.nix/packages/kyojin/hip-blocking-sync.patch
T
alex 41a50393f9
CI / check (push) Has been cancelled
perf(kyojin): allow blocking HIP waits in the Qwen server
Bead: nix-overlay-j2g
2026-11-18 21:16:42 +03:00

24 lines
1.2 KiB
Diff

--- a/tools/qwen/serve.py
+++ b/tools/qwen/serve.py
@@ -826,6 +826,20 @@
# first token 81.3 / 84.1 s instead of 81.2 / 81.5 s
os.environ.setdefault("EXL3_CIL_DPRIO_CHUNK_S", "0.4")
from exllamav3 import Cache, Config, Generator, Job, Model, Tokenizer
+ if torch.version.hip and os.environ.get("KYOJIN_HIP_BLOCKING_SYNC") == "1":
+ import ctypes
+ from exllamav3.util.hip_lib import load_hip_runtime
+
+ hip = load_hip_runtime()
+ flags = ctypes.c_uint()
+ status = hip.hipGetDeviceFlags(ctypes.byref(flags))
+ if status:
+ raise RuntimeError(f"hipGetDeviceFlags failed with status {status}")
+ # Replace only hipDeviceScheduleMask (7) with hipDeviceScheduleBlockingSync (4).
+ status = hip.hipSetDeviceFlags(ctypes.c_uint((flags.value & ~7) | 4))
+ if status:
+ raise RuntimeError(f"hipSetDeviceFlags failed with status {status}")
+ print("qserve: HIP blocking host synchronization enabled", flush=True)
torch.set_grad_enabled(False)
self.torch, self.Generator, self.Job = torch, Generator, Job
self.sessions = sessions