perf(kyojin): allow blocking HIP waits in the Qwen server
CI / check (push) Has been cancelled

Bead: nix-overlay-j2g
This commit is contained in:
2026-11-18 21:16:42 +03:00
parent 2b8eddf7d2
commit 41a50393f9
3 changed files with 28 additions and 1 deletions
+23
View File
@@ -0,0 +1,23 @@
--- a/tools/qwen/serve.py
+++ b/tools/qwen/serve.py
@@ -826,6 +826,20 @@
# first token 81.3 / 84.1 s instead of 81.2 / 81.5 s
os.environ.setdefault("EXL3_CIL_DPRIO_CHUNK_S", "0.4")
from exllamav3 import Cache, Config, Generator, Job, Model, Tokenizer
+ if torch.version.hip and os.environ.get("KYOJIN_HIP_BLOCKING_SYNC") == "1":
+ import ctypes
+ from exllamav3.util.hip_lib import load_hip_runtime
+
+ hip = load_hip_runtime()
+ flags = ctypes.c_uint()
+ status = hip.hipGetDeviceFlags(ctypes.byref(flags))
+ if status:
+ raise RuntimeError(f"hipGetDeviceFlags failed with status {status}")
+ # Replace only hipDeviceScheduleMask (7) with hipDeviceScheduleBlockingSync (4).
+ status = hip.hipSetDeviceFlags(ctypes.c_uint((flags.value & ~7) | 4))
+ if status:
+ raise RuntimeError(f"hipSetDeviceFlags failed with status {status}")
+ print("qserve: HIP blocking host synchronization enabled", flush=True)
torch.set_grad_enabled(False)
self.torch, self.Generator, self.Job = torch, Generator, Job
self.sessions = sessions