24 lines
1.2 KiB
Diff
24 lines
1.2 KiB
Diff
--- a/tools/qwen/serve.py
|
|
+++ b/tools/qwen/serve.py
|
|
@@ -826,6 +826,20 @@
|
|
# first token 81.3 / 84.1 s instead of 81.2 / 81.5 s
|
|
os.environ.setdefault("EXL3_CIL_DPRIO_CHUNK_S", "0.4")
|
|
from exllamav3 import Cache, Config, Generator, Job, Model, Tokenizer
|
|
+ if torch.version.hip and os.environ.get("KYOJIN_HIP_BLOCKING_SYNC") == "1":
|
|
+ import ctypes
|
|
+ from exllamav3.util.hip_lib import load_hip_runtime
|
|
+
|
|
+ hip = load_hip_runtime()
|
|
+ flags = ctypes.c_uint()
|
|
+ status = hip.hipGetDeviceFlags(ctypes.byref(flags))
|
|
+ if status:
|
|
+ raise RuntimeError(f"hipGetDeviceFlags failed with status {status}")
|
|
+ # Replace only hipDeviceScheduleMask (7) with hipDeviceScheduleBlockingSync (4).
|
|
+ status = hip.hipSetDeviceFlags(ctypes.c_uint((flags.value & ~7) | 4))
|
|
+ if status:
|
|
+ raise RuntimeError(f"hipSetDeviceFlags failed with status {status}")
|
|
+ print("qserve: HIP blocking host synchronization enabled", flush=True)
|
|
torch.set_grad_enabled(False)
|
|
self.torch, self.Generator, self.Job = torch, Generator, Job
|
|
self.sessions = sessions
|