--- a/tools/qwen/serve.py +++ b/tools/qwen/serve.py @@ -826,6 +826,20 @@ # first token 81.3 / 84.1 s instead of 81.2 / 81.5 s os.environ.setdefault("EXL3_CIL_DPRIO_CHUNK_S", "0.4") from exllamav3 import Cache, Config, Generator, Job, Model, Tokenizer + if torch.version.hip and os.environ.get("KYOJIN_HIP_BLOCKING_SYNC") == "1": + import ctypes + from exllamav3.util.hip_lib import load_hip_runtime + + hip = load_hip_runtime() + flags = ctypes.c_uint() + status = hip.hipGetDeviceFlags(ctypes.byref(flags)) + if status: + raise RuntimeError(f"hipGetDeviceFlags failed with status {status}") + # Replace only hipDeviceScheduleMask (7) with hipDeviceScheduleBlockingSync (4). + status = hip.hipSetDeviceFlags(ctypes.c_uint((flags.value & ~7) | 4)) + if status: + raise RuntimeError(f"hipSetDeviceFlags failed with status {status}") + print("qserve: HIP blocking host synchronization enabled", flush=True) torch.set_grad_enabled(False) self.torch, self.Generator, self.Job = torch, Generator, Job self.sessions = sessions