Bead: nix-overlay-j2g
This commit is contained in:
@@ -0,0 +1,23 @@
|
||||
--- a/tools/qwen/serve.py
|
||||
+++ b/tools/qwen/serve.py
|
||||
@@ -826,6 +826,20 @@
|
||||
# first token 81.3 / 84.1 s instead of 81.2 / 81.5 s
|
||||
os.environ.setdefault("EXL3_CIL_DPRIO_CHUNK_S", "0.4")
|
||||
from exllamav3 import Cache, Config, Generator, Job, Model, Tokenizer
|
||||
+ if torch.version.hip and os.environ.get("KYOJIN_HIP_BLOCKING_SYNC") == "1":
|
||||
+ import ctypes
|
||||
+ from exllamav3.util.hip_lib import load_hip_runtime
|
||||
+
|
||||
+ hip = load_hip_runtime()
|
||||
+ flags = ctypes.c_uint()
|
||||
+ status = hip.hipGetDeviceFlags(ctypes.byref(flags))
|
||||
+ if status:
|
||||
+ raise RuntimeError(f"hipGetDeviceFlags failed with status {status}")
|
||||
+ # Replace only hipDeviceScheduleMask (7) with hipDeviceScheduleBlockingSync (4).
|
||||
+ status = hip.hipSetDeviceFlags(ctypes.c_uint((flags.value & ~7) | 4))
|
||||
+ if status:
|
||||
+ raise RuntimeError(f"hipSetDeviceFlags failed with status {status}")
|
||||
+ print("qserve: HIP blocking host synchronization enabled", flush=True)
|
||||
torch.set_grad_enabled(False)
|
||||
self.torch, self.Generator, self.Job = torch, Generator, Job
|
||||
self.sessions = sessions
|
||||
Reference in New Issue
Block a user