126 lines
6.7 KiB
Diff
126 lines
6.7 KiB
Diff
--- a/tools/qwen/serve.py
|
|
+++ b/tools/qwen/serve.py
|
|
@@ -663,7 +663,7 @@
|
|
CUDA_SERVE_ENV = (("EXL3_NC2_ASYNC_STASH", "1"), ("EXL3_STASH_SIDE", "1"), ("EXL3_PF_EMB_UPLOAD", "1"), ("EXL3_CUDA_GRAPH", "1"), ("EXL3_DRAFT_VOCAB", "98304"), ("QWSPEC_THF", "0.3"), ("QWSPEC_THV", "0.1"), ("QWSPEC_THFD", "0.6"), ("QWSPEC_THVD", "0.4"))
|
|
|
|
HC_GS_SIDECAR = "hc_gs_sidecar.safetensors"
|
|
-# The sidecar of the published pack. A file with another hash (wrong pack, damaged download) is not used: plain group rounding, one log line.
|
|
+# Default checksum for packs without a local calibration.
|
|
HC_GS_SIDECAR_SHA256 = "473e7547adaad190788634556ce6b32614bdb280ccedadbed8789d39b59dcc8f"
|
|
|
|
|
|
@@ -676,22 +676,41 @@
|
|
return h.hexdigest()
|
|
|
|
|
|
-def configure_hc_sidecar(model_path: str) -> str:
|
|
- """Point EXL3_GR_GS_SIDECAR at <model_path>/hc_gs_sidecar.safetensors when group scales are on, the variable is unset and
|
|
- the file exists. Call before importing exllamav3 (the engine reads it at import). Returns the log line."""
|
|
+def _configure_hc_sidecar(model_path: str) -> tuple[str, str | None]:
|
|
+ """Select the mixer codes before engine import. Keep the digest for saved KV state."""
|
|
+ plain = "hc mixer: group scales, plain rounding"
|
|
if os.environ.get("EXL3_GR_GS", "0") == "0":
|
|
- return "hc mixer: per-row int8 scales (EXL3_GR_GS=0)"
|
|
+ return "hc mixer: per-row int8 scales (EXL3_GR_GS=0)", None
|
|
env = os.environ.get("EXL3_GR_GS_SIDECAR")
|
|
if env:
|
|
- return f"hc mixer: group scales, codes from {env} (EXL3_GR_GS_SIDECAR)"
|
|
+ try:
|
|
+ digest = _sha256_file(env)
|
|
+ except OSError:
|
|
+ digest = None # The engine already falls back to plain codes for unreadable overrides.
|
|
+ return f"hc mixer: group scales, codes from {env} (EXL3_GR_GS_SIDECAR)", digest
|
|
+
|
|
f = Path(model_path) / HC_GS_SIDECAR
|
|
- if f.is_file():
|
|
- got = _sha256_file(f)
|
|
- if got != HC_GS_SIDECAR_SHA256:
|
|
- return f"hc mixer: group scales, plain rounding ({f} has sha256 {got[:12]}, expected {HC_GS_SIDECAR_SHA256[:12]}: sidecar not used)"
|
|
- os.environ["EXL3_GR_GS_SIDECAR"] = str(f)
|
|
- return f"hc mixer: group scales, codes from {f}"
|
|
- return f"hc mixer: group scales, plain rounding (no {HC_GS_SIDECAR} in the model directory)"
|
|
+ if not f.is_file():
|
|
+ return f"{plain} (no {HC_GS_SIDECAR} in the model directory)", None
|
|
+ checksum = f.with_suffix(f.suffix + ".sha256")
|
|
+ try:
|
|
+ expected = checksum.read_text(encoding="ascii").strip().lower()
|
|
+ except FileNotFoundError:
|
|
+ expected = HC_GS_SIDECAR_SHA256
|
|
+ except (OSError, UnicodeError):
|
|
+ return f"{plain} (unreadable {checksum})", None
|
|
+ if not re.fullmatch(r"[0-9a-f]{64}", expected):
|
|
+ return f"{plain} (invalid SHA-256 in {checksum})", None
|
|
+ got = _sha256_file(f)
|
|
+ if got != expected:
|
|
+ return f"{plain} ({f} has sha256 {got[:12]}, expected {expected[:12]}: sidecar not used)", None
|
|
+ os.environ["EXL3_GR_GS_SIDECAR"] = str(f)
|
|
+ return f"hc mixer: group scales, codes from {f}", got
|
|
+
|
|
+
|
|
+def configure_hc_sidecar(model_path: str) -> str:
|
|
+ """Select mixer codes and return the diagnostic used by the standalone tools."""
|
|
+ return _configure_hc_sidecar(model_path)[0]
|
|
|
|
|
|
DRAFT2S_IN_PACK = "draft2s.st" # safetensors layout, but not *.safetensors so the weight loader never indexes it
|
|
@@ -774,7 +793,8 @@
|
|
continue
|
|
os.environ.setdefault(k, v)
|
|
check_latched_env()
|
|
- print("qserve:", configure_hc_sidecar(model_path), flush=True)
|
|
+ hc_message, self.hc_sidecar_digest = _configure_hc_sidecar(model_path)
|
|
+ print("qserve:", hc_message, flush=True)
|
|
print("qserve:", configure_draft2s(model_path), flush=True)
|
|
if on_cuda:
|
|
for k, v in CUDA_SERVE_ENV:
|
|
@@ -2373,12 +2393,13 @@
|
|
env = {k: v for k, v in sorted(os.environ.items()) if k.startswith(("EXL3_", "MPW"))}
|
|
print(f"qserve: effective env {env}", flush=True)
|
|
from exllamav3.generator.slot_store import SlotStore
|
|
+ cache_model_id = args.model_id if not engine.hc_sidecar_digest else f"{args.model_id}:hc:{engine.hc_sidecar_digest}"
|
|
engine.slot_store = SlotStore(engine.generator, [c for c in (engine.cache, engine.draft_cache) if c is not None],
|
|
- args.slot_save_path, args.model_id)
|
|
+ args.slot_save_path, cache_model_id)
|
|
from exllamav3.util.disk_store import env_dir
|
|
if env_dir():
|
|
from exllamav3.generator.prefix_disk import PagedPrefixDisk, weights_identity
|
|
- engine.prefix_disk = PagedPrefixDisk(engine.slot_store, model_id=args.model_id, extra=weights_identity(args.model))
|
|
+ engine.prefix_disk = PagedPrefixDisk(engine.slot_store, model_id=cache_model_id, extra=weights_identity(args.model))
|
|
print(f"qserve: prefix disk {engine.prefix_disk.disk.dir} ({len(engine.prefix_disk.pages)} pages, "
|
|
f"{len(engine.prefix_disk.stashes)} checkpoints, cap {engine.prefix_disk.disk.cap >> 30} GiB)", flush=True)
|
|
startup_health.stage("warm-up")
|
|
--- a/tools/qwen/test_serve.py
|
|
+++ b/tools/qwen/test_serve.py
|
|
@@ -930,6 +930,31 @@
|
|
self.assertEqual(serve._partial_tag("abc", ["</think>"]), 0)
|
|
|
|
|
|
+class HCSidecarTests(unittest.TestCase):
|
|
+ def test_discovery_and_integrity(self):
|
|
+ import hashlib
|
|
+ import tempfile
|
|
+ with tempfile.TemporaryDirectory() as folder, unittest.mock.patch.dict(os.environ, {"EXL3_GR_GS": "1"}, clear=True):
|
|
+ file = Path(folder) / serve.HC_GS_SIDECAR
|
|
+ file.write_bytes(b"calibrated codes")
|
|
+ digest = hashlib.sha256(file.read_bytes()).hexdigest()
|
|
+ checksum = file.with_suffix(".safetensors.sha256")
|
|
+ for value in (None, "bad", "0" * 64):
|
|
+ if value is not None:
|
|
+ checksum.write_text(value)
|
|
+ self.assertIsNone(serve._configure_hc_sidecar(folder)[1])
|
|
+ self.assertNotIn("EXL3_GR_GS_SIDECAR", os.environ)
|
|
+ checksum.write_text(digest + "\n")
|
|
+ self.assertEqual(serve._configure_hc_sidecar(folder)[1], digest)
|
|
+ self.assertEqual(os.environ.pop("EXL3_GR_GS_SIDECAR"), str(file))
|
|
+ checksum.unlink()
|
|
+ with unittest.mock.patch.object(serve, "HC_GS_SIDECAR_SHA256", digest):
|
|
+ self.assertIn(str(file), serve.configure_hc_sidecar(folder))
|
|
+ self.assertEqual(serve._configure_hc_sidecar(folder)[1], digest) # explicit path
|
|
+ os.environ["EXL3_GR_GS"] = "0"
|
|
+ self.assertIsNone(serve._configure_hc_sidecar(folder)[1])
|
|
+
|
|
+
|
|
class ServeEnvDefaultsTest(unittest.TestCase):
|
|
def test_prefill_set_is_default_and_overridable(self):
|
|
d = dict(serve.SERVE_ENV)
|