Files
millerson-overlay.nix/packages/kyojin/hc-sidecar-discovery.patch
T
alex 303eaa945e
CI / check (push) Has been cancelled
fix(kyojin): load calibrated HC sidecars from the model directory
Bead: nix-overlay-kzh
2026-11-19 14:58:32 +03:00

126 lines
6.7 KiB
Diff

--- a/tools/qwen/serve.py
+++ b/tools/qwen/serve.py
@@ -663,7 +663,7 @@
CUDA_SERVE_ENV = (("EXL3_NC2_ASYNC_STASH", "1"), ("EXL3_STASH_SIDE", "1"), ("EXL3_PF_EMB_UPLOAD", "1"), ("EXL3_CUDA_GRAPH", "1"), ("EXL3_DRAFT_VOCAB", "98304"), ("QWSPEC_THF", "0.3"), ("QWSPEC_THV", "0.1"), ("QWSPEC_THFD", "0.6"), ("QWSPEC_THVD", "0.4"))
HC_GS_SIDECAR = "hc_gs_sidecar.safetensors"
-# The sidecar of the published pack. A file with another hash (wrong pack, damaged download) is not used: plain group rounding, one log line.
+# Default checksum for packs without a local calibration.
HC_GS_SIDECAR_SHA256 = "473e7547adaad190788634556ce6b32614bdb280ccedadbed8789d39b59dcc8f"
@@ -676,22 +676,41 @@
return h.hexdigest()
-def configure_hc_sidecar(model_path: str) -> str:
- """Point EXL3_GR_GS_SIDECAR at <model_path>/hc_gs_sidecar.safetensors when group scales are on, the variable is unset and
- the file exists. Call before importing exllamav3 (the engine reads it at import). Returns the log line."""
+def _configure_hc_sidecar(model_path: str) -> tuple[str, str | None]:
+ """Select the mixer codes before engine import. Keep the digest for saved KV state."""
+ plain = "hc mixer: group scales, plain rounding"
if os.environ.get("EXL3_GR_GS", "0") == "0":
- return "hc mixer: per-row int8 scales (EXL3_GR_GS=0)"
+ return "hc mixer: per-row int8 scales (EXL3_GR_GS=0)", None
env = os.environ.get("EXL3_GR_GS_SIDECAR")
if env:
- return f"hc mixer: group scales, codes from {env} (EXL3_GR_GS_SIDECAR)"
+ try:
+ digest = _sha256_file(env)
+ except OSError:
+ digest = None # The engine already falls back to plain codes for unreadable overrides.
+ return f"hc mixer: group scales, codes from {env} (EXL3_GR_GS_SIDECAR)", digest
+
f = Path(model_path) / HC_GS_SIDECAR
- if f.is_file():
- got = _sha256_file(f)
- if got != HC_GS_SIDECAR_SHA256:
- return f"hc mixer: group scales, plain rounding ({f} has sha256 {got[:12]}, expected {HC_GS_SIDECAR_SHA256[:12]}: sidecar not used)"
- os.environ["EXL3_GR_GS_SIDECAR"] = str(f)
- return f"hc mixer: group scales, codes from {f}"
- return f"hc mixer: group scales, plain rounding (no {HC_GS_SIDECAR} in the model directory)"
+ if not f.is_file():
+ return f"{plain} (no {HC_GS_SIDECAR} in the model directory)", None
+ checksum = f.with_suffix(f.suffix + ".sha256")
+ try:
+ expected = checksum.read_text(encoding="ascii").strip().lower()
+ except FileNotFoundError:
+ expected = HC_GS_SIDECAR_SHA256
+ except (OSError, UnicodeError):
+ return f"{plain} (unreadable {checksum})", None
+ if not re.fullmatch(r"[0-9a-f]{64}", expected):
+ return f"{plain} (invalid SHA-256 in {checksum})", None
+ got = _sha256_file(f)
+ if got != expected:
+ return f"{plain} ({f} has sha256 {got[:12]}, expected {expected[:12]}: sidecar not used)", None
+ os.environ["EXL3_GR_GS_SIDECAR"] = str(f)
+ return f"hc mixer: group scales, codes from {f}", got
+
+
+def configure_hc_sidecar(model_path: str) -> str:
+ """Select mixer codes and return the diagnostic used by the standalone tools."""
+ return _configure_hc_sidecar(model_path)[0]
DRAFT2S_IN_PACK = "draft2s.st" # safetensors layout, but not *.safetensors so the weight loader never indexes it
@@ -774,7 +793,8 @@
continue
os.environ.setdefault(k, v)
check_latched_env()
- print("qserve:", configure_hc_sidecar(model_path), flush=True)
+ hc_message, self.hc_sidecar_digest = _configure_hc_sidecar(model_path)
+ print("qserve:", hc_message, flush=True)
print("qserve:", configure_draft2s(model_path), flush=True)
if on_cuda:
for k, v in CUDA_SERVE_ENV:
@@ -2373,12 +2393,13 @@
env = {k: v for k, v in sorted(os.environ.items()) if k.startswith(("EXL3_", "MPW"))}
print(f"qserve: effective env {env}", flush=True)
from exllamav3.generator.slot_store import SlotStore
+ cache_model_id = args.model_id if not engine.hc_sidecar_digest else f"{args.model_id}:hc:{engine.hc_sidecar_digest}"
engine.slot_store = SlotStore(engine.generator, [c for c in (engine.cache, engine.draft_cache) if c is not None],
- args.slot_save_path, args.model_id)
+ args.slot_save_path, cache_model_id)
from exllamav3.util.disk_store import env_dir
if env_dir():
from exllamav3.generator.prefix_disk import PagedPrefixDisk, weights_identity
- engine.prefix_disk = PagedPrefixDisk(engine.slot_store, model_id=args.model_id, extra=weights_identity(args.model))
+ engine.prefix_disk = PagedPrefixDisk(engine.slot_store, model_id=cache_model_id, extra=weights_identity(args.model))
print(f"qserve: prefix disk {engine.prefix_disk.disk.dir} ({len(engine.prefix_disk.pages)} pages, "
f"{len(engine.prefix_disk.stashes)} checkpoints, cap {engine.prefix_disk.disk.cap >> 30} GiB)", flush=True)
startup_health.stage("warm-up")
--- a/tools/qwen/test_serve.py
+++ b/tools/qwen/test_serve.py
@@ -930,6 +930,31 @@
self.assertEqual(serve._partial_tag("abc", ["</think>"]), 0)
+class HCSidecarTests(unittest.TestCase):
+ def test_discovery_and_integrity(self):
+ import hashlib
+ import tempfile
+ with tempfile.TemporaryDirectory() as folder, unittest.mock.patch.dict(os.environ, {"EXL3_GR_GS": "1"}, clear=True):
+ file = Path(folder) / serve.HC_GS_SIDECAR
+ file.write_bytes(b"calibrated codes")
+ digest = hashlib.sha256(file.read_bytes()).hexdigest()
+ checksum = file.with_suffix(".safetensors.sha256")
+ for value in (None, "bad", "0" * 64):
+ if value is not None:
+ checksum.write_text(value)
+ self.assertIsNone(serve._configure_hc_sidecar(folder)[1])
+ self.assertNotIn("EXL3_GR_GS_SIDECAR", os.environ)
+ checksum.write_text(digest + "\n")
+ self.assertEqual(serve._configure_hc_sidecar(folder)[1], digest)
+ self.assertEqual(os.environ.pop("EXL3_GR_GS_SIDECAR"), str(file))
+ checksum.unlink()
+ with unittest.mock.patch.object(serve, "HC_GS_SIDECAR_SHA256", digest):
+ self.assertIn(str(file), serve.configure_hc_sidecar(folder))
+ self.assertEqual(serve._configure_hc_sidecar(folder)[1], digest) # explicit path
+ os.environ["EXL3_GR_GS"] = "0"
+ self.assertIsNone(serve._configure_hc_sidecar(folder)[1])
+
+
class ServeEnvDefaultsTest(unittest.TestCase):
def test_prefill_set_is_default_and_overridable(self):
d = dict(serve.SERVE_ENV)