--- a/tools/qwen/serve.py +++ b/tools/qwen/serve.py @@ -663,7 +663,7 @@ CUDA_SERVE_ENV = (("EXL3_NC2_ASYNC_STASH", "1"), ("EXL3_STASH_SIDE", "1"), ("EXL3_PF_EMB_UPLOAD", "1"), ("EXL3_CUDA_GRAPH", "1"), ("EXL3_DRAFT_VOCAB", "98304"), ("QWSPEC_THF", "0.3"), ("QWSPEC_THV", "0.1"), ("QWSPEC_THFD", "0.6"), ("QWSPEC_THVD", "0.4")) HC_GS_SIDECAR = "hc_gs_sidecar.safetensors" -# The sidecar of the published pack. A file with another hash (wrong pack, damaged download) is not used: plain group rounding, one log line. +# Default checksum for packs without a local calibration. HC_GS_SIDECAR_SHA256 = "473e7547adaad190788634556ce6b32614bdb280ccedadbed8789d39b59dcc8f" @@ -676,22 +676,41 @@ return h.hexdigest() -def configure_hc_sidecar(model_path: str) -> str: - """Point EXL3_GR_GS_SIDECAR at /hc_gs_sidecar.safetensors when group scales are on, the variable is unset and - the file exists. Call before importing exllamav3 (the engine reads it at import). Returns the log line.""" +def _configure_hc_sidecar(model_path: str) -> tuple[str, str | None]: + """Select the mixer codes before engine import. Keep the digest for saved KV state.""" + plain = "hc mixer: group scales, plain rounding" if os.environ.get("EXL3_GR_GS", "0") == "0": - return "hc mixer: per-row int8 scales (EXL3_GR_GS=0)" + return "hc mixer: per-row int8 scales (EXL3_GR_GS=0)", None env = os.environ.get("EXL3_GR_GS_SIDECAR") if env: - return f"hc mixer: group scales, codes from {env} (EXL3_GR_GS_SIDECAR)" + try: + digest = _sha256_file(env) + except OSError: + digest = None # The engine already falls back to plain codes for unreadable overrides. + return f"hc mixer: group scales, codes from {env} (EXL3_GR_GS_SIDECAR)", digest + f = Path(model_path) / HC_GS_SIDECAR - if f.is_file(): - got = _sha256_file(f) - if got != HC_GS_SIDECAR_SHA256: - return f"hc mixer: group scales, plain rounding ({f} has sha256 {got[:12]}, expected {HC_GS_SIDECAR_SHA256[:12]}: sidecar not used)" - os.environ["EXL3_GR_GS_SIDECAR"] = str(f) - return f"hc mixer: group scales, codes from {f}" - return f"hc mixer: group scales, plain rounding (no {HC_GS_SIDECAR} in the model directory)" + if not f.is_file(): + return f"{plain} (no {HC_GS_SIDECAR} in the model directory)", None + checksum = f.with_suffix(f.suffix + ".sha256") + try: + expected = checksum.read_text(encoding="ascii").strip().lower() + except FileNotFoundError: + expected = HC_GS_SIDECAR_SHA256 + except (OSError, UnicodeError): + return f"{plain} (unreadable {checksum})", None + if not re.fullmatch(r"[0-9a-f]{64}", expected): + return f"{plain} (invalid SHA-256 in {checksum})", None + got = _sha256_file(f) + if got != expected: + return f"{plain} ({f} has sha256 {got[:12]}, expected {expected[:12]}: sidecar not used)", None + os.environ["EXL3_GR_GS_SIDECAR"] = str(f) + return f"hc mixer: group scales, codes from {f}", got + + +def configure_hc_sidecar(model_path: str) -> str: + """Select mixer codes and return the diagnostic used by the standalone tools.""" + return _configure_hc_sidecar(model_path)[0] DRAFT2S_IN_PACK = "draft2s.st" # safetensors layout, but not *.safetensors so the weight loader never indexes it @@ -774,7 +793,8 @@ continue os.environ.setdefault(k, v) check_latched_env() - print("qserve:", configure_hc_sidecar(model_path), flush=True) + hc_message, self.hc_sidecar_digest = _configure_hc_sidecar(model_path) + print("qserve:", hc_message, flush=True) print("qserve:", configure_draft2s(model_path), flush=True) if on_cuda: for k, v in CUDA_SERVE_ENV: @@ -2373,12 +2393,13 @@ env = {k: v for k, v in sorted(os.environ.items()) if k.startswith(("EXL3_", "MPW"))} print(f"qserve: effective env {env}", flush=True) from exllamav3.generator.slot_store import SlotStore + cache_model_id = args.model_id if not engine.hc_sidecar_digest else f"{args.model_id}:hc:{engine.hc_sidecar_digest}" engine.slot_store = SlotStore(engine.generator, [c for c in (engine.cache, engine.draft_cache) if c is not None], - args.slot_save_path, args.model_id) + args.slot_save_path, cache_model_id) from exllamav3.util.disk_store import env_dir if env_dir(): from exllamav3.generator.prefix_disk import PagedPrefixDisk, weights_identity - engine.prefix_disk = PagedPrefixDisk(engine.slot_store, model_id=args.model_id, extra=weights_identity(args.model)) + engine.prefix_disk = PagedPrefixDisk(engine.slot_store, model_id=cache_model_id, extra=weights_identity(args.model)) print(f"qserve: prefix disk {engine.prefix_disk.disk.dir} ({len(engine.prefix_disk.pages)} pages, " f"{len(engine.prefix_disk.stashes)} checkpoints, cap {engine.prefix_disk.disk.cap >> 30} GiB)", flush=True) startup_health.stage("warm-up") --- a/tools/qwen/test_serve.py +++ b/tools/qwen/test_serve.py @@ -930,6 +930,31 @@ self.assertEqual(serve._partial_tag("abc", [""]), 0) +class HCSidecarTests(unittest.TestCase): + def test_discovery_and_integrity(self): + import hashlib + import tempfile + with tempfile.TemporaryDirectory() as folder, unittest.mock.patch.dict(os.environ, {"EXL3_GR_GS": "1"}, clear=True): + file = Path(folder) / serve.HC_GS_SIDECAR + file.write_bytes(b"calibrated codes") + digest = hashlib.sha256(file.read_bytes()).hexdigest() + checksum = file.with_suffix(".safetensors.sha256") + for value in (None, "bad", "0" * 64): + if value is not None: + checksum.write_text(value) + self.assertIsNone(serve._configure_hc_sidecar(folder)[1]) + self.assertNotIn("EXL3_GR_GS_SIDECAR", os.environ) + checksum.write_text(digest + "\n") + self.assertEqual(serve._configure_hc_sidecar(folder)[1], digest) + self.assertEqual(os.environ.pop("EXL3_GR_GS_SIDECAR"), str(file)) + checksum.unlink() + with unittest.mock.patch.object(serve, "HC_GS_SIDECAR_SHA256", digest): + self.assertIn(str(file), serve.configure_hc_sidecar(folder)) + self.assertEqual(serve._configure_hc_sidecar(folder)[1], digest) # explicit path + os.environ["EXL3_GR_GS"] = "0" + self.assertIsNone(serve._configure_hc_sidecar(folder)[1]) + + class ServeEnvDefaultsTest(unittest.TestCase): def test_prefill_set_is_default_and_overridable(self): d = dict(serve.SERVE_ENV)