From d7ed5b56f5e27074c41cb82ea6fbb678d3e3bf4e Mon Sep 17 00:00:00 2001 From: Richard Wang Date: Sat, 10 Oct 2026 19:01:01 -0700 Subject: [PATCH] [diffusion] fix: keep the overlay probe from stalling cached startup (#43590) --- .../runtime/utils/hf_diffusers_utils.py | 27 ++++++++++- .../test/unit/test_hf_diffusers_utils.py | 45 +++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/python/sglang/multimodal_gen/runtime/utils/hf_diffusers_utils.py b/python/sglang/multimodal_gen/runtime/utils/hf_diffusers_utils.py index 3bbabdc533..10f67979cc 100644 --- a/python/sglang/multimodal_gen/runtime/utils/hf_diffusers_utils.py +++ b/python/sglang/multimodal_gen/runtime/utils/hf_diffusers_utils.py @@ -782,6 +782,29 @@ def _resolve_remote_repo_model_index_path( raise +def _download_overlay_manifest(repo_id: str, filename: str) -> str: + """``hf_hub_download`` for the overlay manifest that most repos lack. + + For a file the repo never had, ``hf_hub_download`` waits out each Hub rate + limit before giving up, which can stall the startup of a fully cached + checkpoint for minutes. Overlay repos hold only metadata, so a complete + cached snapshot without the manifest is a regular checkpoint, which + maybe_download_model serves from the cache, and needs no Hub request. + Offline, ``hf_hub_download`` already answers from the cache without waiting. + """ + from huggingface_hub import is_offline_mode, try_to_load_from_cache + + if not envs.SGLANG_USE_MODELSCOPE.get() and not is_offline_mode(): + cached_index = try_to_load_from_cache(repo_id, "model_index.json") + if isinstance(cached_index, str): + snapshot = os.path.dirname(cached_index) + if not os.path.exists( + os.path.join(snapshot, filename) + ) and _verify_diffusers_model_complete(snapshot): + raise EntryNotFoundError(f"{filename} not found in {repo_id}") + return hf_hub_download(repo_id=repo_id, filename=filename) + + def maybe_download_model_index(model_name_or_path: str) -> dict[str, Any]: """ Download and extract just the model_index.json for a Hugging Face model. @@ -795,7 +818,7 @@ def maybe_download_model_index(model_name_or_path: str) -> dict[str, Any]: overlay_config = maybe_load_overlay_model_index( model_name_or_path, snapshot_download_fn=snapshot_download, - hf_hub_download_fn=hf_hub_download, + hf_hub_download_fn=_download_overlay_manifest, ) if overlay_config is not None: return overlay_config @@ -892,7 +915,7 @@ def maybe_download_model( download=download, allow_patterns=allow_patterns, snapshot_download_fn=snapshot_download, - hf_hub_download_fn=hf_hub_download, + hf_hub_download_fn=_download_overlay_manifest, verify_diffusers_model_complete_fn=_verify_diffusers_model_complete, base_model_download_fn=maybe_download_model, ) diff --git a/python/sglang/multimodal_gen/test/unit/test_hf_diffusers_utils.py b/python/sglang/multimodal_gen/test/unit/test_hf_diffusers_utils.py index 577c6ff15e..46f130f914 100644 --- a/python/sglang/multimodal_gen/test/unit/test_hf_diffusers_utils.py +++ b/python/sglang/multimodal_gen/test/unit/test_hf_diffusers_utils.py @@ -2,6 +2,7 @@ import json from types import SimpleNamespace from unittest.mock import patch +import huggingface_hub import modelscope import pytest from huggingface_hub.errors import LocalEntryNotFoundError @@ -352,6 +353,50 @@ def test_force_diffusers_model_stub_keeps_its_existing_path( assert calls == ["probe", "download", "download"] +@pytest.mark.parametrize( + "cached, expected_downloads", + [ + ("complete", ["model_index.json"]), + ("manifest", ["_overlay/overlay_manifest.json"]), + ("metadata", ["_overlay/overlay_manifest.json", "model_index.json"]), + ], +) +def test_complete_cached_checkpoint_skips_overlay_probe( + monkeypatch, tmp_path, cached, expected_downloads +): + """hf_hub_download waits out Hub rate limits before reporting a file the repo + never had, so a complete cached checkpoint must not probe its overlay manifest. + A cached manifest still marks an overlay repo, and a metadata-only snapshot, + which an overlay repo leaves, still asks the Hub.""" + repo_cache = tmp_path / "models--org--repo" + snapshot = repo_cache / "snapshots" / ("a" * 40) + (snapshot / "_overlay").mkdir(parents=True) + (repo_cache / "refs").mkdir() + (repo_cache / "refs" / "main").write_text("a" * 40) + _write_model_index(snapshot) + if cached != "metadata": + _populate_components(snapshot, ("scheduler", "tokenizer"), weights=False) + _populate_components(snapshot, ("text_encoder", "transformer", "vae")) + if cached == "manifest": + (snapshot / "_overlay" / "overlay_manifest.json").write_text( + '{"source_model_id": "org/source"}' + ) + downloads = [] + + def fake_hf_hub_download(repo_id, filename, **kwargs): + downloads.append(filename) + return str(snapshot / filename) + + monkeypatch.setattr(huggingface_hub.constants, "HF_HUB_CACHE", str(tmp_path)) + monkeypatch.setattr(huggingface_hub.constants, "HF_HUB_OFFLINE", False) + monkeypatch.setattr(hf_diffusers_utils, "hf_hub_download", fake_hf_hub_download) + + config = hf_diffusers_utils.maybe_download_model_index("org/repo") + + assert config["_class_name"] == "LongLive2Pipeline" + assert downloads == expected_downloads + + def test_modelscope_file_download_preserves_local_dir(monkeypatch, tmp_path): calls = []