[CI] Avoid Hugging Face Hub API calls when the local model cache is complete (#43229)

This commit is contained in:
Liangsheng Yin
2026-10-08 16:27:31 -07:00
committed by GitHub
parent baee5f3b62
commit 2e64740442
6 changed files with 171 additions and 49 deletions
@@ -412,8 +412,11 @@ def _validate_sharded_model(
corrupted_files.append(f)
if group_info["suffix"] == "safetensors":
# Qwen3.5 names shards `model.safetensors-00001-of-00014.safetensors`,
# so the prefix may already end with the suffix.
index_stem = group_info["prefix"].removesuffix(".safetensors")
index_file = os.path.join(
snapshot_dir, f"{group_info['prefix']}.safetensors.index.json"
snapshot_dir, f"{index_stem}.safetensors.index.json"
)
if not os.path.exists(index_file):
return (
+17 -6
View File
@@ -36,6 +36,7 @@ import numpy as np
import safetensors.torch
import torch
from huggingface_hub import HfFileSystem, hf_hub_download, snapshot_download
from huggingface_hub.errors import HfHubHTTPError
from pydantic import BaseModel, ConfigDict, ValidationInfo, model_validator
from tqdm.auto import tqdm
@@ -521,11 +522,10 @@ def _find_local_hf_snapshot_dir_unlocked(
),
)
rev_to_use = revision
if not rev_to_use:
ref_main = os.path.join(repo_folder, "refs", "main")
if os.path.isfile(ref_main):
with open(ref_main) as f:
rev_to_use = f.read().strip()
ref_path = os.path.join(repo_folder, "refs", revision or "main")
if os.path.isfile(ref_path):
with open(ref_path) as f:
rev_to_use = f.read().strip()
if rev_to_use:
rev_dir = os.path.join(repo_folder, "snapshots", rev_to_use)
if os.path.isdir(rev_dir):
@@ -664,7 +664,18 @@ def download_weights_from_hf(
if not huggingface_hub.constants.HF_HUB_OFFLINE:
# Before we download we look at what is available:
fs = HfFileSystem()
file_list = fs.ls(model_name_or_path, detail=False, revision=revision)
try:
file_list = fs.ls(model_name_or_path, detail=False, revision=revision)
except HfHubHTTPError as e:
# Fail open (e.g. a 429 rate limit): pick the format from the local
# snapshot; snapshot_download below re-raises errors it cannot recover.
logger.warning(
"Listing %s on the Hub failed, using the local snapshot: %s",
model_name_or_path,
e,
)
local_dir = find_local_repo_dir(model_name_or_path, revision)
file_list = os.listdir(local_dir) if local_dir else []
# depending on what is available we download different things
for pattern in allow_patterns:
+6 -6
View File
@@ -4256,12 +4256,12 @@ def find_local_repo_dir(repo_id: str, revision: Optional[str] = None) -> Optiona
hf.constants.REPO_ID_SEPARATOR.join(["models", *repo_id.split("/")]),
)
# Get revision from main ref if not specified
if not revision:
ref_path = os.path.join(cache_path, "refs", "main")
if os.path.isfile(ref_path):
with open(ref_path) as f:
revision = f.read().strip()
# A branch or tag name (default "main") maps to a commit through refs/;
# snapshots/ is keyed by commit only.
ref_path = os.path.join(cache_path, "refs", revision or "main")
if os.path.isfile(ref_path):
with open(ref_path) as f:
revision = f.read().strip()
# List files from revision directory
if revision:
+124 -24
View File
@@ -401,18 +401,7 @@ def _try_enable_offline_mode_if_cache_complete(
) -> Optional[str]:
"""Set HF_HUB_OFFLINE=1 in `env` if the model cache validates; return the
per-run marker path, or None if offline mode was not enabled.
Markers are per-run and not shared across runners. A marker is only a hint;
the current launch's requirements (e.g. hf_quant_config.json) are revalidated.
"""
from sglang.srt.model_loader.ci_weight_validation import (
_get_per_run_marker_path,
_read_per_run_marker,
_write_per_run_marker,
validate_cache_lightweight,
)
from sglang.srt.utils import find_local_repo_dir
other_args = other_args or []
# Skip offline mode for LoRA scenarios (dynamic adapter loading may need online access)
@@ -428,6 +417,40 @@ def _try_enable_offline_mode_if_cache_complete(
)
return None
# Detect before the marker check so the current launch's requirements are known.
requires_hf_quant_config = False
for i, arg in enumerate(other_args):
if arg == "--quantization" and i + 1 < len(other_args):
quant_value = other_args[i + 1].lower()
if quant_value in ["modelopt_fp4", "modelopt_fp8", "modelopt"]:
requires_hf_quant_config = True
break
marker_path = _validate_cache_for_offline(
model_name_or_path, requires_hf_quant_config
)
if marker_path is not None:
env["HF_HUB_OFFLINE"] = "1"
return marker_path
def _validate_cache_for_offline(
model_name_or_path: str, requires_hf_quant_config: bool
) -> Optional[str]:
"""Return the per-run marker path if the model's local cache is complete
enough to load offline, else None.
Markers are per-run and not shared across runners. A marker is only a hint;
the current launch's requirements (e.g. hf_quant_config.json) are revalidated.
"""
from sglang.srt.model_loader.ci_weight_validation import (
_get_per_run_marker_path,
_read_per_run_marker,
_write_per_run_marker,
validate_cache_lightweight,
)
from sglang.srt.utils import find_local_repo_dir
# Skip if already a local path
if os.path.isdir(model_name_or_path):
return None
@@ -440,15 +463,6 @@ def _try_enable_offline_mode_if_cache_complete(
except Exception:
return None
# Detect before the marker check so the current launch's requirements are known.
requires_hf_quant_config = False
for i, arg in enumerate(other_args):
if arg == "--quantization" and i + 1 < len(other_args):
quant_value = other_args[i + 1].lower()
if quant_value in ["modelopt_fp4", "modelopt_fp8", "modelopt"]:
requires_hf_quant_config = True
break
# Check per-run marker (fast hint - snapshot validated earlier in this run)
per_run_marker = _read_per_run_marker(snapshot_dir)
if per_run_marker is not None:
@@ -466,7 +480,6 @@ def _try_enable_offline_mode_if_cache_complete(
return None
# Marker exists and current validation passed
env["HF_HUB_OFFLINE"] = "1"
marker_path = _get_per_run_marker_path(snapshot_dir)
print(
f"CI_OFFLINE: Per-run marker found and current validation passed "
@@ -487,9 +500,6 @@ def _try_enable_offline_mode_if_cache_complete(
)
return None
# Validation passed - enable offline mode and write per-run marker
env["HF_HUB_OFFLINE"] = "1"
# Write per-run marker for subsequent tests in this run
_write_per_run_marker(snapshot_dir, model_name_or_path)
@@ -506,6 +516,94 @@ def _try_enable_offline_mode_if_cache_complete(
return marker_path
@contextlib.contextmanager
def _hf_offline_if_cache_complete(engine_kwargs: dict):
"""Run an in-process Engine startup with HF_HUB_OFFLINE=1 when every Hub
repo it loads validates in the local cache.
A scheduler that fails at startup kills the test process, so there is no
online retry as in popen_launch_server; any doubt keeps the startup online.
"""
import huggingface_hub.constants
if "server_args" in engine_kwargs:
server_args = engine_kwargs["server_args"]
model_path = server_args.model_path
tokenizer_path = server_args.tokenizer_path
draft_path = server_args.speculative_draft_model_path
quantization = server_args.quantization
uses_lora = server_args.enable_lora or bool(server_args.lora_paths)
else:
model_path = engine_kwargs.get("model_path")
tokenizer_path = engine_kwargs.get("tokenizer_path")
draft_path = engine_kwargs.get("speculative_draft_model_path")
quantization = engine_kwargs.get("quantization")
uses_lora = bool(
engine_kwargs.get("enable_lora") or engine_kwargs.get("lora_paths")
)
# Adapters can be loaded by repo id after startup, from an offline scheduler.
if uses_lora or os.environ.get("HF_HUB_OFFLINE") == "1":
yield
return
requires_hf_quant_config = quantization in [
"modelopt_fp4",
"modelopt_fp8",
"modelopt",
]
repo_ids = {
path
for path in (model_path, tokenizer_path, draft_path)
if path is not None and not os.path.isdir(path)
}
if not repo_ids or not all(
_validate_cache_for_offline(
repo_id, requires_hf_quant_config and repo_id == model_path
)
for repo_id in repo_ids
):
yield
return
# Spawned children read the env at import; this process already imported
# huggingface_hub, which reads it once into a module constant.
prev_env = os.environ.get("HF_HUB_OFFLINE")
prev_constant = huggingface_hub.constants.HF_HUB_OFFLINE
os.environ["HF_HUB_OFFLINE"] = "1"
huggingface_hub.constants.HF_HUB_OFFLINE = True
try:
yield
finally:
if prev_env is None:
os.environ.pop("HF_HUB_OFFLINE", None)
else:
os.environ["HF_HUB_OFFLINE"] = prev_env
huggingface_hub.constants.HF_HUB_OFFLINE = prev_constant
def _install_ci_engine_offline_hook():
"""In CI, wrap Engine.__init__ once per process with _hf_offline_if_cache_complete."""
try:
from sglang.srt.entrypoints.engine import Engine
except ImportError as e:
# A test that never builds an Engine must not fail on this.
print(f"CI_OFFLINE: Engine import failed, skip the offline hook: {e}")
return
orig_init = Engine.__init__
if getattr(orig_init, "_ci_offline_wrapped", False):
return
@wraps(orig_init)
def init(self, **kwargs):
with _hf_offline_if_cache_complete(kwargs):
orig_init(self, **kwargs)
init._ci_offline_wrapped = True
Engine.__init__ = init
def _create_clean_subprocess_env(env: dict) -> dict:
"""Create a clean subprocess environment without internal CI keys."""
child_env = env.copy()
@@ -2301,6 +2399,8 @@ class CustomTestCase(unittest.TestCase):
def safe_setUpClass(klass):
try:
_wait_for_gpu_idle_in_ci()
if is_in_ci():
_install_ci_engine_offline_hook()
orig_func(klass)
except Exception:
# Best-effort cleanup; suppress teardown errors so the