minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[CI] Speed up dependency install: dual-ABI Rust ext cache and prevalidation pruning (#33619)

Browse Source
This commit is contained in:
Liangsheng Yin
2026-08-04 20:33:48 -07:00
committed by GitHub
parent 6c05aaae7e
commit 1033cae8d5
11 changed files with 155 additions and 1371 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/model_loader/ci_weight_validation.py
-908
View File
@@ -31,92 +31,6 @@ from sglang.srt.utils import log_info_on_rank0
logger = logging.getLogger(__name__)
# Validation marker version - increment when validation logic changes
# v2: Added trust_remote_code module validation (modeling_*.py must exist in snapshot)
# v3: Added remote file existence checks for hf_quant_config.json
# v5: Invalidate all previous markers to force fresh validation
VALIDATION_MARKER_VERSION = "5"
def _remote_file_exists(
repo_id: str, filename: str, revision: Optional[str], allow_remote_check: bool
) -> Optional[bool]:
"""
Check if a file exists on Hugging Face Hub for a specific revision.
Args:
repo_id: Repository ID (e.g., "meta-llama/Llama-2-7b-hf")
filename: File name to check (e.g., "hf_quant_config.json")
revision: Git revision (commit hash, branch, or tag). None means default branch.
allow_remote_check: Whether remote checks are allowed (e.g., CI validation phase)
Returns:
True if file exists on hub, False if it doesn't exist, None if we cannot determine
(network error or remote check not allowed - be conservative and assume incomplete)
"""
if not allow_remote_check:
logger.debug(
"Remote check disabled for %s/%s, returning None (unknown)",
repo_id,
filename,
)
return None
try:
from huggingface_hub import HfApi
api = HfApi()
exists = api.file_exists(repo_id=repo_id, filename=filename, revision=revision)
logger.debug(
"Remote file check: %s/%s (revision=%s) exists=%s",
repo_id,
filename,
revision or "default",
exists,
)
return exists
except Exception as e:
# Network errors, auth issues, repo not found, etc.
# Return None (unknown) - caller will treat as optional
logger.debug(
"Failed to check remote file existence for %s/%s (revision=%s): %s. "
"Will treat as optional.",
repo_id,
filename,
revision or "default",
e,
)
return None
def _get_validation_marker_path(snapshot_dir: str) -> Optional[str]:
"""
Get the path to validation marker file for a snapshot.
Marker is stored in /tmp to avoid permission issues with HF cache directory.
Marker key is sha256(snapshot_dir) to avoid any collisions regardless of
model_name_or_path format.
Args:
snapshot_dir: Path to snapshot directory
Returns:
Path to marker file or None if snapshot_dir is invalid
"""
if not snapshot_dir or not os.path.isdir(snapshot_dir):
return None
# Normalize path to avoid marker misses due to trailing slashes or symlinks
# realpath resolves symlinks, rstrip removes trailing slashes
normalized_dir = os.path.realpath(snapshot_dir).rstrip("/")
# Use sha256 of normalized snapshot_dir path as unique key
# This avoids any collision issues with repo naming or snapshot hash reuse
dir_hash = hashlib.sha256(normalized_dir.encode("utf-8")).hexdigest()[:12]
# Store in /tmp with directory hash
return f"/tmp/sglang_hf_validation_{dir_hash}.json"
def _get_per_run_marker_dir() -> str:
"""
@@ -248,828 +162,6 @@ def _write_per_run_marker(
pass
def _remove_per_run_marker(snapshot_dir: str) -> None:
"""
Remove per-run validation marker for a snapshot.
Args:
snapshot_dir: Path to snapshot directory
"""
marker_path = _get_per_run_marker_path(snapshot_dir)
if marker_path and os.path.exists(marker_path):
try:
os.remove(marker_path)
logger.debug("Removed per-run marker: %s", marker_path)
except Exception as e:
logger.warning("Failed to remove per-run marker %s: %s", marker_path, e)
def _read_validation_marker(snapshot_dir: str) -> Optional[dict]:
"""
Read validation marker for a snapshot.
Args:
snapshot_dir: Path to snapshot directory
Returns:
Marker dict with keys: version, validated_at, validation_passed
None if marker doesn't exist or is invalid or validation_passed is not True
"""
marker_path = _get_validation_marker_path(snapshot_dir)
if not marker_path:
return None
if not os.path.exists(marker_path):
return None
try:
with open(marker_path, "r", encoding="utf-8") as f:
marker = json.load(f)
# Validate marker structure
if not isinstance(marker, dict):
return None
required_keys = ["version", "validated_at", "validation_passed"]
if not all(key in marker for key in required_keys):
return None
# Check version match
if marker["version"] != VALIDATION_MARKER_VERSION:
logger.debug(
"Validation marker version mismatch: %s != %s, will re-validate",
marker["version"],
VALIDATION_MARKER_VERSION,
)
return None
# Explicitly check validation_passed is True (defensive check)
# Even though we only write markers on success, this guards against
# manual edits or future code changes
if marker.get("validation_passed") is not True:
logger.debug(
"Validation marker has validation_passed=%s, treating as invalid",
marker.get("validation_passed"),
)
return None
return marker
except (json.JSONDecodeError, OSError) as e:
logger.debug("Failed to read validation marker at %s: %s", marker_path, e)
return None
def _write_validation_marker(snapshot_dir: str, passed: bool) -> None:
"""
Write validation marker for a snapshot (atomic write).
IMPORTANT: We only cache successful validations. Failed validations are NOT
cached to allow retry after files are downloaded.
Args:
snapshot_dir: Path to snapshot directory
passed: Whether validation passed
"""
if not passed:
# Don't cache failures - allow retry on next launch
return
marker_path = _get_validation_marker_path(snapshot_dir)
if not marker_path:
logger.debug("Cannot write marker: invalid snapshot_dir")
return
from datetime import datetime
marker = {
"version": VALIDATION_MARKER_VERSION,
"validated_at": datetime.utcnow().isoformat() + "Z",
"validation_passed": passed,
}
try:
# Atomic write: write to temp file then os.replace
marker_dir = os.path.dirname(marker_path)
os.makedirs(marker_dir, exist_ok=True)
with tempfile.NamedTemporaryFile(
mode="w",
encoding="utf-8",
dir=marker_dir,
delete=False,
suffix=".tmp",
) as f:
temp_path = f.name
json.dump(marker, f, indent=2)
# Atomic replace (overwrites existing file if any)
os.replace(temp_path, marker_path)
logger.debug("Wrote validation marker to %s (passed=%s)", marker_path, passed)
except Exception as e:
logger.warning("Failed to write validation marker to %s: %s", marker_path, e)
# Clean up temp file if it exists
try:
if "temp_path" in locals() and os.path.exists(temp_path):
os.remove(temp_path)
except Exception:
pass
def _validate_json_file(file_path: str, file_name: str) -> bool:
"""
Validate that a JSON file exists, is non-empty, and can be parsed.
Args:
file_path: Path to the JSON file
file_name: Name of the file (for logging)
Returns:
True if the file is valid, False otherwise
"""
if not os.path.exists(file_path):
logger.debug("CI cache validation: %s not found at %s", file_name, file_path)
return False
if not os.path.isfile(file_path):
logger.warning(
"CI cache validation: %s is not a file: %s", file_name, file_path
)
return False
# Check if file is non-empty
try:
file_size = os.path.getsize(file_path)
if file_size == 0:
logger.warning("CI cache validation: %s is empty: %s", file_name, file_path)
return False
except OSError as e:
logger.warning("CI cache validation: Cannot get size of %s: %s", file_name, e)
return False
# Try to parse JSON
try:
with open(file_path, "r", encoding="utf-8") as f:
json.load(f)
return True
except json.JSONDecodeError as e:
logger.warning(
"CI cache validation: %s is not valid JSON: %s - %s",
file_name,
file_path,
e,
)
return False
except Exception as e:
logger.warning(
"CI cache validation: Failed to read %s: %s - %s",
file_name,
file_path,
e,
)
return False
def _validate_config_and_tokenizer_files(
snapshot_dir: str,
model_id: Optional[str] = None,
revision: Optional[str] = None,
allow_remote_check: bool = False,
) -> Tuple[bool, List[str]]:
"""
Validate that critical config and tokenizer files exist and are valid.
This checks for:
- config.json (required)
- tokenizer_config.json (required)
- generation_config.json (optional but validated if present)
- hf_quant_config.json (conditionally required based on Hub) - for FP4/FP8/ModelOpt
- quantize_config.json / quant_config.json (optional but validated if present) - for AWQ/GPTQ
- params.json (optional but validated if present) - for Mistral native format
- preprocessor_config.json (optional but validated if present) - for vision models
- trust_remote_code dynamic modules (required if auto_map present in config.json)
- At least one tokenizer file: tokenizer.json, tokenizer.model, or tiktoken.model
Args:
snapshot_dir: Path to the model snapshot directory
model_id: Model repository ID (e.g., "meta-llama/Llama-2-7b-hf"), used for remote checks
revision: Git revision (commit hash), used for remote checks
allow_remote_check: Whether to check Hub for file existence to determine requirements
Returns:
Tuple of (is_valid, missing_files)
- is_valid: True if all required files are present and valid
- missing_files: List of missing or invalid file names
"""
missing_files = []
# Check required config files
required_files = [
"config.json",
"tokenizer_config.json",
]
for file_name in required_files:
file_path = os.path.join(snapshot_dir, file_name)
if not _validate_json_file(file_path, file_name):
missing_files.append(file_name)
# Check optional generation_config.json (validate if exists)
generation_config_path = os.path.join(snapshot_dir, "generation_config.json")
if os.path.exists(generation_config_path):
if not _validate_json_file(generation_config_path, "generation_config.json"):
missing_files.append("generation_config.json (exists but invalid)")
# Check hf_quant_config.json with remote existence check
# This file is needed for quantized models (FP4/FP8/ModelOpt)
# Example: nvidia/Llama-3.1-8B-Instruct-FP8, nvidia/DeepSeek-V3-0324-FP4
hf_quant_config_path = os.path.join(snapshot_dir, "hf_quant_config.json")
local_hf_quant_exists = os.path.exists(hf_quant_config_path)
# Check if file exists on Hub for this revision
# Only do remote check if model_id looks like a HF repo_id (org/model format)
# Skip if it's a local path (absolute path or doesn't contain '/')
remote_hf_quant_exists = None
is_hf_repo = (
model_id is not None
and "/" in model_id
and not os.path.isabs(model_id)
and not model_id.startswith("/")
)
if is_hf_repo and allow_remote_check:
remote_hf_quant_exists = _remote_file_exists(
repo_id=model_id,
filename="hf_quant_config.json",
revision=revision,
allow_remote_check=allow_remote_check,
)
# Apply conditional requirement logic
if remote_hf_quant_exists is True:
# Hub has this file for this revision - it's REQUIRED
if not local_hf_quant_exists:
missing_files.append(
f"hf_quant_config.json (required: exists on Hub for revision {revision or 'default'} but missing locally)"
)
log_info_on_rank0(
logger,
f"Hub has hf_quant_config.json for {model_id} revision {revision or 'default'} "
f"but local snapshot missing it. Cache incomplete, will not write marker.",
)
elif not _validate_json_file(hf_quant_config_path, "hf_quant_config.json"):
missing_files.append("hf_quant_config.json (exists but invalid)")
elif remote_hf_quant_exists is False:
# Hub doesn't have this file - it's OPTIONAL
# Only validate if it happens to exist locally
if local_hf_quant_exists:
if not _validate_json_file(hf_quant_config_path, "hf_quant_config.json"):
missing_files.append("hf_quant_config.json (exists but invalid)")
else:
# remote_hf_quant_exists is None - unknown (network error or remote check disabled)
# Treat as OPTIONAL - only enforce when we can positively confirm Hub has it
if local_hf_quant_exists:
# Local file exists - validate it
if not _validate_json_file(hf_quant_config_path, "hf_quant_config.json"):
missing_files.append("hf_quant_config.json (exists but invalid)")
# If local file missing and remote unknown, just log it - don't block marker
logger.debug(
"Cannot verify hf_quant_config.json on Hub for %s (revision=%s), "
"treating as optional since remote status unknown",
model_id or "unknown",
revision or "default",
)
# Check optional quantize_config.json / quant_config.json (validate if exists)
# These files are needed for AWQ/GPTQ/AutoRound quantized models
# Example: TheBloke/Llama-2-7B-AWQ, casperhansen/vicuna-7b-v1.5-awq
for quant_config_name in ["quantize_config.json", "quant_config.json"]:
quant_config_path = os.path.join(snapshot_dir, quant_config_name)
if os.path.exists(quant_config_path):
if not _validate_json_file(quant_config_path, quant_config_name):
missing_files.append(f"{quant_config_name} (exists but invalid)")
break # Only need to check one of these
# Check optional params.json (validate if exists)
# This file is needed for Mistral native format models
# Example: mistralai/Mistral-7B-v0.1
params_json_path = os.path.join(snapshot_dir, "params.json")
if os.path.exists(params_json_path):
if not _validate_json_file(params_json_path, "params.json"):
missing_files.append("params.json (exists but invalid)")
# Check optional preprocessor_config.json (validate if exists)
# This file is needed for vision/multimodal models
# Example: llava-hf/llava-1.5-7b-hf, Qwen/Qwen2-VL-7B-Instruct
preprocessor_config_path = os.path.join(snapshot_dir, "preprocessor_config.json")
if os.path.exists(preprocessor_config_path):
if not _validate_json_file(
preprocessor_config_path, "preprocessor_config.json"
):
missing_files.append("preprocessor_config.json (exists but invalid)")
# Check for trust_remote_code dynamic module files if needed
# When auto_map exists in config.json, the model requires custom Python files
# These files must be present for offline mode to work
config_path = os.path.join(snapshot_dir, "config.json")
if os.path.exists(config_path):
try:
with open(config_path, "r", encoding="utf-8") as f:
config = json.load(f)
auto_map = config.get("auto_map", {})
if auto_map and isinstance(auto_map, dict):
# Extract Python module files from auto_map
# auto_map format: {"AutoConfig": "configuration_xxx.ConfigClass", ...}
# We need to check if the .py files exist
custom_files = set()
for key, value in auto_map.items():
if isinstance(value, str) and "." in value:
# Extract module name (e.g., "configuration_xxx" from "configuration_xxx.ConfigClass")
module_name = value.split(".")[0]
custom_files.add(f"{module_name}.py")
# Check if all custom files exist in snapshot directory
# NOTE: Some models (like nvidia/DeepSeek-V3-0324-FP4) have auto_map
# but don't include modeling_*.py in their repo, relying on transformers
# to fetch it from the base model. We MUST mark these as missing to
# prevent offline mode, which would fail to load the dynamic modules.
for custom_file in custom_files:
custom_file_path = os.path.join(snapshot_dir, custom_file)
if not os.path.exists(custom_file_path):
missing_files.append(
f"{custom_file} (required for trust_remote_code)"
)
logger.debug(
f"Custom module file not in snapshot: {custom_file} for {snapshot_dir}"
)
elif not os.path.isfile(custom_file_path):
missing_files.append(f"{custom_file} (exists but not a file)")
except (json.JSONDecodeError, OSError, KeyError) as e:
# If we can't read config.json, it will be caught by earlier validation
logger.debug("Failed to check auto_map in config.json: %s", e)
# Check for at least one tokenizer file
tokenizer_files = [
"tokenizer.json",
"tokenizer.model",
"tiktoken.model",
]
tokenizer_found = False
for tokenizer_file in tokenizer_files:
tokenizer_path = os.path.join(snapshot_dir, tokenizer_file)
if os.path.exists(tokenizer_path) and os.path.isfile(tokenizer_path):
# For tokenizer.json, validate it's proper JSON
if tokenizer_file == "tokenizer.json":
if _validate_json_file(tokenizer_path, tokenizer_file):
tokenizer_found = True
break
else:
# For .model files, just check they're non-empty
try:
if os.path.getsize(tokenizer_path) > 0:
tokenizer_found = True
break
except OSError:
pass
if not tokenizer_found:
missing_files.append("tokenizer file")
is_valid = len(missing_files) == 0
return is_valid, missing_files
def ci_validate_cache_and_enable_offline_if_complete(
snapshot_dir: str,
weight_files: List[str],
model_name_or_path: str,
) -> bool:
"""
Validate local cache completeness (config/tokenizer/weights) and determine
if offline mode can be safely enabled.
This function uses a snapshot-level marker to cache validation results,
so the heavy validation is done at most once per snapshot per runner.
This function checks:
1. Validation marker (if exists and version matches, skip re-validation)
2. Config and tokenizer files (config.json, tokenizer_config.json, etc.)
3. Weight files (safetensors shards, index files, corruption check)
If all are present and valid, it returns True to signal that offline
mode can be safely enabled.
IMPORTANT: This should be called BEFORE any HF operations, and if it
returns True, the caller should set HF_HUB_OFFLINE=1 for the server
subprocess env ONLY (not global environment).
Args:
snapshot_dir: Path to the model snapshot directory
weight_files: List of weight file paths to validate (must be non-empty)
model_name_or_path: Model identifier for logging
Returns:
True if cache is complete and offline mode can be enabled, False otherwise
"""
# Guard: weight_files is required
if not weight_files:
log_info_on_rank0(
logger,
f"CI_OFFLINE: No weight files provided, skip offline, keep online allowed - {model_name_or_path}",
)
return False
# Fast-path: Check if validation marker exists and is valid
# We only cache successful validations, so if marker exists, it means cache is complete
marker = _read_validation_marker(snapshot_dir)
if marker is not None:
marker_path = _get_validation_marker_path(snapshot_dir)
marker_name = os.path.basename(marker_path) if marker_path else "unknown"
log_info_on_rank0(
logger,
f"CI_OFFLINE: Marker hit (marker={marker_name}), skip re-validation, offline mode will be enabled - {model_name_or_path}",
)
return True
# No marker - perform full validation
# (Failures are not cached, so we'll retry validation each time until success)
# Extract revision (snapshot hash) from snapshot_dir path
# snapshot_dir format: /path/to/cache/models--org--model/snapshots/<commit_hash>
revision = os.path.basename(snapshot_dir)
# Only allow remote checks if we're not in offline mode
# This avoids unnecessary API calls and warnings in offline CI environments
import huggingface_hub.constants
allow_remote_check = not huggingface_hub.constants.HF_HUB_OFFLINE
log_info_on_rank0(
logger,
f"CI_OFFLINE: No marker found, performing full validation "
f"(snapshot={revision}, allow_remote_check={allow_remote_check}) - {model_name_or_path}",
)
# Validate config and tokenizer files with remote existence checks
config_valid, missing_config_files = _validate_config_and_tokenizer_files(
snapshot_dir=snapshot_dir,
model_id=model_name_or_path,
revision=revision,
allow_remote_check=allow_remote_check,
)
if not config_valid:
log_info_on_rank0(
logger,
f"CI_OFFLINE: Missing config/tokenizer files {missing_config_files}, skip offline, keep online allowed - {model_name_or_path}",
)
# Don't write marker for failures - allow retry after download
return False
# Validate weight files using existing validation from PR #15216
# This checks for missing shards, corrupted safetensors, etc.
weights_valid, error_msg, _ = _validate_sharded_model(snapshot_dir, weight_files)
if not weights_valid:
log_info_on_rank0(
logger,
f"CI_OFFLINE: Weight validation failed ({error_msg}), skip offline, keep online allowed - {model_name_or_path}",
)
# Don't write marker for failures - allow retry after download
return False
log_info_on_rank0(
logger,
f"CI_OFFLINE: Cache validation PASSED, offline mode will be enabled - {model_name_or_path}",
)
# Write marker with passed=True for future reuse
# (Failures are not cached, so this only happens on success)
_write_validation_marker(snapshot_dir, passed=True)
return True
def _infer_component_type(component_name: str, component_info: list) -> str:
"""
Infer component type from component name and info.
Args:
component_name: Name of the component (e.g., "scheduler", "tokenizer")
component_info: Component info from model_index.json (e.g., ["diffusers", "SchedulerClass"])
Returns:
Component type string for validation rules
"""
# Normalize component name for type detection
name_lower = component_name.lower()
# Infer type based on name
if "scheduler" in name_lower:
return "scheduler"
elif "tokenizer" in name_lower:
return "tokenizer"
elif "image_processor" in name_lower:
return "image_processor"
elif "feature_extractor" in name_lower:
return "feature_extractor"
elif "processor" in name_lower:
return "processor"
else:
# Default to model component (needs config.json + weights)
return "model"
def _check_component_config(
component_dir: str, component_type: str
) -> Tuple[bool, List[str]]:
"""
Check if component has required config files based on type.
Args:
component_dir: Path to component directory
component_type: Type of component (scheduler, tokenizer, processor, model, etc.)
Returns:
Tuple of (has_valid_config, list_of_candidates_tried)
"""
if component_type == "scheduler":
# Scheduler: scheduler_config.json or config.json
candidates = ["scheduler_config.json", "config.json"]
for candidate in candidates:
candidate_path = os.path.join(component_dir, candidate)
if _validate_json_file(candidate_path, candidate):
return True, candidates
return False, candidates
elif component_type == "tokenizer":
# Tokenizer must have actual tokenizer files (not just tokenizer_config.json)
# Valid combinations:
# - tokenizer.json
# - tokenizer.model
# - vocab.json + merges.txt
candidates = [
"tokenizer.json",
"tokenizer.model",
"vocab.json+merges.txt",
]
# Check tokenizer.json (validate as JSON)
tokenizer_json_path = os.path.join(component_dir, "tokenizer.json")
if _validate_json_file(tokenizer_json_path, "tokenizer.json"):
return True, candidates
# Check tokenizer.model (non-empty file)
tokenizer_model_path = os.path.join(component_dir, "tokenizer.model")
if os.path.exists(tokenizer_model_path) and os.path.isfile(
tokenizer_model_path
):
try:
if os.path.getsize(tokenizer_model_path) > 0:
return True, candidates
except OSError:
pass
# Check vocab.json + merges.txt pair
vocab_path = os.path.join(component_dir, "vocab.json")
merges_path = os.path.join(component_dir, "merges.txt")
if _validate_json_file(vocab_path, "vocab.json") and os.path.exists(
merges_path
):
return True, candidates
return False, candidates
elif component_type in ["processor", "feature_extractor", "image_processor"]:
# Processor/feature_extractor/image_processor: preprocessor_config.json or config.json
candidates = ["preprocessor_config.json", "config.json"]
for candidate in candidates:
candidate_path = os.path.join(component_dir, candidate)
if _validate_json_file(candidate_path, candidate):
return True, candidates
return False, candidates
else:
# Default model components: config.json
candidates = ["config.json"]
config_path = os.path.join(component_dir, "config.json")
if _validate_json_file(config_path, "config.json"):
return True, candidates
return False, candidates
def _check_component_weights(component_dir: str) -> bool:
"""
Check if component directory has weight files.
Args:
component_dir: Path to component directory
Returns:
True if weight files found, False otherwise
"""
weight_patterns = ["*.safetensors", "*.bin", "*.pt", "*.pth"]
for pattern in weight_patterns:
weight_files = glob_module.glob(os.path.join(component_dir, pattern))
if weight_files:
return True
return False
def _format_component_list(components: List[str], max_show: int = 5) -> str:
"""
Format component list with truncation.
Args:
components: List of component names
max_show: Maximum number to show before truncating
Returns:
Formatted string like "comp1, comp2, comp3" or "comp1, comp2, +3 more"
"""
if len(components) <= max_show:
return ", ".join(components)
else:
shown = components[:max_show]
remaining = len(components) - max_show
return f"{', '.join(shown)}, +{remaining} more"
def _validate_diffusion_model(
snapshot_dir: str,
) -> Tuple[bool, Optional[str]]:
"""
Validate diffusion model (diffusers pipeline) cache completeness.
This validation is based on model_index.json as the single source of truth.
Error reporting uses coarse-grained error codes unless verbose mode is enabled.
Error codes:
- DIFFUSERS_INVALID_INDEX: model_index.json missing or corrupted
- DIFFUSERS_INVALID_COMPONENTS: model_index.json has no valid components
- DIFFUSERS_MISSING_COMPONENT: component directory or config missing
- DIFFUSERS_MISSING_WEIGHTS: component weights missing
Args:
snapshot_dir: Path to the model snapshot directory
Returns:
Tuple of (is_valid, error_message)
- (True, None) if validation passed
- (False, error_code_with_components) if validation failed
"""
# Check verbose mode from environment
verbose = os.environ.get("SGLANG_CI_VALIDATE_VERBOSE") == "1"
# 1. Check for model_index.json (required for diffusers models)
model_index_path = os.path.join(snapshot_dir, "model_index.json")
if not os.path.exists(model_index_path):
return False, "DIFFUSERS_INVALID_INDEX: model_index.json not found"
# Parse model_index.json
try:
with open(model_index_path, "r", encoding="utf-8") as f:
model_index = json.load(f)
except (json.JSONDecodeError, OSError) as e:
if verbose:
return False, f"DIFFUSERS_INVALID_INDEX: model_index.json parse error - {e}"
return False, "DIFFUSERS_INVALID_INDEX: model_index.json corrupted"
# 2. Extract components (non-underscore keys with list values)
components = {
k: v
for k, v in model_index.items()
if not k.startswith("_") and isinstance(v, list)
}
if not components:
return False, "DIFFUSERS_INVALID_COMPONENTS: no valid components defined"
# Categorize errors by type
missing_dirs = []
missing_configs = []
missing_configs_verbose = []
missing_weights = []
# 3. Validate each component
for component_name, component_info in components.items():
component_dir = os.path.join(snapshot_dir, component_name)
# Component directory must exist
if not os.path.isdir(component_dir):
missing_dirs.append(component_name)
continue
# Infer component type for validation rules
component_type = _infer_component_type(component_name, component_info)
# Check for required config files based on component type
has_valid_config, config_candidates = _check_component_config(
component_dir, component_type
)
if not has_valid_config:
missing_configs.append(component_name)
if verbose:
candidates_str = ", ".join(config_candidates)
missing_configs_verbose.append(
f"{component_name} (tried: {candidates_str})"
)
continue
# 4. Check for weights if component needs them
# These components don't require weight files (config-only)
needs_weights = component_type not in [
"scheduler",
"tokenizer",
"processor",
"feature_extractor",
"image_processor",
]
if needs_weights:
has_weights = _check_component_weights(component_dir)
if not has_weights:
missing_weights.append(component_name)
# 5. Build error message based on categorized errors
if missing_dirs or missing_configs or missing_weights:
errors = []
if missing_dirs:
dir_str = _format_component_list(missing_dirs)
if verbose:
errors.append(f"DIFFUSERS_MISSING_COMPONENT (dirs): {dir_str}")
else:
errors.append(f"DIFFUSERS_MISSING_COMPONENT(dir): {dir_str}")
if missing_configs:
if verbose:
config_str = "; ".join(missing_configs_verbose)
errors.append(f"DIFFUSERS_MISSING_COMPONENT (configs): {config_str}")
else:
config_str = _format_component_list(missing_configs)
errors.append(f"DIFFUSERS_MISSING_COMPONENT(cfg): {config_str}")
if missing_weights:
weight_str = _format_component_list(missing_weights)
errors.append(f"DIFFUSERS_MISSING_WEIGHTS: {weight_str}")
return False, " | ".join(errors)
return True, None
def validate_cache_with_detailed_reason(
snapshot_dir: str, weight_files: List[str], model_name_or_path: str
) -> Tuple[bool, Optional[str]]:
"""
Validate cache and return detailed reason for failure.
This function performs validation without relying on shared validation markers.
Used by prevalidate_cached_models.py to provide detailed feedback.
Args:
snapshot_dir: Path to the model snapshot directory
weight_files: List of weight file paths to validate
model_name_or_path: Model identifier for logging
Returns:
Tuple of (success, reason):
- (True, None) if validation passed
- (False, reason_str) if validation failed with specific reason
"""
# Guard: weight_files is required
if not weight_files:
return False, "No weight files provided"
# Perform full validation and capture failure reasons
revision = os.path.basename(snapshot_dir)
# Read from environment variable instead of huggingface_hub.constants
allow_remote_check = os.environ.get("HF_HUB_OFFLINE") != "1"
# Validate config and tokenizer files
config_valid, missing_config_files = _validate_config_and_tokenizer_files(
snapshot_dir=snapshot_dir,
model_id=model_name_or_path,
revision=revision,
allow_remote_check=allow_remote_check,
)
if not config_valid:
missing_files_str = ", ".join(missing_config_files)
return False, f"Missing config/tokenizer files: {missing_files_str}"
# Validate weight files
weights_valid, error_msg, _ = _validate_sharded_model(snapshot_dir, weight_files)
if not weights_valid:
return False, f"Weight validation failed: {error_msg}"
# All validations passed
return True, None
def validate_cache_lightweight(
snapshot_dir: str, requires_hf_quant_config: bool = False
) -> bool: