From cdab6c1bcc304060ca215c2d2abf27ed147529d2 Mon Sep 17 00:00:00 2001 From: Mick Date: Sat, 24 Oct 2026 12:17:46 +0800 Subject: [PATCH] [Fix] diffusion/loader: Preserve checkpoints on read failures (#46095) Co-authored-by: Mick Qian --- .../sglang-diffusion/deployment_cookbook.mdx | 8 +++ .../runtime/loader/rank_local_checkpoint.py | 4 +- .../runtime/loader/weight_utils.py | 44 ++++----------- .../test/unit/test_weight_utils.py | 53 +++++++++++++++++++ 4 files changed, 74 insertions(+), 35 deletions(-) diff --git a/docs/docs/sglang-diffusion/deployment_cookbook.mdx b/docs/docs/sglang-diffusion/deployment_cookbook.mdx index 2a7c3334e0..ad50d17760 100644 --- a/docs/docs/sglang-diffusion/deployment_cookbook.mdx +++ b/docs/docs/sglang-diffusion/deployment_cookbook.mdx @@ -77,6 +77,14 @@ livenessProbe: See [Health endpoints](/docs/sglang-diffusion/api/cli#health-endpoints) for the status-code contract and warmup-mode behavior. +## Checkpoint read failures + +Safetensors header validation fails without deleting checkpoint files, symlinks, +or shared cache blobs. A read error can indicate permissions or storage I/O +problems, not just an incomplete download. Check the underlying error before +repairing or re-downloading weights. Do not remove shared cache files while other +replicas may be using them. + ## Stable Model Identity Use `--served-model-name` when the public model name must remain stable across replicas, hosts, or checkpoint mount paths: diff --git a/python/sglang/multimodal_gen/runtime/loader/rank_local_checkpoint.py b/python/sglang/multimodal_gen/runtime/loader/rank_local_checkpoint.py index a5334dc9f8..cbaca8da13 100644 --- a/python/sglang/multimodal_gen/runtime/loader/rank_local_checkpoint.py +++ b/python/sglang/multimodal_gen/runtime/loader/rank_local_checkpoint.py @@ -129,8 +129,8 @@ def _collect_safetensors_sources( ] | None ): - corrupted_files, duplicate_files_by_key = _scan_safetensors_files(weight_files) - if corrupted_files or duplicate_files_by_key: + unreadable_files, duplicate_files_by_key = _scan_safetensors_files(weight_files) + if unreadable_files or duplicate_files_by_key: return None sources_by_target: dict[str, list[SafetensorsSource]] = defaultdict(list) diff --git a/python/sglang/multimodal_gen/runtime/loader/weight_utils.py b/python/sglang/multimodal_gen/runtime/loader/weight_utils.py index 6ae0d4f0a7..9cf29578dd 100644 --- a/python/sglang/multimodal_gen/runtime/loader/weight_utils.py +++ b/python/sglang/multimodal_gen/runtime/loader/weight_utils.py @@ -168,7 +168,7 @@ def _scan_safetensors_files( hf_weights_files: list[str], ) -> tuple[list[str], dict[str, set[str]]]: """Validate headers and detect cross-file duplicate keys in one pass.""" - corrupted_files: list[str] = [] + unreadable_files: list[str] = [] key_to_file: dict[str, str] = {} duplicate_files_by_key: dict[str, set[str]] = defaultdict(set) @@ -183,14 +183,14 @@ def _scan_safetensors_files( duplicate_files_by_key[name].update((previous_file, st_file)) except Exception as e: logger.error( - "Corrupted safetensors file detected: %s - %s: %s", + "Failed to read safetensors file: %s - %s: %s", st_file, type(e).__name__, str(e), ) - corrupted_files.append(st_file) + unreadable_files.append(st_file) - return corrupted_files, duplicate_files_by_key + return unreadable_files, duplicate_files_by_key def _raise_if_duplicate_safetensors_keys( @@ -257,36 +257,14 @@ def safetensors_weights_iterator( requested=requested, needs_key_filter=key_filter is not None ) - # Validate files before loading - corrupted_files, duplicate_files_by_key = _scan_safetensors_files(hf_weights_files) - - if corrupted_files: - # Delete corrupted files (both symlink and blob if applicable) - for file_path in corrupted_files: - try: - if os.path.islink(file_path): - blob_path = os.path.realpath(file_path) - os.remove(file_path) - logger.info( - "Removed corrupted symlink: %s", os.path.basename(file_path) - ) - if os.path.exists(blob_path): - os.remove(blob_path) - logger.info( - "Removed corrupted blob: %s", os.path.basename(blob_path) - ) - elif os.path.isfile(file_path): - os.remove(file_path) - logger.info( - "Removed corrupted file: %s", os.path.basename(file_path) - ) - except Exception as e: - logger.warning("Failed to remove corrupted file %s: %s", file_path, e) - + # validation must not modify checkpoints or shared cache blobs + unreadable_files, duplicate_files_by_key = _scan_safetensors_files(hf_weights_files) + if unreadable_files: raise RuntimeError( - f"Found {len(corrupted_files)} corrupted safetensors file(s). " - f"Files have been removed: {[os.path.basename(f) for f in corrupted_files]}. " - "Please retry - the files will be re-downloaded automatically." + f"Cannot read {len(unreadable_files)} safetensors file(s): " + f"{[os.path.basename(f) for f in unreadable_files]}. " + "Files were not modified. Check permissions and checkpoint integrity; " + "repair or re-download the checkpoint if needed." ) _raise_if_duplicate_safetensors_keys(duplicate_files_by_key) diff --git a/python/sglang/multimodal_gen/test/unit/test_weight_utils.py b/python/sglang/multimodal_gen/test/unit/test_weight_utils.py index a349e9dbba..e2028a0f8b 100644 --- a/python/sglang/multimodal_gen/test/unit/test_weight_utils.py +++ b/python/sglang/multimodal_gen/test/unit/test_weight_utils.py @@ -11,6 +11,7 @@ from unittest.mock import Mock, patch import torch from huggingface_hub.errors import LocalEntryNotFoundError +from safetensors import safe_open from safetensors.torch import save_file as safetensors_save_file from sglang.multimodal_gen.runtime.loader.utils import ( @@ -22,6 +23,7 @@ from sglang.multimodal_gen.runtime.loader.utils import ( from sglang.multimodal_gen.runtime.loader.weight_utils import ( _disable_runai_streamer_rank_discovery_collective, get_lock, + safetensors_weights_iterator, ) from sglang.multimodal_gen.runtime.weights.source import ( filter_duplicate_precision_variant_safetensors, @@ -433,5 +435,56 @@ class TestDiffusionWeightLock(unittest.TestCase): self.assertTrue(os.path.exists(lock.lock_file)) +class TestCheckpointReadFailure(unittest.TestCase): + def test_failed_read_never_removes_checkpoint_or_shared_blob(self): + for symlink in (False, True): + for error in (None, PermissionError("denied"), OSError("I/O error")): + with ( + self.subTest(symlink=symlink, error=error), + tempfile.TemporaryDirectory() as tmpdir, + ): + root = Path(tmpdir) + blob = root / "blob" + if error is None: + blob.write_bytes(b"incomplete safetensors") + else: + safetensors_save_file({"weight": torch.ones(2)}, blob) + original = blob.read_bytes() + path = root / "model.safetensors" + if symlink: + path.symlink_to(blob) + else: + blob.rename(path) + blob = path + other_snapshot = root / "other.safetensors" + other_snapshot.symlink_to(blob) + + with patch( + "sglang.multimodal_gen.runtime.loader.weight_utils.safe_open", + side_effect=error, + wraps=safe_open, + ): + with self.assertRaises(RuntimeError): + next( + safetensors_weights_iterator( + [str(path)], use_runai_model_streamer=False + ) + ) + + self.assertTrue( + path.exists(), "loader removed the requested checkpoint" + ) + self.assertEqual(path.is_symlink(), symlink) + self.assertEqual(blob.read_bytes(), original) + self.assertEqual(other_snapshot.read_bytes(), original) + if error is not None: + loaded = dict( + safetensors_weights_iterator( + [str(path)], use_runai_model_streamer=False + ) + ) + torch.testing.assert_close(loaded["weight"], torch.ones(2)) + + if __name__ == "__main__": unittest.main()