[Fix] diffusion/loader: Preserve checkpoints on read failures (#46095)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Mick
2026-10-24 12:17:46 +08:00
committed by GitHub
co-authored by Mick Qian
parent f2c634961b
commit cdab6c1bcc
4 changed files with 74 additions and 35 deletions
@@ -129,8 +129,8 @@ def _collect_safetensors_sources(
]
| None
):
corrupted_files, duplicate_files_by_key = _scan_safetensors_files(weight_files)
if corrupted_files or duplicate_files_by_key:
unreadable_files, duplicate_files_by_key = _scan_safetensors_files(weight_files)
if unreadable_files or duplicate_files_by_key:
return None
sources_by_target: dict[str, list[SafetensorsSource]] = defaultdict(list)
@@ -168,7 +168,7 @@ def _scan_safetensors_files(
hf_weights_files: list[str],
) -> tuple[list[str], dict[str, set[str]]]:
"""Validate headers and detect cross-file duplicate keys in one pass."""
corrupted_files: list[str] = []
unreadable_files: list[str] = []
key_to_file: dict[str, str] = {}
duplicate_files_by_key: dict[str, set[str]] = defaultdict(set)
@@ -183,14 +183,14 @@ def _scan_safetensors_files(
duplicate_files_by_key[name].update((previous_file, st_file))
except Exception as e:
logger.error(
"Corrupted safetensors file detected: %s - %s: %s",
"Failed to read safetensors file: %s - %s: %s",
st_file,
type(e).__name__,
str(e),
)
corrupted_files.append(st_file)
unreadable_files.append(st_file)
return corrupted_files, duplicate_files_by_key
return unreadable_files, duplicate_files_by_key
def _raise_if_duplicate_safetensors_keys(
@@ -257,36 +257,14 @@ def safetensors_weights_iterator(
requested=requested, needs_key_filter=key_filter is not None
)
# Validate files before loading
corrupted_files, duplicate_files_by_key = _scan_safetensors_files(hf_weights_files)
if corrupted_files:
# Delete corrupted files (both symlink and blob if applicable)
for file_path in corrupted_files:
try:
if os.path.islink(file_path):
blob_path = os.path.realpath(file_path)
os.remove(file_path)
logger.info(
"Removed corrupted symlink: %s", os.path.basename(file_path)
)
if os.path.exists(blob_path):
os.remove(blob_path)
logger.info(
"Removed corrupted blob: %s", os.path.basename(blob_path)
)
elif os.path.isfile(file_path):
os.remove(file_path)
logger.info(
"Removed corrupted file: %s", os.path.basename(file_path)
)
except Exception as e:
logger.warning("Failed to remove corrupted file %s: %s", file_path, e)
# validation must not modify checkpoints or shared cache blobs
unreadable_files, duplicate_files_by_key = _scan_safetensors_files(hf_weights_files)
if unreadable_files:
raise RuntimeError(
f"Found {len(corrupted_files)} corrupted safetensors file(s). "
f"Files have been removed: {[os.path.basename(f) for f in corrupted_files]}. "
"Please retry - the files will be re-downloaded automatically."
f"Cannot read {len(unreadable_files)} safetensors file(s): "
f"{[os.path.basename(f) for f in unreadable_files]}. "
"Files were not modified. Check permissions and checkpoint integrity; "
"repair or re-download the checkpoint if needed."
)
_raise_if_duplicate_safetensors_keys(duplicate_files_by_key)
@@ -11,6 +11,7 @@ from unittest.mock import Mock, patch
import torch
from huggingface_hub.errors import LocalEntryNotFoundError
from safetensors import safe_open
from safetensors.torch import save_file as safetensors_save_file
from sglang.multimodal_gen.runtime.loader.utils import (
@@ -22,6 +23,7 @@ from sglang.multimodal_gen.runtime.loader.utils import (
from sglang.multimodal_gen.runtime.loader.weight_utils import (
_disable_runai_streamer_rank_discovery_collective,
get_lock,
safetensors_weights_iterator,
)
from sglang.multimodal_gen.runtime.weights.source import (
filter_duplicate_precision_variant_safetensors,
@@ -433,5 +435,56 @@ class TestDiffusionWeightLock(unittest.TestCase):
self.assertTrue(os.path.exists(lock.lock_file))
class TestCheckpointReadFailure(unittest.TestCase):
def test_failed_read_never_removes_checkpoint_or_shared_blob(self):
for symlink in (False, True):
for error in (None, PermissionError("denied"), OSError("I/O error")):
with (
self.subTest(symlink=symlink, error=error),
tempfile.TemporaryDirectory() as tmpdir,
):
root = Path(tmpdir)
blob = root / "blob"
if error is None:
blob.write_bytes(b"incomplete safetensors")
else:
safetensors_save_file({"weight": torch.ones(2)}, blob)
original = blob.read_bytes()
path = root / "model.safetensors"
if symlink:
path.symlink_to(blob)
else:
blob.rename(path)
blob = path
other_snapshot = root / "other.safetensors"
other_snapshot.symlink_to(blob)
with patch(
"sglang.multimodal_gen.runtime.loader.weight_utils.safe_open",
side_effect=error,
wraps=safe_open,
):
with self.assertRaises(RuntimeError):
next(
safetensors_weights_iterator(
[str(path)], use_runai_model_streamer=False
)
)
self.assertTrue(
path.exists(), "loader removed the requested checkpoint"
)
self.assertEqual(path.is_symlink(), symlink)
self.assertEqual(blob.read_bytes(), original)
self.assertEqual(other_snapshot.read_bytes(), original)
if error is not None:
loaded = dict(
safetensors_weights_iterator(
[str(path)], use_runai_model_streamer=False
)
)
torch.testing.assert_close(loaded["weight"], torch.ones(2))
if __name__ == "__main__":
unittest.main()