[Fix] diffusion/loader: Preserve checkpoints on read failures (#46095)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -129,8 +129,8 @@ def _collect_safetensors_sources(
|
||||
]
|
||||
| None
|
||||
):
|
||||
corrupted_files, duplicate_files_by_key = _scan_safetensors_files(weight_files)
|
||||
if corrupted_files or duplicate_files_by_key:
|
||||
unreadable_files, duplicate_files_by_key = _scan_safetensors_files(weight_files)
|
||||
if unreadable_files or duplicate_files_by_key:
|
||||
return None
|
||||
|
||||
sources_by_target: dict[str, list[SafetensorsSource]] = defaultdict(list)
|
||||
|
||||
@@ -168,7 +168,7 @@ def _scan_safetensors_files(
|
||||
hf_weights_files: list[str],
|
||||
) -> tuple[list[str], dict[str, set[str]]]:
|
||||
"""Validate headers and detect cross-file duplicate keys in one pass."""
|
||||
corrupted_files: list[str] = []
|
||||
unreadable_files: list[str] = []
|
||||
key_to_file: dict[str, str] = {}
|
||||
duplicate_files_by_key: dict[str, set[str]] = defaultdict(set)
|
||||
|
||||
@@ -183,14 +183,14 @@ def _scan_safetensors_files(
|
||||
duplicate_files_by_key[name].update((previous_file, st_file))
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Corrupted safetensors file detected: %s - %s: %s",
|
||||
"Failed to read safetensors file: %s - %s: %s",
|
||||
st_file,
|
||||
type(e).__name__,
|
||||
str(e),
|
||||
)
|
||||
corrupted_files.append(st_file)
|
||||
unreadable_files.append(st_file)
|
||||
|
||||
return corrupted_files, duplicate_files_by_key
|
||||
return unreadable_files, duplicate_files_by_key
|
||||
|
||||
|
||||
def _raise_if_duplicate_safetensors_keys(
|
||||
@@ -257,36 +257,14 @@ def safetensors_weights_iterator(
|
||||
requested=requested, needs_key_filter=key_filter is not None
|
||||
)
|
||||
|
||||
# Validate files before loading
|
||||
corrupted_files, duplicate_files_by_key = _scan_safetensors_files(hf_weights_files)
|
||||
|
||||
if corrupted_files:
|
||||
# Delete corrupted files (both symlink and blob if applicable)
|
||||
for file_path in corrupted_files:
|
||||
try:
|
||||
if os.path.islink(file_path):
|
||||
blob_path = os.path.realpath(file_path)
|
||||
os.remove(file_path)
|
||||
logger.info(
|
||||
"Removed corrupted symlink: %s", os.path.basename(file_path)
|
||||
)
|
||||
if os.path.exists(blob_path):
|
||||
os.remove(blob_path)
|
||||
logger.info(
|
||||
"Removed corrupted blob: %s", os.path.basename(blob_path)
|
||||
)
|
||||
elif os.path.isfile(file_path):
|
||||
os.remove(file_path)
|
||||
logger.info(
|
||||
"Removed corrupted file: %s", os.path.basename(file_path)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to remove corrupted file %s: %s", file_path, e)
|
||||
|
||||
# validation must not modify checkpoints or shared cache blobs
|
||||
unreadable_files, duplicate_files_by_key = _scan_safetensors_files(hf_weights_files)
|
||||
if unreadable_files:
|
||||
raise RuntimeError(
|
||||
f"Found {len(corrupted_files)} corrupted safetensors file(s). "
|
||||
f"Files have been removed: {[os.path.basename(f) for f in corrupted_files]}. "
|
||||
"Please retry - the files will be re-downloaded automatically."
|
||||
f"Cannot read {len(unreadable_files)} safetensors file(s): "
|
||||
f"{[os.path.basename(f) for f in unreadable_files]}. "
|
||||
"Files were not modified. Check permissions and checkpoint integrity; "
|
||||
"repair or re-download the checkpoint if needed."
|
||||
)
|
||||
|
||||
_raise_if_duplicate_safetensors_keys(duplicate_files_by_key)
|
||||
|
||||
@@ -11,6 +11,7 @@ from unittest.mock import Mock, patch
|
||||
|
||||
import torch
|
||||
from huggingface_hub.errors import LocalEntryNotFoundError
|
||||
from safetensors import safe_open
|
||||
from safetensors.torch import save_file as safetensors_save_file
|
||||
|
||||
from sglang.multimodal_gen.runtime.loader.utils import (
|
||||
@@ -22,6 +23,7 @@ from sglang.multimodal_gen.runtime.loader.utils import (
|
||||
from sglang.multimodal_gen.runtime.loader.weight_utils import (
|
||||
_disable_runai_streamer_rank_discovery_collective,
|
||||
get_lock,
|
||||
safetensors_weights_iterator,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.weights.source import (
|
||||
filter_duplicate_precision_variant_safetensors,
|
||||
@@ -433,5 +435,56 @@ class TestDiffusionWeightLock(unittest.TestCase):
|
||||
self.assertTrue(os.path.exists(lock.lock_file))
|
||||
|
||||
|
||||
class TestCheckpointReadFailure(unittest.TestCase):
|
||||
def test_failed_read_never_removes_checkpoint_or_shared_blob(self):
|
||||
for symlink in (False, True):
|
||||
for error in (None, PermissionError("denied"), OSError("I/O error")):
|
||||
with (
|
||||
self.subTest(symlink=symlink, error=error),
|
||||
tempfile.TemporaryDirectory() as tmpdir,
|
||||
):
|
||||
root = Path(tmpdir)
|
||||
blob = root / "blob"
|
||||
if error is None:
|
||||
blob.write_bytes(b"incomplete safetensors")
|
||||
else:
|
||||
safetensors_save_file({"weight": torch.ones(2)}, blob)
|
||||
original = blob.read_bytes()
|
||||
path = root / "model.safetensors"
|
||||
if symlink:
|
||||
path.symlink_to(blob)
|
||||
else:
|
||||
blob.rename(path)
|
||||
blob = path
|
||||
other_snapshot = root / "other.safetensors"
|
||||
other_snapshot.symlink_to(blob)
|
||||
|
||||
with patch(
|
||||
"sglang.multimodal_gen.runtime.loader.weight_utils.safe_open",
|
||||
side_effect=error,
|
||||
wraps=safe_open,
|
||||
):
|
||||
with self.assertRaises(RuntimeError):
|
||||
next(
|
||||
safetensors_weights_iterator(
|
||||
[str(path)], use_runai_model_streamer=False
|
||||
)
|
||||
)
|
||||
|
||||
self.assertTrue(
|
||||
path.exists(), "loader removed the requested checkpoint"
|
||||
)
|
||||
self.assertEqual(path.is_symlink(), symlink)
|
||||
self.assertEqual(blob.read_bytes(), original)
|
||||
self.assertEqual(other_snapshot.read_bytes(), original)
|
||||
if error is not None:
|
||||
loaded = dict(
|
||||
safetensors_weights_iterator(
|
||||
[str(path)], use_runai_model_streamer=False
|
||||
)
|
||||
)
|
||||
torch.testing.assert_close(loaded["weight"], torch.ones(2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user