[Fix] Fail fast when a safetensors index references missing shard files (#32279)

This commit is contained in:
Liangsheng Yin
2026-07-24 02:18:56 -07:00
committed by GitHub
parent b954e9cf3d
commit f4f15162bc
2 changed files with 98 additions and 0 deletions
@@ -686,6 +686,16 @@ def filter_duplicate_safetensors_files(
weight_files_in_index = set()
for weight_name in weight_map:
weight_files_in_index.add(os.path.join(hf_folder, weight_map[weight_name]))
# Fail fast if the index references shard files that are not on disk (e.g. an
# incomplete or interrupted download). Otherwise those shards are silently
# dropped and the model loads with uninitialized weights.
missing_files = sorted(f for f in weight_files_in_index if not os.path.isfile(f))
if missing_files:
raise RuntimeError(
f"{index_file} references {len(missing_files)} shard file(s) missing "
f"from {hf_folder} (incomplete download?): "
f"{[os.path.basename(f) for f in missing_files]}"
)
# Filter out any fields that are not found in the index file.
hf_weights_files = [f for f in hf_weights_files if f in weight_files_in_index]
return hf_weights_files