[Model Loader] Stop checkpoint prefetch after iterator completion (#41588)

Co-authored-by: Hrithvik Alex <hrithvik@baseten.co>
This commit is contained in:
Hrithvik Alex
2026-09-28 16:27:09 -07:00
committed by GitHub
co-authored by Hrithvik Alex
parent 9654e5c4bc
commit f124c5ceec
@@ -1138,8 +1138,9 @@ def safetensors_weights_iterator(
not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0
)
prefetch_handle = None
if prefetch and not disable_mmap:
_prefetch_all_checkpoints(
prefetch_handle = _prefetch_all_checkpoints(
sorted(hf_weights_files), num_threads=prefetch_num_threads
)
@@ -1161,6 +1162,8 @@ def safetensors_weights_iterator(
yield name, f.get_tensor(name)
if drop_cache_after_load:
_drop_file_cache_after_load(st_file)
if prefetch_handle is not None:
prefetch_handle.stop()
def fastsafetensors_weights_iterator(
@@ -1234,8 +1237,9 @@ def buffered_multi_thread_safetensors_weights_iterator(
max_workers loading concurrently + 1 prefetched and ready to yield.
Peak CPU RAM ≈ (max_workers + 2) × shard_file_size.
"""
prefetch_handle = None
if prefetch and not disable_mmap:
_prefetch_all_checkpoints(
prefetch_handle = _prefetch_all_checkpoints(
sorted(hf_weights_files), num_threads=prefetch_num_threads
)
enable_tqdm = (
@@ -1287,6 +1291,8 @@ def buffered_multi_thread_safetensors_weights_iterator(
# but later mmap-backed tensor access may fault pages again.
_drop_file_cache_after_load(st_file)
pbar.update(1)
if prefetch_handle is not None:
prefetch_handle.stop()
def _load_pt_file(bin_file: str) -> dict: