[Model Loader] Stop checkpoint prefetch after iterator completion (#41588)
Co-authored-by: Hrithvik Alex <hrithvik@baseten.co>
This commit is contained in:
co-authored by
Hrithvik Alex
parent
9654e5c4bc
commit
f124c5ceec
@@ -1138,8 +1138,9 @@ def safetensors_weights_iterator(
|
||||
not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0
|
||||
)
|
||||
|
||||
prefetch_handle = None
|
||||
if prefetch and not disable_mmap:
|
||||
_prefetch_all_checkpoints(
|
||||
prefetch_handle = _prefetch_all_checkpoints(
|
||||
sorted(hf_weights_files), num_threads=prefetch_num_threads
|
||||
)
|
||||
|
||||
@@ -1161,6 +1162,8 @@ def safetensors_weights_iterator(
|
||||
yield name, f.get_tensor(name)
|
||||
if drop_cache_after_load:
|
||||
_drop_file_cache_after_load(st_file)
|
||||
if prefetch_handle is not None:
|
||||
prefetch_handle.stop()
|
||||
|
||||
|
||||
def fastsafetensors_weights_iterator(
|
||||
@@ -1234,8 +1237,9 @@ def buffered_multi_thread_safetensors_weights_iterator(
|
||||
max_workers loading concurrently + 1 prefetched and ready to yield.
|
||||
Peak CPU RAM ≈ (max_workers + 2) × shard_file_size.
|
||||
"""
|
||||
prefetch_handle = None
|
||||
if prefetch and not disable_mmap:
|
||||
_prefetch_all_checkpoints(
|
||||
prefetch_handle = _prefetch_all_checkpoints(
|
||||
sorted(hf_weights_files), num_threads=prefetch_num_threads
|
||||
)
|
||||
enable_tqdm = (
|
||||
@@ -1287,6 +1291,8 @@ def buffered_multi_thread_safetensors_weights_iterator(
|
||||
# but later mmap-backed tensor access may fault pages again.
|
||||
_drop_file_cache_after_load(st_file)
|
||||
pbar.update(1)
|
||||
if prefetch_handle is not None:
|
||||
prefetch_handle.stop()
|
||||
|
||||
|
||||
def _load_pt_file(bin_file: str) -> dict:
|
||||
|
||||
Reference in New Issue
Block a user