diff --git a/python/sglang/srt/model_loader/weight_utils.py b/python/sglang/srt/model_loader/weight_utils.py index 8e34d94f7d..82951f9f19 100644 --- a/python/sglang/srt/model_loader/weight_utils.py +++ b/python/sglang/srt/model_loader/weight_utils.py @@ -1138,8 +1138,9 @@ def safetensors_weights_iterator( not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0 ) + prefetch_handle = None if prefetch and not disable_mmap: - _prefetch_all_checkpoints( + prefetch_handle = _prefetch_all_checkpoints( sorted(hf_weights_files), num_threads=prefetch_num_threads ) @@ -1161,6 +1162,8 @@ def safetensors_weights_iterator( yield name, f.get_tensor(name) if drop_cache_after_load: _drop_file_cache_after_load(st_file) + if prefetch_handle is not None: + prefetch_handle.stop() def fastsafetensors_weights_iterator( @@ -1234,8 +1237,9 @@ def buffered_multi_thread_safetensors_weights_iterator( max_workers loading concurrently + 1 prefetched and ready to yield. Peak CPU RAM ≈ (max_workers + 2) × shard_file_size. """ + prefetch_handle = None if prefetch and not disable_mmap: - _prefetch_all_checkpoints( + prefetch_handle = _prefetch_all_checkpoints( sorted(hf_weights_files), num_threads=prefetch_num_threads ) enable_tqdm = ( @@ -1287,6 +1291,8 @@ def buffered_multi_thread_safetensors_weights_iterator( # but later mmap-backed tensor access may fault pages again. _drop_file_cache_after_load(st_file) pbar.update(1) + if prefetch_handle is not None: + prefetch_handle.stop() def _load_pt_file(bin_file: str) -> dict: