From 6d35cee7fc92d0ae0bdd3fa16e7767c2c361d5b1 Mon Sep 17 00:00:00 2001 From: cctry Date: Fri, 25 Sep 2026 12:18:08 -0700 Subject: [PATCH] Fix multimodal feature offload races (#40621) Co-authored-by: cctry Co-authored-by: Yongji Wu Co-authored-by: Zhiqiang Xie --- python/sglang/srt/managers/mm_schedule.py | 33 ++++++++--- python/sglang/srt/managers/mm_utils.py | 68 ++++++++++++++++------- 2 files changed, 74 insertions(+), 27 deletions(-) diff --git a/python/sglang/srt/managers/mm_schedule.py b/python/sglang/srt/managers/mm_schedule.py index 13dc186976..0bc0615606 100644 --- a/python/sglang/srt/managers/mm_schedule.py +++ b/python/sglang/srt/managers/mm_schedule.py @@ -19,6 +19,7 @@ _is_npu = is_npu() _is_xpu = is_xpu() embedding_cache: Optional[MultiModalStaticCache] = None +host_offload_event: Optional[torch.cuda.Event] = None def init_mm_embedding_cache(max_size: int = 0): @@ -83,6 +84,12 @@ def _get_precomputed_embedding( If some but not all have precomputed_embeddings, raise NotImplementedError. If none have precomputed_embeddings, return None. """ + if host_offload_event is not None and any( + isinstance(item.precomputed_embeddings, torch.Tensor) + and item.precomputed_embeddings.is_cpu + for item in items + ): + host_offload_event.synchronize() precomputed_embeddings = [] max_iterations = min(len(items_size) - 1, len(prefix_length)) @@ -209,9 +216,22 @@ def _can_skip_pre_embed_feature_move(data_embedding_func: DataEmbeddingFunc) -> def _move_items_to_device( - items: List[MultimodalDataItem], device: torch.device + items: List[MultimodalDataItem], + device: torch.device, + data_embedding_func: DataEmbeddingFunc, ) -> None: - """Move item features to the target device (in-place, non-blocking).""" + """Wait for feature readiness and upload unless the encoder defers the move.""" + defer_move = _can_skip_pre_embed_feature_move(data_embedding_func) + if host_offload_event is not None and any( + isinstance(item.feature, torch.Tensor) and item.feature.is_cpu for item in items + ): + if defer_move or device.type != "cuda": + # Deferred encoders can read CPU subsets of a mixed-device batch. + host_offload_event.synchronize() + else: + torch.cuda.current_stream(device).wait_event(host_offload_event) + if defer_move: + return for item in items: if isinstance(item.feature, torch.Tensor) and item.feature.device != device: item.feature = item.feature.to(device, non_blocking=True) @@ -270,8 +290,7 @@ def _get_chunked_embedding_full( embedding_per_req = None if embedding_per_req is None: - if not _can_skip_pre_embed_feature_move(data_embedding_func): - _move_items_to_device(embedding_items_per_req, device) + _move_items_to_device(embedding_items_per_req, device, data_embedding_func) embedding = data_embedding_func(embedding_items_per_req) if isinstance(embedding, list): # This path caches the combined per-request embedding, so the @@ -383,8 +402,7 @@ def _batch_encode_per_image_misses( miss_items = [unique_misses[key][0] for key in ordered_cache_keys] token_counts = [unique_misses[key][1] for key in ordered_cache_keys] - if not _can_skip_pre_embed_feature_move(data_embedding_func): - _move_items_to_device(miss_items, device) + _move_items_to_device(miss_items, device, data_embedding_func) all_miss_embedding = data_embedding_func(miss_items) if isinstance(all_miss_embedding, list): @@ -461,8 +479,7 @@ def _get_chunked_embedding_by_item( if miss_items: miss_item_list = [item for _, item, _, _ in miss_items] - if not _can_skip_pre_embed_feature_move(data_embedding_func): - _move_items_to_device(miss_item_list, device) + _move_items_to_device(miss_item_list, device, data_embedding_func) all_miss_embedding = data_embedding_func(miss_item_list) if isinstance(all_miss_embedding, list): diff --git a/python/sglang/srt/managers/mm_utils.py b/python/sglang/srt/managers/mm_utils.py index 41805b258c..7b17e75c94 100644 --- a/python/sglang/srt/managers/mm_utils.py +++ b/python/sglang/srt/managers/mm_utils.py @@ -20,6 +20,7 @@ from torch import nn from sglang.kernels.ops.memory.gpu_tensor_hash import gpu_tensor_hash from sglang.srt.environ import envs +from sglang.srt.managers import mm_schedule from sglang.srt.managers.io_struct import ( BaseBatchReq, TokenizedEmbeddingReqInput, @@ -661,6 +662,7 @@ def general_mm_embed_routine( data_embedding_funcs: Dict[Modality, DataEmbeddingFunc] = None, placeholder_tokens: Optional[dict[Modality, List[int]]] = None, use_deepstack: Dict[Modality, bool] = {}, + feature_dtypes: Optional[Dict[Modality, torch.dtype]] = None, **kwargs, ) -> torch.Tensor: """ @@ -673,6 +675,7 @@ def general_mm_embed_routine( data_embedding_funcs: A dictionary mapping from modality type to the corresponding embedding function. placeholder_tokens: Token IDs for multimodal placeholders use_deepstack: Whether to use deepstack embeddings for each modality, default False + feature_dtypes: Optional per-modality dtypes for preparing CUDA features before cache lookup. **kwargs: Additional arguments passed to language model Returns: @@ -689,6 +692,23 @@ def general_mm_embed_routine( mm_inputs_list = [ mm_input for mm_input in forward_batch.mm_inputs if mm_input is not None ] + if feature_dtypes: + for mm_input in mm_inputs_list: + for item in mm_input.mm_items: + dtype = feature_dtypes.get(item.modality) + feature = item.feature + if ( + dtype is not None + and isinstance(feature, torch.Tensor) + and feature.is_cuda + and feature.dtype != dtype + ): + # The cast may still read the source after replacing it. + feature.record_stream( + torch.cuda.current_stream(feature.device) + ) + item.feature = feature.to(dtype=dtype) + del feature extend_prefix_lens = [ prefix_len for i, prefix_len in enumerate(forward_batch.extend_prefix_lens_cpu) @@ -742,27 +762,37 @@ def general_mm_embed_routine( # best-effort, offloading to CPU ensures we have a reliable fallback # if a cache miss occurs in subsequent chunks, while still freeing up # critical GPU memory. - if mm_inputs_list: + if mm_inputs_list and input_embeds.is_cuda: + language_only = get_disagg().language_only + stream = torch.cuda.current_stream(input_embeds.device) + offloaded = False for mm_input_obj in mm_inputs_list: - if mm_input_obj and hasattr(mm_input_obj, "mm_items"): - for mm_item in mm_input_obj.mm_items: - feature = getattr(mm_item, "feature", None) - if isinstance(feature, torch.Tensor) and feature.is_cuda: - mm_item.feature = feature.to("cpu", non_blocking=True) - if get_disagg().language_only: - precomputed_embeddings = getattr( - mm_item, "precomputed_embeddings", None + for mm_item in mm_input_obj.mm_items: + feature = mm_item.feature + if isinstance(feature, torch.Tensor) and feature.is_cuda: + # The transport reconstructed this block on the scheduler stream + # and nothing else pins it to ours: without record_stream the + # allocator recycles it into the next scheduler allocation while + # the encoder read and this copy are still queued here. + feature.record_stream(stream) + mm_item.feature = feature.to("cpu", non_blocking=True) + offloaded = True + if language_only: + precomputed = mm_item.precomputed_embeddings + if ( + isinstance(precomputed, torch.Tensor) + and precomputed.is_cuda + and not mm_item.keep_device_embedding + ): + precomputed.record_stream(stream) + mm_item.precomputed_embeddings = precomputed.to( + "cpu", non_blocking=True ) - if ( - isinstance(precomputed_embeddings, torch.Tensor) - and precomputed_embeddings.is_cuda - and not mm_item.keep_device_embedding - ): - mm_item.precomputed_embeddings = ( - precomputed_embeddings.to( - "cpu", non_blocking=True - ) - ) + offloaded = True + if offloaded: + if mm_schedule.host_offload_event is None: + mm_schedule.host_offload_event = torch.cuda.Event() + mm_schedule.host_offload_event.record(stream) forward_batch.mm_inputs = None forward_batch.mm_input_embeds = ( input_embeds.clone()