Fix multimodal feature offload races (#40621)

Co-authored-by: cctry <cctry@fb.com>
Co-authored-by: Yongji Wu <yongji@meta.com>
Co-authored-by: Zhiqiang Xie <xiezhq@stanford.edu>
This commit is contained in:
cctry
2026-09-25 12:18:08 -07:00
committed by GitHub
co-authored by cctry Yongji Wu Zhiqiang Xie
parent 671000aee6
commit 6d35cee7fc
2 changed files with 74 additions and 27 deletions
+25 -8
View File
@@ -19,6 +19,7 @@ _is_npu = is_npu()
_is_xpu = is_xpu()
embedding_cache: Optional[MultiModalStaticCache] = None
host_offload_event: Optional[torch.cuda.Event] = None
def init_mm_embedding_cache(max_size: int = 0):
@@ -83,6 +84,12 @@ def _get_precomputed_embedding(
If some but not all have precomputed_embeddings, raise NotImplementedError.
If none have precomputed_embeddings, return None.
"""
if host_offload_event is not None and any(
isinstance(item.precomputed_embeddings, torch.Tensor)
and item.precomputed_embeddings.is_cpu
for item in items
):
host_offload_event.synchronize()
precomputed_embeddings = []
max_iterations = min(len(items_size) - 1, len(prefix_length))
@@ -209,9 +216,22 @@ def _can_skip_pre_embed_feature_move(data_embedding_func: DataEmbeddingFunc) ->
def _move_items_to_device(
items: List[MultimodalDataItem], device: torch.device
items: List[MultimodalDataItem],
device: torch.device,
data_embedding_func: DataEmbeddingFunc,
) -> None:
"""Move item features to the target device (in-place, non-blocking)."""
"""Wait for feature readiness and upload unless the encoder defers the move."""
defer_move = _can_skip_pre_embed_feature_move(data_embedding_func)
if host_offload_event is not None and any(
isinstance(item.feature, torch.Tensor) and item.feature.is_cpu for item in items
):
if defer_move or device.type != "cuda":
# Deferred encoders can read CPU subsets of a mixed-device batch.
host_offload_event.synchronize()
else:
torch.cuda.current_stream(device).wait_event(host_offload_event)
if defer_move:
return
for item in items:
if isinstance(item.feature, torch.Tensor) and item.feature.device != device:
item.feature = item.feature.to(device, non_blocking=True)
@@ -270,8 +290,7 @@ def _get_chunked_embedding_full(
embedding_per_req = None
if embedding_per_req is None:
if not _can_skip_pre_embed_feature_move(data_embedding_func):
_move_items_to_device(embedding_items_per_req, device)
_move_items_to_device(embedding_items_per_req, device, data_embedding_func)
embedding = data_embedding_func(embedding_items_per_req)
if isinstance(embedding, list):
# This path caches the combined per-request embedding, so the
@@ -383,8 +402,7 @@ def _batch_encode_per_image_misses(
miss_items = [unique_misses[key][0] for key in ordered_cache_keys]
token_counts = [unique_misses[key][1] for key in ordered_cache_keys]
if not _can_skip_pre_embed_feature_move(data_embedding_func):
_move_items_to_device(miss_items, device)
_move_items_to_device(miss_items, device, data_embedding_func)
all_miss_embedding = data_embedding_func(miss_items)
if isinstance(all_miss_embedding, list):
@@ -461,8 +479,7 @@ def _get_chunked_embedding_by_item(
if miss_items:
miss_item_list = [item for _, item, _, _ in miss_items]
if not _can_skip_pre_embed_feature_move(data_embedding_func):
_move_items_to_device(miss_item_list, device)
_move_items_to_device(miss_item_list, device, data_embedding_func)
all_miss_embedding = data_embedding_func(miss_item_list)
if isinstance(all_miss_embedding, list):
+49 -19
View File
@@ -20,6 +20,7 @@ from torch import nn
from sglang.kernels.ops.memory.gpu_tensor_hash import gpu_tensor_hash
from sglang.srt.environ import envs
from sglang.srt.managers import mm_schedule
from sglang.srt.managers.io_struct import (
BaseBatchReq,
TokenizedEmbeddingReqInput,
@@ -661,6 +662,7 @@ def general_mm_embed_routine(
data_embedding_funcs: Dict[Modality, DataEmbeddingFunc] = None,
placeholder_tokens: Optional[dict[Modality, List[int]]] = None,
use_deepstack: Dict[Modality, bool] = {},
feature_dtypes: Optional[Dict[Modality, torch.dtype]] = None,
**kwargs,
) -> torch.Tensor:
"""
@@ -673,6 +675,7 @@ def general_mm_embed_routine(
data_embedding_funcs: A dictionary mapping from modality type to the corresponding embedding function.
placeholder_tokens: Token IDs for multimodal placeholders
use_deepstack: Whether to use deepstack embeddings for each modality, default False
feature_dtypes: Optional per-modality dtypes for preparing CUDA features before cache lookup.
**kwargs: Additional arguments passed to language model
Returns:
@@ -689,6 +692,23 @@ def general_mm_embed_routine(
mm_inputs_list = [
mm_input for mm_input in forward_batch.mm_inputs if mm_input is not None
]
if feature_dtypes:
for mm_input in mm_inputs_list:
for item in mm_input.mm_items:
dtype = feature_dtypes.get(item.modality)
feature = item.feature
if (
dtype is not None
and isinstance(feature, torch.Tensor)
and feature.is_cuda
and feature.dtype != dtype
):
# The cast may still read the source after replacing it.
feature.record_stream(
torch.cuda.current_stream(feature.device)
)
item.feature = feature.to(dtype=dtype)
del feature
extend_prefix_lens = [
prefix_len
for i, prefix_len in enumerate(forward_batch.extend_prefix_lens_cpu)
@@ -742,27 +762,37 @@ def general_mm_embed_routine(
# best-effort, offloading to CPU ensures we have a reliable fallback
# if a cache miss occurs in subsequent chunks, while still freeing up
# critical GPU memory.
if mm_inputs_list:
if mm_inputs_list and input_embeds.is_cuda:
language_only = get_disagg().language_only
stream = torch.cuda.current_stream(input_embeds.device)
offloaded = False
for mm_input_obj in mm_inputs_list:
if mm_input_obj and hasattr(mm_input_obj, "mm_items"):
for mm_item in mm_input_obj.mm_items:
feature = getattr(mm_item, "feature", None)
if isinstance(feature, torch.Tensor) and feature.is_cuda:
mm_item.feature = feature.to("cpu", non_blocking=True)
if get_disagg().language_only:
precomputed_embeddings = getattr(
mm_item, "precomputed_embeddings", None
for mm_item in mm_input_obj.mm_items:
feature = mm_item.feature
if isinstance(feature, torch.Tensor) and feature.is_cuda:
# The transport reconstructed this block on the scheduler stream
# and nothing else pins it to ours: without record_stream the
# allocator recycles it into the next scheduler allocation while
# the encoder read and this copy are still queued here.
feature.record_stream(stream)
mm_item.feature = feature.to("cpu", non_blocking=True)
offloaded = True
if language_only:
precomputed = mm_item.precomputed_embeddings
if (
isinstance(precomputed, torch.Tensor)
and precomputed.is_cuda
and not mm_item.keep_device_embedding
):
precomputed.record_stream(stream)
mm_item.precomputed_embeddings = precomputed.to(
"cpu", non_blocking=True
)
if (
isinstance(precomputed_embeddings, torch.Tensor)
and precomputed_embeddings.is_cuda
and not mm_item.keep_device_embedding
):
mm_item.precomputed_embeddings = (
precomputed_embeddings.to(
"cpu", non_blocking=True
)
)
offloaded = True
if offloaded:
if mm_schedule.host_offload_event is None:
mm_schedule.host_offload_event = torch.cuda.Event()
mm_schedule.host_offload_event.record(stream)
forward_batch.mm_inputs = None
forward_batch.mm_input_embeds = (
input_embeds.clone()