[VLM] fix: repair multimodal VMM lease cleanup and source-stream ordering (#43475)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Mick
2026-10-11 19:13:39 +08:00
committed by GitHub
co-authored by Mick Qian
parent efd14cfb66
commit 04f740580c
19 changed files with 576 additions and 113 deletions
+3 -24
View File
@@ -9,7 +9,7 @@ from sglang.srt.managers.schedule_batch import MultimodalDataItem
from sglang.srt.mem_cache.multimodal_cache import EmbeddingResult, MultiModalStaticCache
from sglang.srt.multimodal.evs import EVSEmbeddingResult
from sglang.srt.multimodal.transport.cuda_ipc import BORROW_CUDA_IPC_FEATURE_KEY
from sglang.srt.runtime_context import get_parallel, get_schedule
from sglang.srt.runtime_context import get_schedule
from sglang.srt.utils import is_hip, is_npu, is_xpu
from sglang.srt.utils.async_probe import maybe_assert_sum
from sglang.utils import logger
@@ -243,26 +243,6 @@ def _move_items_to_device(
item.feature = item.feature.to(device, non_blocking=True)
def _acknowledge_deferred_cuda_ipc_cache_hits(
items: List[MultimodalDataItem],
) -> None:
"""Release lazy Kimi IPC slices when a cached embedding skips ViT.
On an encoder-DP miss, exactly one rank copies an image and acknowledges
the full TP group. On a cache hit no rank copies it, so rank zero performs
the equivalent single acknowledgement. This preserves the fixed-pool
lifecycle without reintroducing an unnecessary GPU-to-GPU copy.
"""
parallel = get_parallel()
if parallel.attn_tp_rank != 0:
return
# The pool's recycler counts the whole TP group, so the acknowledgement must
# match that count even when an attention subgroup is smaller.
consumer_count = max(parallel.tp_size, 1)
for item in items:
item.acknowledge_deferred_cuda_ipc_feature(consumer_count)
def _item_overlap(
item: MultimodalDataItem, chunk_start: int, chunk_end: int
) -> Optional[int]:
@@ -330,8 +310,8 @@ def _get_chunked_embedding_full(
else embedding
)
embedding_cache.set(embedding_items_hash, embedding_per_req)
else:
_acknowledge_deferred_cuda_ipc_cache_hits(embedding_items_per_req)
# cache hits keep raw-input leases until request cleanup for eviction/re-prefill
if isinstance(embedding_per_req, EVSEmbeddingResult):
item = embedding_items_per_req[0]
@@ -492,7 +472,6 @@ def _get_chunked_embedding_by_item(
cached_token_count = _embedding_token_count(cached_embedding)
if cached_token_count == expected_token_count:
cached_embeddings[idx] = cached_embedding
_acknowledge_deferred_cuda_ipc_cache_hits([item])
else:
_discard_mismatched_cached_embedding(
item.hash, expected_token_count, cached_token_count
+16 -3
View File
@@ -1077,6 +1077,8 @@ class Req(ReqDllmMixin):
self.dllm_initialized: bool = False
self.session = session
# media added by this turn, excluding the session's borrowed history
self._session_mm_items: List[MultimodalDataItem] = []
self.session_id = session_id
# Used by the session radix cache to reject registration after a close/reopen.
self.session_generation: Optional[int] = None
@@ -1519,6 +1521,7 @@ class Req(ReqDllmMixin):
def _extend_session_image_inputs(self, image_inputs):
"""Append media while preserving the saved session and its position history."""
self._session_mm_items.extend(image_inputs.mm_items)
# Padding can change token values without changing their count.
self.full_untruncated_fill_ids = array("q")
if self.multimodal_inputs is not None:
@@ -1564,6 +1567,18 @@ class Req(ReqDllmMixin):
positions.max() + 1 - positions.shape[1]
).reshape(1, 1)
def release_mm_inputs_on_abort(self) -> None:
"""Release this turn's media without invalidating session history."""
if self.session is None:
if self.multimodal_inputs is not None:
self.multimodal_inputs.release_features()
else:
for item in self._session_mm_items:
item.release_transport_proxies()
item.feature = None
self._session_mm_items.clear()
self.multimodal_inputs = None
def finished(self) -> bool:
# Whether request reached finished condition
return self.finished_reason is not None
@@ -2110,9 +2125,7 @@ class Req(ReqDllmMixin):
logger.error(f"{error_msg}, {self.rid=}")
# Session requests share historical multimodal inputs with their prior
# request. The session owns and releases those features when it closes.
if self.multimodal_inputs is not None and self.session is None:
self.multimodal_inputs.release_features()
self.multimodal_inputs = None
self.release_mm_inputs_on_abort()
self.grammar = None
self.origin_input_ids = array(
"q", [0]
+51 -1
View File
@@ -23,7 +23,7 @@ import sys
import time
from array import array
from collections import deque
from contextlib import contextmanager, nullcontext
from contextlib import ExitStack, contextmanager, nullcontext
from functools import partial
from http import HTTPStatus
from typing import TYPE_CHECKING, Any, Deque, Dict, List, Optional, Set, Tuple, Union
@@ -210,6 +210,7 @@ from sglang.srt.managers.prefill_delayer import (
from sglang.srt.managers.schedule_batch import (
FINISH_ABORT,
MultimodalInputs,
MultimodalProcessorOutput,
NextBatchPlan,
Req,
ScheduleBatch,
@@ -2235,6 +2236,12 @@ class Scheduler(
)
error_msg = f"Multimodal feature reconstruction failed ({details})"
logger.error(error_msg)
if isinstance(
tokenized_req.mm_inputs,
(MultimodalInputs, MultimodalProcessorOutput),
):
for item in tokenized_req.mm_inputs.mm_items:
item.release_transport_proxies()
tokenized_req.mm_inputs = None
request_errors.append(error_msg)
else:
@@ -2779,6 +2786,8 @@ class Scheduler(
continue
# For session requests, keep mm_inputs for the next request
if req.session:
if isinstance(req.finished_reason, FINISH_ABORT):
req.release_mm_inputs_on_abort()
continue
# For non-session requests, clear features and mm_inputs
mm_inputs.release_features()
@@ -2789,6 +2798,23 @@ class Scheduler(
recv_req: TokenizedGenerateReqInput,
*,
mm_input_error: Optional[str] = None,
):
with ExitStack() as mm_cleanup:
if isinstance(
recv_req.mm_inputs, (MultimodalInputs, MultimodalProcessorOutput)
):
for item in recv_req.mm_inputs.mm_items:
mm_cleanup.callback(item.release_transport_proxies)
self._handle_generate_request(
recv_req, mm_cleanup, mm_input_error=mm_input_error
)
def _handle_generate_request(
self,
recv_req: TokenizedGenerateReqInput,
mm_cleanup: ExitStack,
*,
mm_input_error: Optional[str] = None,
):
# Route: normal request / session request / session-not-found
session_id = (
@@ -3047,6 +3073,8 @@ class Scheduler(
req.origin_input_ids, image_inputs, self.pad_input_ids_func
)
req.extend_image_inputs(image_inputs)
# the request now owns cleanup, including rejection after padding
mm_cleanup.pop_all()
self._maybe_compute_mrope_positions(req)
if len(req.origin_input_ids) >= self.max_req_input_len:
@@ -3279,6 +3307,7 @@ class Scheduler(
# detach lives in `StreamingSession.find_active_slot`, which only runs
# while scheduling; a session left in-flight rejects every later request.
if req.session is not None and req.session.streaming:
req.release_mm_inputs_on_abort()
req.session.abort_req()
req.session = None
# `beam_coordinator.validate_and_init` counts the group in ahead of the
@@ -3364,7 +3393,10 @@ class Scheduler(
def _release_aborted_request(self, req: Req) -> None:
"""Drop the cache-side state an aborted request left behind."""
req.release_mm_inputs_on_abort()
self.tree_cache.finish(req.cache_request_handle, CacheRequestOutcome.ABORT)
if req.session is not None and req.session.streaming:
req.session.abort_req()
def _abort_on_queued_limit(self, recv_req: Req) -> bool:
"""Abort an incoming or existing request if the waiting queue is full. Returns True if the incoming request is aborted."""
@@ -3472,6 +3504,23 @@ class Scheduler(
recv_req: TokenizedEmbeddingReqInput,
*,
mm_input_error: Optional[str] = None,
):
with ExitStack() as mm_cleanup:
if isinstance(
recv_req.mm_inputs, (MultimodalInputs, MultimodalProcessorOutput)
):
for item in recv_req.mm_inputs.mm_items:
mm_cleanup.callback(item.release_transport_proxies)
self._handle_embedding_request(
recv_req, mm_cleanup, mm_input_error=mm_input_error
)
def _handle_embedding_request(
self,
recv_req: TokenizedEmbeddingReqInput,
mm_cleanup: ExitStack,
*,
mm_input_error: Optional[str] = None,
):
req = Req(
recv_req.rid,
@@ -3528,6 +3577,7 @@ class Scheduler(
)
req.extend_image_inputs(image_inputs)
mm_cleanup.pop_all()
self._maybe_compute_mrope_positions(req)
if len(req.origin_input_ids) >= self.max_req_input_len:
+1 -1
View File
@@ -4597,7 +4597,7 @@ class DeepseekV4ForCausalLM(nn.Module):
spans = []
device, dtype = self.image_start.device, self.image_start.dtype
for item in items:
item.reconstruct(device.index, ipc_consumer_count=self.tp_size)
item.reconstruct(device.index)
h, w = int(item.n_vit_h), int(item.n_vit_w)
pixels = torch.as_tensor(item.feature, device=device)
plan = item.model_specific_data.get(GPU_PLAN_KEY)
+3 -11
View File
@@ -49,7 +49,6 @@ from sglang.srt.multimodal.mm_utils import (
from sglang.srt.runtime_context import (
get_exec,
get_mm,
get_parallel,
)
from sglang.srt.utils import add_prefix, is_cuda, is_npu
@@ -724,14 +723,9 @@ class KimiK25ForConditionalGeneration(nn.Module):
CUDA IPC features are intentionally reconstructed after the image
assignment. Each image therefore crosses the tokenizer/scheduler
boundary once instead of once per TP rank. The selected consumer
acknowledges the entire TP group so the bounded IPC pool remains
recyclable.
boundary once instead of once per TP rank. Unused ranks keep their
lease until request cleanup so cache eviction can re-run the encoder.
"""
# Match the configured TP consumer count captured when the
# tokenizer creates MmItemMemoryPool. A live attention subgroup
# size could leave acknowledgements missing and strand the lease.
ipc_consumer_count = max(get_parallel().tp_size, 1)
device_index = device.index
if device.type == "cuda" and device_index is None:
device_index = torch.cuda.current_device()
@@ -740,9 +734,7 @@ class KimiK25ForConditionalGeneration(nn.Module):
for image_index in image_indices:
item = items[image_index]
if device.type == "cuda":
item.reconstruct(
device_index, ipc_consumer_count=ipc_consumer_count
)
item.reconstruct(device_index)
feature = item.feature
if not isinstance(feature, torch.Tensor):
raise TypeError(
+1 -7
View File
@@ -3613,10 +3613,6 @@ class KimiK3ForConditionalGeneration(nn.Module):
LOCAL_PREPROCESSED_KEY,
)
# Match the configured TP consumer count captured when the
# tokenizer creates MmItemMemoryPool. A live attention subgroup
# size could leave acknowledgements missing and strand the lease.
ipc_consumer_count = max(get_parallel().tp_size, 1)
device_index = device.index
if device.type == "cuda" and device_index is None:
device_index = torch.cuda.current_device()
@@ -3625,9 +3621,7 @@ class KimiK3ForConditionalGeneration(nn.Module):
for image_index in image_indices:
item = items[image_index]
if device.type == "cuda":
item.reconstruct(
device_index, ipc_consumer_count=ipc_consumer_count
)
item.reconstruct(device_index)
selected_items.append(item)
locally_preprocessed = [
+1 -1
View File
@@ -1521,7 +1521,7 @@ class Qwen3VLForConditionalGeneration(nn.Module):
(item, proxy, feature_offset, borrowed.shape[0])
)
elif RETAINED_CUDA_IPC_FEATURE_PROXY_KEY not in model_specific_data:
item.reconstruct(device_index, ipc_consumer_count=consumer_count)
item.reconstruct(device_index)
features.append(item.feature)
feature_offset += item.feature.shape[0]
try:
@@ -44,6 +44,7 @@ from sglang.srt.multimodal.transport.cuda_ipc import (
MmItemMemoryPool,
get_mm_feature_pool_size_per_worker,
)
from sglang.srt.multimodal.transport.memory_pool import get_mm_feature_consumer_count
from sglang.srt.runtime_context import (
get_exec,
get_mm,
@@ -450,7 +451,7 @@ class BaseMultimodalProcessor(ABC):
per_worker_pool_size,
MM_ITEM_MEMORY_POOL_RECYCLE_INTERVAL,
self.server_args.base_gpu_id,
self.server_args.tp_size,
get_mm_feature_consumer_count(),
)
@property
@@ -16,6 +16,14 @@ DATA_ALIGNMENT = 256
DEFAULT_MAX_INFLIGHT_SLICES = 4096
def get_mm_feature_consumer_count() -> int:
"""Number of scheduler ranks receiving one routed multimodal request."""
parallel = get_parallel()
if parallel.attn_dp_enabled:
return parallel.tp_size // parallel.num_dp_ranks
return parallel.tp_size
def align_up(value: int, alignment: int) -> int:
return ((value + alignment - 1) // alignment) * alignment
@@ -59,9 +67,8 @@ def resolve_consumer_rank(
return 0
if consumer_rank is None:
try:
# Use the global TP rank. An attention/DCP subgroup rank can alias
# another consumer's acknowledgement slot.
rank = int(get_parallel().tp_rank)
# DP groups are contiguous; attention/DCP subgroup ranks can alias
rank = int(get_parallel().tp_rank) % total_consumer_count
except Exception as exc:
raise RuntimeError(
f"Cannot resolve the {transport_name} consumer rank before "
@@ -80,6 +80,7 @@ class StreamingSession:
if slot is None or not slot.kv.holds_kv:
return None
if req.to_finish is not None:
req.release_mm_inputs_on_abort()
req.session.abort_req()
req.session = None
return None
@@ -160,6 +161,7 @@ class StreamingSession:
# Hand the record and the tree lock back; the caller releases them.
del self.slots[req.session.session_id]
req.lock, slot.lock = slot.lock, None
req.release_mm_inputs_on_abort()
req.session.abort_req()
return False
@@ -17,6 +17,7 @@ from sglang.srt.managers.schedule_batch import (
MultimodalDataItem,
MultimodalProcessorOutput,
)
from sglang.srt.multimodal.transport.memory_pool import get_mm_feature_consumer_count
from sglang.srt.runtime_context import (
get_mm,
get_parallel,
@@ -192,9 +193,7 @@ def _contains_tensor_container(value) -> bool:
def get_vmm_feature_consumer_count() -> int:
if get_parallel().attn_dp_enabled:
return get_parallel().tp_size // get_parallel().num_dp_ranks
return get_parallel().tp_size
return get_mm_feature_consumer_count()
class CudaVmmMemoryPool:
@@ -402,6 +401,10 @@ class CudaVmmMemoryPool:
producer_stream = None
copy_synchronized = False
try:
if tensor.is_cuda:
self._publish_stream.wait_stream(
torch.cuda.current_stream(tensor.device)
)
with (
torch.cuda.device(self.device_index),
torch.cuda.stream(self._publish_stream),
@@ -477,6 +480,8 @@ class CudaVmmMemoryPool:
producer_stream = None
copy_synchronized = False
try:
for device in {tensor.device for tensor in tensors if tensor.is_cuda}:
self._publish_stream.wait_stream(torch.cuda.current_stream(device))
with (
torch.cuda.device(self.device_index),
torch.cuda.stream(self._publish_stream),
@@ -604,20 +609,19 @@ class CudaVmmMemoryPool:
def _recycle_loop(self) -> None:
while not self._stop_recycler.wait(self._recycle_interval):
try:
with self._lock:
self._recycle_chunks()
self._merge_chunks()
self._recycle_chunks()
except Exception as error:
logger.exception("CUDA VMM multimodal pool recycle failed")
self._pool_error = error
self._stop_recycler.set()
def _recycle_chunks(self) -> None:
if not self.occupied_chunks:
with self._lock:
chunks = tuple(self.occupied_chunks)
if not chunks:
return
remaining = []
recycled = []
# CUDA polling must not hold the allocator lock needed by publishers
with (
torch.cuda.device(self.device_index),
torch.cuda.stream(self._recycle_stream),
@@ -628,23 +632,23 @@ class CudaVmmMemoryPool:
chunk.start : chunk.start
+ self.consumer_count * _CONTROL_WORD_BYTES
].view(torch.int32)
for chunk in self.occupied_chunks
for chunk in chunks
]
)
acknowledgement_counts = (
torch.count_nonzero(acknowledgement_words, dim=1).cpu().tolist()
)
for chunk, acknowledgement_count in zip(
self.occupied_chunks, acknowledgement_counts, strict=True
):
if acknowledgement_count == self.consumer_count:
recycled.append(_CudaVmmMemoryChunk(chunk.start, chunk.end))
else:
remaining.append(chunk)
self.available_chunks.extend(recycled)
self.occupied_chunks = remaining
with self._lock:
live = {chunk.start: chunk for chunk in self.occupied_chunks}
for chunk, count in zip(chunks, acknowledgement_counts, strict=True):
# cancellation may have freed and republished the same offset
if count == self.consumer_count and live.get(chunk.start) is chunk:
self.occupied_chunks.remove(chunk)
self.available_chunks.append(
_CudaVmmMemoryChunk(chunk.start, chunk.end)
)
self._merge_chunks()
def _merge_chunks(self) -> None:
merged = []
@@ -809,6 +813,10 @@ class CudaVmmTensorTransportProxy(CudaIpcTensorTransportProxy):
device_index=device_index,
)
def borrow_on_target_device(self, rebuild_device_idx: int) -> None:
# VMM copies into consumer-owned storage; the IPC borrow protocol differs
return None
def _acknowledgement_range(self, consumer_count: int) -> tuple[int, int]:
if consumer_count <= 0:
raise ValueError("consumer_count must be positive")
@@ -816,6 +824,10 @@ class CudaVmmTensorTransportProxy(CudaIpcTensorTransportProxy):
return 0, self.consumer_count
parallel = get_parallel()
if consumer_count == 1:
# attention/DCP subgroup ranks can alias; DP groups are contiguous
slot = parallel.tp_rank % self.consumer_count
return slot, slot + 1
group_start = parallel.attn_cp_rank * parallel.attn_tp_size
group_end = group_start + parallel.attn_tp_size
if not 0 <= group_start < group_end <= self.consumer_count:
@@ -824,9 +836,6 @@ class CudaVmmTensorTransportProxy(CudaIpcTensorTransportProxy):
f"[{group_start}, {group_end}) is outside "
f"consumer_count={self.consumer_count}"
)
if consumer_count == 1:
slot = group_start + parallel.attn_tp_rank
return slot, slot + 1
if consumer_count == parallel.attn_tp_size:
return group_start, group_end
raise ValueError(
@@ -1033,18 +1042,49 @@ class CudaVmmFeatureTransport:
if self._publisher_executor is None:
raise RuntimeError("CUDA VMM feature transport is shutting down")
future = asyncio.get_running_loop().run_in_executor(
self._publisher_executor,
self.prepare_for_dispatch,
mm_inputs_batch,
# CUDA current streams are thread-local; carry readiness into the publisher.
ready_events = []
devices = {
tensor.device
for mm_inputs in mm_inputs_batch
if mm_inputs is not None
for item in mm_inputs.mm_items
for tensor in (item.feature, item.precomputed_embeddings)
if isinstance(tensor, torch.Tensor) and tensor.is_cuda
}
for device in devices:
event = torch.cuda.Event()
event.record(torch.cuda.current_stream(device))
ready_events.append((device, event))
future = self._publisher_executor.submit(
self._prepare_after_source_events, mm_inputs_batch, ready_events
)
try:
return await asyncio.shield(future)
return await asyncio.shield(asyncio.wrap_future(future))
except asyncio.CancelledError:
prepared_mm_items = await future
self.cancel_for_dispatch(prepared_mm_items)
# cleanup must survive a second task cancellation or loop shutdown
future.add_done_callback(self._cancel_completed_publish)
raise
def _prepare_after_source_events(
self,
mm_inputs_batch: Iterable[MultimodalProcessorOutput | None],
ready_events: list[tuple[torch.device, torch.cuda.Event]],
) -> list[MultimodalDataItem]:
for device, event in ready_events:
# Both VMM copies and a pool-full CPU fallback must see ready input.
torch.cuda.current_stream(device).wait_event(event)
return self.prepare_for_dispatch(mm_inputs_batch)
def _cancel_completed_publish(self, future: concurrent.futures.Future) -> None:
if future.cancelled() or future.exception() is not None:
# failed publication already rolls back its slices
return
try:
self.cancel_for_dispatch(future.result())
except Exception:
logger.exception("Failed to cancel abandoned CUDA VMM publication")
def prepare_for_dispatch(
self,
mm_inputs_batch: Iterable[MultimodalProcessorOutput | None],