diff --git a/python/sglang/srt/managers/mm_schedule.py b/python/sglang/srt/managers/mm_schedule.py index 749d5d9c71..cfbdd2f95a 100644 --- a/python/sglang/srt/managers/mm_schedule.py +++ b/python/sglang/srt/managers/mm_schedule.py @@ -9,7 +9,7 @@ from sglang.srt.managers.schedule_batch import MultimodalDataItem from sglang.srt.mem_cache.multimodal_cache import EmbeddingResult, MultiModalStaticCache from sglang.srt.multimodal.evs import EVSEmbeddingResult from sglang.srt.multimodal.transport.cuda_ipc import BORROW_CUDA_IPC_FEATURE_KEY -from sglang.srt.runtime_context import get_parallel, get_schedule +from sglang.srt.runtime_context import get_schedule from sglang.srt.utils import is_hip, is_npu, is_xpu from sglang.srt.utils.async_probe import maybe_assert_sum from sglang.utils import logger @@ -243,26 +243,6 @@ def _move_items_to_device( item.feature = item.feature.to(device, non_blocking=True) -def _acknowledge_deferred_cuda_ipc_cache_hits( - items: List[MultimodalDataItem], -) -> None: - """Release lazy Kimi IPC slices when a cached embedding skips ViT. - - On an encoder-DP miss, exactly one rank copies an image and acknowledges - the full TP group. On a cache hit no rank copies it, so rank zero performs - the equivalent single acknowledgement. This preserves the fixed-pool - lifecycle without reintroducing an unnecessary GPU-to-GPU copy. - """ - parallel = get_parallel() - if parallel.attn_tp_rank != 0: - return - # The pool's recycler counts the whole TP group, so the acknowledgement must - # match that count even when an attention subgroup is smaller. - consumer_count = max(parallel.tp_size, 1) - for item in items: - item.acknowledge_deferred_cuda_ipc_feature(consumer_count) - - def _item_overlap( item: MultimodalDataItem, chunk_start: int, chunk_end: int ) -> Optional[int]: @@ -330,8 +310,8 @@ def _get_chunked_embedding_full( else embedding ) embedding_cache.set(embedding_items_hash, embedding_per_req) - else: - _acknowledge_deferred_cuda_ipc_cache_hits(embedding_items_per_req) + + # cache hits keep raw-input leases until request cleanup for eviction/re-prefill if isinstance(embedding_per_req, EVSEmbeddingResult): item = embedding_items_per_req[0] @@ -492,7 +472,6 @@ def _get_chunked_embedding_by_item( cached_token_count = _embedding_token_count(cached_embedding) if cached_token_count == expected_token_count: cached_embeddings[idx] = cached_embedding - _acknowledge_deferred_cuda_ipc_cache_hits([item]) else: _discard_mismatched_cached_embedding( item.hash, expected_token_count, cached_token_count diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index a964f40505..7499375032 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -1077,6 +1077,8 @@ class Req(ReqDllmMixin): self.dllm_initialized: bool = False self.session = session + # media added by this turn, excluding the session's borrowed history + self._session_mm_items: List[MultimodalDataItem] = [] self.session_id = session_id # Used by the session radix cache to reject registration after a close/reopen. self.session_generation: Optional[int] = None @@ -1519,6 +1521,7 @@ class Req(ReqDllmMixin): def _extend_session_image_inputs(self, image_inputs): """Append media while preserving the saved session and its position history.""" + self._session_mm_items.extend(image_inputs.mm_items) # Padding can change token values without changing their count. self.full_untruncated_fill_ids = array("q") if self.multimodal_inputs is not None: @@ -1564,6 +1567,18 @@ class Req(ReqDllmMixin): positions.max() + 1 - positions.shape[1] ).reshape(1, 1) + def release_mm_inputs_on_abort(self) -> None: + """Release this turn's media without invalidating session history.""" + if self.session is None: + if self.multimodal_inputs is not None: + self.multimodal_inputs.release_features() + else: + for item in self._session_mm_items: + item.release_transport_proxies() + item.feature = None + self._session_mm_items.clear() + self.multimodal_inputs = None + def finished(self) -> bool: # Whether request reached finished condition return self.finished_reason is not None @@ -2110,9 +2125,7 @@ class Req(ReqDllmMixin): logger.error(f"{error_msg}, {self.rid=}") # Session requests share historical multimodal inputs with their prior # request. The session owns and releases those features when it closes. - if self.multimodal_inputs is not None and self.session is None: - self.multimodal_inputs.release_features() - self.multimodal_inputs = None + self.release_mm_inputs_on_abort() self.grammar = None self.origin_input_ids = array( "q", [0] diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 601306bf00..ca6a0ad21f 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -23,7 +23,7 @@ import sys import time from array import array from collections import deque -from contextlib import contextmanager, nullcontext +from contextlib import ExitStack, contextmanager, nullcontext from functools import partial from http import HTTPStatus from typing import TYPE_CHECKING, Any, Deque, Dict, List, Optional, Set, Tuple, Union @@ -210,6 +210,7 @@ from sglang.srt.managers.prefill_delayer import ( from sglang.srt.managers.schedule_batch import ( FINISH_ABORT, MultimodalInputs, + MultimodalProcessorOutput, NextBatchPlan, Req, ScheduleBatch, @@ -2235,6 +2236,12 @@ class Scheduler( ) error_msg = f"Multimodal feature reconstruction failed ({details})" logger.error(error_msg) + if isinstance( + tokenized_req.mm_inputs, + (MultimodalInputs, MultimodalProcessorOutput), + ): + for item in tokenized_req.mm_inputs.mm_items: + item.release_transport_proxies() tokenized_req.mm_inputs = None request_errors.append(error_msg) else: @@ -2779,6 +2786,8 @@ class Scheduler( continue # For session requests, keep mm_inputs for the next request if req.session: + if isinstance(req.finished_reason, FINISH_ABORT): + req.release_mm_inputs_on_abort() continue # For non-session requests, clear features and mm_inputs mm_inputs.release_features() @@ -2789,6 +2798,23 @@ class Scheduler( recv_req: TokenizedGenerateReqInput, *, mm_input_error: Optional[str] = None, + ): + with ExitStack() as mm_cleanup: + if isinstance( + recv_req.mm_inputs, (MultimodalInputs, MultimodalProcessorOutput) + ): + for item in recv_req.mm_inputs.mm_items: + mm_cleanup.callback(item.release_transport_proxies) + self._handle_generate_request( + recv_req, mm_cleanup, mm_input_error=mm_input_error + ) + + def _handle_generate_request( + self, + recv_req: TokenizedGenerateReqInput, + mm_cleanup: ExitStack, + *, + mm_input_error: Optional[str] = None, ): # Route: normal request / session request / session-not-found session_id = ( @@ -3047,6 +3073,8 @@ class Scheduler( req.origin_input_ids, image_inputs, self.pad_input_ids_func ) req.extend_image_inputs(image_inputs) + # the request now owns cleanup, including rejection after padding + mm_cleanup.pop_all() self._maybe_compute_mrope_positions(req) if len(req.origin_input_ids) >= self.max_req_input_len: @@ -3279,6 +3307,7 @@ class Scheduler( # detach lives in `StreamingSession.find_active_slot`, which only runs # while scheduling; a session left in-flight rejects every later request. if req.session is not None and req.session.streaming: + req.release_mm_inputs_on_abort() req.session.abort_req() req.session = None # `beam_coordinator.validate_and_init` counts the group in ahead of the @@ -3364,7 +3393,10 @@ class Scheduler( def _release_aborted_request(self, req: Req) -> None: """Drop the cache-side state an aborted request left behind.""" + req.release_mm_inputs_on_abort() self.tree_cache.finish(req.cache_request_handle, CacheRequestOutcome.ABORT) + if req.session is not None and req.session.streaming: + req.session.abort_req() def _abort_on_queued_limit(self, recv_req: Req) -> bool: """Abort an incoming or existing request if the waiting queue is full. Returns True if the incoming request is aborted.""" @@ -3472,6 +3504,23 @@ class Scheduler( recv_req: TokenizedEmbeddingReqInput, *, mm_input_error: Optional[str] = None, + ): + with ExitStack() as mm_cleanup: + if isinstance( + recv_req.mm_inputs, (MultimodalInputs, MultimodalProcessorOutput) + ): + for item in recv_req.mm_inputs.mm_items: + mm_cleanup.callback(item.release_transport_proxies) + self._handle_embedding_request( + recv_req, mm_cleanup, mm_input_error=mm_input_error + ) + + def _handle_embedding_request( + self, + recv_req: TokenizedEmbeddingReqInput, + mm_cleanup: ExitStack, + *, + mm_input_error: Optional[str] = None, ): req = Req( recv_req.rid, @@ -3528,6 +3577,7 @@ class Scheduler( ) req.extend_image_inputs(image_inputs) + mm_cleanup.pop_all() self._maybe_compute_mrope_positions(req) if len(req.origin_input_ids) >= self.max_req_input_len: diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 635a15666f..cd8db6534d 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -4597,7 +4597,7 @@ class DeepseekV4ForCausalLM(nn.Module): spans = [] device, dtype = self.image_start.device, self.image_start.dtype for item in items: - item.reconstruct(device.index, ipc_consumer_count=self.tp_size) + item.reconstruct(device.index) h, w = int(item.n_vit_h), int(item.n_vit_w) pixels = torch.as_tensor(item.feature, device=device) plan = item.model_specific_data.get(GPU_PLAN_KEY) diff --git a/python/sglang/srt/models/kimi_k25.py b/python/sglang/srt/models/kimi_k25.py index 09cc7952c4..711e723fd9 100644 --- a/python/sglang/srt/models/kimi_k25.py +++ b/python/sglang/srt/models/kimi_k25.py @@ -49,7 +49,6 @@ from sglang.srt.multimodal.mm_utils import ( from sglang.srt.runtime_context import ( get_exec, get_mm, - get_parallel, ) from sglang.srt.utils import add_prefix, is_cuda, is_npu @@ -724,14 +723,9 @@ class KimiK25ForConditionalGeneration(nn.Module): CUDA IPC features are intentionally reconstructed after the image assignment. Each image therefore crosses the tokenizer/scheduler - boundary once instead of once per TP rank. The selected consumer - acknowledges the entire TP group so the bounded IPC pool remains - recyclable. + boundary once instead of once per TP rank. Unused ranks keep their + lease until request cleanup so cache eviction can re-run the encoder. """ - # Match the configured TP consumer count captured when the - # tokenizer creates MmItemMemoryPool. A live attention subgroup - # size could leave acknowledgements missing and strand the lease. - ipc_consumer_count = max(get_parallel().tp_size, 1) device_index = device.index if device.type == "cuda" and device_index is None: device_index = torch.cuda.current_device() @@ -740,9 +734,7 @@ class KimiK25ForConditionalGeneration(nn.Module): for image_index in image_indices: item = items[image_index] if device.type == "cuda": - item.reconstruct( - device_index, ipc_consumer_count=ipc_consumer_count - ) + item.reconstruct(device_index) feature = item.feature if not isinstance(feature, torch.Tensor): raise TypeError( diff --git a/python/sglang/srt/models/kimi_k3.py b/python/sglang/srt/models/kimi_k3.py index 47aa75712f..1979a2e76d 100644 --- a/python/sglang/srt/models/kimi_k3.py +++ b/python/sglang/srt/models/kimi_k3.py @@ -3613,10 +3613,6 @@ class KimiK3ForConditionalGeneration(nn.Module): LOCAL_PREPROCESSED_KEY, ) - # Match the configured TP consumer count captured when the - # tokenizer creates MmItemMemoryPool. A live attention subgroup - # size could leave acknowledgements missing and strand the lease. - ipc_consumer_count = max(get_parallel().tp_size, 1) device_index = device.index if device.type == "cuda" and device_index is None: device_index = torch.cuda.current_device() @@ -3625,9 +3621,7 @@ class KimiK3ForConditionalGeneration(nn.Module): for image_index in image_indices: item = items[image_index] if device.type == "cuda": - item.reconstruct( - device_index, ipc_consumer_count=ipc_consumer_count - ) + item.reconstruct(device_index) selected_items.append(item) locally_preprocessed = [ diff --git a/python/sglang/srt/models/qwen3_vl.py b/python/sglang/srt/models/qwen3_vl.py index 9e86550656..9b89678ac3 100644 --- a/python/sglang/srt/models/qwen3_vl.py +++ b/python/sglang/srt/models/qwen3_vl.py @@ -1521,7 +1521,7 @@ class Qwen3VLForConditionalGeneration(nn.Module): (item, proxy, feature_offset, borrowed.shape[0]) ) elif RETAINED_CUDA_IPC_FEATURE_PROXY_KEY not in model_specific_data: - item.reconstruct(device_index, ipc_consumer_count=consumer_count) + item.reconstruct(device_index) features.append(item.feature) feature_offset += item.feature.shape[0] try: diff --git a/python/sglang/srt/multimodal/processors/base_processor.py b/python/sglang/srt/multimodal/processors/base_processor.py index 7b9e979f5f..4e160a764c 100644 --- a/python/sglang/srt/multimodal/processors/base_processor.py +++ b/python/sglang/srt/multimodal/processors/base_processor.py @@ -44,6 +44,7 @@ from sglang.srt.multimodal.transport.cuda_ipc import ( MmItemMemoryPool, get_mm_feature_pool_size_per_worker, ) +from sglang.srt.multimodal.transport.memory_pool import get_mm_feature_consumer_count from sglang.srt.runtime_context import ( get_exec, get_mm, @@ -450,7 +451,7 @@ class BaseMultimodalProcessor(ABC): per_worker_pool_size, MM_ITEM_MEMORY_POOL_RECYCLE_INTERVAL, self.server_args.base_gpu_id, - self.server_args.tp_size, + get_mm_feature_consumer_count(), ) @property diff --git a/python/sglang/srt/multimodal/transport/memory_pool.py b/python/sglang/srt/multimodal/transport/memory_pool.py index dc32990fff..cb5b445802 100644 --- a/python/sglang/srt/multimodal/transport/memory_pool.py +++ b/python/sglang/srt/multimodal/transport/memory_pool.py @@ -16,6 +16,14 @@ DATA_ALIGNMENT = 256 DEFAULT_MAX_INFLIGHT_SLICES = 4096 +def get_mm_feature_consumer_count() -> int: + """Number of scheduler ranks receiving one routed multimodal request.""" + parallel = get_parallel() + if parallel.attn_dp_enabled: + return parallel.tp_size // parallel.num_dp_ranks + return parallel.tp_size + + def align_up(value: int, alignment: int) -> int: return ((value + alignment - 1) // alignment) * alignment @@ -59,9 +67,8 @@ def resolve_consumer_rank( return 0 if consumer_rank is None: try: - # Use the global TP rank. An attention/DCP subgroup rank can alias - # another consumer's acknowledgement slot. - rank = int(get_parallel().tp_rank) + # DP groups are contiguous; attention/DCP subgroup ranks can alias + rank = int(get_parallel().tp_rank) % total_consumer_count except Exception as exc: raise RuntimeError( f"Cannot resolve the {transport_name} consumer rank before " diff --git a/python/sglang/srt/session/streaming_session.py b/python/sglang/srt/session/streaming_session.py index b483e51b76..bd18eef1ca 100644 --- a/python/sglang/srt/session/streaming_session.py +++ b/python/sglang/srt/session/streaming_session.py @@ -80,6 +80,7 @@ class StreamingSession: if slot is None or not slot.kv.holds_kv: return None if req.to_finish is not None: + req.release_mm_inputs_on_abort() req.session.abort_req() req.session = None return None @@ -160,6 +161,7 @@ class StreamingSession: # Hand the record and the tree lock back; the caller releases them. del self.slots[req.session.session_id] req.lock, slot.lock = slot.lock, None + req.release_mm_inputs_on_abort() req.session.abort_req() return False diff --git a/python/sglang/srt/utils/cuda_vmm_transport_utils.py b/python/sglang/srt/utils/cuda_vmm_transport_utils.py index e00d36cb6d..2c395c062c 100644 --- a/python/sglang/srt/utils/cuda_vmm_transport_utils.py +++ b/python/sglang/srt/utils/cuda_vmm_transport_utils.py @@ -17,6 +17,7 @@ from sglang.srt.managers.schedule_batch import ( MultimodalDataItem, MultimodalProcessorOutput, ) +from sglang.srt.multimodal.transport.memory_pool import get_mm_feature_consumer_count from sglang.srt.runtime_context import ( get_mm, get_parallel, @@ -192,9 +193,7 @@ def _contains_tensor_container(value) -> bool: def get_vmm_feature_consumer_count() -> int: - if get_parallel().attn_dp_enabled: - return get_parallel().tp_size // get_parallel().num_dp_ranks - return get_parallel().tp_size + return get_mm_feature_consumer_count() class CudaVmmMemoryPool: @@ -402,6 +401,10 @@ class CudaVmmMemoryPool: producer_stream = None copy_synchronized = False try: + if tensor.is_cuda: + self._publish_stream.wait_stream( + torch.cuda.current_stream(tensor.device) + ) with ( torch.cuda.device(self.device_index), torch.cuda.stream(self._publish_stream), @@ -477,6 +480,8 @@ class CudaVmmMemoryPool: producer_stream = None copy_synchronized = False try: + for device in {tensor.device for tensor in tensors if tensor.is_cuda}: + self._publish_stream.wait_stream(torch.cuda.current_stream(device)) with ( torch.cuda.device(self.device_index), torch.cuda.stream(self._publish_stream), @@ -604,20 +609,19 @@ class CudaVmmMemoryPool: def _recycle_loop(self) -> None: while not self._stop_recycler.wait(self._recycle_interval): try: - with self._lock: - self._recycle_chunks() - self._merge_chunks() + self._recycle_chunks() except Exception as error: logger.exception("CUDA VMM multimodal pool recycle failed") self._pool_error = error self._stop_recycler.set() def _recycle_chunks(self) -> None: - if not self.occupied_chunks: + with self._lock: + chunks = tuple(self.occupied_chunks) + if not chunks: return - remaining = [] - recycled = [] + # CUDA polling must not hold the allocator lock needed by publishers with ( torch.cuda.device(self.device_index), torch.cuda.stream(self._recycle_stream), @@ -628,23 +632,23 @@ class CudaVmmMemoryPool: chunk.start : chunk.start + self.consumer_count * _CONTROL_WORD_BYTES ].view(torch.int32) - for chunk in self.occupied_chunks + for chunk in chunks ] ) acknowledgement_counts = ( torch.count_nonzero(acknowledgement_words, dim=1).cpu().tolist() ) - for chunk, acknowledgement_count in zip( - self.occupied_chunks, acknowledgement_counts, strict=True - ): - if acknowledgement_count == self.consumer_count: - recycled.append(_CudaVmmMemoryChunk(chunk.start, chunk.end)) - else: - remaining.append(chunk) - - self.available_chunks.extend(recycled) - self.occupied_chunks = remaining + with self._lock: + live = {chunk.start: chunk for chunk in self.occupied_chunks} + for chunk, count in zip(chunks, acknowledgement_counts, strict=True): + # cancellation may have freed and republished the same offset + if count == self.consumer_count and live.get(chunk.start) is chunk: + self.occupied_chunks.remove(chunk) + self.available_chunks.append( + _CudaVmmMemoryChunk(chunk.start, chunk.end) + ) + self._merge_chunks() def _merge_chunks(self) -> None: merged = [] @@ -809,6 +813,10 @@ class CudaVmmTensorTransportProxy(CudaIpcTensorTransportProxy): device_index=device_index, ) + def borrow_on_target_device(self, rebuild_device_idx: int) -> None: + # VMM copies into consumer-owned storage; the IPC borrow protocol differs + return None + def _acknowledgement_range(self, consumer_count: int) -> tuple[int, int]: if consumer_count <= 0: raise ValueError("consumer_count must be positive") @@ -816,6 +824,10 @@ class CudaVmmTensorTransportProxy(CudaIpcTensorTransportProxy): return 0, self.consumer_count parallel = get_parallel() + if consumer_count == 1: + # attention/DCP subgroup ranks can alias; DP groups are contiguous + slot = parallel.tp_rank % self.consumer_count + return slot, slot + 1 group_start = parallel.attn_cp_rank * parallel.attn_tp_size group_end = group_start + parallel.attn_tp_size if not 0 <= group_start < group_end <= self.consumer_count: @@ -824,9 +836,6 @@ class CudaVmmTensorTransportProxy(CudaIpcTensorTransportProxy): f"[{group_start}, {group_end}) is outside " f"consumer_count={self.consumer_count}" ) - if consumer_count == 1: - slot = group_start + parallel.attn_tp_rank - return slot, slot + 1 if consumer_count == parallel.attn_tp_size: return group_start, group_end raise ValueError( @@ -1033,18 +1042,49 @@ class CudaVmmFeatureTransport: if self._publisher_executor is None: raise RuntimeError("CUDA VMM feature transport is shutting down") - future = asyncio.get_running_loop().run_in_executor( - self._publisher_executor, - self.prepare_for_dispatch, - mm_inputs_batch, + # CUDA current streams are thread-local; carry readiness into the publisher. + ready_events = [] + devices = { + tensor.device + for mm_inputs in mm_inputs_batch + if mm_inputs is not None + for item in mm_inputs.mm_items + for tensor in (item.feature, item.precomputed_embeddings) + if isinstance(tensor, torch.Tensor) and tensor.is_cuda + } + for device in devices: + event = torch.cuda.Event() + event.record(torch.cuda.current_stream(device)) + ready_events.append((device, event)) + future = self._publisher_executor.submit( + self._prepare_after_source_events, mm_inputs_batch, ready_events ) try: - return await asyncio.shield(future) + return await asyncio.shield(asyncio.wrap_future(future)) except asyncio.CancelledError: - prepared_mm_items = await future - self.cancel_for_dispatch(prepared_mm_items) + # cleanup must survive a second task cancellation or loop shutdown + future.add_done_callback(self._cancel_completed_publish) raise + def _prepare_after_source_events( + self, + mm_inputs_batch: Iterable[MultimodalProcessorOutput | None], + ready_events: list[tuple[torch.device, torch.cuda.Event]], + ) -> list[MultimodalDataItem]: + for device, event in ready_events: + # Both VMM copies and a pool-full CPU fallback must see ready input. + torch.cuda.current_stream(device).wait_event(event) + return self.prepare_for_dispatch(mm_inputs_batch) + + def _cancel_completed_publish(self, future: concurrent.futures.Future) -> None: + if future.cancelled() or future.exception() is not None: + # failed publication already rolls back its slices + return + try: + self.cancel_for_dispatch(future.result()) + except Exception: + logger.exception("Failed to cancel abandoned CUDA VMM publication") + def prepare_for_dispatch( self, mm_inputs_batch: Iterable[MultimodalProcessorOutput | None], diff --git a/test/registered/chunked_prefill/test_mm_chunked_embedding_unit.py b/test/registered/chunked_prefill/test_mm_chunked_embedding_unit.py index 79ebf26151..d03811cc5e 100644 --- a/test/registered/chunked_prefill/test_mm_chunked_embedding_unit.py +++ b/test/registered/chunked_prefill/test_mm_chunked_embedding_unit.py @@ -50,9 +50,7 @@ HIDDEN = 16 @pytest.fixture(autouse=True) def single_process_runtime_context(): - # These mm_utils unit tests exercise cache-hit paths that acknowledge - # deferred CUDA IPC through runtime_context. They do not start an engine, so - # pin the runtime topology to a single-process CPU setup. + # Exercise the embedding path without starting an engine. server_args_override = get_context().override_server_args(tp_size=1) server_args_override.install() try: @@ -74,14 +72,6 @@ CHUNKS = [(0, 8), (8, 8), (16, 8), (24, 6)] _CPU = torch.device("cpu") -@pytest.fixture(autouse=True) -def _skip_cuda_ipc_acknowledgement(monkeypatch): - """Keep CPU embedding tests independent of tensor-parallel runtime state.""" - monkeypatch.setattr( - mm_schedule, "_acknowledge_deferred_cuda_ipc_cache_hits", lambda _items: None - ) - - def _num_tokens(item: MultimodalDataItem) -> int: start, end = item.offsets[0] return end - start + 1 diff --git a/test/registered/unit/managers/test_mm_process_config.py b/test/registered/unit/managers/test_mm_process_config.py index a9c4909efc..addcc388dd 100644 --- a/test/registered/unit/managers/test_mm_process_config.py +++ b/test/registered/unit/managers/test_mm_process_config.py @@ -439,6 +439,25 @@ class TestStreamOrderedMmFeaturePool(CustomTestCase): with get_parallel().override(tp_rank=6, attn_tp_rank=2): self.assertEqual(resolve_consumer_rank(8), 6) + def test_attention_dp_uses_only_the_routed_scheduler_group(self): + from sglang.srt.multimodal.transport.memory_pool import ( + get_mm_feature_consumer_count, + resolve_consumer_rank, + ) + from sglang.srt.runtime_context import get_parallel + + with get_parallel().override( + tp_size=8, + attn_dp_enabled=True, + num_dp_ranks=2, + tp_rank=6, + attn_tp_rank=0, + attn_dcp_size=2, + ): + count = get_mm_feature_consumer_count() + self.assertEqual(count, 4) + self.assertEqual(resolve_consumer_rank(count), 2) + def test_complete_group_acknowledges_each_consumer_slot(self): from sglang.srt.multimodal.transport import memory_pool diff --git a/test/registered/unit/managers/test_multimodal_abort_cleanup.py b/test/registered/unit/managers/test_multimodal_abort_cleanup.py index 8ddc42b2a5..2086f15e80 100644 --- a/test/registered/unit/managers/test_multimodal_abort_cleanup.py +++ b/test/registered/unit/managers/test_multimodal_abort_cleanup.py @@ -1,21 +1,321 @@ +import asyncio +import concurrent.futures +import copy import sys +import threading +from contextlib import nullcontext from types import SimpleNamespace from unittest.mock import MagicMock, patch import pytest +import torch +from sglang.srt.disaggregation.utils import DisaggregationMode +from sglang.srt.managers import mm_schedule +from sglang.srt.managers.io_struct import ( + AbortReq, + SessionParams, + TokenizedGenerateReqInput, +) from sglang.srt.managers.schedule_batch import ( Modality, MultimodalDataItem, MultimodalInputs, + MultimodalProcessorOutput, Req, ) +from sglang.srt.managers.scheduler import Scheduler +from sglang.srt.mem_cache.multimodal_cache import EmbeddingResult, MultiModalStaticCache from sglang.srt.multimodal.transport.cuda_ipc import CudaIpcTensorTransportProxy +from sglang.srt.runtime_context import get_context, get_parallel +from sglang.srt.sampling.sampling_params import SamplingParams +from sglang.srt.session.session_controller import Session +from sglang.srt.utils import cuda_vmm_transport_utils as vmm from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=10, suite="base-a-test-cpu") +@pytest.fixture +def vmm_pool(): + # exercise the real slice allocator and ACK protocol with host-backed storage + pool = object.__new__(vmm.CudaVmmMemoryPool) + pool.memory_pool = torch.zeros(1024, dtype=torch.uint8) + pool.consumer_count = 2 + pool.device_index = 0 + pool._recycle_stream = None + pool._lock = threading.Lock() + pool.available_chunks = [vmm._CudaVmmMemoryChunk(0, 1024)] + pool.occupied_chunks = [] + with ( + get_context().override_server_args(tp_size=2), + patch.object(torch.cuda, "device", return_value=nullcontext()), + patch.object(torch.cuda, "stream", return_value=nullcontext()), + patch.object(torch.cuda, "current_device", return_value=0), + patch.object( + vmm.CudaVmmTensorTransportProxy, + "_pool", + return_value=SimpleNamespace(memory=pool.memory_pool), + ), + ): + yield pool + + +def _publish(pool): + chunk = pool._reserve_chunk(1024) + assert chunk is not None, "a previous request stranded the pool slice" + pool.memory_pool.zero_() + pool.occupied_chunks.append(chunk) + proxy = vmm.CudaVmmTensorTransportProxy( + fabric_handle=b"test", + posix_socket_path=None, + allocation_size=1024, + data_offset=256, + data_nbytes=768, + control_offset=0, + consumer_count=2, + shape=(768,), + dtype=torch.uint8, + ) + return [ + MultimodalDataItem(modality=Modality.IMAGE, feature=copy.deepcopy(proxy)), + MultimodalDataItem(modality=Modality.IMAGE, feature=copy.deepcopy(proxy)), + ] + + +def _rank(rank): + return get_parallel().override( + tp_rank=rank, attn_tp_rank=rank, attn_tp_size=2, attn_cp_rank=0, attn_cp_size=1 + ) + + +def _queued_scheduler(req): + scheduler = object.__new__(Scheduler) + scheduler.waiting_queue = [req] + scheduler.chunked_req = scheduler.mm_receiver = scheduler.dllm_config = None + scheduler.disaggregation_mode = DisaggregationMode.NULL + scheduler.tree_cache = MagicMock() + scheduler.beam_coordinator = MagicMock() + scheduler.ipc_channels = MagicMock() + scheduler.grammar_manager = MagicMock() + scheduler.collect_inflight_reqs = lambda: set() + return scheduler + + +def test_queued_abort_recycles_unconsumed_vmm_slices(vmm_pool): + """Client cancellations before prefill must not exhaust the bounded pool.""" + for _ in range(3): + for rank, item in enumerate(_publish(vmm_pool)): + req = Req("queued-image", "", [1], SamplingParams()) + req.multimodal_inputs = MultimodalInputs(mm_items=[item]) + scheduler = _queued_scheduler(req) + with _rank(rank): + scheduler.abort_request(AbortReq(rid=req.rid)) + assert scheduler.waiting_queue == [] + vmm_pool._recycle_chunks() + assert not vmm_pool.occupied_chunks + + +@pytest.mark.parametrize("path", ["full", "per_image", "batch"]) +def test_embedding_hit_cannot_recycle_another_ranks_live_proxy(vmm_pool, path): + """A rank's cache hit must not retire another rank's proxy before cleanup.""" + items = _publish(vmm_pool) + cache = MultiModalStaticCache(1 << 20) + cached = torch.ones(1, 2) + cache.set(cache.combine_hashes([123]), EmbeddingResult(embedding=cached)) + cache.set(123, EmbeddingResult(embedding=cached)) + + def encode(_): + pytest.fail("cache hit unexpectedly ran the encoder") + + for rank, item in enumerate(items): + item.hash = 123 + item.offsets = [(0, 0)] + with _rank(rank), patch.object(mm_schedule, "embedding_cache", cache): + if path == "full": + embedding, _ = mm_schedule._get_chunked_embedding_full( + encode, + [item], + [(0, 0)], + 0, + 1, + torch.tensor([1]), + torch.device("cpu"), + ) + elif path == "per_image": + [embedding] = mm_schedule._get_chunked_embedding_by_item( + encode, + [item], + 0, + 1, + torch.device("cpu"), + ) + else: + request = mm_schedule.PerImageRequestInfo(0, [item], [(0, 0)], 0, 1) + embeddings = mm_schedule._batch_encode_per_image_misses( + encode, + [request], + torch.device("cpu"), + ) + embedding = embeddings[(123, 1)] + assert torch.equal(embedding, cached) + assert not item.feature._consumer_acknowledged, ( + "cache eviction may need this input again" + ) + for rank, item in enumerate(items): + with _rank(rank): + item.release_transport_proxies() + vmm_pool._recycle_chunks() + assert not vmm_pool.occupied_chunks + + _publish(vmm_pool) + for rank, item in enumerate(items): + with _rank(rank): + MultimodalInputs(mm_items=[item]).release_features() + vmm_pool._recycle_chunks() + assert vmm_pool.occupied_chunks, "late cleanup acknowledged a recycled slice" + + +def test_aborted_session_turn_releases_new_media_not_history(vmm_pool): + """An aborted append owns new media, but only borrows historical features.""" + for rank, item in enumerate(_publish(vmm_pool)): + session = Session(100, streaming=True) + session._inflight = True + history = MultimodalDataItem(modality=Modality.IMAGE, feature=torch.ones(3)) + req = Req("session-image", "", [1], SamplingParams(), session=session) + req.multimodal_inputs = MultimodalInputs(mm_items=[history]) + req.extend_image_inputs(MultimodalInputs(mm_items=[item])) + with _rank(rank): + _queued_scheduler(req).abort_request(AbortReq(rid=req.rid)) + assert torch.equal(history.feature, torch.ones(3)) + assert not session._inflight + vmm_pool._recycle_chunks() + assert not vmm_pool.occupied_chunks + + +def test_rejected_session_releases_media_before_request_admission(vmm_pool): + """An invalid session is rejected before the request attaches its media.""" + for rank, item in enumerate(_publish(vmm_pool)): + recv = TokenizedGenerateReqInput( + rid="invalid-session", + input_text="", + input_ids=[1], + input_embeds=None, + mm_inputs=MultimodalProcessorOutput(mm_items=[item]), + token_type_ids=None, + sampling_params=SamplingParams(), + return_logprob=False, + logprob_start_len=-1, + top_logprobs_num=0, + token_ids_logprob=None, + stream=False, + session_params=SessionParams(id="missing"), + ) + scheduler = _queued_scheduler(None) + scheduler.session_controller = {} + scheduler.model_config = SimpleNamespace(vocab_size=10) + scheduler.tokenizer = None + scheduler.init_req_max_new_tokens = lambda req: None + scheduler._add_request_to_queue = lambda req: None + with _rank(rank): + scheduler.handle_generate_request(recv) + vmm_pool._recycle_chunks() + assert not vmm_pool.occupied_chunks + + +def test_other_rank_materialization_failure_releases_local_proxies(vmm_pool): + """A healthy rank still owns lazy slices when another rank rejects the input.""" + for rank, item in enumerate(_publish(vmm_pool)): + recv = TokenizedGenerateReqInput( + rid="peer-failure", + input_text="", + input_ids=[1], + input_embeds=None, + mm_inputs=MultimodalInputs(mm_items=[item]), + token_type_ids=None, + sampling_params=SamplingParams(), + return_logprob=False, + logprob_start_len=-1, + top_logprobs_num=0, + token_ids_logprob=None, + stream=False, + ) + scheduler = _queued_scheduler(None) + scheduler._gather_vmm_materialization_errors = lambda error: [ + None, + "copy failed", + ] + with _rank(rank): + errors = scheduler._materialize_cuda_vmm_inputs(recv) + assert "copy failed" in errors[0] + assert recv.mm_inputs is None + vmm_pool._recycle_chunks() + assert not vmm_pool.occupied_chunks + + +def test_repeated_cancellation_during_publication_does_not_leak(vmm_pool): + """Cancellation must not orphan slices still being published on another thread.""" + transport = object.__new__(vmm.CudaVmmFeatureTransport) + transport.pool = vmm_pool + started, release = threading.Event(), threading.Event() + item = MultimodalDataItem(modality=Modality.IMAGE, feature=torch.ones(2)) + + def publish(_): + started.set() + assert release.wait(timeout=5) + return _publish(vmm_pool)[0].feature + + async def run(): + task = asyncio.create_task( + transport.prepare_for_dispatch_async( + [MultimodalProcessorOutput(mm_items=[item])] + ) + ) + while not started.is_set(): + await asyncio.sleep(0) + task.cancel() + await asyncio.sleep(0) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: + transport._publisher_executor = executor + with patch.object(vmm_pool, "wrap_tensor", side_effect=publish): + try: + asyncio.run(run()) + finally: + release.set() + executor.shutdown(wait=True) + assert not vmm_pool.occupied_chunks + + +def test_recycler_does_not_hold_allocator_lock_or_free_reused_slice(vmm_pool): + """A slow ACK poll cannot block cancellation or retire a republished offset.""" + old = _publish(vmm_pool) + vmm_pool._stop_recycler = threading.Event() + vmm_pool._recycle_interval = 0.001 + vmm_pool._pool_error = None + vmm_pool.memory_pool[:8].view(torch.int32).fill_(1) + stack = torch.stack + + def poll(tensors): + vmm_pool._stop_recycler.set() + assert vmm_pool._lock.acquire(blocking=False), ( + "CUDA poll held the allocator lock" + ) + vmm_pool._lock.release() + snapshot = stack(tensors) + vmm_pool.cancel_proxy(old[0].feature) + _publish(vmm_pool) + return snapshot + + with patch.object(torch, "stack", side_effect=poll): + vmm_pool._recycle_loop() + assert vmm_pool._pool_error is None + assert vmm_pool.occupied_chunks, "old poll result retired a new allocation" + + def _deferred_proxy(): proxy = object.__new__(CudaIpcTensorTransportProxy) proxy.total_consumer_count = 1 @@ -70,6 +370,7 @@ def test_session_abort_preserves_shared_multimodal_features(): req = object.__new__(Req) req.rid = "rejected-session-turn" req.session = object() + req._session_mm_items = [] req.multimodal_inputs = mm_inputs req.grammar = object() req.return_logprob = True diff --git a/test/registered/unit/managers/test_scheduler_timeouts.py b/test/registered/unit/managers/test_scheduler_timeouts.py index 57f83fec6d..b84591f987 100644 --- a/test/registered/unit/managers/test_scheduler_timeouts.py +++ b/test/registered/unit/managers/test_scheduler_timeouts.py @@ -24,7 +24,7 @@ from sglang.test.test_utils import ( maybe_stub_sgl_kernel() -from sglang.srt.managers.schedule_batch import FINISH_ABORT +from sglang.srt.managers.schedule_batch import FINISH_ABORT, Req from sglang.srt.managers.scheduler import Scheduler from sglang.srt.mem_cache.base_prefix_cache import ( CacheRequestHandle, @@ -35,11 +35,15 @@ register_cpu_ci(est_time=12, suite="base-a-test-cpu") class _FakeReq: + release_mm_inputs_on_abort = Req.release_mm_inputs_on_abort + def __init__(self, rid, wait_entry=0.0, forward_entry=0.0, is_finished=False): self.rid = rid self.cache_request_handle = CacheRequestHandle(rid, 0) self.to_finish = None self.beam_group = None + self.session = None + self.multimodal_inputs = None self._finished = is_finished self.output_ids = [] self.weight_version_events = [] diff --git a/test/registered/unit/mem_cache/test_streaming_session_unit.py b/test/registered/unit/mem_cache/test_streaming_session_unit.py index 148bdb8a5a..afc5496412 100644 --- a/test/registered/unit/mem_cache/test_streaming_session_unit.py +++ b/test/registered/unit/mem_cache/test_streaming_session_unit.py @@ -3,7 +3,12 @@ from types import SimpleNamespace import pytest import torch -from sglang.srt.managers.schedule_batch import FINISH_ABORT, FINISH_LENGTH, ReqKvInfo +from sglang.srt.managers.schedule_batch import ( + FINISH_ABORT, + FINISH_LENGTH, + Req, + ReqKvInfo, +) from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator from sglang.srt.mem_cache.base_prefix_cache import ( BasePrefixCache, @@ -82,6 +87,8 @@ class _FakeInnerCache: class _FakeReq: + release_mm_inputs_on_abort = Req.release_mm_inputs_on_abort + def __init__( self, session_id: str, req_pool_idx: int, committed: int, allocated: int ): @@ -108,6 +115,8 @@ class _FakeReq: self.to_finish = None self.finished_reason = None self.finished_len = None + self.multimodal_inputs = None + self._session_mm_items = [] def detach_kv(self): kv, self.kv = self.kv, ReqKvInfo() diff --git a/test/registered/unit/models/test_qwen3_vl_feature_materialization.py b/test/registered/unit/models/test_qwen3_vl_feature_materialization.py index 34262c1adb..0f4302f0c3 100644 --- a/test/registered/unit/models/test_qwen3_vl_feature_materialization.py +++ b/test/registered/unit/models/test_qwen3_vl_feature_materialization.py @@ -286,7 +286,7 @@ class TestQwen3VLFeatureMaterialization(CustomTestCase): self.assertIs(output, encoded) items[0].reconstruct.assert_not_called() - items[1].reconstruct.assert_called_once_with(0, ipc_consumer_count=8) + items[1].reconstruct.assert_called_once_with(0) materialize.assert_called_once_with( [items[1].feature], device=visual.device, dtype=visual.dtype ) diff --git a/test/registered/unit/multimodal/test_cuda_vmm_transport.py b/test/registered/unit/multimodal/test_cuda_vmm_transport.py index 5923d48362..6317c6d45c 100644 --- a/test/registered/unit/multimodal/test_cuda_vmm_transport.py +++ b/test/registered/unit/multimodal/test_cuda_vmm_transport.py @@ -2,6 +2,8 @@ from __future__ import annotations +import asyncio +import concurrent.futures import gc import multiprocessing as mp import os @@ -13,8 +15,14 @@ from unittest.mock import patch import torch +from sglang.srt.managers.schedule_batch import ( + Modality, + MultimodalDataItem, + MultimodalProcessorOutput, +) from sglang.srt.runtime_context import get_parallel from sglang.srt.utils.cuda_vmm_transport_utils import ( + CudaVmmFeatureTransport, CudaVmmMemoryPool, CudaVmmPackedTensorTransportProxy, _imported_pool_cache_clear, @@ -55,13 +63,9 @@ def _produce_vmm_tensor(proxy_queue, consumer_done, result_queue, mode): proxy_queue.put((proxy, expected)) if not consumer_done.wait(timeout=60): raise TimeoutError("consumers did not release the CUDA VMM tensor") - with pool._lock: - pool._recycle_chunks() - pool._merge_chunks() - if pool.occupied_chunks: - raise RuntimeError( - "consumer acknowledgements did not recycle the slice" - ) + pool._recycle_chunks() + if pool.occupied_chunks: + raise RuntimeError("consumer acknowledgements did not recycle the slice") except Exception as exc: # noqa: BLE001 # pragma: no cover result_queue.put(("error", repr(exc))) return @@ -121,6 +125,7 @@ class TestCudaVmmTransport(CustomTestCase): ): torch.cuda.set_device(device) with get_parallel().override( + tp_rank=tp_rank, attn_tp_size=2, attn_tp_rank=tp_rank, attn_cp_size=1, @@ -156,17 +161,64 @@ class TestCudaVmmTransport(CustomTestCase): def test_auto_prefers_fabric_tensor_round_trip_and_recycling(self): self._run_round_trip(mode="auto") + def test_publication_waits_for_source_stream(self): + torch.cuda.set_device(0) + for mode in ("single", "packed", "async", "async_fallback"): + with self.subTest(mode=mode): + pool = CudaVmmMemoryPool(4 << 20, 60, 0, 1, allow_posix_fallback=True) + transport = object.__new__(CudaVmmFeatureTransport) + transport.pool = pool + try: + # Make a wrong-stream copy observably read the previous value. + size = 8 << 20 if mode == "async_fallback" else 1024 + source = torch.zeros(size, dtype=torch.uint8, device="cuda") + torch.cuda.synchronize() + stream = torch.cuda.Stream() + with concurrent.futures.ThreadPoolExecutor(1) as executor: + transport._publisher_executor = executor + with torch.cuda.stream(stream): + torch.cuda._sleep(200_000_000) + source.fill_(73) + if mode == "single": + proxy = pool.wrap_tensor(source) + elif mode == "packed": + proxy = pool.wrap_tensors([source, source])[0] + else: + item = MultimodalDataItem( + modality=Modality.IMAGE, feature=source + ) + asyncio.run( + transport.prepare_for_dispatch_async( + [MultimodalProcessorOutput(mm_items=[item])] + ) + ) + proxy = item.feature + result = ( + proxy + if isinstance(proxy, torch.Tensor) + else proxy.reconstruct_on_target_device(0) + ) + self.assertTrue( + torch.equal( + result.cpu(), torch.full((size,), 73, dtype=torch.uint8) + ) + ) + finally: + torch.cuda.synchronize() + _imported_pool_cache_clear() + pool.shutdown() + def test_reused_chunk_clears_acknowledgements(self): pool = CudaVmmMemoryPool(4 << 20, 60, 0, 2, allow_posix_fallback=True) try: old = pool.wrap_tensor(torch.ones(1024, dtype=torch.uint8, device="cuda:0")) + # VMM must bypass the CUDA IPC zero-copy borrow protocol. + self.assertIsNone(old.borrow_on_target_device(0)) pool.memory_pool[old.control_offset : old.control_offset + 8].view( torch.int32 ).fill_(1) torch.cuda.synchronize(0) - with pool._lock: - pool._recycle_chunks() - pool._merge_chunks() + pool._recycle_chunks() pool.wrap_tensor(torch.ones(100, dtype=torch.uint8, device="cuda:0")) live = pool.wrap_tensor(torch.ones(256, dtype=torch.uint8, device="cuda:0")) @@ -175,8 +227,7 @@ class TestCudaVmmTransport(CustomTestCase): ].view(torch.int32) self.assertTrue(torch.equal(control, torch.zeros_like(control))) - with pool._lock: - pool._recycle_chunks() + pool._recycle_chunks() self.assertIn( live.control_offset, [chunk.start for chunk in pool.occupied_chunks], @@ -237,8 +288,7 @@ class TestCudaVmmTransport(CustomTestCase): for tensor in reconstructed ) ) - with pool._lock: - pool._recycle_chunks() + pool._recycle_chunks() self.assertFalse(pool.occupied_chunks) finally: del reconstructed, proxies, expected, sources @@ -425,8 +475,7 @@ class TestCudaVmmTransport(CustomTestCase): ): proxy.reconstruct_on_target_device(0, consumer_count=1) torch.cuda.synchronize(0) - with pool._lock: - pool._recycle_chunks() + pool._recycle_chunks() self.assertFalse(pool.occupied_chunks) with ( diff --git a/test/registered/unit/multimodal/test_gpu_feature_transport.py b/test/registered/unit/multimodal/test_gpu_feature_transport.py index afa13fb6e6..40c64324f5 100644 --- a/test/registered/unit/multimodal/test_gpu_feature_transport.py +++ b/test/registered/unit/multimodal/test_gpu_feature_transport.py @@ -412,6 +412,11 @@ class TestCudaVmmFeatureTransport(unittest.TestCase): transport.cancel_for_dispatch.assert_not_called() def test_async_publication_keeps_event_loop_responsive(self): + from sglang.srt.managers.schedule_batch import ( + Modality, + MultimodalDataItem, + MultimodalProcessorOutput, + ) from sglang.srt.utils.cuda_vmm_transport_utils import ( CudaVmmFeatureTransport, ) @@ -437,7 +442,11 @@ class TestCudaVmmFeatureTransport(unittest.TestCase): release.set() transport.prepare_for_dispatch = MagicMock(side_effect=block_publication) - mm_inputs = SimpleNamespace(mm_items=[object()]) + mm_inputs = MultimodalProcessorOutput( + mm_items=[ + MultimodalDataItem(modality=Modality.IMAGE, feature=torch.ones(2)) + ] + ) async def run(): watchdog = threading.Thread(target=unblock_if_event_loop_stalls) @@ -447,6 +456,8 @@ class TestCudaVmmFeatureTransport(unittest.TestCase): transport.prepare_for_dispatch_async([mm_inputs]) ) while not started.is_set(): + if task.done(): + await task await asyncio.sleep(0) event_loop_responsive.set() self.assertFalse(task.done()) @@ -519,6 +530,7 @@ class TestCudaVmmFeatureTransport(unittest.TestCase): from sglang.srt.utils import cuda_vmm_transport_utils as vmm pool = object.__new__(vmm.CudaVmmMemoryPool) + pool._lock = threading.Lock() pool.device_index = 0 pool.consumer_count = 2 pool._recycle_stream = object() @@ -1122,6 +1134,7 @@ class TestVmmConsumerCount(unittest.TestCase): proxy = object.__new__(CudaVmmTensorTransportProxy) proxy.consumer_count = 4 with get_parallel().override( + tp_rank=3, attn_tp_size=2, attn_tp_rank=1, attn_cp_size=2,