[VLM] fix: repair multimodal VMM lease cleanup and source-stream ordering (#43475)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Mick
2026-10-11 19:13:39 +08:00
committed by GitHub
co-authored by Mick Qian
parent efd14cfb66
commit 04f740580c
19 changed files with 576 additions and 113 deletions
+3 -24
View File
@@ -9,7 +9,7 @@ from sglang.srt.managers.schedule_batch import MultimodalDataItem
from sglang.srt.mem_cache.multimodal_cache import EmbeddingResult, MultiModalStaticCache
from sglang.srt.multimodal.evs import EVSEmbeddingResult
from sglang.srt.multimodal.transport.cuda_ipc import BORROW_CUDA_IPC_FEATURE_KEY
from sglang.srt.runtime_context import get_parallel, get_schedule
from sglang.srt.runtime_context import get_schedule
from sglang.srt.utils import is_hip, is_npu, is_xpu
from sglang.srt.utils.async_probe import maybe_assert_sum
from sglang.utils import logger
@@ -243,26 +243,6 @@ def _move_items_to_device(
item.feature = item.feature.to(device, non_blocking=True)
def _acknowledge_deferred_cuda_ipc_cache_hits(
items: List[MultimodalDataItem],
) -> None:
"""Release lazy Kimi IPC slices when a cached embedding skips ViT.
On an encoder-DP miss, exactly one rank copies an image and acknowledges
the full TP group. On a cache hit no rank copies it, so rank zero performs
the equivalent single acknowledgement. This preserves the fixed-pool
lifecycle without reintroducing an unnecessary GPU-to-GPU copy.
"""
parallel = get_parallel()
if parallel.attn_tp_rank != 0:
return
# The pool's recycler counts the whole TP group, so the acknowledgement must
# match that count even when an attention subgroup is smaller.
consumer_count = max(parallel.tp_size, 1)
for item in items:
item.acknowledge_deferred_cuda_ipc_feature(consumer_count)
def _item_overlap(
item: MultimodalDataItem, chunk_start: int, chunk_end: int
) -> Optional[int]:
@@ -330,8 +310,8 @@ def _get_chunked_embedding_full(
else embedding
)
embedding_cache.set(embedding_items_hash, embedding_per_req)
else:
_acknowledge_deferred_cuda_ipc_cache_hits(embedding_items_per_req)
# cache hits keep raw-input leases until request cleanup for eviction/re-prefill
if isinstance(embedding_per_req, EVSEmbeddingResult):
item = embedding_items_per_req[0]
@@ -492,7 +472,6 @@ def _get_chunked_embedding_by_item(
cached_token_count = _embedding_token_count(cached_embedding)
if cached_token_count == expected_token_count:
cached_embeddings[idx] = cached_embedding
_acknowledge_deferred_cuda_ipc_cache_hits([item])
else:
_discard_mismatched_cached_embedding(
item.hash, expected_token_count, cached_token_count
+16 -3
View File
@@ -1077,6 +1077,8 @@ class Req(ReqDllmMixin):
self.dllm_initialized: bool = False
self.session = session
# media added by this turn, excluding the session's borrowed history
self._session_mm_items: List[MultimodalDataItem] = []
self.session_id = session_id
# Used by the session radix cache to reject registration after a close/reopen.
self.session_generation: Optional[int] = None
@@ -1519,6 +1521,7 @@ class Req(ReqDllmMixin):
def _extend_session_image_inputs(self, image_inputs):
"""Append media while preserving the saved session and its position history."""
self._session_mm_items.extend(image_inputs.mm_items)
# Padding can change token values without changing their count.
self.full_untruncated_fill_ids = array("q")
if self.multimodal_inputs is not None:
@@ -1564,6 +1567,18 @@ class Req(ReqDllmMixin):
positions.max() + 1 - positions.shape[1]
).reshape(1, 1)
def release_mm_inputs_on_abort(self) -> None:
"""Release this turn's media without invalidating session history."""
if self.session is None:
if self.multimodal_inputs is not None:
self.multimodal_inputs.release_features()
else:
for item in self._session_mm_items:
item.release_transport_proxies()
item.feature = None
self._session_mm_items.clear()
self.multimodal_inputs = None
def finished(self) -> bool:
# Whether request reached finished condition
return self.finished_reason is not None
@@ -2110,9 +2125,7 @@ class Req(ReqDllmMixin):
logger.error(f"{error_msg}, {self.rid=}")
# Session requests share historical multimodal inputs with their prior
# request. The session owns and releases those features when it closes.
if self.multimodal_inputs is not None and self.session is None:
self.multimodal_inputs.release_features()
self.multimodal_inputs = None
self.release_mm_inputs_on_abort()
self.grammar = None
self.origin_input_ids = array(
"q", [0]
+51 -1
View File
@@ -23,7 +23,7 @@ import sys
import time
from array import array
from collections import deque
from contextlib import contextmanager, nullcontext
from contextlib import ExitStack, contextmanager, nullcontext
from functools import partial
from http import HTTPStatus
from typing import TYPE_CHECKING, Any, Deque, Dict, List, Optional, Set, Tuple, Union
@@ -210,6 +210,7 @@ from sglang.srt.managers.prefill_delayer import (
from sglang.srt.managers.schedule_batch import (
FINISH_ABORT,
MultimodalInputs,
MultimodalProcessorOutput,
NextBatchPlan,
Req,
ScheduleBatch,
@@ -2235,6 +2236,12 @@ class Scheduler(
)
error_msg = f"Multimodal feature reconstruction failed ({details})"
logger.error(error_msg)
if isinstance(
tokenized_req.mm_inputs,
(MultimodalInputs, MultimodalProcessorOutput),
):
for item in tokenized_req.mm_inputs.mm_items:
item.release_transport_proxies()
tokenized_req.mm_inputs = None
request_errors.append(error_msg)
else:
@@ -2779,6 +2786,8 @@ class Scheduler(
continue
# For session requests, keep mm_inputs for the next request
if req.session:
if isinstance(req.finished_reason, FINISH_ABORT):
req.release_mm_inputs_on_abort()
continue
# For non-session requests, clear features and mm_inputs
mm_inputs.release_features()
@@ -2789,6 +2798,23 @@ class Scheduler(
recv_req: TokenizedGenerateReqInput,
*,
mm_input_error: Optional[str] = None,
):
with ExitStack() as mm_cleanup:
if isinstance(
recv_req.mm_inputs, (MultimodalInputs, MultimodalProcessorOutput)
):
for item in recv_req.mm_inputs.mm_items:
mm_cleanup.callback(item.release_transport_proxies)
self._handle_generate_request(
recv_req, mm_cleanup, mm_input_error=mm_input_error
)
def _handle_generate_request(
self,
recv_req: TokenizedGenerateReqInput,
mm_cleanup: ExitStack,
*,
mm_input_error: Optional[str] = None,
):
# Route: normal request / session request / session-not-found
session_id = (
@@ -3047,6 +3073,8 @@ class Scheduler(
req.origin_input_ids, image_inputs, self.pad_input_ids_func
)
req.extend_image_inputs(image_inputs)
# the request now owns cleanup, including rejection after padding
mm_cleanup.pop_all()
self._maybe_compute_mrope_positions(req)
if len(req.origin_input_ids) >= self.max_req_input_len:
@@ -3279,6 +3307,7 @@ class Scheduler(
# detach lives in `StreamingSession.find_active_slot`, which only runs
# while scheduling; a session left in-flight rejects every later request.
if req.session is not None and req.session.streaming:
req.release_mm_inputs_on_abort()
req.session.abort_req()
req.session = None
# `beam_coordinator.validate_and_init` counts the group in ahead of the
@@ -3364,7 +3393,10 @@ class Scheduler(
def _release_aborted_request(self, req: Req) -> None:
"""Drop the cache-side state an aborted request left behind."""
req.release_mm_inputs_on_abort()
self.tree_cache.finish(req.cache_request_handle, CacheRequestOutcome.ABORT)
if req.session is not None and req.session.streaming:
req.session.abort_req()
def _abort_on_queued_limit(self, recv_req: Req) -> bool:
"""Abort an incoming or existing request if the waiting queue is full. Returns True if the incoming request is aborted."""
@@ -3472,6 +3504,23 @@ class Scheduler(
recv_req: TokenizedEmbeddingReqInput,
*,
mm_input_error: Optional[str] = None,
):
with ExitStack() as mm_cleanup:
if isinstance(
recv_req.mm_inputs, (MultimodalInputs, MultimodalProcessorOutput)
):
for item in recv_req.mm_inputs.mm_items:
mm_cleanup.callback(item.release_transport_proxies)
self._handle_embedding_request(
recv_req, mm_cleanup, mm_input_error=mm_input_error
)
def _handle_embedding_request(
self,
recv_req: TokenizedEmbeddingReqInput,
mm_cleanup: ExitStack,
*,
mm_input_error: Optional[str] = None,
):
req = Req(
recv_req.rid,
@@ -3528,6 +3577,7 @@ class Scheduler(
)
req.extend_image_inputs(image_inputs)
mm_cleanup.pop_all()
self._maybe_compute_mrope_positions(req)
if len(req.origin_input_ids) >= self.max_req_input_len:
+1 -1
View File
@@ -4597,7 +4597,7 @@ class DeepseekV4ForCausalLM(nn.Module):
spans = []
device, dtype = self.image_start.device, self.image_start.dtype
for item in items:
item.reconstruct(device.index, ipc_consumer_count=self.tp_size)
item.reconstruct(device.index)
h, w = int(item.n_vit_h), int(item.n_vit_w)
pixels = torch.as_tensor(item.feature, device=device)
plan = item.model_specific_data.get(GPU_PLAN_KEY)
+3 -11
View File
@@ -49,7 +49,6 @@ from sglang.srt.multimodal.mm_utils import (
from sglang.srt.runtime_context import (
get_exec,
get_mm,
get_parallel,
)
from sglang.srt.utils import add_prefix, is_cuda, is_npu
@@ -724,14 +723,9 @@ class KimiK25ForConditionalGeneration(nn.Module):
CUDA IPC features are intentionally reconstructed after the image
assignment. Each image therefore crosses the tokenizer/scheduler
boundary once instead of once per TP rank. The selected consumer
acknowledges the entire TP group so the bounded IPC pool remains
recyclable.
boundary once instead of once per TP rank. Unused ranks keep their
lease until request cleanup so cache eviction can re-run the encoder.
"""
# Match the configured TP consumer count captured when the
# tokenizer creates MmItemMemoryPool. A live attention subgroup
# size could leave acknowledgements missing and strand the lease.
ipc_consumer_count = max(get_parallel().tp_size, 1)
device_index = device.index
if device.type == "cuda" and device_index is None:
device_index = torch.cuda.current_device()
@@ -740,9 +734,7 @@ class KimiK25ForConditionalGeneration(nn.Module):
for image_index in image_indices:
item = items[image_index]
if device.type == "cuda":
item.reconstruct(
device_index, ipc_consumer_count=ipc_consumer_count
)
item.reconstruct(device_index)
feature = item.feature
if not isinstance(feature, torch.Tensor):
raise TypeError(
+1 -7
View File
@@ -3613,10 +3613,6 @@ class KimiK3ForConditionalGeneration(nn.Module):
LOCAL_PREPROCESSED_KEY,
)
# Match the configured TP consumer count captured when the
# tokenizer creates MmItemMemoryPool. A live attention subgroup
# size could leave acknowledgements missing and strand the lease.
ipc_consumer_count = max(get_parallel().tp_size, 1)
device_index = device.index
if device.type == "cuda" and device_index is None:
device_index = torch.cuda.current_device()
@@ -3625,9 +3621,7 @@ class KimiK3ForConditionalGeneration(nn.Module):
for image_index in image_indices:
item = items[image_index]
if device.type == "cuda":
item.reconstruct(
device_index, ipc_consumer_count=ipc_consumer_count
)
item.reconstruct(device_index)
selected_items.append(item)
locally_preprocessed = [
+1 -1
View File
@@ -1521,7 +1521,7 @@ class Qwen3VLForConditionalGeneration(nn.Module):
(item, proxy, feature_offset, borrowed.shape[0])
)
elif RETAINED_CUDA_IPC_FEATURE_PROXY_KEY not in model_specific_data:
item.reconstruct(device_index, ipc_consumer_count=consumer_count)
item.reconstruct(device_index)
features.append(item.feature)
feature_offset += item.feature.shape[0]
try:
@@ -44,6 +44,7 @@ from sglang.srt.multimodal.transport.cuda_ipc import (
MmItemMemoryPool,
get_mm_feature_pool_size_per_worker,
)
from sglang.srt.multimodal.transport.memory_pool import get_mm_feature_consumer_count
from sglang.srt.runtime_context import (
get_exec,
get_mm,
@@ -450,7 +451,7 @@ class BaseMultimodalProcessor(ABC):
per_worker_pool_size,
MM_ITEM_MEMORY_POOL_RECYCLE_INTERVAL,
self.server_args.base_gpu_id,
self.server_args.tp_size,
get_mm_feature_consumer_count(),
)
@property
@@ -16,6 +16,14 @@ DATA_ALIGNMENT = 256
DEFAULT_MAX_INFLIGHT_SLICES = 4096
def get_mm_feature_consumer_count() -> int:
"""Number of scheduler ranks receiving one routed multimodal request."""
parallel = get_parallel()
if parallel.attn_dp_enabled:
return parallel.tp_size // parallel.num_dp_ranks
return parallel.tp_size
def align_up(value: int, alignment: int) -> int:
return ((value + alignment - 1) // alignment) * alignment
@@ -59,9 +67,8 @@ def resolve_consumer_rank(
return 0
if consumer_rank is None:
try:
# Use the global TP rank. An attention/DCP subgroup rank can alias
# another consumer's acknowledgement slot.
rank = int(get_parallel().tp_rank)
# DP groups are contiguous; attention/DCP subgroup ranks can alias
rank = int(get_parallel().tp_rank) % total_consumer_count
except Exception as exc:
raise RuntimeError(
f"Cannot resolve the {transport_name} consumer rank before "
@@ -80,6 +80,7 @@ class StreamingSession:
if slot is None or not slot.kv.holds_kv:
return None
if req.to_finish is not None:
req.release_mm_inputs_on_abort()
req.session.abort_req()
req.session = None
return None
@@ -160,6 +161,7 @@ class StreamingSession:
# Hand the record and the tree lock back; the caller releases them.
del self.slots[req.session.session_id]
req.lock, slot.lock = slot.lock, None
req.release_mm_inputs_on_abort()
req.session.abort_req()
return False
@@ -17,6 +17,7 @@ from sglang.srt.managers.schedule_batch import (
MultimodalDataItem,
MultimodalProcessorOutput,
)
from sglang.srt.multimodal.transport.memory_pool import get_mm_feature_consumer_count
from sglang.srt.runtime_context import (
get_mm,
get_parallel,
@@ -192,9 +193,7 @@ def _contains_tensor_container(value) -> bool:
def get_vmm_feature_consumer_count() -> int:
if get_parallel().attn_dp_enabled:
return get_parallel().tp_size // get_parallel().num_dp_ranks
return get_parallel().tp_size
return get_mm_feature_consumer_count()
class CudaVmmMemoryPool:
@@ -402,6 +401,10 @@ class CudaVmmMemoryPool:
producer_stream = None
copy_synchronized = False
try:
if tensor.is_cuda:
self._publish_stream.wait_stream(
torch.cuda.current_stream(tensor.device)
)
with (
torch.cuda.device(self.device_index),
torch.cuda.stream(self._publish_stream),
@@ -477,6 +480,8 @@ class CudaVmmMemoryPool:
producer_stream = None
copy_synchronized = False
try:
for device in {tensor.device for tensor in tensors if tensor.is_cuda}:
self._publish_stream.wait_stream(torch.cuda.current_stream(device))
with (
torch.cuda.device(self.device_index),
torch.cuda.stream(self._publish_stream),
@@ -604,20 +609,19 @@ class CudaVmmMemoryPool:
def _recycle_loop(self) -> None:
while not self._stop_recycler.wait(self._recycle_interval):
try:
with self._lock:
self._recycle_chunks()
self._merge_chunks()
self._recycle_chunks()
except Exception as error:
logger.exception("CUDA VMM multimodal pool recycle failed")
self._pool_error = error
self._stop_recycler.set()
def _recycle_chunks(self) -> None:
if not self.occupied_chunks:
with self._lock:
chunks = tuple(self.occupied_chunks)
if not chunks:
return
remaining = []
recycled = []
# CUDA polling must not hold the allocator lock needed by publishers
with (
torch.cuda.device(self.device_index),
torch.cuda.stream(self._recycle_stream),
@@ -628,23 +632,23 @@ class CudaVmmMemoryPool:
chunk.start : chunk.start
+ self.consumer_count * _CONTROL_WORD_BYTES
].view(torch.int32)
for chunk in self.occupied_chunks
for chunk in chunks
]
)
acknowledgement_counts = (
torch.count_nonzero(acknowledgement_words, dim=1).cpu().tolist()
)
for chunk, acknowledgement_count in zip(
self.occupied_chunks, acknowledgement_counts, strict=True
):
if acknowledgement_count == self.consumer_count:
recycled.append(_CudaVmmMemoryChunk(chunk.start, chunk.end))
else:
remaining.append(chunk)
self.available_chunks.extend(recycled)
self.occupied_chunks = remaining
with self._lock:
live = {chunk.start: chunk for chunk in self.occupied_chunks}
for chunk, count in zip(chunks, acknowledgement_counts, strict=True):
# cancellation may have freed and republished the same offset
if count == self.consumer_count and live.get(chunk.start) is chunk:
self.occupied_chunks.remove(chunk)
self.available_chunks.append(
_CudaVmmMemoryChunk(chunk.start, chunk.end)
)
self._merge_chunks()
def _merge_chunks(self) -> None:
merged = []
@@ -809,6 +813,10 @@ class CudaVmmTensorTransportProxy(CudaIpcTensorTransportProxy):
device_index=device_index,
)
def borrow_on_target_device(self, rebuild_device_idx: int) -> None:
# VMM copies into consumer-owned storage; the IPC borrow protocol differs
return None
def _acknowledgement_range(self, consumer_count: int) -> tuple[int, int]:
if consumer_count <= 0:
raise ValueError("consumer_count must be positive")
@@ -816,6 +824,10 @@ class CudaVmmTensorTransportProxy(CudaIpcTensorTransportProxy):
return 0, self.consumer_count
parallel = get_parallel()
if consumer_count == 1:
# attention/DCP subgroup ranks can alias; DP groups are contiguous
slot = parallel.tp_rank % self.consumer_count
return slot, slot + 1
group_start = parallel.attn_cp_rank * parallel.attn_tp_size
group_end = group_start + parallel.attn_tp_size
if not 0 <= group_start < group_end <= self.consumer_count:
@@ -824,9 +836,6 @@ class CudaVmmTensorTransportProxy(CudaIpcTensorTransportProxy):
f"[{group_start}, {group_end}) is outside "
f"consumer_count={self.consumer_count}"
)
if consumer_count == 1:
slot = group_start + parallel.attn_tp_rank
return slot, slot + 1
if consumer_count == parallel.attn_tp_size:
return group_start, group_end
raise ValueError(
@@ -1033,18 +1042,49 @@ class CudaVmmFeatureTransport:
if self._publisher_executor is None:
raise RuntimeError("CUDA VMM feature transport is shutting down")
future = asyncio.get_running_loop().run_in_executor(
self._publisher_executor,
self.prepare_for_dispatch,
mm_inputs_batch,
# CUDA current streams are thread-local; carry readiness into the publisher.
ready_events = []
devices = {
tensor.device
for mm_inputs in mm_inputs_batch
if mm_inputs is not None
for item in mm_inputs.mm_items
for tensor in (item.feature, item.precomputed_embeddings)
if isinstance(tensor, torch.Tensor) and tensor.is_cuda
}
for device in devices:
event = torch.cuda.Event()
event.record(torch.cuda.current_stream(device))
ready_events.append((device, event))
future = self._publisher_executor.submit(
self._prepare_after_source_events, mm_inputs_batch, ready_events
)
try:
return await asyncio.shield(future)
return await asyncio.shield(asyncio.wrap_future(future))
except asyncio.CancelledError:
prepared_mm_items = await future
self.cancel_for_dispatch(prepared_mm_items)
# cleanup must survive a second task cancellation or loop shutdown
future.add_done_callback(self._cancel_completed_publish)
raise
def _prepare_after_source_events(
self,
mm_inputs_batch: Iterable[MultimodalProcessorOutput | None],
ready_events: list[tuple[torch.device, torch.cuda.Event]],
) -> list[MultimodalDataItem]:
for device, event in ready_events:
# Both VMM copies and a pool-full CPU fallback must see ready input.
torch.cuda.current_stream(device).wait_event(event)
return self.prepare_for_dispatch(mm_inputs_batch)
def _cancel_completed_publish(self, future: concurrent.futures.Future) -> None:
if future.cancelled() or future.exception() is not None:
# failed publication already rolls back its slices
return
try:
self.cancel_for_dispatch(future.result())
except Exception:
logger.exception("Failed to cancel abandoned CUDA VMM publication")
def prepare_for_dispatch(
self,
mm_inputs_batch: Iterable[MultimodalProcessorOutput | None],
@@ -50,9 +50,7 @@ HIDDEN = 16
@pytest.fixture(autouse=True)
def single_process_runtime_context():
# These mm_utils unit tests exercise cache-hit paths that acknowledge
# deferred CUDA IPC through runtime_context. They do not start an engine, so
# pin the runtime topology to a single-process CPU setup.
# Exercise the embedding path without starting an engine.
server_args_override = get_context().override_server_args(tp_size=1)
server_args_override.install()
try:
@@ -74,14 +72,6 @@ CHUNKS = [(0, 8), (8, 8), (16, 8), (24, 6)]
_CPU = torch.device("cpu")
@pytest.fixture(autouse=True)
def _skip_cuda_ipc_acknowledgement(monkeypatch):
"""Keep CPU embedding tests independent of tensor-parallel runtime state."""
monkeypatch.setattr(
mm_schedule, "_acknowledge_deferred_cuda_ipc_cache_hits", lambda _items: None
)
def _num_tokens(item: MultimodalDataItem) -> int:
start, end = item.offsets[0]
return end - start + 1
@@ -439,6 +439,25 @@ class TestStreamOrderedMmFeaturePool(CustomTestCase):
with get_parallel().override(tp_rank=6, attn_tp_rank=2):
self.assertEqual(resolve_consumer_rank(8), 6)
def test_attention_dp_uses_only_the_routed_scheduler_group(self):
from sglang.srt.multimodal.transport.memory_pool import (
get_mm_feature_consumer_count,
resolve_consumer_rank,
)
from sglang.srt.runtime_context import get_parallel
with get_parallel().override(
tp_size=8,
attn_dp_enabled=True,
num_dp_ranks=2,
tp_rank=6,
attn_tp_rank=0,
attn_dcp_size=2,
):
count = get_mm_feature_consumer_count()
self.assertEqual(count, 4)
self.assertEqual(resolve_consumer_rank(count), 2)
def test_complete_group_acknowledges_each_consumer_slot(self):
from sglang.srt.multimodal.transport import memory_pool
@@ -1,21 +1,321 @@
import asyncio
import concurrent.futures
import copy
import sys
import threading
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
import torch
from sglang.srt.disaggregation.utils import DisaggregationMode
from sglang.srt.managers import mm_schedule
from sglang.srt.managers.io_struct import (
AbortReq,
SessionParams,
TokenizedGenerateReqInput,
)
from sglang.srt.managers.schedule_batch import (
Modality,
MultimodalDataItem,
MultimodalInputs,
MultimodalProcessorOutput,
Req,
)
from sglang.srt.managers.scheduler import Scheduler
from sglang.srt.mem_cache.multimodal_cache import EmbeddingResult, MultiModalStaticCache
from sglang.srt.multimodal.transport.cuda_ipc import CudaIpcTensorTransportProxy
from sglang.srt.runtime_context import get_context, get_parallel
from sglang.srt.sampling.sampling_params import SamplingParams
from sglang.srt.session.session_controller import Session
from sglang.srt.utils import cuda_vmm_transport_utils as vmm
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
@pytest.fixture
def vmm_pool():
# exercise the real slice allocator and ACK protocol with host-backed storage
pool = object.__new__(vmm.CudaVmmMemoryPool)
pool.memory_pool = torch.zeros(1024, dtype=torch.uint8)
pool.consumer_count = 2
pool.device_index = 0
pool._recycle_stream = None
pool._lock = threading.Lock()
pool.available_chunks = [vmm._CudaVmmMemoryChunk(0, 1024)]
pool.occupied_chunks = []
with (
get_context().override_server_args(tp_size=2),
patch.object(torch.cuda, "device", return_value=nullcontext()),
patch.object(torch.cuda, "stream", return_value=nullcontext()),
patch.object(torch.cuda, "current_device", return_value=0),
patch.object(
vmm.CudaVmmTensorTransportProxy,
"_pool",
return_value=SimpleNamespace(memory=pool.memory_pool),
),
):
yield pool
def _publish(pool):
chunk = pool._reserve_chunk(1024)
assert chunk is not None, "a previous request stranded the pool slice"
pool.memory_pool.zero_()
pool.occupied_chunks.append(chunk)
proxy = vmm.CudaVmmTensorTransportProxy(
fabric_handle=b"test",
posix_socket_path=None,
allocation_size=1024,
data_offset=256,
data_nbytes=768,
control_offset=0,
consumer_count=2,
shape=(768,),
dtype=torch.uint8,
)
return [
MultimodalDataItem(modality=Modality.IMAGE, feature=copy.deepcopy(proxy)),
MultimodalDataItem(modality=Modality.IMAGE, feature=copy.deepcopy(proxy)),
]
def _rank(rank):
return get_parallel().override(
tp_rank=rank, attn_tp_rank=rank, attn_tp_size=2, attn_cp_rank=0, attn_cp_size=1
)
def _queued_scheduler(req):
scheduler = object.__new__(Scheduler)
scheduler.waiting_queue = [req]
scheduler.chunked_req = scheduler.mm_receiver = scheduler.dllm_config = None
scheduler.disaggregation_mode = DisaggregationMode.NULL
scheduler.tree_cache = MagicMock()
scheduler.beam_coordinator = MagicMock()
scheduler.ipc_channels = MagicMock()
scheduler.grammar_manager = MagicMock()
scheduler.collect_inflight_reqs = lambda: set()
return scheduler
def test_queued_abort_recycles_unconsumed_vmm_slices(vmm_pool):
"""Client cancellations before prefill must not exhaust the bounded pool."""
for _ in range(3):
for rank, item in enumerate(_publish(vmm_pool)):
req = Req("queued-image", "", [1], SamplingParams())
req.multimodal_inputs = MultimodalInputs(mm_items=[item])
scheduler = _queued_scheduler(req)
with _rank(rank):
scheduler.abort_request(AbortReq(rid=req.rid))
assert scheduler.waiting_queue == []
vmm_pool._recycle_chunks()
assert not vmm_pool.occupied_chunks
@pytest.mark.parametrize("path", ["full", "per_image", "batch"])
def test_embedding_hit_cannot_recycle_another_ranks_live_proxy(vmm_pool, path):
"""A rank's cache hit must not retire another rank's proxy before cleanup."""
items = _publish(vmm_pool)
cache = MultiModalStaticCache(1 << 20)
cached = torch.ones(1, 2)
cache.set(cache.combine_hashes([123]), EmbeddingResult(embedding=cached))
cache.set(123, EmbeddingResult(embedding=cached))
def encode(_):
pytest.fail("cache hit unexpectedly ran the encoder")
for rank, item in enumerate(items):
item.hash = 123
item.offsets = [(0, 0)]
with _rank(rank), patch.object(mm_schedule, "embedding_cache", cache):
if path == "full":
embedding, _ = mm_schedule._get_chunked_embedding_full(
encode,
[item],
[(0, 0)],
0,
1,
torch.tensor([1]),
torch.device("cpu"),
)
elif path == "per_image":
[embedding] = mm_schedule._get_chunked_embedding_by_item(
encode,
[item],
0,
1,
torch.device("cpu"),
)
else:
request = mm_schedule.PerImageRequestInfo(0, [item], [(0, 0)], 0, 1)
embeddings = mm_schedule._batch_encode_per_image_misses(
encode,
[request],
torch.device("cpu"),
)
embedding = embeddings[(123, 1)]
assert torch.equal(embedding, cached)
assert not item.feature._consumer_acknowledged, (
"cache eviction may need this input again"
)
for rank, item in enumerate(items):
with _rank(rank):
item.release_transport_proxies()
vmm_pool._recycle_chunks()
assert not vmm_pool.occupied_chunks
_publish(vmm_pool)
for rank, item in enumerate(items):
with _rank(rank):
MultimodalInputs(mm_items=[item]).release_features()
vmm_pool._recycle_chunks()
assert vmm_pool.occupied_chunks, "late cleanup acknowledged a recycled slice"
def test_aborted_session_turn_releases_new_media_not_history(vmm_pool):
"""An aborted append owns new media, but only borrows historical features."""
for rank, item in enumerate(_publish(vmm_pool)):
session = Session(100, streaming=True)
session._inflight = True
history = MultimodalDataItem(modality=Modality.IMAGE, feature=torch.ones(3))
req = Req("session-image", "", [1], SamplingParams(), session=session)
req.multimodal_inputs = MultimodalInputs(mm_items=[history])
req.extend_image_inputs(MultimodalInputs(mm_items=[item]))
with _rank(rank):
_queued_scheduler(req).abort_request(AbortReq(rid=req.rid))
assert torch.equal(history.feature, torch.ones(3))
assert not session._inflight
vmm_pool._recycle_chunks()
assert not vmm_pool.occupied_chunks
def test_rejected_session_releases_media_before_request_admission(vmm_pool):
"""An invalid session is rejected before the request attaches its media."""
for rank, item in enumerate(_publish(vmm_pool)):
recv = TokenizedGenerateReqInput(
rid="invalid-session",
input_text="",
input_ids=[1],
input_embeds=None,
mm_inputs=MultimodalProcessorOutput(mm_items=[item]),
token_type_ids=None,
sampling_params=SamplingParams(),
return_logprob=False,
logprob_start_len=-1,
top_logprobs_num=0,
token_ids_logprob=None,
stream=False,
session_params=SessionParams(id="missing"),
)
scheduler = _queued_scheduler(None)
scheduler.session_controller = {}
scheduler.model_config = SimpleNamespace(vocab_size=10)
scheduler.tokenizer = None
scheduler.init_req_max_new_tokens = lambda req: None
scheduler._add_request_to_queue = lambda req: None
with _rank(rank):
scheduler.handle_generate_request(recv)
vmm_pool._recycle_chunks()
assert not vmm_pool.occupied_chunks
def test_other_rank_materialization_failure_releases_local_proxies(vmm_pool):
"""A healthy rank still owns lazy slices when another rank rejects the input."""
for rank, item in enumerate(_publish(vmm_pool)):
recv = TokenizedGenerateReqInput(
rid="peer-failure",
input_text="",
input_ids=[1],
input_embeds=None,
mm_inputs=MultimodalInputs(mm_items=[item]),
token_type_ids=None,
sampling_params=SamplingParams(),
return_logprob=False,
logprob_start_len=-1,
top_logprobs_num=0,
token_ids_logprob=None,
stream=False,
)
scheduler = _queued_scheduler(None)
scheduler._gather_vmm_materialization_errors = lambda error: [
None,
"copy failed",
]
with _rank(rank):
errors = scheduler._materialize_cuda_vmm_inputs(recv)
assert "copy failed" in errors[0]
assert recv.mm_inputs is None
vmm_pool._recycle_chunks()
assert not vmm_pool.occupied_chunks
def test_repeated_cancellation_during_publication_does_not_leak(vmm_pool):
"""Cancellation must not orphan slices still being published on another thread."""
transport = object.__new__(vmm.CudaVmmFeatureTransport)
transport.pool = vmm_pool
started, release = threading.Event(), threading.Event()
item = MultimodalDataItem(modality=Modality.IMAGE, feature=torch.ones(2))
def publish(_):
started.set()
assert release.wait(timeout=5)
return _publish(vmm_pool)[0].feature
async def run():
task = asyncio.create_task(
transport.prepare_for_dispatch_async(
[MultimodalProcessorOutput(mm_items=[item])]
)
)
while not started.is_set():
await asyncio.sleep(0)
task.cancel()
await asyncio.sleep(0)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
transport._publisher_executor = executor
with patch.object(vmm_pool, "wrap_tensor", side_effect=publish):
try:
asyncio.run(run())
finally:
release.set()
executor.shutdown(wait=True)
assert not vmm_pool.occupied_chunks
def test_recycler_does_not_hold_allocator_lock_or_free_reused_slice(vmm_pool):
"""A slow ACK poll cannot block cancellation or retire a republished offset."""
old = _publish(vmm_pool)
vmm_pool._stop_recycler = threading.Event()
vmm_pool._recycle_interval = 0.001
vmm_pool._pool_error = None
vmm_pool.memory_pool[:8].view(torch.int32).fill_(1)
stack = torch.stack
def poll(tensors):
vmm_pool._stop_recycler.set()
assert vmm_pool._lock.acquire(blocking=False), (
"CUDA poll held the allocator lock"
)
vmm_pool._lock.release()
snapshot = stack(tensors)
vmm_pool.cancel_proxy(old[0].feature)
_publish(vmm_pool)
return snapshot
with patch.object(torch, "stack", side_effect=poll):
vmm_pool._recycle_loop()
assert vmm_pool._pool_error is None
assert vmm_pool.occupied_chunks, "old poll result retired a new allocation"
def _deferred_proxy():
proxy = object.__new__(CudaIpcTensorTransportProxy)
proxy.total_consumer_count = 1
@@ -70,6 +370,7 @@ def test_session_abort_preserves_shared_multimodal_features():
req = object.__new__(Req)
req.rid = "rejected-session-turn"
req.session = object()
req._session_mm_items = []
req.multimodal_inputs = mm_inputs
req.grammar = object()
req.return_logprob = True
@@ -24,7 +24,7 @@ from sglang.test.test_utils import (
maybe_stub_sgl_kernel()
from sglang.srt.managers.schedule_batch import FINISH_ABORT
from sglang.srt.managers.schedule_batch import FINISH_ABORT, Req
from sglang.srt.managers.scheduler import Scheduler
from sglang.srt.mem_cache.base_prefix_cache import (
CacheRequestHandle,
@@ -35,11 +35,15 @@ register_cpu_ci(est_time=12, suite="base-a-test-cpu")
class _FakeReq:
release_mm_inputs_on_abort = Req.release_mm_inputs_on_abort
def __init__(self, rid, wait_entry=0.0, forward_entry=0.0, is_finished=False):
self.rid = rid
self.cache_request_handle = CacheRequestHandle(rid, 0)
self.to_finish = None
self.beam_group = None
self.session = None
self.multimodal_inputs = None
self._finished = is_finished
self.output_ids = []
self.weight_version_events = []
@@ -3,7 +3,12 @@ from types import SimpleNamespace
import pytest
import torch
from sglang.srt.managers.schedule_batch import FINISH_ABORT, FINISH_LENGTH, ReqKvInfo
from sglang.srt.managers.schedule_batch import (
FINISH_ABORT,
FINISH_LENGTH,
Req,
ReqKvInfo,
)
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
from sglang.srt.mem_cache.base_prefix_cache import (
BasePrefixCache,
@@ -82,6 +87,8 @@ class _FakeInnerCache:
class _FakeReq:
release_mm_inputs_on_abort = Req.release_mm_inputs_on_abort
def __init__(
self, session_id: str, req_pool_idx: int, committed: int, allocated: int
):
@@ -108,6 +115,8 @@ class _FakeReq:
self.to_finish = None
self.finished_reason = None
self.finished_len = None
self.multimodal_inputs = None
self._session_mm_items = []
def detach_kv(self):
kv, self.kv = self.kv, ReqKvInfo()
@@ -286,7 +286,7 @@ class TestQwen3VLFeatureMaterialization(CustomTestCase):
self.assertIs(output, encoded)
items[0].reconstruct.assert_not_called()
items[1].reconstruct.assert_called_once_with(0, ipc_consumer_count=8)
items[1].reconstruct.assert_called_once_with(0)
materialize.assert_called_once_with(
[items[1].feature], device=visual.device, dtype=visual.dtype
)
@@ -2,6 +2,8 @@
from __future__ import annotations
import asyncio
import concurrent.futures
import gc
import multiprocessing as mp
import os
@@ -13,8 +15,14 @@ from unittest.mock import patch
import torch
from sglang.srt.managers.schedule_batch import (
Modality,
MultimodalDataItem,
MultimodalProcessorOutput,
)
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils.cuda_vmm_transport_utils import (
CudaVmmFeatureTransport,
CudaVmmMemoryPool,
CudaVmmPackedTensorTransportProxy,
_imported_pool_cache_clear,
@@ -55,13 +63,9 @@ def _produce_vmm_tensor(proxy_queue, consumer_done, result_queue, mode):
proxy_queue.put((proxy, expected))
if not consumer_done.wait(timeout=60):
raise TimeoutError("consumers did not release the CUDA VMM tensor")
with pool._lock:
pool._recycle_chunks()
pool._merge_chunks()
if pool.occupied_chunks:
raise RuntimeError(
"consumer acknowledgements did not recycle the slice"
)
pool._recycle_chunks()
if pool.occupied_chunks:
raise RuntimeError("consumer acknowledgements did not recycle the slice")
except Exception as exc: # noqa: BLE001 # pragma: no cover
result_queue.put(("error", repr(exc)))
return
@@ -121,6 +125,7 @@ class TestCudaVmmTransport(CustomTestCase):
):
torch.cuda.set_device(device)
with get_parallel().override(
tp_rank=tp_rank,
attn_tp_size=2,
attn_tp_rank=tp_rank,
attn_cp_size=1,
@@ -156,17 +161,64 @@ class TestCudaVmmTransport(CustomTestCase):
def test_auto_prefers_fabric_tensor_round_trip_and_recycling(self):
self._run_round_trip(mode="auto")
def test_publication_waits_for_source_stream(self):
torch.cuda.set_device(0)
for mode in ("single", "packed", "async", "async_fallback"):
with self.subTest(mode=mode):
pool = CudaVmmMemoryPool(4 << 20, 60, 0, 1, allow_posix_fallback=True)
transport = object.__new__(CudaVmmFeatureTransport)
transport.pool = pool
try:
# Make a wrong-stream copy observably read the previous value.
size = 8 << 20 if mode == "async_fallback" else 1024
source = torch.zeros(size, dtype=torch.uint8, device="cuda")
torch.cuda.synchronize()
stream = torch.cuda.Stream()
with concurrent.futures.ThreadPoolExecutor(1) as executor:
transport._publisher_executor = executor
with torch.cuda.stream(stream):
torch.cuda._sleep(200_000_000)
source.fill_(73)
if mode == "single":
proxy = pool.wrap_tensor(source)
elif mode == "packed":
proxy = pool.wrap_tensors([source, source])[0]
else:
item = MultimodalDataItem(
modality=Modality.IMAGE, feature=source
)
asyncio.run(
transport.prepare_for_dispatch_async(
[MultimodalProcessorOutput(mm_items=[item])]
)
)
proxy = item.feature
result = (
proxy
if isinstance(proxy, torch.Tensor)
else proxy.reconstruct_on_target_device(0)
)
self.assertTrue(
torch.equal(
result.cpu(), torch.full((size,), 73, dtype=torch.uint8)
)
)
finally:
torch.cuda.synchronize()
_imported_pool_cache_clear()
pool.shutdown()
def test_reused_chunk_clears_acknowledgements(self):
pool = CudaVmmMemoryPool(4 << 20, 60, 0, 2, allow_posix_fallback=True)
try:
old = pool.wrap_tensor(torch.ones(1024, dtype=torch.uint8, device="cuda:0"))
# VMM must bypass the CUDA IPC zero-copy borrow protocol.
self.assertIsNone(old.borrow_on_target_device(0))
pool.memory_pool[old.control_offset : old.control_offset + 8].view(
torch.int32
).fill_(1)
torch.cuda.synchronize(0)
with pool._lock:
pool._recycle_chunks()
pool._merge_chunks()
pool._recycle_chunks()
pool.wrap_tensor(torch.ones(100, dtype=torch.uint8, device="cuda:0"))
live = pool.wrap_tensor(torch.ones(256, dtype=torch.uint8, device="cuda:0"))
@@ -175,8 +227,7 @@ class TestCudaVmmTransport(CustomTestCase):
].view(torch.int32)
self.assertTrue(torch.equal(control, torch.zeros_like(control)))
with pool._lock:
pool._recycle_chunks()
pool._recycle_chunks()
self.assertIn(
live.control_offset,
[chunk.start for chunk in pool.occupied_chunks],
@@ -237,8 +288,7 @@ class TestCudaVmmTransport(CustomTestCase):
for tensor in reconstructed
)
)
with pool._lock:
pool._recycle_chunks()
pool._recycle_chunks()
self.assertFalse(pool.occupied_chunks)
finally:
del reconstructed, proxies, expected, sources
@@ -425,8 +475,7 @@ class TestCudaVmmTransport(CustomTestCase):
):
proxy.reconstruct_on_target_device(0, consumer_count=1)
torch.cuda.synchronize(0)
with pool._lock:
pool._recycle_chunks()
pool._recycle_chunks()
self.assertFalse(pool.occupied_chunks)
with (
@@ -412,6 +412,11 @@ class TestCudaVmmFeatureTransport(unittest.TestCase):
transport.cancel_for_dispatch.assert_not_called()
def test_async_publication_keeps_event_loop_responsive(self):
from sglang.srt.managers.schedule_batch import (
Modality,
MultimodalDataItem,
MultimodalProcessorOutput,
)
from sglang.srt.utils.cuda_vmm_transport_utils import (
CudaVmmFeatureTransport,
)
@@ -437,7 +442,11 @@ class TestCudaVmmFeatureTransport(unittest.TestCase):
release.set()
transport.prepare_for_dispatch = MagicMock(side_effect=block_publication)
mm_inputs = SimpleNamespace(mm_items=[object()])
mm_inputs = MultimodalProcessorOutput(
mm_items=[
MultimodalDataItem(modality=Modality.IMAGE, feature=torch.ones(2))
]
)
async def run():
watchdog = threading.Thread(target=unblock_if_event_loop_stalls)
@@ -447,6 +456,8 @@ class TestCudaVmmFeatureTransport(unittest.TestCase):
transport.prepare_for_dispatch_async([mm_inputs])
)
while not started.is_set():
if task.done():
await task
await asyncio.sleep(0)
event_loop_responsive.set()
self.assertFalse(task.done())
@@ -519,6 +530,7 @@ class TestCudaVmmFeatureTransport(unittest.TestCase):
from sglang.srt.utils import cuda_vmm_transport_utils as vmm
pool = object.__new__(vmm.CudaVmmMemoryPool)
pool._lock = threading.Lock()
pool.device_index = 0
pool.consumer_count = 2
pool._recycle_stream = object()
@@ -1122,6 +1134,7 @@ class TestVmmConsumerCount(unittest.TestCase):
proxy = object.__new__(CudaVmmTensorTransportProxy)
proxy.consumer_count = 4
with get_parallel().override(
tp_rank=3,
attn_tp_size=2,
attn_tp_rank=1,
attn_cp_size=2,