[VLM] fix: repair multimodal VMM lease cleanup and source-stream ordering (#43475)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -9,7 +9,7 @@ from sglang.srt.managers.schedule_batch import MultimodalDataItem
|
||||
from sglang.srt.mem_cache.multimodal_cache import EmbeddingResult, MultiModalStaticCache
|
||||
from sglang.srt.multimodal.evs import EVSEmbeddingResult
|
||||
from sglang.srt.multimodal.transport.cuda_ipc import BORROW_CUDA_IPC_FEATURE_KEY
|
||||
from sglang.srt.runtime_context import get_parallel, get_schedule
|
||||
from sglang.srt.runtime_context import get_schedule
|
||||
from sglang.srt.utils import is_hip, is_npu, is_xpu
|
||||
from sglang.srt.utils.async_probe import maybe_assert_sum
|
||||
from sglang.utils import logger
|
||||
@@ -243,26 +243,6 @@ def _move_items_to_device(
|
||||
item.feature = item.feature.to(device, non_blocking=True)
|
||||
|
||||
|
||||
def _acknowledge_deferred_cuda_ipc_cache_hits(
|
||||
items: List[MultimodalDataItem],
|
||||
) -> None:
|
||||
"""Release lazy Kimi IPC slices when a cached embedding skips ViT.
|
||||
|
||||
On an encoder-DP miss, exactly one rank copies an image and acknowledges
|
||||
the full TP group. On a cache hit no rank copies it, so rank zero performs
|
||||
the equivalent single acknowledgement. This preserves the fixed-pool
|
||||
lifecycle without reintroducing an unnecessary GPU-to-GPU copy.
|
||||
"""
|
||||
parallel = get_parallel()
|
||||
if parallel.attn_tp_rank != 0:
|
||||
return
|
||||
# The pool's recycler counts the whole TP group, so the acknowledgement must
|
||||
# match that count even when an attention subgroup is smaller.
|
||||
consumer_count = max(parallel.tp_size, 1)
|
||||
for item in items:
|
||||
item.acknowledge_deferred_cuda_ipc_feature(consumer_count)
|
||||
|
||||
|
||||
def _item_overlap(
|
||||
item: MultimodalDataItem, chunk_start: int, chunk_end: int
|
||||
) -> Optional[int]:
|
||||
@@ -330,8 +310,8 @@ def _get_chunked_embedding_full(
|
||||
else embedding
|
||||
)
|
||||
embedding_cache.set(embedding_items_hash, embedding_per_req)
|
||||
else:
|
||||
_acknowledge_deferred_cuda_ipc_cache_hits(embedding_items_per_req)
|
||||
|
||||
# cache hits keep raw-input leases until request cleanup for eviction/re-prefill
|
||||
|
||||
if isinstance(embedding_per_req, EVSEmbeddingResult):
|
||||
item = embedding_items_per_req[0]
|
||||
@@ -492,7 +472,6 @@ def _get_chunked_embedding_by_item(
|
||||
cached_token_count = _embedding_token_count(cached_embedding)
|
||||
if cached_token_count == expected_token_count:
|
||||
cached_embeddings[idx] = cached_embedding
|
||||
_acknowledge_deferred_cuda_ipc_cache_hits([item])
|
||||
else:
|
||||
_discard_mismatched_cached_embedding(
|
||||
item.hash, expected_token_count, cached_token_count
|
||||
|
||||
@@ -1077,6 +1077,8 @@ class Req(ReqDllmMixin):
|
||||
self.dllm_initialized: bool = False
|
||||
|
||||
self.session = session
|
||||
# media added by this turn, excluding the session's borrowed history
|
||||
self._session_mm_items: List[MultimodalDataItem] = []
|
||||
self.session_id = session_id
|
||||
# Used by the session radix cache to reject registration after a close/reopen.
|
||||
self.session_generation: Optional[int] = None
|
||||
@@ -1519,6 +1521,7 @@ class Req(ReqDllmMixin):
|
||||
|
||||
def _extend_session_image_inputs(self, image_inputs):
|
||||
"""Append media while preserving the saved session and its position history."""
|
||||
self._session_mm_items.extend(image_inputs.mm_items)
|
||||
# Padding can change token values without changing their count.
|
||||
self.full_untruncated_fill_ids = array("q")
|
||||
if self.multimodal_inputs is not None:
|
||||
@@ -1564,6 +1567,18 @@ class Req(ReqDllmMixin):
|
||||
positions.max() + 1 - positions.shape[1]
|
||||
).reshape(1, 1)
|
||||
|
||||
def release_mm_inputs_on_abort(self) -> None:
|
||||
"""Release this turn's media without invalidating session history."""
|
||||
if self.session is None:
|
||||
if self.multimodal_inputs is not None:
|
||||
self.multimodal_inputs.release_features()
|
||||
else:
|
||||
for item in self._session_mm_items:
|
||||
item.release_transport_proxies()
|
||||
item.feature = None
|
||||
self._session_mm_items.clear()
|
||||
self.multimodal_inputs = None
|
||||
|
||||
def finished(self) -> bool:
|
||||
# Whether request reached finished condition
|
||||
return self.finished_reason is not None
|
||||
@@ -2110,9 +2125,7 @@ class Req(ReqDllmMixin):
|
||||
logger.error(f"{error_msg}, {self.rid=}")
|
||||
# Session requests share historical multimodal inputs with their prior
|
||||
# request. The session owns and releases those features when it closes.
|
||||
if self.multimodal_inputs is not None and self.session is None:
|
||||
self.multimodal_inputs.release_features()
|
||||
self.multimodal_inputs = None
|
||||
self.release_mm_inputs_on_abort()
|
||||
self.grammar = None
|
||||
self.origin_input_ids = array(
|
||||
"q", [0]
|
||||
|
||||
@@ -23,7 +23,7 @@ import sys
|
||||
import time
|
||||
from array import array
|
||||
from collections import deque
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from contextlib import ExitStack, contextmanager, nullcontext
|
||||
from functools import partial
|
||||
from http import HTTPStatus
|
||||
from typing import TYPE_CHECKING, Any, Deque, Dict, List, Optional, Set, Tuple, Union
|
||||
@@ -210,6 +210,7 @@ from sglang.srt.managers.prefill_delayer import (
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
FINISH_ABORT,
|
||||
MultimodalInputs,
|
||||
MultimodalProcessorOutput,
|
||||
NextBatchPlan,
|
||||
Req,
|
||||
ScheduleBatch,
|
||||
@@ -2235,6 +2236,12 @@ class Scheduler(
|
||||
)
|
||||
error_msg = f"Multimodal feature reconstruction failed ({details})"
|
||||
logger.error(error_msg)
|
||||
if isinstance(
|
||||
tokenized_req.mm_inputs,
|
||||
(MultimodalInputs, MultimodalProcessorOutput),
|
||||
):
|
||||
for item in tokenized_req.mm_inputs.mm_items:
|
||||
item.release_transport_proxies()
|
||||
tokenized_req.mm_inputs = None
|
||||
request_errors.append(error_msg)
|
||||
else:
|
||||
@@ -2779,6 +2786,8 @@ class Scheduler(
|
||||
continue
|
||||
# For session requests, keep mm_inputs for the next request
|
||||
if req.session:
|
||||
if isinstance(req.finished_reason, FINISH_ABORT):
|
||||
req.release_mm_inputs_on_abort()
|
||||
continue
|
||||
# For non-session requests, clear features and mm_inputs
|
||||
mm_inputs.release_features()
|
||||
@@ -2789,6 +2798,23 @@ class Scheduler(
|
||||
recv_req: TokenizedGenerateReqInput,
|
||||
*,
|
||||
mm_input_error: Optional[str] = None,
|
||||
):
|
||||
with ExitStack() as mm_cleanup:
|
||||
if isinstance(
|
||||
recv_req.mm_inputs, (MultimodalInputs, MultimodalProcessorOutput)
|
||||
):
|
||||
for item in recv_req.mm_inputs.mm_items:
|
||||
mm_cleanup.callback(item.release_transport_proxies)
|
||||
self._handle_generate_request(
|
||||
recv_req, mm_cleanup, mm_input_error=mm_input_error
|
||||
)
|
||||
|
||||
def _handle_generate_request(
|
||||
self,
|
||||
recv_req: TokenizedGenerateReqInput,
|
||||
mm_cleanup: ExitStack,
|
||||
*,
|
||||
mm_input_error: Optional[str] = None,
|
||||
):
|
||||
# Route: normal request / session request / session-not-found
|
||||
session_id = (
|
||||
@@ -3047,6 +3073,8 @@ class Scheduler(
|
||||
req.origin_input_ids, image_inputs, self.pad_input_ids_func
|
||||
)
|
||||
req.extend_image_inputs(image_inputs)
|
||||
# the request now owns cleanup, including rejection after padding
|
||||
mm_cleanup.pop_all()
|
||||
self._maybe_compute_mrope_positions(req)
|
||||
|
||||
if len(req.origin_input_ids) >= self.max_req_input_len:
|
||||
@@ -3279,6 +3307,7 @@ class Scheduler(
|
||||
# detach lives in `StreamingSession.find_active_slot`, which only runs
|
||||
# while scheduling; a session left in-flight rejects every later request.
|
||||
if req.session is not None and req.session.streaming:
|
||||
req.release_mm_inputs_on_abort()
|
||||
req.session.abort_req()
|
||||
req.session = None
|
||||
# `beam_coordinator.validate_and_init` counts the group in ahead of the
|
||||
@@ -3364,7 +3393,10 @@ class Scheduler(
|
||||
|
||||
def _release_aborted_request(self, req: Req) -> None:
|
||||
"""Drop the cache-side state an aborted request left behind."""
|
||||
req.release_mm_inputs_on_abort()
|
||||
self.tree_cache.finish(req.cache_request_handle, CacheRequestOutcome.ABORT)
|
||||
if req.session is not None and req.session.streaming:
|
||||
req.session.abort_req()
|
||||
|
||||
def _abort_on_queued_limit(self, recv_req: Req) -> bool:
|
||||
"""Abort an incoming or existing request if the waiting queue is full. Returns True if the incoming request is aborted."""
|
||||
@@ -3472,6 +3504,23 @@ class Scheduler(
|
||||
recv_req: TokenizedEmbeddingReqInput,
|
||||
*,
|
||||
mm_input_error: Optional[str] = None,
|
||||
):
|
||||
with ExitStack() as mm_cleanup:
|
||||
if isinstance(
|
||||
recv_req.mm_inputs, (MultimodalInputs, MultimodalProcessorOutput)
|
||||
):
|
||||
for item in recv_req.mm_inputs.mm_items:
|
||||
mm_cleanup.callback(item.release_transport_proxies)
|
||||
self._handle_embedding_request(
|
||||
recv_req, mm_cleanup, mm_input_error=mm_input_error
|
||||
)
|
||||
|
||||
def _handle_embedding_request(
|
||||
self,
|
||||
recv_req: TokenizedEmbeddingReqInput,
|
||||
mm_cleanup: ExitStack,
|
||||
*,
|
||||
mm_input_error: Optional[str] = None,
|
||||
):
|
||||
req = Req(
|
||||
recv_req.rid,
|
||||
@@ -3528,6 +3577,7 @@ class Scheduler(
|
||||
)
|
||||
|
||||
req.extend_image_inputs(image_inputs)
|
||||
mm_cleanup.pop_all()
|
||||
self._maybe_compute_mrope_positions(req)
|
||||
|
||||
if len(req.origin_input_ids) >= self.max_req_input_len:
|
||||
|
||||
@@ -4597,7 +4597,7 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
spans = []
|
||||
device, dtype = self.image_start.device, self.image_start.dtype
|
||||
for item in items:
|
||||
item.reconstruct(device.index, ipc_consumer_count=self.tp_size)
|
||||
item.reconstruct(device.index)
|
||||
h, w = int(item.n_vit_h), int(item.n_vit_w)
|
||||
pixels = torch.as_tensor(item.feature, device=device)
|
||||
plan = item.model_specific_data.get(GPU_PLAN_KEY)
|
||||
|
||||
@@ -49,7 +49,6 @@ from sglang.srt.multimodal.mm_utils import (
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_mm,
|
||||
get_parallel,
|
||||
)
|
||||
from sglang.srt.utils import add_prefix, is_cuda, is_npu
|
||||
|
||||
@@ -724,14 +723,9 @@ class KimiK25ForConditionalGeneration(nn.Module):
|
||||
|
||||
CUDA IPC features are intentionally reconstructed after the image
|
||||
assignment. Each image therefore crosses the tokenizer/scheduler
|
||||
boundary once instead of once per TP rank. The selected consumer
|
||||
acknowledges the entire TP group so the bounded IPC pool remains
|
||||
recyclable.
|
||||
boundary once instead of once per TP rank. Unused ranks keep their
|
||||
lease until request cleanup so cache eviction can re-run the encoder.
|
||||
"""
|
||||
# Match the configured TP consumer count captured when the
|
||||
# tokenizer creates MmItemMemoryPool. A live attention subgroup
|
||||
# size could leave acknowledgements missing and strand the lease.
|
||||
ipc_consumer_count = max(get_parallel().tp_size, 1)
|
||||
device_index = device.index
|
||||
if device.type == "cuda" and device_index is None:
|
||||
device_index = torch.cuda.current_device()
|
||||
@@ -740,9 +734,7 @@ class KimiK25ForConditionalGeneration(nn.Module):
|
||||
for image_index in image_indices:
|
||||
item = items[image_index]
|
||||
if device.type == "cuda":
|
||||
item.reconstruct(
|
||||
device_index, ipc_consumer_count=ipc_consumer_count
|
||||
)
|
||||
item.reconstruct(device_index)
|
||||
feature = item.feature
|
||||
if not isinstance(feature, torch.Tensor):
|
||||
raise TypeError(
|
||||
|
||||
@@ -3613,10 +3613,6 @@ class KimiK3ForConditionalGeneration(nn.Module):
|
||||
LOCAL_PREPROCESSED_KEY,
|
||||
)
|
||||
|
||||
# Match the configured TP consumer count captured when the
|
||||
# tokenizer creates MmItemMemoryPool. A live attention subgroup
|
||||
# size could leave acknowledgements missing and strand the lease.
|
||||
ipc_consumer_count = max(get_parallel().tp_size, 1)
|
||||
device_index = device.index
|
||||
if device.type == "cuda" and device_index is None:
|
||||
device_index = torch.cuda.current_device()
|
||||
@@ -3625,9 +3621,7 @@ class KimiK3ForConditionalGeneration(nn.Module):
|
||||
for image_index in image_indices:
|
||||
item = items[image_index]
|
||||
if device.type == "cuda":
|
||||
item.reconstruct(
|
||||
device_index, ipc_consumer_count=ipc_consumer_count
|
||||
)
|
||||
item.reconstruct(device_index)
|
||||
selected_items.append(item)
|
||||
|
||||
locally_preprocessed = [
|
||||
|
||||
@@ -1521,7 +1521,7 @@ class Qwen3VLForConditionalGeneration(nn.Module):
|
||||
(item, proxy, feature_offset, borrowed.shape[0])
|
||||
)
|
||||
elif RETAINED_CUDA_IPC_FEATURE_PROXY_KEY not in model_specific_data:
|
||||
item.reconstruct(device_index, ipc_consumer_count=consumer_count)
|
||||
item.reconstruct(device_index)
|
||||
features.append(item.feature)
|
||||
feature_offset += item.feature.shape[0]
|
||||
try:
|
||||
|
||||
@@ -44,6 +44,7 @@ from sglang.srt.multimodal.transport.cuda_ipc import (
|
||||
MmItemMemoryPool,
|
||||
get_mm_feature_pool_size_per_worker,
|
||||
)
|
||||
from sglang.srt.multimodal.transport.memory_pool import get_mm_feature_consumer_count
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_mm,
|
||||
@@ -450,7 +451,7 @@ class BaseMultimodalProcessor(ABC):
|
||||
per_worker_pool_size,
|
||||
MM_ITEM_MEMORY_POOL_RECYCLE_INTERVAL,
|
||||
self.server_args.base_gpu_id,
|
||||
self.server_args.tp_size,
|
||||
get_mm_feature_consumer_count(),
|
||||
)
|
||||
|
||||
@property
|
||||
|
||||
@@ -16,6 +16,14 @@ DATA_ALIGNMENT = 256
|
||||
DEFAULT_MAX_INFLIGHT_SLICES = 4096
|
||||
|
||||
|
||||
def get_mm_feature_consumer_count() -> int:
|
||||
"""Number of scheduler ranks receiving one routed multimodal request."""
|
||||
parallel = get_parallel()
|
||||
if parallel.attn_dp_enabled:
|
||||
return parallel.tp_size // parallel.num_dp_ranks
|
||||
return parallel.tp_size
|
||||
|
||||
|
||||
def align_up(value: int, alignment: int) -> int:
|
||||
return ((value + alignment - 1) // alignment) * alignment
|
||||
|
||||
@@ -59,9 +67,8 @@ def resolve_consumer_rank(
|
||||
return 0
|
||||
if consumer_rank is None:
|
||||
try:
|
||||
# Use the global TP rank. An attention/DCP subgroup rank can alias
|
||||
# another consumer's acknowledgement slot.
|
||||
rank = int(get_parallel().tp_rank)
|
||||
# DP groups are contiguous; attention/DCP subgroup ranks can alias
|
||||
rank = int(get_parallel().tp_rank) % total_consumer_count
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
f"Cannot resolve the {transport_name} consumer rank before "
|
||||
|
||||
@@ -80,6 +80,7 @@ class StreamingSession:
|
||||
if slot is None or not slot.kv.holds_kv:
|
||||
return None
|
||||
if req.to_finish is not None:
|
||||
req.release_mm_inputs_on_abort()
|
||||
req.session.abort_req()
|
||||
req.session = None
|
||||
return None
|
||||
@@ -160,6 +161,7 @@ class StreamingSession:
|
||||
# Hand the record and the tree lock back; the caller releases them.
|
||||
del self.slots[req.session.session_id]
|
||||
req.lock, slot.lock = slot.lock, None
|
||||
req.release_mm_inputs_on_abort()
|
||||
req.session.abort_req()
|
||||
return False
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ from sglang.srt.managers.schedule_batch import (
|
||||
MultimodalDataItem,
|
||||
MultimodalProcessorOutput,
|
||||
)
|
||||
from sglang.srt.multimodal.transport.memory_pool import get_mm_feature_consumer_count
|
||||
from sglang.srt.runtime_context import (
|
||||
get_mm,
|
||||
get_parallel,
|
||||
@@ -192,9 +193,7 @@ def _contains_tensor_container(value) -> bool:
|
||||
|
||||
|
||||
def get_vmm_feature_consumer_count() -> int:
|
||||
if get_parallel().attn_dp_enabled:
|
||||
return get_parallel().tp_size // get_parallel().num_dp_ranks
|
||||
return get_parallel().tp_size
|
||||
return get_mm_feature_consumer_count()
|
||||
|
||||
|
||||
class CudaVmmMemoryPool:
|
||||
@@ -402,6 +401,10 @@ class CudaVmmMemoryPool:
|
||||
producer_stream = None
|
||||
copy_synchronized = False
|
||||
try:
|
||||
if tensor.is_cuda:
|
||||
self._publish_stream.wait_stream(
|
||||
torch.cuda.current_stream(tensor.device)
|
||||
)
|
||||
with (
|
||||
torch.cuda.device(self.device_index),
|
||||
torch.cuda.stream(self._publish_stream),
|
||||
@@ -477,6 +480,8 @@ class CudaVmmMemoryPool:
|
||||
producer_stream = None
|
||||
copy_synchronized = False
|
||||
try:
|
||||
for device in {tensor.device for tensor in tensors if tensor.is_cuda}:
|
||||
self._publish_stream.wait_stream(torch.cuda.current_stream(device))
|
||||
with (
|
||||
torch.cuda.device(self.device_index),
|
||||
torch.cuda.stream(self._publish_stream),
|
||||
@@ -604,20 +609,19 @@ class CudaVmmMemoryPool:
|
||||
def _recycle_loop(self) -> None:
|
||||
while not self._stop_recycler.wait(self._recycle_interval):
|
||||
try:
|
||||
with self._lock:
|
||||
self._recycle_chunks()
|
||||
self._merge_chunks()
|
||||
self._recycle_chunks()
|
||||
except Exception as error:
|
||||
logger.exception("CUDA VMM multimodal pool recycle failed")
|
||||
self._pool_error = error
|
||||
self._stop_recycler.set()
|
||||
|
||||
def _recycle_chunks(self) -> None:
|
||||
if not self.occupied_chunks:
|
||||
with self._lock:
|
||||
chunks = tuple(self.occupied_chunks)
|
||||
if not chunks:
|
||||
return
|
||||
|
||||
remaining = []
|
||||
recycled = []
|
||||
# CUDA polling must not hold the allocator lock needed by publishers
|
||||
with (
|
||||
torch.cuda.device(self.device_index),
|
||||
torch.cuda.stream(self._recycle_stream),
|
||||
@@ -628,23 +632,23 @@ class CudaVmmMemoryPool:
|
||||
chunk.start : chunk.start
|
||||
+ self.consumer_count * _CONTROL_WORD_BYTES
|
||||
].view(torch.int32)
|
||||
for chunk in self.occupied_chunks
|
||||
for chunk in chunks
|
||||
]
|
||||
)
|
||||
acknowledgement_counts = (
|
||||
torch.count_nonzero(acknowledgement_words, dim=1).cpu().tolist()
|
||||
)
|
||||
|
||||
for chunk, acknowledgement_count in zip(
|
||||
self.occupied_chunks, acknowledgement_counts, strict=True
|
||||
):
|
||||
if acknowledgement_count == self.consumer_count:
|
||||
recycled.append(_CudaVmmMemoryChunk(chunk.start, chunk.end))
|
||||
else:
|
||||
remaining.append(chunk)
|
||||
|
||||
self.available_chunks.extend(recycled)
|
||||
self.occupied_chunks = remaining
|
||||
with self._lock:
|
||||
live = {chunk.start: chunk for chunk in self.occupied_chunks}
|
||||
for chunk, count in zip(chunks, acknowledgement_counts, strict=True):
|
||||
# cancellation may have freed and republished the same offset
|
||||
if count == self.consumer_count and live.get(chunk.start) is chunk:
|
||||
self.occupied_chunks.remove(chunk)
|
||||
self.available_chunks.append(
|
||||
_CudaVmmMemoryChunk(chunk.start, chunk.end)
|
||||
)
|
||||
self._merge_chunks()
|
||||
|
||||
def _merge_chunks(self) -> None:
|
||||
merged = []
|
||||
@@ -809,6 +813,10 @@ class CudaVmmTensorTransportProxy(CudaIpcTensorTransportProxy):
|
||||
device_index=device_index,
|
||||
)
|
||||
|
||||
def borrow_on_target_device(self, rebuild_device_idx: int) -> None:
|
||||
# VMM copies into consumer-owned storage; the IPC borrow protocol differs
|
||||
return None
|
||||
|
||||
def _acknowledgement_range(self, consumer_count: int) -> tuple[int, int]:
|
||||
if consumer_count <= 0:
|
||||
raise ValueError("consumer_count must be positive")
|
||||
@@ -816,6 +824,10 @@ class CudaVmmTensorTransportProxy(CudaIpcTensorTransportProxy):
|
||||
return 0, self.consumer_count
|
||||
|
||||
parallel = get_parallel()
|
||||
if consumer_count == 1:
|
||||
# attention/DCP subgroup ranks can alias; DP groups are contiguous
|
||||
slot = parallel.tp_rank % self.consumer_count
|
||||
return slot, slot + 1
|
||||
group_start = parallel.attn_cp_rank * parallel.attn_tp_size
|
||||
group_end = group_start + parallel.attn_tp_size
|
||||
if not 0 <= group_start < group_end <= self.consumer_count:
|
||||
@@ -824,9 +836,6 @@ class CudaVmmTensorTransportProxy(CudaIpcTensorTransportProxy):
|
||||
f"[{group_start}, {group_end}) is outside "
|
||||
f"consumer_count={self.consumer_count}"
|
||||
)
|
||||
if consumer_count == 1:
|
||||
slot = group_start + parallel.attn_tp_rank
|
||||
return slot, slot + 1
|
||||
if consumer_count == parallel.attn_tp_size:
|
||||
return group_start, group_end
|
||||
raise ValueError(
|
||||
@@ -1033,18 +1042,49 @@ class CudaVmmFeatureTransport:
|
||||
if self._publisher_executor is None:
|
||||
raise RuntimeError("CUDA VMM feature transport is shutting down")
|
||||
|
||||
future = asyncio.get_running_loop().run_in_executor(
|
||||
self._publisher_executor,
|
||||
self.prepare_for_dispatch,
|
||||
mm_inputs_batch,
|
||||
# CUDA current streams are thread-local; carry readiness into the publisher.
|
||||
ready_events = []
|
||||
devices = {
|
||||
tensor.device
|
||||
for mm_inputs in mm_inputs_batch
|
||||
if mm_inputs is not None
|
||||
for item in mm_inputs.mm_items
|
||||
for tensor in (item.feature, item.precomputed_embeddings)
|
||||
if isinstance(tensor, torch.Tensor) and tensor.is_cuda
|
||||
}
|
||||
for device in devices:
|
||||
event = torch.cuda.Event()
|
||||
event.record(torch.cuda.current_stream(device))
|
||||
ready_events.append((device, event))
|
||||
future = self._publisher_executor.submit(
|
||||
self._prepare_after_source_events, mm_inputs_batch, ready_events
|
||||
)
|
||||
try:
|
||||
return await asyncio.shield(future)
|
||||
return await asyncio.shield(asyncio.wrap_future(future))
|
||||
except asyncio.CancelledError:
|
||||
prepared_mm_items = await future
|
||||
self.cancel_for_dispatch(prepared_mm_items)
|
||||
# cleanup must survive a second task cancellation or loop shutdown
|
||||
future.add_done_callback(self._cancel_completed_publish)
|
||||
raise
|
||||
|
||||
def _prepare_after_source_events(
|
||||
self,
|
||||
mm_inputs_batch: Iterable[MultimodalProcessorOutput | None],
|
||||
ready_events: list[tuple[torch.device, torch.cuda.Event]],
|
||||
) -> list[MultimodalDataItem]:
|
||||
for device, event in ready_events:
|
||||
# Both VMM copies and a pool-full CPU fallback must see ready input.
|
||||
torch.cuda.current_stream(device).wait_event(event)
|
||||
return self.prepare_for_dispatch(mm_inputs_batch)
|
||||
|
||||
def _cancel_completed_publish(self, future: concurrent.futures.Future) -> None:
|
||||
if future.cancelled() or future.exception() is not None:
|
||||
# failed publication already rolls back its slices
|
||||
return
|
||||
try:
|
||||
self.cancel_for_dispatch(future.result())
|
||||
except Exception:
|
||||
logger.exception("Failed to cancel abandoned CUDA VMM publication")
|
||||
|
||||
def prepare_for_dispatch(
|
||||
self,
|
||||
mm_inputs_batch: Iterable[MultimodalProcessorOutput | None],
|
||||
|
||||
@@ -50,9 +50,7 @@ HIDDEN = 16
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def single_process_runtime_context():
|
||||
# These mm_utils unit tests exercise cache-hit paths that acknowledge
|
||||
# deferred CUDA IPC through runtime_context. They do not start an engine, so
|
||||
# pin the runtime topology to a single-process CPU setup.
|
||||
# Exercise the embedding path without starting an engine.
|
||||
server_args_override = get_context().override_server_args(tp_size=1)
|
||||
server_args_override.install()
|
||||
try:
|
||||
@@ -74,14 +72,6 @@ CHUNKS = [(0, 8), (8, 8), (16, 8), (24, 6)]
|
||||
_CPU = torch.device("cpu")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _skip_cuda_ipc_acknowledgement(monkeypatch):
|
||||
"""Keep CPU embedding tests independent of tensor-parallel runtime state."""
|
||||
monkeypatch.setattr(
|
||||
mm_schedule, "_acknowledge_deferred_cuda_ipc_cache_hits", lambda _items: None
|
||||
)
|
||||
|
||||
|
||||
def _num_tokens(item: MultimodalDataItem) -> int:
|
||||
start, end = item.offsets[0]
|
||||
return end - start + 1
|
||||
|
||||
@@ -439,6 +439,25 @@ class TestStreamOrderedMmFeaturePool(CustomTestCase):
|
||||
with get_parallel().override(tp_rank=6, attn_tp_rank=2):
|
||||
self.assertEqual(resolve_consumer_rank(8), 6)
|
||||
|
||||
def test_attention_dp_uses_only_the_routed_scheduler_group(self):
|
||||
from sglang.srt.multimodal.transport.memory_pool import (
|
||||
get_mm_feature_consumer_count,
|
||||
resolve_consumer_rank,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
with get_parallel().override(
|
||||
tp_size=8,
|
||||
attn_dp_enabled=True,
|
||||
num_dp_ranks=2,
|
||||
tp_rank=6,
|
||||
attn_tp_rank=0,
|
||||
attn_dcp_size=2,
|
||||
):
|
||||
count = get_mm_feature_consumer_count()
|
||||
self.assertEqual(count, 4)
|
||||
self.assertEqual(resolve_consumer_rank(count), 2)
|
||||
|
||||
def test_complete_group_acknowledges_each_consumer_slot(self):
|
||||
from sglang.srt.multimodal.transport import memory_pool
|
||||
|
||||
|
||||
@@ -1,21 +1,321 @@
|
||||
import asyncio
|
||||
import concurrent.futures
|
||||
import copy
|
||||
import sys
|
||||
import threading
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.srt.disaggregation.utils import DisaggregationMode
|
||||
from sglang.srt.managers import mm_schedule
|
||||
from sglang.srt.managers.io_struct import (
|
||||
AbortReq,
|
||||
SessionParams,
|
||||
TokenizedGenerateReqInput,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
MultimodalDataItem,
|
||||
MultimodalInputs,
|
||||
MultimodalProcessorOutput,
|
||||
Req,
|
||||
)
|
||||
from sglang.srt.managers.scheduler import Scheduler
|
||||
from sglang.srt.mem_cache.multimodal_cache import EmbeddingResult, MultiModalStaticCache
|
||||
from sglang.srt.multimodal.transport.cuda_ipc import CudaIpcTensorTransportProxy
|
||||
from sglang.srt.runtime_context import get_context, get_parallel
|
||||
from sglang.srt.sampling.sampling_params import SamplingParams
|
||||
from sglang.srt.session.session_controller import Session
|
||||
from sglang.srt.utils import cuda_vmm_transport_utils as vmm
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def vmm_pool():
|
||||
# exercise the real slice allocator and ACK protocol with host-backed storage
|
||||
pool = object.__new__(vmm.CudaVmmMemoryPool)
|
||||
pool.memory_pool = torch.zeros(1024, dtype=torch.uint8)
|
||||
pool.consumer_count = 2
|
||||
pool.device_index = 0
|
||||
pool._recycle_stream = None
|
||||
pool._lock = threading.Lock()
|
||||
pool.available_chunks = [vmm._CudaVmmMemoryChunk(0, 1024)]
|
||||
pool.occupied_chunks = []
|
||||
with (
|
||||
get_context().override_server_args(tp_size=2),
|
||||
patch.object(torch.cuda, "device", return_value=nullcontext()),
|
||||
patch.object(torch.cuda, "stream", return_value=nullcontext()),
|
||||
patch.object(torch.cuda, "current_device", return_value=0),
|
||||
patch.object(
|
||||
vmm.CudaVmmTensorTransportProxy,
|
||||
"_pool",
|
||||
return_value=SimpleNamespace(memory=pool.memory_pool),
|
||||
),
|
||||
):
|
||||
yield pool
|
||||
|
||||
|
||||
def _publish(pool):
|
||||
chunk = pool._reserve_chunk(1024)
|
||||
assert chunk is not None, "a previous request stranded the pool slice"
|
||||
pool.memory_pool.zero_()
|
||||
pool.occupied_chunks.append(chunk)
|
||||
proxy = vmm.CudaVmmTensorTransportProxy(
|
||||
fabric_handle=b"test",
|
||||
posix_socket_path=None,
|
||||
allocation_size=1024,
|
||||
data_offset=256,
|
||||
data_nbytes=768,
|
||||
control_offset=0,
|
||||
consumer_count=2,
|
||||
shape=(768,),
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
return [
|
||||
MultimodalDataItem(modality=Modality.IMAGE, feature=copy.deepcopy(proxy)),
|
||||
MultimodalDataItem(modality=Modality.IMAGE, feature=copy.deepcopy(proxy)),
|
||||
]
|
||||
|
||||
|
||||
def _rank(rank):
|
||||
return get_parallel().override(
|
||||
tp_rank=rank, attn_tp_rank=rank, attn_tp_size=2, attn_cp_rank=0, attn_cp_size=1
|
||||
)
|
||||
|
||||
|
||||
def _queued_scheduler(req):
|
||||
scheduler = object.__new__(Scheduler)
|
||||
scheduler.waiting_queue = [req]
|
||||
scheduler.chunked_req = scheduler.mm_receiver = scheduler.dllm_config = None
|
||||
scheduler.disaggregation_mode = DisaggregationMode.NULL
|
||||
scheduler.tree_cache = MagicMock()
|
||||
scheduler.beam_coordinator = MagicMock()
|
||||
scheduler.ipc_channels = MagicMock()
|
||||
scheduler.grammar_manager = MagicMock()
|
||||
scheduler.collect_inflight_reqs = lambda: set()
|
||||
return scheduler
|
||||
|
||||
|
||||
def test_queued_abort_recycles_unconsumed_vmm_slices(vmm_pool):
|
||||
"""Client cancellations before prefill must not exhaust the bounded pool."""
|
||||
for _ in range(3):
|
||||
for rank, item in enumerate(_publish(vmm_pool)):
|
||||
req = Req("queued-image", "", [1], SamplingParams())
|
||||
req.multimodal_inputs = MultimodalInputs(mm_items=[item])
|
||||
scheduler = _queued_scheduler(req)
|
||||
with _rank(rank):
|
||||
scheduler.abort_request(AbortReq(rid=req.rid))
|
||||
assert scheduler.waiting_queue == []
|
||||
vmm_pool._recycle_chunks()
|
||||
assert not vmm_pool.occupied_chunks
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["full", "per_image", "batch"])
|
||||
def test_embedding_hit_cannot_recycle_another_ranks_live_proxy(vmm_pool, path):
|
||||
"""A rank's cache hit must not retire another rank's proxy before cleanup."""
|
||||
items = _publish(vmm_pool)
|
||||
cache = MultiModalStaticCache(1 << 20)
|
||||
cached = torch.ones(1, 2)
|
||||
cache.set(cache.combine_hashes([123]), EmbeddingResult(embedding=cached))
|
||||
cache.set(123, EmbeddingResult(embedding=cached))
|
||||
|
||||
def encode(_):
|
||||
pytest.fail("cache hit unexpectedly ran the encoder")
|
||||
|
||||
for rank, item in enumerate(items):
|
||||
item.hash = 123
|
||||
item.offsets = [(0, 0)]
|
||||
with _rank(rank), patch.object(mm_schedule, "embedding_cache", cache):
|
||||
if path == "full":
|
||||
embedding, _ = mm_schedule._get_chunked_embedding_full(
|
||||
encode,
|
||||
[item],
|
||||
[(0, 0)],
|
||||
0,
|
||||
1,
|
||||
torch.tensor([1]),
|
||||
torch.device("cpu"),
|
||||
)
|
||||
elif path == "per_image":
|
||||
[embedding] = mm_schedule._get_chunked_embedding_by_item(
|
||||
encode,
|
||||
[item],
|
||||
0,
|
||||
1,
|
||||
torch.device("cpu"),
|
||||
)
|
||||
else:
|
||||
request = mm_schedule.PerImageRequestInfo(0, [item], [(0, 0)], 0, 1)
|
||||
embeddings = mm_schedule._batch_encode_per_image_misses(
|
||||
encode,
|
||||
[request],
|
||||
torch.device("cpu"),
|
||||
)
|
||||
embedding = embeddings[(123, 1)]
|
||||
assert torch.equal(embedding, cached)
|
||||
assert not item.feature._consumer_acknowledged, (
|
||||
"cache eviction may need this input again"
|
||||
)
|
||||
for rank, item in enumerate(items):
|
||||
with _rank(rank):
|
||||
item.release_transport_proxies()
|
||||
vmm_pool._recycle_chunks()
|
||||
assert not vmm_pool.occupied_chunks
|
||||
|
||||
_publish(vmm_pool)
|
||||
for rank, item in enumerate(items):
|
||||
with _rank(rank):
|
||||
MultimodalInputs(mm_items=[item]).release_features()
|
||||
vmm_pool._recycle_chunks()
|
||||
assert vmm_pool.occupied_chunks, "late cleanup acknowledged a recycled slice"
|
||||
|
||||
|
||||
def test_aborted_session_turn_releases_new_media_not_history(vmm_pool):
|
||||
"""An aborted append owns new media, but only borrows historical features."""
|
||||
for rank, item in enumerate(_publish(vmm_pool)):
|
||||
session = Session(100, streaming=True)
|
||||
session._inflight = True
|
||||
history = MultimodalDataItem(modality=Modality.IMAGE, feature=torch.ones(3))
|
||||
req = Req("session-image", "", [1], SamplingParams(), session=session)
|
||||
req.multimodal_inputs = MultimodalInputs(mm_items=[history])
|
||||
req.extend_image_inputs(MultimodalInputs(mm_items=[item]))
|
||||
with _rank(rank):
|
||||
_queued_scheduler(req).abort_request(AbortReq(rid=req.rid))
|
||||
assert torch.equal(history.feature, torch.ones(3))
|
||||
assert not session._inflight
|
||||
vmm_pool._recycle_chunks()
|
||||
assert not vmm_pool.occupied_chunks
|
||||
|
||||
|
||||
def test_rejected_session_releases_media_before_request_admission(vmm_pool):
|
||||
"""An invalid session is rejected before the request attaches its media."""
|
||||
for rank, item in enumerate(_publish(vmm_pool)):
|
||||
recv = TokenizedGenerateReqInput(
|
||||
rid="invalid-session",
|
||||
input_text="",
|
||||
input_ids=[1],
|
||||
input_embeds=None,
|
||||
mm_inputs=MultimodalProcessorOutput(mm_items=[item]),
|
||||
token_type_ids=None,
|
||||
sampling_params=SamplingParams(),
|
||||
return_logprob=False,
|
||||
logprob_start_len=-1,
|
||||
top_logprobs_num=0,
|
||||
token_ids_logprob=None,
|
||||
stream=False,
|
||||
session_params=SessionParams(id="missing"),
|
||||
)
|
||||
scheduler = _queued_scheduler(None)
|
||||
scheduler.session_controller = {}
|
||||
scheduler.model_config = SimpleNamespace(vocab_size=10)
|
||||
scheduler.tokenizer = None
|
||||
scheduler.init_req_max_new_tokens = lambda req: None
|
||||
scheduler._add_request_to_queue = lambda req: None
|
||||
with _rank(rank):
|
||||
scheduler.handle_generate_request(recv)
|
||||
vmm_pool._recycle_chunks()
|
||||
assert not vmm_pool.occupied_chunks
|
||||
|
||||
|
||||
def test_other_rank_materialization_failure_releases_local_proxies(vmm_pool):
|
||||
"""A healthy rank still owns lazy slices when another rank rejects the input."""
|
||||
for rank, item in enumerate(_publish(vmm_pool)):
|
||||
recv = TokenizedGenerateReqInput(
|
||||
rid="peer-failure",
|
||||
input_text="",
|
||||
input_ids=[1],
|
||||
input_embeds=None,
|
||||
mm_inputs=MultimodalInputs(mm_items=[item]),
|
||||
token_type_ids=None,
|
||||
sampling_params=SamplingParams(),
|
||||
return_logprob=False,
|
||||
logprob_start_len=-1,
|
||||
top_logprobs_num=0,
|
||||
token_ids_logprob=None,
|
||||
stream=False,
|
||||
)
|
||||
scheduler = _queued_scheduler(None)
|
||||
scheduler._gather_vmm_materialization_errors = lambda error: [
|
||||
None,
|
||||
"copy failed",
|
||||
]
|
||||
with _rank(rank):
|
||||
errors = scheduler._materialize_cuda_vmm_inputs(recv)
|
||||
assert "copy failed" in errors[0]
|
||||
assert recv.mm_inputs is None
|
||||
vmm_pool._recycle_chunks()
|
||||
assert not vmm_pool.occupied_chunks
|
||||
|
||||
|
||||
def test_repeated_cancellation_during_publication_does_not_leak(vmm_pool):
|
||||
"""Cancellation must not orphan slices still being published on another thread."""
|
||||
transport = object.__new__(vmm.CudaVmmFeatureTransport)
|
||||
transport.pool = vmm_pool
|
||||
started, release = threading.Event(), threading.Event()
|
||||
item = MultimodalDataItem(modality=Modality.IMAGE, feature=torch.ones(2))
|
||||
|
||||
def publish(_):
|
||||
started.set()
|
||||
assert release.wait(timeout=5)
|
||||
return _publish(vmm_pool)[0].feature
|
||||
|
||||
async def run():
|
||||
task = asyncio.create_task(
|
||||
transport.prepare_for_dispatch_async(
|
||||
[MultimodalProcessorOutput(mm_items=[item])]
|
||||
)
|
||||
)
|
||||
while not started.is_set():
|
||||
await asyncio.sleep(0)
|
||||
task.cancel()
|
||||
await asyncio.sleep(0)
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
|
||||
transport._publisher_executor = executor
|
||||
with patch.object(vmm_pool, "wrap_tensor", side_effect=publish):
|
||||
try:
|
||||
asyncio.run(run())
|
||||
finally:
|
||||
release.set()
|
||||
executor.shutdown(wait=True)
|
||||
assert not vmm_pool.occupied_chunks
|
||||
|
||||
|
||||
def test_recycler_does_not_hold_allocator_lock_or_free_reused_slice(vmm_pool):
|
||||
"""A slow ACK poll cannot block cancellation or retire a republished offset."""
|
||||
old = _publish(vmm_pool)
|
||||
vmm_pool._stop_recycler = threading.Event()
|
||||
vmm_pool._recycle_interval = 0.001
|
||||
vmm_pool._pool_error = None
|
||||
vmm_pool.memory_pool[:8].view(torch.int32).fill_(1)
|
||||
stack = torch.stack
|
||||
|
||||
def poll(tensors):
|
||||
vmm_pool._stop_recycler.set()
|
||||
assert vmm_pool._lock.acquire(blocking=False), (
|
||||
"CUDA poll held the allocator lock"
|
||||
)
|
||||
vmm_pool._lock.release()
|
||||
snapshot = stack(tensors)
|
||||
vmm_pool.cancel_proxy(old[0].feature)
|
||||
_publish(vmm_pool)
|
||||
return snapshot
|
||||
|
||||
with patch.object(torch, "stack", side_effect=poll):
|
||||
vmm_pool._recycle_loop()
|
||||
assert vmm_pool._pool_error is None
|
||||
assert vmm_pool.occupied_chunks, "old poll result retired a new allocation"
|
||||
|
||||
|
||||
def _deferred_proxy():
|
||||
proxy = object.__new__(CudaIpcTensorTransportProxy)
|
||||
proxy.total_consumer_count = 1
|
||||
@@ -70,6 +370,7 @@ def test_session_abort_preserves_shared_multimodal_features():
|
||||
req = object.__new__(Req)
|
||||
req.rid = "rejected-session-turn"
|
||||
req.session = object()
|
||||
req._session_mm_items = []
|
||||
req.multimodal_inputs = mm_inputs
|
||||
req.grammar = object()
|
||||
req.return_logprob = True
|
||||
|
||||
@@ -24,7 +24,7 @@ from sglang.test.test_utils import (
|
||||
|
||||
maybe_stub_sgl_kernel()
|
||||
|
||||
from sglang.srt.managers.schedule_batch import FINISH_ABORT
|
||||
from sglang.srt.managers.schedule_batch import FINISH_ABORT, Req
|
||||
from sglang.srt.managers.scheduler import Scheduler
|
||||
from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
CacheRequestHandle,
|
||||
@@ -35,11 +35,15 @@ register_cpu_ci(est_time=12, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class _FakeReq:
|
||||
release_mm_inputs_on_abort = Req.release_mm_inputs_on_abort
|
||||
|
||||
def __init__(self, rid, wait_entry=0.0, forward_entry=0.0, is_finished=False):
|
||||
self.rid = rid
|
||||
self.cache_request_handle = CacheRequestHandle(rid, 0)
|
||||
self.to_finish = None
|
||||
self.beam_group = None
|
||||
self.session = None
|
||||
self.multimodal_inputs = None
|
||||
self._finished = is_finished
|
||||
self.output_ids = []
|
||||
self.weight_version_events = []
|
||||
|
||||
@@ -3,7 +3,12 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.srt.managers.schedule_batch import FINISH_ABORT, FINISH_LENGTH, ReqKvInfo
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
FINISH_ABORT,
|
||||
FINISH_LENGTH,
|
||||
Req,
|
||||
ReqKvInfo,
|
||||
)
|
||||
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
BasePrefixCache,
|
||||
@@ -82,6 +87,8 @@ class _FakeInnerCache:
|
||||
|
||||
|
||||
class _FakeReq:
|
||||
release_mm_inputs_on_abort = Req.release_mm_inputs_on_abort
|
||||
|
||||
def __init__(
|
||||
self, session_id: str, req_pool_idx: int, committed: int, allocated: int
|
||||
):
|
||||
@@ -108,6 +115,8 @@ class _FakeReq:
|
||||
self.to_finish = None
|
||||
self.finished_reason = None
|
||||
self.finished_len = None
|
||||
self.multimodal_inputs = None
|
||||
self._session_mm_items = []
|
||||
|
||||
def detach_kv(self):
|
||||
kv, self.kv = self.kv, ReqKvInfo()
|
||||
|
||||
@@ -286,7 +286,7 @@ class TestQwen3VLFeatureMaterialization(CustomTestCase):
|
||||
|
||||
self.assertIs(output, encoded)
|
||||
items[0].reconstruct.assert_not_called()
|
||||
items[1].reconstruct.assert_called_once_with(0, ipc_consumer_count=8)
|
||||
items[1].reconstruct.assert_called_once_with(0)
|
||||
materialize.assert_called_once_with(
|
||||
[items[1].feature], device=visual.device, dtype=visual.dtype
|
||||
)
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import concurrent.futures
|
||||
import gc
|
||||
import multiprocessing as mp
|
||||
import os
|
||||
@@ -13,8 +15,14 @@ from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
MultimodalDataItem,
|
||||
MultimodalProcessorOutput,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils.cuda_vmm_transport_utils import (
|
||||
CudaVmmFeatureTransport,
|
||||
CudaVmmMemoryPool,
|
||||
CudaVmmPackedTensorTransportProxy,
|
||||
_imported_pool_cache_clear,
|
||||
@@ -55,13 +63,9 @@ def _produce_vmm_tensor(proxy_queue, consumer_done, result_queue, mode):
|
||||
proxy_queue.put((proxy, expected))
|
||||
if not consumer_done.wait(timeout=60):
|
||||
raise TimeoutError("consumers did not release the CUDA VMM tensor")
|
||||
with pool._lock:
|
||||
pool._recycle_chunks()
|
||||
pool._merge_chunks()
|
||||
if pool.occupied_chunks:
|
||||
raise RuntimeError(
|
||||
"consumer acknowledgements did not recycle the slice"
|
||||
)
|
||||
pool._recycle_chunks()
|
||||
if pool.occupied_chunks:
|
||||
raise RuntimeError("consumer acknowledgements did not recycle the slice")
|
||||
except Exception as exc: # noqa: BLE001 # pragma: no cover
|
||||
result_queue.put(("error", repr(exc)))
|
||||
return
|
||||
@@ -121,6 +125,7 @@ class TestCudaVmmTransport(CustomTestCase):
|
||||
):
|
||||
torch.cuda.set_device(device)
|
||||
with get_parallel().override(
|
||||
tp_rank=tp_rank,
|
||||
attn_tp_size=2,
|
||||
attn_tp_rank=tp_rank,
|
||||
attn_cp_size=1,
|
||||
@@ -156,17 +161,64 @@ class TestCudaVmmTransport(CustomTestCase):
|
||||
def test_auto_prefers_fabric_tensor_round_trip_and_recycling(self):
|
||||
self._run_round_trip(mode="auto")
|
||||
|
||||
def test_publication_waits_for_source_stream(self):
|
||||
torch.cuda.set_device(0)
|
||||
for mode in ("single", "packed", "async", "async_fallback"):
|
||||
with self.subTest(mode=mode):
|
||||
pool = CudaVmmMemoryPool(4 << 20, 60, 0, 1, allow_posix_fallback=True)
|
||||
transport = object.__new__(CudaVmmFeatureTransport)
|
||||
transport.pool = pool
|
||||
try:
|
||||
# Make a wrong-stream copy observably read the previous value.
|
||||
size = 8 << 20 if mode == "async_fallback" else 1024
|
||||
source = torch.zeros(size, dtype=torch.uint8, device="cuda")
|
||||
torch.cuda.synchronize()
|
||||
stream = torch.cuda.Stream()
|
||||
with concurrent.futures.ThreadPoolExecutor(1) as executor:
|
||||
transport._publisher_executor = executor
|
||||
with torch.cuda.stream(stream):
|
||||
torch.cuda._sleep(200_000_000)
|
||||
source.fill_(73)
|
||||
if mode == "single":
|
||||
proxy = pool.wrap_tensor(source)
|
||||
elif mode == "packed":
|
||||
proxy = pool.wrap_tensors([source, source])[0]
|
||||
else:
|
||||
item = MultimodalDataItem(
|
||||
modality=Modality.IMAGE, feature=source
|
||||
)
|
||||
asyncio.run(
|
||||
transport.prepare_for_dispatch_async(
|
||||
[MultimodalProcessorOutput(mm_items=[item])]
|
||||
)
|
||||
)
|
||||
proxy = item.feature
|
||||
result = (
|
||||
proxy
|
||||
if isinstance(proxy, torch.Tensor)
|
||||
else proxy.reconstruct_on_target_device(0)
|
||||
)
|
||||
self.assertTrue(
|
||||
torch.equal(
|
||||
result.cpu(), torch.full((size,), 73, dtype=torch.uint8)
|
||||
)
|
||||
)
|
||||
finally:
|
||||
torch.cuda.synchronize()
|
||||
_imported_pool_cache_clear()
|
||||
pool.shutdown()
|
||||
|
||||
def test_reused_chunk_clears_acknowledgements(self):
|
||||
pool = CudaVmmMemoryPool(4 << 20, 60, 0, 2, allow_posix_fallback=True)
|
||||
try:
|
||||
old = pool.wrap_tensor(torch.ones(1024, dtype=torch.uint8, device="cuda:0"))
|
||||
# VMM must bypass the CUDA IPC zero-copy borrow protocol.
|
||||
self.assertIsNone(old.borrow_on_target_device(0))
|
||||
pool.memory_pool[old.control_offset : old.control_offset + 8].view(
|
||||
torch.int32
|
||||
).fill_(1)
|
||||
torch.cuda.synchronize(0)
|
||||
with pool._lock:
|
||||
pool._recycle_chunks()
|
||||
pool._merge_chunks()
|
||||
pool._recycle_chunks()
|
||||
|
||||
pool.wrap_tensor(torch.ones(100, dtype=torch.uint8, device="cuda:0"))
|
||||
live = pool.wrap_tensor(torch.ones(256, dtype=torch.uint8, device="cuda:0"))
|
||||
@@ -175,8 +227,7 @@ class TestCudaVmmTransport(CustomTestCase):
|
||||
].view(torch.int32)
|
||||
self.assertTrue(torch.equal(control, torch.zeros_like(control)))
|
||||
|
||||
with pool._lock:
|
||||
pool._recycle_chunks()
|
||||
pool._recycle_chunks()
|
||||
self.assertIn(
|
||||
live.control_offset,
|
||||
[chunk.start for chunk in pool.occupied_chunks],
|
||||
@@ -237,8 +288,7 @@ class TestCudaVmmTransport(CustomTestCase):
|
||||
for tensor in reconstructed
|
||||
)
|
||||
)
|
||||
with pool._lock:
|
||||
pool._recycle_chunks()
|
||||
pool._recycle_chunks()
|
||||
self.assertFalse(pool.occupied_chunks)
|
||||
finally:
|
||||
del reconstructed, proxies, expected, sources
|
||||
@@ -425,8 +475,7 @@ class TestCudaVmmTransport(CustomTestCase):
|
||||
):
|
||||
proxy.reconstruct_on_target_device(0, consumer_count=1)
|
||||
torch.cuda.synchronize(0)
|
||||
with pool._lock:
|
||||
pool._recycle_chunks()
|
||||
pool._recycle_chunks()
|
||||
self.assertFalse(pool.occupied_chunks)
|
||||
|
||||
with (
|
||||
|
||||
@@ -412,6 +412,11 @@ class TestCudaVmmFeatureTransport(unittest.TestCase):
|
||||
transport.cancel_for_dispatch.assert_not_called()
|
||||
|
||||
def test_async_publication_keeps_event_loop_responsive(self):
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
MultimodalDataItem,
|
||||
MultimodalProcessorOutput,
|
||||
)
|
||||
from sglang.srt.utils.cuda_vmm_transport_utils import (
|
||||
CudaVmmFeatureTransport,
|
||||
)
|
||||
@@ -437,7 +442,11 @@ class TestCudaVmmFeatureTransport(unittest.TestCase):
|
||||
release.set()
|
||||
|
||||
transport.prepare_for_dispatch = MagicMock(side_effect=block_publication)
|
||||
mm_inputs = SimpleNamespace(mm_items=[object()])
|
||||
mm_inputs = MultimodalProcessorOutput(
|
||||
mm_items=[
|
||||
MultimodalDataItem(modality=Modality.IMAGE, feature=torch.ones(2))
|
||||
]
|
||||
)
|
||||
|
||||
async def run():
|
||||
watchdog = threading.Thread(target=unblock_if_event_loop_stalls)
|
||||
@@ -447,6 +456,8 @@ class TestCudaVmmFeatureTransport(unittest.TestCase):
|
||||
transport.prepare_for_dispatch_async([mm_inputs])
|
||||
)
|
||||
while not started.is_set():
|
||||
if task.done():
|
||||
await task
|
||||
await asyncio.sleep(0)
|
||||
event_loop_responsive.set()
|
||||
self.assertFalse(task.done())
|
||||
@@ -519,6 +530,7 @@ class TestCudaVmmFeatureTransport(unittest.TestCase):
|
||||
from sglang.srt.utils import cuda_vmm_transport_utils as vmm
|
||||
|
||||
pool = object.__new__(vmm.CudaVmmMemoryPool)
|
||||
pool._lock = threading.Lock()
|
||||
pool.device_index = 0
|
||||
pool.consumer_count = 2
|
||||
pool._recycle_stream = object()
|
||||
@@ -1122,6 +1134,7 @@ class TestVmmConsumerCount(unittest.TestCase):
|
||||
proxy = object.__new__(CudaVmmTensorTransportProxy)
|
||||
proxy.consumer_count = 4
|
||||
with get_parallel().override(
|
||||
tp_rank=3,
|
||||
attn_tp_size=2,
|
||||
attn_tp_rank=1,
|
||||
attn_cp_size=2,
|
||||
|
||||
Reference in New Issue
Block a user