[VLM] fix: repair multimodal VMM lease cleanup and source-stream ordering (#43475)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -9,7 +9,7 @@ from sglang.srt.managers.schedule_batch import MultimodalDataItem
|
||||
from sglang.srt.mem_cache.multimodal_cache import EmbeddingResult, MultiModalStaticCache
|
||||
from sglang.srt.multimodal.evs import EVSEmbeddingResult
|
||||
from sglang.srt.multimodal.transport.cuda_ipc import BORROW_CUDA_IPC_FEATURE_KEY
|
||||
from sglang.srt.runtime_context import get_parallel, get_schedule
|
||||
from sglang.srt.runtime_context import get_schedule
|
||||
from sglang.srt.utils import is_hip, is_npu, is_xpu
|
||||
from sglang.srt.utils.async_probe import maybe_assert_sum
|
||||
from sglang.utils import logger
|
||||
@@ -243,26 +243,6 @@ def _move_items_to_device(
|
||||
item.feature = item.feature.to(device, non_blocking=True)
|
||||
|
||||
|
||||
def _acknowledge_deferred_cuda_ipc_cache_hits(
|
||||
items: List[MultimodalDataItem],
|
||||
) -> None:
|
||||
"""Release lazy Kimi IPC slices when a cached embedding skips ViT.
|
||||
|
||||
On an encoder-DP miss, exactly one rank copies an image and acknowledges
|
||||
the full TP group. On a cache hit no rank copies it, so rank zero performs
|
||||
the equivalent single acknowledgement. This preserves the fixed-pool
|
||||
lifecycle without reintroducing an unnecessary GPU-to-GPU copy.
|
||||
"""
|
||||
parallel = get_parallel()
|
||||
if parallel.attn_tp_rank != 0:
|
||||
return
|
||||
# The pool's recycler counts the whole TP group, so the acknowledgement must
|
||||
# match that count even when an attention subgroup is smaller.
|
||||
consumer_count = max(parallel.tp_size, 1)
|
||||
for item in items:
|
||||
item.acknowledge_deferred_cuda_ipc_feature(consumer_count)
|
||||
|
||||
|
||||
def _item_overlap(
|
||||
item: MultimodalDataItem, chunk_start: int, chunk_end: int
|
||||
) -> Optional[int]:
|
||||
@@ -330,8 +310,8 @@ def _get_chunked_embedding_full(
|
||||
else embedding
|
||||
)
|
||||
embedding_cache.set(embedding_items_hash, embedding_per_req)
|
||||
else:
|
||||
_acknowledge_deferred_cuda_ipc_cache_hits(embedding_items_per_req)
|
||||
|
||||
# cache hits keep raw-input leases until request cleanup for eviction/re-prefill
|
||||
|
||||
if isinstance(embedding_per_req, EVSEmbeddingResult):
|
||||
item = embedding_items_per_req[0]
|
||||
@@ -492,7 +472,6 @@ def _get_chunked_embedding_by_item(
|
||||
cached_token_count = _embedding_token_count(cached_embedding)
|
||||
if cached_token_count == expected_token_count:
|
||||
cached_embeddings[idx] = cached_embedding
|
||||
_acknowledge_deferred_cuda_ipc_cache_hits([item])
|
||||
else:
|
||||
_discard_mismatched_cached_embedding(
|
||||
item.hash, expected_token_count, cached_token_count
|
||||
|
||||
@@ -1077,6 +1077,8 @@ class Req(ReqDllmMixin):
|
||||
self.dllm_initialized: bool = False
|
||||
|
||||
self.session = session
|
||||
# media added by this turn, excluding the session's borrowed history
|
||||
self._session_mm_items: List[MultimodalDataItem] = []
|
||||
self.session_id = session_id
|
||||
# Used by the session radix cache to reject registration after a close/reopen.
|
||||
self.session_generation: Optional[int] = None
|
||||
@@ -1519,6 +1521,7 @@ class Req(ReqDllmMixin):
|
||||
|
||||
def _extend_session_image_inputs(self, image_inputs):
|
||||
"""Append media while preserving the saved session and its position history."""
|
||||
self._session_mm_items.extend(image_inputs.mm_items)
|
||||
# Padding can change token values without changing their count.
|
||||
self.full_untruncated_fill_ids = array("q")
|
||||
if self.multimodal_inputs is not None:
|
||||
@@ -1564,6 +1567,18 @@ class Req(ReqDllmMixin):
|
||||
positions.max() + 1 - positions.shape[1]
|
||||
).reshape(1, 1)
|
||||
|
||||
def release_mm_inputs_on_abort(self) -> None:
|
||||
"""Release this turn's media without invalidating session history."""
|
||||
if self.session is None:
|
||||
if self.multimodal_inputs is not None:
|
||||
self.multimodal_inputs.release_features()
|
||||
else:
|
||||
for item in self._session_mm_items:
|
||||
item.release_transport_proxies()
|
||||
item.feature = None
|
||||
self._session_mm_items.clear()
|
||||
self.multimodal_inputs = None
|
||||
|
||||
def finished(self) -> bool:
|
||||
# Whether request reached finished condition
|
||||
return self.finished_reason is not None
|
||||
@@ -2110,9 +2125,7 @@ class Req(ReqDllmMixin):
|
||||
logger.error(f"{error_msg}, {self.rid=}")
|
||||
# Session requests share historical multimodal inputs with their prior
|
||||
# request. The session owns and releases those features when it closes.
|
||||
if self.multimodal_inputs is not None and self.session is None:
|
||||
self.multimodal_inputs.release_features()
|
||||
self.multimodal_inputs = None
|
||||
self.release_mm_inputs_on_abort()
|
||||
self.grammar = None
|
||||
self.origin_input_ids = array(
|
||||
"q", [0]
|
||||
|
||||
@@ -23,7 +23,7 @@ import sys
|
||||
import time
|
||||
from array import array
|
||||
from collections import deque
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from contextlib import ExitStack, contextmanager, nullcontext
|
||||
from functools import partial
|
||||
from http import HTTPStatus
|
||||
from typing import TYPE_CHECKING, Any, Deque, Dict, List, Optional, Set, Tuple, Union
|
||||
@@ -210,6 +210,7 @@ from sglang.srt.managers.prefill_delayer import (
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
FINISH_ABORT,
|
||||
MultimodalInputs,
|
||||
MultimodalProcessorOutput,
|
||||
NextBatchPlan,
|
||||
Req,
|
||||
ScheduleBatch,
|
||||
@@ -2235,6 +2236,12 @@ class Scheduler(
|
||||
)
|
||||
error_msg = f"Multimodal feature reconstruction failed ({details})"
|
||||
logger.error(error_msg)
|
||||
if isinstance(
|
||||
tokenized_req.mm_inputs,
|
||||
(MultimodalInputs, MultimodalProcessorOutput),
|
||||
):
|
||||
for item in tokenized_req.mm_inputs.mm_items:
|
||||
item.release_transport_proxies()
|
||||
tokenized_req.mm_inputs = None
|
||||
request_errors.append(error_msg)
|
||||
else:
|
||||
@@ -2779,6 +2786,8 @@ class Scheduler(
|
||||
continue
|
||||
# For session requests, keep mm_inputs for the next request
|
||||
if req.session:
|
||||
if isinstance(req.finished_reason, FINISH_ABORT):
|
||||
req.release_mm_inputs_on_abort()
|
||||
continue
|
||||
# For non-session requests, clear features and mm_inputs
|
||||
mm_inputs.release_features()
|
||||
@@ -2789,6 +2798,23 @@ class Scheduler(
|
||||
recv_req: TokenizedGenerateReqInput,
|
||||
*,
|
||||
mm_input_error: Optional[str] = None,
|
||||
):
|
||||
with ExitStack() as mm_cleanup:
|
||||
if isinstance(
|
||||
recv_req.mm_inputs, (MultimodalInputs, MultimodalProcessorOutput)
|
||||
):
|
||||
for item in recv_req.mm_inputs.mm_items:
|
||||
mm_cleanup.callback(item.release_transport_proxies)
|
||||
self._handle_generate_request(
|
||||
recv_req, mm_cleanup, mm_input_error=mm_input_error
|
||||
)
|
||||
|
||||
def _handle_generate_request(
|
||||
self,
|
||||
recv_req: TokenizedGenerateReqInput,
|
||||
mm_cleanup: ExitStack,
|
||||
*,
|
||||
mm_input_error: Optional[str] = None,
|
||||
):
|
||||
# Route: normal request / session request / session-not-found
|
||||
session_id = (
|
||||
@@ -3047,6 +3073,8 @@ class Scheduler(
|
||||
req.origin_input_ids, image_inputs, self.pad_input_ids_func
|
||||
)
|
||||
req.extend_image_inputs(image_inputs)
|
||||
# the request now owns cleanup, including rejection after padding
|
||||
mm_cleanup.pop_all()
|
||||
self._maybe_compute_mrope_positions(req)
|
||||
|
||||
if len(req.origin_input_ids) >= self.max_req_input_len:
|
||||
@@ -3279,6 +3307,7 @@ class Scheduler(
|
||||
# detach lives in `StreamingSession.find_active_slot`, which only runs
|
||||
# while scheduling; a session left in-flight rejects every later request.
|
||||
if req.session is not None and req.session.streaming:
|
||||
req.release_mm_inputs_on_abort()
|
||||
req.session.abort_req()
|
||||
req.session = None
|
||||
# `beam_coordinator.validate_and_init` counts the group in ahead of the
|
||||
@@ -3364,7 +3393,10 @@ class Scheduler(
|
||||
|
||||
def _release_aborted_request(self, req: Req) -> None:
|
||||
"""Drop the cache-side state an aborted request left behind."""
|
||||
req.release_mm_inputs_on_abort()
|
||||
self.tree_cache.finish(req.cache_request_handle, CacheRequestOutcome.ABORT)
|
||||
if req.session is not None and req.session.streaming:
|
||||
req.session.abort_req()
|
||||
|
||||
def _abort_on_queued_limit(self, recv_req: Req) -> bool:
|
||||
"""Abort an incoming or existing request if the waiting queue is full. Returns True if the incoming request is aborted."""
|
||||
@@ -3472,6 +3504,23 @@ class Scheduler(
|
||||
recv_req: TokenizedEmbeddingReqInput,
|
||||
*,
|
||||
mm_input_error: Optional[str] = None,
|
||||
):
|
||||
with ExitStack() as mm_cleanup:
|
||||
if isinstance(
|
||||
recv_req.mm_inputs, (MultimodalInputs, MultimodalProcessorOutput)
|
||||
):
|
||||
for item in recv_req.mm_inputs.mm_items:
|
||||
mm_cleanup.callback(item.release_transport_proxies)
|
||||
self._handle_embedding_request(
|
||||
recv_req, mm_cleanup, mm_input_error=mm_input_error
|
||||
)
|
||||
|
||||
def _handle_embedding_request(
|
||||
self,
|
||||
recv_req: TokenizedEmbeddingReqInput,
|
||||
mm_cleanup: ExitStack,
|
||||
*,
|
||||
mm_input_error: Optional[str] = None,
|
||||
):
|
||||
req = Req(
|
||||
recv_req.rid,
|
||||
@@ -3528,6 +3577,7 @@ class Scheduler(
|
||||
)
|
||||
|
||||
req.extend_image_inputs(image_inputs)
|
||||
mm_cleanup.pop_all()
|
||||
self._maybe_compute_mrope_positions(req)
|
||||
|
||||
if len(req.origin_input_ids) >= self.max_req_input_len:
|
||||
|
||||
@@ -4597,7 +4597,7 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
spans = []
|
||||
device, dtype = self.image_start.device, self.image_start.dtype
|
||||
for item in items:
|
||||
item.reconstruct(device.index, ipc_consumer_count=self.tp_size)
|
||||
item.reconstruct(device.index)
|
||||
h, w = int(item.n_vit_h), int(item.n_vit_w)
|
||||
pixels = torch.as_tensor(item.feature, device=device)
|
||||
plan = item.model_specific_data.get(GPU_PLAN_KEY)
|
||||
|
||||
@@ -49,7 +49,6 @@ from sglang.srt.multimodal.mm_utils import (
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_mm,
|
||||
get_parallel,
|
||||
)
|
||||
from sglang.srt.utils import add_prefix, is_cuda, is_npu
|
||||
|
||||
@@ -724,14 +723,9 @@ class KimiK25ForConditionalGeneration(nn.Module):
|
||||
|
||||
CUDA IPC features are intentionally reconstructed after the image
|
||||
assignment. Each image therefore crosses the tokenizer/scheduler
|
||||
boundary once instead of once per TP rank. The selected consumer
|
||||
acknowledges the entire TP group so the bounded IPC pool remains
|
||||
recyclable.
|
||||
boundary once instead of once per TP rank. Unused ranks keep their
|
||||
lease until request cleanup so cache eviction can re-run the encoder.
|
||||
"""
|
||||
# Match the configured TP consumer count captured when the
|
||||
# tokenizer creates MmItemMemoryPool. A live attention subgroup
|
||||
# size could leave acknowledgements missing and strand the lease.
|
||||
ipc_consumer_count = max(get_parallel().tp_size, 1)
|
||||
device_index = device.index
|
||||
if device.type == "cuda" and device_index is None:
|
||||
device_index = torch.cuda.current_device()
|
||||
@@ -740,9 +734,7 @@ class KimiK25ForConditionalGeneration(nn.Module):
|
||||
for image_index in image_indices:
|
||||
item = items[image_index]
|
||||
if device.type == "cuda":
|
||||
item.reconstruct(
|
||||
device_index, ipc_consumer_count=ipc_consumer_count
|
||||
)
|
||||
item.reconstruct(device_index)
|
||||
feature = item.feature
|
||||
if not isinstance(feature, torch.Tensor):
|
||||
raise TypeError(
|
||||
|
||||
@@ -3613,10 +3613,6 @@ class KimiK3ForConditionalGeneration(nn.Module):
|
||||
LOCAL_PREPROCESSED_KEY,
|
||||
)
|
||||
|
||||
# Match the configured TP consumer count captured when the
|
||||
# tokenizer creates MmItemMemoryPool. A live attention subgroup
|
||||
# size could leave acknowledgements missing and strand the lease.
|
||||
ipc_consumer_count = max(get_parallel().tp_size, 1)
|
||||
device_index = device.index
|
||||
if device.type == "cuda" and device_index is None:
|
||||
device_index = torch.cuda.current_device()
|
||||
@@ -3625,9 +3621,7 @@ class KimiK3ForConditionalGeneration(nn.Module):
|
||||
for image_index in image_indices:
|
||||
item = items[image_index]
|
||||
if device.type == "cuda":
|
||||
item.reconstruct(
|
||||
device_index, ipc_consumer_count=ipc_consumer_count
|
||||
)
|
||||
item.reconstruct(device_index)
|
||||
selected_items.append(item)
|
||||
|
||||
locally_preprocessed = [
|
||||
|
||||
@@ -1521,7 +1521,7 @@ class Qwen3VLForConditionalGeneration(nn.Module):
|
||||
(item, proxy, feature_offset, borrowed.shape[0])
|
||||
)
|
||||
elif RETAINED_CUDA_IPC_FEATURE_PROXY_KEY not in model_specific_data:
|
||||
item.reconstruct(device_index, ipc_consumer_count=consumer_count)
|
||||
item.reconstruct(device_index)
|
||||
features.append(item.feature)
|
||||
feature_offset += item.feature.shape[0]
|
||||
try:
|
||||
|
||||
@@ -44,6 +44,7 @@ from sglang.srt.multimodal.transport.cuda_ipc import (
|
||||
MmItemMemoryPool,
|
||||
get_mm_feature_pool_size_per_worker,
|
||||
)
|
||||
from sglang.srt.multimodal.transport.memory_pool import get_mm_feature_consumer_count
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_mm,
|
||||
@@ -450,7 +451,7 @@ class BaseMultimodalProcessor(ABC):
|
||||
per_worker_pool_size,
|
||||
MM_ITEM_MEMORY_POOL_RECYCLE_INTERVAL,
|
||||
self.server_args.base_gpu_id,
|
||||
self.server_args.tp_size,
|
||||
get_mm_feature_consumer_count(),
|
||||
)
|
||||
|
||||
@property
|
||||
|
||||
@@ -16,6 +16,14 @@ DATA_ALIGNMENT = 256
|
||||
DEFAULT_MAX_INFLIGHT_SLICES = 4096
|
||||
|
||||
|
||||
def get_mm_feature_consumer_count() -> int:
|
||||
"""Number of scheduler ranks receiving one routed multimodal request."""
|
||||
parallel = get_parallel()
|
||||
if parallel.attn_dp_enabled:
|
||||
return parallel.tp_size // parallel.num_dp_ranks
|
||||
return parallel.tp_size
|
||||
|
||||
|
||||
def align_up(value: int, alignment: int) -> int:
|
||||
return ((value + alignment - 1) // alignment) * alignment
|
||||
|
||||
@@ -59,9 +67,8 @@ def resolve_consumer_rank(
|
||||
return 0
|
||||
if consumer_rank is None:
|
||||
try:
|
||||
# Use the global TP rank. An attention/DCP subgroup rank can alias
|
||||
# another consumer's acknowledgement slot.
|
||||
rank = int(get_parallel().tp_rank)
|
||||
# DP groups are contiguous; attention/DCP subgroup ranks can alias
|
||||
rank = int(get_parallel().tp_rank) % total_consumer_count
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
f"Cannot resolve the {transport_name} consumer rank before "
|
||||
|
||||
@@ -80,6 +80,7 @@ class StreamingSession:
|
||||
if slot is None or not slot.kv.holds_kv:
|
||||
return None
|
||||
if req.to_finish is not None:
|
||||
req.release_mm_inputs_on_abort()
|
||||
req.session.abort_req()
|
||||
req.session = None
|
||||
return None
|
||||
@@ -160,6 +161,7 @@ class StreamingSession:
|
||||
# Hand the record and the tree lock back; the caller releases them.
|
||||
del self.slots[req.session.session_id]
|
||||
req.lock, slot.lock = slot.lock, None
|
||||
req.release_mm_inputs_on_abort()
|
||||
req.session.abort_req()
|
||||
return False
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ from sglang.srt.managers.schedule_batch import (
|
||||
MultimodalDataItem,
|
||||
MultimodalProcessorOutput,
|
||||
)
|
||||
from sglang.srt.multimodal.transport.memory_pool import get_mm_feature_consumer_count
|
||||
from sglang.srt.runtime_context import (
|
||||
get_mm,
|
||||
get_parallel,
|
||||
@@ -192,9 +193,7 @@ def _contains_tensor_container(value) -> bool:
|
||||
|
||||
|
||||
def get_vmm_feature_consumer_count() -> int:
|
||||
if get_parallel().attn_dp_enabled:
|
||||
return get_parallel().tp_size // get_parallel().num_dp_ranks
|
||||
return get_parallel().tp_size
|
||||
return get_mm_feature_consumer_count()
|
||||
|
||||
|
||||
class CudaVmmMemoryPool:
|
||||
@@ -402,6 +401,10 @@ class CudaVmmMemoryPool:
|
||||
producer_stream = None
|
||||
copy_synchronized = False
|
||||
try:
|
||||
if tensor.is_cuda:
|
||||
self._publish_stream.wait_stream(
|
||||
torch.cuda.current_stream(tensor.device)
|
||||
)
|
||||
with (
|
||||
torch.cuda.device(self.device_index),
|
||||
torch.cuda.stream(self._publish_stream),
|
||||
@@ -477,6 +480,8 @@ class CudaVmmMemoryPool:
|
||||
producer_stream = None
|
||||
copy_synchronized = False
|
||||
try:
|
||||
for device in {tensor.device for tensor in tensors if tensor.is_cuda}:
|
||||
self._publish_stream.wait_stream(torch.cuda.current_stream(device))
|
||||
with (
|
||||
torch.cuda.device(self.device_index),
|
||||
torch.cuda.stream(self._publish_stream),
|
||||
@@ -604,20 +609,19 @@ class CudaVmmMemoryPool:
|
||||
def _recycle_loop(self) -> None:
|
||||
while not self._stop_recycler.wait(self._recycle_interval):
|
||||
try:
|
||||
with self._lock:
|
||||
self._recycle_chunks()
|
||||
self._merge_chunks()
|
||||
self._recycle_chunks()
|
||||
except Exception as error:
|
||||
logger.exception("CUDA VMM multimodal pool recycle failed")
|
||||
self._pool_error = error
|
||||
self._stop_recycler.set()
|
||||
|
||||
def _recycle_chunks(self) -> None:
|
||||
if not self.occupied_chunks:
|
||||
with self._lock:
|
||||
chunks = tuple(self.occupied_chunks)
|
||||
if not chunks:
|
||||
return
|
||||
|
||||
remaining = []
|
||||
recycled = []
|
||||
# CUDA polling must not hold the allocator lock needed by publishers
|
||||
with (
|
||||
torch.cuda.device(self.device_index),
|
||||
torch.cuda.stream(self._recycle_stream),
|
||||
@@ -628,23 +632,23 @@ class CudaVmmMemoryPool:
|
||||
chunk.start : chunk.start
|
||||
+ self.consumer_count * _CONTROL_WORD_BYTES
|
||||
].view(torch.int32)
|
||||
for chunk in self.occupied_chunks
|
||||
for chunk in chunks
|
||||
]
|
||||
)
|
||||
acknowledgement_counts = (
|
||||
torch.count_nonzero(acknowledgement_words, dim=1).cpu().tolist()
|
||||
)
|
||||
|
||||
for chunk, acknowledgement_count in zip(
|
||||
self.occupied_chunks, acknowledgement_counts, strict=True
|
||||
):
|
||||
if acknowledgement_count == self.consumer_count:
|
||||
recycled.append(_CudaVmmMemoryChunk(chunk.start, chunk.end))
|
||||
else:
|
||||
remaining.append(chunk)
|
||||
|
||||
self.available_chunks.extend(recycled)
|
||||
self.occupied_chunks = remaining
|
||||
with self._lock:
|
||||
live = {chunk.start: chunk for chunk in self.occupied_chunks}
|
||||
for chunk, count in zip(chunks, acknowledgement_counts, strict=True):
|
||||
# cancellation may have freed and republished the same offset
|
||||
if count == self.consumer_count and live.get(chunk.start) is chunk:
|
||||
self.occupied_chunks.remove(chunk)
|
||||
self.available_chunks.append(
|
||||
_CudaVmmMemoryChunk(chunk.start, chunk.end)
|
||||
)
|
||||
self._merge_chunks()
|
||||
|
||||
def _merge_chunks(self) -> None:
|
||||
merged = []
|
||||
@@ -809,6 +813,10 @@ class CudaVmmTensorTransportProxy(CudaIpcTensorTransportProxy):
|
||||
device_index=device_index,
|
||||
)
|
||||
|
||||
def borrow_on_target_device(self, rebuild_device_idx: int) -> None:
|
||||
# VMM copies into consumer-owned storage; the IPC borrow protocol differs
|
||||
return None
|
||||
|
||||
def _acknowledgement_range(self, consumer_count: int) -> tuple[int, int]:
|
||||
if consumer_count <= 0:
|
||||
raise ValueError("consumer_count must be positive")
|
||||
@@ -816,6 +824,10 @@ class CudaVmmTensorTransportProxy(CudaIpcTensorTransportProxy):
|
||||
return 0, self.consumer_count
|
||||
|
||||
parallel = get_parallel()
|
||||
if consumer_count == 1:
|
||||
# attention/DCP subgroup ranks can alias; DP groups are contiguous
|
||||
slot = parallel.tp_rank % self.consumer_count
|
||||
return slot, slot + 1
|
||||
group_start = parallel.attn_cp_rank * parallel.attn_tp_size
|
||||
group_end = group_start + parallel.attn_tp_size
|
||||
if not 0 <= group_start < group_end <= self.consumer_count:
|
||||
@@ -824,9 +836,6 @@ class CudaVmmTensorTransportProxy(CudaIpcTensorTransportProxy):
|
||||
f"[{group_start}, {group_end}) is outside "
|
||||
f"consumer_count={self.consumer_count}"
|
||||
)
|
||||
if consumer_count == 1:
|
||||
slot = group_start + parallel.attn_tp_rank
|
||||
return slot, slot + 1
|
||||
if consumer_count == parallel.attn_tp_size:
|
||||
return group_start, group_end
|
||||
raise ValueError(
|
||||
@@ -1033,18 +1042,49 @@ class CudaVmmFeatureTransport:
|
||||
if self._publisher_executor is None:
|
||||
raise RuntimeError("CUDA VMM feature transport is shutting down")
|
||||
|
||||
future = asyncio.get_running_loop().run_in_executor(
|
||||
self._publisher_executor,
|
||||
self.prepare_for_dispatch,
|
||||
mm_inputs_batch,
|
||||
# CUDA current streams are thread-local; carry readiness into the publisher.
|
||||
ready_events = []
|
||||
devices = {
|
||||
tensor.device
|
||||
for mm_inputs in mm_inputs_batch
|
||||
if mm_inputs is not None
|
||||
for item in mm_inputs.mm_items
|
||||
for tensor in (item.feature, item.precomputed_embeddings)
|
||||
if isinstance(tensor, torch.Tensor) and tensor.is_cuda
|
||||
}
|
||||
for device in devices:
|
||||
event = torch.cuda.Event()
|
||||
event.record(torch.cuda.current_stream(device))
|
||||
ready_events.append((device, event))
|
||||
future = self._publisher_executor.submit(
|
||||
self._prepare_after_source_events, mm_inputs_batch, ready_events
|
||||
)
|
||||
try:
|
||||
return await asyncio.shield(future)
|
||||
return await asyncio.shield(asyncio.wrap_future(future))
|
||||
except asyncio.CancelledError:
|
||||
prepared_mm_items = await future
|
||||
self.cancel_for_dispatch(prepared_mm_items)
|
||||
# cleanup must survive a second task cancellation or loop shutdown
|
||||
future.add_done_callback(self._cancel_completed_publish)
|
||||
raise
|
||||
|
||||
def _prepare_after_source_events(
|
||||
self,
|
||||
mm_inputs_batch: Iterable[MultimodalProcessorOutput | None],
|
||||
ready_events: list[tuple[torch.device, torch.cuda.Event]],
|
||||
) -> list[MultimodalDataItem]:
|
||||
for device, event in ready_events:
|
||||
# Both VMM copies and a pool-full CPU fallback must see ready input.
|
||||
torch.cuda.current_stream(device).wait_event(event)
|
||||
return self.prepare_for_dispatch(mm_inputs_batch)
|
||||
|
||||
def _cancel_completed_publish(self, future: concurrent.futures.Future) -> None:
|
||||
if future.cancelled() or future.exception() is not None:
|
||||
# failed publication already rolls back its slices
|
||||
return
|
||||
try:
|
||||
self.cancel_for_dispatch(future.result())
|
||||
except Exception:
|
||||
logger.exception("Failed to cancel abandoned CUDA VMM publication")
|
||||
|
||||
def prepare_for_dispatch(
|
||||
self,
|
||||
mm_inputs_batch: Iterable[MultimodalProcessorOutput | None],
|
||||
|
||||
Reference in New Issue
Block a user