[Score] Send stacked GPU embedding overrides through a CUDA IPC pool (#39730)
Co-authored-by: Sundara Raman Ramachandran <sundar24295@gmail.com> Co-authored-by: hnyls2002 <lsyincs@gmail.com>
This commit is contained in:
co-authored by
Sundara Raman Ramachandran
hnyls2002
parent
08550acc28
commit
baa632b049
@@ -40,6 +40,7 @@ class EngineScoreMixin:
|
||||
score_extraction_token_id: Optional[int] = None,
|
||||
temperature: float = 1.0,
|
||||
return_token_logprobs: bool = False,
|
||||
stacked_query_item_embed_overrides: Optional[List[torch.Tensor]] = None,
|
||||
) -> ScoreResult:
|
||||
"""
|
||||
Score items against a query using the loaded model.
|
||||
@@ -84,6 +85,9 @@ class EngineScoreMixin:
|
||||
softmax. Requires apply_softmax=True when different from 1.
|
||||
return_token_logprobs: Include uncalibrated full-vocabulary logprobs for
|
||||
each item's candidates. Only supported for CausalLM, not raw logits.
|
||||
stacked_query_item_embed_overrides: Alternative to query_/item_embed_overrides:
|
||||
one already-stacked [N, hidden] tensor per output sequence, used without
|
||||
a restack. See score_request.
|
||||
|
||||
Returns:
|
||||
ScoreResult with scores (flat ``[num_items x num_labels]`` for pointwise
|
||||
@@ -102,6 +106,7 @@ class EngineScoreMixin:
|
||||
query_embed_overrides=query_embed_overrides,
|
||||
item_embed_overrides=item_embed_overrides,
|
||||
score_extraction_token_id=score_extraction_token_id,
|
||||
stacked_query_item_embed_overrides=stacked_query_item_embed_overrides,
|
||||
request=None,
|
||||
return_pooled_hidden_states=return_pooled_hidden_states,
|
||||
temperature=temperature,
|
||||
@@ -123,6 +128,7 @@ class EngineScoreMixin:
|
||||
score_extraction_token_id: Optional[int] = None,
|
||||
temperature: float = 1.0,
|
||||
return_token_logprobs: bool = False,
|
||||
stacked_query_item_embed_overrides: Optional[List[torch.Tensor]] = None,
|
||||
) -> ScoreResult:
|
||||
"""Asynchronous version of score(). See score() for full documentation."""
|
||||
return await self.tokenizer_manager.score_request(
|
||||
@@ -135,6 +141,7 @@ class EngineScoreMixin:
|
||||
query_embed_overrides=query_embed_overrides,
|
||||
item_embed_overrides=item_embed_overrides,
|
||||
score_extraction_token_id=score_extraction_token_id,
|
||||
stacked_query_item_embed_overrides=stacked_query_item_embed_overrides,
|
||||
request=None,
|
||||
return_pooled_hidden_states=return_pooled_hidden_states,
|
||||
temperature=temperature,
|
||||
|
||||
@@ -23,6 +23,8 @@ from typing import List
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.utils.cuda_ipc_transport_utils import CudaIpcTensorTransportProxy
|
||||
|
||||
|
||||
class PositionalEmbeds(msgspec.Struct, array_like=True):
|
||||
"""Embeddings to place at specific token positions.
|
||||
@@ -36,7 +38,8 @@ class PositionalEmbeds(msgspec.Struct, array_like=True):
|
||||
positions: List of positions where embeddings should be injected.
|
||||
"""
|
||||
|
||||
embeds: torch.Tensor
|
||||
# torch.Tensor or CudaIpcTensorTransportProxy; msgspec cannot type that union.
|
||||
embeds: object
|
||||
positions: List[int]
|
||||
|
||||
def __post_init__(self):
|
||||
@@ -51,8 +54,15 @@ class PositionalEmbeds(msgspec.Struct, array_like=True):
|
||||
else:
|
||||
# [1, hidden_dim] (already has the leading dim) → plain concat.
|
||||
self.embeds = torch.cat(self.embeds, dim=0)
|
||||
if self.embeds.shape[0] != len(self.positions):
|
||||
if isinstance(self.embeds, torch.Tensor) and self.embeds.shape[0] != len(
|
||||
self.positions
|
||||
):
|
||||
raise ValueError(
|
||||
f"embeds length ({self.embeds.shape[0]}) != "
|
||||
f"positions length ({len(self.positions)})"
|
||||
)
|
||||
|
||||
def reconstruct(self, target_device: int) -> None:
|
||||
"""Copy embeds out of the CUDA IPC pool onto target_device and release the slot."""
|
||||
if isinstance(self.embeds, CudaIpcTensorTransportProxy):
|
||||
self.embeds = self.embeds.reconstruct_on_target_device(target_device)
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Send CUDA override embeds to the scheduler through a CUDA IPC pool, not by value."""
|
||||
|
||||
import logging
|
||||
from typing import Iterable, List, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.managers.embed_types import PositionalEmbeds
|
||||
from sglang.srt.multimodal.transport.cuda_ipc import (
|
||||
MM_FEATURE_CACHE_SIZE,
|
||||
MM_ITEM_MEMORY_POOL_RECYCLE_INTERVAL,
|
||||
CudaIpcTensorTransportProxy,
|
||||
MmItemMemoryPool,
|
||||
get_mm_feature_pool_size_per_worker,
|
||||
)
|
||||
from sglang.srt.multimodal.transport.memory_pool import get_mm_feature_consumer_count
|
||||
from sglang.srt.runtime_context import get_parallel, get_serving
|
||||
from sglang.srt.utils import is_cuda
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PositionalEmbedsTransport:
|
||||
def __init__(self, base_gpu_id: int, mm_processor) -> None:
|
||||
parallel = get_parallel()
|
||||
# By value otherwise: IPC needs one node, every pipeline stage would ack the
|
||||
# slot, and the msgpack IPC encoder cannot encode the proxy.
|
||||
self.enabled = (
|
||||
is_cuda()
|
||||
and parallel.nnodes == 1
|
||||
and parallel.pp_size == 1
|
||||
and envs.SGLANG_USE_PICKLE_IPC.get()
|
||||
)
|
||||
self._base_gpu_id = base_gpu_id
|
||||
# A multimodal model with --mm-feature-transport=cuda_ipc already has a pool.
|
||||
self._pool: Optional[MmItemMemoryPool] = getattr(
|
||||
mm_processor, "cudaipc_mmfeature_pool", None
|
||||
)
|
||||
self._use_pool_handle_cache = envs.SGLANG_USE_IPC_POOL_HANDLE_CACHE.get()
|
||||
|
||||
def _get_pool(self) -> MmItemMemoryPool:
|
||||
if self._pool is None:
|
||||
pool_size = get_mm_feature_pool_size_per_worker(
|
||||
MM_FEATURE_CACHE_SIZE, get_serving().tokenizer_worker_num
|
||||
)
|
||||
logger.info(
|
||||
"Creating a %.0f MiB CUDA IPC pool on GPU %d for GPU-resident "
|
||||
"embedding overrides (SGLANG_MM_FEATURE_CACHE_MB).",
|
||||
pool_size / (1024 * 1024),
|
||||
self._base_gpu_id,
|
||||
)
|
||||
self._pool = MmItemMemoryPool(
|
||||
pool_size,
|
||||
MM_ITEM_MEMORY_POOL_RECYCLE_INTERVAL,
|
||||
self._base_gpu_id,
|
||||
get_mm_feature_consumer_count(),
|
||||
)
|
||||
return self._pool
|
||||
|
||||
def prepare_for_dispatch(
|
||||
self, tokenized_objs: Iterable
|
||||
) -> List[CudaIpcTensorTransportProxy]:
|
||||
"""Returns the proxies to cancel if dispatch fails; a full pool means by value."""
|
||||
if not self.enabled:
|
||||
return []
|
||||
proxies = []
|
||||
try:
|
||||
for tokenized_obj in tokenized_objs:
|
||||
overrides = tokenized_obj.positional_embed_overrides
|
||||
if overrides is None or not (
|
||||
isinstance(overrides.embeds, torch.Tensor)
|
||||
and overrides.embeds.is_cuda
|
||||
and overrides.embeds.numel() > 0
|
||||
):
|
||||
continue
|
||||
proxy = self._get_pool().wrap_tensor(
|
||||
overrides.embeds, use_pool_handle_cache=self._use_pool_handle_cache
|
||||
)
|
||||
if proxy is None:
|
||||
continue
|
||||
proxies.append(proxy)
|
||||
# A new object: requests may share the caller's PositionalEmbeds.
|
||||
tokenized_obj.positional_embed_overrides = PositionalEmbeds(
|
||||
embeds=proxy, positions=overrides.positions
|
||||
)
|
||||
except BaseException:
|
||||
self.cancel_for_dispatch(proxies)
|
||||
raise
|
||||
return proxies
|
||||
|
||||
def cancel_for_dispatch(self, proxies: List[CudaIpcTensorTransportProxy]) -> None:
|
||||
for proxy in proxies:
|
||||
self._pool.cancel_proxy(proxy)
|
||||
@@ -1367,7 +1367,11 @@ class Req(ReqDllmMixin):
|
||||
# retracted request is rebootstrapped. Set in pause_generation(retract)
|
||||
# and consumed in the decode transfer commit; never plumbed to prefill.
|
||||
self.pd_rebootstrap_forced_output_id: Optional[int] = None
|
||||
self.skip_radix_cache_insert = bootstrap_host == FAKE_BOOTSTRAP_HOST
|
||||
# Override embeds give KV that the token ids alone do not determine.
|
||||
self.skip_radix_cache_insert = (
|
||||
bootstrap_host == FAKE_BOOTSTRAP_HOST
|
||||
or positional_embed_overrides is not None
|
||||
)
|
||||
# Set when the request's in-flight forward ran on KV that never landed:
|
||||
# that forward's output is dropped and the request finishes with it.
|
||||
self.discard_output_reason: Optional[FINISH_ABORT] = None
|
||||
|
||||
@@ -2140,6 +2140,13 @@ class Scheduler(
|
||||
self.session_controller.maybe_reap(now)
|
||||
|
||||
for recv_req in recv_reqs:
|
||||
# Before any early exit, so every request releases its CUDA IPC pool slot.
|
||||
for tokenized_req in self._tokenized_requests(recv_req):
|
||||
if tokenized_req.positional_embed_overrides is not None:
|
||||
tokenized_req.positional_embed_overrides.reconstruct(
|
||||
get_device().gpu_id
|
||||
)
|
||||
|
||||
vmm_errors = None
|
||||
if get_mm().mm_feature_transport == "cuda_vmm":
|
||||
vmm_errors = self._materialize_cuda_vmm_inputs(recv_req)
|
||||
|
||||
@@ -110,6 +110,7 @@ from sglang.srt.managers.io_struct import (
|
||||
from sglang.srt.managers.load_snapshot import create_load_snapshot_reader
|
||||
from sglang.srt.managers.mm_utils import wrap_shm_features
|
||||
from sglang.srt.managers.multimodal_processor import get_mm_processor, import_processors
|
||||
from sglang.srt.managers.positional_embeds_transport import PositionalEmbedsTransport
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
MultimodalDataItem,
|
||||
get_request_return_hidden_states_mode,
|
||||
@@ -559,6 +560,9 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
# Init request dispatcher
|
||||
self.init_request_dispatcher()
|
||||
|
||||
# Init the CUDA IPC transport for GPU-resident embedding overrides
|
||||
self.init_positional_embeds_transport()
|
||||
|
||||
# Construct this last so later initialization failures cannot orphan
|
||||
# the transport's recycler thread.
|
||||
self.cuda_vmm_feature_transport = CudaVmmFeatureTransport(
|
||||
@@ -857,6 +861,11 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
if self.enable_metrics:
|
||||
self.metrics_collector.emit_startup_time(startup_time)
|
||||
|
||||
def init_positional_embeds_transport(self):
|
||||
self.positional_embeds_transport = PositionalEmbedsTransport(
|
||||
self.server_args.base_gpu_id, self.mm_processor
|
||||
)
|
||||
|
||||
def init_request_dispatcher(self):
|
||||
self._result_dispatcher = TypeBasedDispatcher(
|
||||
[
|
||||
@@ -1758,6 +1767,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
tokenized_obj: Union[TokenizedGenerateReqInput, TokenizedEmbeddingReqInput],
|
||||
):
|
||||
prepared_mm_items = []
|
||||
prepared_embeds = []
|
||||
dispatched = False
|
||||
try:
|
||||
prepared_mm_items = (
|
||||
@@ -1765,6 +1775,9 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
(tokenized_obj.mm_inputs,)
|
||||
)
|
||||
)
|
||||
prepared_embeds = self.positional_embeds_transport.prepare_for_dispatch(
|
||||
(tokenized_obj,)
|
||||
)
|
||||
tokenized_obj.time_stats.set_api_server_dispatch_time()
|
||||
tokenized_obj = wrap_shm_features(tokenized_obj)
|
||||
time_stats = tokenized_obj.time_stats
|
||||
@@ -1780,6 +1793,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
finally:
|
||||
if not dispatched:
|
||||
self.cuda_vmm_feature_transport.cancel_for_dispatch(prepared_mm_items)
|
||||
self.positional_embeds_transport.cancel_for_dispatch(prepared_embeds)
|
||||
|
||||
def _mark_state_dispatched(self, rid: str):
|
||||
"""Record that *rid* reached the scheduler.
|
||||
@@ -1799,6 +1813,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
):
|
||||
"""Send a batch of tokenized requests as a single batched request to the scheduler."""
|
||||
prepared_mm_items = []
|
||||
prepared_embeds = []
|
||||
dispatched = False
|
||||
try:
|
||||
prepared_mm_items = (
|
||||
@@ -1806,6 +1821,9 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
tokenized_obj.mm_inputs for tokenized_obj in tokenized_objs
|
||||
)
|
||||
)
|
||||
prepared_embeds = self.positional_embeds_transport.prepare_for_dispatch(
|
||||
tokenized_objs
|
||||
)
|
||||
|
||||
set_time_batch(tokenized_objs, "set_api_server_dispatch_time")
|
||||
time_stats = [tokenized_obj.time_stats for tokenized_obj in tokenized_objs]
|
||||
@@ -1827,6 +1845,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
finally:
|
||||
if not dispatched:
|
||||
self.cuda_vmm_feature_transport.cancel_for_dispatch(prepared_mm_items)
|
||||
self.positional_embeds_transport.cancel_for_dispatch(prepared_embeds)
|
||||
|
||||
def _coalesce_streaming_chunks(
|
||||
self,
|
||||
|
||||
@@ -479,6 +479,18 @@ class TokenizerManagerScoreMixin:
|
||||
)
|
||||
return embeds, positions
|
||||
|
||||
def _placeholder_positions(
|
||||
self,
|
||||
token_ids: List[int],
|
||||
embed_override_token_id: int,
|
||||
position_offset: int = 0,
|
||||
) -> List[int]:
|
||||
return [
|
||||
idx + position_offset
|
||||
for idx, tok in enumerate(token_ids)
|
||||
if tok == embed_override_token_id
|
||||
]
|
||||
|
||||
def _resolve_embed_overrides_for_request(
|
||||
self,
|
||||
query: List[int],
|
||||
@@ -523,12 +535,24 @@ class TokenizerManagerScoreMixin:
|
||||
embed_override_token_id: Optional[int],
|
||||
query_embed_overrides: Optional[List[torch.Tensor]],
|
||||
item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]],
|
||||
stacked_query_item_embed_overrides: Optional[
|
||||
List[Optional[torch.Tensor]]
|
||||
] = None,
|
||||
) -> Tuple[None, List[List[int]], Optional[list], Optional[List[int]]]:
|
||||
"""Build input_ids and resolve embed overrides for token-ID inputs.
|
||||
|
||||
Multi-item-scoring and single-item modes differ only in how input_ids
|
||||
are assembled and what position offset each item gets.
|
||||
"""
|
||||
if stacked_query_item_embed_overrides is not None:
|
||||
return self._build_token_id_inputs_stacked(
|
||||
query,
|
||||
items,
|
||||
use_multi_item_scoring,
|
||||
embed_override_token_id,
|
||||
stacked_query_item_embed_overrides,
|
||||
)
|
||||
|
||||
# Both query and items are token IDs
|
||||
has_embeds = (
|
||||
query_embed_overrides is not None or item_embed_overrides is not None
|
||||
@@ -623,6 +647,63 @@ class TokenizerManagerScoreMixin:
|
||||
None,
|
||||
)
|
||||
|
||||
def _build_token_id_inputs_stacked(
|
||||
self,
|
||||
query: List[int],
|
||||
items: List[List[int]],
|
||||
use_multi_item_scoring: bool,
|
||||
embed_override_token_id: int,
|
||||
stacked_query_item_embed_overrides: List[Optional[torch.Tensor]],
|
||||
) -> Tuple[None, List[List[int]], Optional[list], Optional[List[int]]]:
|
||||
"""On the single-item path each item's tensor holds the query rows, then its own."""
|
||||
if use_multi_item_scoring:
|
||||
delimiter_token_id = MIS_DELIMITER_TOKEN_ID
|
||||
combined_input_ids, delimiter_indices = (
|
||||
self._build_multi_item_token_sequence(query, items, delimiter_token_id)
|
||||
)
|
||||
input_ids = [combined_input_ids]
|
||||
|
||||
embeds = stacked_query_item_embed_overrides[0]
|
||||
if embeds is None:
|
||||
return None, input_ids, None, delimiter_indices
|
||||
|
||||
positions = self._placeholder_positions(query, embed_override_token_id)
|
||||
current_offset = len(query) + 1 # +1 for first delimiter
|
||||
for item in items:
|
||||
positions.extend(
|
||||
self._placeholder_positions(
|
||||
item, embed_override_token_id, current_offset
|
||||
)
|
||||
)
|
||||
current_offset += len(item) + 1 # +1 for delimiter
|
||||
|
||||
pe = [PositionalEmbeds(embeds=embeds, positions=positions)]
|
||||
return None, input_ids, pe, delimiter_indices
|
||||
|
||||
input_ids = [query + item for item in items]
|
||||
q_positions = self._placeholder_positions(query, embed_override_token_id)
|
||||
|
||||
positional_embed_overrides: List[Optional[PositionalEmbeds]] = []
|
||||
any_overrides = False
|
||||
for item, embeds in zip(items, stacked_query_item_embed_overrides):
|
||||
if embeds is None:
|
||||
positional_embed_overrides.append(None)
|
||||
continue
|
||||
i_positions = self._placeholder_positions(
|
||||
item, embed_override_token_id, len(query)
|
||||
)
|
||||
positional_embed_overrides.append(
|
||||
PositionalEmbeds(embeds=embeds, positions=q_positions + i_positions)
|
||||
)
|
||||
any_overrides = True
|
||||
|
||||
return (
|
||||
None,
|
||||
input_ids,
|
||||
positional_embed_overrides if any_overrides else None,
|
||||
None,
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Multi-position pooling readout (setwise scoring)
|
||||
# ------------------------------------------------------------------
|
||||
@@ -818,6 +899,9 @@ class TokenizerManagerScoreMixin:
|
||||
embed_override_token_id: Optional[int] = None,
|
||||
query_embed_overrides: Optional[List[torch.Tensor]] = None,
|
||||
item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None,
|
||||
stacked_query_item_embed_overrides: Optional[
|
||||
List[Optional[torch.Tensor]]
|
||||
] = None,
|
||||
request: Optional[Any] = None,
|
||||
return_pooled_hidden_states: bool = False,
|
||||
score_extraction_token_id: Optional[int] = None,
|
||||
@@ -852,6 +936,9 @@ class TokenizerManagerScoreMixin:
|
||||
|
||||
return_pooled_hidden_states is only supported for non-generation models
|
||||
(SequenceClassification, RewardModel); raises ValueError for CausalLM.
|
||||
|
||||
stacked_query_item_embed_overrides replaces query_/item_embed_overrides with one
|
||||
[N, hidden] tensor per sequence (per item, or one under --enable-mis), query rows first.
|
||||
"""
|
||||
is_generation = self.is_generation
|
||||
|
||||
@@ -910,20 +997,43 @@ class TokenizerManagerScoreMixin:
|
||||
query = [] if items_are_token_ids else ""
|
||||
|
||||
has_embeds = (
|
||||
query_embed_overrides is not None or item_embed_overrides is not None
|
||||
query_embed_overrides is not None
|
||||
or item_embed_overrides is not None
|
||||
or stacked_query_item_embed_overrides is not None
|
||||
)
|
||||
if has_embeds and embed_override_token_id is None:
|
||||
raise ValueError(
|
||||
"embed_override_token_id is required when query_embed_overrides "
|
||||
"or item_embed_overrides are supplied."
|
||||
"embed_override_token_id is required when query_embed_overrides, "
|
||||
"item_embed_overrides, or stacked_query_item_embed_overrides are supplied."
|
||||
)
|
||||
if item_first and has_embeds:
|
||||
raise ValueError("item_first is not supported when embeddings are supplied")
|
||||
if stacked_query_item_embed_overrides is not None and (
|
||||
query_embed_overrides is not None or item_embed_overrides is not None
|
||||
):
|
||||
raise ValueError(
|
||||
"stacked_query_item_embed_overrides is mutually exclusive with "
|
||||
"query_embed_overrides / item_embed_overrides."
|
||||
)
|
||||
if item_embed_overrides is not None and len(item_embed_overrides) != len(items):
|
||||
raise ValueError(
|
||||
f"item_embed_overrides length ({len(item_embed_overrides)}) "
|
||||
f"must match items length ({len(items)})."
|
||||
)
|
||||
if stacked_query_item_embed_overrides is not None:
|
||||
if isinstance(query, str) or isinstance(items, str):
|
||||
raise ValueError(
|
||||
"stacked_query_item_embed_overrides requires token-id query/items, "
|
||||
"not str."
|
||||
)
|
||||
expected = 1 if get_exec().features.enable_mis else len(items)
|
||||
if len(stacked_query_item_embed_overrides) != expected:
|
||||
raise ValueError(
|
||||
f"stacked_query_item_embed_overrides length "
|
||||
f"({len(stacked_query_item_embed_overrides)}) must be {expected} "
|
||||
f"(1 for multi-item scoring, else len(items))."
|
||||
)
|
||||
|
||||
# Check if multi-item scoring is enabled
|
||||
use_multi_item_scoring = get_exec().features.enable_mis
|
||||
|
||||
@@ -995,6 +1105,7 @@ class TokenizerManagerScoreMixin:
|
||||
embed_override_token_id,
|
||||
query_embed_overrides,
|
||||
item_embed_overrides,
|
||||
stacked_query_item_embed_overrides,
|
||||
)
|
||||
)
|
||||
elif (
|
||||
@@ -1025,6 +1136,7 @@ class TokenizerManagerScoreMixin:
|
||||
embed_override_token_id,
|
||||
query_embed_overrides,
|
||||
item_embed_overrides,
|
||||
stacked_query_item_embed_overrides,
|
||||
)
|
||||
)
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user