From baa632b04970742e7622afe308f270c86720de84 Mon Sep 17 00:00:00 2001 From: Chuanrui Zhu <47195419+chuanrui@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:46:50 -0700 Subject: [PATCH] [Score] Send stacked GPU embedding overrides through a CUDA IPC pool (#39730) Co-authored-by: Sundara Raman Ramachandran Co-authored-by: hnyls2002 --- .../srt/entrypoints/engine_score_mixin.py | 7 ++ python/sglang/srt/managers/embed_types.py | 14 ++- .../managers/positional_embeds_transport.py | 94 ++++++++++++++ python/sglang/srt/managers/schedule_batch.py | 6 +- python/sglang/srt/managers/scheduler.py | 7 ++ .../sglang/srt/managers/tokenizer_manager.py | 19 +++ .../managers/tokenizer_manager_score_mixin.py | 118 +++++++++++++++++- .../prefill_only/test_score_engine.py | 34 +++++ .../unit/managers/test_embed_overrides.py | 53 ++++++++ .../test_tokenizer_manager_rid_cleanup.py | 1 + .../multimodal/test_gpu_feature_transport.py | 8 +- 11 files changed, 353 insertions(+), 8 deletions(-) create mode 100644 python/sglang/srt/managers/positional_embeds_transport.py diff --git a/python/sglang/srt/entrypoints/engine_score_mixin.py b/python/sglang/srt/entrypoints/engine_score_mixin.py index d8f35318a2..75a4125d07 100644 --- a/python/sglang/srt/entrypoints/engine_score_mixin.py +++ b/python/sglang/srt/entrypoints/engine_score_mixin.py @@ -40,6 +40,7 @@ class EngineScoreMixin: score_extraction_token_id: Optional[int] = None, temperature: float = 1.0, return_token_logprobs: bool = False, + stacked_query_item_embed_overrides: Optional[List[torch.Tensor]] = None, ) -> ScoreResult: """ Score items against a query using the loaded model. @@ -84,6 +85,9 @@ class EngineScoreMixin: softmax. Requires apply_softmax=True when different from 1. return_token_logprobs: Include uncalibrated full-vocabulary logprobs for each item's candidates. Only supported for CausalLM, not raw logits. + stacked_query_item_embed_overrides: Alternative to query_/item_embed_overrides: + one already-stacked [N, hidden] tensor per output sequence, used without + a restack. See score_request. Returns: ScoreResult with scores (flat ``[num_items x num_labels]`` for pointwise @@ -102,6 +106,7 @@ class EngineScoreMixin: query_embed_overrides=query_embed_overrides, item_embed_overrides=item_embed_overrides, score_extraction_token_id=score_extraction_token_id, + stacked_query_item_embed_overrides=stacked_query_item_embed_overrides, request=None, return_pooled_hidden_states=return_pooled_hidden_states, temperature=temperature, @@ -123,6 +128,7 @@ class EngineScoreMixin: score_extraction_token_id: Optional[int] = None, temperature: float = 1.0, return_token_logprobs: bool = False, + stacked_query_item_embed_overrides: Optional[List[torch.Tensor]] = None, ) -> ScoreResult: """Asynchronous version of score(). See score() for full documentation.""" return await self.tokenizer_manager.score_request( @@ -135,6 +141,7 @@ class EngineScoreMixin: query_embed_overrides=query_embed_overrides, item_embed_overrides=item_embed_overrides, score_extraction_token_id=score_extraction_token_id, + stacked_query_item_embed_overrides=stacked_query_item_embed_overrides, request=None, return_pooled_hidden_states=return_pooled_hidden_states, temperature=temperature, diff --git a/python/sglang/srt/managers/embed_types.py b/python/sglang/srt/managers/embed_types.py index 1846f3c399..8d8974dbac 100644 --- a/python/sglang/srt/managers/embed_types.py +++ b/python/sglang/srt/managers/embed_types.py @@ -23,6 +23,8 @@ from typing import List import msgspec import torch +from sglang.srt.utils.cuda_ipc_transport_utils import CudaIpcTensorTransportProxy + class PositionalEmbeds(msgspec.Struct, array_like=True): """Embeddings to place at specific token positions. @@ -36,7 +38,8 @@ class PositionalEmbeds(msgspec.Struct, array_like=True): positions: List of positions where embeddings should be injected. """ - embeds: torch.Tensor + # torch.Tensor or CudaIpcTensorTransportProxy; msgspec cannot type that union. + embeds: object positions: List[int] def __post_init__(self): @@ -51,8 +54,15 @@ class PositionalEmbeds(msgspec.Struct, array_like=True): else: # [1, hidden_dim] (already has the leading dim) → plain concat. self.embeds = torch.cat(self.embeds, dim=0) - if self.embeds.shape[0] != len(self.positions): + if isinstance(self.embeds, torch.Tensor) and self.embeds.shape[0] != len( + self.positions + ): raise ValueError( f"embeds length ({self.embeds.shape[0]}) != " f"positions length ({len(self.positions)})" ) + + def reconstruct(self, target_device: int) -> None: + """Copy embeds out of the CUDA IPC pool onto target_device and release the slot.""" + if isinstance(self.embeds, CudaIpcTensorTransportProxy): + self.embeds = self.embeds.reconstruct_on_target_device(target_device) diff --git a/python/sglang/srt/managers/positional_embeds_transport.py b/python/sglang/srt/managers/positional_embeds_transport.py new file mode 100644 index 0000000000..2f41bb6b7a --- /dev/null +++ b/python/sglang/srt/managers/positional_embeds_transport.py @@ -0,0 +1,94 @@ +"""Send CUDA override embeds to the scheduler through a CUDA IPC pool, not by value.""" + +import logging +from typing import Iterable, List, Optional + +import torch + +from sglang.srt.environ import envs +from sglang.srt.managers.embed_types import PositionalEmbeds +from sglang.srt.multimodal.transport.cuda_ipc import ( + MM_FEATURE_CACHE_SIZE, + MM_ITEM_MEMORY_POOL_RECYCLE_INTERVAL, + CudaIpcTensorTransportProxy, + MmItemMemoryPool, + get_mm_feature_pool_size_per_worker, +) +from sglang.srt.multimodal.transport.memory_pool import get_mm_feature_consumer_count +from sglang.srt.runtime_context import get_parallel, get_serving +from sglang.srt.utils import is_cuda + +logger = logging.getLogger(__name__) + + +class PositionalEmbedsTransport: + def __init__(self, base_gpu_id: int, mm_processor) -> None: + parallel = get_parallel() + # By value otherwise: IPC needs one node, every pipeline stage would ack the + # slot, and the msgpack IPC encoder cannot encode the proxy. + self.enabled = ( + is_cuda() + and parallel.nnodes == 1 + and parallel.pp_size == 1 + and envs.SGLANG_USE_PICKLE_IPC.get() + ) + self._base_gpu_id = base_gpu_id + # A multimodal model with --mm-feature-transport=cuda_ipc already has a pool. + self._pool: Optional[MmItemMemoryPool] = getattr( + mm_processor, "cudaipc_mmfeature_pool", None + ) + self._use_pool_handle_cache = envs.SGLANG_USE_IPC_POOL_HANDLE_CACHE.get() + + def _get_pool(self) -> MmItemMemoryPool: + if self._pool is None: + pool_size = get_mm_feature_pool_size_per_worker( + MM_FEATURE_CACHE_SIZE, get_serving().tokenizer_worker_num + ) + logger.info( + "Creating a %.0f MiB CUDA IPC pool on GPU %d for GPU-resident " + "embedding overrides (SGLANG_MM_FEATURE_CACHE_MB).", + pool_size / (1024 * 1024), + self._base_gpu_id, + ) + self._pool = MmItemMemoryPool( + pool_size, + MM_ITEM_MEMORY_POOL_RECYCLE_INTERVAL, + self._base_gpu_id, + get_mm_feature_consumer_count(), + ) + return self._pool + + def prepare_for_dispatch( + self, tokenized_objs: Iterable + ) -> List[CudaIpcTensorTransportProxy]: + """Returns the proxies to cancel if dispatch fails; a full pool means by value.""" + if not self.enabled: + return [] + proxies = [] + try: + for tokenized_obj in tokenized_objs: + overrides = tokenized_obj.positional_embed_overrides + if overrides is None or not ( + isinstance(overrides.embeds, torch.Tensor) + and overrides.embeds.is_cuda + and overrides.embeds.numel() > 0 + ): + continue + proxy = self._get_pool().wrap_tensor( + overrides.embeds, use_pool_handle_cache=self._use_pool_handle_cache + ) + if proxy is None: + continue + proxies.append(proxy) + # A new object: requests may share the caller's PositionalEmbeds. + tokenized_obj.positional_embed_overrides = PositionalEmbeds( + embeds=proxy, positions=overrides.positions + ) + except BaseException: + self.cancel_for_dispatch(proxies) + raise + return proxies + + def cancel_for_dispatch(self, proxies: List[CudaIpcTensorTransportProxy]) -> None: + for proxy in proxies: + self._pool.cancel_proxy(proxy) diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index 009a97c39b..fe7ac1bfd1 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -1367,7 +1367,11 @@ class Req(ReqDllmMixin): # retracted request is rebootstrapped. Set in pause_generation(retract) # and consumed in the decode transfer commit; never plumbed to prefill. self.pd_rebootstrap_forced_output_id: Optional[int] = None - self.skip_radix_cache_insert = bootstrap_host == FAKE_BOOTSTRAP_HOST + # Override embeds give KV that the token ids alone do not determine. + self.skip_radix_cache_insert = ( + bootstrap_host == FAKE_BOOTSTRAP_HOST + or positional_embed_overrides is not None + ) # Set when the request's in-flight forward ran on KV that never landed: # that forward's output is dropped and the request finishes with it. self.discard_output_reason: Optional[FINISH_ABORT] = None diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index eff66f6c33..79efa0bacc 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -2140,6 +2140,13 @@ class Scheduler( self.session_controller.maybe_reap(now) for recv_req in recv_reqs: + # Before any early exit, so every request releases its CUDA IPC pool slot. + for tokenized_req in self._tokenized_requests(recv_req): + if tokenized_req.positional_embed_overrides is not None: + tokenized_req.positional_embed_overrides.reconstruct( + get_device().gpu_id + ) + vmm_errors = None if get_mm().mm_feature_transport == "cuda_vmm": vmm_errors = self._materialize_cuda_vmm_inputs(recv_req) diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 343116267d..2b2c405fd4 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -110,6 +110,7 @@ from sglang.srt.managers.io_struct import ( from sglang.srt.managers.load_snapshot import create_load_snapshot_reader from sglang.srt.managers.mm_utils import wrap_shm_features from sglang.srt.managers.multimodal_processor import get_mm_processor, import_processors +from sglang.srt.managers.positional_embeds_transport import PositionalEmbedsTransport from sglang.srt.managers.schedule_batch import ( MultimodalDataItem, get_request_return_hidden_states_mode, @@ -559,6 +560,9 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): # Init request dispatcher self.init_request_dispatcher() + # Init the CUDA IPC transport for GPU-resident embedding overrides + self.init_positional_embeds_transport() + # Construct this last so later initialization failures cannot orphan # the transport's recycler thread. self.cuda_vmm_feature_transport = CudaVmmFeatureTransport( @@ -857,6 +861,11 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): if self.enable_metrics: self.metrics_collector.emit_startup_time(startup_time) + def init_positional_embeds_transport(self): + self.positional_embeds_transport = PositionalEmbedsTransport( + self.server_args.base_gpu_id, self.mm_processor + ) + def init_request_dispatcher(self): self._result_dispatcher = TypeBasedDispatcher( [ @@ -1758,6 +1767,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): tokenized_obj: Union[TokenizedGenerateReqInput, TokenizedEmbeddingReqInput], ): prepared_mm_items = [] + prepared_embeds = [] dispatched = False try: prepared_mm_items = ( @@ -1765,6 +1775,9 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): (tokenized_obj.mm_inputs,) ) ) + prepared_embeds = self.positional_embeds_transport.prepare_for_dispatch( + (tokenized_obj,) + ) tokenized_obj.time_stats.set_api_server_dispatch_time() tokenized_obj = wrap_shm_features(tokenized_obj) time_stats = tokenized_obj.time_stats @@ -1780,6 +1793,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): finally: if not dispatched: self.cuda_vmm_feature_transport.cancel_for_dispatch(prepared_mm_items) + self.positional_embeds_transport.cancel_for_dispatch(prepared_embeds) def _mark_state_dispatched(self, rid: str): """Record that *rid* reached the scheduler. @@ -1799,6 +1813,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): ): """Send a batch of tokenized requests as a single batched request to the scheduler.""" prepared_mm_items = [] + prepared_embeds = [] dispatched = False try: prepared_mm_items = ( @@ -1806,6 +1821,9 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): tokenized_obj.mm_inputs for tokenized_obj in tokenized_objs ) ) + prepared_embeds = self.positional_embeds_transport.prepare_for_dispatch( + tokenized_objs + ) set_time_batch(tokenized_objs, "set_api_server_dispatch_time") time_stats = [tokenized_obj.time_stats for tokenized_obj in tokenized_objs] @@ -1827,6 +1845,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): finally: if not dispatched: self.cuda_vmm_feature_transport.cancel_for_dispatch(prepared_mm_items) + self.positional_embeds_transport.cancel_for_dispatch(prepared_embeds) def _coalesce_streaming_chunks( self, diff --git a/python/sglang/srt/managers/tokenizer_manager_score_mixin.py b/python/sglang/srt/managers/tokenizer_manager_score_mixin.py index 8bb47418b6..095e6f2005 100644 --- a/python/sglang/srt/managers/tokenizer_manager_score_mixin.py +++ b/python/sglang/srt/managers/tokenizer_manager_score_mixin.py @@ -479,6 +479,18 @@ class TokenizerManagerScoreMixin: ) return embeds, positions + def _placeholder_positions( + self, + token_ids: List[int], + embed_override_token_id: int, + position_offset: int = 0, + ) -> List[int]: + return [ + idx + position_offset + for idx, tok in enumerate(token_ids) + if tok == embed_override_token_id + ] + def _resolve_embed_overrides_for_request( self, query: List[int], @@ -523,12 +535,24 @@ class TokenizerManagerScoreMixin: embed_override_token_id: Optional[int], query_embed_overrides: Optional[List[torch.Tensor]], item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]], + stacked_query_item_embed_overrides: Optional[ + List[Optional[torch.Tensor]] + ] = None, ) -> Tuple[None, List[List[int]], Optional[list], Optional[List[int]]]: """Build input_ids and resolve embed overrides for token-ID inputs. Multi-item-scoring and single-item modes differ only in how input_ids are assembled and what position offset each item gets. """ + if stacked_query_item_embed_overrides is not None: + return self._build_token_id_inputs_stacked( + query, + items, + use_multi_item_scoring, + embed_override_token_id, + stacked_query_item_embed_overrides, + ) + # Both query and items are token IDs has_embeds = ( query_embed_overrides is not None or item_embed_overrides is not None @@ -623,6 +647,63 @@ class TokenizerManagerScoreMixin: None, ) + def _build_token_id_inputs_stacked( + self, + query: List[int], + items: List[List[int]], + use_multi_item_scoring: bool, + embed_override_token_id: int, + stacked_query_item_embed_overrides: List[Optional[torch.Tensor]], + ) -> Tuple[None, List[List[int]], Optional[list], Optional[List[int]]]: + """On the single-item path each item's tensor holds the query rows, then its own.""" + if use_multi_item_scoring: + delimiter_token_id = MIS_DELIMITER_TOKEN_ID + combined_input_ids, delimiter_indices = ( + self._build_multi_item_token_sequence(query, items, delimiter_token_id) + ) + input_ids = [combined_input_ids] + + embeds = stacked_query_item_embed_overrides[0] + if embeds is None: + return None, input_ids, None, delimiter_indices + + positions = self._placeholder_positions(query, embed_override_token_id) + current_offset = len(query) + 1 # +1 for first delimiter + for item in items: + positions.extend( + self._placeholder_positions( + item, embed_override_token_id, current_offset + ) + ) + current_offset += len(item) + 1 # +1 for delimiter + + pe = [PositionalEmbeds(embeds=embeds, positions=positions)] + return None, input_ids, pe, delimiter_indices + + input_ids = [query + item for item in items] + q_positions = self._placeholder_positions(query, embed_override_token_id) + + positional_embed_overrides: List[Optional[PositionalEmbeds]] = [] + any_overrides = False + for item, embeds in zip(items, stacked_query_item_embed_overrides): + if embeds is None: + positional_embed_overrides.append(None) + continue + i_positions = self._placeholder_positions( + item, embed_override_token_id, len(query) + ) + positional_embed_overrides.append( + PositionalEmbeds(embeds=embeds, positions=q_positions + i_positions) + ) + any_overrides = True + + return ( + None, + input_ids, + positional_embed_overrides if any_overrides else None, + None, + ) + # ------------------------------------------------------------------ # Multi-position pooling readout (setwise scoring) # ------------------------------------------------------------------ @@ -818,6 +899,9 @@ class TokenizerManagerScoreMixin: embed_override_token_id: Optional[int] = None, query_embed_overrides: Optional[List[torch.Tensor]] = None, item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None, + stacked_query_item_embed_overrides: Optional[ + List[Optional[torch.Tensor]] + ] = None, request: Optional[Any] = None, return_pooled_hidden_states: bool = False, score_extraction_token_id: Optional[int] = None, @@ -852,6 +936,9 @@ class TokenizerManagerScoreMixin: return_pooled_hidden_states is only supported for non-generation models (SequenceClassification, RewardModel); raises ValueError for CausalLM. + + stacked_query_item_embed_overrides replaces query_/item_embed_overrides with one + [N, hidden] tensor per sequence (per item, or one under --enable-mis), query rows first. """ is_generation = self.is_generation @@ -910,20 +997,43 @@ class TokenizerManagerScoreMixin: query = [] if items_are_token_ids else "" has_embeds = ( - query_embed_overrides is not None or item_embed_overrides is not None + query_embed_overrides is not None + or item_embed_overrides is not None + or stacked_query_item_embed_overrides is not None ) if has_embeds and embed_override_token_id is None: raise ValueError( - "embed_override_token_id is required when query_embed_overrides " - "or item_embed_overrides are supplied." + "embed_override_token_id is required when query_embed_overrides, " + "item_embed_overrides, or stacked_query_item_embed_overrides are supplied." ) if item_first and has_embeds: raise ValueError("item_first is not supported when embeddings are supplied") + if stacked_query_item_embed_overrides is not None and ( + query_embed_overrides is not None or item_embed_overrides is not None + ): + raise ValueError( + "stacked_query_item_embed_overrides is mutually exclusive with " + "query_embed_overrides / item_embed_overrides." + ) if item_embed_overrides is not None and len(item_embed_overrides) != len(items): raise ValueError( f"item_embed_overrides length ({len(item_embed_overrides)}) " f"must match items length ({len(items)})." ) + if stacked_query_item_embed_overrides is not None: + if isinstance(query, str) or isinstance(items, str): + raise ValueError( + "stacked_query_item_embed_overrides requires token-id query/items, " + "not str." + ) + expected = 1 if get_exec().features.enable_mis else len(items) + if len(stacked_query_item_embed_overrides) != expected: + raise ValueError( + f"stacked_query_item_embed_overrides length " + f"({len(stacked_query_item_embed_overrides)}) must be {expected} " + f"(1 for multi-item scoring, else len(items))." + ) + # Check if multi-item scoring is enabled use_multi_item_scoring = get_exec().features.enable_mis @@ -995,6 +1105,7 @@ class TokenizerManagerScoreMixin: embed_override_token_id, query_embed_overrides, item_embed_overrides, + stacked_query_item_embed_overrides, ) ) elif ( @@ -1025,6 +1136,7 @@ class TokenizerManagerScoreMixin: embed_override_token_id, query_embed_overrides, item_embed_overrides, + stacked_query_item_embed_overrides, ) ) else: diff --git a/test/registered/prefill_only/test_score_engine.py b/test/registered/prefill_only/test_score_engine.py index df45a4375d..a81c03bbd6 100644 --- a/test/registered/prefill_only/test_score_engine.py +++ b/test/registered/prefill_only/test_score_engine.py @@ -30,6 +30,7 @@ register_cuda_ci(est_time=101, stage="base-b", runner_config="1-gpu-small") _CAUSAL_LM_MODEL = os.environ.get("TEST_MODEL_NAME", DEFAULT_SMALL_MODEL_NAME_FOR_TEST) _SEQCLS_MODEL = os.environ.get("TEST_CLASSIFICATION_BASE_MODEL", "Qwen/Qwen3-0.6B") +_PLACEHOLDER = 50 # --------------------------------------------------------------------------- @@ -279,6 +280,39 @@ class TestCausalLMScoring(CustomTestCase): torch.tensor(cold), torch.tensor(warm), atol=1e-3, rtol=1e-2 ) + def test_stacked_embed_overrides_match_by_value(self): + hidden = self.engine.tokenizer_manager.model_config.hidden_size + gen = torch.Generator().manual_seed(0) + query = [_PLACEHOLDER, 1000, _PLACEHOLDER, 1001] + items = [[2000 + i, _PLACEHOLDER, 3000 + i] for i in range(3)] + query_vecs = 0.02 * torch.randn(2, hidden, generator=gen) + item_vecs = [0.02 * torch.randn(1, hidden, generator=gen) for _ in items] + kwargs = dict(query=query, items=items, label_token_ids=[33, 32]) + + without = self.engine.score(**kwargs).scores + by_value = self.engine.score( + embed_override_token_id=_PLACEHOLDER, + query_embed_overrides=list(query_vecs), + item_embed_overrides=[list(v) for v in item_vecs], + **kwargs, + ).scores + stacked = self.engine.score( + embed_override_token_id=_PLACEHOLDER, + stacked_query_item_embed_overrides=[ + torch.cat([query_vecs, v]).cuda() for v in item_vecs + ], + **kwargs, + ).scores + after = self.engine.score(**kwargs).scores + + torch.testing.assert_close(torch.tensor(stacked), torch.tensor(by_value)) + self.assertFalse(torch.allclose(torch.tensor(stacked), torch.tensor(without))) + # Override KV must stay out of the radix cache. + torch.testing.assert_close(torch.tensor(after), torch.tensor(without)) + self.assertIsNotNone( + self.engine.tokenizer_manager.positional_embeds_transport._pool + ) + # --------------------------------------------------------------------------- # SequenceClassification — single-item mode diff --git a/test/registered/unit/managers/test_embed_overrides.py b/test/registered/unit/managers/test_embed_overrides.py index e00a1d1fc4..b46e927089 100644 --- a/test/registered/unit/managers/test_embed_overrides.py +++ b/test/registered/unit/managers/test_embed_overrides.py @@ -537,6 +537,59 @@ class TestBuildTokenIdInputs(CustomTestCase): self.assertEqual(pe.embeds.shape[0], 2) +# ======================================================================== +# Score mixin: _build_token_id_inputs stacked path +# ======================================================================== + + +def _stacked(n_rows: int, val: float = 1.0) -> torch.Tensor: + return torch.full((n_rows, HIDDEN_DIM), val, dtype=torch.float32) + + +class TestBuildTokenIdInputsStacked(CustomTestCase): + def setUp(self): + self.mixin = _FakeMixin() + + def test_single_item_stacked_none_slot(self): + stacked = _stacked(2, 5.0) + _, input_ids, pe_overrides, _ = self.mixin._build_token_id_inputs( + query=[50, 10], + items=[[20, 50], [30, 40]], + item_first=False, + use_multi_item_scoring=False, + embed_override_token_id=50, + query_embed_overrides=None, + item_embed_overrides=None, + stacked_query_item_embed_overrides=[stacked, None], + ) + self.assertEqual(input_ids, [[50, 10, 20, 50], [50, 10, 30, 40]]) + self.assertEqual(len(pe_overrides), 2) + self.assertIs(pe_overrides[0].embeds, stacked) + self.assertEqual(pe_overrides[0].positions, [0, 3]) + self.assertIsNone(pe_overrides[1]) + + def test_multi_item_stacked_used_as_is(self): + # One fused sequence; placeholders: query pos 0, item[0] pos 4 + # (query_len 2 + delim 1 + idx 1). item[1] has no placeholder. + stacked = _stacked(2, 9.0) + _, input_ids, pe_overrides, delim = self.mixin._build_token_id_inputs( + query=[50, 10], + items=[[20, 50], [30, 40]], + item_first=False, + use_multi_item_scoring=True, + embed_override_token_id=50, + query_embed_overrides=None, + item_embed_overrides=None, + stacked_query_item_embed_overrides=[stacked], + ) + self.assertEqual( + input_ids, [[50, 10, DELIM_TOKEN, 20, 50, DELIM_TOKEN, 30, 40, DELIM_TOKEN]] + ) + self.assertEqual(len(pe_overrides), 1) + self.assertEqual(pe_overrides[0].positions, [0, 4]) + self.assertIs(pe_overrides[0].embeds, stacked) + + # ======================================================================== # Score mixin: score_request validation # ======================================================================== diff --git a/test/registered/unit/managers/test_tokenizer_manager_rid_cleanup.py b/test/registered/unit/managers/test_tokenizer_manager_rid_cleanup.py index d2258b6599..6d42f3b7d3 100644 --- a/test/registered/unit/managers/test_tokenizer_manager_rid_cleanup.py +++ b/test/registered/unit/managers/test_tokenizer_manager_rid_cleanup.py @@ -1222,6 +1222,7 @@ class TestDisconnectAfterDispatchAbortsRequest(CustomTestCase): tm.cuda_vmm_feature_transport.prepare_for_dispatch_async = AsyncMock( return_value=[] ) + tm.positional_embeds_transport = Mock() tm._dispatch_to_scheduler = Mock() rid = "disconnect_zombie" obj = _make_generate_obj(rid, is_single=True) diff --git a/test/registered/unit/multimodal/test_gpu_feature_transport.py b/test/registered/unit/multimodal/test_gpu_feature_transport.py index 40c64324f5..f67d717a7d 100644 --- a/test/registered/unit/multimodal/test_gpu_feature_transport.py +++ b/test/registered/unit/multimodal/test_gpu_feature_transport.py @@ -363,6 +363,7 @@ class TestCudaVmmFeatureTransport(unittest.TestCase): ) transport.prepare_for_dispatch_async = AsyncMock(return_value=items) manager.cuda_vmm_feature_transport = transport + manager.positional_embeds_transport = MagicMock() with ( patch.object(tokenizer_manager, "wrap_shm_features", lambda obj: obj), @@ -401,6 +402,7 @@ class TestCudaVmmFeatureTransport(unittest.TestCase): ) transport.prepare_for_dispatch_async = AsyncMock(return_value=items) manager.cuda_vmm_feature_transport = transport + manager.positional_embeds_transport = MagicMock() with ( patch.object(tokenizer_manager, "wrap_shm_features", lambda obj: obj), @@ -739,6 +741,7 @@ class TestSchedulerMmTransportBoundary(CustomTestCase): class TokenizedRequest: def __init__(self): self.mm_inputs = object() + self.positional_embed_overrides = None scheduler = object.__new__(scheduler_module.Scheduler) scheduler.model_config = SimpleNamespace(requires_mm_token_modalities=False) @@ -798,7 +801,7 @@ class TestSchedulerMmTransportBoundary(CustomTestCase): self._prepare_scheduler(scheduler) raw_inputs = object() materialized = object() - request = SimpleNamespace(mm_inputs=raw_inputs) + request = SimpleNamespace(mm_inputs=raw_inputs, positional_embed_overrides=None) with ( patch.object( @@ -831,6 +834,7 @@ class TestSchedulerMmTransportBoundary(CustomTestCase): class TokenizedRequest: def __init__(self, mm_inputs): self.mm_inputs = mm_inputs + self.positional_embed_overrides = None class BatchRequest: def __init__(self, batch): @@ -1030,7 +1034,7 @@ class TestSchedulerMmTransportBoundary(CustomTestCase): from sglang.srt.managers import scheduler as scheduler_module class TokenizedRequest: - pass + positional_embed_overrides = None class EmbeddingRequest: pass