From fd86688428c5484b104a94d58e869b7ea277ccc7 Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Mon, 19 Oct 2026 10:45:10 +0800 Subject: [PATCH] PR #40219: DeepSeek V4.1 vision CP and single-owner image encoding Source: https://github.com/sgl-project/sglang/pull/40219 (applied via GitHub .diff, 3way) Adaptations for v0.5.21 base: - attn_dp_enabled_of(cfg) -> cfg.enable_dp_attention (#41818 not in base) - dropped _validate_joint_schema_request call (#42721 Clef, not in base) - moved Backend import before the TRT-LLM prefill-graph check --- .../sglang/srt/arg_groups/deepseek_v4_hook.py | 53 +- .../sglang/srt/managers/mm_owner_embedding.py | 464 ++++++++++++ python/sglang/srt/managers/mm_schedule.py | 45 +- python/sglang/srt/managers/mm_utils.py | 11 +- python/sglang/srt/managers/schedule_policy.py | 9 + python/sglang/srt/managers/scheduler.py | 2 + .../sglang/srt/managers/tokenizer_manager.py | 10 + .../sglang/srt/model_executor/model_runner.py | 1 + .../model_runner_components/misc_utils.py | 21 + .../srt/model_executor/runner/eager_runner.py | 38 +- python/sglang/srt/models/deepseek_v4.py | 122 ++- python/sglang/srt/models/deepseek_v4_nextn.py | 1 + .../unit/managers/test_embed_overrides.py | 82 +++ .../unit/managers/test_mm_owner_embedding.py | 693 ++++++++++++++++++ .../unit/managers/test_prefill_adder.py | 123 +++- .../test_deepseek_v41_vision_cp_inputs.py | 247 +++++++ .../unit/server_args/test_server_args.py | 78 ++ 17 files changed, 1956 insertions(+), 44 deletions(-) create mode 100644 python/sglang/srt/managers/mm_owner_embedding.py create mode 100644 test/registered/unit/managers/test_mm_owner_embedding.py create mode 100644 test/registered/unit/models/test_deepseek_v41_vision_cp_inputs.py diff --git a/python/sglang/srt/arg_groups/deepseek_v4_hook.py b/python/sglang/srt/arg_groups/deepseek_v4_hook.py index c2cd5c0526..4c418a2edf 100644 --- a/python/sglang/srt/arg_groups/deepseek_v4_hook.py +++ b/python/sglang/srt/arg_groups/deepseek_v4_hook.py @@ -172,15 +172,25 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None: ) cfg = resolving_view(server_args) - if model_config_of(server_args).hf_config.model_type != "deepseek_v41": + hf_config = model_config_of(server_args).hf_config + if hf_config.model_type != "deepseek_v41": if cfg.enable_encoder_swa_bounded_replay: raise ValueError( "--enable-encoder-swa-bounded-replay requires DeepSeek-V4.1" ) return - if cfg.enable_encoder_swa_bounded_replay: - from sglang.srt.model_executor.cuda_graph_config import Backend + from sglang.srt.model_executor.cuda_graph_config import Backend + if hf_config.vision_n_layers > 0 and cfg.enable_prefill_cp: + _validate_deepseek_v41_vision_prefill_cp(server_args) + if ( + cfg.dsv4_attn_backend == "trtllm" + and cfg.cuda_graph_config.prefill.backend != Backend.DISABLED + ): + raise ValueError( + "DeepSeek-V4.1 TRT-LLM requires --cuda-graph-backend-prefill disabled" + ) + if cfg.enable_encoder_swa_bounded_replay: incompatible = ( ("non-CUDA hardware", not get_platform().is_cuda), ( @@ -188,7 +198,8 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None: cfg.cuda_graph_config.prefill.backend != Backend.DISABLED, ), ("DP attention", cfg.enable_dp_attention), - ("context parallelism", cfg.attn_cp_size > 1), + # Prefill CP declares attn_cp_size and DP attention only later. + ("context parallelism", cfg.attn_cp_size > 1 or cfg.enable_prefill_cp), ("external cache linker", cfg.enable_unified_cache_external_linker), ("unified memory", cfg.enable_unified_memory), ("PD disaggregation", cfg.disaggregation_mode != "null"), @@ -287,3 +298,37 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None: "--enable-decoder-swa-bounded-replay cannot be combined with " f"{feature} yet; disable one of them." ) + + +def _validate_deepseek_v41_vision_prefill_cp(server_args: ServerArgs) -> None: + cfg = resolving_view(server_args) + if cfg.cp_strategy != "interleave": + raise ValueError( + "DeepSeek-V4.1 vision with prefill CP requires --cp-strategy " + f"interleave; got {cfg.cp_strategy!r}." + ) + if cfg.cuda_graph_config.prefill.backend != Backend.DISABLED: + locked = getattr(server_args, "_cuda_graph_config_locked", set()) + if (Phase.PREFILL, "backend") in locked: + raise ValueError( + "DeepSeek-V4.1 vision with prefill CP runs eager prefill; remove " + "the explicit prefill CUDA graph backend." + ) + declare_resolution( + server_args, + "validate_deepseek_v41_features", + cuda_graph_config=with_phase( + cfg.cuda_graph_config, Phase.PREFILL, backend=Backend.DISABLED + ), + ) + logger.warning( + "Disabling the prefill CUDA graph for DeepSeek-V4.1 vision with prefill CP." + ) + if ( + str(cfg.speculative_algorithm).upper() == "DSPARK" + and cfg.enable_decoder_swa_bounded_replay + ): + raise ValueError( + "DeepSeek-V4.1 vision with prefill CP does not support DSpark together " + "with --enable-decoder-swa-bounded-replay yet." + ) diff --git a/python/sglang/srt/managers/mm_owner_embedding.py b/python/sglang/srt/managers/mm_owner_embedding.py new file mode 100644 index 0000000000..b7fc5ea372 --- /dev/null +++ b/python/sglang/srt/managers/mm_owner_embedding.py @@ -0,0 +1,464 @@ +"""One owner rank encodes each image span and broadcasts it to the ranks that +run the same prefill chunk; every agreement precedes the payload it guards.""" + +from __future__ import annotations + +import logging +from contextlib import contextmanager +from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence, Tuple + +import msgspec +import torch + +from sglang.srt.distributed.device_communicators.pynccl_allocator import ( + disable_symmetric_memory_context, + restore_symmetric_memory_context, +) +from sglang.srt.mem_cache.multimodal_cache import EmbeddingResult, MultiModalStaticCache + +logger = logging.getLogger(__name__) + +SpanKey = Tuple[Optional[int], int] +SpanEncoder = Callable[[List[Any]], torch.Tensor | List[torch.Tensor]] + +LOCAL_HIT = 0 +OWNER_CACHE_BROADCAST = 1 +OWNER_ENCODE_BROADCAST = 2 + +PHASE_PREPARE = "prepare" +PHASE_FEATURES = "features" +PHASE_FINALIZE = "finalize" + + +class MmOwnerProtocolError(RuntimeError): + """Raised with identical text on every group member after a group-agreed failure.""" + + +class ImageSpanRequest(msgspec.Struct, frozen=True): + hash: Optional[int] + span_len: int + item: Any + duplicates: List[Any] = [] + + +class ImageSpanKey(msgspec.Struct, frozen=True): + hash: Optional[int] + span_len: int + geometry: Optional[Tuple[Any, ...]] + + +class RankManifest(msgspec.Struct, frozen=True): + rank: int + keys: List[ImageSpanKey] + cached: List[bool] + dtype: str + width: int + rids: List[str] + error: Optional[str] = None + + +class OwnerPlan(msgspec.Struct, frozen=True): + actions: List[int] = [] + owners: List[int] = [] + error: Optional[str] = None + + +class RankStatus(msgspec.Struct, frozen=True): + error: Optional[str] = None + + +def select_owner_group(parallel) -> Optional[Any]: + """The group whose members all execute the same requests; None keeps each + rank encoding its own images.""" + replication = parallel.tp_size // parallel.attn_dp_size + if replication <= 1: + return None + if parallel.attn_cp_size == 1: + group = parallel.attn_tp_group + elif parallel.attn_dp_size == 1 and parallel.attn_cp_size == parallel.tp_size: + group = parallel.attn_cp_group + else: + return None + return group if group.world_size == replication else None + + +def has_owner_span_work( + mm_inputs: Sequence[Any], + extend_prefix_lens: Sequence[int], + extend_seq_lens: Sequence[int], +) -> bool: + """Host-side mirror of the per-image scheduling path: does any raw + single-span image overlap the chunk on every rank of the group.""" + for mm_input, prefix_len, extend_len in zip( + mm_inputs, extend_prefix_lens, extend_seq_lens + ): + if mm_input is None or extend_len <= 0: + continue + items = [item for item in mm_input.mm_items if item is not None] + if any( + item.precomputed_embeddings is not None or len(item.offsets) != 1 + for item in items + ): + continue + for item in items: + start, end = item.offsets[0] + if end >= prefix_len and start < prefix_len + extend_len: + return True + return False + + +class MmOwnerSession(msgspec.Struct): + group: Any + device: Any + dtype: Any + width: int + rids: List[str] + signature: Any + phase: str = PHASE_PREPARE + in_collective: bool = False + + def resolve( + self, + requests: Sequence[ImageSpanRequest], + cache: MultiModalStaticCache, + encode: SpanEncoder, + ) -> Dict[SpanKey, torch.Tensor]: + # Owners allocate different amounts than receivers, so none of these + # buffers may come out of a symmetric pool. + saved_context = disable_symmetric_memory_context() + try: + return _resolve_owner_features(self, requests, cache, encode) + finally: + restore_symmetric_memory_context(saved_context) + + def features_ready(self) -> None: + self._complete() + self.phase = PHASE_FINALIZE + + @contextmanager + def uncaptured(self) -> Iterator[None]: + # A failure inside a collective leaves the group in an unknown state; + # no later exchange may try to agree on it. + self.in_collective = True + yield + self.in_collective = False + + @contextmanager + def fence(self) -> Iterator[None]: + try: + yield + except Exception as exc: + self._fail(exc) + raise + self._complete() + + def _fail(self, exc: BaseException) -> None: + if self.in_collective or isinstance(exc, MmOwnerProtocolError): + raise exc + text = _describe(self, self.phase, exc) + if self.phase == PHASE_PREPARE: + try: + _exchange_manifest(self, _manifest(self, [], [], error=text)) + except MmOwnerProtocolError as agreed: + raise agreed from exc + _exchange_status(self, text, exc) + + def _complete(self) -> None: + if self.phase == PHASE_PREPARE: + raise RuntimeError( + f"owner protocol {self.phase} completed without a manifest exchange" + ) + _drain_and_agree(self, self.phase) + + +def _manifest( + session: MmOwnerSession, + keys: List[ImageSpanKey], + cached: List[bool], + error: Optional[str] = None, +) -> RankManifest: + return RankManifest( + rank=session.group.rank_in_group, + keys=keys, + cached=cached, + dtype=str(session.dtype), + width=session.width, + rids=list(session.rids), + error=error, + ) + + +def _exchange_manifest(session: MmOwnerSession, manifest: RankManifest) -> OwnerPlan: + group = session.group + with session.uncaptured(): + manifests = group.all_gather_object(manifest) + plan = _plan_or_error(session, manifests) if group.rank_in_group == 0 else None + plan = group.broadcast_object(plan, src=0) + session.phase = PHASE_FEATURES + if plan.error is not None: + raise MmOwnerProtocolError(plan.error) + return plan + + +def _plan_or_error(session: MmOwnerSession, manifests: List[RankManifest]) -> OwnerPlan: + try: + return _make_plan(manifests) + except Exception as exc: + return OwnerPlan(error=_describe(session, "plan", exc)) + + +def _exchange_status( + session: MmOwnerSession, error: Optional[str], cause: Optional[BaseException] +) -> None: + with session.uncaptured(): + statuses = session.group.all_gather_object(RankStatus(error=error)) + for status in statuses: + if status.error is not None: + raise MmOwnerProtocolError(status.error) from cause + + +def _drain_and_agree( + session: MmOwnerSession, stage: str, step: Optional[Callable[[], Any]] = None +) -> Any: + result, error, cause = None, None, None + try: + result = step() if step is not None else None + _synchronize(session.device) + except Exception as exc: + error, cause = _describe(session, stage, exc), exc + _exchange_status(session, error, cause) + return result + + +def _resolve_owner_features( + session: MmOwnerSession, + requests: Sequence[ImageSpanRequest], + cache: MultiModalStaticCache, + encode: SpanEncoder, +) -> Dict[SpanKey, torch.Tensor]: + group = session.group + features: Dict[SpanKey, torch.Tensor] = {} + keys: List[ImageSpanKey] = [] + cached: List[bool] = [] + error = None + try: + keys, cached = _pin_local_cache(session, requests, cache, features) + except Exception as exc: + error = _describe(session, "manifest", exc) + plan = _exchange_manifest(session, _manifest(session, keys, cached, error)) + + if all(action == LOCAL_HIT for action in plan.actions): + return features + + buffers = _drain_and_agree( + session, + "encode", + lambda: _prepare_transfers(session, requests, keys, plan, features, encode), + ) + + with session.uncaptured(): + for index, (action, owner) in enumerate(zip(plan.actions, plan.owners)): + if action != LOCAL_HIT: + group.broadcast(buffers[index], src=owner) + + for index, key in enumerate(keys): + if plan.actions[index] == LOCAL_HIT: + continue + span = buffers[index] + features[(key.hash, key.span_len)] = span + cache.set(key.hash, EmbeddingResult(embedding=span)) + return features + + +def _pin_local_cache( + session: MmOwnerSession, + requests: Sequence[ImageSpanRequest], + cache: MultiModalStaticCache, + features: Dict[SpanKey, torch.Tensor], +) -> Tuple[List[ImageSpanKey], List[bool]]: + keys: List[ImageSpanKey] = [] + cached: List[bool] = [] + for request in requests: + if request.hash is None: + raise ValueError( + f"image span of {request.span_len} tokens has no content hash" + ) + geometry = session.signature(request.item, request.span_len) + for duplicate in request.duplicates: + other = session.signature(duplicate, request.span_len) + if other != geometry: + raise ValueError( + f"image hash {request.hash} ({request.span_len} tokens) occurs " + f"with different geometry: {geometry} vs {other}" + ) + keys.append( + ImageSpanKey( + hash=request.hash, span_len=request.span_len, geometry=geometry + ) + ) + span = _valid_cached_span(session, cache, request) + if span is not None: + features[(request.hash, request.span_len)] = span + cached.append(span is not None) + return keys, cached + + +def _valid_cached_span( + session: MmOwnerSession, + cache: MultiModalStaticCache, + request: ImageSpanRequest, +) -> Optional[torch.Tensor]: + entry = cache.get_single(request.hash) + if entry is None: + return None + span = entry.embedding + if ( + span.dim() == 2 + and span.shape[0] == request.span_len + and span.shape[1] == session.width + and span.dtype == session.dtype + and span.device == session.device + ): + return span + logger.warning( + "Discarding cached multimodal embedding that cannot serve the current " + "image span: cache_key=%s expected=(%d, %d, %s) cached=(%s, %s).", + request.hash, + request.span_len, + session.width, + session.dtype, + tuple(span.shape), + span.dtype, + ) + cache.free(request.hash, None) + return None + + +def _make_plan(manifests: List[RankManifest]) -> OwnerPlan: + for manifest in manifests: + if manifest.error is not None: + return OwnerPlan(error=manifest.error) + lead = manifests[0] + for manifest in manifests[1:]: + if (manifest.keys, manifest.dtype, manifest.width, manifest.rids) != ( + lead.keys, + lead.dtype, + lead.width, + lead.rids, + ): + return OwnerPlan( + error=( + "image manifest mismatch between group ranks 0 and " + f"{manifest.rank}: rids={lead.rids} vs {manifest.rids}, " + f"keys={lead.keys} vs {manifest.keys}, " + f"dtype={lead.dtype} vs {manifest.dtype}, " + f"width={lead.width} vs {manifest.width}" + ), + ) + replication = len(manifests) + actions: List[int] = [] + owners: List[int] = [] + for index, key in enumerate(lead.keys): + owner = key.hash % replication + if all(manifest.cached[index] for manifest in manifests): + action = LOCAL_HIT + elif manifests[owner].cached[index]: + action = OWNER_CACHE_BROADCAST + else: + action = OWNER_ENCODE_BROADCAST + actions.append(action) + owners.append(owner) + return OwnerPlan(actions=actions, owners=owners) + + +def _prepare_transfers( + session: MmOwnerSession, + requests: Sequence[ImageSpanRequest], + keys: List[ImageSpanKey], + plan: OwnerPlan, + features: Dict[SpanKey, torch.Tensor], + encode: SpanEncoder, +) -> Dict[int, torch.Tensor]: + rank = session.group.rank_in_group + buffers: Dict[int, torch.Tensor] = {} + owned: List[int] = [] + for index, (action, owner) in enumerate(zip(plan.actions, plan.owners)): + if action == LOCAL_HIT: + continue + if owner != rank: + try: + buffers[index] = _new_span_buffer(session, keys[index]) + except Exception as exc: + raise RuntimeError( + f"receive buffer for image hash {keys[index].hash} shape " + f"{(keys[index].span_len, session.width)} {session.dtype} " + f"failed: {type(exc).__name__}: {exc}" + ) from exc + elif action == OWNER_CACHE_BROADCAST: + key = (keys[index].hash, keys[index].span_len) + buffers[index] = features[key].contiguous() + else: + owned.append(index) + if owned: + try: + encoded = encode([requests[index].item for index in owned]) + except Exception as exc: + raise RuntimeError( + f"owner encode of image hashes {[keys[i].hash for i in owned]} failed: " + f"{type(exc).__name__}: {exc}" + ) from exc + spans = _split_spans(encoded, [keys[index].span_len for index in owned]) + for index, span in zip(owned, spans): + buffers[index] = _validated_span(session, keys[index], span) + return buffers + + +def _new_span_buffer(session: MmOwnerSession, key: ImageSpanKey) -> torch.Tensor: + return torch.empty( + (key.span_len, session.width), device=session.device, dtype=session.dtype + ) + + +def _split_spans( + encoded: torch.Tensor | List[torch.Tensor], span_lens: List[int] +) -> List[torch.Tensor]: + if isinstance(encoded, list): + if len(encoded) != len(span_lens): + raise ValueError( + f"encoder returned {len(encoded)} spans for {len(span_lens)} images" + ) + return [span.reshape(-1, span.shape[-1]) for span in encoded] + encoded = encoded.reshape(-1, encoded.shape[-1]) + if encoded.shape[0] != sum(span_lens): + raise ValueError( + f"encoder returned {encoded.shape[0]} rows for spans of {span_lens}" + ) + return list(torch.split(encoded, span_lens, dim=0)) + + +def _validated_span( + session: MmOwnerSession, key: ImageSpanKey, span: torch.Tensor +) -> torch.Tensor: + expected = (key.span_len, session.width) + if tuple(span.shape) != expected or span.dtype != session.dtype: + raise ValueError( + f"encoded span for hash={key.hash} has shape {tuple(span.shape)} " + f"dtype {span.dtype}; expected {expected} {session.dtype}" + ) + if span.device != session.device: + span = span.to(session.device) + return span.contiguous() + + +def _synchronize(device) -> None: + if device.type == "cuda": + torch.cuda.current_stream(device).synchronize() + + +def _describe(session: MmOwnerSession, stage: str, exc: BaseException) -> str: + return ( + f"multimodal owner protocol failed during {stage} on group rank " + f"{session.group.rank_in_group} (global rank " + f"{session.group.ranks[session.group.rank_in_group]}, rids={list(session.rids)}): " + f"{type(exc).__name__}: {exc}" + ) diff --git a/python/sglang/srt/managers/mm_schedule.py b/python/sglang/srt/managers/mm_schedule.py index 0bc0615606..59cd330e81 100644 --- a/python/sglang/srt/managers/mm_schedule.py +++ b/python/sglang/srt/managers/mm_schedule.py @@ -5,6 +5,7 @@ from typing import Callable, Dict, List, Optional, Tuple import torch +from sglang.srt.managers.mm_owner_embedding import ImageSpanRequest, MmOwnerSession from sglang.srt.managers.schedule_batch import MultimodalDataItem from sglang.srt.mem_cache.multimodal_cache import EmbeddingResult, MultiModalStaticCache from sglang.srt.multimodal.evs import EVSEmbeddingResult @@ -430,6 +431,37 @@ def _batch_encode_per_image_misses( return hash_to_embedding +def _collect_image_span_requests( + per_image_requests: List[PerImageRequestInfo], +) -> List[ImageSpanRequest]: + spans: Dict[Tuple[Optional[int], int], ImageSpanRequest] = {} + for req_info in per_image_requests: + chunk_start = req_info.extend_prefix_len + chunk_end = chunk_start + req_info.extend_seq_len # exclusive + req_info.overlapping = [ + (idx, item, token_count) + for idx, item in enumerate(req_info.items) + if (token_count := _item_overlap(item, chunk_start, chunk_end)) is not None + ] + for _idx, item, token_count in req_info.overlapping: + span = spans.get((item.hash, token_count)) + if span is None: + spans[(item.hash, token_count)] = ImageSpanRequest( + hash=item.hash, span_len=token_count, item=item + ) + elif span.item is not item: + span.duplicates.append(item) + return list(spans.values()) + + +def _owner_span_encoder(data_embedding_func: DataEmbeddingFunc, device: torch.device): + def encode(items: List[MultimodalDataItem]): + _move_items_to_device(items, device, data_embedding_func) + return data_embedding_func(items) + + return encode + + def _get_chunked_embedding_by_item( data_embedding_func: DataEmbeddingFunc, embedding_items_per_req: List[MultimodalDataItem], @@ -554,6 +586,7 @@ def _get_chunked_prefill_embedding( extend_length: List[int], items_offset_list: List[List[Tuple[int, int]]], input_ids: torch.Tensor, + mm_owner: Optional[MmOwnerSession] = None, ) -> tuple[torch.Tensor | None, torch.Tensor]: """ Chunked prefill embedding: encode items across all requests and extract @@ -615,7 +648,15 @@ def _get_chunked_prefill_embedding( # Phase 1: batch encode all per-image cache misses in ONE ViT call hash_to_embedding: Dict[Tuple[Optional[int], int], torch.Tensor] = {} - if per_image_requests: + if per_image_requests and mm_owner is not None: + # The owner protocol must see every overlapping span before any local + # cache filtering: a rank-local hit can never skip a group collective. + hash_to_embedding = mm_owner.resolve( + _collect_image_span_requests(per_image_requests), + cache=embedding_cache, + encode=_owner_span_encoder(data_embedding_func, device), + ) + elif per_image_requests: hash_to_embedding = _batch_encode_per_image_misses( data_embedding_func, per_image_requests, device ) @@ -718,6 +759,7 @@ def get_embedding_and_mask( prefix_length: List[int], extend_length: List[int], items_offset_list: List[List[Tuple[int, int]]], + mm_owner: Optional[MmOwnerSession] = None, ) -> Tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor]: """ Generate multimodal embeddings and create a mask for identifying their positions in the input sequence. @@ -758,6 +800,7 @@ def get_embedding_and_mask( extend_length, items_offset_list, input_ids, + mm_owner=mm_owner, ) if embedding is None: return None, None, input_ids diff --git a/python/sglang/srt/managers/mm_utils.py b/python/sglang/srt/managers/mm_utils.py index 7b17e75c94..358302cd64 100644 --- a/python/sglang/srt/managers/mm_utils.py +++ b/python/sglang/srt/managers/mm_utils.py @@ -11,6 +11,7 @@ import sys from abc import abstractmethod from array import array from collections import defaultdict +from contextlib import nullcontext from multiprocessing import shared_memory from typing import Any, Callable, Dict, List, Optional, Tuple @@ -26,6 +27,7 @@ from sglang.srt.managers.io_struct import ( TokenizedEmbeddingReqInput, TokenizedGenerateReqInput, ) +from sglang.srt.managers.mm_owner_embedding import MmOwnerSession # Preserve the existing initialization import for downstream callers. from sglang.srt.managers.mm_schedule import ( @@ -416,6 +418,7 @@ def embed_mm_inputs( data_embedding_func_mapping: Dict[Modality, DataEmbeddingFunc] = None, placeholder_tokens: dict[Modality, List[int]] = None, use_deepstack: Dict[Modality, bool] = {}, + mm_owner: Optional[MmOwnerSession] = None, ) -> Optional[torch.Tensor]: """ Embed multimodal inputs and integrate them with text token embeddings. @@ -497,6 +500,7 @@ def embed_mm_inputs( prefix_length=extend_prefix_lens, extend_length=extend_seq_lens, items_offset_list=items_offsets, + mm_owner=mm_owner, ) if use_deepstack.get(modality, None) and embedding is not None: @@ -517,7 +521,12 @@ def embed_mm_inputs( # filled with the hash values of the multimodal for the prefix matching in the radix attention. # There values are useless because their embeddings will be replaced by vision embeddings anyway. input_ids.clamp_(min=0, max=vocab_size - 1) - input_embeds = input_embedding(input_ids) + if mm_owner is not None: + # The text embedding may all-reduce across TP; a rank-local failure in + # feature preparation has to be agreed on before any rank enters it. + mm_owner.features_ready() + with mm_owner.uncaptured() if mm_owner is not None else nullcontext(): + input_embeds = input_embedding(input_ids) # deepstack embedding if use_deepstack: diff --git a/python/sglang/srt/managers/schedule_policy.py b/python/sglang/srt/managers/schedule_policy.py index d5f1b80ae7..483da8e8c6 100644 --- a/python/sglang/srt/managers/schedule_policy.py +++ b/python/sglang/srt/managers/schedule_policy.py @@ -1111,6 +1111,15 @@ class PrefillAdder: else AddReqResult.CONTINUE ) + def can_share_extend_batch(self, req: Req) -> bool: + # Token embedding overrides embed the batch's raw input_ids before the + # model runs, and that lookup cannot index multimodal placeholder hash IDs. + if req.positional_embed_overrides is not None: + return all(r.multimodal_inputs is None for r in self.can_run_list) + if req.multimodal_inputs is not None: + return all(r.positional_embed_overrides is None for r in self.can_run_list) + return True + def add_chunked_req(self, req: Req): if self.dllm_config is not None: _rem_tokens = self._get_dllm_remain_tokens(req) diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index a7d8baeecc..267c03ffb7 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -3954,6 +3954,8 @@ class Scheduler( for req in self.waiting_queue: if self.enable_lora and not self.can_schedule_lora_req(req, running_loras): continue + if not adder.can_share_extend_batch(req): + break # A forward batch runs one pooling mode, so setwise readout requests # (token_indices_to_pool) cannot share a batch with last-token ones. diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index eb74ca0223..f49c70f61f 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -1283,6 +1283,16 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): raise ValueError( "encoder SWA replay cannot return cached prompt logprobs" ) + requests_embed_overrides = obj.positional_embed_overrides is not None or ( + isinstance(obj, EmbeddingReqInput) + and obj.embed_overrides is not None + and obj.embed_override_token_id is not None + ) + if requests_embed_overrides and obj.contains_mm_input(): + raise ValueError( + "embedding overrides cannot be combined with image, video, or audio " + "inputs" + ) _max_req_len = self.context_len input_token_num = len(input_ids) if input_ids is not None else 0 input_token_num += self.num_reserved_tokens diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 604a72f4ed..1c974a4310 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -1670,6 +1670,7 @@ class ModelRunner: forward_batch.replace_embeds is not None and forward_batch.replace_positions is not None ): + misc_utils.validate_replace_embeds_batch(forward_batch) # Token embedding overrides: get base embeddings, scatter replacements if "input_embeds" not in kwargs: embed_layer = self.model.get_input_embeddings() diff --git a/python/sglang/srt/model_executor/model_runner_components/misc_utils.py b/python/sglang/srt/model_executor/model_runner_components/misc_utils.py index f510693e54..8bfeaf24f0 100644 --- a/python/sglang/srt/model_executor/model_runner_components/misc_utils.py +++ b/python/sglang/srt/model_executor/model_runner_components/misc_utils.py @@ -19,6 +19,7 @@ from sglang.srt.server_args import CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACK if TYPE_CHECKING: from sglang.srt.configs.model_config import ModelConfig + from sglang.srt.model_executor.forward_batch_info import ForwardBatch logger = logging.getLogger(__name__) @@ -99,6 +100,26 @@ def resolve_pp_proxy_residual_num_blocks( return (start_layer + block_size - 1) // block_size +def validate_replace_embeds_batch(forward_batch: ForwardBatch) -> None: + if forward_batch.mm_inputs is None: + return + for mm_inputs, prefix_len, extend_len in zip( + forward_batch.mm_inputs, + forward_batch.extend_prefix_lens_cpu, + forward_batch.extend_seq_lens_cpu, + ): + if mm_inputs is None: + continue + chunk_end = prefix_len + extend_len + for item in mm_inputs.mm_items: + for start, end in item.offsets or (): + if start < chunk_end and end >= prefix_len: + raise ValueError( + "Token embedding overrides cannot share an extend batch with " + "multimodal placeholders" + ) + + @runtime_checkable class _SupportsDSparkPPProxy(Protocol): def get_pp_proxy_dspark_hidden_size(self) -> int: ... diff --git a/python/sglang/srt/model_executor/runner/eager_runner.py b/python/sglang/srt/model_executor/runner/eager_runner.py index acb438eb57..642ce39f83 100644 --- a/python/sglang/srt/model_executor/runner/eager_runner.py +++ b/python/sglang/srt/model_executor/runner/eager_runner.py @@ -384,18 +384,34 @@ class EagerRunner(BaseRunner): """ model = self.model_runner.model - input_embeds = kwargs.get("input_embeds") - if input_embeds is None: - input_embeds = model.get_input_embeddings()(forward_batch.input_ids) - with cp_shard_model_inputs( - input_embeds, - forward_batch.positions, - forward_batch, - forward_batch.input_ids, - ) as (sharded_input_embeds, sharded_positions, model_input_ids): - model_kwargs = {"input_embeds": sharded_input_embeds} + input_ids = forward_batch.input_ids + if prepare_inputs := getattr(model, "prepare_cp_inputs", None): + input_embeds, positions, model_kwargs = prepare_inputs( + forward_batch, **kwargs + ) + else: + input_embeds = kwargs.get("input_embeds") + if hasattr(model, "prepare_model_inputs"): + # Multimodal offsets are request-global, so the merge and the + # placeholder-ID remap must see the full extend layout first. + input_ids, input_embeds = model.prepare_model_inputs( + input_ids=input_ids, + forward_batch=forward_batch, + input_embeds=input_embeds, + ) + if input_embeds is None: + input_embeds = model.get_input_embeddings()(input_ids) + positions = forward_batch.positions + model_kwargs = {} if (pp_proxy_tensors := kwargs.get("pp_proxy_tensors")) is not None: model_kwargs["pp_proxy_tensors"] = pp_proxy_tensors + with cp_shard_model_inputs( + input_embeds, + positions, + forward_batch, + input_ids, + ) as (sharded_input_embeds, sharded_positions, model_input_ids): + model_kwargs["input_embeds"] = sharded_input_embeds hidden_states = model.model( model_input_ids, sharded_positions, @@ -436,7 +452,7 @@ class EagerRunner(BaseRunner): if aux_hidden_states is None: logits_kwargs["hidden_states_before_norm"] = hidden_states_before_norm return model.logits_processor( - forward_batch.input_ids, + input_ids, hidden_states, model.lm_head, forward_batch, diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index cde005d4bd..af783a283e 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -70,6 +70,7 @@ from sglang.srt.layers.attention.dsv4.dsv41_sparse import ( DeepseekV41Indexer, ) from sglang.srt.layers.attention.dsv4.indexer import C4Indexer +from sglang.srt.layers.cp.base import is_zigzag from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx from sglang.srt.layers.cp.utils import ( cp_gather_full_sequence_states, @@ -121,6 +122,11 @@ from sglang.srt.layers.quantization.mxfp8_input import Mxfp8SwizzledInput from sglang.srt.layers.rotary_embedding import get_rope_wrapper from sglang.srt.layers.utils import PPMissingLayer, get_layer_id from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding +from sglang.srt.managers.mm_owner_embedding import ( + MmOwnerSession, + has_owner_span_work, + select_owner_group, +) from sglang.srt.managers.mm_utils import ( MultiModalityDataPaddingPatternMultimodalTokens, embed_mm_inputs, @@ -4893,13 +4899,21 @@ class DeepseekV4ForCausalLM(nn.Module): self.determine_num_fused_shared_experts() self.vision = None if config.model_type == "deepseek_v41" and config.vision_n_layers > 0: - if ( - get_parallel().attn_cp_size != 1 - or get_parallel().pp_group.world_size != 1 - or not get_moe_a2a_backend().is_none() + if get_parallel().pp_group.world_size != 1 or not ( + get_moe_a2a_backend().is_none() + or (_is_cuda and get_moe_a2a_backend().is_megamoe()) ): raise ValueError( - "V4.1 vision currently supports TP/EP/DP without CP, PP or MoE A2A" + "V4.1 vision supports TP/EP/DP without PP; " + "MoE A2A requires CUDA MegaMoE" + ) + if get_parallel().attn_cp_size != 1 and ( + _is_npu or is_zigzag() or not get_moe_a2a_backend().is_none() + ): + raise ValueError( + "V4.1 vision context parallelism requires the CUDA interleave " + "strategy without MoE A2A; NPU, zigzag CP and MegaMoE are not " + "supported yet" ) args = SimpleNamespace(**vars(config), dim=config.hidden_size) @@ -4908,6 +4922,11 @@ class DeepseekV4ForCausalLM(nn.Module): self.image_start = nn.Parameter(torch.empty(config.hidden_size)) self.image_end = nn.Parameter(torch.empty(config.hidden_size)) self.image_newline = nn.Parameter(torch.empty(config.hidden_size)) + self.mm_owner_group = ( + select_owner_group(get_parallel()) + if self.vision is not None and _is_cuda + else None + ) self.model = DeepseekV4Model( config, quant_config, prefix=add_prefix("model", prefix) ) @@ -5042,7 +5061,42 @@ class DeepseekV4ForCausalLM(nn.Module): spans.append(span) return spans - def _prepare_mm_embeddings(self, input_ids, forward_batch): + def _image_span_signature(self, item, span_len: int): + h, w = int(item.n_vit_h), int(item.n_vit_w) + r = self.config.vision_downsample_ratio + lh, lw = (h + r - 1) // r, (w + r - 1) // r + expected = 2 + lh * (lw + 1) + if expected != span_len: + raise ValueError( + f"image grid {(h, w)} yields {expected} span tokens, " + f"placeholder has {span_len}" + ) + plan = item.model_specific_data.get(GPU_PLAN_KEY) + feature = item.feature + return ( + h, + w, + tuple(feature.shape) if isinstance(feature, torch.Tensor) else None, + None if plan is None else tuple(sorted(plan.items())), + ) + + def _mm_owner_session(self, forward_batch) -> Optional[MmOwnerSession]: + if self.mm_owner_group is None or not has_owner_span_work( + forward_batch.mm_inputs, + forward_batch.extend_prefix_lens_cpu, + forward_batch.extend_seq_lens_cpu, + ): + return None + return MmOwnerSession( + group=self.mm_owner_group, + device=self.image_start.device, + dtype=self.image_start.dtype, + width=self.config.hidden_size, + rids=list(forward_batch.rids or ()), + signature=self._image_span_signature, + ) + + def _prepare_mm_embeddings(self, input_ids, forward_batch, mm_owner): # Keep scheduler hash IDs intact: the shared embedder clamps its input in place. input_embeds, _ = embed_mm_inputs( mm_inputs_list=[ @@ -5054,6 +5108,7 @@ class DeepseekV4ForCausalLM(nn.Module): input_ids=input_ids.clone(), input_embedding=self.get_input_embeddings(), multimodal_model=self, + mm_owner=mm_owner, ) forward_batch.mm_input_embeds = input_embeds return input_embeds @@ -5061,6 +5116,39 @@ class DeepseekV4ForCausalLM(nn.Module): def get_input_embeddings(self) -> nn.Module: return self.model.get_input_embeddings() + def prepare_model_inputs( + self, + input_ids: torch.Tensor, + forward_batch: ForwardBatch, + input_embeds: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + if self.vision is None: + return input_ids, input_embeds + has_images = ( + not forward_batch.forward_mode.is_decode() + and not forward_batch.forward_mode.is_target_verify() + and forward_batch.mm_inputs is not None + and any(x is not None for x in forward_batch.mm_inputs) + ) + if has_images and input_embeds is not None: + raise ValueError("Cannot combine input_embeds and image inputs") + mm_owner = self._mm_owner_session(forward_batch) if has_images else None + with mm_owner.fence() if mm_owner is not None else nullcontext(): + if has_images: + input_embeds = self._prepare_mm_embeddings( + input_ids, forward_batch, mm_owner + ) + if not ( + forward_batch.forward_mode.is_decode_or_idle() + or forward_batch.forward_mode.is_target_verify() + ): + # Decode/verify IDs are already vocabulary IDs; remap prompt image + # hashes for Engram and routing. + input_ids = input_ids.masked_fill( + input_ids >= MM_PAD_SHIFT_VALUE, self.config.image_token_id + ) + return input_ids, input_embeds + def set_dspark_layers_to_capture(self, layer_ids: List[int]) -> None: if not self.pp_group.is_last_rank: return @@ -5114,25 +5202,9 @@ class DeepseekV4ForCausalLM(nn.Module): input_embeds: Optional[torch.Tensor] = None, pp_proxy_tensors: Optional[PPProxyTensors] = None, ) -> torch.Tensor: - if ( - self.vision is not None - and not forward_batch.forward_mode.is_decode() - and not forward_batch.forward_mode.is_target_verify() - and forward_batch.mm_inputs is not None - and any(x is not None for x in forward_batch.mm_inputs) - ): - if input_embeds is not None: - raise ValueError("Cannot combine input_embeds and image inputs") - input_embeds = self._prepare_mm_embeddings(input_ids, forward_batch) - if self.vision is not None and not ( - forward_batch.forward_mode.is_decode_or_idle() - or forward_batch.forward_mode.is_target_verify() - ): - # Decode/verify IDs are already vocabulary IDs; remap prompt image - # hashes for Engram and routing. - input_ids = input_ids.masked_fill( - input_ids >= MM_PAD_SHIFT_VALUE, self.config.image_token_id - ) + input_ids, input_embeds = self.prepare_model_inputs( + input_ids=input_ids, forward_batch=forward_batch, input_embeds=input_embeds + ) with get_attn_tp_context().maybe_input_scattered(forward_batch): hidden_states = self.model.forward( diff --git a/python/sglang/srt/models/deepseek_v4_nextn.py b/python/sglang/srt/models/deepseek_v4_nextn.py index 6694abb603..d35f884545 100644 --- a/python/sglang/srt/models/deepseek_v4_nextn.py +++ b/python/sglang/srt/models/deepseek_v4_nextn.py @@ -222,6 +222,7 @@ class DeepseekV4ForCausalLMNextN(DeepseekV4ForCausalLM): self.quant_config = quant_config self.wo_a_fp8 = wo_a_fp8_gemm_enabled(quant_config) self.determine_num_fused_shared_experts() + self.vision = None self.model = DeepseekV4ModelNextN( config, quant_config, prefix=add_prefix("model", prefix) diff --git a/test/registered/unit/managers/test_embed_overrides.py b/test/registered/unit/managers/test_embed_overrides.py index e00a1d1fc4..2c7eccaced 100644 --- a/test/registered/unit/managers/test_embed_overrides.py +++ b/test/registered/unit/managers/test_embed_overrides.py @@ -9,6 +9,7 @@ Covers: """ import unittest +from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock import torch @@ -17,10 +18,16 @@ from sglang.srt.constants import MIS_DELIMITER_TOKEN_ID from sglang.srt.entrypoints.openai.utils import convert_embeds_to_tensors from sglang.srt.managers.embed_types import PositionalEmbeds from sglang.srt.managers.io_struct import EmbeddingReqInput, GenerateReqInput +from sglang.srt.managers.schedule_batch import ( + Modality, + MultimodalDataItem, + MultimodalInputs, +) from sglang.srt.managers.tokenizer_manager import TokenizerManager from sglang.srt.managers.tokenizer_manager_score_mixin import ( TokenizerManagerScoreMixin, ) +from sglang.srt.model_executor.model_runner import ModelRunner from sglang.srt.runtime_context import publish, reset_context from sglang.srt.server_args import ServerArgs from sglang.test.ci.ci_register import register_cpu_ci @@ -642,5 +649,80 @@ class TestScoreRequestValidation(CustomTestCase): ) +class TestEmbedOverridesRejectMultimodal(CustomTestCase): + def setUp(self): + reset_context() + self.addCleanup(reset_context) + publish(ServerArgs(model_path="dummy"), role="tokenizer") + self.manager = TokenizerManager.__new__(TokenizerManager) + self.manager.context_len = 128 + self.manager.num_reserved_tokens = 0 + self.manager.allow_auto_truncate = False + self.manager.validate_total_tokens = False + self.manager.model_config = SimpleNamespace(joint_head_config=None) + + def test_requests_with_overrides_and_images_are_rejected(self): + """EmbeddingReqInput resolves embed_overrides only after validation, so + the unresolved form must be caught at admission too.""" + overrides = PositionalEmbeds(embeds=[_vec()], positions=[1]) + for build in ( + lambda **mm: GenerateReqInput( + input_ids=[10, 50, 20], + sampling_params={}, + positional_embed_overrides=overrides, + **mm, + ), + lambda **mm: EmbeddingReqInput( + input_ids=[10, 50, 20], + sampling_params={}, + embed_override_token_id=50, + embed_overrides=[_vec()], + **mm, + ), + ): + req = build(image_data=["image.png"]) + self.manager.is_generation = isinstance(req, GenerateReqInput) + with self.assertRaisesRegex(ValueError, "overrides cannot be combined"): + self.manager._validate_one_request(req, req.input_ids) + text_only = build() + self.manager._validate_one_request(text_only, text_only.input_ids) + + def test_mixed_extend_batch_is_rejected_before_embedding_lookup(self): + embed_layer = MagicMock( + side_effect=AssertionError("embedding lookup must not run") + ) + runner = SimpleNamespace( + _pp_kwargs=lambda pp_proxy_tensors: {}, + model=SimpleNamespace(get_input_embeddings=lambda: embed_layer), + is_generation=True, + ) + image = MultimodalDataItem( + modality=Modality.IMAGE, feature=torch.zeros(1), offsets=[(0, 1)] + ) + image.set_hash(1234) + forward_batch = SimpleNamespace( + input_embeds=None, + input_ids=torch.tensor([1, 2, image.pad_value, image.pad_value]), + replace_embeds=torch.full((1, HIDDEN_DIM), 5.0), + replace_positions=torch.tensor([0]), + mm_inputs=[None, MultimodalInputs(mm_items=[image])], + extend_prefix_lens_cpu=[0, 0], + extend_seq_lens_cpu=[2, 2], + ) + + with self.assertRaisesRegex(ValueError, "cannot share an extend batch"): + ModelRunner._extend_forward_kwargs(runner, forward_batch, None) + embed_layer.assert_not_called() + + # A decoding image request in a mixed chunk has no placeholder rows here. + forward_batch.input_ids = torch.tensor([1, 2, 3]) + forward_batch.extend_prefix_lens_cpu = [0, 5] + forward_batch.extend_seq_lens_cpu = [2, 1] + embed_layer.side_effect = None + embed_layer.return_value = torch.zeros(3, HIDDEN_DIM) + kwargs = ModelRunner._extend_forward_kwargs(runner, forward_batch, None) + self.assertTrue(torch.equal(kwargs["input_embeds"][0], _vec(5.0))) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/managers/test_mm_owner_embedding.py b/test/registered/unit/managers/test_mm_owner_embedding.py new file mode 100644 index 0000000000..084ebe30f3 --- /dev/null +++ b/test/registered/unit/managers/test_mm_owner_embedding.py @@ -0,0 +1,693 @@ +"""Owner-encoded image spans: real gloo groups, production coordinator and entrypoints.""" + +import json +import sys +from contextlib import ExitStack, contextmanager, nullcontext +from datetime import timedelta +from pathlib import Path +from tempfile import TemporaryDirectory +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +import torch +import torch.distributed as dist +import torch.multiprocessing +from torch import nn + +from sglang.srt.distributed import parallel_state +from sglang.srt.layers.cp.base import init_cp_strategy +from sglang.srt.layers.cp.utils import prepare_cp_forward +from sglang.srt.managers import mm_owner_embedding, mm_schedule +from sglang.srt.managers.mm_owner_embedding import ( + MmOwnerProtocolError, + select_owner_group, +) +from sglang.srt.managers.schedule_batch import ( + Modality, + MultimodalDataItem, + MultimodalInputs, +) +from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.srt.model_executor.runner.eager_runner import EagerRunner +from sglang.srt.models.deepseek_v4 import DeepseekV4ForCausalLM +from sglang.srt.runtime_context import get_parallel +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=60, suite="base-a-test-cpu") + +HIDDEN = 8 +VOCAB = 64 +IMAGE_TOKEN_ID = 7 +CACHE_TAG = 50 +DOWNSAMPLE = 2 +GLOO_TIMEOUT = timedelta(seconds=90) + + +def _span(item_hash: int, rows: int, tag: int) -> torch.Tensor: + """Distinguishable full span: the producing rank is visible in the values.""" + base = float(item_hash * 1000 + tag * 100) + return torch.arange(rows * HIDDEN, dtype=torch.float32).view(rows, HIDDEN) + base + + +def _grid(rows: int): + """A ViT grid whose downsampled span has exactly ``rows`` tokens.""" + return (0, 0) if rows == 2 else (DOWNSAMPLE, DOWNSAMPLE * (rows - 3)) + + +def _image(item_hash: int, start: int, rows: int) -> MultimodalDataItem: + h, w = _grid(rows) + item = MultimodalDataItem( + modality=Modality.IMAGE, + feature=torch.zeros(1), + offsets=[(start, start + rows - 1)], + model_specific_data={"n_vit_h": h, "n_vit_w": w}, + ) + item.set_hash(item_hash) + return item + + +def _request(parts, prefix_len: int = 0, extend_len=None, rid: str = "rid"): + """``parts`` mixes token ids with ``(hash, rows)`` images.""" + ids, items = [], [] + for part in parts: + if isinstance(part, tuple): + item = _image(part[0], len(ids), part[1]) + items.append(item) + ids.extend([item.pad_value] * part[1]) + else: + ids.append(part) + if extend_len is None: + extend_len = len(ids) - prefix_len + return SimpleNamespace( + ids=ids, items=items, prefix_len=prefix_len, extend_len=extend_len, rid=rid + ) + + +def _chunk_ids(req): + return req.ids[req.prefix_len : req.prefix_len + req.extend_len] + + +def _batch(requests, mode=ForwardMode.EXTEND): + return SimpleNamespace( + forward_mode=mode, + mm_inputs=[ + MultimodalInputs(mm_items=req.items, im_token_id=IMAGE_TOKEN_ID) + if req.items + else None + for req in requests + ], + extend_prefix_lens_cpu=[req.prefix_len for req in requests], + extend_seq_lens_cpu=[req.extend_len for req in requests], + seq_lens_cpu=[req.prefix_len + req.extend_len for req in requests], + input_ids=torch.tensor( + [tok for req in requests for tok in _chunk_ids(req)], dtype=torch.long + ), + positions=torch.cat( + [ + torch.arange(req.prefix_len, req.prefix_len + req.extend_len) + for req in requests + ] + ), + mm_input_embeds=None, + attn_cp_metadata=None, + global_num_tokens_cpu=None, + out_cache_loc=None, + input_ids_global=torch.zeros(1, dtype=torch.long), + rids=[req.rid for req in requests], + ) + + +def _expected_embeds(embed: nn.Embedding, requests, tags): + """``tags`` maps image hash to the tag of the span every rank must end up with.""" + rows = [] + with torch.no_grad(): + for req in requests: + ids = torch.tensor(_chunk_ids(req), dtype=torch.long) + chunk = nn.functional.embedding(ids.clamp(max=VOCAB - 1), embed.weight) + chunk_end = req.prefix_len + req.extend_len + for item in req.items: + start, end = item.offsets[0] + lo = max(start, req.prefix_len) - start + hi = min(end + 1, chunk_end) - start + if lo >= hi: + continue + span = _span(item.hash, end - start + 1, tags[item.hash]) + dest = max(start, req.prefix_len) - req.prefix_len + chunk[dest : dest + hi - lo] = span[lo:hi] + rows.append(chunk) + return torch.cat(rows) + + +def _seed_cache(item_hash: int, rows: int, tag: int) -> None: + mm_schedule.embedding_cache.set( + item_hash, mm_schedule.EmbeddingResult(embedding=_span(item_hash, rows, tag)) + ) + + +class _RecordingBody: + def __init__(self, embed): + self.embed = embed + self.calls = [] + + def get_input_embeddings(self): + return self.embed + + def __call__(self, input_ids, positions, forward_batch, input_embeds=None): + self.calls.append(input_ids) + return input_embeds, input_embeds + + +class _ReducingEmbedding(nn.Embedding): + """Stands in for the TP vocab embedding: a real all-reduce on the device group.""" + + def __init__(self, coordinator): + torch.manual_seed(0) + super().__init__(VOCAB, HIDDEN) + self.coordinator = coordinator + self.calls = 0 + + def forward(self, input_ids): + self.calls += 1 + if self.coordinator is not None: + dist.all_reduce(torch.zeros(1), group=self.coordinator.device_group) + return super().forward(input_ids) + + +class _VisionStub(DeepseekV4ForCausalLM): + def __init__(self, embed, owner_group, tag: int): + nn.Module.__init__(self) + self.config = SimpleNamespace( + image_token_id=IMAGE_TOKEN_ID, + hidden_size=HIDDEN, + vision_downsample_ratio=DOWNSAMPLE, + ) + self.vision = object() + self.tp_size = 1 + self.mm_owner_group = owner_group + self.image_start = nn.Parameter(torch.zeros(HIDDEN)) + self.model = _RecordingBody(embed) + self.pp_group = SimpleNamespace(is_last_rank=True) + self.lm_head = object() + self.capture_aux_hidden_states = False + self.logits_calls = [] + self.tag = tag + self.encoded = [] + + def get_image_feature(self, items): + self.encoded.extend(item.hash for item in items) + return [ + _span(item.hash, item.offsets[0][1] - item.offsets[0][0] + 1, self.tag) + for item in items + ] + + def logits_processor( + self, + input_ids, + hidden_states, + lm_head, + logits_metadata, + aux_hidden_states=None, + hidden_states_before_norm=None, + ): + self.logits_calls.append(hidden_states) + return object() + + def prepare(self, forward_batch): + with torch.no_grad(): + return self.prepare_model_inputs( + input_ids=forward_batch.input_ids, + forward_batch=forward_batch, + input_embeds=None, + ) + + +class _TracedGroup: + def __init__(self, inner): + self.inner = inner + self.trace = [] + + @property + def world_size(self): + return self.inner.world_size + + @property + def rank_in_group(self): + return self.inner.rank_in_group + + @property + def ranks(self): + return self.inner.ranks + + def all_gather_object(self, obj): + self.trace.append(["all_gather_object", type(obj).__name__, self.world_size]) + return self.inner.all_gather_object(obj) + + def broadcast_object(self, obj=None, src=0): + self.trace.append(["broadcast_object", src, self.world_size]) + return self.inner.broadcast_object(obj, src=src) + + def broadcast(self, tensor, src=0): + self.trace.append(["broadcast", src, list(tensor.shape), self.world_size]) + return self.inner.broadcast(tensor, src=src) + + +class _ForbiddenGroup: + world_size = 4 + rank_in_group = 0 + ranks = [0, 1, 2, 3] + + def __getattr__(self, name): + raise AssertionError(f"collective {name!r} reached on a fast path") + + +def _coordinator(group_ranks, rank): + return parallel_state.GroupCoordinator( + group_ranks=group_ranks, + local_rank=rank, + torch_distributed_backend="gloo", + use_pynccl=False, + use_mscclpp=False, + use_custom_allreduce=False, + use_torch_symm_mem_all_reduce=False, + use_hpu_communicator=False, + use_xpu_communicator=False, + use_npu_communicator=False, + use_message_queue_broadcaster=False, + group_name="mm_owner_test", + gloo_timeout=GLOO_TIMEOUT, + ) + + +def _init_rank(rank: int, world_size: int, init_file: str) -> None: + torch.set_num_threads(1) + dist.init_process_group( + backend="gloo", + init_method=Path(init_file).as_uri(), + rank=rank, + world_size=world_size, + timeout=GLOO_TIMEOUT, + ) + parallel_state._MODEL_PARALLEL_GROUP_TIMEOUT = GLOO_TIMEOUT + + +def _run_ranks(world_size: int, target): + with TemporaryDirectory() as directory: + init_file = str(Path(directory) / "gloo-init") + torch.multiprocessing.spawn( + _rank_main, + args=(world_size, init_file, directory, target), + nprocs=world_size, + ) + return [ + json.loads((Path(directory) / f"rank{rank}.json").read_text()) + for rank in range(world_size) + ] + + +def _rank_main(rank, world_size, init_file, directory, target): + _init_rank(rank, world_size, init_file) + try: + result = target(rank, world_size) + (Path(directory) / f"rank{rank}.json").write_text(json.dumps(result)) + finally: + dist.destroy_process_group() + + +@torch.no_grad() +def _run_cp_extend(model, forward_batch, coordinator): + def gather(output, input_tensor): + dist.all_gather_into_tensor( + output, input_tensor, group=coordinator.device_group + ) + + runner = EagerRunner.__new__(EagerRunner) + runner.model_runner = SimpleNamespace(model=model) + with ( + patch("torch.cuda.current_stream", return_value=None), + patch( + "sglang.srt.layers.cp.interleave.attn_cp_all_gather_into_tensor", + side_effect=gather, + ), + patch( + "sglang.srt.layers.cp.interleave.is_allocation_symmetric", + return_value=False, + ), + patch( + "sglang.srt.layers.cp.interleave.use_symmetric_memory", + return_value=torch.no_grad(), + ), + ): + prepare_cp_forward(forward_batch) + runner._execute_extend_cp(forward_batch, {}) + + +@contextmanager +def _interleave_cp(cp_size, **parallel): + init_cp_strategy(enable_prefill_cp=True, cp_size=cp_size, cp_strategy="interleave") + try: + with get_parallel().override( + attn_cp_size=cp_size, enable_cp_tp_group_sharing=False, **parallel + ): + yield + finally: + init_cp_strategy(enable_prefill_cp=False, cp_size=1, cp_strategy="interleave") + + +MANIFEST = ["all_gather_object", "RankManifest", 4] +PLAN = ["broadcast_object", 0, 4] +STATUS = ["all_gather_object", "RankStatus", 4] + + +def _bcast(src, rows, size=4): + return ["broadcast", src, [rows, HIDDEN], size] + + +def _cold(*payloads): + return [MANIFEST, PLAN, STATUS, *payloads, STATUS, STATUS] + + +# Global rank 0 sits outside the owner group, so a group-local source index that +# leaks out as a global rank is caught. +A, B, C, D, E = 400, 401, 402, 403, 404 # owner = hash % 4 -> group ranks 0,1,2,3,0 +ROWS = {A: 3, B: 4, C: 2, D: 3, E: 6} + + +def _owner_lifetime_program(rank, world_size): + group = _coordinator([[0], [1, 2, 3, 4]], rank) + if rank == 0: + return {"trace": [], "encoded": [], "outside": True} + traced = _TracedGroup(group) + local = group.rank_in_group + owner_tag = {key: 1 + key % 4 for key in ROWS} + mm_schedule.init_mm_embedding_cache(1 << 20) + embed = _ReducingEmbedding(group) + model = _VisionStub(embed, traced, tag=rank) + + # A valid only on its owner, B valid only on a non-owner, C valid + # everywhere, D stale on its owner. + if local == 0: + _seed_cache(A, ROWS[A], CACHE_TAG + rank) + if local == 2: + _seed_cache(B, ROWS[B], CACHE_TAG + rank) + _seed_cache(C, ROWS[C], CACHE_TAG) + if local == 3: + _seed_cache(D, ROWS[D] + 2, CACHE_TAG + rank) + forwards = [ + ( + [ + _request([10, 11, (A, ROWS[A]), 12, (B, ROWS[B]), 13], rid="r1"), + _request([20, (C, ROWS[C]), 21, (D, ROWS[D]), 22], rid="r2"), + ], + {A: CACHE_TAG + 1, B: owner_tag[B], C: CACHE_TAG, D: owner_tag[D]}, + ), + ( + [ + _request([10, 11, (A, ROWS[A]), (B, ROWS[B])], rid="r3"), + _request([20, (A, ROWS[A]), (C, ROWS[C]), (D, ROWS[D])], rid="r4"), + ], + {A: CACHE_TAG + 1, B: owner_tag[B], C: owner_tag[C], D: owner_tag[D]}, + ), + ] + [ + ( + [_request([30, 31, (E, ROWS[E]), 32], prefix, extend, rid="r5")], + {E: owner_tag[E]}, + ) + for prefix, extend in ((0, 5), (5, 4)) + ] + encoded, traces = [], [] + for index, (requests, tags) in enumerate(forwards): + if index == 1: + # Local 2 loses every entry and admits nothing new; B's owner evicts B. + if local == 2: + mm_schedule.init_mm_embedding_cache(0) + if local == 1: + mm_schedule.embedding_cache.free(B, None) + model.encoded.clear() + traced.trace.clear() + _, embeds = model.prepare(_batch(requests)) + assert torch.equal(embeds, _expected_embeds(embed, requests, tags)) + encoded.append(list(model.encoded)) + traces.append(list(traced.trace)) + return {"trace": traces, "encoded": encoded, "outside": False} + + +def test_owner_actions_and_cache_lifetime_across_asymmetric_ranks(): + results = _run_ranks(5, _owner_lifetime_program) + members = range(1, 5) + assert all(results[rank]["trace"] == results[1]["trace"] for rank in members) + encoded = [results[rank]["encoded"] for rank in members] + trace = results[1]["trace"] + + # Forward 1: A cache-broadcast, B and D owner-encoded, C a local hit. + assert [e[0] for e in encoded] == [[], [B], [], [D]] + assert trace[0] == _cold(_bcast(0, ROWS[A]), _bcast(1, ROWS[B]), _bcast(3, ROWS[D])) + + # Forward 2: evicted owners of B and C re-encode once; A recurs but moves once. + assert [e[1] for e in encoded] == [[], [B], [C], []] + assert trace[1] == _cold(*(_bcast(key % 4, ROWS[key]) for key in (A, B, C, D))) + + # Forwards 3 and 4: the chunk-crossing span is encoded fully once. + assert [e[2] for e in encoded] == [[E], [], [], []] + assert [e[3] for e in encoded] == [[], [], [], []] + assert trace[2] == trace[3] == _cold(_bcast(0, ROWS[E])) + assert results[0]["outside"] + + +def _cp8_program(rank, world_size): + tp8 = _coordinator([list(range(8))], rank) + singles = _coordinator([[r] for r in range(8)], rank) + attn_cp = _TracedGroup(tp8) + with _interleave_cp( + 8, + tp_size=8, + attn_dp_size=1, + attn_cp_rank=rank, + tp_group=_TracedGroup(tp8), + attn_tp_group=_TracedGroup(singles), + attn_cp_group=attn_cp, + ): + selected = select_owner_group(get_parallel()) + assert selected is attn_cp + mm_schedule.init_mm_embedding_cache(1 << 20) + torch.manual_seed(0) + embed = nn.Embedding(VOCAB, HIDDEN) + model = _VisionStub(embed, selected, tag=rank) + w = 803 # owner 3; rows land on ranks 6,7,0,1,2,3 so ranks 4,5 hold no image row + requests = [ + _request([10, 11, 12, 13, 14], rid="p1"), + _request([20, (w, 6), 21, 22], rid="p2"), + ] + _run_cp_extend(model, _batch(requests), tp8) + (logits,) = model.logits_calls + assert torch.equal(logits, _expected_embeds(embed, requests, {w: 3})) + (body,) = model.model.calls + return { + "encoded": list(model.encoded), + "trace": list(attn_cp.trace), + "image_rows": int((body == IMAGE_TOKEN_ID).sum()), + } + + +def test_tp8_cp8_encodes_once_and_merges_before_shard(): + results = _run_ranks(8, _cp8_program) + assert [r["encoded"] for r in results] == [[], [], [], [803], [], [], [], []] + assert all(r["trace"] == results[0]["trace"] for r in results) + assert [op for op in results[0]["trace"] if op[0] == "broadcast"] == [ + _bcast(3, 6, 8) + ] + assert [r["image_rows"] for r in results] == [1, 1, 1, 1, 0, 0, 1, 1] + + +def _raise_on(on_rank, target, name, exc_type=RuntimeError, after=0): + def inject(rank, batch): + if rank != on_rank: + return nullcontext() + original = getattr(target, name) + calls = [] + + def fail(*args, **kwargs): + calls.append(None) + if len(calls) > after: + raise exc_type(f"injected {name}") + return original(*args, **kwargs) + + return patch.object(target, name, fail) + + return inject + + +def _remap_oom_on(on_rank): + def inject(rank, batch): + if rank != on_rank: + return nullcontext() + original = torch.Tensor.masked_fill + + def masked_fill(self, mask, value): + if self is batch.input_ids: + raise torch.OutOfMemoryError("injected remap") + return original(self, mask, value) + + return patch.object(torch.Tensor, "masked_fill", masked_fill) + + return inject + + +def _failure(requests, trace, errors, inject=None, vocab_calls=0, cp=False): + return SimpleNamespace( + requests=requests, + trace=trace, + errors=errors, + inject=inject or (lambda rank, batch: nullcontext()), + vocab_calls=vocab_calls, + cp=cp, + ) + + +F, G = 300, 301 # owners 0 and 1 in a four-rank group +AT_PLAN = [MANIFEST, PLAN] +AT_FINALIZE_F = _cold(_bcast(0, 3)) + +FAILURES = { + "prepare": _failure( + [[10, (G, 3), 11]], + AT_PLAN, + ["during prepare on group rank 1", "OutOfMemoryError: injected as_tensor"], + _raise_on(1, torch, "as_tensor", torch.OutOfMemoryError), + ), + "remap": _failure( + [[10, (F, 3), 11]], + AT_FINALIZE_F, + ["during finalize on group rank 1", "injected remap"], + _remap_oom_on(1), + vocab_calls=1, + ), + "final_sync": _failure( + [[10, (F, 3), 11]], + AT_FINALIZE_F, + ["during finalize on group rank 1", "injected _synchronize"], + _raise_on(1, mm_owner_embedding, "_synchronize", after=2), + vocab_calls=1, + ), + "leader_plan": _failure( + [[10, (G, 3), 11]], + AT_PLAN, + ["during plan on group rank 0", "MemoryError: injected _make_plan"], + _raise_on(0, mm_owner_embedding, "_make_plan", MemoryError), + ), + "remap_cp": _failure( + [[10, (F, 3), 11]], + AT_FINALIZE_F, + ["during finalize on group rank 1", "injected remap"], + _remap_oom_on(1), + vocab_calls=1, + cp=True, + ), +} + + +def _failure_program(rank, world_size): + group = _coordinator([[0, 1, 2, 3]], rank) + out = {} + for name, case in FAILURES.items(): + mm_schedule.init_mm_embedding_cache(1 << 20) + traced = _TracedGroup(group) + embed = _ReducingEmbedding(None if case.cp else group) + model = _VisionStub(embed, traced, tag=rank) + batch = _batch([_request(parts, rid=name) for parts in case.requests]) + with ExitStack() as stack, pytest.raises(MmOwnerProtocolError) as raised: + stack.enter_context(case.inject(rank, batch)) + if case.cp: + stack.enter_context( + _interleave_cp(4, attn_cp_rank=rank, attn_cp_group=traced) + ) + _run_cp_extend(model, batch, group) + else: + model.prepare(batch) + out[name] = { + "error": str(raised.value), + "trace": traced.trace, + "vocab_calls": embed.calls, + "body_calls": len(model.model.calls), + } + return out + + +def test_failures_agree_before_payload_text_embedding_or_body(): + results = _run_ranks(4, _failure_program) + for name, case in FAILURES.items(): + runs = [r[name] for r in results] + error = runs[0]["error"] + assert all(run["error"] == error for run in runs), (name, runs) + assert all(run["trace"] == case.trace for run in runs), (name, runs) + for fragment in [*case.errors, f"rids=['{name}'"]: + assert fragment in error, (name, fragment, error) + assert [run["vocab_calls"] for run in runs] == [case.vocab_calls] * 4, name + assert [run["body_calls"] for run in runs] == [0] * 4, name + + +def test_text_decode_prefilled_future_and_precomputed_paths_pay_no_collective(): + mm_schedule.init_mm_embedding_cache(1 << 20) + torch.manual_seed(0) + embed = nn.Embedding(VOCAB, HIDDEN) + owner_model = _VisionStub(embed, _ForbiddenGroup(), tag=0) + legacy_model = _VisionStub(embed, None, tag=0) + + text_only = _batch([_request([10, 11], rid="t0"), _request([20, 21, 22], rid="t1")]) + assert owner_model.prepare(text_only)[1] is None + + with_image = [_request([10, (A, 3), 11, 12], rid="v0")] + for mode in (ForwardMode.DECODE, ForwardMode.TARGET_VERIFY): + assert owner_model.prepare(_batch(with_image, mode))[1] is None + + for request, chunk_ids in ( + (_request([10, (A, 3), 11, 12], prefix_len=5, rid="pf"), [12]), + (_request([10, 11, 12, 13, (A, 3)], extend_len=2, rid="fut"), [10, 11]), + ): + ids, embeds = owner_model.prepare(_batch([request])) + assert torch.equal(embeds, legacy_model.prepare(_batch([request]))[1]) + assert torch.equal(ids, torch.tensor(chunk_ids)) + + precomputed = _request([10, 11, 12, 13], rid="pc") + item = MultimodalDataItem( + modality=Modality.IMAGE, + precomputed_embeddings=torch.full((3, HIDDEN), 5.0), + offsets=[(1, 3)], + ) + item.set_hash(A) + precomputed.items = [item] + precomputed.ids[1:4] = [item.pad_value] * 3 + _, embeds = owner_model.prepare(_batch([precomputed])) + assert torch.equal(embeds[1:4], torch.full((3, HIDDEN), 5.0)) + + overlapping = [_request([10, (A, 3), 11, 12], prefix_len=2, rid="ov")] + with pytest.raises(AssertionError, match="collective"): + owner_model.prepare(_batch(overlapping)) + + +def _parallel(tp_size, attn_dp_size, attn_cp_size, attn_tp_size, attn_cp_group_size): + return SimpleNamespace( + tp_size=tp_size, + attn_dp_size=attn_dp_size, + attn_cp_size=attn_cp_size, + tp_group=SimpleNamespace(world_size=tp_size, name="tp"), + attn_tp_group=SimpleNamespace(world_size=attn_tp_size, name="attn_tp"), + attn_cp_group=SimpleNamespace(world_size=attn_cp_group_size, name="attn_cp"), + ) + + +def test_select_owner_group_follows_the_replication_domain(): + for layout, expected in ( + ((8, 1, 1, 8, 1), "attn_tp"), + ((8, 1, 8, 1, 8), "attn_cp"), + ((8, 2, 1, 4, 1), "attn_tp"), + ((1, 1, 1, 1, 1), None), + ((8, 2, 4, 1, 4), None), + ((8, 1, 4, 2, 4), None), + ): + group = select_owner_group(_parallel(*layout)) + assert (group and group.name) == expected, layout + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__])) diff --git a/test/registered/unit/managers/test_prefill_adder.py b/test/registered/unit/managers/test_prefill_adder.py index cf5d25fba1..084539e9fd 100644 --- a/test/registered/unit/managers/test_prefill_adder.py +++ b/test/registered/unit/managers/test_prefill_adder.py @@ -30,7 +30,13 @@ from sglang.srt.runtime_context import get_context from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler from sglang.srt.utils.common import Range from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.test_utils import CustomTestCase +from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel + +maybe_stub_sgl_kernel() + +import sglang.srt.managers.scheduler as scheduler_module +from sglang.srt.disaggregation.utils import DisaggregationMode +from sglang.srt.managers.scheduler import Scheduler register_cpu_ci(est_time=11, suite="base-a-test-cpu") @@ -115,7 +121,10 @@ class TestPrefillAdder(CustomTestCase): req.prefix_indices = [] req.full_untruncated_fill_ids = [] req.output_ids = [0] * output_len - req.sampling_params = SimpleNamespace(max_new_tokens=max_new_tokens) + req.sampling_params = SimpleNamespace( + max_new_tokens=max_new_tokens, ignore_eos=False + ) + req.token_indices_to_pool = None req.time_stats = SimpleNamespace(wait_queue_entry_time=wait_time) req.retracted_stain = False req.host_hit_length = 0 @@ -298,6 +307,116 @@ class TestPrefillAdder(CustomTestCase): ) self.assertEqual(adder.can_run_list, [first]) + def create_admission_scheduler(self, *, chunked_req) -> Scheduler: + allocator = self.create_token_allocator(available_size=4096) + allocator.page_size = 1 + self.mock_tree_cache.supports_mamba.return_value = False + self.mock_tree_cache.is_tree_cache.return_value = False + self.mock_tree_cache.supports_fast_match_prefix.return_value = False + self.mock_tree_cache.storage_prefetch_retries = None + self.mock_tree_cache.supports_prefix_sharing.return_value = False + scheduler = Scheduler.__new__(Scheduler) + scheduler.grammar_manager = SimpleNamespace(has_waiting_grammars=lambda: False) + scheduler.enable_priority_preemption = False + scheduler.enable_priority_scheduling = False + scheduler.is_hybrid_swa = False + scheduler.min_free_slots_delayer = None + scheduler.get_num_allocatable_reqs = lambda *args, **kwargs: 64 + scheduler.policy = SchedulePolicy( + policy="fcfs", + tree_cache=self.mock_tree_cache, + enable_hierarchical_cache=False, + enable_priority_scheduling=False, + schedule_low_priority_values_first=False, + ) + scheduler.processed_tokens_counter = 0 + scheduler.chunked_prefill_size = 16 + scheduler.dynamic_chunk_sizer = None + scheduler.tp_worker = SimpleNamespace( + model_runner=SimpleNamespace(attn_backend=object(), prefill_aware_swa=False) + ) + scheduler.page_size = 1 + scheduler.tree_cache = self.mock_tree_cache + scheduler.token_to_kv_pool_allocator = allocator + scheduler.new_token_ratio_tracker = SimpleNamespace(current=1.0) + scheduler.max_prefill_tokens = 16384 + scheduler.is_mixed_chunk = False + scheduler.priority_scheduling_preemption_threshold = 0 + scheduler.max_prefill_bs = 64 + scheduler.max_running_requests = 64 + scheduler.dllm_config = None + scheduler.enable_lora = False + scheduler.req_to_token_pool = SimpleNamespace() + scheduler.disaggregation_mode = DisaggregationMode.NULL + scheduler.enable_hicache_storage = False + scheduler.enable_lmcache = False + scheduler.enable_hierarchical_cache = False + scheduler.enable_unified_cache_external_linker = False + scheduler.truncation_align_size = None + scheduler.model_config = None + scheduler.enable_overlap = False + scheduler.spec_algorithm = None + scheduler.load_inquirer = MagicMock() + scheduler.chunked_req = chunked_req + scheduler.waiting_queue = [] + return scheduler + + def run_admission_pass(self, scheduler: Scheduler) -> list: + running_batch = self.create_running_batch() + running_batch.batch_is_full = False + with ( + patch.object(scheduler_module, "ScheduleBatch") as schedule_batch, + patch.object(scheduler_module, "PrefillStats"), + patch.object(scheduler_module, "set_time_batch"), + ): + new_batch, _ = scheduler._get_new_batch_prefill_raw(None, running_batch) + if new_batch is None: + return [] + admitted = list(schedule_batch.init_new.call_args.args[0]) + for req in admitted: + req.prefix_len = req.extend_end + return admitted + + def test_fcfs_admits_override_request_once_image_continuation_drains(self): + """An override request at the queue head must be admitted once the image + chunk ahead of it drains, even while more image requests keep arriving.""" + + def tagged(rid, length, *, multimodal=False, overrides=False): + req = self.create_shared_req(rid) + req.origin_input_ids = list(range(length)) + req.full_untruncated_fill_ids = list(range(length)) + req.multimodal_inputs = object() if multimodal else None + req.positional_embed_overrides = object() if overrides else None + req.beam_group = None + req.inflight_middle_chunks = 0 + req.kv = SimpleNamespace(holds_mamba=False) + return req + + continuation = tagged("image-continuation", 20, multimodal=True) + continuation.prefix_len = 16 + scheduler = self.create_admission_scheduler(chunked_req=continuation) + text = tagged("text", 2) + override = tagged("override", 4, overrides=True) + scheduler.waiting_queue = [text, override] + + admitted_at = None + for pass_index in range(6): + scheduler.waiting_queue.append( + tagged(f"image-{pass_index}", 16, multimodal=True) + ) + admitted = self.run_admission_pass(scheduler) + if pass_index == 0: + self.assertEqual(admitted, [continuation, text]) + self.assertFalse( + any(r.multimodal_inputs is not None for r in admitted) + and any(r.positional_embed_overrides is not None for r in admitted) + ) + if any(r is override for r in admitted): + admitted_at = pass_index + break + self.assertIsNotNone(admitted_at) + self.assertNotIn(override, scheduler.waiting_queue) + def test_shared_admission_rechecks_after_prefix_lock(self): adder = self.create_shared_adder() self.assertIsNotNone(adder.token_to_kv_pool_allocator.alloc(24)) diff --git a/test/registered/unit/models/test_deepseek_v41_vision_cp_inputs.py b/test/registered/unit/models/test_deepseek_v41_vision_cp_inputs.py new file mode 100644 index 0000000000..e8708e7d9f --- /dev/null +++ b/test/registered/unit/models/test_deepseek_v41_vision_cp_inputs.py @@ -0,0 +1,247 @@ +"""Vision inputs under prefill CP merge on the full extend layout before the shard.""" + +import unittest +from contextlib import contextmanager +from types import SimpleNamespace +from unittest.mock import patch + +import torch +from torch import nn + +from sglang.srt.layers.cp.base import init_cp_strategy +from sglang.srt.layers.cp.utils import prepare_cp_forward +from sglang.srt.managers import mm_schedule +from sglang.srt.managers.schedule_batch import ( + Modality, + MultimodalDataItem, + MultimodalInputs, +) +from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.srt.model_executor.runner.eager_runner import EagerRunner +from sglang.srt.models.deepseek_v4 import DeepseekV4ForCausalLM +from sglang.srt.runtime_context import get_parallel +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=15, suite="base-a-test-cpu") + +HIDDEN = 8 +VOCAB = 64 +IMAGE_TOKEN_ID = 7 +CP_SIZE = 4 +# (prefix_len, extend_len) per request. Request 1 carries one image whose span +# [2, 8] starts inside its prefix, so only span rows 1..6 land in this chunk. +CHUNKS = [(0, 7), (3, 9), (1, 5)] +IMAGE_OFFSET = (2, 8) +IMAGE_HASH = 12345 +NUM_TOKENS = sum(extend_len for _, extend_len in CHUNKS) +# 21 tokens over 4 ranks give logical [6, 5, 5, 5], padded to the CP alignment. +PHYSICAL_ROWS = 8 +IMAGE_ROWS = torch.arange(7, 13) +POSITIONS = torch.cat([torch.arange(p, p + n) for p, n in CHUNKS]) + + +def _image_span(item: MultimodalDataItem) -> torch.Tensor: + start, end = item.offsets[0] + rows = end - start + 1 + return torch.arange(rows * HIDDEN, dtype=torch.float32).view(rows, HIDDEN) + 100.0 + + +def _pad(x: torch.Tensor) -> torch.Tensor: + return torch.cat([x, x.new_zeros(PHYSICAL_ROWS - x.shape[0], *x.shape[1:])]) + + +class _RecordingBody: + def __init__(self, embed: nn.Embedding): + self.embed = embed + self.calls = [] + + def get_input_embeddings(self): + return self.embed + + def __call__(self, input_ids, positions, forward_batch, input_embeds=None): + self.calls.append( + SimpleNamespace( + input_ids=input_ids, + input_embeds=input_embeds, + input_ids_global=forward_batch.input_ids_global, + ) + ) + return input_embeds, input_embeds + + +class _VisionStub(DeepseekV4ForCausalLM): + def __init__(self, embed: nn.Embedding): + nn.Module.__init__(self) + self.config = SimpleNamespace(image_token_id=IMAGE_TOKEN_ID) + self.vision = object() + self.tp_size = 1 + self.mm_owner_group = None + self.model = _RecordingBody(embed) + self.pp_group = SimpleNamespace(is_last_rank=True) + self.lm_head = object() + self.capture_aux_hidden_states = False + self.logits_calls = [] + + def get_image_feature(self, items): + return [_image_span(item) for item in items] + + def logits_processor( + self, + input_ids, + hidden_states, + lm_head, + logits_metadata, + aux_hidden_states=None, + hidden_states_before_norm=None, + ): + self.logits_calls.append( + SimpleNamespace(input_ids=input_ids, hidden_states=hidden_states) + ) + return object() + + +def _build_batch(): + item = MultimodalDataItem( + modality=Modality.IMAGE, feature=torch.zeros(1), offsets=[IMAGE_OFFSET] + ) + item.set_hash(IMAGE_HASH) + ids = list(range(10, 17)) + ids += [item.pad_value] * len(IMAGE_ROWS) + [20, 21, 22] + ids += list(range(30, 35)) + forward_batch = SimpleNamespace( + forward_mode=ForwardMode.EXTEND, + mm_inputs=[ + MultimodalInputs(mm_items=[]), + MultimodalInputs(mm_items=[item], im_token_id=IMAGE_TOKEN_ID), + None, + ], + extend_prefix_lens_cpu=[prefix for prefix, _ in CHUNKS], + extend_seq_lens_cpu=[extend_len for _, extend_len in CHUNKS], + seq_lens_cpu=[prefix + extend_len for prefix, extend_len in CHUNKS], + input_ids=torch.tensor(ids, dtype=torch.long), + positions=POSITIONS.clone(), + mm_input_embeds=None, + attn_cp_metadata=None, + global_num_tokens_cpu=None, + out_cache_loc=None, + input_ids_global=torch.zeros(1, dtype=torch.long), + ) + return forward_batch, item + + +def _expected_embeds(embed, scheduler_ids, item): + with torch.no_grad(): + full = embed(scheduler_ids.clamp(max=VOCAB - 1)) + full[IMAGE_ROWS] = _image_span(item)[1:7] + return full + + +def _canonical(scheduler_ids): + canonical = scheduler_ids.clone() + canonical[IMAGE_ROWS] = IMAGE_TOKEN_ID + return canonical + + +class TestDeepseekV41VisionPrefillCPInputs(CustomTestCase): + def setUp(self): + mm_schedule.init_mm_embedding_cache(1 << 20) + init_cp_strategy( + enable_prefill_cp=True, cp_size=CP_SIZE, cp_strategy="interleave" + ) + torch.manual_seed(0) + self.embed = nn.Embedding(VOCAB, HIDDEN) + self.model = _VisionStub(self.embed) + + def tearDown(self): + init_cp_strategy(enable_prefill_cp=False, cp_size=1, cp_strategy="interleave") + + @contextmanager + def _cp_collectives(self, full: torch.Tensor, rank: int): + def all_gather(output, input_tensor): + # Peers contribute their expected shards; this rank's rows come from + # what the runner actually handed to the collective. + output.zero_() + for peer in range(CP_SIZE): + rows = full[peer::CP_SIZE] + output[peer * PHYSICAL_ROWS : peer * PHYSICAL_ROWS + rows.shape[0]] = ( + rows + ) + output[rank * PHYSICAL_ROWS : (rank + 1) * PHYSICAL_ROWS] = input_tensor + + with ( + patch("torch.cuda.current_stream", return_value=None), + patch( + "sglang.srt.layers.cp.interleave.attn_cp_all_gather_into_tensor", + side_effect=all_gather, + ), + patch( + "sglang.srt.layers.cp.interleave.is_allocation_symmetric", + return_value=False, + ), + patch( + "sglang.srt.layers.cp.interleave.use_symmetric_memory", + return_value=torch.no_grad(), + ), + ): + yield + + def test_cp_runner_merges_before_shard(self): + runner = EagerRunner.__new__(EagerRunner) + runner.model_runner = SimpleNamespace(model=self.model) + padded = torch.zeros(CP_SIZE * PHYSICAL_ROWS, dtype=torch.long) + + for rank in range(CP_SIZE): + forward_batch, item = _build_batch() + scheduler_ids = forward_batch.input_ids.clone() + canonical = _canonical(scheduler_ids) + full = _expected_embeds(self.embed, scheduler_ids, item) + padded[:NUM_TOKENS] = canonical + rank_major_ids = padded.view(-1, CP_SIZE).T.flatten() + self.model.model.calls.clear() + self.model.logits_calls.clear() + + with ( + get_parallel().override( + attn_cp_rank=rank, + attn_cp_size=CP_SIZE, + attn_cp_group=object(), + enable_cp_tp_group_sharing=False, + ), + self._cp_collectives(full, rank), + torch.no_grad(), + ): + prepare_cp_forward(forward_batch) + runner._execute_extend_cp(forward_batch, {}) + + with self.subTest(rank=rank): + metadata = forward_batch.attn_cp_metadata + self.assertEqual(metadata.per_rank_actual_token, [PHYSICAL_ROWS] * 4) + (body,) = self.model.model.calls + self.assertTrue( + torch.equal(body.input_ids, _pad(canonical[rank::CP_SIZE])) + ) + self.assertTrue( + torch.equal(body.input_embeds, _pad(full[rank::CP_SIZE])) + ) + self.assertTrue(torch.equal(body.input_ids_global, rank_major_ids)) + + (logits,) = self.model.logits_calls + self.assertTrue(torch.equal(logits.input_ids, canonical)) + self.assertTrue(torch.equal(logits.hidden_states, full)) + + self.assertTrue(torch.equal(forward_batch.mm_input_embeds, full)) + self.assertTrue(torch.equal(forward_batch.input_ids, scheduler_ids)) + + def test_external_embeddings_with_images_are_rejected(self): + forward_batch, _ = _build_batch() + with self.assertRaisesRegex(ValueError, "Cannot combine"): + self.model.prepare_model_inputs( + input_ids=forward_batch.input_ids, + forward_batch=forward_batch, + input_embeds=torch.zeros(NUM_TOKENS, HIDDEN), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index 67010b1306..38ffa51c6f 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -29,6 +29,7 @@ from sglang.srt.arg_groups.cuda_graph_hook import ( finalize_cuda_graph_prefill_max_context, handle_cuda_graph_config, ) +from sglang.srt.arg_groups.deepseek_v4_hook import validate_deepseek_v41_features from sglang.srt.arg_groups.hicache_hook import ( handle_hicache, handle_hicache_ratio_default, @@ -45,6 +46,7 @@ from sglang.srt.arg_groups.kv_cache_hook import ( ) from sglang.srt.arg_groups.mamba_hook import handle_mamba_backend from sglang.srt.arg_groups.memory_hook import handle_gpu_memory_settings +from sglang.srt.arg_groups.model_hook import handle_model_specific_adjustments from sglang.srt.arg_groups.model_path_hook import handle_load_format from sglang.srt.arg_groups.moe_hook import ( handle_a2a_moe, @@ -4035,6 +4037,82 @@ class TestDcpCommBackendDefault(CustomTestCase): ) +class TestDeepseekV41VisionPrefillCPArgs(CustomTestCase): + def _args( + self, + *, + vision_n_layers=2, + prefill_backend=Backend.DISABLED, + lock_prefill_backend=False, + **overrides, + ): + fields = dict( + model_path="dummy", + enable_prefill_cp=True, + cp_strategy="interleave", + tp_size=2, + ) + fields.update(overrides) + server_args = ServerArgs(**fields) + server_args._model_config = SimpleNamespace( + hf_config=SimpleNamespace( + architectures=["DeepseekV4ForCausalLM"], + model_type="deepseek_v41", + vision_n_layers=vision_n_layers, + ), + nvfp4_moe_meta=None, + is_fp4_experts=False, + decision_config=None, + ) + # The dummy path does not initialize phase configs. + server_args.cuda_graph_config = CudaGraphConfig( + decode=PhaseConfig(backend=Backend.FULL, max_bs=512), + prefill=PhaseConfig(backend=prefill_backend, max_bs=512), + ) + server_args._resolved_overrides = [] + server_args._cuda_graph_config_locked = ( + {(Phase.PREFILL, "backend")} if lock_prefill_backend else set() + ) + return server_args + + @override_platform(is_cuda=True, is_hip=False) + def test_guard_matrix_in_model_hook_order(self): + for overrides, error in ( + ( + dict(enable_encoder_swa_bounded_replay=True), + "encoder-swa-bounded-replay does not support context parallelism", + ), + (dict(cp_strategy="zigzag"), "requires --cp-strategy interleave"), + ( + dict(prefill_backend=Backend.BREAKABLE, lock_prefill_backend=True), + "runs eager prefill", + ), + ( + dict( + speculative_algorithm="DSPARK", + enable_decoder_swa_bounded_replay=True, + ), + "DSpark.*decoder-swa-bounded-replay", + ), + ): + args = self._args( + max_running_requests=4, chunked_prefill_size=128, **overrides + ) + with self.subTest(error=error), self.assertRaisesRegex(ValueError, error): + handle_model_specific_adjustments(args) + + def test_text_zigzag_and_default_prefill_graph_are_accepted(self): + validate_deepseek_v41_features( + self._args(cp_strategy="zigzag", vision_n_layers=0) + ) + args = self._args(prefill_backend=Backend.BREAKABLE) + validate_deepseek_v41_features(args) + self.assertEqual( + resolution_result(args, "cuda_graph_config").prefill.backend, + Backend.DISABLED, + ) + + class TestParserChoices(CustomTestCase): """The choices come from dependency-free name lists, but `cli/serve.py` loads plugins before parsing, so a plugin's parser must still be accepted."""