PR #40219: DeepSeek V4.1 vision CP and single-owner image encoding

Source: https://github.com/sgl-project/sglang/pull/40219 (applied via GitHub .diff, 3way)
Adaptations for v0.5.21 base:
- attn_dp_enabled_of(cfg) -> cfg.enable_dp_attention (#41818 not in base)
- dropped _validate_joint_schema_request call (#42721 Clef, not in base)
- moved Backend import before the TRT-LLM prefill-graph check
This commit is contained in:
2026-10-19 10:45:10 +08:00
parent da343ba5d1
commit fd86688428
17 changed files with 1956 additions and 44 deletions
@@ -172,15 +172,25 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
)
cfg = resolving_view(server_args)
if model_config_of(server_args).hf_config.model_type != "deepseek_v41":
hf_config = model_config_of(server_args).hf_config
if hf_config.model_type != "deepseek_v41":
if cfg.enable_encoder_swa_bounded_replay:
raise ValueError(
"--enable-encoder-swa-bounded-replay requires DeepSeek-V4.1"
)
return
if cfg.enable_encoder_swa_bounded_replay:
from sglang.srt.model_executor.cuda_graph_config import Backend
from sglang.srt.model_executor.cuda_graph_config import Backend
if hf_config.vision_n_layers > 0 and cfg.enable_prefill_cp:
_validate_deepseek_v41_vision_prefill_cp(server_args)
if (
cfg.dsv4_attn_backend == "trtllm"
and cfg.cuda_graph_config.prefill.backend != Backend.DISABLED
):
raise ValueError(
"DeepSeek-V4.1 TRT-LLM requires --cuda-graph-backend-prefill disabled"
)
if cfg.enable_encoder_swa_bounded_replay:
incompatible = (
("non-CUDA hardware", not get_platform().is_cuda),
(
@@ -188,7 +198,8 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
cfg.cuda_graph_config.prefill.backend != Backend.DISABLED,
),
("DP attention", cfg.enable_dp_attention),
("context parallelism", cfg.attn_cp_size > 1),
# Prefill CP declares attn_cp_size and DP attention only later.
("context parallelism", cfg.attn_cp_size > 1 or cfg.enable_prefill_cp),
("external cache linker", cfg.enable_unified_cache_external_linker),
("unified memory", cfg.enable_unified_memory),
("PD disaggregation", cfg.disaggregation_mode != "null"),
@@ -287,3 +298,37 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
"--enable-decoder-swa-bounded-replay cannot be combined with "
f"{feature} yet; disable one of them."
)
def _validate_deepseek_v41_vision_prefill_cp(server_args: ServerArgs) -> None:
cfg = resolving_view(server_args)
if cfg.cp_strategy != "interleave":
raise ValueError(
"DeepSeek-V4.1 vision with prefill CP requires --cp-strategy "
f"interleave; got {cfg.cp_strategy!r}."
)
if cfg.cuda_graph_config.prefill.backend != Backend.DISABLED:
locked = getattr(server_args, "_cuda_graph_config_locked", set())
if (Phase.PREFILL, "backend") in locked:
raise ValueError(
"DeepSeek-V4.1 vision with prefill CP runs eager prefill; remove "
"the explicit prefill CUDA graph backend."
)
declare_resolution(
server_args,
"validate_deepseek_v41_features",
cuda_graph_config=with_phase(
cfg.cuda_graph_config, Phase.PREFILL, backend=Backend.DISABLED
),
)
logger.warning(
"Disabling the prefill CUDA graph for DeepSeek-V4.1 vision with prefill CP."
)
if (
str(cfg.speculative_algorithm).upper() == "DSPARK"
and cfg.enable_decoder_swa_bounded_replay
):
raise ValueError(
"DeepSeek-V4.1 vision with prefill CP does not support DSpark together "
"with --enable-decoder-swa-bounded-replay yet."
)
@@ -0,0 +1,464 @@
"""One owner rank encodes each image span and broadcasts it to the ranks that
run the same prefill chunk; every agreement precedes the payload it guards."""
from __future__ import annotations
import logging
from contextlib import contextmanager
from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence, Tuple
import msgspec
import torch
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
disable_symmetric_memory_context,
restore_symmetric_memory_context,
)
from sglang.srt.mem_cache.multimodal_cache import EmbeddingResult, MultiModalStaticCache
logger = logging.getLogger(__name__)
SpanKey = Tuple[Optional[int], int]
SpanEncoder = Callable[[List[Any]], torch.Tensor | List[torch.Tensor]]
LOCAL_HIT = 0
OWNER_CACHE_BROADCAST = 1
OWNER_ENCODE_BROADCAST = 2
PHASE_PREPARE = "prepare"
PHASE_FEATURES = "features"
PHASE_FINALIZE = "finalize"
class MmOwnerProtocolError(RuntimeError):
"""Raised with identical text on every group member after a group-agreed failure."""
class ImageSpanRequest(msgspec.Struct, frozen=True):
hash: Optional[int]
span_len: int
item: Any
duplicates: List[Any] = []
class ImageSpanKey(msgspec.Struct, frozen=True):
hash: Optional[int]
span_len: int
geometry: Optional[Tuple[Any, ...]]
class RankManifest(msgspec.Struct, frozen=True):
rank: int
keys: List[ImageSpanKey]
cached: List[bool]
dtype: str
width: int
rids: List[str]
error: Optional[str] = None
class OwnerPlan(msgspec.Struct, frozen=True):
actions: List[int] = []
owners: List[int] = []
error: Optional[str] = None
class RankStatus(msgspec.Struct, frozen=True):
error: Optional[str] = None
def select_owner_group(parallel) -> Optional[Any]:
"""The group whose members all execute the same requests; None keeps each
rank encoding its own images."""
replication = parallel.tp_size // parallel.attn_dp_size
if replication <= 1:
return None
if parallel.attn_cp_size == 1:
group = parallel.attn_tp_group
elif parallel.attn_dp_size == 1 and parallel.attn_cp_size == parallel.tp_size:
group = parallel.attn_cp_group
else:
return None
return group if group.world_size == replication else None
def has_owner_span_work(
mm_inputs: Sequence[Any],
extend_prefix_lens: Sequence[int],
extend_seq_lens: Sequence[int],
) -> bool:
"""Host-side mirror of the per-image scheduling path: does any raw
single-span image overlap the chunk on every rank of the group."""
for mm_input, prefix_len, extend_len in zip(
mm_inputs, extend_prefix_lens, extend_seq_lens
):
if mm_input is None or extend_len <= 0:
continue
items = [item for item in mm_input.mm_items if item is not None]
if any(
item.precomputed_embeddings is not None or len(item.offsets) != 1
for item in items
):
continue
for item in items:
start, end = item.offsets[0]
if end >= prefix_len and start < prefix_len + extend_len:
return True
return False
class MmOwnerSession(msgspec.Struct):
group: Any
device: Any
dtype: Any
width: int
rids: List[str]
signature: Any
phase: str = PHASE_PREPARE
in_collective: bool = False
def resolve(
self,
requests: Sequence[ImageSpanRequest],
cache: MultiModalStaticCache,
encode: SpanEncoder,
) -> Dict[SpanKey, torch.Tensor]:
# Owners allocate different amounts than receivers, so none of these
# buffers may come out of a symmetric pool.
saved_context = disable_symmetric_memory_context()
try:
return _resolve_owner_features(self, requests, cache, encode)
finally:
restore_symmetric_memory_context(saved_context)
def features_ready(self) -> None:
self._complete()
self.phase = PHASE_FINALIZE
@contextmanager
def uncaptured(self) -> Iterator[None]:
# A failure inside a collective leaves the group in an unknown state;
# no later exchange may try to agree on it.
self.in_collective = True
yield
self.in_collective = False
@contextmanager
def fence(self) -> Iterator[None]:
try:
yield
except Exception as exc:
self._fail(exc)
raise
self._complete()
def _fail(self, exc: BaseException) -> None:
if self.in_collective or isinstance(exc, MmOwnerProtocolError):
raise exc
text = _describe(self, self.phase, exc)
if self.phase == PHASE_PREPARE:
try:
_exchange_manifest(self, _manifest(self, [], [], error=text))
except MmOwnerProtocolError as agreed:
raise agreed from exc
_exchange_status(self, text, exc)
def _complete(self) -> None:
if self.phase == PHASE_PREPARE:
raise RuntimeError(
f"owner protocol {self.phase} completed without a manifest exchange"
)
_drain_and_agree(self, self.phase)
def _manifest(
session: MmOwnerSession,
keys: List[ImageSpanKey],
cached: List[bool],
error: Optional[str] = None,
) -> RankManifest:
return RankManifest(
rank=session.group.rank_in_group,
keys=keys,
cached=cached,
dtype=str(session.dtype),
width=session.width,
rids=list(session.rids),
error=error,
)
def _exchange_manifest(session: MmOwnerSession, manifest: RankManifest) -> OwnerPlan:
group = session.group
with session.uncaptured():
manifests = group.all_gather_object(manifest)
plan = _plan_or_error(session, manifests) if group.rank_in_group == 0 else None
plan = group.broadcast_object(plan, src=0)
session.phase = PHASE_FEATURES
if plan.error is not None:
raise MmOwnerProtocolError(plan.error)
return plan
def _plan_or_error(session: MmOwnerSession, manifests: List[RankManifest]) -> OwnerPlan:
try:
return _make_plan(manifests)
except Exception as exc:
return OwnerPlan(error=_describe(session, "plan", exc))
def _exchange_status(
session: MmOwnerSession, error: Optional[str], cause: Optional[BaseException]
) -> None:
with session.uncaptured():
statuses = session.group.all_gather_object(RankStatus(error=error))
for status in statuses:
if status.error is not None:
raise MmOwnerProtocolError(status.error) from cause
def _drain_and_agree(
session: MmOwnerSession, stage: str, step: Optional[Callable[[], Any]] = None
) -> Any:
result, error, cause = None, None, None
try:
result = step() if step is not None else None
_synchronize(session.device)
except Exception as exc:
error, cause = _describe(session, stage, exc), exc
_exchange_status(session, error, cause)
return result
def _resolve_owner_features(
session: MmOwnerSession,
requests: Sequence[ImageSpanRequest],
cache: MultiModalStaticCache,
encode: SpanEncoder,
) -> Dict[SpanKey, torch.Tensor]:
group = session.group
features: Dict[SpanKey, torch.Tensor] = {}
keys: List[ImageSpanKey] = []
cached: List[bool] = []
error = None
try:
keys, cached = _pin_local_cache(session, requests, cache, features)
except Exception as exc:
error = _describe(session, "manifest", exc)
plan = _exchange_manifest(session, _manifest(session, keys, cached, error))
if all(action == LOCAL_HIT for action in plan.actions):
return features
buffers = _drain_and_agree(
session,
"encode",
lambda: _prepare_transfers(session, requests, keys, plan, features, encode),
)
with session.uncaptured():
for index, (action, owner) in enumerate(zip(plan.actions, plan.owners)):
if action != LOCAL_HIT:
group.broadcast(buffers[index], src=owner)
for index, key in enumerate(keys):
if plan.actions[index] == LOCAL_HIT:
continue
span = buffers[index]
features[(key.hash, key.span_len)] = span
cache.set(key.hash, EmbeddingResult(embedding=span))
return features
def _pin_local_cache(
session: MmOwnerSession,
requests: Sequence[ImageSpanRequest],
cache: MultiModalStaticCache,
features: Dict[SpanKey, torch.Tensor],
) -> Tuple[List[ImageSpanKey], List[bool]]:
keys: List[ImageSpanKey] = []
cached: List[bool] = []
for request in requests:
if request.hash is None:
raise ValueError(
f"image span of {request.span_len} tokens has no content hash"
)
geometry = session.signature(request.item, request.span_len)
for duplicate in request.duplicates:
other = session.signature(duplicate, request.span_len)
if other != geometry:
raise ValueError(
f"image hash {request.hash} ({request.span_len} tokens) occurs "
f"with different geometry: {geometry} vs {other}"
)
keys.append(
ImageSpanKey(
hash=request.hash, span_len=request.span_len, geometry=geometry
)
)
span = _valid_cached_span(session, cache, request)
if span is not None:
features[(request.hash, request.span_len)] = span
cached.append(span is not None)
return keys, cached
def _valid_cached_span(
session: MmOwnerSession,
cache: MultiModalStaticCache,
request: ImageSpanRequest,
) -> Optional[torch.Tensor]:
entry = cache.get_single(request.hash)
if entry is None:
return None
span = entry.embedding
if (
span.dim() == 2
and span.shape[0] == request.span_len
and span.shape[1] == session.width
and span.dtype == session.dtype
and span.device == session.device
):
return span
logger.warning(
"Discarding cached multimodal embedding that cannot serve the current "
"image span: cache_key=%s expected=(%d, %d, %s) cached=(%s, %s).",
request.hash,
request.span_len,
session.width,
session.dtype,
tuple(span.shape),
span.dtype,
)
cache.free(request.hash, None)
return None
def _make_plan(manifests: List[RankManifest]) -> OwnerPlan:
for manifest in manifests:
if manifest.error is not None:
return OwnerPlan(error=manifest.error)
lead = manifests[0]
for manifest in manifests[1:]:
if (manifest.keys, manifest.dtype, manifest.width, manifest.rids) != (
lead.keys,
lead.dtype,
lead.width,
lead.rids,
):
return OwnerPlan(
error=(
"image manifest mismatch between group ranks 0 and "
f"{manifest.rank}: rids={lead.rids} vs {manifest.rids}, "
f"keys={lead.keys} vs {manifest.keys}, "
f"dtype={lead.dtype} vs {manifest.dtype}, "
f"width={lead.width} vs {manifest.width}"
),
)
replication = len(manifests)
actions: List[int] = []
owners: List[int] = []
for index, key in enumerate(lead.keys):
owner = key.hash % replication
if all(manifest.cached[index] for manifest in manifests):
action = LOCAL_HIT
elif manifests[owner].cached[index]:
action = OWNER_CACHE_BROADCAST
else:
action = OWNER_ENCODE_BROADCAST
actions.append(action)
owners.append(owner)
return OwnerPlan(actions=actions, owners=owners)
def _prepare_transfers(
session: MmOwnerSession,
requests: Sequence[ImageSpanRequest],
keys: List[ImageSpanKey],
plan: OwnerPlan,
features: Dict[SpanKey, torch.Tensor],
encode: SpanEncoder,
) -> Dict[int, torch.Tensor]:
rank = session.group.rank_in_group
buffers: Dict[int, torch.Tensor] = {}
owned: List[int] = []
for index, (action, owner) in enumerate(zip(plan.actions, plan.owners)):
if action == LOCAL_HIT:
continue
if owner != rank:
try:
buffers[index] = _new_span_buffer(session, keys[index])
except Exception as exc:
raise RuntimeError(
f"receive buffer for image hash {keys[index].hash} shape "
f"{(keys[index].span_len, session.width)} {session.dtype} "
f"failed: {type(exc).__name__}: {exc}"
) from exc
elif action == OWNER_CACHE_BROADCAST:
key = (keys[index].hash, keys[index].span_len)
buffers[index] = features[key].contiguous()
else:
owned.append(index)
if owned:
try:
encoded = encode([requests[index].item for index in owned])
except Exception as exc:
raise RuntimeError(
f"owner encode of image hashes {[keys[i].hash for i in owned]} failed: "
f"{type(exc).__name__}: {exc}"
) from exc
spans = _split_spans(encoded, [keys[index].span_len for index in owned])
for index, span in zip(owned, spans):
buffers[index] = _validated_span(session, keys[index], span)
return buffers
def _new_span_buffer(session: MmOwnerSession, key: ImageSpanKey) -> torch.Tensor:
return torch.empty(
(key.span_len, session.width), device=session.device, dtype=session.dtype
)
def _split_spans(
encoded: torch.Tensor | List[torch.Tensor], span_lens: List[int]
) -> List[torch.Tensor]:
if isinstance(encoded, list):
if len(encoded) != len(span_lens):
raise ValueError(
f"encoder returned {len(encoded)} spans for {len(span_lens)} images"
)
return [span.reshape(-1, span.shape[-1]) for span in encoded]
encoded = encoded.reshape(-1, encoded.shape[-1])
if encoded.shape[0] != sum(span_lens):
raise ValueError(
f"encoder returned {encoded.shape[0]} rows for spans of {span_lens}"
)
return list(torch.split(encoded, span_lens, dim=0))
def _validated_span(
session: MmOwnerSession, key: ImageSpanKey, span: torch.Tensor
) -> torch.Tensor:
expected = (key.span_len, session.width)
if tuple(span.shape) != expected or span.dtype != session.dtype:
raise ValueError(
f"encoded span for hash={key.hash} has shape {tuple(span.shape)} "
f"dtype {span.dtype}; expected {expected} {session.dtype}"
)
if span.device != session.device:
span = span.to(session.device)
return span.contiguous()
def _synchronize(device) -> None:
if device.type == "cuda":
torch.cuda.current_stream(device).synchronize()
def _describe(session: MmOwnerSession, stage: str, exc: BaseException) -> str:
return (
f"multimodal owner protocol failed during {stage} on group rank "
f"{session.group.rank_in_group} (global rank "
f"{session.group.ranks[session.group.rank_in_group]}, rids={list(session.rids)}): "
f"{type(exc).__name__}: {exc}"
)
+44 -1
View File
@@ -5,6 +5,7 @@ from typing import Callable, Dict, List, Optional, Tuple
import torch
from sglang.srt.managers.mm_owner_embedding import ImageSpanRequest, MmOwnerSession
from sglang.srt.managers.schedule_batch import MultimodalDataItem
from sglang.srt.mem_cache.multimodal_cache import EmbeddingResult, MultiModalStaticCache
from sglang.srt.multimodal.evs import EVSEmbeddingResult
@@ -430,6 +431,37 @@ def _batch_encode_per_image_misses(
return hash_to_embedding
def _collect_image_span_requests(
per_image_requests: List[PerImageRequestInfo],
) -> List[ImageSpanRequest]:
spans: Dict[Tuple[Optional[int], int], ImageSpanRequest] = {}
for req_info in per_image_requests:
chunk_start = req_info.extend_prefix_len
chunk_end = chunk_start + req_info.extend_seq_len # exclusive
req_info.overlapping = [
(idx, item, token_count)
for idx, item in enumerate(req_info.items)
if (token_count := _item_overlap(item, chunk_start, chunk_end)) is not None
]
for _idx, item, token_count in req_info.overlapping:
span = spans.get((item.hash, token_count))
if span is None:
spans[(item.hash, token_count)] = ImageSpanRequest(
hash=item.hash, span_len=token_count, item=item
)
elif span.item is not item:
span.duplicates.append(item)
return list(spans.values())
def _owner_span_encoder(data_embedding_func: DataEmbeddingFunc, device: torch.device):
def encode(items: List[MultimodalDataItem]):
_move_items_to_device(items, device, data_embedding_func)
return data_embedding_func(items)
return encode
def _get_chunked_embedding_by_item(
data_embedding_func: DataEmbeddingFunc,
embedding_items_per_req: List[MultimodalDataItem],
@@ -554,6 +586,7 @@ def _get_chunked_prefill_embedding(
extend_length: List[int],
items_offset_list: List[List[Tuple[int, int]]],
input_ids: torch.Tensor,
mm_owner: Optional[MmOwnerSession] = None,
) -> tuple[torch.Tensor | None, torch.Tensor]:
"""
Chunked prefill embedding: encode items across all requests and extract
@@ -615,7 +648,15 @@ def _get_chunked_prefill_embedding(
# Phase 1: batch encode all per-image cache misses in ONE ViT call
hash_to_embedding: Dict[Tuple[Optional[int], int], torch.Tensor] = {}
if per_image_requests:
if per_image_requests and mm_owner is not None:
# The owner protocol must see every overlapping span before any local
# cache filtering: a rank-local hit can never skip a group collective.
hash_to_embedding = mm_owner.resolve(
_collect_image_span_requests(per_image_requests),
cache=embedding_cache,
encode=_owner_span_encoder(data_embedding_func, device),
)
elif per_image_requests:
hash_to_embedding = _batch_encode_per_image_misses(
data_embedding_func, per_image_requests, device
)
@@ -718,6 +759,7 @@ def get_embedding_and_mask(
prefix_length: List[int],
extend_length: List[int],
items_offset_list: List[List[Tuple[int, int]]],
mm_owner: Optional[MmOwnerSession] = None,
) -> Tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor]:
"""
Generate multimodal embeddings and create a mask for identifying their positions in the input sequence.
@@ -758,6 +800,7 @@ def get_embedding_and_mask(
extend_length,
items_offset_list,
input_ids,
mm_owner=mm_owner,
)
if embedding is None:
return None, None, input_ids
+10 -1
View File
@@ -11,6 +11,7 @@ import sys
from abc import abstractmethod
from array import array
from collections import defaultdict
from contextlib import nullcontext
from multiprocessing import shared_memory
from typing import Any, Callable, Dict, List, Optional, Tuple
@@ -26,6 +27,7 @@ from sglang.srt.managers.io_struct import (
TokenizedEmbeddingReqInput,
TokenizedGenerateReqInput,
)
from sglang.srt.managers.mm_owner_embedding import MmOwnerSession
# Preserve the existing initialization import for downstream callers.
from sglang.srt.managers.mm_schedule import (
@@ -416,6 +418,7 @@ def embed_mm_inputs(
data_embedding_func_mapping: Dict[Modality, DataEmbeddingFunc] = None,
placeholder_tokens: dict[Modality, List[int]] = None,
use_deepstack: Dict[Modality, bool] = {},
mm_owner: Optional[MmOwnerSession] = None,
) -> Optional[torch.Tensor]:
"""
Embed multimodal inputs and integrate them with text token embeddings.
@@ -497,6 +500,7 @@ def embed_mm_inputs(
prefix_length=extend_prefix_lens,
extend_length=extend_seq_lens,
items_offset_list=items_offsets,
mm_owner=mm_owner,
)
if use_deepstack.get(modality, None) and embedding is not None:
@@ -517,7 +521,12 @@ def embed_mm_inputs(
# filled with the hash values of the multimodal for the prefix matching in the radix attention.
# There values are useless because their embeddings will be replaced by vision embeddings anyway.
input_ids.clamp_(min=0, max=vocab_size - 1)
input_embeds = input_embedding(input_ids)
if mm_owner is not None:
# The text embedding may all-reduce across TP; a rank-local failure in
# feature preparation has to be agreed on before any rank enters it.
mm_owner.features_ready()
with mm_owner.uncaptured() if mm_owner is not None else nullcontext():
input_embeds = input_embedding(input_ids)
# deepstack embedding
if use_deepstack:
@@ -1111,6 +1111,15 @@ class PrefillAdder:
else AddReqResult.CONTINUE
)
def can_share_extend_batch(self, req: Req) -> bool:
# Token embedding overrides embed the batch's raw input_ids before the
# model runs, and that lookup cannot index multimodal placeholder hash IDs.
if req.positional_embed_overrides is not None:
return all(r.multimodal_inputs is None for r in self.can_run_list)
if req.multimodal_inputs is not None:
return all(r.positional_embed_overrides is None for r in self.can_run_list)
return True
def add_chunked_req(self, req: Req):
if self.dllm_config is not None:
_rem_tokens = self._get_dllm_remain_tokens(req)
+2
View File
@@ -3954,6 +3954,8 @@ class Scheduler(
for req in self.waiting_queue:
if self.enable_lora and not self.can_schedule_lora_req(req, running_loras):
continue
if not adder.can_share_extend_batch(req):
break
# A forward batch runs one pooling mode, so setwise readout requests
# (token_indices_to_pool) cannot share a batch with last-token ones.
@@ -1283,6 +1283,16 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
raise ValueError(
"encoder SWA replay cannot return cached prompt logprobs"
)
requests_embed_overrides = obj.positional_embed_overrides is not None or (
isinstance(obj, EmbeddingReqInput)
and obj.embed_overrides is not None
and obj.embed_override_token_id is not None
)
if requests_embed_overrides and obj.contains_mm_input():
raise ValueError(
"embedding overrides cannot be combined with image, video, or audio "
"inputs"
)
_max_req_len = self.context_len
input_token_num = len(input_ids) if input_ids is not None else 0
input_token_num += self.num_reserved_tokens
@@ -1670,6 +1670,7 @@ class ModelRunner:
forward_batch.replace_embeds is not None
and forward_batch.replace_positions is not None
):
misc_utils.validate_replace_embeds_batch(forward_batch)
# Token embedding overrides: get base embeddings, scatter replacements
if "input_embeds" not in kwargs:
embed_layer = self.model.get_input_embeddings()
@@ -19,6 +19,7 @@ from sglang.srt.server_args import CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACK
if TYPE_CHECKING:
from sglang.srt.configs.model_config import ModelConfig
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
logger = logging.getLogger(__name__)
@@ -99,6 +100,26 @@ def resolve_pp_proxy_residual_num_blocks(
return (start_layer + block_size - 1) // block_size
def validate_replace_embeds_batch(forward_batch: ForwardBatch) -> None:
if forward_batch.mm_inputs is None:
return
for mm_inputs, prefix_len, extend_len in zip(
forward_batch.mm_inputs,
forward_batch.extend_prefix_lens_cpu,
forward_batch.extend_seq_lens_cpu,
):
if mm_inputs is None:
continue
chunk_end = prefix_len + extend_len
for item in mm_inputs.mm_items:
for start, end in item.offsets or ():
if start < chunk_end and end >= prefix_len:
raise ValueError(
"Token embedding overrides cannot share an extend batch with "
"multimodal placeholders"
)
@runtime_checkable
class _SupportsDSparkPPProxy(Protocol):
def get_pp_proxy_dspark_hidden_size(self) -> int: ...
@@ -384,18 +384,34 @@ class EagerRunner(BaseRunner):
"""
model = self.model_runner.model
input_embeds = kwargs.get("input_embeds")
if input_embeds is None:
input_embeds = model.get_input_embeddings()(forward_batch.input_ids)
with cp_shard_model_inputs(
input_embeds,
forward_batch.positions,
forward_batch,
forward_batch.input_ids,
) as (sharded_input_embeds, sharded_positions, model_input_ids):
model_kwargs = {"input_embeds": sharded_input_embeds}
input_ids = forward_batch.input_ids
if prepare_inputs := getattr(model, "prepare_cp_inputs", None):
input_embeds, positions, model_kwargs = prepare_inputs(
forward_batch, **kwargs
)
else:
input_embeds = kwargs.get("input_embeds")
if hasattr(model, "prepare_model_inputs"):
# Multimodal offsets are request-global, so the merge and the
# placeholder-ID remap must see the full extend layout first.
input_ids, input_embeds = model.prepare_model_inputs(
input_ids=input_ids,
forward_batch=forward_batch,
input_embeds=input_embeds,
)
if input_embeds is None:
input_embeds = model.get_input_embeddings()(input_ids)
positions = forward_batch.positions
model_kwargs = {}
if (pp_proxy_tensors := kwargs.get("pp_proxy_tensors")) is not None:
model_kwargs["pp_proxy_tensors"] = pp_proxy_tensors
with cp_shard_model_inputs(
input_embeds,
positions,
forward_batch,
input_ids,
) as (sharded_input_embeds, sharded_positions, model_input_ids):
model_kwargs["input_embeds"] = sharded_input_embeds
hidden_states = model.model(
model_input_ids,
sharded_positions,
@@ -436,7 +452,7 @@ class EagerRunner(BaseRunner):
if aux_hidden_states is None:
logits_kwargs["hidden_states_before_norm"] = hidden_states_before_norm
return model.logits_processor(
forward_batch.input_ids,
input_ids,
hidden_states,
model.lm_head,
forward_batch,
+97 -25
View File
@@ -70,6 +70,7 @@ from sglang.srt.layers.attention.dsv4.dsv41_sparse import (
DeepseekV41Indexer,
)
from sglang.srt.layers.attention.dsv4.indexer import C4Indexer
from sglang.srt.layers.cp.base import is_zigzag
from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx
from sglang.srt.layers.cp.utils import (
cp_gather_full_sequence_states,
@@ -121,6 +122,11 @@ from sglang.srt.layers.quantization.mxfp8_input import Mxfp8SwizzledInput
from sglang.srt.layers.rotary_embedding import get_rope_wrapper
from sglang.srt.layers.utils import PPMissingLayer, get_layer_id
from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding
from sglang.srt.managers.mm_owner_embedding import (
MmOwnerSession,
has_owner_span_work,
select_owner_group,
)
from sglang.srt.managers.mm_utils import (
MultiModalityDataPaddingPatternMultimodalTokens,
embed_mm_inputs,
@@ -4893,13 +4899,21 @@ class DeepseekV4ForCausalLM(nn.Module):
self.determine_num_fused_shared_experts()
self.vision = None
if config.model_type == "deepseek_v41" and config.vision_n_layers > 0:
if (
get_parallel().attn_cp_size != 1
or get_parallel().pp_group.world_size != 1
or not get_moe_a2a_backend().is_none()
if get_parallel().pp_group.world_size != 1 or not (
get_moe_a2a_backend().is_none()
or (_is_cuda and get_moe_a2a_backend().is_megamoe())
):
raise ValueError(
"V4.1 vision currently supports TP/EP/DP without CP, PP or MoE A2A"
"V4.1 vision supports TP/EP/DP without PP; "
"MoE A2A requires CUDA MegaMoE"
)
if get_parallel().attn_cp_size != 1 and (
_is_npu or is_zigzag() or not get_moe_a2a_backend().is_none()
):
raise ValueError(
"V4.1 vision context parallelism requires the CUDA interleave "
"strategy without MoE A2A; NPU, zigzag CP and MegaMoE are not "
"supported yet"
)
args = SimpleNamespace(**vars(config), dim=config.hidden_size)
@@ -4908,6 +4922,11 @@ class DeepseekV4ForCausalLM(nn.Module):
self.image_start = nn.Parameter(torch.empty(config.hidden_size))
self.image_end = nn.Parameter(torch.empty(config.hidden_size))
self.image_newline = nn.Parameter(torch.empty(config.hidden_size))
self.mm_owner_group = (
select_owner_group(get_parallel())
if self.vision is not None and _is_cuda
else None
)
self.model = DeepseekV4Model(
config, quant_config, prefix=add_prefix("model", prefix)
)
@@ -5042,7 +5061,42 @@ class DeepseekV4ForCausalLM(nn.Module):
spans.append(span)
return spans
def _prepare_mm_embeddings(self, input_ids, forward_batch):
def _image_span_signature(self, item, span_len: int):
h, w = int(item.n_vit_h), int(item.n_vit_w)
r = self.config.vision_downsample_ratio
lh, lw = (h + r - 1) // r, (w + r - 1) // r
expected = 2 + lh * (lw + 1)
if expected != span_len:
raise ValueError(
f"image grid {(h, w)} yields {expected} span tokens, "
f"placeholder has {span_len}"
)
plan = item.model_specific_data.get(GPU_PLAN_KEY)
feature = item.feature
return (
h,
w,
tuple(feature.shape) if isinstance(feature, torch.Tensor) else None,
None if plan is None else tuple(sorted(plan.items())),
)
def _mm_owner_session(self, forward_batch) -> Optional[MmOwnerSession]:
if self.mm_owner_group is None or not has_owner_span_work(
forward_batch.mm_inputs,
forward_batch.extend_prefix_lens_cpu,
forward_batch.extend_seq_lens_cpu,
):
return None
return MmOwnerSession(
group=self.mm_owner_group,
device=self.image_start.device,
dtype=self.image_start.dtype,
width=self.config.hidden_size,
rids=list(forward_batch.rids or ()),
signature=self._image_span_signature,
)
def _prepare_mm_embeddings(self, input_ids, forward_batch, mm_owner):
# Keep scheduler hash IDs intact: the shared embedder clamps its input in place.
input_embeds, _ = embed_mm_inputs(
mm_inputs_list=[
@@ -5054,6 +5108,7 @@ class DeepseekV4ForCausalLM(nn.Module):
input_ids=input_ids.clone(),
input_embedding=self.get_input_embeddings(),
multimodal_model=self,
mm_owner=mm_owner,
)
forward_batch.mm_input_embeds = input_embeds
return input_embeds
@@ -5061,6 +5116,39 @@ class DeepseekV4ForCausalLM(nn.Module):
def get_input_embeddings(self) -> nn.Module:
return self.model.get_input_embeddings()
def prepare_model_inputs(
self,
input_ids: torch.Tensor,
forward_batch: ForwardBatch,
input_embeds: Optional[torch.Tensor],
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
if self.vision is None:
return input_ids, input_embeds
has_images = (
not forward_batch.forward_mode.is_decode()
and not forward_batch.forward_mode.is_target_verify()
and forward_batch.mm_inputs is not None
and any(x is not None for x in forward_batch.mm_inputs)
)
if has_images and input_embeds is not None:
raise ValueError("Cannot combine input_embeds and image inputs")
mm_owner = self._mm_owner_session(forward_batch) if has_images else None
with mm_owner.fence() if mm_owner is not None else nullcontext():
if has_images:
input_embeds = self._prepare_mm_embeddings(
input_ids, forward_batch, mm_owner
)
if not (
forward_batch.forward_mode.is_decode_or_idle()
or forward_batch.forward_mode.is_target_verify()
):
# Decode/verify IDs are already vocabulary IDs; remap prompt image
# hashes for Engram and routing.
input_ids = input_ids.masked_fill(
input_ids >= MM_PAD_SHIFT_VALUE, self.config.image_token_id
)
return input_ids, input_embeds
def set_dspark_layers_to_capture(self, layer_ids: List[int]) -> None:
if not self.pp_group.is_last_rank:
return
@@ -5114,25 +5202,9 @@ class DeepseekV4ForCausalLM(nn.Module):
input_embeds: Optional[torch.Tensor] = None,
pp_proxy_tensors: Optional[PPProxyTensors] = None,
) -> torch.Tensor:
if (
self.vision is not None
and not forward_batch.forward_mode.is_decode()
and not forward_batch.forward_mode.is_target_verify()
and forward_batch.mm_inputs is not None
and any(x is not None for x in forward_batch.mm_inputs)
):
if input_embeds is not None:
raise ValueError("Cannot combine input_embeds and image inputs")
input_embeds = self._prepare_mm_embeddings(input_ids, forward_batch)
if self.vision is not None and not (
forward_batch.forward_mode.is_decode_or_idle()
or forward_batch.forward_mode.is_target_verify()
):
# Decode/verify IDs are already vocabulary IDs; remap prompt image
# hashes for Engram and routing.
input_ids = input_ids.masked_fill(
input_ids >= MM_PAD_SHIFT_VALUE, self.config.image_token_id
)
input_ids, input_embeds = self.prepare_model_inputs(
input_ids=input_ids, forward_batch=forward_batch, input_embeds=input_embeds
)
with get_attn_tp_context().maybe_input_scattered(forward_batch):
hidden_states = self.model.forward(
@@ -222,6 +222,7 @@ class DeepseekV4ForCausalLMNextN(DeepseekV4ForCausalLM):
self.quant_config = quant_config
self.wo_a_fp8 = wo_a_fp8_gemm_enabled(quant_config)
self.determine_num_fused_shared_experts()
self.vision = None
self.model = DeepseekV4ModelNextN(
config, quant_config, prefix=add_prefix("model", prefix)
@@ -9,6 +9,7 @@ Covers:
"""
import unittest
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
import torch
@@ -17,10 +18,16 @@ from sglang.srt.constants import MIS_DELIMITER_TOKEN_ID
from sglang.srt.entrypoints.openai.utils import convert_embeds_to_tensors
from sglang.srt.managers.embed_types import PositionalEmbeds
from sglang.srt.managers.io_struct import EmbeddingReqInput, GenerateReqInput
from sglang.srt.managers.schedule_batch import (
Modality,
MultimodalDataItem,
MultimodalInputs,
)
from sglang.srt.managers.tokenizer_manager import TokenizerManager
from sglang.srt.managers.tokenizer_manager_score_mixin import (
TokenizerManagerScoreMixin,
)
from sglang.srt.model_executor.model_runner import ModelRunner
from sglang.srt.runtime_context import publish, reset_context
from sglang.srt.server_args import ServerArgs
from sglang.test.ci.ci_register import register_cpu_ci
@@ -642,5 +649,80 @@ class TestScoreRequestValidation(CustomTestCase):
)
class TestEmbedOverridesRejectMultimodal(CustomTestCase):
def setUp(self):
reset_context()
self.addCleanup(reset_context)
publish(ServerArgs(model_path="dummy"), role="tokenizer")
self.manager = TokenizerManager.__new__(TokenizerManager)
self.manager.context_len = 128
self.manager.num_reserved_tokens = 0
self.manager.allow_auto_truncate = False
self.manager.validate_total_tokens = False
self.manager.model_config = SimpleNamespace(joint_head_config=None)
def test_requests_with_overrides_and_images_are_rejected(self):
"""EmbeddingReqInput resolves embed_overrides only after validation, so
the unresolved form must be caught at admission too."""
overrides = PositionalEmbeds(embeds=[_vec()], positions=[1])
for build in (
lambda **mm: GenerateReqInput(
input_ids=[10, 50, 20],
sampling_params={},
positional_embed_overrides=overrides,
**mm,
),
lambda **mm: EmbeddingReqInput(
input_ids=[10, 50, 20],
sampling_params={},
embed_override_token_id=50,
embed_overrides=[_vec()],
**mm,
),
):
req = build(image_data=["image.png"])
self.manager.is_generation = isinstance(req, GenerateReqInput)
with self.assertRaisesRegex(ValueError, "overrides cannot be combined"):
self.manager._validate_one_request(req, req.input_ids)
text_only = build()
self.manager._validate_one_request(text_only, text_only.input_ids)
def test_mixed_extend_batch_is_rejected_before_embedding_lookup(self):
embed_layer = MagicMock(
side_effect=AssertionError("embedding lookup must not run")
)
runner = SimpleNamespace(
_pp_kwargs=lambda pp_proxy_tensors: {},
model=SimpleNamespace(get_input_embeddings=lambda: embed_layer),
is_generation=True,
)
image = MultimodalDataItem(
modality=Modality.IMAGE, feature=torch.zeros(1), offsets=[(0, 1)]
)
image.set_hash(1234)
forward_batch = SimpleNamespace(
input_embeds=None,
input_ids=torch.tensor([1, 2, image.pad_value, image.pad_value]),
replace_embeds=torch.full((1, HIDDEN_DIM), 5.0),
replace_positions=torch.tensor([0]),
mm_inputs=[None, MultimodalInputs(mm_items=[image])],
extend_prefix_lens_cpu=[0, 0],
extend_seq_lens_cpu=[2, 2],
)
with self.assertRaisesRegex(ValueError, "cannot share an extend batch"):
ModelRunner._extend_forward_kwargs(runner, forward_batch, None)
embed_layer.assert_not_called()
# A decoding image request in a mixed chunk has no placeholder rows here.
forward_batch.input_ids = torch.tensor([1, 2, 3])
forward_batch.extend_prefix_lens_cpu = [0, 5]
forward_batch.extend_seq_lens_cpu = [2, 1]
embed_layer.side_effect = None
embed_layer.return_value = torch.zeros(3, HIDDEN_DIM)
kwargs = ModelRunner._extend_forward_kwargs(runner, forward_batch, None)
self.assertTrue(torch.equal(kwargs["input_embeds"][0], _vec(5.0)))
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,693 @@
"""Owner-encoded image spans: real gloo groups, production coordinator and entrypoints."""
import json
import sys
from contextlib import ExitStack, contextmanager, nullcontext
from datetime import timedelta
from pathlib import Path
from tempfile import TemporaryDirectory
from types import SimpleNamespace
from unittest.mock import patch
import pytest
import torch
import torch.distributed as dist
import torch.multiprocessing
from torch import nn
from sglang.srt.distributed import parallel_state
from sglang.srt.layers.cp.base import init_cp_strategy
from sglang.srt.layers.cp.utils import prepare_cp_forward
from sglang.srt.managers import mm_owner_embedding, mm_schedule
from sglang.srt.managers.mm_owner_embedding import (
MmOwnerProtocolError,
select_owner_group,
)
from sglang.srt.managers.schedule_batch import (
Modality,
MultimodalDataItem,
MultimodalInputs,
)
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.model_executor.runner.eager_runner import EagerRunner
from sglang.srt.models.deepseek_v4 import DeepseekV4ForCausalLM
from sglang.srt.runtime_context import get_parallel
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=60, suite="base-a-test-cpu")
HIDDEN = 8
VOCAB = 64
IMAGE_TOKEN_ID = 7
CACHE_TAG = 50
DOWNSAMPLE = 2
GLOO_TIMEOUT = timedelta(seconds=90)
def _span(item_hash: int, rows: int, tag: int) -> torch.Tensor:
"""Distinguishable full span: the producing rank is visible in the values."""
base = float(item_hash * 1000 + tag * 100)
return torch.arange(rows * HIDDEN, dtype=torch.float32).view(rows, HIDDEN) + base
def _grid(rows: int):
"""A ViT grid whose downsampled span has exactly ``rows`` tokens."""
return (0, 0) if rows == 2 else (DOWNSAMPLE, DOWNSAMPLE * (rows - 3))
def _image(item_hash: int, start: int, rows: int) -> MultimodalDataItem:
h, w = _grid(rows)
item = MultimodalDataItem(
modality=Modality.IMAGE,
feature=torch.zeros(1),
offsets=[(start, start + rows - 1)],
model_specific_data={"n_vit_h": h, "n_vit_w": w},
)
item.set_hash(item_hash)
return item
def _request(parts, prefix_len: int = 0, extend_len=None, rid: str = "rid"):
"""``parts`` mixes token ids with ``(hash, rows)`` images."""
ids, items = [], []
for part in parts:
if isinstance(part, tuple):
item = _image(part[0], len(ids), part[1])
items.append(item)
ids.extend([item.pad_value] * part[1])
else:
ids.append(part)
if extend_len is None:
extend_len = len(ids) - prefix_len
return SimpleNamespace(
ids=ids, items=items, prefix_len=prefix_len, extend_len=extend_len, rid=rid
)
def _chunk_ids(req):
return req.ids[req.prefix_len : req.prefix_len + req.extend_len]
def _batch(requests, mode=ForwardMode.EXTEND):
return SimpleNamespace(
forward_mode=mode,
mm_inputs=[
MultimodalInputs(mm_items=req.items, im_token_id=IMAGE_TOKEN_ID)
if req.items
else None
for req in requests
],
extend_prefix_lens_cpu=[req.prefix_len for req in requests],
extend_seq_lens_cpu=[req.extend_len for req in requests],
seq_lens_cpu=[req.prefix_len + req.extend_len for req in requests],
input_ids=torch.tensor(
[tok for req in requests for tok in _chunk_ids(req)], dtype=torch.long
),
positions=torch.cat(
[
torch.arange(req.prefix_len, req.prefix_len + req.extend_len)
for req in requests
]
),
mm_input_embeds=None,
attn_cp_metadata=None,
global_num_tokens_cpu=None,
out_cache_loc=None,
input_ids_global=torch.zeros(1, dtype=torch.long),
rids=[req.rid for req in requests],
)
def _expected_embeds(embed: nn.Embedding, requests, tags):
"""``tags`` maps image hash to the tag of the span every rank must end up with."""
rows = []
with torch.no_grad():
for req in requests:
ids = torch.tensor(_chunk_ids(req), dtype=torch.long)
chunk = nn.functional.embedding(ids.clamp(max=VOCAB - 1), embed.weight)
chunk_end = req.prefix_len + req.extend_len
for item in req.items:
start, end = item.offsets[0]
lo = max(start, req.prefix_len) - start
hi = min(end + 1, chunk_end) - start
if lo >= hi:
continue
span = _span(item.hash, end - start + 1, tags[item.hash])
dest = max(start, req.prefix_len) - req.prefix_len
chunk[dest : dest + hi - lo] = span[lo:hi]
rows.append(chunk)
return torch.cat(rows)
def _seed_cache(item_hash: int, rows: int, tag: int) -> None:
mm_schedule.embedding_cache.set(
item_hash, mm_schedule.EmbeddingResult(embedding=_span(item_hash, rows, tag))
)
class _RecordingBody:
def __init__(self, embed):
self.embed = embed
self.calls = []
def get_input_embeddings(self):
return self.embed
def __call__(self, input_ids, positions, forward_batch, input_embeds=None):
self.calls.append(input_ids)
return input_embeds, input_embeds
class _ReducingEmbedding(nn.Embedding):
"""Stands in for the TP vocab embedding: a real all-reduce on the device group."""
def __init__(self, coordinator):
torch.manual_seed(0)
super().__init__(VOCAB, HIDDEN)
self.coordinator = coordinator
self.calls = 0
def forward(self, input_ids):
self.calls += 1
if self.coordinator is not None:
dist.all_reduce(torch.zeros(1), group=self.coordinator.device_group)
return super().forward(input_ids)
class _VisionStub(DeepseekV4ForCausalLM):
def __init__(self, embed, owner_group, tag: int):
nn.Module.__init__(self)
self.config = SimpleNamespace(
image_token_id=IMAGE_TOKEN_ID,
hidden_size=HIDDEN,
vision_downsample_ratio=DOWNSAMPLE,
)
self.vision = object()
self.tp_size = 1
self.mm_owner_group = owner_group
self.image_start = nn.Parameter(torch.zeros(HIDDEN))
self.model = _RecordingBody(embed)
self.pp_group = SimpleNamespace(is_last_rank=True)
self.lm_head = object()
self.capture_aux_hidden_states = False
self.logits_calls = []
self.tag = tag
self.encoded = []
def get_image_feature(self, items):
self.encoded.extend(item.hash for item in items)
return [
_span(item.hash, item.offsets[0][1] - item.offsets[0][0] + 1, self.tag)
for item in items
]
def logits_processor(
self,
input_ids,
hidden_states,
lm_head,
logits_metadata,
aux_hidden_states=None,
hidden_states_before_norm=None,
):
self.logits_calls.append(hidden_states)
return object()
def prepare(self, forward_batch):
with torch.no_grad():
return self.prepare_model_inputs(
input_ids=forward_batch.input_ids,
forward_batch=forward_batch,
input_embeds=None,
)
class _TracedGroup:
def __init__(self, inner):
self.inner = inner
self.trace = []
@property
def world_size(self):
return self.inner.world_size
@property
def rank_in_group(self):
return self.inner.rank_in_group
@property
def ranks(self):
return self.inner.ranks
def all_gather_object(self, obj):
self.trace.append(["all_gather_object", type(obj).__name__, self.world_size])
return self.inner.all_gather_object(obj)
def broadcast_object(self, obj=None, src=0):
self.trace.append(["broadcast_object", src, self.world_size])
return self.inner.broadcast_object(obj, src=src)
def broadcast(self, tensor, src=0):
self.trace.append(["broadcast", src, list(tensor.shape), self.world_size])
return self.inner.broadcast(tensor, src=src)
class _ForbiddenGroup:
world_size = 4
rank_in_group = 0
ranks = [0, 1, 2, 3]
def __getattr__(self, name):
raise AssertionError(f"collective {name!r} reached on a fast path")
def _coordinator(group_ranks, rank):
return parallel_state.GroupCoordinator(
group_ranks=group_ranks,
local_rank=rank,
torch_distributed_backend="gloo",
use_pynccl=False,
use_mscclpp=False,
use_custom_allreduce=False,
use_torch_symm_mem_all_reduce=False,
use_hpu_communicator=False,
use_xpu_communicator=False,
use_npu_communicator=False,
use_message_queue_broadcaster=False,
group_name="mm_owner_test",
gloo_timeout=GLOO_TIMEOUT,
)
def _init_rank(rank: int, world_size: int, init_file: str) -> None:
torch.set_num_threads(1)
dist.init_process_group(
backend="gloo",
init_method=Path(init_file).as_uri(),
rank=rank,
world_size=world_size,
timeout=GLOO_TIMEOUT,
)
parallel_state._MODEL_PARALLEL_GROUP_TIMEOUT = GLOO_TIMEOUT
def _run_ranks(world_size: int, target):
with TemporaryDirectory() as directory:
init_file = str(Path(directory) / "gloo-init")
torch.multiprocessing.spawn(
_rank_main,
args=(world_size, init_file, directory, target),
nprocs=world_size,
)
return [
json.loads((Path(directory) / f"rank{rank}.json").read_text())
for rank in range(world_size)
]
def _rank_main(rank, world_size, init_file, directory, target):
_init_rank(rank, world_size, init_file)
try:
result = target(rank, world_size)
(Path(directory) / f"rank{rank}.json").write_text(json.dumps(result))
finally:
dist.destroy_process_group()
@torch.no_grad()
def _run_cp_extend(model, forward_batch, coordinator):
def gather(output, input_tensor):
dist.all_gather_into_tensor(
output, input_tensor, group=coordinator.device_group
)
runner = EagerRunner.__new__(EagerRunner)
runner.model_runner = SimpleNamespace(model=model)
with (
patch("torch.cuda.current_stream", return_value=None),
patch(
"sglang.srt.layers.cp.interleave.attn_cp_all_gather_into_tensor",
side_effect=gather,
),
patch(
"sglang.srt.layers.cp.interleave.is_allocation_symmetric",
return_value=False,
),
patch(
"sglang.srt.layers.cp.interleave.use_symmetric_memory",
return_value=torch.no_grad(),
),
):
prepare_cp_forward(forward_batch)
runner._execute_extend_cp(forward_batch, {})
@contextmanager
def _interleave_cp(cp_size, **parallel):
init_cp_strategy(enable_prefill_cp=True, cp_size=cp_size, cp_strategy="interleave")
try:
with get_parallel().override(
attn_cp_size=cp_size, enable_cp_tp_group_sharing=False, **parallel
):
yield
finally:
init_cp_strategy(enable_prefill_cp=False, cp_size=1, cp_strategy="interleave")
MANIFEST = ["all_gather_object", "RankManifest", 4]
PLAN = ["broadcast_object", 0, 4]
STATUS = ["all_gather_object", "RankStatus", 4]
def _bcast(src, rows, size=4):
return ["broadcast", src, [rows, HIDDEN], size]
def _cold(*payloads):
return [MANIFEST, PLAN, STATUS, *payloads, STATUS, STATUS]
# Global rank 0 sits outside the owner group, so a group-local source index that
# leaks out as a global rank is caught.
A, B, C, D, E = 400, 401, 402, 403, 404 # owner = hash % 4 -> group ranks 0,1,2,3,0
ROWS = {A: 3, B: 4, C: 2, D: 3, E: 6}
def _owner_lifetime_program(rank, world_size):
group = _coordinator([[0], [1, 2, 3, 4]], rank)
if rank == 0:
return {"trace": [], "encoded": [], "outside": True}
traced = _TracedGroup(group)
local = group.rank_in_group
owner_tag = {key: 1 + key % 4 for key in ROWS}
mm_schedule.init_mm_embedding_cache(1 << 20)
embed = _ReducingEmbedding(group)
model = _VisionStub(embed, traced, tag=rank)
# A valid only on its owner, B valid only on a non-owner, C valid
# everywhere, D stale on its owner.
if local == 0:
_seed_cache(A, ROWS[A], CACHE_TAG + rank)
if local == 2:
_seed_cache(B, ROWS[B], CACHE_TAG + rank)
_seed_cache(C, ROWS[C], CACHE_TAG)
if local == 3:
_seed_cache(D, ROWS[D] + 2, CACHE_TAG + rank)
forwards = [
(
[
_request([10, 11, (A, ROWS[A]), 12, (B, ROWS[B]), 13], rid="r1"),
_request([20, (C, ROWS[C]), 21, (D, ROWS[D]), 22], rid="r2"),
],
{A: CACHE_TAG + 1, B: owner_tag[B], C: CACHE_TAG, D: owner_tag[D]},
),
(
[
_request([10, 11, (A, ROWS[A]), (B, ROWS[B])], rid="r3"),
_request([20, (A, ROWS[A]), (C, ROWS[C]), (D, ROWS[D])], rid="r4"),
],
{A: CACHE_TAG + 1, B: owner_tag[B], C: owner_tag[C], D: owner_tag[D]},
),
] + [
(
[_request([30, 31, (E, ROWS[E]), 32], prefix, extend, rid="r5")],
{E: owner_tag[E]},
)
for prefix, extend in ((0, 5), (5, 4))
]
encoded, traces = [], []
for index, (requests, tags) in enumerate(forwards):
if index == 1:
# Local 2 loses every entry and admits nothing new; B's owner evicts B.
if local == 2:
mm_schedule.init_mm_embedding_cache(0)
if local == 1:
mm_schedule.embedding_cache.free(B, None)
model.encoded.clear()
traced.trace.clear()
_, embeds = model.prepare(_batch(requests))
assert torch.equal(embeds, _expected_embeds(embed, requests, tags))
encoded.append(list(model.encoded))
traces.append(list(traced.trace))
return {"trace": traces, "encoded": encoded, "outside": False}
def test_owner_actions_and_cache_lifetime_across_asymmetric_ranks():
results = _run_ranks(5, _owner_lifetime_program)
members = range(1, 5)
assert all(results[rank]["trace"] == results[1]["trace"] for rank in members)
encoded = [results[rank]["encoded"] for rank in members]
trace = results[1]["trace"]
# Forward 1: A cache-broadcast, B and D owner-encoded, C a local hit.
assert [e[0] for e in encoded] == [[], [B], [], [D]]
assert trace[0] == _cold(_bcast(0, ROWS[A]), _bcast(1, ROWS[B]), _bcast(3, ROWS[D]))
# Forward 2: evicted owners of B and C re-encode once; A recurs but moves once.
assert [e[1] for e in encoded] == [[], [B], [C], []]
assert trace[1] == _cold(*(_bcast(key % 4, ROWS[key]) for key in (A, B, C, D)))
# Forwards 3 and 4: the chunk-crossing span is encoded fully once.
assert [e[2] for e in encoded] == [[E], [], [], []]
assert [e[3] for e in encoded] == [[], [], [], []]
assert trace[2] == trace[3] == _cold(_bcast(0, ROWS[E]))
assert results[0]["outside"]
def _cp8_program(rank, world_size):
tp8 = _coordinator([list(range(8))], rank)
singles = _coordinator([[r] for r in range(8)], rank)
attn_cp = _TracedGroup(tp8)
with _interleave_cp(
8,
tp_size=8,
attn_dp_size=1,
attn_cp_rank=rank,
tp_group=_TracedGroup(tp8),
attn_tp_group=_TracedGroup(singles),
attn_cp_group=attn_cp,
):
selected = select_owner_group(get_parallel())
assert selected is attn_cp
mm_schedule.init_mm_embedding_cache(1 << 20)
torch.manual_seed(0)
embed = nn.Embedding(VOCAB, HIDDEN)
model = _VisionStub(embed, selected, tag=rank)
w = 803 # owner 3; rows land on ranks 6,7,0,1,2,3 so ranks 4,5 hold no image row
requests = [
_request([10, 11, 12, 13, 14], rid="p1"),
_request([20, (w, 6), 21, 22], rid="p2"),
]
_run_cp_extend(model, _batch(requests), tp8)
(logits,) = model.logits_calls
assert torch.equal(logits, _expected_embeds(embed, requests, {w: 3}))
(body,) = model.model.calls
return {
"encoded": list(model.encoded),
"trace": list(attn_cp.trace),
"image_rows": int((body == IMAGE_TOKEN_ID).sum()),
}
def test_tp8_cp8_encodes_once_and_merges_before_shard():
results = _run_ranks(8, _cp8_program)
assert [r["encoded"] for r in results] == [[], [], [], [803], [], [], [], []]
assert all(r["trace"] == results[0]["trace"] for r in results)
assert [op for op in results[0]["trace"] if op[0] == "broadcast"] == [
_bcast(3, 6, 8)
]
assert [r["image_rows"] for r in results] == [1, 1, 1, 1, 0, 0, 1, 1]
def _raise_on(on_rank, target, name, exc_type=RuntimeError, after=0):
def inject(rank, batch):
if rank != on_rank:
return nullcontext()
original = getattr(target, name)
calls = []
def fail(*args, **kwargs):
calls.append(None)
if len(calls) > after:
raise exc_type(f"injected {name}")
return original(*args, **kwargs)
return patch.object(target, name, fail)
return inject
def _remap_oom_on(on_rank):
def inject(rank, batch):
if rank != on_rank:
return nullcontext()
original = torch.Tensor.masked_fill
def masked_fill(self, mask, value):
if self is batch.input_ids:
raise torch.OutOfMemoryError("injected remap")
return original(self, mask, value)
return patch.object(torch.Tensor, "masked_fill", masked_fill)
return inject
def _failure(requests, trace, errors, inject=None, vocab_calls=0, cp=False):
return SimpleNamespace(
requests=requests,
trace=trace,
errors=errors,
inject=inject or (lambda rank, batch: nullcontext()),
vocab_calls=vocab_calls,
cp=cp,
)
F, G = 300, 301 # owners 0 and 1 in a four-rank group
AT_PLAN = [MANIFEST, PLAN]
AT_FINALIZE_F = _cold(_bcast(0, 3))
FAILURES = {
"prepare": _failure(
[[10, (G, 3), 11]],
AT_PLAN,
["during prepare on group rank 1", "OutOfMemoryError: injected as_tensor"],
_raise_on(1, torch, "as_tensor", torch.OutOfMemoryError),
),
"remap": _failure(
[[10, (F, 3), 11]],
AT_FINALIZE_F,
["during finalize on group rank 1", "injected remap"],
_remap_oom_on(1),
vocab_calls=1,
),
"final_sync": _failure(
[[10, (F, 3), 11]],
AT_FINALIZE_F,
["during finalize on group rank 1", "injected _synchronize"],
_raise_on(1, mm_owner_embedding, "_synchronize", after=2),
vocab_calls=1,
),
"leader_plan": _failure(
[[10, (G, 3), 11]],
AT_PLAN,
["during plan on group rank 0", "MemoryError: injected _make_plan"],
_raise_on(0, mm_owner_embedding, "_make_plan", MemoryError),
),
"remap_cp": _failure(
[[10, (F, 3), 11]],
AT_FINALIZE_F,
["during finalize on group rank 1", "injected remap"],
_remap_oom_on(1),
vocab_calls=1,
cp=True,
),
}
def _failure_program(rank, world_size):
group = _coordinator([[0, 1, 2, 3]], rank)
out = {}
for name, case in FAILURES.items():
mm_schedule.init_mm_embedding_cache(1 << 20)
traced = _TracedGroup(group)
embed = _ReducingEmbedding(None if case.cp else group)
model = _VisionStub(embed, traced, tag=rank)
batch = _batch([_request(parts, rid=name) for parts in case.requests])
with ExitStack() as stack, pytest.raises(MmOwnerProtocolError) as raised:
stack.enter_context(case.inject(rank, batch))
if case.cp:
stack.enter_context(
_interleave_cp(4, attn_cp_rank=rank, attn_cp_group=traced)
)
_run_cp_extend(model, batch, group)
else:
model.prepare(batch)
out[name] = {
"error": str(raised.value),
"trace": traced.trace,
"vocab_calls": embed.calls,
"body_calls": len(model.model.calls),
}
return out
def test_failures_agree_before_payload_text_embedding_or_body():
results = _run_ranks(4, _failure_program)
for name, case in FAILURES.items():
runs = [r[name] for r in results]
error = runs[0]["error"]
assert all(run["error"] == error for run in runs), (name, runs)
assert all(run["trace"] == case.trace for run in runs), (name, runs)
for fragment in [*case.errors, f"rids=['{name}'"]:
assert fragment in error, (name, fragment, error)
assert [run["vocab_calls"] for run in runs] == [case.vocab_calls] * 4, name
assert [run["body_calls"] for run in runs] == [0] * 4, name
def test_text_decode_prefilled_future_and_precomputed_paths_pay_no_collective():
mm_schedule.init_mm_embedding_cache(1 << 20)
torch.manual_seed(0)
embed = nn.Embedding(VOCAB, HIDDEN)
owner_model = _VisionStub(embed, _ForbiddenGroup(), tag=0)
legacy_model = _VisionStub(embed, None, tag=0)
text_only = _batch([_request([10, 11], rid="t0"), _request([20, 21, 22], rid="t1")])
assert owner_model.prepare(text_only)[1] is None
with_image = [_request([10, (A, 3), 11, 12], rid="v0")]
for mode in (ForwardMode.DECODE, ForwardMode.TARGET_VERIFY):
assert owner_model.prepare(_batch(with_image, mode))[1] is None
for request, chunk_ids in (
(_request([10, (A, 3), 11, 12], prefix_len=5, rid="pf"), [12]),
(_request([10, 11, 12, 13, (A, 3)], extend_len=2, rid="fut"), [10, 11]),
):
ids, embeds = owner_model.prepare(_batch([request]))
assert torch.equal(embeds, legacy_model.prepare(_batch([request]))[1])
assert torch.equal(ids, torch.tensor(chunk_ids))
precomputed = _request([10, 11, 12, 13], rid="pc")
item = MultimodalDataItem(
modality=Modality.IMAGE,
precomputed_embeddings=torch.full((3, HIDDEN), 5.0),
offsets=[(1, 3)],
)
item.set_hash(A)
precomputed.items = [item]
precomputed.ids[1:4] = [item.pad_value] * 3
_, embeds = owner_model.prepare(_batch([precomputed]))
assert torch.equal(embeds[1:4], torch.full((3, HIDDEN), 5.0))
overlapping = [_request([10, (A, 3), 11, 12], prefix_len=2, rid="ov")]
with pytest.raises(AssertionError, match="collective"):
owner_model.prepare(_batch(overlapping))
def _parallel(tp_size, attn_dp_size, attn_cp_size, attn_tp_size, attn_cp_group_size):
return SimpleNamespace(
tp_size=tp_size,
attn_dp_size=attn_dp_size,
attn_cp_size=attn_cp_size,
tp_group=SimpleNamespace(world_size=tp_size, name="tp"),
attn_tp_group=SimpleNamespace(world_size=attn_tp_size, name="attn_tp"),
attn_cp_group=SimpleNamespace(world_size=attn_cp_group_size, name="attn_cp"),
)
def test_select_owner_group_follows_the_replication_domain():
for layout, expected in (
((8, 1, 1, 8, 1), "attn_tp"),
((8, 1, 8, 1, 8), "attn_cp"),
((8, 2, 1, 4, 1), "attn_tp"),
((1, 1, 1, 1, 1), None),
((8, 2, 4, 1, 4), None),
((8, 1, 4, 2, 4), None),
):
group = select_owner_group(_parallel(*layout))
assert (group and group.name) == expected, layout
if __name__ == "__main__":
sys.exit(pytest.main([__file__]))
@@ -30,7 +30,13 @@ from sglang.srt.runtime_context import get_context
from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler
from sglang.srt.utils.common import Range
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel
maybe_stub_sgl_kernel()
import sglang.srt.managers.scheduler as scheduler_module
from sglang.srt.disaggregation.utils import DisaggregationMode
from sglang.srt.managers.scheduler import Scheduler
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
@@ -115,7 +121,10 @@ class TestPrefillAdder(CustomTestCase):
req.prefix_indices = []
req.full_untruncated_fill_ids = []
req.output_ids = [0] * output_len
req.sampling_params = SimpleNamespace(max_new_tokens=max_new_tokens)
req.sampling_params = SimpleNamespace(
max_new_tokens=max_new_tokens, ignore_eos=False
)
req.token_indices_to_pool = None
req.time_stats = SimpleNamespace(wait_queue_entry_time=wait_time)
req.retracted_stain = False
req.host_hit_length = 0
@@ -298,6 +307,116 @@ class TestPrefillAdder(CustomTestCase):
)
self.assertEqual(adder.can_run_list, [first])
def create_admission_scheduler(self, *, chunked_req) -> Scheduler:
allocator = self.create_token_allocator(available_size=4096)
allocator.page_size = 1
self.mock_tree_cache.supports_mamba.return_value = False
self.mock_tree_cache.is_tree_cache.return_value = False
self.mock_tree_cache.supports_fast_match_prefix.return_value = False
self.mock_tree_cache.storage_prefetch_retries = None
self.mock_tree_cache.supports_prefix_sharing.return_value = False
scheduler = Scheduler.__new__(Scheduler)
scheduler.grammar_manager = SimpleNamespace(has_waiting_grammars=lambda: False)
scheduler.enable_priority_preemption = False
scheduler.enable_priority_scheduling = False
scheduler.is_hybrid_swa = False
scheduler.min_free_slots_delayer = None
scheduler.get_num_allocatable_reqs = lambda *args, **kwargs: 64
scheduler.policy = SchedulePolicy(
policy="fcfs",
tree_cache=self.mock_tree_cache,
enable_hierarchical_cache=False,
enable_priority_scheduling=False,
schedule_low_priority_values_first=False,
)
scheduler.processed_tokens_counter = 0
scheduler.chunked_prefill_size = 16
scheduler.dynamic_chunk_sizer = None
scheduler.tp_worker = SimpleNamespace(
model_runner=SimpleNamespace(attn_backend=object(), prefill_aware_swa=False)
)
scheduler.page_size = 1
scheduler.tree_cache = self.mock_tree_cache
scheduler.token_to_kv_pool_allocator = allocator
scheduler.new_token_ratio_tracker = SimpleNamespace(current=1.0)
scheduler.max_prefill_tokens = 16384
scheduler.is_mixed_chunk = False
scheduler.priority_scheduling_preemption_threshold = 0
scheduler.max_prefill_bs = 64
scheduler.max_running_requests = 64
scheduler.dllm_config = None
scheduler.enable_lora = False
scheduler.req_to_token_pool = SimpleNamespace()
scheduler.disaggregation_mode = DisaggregationMode.NULL
scheduler.enable_hicache_storage = False
scheduler.enable_lmcache = False
scheduler.enable_hierarchical_cache = False
scheduler.enable_unified_cache_external_linker = False
scheduler.truncation_align_size = None
scheduler.model_config = None
scheduler.enable_overlap = False
scheduler.spec_algorithm = None
scheduler.load_inquirer = MagicMock()
scheduler.chunked_req = chunked_req
scheduler.waiting_queue = []
return scheduler
def run_admission_pass(self, scheduler: Scheduler) -> list:
running_batch = self.create_running_batch()
running_batch.batch_is_full = False
with (
patch.object(scheduler_module, "ScheduleBatch") as schedule_batch,
patch.object(scheduler_module, "PrefillStats"),
patch.object(scheduler_module, "set_time_batch"),
):
new_batch, _ = scheduler._get_new_batch_prefill_raw(None, running_batch)
if new_batch is None:
return []
admitted = list(schedule_batch.init_new.call_args.args[0])
for req in admitted:
req.prefix_len = req.extend_end
return admitted
def test_fcfs_admits_override_request_once_image_continuation_drains(self):
"""An override request at the queue head must be admitted once the image
chunk ahead of it drains, even while more image requests keep arriving."""
def tagged(rid, length, *, multimodal=False, overrides=False):
req = self.create_shared_req(rid)
req.origin_input_ids = list(range(length))
req.full_untruncated_fill_ids = list(range(length))
req.multimodal_inputs = object() if multimodal else None
req.positional_embed_overrides = object() if overrides else None
req.beam_group = None
req.inflight_middle_chunks = 0
req.kv = SimpleNamespace(holds_mamba=False)
return req
continuation = tagged("image-continuation", 20, multimodal=True)
continuation.prefix_len = 16
scheduler = self.create_admission_scheduler(chunked_req=continuation)
text = tagged("text", 2)
override = tagged("override", 4, overrides=True)
scheduler.waiting_queue = [text, override]
admitted_at = None
for pass_index in range(6):
scheduler.waiting_queue.append(
tagged(f"image-{pass_index}", 16, multimodal=True)
)
admitted = self.run_admission_pass(scheduler)
if pass_index == 0:
self.assertEqual(admitted, [continuation, text])
self.assertFalse(
any(r.multimodal_inputs is not None for r in admitted)
and any(r.positional_embed_overrides is not None for r in admitted)
)
if any(r is override for r in admitted):
admitted_at = pass_index
break
self.assertIsNotNone(admitted_at)
self.assertNotIn(override, scheduler.waiting_queue)
def test_shared_admission_rechecks_after_prefix_lock(self):
adder = self.create_shared_adder()
self.assertIsNotNone(adder.token_to_kv_pool_allocator.alloc(24))
@@ -0,0 +1,247 @@
"""Vision inputs under prefill CP merge on the full extend layout before the shard."""
import unittest
from contextlib import contextmanager
from types import SimpleNamespace
from unittest.mock import patch
import torch
from torch import nn
from sglang.srt.layers.cp.base import init_cp_strategy
from sglang.srt.layers.cp.utils import prepare_cp_forward
from sglang.srt.managers import mm_schedule
from sglang.srt.managers.schedule_batch import (
Modality,
MultimodalDataItem,
MultimodalInputs,
)
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.model_executor.runner.eager_runner import EagerRunner
from sglang.srt.models.deepseek_v4 import DeepseekV4ForCausalLM
from sglang.srt.runtime_context import get_parallel
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=15, suite="base-a-test-cpu")
HIDDEN = 8
VOCAB = 64
IMAGE_TOKEN_ID = 7
CP_SIZE = 4
# (prefix_len, extend_len) per request. Request 1 carries one image whose span
# [2, 8] starts inside its prefix, so only span rows 1..6 land in this chunk.
CHUNKS = [(0, 7), (3, 9), (1, 5)]
IMAGE_OFFSET = (2, 8)
IMAGE_HASH = 12345
NUM_TOKENS = sum(extend_len for _, extend_len in CHUNKS)
# 21 tokens over 4 ranks give logical [6, 5, 5, 5], padded to the CP alignment.
PHYSICAL_ROWS = 8
IMAGE_ROWS = torch.arange(7, 13)
POSITIONS = torch.cat([torch.arange(p, p + n) for p, n in CHUNKS])
def _image_span(item: MultimodalDataItem) -> torch.Tensor:
start, end = item.offsets[0]
rows = end - start + 1
return torch.arange(rows * HIDDEN, dtype=torch.float32).view(rows, HIDDEN) + 100.0
def _pad(x: torch.Tensor) -> torch.Tensor:
return torch.cat([x, x.new_zeros(PHYSICAL_ROWS - x.shape[0], *x.shape[1:])])
class _RecordingBody:
def __init__(self, embed: nn.Embedding):
self.embed = embed
self.calls = []
def get_input_embeddings(self):
return self.embed
def __call__(self, input_ids, positions, forward_batch, input_embeds=None):
self.calls.append(
SimpleNamespace(
input_ids=input_ids,
input_embeds=input_embeds,
input_ids_global=forward_batch.input_ids_global,
)
)
return input_embeds, input_embeds
class _VisionStub(DeepseekV4ForCausalLM):
def __init__(self, embed: nn.Embedding):
nn.Module.__init__(self)
self.config = SimpleNamespace(image_token_id=IMAGE_TOKEN_ID)
self.vision = object()
self.tp_size = 1
self.mm_owner_group = None
self.model = _RecordingBody(embed)
self.pp_group = SimpleNamespace(is_last_rank=True)
self.lm_head = object()
self.capture_aux_hidden_states = False
self.logits_calls = []
def get_image_feature(self, items):
return [_image_span(item) for item in items]
def logits_processor(
self,
input_ids,
hidden_states,
lm_head,
logits_metadata,
aux_hidden_states=None,
hidden_states_before_norm=None,
):
self.logits_calls.append(
SimpleNamespace(input_ids=input_ids, hidden_states=hidden_states)
)
return object()
def _build_batch():
item = MultimodalDataItem(
modality=Modality.IMAGE, feature=torch.zeros(1), offsets=[IMAGE_OFFSET]
)
item.set_hash(IMAGE_HASH)
ids = list(range(10, 17))
ids += [item.pad_value] * len(IMAGE_ROWS) + [20, 21, 22]
ids += list(range(30, 35))
forward_batch = SimpleNamespace(
forward_mode=ForwardMode.EXTEND,
mm_inputs=[
MultimodalInputs(mm_items=[]),
MultimodalInputs(mm_items=[item], im_token_id=IMAGE_TOKEN_ID),
None,
],
extend_prefix_lens_cpu=[prefix for prefix, _ in CHUNKS],
extend_seq_lens_cpu=[extend_len for _, extend_len in CHUNKS],
seq_lens_cpu=[prefix + extend_len for prefix, extend_len in CHUNKS],
input_ids=torch.tensor(ids, dtype=torch.long),
positions=POSITIONS.clone(),
mm_input_embeds=None,
attn_cp_metadata=None,
global_num_tokens_cpu=None,
out_cache_loc=None,
input_ids_global=torch.zeros(1, dtype=torch.long),
)
return forward_batch, item
def _expected_embeds(embed, scheduler_ids, item):
with torch.no_grad():
full = embed(scheduler_ids.clamp(max=VOCAB - 1))
full[IMAGE_ROWS] = _image_span(item)[1:7]
return full
def _canonical(scheduler_ids):
canonical = scheduler_ids.clone()
canonical[IMAGE_ROWS] = IMAGE_TOKEN_ID
return canonical
class TestDeepseekV41VisionPrefillCPInputs(CustomTestCase):
def setUp(self):
mm_schedule.init_mm_embedding_cache(1 << 20)
init_cp_strategy(
enable_prefill_cp=True, cp_size=CP_SIZE, cp_strategy="interleave"
)
torch.manual_seed(0)
self.embed = nn.Embedding(VOCAB, HIDDEN)
self.model = _VisionStub(self.embed)
def tearDown(self):
init_cp_strategy(enable_prefill_cp=False, cp_size=1, cp_strategy="interleave")
@contextmanager
def _cp_collectives(self, full: torch.Tensor, rank: int):
def all_gather(output, input_tensor):
# Peers contribute their expected shards; this rank's rows come from
# what the runner actually handed to the collective.
output.zero_()
for peer in range(CP_SIZE):
rows = full[peer::CP_SIZE]
output[peer * PHYSICAL_ROWS : peer * PHYSICAL_ROWS + rows.shape[0]] = (
rows
)
output[rank * PHYSICAL_ROWS : (rank + 1) * PHYSICAL_ROWS] = input_tensor
with (
patch("torch.cuda.current_stream", return_value=None),
patch(
"sglang.srt.layers.cp.interleave.attn_cp_all_gather_into_tensor",
side_effect=all_gather,
),
patch(
"sglang.srt.layers.cp.interleave.is_allocation_symmetric",
return_value=False,
),
patch(
"sglang.srt.layers.cp.interleave.use_symmetric_memory",
return_value=torch.no_grad(),
),
):
yield
def test_cp_runner_merges_before_shard(self):
runner = EagerRunner.__new__(EagerRunner)
runner.model_runner = SimpleNamespace(model=self.model)
padded = torch.zeros(CP_SIZE * PHYSICAL_ROWS, dtype=torch.long)
for rank in range(CP_SIZE):
forward_batch, item = _build_batch()
scheduler_ids = forward_batch.input_ids.clone()
canonical = _canonical(scheduler_ids)
full = _expected_embeds(self.embed, scheduler_ids, item)
padded[:NUM_TOKENS] = canonical
rank_major_ids = padded.view(-1, CP_SIZE).T.flatten()
self.model.model.calls.clear()
self.model.logits_calls.clear()
with (
get_parallel().override(
attn_cp_rank=rank,
attn_cp_size=CP_SIZE,
attn_cp_group=object(),
enable_cp_tp_group_sharing=False,
),
self._cp_collectives(full, rank),
torch.no_grad(),
):
prepare_cp_forward(forward_batch)
runner._execute_extend_cp(forward_batch, {})
with self.subTest(rank=rank):
metadata = forward_batch.attn_cp_metadata
self.assertEqual(metadata.per_rank_actual_token, [PHYSICAL_ROWS] * 4)
(body,) = self.model.model.calls
self.assertTrue(
torch.equal(body.input_ids, _pad(canonical[rank::CP_SIZE]))
)
self.assertTrue(
torch.equal(body.input_embeds, _pad(full[rank::CP_SIZE]))
)
self.assertTrue(torch.equal(body.input_ids_global, rank_major_ids))
(logits,) = self.model.logits_calls
self.assertTrue(torch.equal(logits.input_ids, canonical))
self.assertTrue(torch.equal(logits.hidden_states, full))
self.assertTrue(torch.equal(forward_batch.mm_input_embeds, full))
self.assertTrue(torch.equal(forward_batch.input_ids, scheduler_ids))
def test_external_embeddings_with_images_are_rejected(self):
forward_batch, _ = _build_batch()
with self.assertRaisesRegex(ValueError, "Cannot combine"):
self.model.prepare_model_inputs(
input_ids=forward_batch.input_ids,
forward_batch=forward_batch,
input_embeds=torch.zeros(NUM_TOKENS, HIDDEN),
)
if __name__ == "__main__":
unittest.main()
@@ -29,6 +29,7 @@ from sglang.srt.arg_groups.cuda_graph_hook import (
finalize_cuda_graph_prefill_max_context,
handle_cuda_graph_config,
)
from sglang.srt.arg_groups.deepseek_v4_hook import validate_deepseek_v41_features
from sglang.srt.arg_groups.hicache_hook import (
handle_hicache,
handle_hicache_ratio_default,
@@ -45,6 +46,7 @@ from sglang.srt.arg_groups.kv_cache_hook import (
)
from sglang.srt.arg_groups.mamba_hook import handle_mamba_backend
from sglang.srt.arg_groups.memory_hook import handle_gpu_memory_settings
from sglang.srt.arg_groups.model_hook import handle_model_specific_adjustments
from sglang.srt.arg_groups.model_path_hook import handle_load_format
from sglang.srt.arg_groups.moe_hook import (
handle_a2a_moe,
@@ -4035,6 +4037,82 @@ class TestDcpCommBackendDefault(CustomTestCase):
)
class TestDeepseekV41VisionPrefillCPArgs(CustomTestCase):
def _args(
self,
*,
vision_n_layers=2,
prefill_backend=Backend.DISABLED,
lock_prefill_backend=False,
**overrides,
):
fields = dict(
model_path="dummy",
enable_prefill_cp=True,
cp_strategy="interleave",
tp_size=2,
)
fields.update(overrides)
server_args = ServerArgs(**fields)
server_args._model_config = SimpleNamespace(
hf_config=SimpleNamespace(
architectures=["DeepseekV4ForCausalLM"],
model_type="deepseek_v41",
vision_n_layers=vision_n_layers,
),
nvfp4_moe_meta=None,
is_fp4_experts=False,
decision_config=None,
)
# The dummy path does not initialize phase configs.
server_args.cuda_graph_config = CudaGraphConfig(
decode=PhaseConfig(backend=Backend.FULL, max_bs=512),
prefill=PhaseConfig(backend=prefill_backend, max_bs=512),
)
server_args._resolved_overrides = []
server_args._cuda_graph_config_locked = (
{(Phase.PREFILL, "backend")} if lock_prefill_backend else set()
)
return server_args
@override_platform(is_cuda=True, is_hip=False)
def test_guard_matrix_in_model_hook_order(self):
for overrides, error in (
(
dict(enable_encoder_swa_bounded_replay=True),
"encoder-swa-bounded-replay does not support context parallelism",
),
(dict(cp_strategy="zigzag"), "requires --cp-strategy interleave"),
(
dict(prefill_backend=Backend.BREAKABLE, lock_prefill_backend=True),
"runs eager prefill",
),
(
dict(
speculative_algorithm="DSPARK",
enable_decoder_swa_bounded_replay=True,
),
"DSpark.*decoder-swa-bounded-replay",
),
):
args = self._args(
max_running_requests=4, chunked_prefill_size=128, **overrides
)
with self.subTest(error=error), self.assertRaisesRegex(ValueError, error):
handle_model_specific_adjustments(args)
def test_text_zigzag_and_default_prefill_graph_are_accepted(self):
validate_deepseek_v41_features(
self._args(cp_strategy="zigzag", vision_n_layers=0)
)
args = self._args(prefill_backend=Backend.BREAKABLE)
validate_deepseek_v41_features(args)
self.assertEqual(
resolution_result(args, "cuda_graph_config").prefill.backend,
Backend.DISABLED,
)
class TestParserChoices(CustomTestCase):
"""The choices come from dependency-free name lists, but `cli/serve.py`
loads plugins before parsing, so a plugin's parser must still be accepted."""