PR #40219: DeepSeek V4.1 vision CP and single-owner image encoding

Source: https://github.com/sgl-project/sglang/pull/40219 (applied via GitHub .diff, 3way)
Adaptations for v0.5.21 base:
- attn_dp_enabled_of(cfg) -> cfg.enable_dp_attention (#41818 not in base)
- dropped _validate_joint_schema_request call (#42721 Clef, not in base)
- moved Backend import before the TRT-LLM prefill-graph check
This commit is contained in:
2026-10-19 10:45:10 +08:00
parent da343ba5d1
commit fd86688428
17 changed files with 1956 additions and 44 deletions
@@ -172,15 +172,25 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
)
cfg = resolving_view(server_args)
if model_config_of(server_args).hf_config.model_type != "deepseek_v41":
hf_config = model_config_of(server_args).hf_config
if hf_config.model_type != "deepseek_v41":
if cfg.enable_encoder_swa_bounded_replay:
raise ValueError(
"--enable-encoder-swa-bounded-replay requires DeepSeek-V4.1"
)
return
if cfg.enable_encoder_swa_bounded_replay:
from sglang.srt.model_executor.cuda_graph_config import Backend
from sglang.srt.model_executor.cuda_graph_config import Backend
if hf_config.vision_n_layers > 0 and cfg.enable_prefill_cp:
_validate_deepseek_v41_vision_prefill_cp(server_args)
if (
cfg.dsv4_attn_backend == "trtllm"
and cfg.cuda_graph_config.prefill.backend != Backend.DISABLED
):
raise ValueError(
"DeepSeek-V4.1 TRT-LLM requires --cuda-graph-backend-prefill disabled"
)
if cfg.enable_encoder_swa_bounded_replay:
incompatible = (
("non-CUDA hardware", not get_platform().is_cuda),
(
@@ -188,7 +198,8 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
cfg.cuda_graph_config.prefill.backend != Backend.DISABLED,
),
("DP attention", cfg.enable_dp_attention),
("context parallelism", cfg.attn_cp_size > 1),
# Prefill CP declares attn_cp_size and DP attention only later.
("context parallelism", cfg.attn_cp_size > 1 or cfg.enable_prefill_cp),
("external cache linker", cfg.enable_unified_cache_external_linker),
("unified memory", cfg.enable_unified_memory),
("PD disaggregation", cfg.disaggregation_mode != "null"),
@@ -287,3 +298,37 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
"--enable-decoder-swa-bounded-replay cannot be combined with "
f"{feature} yet; disable one of them."
)
def _validate_deepseek_v41_vision_prefill_cp(server_args: ServerArgs) -> None:
cfg = resolving_view(server_args)
if cfg.cp_strategy != "interleave":
raise ValueError(
"DeepSeek-V4.1 vision with prefill CP requires --cp-strategy "
f"interleave; got {cfg.cp_strategy!r}."
)
if cfg.cuda_graph_config.prefill.backend != Backend.DISABLED:
locked = getattr(server_args, "_cuda_graph_config_locked", set())
if (Phase.PREFILL, "backend") in locked:
raise ValueError(
"DeepSeek-V4.1 vision with prefill CP runs eager prefill; remove "
"the explicit prefill CUDA graph backend."
)
declare_resolution(
server_args,
"validate_deepseek_v41_features",
cuda_graph_config=with_phase(
cfg.cuda_graph_config, Phase.PREFILL, backend=Backend.DISABLED
),
)
logger.warning(
"Disabling the prefill CUDA graph for DeepSeek-V4.1 vision with prefill CP."
)
if (
str(cfg.speculative_algorithm).upper() == "DSPARK"
and cfg.enable_decoder_swa_bounded_replay
):
raise ValueError(
"DeepSeek-V4.1 vision with prefill CP does not support DSpark together "
"with --enable-decoder-swa-bounded-replay yet."
)
@@ -0,0 +1,464 @@
"""One owner rank encodes each image span and broadcasts it to the ranks that
run the same prefill chunk; every agreement precedes the payload it guards."""
from __future__ import annotations
import logging
from contextlib import contextmanager
from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence, Tuple
import msgspec
import torch
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
disable_symmetric_memory_context,
restore_symmetric_memory_context,
)
from sglang.srt.mem_cache.multimodal_cache import EmbeddingResult, MultiModalStaticCache
logger = logging.getLogger(__name__)
SpanKey = Tuple[Optional[int], int]
SpanEncoder = Callable[[List[Any]], torch.Tensor | List[torch.Tensor]]
LOCAL_HIT = 0
OWNER_CACHE_BROADCAST = 1
OWNER_ENCODE_BROADCAST = 2
PHASE_PREPARE = "prepare"
PHASE_FEATURES = "features"
PHASE_FINALIZE = "finalize"
class MmOwnerProtocolError(RuntimeError):
"""Raised with identical text on every group member after a group-agreed failure."""
class ImageSpanRequest(msgspec.Struct, frozen=True):
hash: Optional[int]
span_len: int
item: Any
duplicates: List[Any] = []
class ImageSpanKey(msgspec.Struct, frozen=True):
hash: Optional[int]
span_len: int
geometry: Optional[Tuple[Any, ...]]
class RankManifest(msgspec.Struct, frozen=True):
rank: int
keys: List[ImageSpanKey]
cached: List[bool]
dtype: str
width: int
rids: List[str]
error: Optional[str] = None
class OwnerPlan(msgspec.Struct, frozen=True):
actions: List[int] = []
owners: List[int] = []
error: Optional[str] = None
class RankStatus(msgspec.Struct, frozen=True):
error: Optional[str] = None
def select_owner_group(parallel) -> Optional[Any]:
"""The group whose members all execute the same requests; None keeps each
rank encoding its own images."""
replication = parallel.tp_size // parallel.attn_dp_size
if replication <= 1:
return None
if parallel.attn_cp_size == 1:
group = parallel.attn_tp_group
elif parallel.attn_dp_size == 1 and parallel.attn_cp_size == parallel.tp_size:
group = parallel.attn_cp_group
else:
return None
return group if group.world_size == replication else None
def has_owner_span_work(
mm_inputs: Sequence[Any],
extend_prefix_lens: Sequence[int],
extend_seq_lens: Sequence[int],
) -> bool:
"""Host-side mirror of the per-image scheduling path: does any raw
single-span image overlap the chunk on every rank of the group."""
for mm_input, prefix_len, extend_len in zip(
mm_inputs, extend_prefix_lens, extend_seq_lens
):
if mm_input is None or extend_len <= 0:
continue
items = [item for item in mm_input.mm_items if item is not None]
if any(
item.precomputed_embeddings is not None or len(item.offsets) != 1
for item in items
):
continue
for item in items:
start, end = item.offsets[0]
if end >= prefix_len and start < prefix_len + extend_len:
return True
return False
class MmOwnerSession(msgspec.Struct):
group: Any
device: Any
dtype: Any
width: int
rids: List[str]
signature: Any
phase: str = PHASE_PREPARE
in_collective: bool = False
def resolve(
self,
requests: Sequence[ImageSpanRequest],
cache: MultiModalStaticCache,
encode: SpanEncoder,
) -> Dict[SpanKey, torch.Tensor]:
# Owners allocate different amounts than receivers, so none of these
# buffers may come out of a symmetric pool.
saved_context = disable_symmetric_memory_context()
try:
return _resolve_owner_features(self, requests, cache, encode)
finally:
restore_symmetric_memory_context(saved_context)
def features_ready(self) -> None:
self._complete()
self.phase = PHASE_FINALIZE
@contextmanager
def uncaptured(self) -> Iterator[None]:
# A failure inside a collective leaves the group in an unknown state;
# no later exchange may try to agree on it.
self.in_collective = True
yield
self.in_collective = False
@contextmanager
def fence(self) -> Iterator[None]:
try:
yield
except Exception as exc:
self._fail(exc)
raise
self._complete()
def _fail(self, exc: BaseException) -> None:
if self.in_collective or isinstance(exc, MmOwnerProtocolError):
raise exc
text = _describe(self, self.phase, exc)
if self.phase == PHASE_PREPARE:
try:
_exchange_manifest(self, _manifest(self, [], [], error=text))
except MmOwnerProtocolError as agreed:
raise agreed from exc
_exchange_status(self, text, exc)
def _complete(self) -> None:
if self.phase == PHASE_PREPARE:
raise RuntimeError(
f"owner protocol {self.phase} completed without a manifest exchange"
)
_drain_and_agree(self, self.phase)
def _manifest(
session: MmOwnerSession,
keys: List[ImageSpanKey],
cached: List[bool],
error: Optional[str] = None,
) -> RankManifest:
return RankManifest(
rank=session.group.rank_in_group,
keys=keys,
cached=cached,
dtype=str(session.dtype),
width=session.width,
rids=list(session.rids),
error=error,
)
def _exchange_manifest(session: MmOwnerSession, manifest: RankManifest) -> OwnerPlan:
group = session.group
with session.uncaptured():
manifests = group.all_gather_object(manifest)
plan = _plan_or_error(session, manifests) if group.rank_in_group == 0 else None
plan = group.broadcast_object(plan, src=0)
session.phase = PHASE_FEATURES
if plan.error is not None:
raise MmOwnerProtocolError(plan.error)
return plan
def _plan_or_error(session: MmOwnerSession, manifests: List[RankManifest]) -> OwnerPlan:
try:
return _make_plan(manifests)
except Exception as exc:
return OwnerPlan(error=_describe(session, "plan", exc))
def _exchange_status(
session: MmOwnerSession, error: Optional[str], cause: Optional[BaseException]
) -> None:
with session.uncaptured():
statuses = session.group.all_gather_object(RankStatus(error=error))
for status in statuses:
if status.error is not None:
raise MmOwnerProtocolError(status.error) from cause
def _drain_and_agree(
session: MmOwnerSession, stage: str, step: Optional[Callable[[], Any]] = None
) -> Any:
result, error, cause = None, None, None
try:
result = step() if step is not None else None
_synchronize(session.device)
except Exception as exc:
error, cause = _describe(session, stage, exc), exc
_exchange_status(session, error, cause)
return result
def _resolve_owner_features(
session: MmOwnerSession,
requests: Sequence[ImageSpanRequest],
cache: MultiModalStaticCache,
encode: SpanEncoder,
) -> Dict[SpanKey, torch.Tensor]:
group = session.group
features: Dict[SpanKey, torch.Tensor] = {}
keys: List[ImageSpanKey] = []
cached: List[bool] = []
error = None
try:
keys, cached = _pin_local_cache(session, requests, cache, features)
except Exception as exc:
error = _describe(session, "manifest", exc)
plan = _exchange_manifest(session, _manifest(session, keys, cached, error))
if all(action == LOCAL_HIT for action in plan.actions):
return features
buffers = _drain_and_agree(
session,
"encode",
lambda: _prepare_transfers(session, requests, keys, plan, features, encode),
)
with session.uncaptured():
for index, (action, owner) in enumerate(zip(plan.actions, plan.owners)):
if action != LOCAL_HIT:
group.broadcast(buffers[index], src=owner)
for index, key in enumerate(keys):
if plan.actions[index] == LOCAL_HIT:
continue
span = buffers[index]
features[(key.hash, key.span_len)] = span
cache.set(key.hash, EmbeddingResult(embedding=span))
return features
def _pin_local_cache(
session: MmOwnerSession,
requests: Sequence[ImageSpanRequest],
cache: MultiModalStaticCache,
features: Dict[SpanKey, torch.Tensor],
) -> Tuple[List[ImageSpanKey], List[bool]]:
keys: List[ImageSpanKey] = []
cached: List[bool] = []
for request in requests:
if request.hash is None:
raise ValueError(
f"image span of {request.span_len} tokens has no content hash"
)
geometry = session.signature(request.item, request.span_len)
for duplicate in request.duplicates:
other = session.signature(duplicate, request.span_len)
if other != geometry:
raise ValueError(
f"image hash {request.hash} ({request.span_len} tokens) occurs "
f"with different geometry: {geometry} vs {other}"
)
keys.append(
ImageSpanKey(
hash=request.hash, span_len=request.span_len, geometry=geometry
)
)
span = _valid_cached_span(session, cache, request)
if span is not None:
features[(request.hash, request.span_len)] = span
cached.append(span is not None)
return keys, cached
def _valid_cached_span(
session: MmOwnerSession,
cache: MultiModalStaticCache,
request: ImageSpanRequest,
) -> Optional[torch.Tensor]:
entry = cache.get_single(request.hash)
if entry is None:
return None
span = entry.embedding
if (
span.dim() == 2
and span.shape[0] == request.span_len
and span.shape[1] == session.width
and span.dtype == session.dtype
and span.device == session.device
):
return span
logger.warning(
"Discarding cached multimodal embedding that cannot serve the current "
"image span: cache_key=%s expected=(%d, %d, %s) cached=(%s, %s).",
request.hash,
request.span_len,
session.width,
session.dtype,
tuple(span.shape),
span.dtype,
)
cache.free(request.hash, None)
return None
def _make_plan(manifests: List[RankManifest]) -> OwnerPlan:
for manifest in manifests:
if manifest.error is not None:
return OwnerPlan(error=manifest.error)
lead = manifests[0]
for manifest in manifests[1:]:
if (manifest.keys, manifest.dtype, manifest.width, manifest.rids) != (
lead.keys,
lead.dtype,
lead.width,
lead.rids,
):
return OwnerPlan(
error=(
"image manifest mismatch between group ranks 0 and "
f"{manifest.rank}: rids={lead.rids} vs {manifest.rids}, "
f"keys={lead.keys} vs {manifest.keys}, "
f"dtype={lead.dtype} vs {manifest.dtype}, "
f"width={lead.width} vs {manifest.width}"
),
)
replication = len(manifests)
actions: List[int] = []
owners: List[int] = []
for index, key in enumerate(lead.keys):
owner = key.hash % replication
if all(manifest.cached[index] for manifest in manifests):
action = LOCAL_HIT
elif manifests[owner].cached[index]:
action = OWNER_CACHE_BROADCAST
else:
action = OWNER_ENCODE_BROADCAST
actions.append(action)
owners.append(owner)
return OwnerPlan(actions=actions, owners=owners)
def _prepare_transfers(
session: MmOwnerSession,
requests: Sequence[ImageSpanRequest],
keys: List[ImageSpanKey],
plan: OwnerPlan,
features: Dict[SpanKey, torch.Tensor],
encode: SpanEncoder,
) -> Dict[int, torch.Tensor]:
rank = session.group.rank_in_group
buffers: Dict[int, torch.Tensor] = {}
owned: List[int] = []
for index, (action, owner) in enumerate(zip(plan.actions, plan.owners)):
if action == LOCAL_HIT:
continue
if owner != rank:
try:
buffers[index] = _new_span_buffer(session, keys[index])
except Exception as exc:
raise RuntimeError(
f"receive buffer for image hash {keys[index].hash} shape "
f"{(keys[index].span_len, session.width)} {session.dtype} "
f"failed: {type(exc).__name__}: {exc}"
) from exc
elif action == OWNER_CACHE_BROADCAST:
key = (keys[index].hash, keys[index].span_len)
buffers[index] = features[key].contiguous()
else:
owned.append(index)
if owned:
try:
encoded = encode([requests[index].item for index in owned])
except Exception as exc:
raise RuntimeError(
f"owner encode of image hashes {[keys[i].hash for i in owned]} failed: "
f"{type(exc).__name__}: {exc}"
) from exc
spans = _split_spans(encoded, [keys[index].span_len for index in owned])
for index, span in zip(owned, spans):
buffers[index] = _validated_span(session, keys[index], span)
return buffers
def _new_span_buffer(session: MmOwnerSession, key: ImageSpanKey) -> torch.Tensor:
return torch.empty(
(key.span_len, session.width), device=session.device, dtype=session.dtype
)
def _split_spans(
encoded: torch.Tensor | List[torch.Tensor], span_lens: List[int]
) -> List[torch.Tensor]:
if isinstance(encoded, list):
if len(encoded) != len(span_lens):
raise ValueError(
f"encoder returned {len(encoded)} spans for {len(span_lens)} images"
)
return [span.reshape(-1, span.shape[-1]) for span in encoded]
encoded = encoded.reshape(-1, encoded.shape[-1])
if encoded.shape[0] != sum(span_lens):
raise ValueError(
f"encoder returned {encoded.shape[0]} rows for spans of {span_lens}"
)
return list(torch.split(encoded, span_lens, dim=0))
def _validated_span(
session: MmOwnerSession, key: ImageSpanKey, span: torch.Tensor
) -> torch.Tensor:
expected = (key.span_len, session.width)
if tuple(span.shape) != expected or span.dtype != session.dtype:
raise ValueError(
f"encoded span for hash={key.hash} has shape {tuple(span.shape)} "
f"dtype {span.dtype}; expected {expected} {session.dtype}"
)
if span.device != session.device:
span = span.to(session.device)
return span.contiguous()
def _synchronize(device) -> None:
if device.type == "cuda":
torch.cuda.current_stream(device).synchronize()
def _describe(session: MmOwnerSession, stage: str, exc: BaseException) -> str:
return (
f"multimodal owner protocol failed during {stage} on group rank "
f"{session.group.rank_in_group} (global rank "
f"{session.group.ranks[session.group.rank_in_group]}, rids={list(session.rids)}): "
f"{type(exc).__name__}: {exc}"
)
+44 -1
View File
@@ -5,6 +5,7 @@ from typing import Callable, Dict, List, Optional, Tuple
import torch
from sglang.srt.managers.mm_owner_embedding import ImageSpanRequest, MmOwnerSession
from sglang.srt.managers.schedule_batch import MultimodalDataItem
from sglang.srt.mem_cache.multimodal_cache import EmbeddingResult, MultiModalStaticCache
from sglang.srt.multimodal.evs import EVSEmbeddingResult
@@ -430,6 +431,37 @@ def _batch_encode_per_image_misses(
return hash_to_embedding
def _collect_image_span_requests(
per_image_requests: List[PerImageRequestInfo],
) -> List[ImageSpanRequest]:
spans: Dict[Tuple[Optional[int], int], ImageSpanRequest] = {}
for req_info in per_image_requests:
chunk_start = req_info.extend_prefix_len
chunk_end = chunk_start + req_info.extend_seq_len # exclusive
req_info.overlapping = [
(idx, item, token_count)
for idx, item in enumerate(req_info.items)
if (token_count := _item_overlap(item, chunk_start, chunk_end)) is not None
]
for _idx, item, token_count in req_info.overlapping:
span = spans.get((item.hash, token_count))
if span is None:
spans[(item.hash, token_count)] = ImageSpanRequest(
hash=item.hash, span_len=token_count, item=item
)
elif span.item is not item:
span.duplicates.append(item)
return list(spans.values())
def _owner_span_encoder(data_embedding_func: DataEmbeddingFunc, device: torch.device):
def encode(items: List[MultimodalDataItem]):
_move_items_to_device(items, device, data_embedding_func)
return data_embedding_func(items)
return encode
def _get_chunked_embedding_by_item(
data_embedding_func: DataEmbeddingFunc,
embedding_items_per_req: List[MultimodalDataItem],
@@ -554,6 +586,7 @@ def _get_chunked_prefill_embedding(
extend_length: List[int],
items_offset_list: List[List[Tuple[int, int]]],
input_ids: torch.Tensor,
mm_owner: Optional[MmOwnerSession] = None,
) -> tuple[torch.Tensor | None, torch.Tensor]:
"""
Chunked prefill embedding: encode items across all requests and extract
@@ -615,7 +648,15 @@ def _get_chunked_prefill_embedding(
# Phase 1: batch encode all per-image cache misses in ONE ViT call
hash_to_embedding: Dict[Tuple[Optional[int], int], torch.Tensor] = {}
if per_image_requests:
if per_image_requests and mm_owner is not None:
# The owner protocol must see every overlapping span before any local
# cache filtering: a rank-local hit can never skip a group collective.
hash_to_embedding = mm_owner.resolve(
_collect_image_span_requests(per_image_requests),
cache=embedding_cache,
encode=_owner_span_encoder(data_embedding_func, device),
)
elif per_image_requests:
hash_to_embedding = _batch_encode_per_image_misses(
data_embedding_func, per_image_requests, device
)
@@ -718,6 +759,7 @@ def get_embedding_and_mask(
prefix_length: List[int],
extend_length: List[int],
items_offset_list: List[List[Tuple[int, int]]],
mm_owner: Optional[MmOwnerSession] = None,
) -> Tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor]:
"""
Generate multimodal embeddings and create a mask for identifying their positions in the input sequence.
@@ -758,6 +800,7 @@ def get_embedding_and_mask(
extend_length,
items_offset_list,
input_ids,
mm_owner=mm_owner,
)
if embedding is None:
return None, None, input_ids
+10 -1
View File
@@ -11,6 +11,7 @@ import sys
from abc import abstractmethod
from array import array
from collections import defaultdict
from contextlib import nullcontext
from multiprocessing import shared_memory
from typing import Any, Callable, Dict, List, Optional, Tuple
@@ -26,6 +27,7 @@ from sglang.srt.managers.io_struct import (
TokenizedEmbeddingReqInput,
TokenizedGenerateReqInput,
)
from sglang.srt.managers.mm_owner_embedding import MmOwnerSession
# Preserve the existing initialization import for downstream callers.
from sglang.srt.managers.mm_schedule import (
@@ -416,6 +418,7 @@ def embed_mm_inputs(
data_embedding_func_mapping: Dict[Modality, DataEmbeddingFunc] = None,
placeholder_tokens: dict[Modality, List[int]] = None,
use_deepstack: Dict[Modality, bool] = {},
mm_owner: Optional[MmOwnerSession] = None,
) -> Optional[torch.Tensor]:
"""
Embed multimodal inputs and integrate them with text token embeddings.
@@ -497,6 +500,7 @@ def embed_mm_inputs(
prefix_length=extend_prefix_lens,
extend_length=extend_seq_lens,
items_offset_list=items_offsets,
mm_owner=mm_owner,
)
if use_deepstack.get(modality, None) and embedding is not None:
@@ -517,7 +521,12 @@ def embed_mm_inputs(
# filled with the hash values of the multimodal for the prefix matching in the radix attention.
# There values are useless because their embeddings will be replaced by vision embeddings anyway.
input_ids.clamp_(min=0, max=vocab_size - 1)
input_embeds = input_embedding(input_ids)
if mm_owner is not None:
# The text embedding may all-reduce across TP; a rank-local failure in
# feature preparation has to be agreed on before any rank enters it.
mm_owner.features_ready()
with mm_owner.uncaptured() if mm_owner is not None else nullcontext():
input_embeds = input_embedding(input_ids)
# deepstack embedding
if use_deepstack:
@@ -1111,6 +1111,15 @@ class PrefillAdder:
else AddReqResult.CONTINUE
)
def can_share_extend_batch(self, req: Req) -> bool:
# Token embedding overrides embed the batch's raw input_ids before the
# model runs, and that lookup cannot index multimodal placeholder hash IDs.
if req.positional_embed_overrides is not None:
return all(r.multimodal_inputs is None for r in self.can_run_list)
if req.multimodal_inputs is not None:
return all(r.positional_embed_overrides is None for r in self.can_run_list)
return True
def add_chunked_req(self, req: Req):
if self.dllm_config is not None:
_rem_tokens = self._get_dllm_remain_tokens(req)
+2
View File
@@ -3954,6 +3954,8 @@ class Scheduler(
for req in self.waiting_queue:
if self.enable_lora and not self.can_schedule_lora_req(req, running_loras):
continue
if not adder.can_share_extend_batch(req):
break
# A forward batch runs one pooling mode, so setwise readout requests
# (token_indices_to_pool) cannot share a batch with last-token ones.
@@ -1283,6 +1283,16 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
raise ValueError(
"encoder SWA replay cannot return cached prompt logprobs"
)
requests_embed_overrides = obj.positional_embed_overrides is not None or (
isinstance(obj, EmbeddingReqInput)
and obj.embed_overrides is not None
and obj.embed_override_token_id is not None
)
if requests_embed_overrides and obj.contains_mm_input():
raise ValueError(
"embedding overrides cannot be combined with image, video, or audio "
"inputs"
)
_max_req_len = self.context_len
input_token_num = len(input_ids) if input_ids is not None else 0
input_token_num += self.num_reserved_tokens
@@ -1670,6 +1670,7 @@ class ModelRunner:
forward_batch.replace_embeds is not None
and forward_batch.replace_positions is not None
):
misc_utils.validate_replace_embeds_batch(forward_batch)
# Token embedding overrides: get base embeddings, scatter replacements
if "input_embeds" not in kwargs:
embed_layer = self.model.get_input_embeddings()
@@ -19,6 +19,7 @@ from sglang.srt.server_args import CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACK
if TYPE_CHECKING:
from sglang.srt.configs.model_config import ModelConfig
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
logger = logging.getLogger(__name__)
@@ -99,6 +100,26 @@ def resolve_pp_proxy_residual_num_blocks(
return (start_layer + block_size - 1) // block_size
def validate_replace_embeds_batch(forward_batch: ForwardBatch) -> None:
if forward_batch.mm_inputs is None:
return
for mm_inputs, prefix_len, extend_len in zip(
forward_batch.mm_inputs,
forward_batch.extend_prefix_lens_cpu,
forward_batch.extend_seq_lens_cpu,
):
if mm_inputs is None:
continue
chunk_end = prefix_len + extend_len
for item in mm_inputs.mm_items:
for start, end in item.offsets or ():
if start < chunk_end and end >= prefix_len:
raise ValueError(
"Token embedding overrides cannot share an extend batch with "
"multimodal placeholders"
)
@runtime_checkable
class _SupportsDSparkPPProxy(Protocol):
def get_pp_proxy_dspark_hidden_size(self) -> int: ...
@@ -384,18 +384,34 @@ class EagerRunner(BaseRunner):
"""
model = self.model_runner.model
input_embeds = kwargs.get("input_embeds")
if input_embeds is None:
input_embeds = model.get_input_embeddings()(forward_batch.input_ids)
with cp_shard_model_inputs(
input_embeds,
forward_batch.positions,
forward_batch,
forward_batch.input_ids,
) as (sharded_input_embeds, sharded_positions, model_input_ids):
model_kwargs = {"input_embeds": sharded_input_embeds}
input_ids = forward_batch.input_ids
if prepare_inputs := getattr(model, "prepare_cp_inputs", None):
input_embeds, positions, model_kwargs = prepare_inputs(
forward_batch, **kwargs
)
else:
input_embeds = kwargs.get("input_embeds")
if hasattr(model, "prepare_model_inputs"):
# Multimodal offsets are request-global, so the merge and the
# placeholder-ID remap must see the full extend layout first.
input_ids, input_embeds = model.prepare_model_inputs(
input_ids=input_ids,
forward_batch=forward_batch,
input_embeds=input_embeds,
)
if input_embeds is None:
input_embeds = model.get_input_embeddings()(input_ids)
positions = forward_batch.positions
model_kwargs = {}
if (pp_proxy_tensors := kwargs.get("pp_proxy_tensors")) is not None:
model_kwargs["pp_proxy_tensors"] = pp_proxy_tensors
with cp_shard_model_inputs(
input_embeds,
positions,
forward_batch,
input_ids,
) as (sharded_input_embeds, sharded_positions, model_input_ids):
model_kwargs["input_embeds"] = sharded_input_embeds
hidden_states = model.model(
model_input_ids,
sharded_positions,
@@ -436,7 +452,7 @@ class EagerRunner(BaseRunner):
if aux_hidden_states is None:
logits_kwargs["hidden_states_before_norm"] = hidden_states_before_norm
return model.logits_processor(
forward_batch.input_ids,
input_ids,
hidden_states,
model.lm_head,
forward_batch,
+97 -25
View File
@@ -70,6 +70,7 @@ from sglang.srt.layers.attention.dsv4.dsv41_sparse import (
DeepseekV41Indexer,
)
from sglang.srt.layers.attention.dsv4.indexer import C4Indexer
from sglang.srt.layers.cp.base import is_zigzag
from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx
from sglang.srt.layers.cp.utils import (
cp_gather_full_sequence_states,
@@ -121,6 +122,11 @@ from sglang.srt.layers.quantization.mxfp8_input import Mxfp8SwizzledInput
from sglang.srt.layers.rotary_embedding import get_rope_wrapper
from sglang.srt.layers.utils import PPMissingLayer, get_layer_id
from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding
from sglang.srt.managers.mm_owner_embedding import (
MmOwnerSession,
has_owner_span_work,
select_owner_group,
)
from sglang.srt.managers.mm_utils import (
MultiModalityDataPaddingPatternMultimodalTokens,
embed_mm_inputs,
@@ -4893,13 +4899,21 @@ class DeepseekV4ForCausalLM(nn.Module):
self.determine_num_fused_shared_experts()
self.vision = None
if config.model_type == "deepseek_v41" and config.vision_n_layers > 0:
if (
get_parallel().attn_cp_size != 1
or get_parallel().pp_group.world_size != 1
or not get_moe_a2a_backend().is_none()
if get_parallel().pp_group.world_size != 1 or not (
get_moe_a2a_backend().is_none()
or (_is_cuda and get_moe_a2a_backend().is_megamoe())
):
raise ValueError(
"V4.1 vision currently supports TP/EP/DP without CP, PP or MoE A2A"
"V4.1 vision supports TP/EP/DP without PP; "
"MoE A2A requires CUDA MegaMoE"
)
if get_parallel().attn_cp_size != 1 and (
_is_npu or is_zigzag() or not get_moe_a2a_backend().is_none()
):
raise ValueError(
"V4.1 vision context parallelism requires the CUDA interleave "
"strategy without MoE A2A; NPU, zigzag CP and MegaMoE are not "
"supported yet"
)
args = SimpleNamespace(**vars(config), dim=config.hidden_size)
@@ -4908,6 +4922,11 @@ class DeepseekV4ForCausalLM(nn.Module):
self.image_start = nn.Parameter(torch.empty(config.hidden_size))
self.image_end = nn.Parameter(torch.empty(config.hidden_size))
self.image_newline = nn.Parameter(torch.empty(config.hidden_size))
self.mm_owner_group = (
select_owner_group(get_parallel())
if self.vision is not None and _is_cuda
else None
)
self.model = DeepseekV4Model(
config, quant_config, prefix=add_prefix("model", prefix)
)
@@ -5042,7 +5061,42 @@ class DeepseekV4ForCausalLM(nn.Module):
spans.append(span)
return spans
def _prepare_mm_embeddings(self, input_ids, forward_batch):
def _image_span_signature(self, item, span_len: int):
h, w = int(item.n_vit_h), int(item.n_vit_w)
r = self.config.vision_downsample_ratio
lh, lw = (h + r - 1) // r, (w + r - 1) // r
expected = 2 + lh * (lw + 1)
if expected != span_len:
raise ValueError(
f"image grid {(h, w)} yields {expected} span tokens, "
f"placeholder has {span_len}"
)
plan = item.model_specific_data.get(GPU_PLAN_KEY)
feature = item.feature
return (
h,
w,
tuple(feature.shape) if isinstance(feature, torch.Tensor) else None,
None if plan is None else tuple(sorted(plan.items())),
)
def _mm_owner_session(self, forward_batch) -> Optional[MmOwnerSession]:
if self.mm_owner_group is None or not has_owner_span_work(
forward_batch.mm_inputs,
forward_batch.extend_prefix_lens_cpu,
forward_batch.extend_seq_lens_cpu,
):
return None
return MmOwnerSession(
group=self.mm_owner_group,
device=self.image_start.device,
dtype=self.image_start.dtype,
width=self.config.hidden_size,
rids=list(forward_batch.rids or ()),
signature=self._image_span_signature,
)
def _prepare_mm_embeddings(self, input_ids, forward_batch, mm_owner):
# Keep scheduler hash IDs intact: the shared embedder clamps its input in place.
input_embeds, _ = embed_mm_inputs(
mm_inputs_list=[
@@ -5054,6 +5108,7 @@ class DeepseekV4ForCausalLM(nn.Module):
input_ids=input_ids.clone(),
input_embedding=self.get_input_embeddings(),
multimodal_model=self,
mm_owner=mm_owner,
)
forward_batch.mm_input_embeds = input_embeds
return input_embeds
@@ -5061,6 +5116,39 @@ class DeepseekV4ForCausalLM(nn.Module):
def get_input_embeddings(self) -> nn.Module:
return self.model.get_input_embeddings()
def prepare_model_inputs(
self,
input_ids: torch.Tensor,
forward_batch: ForwardBatch,
input_embeds: Optional[torch.Tensor],
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
if self.vision is None:
return input_ids, input_embeds
has_images = (
not forward_batch.forward_mode.is_decode()
and not forward_batch.forward_mode.is_target_verify()
and forward_batch.mm_inputs is not None
and any(x is not None for x in forward_batch.mm_inputs)
)
if has_images and input_embeds is not None:
raise ValueError("Cannot combine input_embeds and image inputs")
mm_owner = self._mm_owner_session(forward_batch) if has_images else None
with mm_owner.fence() if mm_owner is not None else nullcontext():
if has_images:
input_embeds = self._prepare_mm_embeddings(
input_ids, forward_batch, mm_owner
)
if not (
forward_batch.forward_mode.is_decode_or_idle()
or forward_batch.forward_mode.is_target_verify()
):
# Decode/verify IDs are already vocabulary IDs; remap prompt image
# hashes for Engram and routing.
input_ids = input_ids.masked_fill(
input_ids >= MM_PAD_SHIFT_VALUE, self.config.image_token_id
)
return input_ids, input_embeds
def set_dspark_layers_to_capture(self, layer_ids: List[int]) -> None:
if not self.pp_group.is_last_rank:
return
@@ -5114,25 +5202,9 @@ class DeepseekV4ForCausalLM(nn.Module):
input_embeds: Optional[torch.Tensor] = None,
pp_proxy_tensors: Optional[PPProxyTensors] = None,
) -> torch.Tensor:
if (
self.vision is not None
and not forward_batch.forward_mode.is_decode()
and not forward_batch.forward_mode.is_target_verify()
and forward_batch.mm_inputs is not None
and any(x is not None for x in forward_batch.mm_inputs)
):
if input_embeds is not None:
raise ValueError("Cannot combine input_embeds and image inputs")
input_embeds = self._prepare_mm_embeddings(input_ids, forward_batch)
if self.vision is not None and not (
forward_batch.forward_mode.is_decode_or_idle()
or forward_batch.forward_mode.is_target_verify()
):
# Decode/verify IDs are already vocabulary IDs; remap prompt image
# hashes for Engram and routing.
input_ids = input_ids.masked_fill(
input_ids >= MM_PAD_SHIFT_VALUE, self.config.image_token_id
)
input_ids, input_embeds = self.prepare_model_inputs(
input_ids=input_ids, forward_batch=forward_batch, input_embeds=input_embeds
)
with get_attn_tp_context().maybe_input_scattered(forward_batch):
hidden_states = self.model.forward(
@@ -222,6 +222,7 @@ class DeepseekV4ForCausalLMNextN(DeepseekV4ForCausalLM):
self.quant_config = quant_config
self.wo_a_fp8 = wo_a_fp8_gemm_enabled(quant_config)
self.determine_num_fused_shared_experts()
self.vision = None
self.model = DeepseekV4ModelNextN(
config, quant_config, prefix=add_prefix("model", prefix)