PR #40219: DeepSeek V4.1 vision CP and single-owner image encoding
Source: https://github.com/sgl-project/sglang/pull/40219 (applied via GitHub .diff, 3way) Adaptations for v0.5.21 base: - attn_dp_enabled_of(cfg) -> cfg.enable_dp_attention (#41818 not in base) - dropped _validate_joint_schema_request call (#42721 Clef, not in base) - moved Backend import before the TRT-LLM prefill-graph check
This commit is contained in:
@@ -172,15 +172,25 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
)
|
||||
|
||||
cfg = resolving_view(server_args)
|
||||
if model_config_of(server_args).hf_config.model_type != "deepseek_v41":
|
||||
hf_config = model_config_of(server_args).hf_config
|
||||
if hf_config.model_type != "deepseek_v41":
|
||||
if cfg.enable_encoder_swa_bounded_replay:
|
||||
raise ValueError(
|
||||
"--enable-encoder-swa-bounded-replay requires DeepSeek-V4.1"
|
||||
)
|
||||
return
|
||||
if cfg.enable_encoder_swa_bounded_replay:
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend
|
||||
|
||||
if hf_config.vision_n_layers > 0 and cfg.enable_prefill_cp:
|
||||
_validate_deepseek_v41_vision_prefill_cp(server_args)
|
||||
if (
|
||||
cfg.dsv4_attn_backend == "trtllm"
|
||||
and cfg.cuda_graph_config.prefill.backend != Backend.DISABLED
|
||||
):
|
||||
raise ValueError(
|
||||
"DeepSeek-V4.1 TRT-LLM requires --cuda-graph-backend-prefill disabled"
|
||||
)
|
||||
if cfg.enable_encoder_swa_bounded_replay:
|
||||
incompatible = (
|
||||
("non-CUDA hardware", not get_platform().is_cuda),
|
||||
(
|
||||
@@ -188,7 +198,8 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
cfg.cuda_graph_config.prefill.backend != Backend.DISABLED,
|
||||
),
|
||||
("DP attention", cfg.enable_dp_attention),
|
||||
("context parallelism", cfg.attn_cp_size > 1),
|
||||
# Prefill CP declares attn_cp_size and DP attention only later.
|
||||
("context parallelism", cfg.attn_cp_size > 1 or cfg.enable_prefill_cp),
|
||||
("external cache linker", cfg.enable_unified_cache_external_linker),
|
||||
("unified memory", cfg.enable_unified_memory),
|
||||
("PD disaggregation", cfg.disaggregation_mode != "null"),
|
||||
@@ -287,3 +298,37 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
"--enable-decoder-swa-bounded-replay cannot be combined with "
|
||||
f"{feature} yet; disable one of them."
|
||||
)
|
||||
|
||||
|
||||
def _validate_deepseek_v41_vision_prefill_cp(server_args: ServerArgs) -> None:
|
||||
cfg = resolving_view(server_args)
|
||||
if cfg.cp_strategy != "interleave":
|
||||
raise ValueError(
|
||||
"DeepSeek-V4.1 vision with prefill CP requires --cp-strategy "
|
||||
f"interleave; got {cfg.cp_strategy!r}."
|
||||
)
|
||||
if cfg.cuda_graph_config.prefill.backend != Backend.DISABLED:
|
||||
locked = getattr(server_args, "_cuda_graph_config_locked", set())
|
||||
if (Phase.PREFILL, "backend") in locked:
|
||||
raise ValueError(
|
||||
"DeepSeek-V4.1 vision with prefill CP runs eager prefill; remove "
|
||||
"the explicit prefill CUDA graph backend."
|
||||
)
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"validate_deepseek_v41_features",
|
||||
cuda_graph_config=with_phase(
|
||||
cfg.cuda_graph_config, Phase.PREFILL, backend=Backend.DISABLED
|
||||
),
|
||||
)
|
||||
logger.warning(
|
||||
"Disabling the prefill CUDA graph for DeepSeek-V4.1 vision with prefill CP."
|
||||
)
|
||||
if (
|
||||
str(cfg.speculative_algorithm).upper() == "DSPARK"
|
||||
and cfg.enable_decoder_swa_bounded_replay
|
||||
):
|
||||
raise ValueError(
|
||||
"DeepSeek-V4.1 vision with prefill CP does not support DSpark together "
|
||||
"with --enable-decoder-swa-bounded-replay yet."
|
||||
)
|
||||
|
||||
@@ -0,0 +1,464 @@
|
||||
"""One owner rank encodes each image span and broadcasts it to the ranks that
|
||||
run the same prefill chunk; every agreement precedes the payload it guards."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from contextlib import contextmanager
|
||||
from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence, Tuple
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
disable_symmetric_memory_context,
|
||||
restore_symmetric_memory_context,
|
||||
)
|
||||
from sglang.srt.mem_cache.multimodal_cache import EmbeddingResult, MultiModalStaticCache
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SpanKey = Tuple[Optional[int], int]
|
||||
SpanEncoder = Callable[[List[Any]], torch.Tensor | List[torch.Tensor]]
|
||||
|
||||
LOCAL_HIT = 0
|
||||
OWNER_CACHE_BROADCAST = 1
|
||||
OWNER_ENCODE_BROADCAST = 2
|
||||
|
||||
PHASE_PREPARE = "prepare"
|
||||
PHASE_FEATURES = "features"
|
||||
PHASE_FINALIZE = "finalize"
|
||||
|
||||
|
||||
class MmOwnerProtocolError(RuntimeError):
|
||||
"""Raised with identical text on every group member after a group-agreed failure."""
|
||||
|
||||
|
||||
class ImageSpanRequest(msgspec.Struct, frozen=True):
|
||||
hash: Optional[int]
|
||||
span_len: int
|
||||
item: Any
|
||||
duplicates: List[Any] = []
|
||||
|
||||
|
||||
class ImageSpanKey(msgspec.Struct, frozen=True):
|
||||
hash: Optional[int]
|
||||
span_len: int
|
||||
geometry: Optional[Tuple[Any, ...]]
|
||||
|
||||
|
||||
class RankManifest(msgspec.Struct, frozen=True):
|
||||
rank: int
|
||||
keys: List[ImageSpanKey]
|
||||
cached: List[bool]
|
||||
dtype: str
|
||||
width: int
|
||||
rids: List[str]
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
class OwnerPlan(msgspec.Struct, frozen=True):
|
||||
actions: List[int] = []
|
||||
owners: List[int] = []
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
class RankStatus(msgspec.Struct, frozen=True):
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
def select_owner_group(parallel) -> Optional[Any]:
|
||||
"""The group whose members all execute the same requests; None keeps each
|
||||
rank encoding its own images."""
|
||||
replication = parallel.tp_size // parallel.attn_dp_size
|
||||
if replication <= 1:
|
||||
return None
|
||||
if parallel.attn_cp_size == 1:
|
||||
group = parallel.attn_tp_group
|
||||
elif parallel.attn_dp_size == 1 and parallel.attn_cp_size == parallel.tp_size:
|
||||
group = parallel.attn_cp_group
|
||||
else:
|
||||
return None
|
||||
return group if group.world_size == replication else None
|
||||
|
||||
|
||||
def has_owner_span_work(
|
||||
mm_inputs: Sequence[Any],
|
||||
extend_prefix_lens: Sequence[int],
|
||||
extend_seq_lens: Sequence[int],
|
||||
) -> bool:
|
||||
"""Host-side mirror of the per-image scheduling path: does any raw
|
||||
single-span image overlap the chunk on every rank of the group."""
|
||||
for mm_input, prefix_len, extend_len in zip(
|
||||
mm_inputs, extend_prefix_lens, extend_seq_lens
|
||||
):
|
||||
if mm_input is None or extend_len <= 0:
|
||||
continue
|
||||
items = [item for item in mm_input.mm_items if item is not None]
|
||||
if any(
|
||||
item.precomputed_embeddings is not None or len(item.offsets) != 1
|
||||
for item in items
|
||||
):
|
||||
continue
|
||||
for item in items:
|
||||
start, end = item.offsets[0]
|
||||
if end >= prefix_len and start < prefix_len + extend_len:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class MmOwnerSession(msgspec.Struct):
|
||||
group: Any
|
||||
device: Any
|
||||
dtype: Any
|
||||
width: int
|
||||
rids: List[str]
|
||||
signature: Any
|
||||
phase: str = PHASE_PREPARE
|
||||
in_collective: bool = False
|
||||
|
||||
def resolve(
|
||||
self,
|
||||
requests: Sequence[ImageSpanRequest],
|
||||
cache: MultiModalStaticCache,
|
||||
encode: SpanEncoder,
|
||||
) -> Dict[SpanKey, torch.Tensor]:
|
||||
# Owners allocate different amounts than receivers, so none of these
|
||||
# buffers may come out of a symmetric pool.
|
||||
saved_context = disable_symmetric_memory_context()
|
||||
try:
|
||||
return _resolve_owner_features(self, requests, cache, encode)
|
||||
finally:
|
||||
restore_symmetric_memory_context(saved_context)
|
||||
|
||||
def features_ready(self) -> None:
|
||||
self._complete()
|
||||
self.phase = PHASE_FINALIZE
|
||||
|
||||
@contextmanager
|
||||
def uncaptured(self) -> Iterator[None]:
|
||||
# A failure inside a collective leaves the group in an unknown state;
|
||||
# no later exchange may try to agree on it.
|
||||
self.in_collective = True
|
||||
yield
|
||||
self.in_collective = False
|
||||
|
||||
@contextmanager
|
||||
def fence(self) -> Iterator[None]:
|
||||
try:
|
||||
yield
|
||||
except Exception as exc:
|
||||
self._fail(exc)
|
||||
raise
|
||||
self._complete()
|
||||
|
||||
def _fail(self, exc: BaseException) -> None:
|
||||
if self.in_collective or isinstance(exc, MmOwnerProtocolError):
|
||||
raise exc
|
||||
text = _describe(self, self.phase, exc)
|
||||
if self.phase == PHASE_PREPARE:
|
||||
try:
|
||||
_exchange_manifest(self, _manifest(self, [], [], error=text))
|
||||
except MmOwnerProtocolError as agreed:
|
||||
raise agreed from exc
|
||||
_exchange_status(self, text, exc)
|
||||
|
||||
def _complete(self) -> None:
|
||||
if self.phase == PHASE_PREPARE:
|
||||
raise RuntimeError(
|
||||
f"owner protocol {self.phase} completed without a manifest exchange"
|
||||
)
|
||||
_drain_and_agree(self, self.phase)
|
||||
|
||||
|
||||
def _manifest(
|
||||
session: MmOwnerSession,
|
||||
keys: List[ImageSpanKey],
|
||||
cached: List[bool],
|
||||
error: Optional[str] = None,
|
||||
) -> RankManifest:
|
||||
return RankManifest(
|
||||
rank=session.group.rank_in_group,
|
||||
keys=keys,
|
||||
cached=cached,
|
||||
dtype=str(session.dtype),
|
||||
width=session.width,
|
||||
rids=list(session.rids),
|
||||
error=error,
|
||||
)
|
||||
|
||||
|
||||
def _exchange_manifest(session: MmOwnerSession, manifest: RankManifest) -> OwnerPlan:
|
||||
group = session.group
|
||||
with session.uncaptured():
|
||||
manifests = group.all_gather_object(manifest)
|
||||
plan = _plan_or_error(session, manifests) if group.rank_in_group == 0 else None
|
||||
plan = group.broadcast_object(plan, src=0)
|
||||
session.phase = PHASE_FEATURES
|
||||
if plan.error is not None:
|
||||
raise MmOwnerProtocolError(plan.error)
|
||||
return plan
|
||||
|
||||
|
||||
def _plan_or_error(session: MmOwnerSession, manifests: List[RankManifest]) -> OwnerPlan:
|
||||
try:
|
||||
return _make_plan(manifests)
|
||||
except Exception as exc:
|
||||
return OwnerPlan(error=_describe(session, "plan", exc))
|
||||
|
||||
|
||||
def _exchange_status(
|
||||
session: MmOwnerSession, error: Optional[str], cause: Optional[BaseException]
|
||||
) -> None:
|
||||
with session.uncaptured():
|
||||
statuses = session.group.all_gather_object(RankStatus(error=error))
|
||||
for status in statuses:
|
||||
if status.error is not None:
|
||||
raise MmOwnerProtocolError(status.error) from cause
|
||||
|
||||
|
||||
def _drain_and_agree(
|
||||
session: MmOwnerSession, stage: str, step: Optional[Callable[[], Any]] = None
|
||||
) -> Any:
|
||||
result, error, cause = None, None, None
|
||||
try:
|
||||
result = step() if step is not None else None
|
||||
_synchronize(session.device)
|
||||
except Exception as exc:
|
||||
error, cause = _describe(session, stage, exc), exc
|
||||
_exchange_status(session, error, cause)
|
||||
return result
|
||||
|
||||
|
||||
def _resolve_owner_features(
|
||||
session: MmOwnerSession,
|
||||
requests: Sequence[ImageSpanRequest],
|
||||
cache: MultiModalStaticCache,
|
||||
encode: SpanEncoder,
|
||||
) -> Dict[SpanKey, torch.Tensor]:
|
||||
group = session.group
|
||||
features: Dict[SpanKey, torch.Tensor] = {}
|
||||
keys: List[ImageSpanKey] = []
|
||||
cached: List[bool] = []
|
||||
error = None
|
||||
try:
|
||||
keys, cached = _pin_local_cache(session, requests, cache, features)
|
||||
except Exception as exc:
|
||||
error = _describe(session, "manifest", exc)
|
||||
plan = _exchange_manifest(session, _manifest(session, keys, cached, error))
|
||||
|
||||
if all(action == LOCAL_HIT for action in plan.actions):
|
||||
return features
|
||||
|
||||
buffers = _drain_and_agree(
|
||||
session,
|
||||
"encode",
|
||||
lambda: _prepare_transfers(session, requests, keys, plan, features, encode),
|
||||
)
|
||||
|
||||
with session.uncaptured():
|
||||
for index, (action, owner) in enumerate(zip(plan.actions, plan.owners)):
|
||||
if action != LOCAL_HIT:
|
||||
group.broadcast(buffers[index], src=owner)
|
||||
|
||||
for index, key in enumerate(keys):
|
||||
if plan.actions[index] == LOCAL_HIT:
|
||||
continue
|
||||
span = buffers[index]
|
||||
features[(key.hash, key.span_len)] = span
|
||||
cache.set(key.hash, EmbeddingResult(embedding=span))
|
||||
return features
|
||||
|
||||
|
||||
def _pin_local_cache(
|
||||
session: MmOwnerSession,
|
||||
requests: Sequence[ImageSpanRequest],
|
||||
cache: MultiModalStaticCache,
|
||||
features: Dict[SpanKey, torch.Tensor],
|
||||
) -> Tuple[List[ImageSpanKey], List[bool]]:
|
||||
keys: List[ImageSpanKey] = []
|
||||
cached: List[bool] = []
|
||||
for request in requests:
|
||||
if request.hash is None:
|
||||
raise ValueError(
|
||||
f"image span of {request.span_len} tokens has no content hash"
|
||||
)
|
||||
geometry = session.signature(request.item, request.span_len)
|
||||
for duplicate in request.duplicates:
|
||||
other = session.signature(duplicate, request.span_len)
|
||||
if other != geometry:
|
||||
raise ValueError(
|
||||
f"image hash {request.hash} ({request.span_len} tokens) occurs "
|
||||
f"with different geometry: {geometry} vs {other}"
|
||||
)
|
||||
keys.append(
|
||||
ImageSpanKey(
|
||||
hash=request.hash, span_len=request.span_len, geometry=geometry
|
||||
)
|
||||
)
|
||||
span = _valid_cached_span(session, cache, request)
|
||||
if span is not None:
|
||||
features[(request.hash, request.span_len)] = span
|
||||
cached.append(span is not None)
|
||||
return keys, cached
|
||||
|
||||
|
||||
def _valid_cached_span(
|
||||
session: MmOwnerSession,
|
||||
cache: MultiModalStaticCache,
|
||||
request: ImageSpanRequest,
|
||||
) -> Optional[torch.Tensor]:
|
||||
entry = cache.get_single(request.hash)
|
||||
if entry is None:
|
||||
return None
|
||||
span = entry.embedding
|
||||
if (
|
||||
span.dim() == 2
|
||||
and span.shape[0] == request.span_len
|
||||
and span.shape[1] == session.width
|
||||
and span.dtype == session.dtype
|
||||
and span.device == session.device
|
||||
):
|
||||
return span
|
||||
logger.warning(
|
||||
"Discarding cached multimodal embedding that cannot serve the current "
|
||||
"image span: cache_key=%s expected=(%d, %d, %s) cached=(%s, %s).",
|
||||
request.hash,
|
||||
request.span_len,
|
||||
session.width,
|
||||
session.dtype,
|
||||
tuple(span.shape),
|
||||
span.dtype,
|
||||
)
|
||||
cache.free(request.hash, None)
|
||||
return None
|
||||
|
||||
|
||||
def _make_plan(manifests: List[RankManifest]) -> OwnerPlan:
|
||||
for manifest in manifests:
|
||||
if manifest.error is not None:
|
||||
return OwnerPlan(error=manifest.error)
|
||||
lead = manifests[0]
|
||||
for manifest in manifests[1:]:
|
||||
if (manifest.keys, manifest.dtype, manifest.width, manifest.rids) != (
|
||||
lead.keys,
|
||||
lead.dtype,
|
||||
lead.width,
|
||||
lead.rids,
|
||||
):
|
||||
return OwnerPlan(
|
||||
error=(
|
||||
"image manifest mismatch between group ranks 0 and "
|
||||
f"{manifest.rank}: rids={lead.rids} vs {manifest.rids}, "
|
||||
f"keys={lead.keys} vs {manifest.keys}, "
|
||||
f"dtype={lead.dtype} vs {manifest.dtype}, "
|
||||
f"width={lead.width} vs {manifest.width}"
|
||||
),
|
||||
)
|
||||
replication = len(manifests)
|
||||
actions: List[int] = []
|
||||
owners: List[int] = []
|
||||
for index, key in enumerate(lead.keys):
|
||||
owner = key.hash % replication
|
||||
if all(manifest.cached[index] for manifest in manifests):
|
||||
action = LOCAL_HIT
|
||||
elif manifests[owner].cached[index]:
|
||||
action = OWNER_CACHE_BROADCAST
|
||||
else:
|
||||
action = OWNER_ENCODE_BROADCAST
|
||||
actions.append(action)
|
||||
owners.append(owner)
|
||||
return OwnerPlan(actions=actions, owners=owners)
|
||||
|
||||
|
||||
def _prepare_transfers(
|
||||
session: MmOwnerSession,
|
||||
requests: Sequence[ImageSpanRequest],
|
||||
keys: List[ImageSpanKey],
|
||||
plan: OwnerPlan,
|
||||
features: Dict[SpanKey, torch.Tensor],
|
||||
encode: SpanEncoder,
|
||||
) -> Dict[int, torch.Tensor]:
|
||||
rank = session.group.rank_in_group
|
||||
buffers: Dict[int, torch.Tensor] = {}
|
||||
owned: List[int] = []
|
||||
for index, (action, owner) in enumerate(zip(plan.actions, plan.owners)):
|
||||
if action == LOCAL_HIT:
|
||||
continue
|
||||
if owner != rank:
|
||||
try:
|
||||
buffers[index] = _new_span_buffer(session, keys[index])
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
f"receive buffer for image hash {keys[index].hash} shape "
|
||||
f"{(keys[index].span_len, session.width)} {session.dtype} "
|
||||
f"failed: {type(exc).__name__}: {exc}"
|
||||
) from exc
|
||||
elif action == OWNER_CACHE_BROADCAST:
|
||||
key = (keys[index].hash, keys[index].span_len)
|
||||
buffers[index] = features[key].contiguous()
|
||||
else:
|
||||
owned.append(index)
|
||||
if owned:
|
||||
try:
|
||||
encoded = encode([requests[index].item for index in owned])
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
f"owner encode of image hashes {[keys[i].hash for i in owned]} failed: "
|
||||
f"{type(exc).__name__}: {exc}"
|
||||
) from exc
|
||||
spans = _split_spans(encoded, [keys[index].span_len for index in owned])
|
||||
for index, span in zip(owned, spans):
|
||||
buffers[index] = _validated_span(session, keys[index], span)
|
||||
return buffers
|
||||
|
||||
|
||||
def _new_span_buffer(session: MmOwnerSession, key: ImageSpanKey) -> torch.Tensor:
|
||||
return torch.empty(
|
||||
(key.span_len, session.width), device=session.device, dtype=session.dtype
|
||||
)
|
||||
|
||||
|
||||
def _split_spans(
|
||||
encoded: torch.Tensor | List[torch.Tensor], span_lens: List[int]
|
||||
) -> List[torch.Tensor]:
|
||||
if isinstance(encoded, list):
|
||||
if len(encoded) != len(span_lens):
|
||||
raise ValueError(
|
||||
f"encoder returned {len(encoded)} spans for {len(span_lens)} images"
|
||||
)
|
||||
return [span.reshape(-1, span.shape[-1]) for span in encoded]
|
||||
encoded = encoded.reshape(-1, encoded.shape[-1])
|
||||
if encoded.shape[0] != sum(span_lens):
|
||||
raise ValueError(
|
||||
f"encoder returned {encoded.shape[0]} rows for spans of {span_lens}"
|
||||
)
|
||||
return list(torch.split(encoded, span_lens, dim=0))
|
||||
|
||||
|
||||
def _validated_span(
|
||||
session: MmOwnerSession, key: ImageSpanKey, span: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
expected = (key.span_len, session.width)
|
||||
if tuple(span.shape) != expected or span.dtype != session.dtype:
|
||||
raise ValueError(
|
||||
f"encoded span for hash={key.hash} has shape {tuple(span.shape)} "
|
||||
f"dtype {span.dtype}; expected {expected} {session.dtype}"
|
||||
)
|
||||
if span.device != session.device:
|
||||
span = span.to(session.device)
|
||||
return span.contiguous()
|
||||
|
||||
|
||||
def _synchronize(device) -> None:
|
||||
if device.type == "cuda":
|
||||
torch.cuda.current_stream(device).synchronize()
|
||||
|
||||
|
||||
def _describe(session: MmOwnerSession, stage: str, exc: BaseException) -> str:
|
||||
return (
|
||||
f"multimodal owner protocol failed during {stage} on group rank "
|
||||
f"{session.group.rank_in_group} (global rank "
|
||||
f"{session.group.ranks[session.group.rank_in_group]}, rids={list(session.rids)}): "
|
||||
f"{type(exc).__name__}: {exc}"
|
||||
)
|
||||
@@ -5,6 +5,7 @@ from typing import Callable, Dict, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.managers.mm_owner_embedding import ImageSpanRequest, MmOwnerSession
|
||||
from sglang.srt.managers.schedule_batch import MultimodalDataItem
|
||||
from sglang.srt.mem_cache.multimodal_cache import EmbeddingResult, MultiModalStaticCache
|
||||
from sglang.srt.multimodal.evs import EVSEmbeddingResult
|
||||
@@ -430,6 +431,37 @@ def _batch_encode_per_image_misses(
|
||||
return hash_to_embedding
|
||||
|
||||
|
||||
def _collect_image_span_requests(
|
||||
per_image_requests: List[PerImageRequestInfo],
|
||||
) -> List[ImageSpanRequest]:
|
||||
spans: Dict[Tuple[Optional[int], int], ImageSpanRequest] = {}
|
||||
for req_info in per_image_requests:
|
||||
chunk_start = req_info.extend_prefix_len
|
||||
chunk_end = chunk_start + req_info.extend_seq_len # exclusive
|
||||
req_info.overlapping = [
|
||||
(idx, item, token_count)
|
||||
for idx, item in enumerate(req_info.items)
|
||||
if (token_count := _item_overlap(item, chunk_start, chunk_end)) is not None
|
||||
]
|
||||
for _idx, item, token_count in req_info.overlapping:
|
||||
span = spans.get((item.hash, token_count))
|
||||
if span is None:
|
||||
spans[(item.hash, token_count)] = ImageSpanRequest(
|
||||
hash=item.hash, span_len=token_count, item=item
|
||||
)
|
||||
elif span.item is not item:
|
||||
span.duplicates.append(item)
|
||||
return list(spans.values())
|
||||
|
||||
|
||||
def _owner_span_encoder(data_embedding_func: DataEmbeddingFunc, device: torch.device):
|
||||
def encode(items: List[MultimodalDataItem]):
|
||||
_move_items_to_device(items, device, data_embedding_func)
|
||||
return data_embedding_func(items)
|
||||
|
||||
return encode
|
||||
|
||||
|
||||
def _get_chunked_embedding_by_item(
|
||||
data_embedding_func: DataEmbeddingFunc,
|
||||
embedding_items_per_req: List[MultimodalDataItem],
|
||||
@@ -554,6 +586,7 @@ def _get_chunked_prefill_embedding(
|
||||
extend_length: List[int],
|
||||
items_offset_list: List[List[Tuple[int, int]]],
|
||||
input_ids: torch.Tensor,
|
||||
mm_owner: Optional[MmOwnerSession] = None,
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor]:
|
||||
"""
|
||||
Chunked prefill embedding: encode items across all requests and extract
|
||||
@@ -615,7 +648,15 @@ def _get_chunked_prefill_embedding(
|
||||
|
||||
# Phase 1: batch encode all per-image cache misses in ONE ViT call
|
||||
hash_to_embedding: Dict[Tuple[Optional[int], int], torch.Tensor] = {}
|
||||
if per_image_requests:
|
||||
if per_image_requests and mm_owner is not None:
|
||||
# The owner protocol must see every overlapping span before any local
|
||||
# cache filtering: a rank-local hit can never skip a group collective.
|
||||
hash_to_embedding = mm_owner.resolve(
|
||||
_collect_image_span_requests(per_image_requests),
|
||||
cache=embedding_cache,
|
||||
encode=_owner_span_encoder(data_embedding_func, device),
|
||||
)
|
||||
elif per_image_requests:
|
||||
hash_to_embedding = _batch_encode_per_image_misses(
|
||||
data_embedding_func, per_image_requests, device
|
||||
)
|
||||
@@ -718,6 +759,7 @@ def get_embedding_and_mask(
|
||||
prefix_length: List[int],
|
||||
extend_length: List[int],
|
||||
items_offset_list: List[List[Tuple[int, int]]],
|
||||
mm_owner: Optional[MmOwnerSession] = None,
|
||||
) -> Tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor]:
|
||||
"""
|
||||
Generate multimodal embeddings and create a mask for identifying their positions in the input sequence.
|
||||
@@ -758,6 +800,7 @@ def get_embedding_and_mask(
|
||||
extend_length,
|
||||
items_offset_list,
|
||||
input_ids,
|
||||
mm_owner=mm_owner,
|
||||
)
|
||||
if embedding is None:
|
||||
return None, None, input_ids
|
||||
|
||||
@@ -11,6 +11,7 @@ import sys
|
||||
from abc import abstractmethod
|
||||
from array import array
|
||||
from collections import defaultdict
|
||||
from contextlib import nullcontext
|
||||
from multiprocessing import shared_memory
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
@@ -26,6 +27,7 @@ from sglang.srt.managers.io_struct import (
|
||||
TokenizedEmbeddingReqInput,
|
||||
TokenizedGenerateReqInput,
|
||||
)
|
||||
from sglang.srt.managers.mm_owner_embedding import MmOwnerSession
|
||||
|
||||
# Preserve the existing initialization import for downstream callers.
|
||||
from sglang.srt.managers.mm_schedule import (
|
||||
@@ -416,6 +418,7 @@ def embed_mm_inputs(
|
||||
data_embedding_func_mapping: Dict[Modality, DataEmbeddingFunc] = None,
|
||||
placeholder_tokens: dict[Modality, List[int]] = None,
|
||||
use_deepstack: Dict[Modality, bool] = {},
|
||||
mm_owner: Optional[MmOwnerSession] = None,
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""
|
||||
Embed multimodal inputs and integrate them with text token embeddings.
|
||||
@@ -497,6 +500,7 @@ def embed_mm_inputs(
|
||||
prefix_length=extend_prefix_lens,
|
||||
extend_length=extend_seq_lens,
|
||||
items_offset_list=items_offsets,
|
||||
mm_owner=mm_owner,
|
||||
)
|
||||
|
||||
if use_deepstack.get(modality, None) and embedding is not None:
|
||||
@@ -517,7 +521,12 @@ def embed_mm_inputs(
|
||||
# filled with the hash values of the multimodal for the prefix matching in the radix attention.
|
||||
# There values are useless because their embeddings will be replaced by vision embeddings anyway.
|
||||
input_ids.clamp_(min=0, max=vocab_size - 1)
|
||||
input_embeds = input_embedding(input_ids)
|
||||
if mm_owner is not None:
|
||||
# The text embedding may all-reduce across TP; a rank-local failure in
|
||||
# feature preparation has to be agreed on before any rank enters it.
|
||||
mm_owner.features_ready()
|
||||
with mm_owner.uncaptured() if mm_owner is not None else nullcontext():
|
||||
input_embeds = input_embedding(input_ids)
|
||||
|
||||
# deepstack embedding
|
||||
if use_deepstack:
|
||||
|
||||
@@ -1111,6 +1111,15 @@ class PrefillAdder:
|
||||
else AddReqResult.CONTINUE
|
||||
)
|
||||
|
||||
def can_share_extend_batch(self, req: Req) -> bool:
|
||||
# Token embedding overrides embed the batch's raw input_ids before the
|
||||
# model runs, and that lookup cannot index multimodal placeholder hash IDs.
|
||||
if req.positional_embed_overrides is not None:
|
||||
return all(r.multimodal_inputs is None for r in self.can_run_list)
|
||||
if req.multimodal_inputs is not None:
|
||||
return all(r.positional_embed_overrides is None for r in self.can_run_list)
|
||||
return True
|
||||
|
||||
def add_chunked_req(self, req: Req):
|
||||
if self.dllm_config is not None:
|
||||
_rem_tokens = self._get_dllm_remain_tokens(req)
|
||||
|
||||
@@ -3954,6 +3954,8 @@ class Scheduler(
|
||||
for req in self.waiting_queue:
|
||||
if self.enable_lora and not self.can_schedule_lora_req(req, running_loras):
|
||||
continue
|
||||
if not adder.can_share_extend_batch(req):
|
||||
break
|
||||
|
||||
# A forward batch runs one pooling mode, so setwise readout requests
|
||||
# (token_indices_to_pool) cannot share a batch with last-token ones.
|
||||
|
||||
@@ -1283,6 +1283,16 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
raise ValueError(
|
||||
"encoder SWA replay cannot return cached prompt logprobs"
|
||||
)
|
||||
requests_embed_overrides = obj.positional_embed_overrides is not None or (
|
||||
isinstance(obj, EmbeddingReqInput)
|
||||
and obj.embed_overrides is not None
|
||||
and obj.embed_override_token_id is not None
|
||||
)
|
||||
if requests_embed_overrides and obj.contains_mm_input():
|
||||
raise ValueError(
|
||||
"embedding overrides cannot be combined with image, video, or audio "
|
||||
"inputs"
|
||||
)
|
||||
_max_req_len = self.context_len
|
||||
input_token_num = len(input_ids) if input_ids is not None else 0
|
||||
input_token_num += self.num_reserved_tokens
|
||||
|
||||
@@ -1670,6 +1670,7 @@ class ModelRunner:
|
||||
forward_batch.replace_embeds is not None
|
||||
and forward_batch.replace_positions is not None
|
||||
):
|
||||
misc_utils.validate_replace_embeds_batch(forward_batch)
|
||||
# Token embedding overrides: get base embeddings, scatter replacements
|
||||
if "input_embeds" not in kwargs:
|
||||
embed_layer = self.model.get_input_embeddings()
|
||||
|
||||
@@ -19,6 +19,7 @@ from sglang.srt.server_args import CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACK
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.configs.model_config import ModelConfig
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -99,6 +100,26 @@ def resolve_pp_proxy_residual_num_blocks(
|
||||
return (start_layer + block_size - 1) // block_size
|
||||
|
||||
|
||||
def validate_replace_embeds_batch(forward_batch: ForwardBatch) -> None:
|
||||
if forward_batch.mm_inputs is None:
|
||||
return
|
||||
for mm_inputs, prefix_len, extend_len in zip(
|
||||
forward_batch.mm_inputs,
|
||||
forward_batch.extend_prefix_lens_cpu,
|
||||
forward_batch.extend_seq_lens_cpu,
|
||||
):
|
||||
if mm_inputs is None:
|
||||
continue
|
||||
chunk_end = prefix_len + extend_len
|
||||
for item in mm_inputs.mm_items:
|
||||
for start, end in item.offsets or ():
|
||||
if start < chunk_end and end >= prefix_len:
|
||||
raise ValueError(
|
||||
"Token embedding overrides cannot share an extend batch with "
|
||||
"multimodal placeholders"
|
||||
)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class _SupportsDSparkPPProxy(Protocol):
|
||||
def get_pp_proxy_dspark_hidden_size(self) -> int: ...
|
||||
|
||||
@@ -384,18 +384,34 @@ class EagerRunner(BaseRunner):
|
||||
"""
|
||||
model = self.model_runner.model
|
||||
|
||||
input_embeds = kwargs.get("input_embeds")
|
||||
if input_embeds is None:
|
||||
input_embeds = model.get_input_embeddings()(forward_batch.input_ids)
|
||||
with cp_shard_model_inputs(
|
||||
input_embeds,
|
||||
forward_batch.positions,
|
||||
forward_batch,
|
||||
forward_batch.input_ids,
|
||||
) as (sharded_input_embeds, sharded_positions, model_input_ids):
|
||||
model_kwargs = {"input_embeds": sharded_input_embeds}
|
||||
input_ids = forward_batch.input_ids
|
||||
if prepare_inputs := getattr(model, "prepare_cp_inputs", None):
|
||||
input_embeds, positions, model_kwargs = prepare_inputs(
|
||||
forward_batch, **kwargs
|
||||
)
|
||||
else:
|
||||
input_embeds = kwargs.get("input_embeds")
|
||||
if hasattr(model, "prepare_model_inputs"):
|
||||
# Multimodal offsets are request-global, so the merge and the
|
||||
# placeholder-ID remap must see the full extend layout first.
|
||||
input_ids, input_embeds = model.prepare_model_inputs(
|
||||
input_ids=input_ids,
|
||||
forward_batch=forward_batch,
|
||||
input_embeds=input_embeds,
|
||||
)
|
||||
if input_embeds is None:
|
||||
input_embeds = model.get_input_embeddings()(input_ids)
|
||||
positions = forward_batch.positions
|
||||
model_kwargs = {}
|
||||
if (pp_proxy_tensors := kwargs.get("pp_proxy_tensors")) is not None:
|
||||
model_kwargs["pp_proxy_tensors"] = pp_proxy_tensors
|
||||
with cp_shard_model_inputs(
|
||||
input_embeds,
|
||||
positions,
|
||||
forward_batch,
|
||||
input_ids,
|
||||
) as (sharded_input_embeds, sharded_positions, model_input_ids):
|
||||
model_kwargs["input_embeds"] = sharded_input_embeds
|
||||
hidden_states = model.model(
|
||||
model_input_ids,
|
||||
sharded_positions,
|
||||
@@ -436,7 +452,7 @@ class EagerRunner(BaseRunner):
|
||||
if aux_hidden_states is None:
|
||||
logits_kwargs["hidden_states_before_norm"] = hidden_states_before_norm
|
||||
return model.logits_processor(
|
||||
forward_batch.input_ids,
|
||||
input_ids,
|
||||
hidden_states,
|
||||
model.lm_head,
|
||||
forward_batch,
|
||||
|
||||
@@ -70,6 +70,7 @@ from sglang.srt.layers.attention.dsv4.dsv41_sparse import (
|
||||
DeepseekV41Indexer,
|
||||
)
|
||||
from sglang.srt.layers.attention.dsv4.indexer import C4Indexer
|
||||
from sglang.srt.layers.cp.base import is_zigzag
|
||||
from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx
|
||||
from sglang.srt.layers.cp.utils import (
|
||||
cp_gather_full_sequence_states,
|
||||
@@ -121,6 +122,11 @@ from sglang.srt.layers.quantization.mxfp8_input import Mxfp8SwizzledInput
|
||||
from sglang.srt.layers.rotary_embedding import get_rope_wrapper
|
||||
from sglang.srt.layers.utils import PPMissingLayer, get_layer_id
|
||||
from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from sglang.srt.managers.mm_owner_embedding import (
|
||||
MmOwnerSession,
|
||||
has_owner_span_work,
|
||||
select_owner_group,
|
||||
)
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
embed_mm_inputs,
|
||||
@@ -4893,13 +4899,21 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
self.determine_num_fused_shared_experts()
|
||||
self.vision = None
|
||||
if config.model_type == "deepseek_v41" and config.vision_n_layers > 0:
|
||||
if (
|
||||
get_parallel().attn_cp_size != 1
|
||||
or get_parallel().pp_group.world_size != 1
|
||||
or not get_moe_a2a_backend().is_none()
|
||||
if get_parallel().pp_group.world_size != 1 or not (
|
||||
get_moe_a2a_backend().is_none()
|
||||
or (_is_cuda and get_moe_a2a_backend().is_megamoe())
|
||||
):
|
||||
raise ValueError(
|
||||
"V4.1 vision currently supports TP/EP/DP without CP, PP or MoE A2A"
|
||||
"V4.1 vision supports TP/EP/DP without PP; "
|
||||
"MoE A2A requires CUDA MegaMoE"
|
||||
)
|
||||
if get_parallel().attn_cp_size != 1 and (
|
||||
_is_npu or is_zigzag() or not get_moe_a2a_backend().is_none()
|
||||
):
|
||||
raise ValueError(
|
||||
"V4.1 vision context parallelism requires the CUDA interleave "
|
||||
"strategy without MoE A2A; NPU, zigzag CP and MegaMoE are not "
|
||||
"supported yet"
|
||||
)
|
||||
|
||||
args = SimpleNamespace(**vars(config), dim=config.hidden_size)
|
||||
@@ -4908,6 +4922,11 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
self.image_start = nn.Parameter(torch.empty(config.hidden_size))
|
||||
self.image_end = nn.Parameter(torch.empty(config.hidden_size))
|
||||
self.image_newline = nn.Parameter(torch.empty(config.hidden_size))
|
||||
self.mm_owner_group = (
|
||||
select_owner_group(get_parallel())
|
||||
if self.vision is not None and _is_cuda
|
||||
else None
|
||||
)
|
||||
self.model = DeepseekV4Model(
|
||||
config, quant_config, prefix=add_prefix("model", prefix)
|
||||
)
|
||||
@@ -5042,7 +5061,42 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
spans.append(span)
|
||||
return spans
|
||||
|
||||
def _prepare_mm_embeddings(self, input_ids, forward_batch):
|
||||
def _image_span_signature(self, item, span_len: int):
|
||||
h, w = int(item.n_vit_h), int(item.n_vit_w)
|
||||
r = self.config.vision_downsample_ratio
|
||||
lh, lw = (h + r - 1) // r, (w + r - 1) // r
|
||||
expected = 2 + lh * (lw + 1)
|
||||
if expected != span_len:
|
||||
raise ValueError(
|
||||
f"image grid {(h, w)} yields {expected} span tokens, "
|
||||
f"placeholder has {span_len}"
|
||||
)
|
||||
plan = item.model_specific_data.get(GPU_PLAN_KEY)
|
||||
feature = item.feature
|
||||
return (
|
||||
h,
|
||||
w,
|
||||
tuple(feature.shape) if isinstance(feature, torch.Tensor) else None,
|
||||
None if plan is None else tuple(sorted(plan.items())),
|
||||
)
|
||||
|
||||
def _mm_owner_session(self, forward_batch) -> Optional[MmOwnerSession]:
|
||||
if self.mm_owner_group is None or not has_owner_span_work(
|
||||
forward_batch.mm_inputs,
|
||||
forward_batch.extend_prefix_lens_cpu,
|
||||
forward_batch.extend_seq_lens_cpu,
|
||||
):
|
||||
return None
|
||||
return MmOwnerSession(
|
||||
group=self.mm_owner_group,
|
||||
device=self.image_start.device,
|
||||
dtype=self.image_start.dtype,
|
||||
width=self.config.hidden_size,
|
||||
rids=list(forward_batch.rids or ()),
|
||||
signature=self._image_span_signature,
|
||||
)
|
||||
|
||||
def _prepare_mm_embeddings(self, input_ids, forward_batch, mm_owner):
|
||||
# Keep scheduler hash IDs intact: the shared embedder clamps its input in place.
|
||||
input_embeds, _ = embed_mm_inputs(
|
||||
mm_inputs_list=[
|
||||
@@ -5054,6 +5108,7 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
input_ids=input_ids.clone(),
|
||||
input_embedding=self.get_input_embeddings(),
|
||||
multimodal_model=self,
|
||||
mm_owner=mm_owner,
|
||||
)
|
||||
forward_batch.mm_input_embeds = input_embeds
|
||||
return input_embeds
|
||||
@@ -5061,6 +5116,39 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
def get_input_embeddings(self) -> nn.Module:
|
||||
return self.model.get_input_embeddings()
|
||||
|
||||
def prepare_model_inputs(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
input_embeds: Optional[torch.Tensor],
|
||||
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
if self.vision is None:
|
||||
return input_ids, input_embeds
|
||||
has_images = (
|
||||
not forward_batch.forward_mode.is_decode()
|
||||
and not forward_batch.forward_mode.is_target_verify()
|
||||
and forward_batch.mm_inputs is not None
|
||||
and any(x is not None for x in forward_batch.mm_inputs)
|
||||
)
|
||||
if has_images and input_embeds is not None:
|
||||
raise ValueError("Cannot combine input_embeds and image inputs")
|
||||
mm_owner = self._mm_owner_session(forward_batch) if has_images else None
|
||||
with mm_owner.fence() if mm_owner is not None else nullcontext():
|
||||
if has_images:
|
||||
input_embeds = self._prepare_mm_embeddings(
|
||||
input_ids, forward_batch, mm_owner
|
||||
)
|
||||
if not (
|
||||
forward_batch.forward_mode.is_decode_or_idle()
|
||||
or forward_batch.forward_mode.is_target_verify()
|
||||
):
|
||||
# Decode/verify IDs are already vocabulary IDs; remap prompt image
|
||||
# hashes for Engram and routing.
|
||||
input_ids = input_ids.masked_fill(
|
||||
input_ids >= MM_PAD_SHIFT_VALUE, self.config.image_token_id
|
||||
)
|
||||
return input_ids, input_embeds
|
||||
|
||||
def set_dspark_layers_to_capture(self, layer_ids: List[int]) -> None:
|
||||
if not self.pp_group.is_last_rank:
|
||||
return
|
||||
@@ -5114,25 +5202,9 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
input_embeds: Optional[torch.Tensor] = None,
|
||||
pp_proxy_tensors: Optional[PPProxyTensors] = None,
|
||||
) -> torch.Tensor:
|
||||
if (
|
||||
self.vision is not None
|
||||
and not forward_batch.forward_mode.is_decode()
|
||||
and not forward_batch.forward_mode.is_target_verify()
|
||||
and forward_batch.mm_inputs is not None
|
||||
and any(x is not None for x in forward_batch.mm_inputs)
|
||||
):
|
||||
if input_embeds is not None:
|
||||
raise ValueError("Cannot combine input_embeds and image inputs")
|
||||
input_embeds = self._prepare_mm_embeddings(input_ids, forward_batch)
|
||||
if self.vision is not None and not (
|
||||
forward_batch.forward_mode.is_decode_or_idle()
|
||||
or forward_batch.forward_mode.is_target_verify()
|
||||
):
|
||||
# Decode/verify IDs are already vocabulary IDs; remap prompt image
|
||||
# hashes for Engram and routing.
|
||||
input_ids = input_ids.masked_fill(
|
||||
input_ids >= MM_PAD_SHIFT_VALUE, self.config.image_token_id
|
||||
)
|
||||
input_ids, input_embeds = self.prepare_model_inputs(
|
||||
input_ids=input_ids, forward_batch=forward_batch, input_embeds=input_embeds
|
||||
)
|
||||
|
||||
with get_attn_tp_context().maybe_input_scattered(forward_batch):
|
||||
hidden_states = self.model.forward(
|
||||
|
||||
@@ -222,6 +222,7 @@ class DeepseekV4ForCausalLMNextN(DeepseekV4ForCausalLM):
|
||||
self.quant_config = quant_config
|
||||
self.wo_a_fp8 = wo_a_fp8_gemm_enabled(quant_config)
|
||||
self.determine_num_fused_shared_experts()
|
||||
self.vision = None
|
||||
|
||||
self.model = DeepseekV4ModelNextN(
|
||||
config, quant_config, prefix=add_prefix("model", prefix)
|
||||
|
||||
Reference in New Issue
Block a user