PR #40219: DeepSeek V4.1 vision CP and single-owner image encoding
Source: https://github.com/sgl-project/sglang/pull/40219 (applied via GitHub .diff, 3way) Adaptations for v0.5.21 base: - attn_dp_enabled_of(cfg) -> cfg.enable_dp_attention (#41818 not in base) - dropped _validate_joint_schema_request call (#42721 Clef, not in base) - moved Backend import before the TRT-LLM prefill-graph check
This commit is contained in:
@@ -172,15 +172,25 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
)
|
||||
|
||||
cfg = resolving_view(server_args)
|
||||
if model_config_of(server_args).hf_config.model_type != "deepseek_v41":
|
||||
hf_config = model_config_of(server_args).hf_config
|
||||
if hf_config.model_type != "deepseek_v41":
|
||||
if cfg.enable_encoder_swa_bounded_replay:
|
||||
raise ValueError(
|
||||
"--enable-encoder-swa-bounded-replay requires DeepSeek-V4.1"
|
||||
)
|
||||
return
|
||||
if cfg.enable_encoder_swa_bounded_replay:
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend
|
||||
|
||||
if hf_config.vision_n_layers > 0 and cfg.enable_prefill_cp:
|
||||
_validate_deepseek_v41_vision_prefill_cp(server_args)
|
||||
if (
|
||||
cfg.dsv4_attn_backend == "trtllm"
|
||||
and cfg.cuda_graph_config.prefill.backend != Backend.DISABLED
|
||||
):
|
||||
raise ValueError(
|
||||
"DeepSeek-V4.1 TRT-LLM requires --cuda-graph-backend-prefill disabled"
|
||||
)
|
||||
if cfg.enable_encoder_swa_bounded_replay:
|
||||
incompatible = (
|
||||
("non-CUDA hardware", not get_platform().is_cuda),
|
||||
(
|
||||
@@ -188,7 +198,8 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
cfg.cuda_graph_config.prefill.backend != Backend.DISABLED,
|
||||
),
|
||||
("DP attention", cfg.enable_dp_attention),
|
||||
("context parallelism", cfg.attn_cp_size > 1),
|
||||
# Prefill CP declares attn_cp_size and DP attention only later.
|
||||
("context parallelism", cfg.attn_cp_size > 1 or cfg.enable_prefill_cp),
|
||||
("external cache linker", cfg.enable_unified_cache_external_linker),
|
||||
("unified memory", cfg.enable_unified_memory),
|
||||
("PD disaggregation", cfg.disaggregation_mode != "null"),
|
||||
@@ -287,3 +298,37 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
"--enable-decoder-swa-bounded-replay cannot be combined with "
|
||||
f"{feature} yet; disable one of them."
|
||||
)
|
||||
|
||||
|
||||
def _validate_deepseek_v41_vision_prefill_cp(server_args: ServerArgs) -> None:
|
||||
cfg = resolving_view(server_args)
|
||||
if cfg.cp_strategy != "interleave":
|
||||
raise ValueError(
|
||||
"DeepSeek-V4.1 vision with prefill CP requires --cp-strategy "
|
||||
f"interleave; got {cfg.cp_strategy!r}."
|
||||
)
|
||||
if cfg.cuda_graph_config.prefill.backend != Backend.DISABLED:
|
||||
locked = getattr(server_args, "_cuda_graph_config_locked", set())
|
||||
if (Phase.PREFILL, "backend") in locked:
|
||||
raise ValueError(
|
||||
"DeepSeek-V4.1 vision with prefill CP runs eager prefill; remove "
|
||||
"the explicit prefill CUDA graph backend."
|
||||
)
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"validate_deepseek_v41_features",
|
||||
cuda_graph_config=with_phase(
|
||||
cfg.cuda_graph_config, Phase.PREFILL, backend=Backend.DISABLED
|
||||
),
|
||||
)
|
||||
logger.warning(
|
||||
"Disabling the prefill CUDA graph for DeepSeek-V4.1 vision with prefill CP."
|
||||
)
|
||||
if (
|
||||
str(cfg.speculative_algorithm).upper() == "DSPARK"
|
||||
and cfg.enable_decoder_swa_bounded_replay
|
||||
):
|
||||
raise ValueError(
|
||||
"DeepSeek-V4.1 vision with prefill CP does not support DSpark together "
|
||||
"with --enable-decoder-swa-bounded-replay yet."
|
||||
)
|
||||
|
||||
@@ -0,0 +1,464 @@
|
||||
"""One owner rank encodes each image span and broadcasts it to the ranks that
|
||||
run the same prefill chunk; every agreement precedes the payload it guards."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from contextlib import contextmanager
|
||||
from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence, Tuple
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
disable_symmetric_memory_context,
|
||||
restore_symmetric_memory_context,
|
||||
)
|
||||
from sglang.srt.mem_cache.multimodal_cache import EmbeddingResult, MultiModalStaticCache
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SpanKey = Tuple[Optional[int], int]
|
||||
SpanEncoder = Callable[[List[Any]], torch.Tensor | List[torch.Tensor]]
|
||||
|
||||
LOCAL_HIT = 0
|
||||
OWNER_CACHE_BROADCAST = 1
|
||||
OWNER_ENCODE_BROADCAST = 2
|
||||
|
||||
PHASE_PREPARE = "prepare"
|
||||
PHASE_FEATURES = "features"
|
||||
PHASE_FINALIZE = "finalize"
|
||||
|
||||
|
||||
class MmOwnerProtocolError(RuntimeError):
|
||||
"""Raised with identical text on every group member after a group-agreed failure."""
|
||||
|
||||
|
||||
class ImageSpanRequest(msgspec.Struct, frozen=True):
|
||||
hash: Optional[int]
|
||||
span_len: int
|
||||
item: Any
|
||||
duplicates: List[Any] = []
|
||||
|
||||
|
||||
class ImageSpanKey(msgspec.Struct, frozen=True):
|
||||
hash: Optional[int]
|
||||
span_len: int
|
||||
geometry: Optional[Tuple[Any, ...]]
|
||||
|
||||
|
||||
class RankManifest(msgspec.Struct, frozen=True):
|
||||
rank: int
|
||||
keys: List[ImageSpanKey]
|
||||
cached: List[bool]
|
||||
dtype: str
|
||||
width: int
|
||||
rids: List[str]
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
class OwnerPlan(msgspec.Struct, frozen=True):
|
||||
actions: List[int] = []
|
||||
owners: List[int] = []
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
class RankStatus(msgspec.Struct, frozen=True):
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
def select_owner_group(parallel) -> Optional[Any]:
|
||||
"""The group whose members all execute the same requests; None keeps each
|
||||
rank encoding its own images."""
|
||||
replication = parallel.tp_size // parallel.attn_dp_size
|
||||
if replication <= 1:
|
||||
return None
|
||||
if parallel.attn_cp_size == 1:
|
||||
group = parallel.attn_tp_group
|
||||
elif parallel.attn_dp_size == 1 and parallel.attn_cp_size == parallel.tp_size:
|
||||
group = parallel.attn_cp_group
|
||||
else:
|
||||
return None
|
||||
return group if group.world_size == replication else None
|
||||
|
||||
|
||||
def has_owner_span_work(
|
||||
mm_inputs: Sequence[Any],
|
||||
extend_prefix_lens: Sequence[int],
|
||||
extend_seq_lens: Sequence[int],
|
||||
) -> bool:
|
||||
"""Host-side mirror of the per-image scheduling path: does any raw
|
||||
single-span image overlap the chunk on every rank of the group."""
|
||||
for mm_input, prefix_len, extend_len in zip(
|
||||
mm_inputs, extend_prefix_lens, extend_seq_lens
|
||||
):
|
||||
if mm_input is None or extend_len <= 0:
|
||||
continue
|
||||
items = [item for item in mm_input.mm_items if item is not None]
|
||||
if any(
|
||||
item.precomputed_embeddings is not None or len(item.offsets) != 1
|
||||
for item in items
|
||||
):
|
||||
continue
|
||||
for item in items:
|
||||
start, end = item.offsets[0]
|
||||
if end >= prefix_len and start < prefix_len + extend_len:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class MmOwnerSession(msgspec.Struct):
|
||||
group: Any
|
||||
device: Any
|
||||
dtype: Any
|
||||
width: int
|
||||
rids: List[str]
|
||||
signature: Any
|
||||
phase: str = PHASE_PREPARE
|
||||
in_collective: bool = False
|
||||
|
||||
def resolve(
|
||||
self,
|
||||
requests: Sequence[ImageSpanRequest],
|
||||
cache: MultiModalStaticCache,
|
||||
encode: SpanEncoder,
|
||||
) -> Dict[SpanKey, torch.Tensor]:
|
||||
# Owners allocate different amounts than receivers, so none of these
|
||||
# buffers may come out of a symmetric pool.
|
||||
saved_context = disable_symmetric_memory_context()
|
||||
try:
|
||||
return _resolve_owner_features(self, requests, cache, encode)
|
||||
finally:
|
||||
restore_symmetric_memory_context(saved_context)
|
||||
|
||||
def features_ready(self) -> None:
|
||||
self._complete()
|
||||
self.phase = PHASE_FINALIZE
|
||||
|
||||
@contextmanager
|
||||
def uncaptured(self) -> Iterator[None]:
|
||||
# A failure inside a collective leaves the group in an unknown state;
|
||||
# no later exchange may try to agree on it.
|
||||
self.in_collective = True
|
||||
yield
|
||||
self.in_collective = False
|
||||
|
||||
@contextmanager
|
||||
def fence(self) -> Iterator[None]:
|
||||
try:
|
||||
yield
|
||||
except Exception as exc:
|
||||
self._fail(exc)
|
||||
raise
|
||||
self._complete()
|
||||
|
||||
def _fail(self, exc: BaseException) -> None:
|
||||
if self.in_collective or isinstance(exc, MmOwnerProtocolError):
|
||||
raise exc
|
||||
text = _describe(self, self.phase, exc)
|
||||
if self.phase == PHASE_PREPARE:
|
||||
try:
|
||||
_exchange_manifest(self, _manifest(self, [], [], error=text))
|
||||
except MmOwnerProtocolError as agreed:
|
||||
raise agreed from exc
|
||||
_exchange_status(self, text, exc)
|
||||
|
||||
def _complete(self) -> None:
|
||||
if self.phase == PHASE_PREPARE:
|
||||
raise RuntimeError(
|
||||
f"owner protocol {self.phase} completed without a manifest exchange"
|
||||
)
|
||||
_drain_and_agree(self, self.phase)
|
||||
|
||||
|
||||
def _manifest(
|
||||
session: MmOwnerSession,
|
||||
keys: List[ImageSpanKey],
|
||||
cached: List[bool],
|
||||
error: Optional[str] = None,
|
||||
) -> RankManifest:
|
||||
return RankManifest(
|
||||
rank=session.group.rank_in_group,
|
||||
keys=keys,
|
||||
cached=cached,
|
||||
dtype=str(session.dtype),
|
||||
width=session.width,
|
||||
rids=list(session.rids),
|
||||
error=error,
|
||||
)
|
||||
|
||||
|
||||
def _exchange_manifest(session: MmOwnerSession, manifest: RankManifest) -> OwnerPlan:
|
||||
group = session.group
|
||||
with session.uncaptured():
|
||||
manifests = group.all_gather_object(manifest)
|
||||
plan = _plan_or_error(session, manifests) if group.rank_in_group == 0 else None
|
||||
plan = group.broadcast_object(plan, src=0)
|
||||
session.phase = PHASE_FEATURES
|
||||
if plan.error is not None:
|
||||
raise MmOwnerProtocolError(plan.error)
|
||||
return plan
|
||||
|
||||
|
||||
def _plan_or_error(session: MmOwnerSession, manifests: List[RankManifest]) -> OwnerPlan:
|
||||
try:
|
||||
return _make_plan(manifests)
|
||||
except Exception as exc:
|
||||
return OwnerPlan(error=_describe(session, "plan", exc))
|
||||
|
||||
|
||||
def _exchange_status(
|
||||
session: MmOwnerSession, error: Optional[str], cause: Optional[BaseException]
|
||||
) -> None:
|
||||
with session.uncaptured():
|
||||
statuses = session.group.all_gather_object(RankStatus(error=error))
|
||||
for status in statuses:
|
||||
if status.error is not None:
|
||||
raise MmOwnerProtocolError(status.error) from cause
|
||||
|
||||
|
||||
def _drain_and_agree(
|
||||
session: MmOwnerSession, stage: str, step: Optional[Callable[[], Any]] = None
|
||||
) -> Any:
|
||||
result, error, cause = None, None, None
|
||||
try:
|
||||
result = step() if step is not None else None
|
||||
_synchronize(session.device)
|
||||
except Exception as exc:
|
||||
error, cause = _describe(session, stage, exc), exc
|
||||
_exchange_status(session, error, cause)
|
||||
return result
|
||||
|
||||
|
||||
def _resolve_owner_features(
|
||||
session: MmOwnerSession,
|
||||
requests: Sequence[ImageSpanRequest],
|
||||
cache: MultiModalStaticCache,
|
||||
encode: SpanEncoder,
|
||||
) -> Dict[SpanKey, torch.Tensor]:
|
||||
group = session.group
|
||||
features: Dict[SpanKey, torch.Tensor] = {}
|
||||
keys: List[ImageSpanKey] = []
|
||||
cached: List[bool] = []
|
||||
error = None
|
||||
try:
|
||||
keys, cached = _pin_local_cache(session, requests, cache, features)
|
||||
except Exception as exc:
|
||||
error = _describe(session, "manifest", exc)
|
||||
plan = _exchange_manifest(session, _manifest(session, keys, cached, error))
|
||||
|
||||
if all(action == LOCAL_HIT for action in plan.actions):
|
||||
return features
|
||||
|
||||
buffers = _drain_and_agree(
|
||||
session,
|
||||
"encode",
|
||||
lambda: _prepare_transfers(session, requests, keys, plan, features, encode),
|
||||
)
|
||||
|
||||
with session.uncaptured():
|
||||
for index, (action, owner) in enumerate(zip(plan.actions, plan.owners)):
|
||||
if action != LOCAL_HIT:
|
||||
group.broadcast(buffers[index], src=owner)
|
||||
|
||||
for index, key in enumerate(keys):
|
||||
if plan.actions[index] == LOCAL_HIT:
|
||||
continue
|
||||
span = buffers[index]
|
||||
features[(key.hash, key.span_len)] = span
|
||||
cache.set(key.hash, EmbeddingResult(embedding=span))
|
||||
return features
|
||||
|
||||
|
||||
def _pin_local_cache(
|
||||
session: MmOwnerSession,
|
||||
requests: Sequence[ImageSpanRequest],
|
||||
cache: MultiModalStaticCache,
|
||||
features: Dict[SpanKey, torch.Tensor],
|
||||
) -> Tuple[List[ImageSpanKey], List[bool]]:
|
||||
keys: List[ImageSpanKey] = []
|
||||
cached: List[bool] = []
|
||||
for request in requests:
|
||||
if request.hash is None:
|
||||
raise ValueError(
|
||||
f"image span of {request.span_len} tokens has no content hash"
|
||||
)
|
||||
geometry = session.signature(request.item, request.span_len)
|
||||
for duplicate in request.duplicates:
|
||||
other = session.signature(duplicate, request.span_len)
|
||||
if other != geometry:
|
||||
raise ValueError(
|
||||
f"image hash {request.hash} ({request.span_len} tokens) occurs "
|
||||
f"with different geometry: {geometry} vs {other}"
|
||||
)
|
||||
keys.append(
|
||||
ImageSpanKey(
|
||||
hash=request.hash, span_len=request.span_len, geometry=geometry
|
||||
)
|
||||
)
|
||||
span = _valid_cached_span(session, cache, request)
|
||||
if span is not None:
|
||||
features[(request.hash, request.span_len)] = span
|
||||
cached.append(span is not None)
|
||||
return keys, cached
|
||||
|
||||
|
||||
def _valid_cached_span(
|
||||
session: MmOwnerSession,
|
||||
cache: MultiModalStaticCache,
|
||||
request: ImageSpanRequest,
|
||||
) -> Optional[torch.Tensor]:
|
||||
entry = cache.get_single(request.hash)
|
||||
if entry is None:
|
||||
return None
|
||||
span = entry.embedding
|
||||
if (
|
||||
span.dim() == 2
|
||||
and span.shape[0] == request.span_len
|
||||
and span.shape[1] == session.width
|
||||
and span.dtype == session.dtype
|
||||
and span.device == session.device
|
||||
):
|
||||
return span
|
||||
logger.warning(
|
||||
"Discarding cached multimodal embedding that cannot serve the current "
|
||||
"image span: cache_key=%s expected=(%d, %d, %s) cached=(%s, %s).",
|
||||
request.hash,
|
||||
request.span_len,
|
||||
session.width,
|
||||
session.dtype,
|
||||
tuple(span.shape),
|
||||
span.dtype,
|
||||
)
|
||||
cache.free(request.hash, None)
|
||||
return None
|
||||
|
||||
|
||||
def _make_plan(manifests: List[RankManifest]) -> OwnerPlan:
|
||||
for manifest in manifests:
|
||||
if manifest.error is not None:
|
||||
return OwnerPlan(error=manifest.error)
|
||||
lead = manifests[0]
|
||||
for manifest in manifests[1:]:
|
||||
if (manifest.keys, manifest.dtype, manifest.width, manifest.rids) != (
|
||||
lead.keys,
|
||||
lead.dtype,
|
||||
lead.width,
|
||||
lead.rids,
|
||||
):
|
||||
return OwnerPlan(
|
||||
error=(
|
||||
"image manifest mismatch between group ranks 0 and "
|
||||
f"{manifest.rank}: rids={lead.rids} vs {manifest.rids}, "
|
||||
f"keys={lead.keys} vs {manifest.keys}, "
|
||||
f"dtype={lead.dtype} vs {manifest.dtype}, "
|
||||
f"width={lead.width} vs {manifest.width}"
|
||||
),
|
||||
)
|
||||
replication = len(manifests)
|
||||
actions: List[int] = []
|
||||
owners: List[int] = []
|
||||
for index, key in enumerate(lead.keys):
|
||||
owner = key.hash % replication
|
||||
if all(manifest.cached[index] for manifest in manifests):
|
||||
action = LOCAL_HIT
|
||||
elif manifests[owner].cached[index]:
|
||||
action = OWNER_CACHE_BROADCAST
|
||||
else:
|
||||
action = OWNER_ENCODE_BROADCAST
|
||||
actions.append(action)
|
||||
owners.append(owner)
|
||||
return OwnerPlan(actions=actions, owners=owners)
|
||||
|
||||
|
||||
def _prepare_transfers(
|
||||
session: MmOwnerSession,
|
||||
requests: Sequence[ImageSpanRequest],
|
||||
keys: List[ImageSpanKey],
|
||||
plan: OwnerPlan,
|
||||
features: Dict[SpanKey, torch.Tensor],
|
||||
encode: SpanEncoder,
|
||||
) -> Dict[int, torch.Tensor]:
|
||||
rank = session.group.rank_in_group
|
||||
buffers: Dict[int, torch.Tensor] = {}
|
||||
owned: List[int] = []
|
||||
for index, (action, owner) in enumerate(zip(plan.actions, plan.owners)):
|
||||
if action == LOCAL_HIT:
|
||||
continue
|
||||
if owner != rank:
|
||||
try:
|
||||
buffers[index] = _new_span_buffer(session, keys[index])
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
f"receive buffer for image hash {keys[index].hash} shape "
|
||||
f"{(keys[index].span_len, session.width)} {session.dtype} "
|
||||
f"failed: {type(exc).__name__}: {exc}"
|
||||
) from exc
|
||||
elif action == OWNER_CACHE_BROADCAST:
|
||||
key = (keys[index].hash, keys[index].span_len)
|
||||
buffers[index] = features[key].contiguous()
|
||||
else:
|
||||
owned.append(index)
|
||||
if owned:
|
||||
try:
|
||||
encoded = encode([requests[index].item for index in owned])
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
f"owner encode of image hashes {[keys[i].hash for i in owned]} failed: "
|
||||
f"{type(exc).__name__}: {exc}"
|
||||
) from exc
|
||||
spans = _split_spans(encoded, [keys[index].span_len for index in owned])
|
||||
for index, span in zip(owned, spans):
|
||||
buffers[index] = _validated_span(session, keys[index], span)
|
||||
return buffers
|
||||
|
||||
|
||||
def _new_span_buffer(session: MmOwnerSession, key: ImageSpanKey) -> torch.Tensor:
|
||||
return torch.empty(
|
||||
(key.span_len, session.width), device=session.device, dtype=session.dtype
|
||||
)
|
||||
|
||||
|
||||
def _split_spans(
|
||||
encoded: torch.Tensor | List[torch.Tensor], span_lens: List[int]
|
||||
) -> List[torch.Tensor]:
|
||||
if isinstance(encoded, list):
|
||||
if len(encoded) != len(span_lens):
|
||||
raise ValueError(
|
||||
f"encoder returned {len(encoded)} spans for {len(span_lens)} images"
|
||||
)
|
||||
return [span.reshape(-1, span.shape[-1]) for span in encoded]
|
||||
encoded = encoded.reshape(-1, encoded.shape[-1])
|
||||
if encoded.shape[0] != sum(span_lens):
|
||||
raise ValueError(
|
||||
f"encoder returned {encoded.shape[0]} rows for spans of {span_lens}"
|
||||
)
|
||||
return list(torch.split(encoded, span_lens, dim=0))
|
||||
|
||||
|
||||
def _validated_span(
|
||||
session: MmOwnerSession, key: ImageSpanKey, span: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
expected = (key.span_len, session.width)
|
||||
if tuple(span.shape) != expected or span.dtype != session.dtype:
|
||||
raise ValueError(
|
||||
f"encoded span for hash={key.hash} has shape {tuple(span.shape)} "
|
||||
f"dtype {span.dtype}; expected {expected} {session.dtype}"
|
||||
)
|
||||
if span.device != session.device:
|
||||
span = span.to(session.device)
|
||||
return span.contiguous()
|
||||
|
||||
|
||||
def _synchronize(device) -> None:
|
||||
if device.type == "cuda":
|
||||
torch.cuda.current_stream(device).synchronize()
|
||||
|
||||
|
||||
def _describe(session: MmOwnerSession, stage: str, exc: BaseException) -> str:
|
||||
return (
|
||||
f"multimodal owner protocol failed during {stage} on group rank "
|
||||
f"{session.group.rank_in_group} (global rank "
|
||||
f"{session.group.ranks[session.group.rank_in_group]}, rids={list(session.rids)}): "
|
||||
f"{type(exc).__name__}: {exc}"
|
||||
)
|
||||
@@ -5,6 +5,7 @@ from typing import Callable, Dict, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.managers.mm_owner_embedding import ImageSpanRequest, MmOwnerSession
|
||||
from sglang.srt.managers.schedule_batch import MultimodalDataItem
|
||||
from sglang.srt.mem_cache.multimodal_cache import EmbeddingResult, MultiModalStaticCache
|
||||
from sglang.srt.multimodal.evs import EVSEmbeddingResult
|
||||
@@ -430,6 +431,37 @@ def _batch_encode_per_image_misses(
|
||||
return hash_to_embedding
|
||||
|
||||
|
||||
def _collect_image_span_requests(
|
||||
per_image_requests: List[PerImageRequestInfo],
|
||||
) -> List[ImageSpanRequest]:
|
||||
spans: Dict[Tuple[Optional[int], int], ImageSpanRequest] = {}
|
||||
for req_info in per_image_requests:
|
||||
chunk_start = req_info.extend_prefix_len
|
||||
chunk_end = chunk_start + req_info.extend_seq_len # exclusive
|
||||
req_info.overlapping = [
|
||||
(idx, item, token_count)
|
||||
for idx, item in enumerate(req_info.items)
|
||||
if (token_count := _item_overlap(item, chunk_start, chunk_end)) is not None
|
||||
]
|
||||
for _idx, item, token_count in req_info.overlapping:
|
||||
span = spans.get((item.hash, token_count))
|
||||
if span is None:
|
||||
spans[(item.hash, token_count)] = ImageSpanRequest(
|
||||
hash=item.hash, span_len=token_count, item=item
|
||||
)
|
||||
elif span.item is not item:
|
||||
span.duplicates.append(item)
|
||||
return list(spans.values())
|
||||
|
||||
|
||||
def _owner_span_encoder(data_embedding_func: DataEmbeddingFunc, device: torch.device):
|
||||
def encode(items: List[MultimodalDataItem]):
|
||||
_move_items_to_device(items, device, data_embedding_func)
|
||||
return data_embedding_func(items)
|
||||
|
||||
return encode
|
||||
|
||||
|
||||
def _get_chunked_embedding_by_item(
|
||||
data_embedding_func: DataEmbeddingFunc,
|
||||
embedding_items_per_req: List[MultimodalDataItem],
|
||||
@@ -554,6 +586,7 @@ def _get_chunked_prefill_embedding(
|
||||
extend_length: List[int],
|
||||
items_offset_list: List[List[Tuple[int, int]]],
|
||||
input_ids: torch.Tensor,
|
||||
mm_owner: Optional[MmOwnerSession] = None,
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor]:
|
||||
"""
|
||||
Chunked prefill embedding: encode items across all requests and extract
|
||||
@@ -615,7 +648,15 @@ def _get_chunked_prefill_embedding(
|
||||
|
||||
# Phase 1: batch encode all per-image cache misses in ONE ViT call
|
||||
hash_to_embedding: Dict[Tuple[Optional[int], int], torch.Tensor] = {}
|
||||
if per_image_requests:
|
||||
if per_image_requests and mm_owner is not None:
|
||||
# The owner protocol must see every overlapping span before any local
|
||||
# cache filtering: a rank-local hit can never skip a group collective.
|
||||
hash_to_embedding = mm_owner.resolve(
|
||||
_collect_image_span_requests(per_image_requests),
|
||||
cache=embedding_cache,
|
||||
encode=_owner_span_encoder(data_embedding_func, device),
|
||||
)
|
||||
elif per_image_requests:
|
||||
hash_to_embedding = _batch_encode_per_image_misses(
|
||||
data_embedding_func, per_image_requests, device
|
||||
)
|
||||
@@ -718,6 +759,7 @@ def get_embedding_and_mask(
|
||||
prefix_length: List[int],
|
||||
extend_length: List[int],
|
||||
items_offset_list: List[List[Tuple[int, int]]],
|
||||
mm_owner: Optional[MmOwnerSession] = None,
|
||||
) -> Tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor]:
|
||||
"""
|
||||
Generate multimodal embeddings and create a mask for identifying their positions in the input sequence.
|
||||
@@ -758,6 +800,7 @@ def get_embedding_and_mask(
|
||||
extend_length,
|
||||
items_offset_list,
|
||||
input_ids,
|
||||
mm_owner=mm_owner,
|
||||
)
|
||||
if embedding is None:
|
||||
return None, None, input_ids
|
||||
|
||||
@@ -11,6 +11,7 @@ import sys
|
||||
from abc import abstractmethod
|
||||
from array import array
|
||||
from collections import defaultdict
|
||||
from contextlib import nullcontext
|
||||
from multiprocessing import shared_memory
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
@@ -26,6 +27,7 @@ from sglang.srt.managers.io_struct import (
|
||||
TokenizedEmbeddingReqInput,
|
||||
TokenizedGenerateReqInput,
|
||||
)
|
||||
from sglang.srt.managers.mm_owner_embedding import MmOwnerSession
|
||||
|
||||
# Preserve the existing initialization import for downstream callers.
|
||||
from sglang.srt.managers.mm_schedule import (
|
||||
@@ -416,6 +418,7 @@ def embed_mm_inputs(
|
||||
data_embedding_func_mapping: Dict[Modality, DataEmbeddingFunc] = None,
|
||||
placeholder_tokens: dict[Modality, List[int]] = None,
|
||||
use_deepstack: Dict[Modality, bool] = {},
|
||||
mm_owner: Optional[MmOwnerSession] = None,
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""
|
||||
Embed multimodal inputs and integrate them with text token embeddings.
|
||||
@@ -497,6 +500,7 @@ def embed_mm_inputs(
|
||||
prefix_length=extend_prefix_lens,
|
||||
extend_length=extend_seq_lens,
|
||||
items_offset_list=items_offsets,
|
||||
mm_owner=mm_owner,
|
||||
)
|
||||
|
||||
if use_deepstack.get(modality, None) and embedding is not None:
|
||||
@@ -517,7 +521,12 @@ def embed_mm_inputs(
|
||||
# filled with the hash values of the multimodal for the prefix matching in the radix attention.
|
||||
# There values are useless because their embeddings will be replaced by vision embeddings anyway.
|
||||
input_ids.clamp_(min=0, max=vocab_size - 1)
|
||||
input_embeds = input_embedding(input_ids)
|
||||
if mm_owner is not None:
|
||||
# The text embedding may all-reduce across TP; a rank-local failure in
|
||||
# feature preparation has to be agreed on before any rank enters it.
|
||||
mm_owner.features_ready()
|
||||
with mm_owner.uncaptured() if mm_owner is not None else nullcontext():
|
||||
input_embeds = input_embedding(input_ids)
|
||||
|
||||
# deepstack embedding
|
||||
if use_deepstack:
|
||||
|
||||
@@ -1111,6 +1111,15 @@ class PrefillAdder:
|
||||
else AddReqResult.CONTINUE
|
||||
)
|
||||
|
||||
def can_share_extend_batch(self, req: Req) -> bool:
|
||||
# Token embedding overrides embed the batch's raw input_ids before the
|
||||
# model runs, and that lookup cannot index multimodal placeholder hash IDs.
|
||||
if req.positional_embed_overrides is not None:
|
||||
return all(r.multimodal_inputs is None for r in self.can_run_list)
|
||||
if req.multimodal_inputs is not None:
|
||||
return all(r.positional_embed_overrides is None for r in self.can_run_list)
|
||||
return True
|
||||
|
||||
def add_chunked_req(self, req: Req):
|
||||
if self.dllm_config is not None:
|
||||
_rem_tokens = self._get_dllm_remain_tokens(req)
|
||||
|
||||
@@ -3954,6 +3954,8 @@ class Scheduler(
|
||||
for req in self.waiting_queue:
|
||||
if self.enable_lora and not self.can_schedule_lora_req(req, running_loras):
|
||||
continue
|
||||
if not adder.can_share_extend_batch(req):
|
||||
break
|
||||
|
||||
# A forward batch runs one pooling mode, so setwise readout requests
|
||||
# (token_indices_to_pool) cannot share a batch with last-token ones.
|
||||
|
||||
@@ -1283,6 +1283,16 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
raise ValueError(
|
||||
"encoder SWA replay cannot return cached prompt logprobs"
|
||||
)
|
||||
requests_embed_overrides = obj.positional_embed_overrides is not None or (
|
||||
isinstance(obj, EmbeddingReqInput)
|
||||
and obj.embed_overrides is not None
|
||||
and obj.embed_override_token_id is not None
|
||||
)
|
||||
if requests_embed_overrides and obj.contains_mm_input():
|
||||
raise ValueError(
|
||||
"embedding overrides cannot be combined with image, video, or audio "
|
||||
"inputs"
|
||||
)
|
||||
_max_req_len = self.context_len
|
||||
input_token_num = len(input_ids) if input_ids is not None else 0
|
||||
input_token_num += self.num_reserved_tokens
|
||||
|
||||
@@ -1670,6 +1670,7 @@ class ModelRunner:
|
||||
forward_batch.replace_embeds is not None
|
||||
and forward_batch.replace_positions is not None
|
||||
):
|
||||
misc_utils.validate_replace_embeds_batch(forward_batch)
|
||||
# Token embedding overrides: get base embeddings, scatter replacements
|
||||
if "input_embeds" not in kwargs:
|
||||
embed_layer = self.model.get_input_embeddings()
|
||||
|
||||
@@ -19,6 +19,7 @@ from sglang.srt.server_args import CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACK
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.configs.model_config import ModelConfig
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -99,6 +100,26 @@ def resolve_pp_proxy_residual_num_blocks(
|
||||
return (start_layer + block_size - 1) // block_size
|
||||
|
||||
|
||||
def validate_replace_embeds_batch(forward_batch: ForwardBatch) -> None:
|
||||
if forward_batch.mm_inputs is None:
|
||||
return
|
||||
for mm_inputs, prefix_len, extend_len in zip(
|
||||
forward_batch.mm_inputs,
|
||||
forward_batch.extend_prefix_lens_cpu,
|
||||
forward_batch.extend_seq_lens_cpu,
|
||||
):
|
||||
if mm_inputs is None:
|
||||
continue
|
||||
chunk_end = prefix_len + extend_len
|
||||
for item in mm_inputs.mm_items:
|
||||
for start, end in item.offsets or ():
|
||||
if start < chunk_end and end >= prefix_len:
|
||||
raise ValueError(
|
||||
"Token embedding overrides cannot share an extend batch with "
|
||||
"multimodal placeholders"
|
||||
)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class _SupportsDSparkPPProxy(Protocol):
|
||||
def get_pp_proxy_dspark_hidden_size(self) -> int: ...
|
||||
|
||||
@@ -384,18 +384,34 @@ class EagerRunner(BaseRunner):
|
||||
"""
|
||||
model = self.model_runner.model
|
||||
|
||||
input_embeds = kwargs.get("input_embeds")
|
||||
if input_embeds is None:
|
||||
input_embeds = model.get_input_embeddings()(forward_batch.input_ids)
|
||||
with cp_shard_model_inputs(
|
||||
input_embeds,
|
||||
forward_batch.positions,
|
||||
forward_batch,
|
||||
forward_batch.input_ids,
|
||||
) as (sharded_input_embeds, sharded_positions, model_input_ids):
|
||||
model_kwargs = {"input_embeds": sharded_input_embeds}
|
||||
input_ids = forward_batch.input_ids
|
||||
if prepare_inputs := getattr(model, "prepare_cp_inputs", None):
|
||||
input_embeds, positions, model_kwargs = prepare_inputs(
|
||||
forward_batch, **kwargs
|
||||
)
|
||||
else:
|
||||
input_embeds = kwargs.get("input_embeds")
|
||||
if hasattr(model, "prepare_model_inputs"):
|
||||
# Multimodal offsets are request-global, so the merge and the
|
||||
# placeholder-ID remap must see the full extend layout first.
|
||||
input_ids, input_embeds = model.prepare_model_inputs(
|
||||
input_ids=input_ids,
|
||||
forward_batch=forward_batch,
|
||||
input_embeds=input_embeds,
|
||||
)
|
||||
if input_embeds is None:
|
||||
input_embeds = model.get_input_embeddings()(input_ids)
|
||||
positions = forward_batch.positions
|
||||
model_kwargs = {}
|
||||
if (pp_proxy_tensors := kwargs.get("pp_proxy_tensors")) is not None:
|
||||
model_kwargs["pp_proxy_tensors"] = pp_proxy_tensors
|
||||
with cp_shard_model_inputs(
|
||||
input_embeds,
|
||||
positions,
|
||||
forward_batch,
|
||||
input_ids,
|
||||
) as (sharded_input_embeds, sharded_positions, model_input_ids):
|
||||
model_kwargs["input_embeds"] = sharded_input_embeds
|
||||
hidden_states = model.model(
|
||||
model_input_ids,
|
||||
sharded_positions,
|
||||
@@ -436,7 +452,7 @@ class EagerRunner(BaseRunner):
|
||||
if aux_hidden_states is None:
|
||||
logits_kwargs["hidden_states_before_norm"] = hidden_states_before_norm
|
||||
return model.logits_processor(
|
||||
forward_batch.input_ids,
|
||||
input_ids,
|
||||
hidden_states,
|
||||
model.lm_head,
|
||||
forward_batch,
|
||||
|
||||
@@ -70,6 +70,7 @@ from sglang.srt.layers.attention.dsv4.dsv41_sparse import (
|
||||
DeepseekV41Indexer,
|
||||
)
|
||||
from sglang.srt.layers.attention.dsv4.indexer import C4Indexer
|
||||
from sglang.srt.layers.cp.base import is_zigzag
|
||||
from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx
|
||||
from sglang.srt.layers.cp.utils import (
|
||||
cp_gather_full_sequence_states,
|
||||
@@ -121,6 +122,11 @@ from sglang.srt.layers.quantization.mxfp8_input import Mxfp8SwizzledInput
|
||||
from sglang.srt.layers.rotary_embedding import get_rope_wrapper
|
||||
from sglang.srt.layers.utils import PPMissingLayer, get_layer_id
|
||||
from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from sglang.srt.managers.mm_owner_embedding import (
|
||||
MmOwnerSession,
|
||||
has_owner_span_work,
|
||||
select_owner_group,
|
||||
)
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
embed_mm_inputs,
|
||||
@@ -4893,13 +4899,21 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
self.determine_num_fused_shared_experts()
|
||||
self.vision = None
|
||||
if config.model_type == "deepseek_v41" and config.vision_n_layers > 0:
|
||||
if (
|
||||
get_parallel().attn_cp_size != 1
|
||||
or get_parallel().pp_group.world_size != 1
|
||||
or not get_moe_a2a_backend().is_none()
|
||||
if get_parallel().pp_group.world_size != 1 or not (
|
||||
get_moe_a2a_backend().is_none()
|
||||
or (_is_cuda and get_moe_a2a_backend().is_megamoe())
|
||||
):
|
||||
raise ValueError(
|
||||
"V4.1 vision currently supports TP/EP/DP without CP, PP or MoE A2A"
|
||||
"V4.1 vision supports TP/EP/DP without PP; "
|
||||
"MoE A2A requires CUDA MegaMoE"
|
||||
)
|
||||
if get_parallel().attn_cp_size != 1 and (
|
||||
_is_npu or is_zigzag() or not get_moe_a2a_backend().is_none()
|
||||
):
|
||||
raise ValueError(
|
||||
"V4.1 vision context parallelism requires the CUDA interleave "
|
||||
"strategy without MoE A2A; NPU, zigzag CP and MegaMoE are not "
|
||||
"supported yet"
|
||||
)
|
||||
|
||||
args = SimpleNamespace(**vars(config), dim=config.hidden_size)
|
||||
@@ -4908,6 +4922,11 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
self.image_start = nn.Parameter(torch.empty(config.hidden_size))
|
||||
self.image_end = nn.Parameter(torch.empty(config.hidden_size))
|
||||
self.image_newline = nn.Parameter(torch.empty(config.hidden_size))
|
||||
self.mm_owner_group = (
|
||||
select_owner_group(get_parallel())
|
||||
if self.vision is not None and _is_cuda
|
||||
else None
|
||||
)
|
||||
self.model = DeepseekV4Model(
|
||||
config, quant_config, prefix=add_prefix("model", prefix)
|
||||
)
|
||||
@@ -5042,7 +5061,42 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
spans.append(span)
|
||||
return spans
|
||||
|
||||
def _prepare_mm_embeddings(self, input_ids, forward_batch):
|
||||
def _image_span_signature(self, item, span_len: int):
|
||||
h, w = int(item.n_vit_h), int(item.n_vit_w)
|
||||
r = self.config.vision_downsample_ratio
|
||||
lh, lw = (h + r - 1) // r, (w + r - 1) // r
|
||||
expected = 2 + lh * (lw + 1)
|
||||
if expected != span_len:
|
||||
raise ValueError(
|
||||
f"image grid {(h, w)} yields {expected} span tokens, "
|
||||
f"placeholder has {span_len}"
|
||||
)
|
||||
plan = item.model_specific_data.get(GPU_PLAN_KEY)
|
||||
feature = item.feature
|
||||
return (
|
||||
h,
|
||||
w,
|
||||
tuple(feature.shape) if isinstance(feature, torch.Tensor) else None,
|
||||
None if plan is None else tuple(sorted(plan.items())),
|
||||
)
|
||||
|
||||
def _mm_owner_session(self, forward_batch) -> Optional[MmOwnerSession]:
|
||||
if self.mm_owner_group is None or not has_owner_span_work(
|
||||
forward_batch.mm_inputs,
|
||||
forward_batch.extend_prefix_lens_cpu,
|
||||
forward_batch.extend_seq_lens_cpu,
|
||||
):
|
||||
return None
|
||||
return MmOwnerSession(
|
||||
group=self.mm_owner_group,
|
||||
device=self.image_start.device,
|
||||
dtype=self.image_start.dtype,
|
||||
width=self.config.hidden_size,
|
||||
rids=list(forward_batch.rids or ()),
|
||||
signature=self._image_span_signature,
|
||||
)
|
||||
|
||||
def _prepare_mm_embeddings(self, input_ids, forward_batch, mm_owner):
|
||||
# Keep scheduler hash IDs intact: the shared embedder clamps its input in place.
|
||||
input_embeds, _ = embed_mm_inputs(
|
||||
mm_inputs_list=[
|
||||
@@ -5054,6 +5108,7 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
input_ids=input_ids.clone(),
|
||||
input_embedding=self.get_input_embeddings(),
|
||||
multimodal_model=self,
|
||||
mm_owner=mm_owner,
|
||||
)
|
||||
forward_batch.mm_input_embeds = input_embeds
|
||||
return input_embeds
|
||||
@@ -5061,6 +5116,39 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
def get_input_embeddings(self) -> nn.Module:
|
||||
return self.model.get_input_embeddings()
|
||||
|
||||
def prepare_model_inputs(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
input_embeds: Optional[torch.Tensor],
|
||||
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
if self.vision is None:
|
||||
return input_ids, input_embeds
|
||||
has_images = (
|
||||
not forward_batch.forward_mode.is_decode()
|
||||
and not forward_batch.forward_mode.is_target_verify()
|
||||
and forward_batch.mm_inputs is not None
|
||||
and any(x is not None for x in forward_batch.mm_inputs)
|
||||
)
|
||||
if has_images and input_embeds is not None:
|
||||
raise ValueError("Cannot combine input_embeds and image inputs")
|
||||
mm_owner = self._mm_owner_session(forward_batch) if has_images else None
|
||||
with mm_owner.fence() if mm_owner is not None else nullcontext():
|
||||
if has_images:
|
||||
input_embeds = self._prepare_mm_embeddings(
|
||||
input_ids, forward_batch, mm_owner
|
||||
)
|
||||
if not (
|
||||
forward_batch.forward_mode.is_decode_or_idle()
|
||||
or forward_batch.forward_mode.is_target_verify()
|
||||
):
|
||||
# Decode/verify IDs are already vocabulary IDs; remap prompt image
|
||||
# hashes for Engram and routing.
|
||||
input_ids = input_ids.masked_fill(
|
||||
input_ids >= MM_PAD_SHIFT_VALUE, self.config.image_token_id
|
||||
)
|
||||
return input_ids, input_embeds
|
||||
|
||||
def set_dspark_layers_to_capture(self, layer_ids: List[int]) -> None:
|
||||
if not self.pp_group.is_last_rank:
|
||||
return
|
||||
@@ -5114,25 +5202,9 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
input_embeds: Optional[torch.Tensor] = None,
|
||||
pp_proxy_tensors: Optional[PPProxyTensors] = None,
|
||||
) -> torch.Tensor:
|
||||
if (
|
||||
self.vision is not None
|
||||
and not forward_batch.forward_mode.is_decode()
|
||||
and not forward_batch.forward_mode.is_target_verify()
|
||||
and forward_batch.mm_inputs is not None
|
||||
and any(x is not None for x in forward_batch.mm_inputs)
|
||||
):
|
||||
if input_embeds is not None:
|
||||
raise ValueError("Cannot combine input_embeds and image inputs")
|
||||
input_embeds = self._prepare_mm_embeddings(input_ids, forward_batch)
|
||||
if self.vision is not None and not (
|
||||
forward_batch.forward_mode.is_decode_or_idle()
|
||||
or forward_batch.forward_mode.is_target_verify()
|
||||
):
|
||||
# Decode/verify IDs are already vocabulary IDs; remap prompt image
|
||||
# hashes for Engram and routing.
|
||||
input_ids = input_ids.masked_fill(
|
||||
input_ids >= MM_PAD_SHIFT_VALUE, self.config.image_token_id
|
||||
)
|
||||
input_ids, input_embeds = self.prepare_model_inputs(
|
||||
input_ids=input_ids, forward_batch=forward_batch, input_embeds=input_embeds
|
||||
)
|
||||
|
||||
with get_attn_tp_context().maybe_input_scattered(forward_batch):
|
||||
hidden_states = self.model.forward(
|
||||
|
||||
@@ -222,6 +222,7 @@ class DeepseekV4ForCausalLMNextN(DeepseekV4ForCausalLM):
|
||||
self.quant_config = quant_config
|
||||
self.wo_a_fp8 = wo_a_fp8_gemm_enabled(quant_config)
|
||||
self.determine_num_fused_shared_experts()
|
||||
self.vision = None
|
||||
|
||||
self.model = DeepseekV4ModelNextN(
|
||||
config, quant_config, prefix=add_prefix("model", prefix)
|
||||
|
||||
@@ -9,6 +9,7 @@ Covers:
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import torch
|
||||
@@ -17,10 +18,16 @@ from sglang.srt.constants import MIS_DELIMITER_TOKEN_ID
|
||||
from sglang.srt.entrypoints.openai.utils import convert_embeds_to_tensors
|
||||
from sglang.srt.managers.embed_types import PositionalEmbeds
|
||||
from sglang.srt.managers.io_struct import EmbeddingReqInput, GenerateReqInput
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
MultimodalDataItem,
|
||||
MultimodalInputs,
|
||||
)
|
||||
from sglang.srt.managers.tokenizer_manager import TokenizerManager
|
||||
from sglang.srt.managers.tokenizer_manager_score_mixin import (
|
||||
TokenizerManagerScoreMixin,
|
||||
)
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
from sglang.srt.runtime_context import publish, reset_context
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
@@ -642,5 +649,80 @@ class TestScoreRequestValidation(CustomTestCase):
|
||||
)
|
||||
|
||||
|
||||
class TestEmbedOverridesRejectMultimodal(CustomTestCase):
|
||||
def setUp(self):
|
||||
reset_context()
|
||||
self.addCleanup(reset_context)
|
||||
publish(ServerArgs(model_path="dummy"), role="tokenizer")
|
||||
self.manager = TokenizerManager.__new__(TokenizerManager)
|
||||
self.manager.context_len = 128
|
||||
self.manager.num_reserved_tokens = 0
|
||||
self.manager.allow_auto_truncate = False
|
||||
self.manager.validate_total_tokens = False
|
||||
self.manager.model_config = SimpleNamespace(joint_head_config=None)
|
||||
|
||||
def test_requests_with_overrides_and_images_are_rejected(self):
|
||||
"""EmbeddingReqInput resolves embed_overrides only after validation, so
|
||||
the unresolved form must be caught at admission too."""
|
||||
overrides = PositionalEmbeds(embeds=[_vec()], positions=[1])
|
||||
for build in (
|
||||
lambda **mm: GenerateReqInput(
|
||||
input_ids=[10, 50, 20],
|
||||
sampling_params={},
|
||||
positional_embed_overrides=overrides,
|
||||
**mm,
|
||||
),
|
||||
lambda **mm: EmbeddingReqInput(
|
||||
input_ids=[10, 50, 20],
|
||||
sampling_params={},
|
||||
embed_override_token_id=50,
|
||||
embed_overrides=[_vec()],
|
||||
**mm,
|
||||
),
|
||||
):
|
||||
req = build(image_data=["image.png"])
|
||||
self.manager.is_generation = isinstance(req, GenerateReqInput)
|
||||
with self.assertRaisesRegex(ValueError, "overrides cannot be combined"):
|
||||
self.manager._validate_one_request(req, req.input_ids)
|
||||
text_only = build()
|
||||
self.manager._validate_one_request(text_only, text_only.input_ids)
|
||||
|
||||
def test_mixed_extend_batch_is_rejected_before_embedding_lookup(self):
|
||||
embed_layer = MagicMock(
|
||||
side_effect=AssertionError("embedding lookup must not run")
|
||||
)
|
||||
runner = SimpleNamespace(
|
||||
_pp_kwargs=lambda pp_proxy_tensors: {},
|
||||
model=SimpleNamespace(get_input_embeddings=lambda: embed_layer),
|
||||
is_generation=True,
|
||||
)
|
||||
image = MultimodalDataItem(
|
||||
modality=Modality.IMAGE, feature=torch.zeros(1), offsets=[(0, 1)]
|
||||
)
|
||||
image.set_hash(1234)
|
||||
forward_batch = SimpleNamespace(
|
||||
input_embeds=None,
|
||||
input_ids=torch.tensor([1, 2, image.pad_value, image.pad_value]),
|
||||
replace_embeds=torch.full((1, HIDDEN_DIM), 5.0),
|
||||
replace_positions=torch.tensor([0]),
|
||||
mm_inputs=[None, MultimodalInputs(mm_items=[image])],
|
||||
extend_prefix_lens_cpu=[0, 0],
|
||||
extend_seq_lens_cpu=[2, 2],
|
||||
)
|
||||
|
||||
with self.assertRaisesRegex(ValueError, "cannot share an extend batch"):
|
||||
ModelRunner._extend_forward_kwargs(runner, forward_batch, None)
|
||||
embed_layer.assert_not_called()
|
||||
|
||||
# A decoding image request in a mixed chunk has no placeholder rows here.
|
||||
forward_batch.input_ids = torch.tensor([1, 2, 3])
|
||||
forward_batch.extend_prefix_lens_cpu = [0, 5]
|
||||
forward_batch.extend_seq_lens_cpu = [2, 1]
|
||||
embed_layer.side_effect = None
|
||||
embed_layer.return_value = torch.zeros(3, HIDDEN_DIM)
|
||||
kwargs = ModelRunner._extend_forward_kwargs(runner, forward_batch, None)
|
||||
self.assertTrue(torch.equal(kwargs["input_embeds"][0], _vec(5.0)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -0,0 +1,693 @@
|
||||
"""Owner-encoded image spans: real gloo groups, production coordinator and entrypoints."""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from contextlib import ExitStack, contextmanager, nullcontext
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import torch.multiprocessing
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.distributed import parallel_state
|
||||
from sglang.srt.layers.cp.base import init_cp_strategy
|
||||
from sglang.srt.layers.cp.utils import prepare_cp_forward
|
||||
from sglang.srt.managers import mm_owner_embedding, mm_schedule
|
||||
from sglang.srt.managers.mm_owner_embedding import (
|
||||
MmOwnerProtocolError,
|
||||
select_owner_group,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
MultimodalDataItem,
|
||||
MultimodalInputs,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.model_executor.runner.eager_runner import EagerRunner
|
||||
from sglang.srt.models.deepseek_v4 import DeepseekV4ForCausalLM
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=60, suite="base-a-test-cpu")
|
||||
|
||||
HIDDEN = 8
|
||||
VOCAB = 64
|
||||
IMAGE_TOKEN_ID = 7
|
||||
CACHE_TAG = 50
|
||||
DOWNSAMPLE = 2
|
||||
GLOO_TIMEOUT = timedelta(seconds=90)
|
||||
|
||||
|
||||
def _span(item_hash: int, rows: int, tag: int) -> torch.Tensor:
|
||||
"""Distinguishable full span: the producing rank is visible in the values."""
|
||||
base = float(item_hash * 1000 + tag * 100)
|
||||
return torch.arange(rows * HIDDEN, dtype=torch.float32).view(rows, HIDDEN) + base
|
||||
|
||||
|
||||
def _grid(rows: int):
|
||||
"""A ViT grid whose downsampled span has exactly ``rows`` tokens."""
|
||||
return (0, 0) if rows == 2 else (DOWNSAMPLE, DOWNSAMPLE * (rows - 3))
|
||||
|
||||
|
||||
def _image(item_hash: int, start: int, rows: int) -> MultimodalDataItem:
|
||||
h, w = _grid(rows)
|
||||
item = MultimodalDataItem(
|
||||
modality=Modality.IMAGE,
|
||||
feature=torch.zeros(1),
|
||||
offsets=[(start, start + rows - 1)],
|
||||
model_specific_data={"n_vit_h": h, "n_vit_w": w},
|
||||
)
|
||||
item.set_hash(item_hash)
|
||||
return item
|
||||
|
||||
|
||||
def _request(parts, prefix_len: int = 0, extend_len=None, rid: str = "rid"):
|
||||
"""``parts`` mixes token ids with ``(hash, rows)`` images."""
|
||||
ids, items = [], []
|
||||
for part in parts:
|
||||
if isinstance(part, tuple):
|
||||
item = _image(part[0], len(ids), part[1])
|
||||
items.append(item)
|
||||
ids.extend([item.pad_value] * part[1])
|
||||
else:
|
||||
ids.append(part)
|
||||
if extend_len is None:
|
||||
extend_len = len(ids) - prefix_len
|
||||
return SimpleNamespace(
|
||||
ids=ids, items=items, prefix_len=prefix_len, extend_len=extend_len, rid=rid
|
||||
)
|
||||
|
||||
|
||||
def _chunk_ids(req):
|
||||
return req.ids[req.prefix_len : req.prefix_len + req.extend_len]
|
||||
|
||||
|
||||
def _batch(requests, mode=ForwardMode.EXTEND):
|
||||
return SimpleNamespace(
|
||||
forward_mode=mode,
|
||||
mm_inputs=[
|
||||
MultimodalInputs(mm_items=req.items, im_token_id=IMAGE_TOKEN_ID)
|
||||
if req.items
|
||||
else None
|
||||
for req in requests
|
||||
],
|
||||
extend_prefix_lens_cpu=[req.prefix_len for req in requests],
|
||||
extend_seq_lens_cpu=[req.extend_len for req in requests],
|
||||
seq_lens_cpu=[req.prefix_len + req.extend_len for req in requests],
|
||||
input_ids=torch.tensor(
|
||||
[tok for req in requests for tok in _chunk_ids(req)], dtype=torch.long
|
||||
),
|
||||
positions=torch.cat(
|
||||
[
|
||||
torch.arange(req.prefix_len, req.prefix_len + req.extend_len)
|
||||
for req in requests
|
||||
]
|
||||
),
|
||||
mm_input_embeds=None,
|
||||
attn_cp_metadata=None,
|
||||
global_num_tokens_cpu=None,
|
||||
out_cache_loc=None,
|
||||
input_ids_global=torch.zeros(1, dtype=torch.long),
|
||||
rids=[req.rid for req in requests],
|
||||
)
|
||||
|
||||
|
||||
def _expected_embeds(embed: nn.Embedding, requests, tags):
|
||||
"""``tags`` maps image hash to the tag of the span every rank must end up with."""
|
||||
rows = []
|
||||
with torch.no_grad():
|
||||
for req in requests:
|
||||
ids = torch.tensor(_chunk_ids(req), dtype=torch.long)
|
||||
chunk = nn.functional.embedding(ids.clamp(max=VOCAB - 1), embed.weight)
|
||||
chunk_end = req.prefix_len + req.extend_len
|
||||
for item in req.items:
|
||||
start, end = item.offsets[0]
|
||||
lo = max(start, req.prefix_len) - start
|
||||
hi = min(end + 1, chunk_end) - start
|
||||
if lo >= hi:
|
||||
continue
|
||||
span = _span(item.hash, end - start + 1, tags[item.hash])
|
||||
dest = max(start, req.prefix_len) - req.prefix_len
|
||||
chunk[dest : dest + hi - lo] = span[lo:hi]
|
||||
rows.append(chunk)
|
||||
return torch.cat(rows)
|
||||
|
||||
|
||||
def _seed_cache(item_hash: int, rows: int, tag: int) -> None:
|
||||
mm_schedule.embedding_cache.set(
|
||||
item_hash, mm_schedule.EmbeddingResult(embedding=_span(item_hash, rows, tag))
|
||||
)
|
||||
|
||||
|
||||
class _RecordingBody:
|
||||
def __init__(self, embed):
|
||||
self.embed = embed
|
||||
self.calls = []
|
||||
|
||||
def get_input_embeddings(self):
|
||||
return self.embed
|
||||
|
||||
def __call__(self, input_ids, positions, forward_batch, input_embeds=None):
|
||||
self.calls.append(input_ids)
|
||||
return input_embeds, input_embeds
|
||||
|
||||
|
||||
class _ReducingEmbedding(nn.Embedding):
|
||||
"""Stands in for the TP vocab embedding: a real all-reduce on the device group."""
|
||||
|
||||
def __init__(self, coordinator):
|
||||
torch.manual_seed(0)
|
||||
super().__init__(VOCAB, HIDDEN)
|
||||
self.coordinator = coordinator
|
||||
self.calls = 0
|
||||
|
||||
def forward(self, input_ids):
|
||||
self.calls += 1
|
||||
if self.coordinator is not None:
|
||||
dist.all_reduce(torch.zeros(1), group=self.coordinator.device_group)
|
||||
return super().forward(input_ids)
|
||||
|
||||
|
||||
class _VisionStub(DeepseekV4ForCausalLM):
|
||||
def __init__(self, embed, owner_group, tag: int):
|
||||
nn.Module.__init__(self)
|
||||
self.config = SimpleNamespace(
|
||||
image_token_id=IMAGE_TOKEN_ID,
|
||||
hidden_size=HIDDEN,
|
||||
vision_downsample_ratio=DOWNSAMPLE,
|
||||
)
|
||||
self.vision = object()
|
||||
self.tp_size = 1
|
||||
self.mm_owner_group = owner_group
|
||||
self.image_start = nn.Parameter(torch.zeros(HIDDEN))
|
||||
self.model = _RecordingBody(embed)
|
||||
self.pp_group = SimpleNamespace(is_last_rank=True)
|
||||
self.lm_head = object()
|
||||
self.capture_aux_hidden_states = False
|
||||
self.logits_calls = []
|
||||
self.tag = tag
|
||||
self.encoded = []
|
||||
|
||||
def get_image_feature(self, items):
|
||||
self.encoded.extend(item.hash for item in items)
|
||||
return [
|
||||
_span(item.hash, item.offsets[0][1] - item.offsets[0][0] + 1, self.tag)
|
||||
for item in items
|
||||
]
|
||||
|
||||
def logits_processor(
|
||||
self,
|
||||
input_ids,
|
||||
hidden_states,
|
||||
lm_head,
|
||||
logits_metadata,
|
||||
aux_hidden_states=None,
|
||||
hidden_states_before_norm=None,
|
||||
):
|
||||
self.logits_calls.append(hidden_states)
|
||||
return object()
|
||||
|
||||
def prepare(self, forward_batch):
|
||||
with torch.no_grad():
|
||||
return self.prepare_model_inputs(
|
||||
input_ids=forward_batch.input_ids,
|
||||
forward_batch=forward_batch,
|
||||
input_embeds=None,
|
||||
)
|
||||
|
||||
|
||||
class _TracedGroup:
|
||||
def __init__(self, inner):
|
||||
self.inner = inner
|
||||
self.trace = []
|
||||
|
||||
@property
|
||||
def world_size(self):
|
||||
return self.inner.world_size
|
||||
|
||||
@property
|
||||
def rank_in_group(self):
|
||||
return self.inner.rank_in_group
|
||||
|
||||
@property
|
||||
def ranks(self):
|
||||
return self.inner.ranks
|
||||
|
||||
def all_gather_object(self, obj):
|
||||
self.trace.append(["all_gather_object", type(obj).__name__, self.world_size])
|
||||
return self.inner.all_gather_object(obj)
|
||||
|
||||
def broadcast_object(self, obj=None, src=0):
|
||||
self.trace.append(["broadcast_object", src, self.world_size])
|
||||
return self.inner.broadcast_object(obj, src=src)
|
||||
|
||||
def broadcast(self, tensor, src=0):
|
||||
self.trace.append(["broadcast", src, list(tensor.shape), self.world_size])
|
||||
return self.inner.broadcast(tensor, src=src)
|
||||
|
||||
|
||||
class _ForbiddenGroup:
|
||||
world_size = 4
|
||||
rank_in_group = 0
|
||||
ranks = [0, 1, 2, 3]
|
||||
|
||||
def __getattr__(self, name):
|
||||
raise AssertionError(f"collective {name!r} reached on a fast path")
|
||||
|
||||
|
||||
def _coordinator(group_ranks, rank):
|
||||
return parallel_state.GroupCoordinator(
|
||||
group_ranks=group_ranks,
|
||||
local_rank=rank,
|
||||
torch_distributed_backend="gloo",
|
||||
use_pynccl=False,
|
||||
use_mscclpp=False,
|
||||
use_custom_allreduce=False,
|
||||
use_torch_symm_mem_all_reduce=False,
|
||||
use_hpu_communicator=False,
|
||||
use_xpu_communicator=False,
|
||||
use_npu_communicator=False,
|
||||
use_message_queue_broadcaster=False,
|
||||
group_name="mm_owner_test",
|
||||
gloo_timeout=GLOO_TIMEOUT,
|
||||
)
|
||||
|
||||
|
||||
def _init_rank(rank: int, world_size: int, init_file: str) -> None:
|
||||
torch.set_num_threads(1)
|
||||
dist.init_process_group(
|
||||
backend="gloo",
|
||||
init_method=Path(init_file).as_uri(),
|
||||
rank=rank,
|
||||
world_size=world_size,
|
||||
timeout=GLOO_TIMEOUT,
|
||||
)
|
||||
parallel_state._MODEL_PARALLEL_GROUP_TIMEOUT = GLOO_TIMEOUT
|
||||
|
||||
|
||||
def _run_ranks(world_size: int, target):
|
||||
with TemporaryDirectory() as directory:
|
||||
init_file = str(Path(directory) / "gloo-init")
|
||||
torch.multiprocessing.spawn(
|
||||
_rank_main,
|
||||
args=(world_size, init_file, directory, target),
|
||||
nprocs=world_size,
|
||||
)
|
||||
return [
|
||||
json.loads((Path(directory) / f"rank{rank}.json").read_text())
|
||||
for rank in range(world_size)
|
||||
]
|
||||
|
||||
|
||||
def _rank_main(rank, world_size, init_file, directory, target):
|
||||
_init_rank(rank, world_size, init_file)
|
||||
try:
|
||||
result = target(rank, world_size)
|
||||
(Path(directory) / f"rank{rank}.json").write_text(json.dumps(result))
|
||||
finally:
|
||||
dist.destroy_process_group()
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def _run_cp_extend(model, forward_batch, coordinator):
|
||||
def gather(output, input_tensor):
|
||||
dist.all_gather_into_tensor(
|
||||
output, input_tensor, group=coordinator.device_group
|
||||
)
|
||||
|
||||
runner = EagerRunner.__new__(EagerRunner)
|
||||
runner.model_runner = SimpleNamespace(model=model)
|
||||
with (
|
||||
patch("torch.cuda.current_stream", return_value=None),
|
||||
patch(
|
||||
"sglang.srt.layers.cp.interleave.attn_cp_all_gather_into_tensor",
|
||||
side_effect=gather,
|
||||
),
|
||||
patch(
|
||||
"sglang.srt.layers.cp.interleave.is_allocation_symmetric",
|
||||
return_value=False,
|
||||
),
|
||||
patch(
|
||||
"sglang.srt.layers.cp.interleave.use_symmetric_memory",
|
||||
return_value=torch.no_grad(),
|
||||
),
|
||||
):
|
||||
prepare_cp_forward(forward_batch)
|
||||
runner._execute_extend_cp(forward_batch, {})
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _interleave_cp(cp_size, **parallel):
|
||||
init_cp_strategy(enable_prefill_cp=True, cp_size=cp_size, cp_strategy="interleave")
|
||||
try:
|
||||
with get_parallel().override(
|
||||
attn_cp_size=cp_size, enable_cp_tp_group_sharing=False, **parallel
|
||||
):
|
||||
yield
|
||||
finally:
|
||||
init_cp_strategy(enable_prefill_cp=False, cp_size=1, cp_strategy="interleave")
|
||||
|
||||
|
||||
MANIFEST = ["all_gather_object", "RankManifest", 4]
|
||||
PLAN = ["broadcast_object", 0, 4]
|
||||
STATUS = ["all_gather_object", "RankStatus", 4]
|
||||
|
||||
|
||||
def _bcast(src, rows, size=4):
|
||||
return ["broadcast", src, [rows, HIDDEN], size]
|
||||
|
||||
|
||||
def _cold(*payloads):
|
||||
return [MANIFEST, PLAN, STATUS, *payloads, STATUS, STATUS]
|
||||
|
||||
|
||||
# Global rank 0 sits outside the owner group, so a group-local source index that
|
||||
# leaks out as a global rank is caught.
|
||||
A, B, C, D, E = 400, 401, 402, 403, 404 # owner = hash % 4 -> group ranks 0,1,2,3,0
|
||||
ROWS = {A: 3, B: 4, C: 2, D: 3, E: 6}
|
||||
|
||||
|
||||
def _owner_lifetime_program(rank, world_size):
|
||||
group = _coordinator([[0], [1, 2, 3, 4]], rank)
|
||||
if rank == 0:
|
||||
return {"trace": [], "encoded": [], "outside": True}
|
||||
traced = _TracedGroup(group)
|
||||
local = group.rank_in_group
|
||||
owner_tag = {key: 1 + key % 4 for key in ROWS}
|
||||
mm_schedule.init_mm_embedding_cache(1 << 20)
|
||||
embed = _ReducingEmbedding(group)
|
||||
model = _VisionStub(embed, traced, tag=rank)
|
||||
|
||||
# A valid only on its owner, B valid only on a non-owner, C valid
|
||||
# everywhere, D stale on its owner.
|
||||
if local == 0:
|
||||
_seed_cache(A, ROWS[A], CACHE_TAG + rank)
|
||||
if local == 2:
|
||||
_seed_cache(B, ROWS[B], CACHE_TAG + rank)
|
||||
_seed_cache(C, ROWS[C], CACHE_TAG)
|
||||
if local == 3:
|
||||
_seed_cache(D, ROWS[D] + 2, CACHE_TAG + rank)
|
||||
forwards = [
|
||||
(
|
||||
[
|
||||
_request([10, 11, (A, ROWS[A]), 12, (B, ROWS[B]), 13], rid="r1"),
|
||||
_request([20, (C, ROWS[C]), 21, (D, ROWS[D]), 22], rid="r2"),
|
||||
],
|
||||
{A: CACHE_TAG + 1, B: owner_tag[B], C: CACHE_TAG, D: owner_tag[D]},
|
||||
),
|
||||
(
|
||||
[
|
||||
_request([10, 11, (A, ROWS[A]), (B, ROWS[B])], rid="r3"),
|
||||
_request([20, (A, ROWS[A]), (C, ROWS[C]), (D, ROWS[D])], rid="r4"),
|
||||
],
|
||||
{A: CACHE_TAG + 1, B: owner_tag[B], C: owner_tag[C], D: owner_tag[D]},
|
||||
),
|
||||
] + [
|
||||
(
|
||||
[_request([30, 31, (E, ROWS[E]), 32], prefix, extend, rid="r5")],
|
||||
{E: owner_tag[E]},
|
||||
)
|
||||
for prefix, extend in ((0, 5), (5, 4))
|
||||
]
|
||||
encoded, traces = [], []
|
||||
for index, (requests, tags) in enumerate(forwards):
|
||||
if index == 1:
|
||||
# Local 2 loses every entry and admits nothing new; B's owner evicts B.
|
||||
if local == 2:
|
||||
mm_schedule.init_mm_embedding_cache(0)
|
||||
if local == 1:
|
||||
mm_schedule.embedding_cache.free(B, None)
|
||||
model.encoded.clear()
|
||||
traced.trace.clear()
|
||||
_, embeds = model.prepare(_batch(requests))
|
||||
assert torch.equal(embeds, _expected_embeds(embed, requests, tags))
|
||||
encoded.append(list(model.encoded))
|
||||
traces.append(list(traced.trace))
|
||||
return {"trace": traces, "encoded": encoded, "outside": False}
|
||||
|
||||
|
||||
def test_owner_actions_and_cache_lifetime_across_asymmetric_ranks():
|
||||
results = _run_ranks(5, _owner_lifetime_program)
|
||||
members = range(1, 5)
|
||||
assert all(results[rank]["trace"] == results[1]["trace"] for rank in members)
|
||||
encoded = [results[rank]["encoded"] for rank in members]
|
||||
trace = results[1]["trace"]
|
||||
|
||||
# Forward 1: A cache-broadcast, B and D owner-encoded, C a local hit.
|
||||
assert [e[0] for e in encoded] == [[], [B], [], [D]]
|
||||
assert trace[0] == _cold(_bcast(0, ROWS[A]), _bcast(1, ROWS[B]), _bcast(3, ROWS[D]))
|
||||
|
||||
# Forward 2: evicted owners of B and C re-encode once; A recurs but moves once.
|
||||
assert [e[1] for e in encoded] == [[], [B], [C], []]
|
||||
assert trace[1] == _cold(*(_bcast(key % 4, ROWS[key]) for key in (A, B, C, D)))
|
||||
|
||||
# Forwards 3 and 4: the chunk-crossing span is encoded fully once.
|
||||
assert [e[2] for e in encoded] == [[E], [], [], []]
|
||||
assert [e[3] for e in encoded] == [[], [], [], []]
|
||||
assert trace[2] == trace[3] == _cold(_bcast(0, ROWS[E]))
|
||||
assert results[0]["outside"]
|
||||
|
||||
|
||||
def _cp8_program(rank, world_size):
|
||||
tp8 = _coordinator([list(range(8))], rank)
|
||||
singles = _coordinator([[r] for r in range(8)], rank)
|
||||
attn_cp = _TracedGroup(tp8)
|
||||
with _interleave_cp(
|
||||
8,
|
||||
tp_size=8,
|
||||
attn_dp_size=1,
|
||||
attn_cp_rank=rank,
|
||||
tp_group=_TracedGroup(tp8),
|
||||
attn_tp_group=_TracedGroup(singles),
|
||||
attn_cp_group=attn_cp,
|
||||
):
|
||||
selected = select_owner_group(get_parallel())
|
||||
assert selected is attn_cp
|
||||
mm_schedule.init_mm_embedding_cache(1 << 20)
|
||||
torch.manual_seed(0)
|
||||
embed = nn.Embedding(VOCAB, HIDDEN)
|
||||
model = _VisionStub(embed, selected, tag=rank)
|
||||
w = 803 # owner 3; rows land on ranks 6,7,0,1,2,3 so ranks 4,5 hold no image row
|
||||
requests = [
|
||||
_request([10, 11, 12, 13, 14], rid="p1"),
|
||||
_request([20, (w, 6), 21, 22], rid="p2"),
|
||||
]
|
||||
_run_cp_extend(model, _batch(requests), tp8)
|
||||
(logits,) = model.logits_calls
|
||||
assert torch.equal(logits, _expected_embeds(embed, requests, {w: 3}))
|
||||
(body,) = model.model.calls
|
||||
return {
|
||||
"encoded": list(model.encoded),
|
||||
"trace": list(attn_cp.trace),
|
||||
"image_rows": int((body == IMAGE_TOKEN_ID).sum()),
|
||||
}
|
||||
|
||||
|
||||
def test_tp8_cp8_encodes_once_and_merges_before_shard():
|
||||
results = _run_ranks(8, _cp8_program)
|
||||
assert [r["encoded"] for r in results] == [[], [], [], [803], [], [], [], []]
|
||||
assert all(r["trace"] == results[0]["trace"] for r in results)
|
||||
assert [op for op in results[0]["trace"] if op[0] == "broadcast"] == [
|
||||
_bcast(3, 6, 8)
|
||||
]
|
||||
assert [r["image_rows"] for r in results] == [1, 1, 1, 1, 0, 0, 1, 1]
|
||||
|
||||
|
||||
def _raise_on(on_rank, target, name, exc_type=RuntimeError, after=0):
|
||||
def inject(rank, batch):
|
||||
if rank != on_rank:
|
||||
return nullcontext()
|
||||
original = getattr(target, name)
|
||||
calls = []
|
||||
|
||||
def fail(*args, **kwargs):
|
||||
calls.append(None)
|
||||
if len(calls) > after:
|
||||
raise exc_type(f"injected {name}")
|
||||
return original(*args, **kwargs)
|
||||
|
||||
return patch.object(target, name, fail)
|
||||
|
||||
return inject
|
||||
|
||||
|
||||
def _remap_oom_on(on_rank):
|
||||
def inject(rank, batch):
|
||||
if rank != on_rank:
|
||||
return nullcontext()
|
||||
original = torch.Tensor.masked_fill
|
||||
|
||||
def masked_fill(self, mask, value):
|
||||
if self is batch.input_ids:
|
||||
raise torch.OutOfMemoryError("injected remap")
|
||||
return original(self, mask, value)
|
||||
|
||||
return patch.object(torch.Tensor, "masked_fill", masked_fill)
|
||||
|
||||
return inject
|
||||
|
||||
|
||||
def _failure(requests, trace, errors, inject=None, vocab_calls=0, cp=False):
|
||||
return SimpleNamespace(
|
||||
requests=requests,
|
||||
trace=trace,
|
||||
errors=errors,
|
||||
inject=inject or (lambda rank, batch: nullcontext()),
|
||||
vocab_calls=vocab_calls,
|
||||
cp=cp,
|
||||
)
|
||||
|
||||
|
||||
F, G = 300, 301 # owners 0 and 1 in a four-rank group
|
||||
AT_PLAN = [MANIFEST, PLAN]
|
||||
AT_FINALIZE_F = _cold(_bcast(0, 3))
|
||||
|
||||
FAILURES = {
|
||||
"prepare": _failure(
|
||||
[[10, (G, 3), 11]],
|
||||
AT_PLAN,
|
||||
["during prepare on group rank 1", "OutOfMemoryError: injected as_tensor"],
|
||||
_raise_on(1, torch, "as_tensor", torch.OutOfMemoryError),
|
||||
),
|
||||
"remap": _failure(
|
||||
[[10, (F, 3), 11]],
|
||||
AT_FINALIZE_F,
|
||||
["during finalize on group rank 1", "injected remap"],
|
||||
_remap_oom_on(1),
|
||||
vocab_calls=1,
|
||||
),
|
||||
"final_sync": _failure(
|
||||
[[10, (F, 3), 11]],
|
||||
AT_FINALIZE_F,
|
||||
["during finalize on group rank 1", "injected _synchronize"],
|
||||
_raise_on(1, mm_owner_embedding, "_synchronize", after=2),
|
||||
vocab_calls=1,
|
||||
),
|
||||
"leader_plan": _failure(
|
||||
[[10, (G, 3), 11]],
|
||||
AT_PLAN,
|
||||
["during plan on group rank 0", "MemoryError: injected _make_plan"],
|
||||
_raise_on(0, mm_owner_embedding, "_make_plan", MemoryError),
|
||||
),
|
||||
"remap_cp": _failure(
|
||||
[[10, (F, 3), 11]],
|
||||
AT_FINALIZE_F,
|
||||
["during finalize on group rank 1", "injected remap"],
|
||||
_remap_oom_on(1),
|
||||
vocab_calls=1,
|
||||
cp=True,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _failure_program(rank, world_size):
|
||||
group = _coordinator([[0, 1, 2, 3]], rank)
|
||||
out = {}
|
||||
for name, case in FAILURES.items():
|
||||
mm_schedule.init_mm_embedding_cache(1 << 20)
|
||||
traced = _TracedGroup(group)
|
||||
embed = _ReducingEmbedding(None if case.cp else group)
|
||||
model = _VisionStub(embed, traced, tag=rank)
|
||||
batch = _batch([_request(parts, rid=name) for parts in case.requests])
|
||||
with ExitStack() as stack, pytest.raises(MmOwnerProtocolError) as raised:
|
||||
stack.enter_context(case.inject(rank, batch))
|
||||
if case.cp:
|
||||
stack.enter_context(
|
||||
_interleave_cp(4, attn_cp_rank=rank, attn_cp_group=traced)
|
||||
)
|
||||
_run_cp_extend(model, batch, group)
|
||||
else:
|
||||
model.prepare(batch)
|
||||
out[name] = {
|
||||
"error": str(raised.value),
|
||||
"trace": traced.trace,
|
||||
"vocab_calls": embed.calls,
|
||||
"body_calls": len(model.model.calls),
|
||||
}
|
||||
return out
|
||||
|
||||
|
||||
def test_failures_agree_before_payload_text_embedding_or_body():
|
||||
results = _run_ranks(4, _failure_program)
|
||||
for name, case in FAILURES.items():
|
||||
runs = [r[name] for r in results]
|
||||
error = runs[0]["error"]
|
||||
assert all(run["error"] == error for run in runs), (name, runs)
|
||||
assert all(run["trace"] == case.trace for run in runs), (name, runs)
|
||||
for fragment in [*case.errors, f"rids=['{name}'"]:
|
||||
assert fragment in error, (name, fragment, error)
|
||||
assert [run["vocab_calls"] for run in runs] == [case.vocab_calls] * 4, name
|
||||
assert [run["body_calls"] for run in runs] == [0] * 4, name
|
||||
|
||||
|
||||
def test_text_decode_prefilled_future_and_precomputed_paths_pay_no_collective():
|
||||
mm_schedule.init_mm_embedding_cache(1 << 20)
|
||||
torch.manual_seed(0)
|
||||
embed = nn.Embedding(VOCAB, HIDDEN)
|
||||
owner_model = _VisionStub(embed, _ForbiddenGroup(), tag=0)
|
||||
legacy_model = _VisionStub(embed, None, tag=0)
|
||||
|
||||
text_only = _batch([_request([10, 11], rid="t0"), _request([20, 21, 22], rid="t1")])
|
||||
assert owner_model.prepare(text_only)[1] is None
|
||||
|
||||
with_image = [_request([10, (A, 3), 11, 12], rid="v0")]
|
||||
for mode in (ForwardMode.DECODE, ForwardMode.TARGET_VERIFY):
|
||||
assert owner_model.prepare(_batch(with_image, mode))[1] is None
|
||||
|
||||
for request, chunk_ids in (
|
||||
(_request([10, (A, 3), 11, 12], prefix_len=5, rid="pf"), [12]),
|
||||
(_request([10, 11, 12, 13, (A, 3)], extend_len=2, rid="fut"), [10, 11]),
|
||||
):
|
||||
ids, embeds = owner_model.prepare(_batch([request]))
|
||||
assert torch.equal(embeds, legacy_model.prepare(_batch([request]))[1])
|
||||
assert torch.equal(ids, torch.tensor(chunk_ids))
|
||||
|
||||
precomputed = _request([10, 11, 12, 13], rid="pc")
|
||||
item = MultimodalDataItem(
|
||||
modality=Modality.IMAGE,
|
||||
precomputed_embeddings=torch.full((3, HIDDEN), 5.0),
|
||||
offsets=[(1, 3)],
|
||||
)
|
||||
item.set_hash(A)
|
||||
precomputed.items = [item]
|
||||
precomputed.ids[1:4] = [item.pad_value] * 3
|
||||
_, embeds = owner_model.prepare(_batch([precomputed]))
|
||||
assert torch.equal(embeds[1:4], torch.full((3, HIDDEN), 5.0))
|
||||
|
||||
overlapping = [_request([10, (A, 3), 11, 12], prefix_len=2, rid="ov")]
|
||||
with pytest.raises(AssertionError, match="collective"):
|
||||
owner_model.prepare(_batch(overlapping))
|
||||
|
||||
|
||||
def _parallel(tp_size, attn_dp_size, attn_cp_size, attn_tp_size, attn_cp_group_size):
|
||||
return SimpleNamespace(
|
||||
tp_size=tp_size,
|
||||
attn_dp_size=attn_dp_size,
|
||||
attn_cp_size=attn_cp_size,
|
||||
tp_group=SimpleNamespace(world_size=tp_size, name="tp"),
|
||||
attn_tp_group=SimpleNamespace(world_size=attn_tp_size, name="attn_tp"),
|
||||
attn_cp_group=SimpleNamespace(world_size=attn_cp_group_size, name="attn_cp"),
|
||||
)
|
||||
|
||||
|
||||
def test_select_owner_group_follows_the_replication_domain():
|
||||
for layout, expected in (
|
||||
((8, 1, 1, 8, 1), "attn_tp"),
|
||||
((8, 1, 8, 1, 8), "attn_cp"),
|
||||
((8, 2, 1, 4, 1), "attn_tp"),
|
||||
((1, 1, 1, 1, 1), None),
|
||||
((8, 2, 4, 1, 4), None),
|
||||
((8, 1, 4, 2, 4), None),
|
||||
):
|
||||
group = select_owner_group(_parallel(*layout))
|
||||
assert (group and group.name) == expected, layout
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__]))
|
||||
@@ -30,7 +30,13 @@ from sglang.srt.runtime_context import get_context
|
||||
from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler
|
||||
from sglang.srt.utils.common import Range
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel
|
||||
|
||||
maybe_stub_sgl_kernel()
|
||||
|
||||
import sglang.srt.managers.scheduler as scheduler_module
|
||||
from sglang.srt.disaggregation.utils import DisaggregationMode
|
||||
from sglang.srt.managers.scheduler import Scheduler
|
||||
|
||||
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
|
||||
|
||||
@@ -115,7 +121,10 @@ class TestPrefillAdder(CustomTestCase):
|
||||
req.prefix_indices = []
|
||||
req.full_untruncated_fill_ids = []
|
||||
req.output_ids = [0] * output_len
|
||||
req.sampling_params = SimpleNamespace(max_new_tokens=max_new_tokens)
|
||||
req.sampling_params = SimpleNamespace(
|
||||
max_new_tokens=max_new_tokens, ignore_eos=False
|
||||
)
|
||||
req.token_indices_to_pool = None
|
||||
req.time_stats = SimpleNamespace(wait_queue_entry_time=wait_time)
|
||||
req.retracted_stain = False
|
||||
req.host_hit_length = 0
|
||||
@@ -298,6 +307,116 @@ class TestPrefillAdder(CustomTestCase):
|
||||
)
|
||||
self.assertEqual(adder.can_run_list, [first])
|
||||
|
||||
def create_admission_scheduler(self, *, chunked_req) -> Scheduler:
|
||||
allocator = self.create_token_allocator(available_size=4096)
|
||||
allocator.page_size = 1
|
||||
self.mock_tree_cache.supports_mamba.return_value = False
|
||||
self.mock_tree_cache.is_tree_cache.return_value = False
|
||||
self.mock_tree_cache.supports_fast_match_prefix.return_value = False
|
||||
self.mock_tree_cache.storage_prefetch_retries = None
|
||||
self.mock_tree_cache.supports_prefix_sharing.return_value = False
|
||||
scheduler = Scheduler.__new__(Scheduler)
|
||||
scheduler.grammar_manager = SimpleNamespace(has_waiting_grammars=lambda: False)
|
||||
scheduler.enable_priority_preemption = False
|
||||
scheduler.enable_priority_scheduling = False
|
||||
scheduler.is_hybrid_swa = False
|
||||
scheduler.min_free_slots_delayer = None
|
||||
scheduler.get_num_allocatable_reqs = lambda *args, **kwargs: 64
|
||||
scheduler.policy = SchedulePolicy(
|
||||
policy="fcfs",
|
||||
tree_cache=self.mock_tree_cache,
|
||||
enable_hierarchical_cache=False,
|
||||
enable_priority_scheduling=False,
|
||||
schedule_low_priority_values_first=False,
|
||||
)
|
||||
scheduler.processed_tokens_counter = 0
|
||||
scheduler.chunked_prefill_size = 16
|
||||
scheduler.dynamic_chunk_sizer = None
|
||||
scheduler.tp_worker = SimpleNamespace(
|
||||
model_runner=SimpleNamespace(attn_backend=object(), prefill_aware_swa=False)
|
||||
)
|
||||
scheduler.page_size = 1
|
||||
scheduler.tree_cache = self.mock_tree_cache
|
||||
scheduler.token_to_kv_pool_allocator = allocator
|
||||
scheduler.new_token_ratio_tracker = SimpleNamespace(current=1.0)
|
||||
scheduler.max_prefill_tokens = 16384
|
||||
scheduler.is_mixed_chunk = False
|
||||
scheduler.priority_scheduling_preemption_threshold = 0
|
||||
scheduler.max_prefill_bs = 64
|
||||
scheduler.max_running_requests = 64
|
||||
scheduler.dllm_config = None
|
||||
scheduler.enable_lora = False
|
||||
scheduler.req_to_token_pool = SimpleNamespace()
|
||||
scheduler.disaggregation_mode = DisaggregationMode.NULL
|
||||
scheduler.enable_hicache_storage = False
|
||||
scheduler.enable_lmcache = False
|
||||
scheduler.enable_hierarchical_cache = False
|
||||
scheduler.enable_unified_cache_external_linker = False
|
||||
scheduler.truncation_align_size = None
|
||||
scheduler.model_config = None
|
||||
scheduler.enable_overlap = False
|
||||
scheduler.spec_algorithm = None
|
||||
scheduler.load_inquirer = MagicMock()
|
||||
scheduler.chunked_req = chunked_req
|
||||
scheduler.waiting_queue = []
|
||||
return scheduler
|
||||
|
||||
def run_admission_pass(self, scheduler: Scheduler) -> list:
|
||||
running_batch = self.create_running_batch()
|
||||
running_batch.batch_is_full = False
|
||||
with (
|
||||
patch.object(scheduler_module, "ScheduleBatch") as schedule_batch,
|
||||
patch.object(scheduler_module, "PrefillStats"),
|
||||
patch.object(scheduler_module, "set_time_batch"),
|
||||
):
|
||||
new_batch, _ = scheduler._get_new_batch_prefill_raw(None, running_batch)
|
||||
if new_batch is None:
|
||||
return []
|
||||
admitted = list(schedule_batch.init_new.call_args.args[0])
|
||||
for req in admitted:
|
||||
req.prefix_len = req.extend_end
|
||||
return admitted
|
||||
|
||||
def test_fcfs_admits_override_request_once_image_continuation_drains(self):
|
||||
"""An override request at the queue head must be admitted once the image
|
||||
chunk ahead of it drains, even while more image requests keep arriving."""
|
||||
|
||||
def tagged(rid, length, *, multimodal=False, overrides=False):
|
||||
req = self.create_shared_req(rid)
|
||||
req.origin_input_ids = list(range(length))
|
||||
req.full_untruncated_fill_ids = list(range(length))
|
||||
req.multimodal_inputs = object() if multimodal else None
|
||||
req.positional_embed_overrides = object() if overrides else None
|
||||
req.beam_group = None
|
||||
req.inflight_middle_chunks = 0
|
||||
req.kv = SimpleNamespace(holds_mamba=False)
|
||||
return req
|
||||
|
||||
continuation = tagged("image-continuation", 20, multimodal=True)
|
||||
continuation.prefix_len = 16
|
||||
scheduler = self.create_admission_scheduler(chunked_req=continuation)
|
||||
text = tagged("text", 2)
|
||||
override = tagged("override", 4, overrides=True)
|
||||
scheduler.waiting_queue = [text, override]
|
||||
|
||||
admitted_at = None
|
||||
for pass_index in range(6):
|
||||
scheduler.waiting_queue.append(
|
||||
tagged(f"image-{pass_index}", 16, multimodal=True)
|
||||
)
|
||||
admitted = self.run_admission_pass(scheduler)
|
||||
if pass_index == 0:
|
||||
self.assertEqual(admitted, [continuation, text])
|
||||
self.assertFalse(
|
||||
any(r.multimodal_inputs is not None for r in admitted)
|
||||
and any(r.positional_embed_overrides is not None for r in admitted)
|
||||
)
|
||||
if any(r is override for r in admitted):
|
||||
admitted_at = pass_index
|
||||
break
|
||||
self.assertIsNotNone(admitted_at)
|
||||
self.assertNotIn(override, scheduler.waiting_queue)
|
||||
|
||||
def test_shared_admission_rechecks_after_prefix_lock(self):
|
||||
adder = self.create_shared_adder()
|
||||
self.assertIsNotNone(adder.token_to_kv_pool_allocator.alloc(24))
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
"""Vision inputs under prefill CP merge on the full extend layout before the shard."""
|
||||
|
||||
import unittest
|
||||
from contextlib import contextmanager
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.layers.cp.base import init_cp_strategy
|
||||
from sglang.srt.layers.cp.utils import prepare_cp_forward
|
||||
from sglang.srt.managers import mm_schedule
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
MultimodalDataItem,
|
||||
MultimodalInputs,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.model_executor.runner.eager_runner import EagerRunner
|
||||
from sglang.srt.models.deepseek_v4 import DeepseekV4ForCausalLM
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=15, suite="base-a-test-cpu")
|
||||
|
||||
HIDDEN = 8
|
||||
VOCAB = 64
|
||||
IMAGE_TOKEN_ID = 7
|
||||
CP_SIZE = 4
|
||||
# (prefix_len, extend_len) per request. Request 1 carries one image whose span
|
||||
# [2, 8] starts inside its prefix, so only span rows 1..6 land in this chunk.
|
||||
CHUNKS = [(0, 7), (3, 9), (1, 5)]
|
||||
IMAGE_OFFSET = (2, 8)
|
||||
IMAGE_HASH = 12345
|
||||
NUM_TOKENS = sum(extend_len for _, extend_len in CHUNKS)
|
||||
# 21 tokens over 4 ranks give logical [6, 5, 5, 5], padded to the CP alignment.
|
||||
PHYSICAL_ROWS = 8
|
||||
IMAGE_ROWS = torch.arange(7, 13)
|
||||
POSITIONS = torch.cat([torch.arange(p, p + n) for p, n in CHUNKS])
|
||||
|
||||
|
||||
def _image_span(item: MultimodalDataItem) -> torch.Tensor:
|
||||
start, end = item.offsets[0]
|
||||
rows = end - start + 1
|
||||
return torch.arange(rows * HIDDEN, dtype=torch.float32).view(rows, HIDDEN) + 100.0
|
||||
|
||||
|
||||
def _pad(x: torch.Tensor) -> torch.Tensor:
|
||||
return torch.cat([x, x.new_zeros(PHYSICAL_ROWS - x.shape[0], *x.shape[1:])])
|
||||
|
||||
|
||||
class _RecordingBody:
|
||||
def __init__(self, embed: nn.Embedding):
|
||||
self.embed = embed
|
||||
self.calls = []
|
||||
|
||||
def get_input_embeddings(self):
|
||||
return self.embed
|
||||
|
||||
def __call__(self, input_ids, positions, forward_batch, input_embeds=None):
|
||||
self.calls.append(
|
||||
SimpleNamespace(
|
||||
input_ids=input_ids,
|
||||
input_embeds=input_embeds,
|
||||
input_ids_global=forward_batch.input_ids_global,
|
||||
)
|
||||
)
|
||||
return input_embeds, input_embeds
|
||||
|
||||
|
||||
class _VisionStub(DeepseekV4ForCausalLM):
|
||||
def __init__(self, embed: nn.Embedding):
|
||||
nn.Module.__init__(self)
|
||||
self.config = SimpleNamespace(image_token_id=IMAGE_TOKEN_ID)
|
||||
self.vision = object()
|
||||
self.tp_size = 1
|
||||
self.mm_owner_group = None
|
||||
self.model = _RecordingBody(embed)
|
||||
self.pp_group = SimpleNamespace(is_last_rank=True)
|
||||
self.lm_head = object()
|
||||
self.capture_aux_hidden_states = False
|
||||
self.logits_calls = []
|
||||
|
||||
def get_image_feature(self, items):
|
||||
return [_image_span(item) for item in items]
|
||||
|
||||
def logits_processor(
|
||||
self,
|
||||
input_ids,
|
||||
hidden_states,
|
||||
lm_head,
|
||||
logits_metadata,
|
||||
aux_hidden_states=None,
|
||||
hidden_states_before_norm=None,
|
||||
):
|
||||
self.logits_calls.append(
|
||||
SimpleNamespace(input_ids=input_ids, hidden_states=hidden_states)
|
||||
)
|
||||
return object()
|
||||
|
||||
|
||||
def _build_batch():
|
||||
item = MultimodalDataItem(
|
||||
modality=Modality.IMAGE, feature=torch.zeros(1), offsets=[IMAGE_OFFSET]
|
||||
)
|
||||
item.set_hash(IMAGE_HASH)
|
||||
ids = list(range(10, 17))
|
||||
ids += [item.pad_value] * len(IMAGE_ROWS) + [20, 21, 22]
|
||||
ids += list(range(30, 35))
|
||||
forward_batch = SimpleNamespace(
|
||||
forward_mode=ForwardMode.EXTEND,
|
||||
mm_inputs=[
|
||||
MultimodalInputs(mm_items=[]),
|
||||
MultimodalInputs(mm_items=[item], im_token_id=IMAGE_TOKEN_ID),
|
||||
None,
|
||||
],
|
||||
extend_prefix_lens_cpu=[prefix for prefix, _ in CHUNKS],
|
||||
extend_seq_lens_cpu=[extend_len for _, extend_len in CHUNKS],
|
||||
seq_lens_cpu=[prefix + extend_len for prefix, extend_len in CHUNKS],
|
||||
input_ids=torch.tensor(ids, dtype=torch.long),
|
||||
positions=POSITIONS.clone(),
|
||||
mm_input_embeds=None,
|
||||
attn_cp_metadata=None,
|
||||
global_num_tokens_cpu=None,
|
||||
out_cache_loc=None,
|
||||
input_ids_global=torch.zeros(1, dtype=torch.long),
|
||||
)
|
||||
return forward_batch, item
|
||||
|
||||
|
||||
def _expected_embeds(embed, scheduler_ids, item):
|
||||
with torch.no_grad():
|
||||
full = embed(scheduler_ids.clamp(max=VOCAB - 1))
|
||||
full[IMAGE_ROWS] = _image_span(item)[1:7]
|
||||
return full
|
||||
|
||||
|
||||
def _canonical(scheduler_ids):
|
||||
canonical = scheduler_ids.clone()
|
||||
canonical[IMAGE_ROWS] = IMAGE_TOKEN_ID
|
||||
return canonical
|
||||
|
||||
|
||||
class TestDeepseekV41VisionPrefillCPInputs(CustomTestCase):
|
||||
def setUp(self):
|
||||
mm_schedule.init_mm_embedding_cache(1 << 20)
|
||||
init_cp_strategy(
|
||||
enable_prefill_cp=True, cp_size=CP_SIZE, cp_strategy="interleave"
|
||||
)
|
||||
torch.manual_seed(0)
|
||||
self.embed = nn.Embedding(VOCAB, HIDDEN)
|
||||
self.model = _VisionStub(self.embed)
|
||||
|
||||
def tearDown(self):
|
||||
init_cp_strategy(enable_prefill_cp=False, cp_size=1, cp_strategy="interleave")
|
||||
|
||||
@contextmanager
|
||||
def _cp_collectives(self, full: torch.Tensor, rank: int):
|
||||
def all_gather(output, input_tensor):
|
||||
# Peers contribute their expected shards; this rank's rows come from
|
||||
# what the runner actually handed to the collective.
|
||||
output.zero_()
|
||||
for peer in range(CP_SIZE):
|
||||
rows = full[peer::CP_SIZE]
|
||||
output[peer * PHYSICAL_ROWS : peer * PHYSICAL_ROWS + rows.shape[0]] = (
|
||||
rows
|
||||
)
|
||||
output[rank * PHYSICAL_ROWS : (rank + 1) * PHYSICAL_ROWS] = input_tensor
|
||||
|
||||
with (
|
||||
patch("torch.cuda.current_stream", return_value=None),
|
||||
patch(
|
||||
"sglang.srt.layers.cp.interleave.attn_cp_all_gather_into_tensor",
|
||||
side_effect=all_gather,
|
||||
),
|
||||
patch(
|
||||
"sglang.srt.layers.cp.interleave.is_allocation_symmetric",
|
||||
return_value=False,
|
||||
),
|
||||
patch(
|
||||
"sglang.srt.layers.cp.interleave.use_symmetric_memory",
|
||||
return_value=torch.no_grad(),
|
||||
),
|
||||
):
|
||||
yield
|
||||
|
||||
def test_cp_runner_merges_before_shard(self):
|
||||
runner = EagerRunner.__new__(EagerRunner)
|
||||
runner.model_runner = SimpleNamespace(model=self.model)
|
||||
padded = torch.zeros(CP_SIZE * PHYSICAL_ROWS, dtype=torch.long)
|
||||
|
||||
for rank in range(CP_SIZE):
|
||||
forward_batch, item = _build_batch()
|
||||
scheduler_ids = forward_batch.input_ids.clone()
|
||||
canonical = _canonical(scheduler_ids)
|
||||
full = _expected_embeds(self.embed, scheduler_ids, item)
|
||||
padded[:NUM_TOKENS] = canonical
|
||||
rank_major_ids = padded.view(-1, CP_SIZE).T.flatten()
|
||||
self.model.model.calls.clear()
|
||||
self.model.logits_calls.clear()
|
||||
|
||||
with (
|
||||
get_parallel().override(
|
||||
attn_cp_rank=rank,
|
||||
attn_cp_size=CP_SIZE,
|
||||
attn_cp_group=object(),
|
||||
enable_cp_tp_group_sharing=False,
|
||||
),
|
||||
self._cp_collectives(full, rank),
|
||||
torch.no_grad(),
|
||||
):
|
||||
prepare_cp_forward(forward_batch)
|
||||
runner._execute_extend_cp(forward_batch, {})
|
||||
|
||||
with self.subTest(rank=rank):
|
||||
metadata = forward_batch.attn_cp_metadata
|
||||
self.assertEqual(metadata.per_rank_actual_token, [PHYSICAL_ROWS] * 4)
|
||||
(body,) = self.model.model.calls
|
||||
self.assertTrue(
|
||||
torch.equal(body.input_ids, _pad(canonical[rank::CP_SIZE]))
|
||||
)
|
||||
self.assertTrue(
|
||||
torch.equal(body.input_embeds, _pad(full[rank::CP_SIZE]))
|
||||
)
|
||||
self.assertTrue(torch.equal(body.input_ids_global, rank_major_ids))
|
||||
|
||||
(logits,) = self.model.logits_calls
|
||||
self.assertTrue(torch.equal(logits.input_ids, canonical))
|
||||
self.assertTrue(torch.equal(logits.hidden_states, full))
|
||||
|
||||
self.assertTrue(torch.equal(forward_batch.mm_input_embeds, full))
|
||||
self.assertTrue(torch.equal(forward_batch.input_ids, scheduler_ids))
|
||||
|
||||
def test_external_embeddings_with_images_are_rejected(self):
|
||||
forward_batch, _ = _build_batch()
|
||||
with self.assertRaisesRegex(ValueError, "Cannot combine"):
|
||||
self.model.prepare_model_inputs(
|
||||
input_ids=forward_batch.input_ids,
|
||||
forward_batch=forward_batch,
|
||||
input_embeds=torch.zeros(NUM_TOKENS, HIDDEN),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -29,6 +29,7 @@ from sglang.srt.arg_groups.cuda_graph_hook import (
|
||||
finalize_cuda_graph_prefill_max_context,
|
||||
handle_cuda_graph_config,
|
||||
)
|
||||
from sglang.srt.arg_groups.deepseek_v4_hook import validate_deepseek_v41_features
|
||||
from sglang.srt.arg_groups.hicache_hook import (
|
||||
handle_hicache,
|
||||
handle_hicache_ratio_default,
|
||||
@@ -45,6 +46,7 @@ from sglang.srt.arg_groups.kv_cache_hook import (
|
||||
)
|
||||
from sglang.srt.arg_groups.mamba_hook import handle_mamba_backend
|
||||
from sglang.srt.arg_groups.memory_hook import handle_gpu_memory_settings
|
||||
from sglang.srt.arg_groups.model_hook import handle_model_specific_adjustments
|
||||
from sglang.srt.arg_groups.model_path_hook import handle_load_format
|
||||
from sglang.srt.arg_groups.moe_hook import (
|
||||
handle_a2a_moe,
|
||||
@@ -4035,6 +4037,82 @@ class TestDcpCommBackendDefault(CustomTestCase):
|
||||
)
|
||||
|
||||
|
||||
class TestDeepseekV41VisionPrefillCPArgs(CustomTestCase):
|
||||
def _args(
|
||||
self,
|
||||
*,
|
||||
vision_n_layers=2,
|
||||
prefill_backend=Backend.DISABLED,
|
||||
lock_prefill_backend=False,
|
||||
**overrides,
|
||||
):
|
||||
fields = dict(
|
||||
model_path="dummy",
|
||||
enable_prefill_cp=True,
|
||||
cp_strategy="interleave",
|
||||
tp_size=2,
|
||||
)
|
||||
fields.update(overrides)
|
||||
server_args = ServerArgs(**fields)
|
||||
server_args._model_config = SimpleNamespace(
|
||||
hf_config=SimpleNamespace(
|
||||
architectures=["DeepseekV4ForCausalLM"],
|
||||
model_type="deepseek_v41",
|
||||
vision_n_layers=vision_n_layers,
|
||||
),
|
||||
nvfp4_moe_meta=None,
|
||||
is_fp4_experts=False,
|
||||
decision_config=None,
|
||||
)
|
||||
# The dummy path does not initialize phase configs.
|
||||
server_args.cuda_graph_config = CudaGraphConfig(
|
||||
decode=PhaseConfig(backend=Backend.FULL, max_bs=512),
|
||||
prefill=PhaseConfig(backend=prefill_backend, max_bs=512),
|
||||
)
|
||||
server_args._resolved_overrides = []
|
||||
server_args._cuda_graph_config_locked = (
|
||||
{(Phase.PREFILL, "backend")} if lock_prefill_backend else set()
|
||||
)
|
||||
return server_args
|
||||
|
||||
@override_platform(is_cuda=True, is_hip=False)
|
||||
def test_guard_matrix_in_model_hook_order(self):
|
||||
for overrides, error in (
|
||||
(
|
||||
dict(enable_encoder_swa_bounded_replay=True),
|
||||
"encoder-swa-bounded-replay does not support context parallelism",
|
||||
),
|
||||
(dict(cp_strategy="zigzag"), "requires --cp-strategy interleave"),
|
||||
(
|
||||
dict(prefill_backend=Backend.BREAKABLE, lock_prefill_backend=True),
|
||||
"runs eager prefill",
|
||||
),
|
||||
(
|
||||
dict(
|
||||
speculative_algorithm="DSPARK",
|
||||
enable_decoder_swa_bounded_replay=True,
|
||||
),
|
||||
"DSpark.*decoder-swa-bounded-replay",
|
||||
),
|
||||
):
|
||||
args = self._args(
|
||||
max_running_requests=4, chunked_prefill_size=128, **overrides
|
||||
)
|
||||
with self.subTest(error=error), self.assertRaisesRegex(ValueError, error):
|
||||
handle_model_specific_adjustments(args)
|
||||
|
||||
def test_text_zigzag_and_default_prefill_graph_are_accepted(self):
|
||||
validate_deepseek_v41_features(
|
||||
self._args(cp_strategy="zigzag", vision_n_layers=0)
|
||||
)
|
||||
args = self._args(prefill_backend=Backend.BREAKABLE)
|
||||
validate_deepseek_v41_features(args)
|
||||
self.assertEqual(
|
||||
resolution_result(args, "cuda_graph_config").prefill.backend,
|
||||
Backend.DISABLED,
|
||||
)
|
||||
|
||||
|
||||
class TestParserChoices(CustomTestCase):
|
||||
"""The choices come from dependency-free name lists, but `cli/serve.py`
|
||||
loads plugins before parsing, so a plugin's parser must still be accepted."""
|
||||
|
||||
Reference in New Issue
Block a user