model: support DeepSeek V4.1 vision with interleave prefill CP

The CP runner bypassed the vision merge and used bare text embeddings.
Merge image features before sharding so request-global offsets stay valid.
Canonicalize model IDs separately to preserve scheduler hash IDs.
Keep unsupported combinations guarded and isolate embedding overrides
from multimodal prefills without starving queued FCFS requests.

python/sglang/srt/arg_groups/deepseek_v4_hook.py

@@ -181,12 +181,15 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
     )
 
     cfg = resolving_view(server_args)
-    if model_config_of(server_args).hf_config.model_type != "deepseek_v41":
+    hf_config = model_config_of(server_args).hf_config
+    if hf_config.model_type != "deepseek_v41":
         if cfg.enable_encoder_swa_bounded_replay:
             raise ValueError(
                 "--enable-encoder-swa-bounded-replay requires DeepSeek-V4.1"
             )
         return
+    if hf_config.vision_n_layers > 0 and cfg.enable_prefill_cp:
+        _validate_deepseek_v41_vision_prefill_cp(server_args)
     if cfg.enable_encoder_swa_bounded_replay:
         from sglang.srt.model_executor.cuda_graph_config import Backend
 
@@ -197,7 +200,8 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
                 cfg.cuda_graph_config.prefill.backend != Backend.DISABLED,
             ),
             ("DP attention", cfg.enable_dp_attention),
-            ("context parallelism", cfg.attn_cp_size > 1),
+            # Prefill CP declares attn_cp_size and DP attention only later.
+            ("context parallelism", cfg.attn_cp_size > 1 or cfg.enable_prefill_cp),
             ("external cache linker", cfg.enable_unified_cache_external_linker),
             ("unified memory", cfg.enable_unified_memory),
             ("PD disaggregation", cfg.disaggregation_mode != "null"),
@@ -306,3 +310,40 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
                     "--enable-decoder-swa-bounded-replay cannot be combined with "
                     f"{feature} yet; disable one of them."
                 )
+
+
+def _validate_deepseek_v41_vision_prefill_cp(server_args: ServerArgs) -> None:
+    from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
+
+    cfg = resolving_view(server_args)
+    if cfg.cp_strategy != "interleave":
+        raise ValueError(
+            "DeepSeek-V4.1 vision with prefill CP requires --cp-strategy "
+            f"interleave; got {cfg.cp_strategy!r}."
+        )
+    if cfg.cuda_graph_config.prefill.backend != Backend.DISABLED:
+        # The CP runner merges image features eagerly; no capture path replays it.
+        locked = getattr(server_args, "_cuda_graph_config_locked", set())
+        if (Phase.PREFILL, "backend") in locked:
+            raise ValueError(
+                "DeepSeek-V4.1 vision with prefill CP runs eager prefill; remove "
+                "the explicit prefill CUDA graph backend."
+            )
+        declare_resolution(
+            server_args,
+            "validate_deepseek_v41_features",
+            cuda_graph_config=with_phase(
+                cfg.cuda_graph_config, Phase.PREFILL, backend=Backend.DISABLED
+            ),
+        )
+        logger.warning(
+            "Disabling the prefill CUDA graph for DeepSeek-V4.1 vision with prefill CP."
+        )
+    if (
+        str(cfg.speculative_algorithm).upper() == "DSPARK"
+        and cfg.enable_decoder_swa_bounded_replay
+    ):
+        raise ValueError(
+            "DeepSeek-V4.1 vision with prefill CP does not support DSpark together "
+            "with --enable-decoder-swa-bounded-replay yet."
+        )

python/sglang/srt/managers/schedule_policy.py

@@ -1019,6 +1019,15 @@ class PrefillAdder:
             else AddReqResult.CONTINUE
         )
 
+    def can_share_extend_batch(self, req: Req) -> bool:
+        # Token embedding overrides embed the batch's raw input_ids before the
+        # model runs, and that lookup cannot index multimodal placeholder hash IDs.
+        if req.positional_embed_overrides is not None:
+            return all(r.multimodal_inputs is None for r in self.can_run_list)
+        if req.multimodal_inputs is not None:
+            return all(r.positional_embed_overrides is None for r in self.can_run_list)
+        return True
+
     def add_chunked_req(self, req: Req):
         if self.dllm_config is not None:
             _rem_tokens = self._get_dllm_remain_tokens()

python/sglang/srt/managers/scheduler.py

@@ -3940,6 +3940,8 @@ class Scheduler(
         for req in self.waiting_queue:
             if self.enable_lora and not self.can_schedule_lora_req(req, running_loras):
                 continue
+            if not adder.can_share_extend_batch(req):
+                break
 
             running_bs = len(running_batch.reqs)
             candidate_beam_width = (

python/sglang/srt/managers/tokenizer_manager.py

@@ -1279,6 +1279,16 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
                 raise ValueError(
                     "encoder SWA replay cannot return cached prompt logprobs"
                 )
+        requests_embed_overrides = obj.positional_embed_overrides is not None or (
+            isinstance(obj, EmbeddingReqInput)
+            and obj.embed_overrides is not None
+            and obj.embed_override_token_id is not None
+        )
+        if requests_embed_overrides and obj.contains_mm_input():
+            raise ValueError(
+                "embedding overrides cannot be combined with image, video, or audio "
+                "inputs"
+            )
         _max_req_len = self.context_len
         input_token_num = len(input_ids) if input_ids is not None else 0
         input_token_num += self.num_reserved_tokens

python/sglang/srt/model_executor/model_runner.py

@@ -1646,6 +1646,7 @@ class ModelRunner:
             forward_batch.replace_embeds is not None
             and forward_batch.replace_positions is not None
         ):
+            misc_utils.validate_replace_embeds_batch(forward_batch)
             # Token embedding overrides: get base embeddings, scatter replacements
             if "input_embeds" not in kwargs:
                 embed_layer = self.model.get_input_embeddings()

python/sglang/srt/model_executor/model_runner_components/misc_utils.py

@@ -18,6 +18,7 @@ from sglang.srt.server_args import CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACK
 
 if TYPE_CHECKING:
     from sglang.srt.configs.model_config import ModelConfig
+    from sglang.srt.model_executor.forward_batch_info import ForwardBatch
 
 logger = logging.getLogger(__name__)
 
@@ -105,3 +106,24 @@ def resolve_pp_proxy_dspark_hidden_size(
     if isinstance(model, _SupportsDSparkPPProxy):
         return model.get_pp_proxy_dspark_hidden_size()
     return 0
+
+
+def validate_replace_embeds_batch(forward_batch: ForwardBatch) -> None:
+    if forward_batch.mm_inputs is None:
+        return
+    for mm_inputs, prefix_len, extend_len in zip(
+        forward_batch.mm_inputs,
+        forward_batch.extend_prefix_lens_cpu,
+        forward_batch.extend_seq_lens_cpu,
+    ):
+        if mm_inputs is None:
+            continue
+        chunk_end = prefix_len + extend_len
+        for item in mm_inputs.mm_items:
+            for start, end in item.offsets or ():
+                if start < chunk_end and end >= prefix_len:
+                    # Placeholder rows carry hash IDs the base embedding lookup cannot index.
+                    raise ValueError(
+                        "Token embedding overrides cannot share an extend batch with "
+                        "multimodal placeholders"
+                    )

python/sglang/srt/model_executor/runner/eager_runner.py

@@ -387,12 +387,13 @@ class EagerRunner(BaseRunner):
 
         input_ids = forward_batch.input_ids
         input_embeds = kwargs.get("input_embeds")
-        # Multimodal spans must be embedded in global token order, before CP
-        # slicing. The model may also normalize image hash IDs for its router.
-        prepare_inputs = getattr(model, "prepare_language_model_inputs", None)
-        if prepare_inputs is not None:
-            input_ids, input_embeds = prepare_inputs(
-                input_ids, forward_batch, input_embeds
+        if hasattr(model, "prepare_model_inputs"):
+            # Multimodal offsets are request-global, so the merge and the
+            # placeholder-ID remap must see the full extend layout first.
+            input_ids, input_embeds = model.prepare_model_inputs(
+                input_ids=input_ids,
+                forward_batch=forward_batch,
+                input_embeds=input_embeds,
             )
         if input_embeds is None:
             input_embeds = model.get_input_embeddings()(input_ids)

python/sglang/srt/models/deepseek_v4.py

@@ -74,6 +74,7 @@ from sglang.srt.layers.communicator_dsa_cp import (
     dsa_cp_gather_hidden_states,
     dsa_cp_reduce_scatter_hidden_states,
 )
+from sglang.srt.layers.cp.base import is_zigzag
 from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx
 from sglang.srt.layers.cp.utils import (
     cp_gather_full_sequence_states,
@@ -4897,14 +4898,18 @@ class DeepseekV4ForCausalLM(nn.Module):
             and not getattr(config, "language_model_only", False)
         ):
             if (
-                get_parallel().attn_cp_size != 1
-                or get_pp_group().world_size != 1
+                get_pp_group().world_size != 1
                 or not _v41_vision_a2a_supported()
             ):
                 raise ValueError(
-                    "V4.1 vision supports TP/EP/DP without CP or PP; "
+                    "V4.1 vision supports TP/EP/DP without PP; "
                     "MoE A2A is supported only with MegaMoE on a PD decode node"
                 )
+            if get_parallel().attn_cp_size != 1 and (_is_npu or is_zigzag()):
+                raise ValueError(
+                    "V4.1 vision context parallelism requires the CUDA interleave "
+                    "strategy; NPU and zigzag CP are not supported yet"
+                )
 
             args = SimpleNamespace(**vars(config), dim=config.hidden_size)
             self.vision = ViT(args)
@@ -5065,6 +5070,34 @@ class DeepseekV4ForCausalLM(nn.Module):
     def get_input_embeddings(self) -> nn.Module:
         return self.model.get_input_embeddings()
 
+    def prepare_model_inputs(
+        self,
+        input_ids: torch.Tensor,
+        forward_batch: ForwardBatch,
+        input_embeds: Optional[torch.Tensor],
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
+        if self.vision is None:
+            return input_ids, input_embeds
+        if (
+            not forward_batch.forward_mode.is_decode()
+            and not forward_batch.forward_mode.is_target_verify()
+            and forward_batch.mm_inputs is not None
+            and any(x is not None for x in forward_batch.mm_inputs)
+        ):
+            if input_embeds is not None:
+                raise ValueError("Cannot combine input_embeds and image inputs")
+            input_embeds = self._prepare_mm_embeddings(input_ids, forward_batch)
+        if not (
+            forward_batch.forward_mode.is_decode_or_idle()
+            or forward_batch.forward_mode.is_target_verify()
+        ):
+            # Decode/verify IDs are already vocabulary IDs; remap prompt image
+            # hashes for Engram and routing.
+            input_ids = input_ids.masked_fill(
+                input_ids >= MM_PAD_SHIFT_VALUE, self.config.image_token_id
+            )
+        return input_ids, input_embeds
+
     def set_dspark_layers_to_capture(self, layer_ids: List[int]) -> None:
         if not self.pp_group.is_last_rank:
             return
@@ -5115,31 +5148,11 @@ class DeepseekV4ForCausalLM(nn.Module):
         input_ids: torch.Tensor,
         forward_batch: ForwardBatch,
         input_embeds: Optional[torch.Tensor] = None,
-    ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
-        """Prepare full-sequence image embeddings and model IDs before CP splits.
-
-        Scheduler hash IDs stay intact for multimodal cache keys; the language
-        model uses image_token_id for Engram masking and visual MoE routing.
-        """
-        if (
-            getattr(self, "vision", None) is not None
-            and not forward_batch.forward_mode.is_decode()
-            and not forward_batch.forward_mode.is_target_verify()
-            and forward_batch.mm_inputs is not None
-            and any(x is not None for x in forward_batch.mm_inputs)
-        ):
-            if input_embeds is not None:
-                raise ValueError("Cannot combine input_embeds and image inputs")
-            input_embeds = self._prepare_mm_embeddings(input_ids, forward_batch)
-        if getattr(self, "vision", None) is not None and not (
-            forward_batch.forward_mode.is_decode_or_idle()
-            or forward_batch.forward_mode.is_target_verify()
-        ):
-            # Decode/verify IDs are already vocabulary IDs; remap prompt image
-            # hashes for Engram and routing.
-            input_ids = input_ids.masked_fill(
-                input_ids >= MM_PAD_SHIFT_VALUE, self.config.image_token_id
-            )
+        pp_proxy_tensors: Optional[PPProxyTensors] = None,
+    ) -> torch.Tensor:
+        input_ids, input_embeds = self.prepare_model_inputs(
+            input_ids=input_ids, forward_batch=forward_batch, input_embeds=input_embeds
+        )
 
         return input_ids, input_embeds
 

python/sglang/srt/models/deepseek_v4_nextn.py

@@ -222,6 +222,7 @@ class DeepseekV4ForCausalLMNextN(DeepseekV4ForCausalLM):
         self.quant_config = quant_config
         self.wo_a_fp8 = wo_a_fp8_gemm_enabled(quant_config)
         self.determine_num_fused_shared_experts()
+        self.vision = None
 
         self.model = DeepseekV4ModelNextN(
             config, quant_config, prefix=add_prefix("model", prefix)