[DSV4.1][PD][1/N] Support prefill node use cp parallel strategy (#40323)

This commit is contained in:
BingjiaWang
2026-10-12 01:05:21 -07:00
committed by GitHub
parent 54409c2fa0
commit 62ff466ac0
9 changed files with 216 additions and 30 deletions
@@ -289,23 +289,41 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
read_ragged_verify_mode,
)
# CP size may still be implicit here; validate_deepseek_v4_cp resolves it.
supported_prefill_cp = (
cfg.disaggregation_mode == "prefill"
and cfg.enable_prefill_cp
and cfg.cp_strategy == "interleave"
)
if (
read_ragged_verify_mode() is not RaggedVerifyMode.STATIC
or cfg.disaggregation_transfer_backend != "mooncake"
or cfg.enable_prefill_cp
or cfg.attn_cp_size != 1
or (
(cfg.enable_prefill_cp or cfg.attn_cp_size != 1)
and not supported_prefill_cp
)
or cfg.dcp_size != 1
):
raise ValueError(
"DeepSeek-V4.1 DSpark PD requires static verify, Mooncake, "
"and CP=1 on both servers. DP attention is supported when "
"DCP=1, and CP=1 on decode. Prefill CP requires "
"--enable-prefill-cp with the interleave strategy. "
"DP attention is supported when "
"both servers use the same block size and target/draft KV layout."
)
prefill_graph = cfg.cuda_graph_config.prefill
if prefill_graph.backend != Backend.DISABLED and prefill_graph.max_seq_len is None:
# The captured low-ratio indexer scores a static context width; 16k
# keeps it inside the candidate window at under 1 ms per layer.
cp_breakable_prefill = (
cfg.enable_prefill_cp
and cfg.cp_strategy == "interleave"
and cfg.tp_size > 1
and prefill_graph.backend == Backend.BREAKABLE
)
if (
prefill_graph.backend != Backend.DISABLED
and prefill_graph.max_seq_len is None
and not cp_breakable_prefill
):
declare_resolution(
server_args,
"validate_deepseek_v41_features",
+4 -2
View File
@@ -1068,12 +1068,14 @@ def handle_language_model_only(server_args: Any):
):
if flag:
raise ValueError(f"--language-model-only cannot be combined with {name}")
if cfg.disaggregation_mode != "null":
hf_config = model_config_of(server_args).hf_config
# V4.1 text-only workers use the standard PD KV transfer path.
if cfg.disaggregation_mode != "null" and hf_config.model_type != "deepseek_v41":
raise ValueError(
"--language-model-only is incompatible with --disaggregation-mode "
"prefill/decode"
)
architectures = model_config_of(server_args).hf_config.architectures
architectures = hf_config.architectures
if not any(
a in server_args.LANGUAGE_MODEL_ONLY_ARCHITECTURES for a in architectures
):
@@ -1214,17 +1214,20 @@ class CommonKVManager(BaseKVManager):
"enable DSpark with the same block size and target/draft KV "
"layout. Upgrade both servers together."
)
if info.attn_cp_size != 1 or self.attn_cp_size != 1:
raise RuntimeError(
"DeepSeek-V4.1 DP-only DSpark PD requires CP=1 on both servers"
)
non_cp_mla_layout = info.attn_cp_size == self.attn_cp_size == 1 and (
self.is_mla_backend or self.is_hybrid_mla_backend
decode_tp_size_aligns_prefill_world_size = (
info.attn_cp_size > 1
and (self.is_mla_backend or self.is_hybrid_mla_backend)
and self.attn_cp_size == 1
and info.attn_tp_size * info.attn_cp_size == self.attn_tp_size
)
if info.attn_tp_size != self.attn_tp_size and not non_cp_mla_layout:
if (
info.attn_tp_size != self.attn_tp_size
and not decode_tp_size_aligns_prefill_world_size
):
raise RuntimeError(
"DeepSeek-V4.1 DSpark PD requires matching attention TP "
"unless both servers use CP=1 with an MLA KV layout"
"DeepSeek-V4.1 DSpark PD requires the same TP size on both "
"servers (including prefill CP ranks for MLA)"
)
if self.dcp_size > 1:
@@ -1530,7 +1530,13 @@ class DeepseekV4AttnBackend(
@property
def low_ratio_prefill_graph(self) -> bool:
return bool(self.low_ratios) and has_dense_fp4_indexer() and is_sm100_or_newer()
"""Whether ratio-1/2 sources use captured projections and indexer metadata."""
return (
bool(self.low_ratios)
and has_dense_fp4_indexer()
and is_sm100_or_newer()
and get_parallel().attn_cp_size == 1
)
def can_run_prefill_cuda_graph(self, forward_batch: ForwardBatch) -> bool:
max_seq_len = _prefill_graph_max_seq_len()
@@ -375,6 +375,7 @@ class EagerRunner(BaseRunner):
model_kwargs = {}
if (pp_proxy_tensors := kwargs.get("pp_proxy_tensors")) is not None:
model_kwargs["pp_proxy_tensors"] = pp_proxy_tensors
with cp_shard_model_inputs(
input_embeds,
positions,
@@ -1417,9 +1417,9 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
batch_max_context_len=batch_max_context_len,
):
return False
if getattr(self, "enable_cp_bcg_capture", False) and is_cp_active(
forward_batch
):
if getattr(self, "enable_cp_bcg_capture", False):
if not is_cp_active(forward_batch):
return False
assert self.prefill_cp_bcg_input is not None
if (
self.prefill_cp_bcg_input.select_replay_bucket_for_batch(
+37 -9
View File
@@ -2749,7 +2749,8 @@ class DeepseekV4DecoderLayer(nn.Module):
is_nextn=is_nextn,
is_deepseek_v4=True,
vl_correction_bias=config.model_type == "deepseek_v41"
and config.vision_n_layers > 0,
and config.vision_n_layers > 0
and not getattr(config, "language_model_only", False),
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -3542,7 +3543,14 @@ class DeepseekV4DecoderLayer(nn.Module):
)
if _use_cp and get_moe_a2a_backend().is_none():
hidden_states = attn_cp_interleave_reduce_scatter(hidden_states)
if self.config.model_type == "deepseek_v41":
parallel = get_parallel()
hidden_states = parallel.tp_group.all_reduce(hidden_states)
hidden_states = hidden_states.tensor_split(parallel.attn_cp_size)[
parallel.attn_cp_rank
].contiguous()
else:
hidden_states = attn_cp_interleave_reduce_scatter(hidden_states)
elif _use_tp_moe_gather:
hidden_states, global_hidden_states = (
get_local_dp_buffer(get_parallel().tp_group),
@@ -4597,7 +4605,11 @@ class DeepseekV4ForCausalLM(nn.Module):
self.wo_a_fp8 = wo_a_fp8_gemm_enabled(quant_config)
self.determine_num_fused_shared_experts()
self.vision = None
if config.model_type == "deepseek_v41" and config.vision_n_layers > 0:
if (
config.model_type == "deepseek_v41"
and config.vision_n_layers > 0
and not getattr(config, "language_model_only", False)
):
if (
get_parallel().attn_cp_size != 1
or get_parallel().pp_group.world_size != 1
@@ -4815,16 +4827,19 @@ class DeepseekV4ForCausalLM(nn.Module):
0 if is_shared_experts_fusion_disabled() else self.config.n_shared_experts
)
def forward(
def prepare_language_model_inputs(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
forward_batch: ForwardBatch,
input_embeds: Optional[torch.Tensor] = None,
pp_proxy_tensors: Optional[PPProxyTensors] = None,
) -> torch.Tensor:
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
"""Prepare full-sequence image embeddings and model IDs before CP splits.
Scheduler hash IDs stay intact for multimodal cache keys; the language
model uses image_token_id for Engram masking and visual MoE routing.
"""
if (
self.vision is not None
getattr(self, "vision", None) is not None
and not forward_batch.forward_mode.is_decode()
and not forward_batch.forward_mode.is_target_verify()
and forward_batch.mm_inputs is not None
@@ -4833,7 +4848,7 @@ class DeepseekV4ForCausalLM(nn.Module):
if input_embeds is not None:
raise ValueError("Cannot combine input_embeds and image inputs")
input_embeds = self._prepare_mm_embeddings(input_ids, forward_batch)
if self.vision is not None and not (
if getattr(self, "vision", None) is not None and not (
forward_batch.forward_mode.is_decode_or_idle()
or forward_batch.forward_mode.is_target_verify()
):
@@ -4843,6 +4858,19 @@ class DeepseekV4ForCausalLM(nn.Module):
input_ids >= MM_PAD_SHIFT_VALUE, self.config.image_token_id
)
return input_ids, input_embeds
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
forward_batch: ForwardBatch,
input_embeds: Optional[torch.Tensor] = None,
pp_proxy_tensors: Optional[PPProxyTensors] = None,
) -> torch.Tensor:
input_ids, input_embeds = self.prepare_language_model_inputs(
input_ids, forward_batch, input_embeds
)
with get_attn_tp_context().maybe_input_scattered(forward_batch):
hidden_states = self.model.forward(
input_ids, positions, forward_batch, input_embeds, pp_proxy_tensors
+1
View File
@@ -323,6 +323,7 @@ class ServerArgs:
return resolved
LANGUAGE_MODEL_ONLY_ARCHITECTURES = (
"DeepseekV4ForCausalLM",
"MuseGlimmerForConditionalGeneration",
"Cosmos3ForConditionalGeneration",
"Cosmos3EdgeForConditionalGeneration",