From 62ff466ac0c2cdd5521acf7d4c673fa7630eb083 Mon Sep 17 00:00:00 2001 From: BingjiaWang Date: Mon, 12 Oct 2026 16:05:21 +0800 Subject: [PATCH] [DSV4.1][PD][1/N] Support prefill node use cp parallel strategy (#40323) --- .../sglang/srt/arg_groups/deepseek_v4_hook.py | 30 ++++- python/sglang/srt/arg_groups/model_hook.py | 6 +- .../sglang/srt/disaggregation/common/conn.py | 21 +-- .../layers/attention/deepseek_v4_backend.py | 8 +- .../srt/model_executor/runner/eager_runner.py | 1 + .../runner/prefill_cuda_graph_runner.py | 6 +- python/sglang/srt/models/deepseek_v4.py | 46 +++++-- python/sglang/srt/server_args.py | 1 + .../models/test_deepseek_v41_pd_cp_dspark.py | 127 ++++++++++++++++++ 9 files changed, 216 insertions(+), 30 deletions(-) create mode 100644 test/registered/e2e/models/test_deepseek_v41_pd_cp_dspark.py diff --git a/python/sglang/srt/arg_groups/deepseek_v4_hook.py b/python/sglang/srt/arg_groups/deepseek_v4_hook.py index 21995ae36f..46abe9721e 100644 --- a/python/sglang/srt/arg_groups/deepseek_v4_hook.py +++ b/python/sglang/srt/arg_groups/deepseek_v4_hook.py @@ -289,23 +289,41 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None: read_ragged_verify_mode, ) + # CP size may still be implicit here; validate_deepseek_v4_cp resolves it. + supported_prefill_cp = ( + cfg.disaggregation_mode == "prefill" + and cfg.enable_prefill_cp + and cfg.cp_strategy == "interleave" + ) if ( read_ragged_verify_mode() is not RaggedVerifyMode.STATIC or cfg.disaggregation_transfer_backend != "mooncake" - or cfg.enable_prefill_cp - or cfg.attn_cp_size != 1 + or ( + (cfg.enable_prefill_cp or cfg.attn_cp_size != 1) + and not supported_prefill_cp + ) or cfg.dcp_size != 1 ): raise ValueError( "DeepSeek-V4.1 DSpark PD requires static verify, Mooncake, " - "and CP=1 on both servers. DP attention is supported when " + "DCP=1, and CP=1 on decode. Prefill CP requires " + "--enable-prefill-cp with the interleave strategy. " + "DP attention is supported when " "both servers use the same block size and target/draft KV layout." ) prefill_graph = cfg.cuda_graph_config.prefill - if prefill_graph.backend != Backend.DISABLED and prefill_graph.max_seq_len is None: - # The captured low-ratio indexer scores a static context width; 16k - # keeps it inside the candidate window at under 1 ms per layer. + cp_breakable_prefill = ( + cfg.enable_prefill_cp + and cfg.cp_strategy == "interleave" + and cfg.tp_size > 1 + and prefill_graph.backend == Backend.BREAKABLE + ) + if ( + prefill_graph.backend != Backend.DISABLED + and prefill_graph.max_seq_len is None + and not cp_breakable_prefill + ): declare_resolution( server_args, "validate_deepseek_v41_features", diff --git a/python/sglang/srt/arg_groups/model_hook.py b/python/sglang/srt/arg_groups/model_hook.py index 05957e9902..b71a4cc535 100644 --- a/python/sglang/srt/arg_groups/model_hook.py +++ b/python/sglang/srt/arg_groups/model_hook.py @@ -1068,12 +1068,14 @@ def handle_language_model_only(server_args: Any): ): if flag: raise ValueError(f"--language-model-only cannot be combined with {name}") - if cfg.disaggregation_mode != "null": + hf_config = model_config_of(server_args).hf_config + # V4.1 text-only workers use the standard PD KV transfer path. + if cfg.disaggregation_mode != "null" and hf_config.model_type != "deepseek_v41": raise ValueError( "--language-model-only is incompatible with --disaggregation-mode " "prefill/decode" ) - architectures = model_config_of(server_args).hf_config.architectures + architectures = hf_config.architectures if not any( a in server_args.LANGUAGE_MODEL_ONLY_ARCHITECTURES for a in architectures ): diff --git a/python/sglang/srt/disaggregation/common/conn.py b/python/sglang/srt/disaggregation/common/conn.py index 24ed9a2205..db2b6d14ac 100644 --- a/python/sglang/srt/disaggregation/common/conn.py +++ b/python/sglang/srt/disaggregation/common/conn.py @@ -1214,17 +1214,20 @@ class CommonKVManager(BaseKVManager): "enable DSpark with the same block size and target/draft KV " "layout. Upgrade both servers together." ) - if info.attn_cp_size != 1 or self.attn_cp_size != 1: - raise RuntimeError( - "DeepSeek-V4.1 DP-only DSpark PD requires CP=1 on both servers" - ) - non_cp_mla_layout = info.attn_cp_size == self.attn_cp_size == 1 and ( - self.is_mla_backend or self.is_hybrid_mla_backend + + decode_tp_size_aligns_prefill_world_size = ( + info.attn_cp_size > 1 + and (self.is_mla_backend or self.is_hybrid_mla_backend) + and self.attn_cp_size == 1 + and info.attn_tp_size * info.attn_cp_size == self.attn_tp_size ) - if info.attn_tp_size != self.attn_tp_size and not non_cp_mla_layout: + if ( + info.attn_tp_size != self.attn_tp_size + and not decode_tp_size_aligns_prefill_world_size + ): raise RuntimeError( - "DeepSeek-V4.1 DSpark PD requires matching attention TP " - "unless both servers use CP=1 with an MLA KV layout" + "DeepSeek-V4.1 DSpark PD requires the same TP size on both " + "servers (including prefill CP ranks for MLA)" ) if self.dcp_size > 1: diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index ffcbedaec9..832c7678b0 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -1530,7 +1530,13 @@ class DeepseekV4AttnBackend( @property def low_ratio_prefill_graph(self) -> bool: - return bool(self.low_ratios) and has_dense_fp4_indexer() and is_sm100_or_newer() + """Whether ratio-1/2 sources use captured projections and indexer metadata.""" + return ( + bool(self.low_ratios) + and has_dense_fp4_indexer() + and is_sm100_or_newer() + and get_parallel().attn_cp_size == 1 + ) def can_run_prefill_cuda_graph(self, forward_batch: ForwardBatch) -> bool: max_seq_len = _prefill_graph_max_seq_len() diff --git a/python/sglang/srt/model_executor/runner/eager_runner.py b/python/sglang/srt/model_executor/runner/eager_runner.py index 7b406aafd5..b1f21a26da 100644 --- a/python/sglang/srt/model_executor/runner/eager_runner.py +++ b/python/sglang/srt/model_executor/runner/eager_runner.py @@ -375,6 +375,7 @@ class EagerRunner(BaseRunner): model_kwargs = {} if (pp_proxy_tensors := kwargs.get("pp_proxy_tensors")) is not None: model_kwargs["pp_proxy_tensors"] = pp_proxy_tensors + with cp_shard_model_inputs( input_embeds, positions, diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index e8e91c21a6..242d0b23e6 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -1417,9 +1417,9 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): batch_max_context_len=batch_max_context_len, ): return False - if getattr(self, "enable_cp_bcg_capture", False) and is_cp_active( - forward_batch - ): + if getattr(self, "enable_cp_bcg_capture", False): + if not is_cp_active(forward_batch): + return False assert self.prefill_cp_bcg_input is not None if ( self.prefill_cp_bcg_input.select_replay_bucket_for_batch( diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index dd6bf85ae0..cce25ceef2 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -2749,7 +2749,8 @@ class DeepseekV4DecoderLayer(nn.Module): is_nextn=is_nextn, is_deepseek_v4=True, vl_correction_bias=config.model_type == "deepseek_v41" - and config.vision_n_layers > 0, + and config.vision_n_layers > 0 + and not getattr(config, "language_model_only", False), ) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) @@ -3542,7 +3543,14 @@ class DeepseekV4DecoderLayer(nn.Module): ) if _use_cp and get_moe_a2a_backend().is_none(): - hidden_states = attn_cp_interleave_reduce_scatter(hidden_states) + if self.config.model_type == "deepseek_v41": + parallel = get_parallel() + hidden_states = parallel.tp_group.all_reduce(hidden_states) + hidden_states = hidden_states.tensor_split(parallel.attn_cp_size)[ + parallel.attn_cp_rank + ].contiguous() + else: + hidden_states = attn_cp_interleave_reduce_scatter(hidden_states) elif _use_tp_moe_gather: hidden_states, global_hidden_states = ( get_local_dp_buffer(get_parallel().tp_group), @@ -4597,7 +4605,11 @@ class DeepseekV4ForCausalLM(nn.Module): self.wo_a_fp8 = wo_a_fp8_gemm_enabled(quant_config) self.determine_num_fused_shared_experts() self.vision = None - if config.model_type == "deepseek_v41" and config.vision_n_layers > 0: + if ( + config.model_type == "deepseek_v41" + and config.vision_n_layers > 0 + and not getattr(config, "language_model_only", False) + ): if ( get_parallel().attn_cp_size != 1 or get_parallel().pp_group.world_size != 1 @@ -4815,16 +4827,19 @@ class DeepseekV4ForCausalLM(nn.Module): 0 if is_shared_experts_fusion_disabled() else self.config.n_shared_experts ) - def forward( + def prepare_language_model_inputs( self, input_ids: torch.Tensor, - positions: torch.Tensor, forward_batch: ForwardBatch, input_embeds: Optional[torch.Tensor] = None, - pp_proxy_tensors: Optional[PPProxyTensors] = None, - ) -> torch.Tensor: + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """Prepare full-sequence image embeddings and model IDs before CP splits. + + Scheduler hash IDs stay intact for multimodal cache keys; the language + model uses image_token_id for Engram masking and visual MoE routing. + """ if ( - self.vision is not None + getattr(self, "vision", None) is not None and not forward_batch.forward_mode.is_decode() and not forward_batch.forward_mode.is_target_verify() and forward_batch.mm_inputs is not None @@ -4833,7 +4848,7 @@ class DeepseekV4ForCausalLM(nn.Module): if input_embeds is not None: raise ValueError("Cannot combine input_embeds and image inputs") input_embeds = self._prepare_mm_embeddings(input_ids, forward_batch) - if self.vision is not None and not ( + if getattr(self, "vision", None) is not None and not ( forward_batch.forward_mode.is_decode_or_idle() or forward_batch.forward_mode.is_target_verify() ): @@ -4843,6 +4858,19 @@ class DeepseekV4ForCausalLM(nn.Module): input_ids >= MM_PAD_SHIFT_VALUE, self.config.image_token_id ) + return input_ids, input_embeds + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + forward_batch: ForwardBatch, + input_embeds: Optional[torch.Tensor] = None, + pp_proxy_tensors: Optional[PPProxyTensors] = None, + ) -> torch.Tensor: + input_ids, input_embeds = self.prepare_language_model_inputs( + input_ids, forward_batch, input_embeds + ) with get_attn_tp_context().maybe_input_scattered(forward_batch): hidden_states = self.model.forward( input_ids, positions, forward_batch, input_embeds, pp_proxy_tensors diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index cfe97d5500..c2ac0926cb 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -323,6 +323,7 @@ class ServerArgs: return resolved LANGUAGE_MODEL_ONLY_ARCHITECTURES = ( + "DeepseekV4ForCausalLM", "MuseGlimmerForConditionalGeneration", "Cosmos3ForConditionalGeneration", "Cosmos3EdgeForConditionalGeneration", diff --git a/test/registered/e2e/models/test_deepseek_v41_pd_cp_dspark.py b/test/registered/e2e/models/test_deepseek_v41_pd_cp_dspark.py new file mode 100644 index 0000000000..8afa163cb5 --- /dev/null +++ b/test/registered/e2e/models/test_deepseek_v41_pd_cp_dspark.py @@ -0,0 +1,127 @@ +"""V4.1 prefill CP + PD disaggregation + DSpark on eight B300 GPUs. + +GPU 0-3: TP4/CP4 prefill. GPU 4-7: TP4 decode. Both workers use the +DeepSeek-V4.1-Flash checkpoint's bundled DSpark draft and Mooncake transfer. +""" + +import unittest + +import requests + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin +from sglang.test.server_fixtures.disaggregation_fixture import ( + PDDisaggregationServerBase, +) +from sglang.test.test_utils import try_cached_model + +# Initial estimate; update after the first B300 CI run. +register_cuda_ci(est_time=1800, stage="extra-b", runner_config="8-gpu-b300") + +MODEL = "deepseek-ai/DeepSeek-V4.1-Flash" + +COMMON_ARGS = [ + "--language-model-only", + "--attention-backend", + "dsv4", + "--moe-a2a-backend", + "none", + "--moe-runner-backend", + "flashinfer_mxfp4", + "--ep-size", + "4", + "--speculative-algorithm", + "DSPARK", + "--speculative-dspark-block-size", + "5", + "--mem-fraction-static", + "0.8", + "--max-running-requests", + "32", + "--cuda-graph-max-bs-decode", + "32", + "--cuda-graph-backend-prefill", + "disabled", + "--context-length", + "16384", + "--model-loader-extra-config", + '{"enable_multithread_load": true, "num_threads": 6}', + "--random-seed", + "0", +] + + +class TestDeepseekV41PDCPDSpark(PDDisaggregationServerBase, GSM8KMixin): + model = MODEL + prefill_tp_size = 4 + decode_tp_size = 4 + decode_base_gpu_id = 4 + + gsm8k_score_threshold = 0.85 + gsm8k_num_examples = 200 + gsm8k_num_threads = 32 + gsm8k_num_shots = 8 + + extra_prefill_args = COMMON_ARGS + [ + "--enable-prefill-cp", + "--attn-cp-size", + "4", + "--cp-strategy", + "interleave", + # Small chunks exercise continuation as well as the P-to-D handoff. + "--chunked-prefill-size", + "1024", + "--max-prefill-tokens", + "4096", + ] + extra_decode_args = COMMON_ARGS + extra_prefill_env = {"SGLANG_RAGGED_VERIFY_MODE": "static"} + extra_decode_env = {"SGLANG_RAGGED_VERIFY_MODE": "static"} + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.model = try_cached_model(MODEL) + # V4.1's target/draft KV transfer requires Mooncake on both sides. + cls.transfer_backend = ["--disaggregation-transfer-backend", "mooncake"] + cls.launch_all() + + def _server_info(self, url): + response = requests.get(url + "/server_info", timeout=30) + response.raise_for_status() + return response.json() + + def test_gsm8k(self): + # Inspect workers directly: the router does not expose both configs. + for url, role, cp_size in ( + (self.prefill_url, "prefill", 4), + (self.decode_url, "decode", 1), + ): + info = self._server_info(url) + self.assertEqual(info["disaggregation_mode"], role) + self.assertEqual(info["disaggregation_transfer_backend"], "mooncake") + self.assertEqual(info["tp_size"], 4) + self.assertEqual(info["attn_cp_size"], cp_size) + self.assertEqual(info["enable_prefill_cp"], role == "prefill") + self.assertEqual(info["speculative_algorithm"], "DSPARK") + self.assertEqual(info["speculative_num_draft_tokens"], 6) + self.assertTrue(info["language_model_only"]) + + # Completion evaluation avoids V4.1 chat-template thinking mode. + # base_url is the PD router, so every request exercises both workers. + super().test_gsm8k() + + # Require evidence that DSpark actually accepted draft tokens. Query D + # directly rather than letting the optional mixin check skip a missing + # metric from the router or the prefill worker. + states = self._server_info(self.decode_url)["internal_states"] + self.assertTrue(states, "Decode worker returned no scheduler state") + for state in states: + self.assertIn("avg_spec_accept_length", state) + accept_length = state["avg_spec_accept_length"] + print(f"V4.1 PD+CP DSpark accept length: {accept_length:.4f}") + self.assertGreater(accept_length, 1.0) + + +if __name__ == "__main__": + unittest.main()