[DSV4.1][PD][1/N] Support prefill node use cp parallel strategy (#40323)
This commit is contained in:
@@ -248,25 +248,43 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
read_ragged_verify_mode,
|
||||
)
|
||||
|
||||
# CP size may still be implicit here; validate_deepseek_v4_cp resolves it.
|
||||
supported_prefill_cp = (
|
||||
cfg.disaggregation_mode == "prefill"
|
||||
and cfg.enable_prefill_cp
|
||||
and cfg.cp_strategy == "interleave"
|
||||
)
|
||||
if (
|
||||
read_ragged_verify_mode() is not RaggedVerifyMode.STATIC
|
||||
or cfg.disaggregation_transfer_backend != "mooncake"
|
||||
or cfg.enable_prefill_cp
|
||||
or cfg.attn_cp_size != 1
|
||||
or (
|
||||
(cfg.enable_prefill_cp or cfg.attn_cp_size != 1)
|
||||
and not supported_prefill_cp
|
||||
)
|
||||
or cfg.dcp_size != 1
|
||||
):
|
||||
raise ValueError(
|
||||
"DeepSeek-V4.1 DSpark PD requires static verify, Mooncake, "
|
||||
"and CP=1 on both servers. DP attention is supported when "
|
||||
"DCP=1, and CP=1 on decode. Prefill CP requires "
|
||||
"--enable-prefill-cp with the interleave strategy. "
|
||||
"DP attention is supported when "
|
||||
"both servers use the same block size and target/draft KV layout."
|
||||
)
|
||||
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
|
||||
|
||||
prefill_graph = cfg.cuda_graph_config.prefill
|
||||
if prefill_graph.backend != Backend.DISABLED and prefill_graph.max_seq_len is None:
|
||||
# The captured low-ratio indexer scores a static context width; 16k
|
||||
# keeps it inside the candidate window at under 1 ms per layer.
|
||||
cp_breakable_prefill = (
|
||||
cfg.enable_prefill_cp
|
||||
and cfg.cp_strategy == "interleave"
|
||||
and cfg.tp_size > 1
|
||||
and prefill_graph.backend == Backend.BREAKABLE
|
||||
)
|
||||
if (
|
||||
prefill_graph.backend != Backend.DISABLED
|
||||
and prefill_graph.max_seq_len is None
|
||||
and not cp_breakable_prefill
|
||||
):
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"validate_deepseek_v41_features",
|
||||
|
||||
@@ -980,12 +980,14 @@ def handle_language_model_only(server_args: Any):
|
||||
):
|
||||
if flag:
|
||||
raise ValueError(f"--language-model-only cannot be combined with {name}")
|
||||
if cfg.disaggregation_mode != "null":
|
||||
hf_config = model_config_of(server_args).hf_config
|
||||
# V4.1 text-only workers use the standard PD KV transfer path.
|
||||
if cfg.disaggregation_mode != "null" and hf_config.model_type != "deepseek_v41":
|
||||
raise ValueError(
|
||||
"--language-model-only is incompatible with --disaggregation-mode "
|
||||
"prefill/decode"
|
||||
)
|
||||
architectures = model_config_of(server_args).hf_config.architectures
|
||||
architectures = hf_config.architectures
|
||||
if not any(
|
||||
a in server_args.LANGUAGE_MODEL_ONLY_ARCHITECTURES for a in architectures
|
||||
):
|
||||
|
||||
@@ -986,17 +986,20 @@ class CommonKVManager(BaseKVManager):
|
||||
"enable DSpark with the same block size and target/draft KV "
|
||||
"layout. Upgrade both servers together."
|
||||
)
|
||||
if info.attn_cp_size != 1 or self.attn_cp_size != 1:
|
||||
raise RuntimeError(
|
||||
"DeepSeek-V4.1 DP-only DSpark PD requires CP=1 on both servers"
|
||||
)
|
||||
non_cp_mla_layout = info.attn_cp_size == self.attn_cp_size == 1 and (
|
||||
self.is_mla_backend or self.is_hybrid_mla_backend
|
||||
|
||||
decode_tp_size_aligns_prefill_world_size = (
|
||||
info.attn_cp_size > 1
|
||||
and (self.is_mla_backend or self.is_hybrid_mla_backend)
|
||||
and self.attn_cp_size == 1
|
||||
and info.attn_tp_size * info.attn_cp_size == self.attn_tp_size
|
||||
)
|
||||
if info.attn_tp_size != self.attn_tp_size and not non_cp_mla_layout:
|
||||
if (
|
||||
info.attn_tp_size != self.attn_tp_size
|
||||
and not decode_tp_size_aligns_prefill_world_size
|
||||
):
|
||||
raise RuntimeError(
|
||||
"DeepSeek-V4.1 DSpark PD requires matching attention TP "
|
||||
"unless both servers use CP=1 with an MLA KV layout"
|
||||
"DeepSeek-V4.1 DSpark PD requires the same TP size on both "
|
||||
"servers (including prefill CP ranks for MLA)"
|
||||
)
|
||||
|
||||
if self.dcp_size > 1:
|
||||
|
||||
@@ -1514,7 +1514,13 @@ class DeepseekV4AttnBackend(
|
||||
|
||||
@property
|
||||
def low_ratio_prefill_graph(self) -> bool:
|
||||
return bool(self.low_ratios) and has_dense_fp4_indexer() and is_sm100_or_newer()
|
||||
"""Whether ratio-1/2 sources use captured projections and indexer metadata."""
|
||||
return (
|
||||
bool(self.low_ratios)
|
||||
and has_dense_fp4_indexer()
|
||||
and is_sm100_or_newer()
|
||||
and get_parallel().attn_cp_size == 1
|
||||
)
|
||||
|
||||
def can_run_prefill_cuda_graph(self, forward_batch: ForwardBatch) -> bool:
|
||||
max_seq_len = _prefill_graph_max_seq_len()
|
||||
|
||||
@@ -405,6 +405,7 @@ class EagerRunner(BaseRunner):
|
||||
model_kwargs = {}
|
||||
if (pp_proxy_tensors := kwargs.get("pp_proxy_tensors")) is not None:
|
||||
model_kwargs["pp_proxy_tensors"] = pp_proxy_tensors
|
||||
|
||||
with cp_shard_model_inputs(
|
||||
input_embeds,
|
||||
positions,
|
||||
|
||||
@@ -1337,9 +1337,9 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
batch_max_context_len=batch_max_context_len,
|
||||
):
|
||||
return False
|
||||
if getattr(self, "enable_cp_bcg_capture", False) and is_cp_active(
|
||||
forward_batch
|
||||
):
|
||||
if getattr(self, "enable_cp_bcg_capture", False):
|
||||
if not is_cp_active(forward_batch):
|
||||
return False
|
||||
assert self.prefill_cp_bcg_input is not None
|
||||
if (
|
||||
self.prefill_cp_bcg_input.select_replay_bucket_for_batch(
|
||||
|
||||
@@ -2652,7 +2652,8 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
is_nextn=is_nextn,
|
||||
is_deepseek_v4=True,
|
||||
vl_correction_bias=config.model_type == "deepseek_v41"
|
||||
and config.vision_n_layers > 0,
|
||||
and config.vision_n_layers > 0
|
||||
and not getattr(config, "language_model_only", False),
|
||||
)
|
||||
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
@@ -4913,7 +4914,11 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
self.wo_a_fp8 = wo_a_fp8_gemm_enabled(quant_config)
|
||||
self.determine_num_fused_shared_experts()
|
||||
self.vision = None
|
||||
if config.model_type == "deepseek_v41" and config.vision_n_layers > 0:
|
||||
if (
|
||||
config.model_type == "deepseek_v41"
|
||||
and config.vision_n_layers > 0
|
||||
and not getattr(config, "language_model_only", False)
|
||||
):
|
||||
if get_parallel().pp_group.world_size != 1 or not (
|
||||
get_moe_a2a_backend().is_none()
|
||||
or (_is_cuda and get_moe_a2a_backend().is_megamoe())
|
||||
@@ -5209,10 +5214,9 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
0 if is_shared_experts_fusion_disabled() else self.config.n_shared_experts
|
||||
)
|
||||
|
||||
def forward(
|
||||
def prepare_language_model_inputs(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
input_embeds: Optional[torch.Tensor] = None,
|
||||
pp_proxy_tensors: Optional[PPProxyTensors] = None,
|
||||
@@ -5221,6 +5225,19 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
input_ids=input_ids, forward_batch=forward_batch, input_embeds=input_embeds
|
||||
)
|
||||
|
||||
return input_ids, input_embeds
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
input_embeds: Optional[torch.Tensor] = None,
|
||||
pp_proxy_tensors: Optional[PPProxyTensors] = None,
|
||||
) -> torch.Tensor:
|
||||
input_ids, input_embeds = self.prepare_language_model_inputs(
|
||||
input_ids, forward_batch, input_embeds
|
||||
)
|
||||
with get_attn_tp_context().maybe_input_scattered(forward_batch):
|
||||
hidden_states = self.model.forward(
|
||||
input_ids, positions, forward_batch, input_embeds, pp_proxy_tensors
|
||||
|
||||
@@ -266,6 +266,7 @@ class ServerArgs:
|
||||
}
|
||||
|
||||
LANGUAGE_MODEL_ONLY_ARCHITECTURES = (
|
||||
"DeepseekV4ForCausalLM",
|
||||
"MuseGlimmerForConditionalGeneration",
|
||||
"Cosmos3ForConditionalGeneration",
|
||||
"Cosmos3EdgeForConditionalGeneration",
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
"""V4.1 prefill CP + PD disaggregation + DSpark on eight B300 GPUs.
|
||||
|
||||
GPU 0-3: TP4/CP4 prefill. GPU 4-7: TP4 decode. Both workers use the
|
||||
DeepSeek-V4.1-Flash checkpoint's bundled DSpark draft and Mooncake transfer.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
|
||||
import requests
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
|
||||
from sglang.test.server_fixtures.disaggregation_fixture import (
|
||||
PDDisaggregationServerBase,
|
||||
)
|
||||
from sglang.test.test_utils import try_cached_model
|
||||
|
||||
# Initial estimate; update after the first B300 CI run.
|
||||
register_cuda_ci(est_time=1800, stage="extra-b", runner_config="8-gpu-b300")
|
||||
|
||||
MODEL = "deepseek-ai/DeepSeek-V4.1-Flash"
|
||||
|
||||
COMMON_ARGS = [
|
||||
"--language-model-only",
|
||||
"--attention-backend",
|
||||
"dsv4",
|
||||
"--moe-a2a-backend",
|
||||
"none",
|
||||
"--moe-runner-backend",
|
||||
"flashinfer_mxfp4",
|
||||
"--ep-size",
|
||||
"4",
|
||||
"--speculative-algorithm",
|
||||
"DSPARK",
|
||||
"--speculative-dspark-block-size",
|
||||
"5",
|
||||
"--mem-fraction-static",
|
||||
"0.8",
|
||||
"--max-running-requests",
|
||||
"32",
|
||||
"--cuda-graph-max-bs-decode",
|
||||
"32",
|
||||
"--cuda-graph-backend-prefill",
|
||||
"disabled",
|
||||
"--context-length",
|
||||
"16384",
|
||||
"--model-loader-extra-config",
|
||||
'{"enable_multithread_load": true, "num_threads": 6}',
|
||||
"--random-seed",
|
||||
"0",
|
||||
]
|
||||
|
||||
|
||||
class TestDeepseekV41PDCPDSpark(PDDisaggregationServerBase, GSM8KMixin):
|
||||
model = MODEL
|
||||
prefill_tp_size = 4
|
||||
decode_tp_size = 4
|
||||
decode_base_gpu_id = 4
|
||||
|
||||
gsm8k_score_threshold = 0.85
|
||||
gsm8k_num_examples = 200
|
||||
gsm8k_num_threads = 32
|
||||
gsm8k_num_shots = 8
|
||||
|
||||
extra_prefill_args = COMMON_ARGS + [
|
||||
"--enable-prefill-cp",
|
||||
"--attn-cp-size",
|
||||
"4",
|
||||
"--cp-strategy",
|
||||
"interleave",
|
||||
# Small chunks exercise continuation as well as the P-to-D handoff.
|
||||
"--chunked-prefill-size",
|
||||
"1024",
|
||||
"--max-prefill-tokens",
|
||||
"4096",
|
||||
]
|
||||
extra_decode_args = COMMON_ARGS
|
||||
extra_prefill_env = {"SGLANG_RAGGED_VERIFY_MODE": "static"}
|
||||
extra_decode_env = {"SGLANG_RAGGED_VERIFY_MODE": "static"}
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
super().setUpClass()
|
||||
cls.model = try_cached_model(MODEL)
|
||||
# V4.1's target/draft KV transfer requires Mooncake on both sides.
|
||||
cls.transfer_backend = ["--disaggregation-transfer-backend", "mooncake"]
|
||||
cls.launch_all()
|
||||
|
||||
def _server_info(self, url):
|
||||
response = requests.get(url + "/server_info", timeout=30)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
def test_gsm8k(self):
|
||||
# Inspect workers directly: the router does not expose both configs.
|
||||
for url, role, cp_size in (
|
||||
(self.prefill_url, "prefill", 4),
|
||||
(self.decode_url, "decode", 1),
|
||||
):
|
||||
info = self._server_info(url)
|
||||
self.assertEqual(info["disaggregation_mode"], role)
|
||||
self.assertEqual(info["disaggregation_transfer_backend"], "mooncake")
|
||||
self.assertEqual(info["tp_size"], 4)
|
||||
self.assertEqual(info["attn_cp_size"], cp_size)
|
||||
self.assertEqual(info["enable_prefill_cp"], role == "prefill")
|
||||
self.assertEqual(info["speculative_algorithm"], "DSPARK")
|
||||
self.assertEqual(info["speculative_num_draft_tokens"], 6)
|
||||
self.assertTrue(info["language_model_only"])
|
||||
|
||||
# Completion evaluation avoids V4.1 chat-template thinking mode.
|
||||
# base_url is the PD router, so every request exercises both workers.
|
||||
super().test_gsm8k()
|
||||
|
||||
# Require evidence that DSpark actually accepted draft tokens. Query D
|
||||
# directly rather than letting the optional mixin check skip a missing
|
||||
# metric from the router or the prefill worker.
|
||||
states = self._server_info(self.decode_url)["internal_states"]
|
||||
self.assertTrue(states, "Decode worker returned no scheduler state")
|
||||
for state in states:
|
||||
self.assertIn("avg_spec_accept_length", state)
|
||||
accept_length = state["avg_spec_accept_length"]
|
||||
print(f"V4.1 PD+CP DSpark accept length: {accept_length:.4f}")
|
||||
self.assertGreater(accept_length, 1.0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user