From fccc61c1c5bdbc63ef15c83286cbdac4c32d9b19 Mon Sep 17 00:00:00 2001 From: Baizhou Zhang Date: Sat, 3 Oct 2026 17:57:36 -1000 Subject: [PATCH] [CP] Support DP x TP x CP for attention with interleave CP strategy (#42038) --- .../autoregressive/DeepSeek/DeepSeek-V3_2.mdx | 2 +- docs/cookbook/autoregressive/GLM/GLM-5.1.mdx | 2 +- docs/cookbook/autoregressive/GLM/GLM-5.2.mdx | 2 +- docs/cookbook/autoregressive/GLM/GLM-5.3.mdx | 8 +- .../arg_groups/model_overrides/deepseek_v2.py | 20 +-- python/sglang/srt/layers/cp/interleave.py | 25 ++-- python/sglang/srt/layers/dp_attention.py | 29 +++- .../srt/layers/layer_boundary/boundary.py | 13 +- .../sglang/srt/layers/layer_boundary/ops.py | 3 - test/registered/cp/test_dsa_prefill_cp.py | 26 +++- .../test_declared_attention_cp.py | 135 +++++++++++++++--- .../layers/test_dp_attention_cp_replicas.py | 93 ++++++++++++ test/registered/unit/test_model_overrides.py | 27 +++- 13 files changed, 318 insertions(+), 67 deletions(-) diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx index 6860fd1a0f..4ad4a72ca2 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx @@ -432,7 +432,7 @@ For production deployments (RBG / LWS-based, DeepEP EP parallelism), see [multi_ Enable prefill context parallelism (CP) on CUDA with `--enable-prefill-cp --cp-strategy interleave` for long-sequence workloads. -Zigzag prefill CP (`--cp-strategy zigzag`) is temporarily unavailable for DeepSeek V3.2, GLM-5, GLM-5.1, GLM-5.2, and GLM-5.3. Use `interleave` for these models and keep `--attn-dp-size 1`; interleave DSA CP does not support `--attn-dp-size` greater than 1. +Zigzag prefill CP (`--cp-strategy zigzag`) is temporarily unavailable for DeepSeek V3.2, GLM-5, GLM-5.1, GLM-5.2, and GLM-5.3. Use `interleave` for these models. Attention DP is supported, with `tp_size / attn_dp_size` CP ranks in each DP group. Dense FFNs retain their configured TP width; `--moe-dense-tp-size 1` selects local dense computation. **Interleave** (`--cp-strategy interleave`): Distributes tokens by `token_idx % cp_size`. It supports fused MoE, FP8 KV cache, and multi-batch prefill. diff --git a/docs/cookbook/autoregressive/GLM/GLM-5.1.mdx b/docs/cookbook/autoregressive/GLM/GLM-5.1.mdx index c069d84c9b..f84faca603 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-5.1.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-5.1.mdx @@ -101,7 +101,7 @@ import { GLM51Deployment } from '/src/snippets/autoregressive/glm-51-deployment. - **AMD GPUs**: BF16 and FP8 checkpoints run on MI300X/MI325X/MI355X at tp=8. On MI355X (gfx950), the MXFP4 checkpoint `amd/GLM-5.1-MXFP4` is also supported at tp=4 with `--kv-cache-dtype fp8_e4m3`. All AMD paths pass `--dsa-prefill-backend tilelang --dsa-decode-backend tilelang`, `--chunked-prefill-size 131072`, and `--watchdog-timeout 1200` (20 minutes for weight loading). FP8 uses approximately half the memory of BF16 (~89 GB/GPU vs ~175 GB/GPU). EAGLE speculative decoding is supported on AMD GPUs: MI300X/MI325X (gfx942) and MI355X (gfx950), but it **requires `--disable-custom-all-reduce`** — the aiter custom all-reduce kernel deadlocks during EAGLE verify at high concurrency, so without this flag the server will hang. - For other configuration tips (MTP, DSA kernel, Context Parallel, HiSparse, NVFP4, Index Cache), see the [DeepSeek-V3.2 cookbook page](../DeepSeek/DeepSeek-V3_2). GLM-5.1 and DeepSeek-V3.2 share the same model structure, so the optimization techniques are common. -- **Prefill CP on CUDA**: Zigzag (`--cp-strategy zigzag`) is temporarily unavailable for GLM-5.1. Use `--enable-prefill-cp --cp-strategy interleave` with `--attn-dp-size 1`. +- **Prefill CP on CUDA**: Zigzag (`--cp-strategy zigzag`) is temporarily unavailable for GLM-5.1. Use `--enable-prefill-cp --cp-strategy interleave`. Attention DP is supported, with `tp_size / attn_dp_size` CP ranks in each DP group. Dense FFNs retain their configured TP width. - Use `--json-model-override-args '{"index_topk_pattern": "FFSFSSSFSSFFFSSSFFFSFSSSSSSFFSFFSFFSSFFFFFFSFFFFFSFFSSSSSSFSFFFSFSSSFSFFSFFSSS"}'` to enable the [IndexCache](https://github.com/THUDM/IndexCache) method for GLM-5.1. This can improve serving efficiency with only a small accuracy loss. If you are running rigorous accuracy evaluations, do not enable this feature. diff --git a/docs/cookbook/autoregressive/GLM/GLM-5.2.mdx b/docs/cookbook/autoregressive/GLM/GLM-5.2.mdx index b6d57f3d2c..103f23cd96 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-5.2.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-5.2.mdx @@ -255,7 +255,7 @@ For the full setup (streaming, tool-use, count_tokens, persisting env in `~/.cla ### 3.5 Context Parallelism -Zigzag prefill CP (`--cp-strategy zigzag`) is temporarily unavailable for GLM-5.2. For prefill CP on CUDA, use `interleave` with `--attn-dp-size 1` as shown below. +Zigzag prefill CP (`--cp-strategy zigzag`) is temporarily unavailable for GLM-5.2. For prefill CP on CUDA, use `interleave` as shown below. Attention DP is supported: each DP group uses `tp_size / attn_dp_size` CP ranks. Dense FFNs retain the configured TP width; use `--moe-dense-tp-size 1` for local dense computation. Prefill context parallelism can help with reduction of TTFT under long context. To enable prefill context parallelism for GLM 5.2, please append the following arguments: diff --git a/docs/cookbook/autoregressive/GLM/GLM-5.3.mdx b/docs/cookbook/autoregressive/GLM/GLM-5.3.mdx index 38dfe435eb..a9fadf7dba 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-5.3.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-5.3.mdx @@ -243,7 +243,7 @@ For the full setup (streaming, tool-use, count_tokens, persisting env in `~/.cla ### 3.5 Context Parallelism -Zigzag prefill CP (`--cp-strategy zigzag`) is temporarily unavailable for GLM-5.3. For prefill CP on CUDA, use `interleave` with `--attn-dp-size 1` as shown below. +Zigzag prefill CP (`--cp-strategy zigzag`) is temporarily unavailable for GLM-5.3. For prefill CP on CUDA, use `interleave` as shown below. Prefill context parallelism can help with reduction of TTFT under long context. To enable prefill context parallelism for GLM 5.3, please append the following arguments: @@ -254,6 +254,12 @@ Prefill context parallelism can help with reduction of TTFT under long context. ``` which splits the sequence equally across `--attn-cp-size` ranks during attention forward. The trade off for prefill CP is that it will introduce extra all-gather operation before indexer-topk and attention kernels, so it will increase latency for decode (in unified deployment) or short prefill. +You can combine interleave CP with attention DP. Each attention-DP group owns +`tp_size / attn_dp_size` CP ranks; for example, `--tp 8 --attn-dp-size 2` +uses CP4 within each DP group. Dense FFNs retain their configured TP width: +the default runs them over the full TP group, while `--moe-dense-tp-size 1` +runs them independently on local token shards. + When deploying with PD Disaggregation, a **prefill worker using the Mooncake transfer backend** can enable the [LayerSplit](https://z.ai/blog/scaling-pain) technique with ```bash --enable-dsa-cache-layer-split \ diff --git a/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py b/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py index 54689473fd..0c756907b8 100644 --- a/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py +++ b/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py @@ -91,30 +91,30 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict: ) overrides["attn_dp_size"] = attn_dp_size overrides["dp_size"] = 1 - overrides["moe_dense_tp_size"] = 1 if cfg.cp_strategy == "zigzag": + overrides["moe_dense_tp_size"] = 1 overrides["moe_a2a_backend"] = "deepep" overrides["ep_size"] = cfg.tp_size logger.warning( "zigzag DSA CP requires moe_dense_tp_size=1, " "moe_a2a_backend=deepep, ep_size=tp_size, batch_size=1." ) - else: - assert attn_dp_size == 1, ( - "interleave DSA CP does not support DP attention." - ) assert cfg.tp_size <= 8, ( "Context parallel only supports single machine (tp_size <= 8). Cross-machine CP has precision issues." ) - # Note(kpham-sgl): Keep attn_tp_size == 1 under DSA CP. - # The DSA / MLA CP gather and reduce-scatter - # (the dsa_cp_* helpers in adapters/context_parallel.py) assume it. - attn_cp_size = cfg.tp_size // attn_dp_size + # Interleave can shard attention heads within each CP rank. + # Keep an explicit CP width; default to attention TP1 as before. + # Zigzag still requires attention TP1. + attn_cp_size = ( + cfg.attn_cp_size + if cfg.cp_strategy == "interleave" and cfg.attn_cp_size > 1 + else cfg.tp_size // attn_dp_size + ) overrides["attn_cp_size"] = attn_cp_size logger.warning( "Enabled DSA context parallel: " f"strategy={cfg.cp_strategy}, attn_dp_size={attn_dp_size}, " - f"moe_dense_tp_size={overrides['moe_dense_tp_size']}, " + f"moe_dense_tp_size={overrides.get('moe_dense_tp_size', cfg.moe_dense_tp_size)}, " f"ep_size={overrides.get('ep_size', cfg.ep_size)}, tp_size={cfg.tp_size}, " f"attn_cp_size={attn_cp_size}, " f"kv_cache_dtype={cfg.kv_cache_dtype}, " diff --git a/python/sglang/srt/layers/cp/interleave.py b/python/sglang/srt/layers/cp/interleave.py index 70e8901ff6..7ef2f7fe52 100644 --- a/python/sglang/srt/layers/cp/interleave.py +++ b/python/sglang/srt/layers/cp/interleave.py @@ -45,23 +45,26 @@ from sglang.srt.layers.cp.base import ( from sglang.srt.layers.cp.padding import pad_local_rows from sglang.srt.layers.dp_attention import ( attn_cp_all_gather_into_tensor, - get_local_dp_buffer, is_allocation_symmetric, ) from sglang.srt.runtime_context import get_parallel def attn_cp_interleave_gather(hidden_states: torch.Tensor): - """Gather equal padded interleave shards in rank order, not token order.""" - attn_dp_size = get_parallel().attn_dp_size - attn_tp_size = get_parallel().attn_tp_size - assert attn_dp_size == 1 and attn_tp_size == 1 - hidden_states, local_hidden_states = ( - get_local_dp_buffer(get_parallel().attn_cp_group), - hidden_states, - ) - attn_cp_all_gather_into_tensor(hidden_states, local_hidden_states) - return hidden_states + """Gather equal padded interleave shards in rank order, not token order. + + Size from the actual shard: the DP scratch length can already describe a + shard when dense FFNs run over TP, and is not a CP collective's output size. + """ + parallel = get_parallel() + with use_symmetric_memory( + parallel.attn_cp_group, disabled=not is_allocation_symmetric() + ): + gathered = hidden_states.new_empty( + (hidden_states.shape[0] * parallel.attn_cp_size, *hidden_states.shape[1:]) + ) + attn_cp_all_gather_into_tensor(gathered, hidden_states.contiguous()) + return gathered @dataclass diff --git a/python/sglang/srt/layers/dp_attention.py b/python/sglang/srt/layers/dp_attention.py index 1942e0e9eb..3a5f46a0a9 100644 --- a/python/sglang/srt/layers/dp_attention.py +++ b/python/sglang/srt/layers/dp_attention.py @@ -1069,6 +1069,15 @@ def can_use_dp_reduce_scatter() -> bool: def dp_reduce_scatter_tensor(output: torch.Tensor, input: torch.Tensor): _note_dp_gather_in_prefill_graph() + parallel = get_parallel() + if parallel.attn_cp_size > 1 and input.shape[0] % parallel.tp_size: + # MAX_LEN padding aligns each DP slot to attention TP, not CP x TP. + # Small decode batches cannot be equally split over the full TP group. + reduced = parallel.tp_group.all_reduce(input) + output.copy_( + reduced.narrow(0, parallel.attn_dp_rank * output.shape[0], output.shape[0]) + ) + return if is_dp_gatherv_active(): # Variable-length combine matching all_gatherv dispatch: scatter the # global (sum_len) tensor back to per-rank token counts. Fall through to @@ -1084,9 +1093,23 @@ def dp_reduce_scatter_tensor(output: torch.Tensor, input: torch.Tensor): get_parallel().tp_rank ] get_parallel().tp_group.reduce_scatter_tensor(scattered_local_tokens, input) - get_parallel().attn_tp_group.all_gather_into_tensor( - output, scattered_local_tokens - ) + if parallel.attn_cp_size > 1: + # TP reduce-scatter splits each DP slot over CP x attention TP. + # Restore both axes: decode (and warmup) holds the full DP slot on + # every CP rank, even though prefill uses context-sharded rows. + cp_local_tokens = scattered_local_tokens + if parallel.attn_tp_size > 1: + cp_local_tokens = output.new_empty( + (output.shape[0] // parallel.attn_cp_size, *output.shape[1:]) + ) + parallel.attn_tp_group.all_gather_into_tensor( + cp_local_tokens, scattered_local_tokens + ) + parallel.attn_cp_group.all_gather_into_tensor(output, cp_local_tokens) + else: + parallel.attn_tp_group.all_gather_into_tensor( + output, scattered_local_tokens + ) # --------------------------------------------------------------------------- diff --git a/python/sglang/srt/layers/layer_boundary/boundary.py b/python/sglang/srt/layers/layer_boundary/boundary.py index cb96d38ae4..7dd22f76a7 100644 --- a/python/sglang/srt/layers/layer_boundary/boundary.py +++ b/python/sglang/srt/layers/layer_boundary/boundary.py @@ -639,12 +639,15 @@ def _select_exit_move( if not produced.may_reduce_scatter: # A complete output: this rank's block of it, nothing summed. return ExitMove(cp_moves.take_back) - # The FFN leaves its sum: only a take-back that sums over the same - # ranks completes it. - if cp_moves.reduce_scatter is None or not _same_ranks( - _sum_group(produced.group), cp_moves.reduce_scatter_group() - ): + if cp_moves.reduce_scatter is None: raise NotImplementedError(f"{produced=} {residual=} {to=}") + # Only a take-back over the FFN's sum group can complete its reduction. + # With attention TP > 1, the full TP sum spans more ranks than CP: + # let the exit complete that sum, then take this rank's CP rows. + if not _same_ranks(_sum_group(produced.group), cp_moves.reduce_scatter_group()): + if produced.always_partial: + raise NotImplementedError(f"{produced=} {residual=} {to=}") + return ExitMove(cp_moves.take_back) return ExitMove(cp_moves.reduce_scatter, output_move_completes_sum=True) if returned == {TokenAxis.ATTN_DP, TokenAxis.ATTN_CP}: # This rank's CP shard, from where the DP gather put it. diff --git a/python/sglang/srt/layers/layer_boundary/ops.py b/python/sglang/srt/layers/layer_boundary/ops.py index 8cd077c60a..61d84e8156 100644 --- a/python/sglang/srt/layers/layer_boundary/ops.py +++ b/python/sglang/srt/layers/layer_boundary/ops.py @@ -134,9 +134,6 @@ def attn_tp_reduce_scatter( def attn_cp_interleave_reduce_scatter(hidden_states: torch.Tensor): """Sum rank-major output onto each rank's equal, padded interleave shard.""" - attn_dp_size = get_parallel().attn_dp_size - attn_tp_size = get_parallel().attn_tp_size - assert attn_dp_size == 1 and attn_tp_size == 1 cp_size = get_parallel().attn_cp_size cp_rank = get_parallel().attn_cp_rank input_hidden_states = hidden_states diff --git a/test/registered/cp/test_dsa_prefill_cp.py b/test/registered/cp/test_dsa_prefill_cp.py index 8671a9a97b..baa4242cd4 100644 --- a/test/registered/cp/test_dsa_prefill_cp.py +++ b/test/registered/cp/test_dsa_prefill_cp.py @@ -1,7 +1,8 @@ import unittest from types import SimpleNamespace -from sglang.srt.utils import kill_process_tree +import requests + from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( @@ -10,6 +11,7 @@ from sglang.test.test_utils import ( CustomTestCase, is_in_ci, popen_launch_server, + terminate_and_kill_process_tree, write_github_step_summary, ) @@ -19,6 +21,10 @@ SERVER_LAUNCH_TIMEOUT = max(DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, 1800) class TestDSACPInterleave(CustomTestCase): + attn_cp_size = 2 + attn_dp_size = 2 + attn_tp_size = 2 + @classmethod def setUpClass(cls): cls.model = GLM52_MODEL_PATH @@ -31,7 +37,9 @@ class TestDSACPInterleave(CustomTestCase): "--cp-strategy", "interleave", "--attn-cp-size", - "8", + str(cls.attn_cp_size), + "--attn-dp-size", + str(cls.attn_dp_size), "--speculative-algorithm", "EAGLE", "--speculative-num-steps", @@ -58,11 +66,21 @@ class TestDSACPInterleave(CustomTestCase): @classmethod def tearDownClass(cls): - kill_process_tree(cls.process.pid) + if getattr(cls, "process", None) is not None: + terminate_and_kill_process_tree(cls.process) def test_a_gsm8k( self, ): # Append an "a" to make this test run first (alphabetically) to warm up the server + response = requests.get(f"{self.base_url}/server_info", timeout=30) + response.raise_for_status() + info = response.json() + self.assertEqual(info["attn_cp_size"], self.attn_cp_size) + self.assertEqual(info["attn_dp_size"], self.attn_dp_size) + self.assertEqual( + info["tp_size"] // (info["attn_dp_size"] * info["attn_cp_size"]), + self.attn_tp_size, + ) args = SimpleNamespace( base_url=self.base_url, model=self.model, @@ -80,7 +98,7 @@ class TestDSACPInterleave(CustomTestCase): write_github_step_summary( f'### test_a_gsm8k (dsa-cp-interleave)\n{metrics["score"]=:.3f}\n' ) - self.assertGreater(metrics["score"], 0.935) + self.assertGreater(metrics["score"], 0.935) if __name__ == "__main__": diff --git a/test/registered/unit/layer_boundary/test_declared_attention_cp.py b/test/registered/unit/layer_boundary/test_declared_attention_cp.py index 6f42cc60f8..4b5d55596f 100644 --- a/test/registered/unit/layer_boundary/test_declared_attention_cp.py +++ b/test/registered/unit/layer_boundary/test_declared_attention_cp.py @@ -75,29 +75,37 @@ class TestAttentionCpBoundary(CustomTestCase): ] @contextmanager - def as_rank(self, cp, collectives, moe_group=None): - """Rank ``cp`` of attention CP 2 (attention DP and TP 1, TP 2), with - the attention-CP collectives the case gives.""" + def as_rank(self, cp, collectives): + """The first attention-TP lane of CP rank ``cp``, with fake CP collectives. + + Other attention-TP lanes hold identical rows after attention's sum. + """ + attn_tp_size = getattr(self, "attn_tp_size", 1) + tp_size = CP_SIZE * attn_tp_size + tp_rank = cp * attn_tp_size + moe_tp_size = getattr(self, "moe_tp_size", tp_size) + moe_start = tp_rank // moe_tp_size * moe_tp_size + moe_group = group("moe", range(moe_start, moe_start + moe_tp_size)) parallel = SimpleNamespace( - tp_size=CP_SIZE, - tp_rank=cp, + tp_size=tp_size, + tp_rank=tp_rank, attn_dp_size=1, attn_dp_rank=0, attn_dp_enabled=False, - attn_tp_size=1, + attn_tp_size=attn_tp_size, attn_tp_rank=0, attn_cp_size=CP_SIZE, attn_cp_rank=cp, enable_prefill_cp=True, - moe_dense_tp_size=1, + moe_dense_tp_size=1 if getattr(self, "sparse", True) else None, moe_dp_size=1, moe_ep_size=1, - moe_tp_size=CP_SIZE, + moe_tp_size=moe_tp_size, dwdp_size=1, enable_attn_tp_input_scattered=False, - tp_group=group("tp", range(CP_SIZE)), - attn_tp_group=group("attn_tp", [cp]), - attn_cp_group=group("attn_cp", range(CP_SIZE)), + tp_group=group("tp", range(tp_size)), + attn_tp_group=group("attn_tp", range(tp_rank, tp_rank + attn_tp_size)), + attn_cp_group=group("attn_cp", range(0, tp_size, attn_tp_size)), ) flags = Flags() with ExitStack() as stack: @@ -122,7 +130,7 @@ class TestAttentionCpBoundary(CustomTestCase): ), ( (comm, "post_experts_reduction_group"), - lambda: moe_group or parallel.tp_group, + lambda: moe_group, ), ( (comm, "get_exec"), @@ -143,9 +151,10 @@ class TestAttentionCpBoundary(CustomTestCase): ((comm, "use_symmetric_memory"), lambda *a, **k: nullcontext()), ((comm, "is_allocation_symmetric"), lambda: False), ( - (dsa_cp, "get_local_dp_buffer"), - lambda g: torch.empty(ROWS * CP_SIZE, HIDDEN).double(), + (dsa_cp, "use_symmetric_memory"), + lambda *a, **k: nullcontext(), ), + ((dsa_cp, "is_allocation_symmetric"), lambda: False), ((dsa_cp, "attn_cp_all_gather_into_tensor"), collectives["gather"]), ( (comm, "attn_cp_reduce_scatter_tensor"), @@ -174,7 +183,7 @@ class TestAttentionCpBoundary(CustomTestCase): return make_test_stages( first=False, last=False, - sparse=True, + sparse=getattr(self, "sparse", True), previous_sparse=False, next_layer_sparse=False, attention_norm=layernorm, @@ -300,16 +309,96 @@ class TestAttentionCpBoundary(CustomTestCase): back[cp], self.values[cp] + self.residuals[cp], rtol=0, atol=0 ) - def test_a_sum_over_other_ranks_is_not_left_to_it(self): - # A MoE group narrower than attention CP, as when MoE DP splits CP. - def unused(*args): - raise AssertionError("not run") + def test_dense_tp_returns_the_same_context_shard(self): + self.sparse = False + for reduce_scatter in (False, True): + with self.subTest(reduce_scatter=reduce_scatter): + _, back, _ = self.run_ranks(use_reduce_scatter=reduce_scatter) + for cp in range(CP_SIZE): + torch.testing.assert_close( + back[cp], self.values[cp] + self.residuals[cp], rtol=0, atol=0 + ) - with self.as_rank( - 0, dict(gather=unused, reduce_scatter=unused), moe_group=group("moe", [0]) + def test_gather_uses_actual_rows_with_attention_dp(self): + # CP partners belong to one DP replica. Neither DP buffer padding nor + # expanded residual width determines the collective's output shape. + parallel = SimpleNamespace( + attn_dp_size=2, + attn_tp_size=1, + attn_cp_size=CP_SIZE, + attn_cp_group=group("attn_cp", [2, 3]), + ) + for shape in ((0, HIDDEN), (5, HIDDEN), (7, 4, HIDDEN)): + with self.subTest(shape=shape): + local = torch.arange(torch.tensor(shape).prod()).reshape(shape).double() + expected = torch.cat((local, local + 1)) + + def gather(output, input_): + self.assertEqual(output.shape, expected.shape) + torch.testing.assert_close(input_, local) + output.copy_(expected) + + with ( + patch.object(dsa_cp, "get_parallel", lambda: parallel), + patch.object( + dsa_cp, "use_symmetric_memory", lambda *a, **k: nullcontext() + ), + patch.object(dsa_cp, "is_allocation_symmetric", lambda: False), + patch.object(dsa_cp, "attn_cp_all_gather_into_tensor", gather), + ): + torch.testing.assert_close( + dsa_cp.attn_cp_interleave_gather(local), expected + ) + + def test_exit_sums_over_the_ffn_group_before_cp_take_back(self): + # DP1 x CP2 x attention TP2 needs a TP4 sum, not CP2. A MoE group + # narrower than CP likewise must not leave its sum to CP. + complete_output = torch.cat(self.values) + + def unused(*args): + raise AssertionError("a complete FFN output needs no CP collective") + + for sparse, attn_tp_size, group_size in ( + (False, 2, 4), + (True, 2, 4), + (True, 1, 1), ): - with self.assertRaises(NotImplementedError): - self.build(use_reduce_scatter=True) + with self.subTest( + sparse=sparse, attn_tp=attn_tp_size, group_size=group_size + ): + self.sparse = sparse + self.attn_tp_size = attn_tp_size + self.moe_tp_size = group_size + expected_group = ( + comm.SumGroup.MOE_OUTPUT if sparse else comm.SumGroup.TP + ) + + def sum_output(value, group, *args, **kwargs): + self.assertEqual(group, expected_group) + torch.testing.assert_close( + value, complete_output / group_size, rtol=0, atol=0 + ) + return value * group_size + + for cp in range(CP_SIZE): + with ( + self.as_rank(cp, dict(gather=unused, reduce_scatter=unused)), + patch_communicator("sum_output", sum_output), + ): + stages = self.build(use_reduce_scatter=True) + stream = ResidualStream(self.residuals[cp]) + with stages.ffn.plan.output.ffn_exit( + self.cp_extend(), stream=stream + ) as exit_: + # Compute returns a partial sum; the exit owns it. + output = complete_output / group_size + back, stream = finish_exit(exit_, output, stream) + torch.testing.assert_close( + back, self.values[cp], rtol=0, atol=0 + ) + torch.testing.assert_close( + stream.residual, self.residuals[cp], rtol=0, atol=0 + ) if __name__ == "__main__": diff --git a/test/registered/unit/layers/test_dp_attention_cp_replicas.py b/test/registered/unit/layers/test_dp_attention_cp_replicas.py index a4eb7800d3..9ca9bb7c02 100644 --- a/test/registered/unit/layers/test_dp_attention_cp_replicas.py +++ b/test/registered/unit/layers/test_dp_attention_cp_replicas.py @@ -32,6 +32,99 @@ PAD_VALUE = 99.0 class TestDpGatherCpReplicas(CustomTestCase): + def test_reduce_scatter_restores_every_cp_and_tp_replica(self): + # TP reduce-scatter leaves DP x CP x attention-TP shards. Decode needs + # the complete DP slot on every CP and attention-TP rank, not just the + # attention-TP gather (which is an identity for DSA CP). + for cp_size, attn_tp_size, rows in ( + (1, 1, 3), + (1, 2, 6), + (2, 1, 6), + (2, 2, 12), + (2, 1, 0), + (2, 2, 0), + (2, 1, 1), + (2, 1, 3), + (4, 1, 1), + (4, 1, 3), + (2, 2, 2), + (2, 2, 6), + (4, 2, 2), + (4, 2, 6), + ): + tp_size = DP_SIZE * cp_size * attn_tp_size + summed = torch.arange(DP_SIZE * rows * HIDDEN).reshape(-1, HIDDEN).float() + for rank in range(tp_size): + dp = rank // (cp_size * attn_tp_size) + cp = rank // attn_tp_size % cp_size + with self.subTest( + cp_size=cp_size, attn_tp_size=attn_tp_size, rank=rank, rows=rows + ): + calls = [] + local_contribution = summed / tp_size + dp_rows = summed[dp * rows : (dp + 1) * rows] + cp_rows = dp_rows.tensor_split(cp_size)[cp] + + def all_reduce(value): + calls.append("all_reduce") + torch.testing.assert_close(value, local_contribution) + return summed.clone() + + def reduce_scatter(output, value): + calls.append("reduce_scatter") + torch.testing.assert_close(value, local_contribution) + output.copy_(summed.tensor_split(tp_size)[rank]) + + def tp_gather(output, value): + calls.append("attn_tp_gather") + self.assertEqual(output.shape[0], value.shape[0] * attn_tp_size) + torch.testing.assert_close( + value, summed.tensor_split(tp_size)[rank] + ) + output.copy_(cp_rows) + + def cp_gather(output, value): + calls.append("attn_cp_gather") + self.assertEqual(output.shape[0], value.shape[0] * cp_size) + torch.testing.assert_close(value, cp_rows) + output.copy_(dp_rows) + + parallel = SimpleNamespace( + tp_size=tp_size, + tp_rank=rank, + attn_dp_size=DP_SIZE, + attn_dp_rank=dp, + attn_cp_size=cp_size, + attn_cp_rank=cp, + attn_tp_size=attn_tp_size, + tp_group=SimpleNamespace( + reduce_scatter_tensor=reduce_scatter, all_reduce=all_reduce + ), + attn_tp_group=SimpleNamespace(all_gather_into_tensor=tp_gather), + attn_cp_group=SimpleNamespace(all_gather_into_tensor=cp_gather), + ) + output = torch.empty_like(dp_rows) + with ( + patch.object( + dp_attention, "get_parallel", return_value=parallel + ), + patch.object(dp_attention, "_note_dp_gather_in_prefill_graph"), + patch.object( + dp_attention, "is_dp_gatherv_active", return_value=False + ), + ): + dp_attention.dp_reduce_scatter_tensor( + output, local_contribution.clone() + ) + torch.testing.assert_close(output, dp_rows) + if rows % (cp_size * attn_tp_size): + self.assertEqual(calls, ["all_reduce"]) + else: + self.assertEqual(calls[0], "reduce_scatter") + self.assertEqual( + calls.count("attn_cp_gather"), int(cp_size > 1) + ) + def run_ranks(self, token_rows, padding, is_partial): """Gather on every rank and check each DP slot holds its group's rows once, then scatter the result back and check every rank's rows.""" diff --git a/test/registered/unit/test_model_overrides.py b/test/registered/unit/test_model_overrides.py index 64ab395b63..33113359a7 100644 --- a/test/registered/unit/test_model_overrides.py +++ b/test/registered/unit/test_model_overrides.py @@ -3511,6 +3511,8 @@ class TestGoldenModelOverrides(_IsolatedPublish): decode_attention_backend=None, enable_prefill_cp=False, dcp_size=1, + attn_cp_size=1, + moe_dense_tp_size=None, ) defaults.update(kw) return SimpleNamespace(**defaults) @@ -3585,18 +3587,35 @@ class TestGoldenModelOverrides(_IsolatedPublish): "attn_cp_size": 8, }, ) - # interleave CP with attention DP must assert - with self.assertRaises(AssertionError): - _deepseek_family_overrides( + # Interleave keeps attention DP and the configured dense TP. + for attn_dp_size, dense_tp_size, cp_size, expected_cp in ( + (1, None, 1, 8), + (2, 8, 1, 4), + (2, 1, 1, 4), + # An explicit CP2 leaves two attention-TP ranks + # within each DP group; it must not become CP4. + (2, 8, 2, 2), + ): + result = _deepseek_family_overrides( _args( enable_prefill_cp=True, cp_strategy="interleave", tp_size=8, dp_size=1, - attn_dp_size=2, + attn_dp_size=attn_dp_size, + attn_cp_size=cp_size, + moe_dense_tp_size=dense_tp_size, + ep_size=1, + moe_a2a_backend="none", + kv_cache_dtype="auto", ), None, ) + self.assertEqual(result["attn_dp_size"], attn_dp_size) + self.assertEqual(result["attn_cp_size"], expected_cp) + self.assertNotIn("moe_dense_tp_size", result) + self.assertNotIn("ep_size", result) + self.assertNotIn("moe_a2a_backend", result) # MLA path on sm100: trtllm_mla fill (all three backends unset) with patch(