From 3bde8eb2d871f3840d317bc670873193b4f63725 Mon Sep 17 00:00:00 2001 From: Baizhou Zhang Date: Fri, 2 Oct 2026 13:50:20 -1000 Subject: [PATCH] [CP 1/5] Remove the CP adapter and separate interleave transport from boundary reduction (#42036) --- python/sglang/srt/layers/cp/interleave.py | 14 ++++++ .../adapters/context_parallel.py | 47 ------------------- .../sglang/srt/layers/layer_boundary/ops.py | 22 ++++++--- .../srt/layers/layer_boundary/prepare.py | 34 ++++++-------- python/sglang/srt/models/deepseek_v4.py | 10 ++-- .../test_declared_attention_cp.py | 4 +- 6 files changed, 50 insertions(+), 81 deletions(-) delete mode 100644 python/sglang/srt/layers/layer_boundary/adapters/context_parallel.py diff --git a/python/sglang/srt/layers/cp/interleave.py b/python/sglang/srt/layers/cp/interleave.py index b5542de611..70e8901ff6 100644 --- a/python/sglang/srt/layers/cp/interleave.py +++ b/python/sglang/srt/layers/cp/interleave.py @@ -45,11 +45,25 @@ from sglang.srt.layers.cp.base import ( from sglang.srt.layers.cp.padding import pad_local_rows from sglang.srt.layers.dp_attention import ( attn_cp_all_gather_into_tensor, + get_local_dp_buffer, is_allocation_symmetric, ) from sglang.srt.runtime_context import get_parallel +def attn_cp_interleave_gather(hidden_states: torch.Tensor): + """Gather equal padded interleave shards in rank order, not token order.""" + attn_dp_size = get_parallel().attn_dp_size + attn_tp_size = get_parallel().attn_tp_size + assert attn_dp_size == 1 and attn_tp_size == 1 + hidden_states, local_hidden_states = ( + get_local_dp_buffer(get_parallel().attn_cp_group), + hidden_states, + ) + attn_cp_all_gather_into_tensor(hidden_states, local_hidden_states) + return hidden_states + + @dataclass class InterleaveContextParallelMetadata(BaseContextParallelMetadata): per_rank_actual_token: Optional[List[int]] = None diff --git a/python/sglang/srt/layers/layer_boundary/adapters/context_parallel.py b/python/sglang/srt/layers/layer_boundary/adapters/context_parallel.py deleted file mode 100644 index 2aaba78c8d..0000000000 --- a/python/sglang/srt/layers/layer_boundary/adapters/context_parallel.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2023-2024 SGLang Team -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - - -import torch - -from sglang.srt.layers.dp_attention import ( - attn_cp_all_gather_into_tensor, - attn_cp_reduce_scatter_tensor, - get_local_dp_buffer, -) -from sglang.srt.runtime_context import get_parallel - - -def attn_cp_gather(hidden_states: torch.Tensor): - attn_dp_size = get_parallel().attn_dp_size - attn_tp_size = get_parallel().attn_tp_size - assert attn_dp_size == 1 and attn_tp_size == 1 - hidden_states, local_hidden_states = ( - get_local_dp_buffer(get_parallel().attn_cp_group), - hidden_states, - ) - attn_cp_all_gather_into_tensor(hidden_states, local_hidden_states) - return hidden_states - - -def attn_cp_reduce_scatter(hidden_states: torch.Tensor): - attn_dp_size = get_parallel().attn_dp_size - attn_tp_size = get_parallel().attn_tp_size - assert attn_dp_size == 1 and attn_tp_size == 1 - cp_size = get_parallel().attn_cp_size - cp_rank = get_parallel().attn_cp_rank - input_hidden_states = hidden_states - hidden_states = hidden_states.tensor_split(cp_size)[cp_rank] - attn_cp_reduce_scatter_tensor(hidden_states, input_hidden_states) - return hidden_states diff --git a/python/sglang/srt/layers/layer_boundary/ops.py b/python/sglang/srt/layers/layer_boundary/ops.py index fb1a2ea7a7..069ebc2808 100644 --- a/python/sglang/srt/layers/layer_boundary/ops.py +++ b/python/sglang/srt/layers/layer_boundary/ops.py @@ -27,6 +27,7 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import ( use_symmetric_memory, ) from sglang.srt.layers.dp_attention import ( + attn_cp_reduce_scatter_tensor, attn_tp_all_gather_into_tensor, attn_tp_reduce_scatter_tensor, dp_gather_partial, @@ -44,9 +45,6 @@ from sglang.srt.layers.layer_boundary.adapters.attention import ( attn_tp_gather, attn_tp_slice, ) -from sglang.srt.layers.layer_boundary.adapters.context_parallel import ( - attn_cp_reduce_scatter, -) from sglang.srt.layers.layer_boundary.layout import ( Layout, TokenAxis, @@ -130,6 +128,19 @@ def attn_tp_reduce_scatter( return local_hidden_states +def attn_cp_interleave_reduce_scatter(hidden_states: torch.Tensor): + """Sum rank-major output onto each rank's equal, padded interleave shard.""" + attn_dp_size = get_parallel().attn_dp_size + attn_tp_size = get_parallel().attn_tp_size + assert attn_dp_size == 1 and attn_tp_size == 1 + cp_size = get_parallel().attn_cp_size + cp_rank = get_parallel().attn_cp_rank + input_hidden_states = hidden_states + hidden_states = hidden_states.tensor_split(cp_size)[cp_rank] + attn_cp_reduce_scatter_tensor(hidden_states, input_hidden_states) + return hidden_states + + def moe_cp_gather( hidden_states: torch.Tensor, rows: List[int], moe_cp_size: int ) -> torch.Tensor: @@ -353,7 +364,7 @@ def attn_cp_reduce_scatter_output( ): """DSA and MLA CP: sum the FFN output over the attention-CP group and keep this rank's shard.""" - return attn_cp_reduce_scatter(hidden_states), residual + return attn_cp_interleave_reduce_scatter(hidden_states), residual def dp_cp_take_back_output( @@ -393,8 +404,7 @@ def moe_cp_take_back_output( If DP>1, further scatter back to the local DP slice. """ # Only scatter back during prefill; decode was never allgathered so no-op. - # Safe w.r.t. empty tensors: same reasoning as _then_moe_cp_gather - # — CP extend always has non-zero tokens per rank, and decode skips this path. + # CP extend has non-zero tokens per rank, and decode skips this path. rows = moe_cp_gathered_rows(forward_batch) if rows is not None: hidden_states = moe_cp_take_back(hidden_states, rows) diff --git a/python/sglang/srt/layers/layer_boundary/prepare.py b/python/sglang/srt/layers/layer_boundary/prepare.py index e104a022e3..562f3f6b78 100644 --- a/python/sglang/srt/layers/layer_boundary/prepare.py +++ b/python/sglang/srt/layers/layer_boundary/prepare.py @@ -24,6 +24,9 @@ from sglang.srt.distributed import ( from sglang.srt.distributed.device_communicators.pynccl_allocator import ( use_symmetric_memory, ) +from sglang.srt.layers.cp.interleave import ( + attn_cp_interleave_gather, +) from sglang.srt.layers.dp_attention import ( attn_tp_all_gather_into_tensor, dp_scatter, @@ -37,13 +40,9 @@ from sglang.srt.layers.layer_boundary.adapters.attention import ( get_attn_tp_context, tp_gather, ) -from sglang.srt.layers.layer_boundary.adapters.context_parallel import ( - attn_cp_gather, -) from sglang.srt.layers.layer_boundary.layout import ( SumGroup, _cp_shard_token_rows, - moe_cp_gathered_rows, ) from sglang.srt.layers.layer_boundary.output import ( DeferredFinalize, @@ -415,7 +414,7 @@ def _then_attn_cp_gather( hidden_states, residual = gather( hidden_states, residual, forward_batch, norm, update=update, cache=cache ) - return attn_cp_gather(hidden_states), residual + return attn_cp_interleave_gather(hidden_states), residual def _then_moe_cp_gather( @@ -428,25 +427,20 @@ def _then_moe_cp_gather( gather: Callable, update: ResidualUpdate = PLAIN_ADD, ): - """Gather for the FFN, then over the MoE-CP group so each rank holds all - tokens of its MoE group (moe_dp_size < attn_cp_size). The residual stays on - this rank's attention rows.""" - # Early return on empty tensor is safe for MOE_CP because: - # - During CP extend: zigzag split guarantees all CP ranks have non-zero tokens, - # so no rank hits this path while others proceed to the allgather. - # - During decode: moe_cp allgather is skipped (guarded by is_context_parallel_extend). - # - CUDA graph warmup: not applicable when --cuda-graph-backend-prefill=disabled is used. - if hidden_states.shape[0] == 0: - return hidden_states, residual + """Read each shard, then gather the MoE group's tokens on its CP path. + StagePlan selects this path only for a CP extend with MoE-CP rows; zigzag + eligibility guarantees nonempty shards. Decode and non-CP batches use the + ordinary path. The residual stays on this rank's attention rows. + """ hidden_states, residual = gather( hidden_states, residual, forward_batch, norm, update=update, cache=cache ) - - rows = moe_cp_gathered_rows(forward_batch) - if rows is not None and hidden_states.shape[0] > 0: - hidden_states = moe_cp_gather(hidden_states, rows, get_moe_cp_size()) - + hidden_states = moe_cp_gather( + hidden_states, + forward_batch.attn_cp_metadata.per_rank_actual_token, + get_moe_cp_size(), + ) return hidden_states, residual diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index d3fa6ff49b..e280a6d0dd 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -71,6 +71,7 @@ from sglang.srt.layers.attention.dsv4.dsv41_sparse import ( ) from sglang.srt.layers.attention.dsv4.indexer import C4Indexer from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx +from sglang.srt.layers.cp.interleave import attn_cp_interleave_gather from sglang.srt.layers.cp.utils import ( cp_gather_full_sequence_states, cp_materialize_global_token_order, @@ -99,10 +100,7 @@ from sglang.srt.layers.dp_attention import ( ) from sglang.srt.layers.engram import Engram, EngramHasher, EngramLayout from sglang.srt.layers.layer_boundary import get_attn_tp_context -from sglang.srt.layers.layer_boundary.adapters.context_parallel import ( - attn_cp_gather, - attn_cp_reduce_scatter, -) +from sglang.srt.layers.layer_boundary.ops import attn_cp_interleave_reduce_scatter from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear from sglang.srt.layers.logits_processor import LogitsMetadata, LogitsProcessor @@ -3916,7 +3914,7 @@ class DeepseekV4DecoderLayer(nn.Module): if _use_cp: moe_a2a_backend = get_moe_a2a_backend() if moe_a2a_backend.is_none(): - hidden_states = attn_cp_gather(hidden_states) + hidden_states = attn_cp_interleave_gather(hidden_states) else: assert ( moe_a2a_backend.is_deepep() @@ -3967,7 +3965,7 @@ class DeepseekV4DecoderLayer(nn.Module): skip_shared_experts=_do_shared_local, ) if _use_cp and get_moe_a2a_backend().is_none(): - hidden_states = attn_cp_reduce_scatter(hidden_states) + hidden_states = attn_cp_interleave_reduce_scatter(hidden_states) elif _use_tp_moe_gather: hidden_states, global_hidden_states = ( get_local_dp_buffer(get_parallel().tp_group), diff --git a/test/registered/unit/layer_boundary/test_declared_attention_cp.py b/test/registered/unit/layer_boundary/test_declared_attention_cp.py index 72a3cfbd7f..83afd4ce81 100644 --- a/test/registered/unit/layer_boundary/test_declared_attention_cp.py +++ b/test/registered/unit/layer_boundary/test_declared_attention_cp.py @@ -17,7 +17,7 @@ import torch from sglang.srt.layers import layer_boundary as comm from sglang.srt.layers import layernorm_sp -from sglang.srt.layers.layer_boundary.adapters import context_parallel as dsa_cp +from sglang.srt.layers.cp import interleave as dsa_cp from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.test.boundary_fixtures import finish_exit, make_test_stages, prepare_input from sglang.test.ci.ci_register import register_cpu_ci @@ -149,7 +149,7 @@ class TestAttentionCpBoundary(CustomTestCase): ), ((dsa_cp, "attn_cp_all_gather_into_tensor"), collectives["gather"]), ( - (dsa_cp, "attn_cp_reduce_scatter_tensor"), + (comm, "attn_cp_reduce_scatter_tensor"), collectives["reduce_scatter"], ), ]: