[CP 1/5] Remove the CP adapter and separate interleave transport from boundary reduction (#42036)

This commit is contained in:
Baizhou Zhang
2026-10-02 16:50:20 -07:00
committed by GitHub
parent cbe070b6bf
commit 3bde8eb2d8
6 changed files with 50 additions and 81 deletions
+14
View File
@@ -45,11 +45,25 @@ from sglang.srt.layers.cp.base import (
from sglang.srt.layers.cp.padding import pad_local_rows
from sglang.srt.layers.dp_attention import (
attn_cp_all_gather_into_tensor,
get_local_dp_buffer,
is_allocation_symmetric,
)
from sglang.srt.runtime_context import get_parallel
def attn_cp_interleave_gather(hidden_states: torch.Tensor):
"""Gather equal padded interleave shards in rank order, not token order."""
attn_dp_size = get_parallel().attn_dp_size
attn_tp_size = get_parallel().attn_tp_size
assert attn_dp_size == 1 and attn_tp_size == 1
hidden_states, local_hidden_states = (
get_local_dp_buffer(get_parallel().attn_cp_group),
hidden_states,
)
attn_cp_all_gather_into_tensor(hidden_states, local_hidden_states)
return hidden_states
@dataclass
class InterleaveContextParallelMetadata(BaseContextParallelMetadata):
per_rank_actual_token: Optional[List[int]] = None
@@ -1,47 +0,0 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
import torch
from sglang.srt.layers.dp_attention import (
attn_cp_all_gather_into_tensor,
attn_cp_reduce_scatter_tensor,
get_local_dp_buffer,
)
from sglang.srt.runtime_context import get_parallel
def attn_cp_gather(hidden_states: torch.Tensor):
attn_dp_size = get_parallel().attn_dp_size
attn_tp_size = get_parallel().attn_tp_size
assert attn_dp_size == 1 and attn_tp_size == 1
hidden_states, local_hidden_states = (
get_local_dp_buffer(get_parallel().attn_cp_group),
hidden_states,
)
attn_cp_all_gather_into_tensor(hidden_states, local_hidden_states)
return hidden_states
def attn_cp_reduce_scatter(hidden_states: torch.Tensor):
attn_dp_size = get_parallel().attn_dp_size
attn_tp_size = get_parallel().attn_tp_size
assert attn_dp_size == 1 and attn_tp_size == 1
cp_size = get_parallel().attn_cp_size
cp_rank = get_parallel().attn_cp_rank
input_hidden_states = hidden_states
hidden_states = hidden_states.tensor_split(cp_size)[cp_rank]
attn_cp_reduce_scatter_tensor(hidden_states, input_hidden_states)
return hidden_states
+16 -6
View File
@@ -27,6 +27,7 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
use_symmetric_memory,
)
from sglang.srt.layers.dp_attention import (
attn_cp_reduce_scatter_tensor,
attn_tp_all_gather_into_tensor,
attn_tp_reduce_scatter_tensor,
dp_gather_partial,
@@ -44,9 +45,6 @@ from sglang.srt.layers.layer_boundary.adapters.attention import (
attn_tp_gather,
attn_tp_slice,
)
from sglang.srt.layers.layer_boundary.adapters.context_parallel import (
attn_cp_reduce_scatter,
)
from sglang.srt.layers.layer_boundary.layout import (
Layout,
TokenAxis,
@@ -130,6 +128,19 @@ def attn_tp_reduce_scatter(
return local_hidden_states
def attn_cp_interleave_reduce_scatter(hidden_states: torch.Tensor):
"""Sum rank-major output onto each rank's equal, padded interleave shard."""
attn_dp_size = get_parallel().attn_dp_size
attn_tp_size = get_parallel().attn_tp_size
assert attn_dp_size == 1 and attn_tp_size == 1
cp_size = get_parallel().attn_cp_size
cp_rank = get_parallel().attn_cp_rank
input_hidden_states = hidden_states
hidden_states = hidden_states.tensor_split(cp_size)[cp_rank]
attn_cp_reduce_scatter_tensor(hidden_states, input_hidden_states)
return hidden_states
def moe_cp_gather(
hidden_states: torch.Tensor, rows: List[int], moe_cp_size: int
) -> torch.Tensor:
@@ -353,7 +364,7 @@ def attn_cp_reduce_scatter_output(
):
"""DSA and MLA CP: sum the FFN output over the attention-CP group and
keep this rank's shard."""
return attn_cp_reduce_scatter(hidden_states), residual
return attn_cp_interleave_reduce_scatter(hidden_states), residual
def dp_cp_take_back_output(
@@ -393,8 +404,7 @@ def moe_cp_take_back_output(
If DP>1, further scatter back to the local DP slice.
"""
# Only scatter back during prefill; decode was never allgathered so no-op.
# Safe w.r.t. empty tensors: same reasoning as _then_moe_cp_gather
# — CP extend always has non-zero tokens per rank, and decode skips this path.
# CP extend has non-zero tokens per rank, and decode skips this path.
rows = moe_cp_gathered_rows(forward_batch)
if rows is not None:
hidden_states = moe_cp_take_back(hidden_states, rows)
@@ -24,6 +24,9 @@ from sglang.srt.distributed import (
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
use_symmetric_memory,
)
from sglang.srt.layers.cp.interleave import (
attn_cp_interleave_gather,
)
from sglang.srt.layers.dp_attention import (
attn_tp_all_gather_into_tensor,
dp_scatter,
@@ -37,13 +40,9 @@ from sglang.srt.layers.layer_boundary.adapters.attention import (
get_attn_tp_context,
tp_gather,
)
from sglang.srt.layers.layer_boundary.adapters.context_parallel import (
attn_cp_gather,
)
from sglang.srt.layers.layer_boundary.layout import (
SumGroup,
_cp_shard_token_rows,
moe_cp_gathered_rows,
)
from sglang.srt.layers.layer_boundary.output import (
DeferredFinalize,
@@ -415,7 +414,7 @@ def _then_attn_cp_gather(
hidden_states, residual = gather(
hidden_states, residual, forward_batch, norm, update=update, cache=cache
)
return attn_cp_gather(hidden_states), residual
return attn_cp_interleave_gather(hidden_states), residual
def _then_moe_cp_gather(
@@ -428,25 +427,20 @@ def _then_moe_cp_gather(
gather: Callable,
update: ResidualUpdate = PLAIN_ADD,
):
"""Gather for the FFN, then over the MoE-CP group so each rank holds all
tokens of its MoE group (moe_dp_size < attn_cp_size). The residual stays on
this rank's attention rows."""
# Early return on empty tensor is safe for MOE_CP because:
# - During CP extend: zigzag split guarantees all CP ranks have non-zero tokens,
# so no rank hits this path while others proceed to the allgather.
# - During decode: moe_cp allgather is skipped (guarded by is_context_parallel_extend).
# - CUDA graph warmup: not applicable when --cuda-graph-backend-prefill=disabled is used.
if hidden_states.shape[0] == 0:
return hidden_states, residual
"""Read each shard, then gather the MoE group's tokens on its CP path.
StagePlan selects this path only for a CP extend with MoE-CP rows; zigzag
eligibility guarantees nonempty shards. Decode and non-CP batches use the
ordinary path. The residual stays on this rank's attention rows.
"""
hidden_states, residual = gather(
hidden_states, residual, forward_batch, norm, update=update, cache=cache
)
rows = moe_cp_gathered_rows(forward_batch)
if rows is not None and hidden_states.shape[0] > 0:
hidden_states = moe_cp_gather(hidden_states, rows, get_moe_cp_size())
hidden_states = moe_cp_gather(
hidden_states,
forward_batch.attn_cp_metadata.per_rank_actual_token,
get_moe_cp_size(),
)
return hidden_states, residual
+4 -6
View File
@@ -71,6 +71,7 @@ from sglang.srt.layers.attention.dsv4.dsv41_sparse import (
)
from sglang.srt.layers.attention.dsv4.indexer import C4Indexer
from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx
from sglang.srt.layers.cp.interleave import attn_cp_interleave_gather
from sglang.srt.layers.cp.utils import (
cp_gather_full_sequence_states,
cp_materialize_global_token_order,
@@ -99,10 +100,7 @@ from sglang.srt.layers.dp_attention import (
)
from sglang.srt.layers.engram import Engram, EngramHasher, EngramLayout
from sglang.srt.layers.layer_boundary import get_attn_tp_context
from sglang.srt.layers.layer_boundary.adapters.context_parallel import (
attn_cp_gather,
attn_cp_reduce_scatter,
)
from sglang.srt.layers.layer_boundary.ops import attn_cp_interleave_reduce_scatter
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear
from sglang.srt.layers.logits_processor import LogitsMetadata, LogitsProcessor
@@ -3916,7 +3914,7 @@ class DeepseekV4DecoderLayer(nn.Module):
if _use_cp:
moe_a2a_backend = get_moe_a2a_backend()
if moe_a2a_backend.is_none():
hidden_states = attn_cp_gather(hidden_states)
hidden_states = attn_cp_interleave_gather(hidden_states)
else:
assert (
moe_a2a_backend.is_deepep()
@@ -3967,7 +3965,7 @@ class DeepseekV4DecoderLayer(nn.Module):
skip_shared_experts=_do_shared_local,
)
if _use_cp and get_moe_a2a_backend().is_none():
hidden_states = attn_cp_reduce_scatter(hidden_states)
hidden_states = attn_cp_interleave_reduce_scatter(hidden_states)
elif _use_tp_moe_gather:
hidden_states, global_hidden_states = (
get_local_dp_buffer(get_parallel().tp_group),