[CP 1/5] Remove the CP adapter and separate interleave transport from boundary reduction (#42036)
This commit is contained in:
@@ -45,11 +45,25 @@ from sglang.srt.layers.cp.base import (
|
||||
from sglang.srt.layers.cp.padding import pad_local_rows
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_cp_all_gather_into_tensor,
|
||||
get_local_dp_buffer,
|
||||
is_allocation_symmetric,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
|
||||
def attn_cp_interleave_gather(hidden_states: torch.Tensor):
|
||||
"""Gather equal padded interleave shards in rank order, not token order."""
|
||||
attn_dp_size = get_parallel().attn_dp_size
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
assert attn_dp_size == 1 and attn_tp_size == 1
|
||||
hidden_states, local_hidden_states = (
|
||||
get_local_dp_buffer(get_parallel().attn_cp_group),
|
||||
hidden_states,
|
||||
)
|
||||
attn_cp_all_gather_into_tensor(hidden_states, local_hidden_states)
|
||||
return hidden_states
|
||||
|
||||
|
||||
@dataclass
|
||||
class InterleaveContextParallelMetadata(BaseContextParallelMetadata):
|
||||
per_rank_actual_token: Optional[List[int]] = None
|
||||
|
||||
@@ -1,47 +0,0 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_cp_all_gather_into_tensor,
|
||||
attn_cp_reduce_scatter_tensor,
|
||||
get_local_dp_buffer,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
|
||||
def attn_cp_gather(hidden_states: torch.Tensor):
|
||||
attn_dp_size = get_parallel().attn_dp_size
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
assert attn_dp_size == 1 and attn_tp_size == 1
|
||||
hidden_states, local_hidden_states = (
|
||||
get_local_dp_buffer(get_parallel().attn_cp_group),
|
||||
hidden_states,
|
||||
)
|
||||
attn_cp_all_gather_into_tensor(hidden_states, local_hidden_states)
|
||||
return hidden_states
|
||||
|
||||
|
||||
def attn_cp_reduce_scatter(hidden_states: torch.Tensor):
|
||||
attn_dp_size = get_parallel().attn_dp_size
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
assert attn_dp_size == 1 and attn_tp_size == 1
|
||||
cp_size = get_parallel().attn_cp_size
|
||||
cp_rank = get_parallel().attn_cp_rank
|
||||
input_hidden_states = hidden_states
|
||||
hidden_states = hidden_states.tensor_split(cp_size)[cp_rank]
|
||||
attn_cp_reduce_scatter_tensor(hidden_states, input_hidden_states)
|
||||
return hidden_states
|
||||
@@ -27,6 +27,7 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
use_symmetric_memory,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_cp_reduce_scatter_tensor,
|
||||
attn_tp_all_gather_into_tensor,
|
||||
attn_tp_reduce_scatter_tensor,
|
||||
dp_gather_partial,
|
||||
@@ -44,9 +45,6 @@ from sglang.srt.layers.layer_boundary.adapters.attention import (
|
||||
attn_tp_gather,
|
||||
attn_tp_slice,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.adapters.context_parallel import (
|
||||
attn_cp_reduce_scatter,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.layout import (
|
||||
Layout,
|
||||
TokenAxis,
|
||||
@@ -130,6 +128,19 @@ def attn_tp_reduce_scatter(
|
||||
return local_hidden_states
|
||||
|
||||
|
||||
def attn_cp_interleave_reduce_scatter(hidden_states: torch.Tensor):
|
||||
"""Sum rank-major output onto each rank's equal, padded interleave shard."""
|
||||
attn_dp_size = get_parallel().attn_dp_size
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
assert attn_dp_size == 1 and attn_tp_size == 1
|
||||
cp_size = get_parallel().attn_cp_size
|
||||
cp_rank = get_parallel().attn_cp_rank
|
||||
input_hidden_states = hidden_states
|
||||
hidden_states = hidden_states.tensor_split(cp_size)[cp_rank]
|
||||
attn_cp_reduce_scatter_tensor(hidden_states, input_hidden_states)
|
||||
return hidden_states
|
||||
|
||||
|
||||
def moe_cp_gather(
|
||||
hidden_states: torch.Tensor, rows: List[int], moe_cp_size: int
|
||||
) -> torch.Tensor:
|
||||
@@ -353,7 +364,7 @@ def attn_cp_reduce_scatter_output(
|
||||
):
|
||||
"""DSA and MLA CP: sum the FFN output over the attention-CP group and
|
||||
keep this rank's shard."""
|
||||
return attn_cp_reduce_scatter(hidden_states), residual
|
||||
return attn_cp_interleave_reduce_scatter(hidden_states), residual
|
||||
|
||||
|
||||
def dp_cp_take_back_output(
|
||||
@@ -393,8 +404,7 @@ def moe_cp_take_back_output(
|
||||
If DP>1, further scatter back to the local DP slice.
|
||||
"""
|
||||
# Only scatter back during prefill; decode was never allgathered so no-op.
|
||||
# Safe w.r.t. empty tensors: same reasoning as _then_moe_cp_gather
|
||||
# — CP extend always has non-zero tokens per rank, and decode skips this path.
|
||||
# CP extend has non-zero tokens per rank, and decode skips this path.
|
||||
rows = moe_cp_gathered_rows(forward_batch)
|
||||
if rows is not None:
|
||||
hidden_states = moe_cp_take_back(hidden_states, rows)
|
||||
|
||||
@@ -24,6 +24,9 @@ from sglang.srt.distributed import (
|
||||
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
use_symmetric_memory,
|
||||
)
|
||||
from sglang.srt.layers.cp.interleave import (
|
||||
attn_cp_interleave_gather,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_gather_into_tensor,
|
||||
dp_scatter,
|
||||
@@ -37,13 +40,9 @@ from sglang.srt.layers.layer_boundary.adapters.attention import (
|
||||
get_attn_tp_context,
|
||||
tp_gather,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.adapters.context_parallel import (
|
||||
attn_cp_gather,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.layout import (
|
||||
SumGroup,
|
||||
_cp_shard_token_rows,
|
||||
moe_cp_gathered_rows,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.output import (
|
||||
DeferredFinalize,
|
||||
@@ -415,7 +414,7 @@ def _then_attn_cp_gather(
|
||||
hidden_states, residual = gather(
|
||||
hidden_states, residual, forward_batch, norm, update=update, cache=cache
|
||||
)
|
||||
return attn_cp_gather(hidden_states), residual
|
||||
return attn_cp_interleave_gather(hidden_states), residual
|
||||
|
||||
|
||||
def _then_moe_cp_gather(
|
||||
@@ -428,25 +427,20 @@ def _then_moe_cp_gather(
|
||||
gather: Callable,
|
||||
update: ResidualUpdate = PLAIN_ADD,
|
||||
):
|
||||
"""Gather for the FFN, then over the MoE-CP group so each rank holds all
|
||||
tokens of its MoE group (moe_dp_size < attn_cp_size). The residual stays on
|
||||
this rank's attention rows."""
|
||||
# Early return on empty tensor is safe for MOE_CP because:
|
||||
# - During CP extend: zigzag split guarantees all CP ranks have non-zero tokens,
|
||||
# so no rank hits this path while others proceed to the allgather.
|
||||
# - During decode: moe_cp allgather is skipped (guarded by is_context_parallel_extend).
|
||||
# - CUDA graph warmup: not applicable when --cuda-graph-backend-prefill=disabled is used.
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, residual
|
||||
"""Read each shard, then gather the MoE group's tokens on its CP path.
|
||||
|
||||
StagePlan selects this path only for a CP extend with MoE-CP rows; zigzag
|
||||
eligibility guarantees nonempty shards. Decode and non-CP batches use the
|
||||
ordinary path. The residual stays on this rank's attention rows.
|
||||
"""
|
||||
hidden_states, residual = gather(
|
||||
hidden_states, residual, forward_batch, norm, update=update, cache=cache
|
||||
)
|
||||
|
||||
rows = moe_cp_gathered_rows(forward_batch)
|
||||
if rows is not None and hidden_states.shape[0] > 0:
|
||||
hidden_states = moe_cp_gather(hidden_states, rows, get_moe_cp_size())
|
||||
|
||||
hidden_states = moe_cp_gather(
|
||||
hidden_states,
|
||||
forward_batch.attn_cp_metadata.per_rank_actual_token,
|
||||
get_moe_cp_size(),
|
||||
)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
|
||||
@@ -71,6 +71,7 @@ from sglang.srt.layers.attention.dsv4.dsv41_sparse import (
|
||||
)
|
||||
from sglang.srt.layers.attention.dsv4.indexer import C4Indexer
|
||||
from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx
|
||||
from sglang.srt.layers.cp.interleave import attn_cp_interleave_gather
|
||||
from sglang.srt.layers.cp.utils import (
|
||||
cp_gather_full_sequence_states,
|
||||
cp_materialize_global_token_order,
|
||||
@@ -99,10 +100,7 @@ from sglang.srt.layers.dp_attention import (
|
||||
)
|
||||
from sglang.srt.layers.engram import Engram, EngramHasher, EngramLayout
|
||||
from sglang.srt.layers.layer_boundary import get_attn_tp_context
|
||||
from sglang.srt.layers.layer_boundary.adapters.context_parallel import (
|
||||
attn_cp_gather,
|
||||
attn_cp_reduce_scatter,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.ops import attn_cp_interleave_reduce_scatter
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsMetadata, LogitsProcessor
|
||||
@@ -3916,7 +3914,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
if _use_cp:
|
||||
moe_a2a_backend = get_moe_a2a_backend()
|
||||
if moe_a2a_backend.is_none():
|
||||
hidden_states = attn_cp_gather(hidden_states)
|
||||
hidden_states = attn_cp_interleave_gather(hidden_states)
|
||||
else:
|
||||
assert (
|
||||
moe_a2a_backend.is_deepep()
|
||||
@@ -3967,7 +3965,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
skip_shared_experts=_do_shared_local,
|
||||
)
|
||||
if _use_cp and get_moe_a2a_backend().is_none():
|
||||
hidden_states = attn_cp_reduce_scatter(hidden_states)
|
||||
hidden_states = attn_cp_interleave_reduce_scatter(hidden_states)
|
||||
elif _use_tp_moe_gather:
|
||||
hidden_states, global_hidden_states = (
|
||||
get_local_dp_buffer(get_parallel().tp_group),
|
||||
|
||||
Reference in New Issue
Block a user