[Refactor] Compare token layouts instead of group sizes when selecting communicator paths (#40869)

This commit is contained in:
Cheng Wan
2026-09-23 14:41:55 -07:00
committed by GitHub
parent e31cbc01a2
commit 7dd9640752
3 changed files with 305 additions and 88 deletions
@@ -0,0 +1,41 @@
# Copyright 2026 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Token layouts of the tensors handed across layer communication boundaries."""
from enum import Enum, auto
from typing import FrozenSet, Mapping
import msgspec
class TokenAxis(Enum):
"""A parallel dimension across which ranks hold different tokens."""
ATTN_DP = auto()
ATTN_CP = auto()
# Each attention-TP rank holds a slice of its group's tokens.
ATTN_TP_SCATTER = auto()
class Layout(msgspec.Struct, frozen=True):
"""The token axes a rank's rows are sharded over. Single-rank axes are left
out, so two layouts are equal exactly when every rank holds the same tokens."""
sharded: FrozenSet[TokenAxis]
@classmethod
def sharded_over(
cls, *axes: TokenAxis, axis_sizes: Mapping[TokenAxis, int]
) -> "Layout":
return cls(frozenset(axis for axis in axes if axis_sizes[axis] > 1))
+97 -88
View File
@@ -15,7 +15,7 @@ import logging
from contextlib import contextmanager
from dataclasses import dataclass
from enum import Enum, auto
from functools import partial
from functools import cached_property, partial
from typing import Callable, Dict, Optional, Tuple, Union
import torch
@@ -35,6 +35,7 @@ from sglang.srt.layers.attention.dsa.utils import (
is_dsa_enable_prefill_cp,
)
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator
from sglang.srt.layers.boundary_layout import Layout, TokenAxis
from sglang.srt.layers.cp.utils import (
is_mla_cp_active,
is_mla_cp_enabled,
@@ -1011,6 +1012,33 @@ class LayerCommunicator:
)
# MOE_FULL gathers across the MoE-CP group, which spans every CP rank when CP is on.
_SCATTER_MODE_SHARDED_AXES = {
ScatterMode.SCATTERED: (
TokenAxis.ATTN_DP,
TokenAxis.ATTN_CP,
TokenAxis.ATTN_TP_SCATTER,
),
ScatterMode.TP_ATTN_FULL: (TokenAxis.ATTN_DP, TokenAxis.ATTN_CP),
ScatterMode.FULL: (TokenAxis.ATTN_CP,),
ScatterMode.MOE_FULL: (),
}
def scatter_mode_layouts(
*, attn_dp_size: int, attn_cp_size: int, attn_tp_size: int
) -> Dict[ScatterMode, Layout]:
axis_sizes = {
TokenAxis.ATTN_DP: attn_dp_size,
TokenAxis.ATTN_CP: attn_cp_size,
TokenAxis.ATTN_TP_SCATTER: attn_tp_size,
}
return {
mode: Layout.sharded_over(*axes, axis_sizes=axis_sizes)
for mode, axes in _SCATTER_MODE_SHARDED_AXES.items()
}
@dataclass
class CommunicateContext:
process_group_sizes: Dict[ScatterMode, int]
@@ -1027,6 +1055,17 @@ class CommunicateContext:
def is_same_group_size(self, a: ScatterMode, b: ScatterMode):
return self.process_group_sizes[a] == self.process_group_sizes[b]
@cached_property
def layouts(self) -> Dict[ScatterMode, Layout]:
return scatter_mode_layouts(
attn_dp_size=self.attn_dp_size,
attn_cp_size=self.attn_cp_size,
attn_tp_size=self.attn_tp_size,
)
def is_same_layout(self, a: ScatterMode, b: ScatterMode):
return self.layouts[a] == self.layouts[b]
@classmethod
def init_new(cls):
attn_tp_rank = get_parallel().attn_tp_rank
@@ -1065,7 +1104,7 @@ class CommunicateSimpleFn:
output_mode: ScatterMode,
context: CommunicateContext,
):
if context.is_same_group_size(input_mode, output_mode):
if context.is_same_layout(input_mode, output_mode):
return CommunicateSimpleFn._trivial
if (input_mode == ScatterMode.SCATTERED) and (
@@ -1140,10 +1179,8 @@ class CommunicateWithAllReduceAndLayerNormFn:
):
if (
context.is_same_group_size(
hidden_states_input_mode, hidden_states_output_mode
)
and context.is_same_group_size(residual_input_mode, residual_output_mode)
context.is_same_layout(hidden_states_input_mode, hidden_states_output_mode)
and context.is_same_layout(residual_input_mode, residual_output_mode)
and context.attn_tp_size == 1
):
return CommunicateWithAllReduceAndLayerNormFn._simple
@@ -1164,61 +1201,36 @@ class CommunicateWithAllReduceAndLayerNormFn:
# attn_tp_size == tp_size > 1, so that gate does not fire.
return CommunicateWithAllReduceAndLayerNormFn._simple
if (
(hidden_states_input_mode == ScatterMode.TP_ATTN_FULL)
and (
residual_input_mode in [ScatterMode.SCATTERED, ScatterMode.TP_ATTN_FULL]
)
and (hidden_states_output_mode == ScatterMode.FULL)
and (residual_output_mode == ScatterMode.TP_ATTN_FULL)
if hidden_states_input_mode == ScatterMode.TP_ATTN_FULL and (
residual_input_mode in (ScatterMode.SCATTERED, ScatterMode.TP_ATTN_FULL)
):
return partial(
CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual,
residual_input_mode=residual_input_mode,
)
fn = {
(
ScatterMode.FULL,
ScatterMode.TP_ATTN_FULL,
): CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual,
(
ScatterMode.MOE_FULL,
ScatterMode.TP_ATTN_FULL,
): CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual_moe,
(
ScatterMode.SCATTERED,
ScatterMode.SCATTERED,
): CommunicateWithAllReduceAndLayerNormFn._scatter_hidden_states_and_residual,
}.get((hidden_states_output_mode, residual_output_mode))
if fn is not None:
return partial(fn, residual_input_mode=residual_input_mode)
if (
(hidden_states_input_mode == ScatterMode.TP_ATTN_FULL)
and (
residual_input_mode in [ScatterMode.SCATTERED, ScatterMode.TP_ATTN_FULL]
)
and (hidden_states_output_mode == ScatterMode.MOE_FULL)
and (residual_output_mode == ScatterMode.TP_ATTN_FULL)
):
return partial(
CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual_moe,
residual_input_mode=residual_input_mode,
)
if (
(hidden_states_input_mode == ScatterMode.TP_ATTN_FULL)
and (
residual_input_mode in [ScatterMode.SCATTERED, ScatterMode.TP_ATTN_FULL]
)
and (hidden_states_output_mode == ScatterMode.SCATTERED)
and (residual_output_mode == ScatterMode.SCATTERED)
):
return partial(
CommunicateWithAllReduceAndLayerNormFn._scatter_hidden_states_and_residual,
residual_input_mode=residual_input_mode,
)
if (
(hidden_states_input_mode == ScatterMode.TP_ATTN_FULL)
and (
residual_input_mode in [ScatterMode.SCATTERED, ScatterMode.TP_ATTN_FULL]
)
and (hidden_states_output_mode == ScatterMode.TP_ATTN_FULL)
and (residual_output_mode == ScatterMode.TP_ATTN_FULL)
and context.attn_tp_size > 1
):
# Used when the dense MLP is tensor-parallelized along the
# attention TP group (``moe_dense_tp_size > 1``): hidden states
# need an all-reduce inside the attention TP group before the
# next layernorm, while staying in TP_ATTN_FULL on both sides.
return (
CommunicateWithAllReduceAndLayerNormFn._tp_attn_all_reduce_and_layernorm
)
if (
hidden_states_output_mode == ScatterMode.TP_ATTN_FULL
and residual_output_mode == ScatterMode.TP_ATTN_FULL
and context.attn_tp_size > 1
):
# Used when the dense MLP is tensor-parallelized along the
# attention TP group (``moe_dense_tp_size > 1``): hidden states
# need an all-reduce inside the attention TP group before the
# next layernorm, while staying in TP_ATTN_FULL on both sides.
return CommunicateWithAllReduceAndLayerNormFn._tp_attn_all_reduce_and_layernorm
raise NotImplementedError(
f"{hidden_states_input_mode=} {residual_input_mode=} {hidden_states_output_mode=} {residual_output_mode=}"
@@ -1474,38 +1486,35 @@ class CommunicateSummableTensorPairFn:
output_mode: ScatterMode,
context: CommunicateContext,
):
if context.is_same_group_size(
if context.is_same_layout(
hidden_states_input_mode, output_mode
) and context.is_same_group_size(residual_input_mode, output_mode):
) and context.is_same_layout(residual_input_mode, output_mode):
return CommunicateSummableTensorPairFn._trivial
if (
(hidden_states_input_mode == ScatterMode.FULL)
and (residual_input_mode == ScatterMode.TP_ATTN_FULL)
and (output_mode == ScatterMode.TP_ATTN_FULL)
):
return CommunicateSummableTensorPairFn._scatter_hidden_states
if (
(hidden_states_input_mode == ScatterMode.SCATTERED)
and (residual_input_mode == ScatterMode.SCATTERED)
and (output_mode == ScatterMode.TP_ATTN_FULL)
):
return CommunicateSummableTensorPairFn._gather
if (
(hidden_states_input_mode == ScatterMode.TP_ATTN_FULL)
and (residual_input_mode == ScatterMode.TP_ATTN_FULL)
and (output_mode == ScatterMode.SCATTERED)
):
return CommunicateSummableTensorPairFn._scatter
if (
(hidden_states_input_mode == ScatterMode.MOE_FULL)
and (residual_input_mode == ScatterMode.TP_ATTN_FULL)
and (output_mode == ScatterMode.TP_ATTN_FULL)
):
return CommunicateSummableTensorPairFn._scatter_hidden_states_moe
fn = {
(
ScatterMode.FULL,
ScatterMode.TP_ATTN_FULL,
ScatterMode.TP_ATTN_FULL,
): CommunicateSummableTensorPairFn._scatter_hidden_states,
(
ScatterMode.SCATTERED,
ScatterMode.SCATTERED,
ScatterMode.TP_ATTN_FULL,
): CommunicateSummableTensorPairFn._gather,
(
ScatterMode.TP_ATTN_FULL,
ScatterMode.TP_ATTN_FULL,
ScatterMode.SCATTERED,
): CommunicateSummableTensorPairFn._scatter,
(
ScatterMode.MOE_FULL,
ScatterMode.TP_ATTN_FULL,
ScatterMode.TP_ATTN_FULL,
): CommunicateSummableTensorPairFn._scatter_hidden_states_moe,
}.get((hidden_states_input_mode, residual_input_mode, output_mode))
if fn is not None:
return fn
raise NotImplementedError(
f"{hidden_states_input_mode=} {residual_input_mode=} {output_mode=}"