[Refactor] Compare token layouts instead of group sizes when selecting communicator paths (#40869)
This commit is contained in:
@@ -0,0 +1,41 @@
|
||||
# Copyright 2026 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""Token layouts of the tensors handed across layer communication boundaries."""
|
||||
|
||||
from enum import Enum, auto
|
||||
from typing import FrozenSet, Mapping
|
||||
|
||||
import msgspec
|
||||
|
||||
|
||||
class TokenAxis(Enum):
|
||||
"""A parallel dimension across which ranks hold different tokens."""
|
||||
|
||||
ATTN_DP = auto()
|
||||
ATTN_CP = auto()
|
||||
# Each attention-TP rank holds a slice of its group's tokens.
|
||||
ATTN_TP_SCATTER = auto()
|
||||
|
||||
|
||||
class Layout(msgspec.Struct, frozen=True):
|
||||
"""The token axes a rank's rows are sharded over. Single-rank axes are left
|
||||
out, so two layouts are equal exactly when every rank holds the same tokens."""
|
||||
|
||||
sharded: FrozenSet[TokenAxis]
|
||||
|
||||
@classmethod
|
||||
def sharded_over(
|
||||
cls, *axes: TokenAxis, axis_sizes: Mapping[TokenAxis, int]
|
||||
) -> "Layout":
|
||||
return cls(frozenset(axis for axis in axes if axis_sizes[axis] > 1))
|
||||
@@ -15,7 +15,7 @@ import logging
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
from functools import partial
|
||||
from functools import cached_property, partial
|
||||
from typing import Callable, Dict, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
@@ -35,6 +35,7 @@ from sglang.srt.layers.attention.dsa.utils import (
|
||||
is_dsa_enable_prefill_cp,
|
||||
)
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator
|
||||
from sglang.srt.layers.boundary_layout import Layout, TokenAxis
|
||||
from sglang.srt.layers.cp.utils import (
|
||||
is_mla_cp_active,
|
||||
is_mla_cp_enabled,
|
||||
@@ -1011,6 +1012,33 @@ class LayerCommunicator:
|
||||
)
|
||||
|
||||
|
||||
# MOE_FULL gathers across the MoE-CP group, which spans every CP rank when CP is on.
|
||||
_SCATTER_MODE_SHARDED_AXES = {
|
||||
ScatterMode.SCATTERED: (
|
||||
TokenAxis.ATTN_DP,
|
||||
TokenAxis.ATTN_CP,
|
||||
TokenAxis.ATTN_TP_SCATTER,
|
||||
),
|
||||
ScatterMode.TP_ATTN_FULL: (TokenAxis.ATTN_DP, TokenAxis.ATTN_CP),
|
||||
ScatterMode.FULL: (TokenAxis.ATTN_CP,),
|
||||
ScatterMode.MOE_FULL: (),
|
||||
}
|
||||
|
||||
|
||||
def scatter_mode_layouts(
|
||||
*, attn_dp_size: int, attn_cp_size: int, attn_tp_size: int
|
||||
) -> Dict[ScatterMode, Layout]:
|
||||
axis_sizes = {
|
||||
TokenAxis.ATTN_DP: attn_dp_size,
|
||||
TokenAxis.ATTN_CP: attn_cp_size,
|
||||
TokenAxis.ATTN_TP_SCATTER: attn_tp_size,
|
||||
}
|
||||
return {
|
||||
mode: Layout.sharded_over(*axes, axis_sizes=axis_sizes)
|
||||
for mode, axes in _SCATTER_MODE_SHARDED_AXES.items()
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class CommunicateContext:
|
||||
process_group_sizes: Dict[ScatterMode, int]
|
||||
@@ -1027,6 +1055,17 @@ class CommunicateContext:
|
||||
def is_same_group_size(self, a: ScatterMode, b: ScatterMode):
|
||||
return self.process_group_sizes[a] == self.process_group_sizes[b]
|
||||
|
||||
@cached_property
|
||||
def layouts(self) -> Dict[ScatterMode, Layout]:
|
||||
return scatter_mode_layouts(
|
||||
attn_dp_size=self.attn_dp_size,
|
||||
attn_cp_size=self.attn_cp_size,
|
||||
attn_tp_size=self.attn_tp_size,
|
||||
)
|
||||
|
||||
def is_same_layout(self, a: ScatterMode, b: ScatterMode):
|
||||
return self.layouts[a] == self.layouts[b]
|
||||
|
||||
@classmethod
|
||||
def init_new(cls):
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
@@ -1065,7 +1104,7 @@ class CommunicateSimpleFn:
|
||||
output_mode: ScatterMode,
|
||||
context: CommunicateContext,
|
||||
):
|
||||
if context.is_same_group_size(input_mode, output_mode):
|
||||
if context.is_same_layout(input_mode, output_mode):
|
||||
return CommunicateSimpleFn._trivial
|
||||
|
||||
if (input_mode == ScatterMode.SCATTERED) and (
|
||||
@@ -1140,10 +1179,8 @@ class CommunicateWithAllReduceAndLayerNormFn:
|
||||
):
|
||||
|
||||
if (
|
||||
context.is_same_group_size(
|
||||
hidden_states_input_mode, hidden_states_output_mode
|
||||
)
|
||||
and context.is_same_group_size(residual_input_mode, residual_output_mode)
|
||||
context.is_same_layout(hidden_states_input_mode, hidden_states_output_mode)
|
||||
and context.is_same_layout(residual_input_mode, residual_output_mode)
|
||||
and context.attn_tp_size == 1
|
||||
):
|
||||
return CommunicateWithAllReduceAndLayerNormFn._simple
|
||||
@@ -1164,61 +1201,36 @@ class CommunicateWithAllReduceAndLayerNormFn:
|
||||
# attn_tp_size == tp_size > 1, so that gate does not fire.
|
||||
return CommunicateWithAllReduceAndLayerNormFn._simple
|
||||
|
||||
if (
|
||||
(hidden_states_input_mode == ScatterMode.TP_ATTN_FULL)
|
||||
and (
|
||||
residual_input_mode in [ScatterMode.SCATTERED, ScatterMode.TP_ATTN_FULL]
|
||||
)
|
||||
and (hidden_states_output_mode == ScatterMode.FULL)
|
||||
and (residual_output_mode == ScatterMode.TP_ATTN_FULL)
|
||||
if hidden_states_input_mode == ScatterMode.TP_ATTN_FULL and (
|
||||
residual_input_mode in (ScatterMode.SCATTERED, ScatterMode.TP_ATTN_FULL)
|
||||
):
|
||||
return partial(
|
||||
CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual,
|
||||
residual_input_mode=residual_input_mode,
|
||||
)
|
||||
fn = {
|
||||
(
|
||||
ScatterMode.FULL,
|
||||
ScatterMode.TP_ATTN_FULL,
|
||||
): CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual,
|
||||
(
|
||||
ScatterMode.MOE_FULL,
|
||||
ScatterMode.TP_ATTN_FULL,
|
||||
): CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual_moe,
|
||||
(
|
||||
ScatterMode.SCATTERED,
|
||||
ScatterMode.SCATTERED,
|
||||
): CommunicateWithAllReduceAndLayerNormFn._scatter_hidden_states_and_residual,
|
||||
}.get((hidden_states_output_mode, residual_output_mode))
|
||||
if fn is not None:
|
||||
return partial(fn, residual_input_mode=residual_input_mode)
|
||||
|
||||
if (
|
||||
(hidden_states_input_mode == ScatterMode.TP_ATTN_FULL)
|
||||
and (
|
||||
residual_input_mode in [ScatterMode.SCATTERED, ScatterMode.TP_ATTN_FULL]
|
||||
)
|
||||
and (hidden_states_output_mode == ScatterMode.MOE_FULL)
|
||||
and (residual_output_mode == ScatterMode.TP_ATTN_FULL)
|
||||
):
|
||||
return partial(
|
||||
CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual_moe,
|
||||
residual_input_mode=residual_input_mode,
|
||||
)
|
||||
|
||||
if (
|
||||
(hidden_states_input_mode == ScatterMode.TP_ATTN_FULL)
|
||||
and (
|
||||
residual_input_mode in [ScatterMode.SCATTERED, ScatterMode.TP_ATTN_FULL]
|
||||
)
|
||||
and (hidden_states_output_mode == ScatterMode.SCATTERED)
|
||||
and (residual_output_mode == ScatterMode.SCATTERED)
|
||||
):
|
||||
return partial(
|
||||
CommunicateWithAllReduceAndLayerNormFn._scatter_hidden_states_and_residual,
|
||||
residual_input_mode=residual_input_mode,
|
||||
)
|
||||
|
||||
if (
|
||||
(hidden_states_input_mode == ScatterMode.TP_ATTN_FULL)
|
||||
and (
|
||||
residual_input_mode in [ScatterMode.SCATTERED, ScatterMode.TP_ATTN_FULL]
|
||||
)
|
||||
and (hidden_states_output_mode == ScatterMode.TP_ATTN_FULL)
|
||||
and (residual_output_mode == ScatterMode.TP_ATTN_FULL)
|
||||
and context.attn_tp_size > 1
|
||||
):
|
||||
# Used when the dense MLP is tensor-parallelized along the
|
||||
# attention TP group (``moe_dense_tp_size > 1``): hidden states
|
||||
# need an all-reduce inside the attention TP group before the
|
||||
# next layernorm, while staying in TP_ATTN_FULL on both sides.
|
||||
return (
|
||||
CommunicateWithAllReduceAndLayerNormFn._tp_attn_all_reduce_and_layernorm
|
||||
)
|
||||
if (
|
||||
hidden_states_output_mode == ScatterMode.TP_ATTN_FULL
|
||||
and residual_output_mode == ScatterMode.TP_ATTN_FULL
|
||||
and context.attn_tp_size > 1
|
||||
):
|
||||
# Used when the dense MLP is tensor-parallelized along the
|
||||
# attention TP group (``moe_dense_tp_size > 1``): hidden states
|
||||
# need an all-reduce inside the attention TP group before the
|
||||
# next layernorm, while staying in TP_ATTN_FULL on both sides.
|
||||
return CommunicateWithAllReduceAndLayerNormFn._tp_attn_all_reduce_and_layernorm
|
||||
|
||||
raise NotImplementedError(
|
||||
f"{hidden_states_input_mode=} {residual_input_mode=} {hidden_states_output_mode=} {residual_output_mode=}"
|
||||
@@ -1474,38 +1486,35 @@ class CommunicateSummableTensorPairFn:
|
||||
output_mode: ScatterMode,
|
||||
context: CommunicateContext,
|
||||
):
|
||||
if context.is_same_group_size(
|
||||
if context.is_same_layout(
|
||||
hidden_states_input_mode, output_mode
|
||||
) and context.is_same_group_size(residual_input_mode, output_mode):
|
||||
) and context.is_same_layout(residual_input_mode, output_mode):
|
||||
return CommunicateSummableTensorPairFn._trivial
|
||||
|
||||
if (
|
||||
(hidden_states_input_mode == ScatterMode.FULL)
|
||||
and (residual_input_mode == ScatterMode.TP_ATTN_FULL)
|
||||
and (output_mode == ScatterMode.TP_ATTN_FULL)
|
||||
):
|
||||
return CommunicateSummableTensorPairFn._scatter_hidden_states
|
||||
|
||||
if (
|
||||
(hidden_states_input_mode == ScatterMode.SCATTERED)
|
||||
and (residual_input_mode == ScatterMode.SCATTERED)
|
||||
and (output_mode == ScatterMode.TP_ATTN_FULL)
|
||||
):
|
||||
return CommunicateSummableTensorPairFn._gather
|
||||
|
||||
if (
|
||||
(hidden_states_input_mode == ScatterMode.TP_ATTN_FULL)
|
||||
and (residual_input_mode == ScatterMode.TP_ATTN_FULL)
|
||||
and (output_mode == ScatterMode.SCATTERED)
|
||||
):
|
||||
return CommunicateSummableTensorPairFn._scatter
|
||||
|
||||
if (
|
||||
(hidden_states_input_mode == ScatterMode.MOE_FULL)
|
||||
and (residual_input_mode == ScatterMode.TP_ATTN_FULL)
|
||||
and (output_mode == ScatterMode.TP_ATTN_FULL)
|
||||
):
|
||||
return CommunicateSummableTensorPairFn._scatter_hidden_states_moe
|
||||
fn = {
|
||||
(
|
||||
ScatterMode.FULL,
|
||||
ScatterMode.TP_ATTN_FULL,
|
||||
ScatterMode.TP_ATTN_FULL,
|
||||
): CommunicateSummableTensorPairFn._scatter_hidden_states,
|
||||
(
|
||||
ScatterMode.SCATTERED,
|
||||
ScatterMode.SCATTERED,
|
||||
ScatterMode.TP_ATTN_FULL,
|
||||
): CommunicateSummableTensorPairFn._gather,
|
||||
(
|
||||
ScatterMode.TP_ATTN_FULL,
|
||||
ScatterMode.TP_ATTN_FULL,
|
||||
ScatterMode.SCATTERED,
|
||||
): CommunicateSummableTensorPairFn._scatter,
|
||||
(
|
||||
ScatterMode.MOE_FULL,
|
||||
ScatterMode.TP_ATTN_FULL,
|
||||
ScatterMode.TP_ATTN_FULL,
|
||||
): CommunicateSummableTensorPairFn._scatter_hidden_states_moe,
|
||||
}.get((hidden_states_input_mode, residual_input_mode, output_mode))
|
||||
if fn is not None:
|
||||
return fn
|
||||
|
||||
raise NotImplementedError(
|
||||
f"{hidden_states_input_mode=} {residual_input_mode=} {output_mode=}"
|
||||
|
||||
Reference in New Issue
Block a user