From 7dd96407524de43be39c39265b62fce412671f63 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Wed, 23 Sep 2026 14:41:55 -0700 Subject: [PATCH] [Refactor] Compare token layouts instead of group sizes when selecting communicator paths (#40869) --- python/sglang/srt/layers/boundary_layout.py | 41 ++++ python/sglang/srt/layers/communicator.py | 185 +++++++++--------- .../unit/layers/test_communicator_layout.py | 167 ++++++++++++++++ 3 files changed, 305 insertions(+), 88 deletions(-) create mode 100644 python/sglang/srt/layers/boundary_layout.py create mode 100644 test/registered/unit/layers/test_communicator_layout.py diff --git a/python/sglang/srt/layers/boundary_layout.py b/python/sglang/srt/layers/boundary_layout.py new file mode 100644 index 0000000000..9615c21b60 --- /dev/null +++ b/python/sglang/srt/layers/boundary_layout.py @@ -0,0 +1,41 @@ +# Copyright 2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Token layouts of the tensors handed across layer communication boundaries.""" + +from enum import Enum, auto +from typing import FrozenSet, Mapping + +import msgspec + + +class TokenAxis(Enum): + """A parallel dimension across which ranks hold different tokens.""" + + ATTN_DP = auto() + ATTN_CP = auto() + # Each attention-TP rank holds a slice of its group's tokens. + ATTN_TP_SCATTER = auto() + + +class Layout(msgspec.Struct, frozen=True): + """The token axes a rank's rows are sharded over. Single-rank axes are left + out, so two layouts are equal exactly when every rank holds the same tokens.""" + + sharded: FrozenSet[TokenAxis] + + @classmethod + def sharded_over( + cls, *axes: TokenAxis, axis_sizes: Mapping[TokenAxis, int] + ) -> "Layout": + return cls(frozenset(axis for axis in axes if axis_sizes[axis] > 1)) diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index eeb6a4604f..76ac7b44fe 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -15,7 +15,7 @@ import logging from contextlib import contextmanager from dataclasses import dataclass from enum import Enum, auto -from functools import partial +from functools import cached_property, partial from typing import Callable, Dict, Optional, Tuple, Union import torch @@ -35,6 +35,7 @@ from sglang.srt.layers.attention.dsa.utils import ( is_dsa_enable_prefill_cp, ) from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator +from sglang.srt.layers.boundary_layout import Layout, TokenAxis from sglang.srt.layers.cp.utils import ( is_mla_cp_active, is_mla_cp_enabled, @@ -1011,6 +1012,33 @@ class LayerCommunicator: ) +# MOE_FULL gathers across the MoE-CP group, which spans every CP rank when CP is on. +_SCATTER_MODE_SHARDED_AXES = { + ScatterMode.SCATTERED: ( + TokenAxis.ATTN_DP, + TokenAxis.ATTN_CP, + TokenAxis.ATTN_TP_SCATTER, + ), + ScatterMode.TP_ATTN_FULL: (TokenAxis.ATTN_DP, TokenAxis.ATTN_CP), + ScatterMode.FULL: (TokenAxis.ATTN_CP,), + ScatterMode.MOE_FULL: (), +} + + +def scatter_mode_layouts( + *, attn_dp_size: int, attn_cp_size: int, attn_tp_size: int +) -> Dict[ScatterMode, Layout]: + axis_sizes = { + TokenAxis.ATTN_DP: attn_dp_size, + TokenAxis.ATTN_CP: attn_cp_size, + TokenAxis.ATTN_TP_SCATTER: attn_tp_size, + } + return { + mode: Layout.sharded_over(*axes, axis_sizes=axis_sizes) + for mode, axes in _SCATTER_MODE_SHARDED_AXES.items() + } + + @dataclass class CommunicateContext: process_group_sizes: Dict[ScatterMode, int] @@ -1027,6 +1055,17 @@ class CommunicateContext: def is_same_group_size(self, a: ScatterMode, b: ScatterMode): return self.process_group_sizes[a] == self.process_group_sizes[b] + @cached_property + def layouts(self) -> Dict[ScatterMode, Layout]: + return scatter_mode_layouts( + attn_dp_size=self.attn_dp_size, + attn_cp_size=self.attn_cp_size, + attn_tp_size=self.attn_tp_size, + ) + + def is_same_layout(self, a: ScatterMode, b: ScatterMode): + return self.layouts[a] == self.layouts[b] + @classmethod def init_new(cls): attn_tp_rank = get_parallel().attn_tp_rank @@ -1065,7 +1104,7 @@ class CommunicateSimpleFn: output_mode: ScatterMode, context: CommunicateContext, ): - if context.is_same_group_size(input_mode, output_mode): + if context.is_same_layout(input_mode, output_mode): return CommunicateSimpleFn._trivial if (input_mode == ScatterMode.SCATTERED) and ( @@ -1140,10 +1179,8 @@ class CommunicateWithAllReduceAndLayerNormFn: ): if ( - context.is_same_group_size( - hidden_states_input_mode, hidden_states_output_mode - ) - and context.is_same_group_size(residual_input_mode, residual_output_mode) + context.is_same_layout(hidden_states_input_mode, hidden_states_output_mode) + and context.is_same_layout(residual_input_mode, residual_output_mode) and context.attn_tp_size == 1 ): return CommunicateWithAllReduceAndLayerNormFn._simple @@ -1164,61 +1201,36 @@ class CommunicateWithAllReduceAndLayerNormFn: # attn_tp_size == tp_size > 1, so that gate does not fire. return CommunicateWithAllReduceAndLayerNormFn._simple - if ( - (hidden_states_input_mode == ScatterMode.TP_ATTN_FULL) - and ( - residual_input_mode in [ScatterMode.SCATTERED, ScatterMode.TP_ATTN_FULL] - ) - and (hidden_states_output_mode == ScatterMode.FULL) - and (residual_output_mode == ScatterMode.TP_ATTN_FULL) + if hidden_states_input_mode == ScatterMode.TP_ATTN_FULL and ( + residual_input_mode in (ScatterMode.SCATTERED, ScatterMode.TP_ATTN_FULL) ): - return partial( - CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual, - residual_input_mode=residual_input_mode, - ) + fn = { + ( + ScatterMode.FULL, + ScatterMode.TP_ATTN_FULL, + ): CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual, + ( + ScatterMode.MOE_FULL, + ScatterMode.TP_ATTN_FULL, + ): CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual_moe, + ( + ScatterMode.SCATTERED, + ScatterMode.SCATTERED, + ): CommunicateWithAllReduceAndLayerNormFn._scatter_hidden_states_and_residual, + }.get((hidden_states_output_mode, residual_output_mode)) + if fn is not None: + return partial(fn, residual_input_mode=residual_input_mode) - if ( - (hidden_states_input_mode == ScatterMode.TP_ATTN_FULL) - and ( - residual_input_mode in [ScatterMode.SCATTERED, ScatterMode.TP_ATTN_FULL] - ) - and (hidden_states_output_mode == ScatterMode.MOE_FULL) - and (residual_output_mode == ScatterMode.TP_ATTN_FULL) - ): - return partial( - CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual_moe, - residual_input_mode=residual_input_mode, - ) - - if ( - (hidden_states_input_mode == ScatterMode.TP_ATTN_FULL) - and ( - residual_input_mode in [ScatterMode.SCATTERED, ScatterMode.TP_ATTN_FULL] - ) - and (hidden_states_output_mode == ScatterMode.SCATTERED) - and (residual_output_mode == ScatterMode.SCATTERED) - ): - return partial( - CommunicateWithAllReduceAndLayerNormFn._scatter_hidden_states_and_residual, - residual_input_mode=residual_input_mode, - ) - - if ( - (hidden_states_input_mode == ScatterMode.TP_ATTN_FULL) - and ( - residual_input_mode in [ScatterMode.SCATTERED, ScatterMode.TP_ATTN_FULL] - ) - and (hidden_states_output_mode == ScatterMode.TP_ATTN_FULL) - and (residual_output_mode == ScatterMode.TP_ATTN_FULL) - and context.attn_tp_size > 1 - ): - # Used when the dense MLP is tensor-parallelized along the - # attention TP group (``moe_dense_tp_size > 1``): hidden states - # need an all-reduce inside the attention TP group before the - # next layernorm, while staying in TP_ATTN_FULL on both sides. - return ( - CommunicateWithAllReduceAndLayerNormFn._tp_attn_all_reduce_and_layernorm - ) + if ( + hidden_states_output_mode == ScatterMode.TP_ATTN_FULL + and residual_output_mode == ScatterMode.TP_ATTN_FULL + and context.attn_tp_size > 1 + ): + # Used when the dense MLP is tensor-parallelized along the + # attention TP group (``moe_dense_tp_size > 1``): hidden states + # need an all-reduce inside the attention TP group before the + # next layernorm, while staying in TP_ATTN_FULL on both sides. + return CommunicateWithAllReduceAndLayerNormFn._tp_attn_all_reduce_and_layernorm raise NotImplementedError( f"{hidden_states_input_mode=} {residual_input_mode=} {hidden_states_output_mode=} {residual_output_mode=}" @@ -1474,38 +1486,35 @@ class CommunicateSummableTensorPairFn: output_mode: ScatterMode, context: CommunicateContext, ): - if context.is_same_group_size( + if context.is_same_layout( hidden_states_input_mode, output_mode - ) and context.is_same_group_size(residual_input_mode, output_mode): + ) and context.is_same_layout(residual_input_mode, output_mode): return CommunicateSummableTensorPairFn._trivial - if ( - (hidden_states_input_mode == ScatterMode.FULL) - and (residual_input_mode == ScatterMode.TP_ATTN_FULL) - and (output_mode == ScatterMode.TP_ATTN_FULL) - ): - return CommunicateSummableTensorPairFn._scatter_hidden_states - - if ( - (hidden_states_input_mode == ScatterMode.SCATTERED) - and (residual_input_mode == ScatterMode.SCATTERED) - and (output_mode == ScatterMode.TP_ATTN_FULL) - ): - return CommunicateSummableTensorPairFn._gather - - if ( - (hidden_states_input_mode == ScatterMode.TP_ATTN_FULL) - and (residual_input_mode == ScatterMode.TP_ATTN_FULL) - and (output_mode == ScatterMode.SCATTERED) - ): - return CommunicateSummableTensorPairFn._scatter - - if ( - (hidden_states_input_mode == ScatterMode.MOE_FULL) - and (residual_input_mode == ScatterMode.TP_ATTN_FULL) - and (output_mode == ScatterMode.TP_ATTN_FULL) - ): - return CommunicateSummableTensorPairFn._scatter_hidden_states_moe + fn = { + ( + ScatterMode.FULL, + ScatterMode.TP_ATTN_FULL, + ScatterMode.TP_ATTN_FULL, + ): CommunicateSummableTensorPairFn._scatter_hidden_states, + ( + ScatterMode.SCATTERED, + ScatterMode.SCATTERED, + ScatterMode.TP_ATTN_FULL, + ): CommunicateSummableTensorPairFn._gather, + ( + ScatterMode.TP_ATTN_FULL, + ScatterMode.TP_ATTN_FULL, + ScatterMode.SCATTERED, + ): CommunicateSummableTensorPairFn._scatter, + ( + ScatterMode.MOE_FULL, + ScatterMode.TP_ATTN_FULL, + ScatterMode.TP_ATTN_FULL, + ): CommunicateSummableTensorPairFn._scatter_hidden_states_moe, + }.get((hidden_states_input_mode, residual_input_mode, output_mode)) + if fn is not None: + return fn raise NotImplementedError( f"{hidden_states_input_mode=} {residual_input_mode=} {output_mode=}" diff --git a/test/registered/unit/layers/test_communicator_layout.py b/test/registered/unit/layers/test_communicator_layout.py new file mode 100644 index 0000000000..0dbad9ea36 --- /dev/null +++ b/test/registered/unit/layers/test_communicator_layout.py @@ -0,0 +1,167 @@ +import unittest +from functools import partial +from unittest.mock import patch + +from sglang.srt.layers import communicator as comm +from sglang.srt.layers.boundary_layout import Layout, TokenAxis +from sglang.srt.layers.communicator import ( + CommunicateContext, + CommunicateSimpleFn, + CommunicateSummableTensorPairFn, + CommunicateWithAllReduceAndLayerNormFn, + ScatterMode, +) +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +SCATTERED = ScatterMode.SCATTERED +TP_ATTN_FULL = ScatterMode.TP_ATTN_FULL +FULL = ScatterMode.FULL +MOE_FULL = ScatterMode.MOE_FULL +Norm = CommunicateWithAllReduceAndLayerNormFn +Pair = CommunicateSummableTensorPairFn + + +def make_context(*, dp=1, cp=1, tp=1): + """A context for attention DP x CP x TP, built without the process-wide + parallel state.""" + return CommunicateContext( + process_group_sizes={}, + attn_tp_rank=0, + attn_tp_size=tp, + attn_dp_size=dp, + attn_cp_rank=0, + attn_cp_size=cp, + tp_size=dp * cp * tp, + tp_rank=0, + ) + + +class TestScatterModeLayouts(CustomTestCase): + def test_modes_shard_nested_axes(self): + layouts = make_context(dp=2, cp=2, tp=2).layouts + self.assertEqual( + layouts[SCATTERED], + Layout( + frozenset( + {TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, TokenAxis.ATTN_TP_SCATTER} + ) + ), + ) + self.assertEqual( + layouts[TP_ATTN_FULL], + Layout(frozenset({TokenAxis.ATTN_DP, TokenAxis.ATTN_CP})), + ) + self.assertEqual(layouts[FULL], Layout(frozenset({TokenAxis.ATTN_CP}))) + self.assertEqual(layouts[MOE_FULL], Layout(frozenset())) + + def test_single_rank_axes_shard_nothing(self): + cases = [ + # (topology, pairs with equal layouts, pairs with different layouts) + ( + dict(tp=2), + [(FULL, TP_ATTN_FULL), (MOE_FULL, FULL)], + [(SCATTERED, TP_ATTN_FULL)], + ), + (dict(dp=2, tp=2), [(MOE_FULL, FULL)], [(FULL, TP_ATTN_FULL)]), + ( + dict(cp=2), + [(SCATTERED, TP_ATTN_FULL), (FULL, TP_ATTN_FULL)], + [(MOE_FULL, FULL)], + ), + (dict(), [(SCATTERED, MOE_FULL)], []), + ] + for topology, same, different in cases: + context = make_context(**topology) + for a, b in same: + with self.subTest(topology=topology, same=(a, b)): + self.assertTrue(context.is_same_layout(a, b)) + for a, b in different: + with self.subTest(topology=topology, different=(a, b)): + self.assertFalse(context.is_same_layout(a, b)) + + +class TestBoundarySelection(CustomTestCase): + def assert_partial(self, fn, func, residual_input_mode): + self.assertIsInstance(fn, partial) + self.assertIs(fn.func, func) + self.assertEqual(fn.keywords, {"residual_input_mode": residual_input_mode}) + + def test_prepare_mlp(self): + dp_tp = make_context(dp=2, tp=2) + self.assert_partial( + Norm.get_fn(TP_ATTN_FULL, SCATTERED, FULL, TP_ATTN_FULL, dp_tp), + Norm._gather_hidden_states_and_residual, + SCATTERED, + ) + self.assert_partial( + Norm.get_fn( + TP_ATTN_FULL, TP_ATTN_FULL, MOE_FULL, TP_ATTN_FULL, make_context(cp=2) + ), + Norm._gather_hidden_states_and_residual_moe, + TP_ATTN_FULL, + ) + self.assert_partial( + Norm.get_fn(TP_ATTN_FULL, TP_ATTN_FULL, SCATTERED, SCATTERED, dp_tp), + Norm._scatter_hidden_states_and_residual, + TP_ATTN_FULL, + ) + + def test_prepare_mlp_without_attention_dp(self): + # FULL and TP_ATTN_FULL share a layout here but select different paths. + tp = make_context(tp=2) + self.assert_partial( + Norm.get_fn(TP_ATTN_FULL, TP_ATTN_FULL, FULL, TP_ATTN_FULL, tp), + Norm._gather_hidden_states_and_residual, + TP_ATTN_FULL, + ) + self.assertIs( + Norm.get_fn(TP_ATTN_FULL, TP_ATTN_FULL, TP_ATTN_FULL, TP_ATTN_FULL, tp), + Norm._tp_attn_all_reduce_and_layernorm, + ) + self.assertIs( + Norm.get_fn(TP_ATTN_FULL, TP_ATTN_FULL, FULL, TP_ATTN_FULL, make_context()), + Norm._simple, + ) + self.assertIs( + Norm.get_fn(SCATTERED, SCATTERED, SCATTERED, SCATTERED, tp), + Norm._simple, + ) + with self.assertRaises(NotImplementedError): + Norm.get_fn(SCATTERED, SCATTERED, TP_ATTN_FULL, TP_ATTN_FULL, tp) + + def test_postprocess(self): + self.assertIs( + Pair.get_fn(FULL, TP_ATTN_FULL, TP_ATTN_FULL, make_context(dp=2)), + Pair._scatter_hidden_states, + ) + self.assertIs( + Pair.get_fn(FULL, TP_ATTN_FULL, TP_ATTN_FULL, make_context(tp=2)), + Pair._trivial, + ) + self.assertIs( + Pair.get_fn(MOE_FULL, TP_ATTN_FULL, TP_ATTN_FULL, make_context(cp=2)), + Pair._scatter_hidden_states_moe, + ) + tp = make_context(tp=2) + self.assertIs(Pair.get_fn(SCATTERED, SCATTERED, TP_ATTN_FULL, tp), Pair._gather) + self.assertIs( + Pair.get_fn(TP_ATTN_FULL, TP_ATTN_FULL, SCATTERED, tp), Pair._scatter + ) + + def test_prepare_attn(self): + with patch.object(comm, "_use_ag_after_qlora", False): + self.assertIs( + CommunicateSimpleFn.get_fn(SCATTERED, TP_ATTN_FULL, make_context(tp=2)), + CommunicateSimpleFn._scattered_to_tp_attn_full, + ) + self.assertIs( + CommunicateSimpleFn.get_fn(SCATTERED, TP_ATTN_FULL, make_context(cp=2)), + CommunicateSimpleFn._trivial, + ) + + +if __name__ == "__main__": + unittest.main()