diff --git a/python/sglang/srt/batch_overlap/two_batch_overlap.py b/python/sglang/srt/batch_overlap/two_batch_overlap.py index 49e7e11da5..05015459a6 100644 --- a/python/sglang/srt/batch_overlap/two_batch_overlap.py +++ b/python/sglang/srt/batch_overlap/two_batch_overlap.py @@ -14,11 +14,11 @@ from sglang.srt.batch_overlap.operations import ( ) from sglang.srt.batch_overlap.operations_strategy import OperationsStrategy from sglang.srt.layers import deep_gemm_wrapper +from sglang.srt.layers.boundary_layout import Layout from sglang.srt.layers.communicator import ( CommunicateContext, - CommunicateSummableTensorPairFn, - ScatterMode, reduce_output, + tbo_split_moves, ) from sglang.srt.layers.moe import ( get_deepep_mode, @@ -935,7 +935,6 @@ def model_forward_maybe_tbo( positions: torch.Tensor, forward_batch: ForwardBatch, hidden_states: torch.Tensor, - input_data_scatter_mode: ScatterMode, residual: Optional[torch.Tensor], zero_allocator: Optional[BumpAllocator] = None, ): @@ -946,7 +945,6 @@ def model_forward_maybe_tbo( residual=residual, zero_allocator=zero_allocator, ) - layer_input_scatter_mode = layers[0].layer_scatter_modes.layer_input_mode operations_strategy = OperationsStrategy.init_new_tbo( layers, forward_batch.global_forward_mode ) @@ -954,8 +952,7 @@ def model_forward_maybe_tbo( return _model_forward_tbo( inputs=inputs, operations_strategy=operations_strategy, - input_data_scatter_mode=input_data_scatter_mode, - layer_input_scatter_mode=layer_input_scatter_mode, + layer_input_rows=layers[0].layer_communicator.input_rows, ) else: return _model_forward_non_tbo(inputs, operations_strategy) @@ -964,14 +961,11 @@ def model_forward_maybe_tbo( def _model_forward_tbo( inputs, operations_strategy: OperationsStrategy, - input_data_scatter_mode: ScatterMode, - layer_input_scatter_mode: ScatterMode, + layer_input_rows: Layout, ): inputs["hidden_states"] = reduce_output(inputs["hidden_states"]) inputs_arr = _model_forward_tbo_split_inputs( - **inputs, - input_data_scatter_mode=input_data_scatter_mode, - layer_input_scatter_mode=layer_input_scatter_mode, + **inputs, layer_input_rows=layer_input_rows ) original_hidden_states_len = inputs["hidden_states"].shape[0] del inputs @@ -1005,25 +999,10 @@ def _model_forward_tbo_split_inputs( positions: torch.Tensor, forward_batch: ForwardBatch, zero_allocator: Optional[BumpAllocator], - input_data_scatter_mode: ScatterMode, - layer_input_scatter_mode: ScatterMode, + layer_input_rows: Layout, ) -> List[Dict]: - tbo_splitter_scatter_mode = ScatterMode.TP_ATTN_FULL context = CommunicateContext.init_new() - # The splitter cuts the attention-TP-full layout; each microbatch then moves - # to the first layer's input layout. - to_splitter = CommunicateSummableTensorPairFn.get_fn( - hidden_states_input_mode=input_data_scatter_mode, - residual_input_mode=input_data_scatter_mode, - output_mode=tbo_splitter_scatter_mode, - context=context, - ) - to_layer_input = CommunicateSummableTensorPairFn.get_fn( - hidden_states_input_mode=tbo_splitter_scatter_mode, - residual_input_mode=tbo_splitter_scatter_mode, - output_mode=layer_input_scatter_mode, - context=context, - ) + to_splitter, to_layer_input = tbo_split_moves(layer_input_rows) hidden_states, residual = to_splitter( hidden_states=hidden_states, diff --git a/python/sglang/srt/layers/boundary_layout.py b/python/sglang/srt/layers/boundary_layout.py index c7af94f9a6..b18c55a997 100644 --- a/python/sglang/srt/layers/boundary_layout.py +++ b/python/sglang/srt/layers/boundary_layout.py @@ -229,11 +229,14 @@ def decoder_layer_sides( leaves_for_next_layer: bool, leaves_for_reduce_scatter: bool, leaves_for_reduce_scatterv: bool, + hands_on_attention_rows: bool = False, ) -> DecoderLayerSides: """An attention followed by an FFN, derived from the groups each computes over. The FFN runs either on the TP group (a dense MLP, or a MoE not dispatched per DP shard) or on this rank's local rows (a MoE dispatched per - DP shard, which completes its own combine, or a dense MLP on every rank).""" + DP shard, which completes its own combine, or a dense MLP on every rank). + A layer on local rows hands on the attention's rows when it is the last, or + with ``hands_on_attention_rows``.""" # Attention computes over the attention-TP ranks of one DP (and CP) shard. attention = Layout.sharded_over( TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes @@ -277,5 +280,9 @@ def decoder_layer_sides( ) ), ffn_residual_rows=local if ffn_on_local_rows else attention, - output_rows=local if ffn_on_local_rows and not is_last_layer else attention, + output_rows=( + local + if ffn_on_local_rows and not (is_last_layer or hands_on_attention_rows) + else attention + ), ) diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index 2fc3842b67..7f583687cd 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -467,6 +467,8 @@ class LayerScatterModes: # Whether the layer before this one has a sparse MLP; None when the modes # were given directly, not planned from the layer sequence. is_previous_layer_sparse: Optional[bool] = None + # Whether the layer after this one has a sparse MLP; None likewise. + is_next_layer_sparse: Optional[bool] = None @classmethod def init_new(cls, **kwargs): @@ -481,6 +483,7 @@ class LayerScatterModes: is_first_layer=context.layer_id == 0, is_last_layer=context.layer_id == context.num_layers - 1, is_previous_layer_sparse=context.is_previous_layer_sparse, + is_next_layer_sparse=context.is_next_layer_sparse, ) @classmethod @@ -854,6 +857,7 @@ class LayerCommunicator: ) # The steps the layer's ordinary batches run. sides = self._declared_sides() + self._declared = sides self._steps = ( self._steps_from_declarations( sides, @@ -903,6 +907,19 @@ class LayerCommunicator: else None ) + @property + def input_rows(self) -> Layout: + """The rows the layer's input and residual arrive on in a batch that + runs its ordinary steps.""" + if self._declared is not None: + return self._declared.input_rows + # Such a batch holds every token on each CP rank. + return scatter_mode_layouts( + attn_dp_size=self._context.attn_dp_size, + attn_cp_size=1, + attn_tp_size=self._context.attn_tp_size, + )[self.layer_scatter_modes.layer_input_mode] + def _input_can_be_scattered(self) -> bool: """Whether a batch may run this layer with input-scattered attention: configured, on pure TP without an a2a backend or a dense MLP on every @@ -938,6 +955,17 @@ class LayerCommunicator: def on_local_rows(sparse: bool) -> bool: return moe_on_local_rows if sparse else dense_on_local_rows + def gathers_for_tbo(sparse: bool, next_sparse: Optional[bool]) -> bool: + # Under two-batch overlap a dense layer on local rows hands the + # sparse layer after it, where the split happens, the attention's + # rows. + return ( + dense_on_local_rows + and get_exec().overlap.enable_two_batch_overlap + and not sparse + and bool(next_sparse) + ) + if not ( self._takes_declared_boundaries and (parallel.attn_cp_size == 1 or _cp_on_declarations()) @@ -949,14 +977,6 @@ class LayerCommunicator: and modes.is_layer_sparse and not _gathers_over_attention_cp() ) - # Under two-batch overlap a dense layer before a sparse one gathers - # its output over attention TP for the split; those layers keep the - # scatter-mode steps. - and not ( - dense_on_local_rows - and parallel.attn_tp_size > 1 - and get_exec().overlap.enable_two_batch_overlap - ) and (modes.is_first_layer or modes.is_previous_layer_sparse is not None) ): return None @@ -974,8 +994,14 @@ class LayerCommunicator: previous_on_local_rows=( not modes.is_first_layer and on_local_rows(modes.is_previous_layer_sparse) + and not gathers_for_tbo( + modes.is_previous_layer_sparse, modes.is_layer_sparse + ) ), is_last_layer=modes.is_last_layer, + hands_on_attention_rows=gathers_for_tbo( + modes.is_layer_sparse, modes.is_next_layer_sparse + ), attention_gathers_local_rows=_use_ag_after_qlora, ffn_group=SumGroup.MOE_OUTPUT if modes.is_layer_sparse else SumGroup.TP, leaves_for_next_layer=self.allow_deferred_ffn_reduction and may_leave, @@ -2153,6 +2179,23 @@ def _token_axis_sizes(*, cp_active: bool = False) -> Dict[TokenAxis, int]: } +def tbo_split_moves(layer_input_rows: Layout) -> Tuple[Callable, Callable]: + """The moves around the two-batch-overlap split, which cuts the attention's + rows: from the rows the first overlapped layer takes to the attention's, + and back again for each half.""" + attention = Layout.sharded_over( + TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=_token_axis_sizes() + ) + pair = CommunicateSummableTensorPairFn + if layer_input_rows == attention: + return pair._trivial, pair._trivial + if layer_input_rows.sharded - attention.sharded == {TokenAxis.ATTN_TP_SCATTER}: + # Each rank's slice: write the residual in and gather over attention + # TP, then take the slice of each half. + return pair._gather, pair._scatter + raise NotImplementedError(f"{layer_input_rows=}") + + def _cp_on_declarations() -> bool: """Whether attention CP is one the declarations cover: a prefill CP that shards tokens, with DSA or MLA attention, or with the FFN input gathered diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 3d935bacea..b41e79e8b3 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -3124,9 +3124,6 @@ class DeepseekV2Model(nn.Module): forward_batch=forward_batch, hidden_states=hidden_states, residual=residual, - input_data_scatter_mode=self.layers[ - normal_end_layer - 1 - ].layer_scatter_modes.layer_output_mode, zero_allocator=zero_allocator, ) diff --git a/python/sglang/srt/models/dots3_common/modeling.py b/python/sglang/srt/models/dots3_common/modeling.py index 6cfc2109d3..170bd39e82 100644 --- a/python/sglang/srt/models/dots3_common/modeling.py +++ b/python/sglang/srt/models/dots3_common/modeling.py @@ -1797,9 +1797,6 @@ class Dots3Model(nn.Module): forward_batch=forward_batch, hidden_states=hidden_states, residual=residual, - input_data_scatter_mode=self.layers[ - normal_end_layer - 1 - ].layer_scatter_modes.layer_output_mode, zero_allocator=zero_allocator, ) diff --git a/python/sglang/srt/models/glm4_moe.py b/python/sglang/srt/models/glm4_moe.py index 2b20e64f84..c12375f409 100644 --- a/python/sglang/srt/models/glm4_moe.py +++ b/python/sglang/srt/models/glm4_moe.py @@ -1124,9 +1124,6 @@ class Glm4MoeModel(nn.Module): forward_batch=forward_batch, hidden_states=hidden_states, residual=residual, - input_data_scatter_mode=self.layers[ - normal_end_layer - 1 - ].layer_scatter_modes.layer_output_mode, ) last_layer = self.layers[self.end_layer - 1] diff --git a/python/sglang/srt/models/glm4_moe_lite.py b/python/sglang/srt/models/glm4_moe_lite.py index 3b4f46ce60..3028e2f0d9 100644 --- a/python/sglang/srt/models/glm4_moe_lite.py +++ b/python/sglang/srt/models/glm4_moe_lite.py @@ -820,9 +820,6 @@ class Glm4MoeLiteModel(nn.Module): forward_batch=forward_batch, hidden_states=hidden_states, residual=residual, - input_data_scatter_mode=self.layers[ - normal_end_layer - 1 - ].layer_scatter_modes.layer_output_mode, zero_allocator=zero_allocator, ) diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index 5f1567ea74..c6ae5b304d 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -1183,9 +1183,6 @@ class Glm5NextModel(nn.Module): forward_batch=forward_batch, hidden_states=hidden_states, residual=residual, - input_data_scatter_mode=self.layers[ - normal_end_layer - 1 - ].layer_scatter_modes.layer_output_mode, zero_allocator=zero_allocator, ) diff --git a/python/sglang/srt/models/mimo_v2.py b/python/sglang/srt/models/mimo_v2.py index dcdefbf80b..144a21f18e 100644 --- a/python/sglang/srt/models/mimo_v2.py +++ b/python/sglang/srt/models/mimo_v2.py @@ -35,7 +35,6 @@ from sglang.srt.layers.aux_hidden_states import ( from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, - ScatterMode, enable_moe_dense_fully_dp, ) from sglang.srt.layers.dp_attention import ( @@ -1060,13 +1059,6 @@ class MiMoV2Model(nn.Module): hidden_states, residual = model_forward_maybe_tbo( layers=self.layers[tbo_start_layer:tbo_end_layer], enable_tbo=True, - input_data_scatter_mode=( - ScatterMode.model_input_output() - if tbo_start_layer == self.start_layer - else self.layers[ - tbo_start_layer - 1 - ].layer_scatter_modes.layer_output_mode - ), positions=positions, forward_batch=forward_batch, hidden_states=hidden_states, diff --git a/python/sglang/srt/models/minimax_m2.py b/python/sglang/srt/models/minimax_m2.py index c680ee8439..28c695d6fc 100644 --- a/python/sglang/srt/models/minimax_m2.py +++ b/python/sglang/srt/models/minimax_m2.py @@ -33,7 +33,6 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, - ScatterMode, ) from sglang.srt.layers.dp_attention import ( attn_tp_all_reduce, @@ -1153,7 +1152,6 @@ class MiniMaxM2Model(nn.Module): hidden_states, residual = model_forward_maybe_tbo( layers=self.layers, enable_tbo=True, - input_data_scatter_mode=ScatterMode.model_input_output(), positions=positions, forward_batch=forward_batch, hidden_states=hidden_states, diff --git a/python/sglang/srt/models/minimax_m3.py b/python/sglang/srt/models/minimax_m3.py index 883eb99dd7..d3900dcbbf 100644 --- a/python/sglang/srt/models/minimax_m3.py +++ b/python/sglang/srt/models/minimax_m3.py @@ -38,7 +38,6 @@ from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, - ScatterMode, enable_moe_dense_fully_dp, ) from sglang.srt.layers.dp_attention import ( @@ -1488,7 +1487,6 @@ class MiniMaxM3Model(nn.Module): hidden_states, residual = model_forward_maybe_tbo( layers=self.layers, enable_tbo=True, - input_data_scatter_mode=ScatterMode.model_input_output(), positions=positions, forward_batch=forward_batch, hidden_states=hidden_states, diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index 6e082a9d77..743d15a06f 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -43,7 +43,6 @@ from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, - ScatterMode, ) from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -1184,7 +1183,6 @@ class Qwen2MoeModel(nn.Module): hidden_states, residual = model_forward_maybe_tbo( layers=self.layers, enable_tbo=True, - input_data_scatter_mode=ScatterMode.model_input_output(), positions=positions, forward_batch=forward_batch, hidden_states=hidden_states, diff --git a/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py b/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py index 88dd8cd118..9543a5e52f 100644 --- a/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py +++ b/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py @@ -9,7 +9,8 @@ from unittest.mock import patch import torch from sglang.srt.batch_overlap import two_batch_overlap as tbo -from sglang.srt.layers.communicator import ScatterMode, UnreducedOutput +from sglang.srt.layers.boundary_layout import Layout +from sglang.srt.layers.communicator import UnreducedOutput from sglang.srt.utils import empty_context from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -58,8 +59,7 @@ class TestTboEntryReducesItsInput(CustomTestCase): operations_strategy=SimpleNamespace( deep_gemm_num_sms=None, operations=[], tbo_delta_stages=0 ), - input_data_scatter_mode=ScatterMode.TP_ATTN_FULL, - layer_input_scatter_mode=ScatterMode.TP_ATTN_FULL, + layer_input_rows=Layout(frozenset()), ) self.assertIs(seen["split"], reduced) diff --git a/test/registered/unit/layers/test_declared_attention_cp.py b/test/registered/unit/layers/test_declared_attention_cp.py index 44efc840bd..7327e41fd4 100644 --- a/test/registered/unit/layers/test_declared_attention_cp.py +++ b/test/registered/unit/layers/test_declared_attention_cp.py @@ -126,6 +126,12 @@ class TestAttentionCpBoundary(CustomTestCase): (comm, "post_experts_reduction_group"), lambda: moe_group or parallel.tp_group, ), + ( + (comm, "get_exec"), + lambda: SimpleNamespace( + overlap=SimpleNamespace(enable_two_batch_overlap=False) + ), + ), ( (comm, "get_spec"), lambda: SimpleNamespace(speculative_algorithm=None), @@ -159,6 +165,7 @@ class TestAttentionCpBoundary(CustomTestCase): is_last_layer=False, is_layer_sparse=True, is_previous_layer_sparse=False, + is_next_layer_sparse=False, ), input_layernorm=layernorm, post_attention_layernorm=layernorm, diff --git a/test/registered/unit/layers/test_declared_decoder_boundary.py b/test/registered/unit/layers/test_declared_decoder_boundary.py index 0ddfc8cb34..62d9ba6c72 100644 --- a/test/registered/unit/layers/test_declared_decoder_boundary.py +++ b/test/registered/unit/layers/test_declared_decoder_boundary.py @@ -153,12 +153,15 @@ class FactsOnly: raise AssertionError(f"read scatter mode {name!r}") -def layer_facts(layer_id, num_layers, *, sparse=False, previous_sparse=False): +def layer_facts( + layer_id, num_layers, *, sparse=False, previous_sparse=False, next_sparse=False +): return FactsOnly( is_layer_sparse=sparse, is_first_layer=layer_id == 0, is_last_layer=layer_id == num_layers - 1, is_previous_layer_sparse=None if layer_id == 0 else previous_sparse, + is_next_layer_sparse=next_sparse, ) @@ -504,11 +507,6 @@ class TestWhichLayersUseDeclarations(CustomTestCase): layer_facts(1, 3), parallel_of(attn_dp=2, attn_tp=1, attn_cp=2), ), - # The layer before a sparse one gathers its output for the split. - "dense MLP fully DP under two-batch overlap": ( - layer_facts(1, 3), - parallel_of(attn_dp=2, attn_tp=2, moe_dense_tp_size=1), - ), } two_batch_overlap = SimpleNamespace( overlap=SimpleNamespace(enable_two_batch_overlap=True) @@ -752,24 +750,21 @@ class TestMhcOnTheDeclarations(CustomTestCase): comm.CommunicateSummableTensorPairFn._trivial, ) - def test_two_batch_overlap_keeps_the_scatter_modes(self): - # A dense layer before a sparse one gathers its output for the split, - # so dense layers on every rank keep their scatter-mode steps. + def test_two_batch_overlap_gathers_on_the_declarations(self): + # The dense layer before a sparse one hands the split the attention's + # rows: MHC writes its output into the streams, then gathers them. parallel = parallel_of(attn_dp=2, attn_tp=2, moe_dense_tp_size=1) - modes = planned_modes( - 1, 3, sparse=False, previous_sparse=False, parallel=parallel + communicator = build_mhc( + layer_facts(1, 3, next_sparse=True), parallel, two_batch_overlap=True + ) + self.assert_step( + communicator._steps.ffn_input, comm._mlp_input_scatter, communicator + ) + self.assert_step( + communicator._steps.ffn_output_move, + comm.CommunicateSummableTensorPairFn._gather, + communicator, ) - declared = build_mhc(modes, parallel) - kept = build_mhc(modes, parallel, two_batch_overlap=True) - # Both run the shared slice onto each rank's rows with MHC's residual. - for communicator in (declared, kept): - self.assert_step( - communicator._steps.ffn_input, comm._mlp_input_scatter, communicator - ) - # The declared layer never reads a scatter mode; the kept one does. - build_mhc(layer_facts(1, 3), parallel) - with self.assertRaises(AssertionError): - build_mhc(layer_facts(1, 3), parallel, two_batch_overlap=True) def test_the_postprocess_writes_the_output_into_the_streams(self): # The last layer also contracts the streams into the hidden states. @@ -820,6 +815,110 @@ class TestMhcOnTheDeclarations(CustomTestCase): build_mhc(modes, parallel, dsa_cp=True, allow_reduce_scatter=True) +class TestTwoBatchOverlap(CustomTestCase): + """Two-batch overlap splits the attention's rows. A dense MLP on every rank + hands the sparse layer after it those rows, and the split moves the input + from the rows the first overlapped layer takes.""" + + def layers(self, *, tbo): + parallel = parallel_of(attn_dp=2, attn_tp=2, moe_dense_tp_size=1) + overlap = SimpleNamespace(overlap=SimpleNamespace(enable_two_batch_overlap=tbo)) + layers = [] + with planning(parallel), patch.object(comm, "get_exec", lambda: overlap): + for i, sparse in enumerate((False, False, True, True)): + modes = LayerScatterModes.init_new( + layer_id=i, + num_layers=4, + is_layer_sparse=sparse, + is_previous_layer_sparse=i > 2, + is_next_layer_sparse=i >= 1, + ) + layers.append( + LayerCommunicator( + layer_scatter_modes=modes, + input_layernorm=Norm(), + post_attention_layernorm=Norm(), + ) + ) + return layers + + def test_the_dense_layer_before_the_split_hands_on_the_attention_rows(self): + pair = comm.CommunicateSummableTensorPairFn + attention = comm.Layout(frozenset({TokenAxis.ATTN_DP})) + local = comm.Layout(frozenset({TokenAxis.ATTN_DP, TokenAxis.ATTN_TP_SCATTER})) + for tbo, gathered in ((True, True), (False, False)): + with self.subTest(two_batch_overlap=tbo): + first, before, after, last = self.layers(tbo=tbo) + for layer in (first, before, after, last): + self.assertIsNotNone(layer._declared) + self.assertIs(first._steps.ffn_output_move, pair._trivial) + if gathered: + self.assertIs(before._steps.ffn_output_move.func, pair._gather) + self.assertEqual(after.input_rows, attention) + self.assertIs( + after._steps.attention_input, comm.CommunicateSimpleFn._trivial + ) + else: + self.assertIs(before._steps.ffn_output_move, pair._trivial) + self.assertEqual(after.input_rows, local) + self.assertIs( + after._steps.attention_input, + comm.CommunicateSimpleFn._scattered_to_tp_attn_full, + ) + # The declarations give the rows the scatter modes planned. + for layer in (first, before, after, last): + layouts = layer._context.layouts + modes = layer.layer_scatter_modes + self.assertEqual(layer.input_rows, layouts[modes.layer_input_mode]) + self.assertEqual( + layer._declared.output_rows, layouts[modes.layer_output_mode] + ) + + def test_the_split_moves_from_the_first_layer_s_rows(self): + pair = comm.CommunicateSummableTensorPairFn + parallel = parallel_of(attn_dp=2, attn_tp=2) + dp = frozenset({TokenAxis.ATTN_DP}) + with planning(parallel): + self.assertEqual( + comm.tbo_split_moves(comm.Layout(dp)), (pair._trivial, pair._trivial) + ) + self.assertEqual( + comm.tbo_split_moves(comm.Layout(dp | {TokenAxis.ATTN_TP_SCATTER})), + (pair._gather, pair._scatter), + ) + with self.assertRaises(NotImplementedError): + comm.tbo_split_moves(comm.Layout(frozenset())) + + def test_a_layer_off_the_declarations_takes_its_rows_from_its_modes(self): + dp = parallel_of(attn_dp=2, attn_tp=2) + direct = LayerScatterModes( + layer_input_mode=ScatterMode.SCATTERED, + attn_mode=ScatterMode.TP_ATTN_FULL, + mlp_mode=ScatterMode.FULL, + middle_residual_mode=ScatterMode.TP_ATTN_FULL, + layer_output_mode=ScatterMode.TP_ATTN_FULL, + ) + communicator = build(direct, dp) + self.assertIsNone(communicator._declared) + self.assertEqual( + communicator.input_rows, + comm.Layout(frozenset({TokenAxis.ATTN_DP, TokenAxis.ATTN_TP_SCATTER})), + ) + # Under attention CP the rows are those of a batch that holds every + # token on each CP rank, which the two-batch-overlap split takes. + cp = parallel_of(attn_dp=1, attn_tp=2, attn_cp=2) + communicator = build(direct, cp) + self.assertIsNone(communicator._declared) + slice_ = comm.Layout(frozenset({TokenAxis.ATTN_TP_SCATTER})) + self.assertEqual(communicator.input_rows, slice_) + pair = comm.CommunicateSummableTensorPairFn + with planning(cp): + self.assertEqual( + comm.tbo_split_moves(communicator.input_rows), + (pair._gather, pair._scatter), + ) + + class TestTheAttentionOutputDecidesItsSum(CustomTestCase): """prepare_mlp completes the attention-TP sum exactly when the attention output's declaration says it is owed, whatever the attention-TP size.""" diff --git a/test/registered/unit/layers/test_dp_attention_cp_gather.py b/test/registered/unit/layers/test_dp_attention_cp_gather.py index 6fed5c265f..d41b5f8537 100644 --- a/test/registered/unit/layers/test_dp_attention_cp_gather.py +++ b/test/registered/unit/layers/test_dp_attention_cp_gather.py @@ -245,6 +245,7 @@ class TestDpCpGather(CustomTestCase): is_last_layer=False, is_layer_sparse=False, is_previous_layer_sparse=False, + is_next_layer_sparse=False, ), input_layernorm=norm, post_attention_layernorm=norm,