diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index 95adeb1b83..d547e04787 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -756,11 +756,17 @@ class LayerCommunicator: self._context.force_layernorm_before_dp_gather = ( force_layernorm_before_dp_gather ) + # The steps the layer's ordinary batches run. sides = self._declared_sides() - if sides is not None: - self._select_declared_boundaries(sides) - else: - self._select_boundaries_from_scatter_modes() + self._steps = ( + _select_boundary_steps( + sides, + fusions=self._select_mlp_input_fusions(), + force_layernorm_before_gather=force_layernorm_before_dp_gather, + ) + if sides is not None + else self._select_boundaries_from_scatter_modes() + ) # The steps a batch with input-scattered attention runs; None where it # cannot run or the steps come from the scatter modes. self._input_scattered_steps = ( @@ -846,46 +852,13 @@ class LayerCommunicator: ), ) - def _select_declared_boundaries(self, sides: DecoderLayerSides) -> None: - """Choose the three boundary sides this layer owns from the - declarations: the attention input, the attention -> FFN steps, and the - FFN output's way on to the rows the layer hands on.""" - self._communicate_simple_fn = _select_attention_input_move( - sides.input_rows, sides.attention - ) - self._mlp_input, fused = _select_ffn_input( - sides.attention_output, - residual=sides.input_rows, - residual_to=sides.ffn_residual_rows, - need=sides.ffn, - force_layernorm_before_gather=self.force_layernorm_before_dp_gather, - fusions=self._select_mlp_input_fusions(), - ) - self._mlp_input_may_return_new_residual = any( - f.may_return_new_residual for f in fused - ) - self._ffn_output = sides.ffn_output - ( - self._postprocess_scatters_to_local_tokens, - postprocess, - ) = _select_ffn_output_move( - sides.ffn_output, residual=sides.ffn_residual_rows, to=sides.output_rows - ) - if postprocess is not None: - self._communicate_summable_tensor_pair_fn = postprocess - self._ffn_sum_is_movable = sides.ffn_output.group is not None - - def _select_boundaries_from_scatter_modes(self) -> None: - self._post_init_communicate() - # Under attention DP, the base postprocess scatters the FFN output back - # to this rank's tokens, which the next layer's input can run instead; - # the MHC and DSA-CP postprocess do more, so theirs stays here. - self._postprocess_scatters_to_local_tokens = ( - self._communicate_summable_tensor_pair_fn - is CommunicateSummableTensorPairFn._scatter_hidden_states - ) + def _select_boundaries_from_scatter_modes(self) -> "BoundarySteps": + """The layer's steps chosen from its scatter modes, for the layers the + declarations do not cover yet.""" + attention_input, postprocess = self._post_init_communicate() + ffn_input, fused = self._select_mlp_input() modes = self.layer_scatter_modes - self._ffn_output = StageOutput( + ffn_output = StageOutput( self._context.layouts[modes.mlp_mode], group=SumGroup.MOE_OUTPUT if modes.is_layer_sparse else SumGroup.TP, leaves_for_next_layer=self.allow_deferred_ffn_reduction, @@ -897,30 +870,41 @@ class LayerCommunicator: self.allow_reduce_scatter or modes.is_layer_sparse ), ) - # Whether the FFN sum is one the next layer's input can take: not when - # the way back is the MoE-CP scatter, nor when a SCATTERED FFN computes - # whole tokens with nothing left to sum. - self._ffn_sum_is_movable = modes.mlp_mode not in ( - ScatterMode.MOE_FULL, - ScatterMode.SCATTERED, - ) - self._mlp_input, self._mlp_input_may_return_new_residual = ( - self._select_mlp_input() + return BoundarySteps( + attention_input=attention_input, + ffn_input=ffn_input, + ffn_output=ffn_output, + # Under attention DP the base postprocess scatters the FFN output + # back to this rank's tokens, which the next layer's input can run + # instead; the MHC and DSA-CP postprocess do more. + ffn_output_move=( + None + if postprocess is CommunicateSummableTensorPairFn._scatter_hidden_states + else postprocess + ), + # Not when the way back is the MoE-CP scatter, nor when a SCATTERED + # FFN computes whole tokens with nothing left to sum. + ffn_sum_is_movable=modes.mlp_mode + not in (ScatterMode.MOE_FULL, ScatterMode.SCATTERED), + fused=fused, + layer_input=_complete_scattered_input, ) - def _post_init_communicate(self): - self._communicate_simple_fn = CommunicateSimpleFn.get_fn( - input_mode=self.layer_scatter_modes.layer_input_mode, - output_mode=self.layer_scatter_modes.attn_mode, - context=self._context, - ) - self._communicate_summable_tensor_pair_fn = ( + def _post_init_communicate(self) -> Tuple[Callable, Callable]: + """The attention input move and the postprocess the scatter modes + choose.""" + return ( + CommunicateSimpleFn.get_fn( + input_mode=self.layer_scatter_modes.layer_input_mode, + output_mode=self.layer_scatter_modes.attn_mode, + context=self._context, + ), CommunicateSummableTensorPairFn.get_fn( hidden_states_input_mode=self.layer_scatter_modes.mlp_mode, residual_input_mode=self.layer_scatter_modes.middle_residual_mode, output_mode=self.layer_scatter_modes.layer_output_mode, context=self._context, - ) + ), ) def prepare_attn_and_capture_last_layer_outputs( @@ -940,7 +924,7 @@ class LayerCommunicator: post_residual_addition=post_residual_addition, ) if captured_last_layer_outputs is not None: - gathered_last_layer_output = self._communicate_simple_fn( + gathered_last_layer_output = self._batch_steps().attention_input( hidden_states=residual, forward_batch=forward_batch, context=self._context, @@ -966,7 +950,7 @@ class LayerCommunicator: the batch is not input-scattered. """ return ( - self._mlp_input_may_return_new_residual + any(f.may_return_new_residual for f in self._batch_steps().fused) and not get_attn_tp_context().input_scattered and apply_flashinfer_allreduce_fusion(residual.shape[0]) ) @@ -1028,16 +1012,11 @@ class LayerCommunicator: ) if get_forward().sp_active: hidden_states = layernorm_sp.sp_entry_scatter(hidden_states) - steps = self._batch_steps() - if steps is not None and steps.layer_input is not None: - hidden_states, residual = steps.layer_input( + layer_input = self._batch_steps().layer_input + if layer_input is not None: + hidden_states, residual = layer_input( hidden_states, residual, self._context ) - elif get_attn_tp_context().input_scattered: - hidden_states, residual = self._tp_reduce_scatter( - hidden_states, - residual, - ) if hidden_states.shape[0] == 0: residual = hidden_states elif owed is not None: @@ -1059,9 +1038,9 @@ class LayerCommunicator: def _in_sp_region(self) -> bool: return self._sp_steps is not None and get_forward().sp_active - def _batch_steps(self) -> Optional["BoundarySteps"]: - """The steps this batch runs in place of the layer's ordinary ones: an - active LayerNorm SP region's, or input-scattered attention's.""" + def _batch_steps(self) -> "BoundarySteps": + """The steps this batch runs: an active LayerNorm SP region's, + input-scattered attention's, or the layer's ordinary ones.""" if self._in_sp_region(): return self._sp_steps if ( @@ -1069,13 +1048,11 @@ class LayerCommunicator: and get_attn_tp_context().input_scattered ): return self._input_scattered_steps - return None + return self._steps def _finish_prepare_attn(self, hidden_states, residual, forward_batch): """Tail every prepare_attn path must run, or ``attn_inputs`` is unset.""" - steps = self._batch_steps() - move = self._communicate_simple_fn if steps is None else steps.attention_input - hidden_states = move( + hidden_states = self._batch_steps().attention_input( hidden_states=hidden_states, forward_batch=forward_batch, context=self._context, @@ -1145,30 +1122,23 @@ class LayerCommunicator: hidden_states, residual, use_attn_tp_group=False ) - def _tp_reduce_scatter( - self, - hidden_states: torch.Tensor, - residual: torch.Tensor, - ) -> Tuple[torch.Tensor, torch.Tensor]: - return tp_reduce_scatter(hidden_states, residual, self._context) - - def _select_mlp_input(self) -> Tuple[Callable, bool]: + def _select_mlp_input(self) -> Tuple[Callable, Tuple["FusedMlpInput", ...]]: """The attention-TP -> FFN boundary's steps, chosen from the layouts, and - whether they include the aiter / flashinfer fused entry. The MHC and - DSA-CP communicators pick their own implementations.""" + the fused kernels they try first. The MHC and DSA-CP communicators pick + their own implementations.""" kind = mlp_input_kind(self.layer_scatter_modes, self._context) residual_input_mode = self.layer_scatter_modes.layer_input_mode if kind is MlpInputKind.NORM: - return _mlp_input_norm, False + return _mlp_input_norm, () if kind is MlpInputKind.ATTN_TP_ALL_REDUCE: - return _mlp_input_attn_tp_all_reduce, False + return _mlp_input_attn_tp_all_reduce, () if kind is MlpInputKind.SCATTER: return ( partial( _mlp_input_scatter, scatters_residual=residual_input_mode == ScatterMode.TP_ATTN_FULL, ), - False, + (), ) # Neither fused kernel runs under attention DP. fusions = ( @@ -1182,7 +1152,7 @@ class LayerCommunicator: ) if kind is MlpInputKind.GATHER_MOE_CP: steps = partial(_mlp_input_gather_moe_cp, gather=steps) - return steps, any(f.may_return_new_residual for f in fusions) + return steps, fusions def _select_mlp_input_fusions(self) -> Tuple["FusedMlpInput", ...]: """The fused kernels that can take the attention -> FFN steps, in the @@ -1222,20 +1192,10 @@ class LayerCommunicator: cache=None, ): self.publish_mlp_lora_layout() - if self._in_sp_region(): - return self._sp_steps.ffn_input( - hidden_states, - residual, - forward_batch, - self.post_attention_layernorm, - self._context, - ) if cache is not None: self._context.cache = cache - steps = self._batch_steps() - mlp_input = self._mlp_input if steps is None else steps.ffn_input - return mlp_input( + return self._batch_steps().ffn_input( hidden_states, residual, forward_batch, @@ -1257,17 +1217,10 @@ class LayerCommunicator: forward_batch: ForwardBatch, ): steps = self._batch_steps() - if steps is not None: - return steps.ffn_output_move( - hidden_states=hidden_states, - residual=residual, - forward_batch=forward_batch, - context=self._context, - ) - if self._postprocess_scatters_to_local_tokens: + if steps.returns_over_dp: step = self._postprocess_dp_step(forward_batch) or _redistribute_output return _to_local_tokens(step, forward_batch, hidden_states), residual - return self._communicate_summable_tensor_pair_fn( + return steps.ffn_output_move( hidden_states=hidden_states, residual=residual, forward_batch=forward_batch, @@ -1278,9 +1231,8 @@ class LayerCommunicator: def _local_token_move_can_go_to_next_layer(self) -> bool: """Whether the next layer's input can run this layer's move of its FFN - output back to this rank's tokens: the base postprocess scatter, outside - an active LayerNorm SP region.""" - return self._postprocess_scatters_to_local_tokens and not self._in_sp_region() + output back to this rank's tokens: the base postprocess scatter.""" + return self._batch_steps().returns_over_dp def _postprocess_dp_step( self, forward_batch: ForwardBatch @@ -1288,12 +1240,13 @@ class LayerCommunicator: """The reduce-scatter that brings this layer's FFN output back to this rank's tokens under attention DP; None when the base postprocess would only scatter, or does not move tokens.""" - if not self._postprocess_scatters_to_local_tokens: + steps = self._batch_steps() + if not steps.returns_over_dp: return None return _reduce_and_redistribute_output_step( forward_batch, - leaves_for_reduce_scatter=self._ffn_output.leaves_for_reduce_scatter, - leaves_for_reduce_scatterv=self._ffn_output.leaves_for_reduce_scatterv, + leaves_for_reduce_scatter=steps.ffn_output.leaves_for_reduce_scatter, + leaves_for_reduce_scatterv=steps.ffn_output.leaves_for_reduce_scatterv, ) def _ffn_leaves_sum_to_reduce_scatter( @@ -1302,10 +1255,7 @@ class LayerCommunicator: """Whether the FFN leaves its sum out because a reduce-scatter completes it: the attention-DP one ``dp_step`` names, or the CP / input-scattered one.""" - steps = self._batch_steps() - if steps is not None: - return steps.ffn_output.leaves_for_reduce_scatter - if not self._ffn_output.leaves_for_reduce_scatter: + if not self._batch_steps().ffn_output.leaves_for_reduce_scatter: return False if dp_step is not None: return True @@ -1319,7 +1269,7 @@ class LayerCommunicator: def ffn_reduction_group(self) -> GroupCoordinator: """The group this layer's FFN output owes its sum over: the MoE output's group on a sparse layer, the TP group a dense MLP reduces over.""" - return _sum_group(self._ffn_output.group) + return _sum_group(self._batch_steps().ffn_output.group) def _select_ffn_completion(self, forward_batch: ForwardBatch) -> "FfnCompletion": """Decide once, before the FFN runs, what it skips and what completes its @@ -1334,8 +1284,7 @@ class LayerCommunicator: dp_step=dp_step, ) steps = self._batch_steps() - ffn_output = self._ffn_output if steps is None else steps.ffn_output - if not ffn_output.leaves_for_next_layer: + if not steps.ffn_output.leaves_for_next_layer: return FfnCompletion( defer_moe_finalize=False, fuse_mlp_allreduce=False, @@ -1349,7 +1298,7 @@ class LayerCommunicator: ) if fuse_mlp_allreduce: group = self.ffn_reduction_group() - if self._postprocess_scatters_to_local_tokens: + if steps.returns_over_dp: # Under attention DP the next layer also brings the sum back to # this rank's tokens. wrap = partial( @@ -1435,7 +1384,7 @@ class LayerCommunicator: def _ffn_sum_can_move_to_next_layer(self) -> bool: # Under the MoE-CP all-gather the fusion path would skip postprocess_layer # and its MoE-CP scatter, leaving hidden_states longer than the residual. - if is_enable_moe_cp_allgather() or not self._ffn_sum_is_movable: + if is_enable_moe_cp_allgather() or not self._batch_steps().ffn_sum_is_movable: return False # The fused residual+LN reduces over a single group. Hybrid EP+TP spans @@ -1914,29 +1863,40 @@ def _token_axis_sizes() -> Dict[TokenAxis, int]: class BoundarySteps(msgspec.Struct, frozen=True): - """The steps one set of declarations chooses for a layer's boundaries: - into the attention, from the attention output to the FFN input, and the - FFN output on to the rows the layer hands on.""" + """The steps a batch runs at a layer's boundaries: into the attention, + from the attention output to the FFN input, and the FFN output on to the + rows the layer hands on.""" attention_input: Callable ffn_input: Callable - ffn_output_move: Callable - # What the FFN exit reads in place of the layer's own FFN output declaration. + # What the FFN exit reads: the FFN output's group and what it may leave. ffn_output: StageOutput + # The postprocess that moves the FFN output on; None when it goes back over + # attention DP, whose step the FFN exit and postprocess choose per batch. + ffn_output_move: Optional[Callable] + # Whether the next layer's input can take the FFN's sum. + ffn_sum_is_movable: bool + # The fused kernels ffn_input tries first. + fused: Tuple["FusedMlpInput", ...] = () # Completes what the layer's input owes before the input norm; None when it # owes nothing. layer_input: Optional[Callable] = None + @property + def returns_over_dp(self) -> bool: + return self.ffn_output_move is None -def _select_boundary_steps(sides: DecoderLayerSides) -> BoundarySteps: - """The steps for a batch's own declarations: nothing moves back over - attention DP and no fused kernel runs, so the steps do not wait on a - per-batch decision.""" + +def _select_boundary_steps( + sides: DecoderLayerSides, + *, + fusions: Tuple["FusedMlpInput", ...] = (), + force_layernorm_before_gather: bool = False, +) -> BoundarySteps: + """The steps a set of declarations chooses.""" returns_over_dp, ffn_output_move = _select_ffn_output_move( sides.ffn_output, residual=sides.ffn_residual_rows, to=sides.output_rows ) - if returns_over_dp or sides.ffn_output.leaves_for_next_layer: - raise NotImplementedError(f"{sides=}") rows = sides.input_rows layer_input = None if sides.input_owes is not None: @@ -1950,24 +1910,39 @@ def _select_boundary_steps(sides: DecoderLayerSides) -> BoundarySteps: raise NotImplementedError(f"{sides=}") layer_input = tp_reduce_scatter rows = Layout(frozenset({TokenAxis.ATTN_TP_SCATTER})) - ffn_input, _ = _select_ffn_input( + ffn_input, fused = _select_ffn_input( sides.attention_output, residual=rows, residual_to=sides.ffn_residual_rows, need=sides.ffn, - force_layernorm_before_gather=False, - fusions=(), + force_layernorm_before_gather=force_layernorm_before_gather, + fusions=fusions, residual_joins_sum=sides.residual_joins_attention_sum, ) return BoundarySteps( attention_input=_select_attention_input_move(rows, sides.attention), ffn_input=ffn_input, - ffn_output_move=ffn_output_move, ffn_output=sides.ffn_output, + ffn_output_move=None if returns_over_dp else ffn_output_move, + ffn_sum_is_movable=sides.ffn_output.group is not None, + fused=fused, layer_input=layer_input, ) +def _complete_scattered_input( + hidden_states: torch.Tensor, + residual: Optional[torch.Tensor], + context: "CommunicateContext", +) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """The scatter-mode path's input completion: with input-scattered + attention the layer's input is a TP partial that a reduce-scatter completes + onto this rank's slice.""" + if get_attn_tp_context().input_scattered: + return tp_reduce_scatter(hidden_states, residual, context) + return hidden_states, residual + + def _select_attention_input_move(rows: Layout, need: StageInput) -> Callable: """How the rows a layer takes become its attention's input: as they are, or gathered over attention TP from each rank's slice, unless the attention diff --git a/python/sglang/srt/layers/communicator_dsa_cp.py b/python/sglang/srt/layers/communicator_dsa_cp.py index aad1f67778..7a0dfd0600 100644 --- a/python/sglang/srt/layers/communicator_dsa_cp.py +++ b/python/sglang/srt/layers/communicator_dsa_cp.py @@ -93,16 +93,18 @@ class DSACPLayerCommunicator(LayerCommunicator): assert self._context.attn_dp_size == 1, ( f"dp_size should be 1 when moe_runner_backend is none" ) - self._communicate_simple_fn = DSACPCommunicateSimpleFn.get_fn( - input_mode=ScatterMode.SCATTERED, - output_mode=ScatterMode.SCATTERED, - context=self._context, - ) - self._communicate_summable_tensor_pair_fn = DSACPCommunicateSummableTensorPairFn.get_fn( - hidden_states_input_mode=self.layer_scatter_modes.mlp_mode, # SCATTERED, FULL - residual_input_mode=ScatterMode.SCATTERED, - output_mode=ScatterMode.SCATTERED, - context=self._context, + return ( + DSACPCommunicateSimpleFn.get_fn( + input_mode=ScatterMode.SCATTERED, + output_mode=ScatterMode.SCATTERED, + context=self._context, + ), + DSACPCommunicateSummableTensorPairFn.get_fn( + hidden_states_input_mode=self.layer_scatter_modes.mlp_mode, # SCATTERED, FULL + residual_input_mode=ScatterMode.SCATTERED, + output_mode=ScatterMode.SCATTERED, + context=self._context, + ), ) def _select_mlp_input(self): @@ -113,7 +115,7 @@ class DSACPLayerCommunicator(LayerCommunicator): residual_output_mode=ScatterMode.SCATTERED, context=self._context, ) - return fn, False + return fn, () class DSACPCommunicateSimpleFn(CommunicateSimpleFn): diff --git a/python/sglang/srt/layers/communicator_mhc.py b/python/sglang/srt/layers/communicator_mhc.py index ec37e36372..f676c6a02d 100644 --- a/python/sglang/srt/layers/communicator_mhc.py +++ b/python/sglang/srt/layers/communicator_mhc.py @@ -441,6 +441,8 @@ class MHCLayerCommunicator(LayerCommunicator): context=self._context, ) ) + # MHC's own prepare_attn and postprocess run these. + return self._communicate_simple_fn, self._communicate_summable_tensor_pair_fn def prepare_attn( self, @@ -494,14 +496,14 @@ class MHCLayerCommunicator(LayerCommunicator): residual_input_mode = self.layer_scatter_modes.layer_input_mode fns = MHCCommunicateWithAllReduceAndLayerNormFn if kind is MlpInputKind.NORM: - return partial(fns._simple, mhc=self.mhc), False + return partial(fns._simple, mhc=self.mhc), () if kind is MlpInputKind.GATHER: fn = fns._gather_hidden_states_and_residual elif kind is MlpInputKind.SCATTER: fn = fns._scatter_hidden_states_and_residual else: raise NotImplementedError(f"MHCLayerCommunicator does not support {kind}") - return partial(fn, residual_input_mode=residual_input_mode, mhc=self.mhc), False + return partial(fn, residual_input_mode=residual_input_mode, mhc=self.mhc), () def postprocess_layer(self, hidden_states, residual, forward_batch): hidden_states, residual = self._communicate_summable_tensor_pair_fn( diff --git a/test/registered/unit/layers/test_communicator_ffn_exit.py b/test/registered/unit/layers/test_communicator_ffn_exit.py index 06ce59cf88..dd1a0add9d 100644 --- a/test/registered/unit/layers/test_communicator_ffn_exit.py +++ b/test/registered/unit/layers/test_communicator_ffn_exit.py @@ -45,6 +45,20 @@ def make_group(scale=3): return types.SimpleNamespace(all_reduce=MagicMock(side_effect=lambda h: h * scale)) +def ordinary_steps(ffn_output, *, returns_over_dp=False): + """A layer's ordinary steps, carrying the FFN output declaration the exit + reads and whether the output goes back over attention DP.""" + return comm.BoundarySteps( + attention_input=comm.CommunicateSimpleFn._trivial, + ffn_input=comm._mlp_input_norm, + ffn_output=ffn_output, + ffn_output_move=( + None if returns_over_dp else comm.CommunicateSummableTensorPairFn._trivial + ), + ffn_sum_is_movable=True, + ) + + def make_communicator( *, fuse, @@ -58,8 +72,13 @@ def make_communicator( """A communicator whose decisions and postprocess are stubbed, built without the process-wide parallel state.""" communicator = cls.__new__(cls) - communicator._ffn_output = StageOutput( - Layout(frozenset()), group=SumGroup.TP, leaves_for_next_layer=allow_deferred + communicator._steps = ordinary_steps( + StageOutput( + Layout(frozenset()), + group=SumGroup.TP, + leaves_for_next_layer=allow_deferred, + ), + returns_over_dp=scatters_to_local_tokens, ) if cls is LayerCommunicator: communicator._ffn_sum_moves_to_next_layer = MagicMock(return_value=fuse) @@ -72,7 +91,6 @@ def make_communicator( communicator.is_last_layer = False communicator._sp_steps = None communicator._input_scattered_steps = None - communicator._postprocess_scatters_to_local_tokens = scatters_to_local_tokens communicator._postprocess_dp_step = MagicMock(return_value=reduce_scatter_step) communicator.ffn_reduction_group = MagicMock(return_value=group or make_group()) communicator.postprocess_layer = MagicMock( @@ -306,13 +324,15 @@ class TestSelectFfnCompletion(CustomTestCase): communicator.is_last_layer = is_last_layer communicator._sp_steps = sp_region_steps() if sp_region else None communicator._input_scattered_steps = None - communicator._postprocess_scatters_to_local_tokens = scatters - communicator._ffn_output = StageOutput( - Layout(frozenset()), - group=SumGroup.MOE_OUTPUT, - leaves_for_next_layer=True, - leaves_for_reduce_scatter=True, - leaves_for_reduce_scatterv=True, + communicator._steps = ordinary_steps( + StageOutput( + Layout(frozenset()), + group=SumGroup.MOE_OUTPUT, + leaves_for_next_layer=True, + leaves_for_reduce_scatter=True, + leaves_for_reduce_scatterv=True, + ), + returns_over_dp=scatters, ) communicator._ffn_sum_moves_to_next_layer = lambda forward_batch, **_: fuse communicator._ffn_leaves_sum_to_reduce_scatter = lambda forward_batch, dp_step: ( diff --git a/test/registered/unit/layers/test_communicator_layout.py b/test/registered/unit/layers/test_communicator_layout.py index 5edb1ac3ea..c1e7c55556 100644 --- a/test/registered/unit/layers/test_communicator_layout.py +++ b/test/registered/unit/layers/test_communicator_layout.py @@ -223,7 +223,7 @@ class TestMlpInputOrder(CustomTestCase): def test_a_gather_runs_the_order_chosen_at_construction(self): for mlp_mode in (FULL, MOE_FULL): with self.subTest(mlp_mode=mlp_mode): - c, (steps, may_fuse) = communicator( + c, (steps, fused) = communicator( (TP_ATTN_FULL, TP_ATTN_FULL, mlp_mode, TP_ATTN_FULL), make_context(tp=2), Fusable(), @@ -239,7 +239,7 @@ class TestMlpInputOrder(CustomTestCase): gathers_residual=False, fusions=(entry,), ) - self.assertTrue(may_fuse) + self.assertEqual([f.run for f in fused], [entry]) def test_no_fused_entry_without_a_fusable_norm_or_under_attention_dp(self): for context, norm in ( @@ -247,10 +247,10 @@ class TestMlpInputOrder(CustomTestCase): (make_context(dp=2, tp=2), Fusable()), ): with self.subTest(dp=context.attn_dp_size): - _, (steps, may_fuse) = communicator(GATHER_LAYOUT, context, norm) + _, (steps, fused) = communicator(GATHER_LAYOUT, context, norm) order = steps.keywords["order"] self.assertEqual(order.keywords.get("fusions", ()), ()) - self.assertFalse(may_fuse) + self.assertEqual(fused, ()) def test_other_kinds_take_their_steps(self): tp = make_context(tp=2) @@ -267,9 +267,9 @@ class TestMlpInputOrder(CustomTestCase): ), ): with self.subTest(expected=expected.__name__): - _, (steps, may_fuse) = communicator(layout, context, Fusable()) + _, (steps, fused) = communicator(layout, context, Fusable()) self.assertIs(steps, expected) - self.assertFalse(may_fuse) + self.assertEqual(fused, ()) _, (steps, _) = communicator( (TP_ATTN_FULL, TP_ATTN_FULL, SCATTERED, SCATTERED), tp, Fusable() ) @@ -297,11 +297,11 @@ class TestMlpInputOrder(CustomTestCase): c.layer_scatter_modes = modes(*layout) c._context = make_context(tp=2) c.post_attention_layernorm = Fusable() - steps, may_fuse = c._select_mlp_input() + steps, fused = c._select_mlp_input() self.assert_order( steps, func, residual_input_mode=TP_ATTN_FULL, mhc=mhc ) - self.assertFalse(may_fuse) + self.assertEqual(fused, ()) if __name__ == "__main__": diff --git a/test/registered/unit/layers/test_declared_decoder_boundary.py b/test/registered/unit/layers/test_declared_decoder_boundary.py index 87e74214e4..6b05969745 100644 --- a/test/registered/unit/layers/test_declared_decoder_boundary.py +++ b/test/registered/unit/layers/test_declared_decoder_boundary.py @@ -274,7 +274,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase): declarations, or none do.""" def declared(self, communicator): - return getattr(communicator._mlp_input, "func", None) in ( + return getattr(communicator._steps.ffn_input, "func", None) in ( comm._mlp_input_dp_partial, comm._mlp_input_dp_replicate, comm._mlp_input_without_dp, @@ -288,10 +288,10 @@ class TestWhichLayersUseDeclarations(CustomTestCase): communicator = build(layer_facts(layer_id, 3), parallel) self.assertTrue(self.declared(communicator)) self.assertIs( - communicator._communicate_simple_fn, + communicator._steps.attention_input, comm.CommunicateSimpleFn._trivial, ) - self.assertTrue(communicator._postprocess_scatters_to_local_tokens) + self.assertTrue(communicator._steps.returns_over_dp) def test_plain_tp(self): for norm, fuses in ((Norm, False), (FusableNorm, True)): @@ -302,18 +302,23 @@ class TestWhichLayersUseDeclarations(CustomTestCase): input_layernorm=norm(), post_attention_layernorm=norm(), ) - self.assertIs(communicator._mlp_input.func, comm._mlp_input_without_dp) + self.assertIs( + communicator._steps.ffn_input.func, comm._mlp_input_without_dp + ) entry = ( communicator._mlp_input_reduce_output_and_update_and_read_residual ) self.assertEqual( - communicator._mlp_input.keywords["fusions"], + communicator._steps.ffn_input.keywords["fusions"], (entry,) if fuses else (), ) - self.assertIs(communicator._mlp_input_may_return_new_residual, fuses) - self.assertFalse(communicator._postprocess_scatters_to_local_tokens) self.assertIs( - communicator._communicate_summable_tensor_pair_fn, + any(f.may_return_new_residual for f in communicator._steps.fused), + fuses, + ) + self.assertFalse(communicator._steps.returns_over_dp) + self.assertIs( + communicator._steps.ffn_output_move, comm.CommunicateSummableTensorPairFn._trivial, ) # One rank: the attention output is complete, only the norm is left. @@ -321,7 +326,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase): single = build(layer_facts(1, 3), one_rank) with planning(one_rank): self.assertIsNotNone(single._declared_sides()) - self.assertIs(single._mlp_input, comm._mlp_input_norm) + self.assertIs(single._steps.ffn_input, comm._mlp_input_norm) def test_the_order_follows_the_attention_output(self): for attn_tp, force, order in ( @@ -335,7 +340,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase): parallel_of(attn_dp=2, attn_tp=attn_tp), force_layernorm_before_dp_gather=force, ) - self.assertIs(communicator._mlp_input.func, order) + self.assertIs(communicator._steps.ffn_input.func, order) def test_a_dense_first_layer_before_sparse_ones(self): parallel = parallel_of(attn_dp=2, attn_tp=2) @@ -361,30 +366,32 @@ class TestWhichLayersUseDeclarations(CustomTestCase): ] self.assertEqual([self.declared(layer) for layer in layers], [True] * 4) if not a2a: - moe = layers[1]._ffn_output + moe = layers[1]._steps.ffn_output self.assertIs(moe.group, SumGroup.MOE_OUTPUT) self.assertTrue(moe.leaves_for_reduce_scatterv) continue # The first a2a layer slices the residual it takes from a # dense layer; the next one takes it already sliced. self.assertEqual( - [layers[i]._mlp_input.func for i in (1, 2)], + [layers[i]._steps.ffn_input.func for i in (1, 2)], [comm._mlp_input_scatter] * 2, ) self.assertEqual( [ - layers[i]._mlp_input.keywords["scatters_residual"] + layers[i]._steps.ffn_input.keywords["scatters_residual"] for i in (1, 2) ], [True, False], ) - self.assertIsNone(layers[1]._ffn_output.group) + self.assertIsNone(layers[1]._steps.ffn_output.group) self.assertIs( - layers[2]._communicate_simple_fn, + layers[2]._steps.attention_input, comm.CommunicateSimpleFn._scattered_to_tp_attn_full, ) - self.assertIs(layers[3]._mlp_input.func, comm._mlp_input_dp_partial) - self.assertTrue(layers[3]._mlp_input.keywords["gathers_residual"]) + self.assertIs( + layers[3]._steps.ffn_input.func, comm._mlp_input_dp_partial + ) + self.assertTrue(layers[3]._steps.ffn_input.keywords["gathers_residual"]) def test_the_last_a2a_layer_folds_the_residual_back(self): parallel = parallel_of(attn_dp=2, attn_tp=2) @@ -396,7 +403,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase): a2a=True, ) self.assertIs( - last._communicate_summable_tensor_pair_fn, + last._steps.ffn_output_move, comm.CommunicateSummableTensorPairFn._gather, ) @@ -407,7 +414,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase): ) with patch.object(comm, "_use_ag_after_qlora", True): layer = build(modes, parallel, a2a=True) - self.assertIs(layer._communicate_simple_fn, comm.CommunicateSimpleFn._trivial) + self.assertIs(layer._steps.attention_input, comm.CommunicateSimpleFn._trivial) def test_layers_that_keep_the_scatter_modes(self): dp = parallel_of(attn_dp=2, attn_tp=2) @@ -611,7 +618,10 @@ class TestFusedKernelsTakeOnlyTheStepsTheyComplete(CustomTestCase): input_layernorm=Norm(), post_attention_layernorm=Norm(), ) - self.assertIs(communicator._mlp_input_may_return_new_residual, expected) + self.assertIs( + any(f.may_return_new_residual for f in communicator._steps.fused), + expected, + ) class TestTheSequenceParallelRegion(CustomTestCase): @@ -624,8 +634,9 @@ class TestTheSequenceParallelRegion(CustomTestCase): SP_STEPS = comm.BoundarySteps( attention_input=comm.CommunicateSimpleFn._trivial, ffn_input=comm._mlp_input_norm, - ffn_output_move=comm.CommunicateSummableTensorPairFn._trivial, ffn_output=sequence_parallel_layer_sides(axis_sizes=SIZES).ffn_output, + ffn_output_move=comm.CommunicateSummableTensorPairFn._trivial, + ffn_sum_is_movable=False, ) def test_the_region_declarations_choose_local_steps(self): @@ -666,7 +677,31 @@ class TestTheSequenceParallelRegion(CustomTestCase): post_attention_layernorm=Norm(), ) self.assertEqual(communicator._sp_steps, self.SP_STEPS) - self.assertIs(communicator._mlp_input.func, comm._mlp_input_without_dp) + self.assertIs( + communicator._steps.ffn_input.func, comm._mlp_input_without_dp + ) + + def test_what_the_ffn_exit_reads_comes_from_the_batch_s_steps(self): + # Inside the region the FFN output is complete: no sum to move. + parallel = parallel_of(attn_dp=1, attn_tp=2) + communicator = build(layer_facts(1, 3), parallel, sp=True) + for active in (False, True): + with ( + self.subTest(sp_active=active), + planning(parallel, sp=True), + patch.object( + comm, "get_forward", lambda: SimpleNamespace(sp_active=active) + ), + patch.object( + comm, + "get_attn_tp_context", + lambda: SimpleNamespace(input_scattered=False), + ), + patch.object(comm, "is_dp_attention_enabled", lambda: False), + ): + self.assertIs( + communicator._ffn_sum_can_move_to_next_layer(), not active + ) def test_without_sp_there_is_no_region(self): with planning(parallel_of(attn_dp=1, attn_tp=2)): @@ -677,15 +712,19 @@ class TestTheSequenceParallelRegion(CustomTestCase): ) self.assertIsNone(communicator._sp_steps) - def test_declarations_left_to_the_ffn_exit_are_not_batch_steps(self): - # A sum the FFN exit may hand to the next layer, or a move back over - # attention DP, waits on a per-batch decision. - for sizes, leaves_next in ( - (self.SIZES, True), - ({**self.SIZES, TokenAxis.ATTN_DP: 2}, False), - ): - with self.subTest(sizes=sizes), self.assertRaises(NotImplementedError): - comm._select_boundary_steps(sides_of(sizes, leaves_next=leaves_next)) + def test_the_same_selector_chooses_a_layer_s_ordinary_steps(self): + # Under attention DP the FFN exit chooses the move back per batch. + kept = comm._select_boundary_steps(sides_of(self.SIZES, leaves_next=True)) + self.assertTrue(kept.ffn_output.leaves_for_next_layer) + self.assertTrue(kept.ffn_sum_is_movable) + self.assertIs( + kept.ffn_output_move, comm.CommunicateSummableTensorPairFn._trivial + ) + returned = comm._select_boundary_steps( + sides_of({**self.SIZES, TokenAxis.ATTN_DP: 2}) + ) + self.assertTrue(returned.returns_over_dp) + self.assertIsNone(returned.ffn_output_move) class TestInputScatteredAttention(CustomTestCase): @@ -780,9 +819,40 @@ class TestInputScatteredAttention(CustomTestCase): ): self.assertIs( communicator._batch_steps(), - communicator._input_scattered_steps if scattered else None, + communicator._input_scattered_steps + if scattered + else communicator._steps, ) + def test_prepare_attn_completes_the_scattered_input(self): + scattered = [] + group = SimpleNamespace( + name="tp", + reduce_scatter_tensor=lambda out, x: ( + scattered.append(x.shape[0]) or out.copy_(x[: out.shape[0]] * 2) + ), + ) + parallel = parallel_of( + attn_dp=1, attn_tp=2, enable_attn_tp_input_scattered=True, tp_group=group + ) + communicator = build(layer_facts(1, 3), parallel, allow_reduce_scatter=True) + with ( + patch.object(comm, "get_parallel", lambda: parallel), + patch.object( + comm, + "get_attn_tp_context", + lambda: SimpleNamespace(input_scattered=True), + ), + patch.object(comm, "get_forward", lambda: SimpleNamespace(sp_active=False)), + ): + hidden, residual = communicator.prepare_attn( + torch.ones(4, HIDDEN), torch.full((4, HIDDEN), 3.0), None + ) + self.assertEqual(scattered, [4]) + # Norm: (2 * (h + r), h + r) on the slice, with h the completed sum. + torch.testing.assert_close(residual, torch.full((2, HIDDEN), 5.0)) + torch.testing.assert_close(hidden, torch.full((2, HIDDEN), 10.0)) + def test_the_last_layer_completes_its_own_sum(self): parallel = parallel_of( attn_dp=1, attn_tp=2, enable_attn_tp_input_scattered=True @@ -800,6 +870,64 @@ class TestInputScatteredAttention(CustomTestCase): self.assertIs(steps.ffn_output.leaves_for_reduce_scatter, hands_on) +class TestOneRepresentation(CustomTestCase): + """Every layer runs from one BoundarySteps per batch, whichever entry chose + its steps: none keeps the steps as separate attributes.""" + + SEPARATE = ( + "_communicate_simple_fn", + "_mlp_input", + "_communicate_summable_tensor_pair_fn", + "_ffn_output", + "_postprocess_scatters_to_local_tokens", + "_ffn_sum_is_movable", + "_mlp_input_may_return_new_residual", + ) + + def test_both_entries_hold_steps_only(self): + parallel = parallel_of(attn_dp=2, attn_tp=2) + direct = LayerScatterModes( + layer_input_mode=ScatterMode.TP_ATTN_FULL, + attn_mode=ScatterMode.TP_ATTN_FULL, + mlp_mode=ScatterMode.FULL, + middle_residual_mode=ScatterMode.TP_ATTN_FULL, + layer_output_mode=ScatterMode.TP_ATTN_FULL, + ) + for name, communicator in ( + ("declarations", build(layer_facts(1, 3), parallel)), + ("scatter modes", build(direct, parallel)), + ): + with self.subTest(name): + self.assertIsInstance(communicator._steps, comm.BoundarySteps) + for attribute in self.SEPARATE: + self.assertFalse(hasattr(communicator, attribute), attribute) + self.assertIs(communicator._batch_steps(), communicator._steps) + # Under attention DP both bring the FFN output back to this + # rank's tokens with the step the FFN exit chooses. + self.assertTrue(communicator._steps.returns_over_dp) + + def test_the_scatter_mode_steps_complete_a_scattered_input(self): + # The scatter-mode path's input completion reads the batch fact itself. + steps = build( + LayerScatterModes( + layer_input_mode=ScatterMode.TP_ATTN_FULL, + attn_mode=ScatterMode.TP_ATTN_FULL, + mlp_mode=ScatterMode.FULL, + middle_residual_mode=ScatterMode.TP_ATTN_FULL, + layer_output_mode=ScatterMode.TP_ATTN_FULL, + ), + parallel_of(attn_dp=1, attn_tp=2), + )._steps + self.assertIs(steps.layer_input, comm._complete_scattered_input) + hidden, residual = torch.ones(2, HIDDEN), torch.ones(2, HIDDEN) + with patch.object( + comm, "get_attn_tp_context", lambda: SimpleNamespace(input_scattered=False) + ): + self.assertEqual( + steps.layer_input(hidden, residual, None), (hidden, residual) + ) + + # --------------------------------------------------------------------------- # Two consecutive layers, rank by rank. diff --git a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py index ef9ec60cde..6975e0cf76 100644 --- a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py +++ b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py @@ -40,15 +40,32 @@ def sp_region_steps(): ) +def _steps(*, ffn_output=None, returns_over_dp=False, ffn_sum_is_movable=True): + return comm.BoundarySteps( + attention_input=comm.CommunicateSimpleFn._trivial, + ffn_input=comm._mlp_input_norm, + ffn_output=ffn_output or StageOutput(Layout(frozenset()), group=SumGroup.TP), + ffn_output_move=( + None if returns_over_dp else comm.CommunicateSummableTensorPairFn._trivial + ), + ffn_sum_is_movable=ffn_sum_is_movable, + ) + + def _fake_communicator(mlp_mode=ScatterMode.TP_ATTN_FULL): communicator = LayerCommunicator.__new__(LayerCommunicator) communicator._speculative_algo = None communicator.layer_scatter_modes = types.SimpleNamespace(mlp_mode=mlp_mode) # Fixed at construction by the scatter-mode path from the MLP's mode. - communicator._ffn_sum_is_movable = mlp_mode not in ( - ScatterMode.MOE_FULL, - ScatterMode.SCATTERED, + communicator._steps = _steps( + ffn_sum_is_movable=mlp_mode + not in ( + ScatterMode.MOE_FULL, + ScatterMode.SCATTERED, + ) ) + communicator._sp_steps = None + communicator._input_scattered_steps = None communicator.is_last_layer = False communicator._context = types.SimpleNamespace(tp_size=4) return communicator @@ -347,14 +364,16 @@ class TestDeferFfnReduction(CustomTestCase): ): communicator = _fake_communicator() communicator.is_last_layer = is_last_layer - communicator._postprocess_scatters_to_local_tokens = scatters_to_local_tokens communicator._sp_steps = sp_region_steps() if sp_active else None communicator._input_scattered_steps = None - communicator._ffn_output = StageOutput( - Layout(frozenset()), - group=SumGroup.MOE_OUTPUT, - leaves_for_reduce_scatter=True, - leaves_for_reduce_scatterv=True, + communicator._steps = _steps( + ffn_output=StageOutput( + Layout(frozenset()), + group=SumGroup.MOE_OUTPUT, + leaves_for_reduce_scatter=True, + leaves_for_reduce_scatterv=True, + ), + returns_over_dp=scatters_to_local_tokens, ) forward_batch = types.SimpleNamespace( input_ids=types.SimpleNamespace(shape=(batch_size,)), diff --git a/test/registered/unit/layers/test_layernorm_sp.py b/test/registered/unit/layers/test_layernorm_sp.py index 0df066fa00..50fb90394b 100644 --- a/test/registered/unit/layers/test_layernorm_sp.py +++ b/test/registered/unit/layers/test_layernorm_sp.py @@ -15,6 +15,9 @@ from sglang.srt.arg_groups.layernorm_sp_hook import validate_layernorm_sp from sglang.srt.layers import communicator as comm from sglang.srt.layers import layernorm_sp from sglang.srt.layers.boundary_layout import ( + Layout, + StageOutput, + SumGroup, TokenAxis, sequence_parallel_layer_sides, ) @@ -167,18 +170,28 @@ class TestSpRegionSteps(CustomTestCase): c = comm.LayerCommunicator.__new__(comm.LayerCommunicator) c._sp_steps = sp_region_steps() c._input_scattered_steps = None - c.layer_scatter_modes = SimpleNamespace(is_first_layer=first_layer) + c.layer_scatter_modes = SimpleNamespace( + is_first_layer=first_layer, is_layer_sparse=False + ) + c.allow_reduce_scatter = False c._attn_input_fusions = () c.input_layernorm = _Norm() c.post_attention_layernorm = _Norm() c.qkv_latent_func = None c._context = SimpleNamespace(cache=None) - c._communicate_simple_fn = MagicMock(side_effect=AssertionError("moved")) - c._communicate_summable_tensor_pair_fn = MagicMock( - side_effect=AssertionError("postprocess ran") - ) + c._steps = self.ordinary_steps(MagicMock(side_effect=AssertionError("moved"))) return c + def ordinary_steps(self, attention_input): + """The layer's steps outside the region, which must not run inside it.""" + return comm.BoundarySteps( + attention_input=attention_input, + ffn_input=MagicMock(side_effect=AssertionError("ordinary FFN input ran")), + ffn_output=StageOutput(Layout(frozenset()), group=SumGroup.TP), + ffn_output_move=MagicMock(side_effect=AssertionError("postprocess ran")), + ffn_sum_is_movable=True, + ) + def run_prepare_attn(self, communicator, mode, hidden, residual): batch = SimpleNamespace(forward_mode=mode) with ( @@ -208,20 +221,19 @@ class TestSpRegionSteps(CustomTestCase): def test_decode_leaves_the_region_closed(self): communicator = self.communicator(first_layer=True) - communicator._communicate_simple_fn = MagicMock( - side_effect=lambda **k: k["hidden_states"] - ) + move = MagicMock(side_effect=lambda **k: k["hidden_states"]) + communicator._steps = self.ordinary_steps(move) (h, _), active, scatter = self.run_prepare_attn( communicator, ForwardMode.DECODE, torch.ones(2, 4), None ) self.assertFalse(active) scatter.assert_not_called() - communicator._communicate_simple_fn.assert_called_once() + move.assert_called_once() def test_a_later_layer_does_not_reopen_the_region(self): communicator = self.communicator(first_layer=False) - communicator._communicate_simple_fn = MagicMock( - side_effect=lambda **k: k["hidden_states"] + communicator._steps = self.ordinary_steps( + MagicMock(side_effect=lambda **k: k["hidden_states"]) ) _, active, scatter = self.run_prepare_attn( communicator, ForwardMode.EXTEND, torch.ones(2, 4), torch.ones(2, 4) diff --git a/test/registered/unit/layers/test_postprocess_reduce_scatterv.py b/test/registered/unit/layers/test_postprocess_reduce_scatterv.py index ecb6245b21..0856aa597b 100644 --- a/test/registered/unit/layers/test_postprocess_reduce_scatterv.py +++ b/test/registered/unit/layers/test_postprocess_reduce_scatterv.py @@ -29,6 +29,18 @@ from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=5, suite="base-a-test-cpu") +def steps(*, ffn_output_move): + """A layer's steps with the given FFN output move; None sends the output + back over attention DP.""" + return comm.BoundarySteps( + attention_input=comm.CommunicateSimpleFn._trivial, + ffn_input=comm._mlp_input_norm, + ffn_output=comm.StageOutput(comm.Layout(frozenset())), + ffn_output_move=ffn_output_move, + ffn_sum_is_movable=False, + ) + + @contextlib.contextmanager def reduce_scatterv_applies(): with ( @@ -143,11 +155,10 @@ class TestPostprocessReduceScatterv(CustomTestCase): communicator._sp_steps = None communicator._input_scattered_steps = None communicator._context = None - communicator._postprocess_scatters_to_local_tokens = False communicator.allow_reduce_scatter = False communicator.layer_scatter_modes = types.SimpleNamespace(is_layer_sparse=True) - communicator._communicate_summable_tensor_pair_fn = lambda **kwargs: ( - seen.update(kwargs) or (None, None) + communicator._steps = steps( + ffn_output_move=lambda **kwargs: seen.update(kwargs) or (None, None) ) communicator.postprocess_layer(None, None, None) self.assertIs(seen["is_layer_sparse"], True) @@ -160,7 +171,7 @@ class TestPostprocessReduceScatterv(CustomTestCase): communicator = LayerCommunicator.__new__(LayerCommunicator) communicator._sp_steps = None communicator._input_scattered_steps = None - communicator._postprocess_scatters_to_local_tokens = True + communicator._steps = steps(ffn_output_move=None) communicator._postprocess_dp_step = lambda forward_batch: step with patch.object( comm, "_to_local_tokens", side_effect=lambda s, fb, h: (s, h) diff --git a/test/registered/unit/layers/test_prepare_attn_steps.py b/test/registered/unit/layers/test_prepare_attn_steps.py index 4a5fe39d3e..dbf9c4c95e 100644 --- a/test/registered/unit/layers/test_prepare_attn_steps.py +++ b/test/registered/unit/layers/test_prepare_attn_steps.py @@ -48,7 +48,14 @@ def communicator(norm): c._context = None c.enable_fused_ar_quant = False c.fused_ar_quant_keep_bf16 = False - c._communicate_simple_fn = lambda hidden_states, **_: hidden_states + # A layer whose attention takes its input as it is and owes nothing on it. + c._steps = comm.BoundarySteps( + attention_input=lambda hidden_states, **_: hidden_states, + ffn_input=comm._mlp_input_norm, + ffn_output=comm.StageOutput(comm.Layout(frozenset())), + ffn_output_move=comm.CommunicateSummableTensorPairFn._trivial, + ffn_sum_is_movable=False, + ) # Construction picks the fused entries; call it under the platform patches. c._attn_input_fusions = c._select_attn_input_fusions() return c diff --git a/test/registered/unit/lora/test_triton_dp_attention_unit.py b/test/registered/unit/lora/test_triton_dp_attention_unit.py index d26320e4ed..f352e4edd7 100644 --- a/test/registered/unit/lora/test_triton_dp_attention_unit.py +++ b/test/registered/unit/lora/test_triton_dp_attention_unit.py @@ -227,10 +227,10 @@ def test_communicator_publishes_layout_at_each_transition( communicator.post_attention_layernorm = None communicator.input_layernorm = lambda x: x communicator.qkv_latent_func = None - communicator._communicate_simple_fn = lambda **kwargs: kwargs["hidden_states"] - communicator._mlp_input = lambda hidden_states, residual, *args: ( - hidden_states, - residual, + communicator._steps = SimpleNamespace( + attention_input=lambda hidden_states, **kwargs: hidden_states, + layer_input=None, + ffn_input=lambda hidden_states, residual, *args: (hidden_states, residual), ) communicator._attn_input_fusions = () monkeypatch.setattr( diff --git a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py index 19620976f2..3c6e2e737d 100644 --- a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py +++ b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py @@ -46,17 +46,22 @@ class DeferringLayer(nn.Module): super().__init__() self.return_topk = return_topk self.layer_communicator = comm.LayerCommunicator.__new__(comm.LayerCommunicator) - self.layer_communicator._ffn_output = comm.StageOutput( - comm.Layout(frozenset()), - group=comm.SumGroup.TP, - leaves_for_next_layer=True, + self.layer_communicator._steps = comm.BoundarySteps( + attention_input=comm.CommunicateSimpleFn._trivial, + ffn_input=comm._mlp_input_norm, + ffn_output=comm.StageOutput( + comm.Layout(frozenset()), + group=comm.SumGroup.TP, + leaves_for_next_layer=True, + ), + ffn_output_move=comm.CommunicateSummableTensorPairFn._trivial, + ffn_sum_is_movable=True, ) self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer = ( lambda batch: defer ) self.layer_communicator._ffn_sum_moves_to_next_layer = lambda batch, **_: defer self.layer_communicator.is_last_layer = False - self.layer_communicator._postprocess_scatters_to_local_tokens = False self.layer_communicator._sp_steps = None self.layer_communicator._input_scattered_steps = None self.layer_communicator.ffn_reduction_group = lambda: GROUP