diff --git a/python/sglang/srt/layers/boundary_layout.py b/python/sglang/srt/layers/boundary_layout.py index 9dc3c95656..c7af94f9a6 100644 --- a/python/sglang/srt/layers/boundary_layout.py +++ b/python/sglang/srt/layers/boundary_layout.py @@ -138,6 +138,52 @@ def sequence_parallel_layer_sides( ) +def scattered_residual_layer_sides( + *, + axis_sizes: Mapping[TokenAxis, int], + ffn_group: SumGroup, + is_first_layer: bool, + is_last_layer: bool, + leaves_for_reduce_scatter: bool, +) -> DecoderLayerSides: + """A decoder layer on an input-scattered batch whose residual stays on + each attention-TP rank's slice (MHC). The attention and the FFN compute on + the full rows. The attention takes the slice and gathers it itself (its + QKV hook, or the handoff before an attention that needs the rows); the + boundary brings the FFN its rows and moves both outputs back onto the + slice: the attention output's TP sum by a reduce-scatter, the FFN's too + when it leaves it (``leaves_for_reduce_scatter``). The first layer's input + is the embedding's partial sum on the full rows; the last layer hands on + the full rows again.""" + attention = Layout.sharded_over( + TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes + ) + local = Layout.sharded_over( + TokenAxis.ATTN_DP, + TokenAxis.ATTN_CP, + TokenAxis.ATTN_TP_SCATTER, + axis_sizes=axis_sizes, + ) + return DecoderLayerSides( + input_rows=attention if is_first_layer else local, + attention=StageInput( + attention, gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}) + ), + attention_output=StageOutput( + attention, group=SumGroup.ATTN_TP, always_leaves=True + ), + ffn=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)), + ffn_output=StageOutput( + attention, + group=ffn_group, + leaves_for_reduce_scatter=leaves_for_reduce_scatter, + ), + ffn_residual_rows=local, + output_rows=attention if is_last_layer else local, + input_owes=SumGroup.TP if is_first_layer else None, + ) + + def input_scattered_layer_sides( *, axis_sizes: Mapping[TokenAxis, int], diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index 22020e9023..2fc3842b67 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -16,7 +16,7 @@ from contextlib import contextmanager from dataclasses import dataclass from enum import Enum, auto from functools import cached_property, partial -from typing import Callable, Dict, List, Optional, Tuple, Union +from typing import Callable, Dict, List, Optional, Protocol, Tuple, Union import msgspec import torch @@ -46,6 +46,7 @@ from sglang.srt.layers.boundary_layout import ( TokenAxis, decoder_layer_sides, input_scattered_layer_sides, + scattered_residual_layer_sides, sequence_parallel_layer_sides, ) from sglang.srt.layers.communicator_dsa_cp import ( @@ -651,6 +652,92 @@ def _attn_input_update_and_read_residual(quant_format: str): return _update_and_read_residual_plain +class ResidualOps(Protocol): + """How a layer writes each stage's output into its residual and reads the + next stage's input from it. The boundary steps complete sums and move + tokens around these operations. AddAndNorm adds and normalizes; MHC's + hyper-connection streams implement the same operations (communicator_mhc).""" + + # The write-back is a plain add, which one rank may run before the sum it + # adds into completes: the DP partial order, and a residual that joins the + # attention output's sum. + adds_plainly: bool + # The layer writes its FFN output into the residual itself instead of + # leaving that to the next layer's input. + updates_residual_after_ffn: bool + + def enter(self, hidden_states: torch.Tensor) -> torch.Tensor: + """The residual the layer stack starts from, given its input.""" + + def read_attention_input(self, residual, norm, quant_format: str) -> Tuple: + """The attention input and the residual, from a residual that already + holds the previous layer's output.""" + + def update_and_read_attention_input( + self, hidden_states, residual, norm, quant_format: str, post_residual_addition + ) -> Tuple: + """Write the previous layer's output into the residual, then read the + attention input from it.""" + + def update_and_read_ffn_input(self, hidden_states, residual, norm) -> Tuple: + """Write the attention output into the residual, then read the FFN + input from it.""" + + def update_residual(self, hidden_states, residual) -> torch.Tensor: + """The residual with the FFN output written into it.""" + + def residual_to_attn_tp_shard(self, residual, context) -> torch.Tensor: + """This attention-TP rank's slice of the residual.""" + + def residual_from_attn_tp_shards(self, residual) -> torch.Tensor: + """The residual gathered from every attention-TP rank's slice.""" + + +class AddAndNorm: + """A plain residual: a stage's output is added into it, and the next + stage's input is its norm. An empty batch skips the norms.""" + + adds_plainly = True + updates_residual_after_ffn = False + + def enter(self, hidden_states): + return hidden_states + + def read_attention_input(self, residual, norm, quant_format): + if residual.shape[0] == 0: + return residual, residual + return _attn_input_update_and_read_residual(quant_format)( + norm, residual, None, None + ) + + def update_and_read_attention_input( + self, hidden_states, residual, norm, quant_format, post_residual_addition + ): + if hidden_states.shape[0] == 0: + return hidden_states, hidden_states + return _attn_input_update_and_read_residual(quant_format)( + norm, hidden_states, residual, post_residual_addition + ) + + def update_and_read_ffn_input(self, hidden_states, residual, norm): + if hidden_states.shape[0] == 0: + return hidden_states, residual + return norm(hidden_states, residual) + + def update_residual(self, hidden_states, residual): + hidden_states += residual + return hidden_states + + def residual_to_attn_tp_shard(self, residual, context): + return _redistribute_to_attn_tp_shards(residual, context) + + def residual_from_attn_tp_shards(self, residual): + return _redistribute_from_attn_tp_shards(residual) + + +ADD_AND_NORM = AddAndNorm() + + class UnreducedOutput(msgspec.Struct, frozen=True): """A layer output that still owes its sum, left for the next layer's input. Hand it to the next layer, or pass it through reduce_output() before reading @@ -730,6 +817,8 @@ class LayerCommunicator: # Whether this class's boundary steps may be chosen from both sides' # declarations; a subclass that picks its own steps says no. _takes_declared_boundaries = True + # A plain residual unless the layer is built with its own. + _residual_ops: ResidualOps = ADD_AND_NORM def __init__( self, @@ -744,6 +833,8 @@ class LayerCommunicator: fused_ar_quant_keep_bf16: bool = False, # False for a layer whose FFN always completes its own reduction. allow_deferred_ffn_reduction: bool = True, + # How the layer writes its residual and reads its stages' inputs. + residual_ops: ResidualOps = ADD_AND_NORM, ): self.layer_scatter_modes = layer_scatter_modes self.input_layernorm = input_layernorm @@ -755,6 +846,7 @@ class LayerCommunicator: self.enable_fused_ar_quant = enable_fused_ar_quant self.fused_ar_quant_keep_bf16 = fused_ar_quant_keep_bf16 self.allow_deferred_ffn_reduction = allow_deferred_ffn_reduction + self._residual_ops = residual_ops self._context = CommunicateContext.init_new() self._context.force_layernorm_before_dp_gather = ( @@ -762,17 +854,8 @@ class LayerCommunicator: ) # The steps the layer's ordinary batches run. sides = self._declared_sides() - if ( - sides is None - and self._takes_declared_boundaries - and (_generic_prefill_cp_shards_tokens() and _gathers_over_attention_cp()) - ): - # The scatter-mode steps have no attention-CP gather. - raise NotImplementedError( - "a DSA or MLA prefill CP layer outside the declarations" - ) self._steps = ( - _select_boundary_steps( + self._steps_from_declarations( sides, fusions=self._select_mlp_input_fusions(), force_layernorm_before_gather=force_layernorm_before_dp_gather, @@ -783,7 +866,7 @@ class LayerCommunicator: # The steps a batch that shards its tokens over attention CP runs; None # without CP or where the steps come from the scatter modes. self._cp_steps = ( - _select_boundary_steps( + self._steps_from_declarations( self._declared_sides(cp_active=True), fusions=self._select_mlp_input_fusions(), force_layernorm_before_gather=force_layernorm_before_dp_gather, @@ -795,14 +878,7 @@ class LayerCommunicator: # The steps a batch with input-scattered attention runs; None where it # cannot run or the steps come from the scatter modes. self._input_scattered_steps = ( - _select_boundary_steps( - input_scattered_layer_sides( - axis_sizes=_token_axis_sizes(), - ffn_group=sides.ffn_output.group, - hands_on_partial=allow_reduce_scatter - and not layer_scatter_modes.is_last_layer, - ) - ) + self._steps_for_input_scattered(sides) if sides is not None and self._input_can_be_scattered() else None ) @@ -820,7 +896,8 @@ class LayerCommunicator: # q_lora, which input-scattered attention needs. self._sp_steps = ( _select_boundary_steps( - sequence_parallel_layer_sides(axis_sizes=_token_axis_sizes()) + sequence_parallel_layer_sides(axis_sizes=_token_axis_sizes()), + residual_ops=residual_ops, ) if layernorm_sp.layernorm_sp_enabled() else None @@ -918,6 +995,7 @@ class LayerCommunicator: attention_input, postprocess = self._post_init_communicate() ffn_input, fused = self._select_mlp_input() modes = self.layer_scatter_modes + ops = self._residual_ops ffn_output = StageOutput( self._context.layouts[modes.mlp_mode], group=SumGroup.MOE_OUTPUT if modes.is_layer_sparse else SumGroup.TP, @@ -940,7 +1018,7 @@ class LayerCommunicator: ffn_output_move=( None if postprocess is CommunicateSummableTensorPairFn._scatter_hidden_states - else postprocess + else partial(postprocess, residual_ops=ops) ), # Not when the way back is the MoE-CP scatter, nor when a SCATTERED # FFN computes whole tokens with nothing left to sum. @@ -950,9 +1028,46 @@ class LayerCommunicator: layer_input=_complete_scattered_input, ) + def _steps_for_input_scattered(self, sides: DecoderLayerSides) -> "BoundarySteps": + """The steps a batch with input-scattered attention runs at this layer, + for the layer's ordinary declarations ``sides``. A plain residual comes + back to the full rows inside the attention output's sum; one whose + write-back is not a plain add stays on each rank's slice.""" + if self._residual_ops.adds_plainly: + scattered = input_scattered_layer_sides( + axis_sizes=_token_axis_sizes(), + ffn_group=sides.ffn_output.group, + hands_on_partial=self.allow_reduce_scatter and not self.is_last_layer, + ) + handoff = _hand_qkv_hook_its_input + else: + scattered = scattered_residual_layer_sides( + axis_sizes=_token_axis_sizes(), + ffn_group=sides.ffn_output.group, + is_first_layer=self.layer_scatter_modes.is_first_layer, + is_last_layer=self.is_last_layer, + leaves_for_reduce_scatter=self.allow_reduce_scatter, + ) + # DSA and hook-less attention take the slice gathered. + handoff = _hand_scattered_input_to_attention + return _select_boundary_steps( + scattered, residual_ops=self._residual_ops, attention_handoff=handoff + ) + + def _steps_from_declarations( + self, sides: DecoderLayerSides, **kwargs + ) -> "BoundarySteps": + """The steps this layer runs for a set of declarations.""" + return _select_boundary_steps(sides, residual_ops=self._residual_ops, **kwargs) + def _post_init_communicate(self) -> Tuple[Callable, Callable]: """The attention input move and the postprocess the scatter modes choose.""" + if _generic_prefill_cp_shards_tokens() and _gathers_over_attention_cp(): + # These tables have no attention-CP gather. + raise NotImplementedError( + "a DSA or MLA prefill CP layer outside the declarations" + ) return ( CommunicateSimpleFn.get_fn( input_mode=self.layer_scatter_modes.layer_input_mode, @@ -1048,7 +1163,8 @@ class LayerCommunicator: post_residual_addition: Optional[torch.Tensor] = None, ): self.publish_attn_lora_layout() - + # The layer stack's first layer starts its residual from its input. + enters = residual is None and self.layer_scatter_modes.is_first_layer # What the previous layer left to complete: an UnreducedOutput, or a # producer's handoff that one of the fused entries consumes. owed = None if isinstance(hidden_states, torch.Tensor) else hidden_states @@ -1084,17 +1200,24 @@ class LayerCommunicator: hidden_states, residual = layer_input( hidden_states, residual, self._context ) - if hidden_states.shape[0] == 0: - residual = hidden_states - elif owed is not None: + ops = self._residual_ops + if enters: + hidden_states, residual = ops.enter(hidden_states), None + if owed is not None and hidden_states.shape[0] != 0: hidden_states = reduce_output(owed) - hidden_states, residual = _attn_input_update_and_read_residual( - quant_format - )(self.input_layernorm, hidden_states, residual, post_residual_addition) + if residual is None: + # The previous layer already wrote its output into the residual. + hidden_states, residual = ops.read_attention_input( + hidden_states, self.input_layernorm, quant_format + ) else: - hidden_states, residual = _attn_input_update_and_read_residual( - quant_format - )(self.input_layernorm, hidden_states, residual, post_residual_addition) + hidden_states, residual = ops.update_and_read_attention_input( + hidden_states, + residual, + self.input_layernorm, + quant_format, + post_residual_addition, + ) return self._finish_prepare_attn( hidden_states=hidden_states, @@ -1122,24 +1245,27 @@ class LayerCommunicator: def _finish_prepare_attn(self, hidden_states, residual, forward_batch): """Tail every prepare_attn path must run, or ``attn_inputs`` is unset.""" - hidden_states = self._batch_steps(forward_batch).attention_input( + steps = self._batch_steps(forward_batch) + hidden_states = steps.attention_input( hidden_states=hidden_states, forward_batch=forward_batch, context=self._context, ) - if self.qkv_latent_func is not None: - attn_inputs = AttentionInputs( - hidden_states, forward_batch, self.qkv_latent_func - ) - get_attn_tp_context().set_attn_inputs(attn_inputs) + hidden_states = steps.attention_handoff( + hidden_states, forward_batch, self.qkv_latent_func + ) return hidden_states, residual def _select_attn_input_fusions(self) -> Tuple[Callable, ...]: """The fused kernels that complete what the previous layer left together with the residual update and the input norm, in the order they are tried. Each takes (owed, residual, forward_batch, post_residual_addition) and - returns None when it does not take the batch.""" - if not hasattr(self.input_layernorm, "forward_with_allreduce_fusion"): + returns None when it does not take the batch. They add the residual + plainly.""" + if not ( + self._residual_ops.adds_plainly + and hasattr(self.input_layernorm, "forward_with_allreduce_fusion") + ): return () self._attn_input_fuses_quant = ( self.enable_fused_ar_quant @@ -1194,19 +1320,20 @@ class LayerCommunicator: def _select_mlp_input(self) -> Tuple[Callable, Tuple["FusedMlpInput", ...]]: """The attention-TP -> FFN boundary's steps, chosen from the layouts, and - the fused kernels they try first. The MHC and DSA-CP communicators pick - their own implementations.""" + the fused kernels they try first.""" kind = mlp_input_kind(self.layer_scatter_modes, self._context) residual_input_mode = self.layer_scatter_modes.layer_input_mode + ops = self._residual_ops if kind is MlpInputKind.NORM: - return _mlp_input_norm, () + return partial(_mlp_input_norm, residual_ops=ops), () if kind is MlpInputKind.ATTN_TP_ALL_REDUCE: - return _mlp_input_attn_tp_all_reduce, () + return partial(_mlp_input_attn_tp_all_reduce, residual_ops=ops), () if kind is MlpInputKind.SCATTER: return ( partial( _mlp_input_scatter, scatters_residual=residual_input_mode == ScatterMode.TP_ATTN_FULL, + residual_ops=ops, ), (), ) @@ -1217,7 +1344,10 @@ class LayerCommunicator: steps = partial( _mlp_input_gather, order=_mlp_input_order( - self._context, residual_input_mode, tuple(f.run for f in fusions) + self._context, + residual_input_mode, + tuple(f.run for f in fusions), + residual_ops=ops, ), ) if kind is MlpInputKind.GATHER_MOE_CP: @@ -1226,8 +1356,11 @@ class LayerCommunicator: def _select_mlp_input_fusions(self) -> Tuple["FusedMlpInput", ...]: """The fused kernels that can take the attention -> FFN steps, in the - order they are tried.""" - if not hasattr(self.post_attention_layernorm, "forward_with_allreduce_fusion"): + order they are tried. They add the residual plainly.""" + if not ( + self._residual_ops.adds_plainly + and hasattr(self.post_attention_layernorm, "forward_with_allreduce_fusion") + ): return () return ( FusedMlpInput( @@ -1286,25 +1419,23 @@ class LayerCommunicator: residual: torch.Tensor, forward_batch: ForwardBatch, ): - steps = self._batch_steps(forward_batch) - if steps.returns_over_dp: - step = self._postprocess_dp_step(forward_batch) or _redistribute_output - return _to_local_tokens(step, forward_batch, hidden_states), residual - return steps.ffn_output_move( - hidden_states=hidden_states, - residual=residual, + return self._complete_ffn_output_now( + hidden_states, + residual, forward_batch=forward_batch, - context=self._context, - allow_reduce_scatter=self.allow_reduce_scatter, - is_layer_sparse=self.layer_scatter_modes.is_layer_sparse, + dp_step=self._postprocess_dp_step(forward_batch), ) def _local_token_move_can_go_to_next_layer( self, forward_batch: ForwardBatch ) -> bool: """Whether the next layer's input can run this layer's move of its FFN - output back to this rank's tokens: the base postprocess scatter.""" - return self._batch_steps(forward_batch).returns_over_dp + output back to this rank's tokens: the base postprocess scatter, when + the next layer's input also writes the output into the residual.""" + return ( + self._batch_steps(forward_batch).returns_over_dp + and not self._residual_ops.updates_residual_after_ffn + ) def _postprocess_dp_step( self, forward_batch: ForwardBatch @@ -1418,16 +1549,27 @@ class LayerCommunicator: forward_batch: ForwardBatch, dp_step: Optional[Callable], ) -> Tuple[torch.Tensor, torch.Tensor]: - """This layer's postprocess, run with the attention-DP step the FFN exit - chose.""" - if self._local_token_move_can_go_to_next_layer(forward_batch): - return ( - _to_local_tokens( - dp_step or _redistribute_output, forward_batch, hidden_states - ), - residual, + """This layer's postprocess, run with the attention-DP step already + chosen: the move back to where the next layer reads the FFN output, then + the write-back into the residual for a layer that does it itself.""" + steps = self._batch_steps(forward_batch) + if steps.returns_over_dp: + hidden_states = _to_local_tokens( + dp_step or _redistribute_output, forward_batch, hidden_states ) - return self.postprocess_layer(hidden_states, residual, forward_batch) + else: + hidden_states, residual = steps.ffn_output_move( + hidden_states=hidden_states, + residual=residual, + forward_batch=forward_batch, + context=self._context, + allow_reduce_scatter=self.allow_reduce_scatter, + is_layer_sparse=self.layer_scatter_modes.is_layer_sparse, + ) + if residual is not None and self._residual_ops.updates_residual_after_ffn: + hidden_states = self._residual_ops.update_residual(hidden_states, residual) + residual = None + return hidden_states, residual def ffn_exit(self, forward_batch: ForwardBatch) -> "FfnExit": """Decide once how this layer's FFN output reduction completes. Use the @@ -1751,7 +1893,9 @@ class CommunicateSimpleFn: def _redistribute_from_attn_tp_shards(tensor: torch.Tensor) -> torch.Tensor: - gathered = get_local_dp_buffer(get_parallel().attn_tp_group) + gathered = get_local_dp_buffer( + get_parallel().attn_tp_group, hidden_size=tensor.shape[-1] + ) attn_tp_all_gather_into_tensor(gathered, tensor) return gathered @@ -1808,10 +1952,11 @@ def _redistribute_input_to_moe_cp( def _mlp_input_reduce_output( - hidden_states: torch.Tensor, forward_batch: ForwardBatch + hidden_states: torch.Tensor, forward_batch: ForwardBatch, may_quantize: bool = True ) -> torch.Tensor: if ( - not forward_batch.forward_mode.is_decode_or_idle() + may_quantize + and not forward_batch.forward_mode.is_decode_or_idle() and get_exec().comm.enable_quant_communications ): return attention_tensor_model_parallel_quant_all_reduce(hidden_states) @@ -1873,17 +2018,21 @@ def _mlp_input_without_dp( *, gathers_residual: bool, fusions: Tuple[Callable, ...], + residual_ops: ResidualOps = ADD_AND_NORM, ): if gathers_residual: - residual = _redistribute_from_attn_tp_shards(residual) + residual = residual_ops.residual_from_attn_tp_shards(residual) for fused in fusions: result = fused(hidden_states, residual, forward_batch) if result is not None: return result - hidden_states = _mlp_input_reduce_output(hidden_states, forward_batch) + # MHC sums its streams in full precision. + hidden_states = _mlp_input_reduce_output( + hidden_states, forward_batch, may_quantize=residual_ops.adds_plainly + ) if _is_npu and context.cache is not None: _ = prepare_weight_cache(hidden_states, context.cache) - return layernorm(hidden_states, residual) + return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm) def _mlp_input_dp_replicate( @@ -1896,12 +2045,14 @@ def _mlp_input_dp_replicate( gathers_residual: bool, reduces_attention_tp: bool, places_cp_shards: bool = False, + residual_ops: ResidualOps = ADD_AND_NORM, ): - """Attention DP: complete the attention-TP sum if it is owed, add and - normalize locally, then gather. With ``places_cp_shards`` each CP rank puts - its shard of the DP group's tokens beside the others' in the group's slot.""" + """Attention DP: complete the attention-TP sum if it is owed, write the + output into the residual and read the FFN input locally, then gather. With + ``places_cp_shards`` each CP rank puts its shard of the DP group's tokens + beside the others' in the group's slot.""" if gathers_residual: - residual = _redistribute_from_attn_tp_shards(residual) + residual = residual_ops.residual_from_attn_tp_shards(residual) if hidden_states.shape[0] != 0: if reduces_attention_tp: hidden_states = attention_tensor_model_parallel_all_reduce(hidden_states) @@ -1909,7 +2060,13 @@ def _mlp_input_dp_replicate( get_parallel().tp_group, disabled=not is_allocation_symmetric(), ): - hidden_states, residual = layernorm(hidden_states, residual) + hidden_states, residual = residual_ops.update_and_read_ffn_input( + hidden_states, residual, layernorm + ) + else: + hidden_states, residual = residual_ops.update_and_read_ffn_input( + hidden_states, residual, layernorm + ) cp_shard_counts = _cp_shard_token_rows(forward_batch) if places_cp_shards else None return ( _redistribute_input_to_dp(hidden_states, forward_batch, cp_shard_counts), @@ -1948,6 +2105,7 @@ def _mlp_input_order( context: CommunicateContext, residual_input_mode: ScatterMode, fusions: Tuple[Callable, ...], + residual_ops: ResidualOps = ADD_AND_NORM, ) -> Callable: """The steps from the attention output to the FFN input, chosen from facts fixed at construction.""" @@ -1956,13 +2114,21 @@ def _mlp_input_order( ) if context.attn_dp_size == 1: return partial( - _mlp_input_without_dp, gathers_residual=gathers_residual, fusions=fusions + _mlp_input_without_dp, + gathers_residual=gathers_residual, + fusions=fusions, + residual_ops=residual_ops, ) - if context.force_layernorm_before_dp_gather or context.attn_tp_size == 1: + if ( + context.force_layernorm_before_dp_gather + or context.attn_tp_size == 1 + or not residual_ops.adds_plainly + ): return partial( _mlp_input_dp_replicate, gathers_residual=gathers_residual, reduces_attention_tp=context.attn_tp_size > 1, + residual_ops=residual_ops, ) return partial(_mlp_input_dp_partial, gathers_residual=gathers_residual) @@ -2049,6 +2215,53 @@ def _same_ranks(a: GroupCoordinator, b: GroupCoordinator) -> bool: return sorted(a.ranks) == sorted(b.ranks) +def _hand_qkv_hook_its_input( + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + qkv_latent_func: Optional[Callable], +) -> torch.Tensor: + """Give the attention's QKV hook the attention input.""" + if qkv_latent_func is not None: + get_attn_tp_context().set_attn_inputs( + AttentionInputs(hidden_states, forward_batch, qkv_latent_func) + ) + return hidden_states + + +def _hand_scattered_input_to_attention( + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + qkv_latent_func: Optional[Callable], +) -> torch.Tensor: + """Input-scattered attention takes each rank's slice, and its QKV hook + gathers the rows after the projection. DSA and attention without a hook + consume full hidden states, so those are gathered here, and the hook is + told they are.""" + ctx = get_attn_tp_context() + if ctx.is_dsa or qkv_latent_func is None: + hidden_states = _tp_all_gather_scattered_rows(hidden_states, forward_batch) + if qkv_latent_func is not None: + ctx.set_attn_inputs( + AttentionInputs( + hidden_states, + forward_batch, + qkv_latent_func, + is_pre_gathered=ctx.is_dsa, + ) + ) + return hidden_states + + +def _tp_all_gather_scattered_rows( + hidden_states: torch.Tensor, forward_batch: ForwardBatch +) -> torch.Tensor: + # Input-scattered attention keeps the same number of tokens on every TP rank. + total_tokens = forward_batch.input_ids.shape[0] + output = hidden_states.new_empty((total_tokens, hidden_states.shape[-1])) + get_parallel().tp_group.all_gather_into_tensor(output, hidden_states) + return output + + class BoundarySteps(msgspec.Struct, frozen=True): """The steps a batch runs at a layer's boundaries: into the attention, from the attention output to the FFN input, and the FFN output on to the @@ -2070,6 +2283,9 @@ class BoundarySteps(msgspec.Struct, frozen=True): # Completes what the layer's input owes before the input norm; None when it # owes nothing. layer_input: Optional[Callable] = None + # Hands the attention its input once attention_input has moved it: + # (hidden_states, forward_batch, qkv_latent_func) -> hidden_states. + attention_handoff: Callable = _hand_qkv_hook_its_input @property def returns_over_dp(self) -> bool: @@ -2082,14 +2298,17 @@ def _select_boundary_steps( fusions: Tuple["FusedMlpInput", ...] = (), force_layernorm_before_gather: bool = False, cp_moves: Optional[CpMoves] = None, + residual_ops: ResidualOps = ADD_AND_NORM, + attention_handoff: Callable = _hand_qkv_hook_its_input, ) -> BoundarySteps: - """The steps a set of declarations chooses; ``cp_moves`` for the ones - that gather over attention CP.""" + """The steps a set of declarations chooses, around the layer's residual + operations; ``cp_moves`` for the ones that gather over attention CP.""" returns_over_dp, ffn_output_move, completes_sum = _select_ffn_output_move( sides.ffn_output, residual=sides.ffn_residual_rows, to=sides.output_rows, cp_moves=cp_moves, + residual_ops=residual_ops, ) rows = sides.input_rows layer_input = None @@ -2113,6 +2332,7 @@ def _select_boundary_steps( fusions=fusions, residual_joins_sum=sides.residual_joins_attention_sum, cp_moves=cp_moves, + residual_ops=residual_ops, ) return BoundarySteps( attention_input=_select_attention_input_move(rows, sides.attention), @@ -2123,6 +2343,7 @@ def _select_boundary_steps( ffn_sum_is_movable=sides.ffn_output.group is not None, fused=fused, layer_input=layer_input, + attention_handoff=attention_handoff, ) @@ -2164,13 +2385,16 @@ def _select_ffn_input( fusions: Tuple[FusedMlpInput, ...], residual_joins_sum: bool = False, cp_moves: Optional[CpMoves] = None, + residual_ops: ResidualOps = ADD_AND_NORM, ) -> Tuple[Callable, Tuple[FusedMlpInput, ...]]: """The steps from the attention output to the FFN input, and the fused kernels they try first: complete the attention-TP sum, move the residual to - the rows it has while the FFN runs, add it and normalize, and bring the rows - to what the FFN's group needs: a gather over attention DP, or each rank's - own slice. A kernel in ``fusions`` is tried only when nothing is gathered or - sliced, and only if it completes the sum the attention output owes.""" + the rows it has while the FFN runs, write the output into it and read the + FFN input, and bring the rows to what the FFN's group needs: a gather over + attention DP, or each rank's own slice. A kernel in ``fusions`` is tried + only when nothing is gathered or sliced, and only if it completes the sum + the attention output owes. A write-back that is not a plain add runs only + after the sum completes.""" # What the attention output owes decides the steps: the attention-TP sum, # always left by the output projection, or nothing. owes_attention_tp = produced.group is SumGroup.ATTN_TP @@ -2193,9 +2417,19 @@ def _select_ffn_input( ): raise NotImplementedError(f"{produced=} {residual=} {need=}") return ( - partial(_mlp_input_scatter, scatters_residual=residual != residual_to), + partial( + _mlp_input_scatter, + scatters_residual=residual != residual_to, + residual_ops=residual_ops, + ), (), ) + if residual_to.sharded - produced.layout.sharded == {TokenAxis.ATTN_TP_SCATTER}: + # The residual stays on each rank's slice while the FFN takes the + # attention's rows (MHC on an input-scattered batch). + if gathered or not owes_attention_tp or residual != residual_to: + raise NotImplementedError(f"{produced=} {residual=} {need=}") + return partial(_mlp_input_on_residual_shard, residual_ops=residual_ops), () if gathered == {TokenAxis.ATTN_CP}: # Each CP rank completes its own chunk, then the CP moves gather them. if cp_moves is None: @@ -2208,6 +2442,7 @@ def _select_ffn_input( force_layernorm_before_gather=force_layernorm_before_gather, fusions=fusions, residual_joins_sum=residual_joins_sum, + residual_ops=residual_ops, ) return partial(cp_moves.gather, gather=on_chunk), fused if ( @@ -2226,10 +2461,12 @@ def _select_ffn_input( gathers_residual = residual != residual_to if not gathered: if not owes_attention_tp: - return _mlp_input_norm, () + return partial(_mlp_input_norm, residual_ops=residual_ops), () if gathers_residual and residual_joins_sum: # Each rank adds its slice of the residual into its share of the # sum, so the all-reduce also brings the residual back to every row. + if not residual_ops.adds_plainly: + raise NotImplementedError(f"{produced=} {residual=} {need=}") return _mlp_input_residual_into_sum, () fused = tuple(f for f in fusions if f.completes is produced.group) return ( @@ -2237,6 +2474,7 @@ def _select_ffn_input( _mlp_input_without_dp, gathers_residual=gathers_residual, fusions=tuple(f.run for f in fused), + residual_ops=residual_ops, ), fused, ) @@ -2246,7 +2484,11 @@ def _select_ffn_input( # Over attention DP and CP, the DP gather puts each CP rank's shard in its # DP group's slot, so the one DP sum gathers both axes. places_cp_shards = TokenAxis.ATTN_CP in gathered - if owes_attention_tp and not force_layernorm_before_gather: + if ( + owes_attention_tp + and not force_layernorm_before_gather + and residual_ops.adds_plainly + ): return ( partial( _mlp_input_dp_partial, @@ -2261,6 +2503,7 @@ def _select_ffn_input( gathers_residual=gathers_residual, reduces_attention_tp=owes_attention_tp, places_cp_shards=places_cp_shards, + residual_ops=residual_ops, ), (), ) @@ -2272,21 +2515,38 @@ def _select_ffn_output_move( residual: Layout, to: Layout, cp_moves: Optional[CpMoves] = None, + residual_ops: ResidualOps = ADD_AND_NORM, ) -> Tuple[bool, Optional[Callable], bool]: """How the FFN output reaches the rows the layer hands on: whether it goes back by undoing the attention-DP gather (the FFN exit and postprocess run that step), or else the postprocess that moves it, None when there is none to choose here; and whether that move also completes the sum the FFN leaves.""" + pair = CommunicateSummableTensorPairFn if produced.layout == residual: if to == residual: - return False, CommunicateSummableTensorPairFn._trivial, False + return False, pair._trivial, False if to.sharded == residual.sharded - {TokenAxis.ATTN_TP_SCATTER}: - # Each rank's slice back to the attention's rows: fold the residual - # into the output, then gather over attention TP. - return False, CommunicateSummableTensorPairFn._gather, False + # Each rank's slice back to the attention's rows: write the output + # into the residual, then gather over attention TP. + return False, partial(pair._gather, residual_ops=residual_ops), False raise NotImplementedError(f"{produced=} {residual=} {to=}") returned = residual.sharded - produced.layout.sharded + if returned == {TokenAxis.ATTN_TP_SCATTER} and to in (residual, produced.layout): + # The residual stays on each rank's slice (MHC on an input-scattered + # batch): a reduce-scatter onto the slice completes the sum the FFN + # leaves; a complete output is only sliced. + sums = produced.leaves_for_reduce_scatter + return ( + False, + partial( + pair._onto_residual_shard, + sums=sums, + gathers_back=to != residual, + residual_ops=residual_ops, + ), + sums, + ) if to != residual or not produced.layout.sharded <= residual.sharded: raise NotImplementedError(f"{produced=} {residual=} {to=}") if returned == {TokenAxis.ATTN_CP}: @@ -2371,11 +2631,10 @@ def _mlp_input_norm( forward_batch: ForwardBatch, layernorm: torch.nn.Module, context: CommunicateContext, + *, + residual_ops: ResidualOps = ADD_AND_NORM, ): - # TODO move these `if shape != 0` into LayerNorm itself - if hidden_states.shape[0] != 0: - hidden_states, residual = layernorm(hidden_states, residual) - return hidden_states, residual + return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm) def _mlp_input_attn_tp_all_reduce( @@ -2384,6 +2643,8 @@ def _mlp_input_attn_tp_all_reduce( forward_batch: ForwardBatch, layernorm: torch.nn.Module, context: CommunicateContext, + *, + residual_ops: ResidualOps = ADD_AND_NORM, ): """All-reduce hidden states inside the attention TP group, then layernorm. @@ -2392,9 +2653,7 @@ def _mlp_input_attn_tp_all_reduce( ``TP_ATTN_FULL`` across the boundary. """ hidden_states = get_parallel().attn_tp_group.all_reduce(hidden_states) - if hidden_states.shape[0] != 0: - hidden_states, residual = layernorm(hidden_states, residual) - return hidden_states, residual + return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm) def _mlp_input_scatter( @@ -2405,14 +2664,35 @@ def _mlp_input_scatter( context: CommunicateContext, *, scatters_residual: bool, + residual_ops: ResidualOps = ADD_AND_NORM, ): hidden_states = _reduce_and_redistribute_output_to_attn_tp_shards( hidden_states, context ) if scatters_residual: - residual = _redistribute_to_attn_tp_shards(residual, context) - if hidden_states.shape[0] != 0: - hidden_states, residual = layernorm(hidden_states, residual) + residual = residual_ops.residual_to_attn_tp_shard(residual, context) + return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm) + + +def _mlp_input_on_residual_shard( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + context: CommunicateContext, + *, + residual_ops: ResidualOps, +): + """The residual stays on each rank's slice while the FFN takes the full + rows: reduce-scatter the attention output onto the slice, which completes + its sum, write it into the residual and read the FFN input there, then + gather the input back into the attention output's rows.""" + if hidden_states.shape[0] == 0: + return hidden_states, hidden_states + shard = hidden_states.tensor_split(context.tp_size)[context.tp_rank] + get_parallel().tp_group.reduce_scatter_tensor(shard, hidden_states) + shard, residual = residual_ops.update_and_read_ffn_input(shard, residual, layernorm) + attn_tp_all_gather_into_tensor(hidden_states, shard) return hidden_states, residual @@ -2681,11 +2961,41 @@ class CommunicateSummableTensorPairFn: residual: torch.Tensor, forward_batch: ForwardBatch, context: CommunicateContext, + residual_ops: ResidualOps = ADD_AND_NORM, **kwargs, ): - hidden_states += residual + hidden_states = residual_ops.update_residual(hidden_states, residual) return _redistribute_from_attn_tp_shards(hidden_states), None + @staticmethod + def _onto_residual_shard( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + context: CommunicateContext, + *, + sums: bool, + gathers_back: bool, + residual_ops: ResidualOps, + **kwargs, + ): + """Bring the FFN output onto the slice of the rows the residual is on: + a reduce-scatter that also completes its sum when ``sums``, else this + rank's slice of the complete output. With ``gathers_back`` it is + written into the residual there and the full rows are gathered back.""" + if sums: + hidden_states, _ = tp_reduce_scatter(hidden_states, None, context) + else: + hidden_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank] + if not gathers_back: + return hidden_states, residual + local_states = residual_ops.update_residual(hidden_states, residual) + hidden_states = local_states.new_empty( + local_states.shape[0] * context.tp_size, *local_states.shape[1:] + ) + get_parallel().tp_group.all_gather_into_tensor(hidden_states, local_states) + return hidden_states, None + @staticmethod def _scatter( hidden_states: torch.Tensor, diff --git a/python/sglang/srt/layers/communicator_mhc.py b/python/sglang/srt/layers/communicator_mhc.py index f676c6a02d..c8dc5a702f 100644 --- a/python/sglang/srt/layers/communicator_mhc.py +++ b/python/sglang/srt/layers/communicator_mhc.py @@ -12,62 +12,28 @@ # limitations under the License. # ============================================================================== from dataclasses import dataclass -from functools import partial from typing import Callable, Optional import torch from sglang.kernels.ops.layernorm.mhc import hc_contract, hc_expand -from sglang.srt.distributed.communication_op import ( - attention_tensor_model_parallel_all_reduce, -) -from sglang.srt.distributed.device_communicators.pynccl_allocator import ( - use_symmetric_memory, -) +from sglang.srt.layers.boundary_layout import DecoderLayerSides, TokenAxis from sglang.srt.layers.communicator import ( - AttentionInputs, - CommunicateContext, - CommunicateSimpleFn, - CommunicateSummableTensorPairFn, - FfnCompletion, + BoundarySteps, LayerCommunicator, LayerScatterModes, - MlpInputKind, ScatterMode, - get_attn_tp_context, - mlp_input_kind, - tp_reduce_scatter, ) -from sglang.srt.layers.dp_attention import ( - attn_tp_all_gather_into_tensor, - attn_tp_reduce_scatter_tensor, - dp_gather_replicate, - dp_reduce_scatter_tensor, - dp_scatter, - get_dp_global_num_tokens, - get_global_dp_buffer, - get_local_dp_buffer_mhc, - is_allocation_symmetric, -) -from sglang.srt.layers.moe import should_use_dp_reduce_scatterv -from sglang.srt.model_executor.forward_batch_info import ForwardBatch -from sglang.srt.runtime_context import get_parallel - - -def tp_all_gather_hidden_states(hidden_states, forward_batch): - assert get_attn_tp_context().input_scattered, ( - "Input scattered guarantees same num tokens in TP group." - ) - total_tokens = forward_batch.input_ids.shape[0] - output = hidden_states.new_empty((total_tokens, hidden_states.shape[-1])) - get_parallel().tp_group.all_gather_into_tensor(output, hidden_states) - - return output @dataclass class MHCState: - """Parameters belong to the owning layer; this state only holds scratch + """A layer's residual as hyper-connection streams, with the residual + operations the boundary steps run (communicator.ResidualOps): the residual + is hc_mult streams, a stage's output is written in with hc_post and the + next stage's input read with hc_pre and the norm. A read produces the + h_res / h_post the next write-back consumes; they move with the tokens. + Parameters belong to the owning layer; this state only holds scratch shared across communication stages.""" hc_mult: int @@ -75,9 +41,19 @@ class MHCState: hc_ffn_pre: Callable hc_post: Callable hc_ffn_post_pre: Optional[Callable] = None + # The last layer's write-back also contracts the streams into the hidden + # states the layer stack hands on. + is_last_layer: bool = False h_res: Optional[torch.Tensor] = None h_post: Optional[torch.Tensor] = None + # hc_post is not a plain add, so it runs only once the sum it writes in + # is complete. + adds_plainly = False + # The FFN output's write-back uses the coefficients this layer's FFN input + # read produced, so the layer runs it itself. + updates_residual_after_ffn = True + @staticmethod def _resolve_out_norm(out_norm): if out_norm is None: @@ -131,261 +107,45 @@ class MHCState: self.h_res = None self.h_post = None + def enter(self, hidden_states): + return hc_expand(hidden_states, self.hc_mult) -class MHCCommunicateWithAllReduceAndLayerNormFn: - @staticmethod - def _scatter_hidden_states_and_residual( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - layernorm: torch.nn.Module, - context: CommunicateContext, - *, - residual_input_mode, - mhc: MHCState, + def read_attention_input(self, residual, norm, quant_format): + if quant_format: + raise NotImplementedError(f"an MHC attention input in {quant_format=}") + return self.attn_split(residual, out_norm=norm) + + def update_and_read_attention_input( + self, hidden_states, residual, norm, quant_format, post_residual_addition ): - input_hidden_states = hidden_states - hidden_states = hidden_states.tensor_split(context.attn_tp_size)[ - context.attn_tp_rank - ] - attn_tp_reduce_scatter_tensor(hidden_states, input_hidden_states) - if residual_input_mode == ScatterMode.TP_ATTN_FULL: - residual = residual.tensor_split(context.attn_tp_size)[context.attn_tp_rank] - mhc.h_res = mhc.h_res.tensor_split(context.attn_tp_size)[ - context.attn_tp_rank - ] - mhc.h_post = mhc.h_post.tensor_split(context.attn_tp_size)[ - context.attn_tp_rank - ] + # An MHC layer's input arrives with the previous FFN output written in. + raise NotImplementedError("an MHC layer takes its input written back") - hidden_states, residual = mhc.attn_to_mlp( - hidden_states, residual, out_norm=layernorm + def update_and_read_ffn_input(self, hidden_states, residual, norm): + return self.attn_to_mlp(hidden_states, residual, out_norm=norm) + + def update_residual(self, hidden_states, residual): + hidden_states = self.mlp_combine(hidden_states, residual) + self.reset_aux() + if self.is_last_layer: + hidden_states = hc_contract(hidden_states, self.hc_mult) + return hidden_states + + def residual_to_attn_tp_shard(self, residual, context): + rank, size = context.attn_tp_rank, context.attn_tp_size + self.h_res = self.h_res.tensor_split(size)[rank] + self.h_post = self.h_post.tensor_split(size)[rank] + return residual.tensor_split(size)[rank] + + def residual_from_attn_tp_shards(self, residual): + raise NotImplementedError( + "Unsupported: h_res/h_post allgather not implemented." ) - return hidden_states, residual - - @staticmethod - def _simple( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - layernorm: torch.nn.Module, - context: CommunicateContext, - *, - mhc: MHCState, - ): - hidden_states, residual = mhc.attn_to_mlp( - hidden_states, residual, out_norm=layernorm - ) - return hidden_states, residual - - @staticmethod - def _tp_all_reduce_with_scattered_residual( - hidden_states: torch.Tensor, - residual: torch.Tensor, - layernorm: torch.nn.Module, - context: CommunicateContext, - *, - mhc: MHCState, - ): - if hidden_states.shape[0] == 0: - return hidden_states, hidden_states - - scatter_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank] - get_parallel().tp_group.reduce_scatter_tensor(scatter_states, hidden_states) - - scatter_states, residual = mhc.attn_to_mlp( - scatter_states, residual, out_norm=layernorm - ) - - attn_tp_all_gather_into_tensor(hidden_states, scatter_states) - - return hidden_states, residual - - @staticmethod - def _gather_hidden_states_and_residual( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - layernorm: torch.nn.Module, - context: CommunicateContext, - *, - residual_input_mode, - mhc: MHCState, - ): - if get_attn_tp_context().input_scattered: - return MHCCommunicateWithAllReduceAndLayerNormFn._tp_all_reduce_with_scattered_residual( - hidden_states, - residual, - layernorm, - context, - mhc=mhc, - ) - - if residual_input_mode == ScatterMode.SCATTERED and context.attn_tp_size > 1: - raise NotImplementedError( - "Unsupported: h_res/h_post allgather not implemented." - ) - - hidden_states = attention_tensor_model_parallel_all_reduce(hidden_states) - if context.attn_dp_size != 1: - if hidden_states.shape[0] != 0: - with use_symmetric_memory( - get_parallel().tp_group, - disabled=not is_allocation_symmetric(), - ): - hidden_states, residual = mhc.attn_to_mlp( - hidden_states, residual, out_norm=layernorm - ) - else: - hidden_states, residual = mhc.attn_to_mlp(hidden_states, residual) - - hidden_states, local_hidden_states = ( - get_global_dp_buffer(get_parallel().tp_group), - hidden_states, - ) - dp_gather_replicate(hidden_states, local_hidden_states, forward_batch) - else: - hidden_states, residual = mhc.attn_to_mlp( - hidden_states, residual, out_norm=layernorm - ) - return hidden_states, residual - - -class MHCCommunicateSummableTensorPairFn(CommunicateSummableTensorPairFn): - @staticmethod - def get_fn( - hidden_states_input_mode: ScatterMode, - residual_input_mode: ScatterMode, - output_mode: ScatterMode, - context: CommunicateContext, - ): - fn = CommunicateSummableTensorPairFn.get_fn( - hidden_states_input_mode, - residual_input_mode, - output_mode, - context, - ) - replacements = { - CommunicateSummableTensorPairFn._trivial: MHCCommunicateSummableTensorPairFn._trivial, - CommunicateSummableTensorPairFn._scatter_hidden_states: MHCCommunicateSummableTensorPairFn._scatter_hidden_states, - CommunicateSummableTensorPairFn._gather: MHCCommunicateSummableTensorPairFn._gather, - CommunicateSummableTensorPairFn._scatter: MHCCommunicateSummableTensorPairFn._scatter, - } - return replacements.get(fn, fn) - - @staticmethod - def _trivial( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - context: CommunicateContext, - *, - mhc: MHCState, - is_last_layer: bool, - **kwargs, - ): - if get_attn_tp_context().input_scattered: - hidden_states, _ = tp_reduce_scatter(hidden_states, None, context) - - hidden_states = mhc.mlp_combine(hidden_states, residual) - if not is_last_layer: - return hidden_states, None - - hidden_states = hc_contract(hidden_states, mhc.hc_mult) - if get_attn_tp_context().input_scattered: - local_states = hidden_states - hidden_states = local_states.new_empty( - local_states.shape[0] * context.tp_size, *local_states.shape[1:] - ) - get_parallel().tp_group.all_gather_into_tensor(hidden_states, local_states) - - return hidden_states, None - - @staticmethod - def _scatter_hidden_states( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - context: CommunicateContext, - allow_reduce_scatter: bool = False, - *, - mhc: MHCState, - is_last_layer: bool, - **kwargs, - ): - hidden_states, global_hidden_states = ( - get_local_dp_buffer_mhc(get_parallel().tp_group, 1), - hidden_states, - ) - # MoE skips its post-expert all-reduce with reduce_scatterv, so this - # scatter must reduce while combining local-expert partial sums. - if should_use_dp_reduce_scatterv(): - get_parallel().tp_group.reduce_scatterv( - global_hidden_states, - output=hidden_states, - sizes=get_dp_global_num_tokens(), - ) - elif allow_reduce_scatter and forward_batch.dp_padding_mode.is_max_len(): - dp_reduce_scatter_tensor(hidden_states, global_hidden_states) - else: - dp_scatter(hidden_states, global_hidden_states, forward_batch) - - hidden_states = mhc.mlp_combine(hidden_states, residual) - if not is_last_layer: - return hidden_states, None - - hidden_states = hc_contract(hidden_states, mhc.hc_mult) - return hidden_states, None - - @staticmethod - def _gather( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - context: CommunicateContext, - *, - mhc: MHCState, - is_last_layer: bool, - **kwargs, - ): - hidden_states = mhc.mlp_combine(hidden_states, residual) - if is_last_layer: - hidden_states = hc_contract(hidden_states, mhc.hc_mult) - - hidden_states, local_hidden_states = ( - get_local_dp_buffer_mhc( - get_parallel().tp_group, 1 if is_last_layer else mhc.hc_mult - ), - hidden_states, - ) - - attn_tp_all_gather_into_tensor(hidden_states, local_hidden_states) - return hidden_states, None - - @staticmethod - def _scatter( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - context: CommunicateContext, - *, - mhc: MHCState, - is_last_layer: bool, - **kwargs, - ): - hidden_states = hidden_states.tensor_split(context.attn_tp_size)[ - context.attn_tp_rank - ] - residual = residual.tensor_split(context.attn_tp_size)[context.attn_tp_rank] - - hidden_states = mhc.mlp_combine(hidden_states, residual) - - return hidden_states, None class MHCLayerCommunicator(LayerCommunicator): - # Chooses its own boundary steps, not from the declarations. - _takes_declared_boundaries = False + """A layer whose residual is hyper-connection streams: the shared boundary + steps, run with MHCState's residual operations.""" def __init__( self, @@ -395,158 +155,50 @@ class MHCLayerCommunicator(LayerCommunicator): allow_reduce_scatter: bool = False, qkv_latent_func: Optional[Callable] = None, *, - is_first_layer: bool, hc_mult: int, hc_attn_pre: Callable, hc_ffn_pre: Callable, hc_post: Callable, hc_ffn_post_pre: Optional[Callable] = None, ): - self.is_first_layer = is_first_layer self.mhc = MHCState( hc_mult=hc_mult, hc_attn_pre=hc_attn_pre, hc_ffn_pre=hc_ffn_pre, hc_post=hc_post, hc_ffn_post_pre=hc_ffn_post_pre, + is_last_layer=layer_scatter_modes.is_last_layer, ) - + # The postprocess writes the FFN output into the streams, so the FFN's + # sum never waits for the next layer. super().__init__( layer_scatter_modes, input_layernorm, post_attention_layernorm, allow_reduce_scatter, qkv_latent_func, + allow_deferred_ffn_reduction=False, + residual_ops=self.mhc, ) + def _steps_from_declarations( + self, sides: DecoderLayerSides, **kwargs + ) -> BoundarySteps: + # MHC has not been run with an FFN input gathered over attention CP. + if ( + TokenAxis.ATTN_CP + in sides.attention_output.layout.sharded - sides.ffn.layout.sharded + ): + raise NotImplementedError( + f"MHCLayerCommunicator with a gather over attention CP: {sides=}" + ) + return super()._steps_from_declarations(sides, **kwargs) + def _post_init_communicate(self): - # Base MOE_FULL callables do not accept ``mhc``, so reject this - # combination at construction. if self.layer_scatter_modes.mlp_mode == ScatterMode.MOE_FULL: raise NotImplementedError( "MHCLayerCommunicator does not support MOE_FULL " "(moe_dp_size < attention_context_parallel_size). Increase " "moe_dp_size to match attention_context_parallel_size." ) - self._communicate_simple_fn = CommunicateSimpleFn.get_fn( - input_mode=self.layer_scatter_modes.layer_input_mode, - output_mode=self.layer_scatter_modes.attn_mode, - context=self._context, - ) - self._communicate_summable_tensor_pair_fn = ( - MHCCommunicateSummableTensorPairFn.get_fn( - hidden_states_input_mode=self.layer_scatter_modes.mlp_mode, - residual_input_mode=self.layer_scatter_modes.middle_residual_mode, - output_mode=self.layer_scatter_modes.layer_output_mode, - context=self._context, - ) - ) - # MHC's own prepare_attn and postprocess run these. - return self._communicate_simple_fn, self._communicate_summable_tensor_pair_fn - - def prepare_attn( - self, - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - ): - self.publish_attn_lora_layout() - if self.is_first_layer: - if get_attn_tp_context().input_scattered: - hidden_states, _ = tp_reduce_scatter( - hidden_states, - None, - self._context, - ) - hidden_states = hc_expand(hidden_states, self.mhc.hc_mult) - - hidden_states, residual = self.mhc.attn_split( - hidden_states, out_norm=self.input_layernorm - ) - - hidden_states = self._communicate_simple_fn( - hidden_states=hidden_states, - forward_batch=forward_batch, - context=self._context, - ) - - # DSA and attention without a QKV hook consume full hidden states, so - # gather them before attention. - ctx = get_attn_tp_context() - dsa_pre_gather = ctx.input_scattered and ctx.is_dsa - no_qkv_latent_pre_gather = ctx.input_scattered and self.qkv_latent_func is None - if dsa_pre_gather or no_qkv_latent_pre_gather: - hidden_states = tp_all_gather_hidden_states(hidden_states, forward_batch) - - if self.qkv_latent_func is not None: - attn_inputs = AttentionInputs( - hidden_states, - forward_batch, - self.qkv_latent_func, - is_pre_gathered=dsa_pre_gather, - ) - ctx.set_attn_inputs(attn_inputs) - - return hidden_states, residual - - def _select_mlp_input(self): - """MHC's own implementation of each boundary kind, applied to this - layer's MHC state.""" - kind = mlp_input_kind(self.layer_scatter_modes, self._context) - residual_input_mode = self.layer_scatter_modes.layer_input_mode - fns = MHCCommunicateWithAllReduceAndLayerNormFn - if kind is MlpInputKind.NORM: - return partial(fns._simple, mhc=self.mhc), () - if kind is MlpInputKind.GATHER: - fn = fns._gather_hidden_states_and_residual - elif kind is MlpInputKind.SCATTER: - fn = fns._scatter_hidden_states_and_residual - else: - raise NotImplementedError(f"MHCLayerCommunicator does not support {kind}") - return partial(fn, residual_input_mode=residual_input_mode, mhc=self.mhc), () - - def postprocess_layer(self, hidden_states, residual, forward_batch): - hidden_states, residual = self._communicate_summable_tensor_pair_fn( - hidden_states=hidden_states, - residual=residual, - forward_batch=forward_batch, - context=self._context, - allow_reduce_scatter=self.allow_reduce_scatter, - mhc=self.mhc, - is_last_layer=self.is_last_layer, - ) - self.mhc.reset_aux() - - return hidden_states, residual - - def _select_ffn_completion(self, forward_batch: ForwardBatch) -> FfnCompletion: - """An MHC layer's own postprocess completes its FFN output, combining the - hyper-connection streams; nothing is left to the next layer.""" - return FfnCompletion( - defer_moe_finalize=False, - fuse_mlp_allreduce=False, - mlp_reduce_scatter=self.should_use_reduce_scatter(forward_batch), - complete=partial( - self._complete_ffn_output_now, - forward_batch=forward_batch, - dp_step=None, - ), - ) - - def should_use_reduce_scatter(self, forward_batch: ForwardBatch): - if not self.allow_reduce_scatter: - return False - if ( - self._communicate_summable_tensor_pair_fn - is MHCCommunicateSummableTensorPairFn._scatter_hidden_states - ): - # reduce_scatterv already combines expert outputs; returning False - # would make RowParallelLinear perform an extra all-reduce. - if should_use_dp_reduce_scatterv(): - return True - if forward_batch.dp_padding_mode.is_max_len(): - return True - - if get_attn_tp_context().input_scattered: - return True - return False + return super()._post_init_communicate() diff --git a/python/sglang/srt/layers/dp_attention.py b/python/sglang/srt/layers/dp_attention.py index 3821ece881..bf6c1e8f67 100644 --- a/python/sglang/srt/layers/dp_attention.py +++ b/python/sglang/srt/layers/dp_attention.py @@ -258,27 +258,14 @@ class _DpGatheredBufferWrapper: return buffer @classmethod - def get_local_dp_buffer(cls, group: GroupCoordinator) -> torch.Tensor: - - dp = get_flags().dp - with use_symmetric_memory(group, disabled=not cls._dp_max_padding): - buffer = torch.empty( - (cls._local_dp_buffer_len, dp.buffer_hidden_size), - dtype=dp.buffer_dtype, - device=dp.buffer_device, - ) - return buffer - - @classmethod - def get_local_dp_buffer_mhc( - cls, group: GroupCoordinator, n: int = 1 + def get_local_dp_buffer( + cls, group: GroupCoordinator, hidden_size: Optional[int] = None ) -> torch.Tensor: - from sglang.srt.runtime_context import get_flags dp = get_flags().dp with use_symmetric_memory(group, disabled=not cls._dp_max_padding): buffer = torch.empty( - (cls._local_dp_buffer_len, dp.buffer_hidden_size * n), + (cls._local_dp_buffer_len, hidden_size or dp.buffer_hidden_size), dtype=dp.buffer_dtype, device=dp.buffer_device, ) @@ -344,12 +331,14 @@ def get_global_dp_buffer(group: GroupCoordinator) -> torch.Tensor: return _DpGatheredBufferWrapper.get_global_dp_buffer(group=group) -def get_local_dp_buffer(group: GroupCoordinator) -> torch.Tensor: - return _DpGatheredBufferWrapper.get_local_dp_buffer(group=group) - - -def get_local_dp_buffer_mhc(group: GroupCoordinator, n: int = 1) -> torch.Tensor: - return _DpGatheredBufferWrapper.get_local_dp_buffer_mhc(group=group, n=n) +def get_local_dp_buffer( + group: GroupCoordinator, hidden_size: Optional[int] = None +) -> torch.Tensor: + """A buffer for this rank's local DP rows, ``hidden_size`` wide (the model's + hidden size by default).""" + return _DpGatheredBufferWrapper.get_local_dp_buffer( + group=group, hidden_size=hidden_size + ) def get_global_dp_buffer_len() -> int: diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index c634008bd4..5f1567ea74 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -795,7 +795,6 @@ class Glm5NextDecoderLayer(nn.Module): if self.config.mhc: mhc_kwargs: Dict[str, Any] = dict( - is_first_layer=(self.layer_id == 0), hc_mult=config.hc_mult, hc_attn_pre=self.hc_attn_pre, hc_ffn_pre=self.hc_ffn_pre, diff --git a/test/registered/unit/layers/test_communicator_ffn_exit.py b/test/registered/unit/layers/test_communicator_ffn_exit.py index 5e3a6553d5..458d47d78d 100644 --- a/test/registered/unit/layers/test_communicator_ffn_exit.py +++ b/test/registered/unit/layers/test_communicator_ffn_exit.py @@ -94,11 +94,8 @@ def make_communicator( communicator._cp_steps = None communicator._postprocess_dp_step = MagicMock(return_value=reduce_scatter_step) communicator.ffn_reduction_group = MagicMock(return_value=group or make_group()) - communicator.postprocess_layer = MagicMock( - side_effect=lambda hidden_states, residual, forward_batch: ( - hidden_states + 1, - residual, - ) + communicator._complete_ffn_output_now = MagicMock( + side_effect=lambda hidden_states, residual, **_: (hidden_states + 1, residual) ) return communicator @@ -126,7 +123,7 @@ class TestFfnExit(CustomTestCase): self.assertEqual(seen, (True, False)) self.assertIsInstance(hidden_states, UnreducedOutput) self.assertIs(residual, self.residual) - communicator.postprocess_layer.assert_not_called() + communicator._complete_ffn_output_now.assert_not_called() def test_reduction_left_to_next_layer_declares_its_group(self): group = make_group() @@ -152,7 +149,7 @@ class TestFfnExit(CustomTestCase): self.assertIs(bound.func, comm._to_local_tokens) self.assertEqual(bound.args, (step, self.forward_batch)) self.assertIs(residual, self.residual) - communicator.postprocess_layer.assert_not_called() + communicator._complete_ffn_output_now.assert_not_called() step.assert_not_called() def test_a_deferred_sum_carries_the_scatter_back_under_attention_dp(self): @@ -165,7 +162,7 @@ class TestFfnExit(CustomTestCase): bound = hidden_states.reduce_and_redistribute self.assertIs(bound.func, comm._all_reduce_then_to_local_tokens) self.assertEqual(bound.args, (group, self.forward_batch)) - communicator.postprocess_layer.assert_not_called() + communicator._complete_ffn_output_now.assert_not_called() group.all_reduce.assert_not_called() def test_postprocess_completes_other_exits(self): @@ -177,7 +174,7 @@ class TestFfnExit(CustomTestCase): seen, (hidden_states, residual) = self.run_exit(communicator) self.assertEqual(seen, (False, reduce_scatter)) self.assertNotIsInstance(hidden_states, UnreducedOutput) - communicator.postprocess_layer.assert_called_once() + communicator._complete_ffn_output_now.assert_called_once() torch.testing.assert_close(hidden_states, self.hidden_states * 2 + 1) self.assertIs(residual, self.residual) @@ -189,7 +186,7 @@ class TestFfnExit(CustomTestCase): self.assertEqual(seen, (False, True)) communicator._ffn_sum_moves_to_next_layer.assert_not_called() - communicator.postprocess_layer.assert_called_once() + communicator._complete_ffn_output_now.assert_called_once() torch.testing.assert_close(hidden_states, self.hidden_states * 2 + 1) def test_finish_applies_the_decision_the_ffn_saw(self): @@ -206,7 +203,7 @@ class TestFfnExit(CustomTestCase): hidden_states, _ = ffn_exit.finish(hidden_states, self.residual) self.assertEqual(seen, (fuse, False)) self.assertEqual(isinstance(hidden_states, UnreducedOutput), fuse) - self.assertEqual(communicator.postprocess_layer.called, not fuse) + self.assertEqual(communicator._complete_ffn_output_now.called, not fuse) def test_flags_are_restored_after_the_ffn(self): before = published_flags() @@ -223,7 +220,7 @@ class TestFfnExit(CustomTestCase): ) seen, _ = self.run_exit(communicator) self.assertEqual(seen, (False, False)) - communicator.postprocess_layer.assert_called_once() + communicator._complete_ffn_output_now.assert_called_once() def test_deferral_implies_fusion_and_passes_the_handoff_through(self): """A deferring communicator publishes both flags, and a non-tensor @@ -244,7 +241,7 @@ class TestFfnExit(CustomTestCase): self.assertEqual( ffn_exit.finish(handoff, self.residual), (handoff, self.residual) ) - communicator.postprocess_layer.assert_not_called() + communicator._complete_ffn_output_now.assert_not_called() self.assertFalse(get_forward().defer_moe_finalize) # The MoE may decline per forward; a tensor then takes the fused exit. diff --git a/test/registered/unit/layers/test_communicator_layout.py b/test/registered/unit/layers/test_communicator_layout.py index c1e7c55556..542821fe2a 100644 --- a/test/registered/unit/layers/test_communicator_layout.py +++ b/test/registered/unit/layers/test_communicator_layout.py @@ -184,10 +184,12 @@ GATHER_LAYOUT = (TP_ATTN_FULL, TP_ATTN_FULL, FULL, TP_ATTN_FULL) class TestMlpInputOrder(CustomTestCase): - def assert_order(self, order, func, **keywords): + def assert_order(self, order, func, residual_ops=comm.ADD_AND_NORM, **keywords): self.assertIsInstance(order, partial) self.assertIs(order.func, func) - self.assertEqual(order.keywords, keywords) + bound = dict(order.keywords) + self.assertIs(bound.pop("residual_ops", comm.ADD_AND_NORM), residual_ops) + self.assertEqual(bound, keywords) def test_facts_fixed_at_construction_pick_the_order(self): fusions = (object(),) @@ -268,41 +270,63 @@ class TestMlpInputOrder(CustomTestCase): ): with self.subTest(expected=expected.__name__): _, (steps, fused) = communicator(layout, context, Fusable()) - self.assertIs(steps, expected) + self.assert_order(steps, expected) self.assertEqual(fused, ()) _, (steps, _) = communicator( (TP_ATTN_FULL, TP_ATTN_FULL, SCATTERED, SCATTERED), tp, Fusable() ) self.assert_order(steps, comm._mlp_input_scatter, scatters_residual=True) - def test_mhc_picks_its_own_implementation_of_the_kind(self): - from sglang.srt.layers.communicator_mhc import ( - MHCCommunicateWithAllReduceAndLayerNormFn as MHC, - ) + def test_mhc_runs_the_shared_steps_with_its_residual(self): from sglang.srt.layers.communicator_mhc import ( MHCLayerCommunicator, + MHCState, ) - mhc = object() - for layout, func in ( - (GATHER_LAYOUT, MHC._gather_hidden_states_and_residual), + mhc = MHCState(hc_mult=2, hc_attn_pre=None, hc_ffn_pre=None, hc_post=None) + for layout, func, order, keywords in ( + ( + GATHER_LAYOUT, + comm._mlp_input_gather, + comm._mlp_input_without_dp, + # The fused add + RMSNorm kernels do not write hc_post. + dict(gathers_residual=False, fusions=()), + ), ( (TP_ATTN_FULL, TP_ATTN_FULL, SCATTERED, SCATTERED), - MHC._scatter_hidden_states_and_residual, + comm._mlp_input_scatter, + None, + dict(scatters_residual=True), ), ): with self.subTest(func=func.__name__): c = MHCLayerCommunicator.__new__(MHCLayerCommunicator) - c.mhc = mhc + c._residual_ops = mhc c.layer_scatter_modes = modes(*layout) c._context = make_context(tp=2) c.post_attention_layernorm = Fusable() steps, fused = c._select_mlp_input() - self.assert_order( - steps, func, residual_input_mode=TP_ATTN_FULL, mhc=mhc - ) + if order is None: + self.assert_order(steps, func, residual_ops=mhc, **keywords) + else: + self.assertIs(steps.func, func) + self.assert_order( + steps.keywords["order"], order, residual_ops=mhc, **keywords + ) self.assertEqual(fused, ()) + def test_mhc_gathers_over_dp_after_its_write_back(self): + from sglang.srt.layers.communicator_mhc import MHCState + + mhc = MHCState(hc_mult=2, hc_attn_pre=None, hc_ffn_pre=None, hc_post=None) + self.assert_order( + comm._mlp_input_order(make_context(dp=2, tp=2), SCATTERED, (), mhc), + comm._mlp_input_dp_replicate, + residual_ops=mhc, + gathers_residual=True, + reduces_attention_tp=True, + ) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/layers/test_declared_decoder_boundary.py b/test/registered/unit/layers/test_declared_decoder_boundary.py index d0b3cdaa9c..0ddfc8cb34 100644 --- a/test/registered/unit/layers/test_declared_decoder_boundary.py +++ b/test/registered/unit/layers/test_declared_decoder_boundary.py @@ -15,11 +15,13 @@ import unittest from contextlib import ExitStack, contextmanager, nullcontext from functools import partial from types import SimpleNamespace -from unittest.mock import patch +from unittest.mock import MagicMock, patch +import msgspec import torch from sglang.srt.layers import communicator as comm +from sglang.srt.layers import communicator_mhc as mhc_module from sglang.srt.layers.boundary_layout import ( SumGroup, TokenAxis, @@ -365,7 +367,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase): single = build(layer_facts(1, 3), one_rank) with planning(one_rank): self.assertIsNotNone(single._declared_sides()) - self.assertIs(single._steps.ffn_input, comm._mlp_input_norm) + self.assertIs(single._steps.ffn_input.func, comm._mlp_input_norm) def test_the_order_follows_the_attention_output(self): for attn_tp, force, order in ( @@ -482,7 +484,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase): a2a=True, ) self.assertIs( - last._steps.ffn_output_move, + last._steps.ffn_output_move.func, comm.CommunicateSummableTensorPairFn._gather, ) @@ -534,10 +536,288 @@ class TestWhichLayersUseDeclarations(CustomTestCase): self.assertFalse(self.declared(build(direct, dp))) def test_subclasses_that_pick_their_own_steps(self): - for cls in (MHCLayerCommunicator, CuteDSLFusionLayerCommunicator): + self.assertFalse(CuteDSLFusionLayerCommunicator._takes_declared_boundaries) + for cls in (LayerCommunicator, MHCLayerCommunicator): with self.subTest(cls.__name__): - self.assertFalse(cls._takes_declared_boundaries) - self.assertTrue(LayerCommunicator._takes_declared_boundaries) + self.assertTrue(cls._takes_declared_boundaries) + + +def build_mhc( + modes, parallel, *, a2a=False, dsa_cp=False, two_batch_overlap=False, **kwargs +): + overlap = SimpleNamespace( + overlap=SimpleNamespace(enable_two_batch_overlap=two_batch_overlap) + ) + with ( + planning(parallel, a2a=a2a, dsa_cp=dsa_cp), + patch.object(comm, "get_exec", lambda: overlap), + ): + return MHCLayerCommunicator( + layer_scatter_modes=modes, + input_layernorm=Norm(), + post_attention_layernorm=Norm(), + hc_mult=2, + hc_attn_pre=lambda *a: None, + hc_ffn_pre=lambda *a: None, + **{"hc_post": lambda *a: None, **kwargs}, + ) + + +class TestMhcOnTheDeclarations(CustomTestCase): + """An MHC layer takes the same declarations and runs the same steps as any + other layer; only the residual operations the steps run are MHC's.""" + + def assert_step(self, step, func, communicator, **keywords): + self.assertIs(step.func, func) + self.assertIs(step.keywords["residual_ops"], communicator.mhc) + for key, value in keywords.items(): + self.assertEqual(step.keywords[key], value, key) + + def test_each_boundary_runs_the_shared_step(self): + for name, parallel, ffn_input, output_move in ( + ( + "attention TP 1", + parallel_of(attn_dp=1, attn_tp=1), + comm._mlp_input_norm, + comm.CommunicateSummableTensorPairFn._trivial, + ), + ( + "the attention-TP sum", + parallel_of(attn_dp=1, attn_tp=2), + comm._mlp_input_without_dp, + comm.CommunicateSummableTensorPairFn._trivial, + ), + # The DP gather runs after hc_post, which is not a plain add. + ( + "a DP gather", + parallel_of(attn_dp=2, attn_tp=2), + comm._mlp_input_dp_replicate, + None, + ), + ): + with self.subTest(name): + communicator = build_mhc(layer_facts(1, 3), parallel) + self.assert_step(communicator._steps.ffn_input, ffn_input, communicator) + self.assertIs(communicator._steps.ffn_output_move, output_move) + # The fused add + RMSNorm kernels do not write hc_post. + self.assertEqual(communicator._steps.fused, ()) + self.assertEqual(communicator._attn_input_fusions, ()) + + def test_the_move_back_over_dp_stays_with_the_layer(self): + parallel = parallel_of(attn_dp=2, attn_tp=2) + communicator = build_mhc(layer_facts(1, 3), parallel, allow_reduce_scatter=True) + self.assertTrue(communicator._steps.returns_over_dp) + self.assertFalse(communicator._steps.ffn_output.leaves_for_next_layer) + # The move back also writes the FFN output into the streams. + self.assertFalse(communicator._local_token_move_can_go_to_next_layer(None)) + max_len = SimpleNamespace( + dp_padding_mode=SimpleNamespace(is_max_len=lambda: True), + forward_mode=SimpleNamespace(is_context_parallel_extend=lambda: False), + ) + with ( + planning(parallel), + patch.object(comm, "get_forward", lambda: SimpleNamespace(sp_active=False)), + patch.object(comm, "should_use_dp_reduce_scatterv", lambda: False), + patch.object(comm, "can_use_dp_reduce_scatter", lambda: True), + ): + self.assertTrue(communicator.should_use_reduce_scatter(max_len)) + + def test_the_exit_runs_the_move_back_over_dp_it_chose(self): + parallel = parallel_of(attn_dp=2, attn_tp=2) + for name, reduce_scatterv, is_max_len, move in ( + ("SUM_LEN", True, False, comm._reduce_and_redistribute_output_varlen), + ("MAX_LEN", False, True, comm._reduce_and_redistribute_output_max_len), + ): + with self.subTest(name): + hc_post = MagicMock(side_effect=lambda h, r, h_res, h_post: h + r) + communicator = build_mhc( + layer_facts(1, 3), + parallel, + allow_reduce_scatter=True, + hc_post=hc_post, + ) + communicator.mhc.h_res = communicator.mhc.h_post = torch.zeros(2) + forward_batch = SimpleNamespace( + dp_padding_mode=SimpleNamespace(is_max_len=lambda: is_max_len), + forward_mode=SimpleNamespace( + is_context_parallel_extend=lambda: False + ), + ) + choose = MagicMock(wraps=comm._reduce_and_redistribute_output_step) + to_local_tokens = MagicMock(side_effect=lambda step, fb, h: h[:2]) + with ( + planning(parallel), + patch.object( + comm, "should_use_dp_reduce_scatterv", lambda: reduce_scatterv + ), + patch.object(comm, "can_use_dp_reduce_scatter", lambda: True), + patch.object(comm, "_reduce_and_redistribute_output_step", choose), + patch.object(comm, "_to_local_tokens", to_local_tokens), + ): + with communicator.ffn_exit(forward_batch) as ffn_exit: + self.assertTrue(ffn_exit.mlp_reduce_scatter) + hidden, residual = ffn_exit.finish( + torch.ones(4, 4), torch.full((2, 4), 2.0) + ) + choose.assert_called_once() + to_local_tokens.assert_called_once() + self.assertIs(to_local_tokens.call_args.args[0], move) + hc_post.assert_called_once() + self.assertIsNone(residual) + torch.testing.assert_close(hidden, torch.full((2, 4), 3.0)) + + def test_a2a_layers_take_their_slice_with_the_coefficients(self): + parallel = parallel_of(attn_dp=2, attn_tp=2) + layers = [ + build_mhc( + planned_modes( + i, + 3, + sparse=sparse, + previous_sparse=previous, + parallel=parallel, + a2a=True, + ), + parallel, + a2a=True, + ) + for i, (sparse, previous) in enumerate( + ((False, False), (True, False), (True, True)) + ) + ] + first_a2a, last = layers[1], layers[2] + self.assert_step( + first_a2a._steps.ffn_input, + comm._mlp_input_scatter, + first_a2a, + scatters_residual=True, + ) + self.assert_step( + last._steps.ffn_input, + comm._mlp_input_scatter, + last, + scatters_residual=False, + ) + self.assert_step( + last._steps.ffn_output_move, + comm.CommunicateSummableTensorPairFn._gather, + last, + ) + state = first_a2a.mhc + state.h_res, state.h_post = torch.arange(4.0), torch.arange(4.0) + 10 + context = SimpleNamespace(attn_tp_rank=1, attn_tp_size=2) + residual = state.residual_to_attn_tp_shard(torch.arange(4.0) + 20, context) + self.assertEqual(residual.tolist(), [22.0, 23.0]) + self.assertEqual(state.h_res.tolist(), [2.0, 3.0]) + self.assertEqual(state.h_post.tolist(), [12.0, 13.0]) + + def test_an_input_scattered_batch_keeps_the_residual_on_the_slice(self): + parallel = parallel_of( + attn_dp=1, attn_tp=2, enable_attn_tp_input_scattered=True + ) + for layer_id in range(3): + with self.subTest(layer_id=layer_id): + communicator = build_mhc( + layer_facts(layer_id, 3), parallel, allow_reduce_scatter=True + ) + steps = communicator._input_scattered_steps + self.assert_step( + steps.ffn_input, comm._mlp_input_on_residual_shard, communicator + ) + self.assert_step( + steps.ffn_output_move, + comm.CommunicateSummableTensorPairFn._onto_residual_shard, + communicator, + sums=True, + gathers_back=layer_id == 2, + ) + self.assertTrue(steps.ffn_output_move_completes_sum) + completes = build_mhc( + layer_facts(layer_id, 3), parallel, allow_reduce_scatter=False + )._input_scattered_steps + self.assertFalse(completes.ffn_output_move.keywords["sums"]) + self.assertFalse(completes.ffn_output_move_completes_sum) + # Only the first layer's input, the embedding's partial sum, + # is completed onto the slice. + self.assertIs( + steps.layer_input, + comm.tp_reduce_scatter if layer_id == 0 else None, + ) + self.assertIs(steps.attention_input, comm.CommunicateSimpleFn._trivial) + self.assertIs( + steps.attention_handoff, comm._hand_scattered_input_to_attention + ) + self.assertIs( + communicator._steps.ffn_output_move, + comm.CommunicateSummableTensorPairFn._trivial, + ) + + def test_two_batch_overlap_keeps_the_scatter_modes(self): + # A dense layer before a sparse one gathers its output for the split, + # so dense layers on every rank keep their scatter-mode steps. + parallel = parallel_of(attn_dp=2, attn_tp=2, moe_dense_tp_size=1) + modes = planned_modes( + 1, 3, sparse=False, previous_sparse=False, parallel=parallel + ) + declared = build_mhc(modes, parallel) + kept = build_mhc(modes, parallel, two_batch_overlap=True) + # Both run the shared slice onto each rank's rows with MHC's residual. + for communicator in (declared, kept): + self.assert_step( + communicator._steps.ffn_input, comm._mlp_input_scatter, communicator + ) + # The declared layer never reads a scatter mode; the kept one does. + build_mhc(layer_facts(1, 3), parallel) + with self.assertRaises(AssertionError): + build_mhc(layer_facts(1, 3), parallel, two_batch_overlap=True) + + def test_the_postprocess_writes_the_output_into_the_streams(self): + # The last layer also contracts the streams into the hidden states. + parallel = parallel_of(attn_dp=1, attn_tp=1) + for layer_id, contracted in ((1, False), (2, True)): + with self.subTest(layer_id=layer_id): + communicator = build_mhc( + layer_facts(layer_id, 3), + parallel, + hc_post=lambda h, r, h_res, h_post: h + r, + ) + communicator.mhc.h_res = communicator.mhc.h_post = torch.zeros(2) + with ( + planning(parallel), + patch.object( + comm, "get_forward", lambda: SimpleNamespace(sp_active=False) + ), + patch.object( + mhc_module, "hc_contract", lambda h, hc_mult: h.sum(-1) + ), + ): + hidden, residual = communicator.postprocess_layer( + torch.ones(2, 4), torch.full((2, 4), 2.0), None + ) + self.assertIsNone(residual) + expected = torch.full((2, 4), 3.0) + torch.testing.assert_close( + hidden, expected.sum(-1) if contracted else expected + ) + # The write-back consumes this layer's coefficients. + self.assertIsNone(communicator.mhc.h_res) + + def test_a_gather_over_attention_cp_is_rejected(self): + # A MoE on the TP group under DSA prefill CP gathers its input over + # attention CP, which MHC has not been run with. + parallel = parallel_of( + attn_dp=1, attn_tp=1, attn_cp=2, enable_prefill_cp=True, moe_dense_tp_size=1 + ) + modes = planned_modes( + 1, 3, sparse=True, previous_sparse=False, parallel=parallel, dsa_cp=True + ) + with self.assertRaises(NotImplementedError): + build_mhc(modes, parallel, dsa_cp=True, allow_reduce_scatter=True) + # A dense layer on every rank computes on its own shard: nothing moves. + modes = planned_modes( + 1, 3, sparse=False, previous_sparse=False, parallel=parallel, dsa_cp=True + ) + build_mhc(modes, parallel, dsa_cp=True, allow_reduce_scatter=True) class TestTheAttentionOutputDecidesItsSum(CustomTestCase): @@ -723,6 +1003,11 @@ class TestTheSequenceParallelRegion(CustomTestCase): ffn_sum_is_movable=False, ) + def unbound(self, steps): + """``steps`` with the plain residual's binding taken off the FFN input.""" + self.assertEqual(steps.ffn_input.keywords, {"residual_ops": comm.ADD_AND_NORM}) + return msgspec.structs.replace(steps, ffn_input=steps.ffn_input.func) + def test_the_region_declarations_choose_local_steps(self): for attn_tp in (2, 4): with self.subTest(attn_tp=attn_tp): @@ -739,7 +1024,7 @@ class TestTheSequenceParallelRegion(CustomTestCase): self.assertEqual(sides.ffn.gathers_itself, local) self.assertIsNone(sides.attention_output.group) self.assertIsNone(sides.ffn_output.group) - steps = comm._select_boundary_steps(sides) + steps = self.unbound(comm._select_boundary_steps(sides)) self.assertEqual( (steps.attention_input, steps.ffn_input, steps.ffn_output_move), ( @@ -760,7 +1045,7 @@ class TestTheSequenceParallelRegion(CustomTestCase): input_layernorm=Norm(), post_attention_layernorm=Norm(), ) - self.assertEqual(communicator._sp_steps, self.SP_STEPS) + self.assertEqual(self.unbound(communicator._sp_steps), self.SP_STEPS) self.assertIs( communicator._steps.ffn_input.func, comm._mlp_input_without_dp ) @@ -1042,7 +1327,7 @@ class TestPrefillCP(CustomTestCase): communicator = build(modes, parallel, dsa_cp=True, allow_reduce_scatter=True) cp, ordinary = communicator._cp_steps, communicator._steps self.assertIs(cp.ffn_input.func, comm._mlp_input_gather_attention_cp) - self.assertIs(cp.ffn_input.keywords["gather"], comm._mlp_input_norm) + self.assertIs(cp.ffn_input.keywords["gather"].func, comm._mlp_input_norm) self.assertIs( cp.ffn_output_move, comm.CommunicateSummableTensorPairFn._reduce_scatter_over_cp, @@ -1052,7 +1337,7 @@ class TestPrefillCP(CustomTestCase): # The dense layer before it ran on the same shard: nothing to move. self.assertIs(cp.attention_input, comm.CommunicateSimpleFn._trivial) # Other batches hold every token on each CP rank: the MoE sums itself. - self.assertIs(ordinary.ffn_input, comm._mlp_input_norm) + self.assertIs(ordinary.ffn_input.func, comm._mlp_input_norm) self.assertIs( ordinary.ffn_output_move, comm.CommunicateSummableTensorPairFn._trivial ) @@ -1109,7 +1394,7 @@ class TestPrefillCP(CustomTestCase): ) communicator = build(modes, parallel, dsa_cp=True, allow_reduce_scatter=True) for steps in (communicator._steps, communicator._cp_steps): - self.assertIs(steps.ffn_input, comm._mlp_input_norm) + self.assertIs(steps.ffn_input.func, comm._mlp_input_norm) self.assertIsNone(steps.ffn_output.group) self.assertIs( steps.ffn_output_move, comm.CommunicateSummableTensorPairFn._trivial @@ -1393,8 +1678,8 @@ def running(*, reduce_scatterv, a2a=False): "get_global_dp_buffer": lambda g: torch.zeros( state().global_rows, HIDDEN, dtype=torch.double ), - "get_local_dp_buffer": lambda g: torch.zeros( - state().local_rows, HIDDEN, dtype=torch.double + "get_local_dp_buffer": lambda g, hidden_size=None: torch.zeros( + state().local_rows, hidden_size or HIDDEN, dtype=torch.double ), "dp_gather_replicate": partial(fake_dp_gather, is_partial=False), "dp_gather_partial": partial(fake_dp_gather, is_partial=True), @@ -1672,5 +1957,47 @@ class TestTwoLayers(CustomTestCase): ) +class TestTheFfnInputReduction(CustomTestCase): + """With quantized communications a prefill reduces the FFN input quantized + for a plain residual; MHC sums its streams in full precision.""" + + def test_only_a_plain_residual_reduces_quantized(self): + exec_ = SimpleNamespace(comm=SimpleNamespace(enable_quant_communications=True)) + batch = SimpleNamespace( + forward_mode=SimpleNamespace(is_decode_or_idle=lambda: False) + ) + for adds_plainly, reduction in ((True, "quant"), (False, "full")): + with self.subTest(adds_plainly=adds_plainly): + ops = SimpleNamespace( + adds_plainly=adds_plainly, + update_and_read_ffn_input=lambda h, r, norm: (h, r), + ) + calls = [] + with ( + patch.object(comm, "get_exec", lambda: exec_), + patch.object( + comm, + "attention_tensor_model_parallel_quant_all_reduce", + lambda h: calls.append("quant") or h, + ), + patch.object( + comm, + "attention_tensor_model_parallel_all_reduce", + lambda h: calls.append("full") or h, + ), + ): + comm._mlp_input_without_dp( + torch.ones(2, HIDDEN), + torch.ones(2, HIDDEN), + batch, + None, + SimpleNamespace(cache=None), + gathers_residual=False, + fusions=(), + residual_ops=ops, + ) + self.assertEqual(calls, [reduction]) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/layers/test_dp_attention_cp_gather.py b/test/registered/unit/layers/test_dp_attention_cp_gather.py index 162cb8c68d..6fed5c265f 100644 --- a/test/registered/unit/layers/test_dp_attention_cp_gather.py +++ b/test/registered/unit/layers/test_dp_attention_cp_gather.py @@ -231,7 +231,9 @@ class TestDpCpGather(CustomTestCase): ), ( (comm, "get_local_dp_buffer"), - lambda group: torch.empty(local_buffer_len, HIDDEN).double(), + lambda group, hidden_size=None: torch.empty( + local_buffer_len, hidden_size or HIDDEN + ).double(), ), ]: stack.enter_context(patch.object(*target, value)) diff --git a/test/registered/unit/layers/test_prepare_attn_steps.py b/test/registered/unit/layers/test_prepare_attn_steps.py index 04705d3c32..43f4c2de15 100644 --- a/test/registered/unit/layers/test_prepare_attn_steps.py +++ b/test/registered/unit/layers/test_prepare_attn_steps.py @@ -42,6 +42,8 @@ class Norm: def communicator(norm): c = comm.LayerCommunicator.__new__(comm.LayerCommunicator) c.input_layernorm = norm + # A layer inside the stack: an absent residual was written back before. + c.layer_scatter_modes = SimpleNamespace(is_first_layer=False) c._sp_steps = None c._input_scattered_steps = None c._cp_steps = None diff --git a/test/registered/unit/lora/test_triton_dp_attention_unit.py b/test/registered/unit/lora/test_triton_dp_attention_unit.py index 5dc8c6f0c3..6f3f07dbb1 100644 --- a/test/registered/unit/lora/test_triton_dp_attention_unit.py +++ b/test/registered/unit/lora/test_triton_dp_attention_unit.py @@ -220,7 +220,9 @@ def test_communicator_publishes_layout_at_each_transition( expected_mlp = expected_attn = initial communicator = LayerCommunicator.__new__(LayerCommunicator) communicator._publish_lora_layout = publish_lora_layout - communicator.layer_scatter_modes = SimpleNamespace(mlp_mode=mlp_mode) + communicator.layer_scatter_modes = SimpleNamespace( + mlp_mode=mlp_mode, is_first_layer=False + ) communicator._context = SimpleNamespace() communicator._sp_steps = None communicator._input_scattered_steps = None @@ -231,6 +233,7 @@ def test_communicator_publishes_layout_at_each_transition( communicator._steps = SimpleNamespace( attention_input=lambda hidden_states, **kwargs: hidden_states, layer_input=None, + attention_handoff=lambda hidden_states, *args: hidden_states, ffn_input=lambda hidden_states, residual, *args: (hidden_states, residual), ) communicator._attn_input_fusions = () diff --git a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py index b9d5f40579..0c99099503 100644 --- a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py +++ b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py @@ -69,9 +69,8 @@ class DeferringLayer(nn.Module): self.layer_communicator._ffn_leaves_sum_to_reduce_scatter = ( lambda batch, dp_step: False ) - self.layer_communicator.postprocess_layer = lambda hidden, residual, batch: ( - all_reduce(hidden), - residual, + self.layer_communicator._complete_ffn_output_now = ( + lambda hidden, residual, **_: (all_reduce(hidden), residual) ) def forward( diff --git a/test/registered/unit/models/test_longcat_flash_shortcut.py b/test/registered/unit/models/test_longcat_flash_shortcut.py index a4651e2973..e85c106247 100644 --- a/test/registered/unit/models/test_longcat_flash_shortcut.py +++ b/test/registered/unit/models/test_longcat_flash_shortcut.py @@ -59,7 +59,7 @@ class TestLongcatShortcut(CustomTestCase): patch.object( comm, "get_local_dp_buffer", - side_effect=lambda group: torch.empty(rows, 3), + side_effect=lambda group, hidden_size=None: torch.empty(rows, 3), ), patch.object( comm,