diff --git a/python/sglang/srt/layers/communicator/__init__.py b/python/sglang/srt/layers/communicator/__init__.py index 4ff31e3fcc..ea64def84b 100644 --- a/python/sglang/srt/layers/communicator/__init__.py +++ b/python/sglang/srt/layers/communicator/__init__.py @@ -25,6 +25,7 @@ from sglang.srt.layers.communicator.boundary import ( InputRead, LayerStage, StageDecl, + StageEntry, StageInput, StageOutput, decoder_layer_edges, @@ -67,16 +68,20 @@ from sglang.srt.layers.communicator.output import ( UnreducedOutput, reduce_output, ) +from sglang.srt.layers.communicator.residual import LayerResidual from sglang.srt.layers.communicator.residual.add_norm import ( - ADD_AND_NORM, + ADD, FUSE_ALLREDUCE_MAX_BATCH_SIZE, + NORM_QUANT_READ, + NORM_READ, + PLAIN_RESIDUAL, ) from sglang.srt.layers.communicator.residual.mhc import ( MHCState, ) __all__ = [ - "ADD_AND_NORM", + "ADD", "AttentionInputs", "BoundarySteps", "CommunicateContext", @@ -92,14 +97,19 @@ __all__ = [ "HandoffOutput", "InputRead", "LayerCommunicator", + "LayerResidual", "LayerScatterModes", "LayerStage", "Layout", "MHCLayerCommunicator", "MHCState", "MixerExit", + "NORM_QUANT_READ", + "NORM_READ", + "PLAIN_RESIDUAL", "ScatterMode", "StageDecl", + "StageEntry", "StageInput", "StageOutput", "SumGroup", diff --git a/python/sglang/srt/layers/communicator/boundary.py b/python/sglang/srt/layers/communicator/boundary.py index 285abefa94..7940efc8bf 100644 --- a/python/sglang/srt/layers/communicator/boundary.py +++ b/python/sglang/srt/layers/communicator/boundary.py @@ -54,8 +54,16 @@ from sglang.srt.layers.communicator.output import ( UnreducedOutput, reduce_output, ) -from sglang.srt.layers.communicator.residual import ResidualOps -from sglang.srt.layers.communicator.residual.add_norm import ADD_AND_NORM +from sglang.srt.layers.communicator.residual import ( + LayerResidual, + StageRead, + StageUpdate, +) +from sglang.srt.layers.communicator.residual.add_norm import ( + ADD, + NORM_QUANT_READ, + NORM_READ, +) from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.runtime_context import get_parallel @@ -68,6 +76,8 @@ class StageInput(msgspec.Struct, frozen=True): # Token axes the consumer gathers over itself when its input arrives # sharded over them. gathers_itself: FrozenSet[TokenAxis] = frozenset() + # How the consumer reads its input from the residual. + read: StageRead = NORM_READ class StageOutput(msgspec.Struct, frozen=True): @@ -93,6 +103,8 @@ class StageOutput(msgspec.Struct, frozen=True): # Whether it leaves the sum to the attention-DP reduce_scatterv whenever that # combine applies, as a MoE block does without any flag. leaves_for_reduce_scatterv: bool = False + # How the producer's output is written into the residual. + update: StageUpdate = ADD class DecoderLayerSides(msgspec.Struct, frozen=True): @@ -156,7 +168,14 @@ def decoder_layer_edges(sides: DecoderLayerSides) -> DecoderLayerEdges: construction (``input_owes``); a sum the previous layer leaves for a batch comes with the value.""" owes = sides.input_owes is not None - arrived = StageOutput(sides.input_rows, group=sides.input_owes, always_leaves=owes) + # The previous layer's FFN output, written in as this layer's FFN writes its + # own. + arrived = StageOutput( + sides.input_rows, + group=sides.input_owes, + always_leaves=owes, + update=sides.ffn_output.update, + ) # Completing what the input owes leaves it and the residual on each # attention-TP rank's slice. attention_rows = ( @@ -187,6 +206,23 @@ def decoder_layer_edges(sides: DecoderLayerSides) -> DecoderLayerEdges: ) +def with_residual( + sides: DecoderLayerSides, residual: LayerResidual +) -> DecoderLayerSides: + """``sides`` with the reads and updates the layer declares for its + attention and its FFN.""" + replace = msgspec.structs.replace + return replace( + sides, + attention=replace(sides.attention, read=residual.attention_read), + attention_output=replace( + sides.attention_output, update=residual.attention_update + ), + ffn=replace(sides.ffn, read=residual.ffn_read), + ffn_output=replace(sides.ffn_output, update=residual.ffn_update), + ) + + def stage_edges( *, previous: Optional[StageOutput], stage: StageDecl, rows: Layout ) -> Tuple[EdgeDecl, EdgeDecl]: @@ -198,16 +234,18 @@ def stage_edges( first, and what it may leave of its sum comes with the value; otherwise the value is complete. The residual follows the input onto a finer slice and stays where it is when the input is gathered.""" + update = previous.update if previous is not None else ADD arrived = ( StageOutput( rows, group=previous.group, always_leaves=previous.always_leaves, leaves_for_next_layer=previous.leaves_for_next_layer, + update=update, ) if previous is not None and (previous.always_leaves or previous.leaves_for_next_layer) - else StageOutput(rows) + else StageOutput(rows, update=update) ) during = stage.input.layout if rows.sharded <= stage.input.layout.sharded else rows return ( @@ -240,7 +278,7 @@ def sequence_parallel_layer_sides( return DecoderLayerSides( input_rows=local, # qkv and gate_up gather the slices into their GEMM. - attention=StageInput(attention, gathers_itself=gathers), + attention=StageInput(attention, gathers_itself=gathers, read=NORM_QUANT_READ), # o_proj and down reduce-scatter out of theirs, whether fused or not. attention_output=StageOutput(local), ffn=StageInput( @@ -281,7 +319,9 @@ def scattered_residual_layer_sides( return DecoderLayerSides( input_rows=attention if is_first_layer else local, attention=StageInput( - attention, gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}) + attention, + gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}), + read=NORM_QUANT_READ, ), attention_output=StageOutput( attention, group=SumGroup.ATTN_TP, always_leaves=True @@ -315,7 +355,9 @@ def input_scattered_layer_sides( return DecoderLayerSides( input_rows=attention, attention=StageInput( - attention, gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}) + attention, + gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}), + read=NORM_QUANT_READ, ), attention_output=StageOutput( attention, group=SumGroup.ATTN_TP, always_leaves=True @@ -384,6 +426,7 @@ def decoder_layer_sides( if attention_gathers_local_rows else frozenset() ), + read=NORM_QUANT_READ, ), # The output projection leaves the attention-TP sum to prepare_mlp. attention_output=StageOutput( @@ -475,19 +518,33 @@ def _cp_moves() -> CpMoves: ) -class BoundarySteps(msgspec.Struct, frozen=True): - """The steps a batch runs at a layer's boundaries: into the attention, - from the attention output to the FFN input, and the FFN output on to the - rows the layer hands on.""" +class StageEntry(msgspec.Struct, frozen=True): + """The boundary into one of a layer's stages, as the layer runs it.""" - # The half into the attention: completes what the input owes, writes the - # previous output into the residual and reads the attention input - # (_attention_input_step); attention_input then moves it. - attention_prepare: Callable - attention_input: Callable - ffn_input: Callable - # The rows ffn_input hands the FFN. - ffn_input_rows: Layout + # Completes what the input owes, writes the previous stage's output into + # the residual and reads this stage's input: + # (hidden_states, residual, forward_batch, norm, context, **call). + prepare: Callable + # The rows prepare hands the stage. + input_rows: Layout + # Moves the input onto the stage's rows after prepare, when prepare does + # not: (hidden_states, forward_batch, context) -> hidden_states. + input_move: Optional[Callable] = None + # Hands the stage its input once it is on its rows: + # (hidden_states, forward_batch, qkv_latent_func) -> hidden_states. + handoff: Optional[Callable] = None + # The fused kernels prepare tries first. + fused: Tuple["FusedMlpInput", ...] = () + + +class BoundarySteps(msgspec.Struct, frozen=True): + """The steps a batch runs at a layer's boundaries: into each of its + stages, and the last stage's output on to the rows the layer hands on.""" + + # The boundary into the layer's attention and into its FFN; None for a + # stage a layer that is one stage does not have. + attention: Optional[StageEntry] + ffn: Optional[StageEntry] # What the FFN exit reads: the FFN output's group and what it may leave. ffn_output: StageOutput # The postprocess that moves the FFN output on; None when it goes back over @@ -497,11 +554,6 @@ class BoundarySteps(msgspec.Struct, frozen=True): ffn_sum_is_movable: bool # Whether ffn_output_move also completes the sum the FFN leaves. ffn_output_move_completes_sum: bool = False - # The fused kernels ffn_input tries first. - fused: Tuple["FusedMlpInput", ...] = () - # Hands the attention its input once attention_input has moved it: - # (hidden_states, forward_batch, qkv_latent_func) -> hidden_states. - attention_handoff: Callable = _hand_qkv_hook_its_input @property def returns_over_dp(self) -> bool: @@ -520,7 +572,8 @@ def _attention_input_step( layer_input: Optional[Callable], fusions: Tuple[Callable, ...], enters_stack: bool, - residual_ops: ResidualOps, + read: StageRead, + update: StageUpdate, ) -> Tuple[torch.Tensor, torch.Tensor]: """A boundary's half into an attention: complete what the previous layer left (a value that owes a sum, or a producer's handoff one of ``fusions`` @@ -549,22 +602,17 @@ def _attention_input_step( if layer_input is not None: hidden_states, residual = layer_input(hidden_states, residual, context) if enters: - hidden_states, residual = residual_ops.enter(hidden_states), None + hidden_states, residual = read.enter(hidden_states), None if owed is not None and hidden_states.shape[0] != 0: hidden_states = reduce_output(owed) if residual is None: # The previous layer already wrote its output into the residual. - return residual_ops.read_attention_input(hidden_states, norm, quant_format) - return residual_ops.update_and_read_attention_input( - hidden_states, residual, norm, quant_format, post_residual_addition + return read.read(hidden_states, norm, quant_format) + return read.update_and_read( + update, hidden_states, residual, norm, quant_format, post_residual_addition ) -def _another_stage(*args, **kwargs): - """The steps of a stage a single-stage layer does not have.""" - raise RuntimeError("this layer is one stage and does not have the other") - - class InputRead(Enum): """How a boundary's consumer reads its input from the residual: with the attention input norm (prepare_attn) or with the FFN input norm and its @@ -610,11 +658,10 @@ class Boundary(msgspec.Struct, frozen=True): @property def input_rows(self) -> Layout: """The rows the consumer is handed: what it needs, still sharded over - the axes it gathers itself.""" + the axes it gathers itself as the rows its input is read on are.""" need = self.edge.need return Layout( - need.layout.sharded - | (self.edge.produced.layout.sharded & need.gathers_itself) + need.layout.sharded | (self.edge.residual_to.sharded & need.gathers_itself) ) @@ -625,7 +672,6 @@ def make_boundary( fusions: Tuple = (), force_layernorm_before_gather: bool = False, cp_moves: Optional[CpMoves] = None, - residual_ops: ResidualOps = ADD_AND_NORM, enters_stack: bool = False, ) -> Boundary: """The steps a layer runs at ``edge``, around the residual operations; @@ -646,7 +692,7 @@ def make_boundary( residual=edge.residual, to=edge.residual_to, cp_moves=cp_moves, - residual_ops=residual_ops, + update=edge.produced.update, ) return Boundary( edge, @@ -663,7 +709,8 @@ def make_boundary( fusions=fusions, residual_joins_sum=edge.residual_joins_sum, cp_moves=cp_moves, - residual_ops=residual_ops, + read=edge.need.read, + update=edge.produced.update, ) return Boundary(edge, prepare=input_step, fused=fused) layer_input = None @@ -683,9 +730,15 @@ def make_boundary( prepare=partial( _attention_input_step, layer_input=layer_input, - fusions=fusions, + # A fused kernel runs the add and the norm itself. + fusions=( + fusions + if edge.produced.update.adds_plainly and edge.need.read.norms_plainly + else () + ), enters_stack=enters_stack, - residual_ops=residual_ops, + read=edge.need.read, + update=edge.produced.update, ), input_move=_select_attention_input_move(edge.residual_to, edge.need), ) @@ -697,7 +750,6 @@ def _select_boundary_steps( fusions: Tuple["FusedMlpInput", ...] = (), force_layernorm_before_gather: bool = False, cp_moves: Optional[CpMoves] = None, - residual_ops: ResidualOps = ADD_AND_NORM, attention_handoff: Callable = _hand_qkv_hook_its_input, attention_fusions: Tuple[Callable, ...] = (), enters_stack: bool = False, @@ -708,35 +760,36 @@ def _select_boundary_steps( input tries ``attention_fusions``, and the layer stack's first layer (``enters_stack``) starts its residual there.""" edges = decoder_layer_edges(sides) - out_of_ffn = make_boundary( - edges.out_of_ffn, reads=None, cp_moves=cp_moves, residual_ops=residual_ops - ) + out_of_ffn = make_boundary(edges.out_of_ffn, reads=None, cp_moves=cp_moves) into_ffn = make_boundary( edges.into_ffn, reads=InputRead.FFN, fusions=fusions, force_layernorm_before_gather=force_layernorm_before_gather, cp_moves=cp_moves, - residual_ops=residual_ops, ) into_attention = make_boundary( edges.into_attention, reads=InputRead.ATTENTION, fusions=attention_fusions, - residual_ops=residual_ops, enters_stack=enters_stack, ) return BoundarySteps( - attention_prepare=into_attention.prepare, - attention_input=into_attention.input_move, - ffn_input=into_ffn.prepare, - ffn_input_rows=into_ffn.input_rows, + attention=StageEntry( + prepare=into_attention.prepare, + input_rows=into_attention.input_rows, + input_move=into_attention.input_move, + handoff=attention_handoff, + ), + ffn=StageEntry( + prepare=into_ffn.prepare, + input_rows=into_ffn.input_rows, + fused=into_ffn.fused, + ), ffn_output=edges.out_of_ffn.produced, ffn_output_move=out_of_ffn.output_move, ffn_output_move_completes_sum=out_of_ffn.output_move_completes_sum, ffn_sum_is_movable=edges.out_of_ffn.produced.group is not None, - fused=into_ffn.fused, - attention_handoff=attention_handoff, ) @@ -765,7 +818,8 @@ def _select_ffn_input( fusions: Tuple[FusedMlpInput, ...], residual_joins_sum: bool = False, cp_moves: Optional[CpMoves] = None, - residual_ops: ResidualOps = ADD_AND_NORM, + read: StageRead = NORM_READ, + update: StageUpdate = ADD, ) -> Tuple[Callable, Tuple[FusedMlpInput, ...]]: """The steps from the attention output to the FFN input, and the fused kernels they try first: complete the attention-TP sum, move the residual to @@ -791,7 +845,8 @@ def _select_ffn_input( fusions=fusions, residual_joins_sum=residual_joins_sum, cp_moves=cp_moves, - residual_ops=residual_ops, + read=read, + update=update, ) return partial(_mlp_input_completing_owed, step=step), fused # What the attention output owes decides the steps: the attention-TP sum, @@ -819,7 +874,8 @@ def _select_ffn_input( partial( _mlp_input_scatter if owes_attention_tp else _mlp_input_slice, scatters_residual=residual != residual_to, - residual_ops=residual_ops, + read=read, + update=update, ), (), ) @@ -828,7 +884,10 @@ def _select_ffn_input( # attention's rows (MHC on an input-scattered batch). if gathered or not owes_attention_tp or residual != residual_to: raise NotImplementedError(f"{produced=} {residual=} {need=}") - return partial(_mlp_input_on_residual_shard, residual_ops=residual_ops), () + return ( + partial(_mlp_input_on_residual_shard, read=read, update=update), + (), + ) if gathered == {TokenAxis.ATTN_CP}: # Each CP rank completes its own chunk, then the CP moves gather them. if cp_moves is None: @@ -841,7 +900,8 @@ def _select_ffn_input( force_layernorm_before_gather=force_layernorm_before_gather, fusions=fusions, residual_joins_sum=residual_joins_sum, - residual_ops=residual_ops, + read=read, + update=update, ) return partial(cp_moves.gather, gather=on_chunk), fused if ( @@ -860,20 +920,26 @@ def _select_ffn_input( gathers_residual = residual != residual_to if not gathered: if not owes_attention_tp: - return partial(_mlp_input_norm, residual_ops=residual_ops), () + return partial(_mlp_input_norm, read=read, update=update), () if gathers_residual and residual_joins_sum: # Each rank adds its slice of the residual into its share of the # sum, so the all-reduce also brings the residual back to every row. - if not residual_ops.adds_plainly: + if not update.adds_plainly: raise NotImplementedError(f"{produced=} {residual=} {need=}") - return _mlp_input_residual_into_sum, () - fused = tuple(f for f in fusions if f.completes is produced.group) + return partial(_mlp_input_residual_into_sum, read=read), () + # A fused kernel runs the add and the norm itself. + fused = ( + tuple(f for f in fusions if f.completes is produced.group) + if update.adds_plainly and read.norms_plainly + else () + ) return ( partial( _mlp_input_without_dp, gathers_residual=gathers_residual, fusions=tuple(f.run for f in fused), - residual_ops=residual_ops, + read=read, + update=update, ), fused, ) @@ -886,13 +952,15 @@ def _select_ffn_input( if ( owes_attention_tp and not force_layernorm_before_gather - and residual_ops.adds_plainly + and update.adds_plainly + and read.norms_plainly ): return ( partial( _mlp_input_dp_partial, gathers_residual=gathers_residual, places_cp_shards=places_cp_shards, + read=read, ), (), ) @@ -902,7 +970,8 @@ def _select_ffn_input( gathers_residual=gathers_residual, reduces_attention_tp=owes_attention_tp, places_cp_shards=places_cp_shards, - residual_ops=residual_ops, + read=read, + update=update, ), (), ) @@ -914,7 +983,7 @@ def _select_ffn_output_move( residual: Layout, to: Layout, cp_moves: Optional[CpMoves] = None, - residual_ops: ResidualOps = ADD_AND_NORM, + update: StageUpdate = ADD, ) -> Tuple[bool, Optional[Callable], bool]: """How the FFN output reaches the rows the layer hands on: whether it goes back by undoing the attention-DP gather (the FFN exit and postprocess run @@ -928,7 +997,7 @@ def _select_ffn_output_move( if to.sharded == residual.sharded - {TokenAxis.ATTN_TP_SCATTER}: # Each rank's slice back to the attention's rows: write the output # into the residual, then gather over attention TP. - return False, partial(pair._gather, residual_ops=residual_ops), False + return False, partial(pair._gather, update=update), False raise NotImplementedError(f"{produced=} {residual=} {to=}") returned = residual.sharded - produced.layout.sharded if returned == {TokenAxis.ATTN_TP_SCATTER} and to in (residual, produced.layout): @@ -942,7 +1011,7 @@ def _select_ffn_output_move( pair._onto_residual_shard, sums=sums, gathers_back=to != residual, - residual_ops=residual_ops, + update=update, ), sums, ) diff --git a/python/sglang/srt/layers/communicator/layer.py b/python/sglang/srt/layers/communicator/layer.py index bb19c50b77..32c8359a55 100644 --- a/python/sglang/srt/layers/communicator/layer.py +++ b/python/sglang/srt/layers/communicator/layer.py @@ -15,7 +15,7 @@ from dataclasses import dataclass from enum import Enum, auto -from functools import partial +from functools import cached_property, partial from typing import Callable, Optional, Tuple, Union import msgspec @@ -33,7 +33,7 @@ from sglang.srt.layers.communicator.boundary import ( FusedMlpInput, InputRead, LayerStage, - _another_stage, + StageEntry, _cp_moves, _select_boundary_steps, decoder_layer_sides, @@ -41,6 +41,7 @@ from sglang.srt.layers.communicator.boundary import ( make_boundary, scattered_residual_layer_sides, sequence_parallel_layer_sides, + with_residual, ) from sglang.srt.layers.communicator.layout import ( CommunicateContext, @@ -73,9 +74,9 @@ from sglang.srt.layers.communicator.output import ( UnreducedOutput, reduce_output, ) -from sglang.srt.layers.communicator.residual import ResidualOps +from sglang.srt.layers.communicator.residual import LayerResidual from sglang.srt.layers.communicator.residual.add_norm import ( - ADD_AND_NORM, + PLAIN_RESIDUAL, aiter_all_reduce_fusion_enabled_for, apply_aiter_all_reduce_fusion, apply_flashinfer_allreduce_fusion, @@ -244,13 +245,68 @@ def _unfused_completion_matches_the_ffn(forward_batch: ForwardBatch) -> bool: return _ffn_has_tokens(forward_batch) and post_experts_sum_is_one_all_reduce() +class StageCommunicator: + """One stage of a layer, its attention or its FFN: the boundary into it, + run with the stage's norm on the steps the layer chose for the batch.""" + + def __init__(self, layer: "LayerCommunicator", name: str, norm_name: str): + self._layer = layer + self._name = name + self._norm_name = norm_name + + @property + def norm(self) -> torch.nn.Module: + """The norm the stage reads its input with, one of the layer's.""" + return getattr(self._layer, self._norm_name) + + def entry( + self, forward_batch: ForwardBatch, steps: Optional[BoundarySteps] = None + ) -> StageEntry: + if steps is None: + steps = self._layer._batch_steps(forward_batch) + entry = getattr(steps, self._name) + if entry is None: + raise NotImplementedError( + f"a layer that is one stage has no {self._name} stage" + ) + return entry + + def prepare( + self, + hidden_states, + residual, + forward_batch: ForwardBatch, + steps: Optional[BoundarySteps] = None, + **call, + ): + """The stage's input and the residual, from the previous stage's output + (``call``: what the stage's read takes, e.g. the attention's + ``quant_format``).""" + entry = self.entry(forward_batch, steps) + context = self._layer._context + hidden_states, residual = entry.prepare( + hidden_states, residual, forward_batch, self.norm, context, **call + ) + if entry.input_move is not None: + hidden_states = entry.input_move( + hidden_states=hidden_states, + forward_batch=forward_batch, + context=context, + ) + if entry.handoff is not None: + hidden_states = entry.handoff( + hidden_states, forward_batch, self._layer.qkv_latent_func + ) + return hidden_states, residual + + class LayerCommunicator: # Communicators built without __init__ (e.g. test doubles) publish no LoRA # layout and try no fused kernel at the FFN exit. _publish_lora_layout: bool = False _ffn_exit_fusions: Tuple[Callable, ...] = () # A plain residual unless the layer is built with its own. - _residual_ops: ResidualOps = ADD_AND_NORM + _residual: LayerResidual = PLAIN_RESIDUAL def __init__( self, @@ -265,8 +321,9 @@ class LayerCommunicator: fused_ar_quant_keep_bf16: bool = False, # False for a layer whose FFN always completes its own reduction. allow_deferred_ffn_reduction: bool = True, - # How the layer writes its residual and reads its stages' inputs. - residual_ops: ResidualOps = ADD_AND_NORM, + # How the layer's attention and FFN read their inputs from the residual + # and write their outputs into it. + residual: LayerResidual = PLAIN_RESIDUAL, # A layer that is one stage of a sequence of stages, instead of an # attention followed by an FFN. stage: Optional["LayerStage"] = None, @@ -281,7 +338,7 @@ class LayerCommunicator: self.enable_fused_ar_quant = enable_fused_ar_quant self.fused_ar_quant_keep_bf16 = fused_ar_quant_keep_bf16 self.allow_deferred_ffn_reduction = allow_deferred_ffn_reduction - self._residual_ops = residual_ops + self._residual = residual self._context = CommunicateContext.init_new() self._context.force_layernorm_before_dp_gather = ( @@ -299,6 +356,11 @@ class LayerCommunicator: get_lora().enable_lora ) if stage is not None: + if residual != PLAIN_RESIDUAL: + raise ValueError( + "a layer that is one stage takes its reads and updates from " + "the stage's declarations" + ) self._init_stage(stage) return # Its two boundaries, for a layer that is one stage. @@ -336,8 +398,10 @@ class LayerCommunicator: # q_lora, which input-scattered attention needs. self._sp_steps = ( _select_boundary_steps( - sequence_parallel_layer_sides(axis_sizes=token_axis_sizes()), - residual_ops=residual_ops, + with_residual( + sequence_parallel_layer_sides(axis_sizes=token_axis_sizes()), + residual, + ), attention_fusions=self._attn_input_fusions, enters_stack=self.layer_scatter_modes.is_first_layer, ) @@ -354,9 +418,9 @@ class LayerCommunicator: ) self._declared = None self._cp_steps = self._input_scattered_steps = self._sp_steps = None - self.stage_edges = stage.edges into_edge, out_edge = stage.edges reads_ffn = stage.reads is InputRead.FFN + self.stage_edges = stage.edges into = make_boundary( into_edge, reads=stage.reads, @@ -366,22 +430,35 @@ class LayerCommunicator: else self._attn_input_fusions ), force_layernorm_before_gather=self.force_layernorm_before_dp_gather, - residual_ops=self._residual_ops, enters_stack=stage.enters_stack, ) - out = make_boundary(out_edge, reads=None, residual_ops=self._residual_ops) + out = make_boundary(out_edge, reads=None) + entry = StageEntry( + prepare=into.prepare, + input_rows=into.input_rows, + input_move=None if reads_ffn else into.input_move, + handoff=None if reads_ffn else _hand_qkv_hook_its_input, + fused=into.fused, + ) self._steps = BoundarySteps( - attention_prepare=_another_stage if reads_ffn else into.prepare, - attention_input=_another_stage if reads_ffn else into.input_move, - ffn_input=into.prepare if reads_ffn else _another_stage, - ffn_input_rows=into.input_rows, + attention=None if reads_ffn else entry, + ffn=entry if reads_ffn else None, ffn_output=out_edge.produced, ffn_output_move=out.output_move, ffn_output_move_completes_sum=out.output_move_completes_sum, ffn_sum_is_movable=out_edge.produced.group is not None, - fused=into.fused, ) + @cached_property + def attn(self) -> StageCommunicator: + """The layer's attention stage.""" + return StageCommunicator(self, "attention", "input_layernorm") + + @cached_property + def ffn(self) -> StageCommunicator: + """The layer's FFN stage.""" + return StageCommunicator(self, "ffn", "post_attention_layernorm") + @property def input_rows(self) -> Layout: """The rows the layer's input and residual arrive on in a batch that @@ -472,7 +549,7 @@ class LayerCommunicator: # No batch shards its tokens and there is no attention DP: as # without CP. may_leave = may_leave_to_reduce_scatter = True - return decoder_layer_sides( + sides = decoder_layer_sides( axis_sizes=token_axis_sizes(cp_active=cp_active), ffn_on_local_rows=on_local_rows(modes.is_layer_sparse), ffn_shards_over_cp=ffn_on_cp_shards, @@ -499,6 +576,7 @@ class LayerCommunicator: ) and may_leave, ) + return with_residual(sides, self._residual) @staticmethod def _refuse_uncovered_cp_moe(moe_on_local_rows: bool, cp_shards: bool) -> None: @@ -532,7 +610,7 @@ class LayerCommunicator: for the layer's ordinary declarations ``sides``. A plain residual comes back to the full rows inside the attention output's sum; one whose write-back is not a plain add stays on each rank's slice.""" - if self._residual_ops.adds_plainly: + if self._residual.attention_update.adds_plainly: scattered = input_scattered_layer_sides( axis_sizes=token_axis_sizes(), ffn_group=sides.ffn_output.group, @@ -550,8 +628,7 @@ class LayerCommunicator: # DSA and hook-less attention take the slice gathered. handoff = _hand_scattered_input_to_attention return _select_boundary_steps( - scattered, - residual_ops=self._residual_ops, + with_residual(scattered, self._residual), attention_handoff=handoff, attention_fusions=self._attn_input_fusions, enters_stack=self.layer_scatter_modes.is_first_layer, @@ -563,7 +640,6 @@ class LayerCommunicator: """The steps this layer runs for a set of declarations.""" return _select_boundary_steps( sides, - residual_ops=self._residual_ops, attention_fusions=self._attn_input_fusions, enters_stack=self.layer_scatter_modes.is_first_layer, **kwargs, @@ -586,9 +662,7 @@ class LayerCommunicator: post_residual_addition=post_residual_addition, ) if captured_last_layer_outputs is not None: - gathered_last_layer_output = self._batch_steps( - forward_batch - ).attention_input( + gathered_last_layer_output = self.attn.entry(forward_batch).input_move( hidden_states=residual, forward_batch=forward_batch, context=self._context, @@ -616,10 +690,7 @@ class LayerCommunicator: the batch is not input-scattered. """ return ( - any( - f.may_return_new_residual - for f in self._batch_steps(forward_batch).fused - ) + any(f.may_return_new_residual for f in self.ffn.entry(forward_batch).fused) and not get_attn_tp_context().input_scattered and apply_flashinfer_allreduce_fusion(residual.shape[0]) ) @@ -637,7 +708,7 @@ class LayerCommunicator: "lora_batch_layout", ( LoRABatchLayout.DP_LOCAL - if TokenAxis.ATTN_DP in steps.ffn_input_rows.sharded + if TokenAxis.ATTN_DP in steps.ffn.input_rows.sharded else LoRABatchLayout.TP_GLOBAL ), ) @@ -660,20 +731,13 @@ class LayerCommunicator: ) if get_forward().sp_active: hidden_states = layernorm_sp.sp_entry_scatter(hidden_states) - hidden_states, residual = self._batch_steps(forward_batch).attention_prepare( + return self.attn.prepare( hidden_states, residual, forward_batch, - self.input_layernorm, - self._context, quant_format=quant_format, post_residual_addition=post_residual_addition, ) - return self._finish_prepare_attn( - hidden_states=hidden_states, - residual=residual, - forward_batch=forward_batch, - ) def _in_sp_region(self) -> bool: return self._sp_steps is not None and get_forward().sp_active @@ -693,27 +757,14 @@ class LayerCommunicator: return self._cp_steps return self._steps - def _finish_prepare_attn(self, hidden_states, residual, forward_batch): - """Tail every prepare_attn path must run, or ``attn_inputs`` is unset.""" - steps = self._batch_steps(forward_batch) - hidden_states = steps.attention_input( - hidden_states=hidden_states, - forward_batch=forward_batch, - context=self._context, - ) - hidden_states = steps.attention_handoff( - hidden_states, forward_batch, self.qkv_latent_func - ) - return hidden_states, residual - def _select_attn_input_fusions(self) -> Tuple[Callable, ...]: """The fused kernels that complete what the previous layer left together with the residual update and the input norm, in the order they are tried. Each takes (owed, residual, forward_batch, post_residual_addition) and returns None when it does not take the batch. They add the residual - plainly.""" + plainly: what arrives is written in as this layer's FFN writes its own.""" if not ( - self._residual_ops.adds_plainly + self._residual.ffn_update.adds_plainly and hasattr(self.input_layernorm, "forward_with_allreduce_fusion") ): return () @@ -772,7 +823,7 @@ class LayerCommunicator: """The fused kernels that can take the attention -> FFN steps, in the order they are tried. They add the residual plainly.""" if not ( - self._residual_ops.adds_plainly + self._residual.attention_update.adds_plainly and hasattr(self.post_attention_layernorm, "forward_with_allreduce_fusion") ): return () @@ -813,13 +864,7 @@ class LayerCommunicator: if cache is not None: self._context.cache = cache - return steps.ffn_input( - hidden_states, - residual, - forward_batch, - self.post_attention_layernorm, - self._context, - ) + return self.ffn.prepare(hidden_states, residual, forward_batch, steps) def maybe_prefetch_next_full_attention_kv( self, @@ -847,10 +892,8 @@ class LayerCommunicator: """Whether the next layer's input can run this layer's move of its FFN output back to this rank's tokens: the base postprocess scatter, when the next layer's input also writes the output into the residual.""" - return ( - self._batch_steps(forward_batch).returns_over_dp - and not self._residual_ops.updates_residual_after_ffn - ) + steps = self._batch_steps(forward_batch) + return steps.returns_over_dp and not steps.ffn_output.update.at_producer def _postprocess_dp_step( self, forward_batch: ForwardBatch @@ -977,8 +1020,9 @@ class LayerCommunicator: allow_reduce_scatter=self.allow_reduce_scatter, is_layer_sparse=self.layer_scatter_modes.is_layer_sparse, ) - if residual is not None and self._residual_ops.updates_residual_after_ffn: - hidden_states = self._residual_ops.update_residual(hidden_states, residual) + update = steps.ffn_output.update + if residual is not None and update.at_producer: + hidden_states = update.update(hidden_states, residual) residual = None return hidden_states, residual @@ -1326,7 +1370,7 @@ class MHCLayerCommunicator(LayerCommunicator): allow_reduce_scatter, qkv_latent_func, allow_deferred_ffn_reduction=False, - residual_ops=self.mhc, + residual=self.mhc.layer_residual(), ) # MHC has not been run with input-scattered attention under attention CP. if self._context.attn_cp_size > 1 and self._input_can_be_scattered(): diff --git a/python/sglang/srt/layers/communicator/ops.py b/python/sglang/srt/layers/communicator/ops.py index f53e74c0be..a7aa7833ec 100644 --- a/python/sglang/srt/layers/communicator/ops.py +++ b/python/sglang/srt/layers/communicator/ops.py @@ -41,8 +41,8 @@ from sglang.srt.layers.communicator.layout import ( moe_cp_gathered_rows, ) from sglang.srt.layers.communicator.output import UnreducedOutput, reduce_output -from sglang.srt.layers.communicator.residual import ResidualOps -from sglang.srt.layers.communicator.residual.add_norm import ADD_AND_NORM +from sglang.srt.layers.communicator.residual import StageRead, StageUpdate +from sglang.srt.layers.communicator.residual.add_norm import ADD, NORM_READ from sglang.srt.layers.communicator_dsa_cp import ( dsa_cp_gather_hidden_states, dsa_cp_reduce_scatter_hidden_states, @@ -249,21 +249,22 @@ def _mlp_input_without_dp( *, gathers_residual: bool, fusions: Tuple[Callable, ...], - residual_ops: ResidualOps = ADD_AND_NORM, + read: StageRead = NORM_READ, + update: StageUpdate = ADD, ): if gathers_residual: - residual = residual_ops.residual_from_attn_tp_shards(residual) + residual = update.residual_from_attn_tp_shards(residual) for fused in fusions: result = fused(hidden_states, residual, forward_batch) if result is not None: return result # MHC sums its streams in full precision. hidden_states = _mlp_input_reduce_output( - hidden_states, forward_batch, may_quantize=residual_ops.adds_plainly + hidden_states, forward_batch, may_quantize=update.adds_plainly ) if _is_npu and context.cache is not None: _ = prepare_weight_cache(hidden_states, context.cache) - return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm) + return read.update_and_read(update, hidden_states, residual, layernorm) def _mlp_input_dp_replicate( @@ -276,14 +277,15 @@ def _mlp_input_dp_replicate( gathers_residual: bool, reduces_attention_tp: bool, places_cp_shards: bool = False, - residual_ops: ResidualOps = ADD_AND_NORM, + read: StageRead = NORM_READ, + update: StageUpdate = ADD, ): """Attention DP: complete the attention-TP sum if it is owed, write the output into the residual and read the FFN input locally, then gather. With ``places_cp_shards`` each CP rank puts its shard of the DP group's tokens beside the others' in the group's slot.""" if gathers_residual: - residual = residual_ops.residual_from_attn_tp_shards(residual) + residual = update.residual_from_attn_tp_shards(residual) if hidden_states.shape[0] != 0: if reduces_attention_tp: hidden_states = attention_tensor_model_parallel_all_reduce(hidden_states) @@ -291,12 +293,12 @@ def _mlp_input_dp_replicate( get_parallel().tp_group, disabled=not is_allocation_symmetric(), ): - hidden_states, residual = residual_ops.update_and_read_ffn_input( - hidden_states, residual, layernorm + hidden_states, residual = read.update_and_read( + update, hidden_states, residual, layernorm ) else: - hidden_states, residual = residual_ops.update_and_read_ffn_input( - hidden_states, residual, layernorm + hidden_states, residual = read.update_and_read( + update, hidden_states, residual, layernorm ) cp_shard_counts = _cp_shard_token_rows(forward_batch) if places_cp_shards else None return ( @@ -314,10 +316,11 @@ def _mlp_input_dp_partial( *, gathers_residual: bool, places_cp_shards: bool = False, + read: StageRead = NORM_READ, ): - """Attention DP: one rank adds the residual, the gather sums it, then - normalize. With ``places_cp_shards`` each CP rank puts its shard of the DP - group's tokens beside the others' in the group's slot.""" + """Attention DP: one rank adds the residual, the gather sums it, then read + the input from the sum. With ``places_cp_shards`` each CP rank puts its + shard of the DP group's tokens beside the others' in the group's slot.""" if gathers_residual: residual = _redistribute_from_attn_tp_shards(residual) if context.attn_tp_rank == 0: @@ -327,8 +330,7 @@ def _mlp_input_dp_partial( hidden_states, forward_batch, cp_shard_counts ) dp_scatter(residual, hidden_states, forward_batch, cp_shard_counts) - if hidden_states.shape[0] != 0: - hidden_states = layernorm(hidden_states) + hidden_states, _ = read.read(hidden_states, layernorm) return hidden_states, residual @@ -402,9 +404,10 @@ def _mlp_input_norm( layernorm: torch.nn.Module, context: CommunicateContext, *, - residual_ops: ResidualOps = ADD_AND_NORM, + read: StageRead = NORM_READ, + update: StageUpdate = ADD, ): - return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm) + return read.update_and_read(update, hidden_states, residual, layernorm) def _mlp_input_scatter( @@ -415,14 +418,15 @@ def _mlp_input_scatter( context: CommunicateContext, *, scatters_residual: bool, - residual_ops: ResidualOps = ADD_AND_NORM, + read: StageRead = NORM_READ, + update: StageUpdate = ADD, ): hidden_states = _reduce_and_redistribute_output_to_attn_tp_shards( hidden_states, context ) if scatters_residual: - residual = residual_ops.residual_to_attn_tp_shard(residual, context) - return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm) + residual = update.residual_to_attn_tp_shard(residual, context) + return read.update_and_read(update, hidden_states, residual, layernorm) def _mlp_input_slice( @@ -433,12 +437,13 @@ def _mlp_input_slice( context: CommunicateContext, *, scatters_residual: bool, - residual_ops: ResidualOps = ADD_AND_NORM, + read: StageRead = NORM_READ, + update: StageUpdate = ADD, ): hidden_states = _redistribute_to_attn_tp_shards(hidden_states, context).clone() if scatters_residual and residual is not None: - residual = residual_ops.residual_to_attn_tp_shard(residual, context) - return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm) + residual = update.residual_to_attn_tp_shard(residual, context) + return read.update_and_read(update, hidden_states, residual, layernorm) def _mlp_input_on_residual_shard( @@ -448,7 +453,8 @@ def _mlp_input_on_residual_shard( layernorm: torch.nn.Module, context: CommunicateContext, *, - residual_ops: ResidualOps, + read: StageRead, + update: StageUpdate, ): """The residual stays on each rank's slice while the FFN takes the full rows: reduce-scatter the attention output onto the slice, which completes @@ -458,7 +464,7 @@ def _mlp_input_on_residual_shard( return hidden_states, hidden_states shard = hidden_states.tensor_split(context.tp_size)[context.tp_rank] get_parallel().tp_group.reduce_scatter_tensor(shard, hidden_states) - shard, residual = residual_ops.update_and_read_ffn_input(shard, residual, layernorm) + shard, residual = read.update_and_read(update, shard, residual, layernorm) attn_tp_all_gather_into_tensor(hidden_states, shard) return hidden_states, residual @@ -469,9 +475,11 @@ def _mlp_input_residual_into_sum( forward_batch: ForwardBatch, layernorm: torch.nn.Module, context: CommunicateContext, + *, + read: StageRead = NORM_READ, ): return _tp_all_reduce_with_scattered_residual( - hidden_states, residual, layernorm, context + hidden_states, residual, layernorm, context, read ) @@ -480,6 +488,7 @@ def _tp_all_reduce_with_scattered_residual( residual: torch.Tensor, layernorm: torch.nn.Module, context: CommunicateContext, + read: StageRead = NORM_READ, ): if hidden_states.shape[0] == 0: return hidden_states, hidden_states @@ -487,8 +496,7 @@ def _tp_all_reduce_with_scattered_residual( scattered_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank] scattered_states += residual residual = tensor_model_parallel_all_reduce(hidden_states) - hidden_states = layernorm(residual) - return hidden_states, residual + return read.read(residual, layernorm) def _mlp_input_gather_attention_cp( @@ -669,10 +677,10 @@ class CommunicateSummableTensorPairFn: residual: torch.Tensor, forward_batch: ForwardBatch, context: CommunicateContext, - residual_ops: ResidualOps = ADD_AND_NORM, + update: StageUpdate = ADD, **kwargs, ): - hidden_states = residual_ops.update_residual(hidden_states, residual) + hidden_states = update.update(hidden_states, residual) return _redistribute_from_attn_tp_shards(hidden_states), None @staticmethod @@ -684,7 +692,7 @@ class CommunicateSummableTensorPairFn: *, sums: bool, gathers_back: bool, - residual_ops: ResidualOps, + update: StageUpdate, **kwargs, ): """Bring the FFN output onto the slice of the rows the residual is on: @@ -697,7 +705,7 @@ class CommunicateSummableTensorPairFn: hidden_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank] if not gathers_back: return hidden_states, residual - local_states = residual_ops.update_residual(hidden_states, residual) + local_states = update.update(hidden_states, residual) hidden_states = local_states.new_empty( local_states.shape[0] * context.tp_size, *local_states.shape[1:] ) diff --git a/python/sglang/srt/layers/communicator/residual/__init__.py b/python/sglang/srt/layers/communicator/residual/__init__.py index e4dfdc0759..810267c3f3 100644 --- a/python/sglang/srt/layers/communicator/residual/__init__.py +++ b/python/sglang/srt/layers/communicator/residual/__init__.py @@ -11,50 +11,77 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""How a layer writes its stages' outputs into the residual and reads the next -stage's input back.""" +"""How a stage writes its output into the residual (its update) and how a +stage reads its input back from it (its read). The producer of a boundary +declares the update, the consumer the read; the boundary steps complete sums +and move tokens around the two.""" -from typing import Protocol, Tuple +from typing import NamedTuple, Optional, Protocol, Tuple import torch -class ResidualOps(Protocol): - """How a layer writes each stage's output into its residual and reads the - next stage's input from it. The boundary steps complete sums and move - tokens around these operations. AddAndNorm adds and normalizes; MHC's - hyper-connection streams implement the same operations (residual.mhc).""" +class StageUpdate(Protocol): + """How a stage's output is written into the residual.""" - # The write-back is a plain add, which one rank may run before the sum it - # adds into completes: the DP partial order, and a residual that joins the - # attention output's sum. + # A plain add, which one rank may run before the sum it adds into + # completes (the DP partial order, a residual that joins the attention + # output's sum), and which a fused add + norm kernel may run. adds_plainly: bool - # The layer writes its FFN output into the residual itself instead of - # leaving that to the next layer's input. - updates_residual_after_ffn: bool + # The stage writes its output into the residual itself at its end, instead + # of leaving that to the next stage's read. + at_producer: bool - def enter(self, hidden_states: torch.Tensor) -> torch.Tensor: - """The residual the layer stack starts from, given its input.""" - - def read_attention_input(self, residual, norm, quant_format: str) -> Tuple: - """The attention input and the residual, from a residual that already - holds the previous layer's output.""" - - def update_and_read_attention_input( - self, hidden_states, residual, norm, quant_format: str, post_residual_addition - ) -> Tuple: - """Write the previous layer's output into the residual, then read the - attention input from it.""" - - def update_and_read_ffn_input(self, hidden_states, residual, norm) -> Tuple: - """Write the attention output into the residual, then read the FFN - input from it.""" - - def update_residual(self, hidden_states, residual) -> torch.Tensor: - """The residual with the FFN output written into it.""" + def update(self, hidden_states, residual) -> torch.Tensor: + """The residual with the output written into it.""" def residual_to_attn_tp_shard(self, residual, context) -> torch.Tensor: - """This attention-TP rank's slice of the residual.""" + """This attention-TP rank's slice of the residual, with whatever the + update reads along with it.""" def residual_from_attn_tp_shards(self, residual) -> torch.Tensor: """The residual gathered from every attention-TP rank's slice.""" + + +class StageRead(Protocol): + """How a stage reads its input from the residual.""" + + # The input is the residual's norm, in the quantization the call asks for, + # and the residual is left as it is: what a fused add + norm kernel computes, + # and what may run on the rows a sum completes onto. + norms_plainly: bool + + def enter(self, hidden_states) -> torch.Tensor: + """The residual the layer stack starts from, given its input.""" + + def read( + self, + residual, + norm, + quant_format: str = "", + post_residual_addition: Optional[torch.Tensor] = None, + ) -> Tuple: + """The input and the residual, from a residual that already holds the + previous stage's output.""" + + def update_and_read( + self, + update: StageUpdate, + hidden_states, + residual, + norm, + quant_format: str = "", + post_residual_addition: Optional[torch.Tensor] = None, + ) -> Tuple: + """Write the previous stage's output into the residual with its + producer's ``update``, then read the input.""" + + +class LayerResidual(NamedTuple): + """The reads and updates a decoder layer declares for its attention and + its FFN.""" + + attention_read: StageRead + attention_update: StageUpdate + ffn_read: StageRead + ffn_update: StageUpdate diff --git a/python/sglang/srt/layers/communicator/residual/add_norm.py b/python/sglang/srt/layers/communicator/residual/add_norm.py index eb4c33f60d..60e560f3f5 100644 --- a/python/sglang/srt/layers/communicator/residual/add_norm.py +++ b/python/sglang/srt/layers/communicator/residual/add_norm.py @@ -23,6 +23,7 @@ from sglang.srt.layers.communicator.adapters.attention import ( _redistribute_to_attn_tp_shards, get_attn_tp_context, ) +from sglang.srt.layers.communicator.residual import LayerResidual from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.flashinfer_comm_fusion import ( is_flashinfer_allreduce_unavailable, @@ -261,43 +262,13 @@ def _attn_input_update_and_read_residual(quant_format: str): return _update_and_read_residual_plain -class AddAndNorm: - """A plain residual: a stage's output is added into it, and the next - stage's input is its norm. An empty batch skips the norms.""" +class Add: + """A plain residual: the output is added into it.""" adds_plainly = True - updates_residual_after_ffn = False + at_producer = False - def enter(self, hidden_states): - return hidden_states - - def read_attention_input(self, residual, norm, quant_format): - if residual.shape[0] == 0: - return residual, residual - return _attn_input_update_and_read_residual(quant_format)( - norm, residual, None, None - ) - - def update_and_read_attention_input( - self, hidden_states, residual, norm, quant_format, post_residual_addition - ): - if hidden_states.shape[0] == 0: - return hidden_states, hidden_states - return _attn_input_update_and_read_residual(quant_format)( - norm, hidden_states, residual, post_residual_addition - ) - - def update_and_read_ffn_input(self, hidden_states, residual, norm): - if residual is None: - # The layer stack starts at this FFN: its input is the residual. - if hidden_states.shape[0] == 0: - return hidden_states, hidden_states - return norm(hidden_states), hidden_states - if hidden_states.shape[0] == 0: - return hidden_states, residual - return norm(hidden_states, residual) - - def update_residual(self, hidden_states, residual): + def update(self, hidden_states, residual): hidden_states += residual return hidden_states @@ -308,4 +279,91 @@ class AddAndNorm: return _redistribute_from_attn_tp_shards(residual) -ADD_AND_NORM = AddAndNorm() +class NormQuantRead: + """The input is the residual's norm, fused with the quantization the + consumer asks for (``quant_format``) and a post-residual addition; a plain + add of the previous output runs in the same kernel. An empty batch skips + the norm.""" + + norms_plainly = True + + def enter(self, hidden_states): + return hidden_states + + def read(self, residual, norm, quant_format="", post_residual_addition=None): + if residual.shape[0] == 0: + return residual, residual + return _attn_input_update_and_read_residual(quant_format)( + norm, residual, None, None + ) + + def update_and_read( + self, + update, + hidden_states, + residual, + norm, + quant_format="", + post_residual_addition=None, + ): + if not update.adds_plainly: + return self.read(update.update(hidden_states, residual), norm, quant_format) + if hidden_states.shape[0] == 0: + return hidden_states, hidden_states + return _attn_input_update_and_read_residual(quant_format)( + norm, hidden_states, residual, post_residual_addition + ) + + +class NormRead: + """The input is the residual's norm; a plain add of the previous output + runs in the same kernel. An empty batch skips the norm.""" + + norms_plainly = True + + def enter(self, hidden_states): + return hidden_states + + def read(self, residual, norm, quant_format="", post_residual_addition=None): + if quant_format or post_residual_addition is not None: + raise NotImplementedError( + f"a norm read with {quant_format=} or a post-residual addition" + ) + if residual.shape[0] == 0: + return residual, residual + return norm(residual), residual + + def update_and_read( + self, + update, + hidden_states, + residual, + norm, + quant_format="", + post_residual_addition=None, + ): + if residual is None: + # The layer stack starts at this stage: its input is the residual. + return self.read(hidden_states, norm, quant_format, post_residual_addition) + if not update.adds_plainly: + return self.read(update.update(hidden_states, residual), norm, quant_format) + if quant_format or post_residual_addition is not None: + raise NotImplementedError( + f"a norm read with {quant_format=} or a post-residual addition" + ) + if hidden_states.shape[0] == 0: + return hidden_states, residual + return norm(hidden_states, residual) + + +ADD = Add() +NORM_QUANT_READ = NormQuantRead() +NORM_READ = NormRead() +# A plain residual: the attention reads with its input norm and the quantization +# it wants, the FFN with its norm, and each stage's output is added. +PLAIN_RESIDUAL = LayerResidual( + attention_read=NORM_QUANT_READ, + attention_update=ADD, + ffn_read=NORM_READ, + ffn_update=ADD, +) diff --git a/python/sglang/srt/layers/communicator/residual/mhc.py b/python/sglang/srt/layers/communicator/residual/mhc.py index aeb598531a..a21d5ea12e 100644 --- a/python/sglang/srt/layers/communicator/residual/mhc.py +++ b/python/sglang/srt/layers/communicator/residual/mhc.py @@ -19,17 +19,18 @@ from typing import Callable, Optional import torch from sglang.kernels.ops.layernorm.mhc import hc_contract, hc_expand +from sglang.srt.layers.communicator.residual import LayerResidual @dataclass class MHCState: - """A layer's residual as hyper-connection streams, with the residual - operations the boundary steps run (communicator.ResidualOps): the residual - is hc_mult streams, a stage's output is written in with hc_post and the - next stage's input read with hc_pre and the norm. A read produces the - h_res / h_post the next write-back consumes; they move with the tokens. - Parameters belong to the owning layer; this state only holds scratch - shared across communication stages.""" + """A layer's residual as hyper-connection streams: the residual is hc_mult + streams, a stage's output is written in with hc_post and the next stage's + input read with hc_pre and the norm. A read produces the h_res / h_post the + next write-back consumes; they move with the tokens. ``layer_residual()`` + gives the reads and updates the layer's two stages declare. Parameters + belong to the owning layer; this state only holds scratch shared across + communication stages.""" hc_mult: int hc_attn_pre: Callable @@ -42,13 +43,6 @@ class MHCState: h_res: Optional[torch.Tensor] = None h_post: Optional[torch.Tensor] = None - # hc_post is not a plain add, so it runs only once the sum it writes in - # is complete. - adds_plainly = False - # The FFN output's write-back uses the coefficients this layer's FFN input - # read produced, so the layer runs it itself. - updates_residual_after_ffn = True - @staticmethod def _resolve_out_norm(out_norm): if out_norm is None: @@ -102,30 +96,6 @@ class MHCState: self.h_res = None self.h_post = None - def enter(self, hidden_states): - return hc_expand(hidden_states, self.hc_mult) - - def read_attention_input(self, residual, norm, quant_format): - if quant_format: - raise NotImplementedError(f"an MHC attention input in {quant_format=}") - return self.attn_split(residual, out_norm=norm) - - def update_and_read_attention_input( - self, hidden_states, residual, norm, quant_format, post_residual_addition - ): - # An MHC layer's input arrives with the previous FFN output written in. - raise NotImplementedError("an MHC layer takes its input written back") - - def update_and_read_ffn_input(self, hidden_states, residual, norm): - return self.attn_to_mlp(hidden_states, residual, out_norm=norm) - - def update_residual(self, hidden_states, residual): - hidden_states = self.mlp_combine(hidden_states, residual) - self.reset_aux() - if self.is_last_layer: - hidden_states = hc_contract(hidden_states, self.hc_mult) - return hidden_states - def residual_to_attn_tp_shard(self, residual, context): rank, size = context.attn_tp_rank, context.attn_tp_size self.h_res = self.h_res.tensor_split(size)[rank] @@ -136,3 +106,98 @@ class MHCState: raise NotImplementedError( "Unsupported: h_res/h_post allgather not implemented." ) + + def layer_residual(self) -> LayerResidual: + return LayerResidual( + attention_read=_AttentionRead(self), + attention_update=_AttentionUpdate(self), + ffn_read=_FfnRead(self), + ffn_update=_FfnUpdate(self), + ) + + +class _AttentionRead: + """hc_pre and the input norm, from streams that already hold the previous + layer's output: an MHC layer takes its input written back.""" + + norms_plainly = False + + def __init__(self, state: MHCState): + self.state = state + + def enter(self, hidden_states): + return hc_expand(hidden_states, self.state.hc_mult) + + def read(self, residual, norm, quant_format="", post_residual_addition=None): + if quant_format: + raise NotImplementedError(f"an MHC attention input in {quant_format=}") + return self.state.attn_split(residual, out_norm=norm) + + def update_and_read(self, update, hidden_states, residual, norm, **kwargs): + raise NotImplementedError("an MHC layer takes its input written back") + + +class _AttentionUpdate: + """hc_post with the coefficients the attention's read produced. It is not a + plain add, so it runs only once the sum it writes in is complete.""" + + adds_plainly = False + at_producer = False + + def __init__(self, state: MHCState): + self.state = state + + def update(self, hidden_states, residual): + return self.state.mlp_combine(hidden_states, residual) + + def residual_to_attn_tp_shard(self, residual, context): + return self.state.residual_to_attn_tp_shard(residual, context) + + def residual_from_attn_tp_shards(self, residual): + return self.state.residual_from_attn_tp_shards(residual) + + +class _FfnRead: + """The attention output's hc_post and the FFN input's hc_pre and norm, fused + in hc_ffn_post_pre when it takes the batch.""" + + norms_plainly = False + + def __init__(self, state: MHCState): + self.state = state + + def enter(self, hidden_states): + return hc_expand(hidden_states, self.state.hc_mult) + + def read(self, residual, norm, quant_format="", post_residual_addition=None): + raise NotImplementedError("an MHC FFN input read without its attention") + + def update_and_read(self, update, hidden_states, residual, norm, **kwargs): + if not isinstance(update, _AttentionUpdate) or update.state is not self.state: + raise NotImplementedError(f"an MHC FFN input after {update=}") + return self.state.attn_to_mlp(hidden_states, residual, out_norm=norm) + + +class _FfnUpdate: + """hc_post with the coefficients the FFN input's read produced, which this + layer runs itself; the last layer also contracts the streams into the + hidden states the layer stack hands on.""" + + adds_plainly = False + at_producer = True + + def __init__(self, state: MHCState): + self.state = state + + def update(self, hidden_states, residual): + hidden_states = self.state.mlp_combine(hidden_states, residual) + self.state.reset_aux() + if self.state.is_last_layer: + hidden_states = hc_contract(hidden_states, self.state.hc_mult) + return hidden_states + + def residual_to_attn_tp_shard(self, residual, context): + return self.state.residual_to_attn_tp_shard(residual, context) + + def residual_from_attn_tp_shards(self, residual): + return self.state.residual_from_attn_tp_shards(residual) diff --git a/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py b/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py index 7eee467cb7..e614ea0d9f 100644 --- a/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py +++ b/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py @@ -361,7 +361,7 @@ class CuteDSLFusionLayerCommunicator(LayerCommunicator): and self._context.tp_size > 1 # The FFN runs on the full rows, not each rank's own slice. and TokenAxis.ATTN_TP_SCATTER - not in self._batch_steps(forward_batch).ffn_input_rows.sharded + not in self._batch_steps(forward_batch).ffn.input_rows.sharded ) diff --git a/python/sglang/srt/models/nemotron_h_utils.py b/python/sglang/srt/models/nemotron_h_utils.py index 0532afc2c8..75bd4f92df 100644 --- a/python/sglang/srt/models/nemotron_h_utils.py +++ b/python/sglang/srt/models/nemotron_h_utils.py @@ -4,6 +4,7 @@ from typing import Optional from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MOE from sglang.srt.layers.communicator import ( + NORM_QUANT_READ, InputRead, LayerCommunicator, LayerScatterModes, @@ -37,11 +38,12 @@ def _stage_kind(pattern: str, layer_idx: int) -> Optional[InputRead]: def _stage_decl(pattern: str, layer_idx: int) -> StageDecl: - """What the stage at ``layer_idx`` declares. A mixer computes on the - attention's rows and leaves its attention-TP sum to an FFN stage after it, - or, when a fused kernel takes it, to a mixer after it. An FFN computes on - the TP group's rows, a MoE dispatched by an a2a backend on this rank's own, - and may leave its sum to a mixer after it.""" + """What the stage at ``layer_idx`` declares. A mixer reads its input as + an attention does, computes on the attention's rows and leaves its + attention-TP sum to an FFN stage after it, or, when a fused kernel takes + it, to a mixer after it. An FFN computes on the TP group's rows, a MoE + dispatched by an a2a backend on this rank's own, and may leave its sum to a + mixer after it.""" axis_sizes = token_axis_sizes() attention = Layout.sharded_over( TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes @@ -50,7 +52,7 @@ def _stage_decl(pattern: str, layer_idx: int) -> StageDecl: if is_attn_layer(pattern[layer_idx]): owes = axis_sizes[TokenAxis.ATTN_TP_SCATTER] > 1 return StageDecl( - StageInput(attention), + StageInput(attention, read=NORM_QUANT_READ), StageOutput( attention, group=SumGroup.ATTN_TP if owes else None, diff --git a/python/sglang/test/communicator_patch.py b/python/sglang/test/communicator_patch.py index 7dfb19b8fd..5db7f4f5c5 100644 --- a/python/sglang/test/communicator_patch.py +++ b/python/sglang/test/communicator_patch.py @@ -10,7 +10,7 @@ from unittest import mock from sglang.srt.layers import communicator from sglang.srt.layers.communicator import boundary, layer, layout, ops, output -from sglang.srt.layers.communicator import residual as residual_ops +from sglang.srt.layers.communicator import residual as residual_contract from sglang.srt.layers.communicator.adapters import attention from sglang.srt.layers.communicator.residual import add_norm, mhc @@ -19,7 +19,7 @@ COMMUNICATOR_MODULES = ( layout, output, attention, - residual_ops, + residual_contract, add_norm, mhc, ops, diff --git a/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py b/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py index 6be3fd8e8c..fc2375c6d8 100644 --- a/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py +++ b/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py @@ -6,7 +6,8 @@ import pytest import torch from sglang.srt.layers.communicator import ( - ADD_AND_NORM, + ADD, + NORM_QUANT_READ, FfnExitFusion, LayerCommunicator, Layout, @@ -51,12 +52,17 @@ def _communicator(): comm._context = SimpleNamespace() comm._sp_steps = comm._input_scattered_steps = comm._cp_steps = None comm._steps = SimpleNamespace( - attention_prepare=partial( - _attention_input_step, - layer_input=None, - fusions=comm._attn_input_fusions, - enters_stack=False, - residual_ops=ADD_AND_NORM, + attention=SimpleNamespace( + prepare=partial( + _attention_input_step, + layer_input=None, + fusions=comm._attn_input_fusions, + enters_stack=False, + read=NORM_QUANT_READ, + update=ADD, + ), + input_move=None, + handoff=None, ) ) return comm @@ -111,14 +117,6 @@ def test_last_layer_consumes_but_does_not_skip_the_pending_all_reduce(eligible): hidden_states = UnreducedOutput(torch.zeros(8, 8)) with ( - patch.object( - CuteDSLFusionLayerCommunicator, - "_finish_prepare_attn", - lambda self, *, hidden_states, residual, forward_batch: ( - hidden_states, - residual, - ), - ), patch_communicator( "reduce_output", lambda *a, **k: pytest.fail("fell through to the unfused path"), @@ -184,7 +182,7 @@ def test_the_fusion_runs_only_on_the_ffn_full_rows(): (frozenset({TokenAxis.ATTN_TP_SCATTER}), False), ): comm._batch_steps = lambda fb, rows=Layout(sharded): SimpleNamespace( - ffn_input_rows=rows + ffn=SimpleNamespace(input_rows=rows) ) assert comm._common_eligible(_DECODE, 8) is eligible @@ -384,14 +382,6 @@ def test_a_handoff_the_kernel_does_not_take_is_completed_then_normed(): patch.object( CuteDSLFusionLayerCommunicator, "_common_eligible", return_value=False ), - patch.object( - CuteDSLFusionLayerCommunicator, - "_finish_prepare_attn", - lambda self, *, hidden_states, residual, forward_batch: ( - hidden_states, - residual, - ), - ), ): hidden, residual = comm.prepare_attn( _handoff(finish), torch.full((2, 8), 2.0), _DECODE diff --git a/test/registered/unit/layers/test_boundary_edges.py b/test/registered/unit/layers/test_boundary_edges.py index 409b361c45..83191166a5 100644 --- a/test/registered/unit/layers/test_boundary_edges.py +++ b/test/registered/unit/layers/test_boundary_edges.py @@ -1,13 +1,15 @@ import itertools import unittest from types import SimpleNamespace -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch import torch from sglang.srt.layers import communicator as comm from sglang.srt.layers.communicator import ( + ADD, EdgeDecl, + FusedMlpInput, InputRead, Layout, StageInput, @@ -260,5 +262,144 @@ class TestNonAlternatingEdges(CustomTestCase): make_boundary(edge, reads=None) +class ProbeRead: + """A read that marks what it reads: the input is the residual plus 100. + ``norms_plainly`` says whether it may stand in for a norm.""" + + def __init__(self, norms_plainly): + self.norms_plainly = norms_plainly + self.reads = 0 + + def enter(self, hidden_states): + return hidden_states + + def read(self, residual, norm, quant_format="", post_residual_addition=None): + self.reads += 1 + return residual + 100, residual + + def update_and_read( + self, + update, + hidden_states, + residual, + norm, + quant_format="", + post_residual_addition=None, + ): + return self.read(update.update(hidden_states, residual), norm) + + +class TestTheConsumerRunsItsDeclaredRead(CustomTestCase): + """Every input step reads the FFN input with the read the consumer + declares. A step that completes the sum onto the rows it reads from, or a + fused add + norm kernel, runs only for a read that leaves the residual as + it is.""" + + def _ffn_input(self, read, *, dp=1, residual_joins_sum=False, fusions=()): + axis_sizes = sizes(dp=dp, tp=2) + attention = rows(axis_sizes, TokenAxis.ATTN_DP) + residual = ( + rows(axis_sizes, TokenAxis.ATTN_DP, TokenAxis.ATTN_TP_SCATTER) + if residual_joins_sum + else attention + ) + need = rows(axis_sizes) if dp > 1 else attention + return comm_boundary._select_ffn_input( + StageOutput(attention, group=SumGroup.ATTN_TP, always_leaves=True), + residual=residual, + residual_to=attention, + need=StageInput(need, read=read), + force_layernorm_before_gather=False, + fusions=fusions, + residual_joins_sum=residual_joins_sum, + read=read, + update=ADD, + ) + + def test_the_dp_partial_reads_the_gathered_sum_with_the_declared_read(self): + read = ProbeRead(norms_plainly=True) + step, _ = self._ffn_input(read, dp=2) + self.assertIs(step.func, comm_ops._mlp_input_dp_partial) + hidden, residual = torch.ones(2, 4), torch.full((2, 4), 3.0) + with ( + patch.object( + comm_ops, + "_reduce_and_redistribute_output_to_dp", + lambda h, forward_batch, cp_shard_counts: h * 2, + ), + patch.object(comm_ops, "dp_scatter", lambda *args: None), + ): + out, out_residual = step( + hidden, residual, None, MagicMock(), SimpleNamespace(attn_tp_rank=0) + ) + self.assertEqual(read.reads, 1) + torch.testing.assert_close(out, torch.full((2, 4), 108.0)) + self.assertIs(out_residual, residual) + + def test_a_read_that_changes_the_residual_replicates_instead(self): + step, _ = self._ffn_input(ProbeRead(norms_plainly=False), dp=2) + self.assertIs(step.func, comm_ops._mlp_input_dp_replicate) + + def test_the_residual_joined_sum_is_read_with_the_declared_read(self): + read = ProbeRead(norms_plainly=True) + step, _ = self._ffn_input(read, residual_joins_sum=True) + self.assertIs(step.func, comm_ops._mlp_input_residual_into_sum) + hidden, residual = torch.ones(4, 4), torch.full((2, 4), 3.0) + with patch.object( + comm_ops, "tensor_model_parallel_all_reduce", lambda h: h * 2 + ): + out, out_residual = step( + hidden, + residual, + None, + MagicMock(), + SimpleNamespace(tp_size=2, tp_rank=0), + ) + self.assertEqual(read.reads, 1) + expected = torch.tensor([8.0, 8.0, 2.0, 2.0])[:, None].expand(4, 4) + torch.testing.assert_close(out_residual, expected) + torch.testing.assert_close(out, expected + 100) + + def test_a_fused_kernel_runs_only_for_a_read_that_leaves_the_residual(self): + fused = FusedMlpInput( + completes=SumGroup.ATTN_TP, run=MagicMock(), may_return_new_residual=True + ) + for norms_plainly in (True, False): + with self.subTest(norms_plainly=norms_plainly): + step, tried = self._ffn_input( + ProbeRead(norms_plainly), fusions=(fused,) + ) + self.assertIs(step.func, comm_ops._mlp_input_without_dp) + expected = (fused,) if norms_plainly else () + self.assertEqual(tried, expected) + self.assertEqual( + step.keywords["fusions"], tuple(f.run for f in expected) + ) + + def test_the_attention_input_tries_its_kernels_only_for_such_a_read(self): + axis_sizes = sizes(tp=2) + attention = rows(axis_sizes) + kernel = MagicMock() + for norms_plainly in (True, False): + with self.subTest(norms_plainly=norms_plainly): + read = ProbeRead(norms_plainly) + edge = EdgeDecl( + produced=StageOutput( + attention, group=SumGroup.TP, leaves_for_next_layer=True + ), + need=StageInput(attention, read=read), + residual=attention, + residual_to=attention, + ) + boundary = make_boundary( + edge, reads=InputRead.ATTENTION, fusions=(kernel,) + ) + self.assertIs(boundary.prepare.keywords["read"], read) + self.assertEqual( + boundary.prepare.keywords["fusions"], + (kernel,) if norms_plainly else (), + ) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/layers/test_communicator_ffn_exit.py b/test/registered/unit/layers/test_communicator_ffn_exit.py index fbea9e12ec..adaa4df2e4 100644 --- a/test/registered/unit/layers/test_communicator_ffn_exit.py +++ b/test/registered/unit/layers/test_communicator_ffn_exit.py @@ -53,20 +53,26 @@ def ordinary_steps(ffn_output, *, returns_over_dp=False): """A layer's ordinary steps, carrying the FFN output declaration the exit reads and whether the output goes back over attention DP.""" return comm.BoundarySteps( - attention_prepare=partial( - comm_boundary._attention_input_step, - layer_input=None, - fusions=(), - enters_stack=False, - residual_ops=comm.ADD_AND_NORM, + attention=comm.StageEntry( + prepare=partial( + comm_boundary._attention_input_step, + layer_input=None, + fusions=(), + enters_stack=False, + read=comm.NORM_QUANT_READ, + update=comm.ADD, + ), + input_rows=comm.Layout(frozenset()), + input_move=comm.CommunicateSimpleFn._trivial, + handoff=comm_ops._hand_qkv_hook_its_input, + ), + ffn=comm.StageEntry( + prepare=comm_ops._mlp_input_norm, input_rows=Layout(frozenset()) ), - attention_input=comm.CommunicateSimpleFn._trivial, - ffn_input=comm_ops._mlp_input_norm, - ffn_input_rows=Layout(frozenset()), ffn_output=ffn_output, - ffn_output_move=( - None if returns_over_dp else comm.CommunicateSummableTensorPairFn._trivial - ), + ffn_output_move=None + if returns_over_dp + else comm.CommunicateSummableTensorPairFn._trivial, ffn_sum_is_movable=True, ) diff --git a/test/registered/unit/layers/test_declared_decoder_boundary.py b/test/registered/unit/layers/test_declared_decoder_boundary.py index 37d518e83f..733f3bd03a 100644 --- a/test/registered/unit/layers/test_declared_decoder_boundary.py +++ b/test/registered/unit/layers/test_declared_decoder_boundary.py @@ -237,7 +237,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase): declarations, or none do.""" def declared(self, communicator): - return getattr(communicator._steps.ffn_input, "func", None) in ( + return getattr(communicator._steps.ffn.prepare, "func", None) in ( comm_ops._mlp_input_dp_partial, comm_ops._mlp_input_dp_replicate, comm_ops._mlp_input_without_dp, @@ -251,7 +251,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase): communicator = build(layer_facts(layer_id, 3), parallel) self.assertTrue(self.declared(communicator)) self.assertIs( - communicator._steps.attention_input, + communicator._steps.attention.input_move, comm.CommunicateSimpleFn._trivial, ) self.assertTrue(communicator._steps.returns_over_dp) @@ -266,17 +266,19 @@ class TestWhichLayersUseDeclarations(CustomTestCase): post_attention_layernorm=norm(), ) self.assertIs( - communicator._steps.ffn_input.func, comm_ops._mlp_input_without_dp + communicator._steps.ffn.prepare.func, comm_ops._mlp_input_without_dp ) entry = ( communicator._mlp_input_reduce_output_and_update_and_read_residual ) self.assertEqual( - communicator._steps.ffn_input.keywords["fusions"], + communicator._steps.ffn.prepare.keywords["fusions"], (entry,) if fuses else (), ) self.assertIs( - any(f.may_return_new_residual for f in communicator._steps.fused), + any( + f.may_return_new_residual for f in communicator._steps.ffn.fused + ), fuses, ) self.assertFalse(communicator._steps.returns_over_dp) @@ -289,7 +291,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase): single = build(layer_facts(1, 3), one_rank) with planning(one_rank): self.assertIsNotNone(single._declared_sides()) - self.assertIs(single._steps.ffn_input.func, comm_ops._mlp_input_norm) + self.assertIs(single._steps.ffn.prepare.func, comm_ops._mlp_input_norm) def test_the_order_follows_the_attention_output(self): for attn_tp, force, order in ( @@ -303,7 +305,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase): parallel_of(attn_dp=2, attn_tp=attn_tp), force_layernorm_before_dp_gather=force, ) - self.assertIs(communicator._steps.ffn_input.func, order) + self.assertIs(communicator._steps.ffn.prepare.func, order) def test_a_dense_first_layer_before_sparse_ones(self): parallel = parallel_of(attn_dp=2, attn_tp=2) @@ -336,25 +338,27 @@ class TestWhichLayersUseDeclarations(CustomTestCase): # The first a2a layer slices the residual it takes from a # dense layer; the next one takes it already sliced. self.assertEqual( - [layers[i]._steps.ffn_input.func for i in (1, 2)], + [layers[i]._steps.ffn.prepare.func for i in (1, 2)], [comm_ops._mlp_input_scatter] * 2, ) self.assertEqual( [ - layers[i]._steps.ffn_input.keywords["scatters_residual"] + layers[i]._steps.ffn.prepare.keywords["scatters_residual"] for i in (1, 2) ], [True, False], ) self.assertIsNone(layers[1]._steps.ffn_output.group) self.assertIs( - layers[2]._steps.attention_input, + layers[2]._steps.attention.input_move, comm.CommunicateSimpleFn._scattered_to_tp_attn_full, ) self.assertIs( - layers[3]._steps.ffn_input.func, comm_ops._mlp_input_dp_partial + layers[3]._steps.ffn.prepare.func, comm_ops._mlp_input_dp_partial + ) + self.assertTrue( + layers[3]._steps.ffn.prepare.keywords["gathers_residual"] ) - self.assertTrue(layers[3]._steps.ffn_input.keywords["gathers_residual"]) def test_a_dense_mlp_on_every_rank(self): # moe_dense_tp_size 1: each rank runs the dense MLP on its own slice, as @@ -389,12 +393,12 @@ class TestWhichLayersUseDeclarations(CustomTestCase): self.assertEqual([self.declared(layer) for layer in layers], [True] * 4) for dense in layers[:2]: self.assertIs( - dense._steps.ffn_input.func, comm_ops._mlp_input_scatter + dense._steps.ffn.prepare.func, comm_ops._mlp_input_scatter ) self.assertIsNone(dense._steps.ffn_output.group) for after_dense in layers[1:3]: self.assertIs( - after_dense._steps.attention_input, + after_dense._steps.attention.input_move, comm.CommunicateSimpleFn._scattered_to_tp_attn_full, ) @@ -419,7 +423,9 @@ class TestWhichLayersUseDeclarations(CustomTestCase): ) with patch_communicator("_use_ag_after_qlora", True): layer = build(modes, parallel, a2a=True) - self.assertIs(layer._steps.attention_input, comm.CommunicateSimpleFn._trivial) + self.assertIs( + layer._steps.attention.input_move, comm.CommunicateSimpleFn._trivial + ) def test_cp_without_prefill_cp(self): """No batch shards its tokens, so no CP steps. Under attention DP the @@ -480,7 +486,14 @@ class TestMhcOnTheDeclarations(CustomTestCase): def assert_step(self, step, func, communicator, **keywords): self.assertIs(step.func, func) - self.assertIs(step.keywords["residual_ops"], communicator.mhc) + residual = communicator._residual + if "read" in step.keywords: + # The FFN input: the attention output's hc_post, then the FFN's read. + self.assertIs(step.keywords["read"], residual.ffn_read) + self.assertIs(step.keywords["update"], residual.attention_update) + else: + # The FFN output's move: its hc_post. + self.assertIs(step.keywords["update"], residual.ffn_update) for key, value in keywords.items(): self.assertEqual(step.keywords[key], value, key) @@ -508,10 +521,12 @@ class TestMhcOnTheDeclarations(CustomTestCase): ): with self.subTest(name): communicator = build_mhc(layer_facts(1, 3), parallel) - self.assert_step(communicator._steps.ffn_input, ffn_input, communicator) + self.assert_step( + communicator._steps.ffn.prepare, ffn_input, communicator + ) self.assertIs(communicator._steps.ffn_output_move, output_move) # The fused add + RMSNorm kernels do not write hc_post. - self.assertEqual(communicator._steps.fused, ()) + self.assertEqual(communicator._steps.ffn.fused, ()) self.assertEqual(communicator._attn_input_fusions, ()) def test_the_move_back_over_dp_stays_with_the_layer(self): @@ -598,13 +613,13 @@ class TestMhcOnTheDeclarations(CustomTestCase): ] first_a2a, last = layers[1], layers[2] self.assert_step( - first_a2a._steps.ffn_input, + first_a2a._steps.ffn.prepare, comm_ops._mlp_input_scatter, first_a2a, scatters_residual=True, ) self.assert_step( - last._steps.ffn_input, + last._steps.ffn.prepare, comm_ops._mlp_input_scatter, last, scatters_residual=False, @@ -633,7 +648,9 @@ class TestMhcOnTheDeclarations(CustomTestCase): ) steps = communicator._input_scattered_steps self.assert_step( - steps.ffn_input, comm_ops._mlp_input_on_residual_shard, communicator + steps.ffn.prepare, + comm_ops._mlp_input_on_residual_shard, + communicator, ) self.assert_step( steps.ffn_output_move, @@ -651,12 +668,14 @@ class TestMhcOnTheDeclarations(CustomTestCase): # Only the first layer's input, the embedding's partial sum, # is completed onto the slice. self.assertIs( - steps.attention_prepare.keywords["layer_input"], + steps.attention.prepare.keywords["layer_input"], comm.tp_reduce_scatter if layer_id == 0 else None, ) - self.assertIs(steps.attention_input, comm.CommunicateSimpleFn._trivial) self.assertIs( - steps.attention_handoff, comm_ops._hand_scattered_input_to_attention + steps.attention.input_move, comm.CommunicateSimpleFn._trivial + ) + self.assertIs( + steps.attention.handoff, comm_ops._hand_scattered_input_to_attention ) self.assertIs( communicator._steps.ffn_output_move, @@ -671,7 +690,7 @@ class TestMhcOnTheDeclarations(CustomTestCase): layer_facts(1, 3, next_sparse=True), parallel, two_batch_overlap=True ) self.assert_step( - communicator._steps.ffn_input, comm_ops._mlp_input_scatter, communicator + communicator._steps.ffn.prepare, comm_ops._mlp_input_scatter, communicator ) self.assert_step( communicator._steps.ffn_output_move, @@ -811,13 +830,14 @@ class TestTwoBatchOverlap(CustomTestCase): self.assertIs(before._steps.ffn_output_move.func, pair._gather) self.assertEqual(after.input_rows, attention) self.assertIs( - after._steps.attention_input, comm.CommunicateSimpleFn._trivial + after._steps.attention.input_move, + comm.CommunicateSimpleFn._trivial, ) else: self.assertIs(before._steps.ffn_output_move, pair._trivial) self.assertEqual(after.input_rows, local) self.assertIs( - after._steps.attention_input, + after._steps.attention.input_move, comm.CommunicateSimpleFn._scattered_to_tp_attn_full, ) @@ -905,7 +925,7 @@ class TestTwoBatchOverlap(CustomTestCase): ) steps = communicator._steps self.assertEqual( - steps.ffn_input_rows, communicator._declared.ffn.layout + steps.ffn.input_rows, communicator._declared.ffn.layout ) published = {} with patch_communicator( @@ -1105,7 +1125,9 @@ class TestFusedKernelsTakeOnlyTheStepsTheyComplete(CustomTestCase): post_attention_layernorm=Norm(), ) self.assertIs( - any(f.may_return_new_residual for f in communicator._steps.fused), + any( + f.may_return_new_residual for f in communicator._steps.ffn.fused + ), expected, ) @@ -1118,11 +1140,16 @@ class TestTheSequenceParallelRegion(CustomTestCase): SIZES = {TokenAxis.ATTN_DP: 1, TokenAxis.ATTN_CP: 1, TokenAxis.ATTN_TP_SCATTER: 2} SP_STEPS = comm.BoundarySteps( - attention_prepare=comm_boundary._attention_input_step, - attention_input=comm.CommunicateSimpleFn._trivial, - ffn_input=comm_ops._mlp_input_norm, - # The linears gather each rank's slice themselves. - ffn_input_rows=comm.Layout(frozenset({TokenAxis.ATTN_TP_SCATTER})), + attention=comm.StageEntry( + prepare=comm_boundary._attention_input_step, + input_rows=comm.Layout(frozenset({TokenAxis.ATTN_TP_SCATTER})), + input_move=comm.CommunicateSimpleFn._trivial, + handoff=comm_ops._hand_qkv_hook_its_input, + ), + ffn=comm.StageEntry( + prepare=comm_ops._mlp_input_norm, + input_rows=comm.Layout(frozenset({TokenAxis.ATTN_TP_SCATTER})), + ), ffn_output=sequence_parallel_layer_sides(axis_sizes=SIZES).ffn_output, ffn_output_move=comm.CommunicateSummableTensorPairFn._trivial, ffn_sum_is_movable=False, @@ -1131,14 +1158,18 @@ class TestTheSequenceParallelRegion(CustomTestCase): def unbound(self, steps): """``steps`` with the plain residual's binding taken off the FFN input and the attention input, whose input owes nothing in the region.""" - self.assertEqual(steps.ffn_input.keywords, {"residual_ops": comm.ADD_AND_NORM}) - attention = steps.attention_prepare.keywords + self.assertEqual( + steps.ffn.prepare.keywords, {"read": comm.NORM_READ, "update": comm.ADD} + ) + attention = steps.attention.prepare.keywords self.assertIsNone(attention["layer_input"]) - self.assertIs(attention["residual_ops"], comm.ADD_AND_NORM) - return msgspec.structs.replace( + self.assertIs(attention["read"], comm.NORM_QUANT_READ) + self.assertIs(attention["update"], comm.ADD) + replace = msgspec.structs.replace + return replace( steps, - ffn_input=steps.ffn_input.func, - attention_prepare=steps.attention_prepare.func, + ffn=replace(steps.ffn, prepare=steps.ffn.prepare.func), + attention=replace(steps.attention, prepare=steps.attention.prepare.func), ) def test_the_region_declarations_choose_local_steps(self): @@ -1159,10 +1190,14 @@ class TestTheSequenceParallelRegion(CustomTestCase): self.assertIsNone(sides.ffn_output.group) steps = self.unbound(comm_boundary._select_boundary_steps(sides)) self.assertEqual( - (steps.attention_input, steps.ffn_input, steps.ffn_output_move), ( - self.SP_STEPS.attention_input, - self.SP_STEPS.ffn_input, + steps.attention.input_move, + steps.ffn.prepare, + steps.ffn_output_move, + ), + ( + self.SP_STEPS.attention.input_move, + self.SP_STEPS.ffn.prepare, self.SP_STEPS.ffn_output_move, ), ) @@ -1179,7 +1214,7 @@ class TestTheSequenceParallelRegion(CustomTestCase): ) self.assertEqual(self.unbound(communicator._sp_steps), self.SP_STEPS) self.assertIs( - communicator._steps.ffn_input.func, comm_ops._mlp_input_without_dp + communicator._steps.ffn.prepare.func, comm_ops._mlp_input_without_dp ) def test_what_the_ffn_exit_reads_comes_from_the_batch_s_steps(self): @@ -1247,11 +1282,18 @@ class TestInputScatteredAttention(CustomTestCase): ) steps = comm_boundary._select_boundary_steps(sides) self.assertIs( - steps.attention_prepare.keywords["layer_input"], + steps.attention.prepare.keywords["layer_input"], comm.tp_reduce_scatter, ) - self.assertIs(steps.attention_input, comm.CommunicateSimpleFn._trivial) - self.assertIs(steps.ffn_input, comm_ops._mlp_input_residual_into_sum) + self.assertIs( + steps.attention.input_move, comm.CommunicateSimpleFn._trivial + ) + slice_ = comm.Layout(frozenset({TokenAxis.ATTN_TP_SCATTER})) + self.assertEqual(steps.attention.input_rows, slice_) + self.assertEqual(steps.ffn.input_rows, comm.Layout(frozenset())) + self.assertIs( + steps.ffn.prepare.func, comm_ops._mlp_input_residual_into_sum + ) self.assertIs( steps.ffn_output_move, comm.CommunicateSummableTensorPairFn._trivial, @@ -1470,7 +1512,7 @@ class TestTheAttentionInputHalf(CustomTestCase): ] self.assertEqual(len(variants), 2) for steps in variants: - bound = steps.attention_prepare.keywords + bound = steps.attention.prepare.keywords self.assertIs(bound["fusions"], fusions) self.assertIs(bound["enters_stack"], layer_id == 0) @@ -1504,8 +1546,8 @@ class TestPrefillCP(CustomTestCase): ) communicator = build(modes, parallel, dsa_cp=True, allow_reduce_scatter=True) cp, ordinary = communicator._cp_steps, communicator._steps - self.assertIs(cp.ffn_input.func, comm_ops._mlp_input_gather_attention_cp) - self.assertIs(cp.ffn_input.keywords["gather"].func, comm_ops._mlp_input_norm) + self.assertIs(cp.ffn.prepare.func, comm_ops._mlp_input_gather_attention_cp) + self.assertIs(cp.ffn.prepare.keywords["gather"].func, comm_ops._mlp_input_norm) self.assertIs( cp.ffn_output_move, comm.CommunicateSummableTensorPairFn._reduce_scatter_over_cp, @@ -1513,9 +1555,9 @@ class TestPrefillCP(CustomTestCase): self.assertTrue(cp.ffn_output.leaves_for_reduce_scatter) self.assertFalse(cp.ffn_output.leaves_for_next_layer) # The dense layer before it ran on the same shard: nothing to move. - self.assertIs(cp.attention_input, comm.CommunicateSimpleFn._trivial) + self.assertIs(cp.attention.input_move, comm.CommunicateSimpleFn._trivial) # Other batches hold every token on each CP rank: the MoE sums itself. - self.assertIs(ordinary.ffn_input.func, comm_ops._mlp_input_norm) + self.assertIs(ordinary.ffn.prepare.func, comm_ops._mlp_input_norm) self.assertIs( ordinary.ffn_output_move, comm.CommunicateSummableTensorPairFn._trivial ) @@ -1573,7 +1615,7 @@ class TestPrefillCP(CustomTestCase): ) communicator = build(modes, parallel, dsa_cp=True, allow_reduce_scatter=True) for steps in (communicator._steps, communicator._cp_steps): - self.assertIs(steps.ffn_input.func, comm_ops._mlp_input_norm) + self.assertIs(steps.ffn.prepare.func, comm_ops._mlp_input_norm) self.assertIsNone(steps.ffn_output.group) self.assertIs( steps.ffn_output_move, comm.CommunicateSummableTensorPairFn._trivial @@ -1582,10 +1624,10 @@ class TestPrefillCP(CustomTestCase): def test_a_cp_extend_gathers_over_cp_and_takes_its_chunk_back(self): communicator = build(layer_facts(1, 3), self.cp_parallel()) cp = communicator._cp_steps - self.assertIs(cp.ffn_input.func, comm_ops._mlp_input_gather_moe_cp) + self.assertIs(cp.ffn.prepare.func, comm_ops._mlp_input_gather_moe_cp) # Each rank completes its own chunk before the gather. self.assertIs( - cp.ffn_input.keywords["gather"].func, comm_ops._mlp_input_without_dp + cp.ffn.prepare.keywords["gather"].func, comm_ops._mlp_input_without_dp ) self.assertIs( cp.ffn_output_move, @@ -1595,7 +1637,7 @@ class TestPrefillCP(CustomTestCase): self.assertFalse(cp.ffn_output.leaves_for_next_layer) self.assertFalse(cp.ffn_output.leaves_for_reduce_scatter) self.assertIs( - communicator._steps.ffn_input.func, comm_ops._mlp_input_without_dp + communicator._steps.ffn.prepare.func, comm_ops._mlp_input_without_dp ) self.assertIs( communicator._steps.ffn_output_move, @@ -1612,13 +1654,15 @@ class TestPrefillCP(CustomTestCase): ) for steps in (communicator._cp_steps, communicator._steps): # The attention's rows are the MoE's: nothing to gather or take back. - self.assertIs(steps.ffn_input.func, comm_ops._mlp_input_without_dp) + self.assertIs(steps.ffn.prepare.func, comm_ops._mlp_input_without_dp) self.assertIs( steps.ffn_output_move, comm.CommunicateSummableTensorPairFn._trivial ) self.assertTrue(steps.ffn_output.leaves_for_next_layer) dense = build(layer_facts(1, 3), parallel, allow_deferred_ffn_reduction=True) - self.assertIs(dense._cp_steps.ffn_input.func, comm_ops._mlp_input_gather_moe_cp) + self.assertIs( + dense._cp_steps.ffn.prepare.func, comm_ops._mlp_input_gather_moe_cp + ) self.assertFalse(dense._cp_steps.ffn_output.leaves_for_next_layer) def test_the_fused_kernels_run_on_each_chunk(self): @@ -1628,7 +1672,7 @@ class TestPrefillCP(CustomTestCase): input_layernorm=FusableNorm(), post_attention_layernorm=FusableNorm(), ) - chunk = communicator._cp_steps.ffn_input.keywords["gather"] + chunk = communicator._cp_steps.ffn.prepare.keywords["gather"] self.assertEqual( chunk.keywords["fusions"], (communicator._mlp_input_reduce_output_and_update_and_read_residual,), @@ -1730,13 +1774,13 @@ class TestPrefillCP(CustomTestCase): parallel = parallel_of(attn_dp=2, attn_tp=2, attn_cp=2, enable_prefill_cp=True) communicator = build(layer_facts(1, 3), parallel) cp, ordinary = communicator._cp_steps, communicator._steps - self.assertIs(cp.ffn_input.func, comm_ops._mlp_input_dp_partial) - self.assertTrue(cp.ffn_input.keywords["places_cp_shards"]) + self.assertIs(cp.ffn.prepare.func, comm_ops._mlp_input_dp_partial) + self.assertTrue(cp.ffn.prepare.keywords["places_cp_shards"]) self.assertIs( cp.ffn_output_move, comm.CommunicateSummableTensorPairFn._take_back_cp_shard ) - self.assertIs(ordinary.ffn_input.func, comm_ops._mlp_input_dp_partial) - self.assertFalse(ordinary.ffn_input.keywords["places_cp_shards"]) + self.assertIs(ordinary.ffn.prepare.func, comm_ops._mlp_input_dp_partial) + self.assertFalse(ordinary.ffn.prepare.keywords["places_cp_shards"]) self.assertTrue(ordinary.returns_over_dp) for steps in (cp, ordinary): self.assertFalse(steps.ffn_output.leaves_for_next_layer) @@ -2357,9 +2401,9 @@ class TestTheFfnInputReduction(CustomTestCase): ) for adds_plainly, reduction in ((True, "quant"), (False, "full")): with self.subTest(adds_plainly=adds_plainly): - ops = SimpleNamespace( - adds_plainly=adds_plainly, - update_and_read_ffn_input=lambda h, r, norm: (h, r), + update = SimpleNamespace(adds_plainly=adds_plainly) + read = SimpleNamespace( + update_and_read=lambda update, h, r, norm: (h, r) ) calls = [] with ( @@ -2381,10 +2425,86 @@ class TestTheFfnInputReduction(CustomTestCase): SimpleNamespace(cache=None), gathers_residual=False, fusions=(), - residual_ops=ops, + read=read, + update=update, ) self.assertEqual(calls, [reduction]) +class TestALayerThatIsOneStage(CustomTestCase): + """A layer that is one stage reads its input and writes its output into the + residual as its stage declares.""" + + SIZES = {TokenAxis.ATTN_DP: 1, TokenAxis.ATTN_CP: 1, TokenAxis.ATTN_TP_SCATTER: 2} + + class ProbeRead: + norms_plainly = True + + def __init__(self): + self.reads = 0 + + def read(self, residual, norm, quant_format="", post_residual_addition=None): + self.reads += 1 + return residual + 100, residual + + def update_and_read( + self, + update, + hidden_states, + residual, + norm, + quant_format="", + post_residual_addition=None, + ): + return self.read(update.update(hidden_states, residual), norm) + + def _stage(self, read, update): + rows = comm.Layout.sharded_over(axis_sizes=self.SIZES) + return comm.LayerStage( + reads=comm.InputRead.FFN, + edges=comm.stage_edges( + previous=None, + stage=comm.StageDecl( + comm.StageInput(rows, read=read), + comm.StageOutput(rows, update=update), + ), + rows=rows, + ), + ) + + def test_its_steps_run_the_declared_read_and_update(self): + read, update = self.ProbeRead(), SimpleNamespace(adds_plainly=True) + communicator = build( + layer_facts(1, 3), + parallel_of(attn_dp=1, attn_tp=2), + stage=self._stage(read, update), + ) + into, out_of = communicator.stage_edges + self.assertIs(into.need.read, read) + self.assertIs(out_of.produced.update, update) + hidden, residual = torch.ones(2, HIDDEN), torch.full((2, HIDDEN), 3.0) + out, out_residual = communicator._steps.ffn.prepare( + hidden, residual, None, Norm(), None + ) + self.assertEqual(read.reads, 1) + torch.testing.assert_close(out_residual, torch.full((2, HIDDEN), 4.0)) + torch.testing.assert_close(out, torch.full((2, HIDDEN), 104.0)) + + def test_it_takes_no_residual_of_its_own(self): + own = comm.LayerResidual( + attention_read=self.ProbeRead(), + attention_update=comm.ADD, + ffn_read=comm.NORM_READ, + ffn_update=comm.ADD, + ) + with self.assertRaises(ValueError): + build( + layer_facts(1, 3), + parallel_of(attn_dp=1, attn_tp=2), + stage=self._stage(comm.NORM_READ, comm.ADD), + residual=own, + ) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/layers/test_dp_attention_cp_gather.py b/test/registered/unit/layers/test_dp_attention_cp_gather.py index 4030cf2307..08f5e4a9f3 100644 --- a/test/registered/unit/layers/test_dp_attention_cp_gather.py +++ b/test/registered/unit/layers/test_dp_attention_cp_gather.py @@ -276,7 +276,7 @@ class TestDpCpGather(CustomTestCase): self.assertIsNotNone(r.communicator._cp_steps, "declared under CP") if ffn_input is not None: steps = r.communicator._batch_steps(r.forward_batch) - self.assertIs(steps.ffn_input.func, ffn_input) + self.assertIs(steps.ffn.prepare.func, ffn_input) hidden_states, residual = rank_inputs(*rank) return r.communicator.prepare_mlp( hidden_states, residual, r.forward_batch diff --git a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py index 97c2c4d907..83d06ffe20 100644 --- a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py +++ b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py @@ -49,20 +49,26 @@ def sp_region_steps(): def _steps(*, ffn_output=None, returns_over_dp=False, ffn_sum_is_movable=True): return comm.BoundarySteps( - attention_prepare=partial( - comm_boundary._attention_input_step, - layer_input=None, - fusions=(), - enters_stack=False, - residual_ops=comm.ADD_AND_NORM, + attention=comm.StageEntry( + prepare=partial( + comm_boundary._attention_input_step, + layer_input=None, + fusions=(), + enters_stack=False, + read=comm.NORM_QUANT_READ, + update=comm.ADD, + ), + input_rows=comm.Layout(frozenset()), + input_move=comm.CommunicateSimpleFn._trivial, + handoff=comm_ops._hand_qkv_hook_its_input, + ), + ffn=comm.StageEntry( + prepare=comm_ops._mlp_input_norm, input_rows=Layout(frozenset()) ), - attention_input=comm.CommunicateSimpleFn._trivial, - ffn_input=comm_ops._mlp_input_norm, - ffn_input_rows=Layout(frozenset()), ffn_output=ffn_output or StageOutput(Layout(frozenset()), group=SumGroup.TP), - ffn_output_move=( - None if returns_over_dp else comm.CommunicateSummableTensorPairFn._trivial - ), + ffn_output_move=None + if returns_over_dp + else comm.CommunicateSummableTensorPairFn._trivial, ffn_sum_is_movable=ffn_sum_is_movable, ) diff --git a/test/registered/unit/layers/test_layernorm_sp.py b/test/registered/unit/layers/test_layernorm_sp.py index 03ef34de0a..12ea23d53a 100644 --- a/test/registered/unit/layers/test_layernorm_sp.py +++ b/test/registered/unit/layers/test_layernorm_sp.py @@ -22,6 +22,7 @@ from sglang.srt.layers.communicator import ( TokenAxis, ) from sglang.srt.layers.communicator import boundary as comm_boundary +from sglang.srt.layers.communicator import ops as comm_ops from sglang.srt.layers.communicator import ( sequence_parallel_layer_sides, ) @@ -191,16 +192,23 @@ class TestSpRegionSteps(CustomTestCase): def ordinary_steps(self, attention_input): """The layer's steps outside the region, which must not run inside it.""" return comm.BoundarySteps( - attention_prepare=partial( - comm_boundary._attention_input_step, - layer_input=None, - fusions=(), - enters_stack=False, - residual_ops=comm.ADD_AND_NORM, + attention=comm.StageEntry( + prepare=partial( + comm_boundary._attention_input_step, + layer_input=None, + fusions=(), + enters_stack=False, + read=comm.NORM_QUANT_READ, + update=comm.ADD, + ), + input_rows=comm.Layout(frozenset()), + input_move=attention_input, + handoff=comm_ops._hand_qkv_hook_its_input, + ), + ffn=comm.StageEntry( + prepare=MagicMock(side_effect=AssertionError("ordinary FFN input ran")), + input_rows=Layout(frozenset()), ), - attention_input=attention_input, - ffn_input=MagicMock(side_effect=AssertionError("ordinary FFN input ran")), - ffn_input_rows=Layout(frozenset()), ffn_output=StageOutput(Layout(frozenset()), group=SumGroup.TP), ffn_output_move=MagicMock(side_effect=AssertionError("postprocess ran")), ffn_sum_is_movable=True, diff --git a/test/registered/unit/layers/test_postprocess_reduce_scatterv.py b/test/registered/unit/layers/test_postprocess_reduce_scatterv.py index 944a37180c..6496649cfb 100644 --- a/test/registered/unit/layers/test_postprocess_reduce_scatterv.py +++ b/test/registered/unit/layers/test_postprocess_reduce_scatterv.py @@ -37,16 +37,22 @@ def steps(*, ffn_output_move): """A layer's steps with the given FFN output move; None sends the output back over attention DP.""" return comm.BoundarySteps( - attention_prepare=partial( - comm_boundary._attention_input_step, - layer_input=None, - fusions=(), - enters_stack=False, - residual_ops=comm.ADD_AND_NORM, + attention=comm.StageEntry( + prepare=partial( + comm_boundary._attention_input_step, + layer_input=None, + fusions=(), + enters_stack=False, + read=comm.NORM_QUANT_READ, + update=comm.ADD, + ), + input_rows=comm.Layout(frozenset()), + input_move=comm.CommunicateSimpleFn._trivial, + handoff=comm_ops._hand_qkv_hook_its_input, + ), + ffn=comm.StageEntry( + prepare=comm_ops._mlp_input_norm, input_rows=comm.Layout(frozenset()) ), - attention_input=comm.CommunicateSimpleFn._trivial, - ffn_input=comm_ops._mlp_input_norm, - ffn_input_rows=comm.Layout(frozenset()), ffn_output=comm.StageOutput(comm.Layout(frozenset())), ffn_output_move=ffn_output_move, ffn_sum_is_movable=False, diff --git a/test/registered/unit/layers/test_prepare_attn_steps.py b/test/registered/unit/layers/test_prepare_attn_steps.py index 4b07af3145..b0bb9dd1e0 100644 --- a/test/registered/unit/layers/test_prepare_attn_steps.py +++ b/test/registered/unit/layers/test_prepare_attn_steps.py @@ -60,16 +60,22 @@ def communicator(norm): c._attn_input_fusions = c._select_attn_input_fusions() # A layer whose attention takes its input as it is and owes nothing on it. c._steps = comm.BoundarySteps( - attention_prepare=partial( - comm_boundary._attention_input_step, - layer_input=None, - fusions=c._attn_input_fusions, - enters_stack=False, - residual_ops=comm.ADD_AND_NORM, + attention=comm.StageEntry( + prepare=partial( + comm_boundary._attention_input_step, + layer_input=None, + fusions=c._attn_input_fusions, + enters_stack=False, + read=comm.NORM_QUANT_READ, + update=comm.ADD, + ), + input_rows=comm.Layout(frozenset()), + input_move=lambda hidden_states, **_: hidden_states, + handoff=comm_ops._hand_qkv_hook_its_input, + ), + ffn=comm.StageEntry( + prepare=comm_ops._mlp_input_norm, input_rows=comm.Layout(frozenset()) ), - attention_input=lambda hidden_states, **_: hidden_states, - ffn_input=comm_ops._mlp_input_norm, - ffn_input_rows=comm.Layout(frozenset()), ffn_output=comm.StageOutput(comm.Layout(frozenset())), ffn_output_move=comm.CommunicateSummableTensorPairFn._trivial, ffn_sum_is_movable=False, @@ -240,12 +246,16 @@ class TestPrepareAttnSteps(CustomTestCase): takes_anything = MagicMock(return_value=("fused", "fused")) c._steps = msgspec.structs.replace( c._steps, - attention_prepare=partial( - comm_boundary._attention_input_step, - layer_input=None, - fusions=(takes_anything,), - enters_stack=False, - residual_ops=comm.ADD_AND_NORM, + attention=msgspec.structs.replace( + c._steps.attention, + prepare=partial( + comm_boundary._attention_input_step, + layer_input=None, + fusions=(takes_anything,), + enters_stack=False, + read=comm.NORM_QUANT_READ, + update=comm.ADD, + ), ), ) partial_sum = torch.ones(3, 4) diff --git a/test/registered/unit/lora/test_triton_dp_attention_unit.py b/test/registered/unit/lora/test_triton_dp_attention_unit.py index 2bda7e75c2..6e3cbc9911 100644 --- a/test/registered/unit/lora/test_triton_dp_attention_unit.py +++ b/test/registered/unit/lora/test_triton_dp_attention_unit.py @@ -10,7 +10,8 @@ import pytest import torch from sglang.srt.layers.communicator import ( - ADD_AND_NORM, + ADD, + NORM_QUANT_READ, LayerCommunicator, Layout, TokenAxis, @@ -238,20 +239,27 @@ def test_communicator_publishes_layout_at_each_transition( gathered, local = Layout(frozenset()), Layout(frozenset({TokenAxis.ATTN_DP})) # The rows of the steps the batch runs decide, not the ordinary steps'. communicator._steps = SimpleNamespace( - ffn_input_rows=local if gathered_over_dp else gathered + ffn=SimpleNamespace(input_rows=local if gathered_over_dp else gathered) ) selected = SimpleNamespace( - attention_prepare=partial( - _attention_input_step, - layer_input=None, - fusions=(), - enters_stack=False, - residual_ops=ADD_AND_NORM, + attention=SimpleNamespace( + prepare=partial( + _attention_input_step, + layer_input=None, + fusions=(), + enters_stack=False, + read=NORM_QUANT_READ, + update=ADD, + ), + input_move=lambda hidden_states, **kwargs: hidden_states, + handoff=lambda hidden_states, *args: hidden_states, + ), + ffn=SimpleNamespace( + prepare=lambda hidden_states, residual, *args: (hidden_states, residual), + input_rows=gathered if gathered_over_dp else local, + input_move=None, + handoff=None, ), - attention_input=lambda hidden_states, **kwargs: hidden_states, - attention_handoff=lambda hidden_states, *args: hidden_states, - ffn_input=lambda hidden_states, residual, *args: (hidden_states, residual), - ffn_input_rows=gathered if gathered_over_dp else local, ) communicator._batch_steps = lambda forward_batch: selected hidden = torch.zeros(num_tokens, 4) diff --git a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py index 434e464816..61f48f2e7d 100644 --- a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py +++ b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py @@ -50,16 +50,22 @@ class DeferringLayer(nn.Module): self.return_topk = return_topk self.layer_communicator = comm.LayerCommunicator.__new__(comm.LayerCommunicator) self.layer_communicator._steps = comm.BoundarySteps( - attention_prepare=partial( - comm_boundary._attention_input_step, - layer_input=None, - fusions=(), - enters_stack=False, - residual_ops=comm.ADD_AND_NORM, + attention=comm.StageEntry( + prepare=partial( + comm_boundary._attention_input_step, + layer_input=None, + fusions=(), + enters_stack=False, + read=comm.NORM_QUANT_READ, + update=comm.ADD, + ), + input_rows=comm.Layout(frozenset()), + input_move=comm.CommunicateSimpleFn._trivial, + handoff=comm_ops._hand_qkv_hook_its_input, + ), + ffn=comm.StageEntry( + prepare=comm_ops._mlp_input_norm, input_rows=comm.Layout(frozenset()) ), - attention_input=comm.CommunicateSimpleFn._trivial, - ffn_input=comm_ops._mlp_input_norm, - ffn_input_rows=comm.Layout(frozenset()), ffn_output=comm.StageOutput( comm.Layout(frozenset()), group=comm.SumGroup.TP, diff --git a/test/registered/unit/models/test_nemotron_h_stages.py b/test/registered/unit/models/test_nemotron_h_stages.py index 65824661f3..4fdcc14c0a 100644 --- a/test/registered/unit/models/test_nemotron_h_stages.py +++ b/test/registered/unit/models/test_nemotron_h_stages.py @@ -6,6 +6,9 @@ from unittest.mock import MagicMock, patch import torch from sglang.srt.layers.communicator import ( + ADD, + NORM_QUANT_READ, + NORM_READ, Layout, MixerExit, StageDecl, @@ -112,6 +115,17 @@ class TestStageEdges(CustomTestCase): (SumGroup.TP, False, True), ) + def test_each_stage_reads_and_writes_as_a_layer_s_attention_or_ffn(self): + for pattern, tp in itertools.product(self.PATTERNS, (1, 2)): + with self.subTest(pattern=pattern, tp=tp): + for kind, layer in zip(pattern, stages(pattern, tp=tp)): + into, out_of = layer.edges + self.assertIs( + into.need.read, + NORM_QUANT_READ if utils.is_attn_layer(kind) else NORM_READ, + ) + self.assertIs(out_of.produced.update, ADD) + def test_the_residual_follows_the_input_onto_a_finer_slice(self): axis_sizes = sizes(dp=2, tp=2) attention = Layout.sharded_over(TokenAxis.ATTN_DP, axis_sizes=axis_sizes)