diff --git a/python/sglang/srt/layers/layer_boundary/boundary.py b/python/sglang/srt/layers/layer_boundary/boundary.py index 585ba53fba..e12aa83af7 100644 --- a/python/sglang/srt/layers/layer_boundary/boundary.py +++ b/python/sglang/srt/layers/layer_boundary/boundary.py @@ -166,11 +166,19 @@ class _TransformedRead: transform: OutputTransform is_plain_norm = False + # The inner read's kernels read the stream directly and would skip the + # transform, so none is offered. + completing_fusions = () + gathering_reads = () @property def reads_before_dp_gather(self): return self.inner.reads_before_dp_gather + @property + def reads_after_attn_tp_gather(self): + return self.inner.reads_after_attn_tp_gather + def init_residual(self, hidden_states): return self.inner.init_residual(hidden_states) @@ -368,8 +376,8 @@ def bind_exit( transport. The receiver binds its read independently. """ update = edge.produced.update - if not getattr(update, "applied_at_exit", False): - if not getattr(update, "outlives_layer", False): + if not update.applied_at_exit: + if not update.outlives_layer: raise NotImplementedError( "a deferred update must guarantee its lifetime across layers" ) @@ -396,7 +404,7 @@ def _read_fusions(read, owes, is_plain_add, *, scatters): return () return tuple( f - for f in getattr(read, "completing_fusions", ()) + for f in read.completing_fusions if f.completes is owes and f.scatters == scatters ) @@ -558,9 +566,7 @@ def _select_entry_step( enters_stack=enters_stack, read=read, read_gathers=( - getattr(read, "gathering_reads", ()) - if is_plain_add or arrives_written - else () + read.gathering_reads if is_plain_add or arrives_written else () ), ), input_move=( diff --git a/python/sglang/srt/layers/layer_boundary/construction.py b/python/sglang/srt/layers/layer_boundary/construction.py index 080377f960..ca400e278c 100644 --- a/python/sglang/srt/layers/layer_boundary/construction.py +++ b/python/sglang/srt/layers/layer_boundary/construction.py @@ -313,6 +313,14 @@ def _bind_stage(declaration, norm, incoming, outgoing, *, final_read=None, **opt raise ValueError("connections do not match the stage declaration") if incoming.entries.keys() != outgoing.exits.keys(): raise ValueError("incoming and outgoing batch variants disagree") + update = declaration.update + if declaration.terminal and not (update.is_plain_add or update.applied_at_exit): + # The final read adds the last output into the residual itself, as a + # plain add, before its norm. + raise NotImplementedError( + f"the layer stack ends on a stage whose {type(update).__name__} " + "update the final read would apply as a plain add" + ) variants = {} for variant, edge in incoming.entries.items(): if ( @@ -321,13 +329,17 @@ def _bind_stage(declaration, norm, incoming, outgoing, *, final_read=None, **opt in edge.produced.layout.sharded - edge.need.layout.sharded and not _cp_gathers_over_attn_cp() ): - raise NotImplementedError("MHC with a gather over attention CP") + raise NotImplementedError( + "an update applied at the stage's exit with a gather over attention CP" + ) attn_input_adapter = None if declaration.kind is StageKind.ATTENTION: + # On an input-scattered batch the step gathers the rows itself for + # an attention whose QKV hook does not gather them after the + # projection. attn_input_adapter = ( _attn_input_scattered if variant is BatchVariant.INPUT_SCATTERED - and not declaration.update.is_plain_add else _attn_input_default ) moves = _cp_moves() if variant is BatchVariant.CONTEXT_PARALLEL else None diff --git a/python/sglang/srt/layers/layer_boundary/exit.py b/python/sglang/srt/layers/layer_boundary/exit.py index 94d0dc6e3b..a47f2cb409 100644 --- a/python/sglang/srt/layers/layer_boundary/exit.py +++ b/python/sglang/srt/layers/layer_boundary/exit.py @@ -271,7 +271,7 @@ class ExitPolicy: hidden_states, steps.output.group, forward_batch, - may_quantize=steps.output.update.is_plain_add, + may_quantize=steps.output.update.quantized_sum, ) if steps.output.transform is not None: hidden_states = steps.output.transform.apply(hidden_states) @@ -476,7 +476,7 @@ class MixerExit: hidden_states, self._group, self._forward_batch, - may_quantize=self._update.is_plain_add, + may_quantize=self._update.quantized_sum, ) return self._stream.record( hidden_states, self._update, declared_sum=self._declared_sum diff --git a/python/sglang/srt/layers/layer_boundary/factories.py b/python/sglang/srt/layers/layer_boundary/factories.py index 2b7c2dd931..f576ce481b 100644 --- a/python/sglang/srt/layers/layer_boundary/factories.py +++ b/python/sglang/srt/layers/layer_boundary/factories.py @@ -45,12 +45,14 @@ from sglang.srt.layers.layer_boundary.layout import ( token_axis_sizes, ) from sglang.srt.layers.layer_boundary.output import OutputTransform -from sglang.srt.layers.layer_boundary.residual import ResidualReadout, ResidualUpdate +from sglang.srt.layers.layer_boundary.residual import ( + ResidualReadout, + ResidualUpdate, +) from sglang.srt.layers.layer_boundary.residual.add_norm import ( NORM_QUANT_READOUT, NORM_READOUT, PLAIN_ADD, - REPLACE_AT_EXIT, ) from sglang.srt.layers.layer_boundary.stage import StageBoundary from sglang.srt.layers.moe import is_moe_input_scattered_across_dp_ranks @@ -150,6 +152,8 @@ def _resolve_ffn( group = SumGroup.MOE_OUTPUT else: group = SumGroup.ATTN_TP if on_attention_rows else SumGroup.TP + # An FFN that writes the next stream itself hands on a complete output. + complete = on_rank_rows or update.writes_stream or output_complete if variant is BatchVariant.SEQUENCE_PARALLEL: return ( StageContract( @@ -172,8 +176,9 @@ def _resolve_ffn( InputContract(full, read=read), OutputContract( attention, - group=group, - may_reduce_scatter=use_reduce_scatter + group=None if complete else group, + may_reduce_scatter=not complete + and use_reduce_scatter and (scattered_residual or not terminal), update=update, transform=output_transform, @@ -199,8 +204,6 @@ def _resolve_ffn( rows = Layout.sharded_over( *((TokenAxis.ATTN_CP,) if on_cp_shards else ()), axis_sizes=axes ) - # An FFN that writes the next stream itself hands on a complete output. - complete = on_rank_rows or update is REPLACE_AT_EXIT or output_complete produced = ( OutputContract(rows, update=update, transform=output_transform) if complete @@ -301,6 +304,51 @@ class StageDeclaration: for source in (self.previous, self.prepared_from): if source is not None and not isinstance(source, StageDeclaration): raise TypeError("stage sources must be declarations") + _check_update(self.update) + _check_read(self.read) + + +# The members every ResidualUpdate and ResidualReadout must set: the binding +# reads each of them, and none has a default that is correct for every +# implementation. +_UPDATE_FACTS = ( + "is_plain_add", + "applied_at_exit", + "outlives_layer", + "writes_stream", + "quantized_sum", +) +_READ_FACTS = ("is_plain_norm", "reads_before_dp_gather", "reads_after_attn_tp_gather") +# The kernels a read supplies, which the binding also reads; empty when it +# supplies none. +_READ_KERNELS = ("completing_fusions", "gathering_reads") + + +def _require(protocol, implementation, facts): + missing = [name for name in facts if not hasattr(implementation, name)] + if missing: + raise TypeError( + f"{type(implementation).__name__} does not set {', '.join(missing)}, " + f"which every {protocol} must" + ) + + +def _check_update(update): + _require("ResidualUpdate", update, _UPDATE_FACTS) + name = type(update).__name__ + if update.writes_stream and (update.is_plain_add or not update.applied_at_exit): + raise ValueError( + f"{name} writes the next stream itself, which only an update " + "applied at its exit and other than a plain add does" + ) + if update.quantized_sum and not update.is_plain_add: + raise ValueError( + f"{name} lets its sum run quantized, which only a plain add does" + ) + + +def _check_read(read): + _require("ResidualReadout", read, _READ_FACTS + _READ_KERNELS) @dataclass(frozen=True) @@ -420,11 +468,13 @@ def _resolve_stage(stage, variant, following=None): if stage.update.applied_at_exit: if stage.sparse and moe_gathers_over_moe_cp(): raise NotImplementedError( - "MHC does not support a MoE gathered over the MoE-CP group" + "an update applied at the stage's exit with a MoE gathered over " + "the MoE-CP group" ) if get_parallel().attn_cp_size > 1 and _input_scattered_possible(): raise NotImplementedError( - "MHC with input-scattered attention under attention CP" + "an update applied at the stage's exit with input-scattered " + "attention under attention CP" ) if stage.kind is StageKind.FFN: declaration, residual, returned = _resolve_ffn( @@ -439,8 +489,7 @@ def _resolve_stage(stage, variant, following=None): ) exit_rows = resolve_exit_rows(stage.exit_rows) if exit_rows is ExitRows.ATTENTION or ( - following is not None - and getattr(following.read, "reads_after_attn_tp_gather", False) + following is not None and following.read.reads_after_attn_tp_gather ): returned = attention elif exit_rows is ExitRows.SLICE: @@ -706,11 +755,12 @@ def layer_stack(*, previous_layers=(), next_layers=(), final_read=None): stack's first. What they build is discarded. next_layers: Likewise for the layers after this stack, whose first declared stage is the consumer of this stack's last. - final_read: The model's final read of the stack's output, when it is - not a plain final norm: the consumer of the last stage when no - later layer declares one. Its ``attn_tp_gather`` gathers the rows - that stage leaves on this rank's attention-TP slice, and with - ``reads_attn_tp_slices`` it reads that slice instead. + final_read: The model's final read of the stack's output (a + ``FinalRead``), when it is not a plain final norm: the consumer of + the last stage when no later layer declares one. Its + ``attn_tp_gather`` gathers the rows that stage leaves on this + rank's attention-TP slice, and with ``reads_attn_tp_slices`` it + reads that slice instead. """ global _stack outer = _stack diff --git a/python/sglang/srt/layers/layer_boundary/prepare.py b/python/sglang/srt/layers/layer_boundary/prepare.py index d0268b381c..519b1a3c68 100644 --- a/python/sglang/srt/layers/layer_boundary/prepare.py +++ b/python/sglang/srt/layers/layer_boundary/prepare.py @@ -135,9 +135,8 @@ def _reduce_update_read( if result is not None: return result if group is SumGroup.ATTN_TP: - # MHC sums its streams in full precision. hidden_states = attn_tp_all_reduce( - hidden_states, forward_batch, may_quantize=update.is_plain_add + hidden_states, forward_batch, may_quantize=update.quantized_sum ) elif group is SumGroup.TP: hidden_states = tensor_model_parallel_all_reduce(hidden_states) diff --git a/python/sglang/srt/layers/layer_boundary/residual/__init__.py b/python/sglang/srt/layers/layer_boundary/residual/__init__.py index 58a6bc9599..adbffd9588 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/__init__.py +++ b/python/sglang/srt/layers/layer_boundary/residual/__init__.py @@ -31,6 +31,13 @@ class ResidualUpdate(Protocol): than leaving the operation for the next prepare. outlives_layer: Whether parameters/state remain valid after leaving the producer layer, including any offload or state reuse. + writes_stream: Whether the producer computes the next stream itself + and writes it at its exit, dropping the previous residual: what + reaches its exit is already complete. + quantized_sum: Whether the sum the contribution owes may be completed + by a quantized all-reduce when quantized communication is enabled. + This is a numerical policy, separate from being a plain add; full + precision is always correct. The shard conversion methods must move any update-associated state together with the residual. Nonlinear updates must not claim is_plain_add. @@ -46,6 +53,12 @@ class ResidualUpdate(Protocol): # Its parameters and state remain valid after leaving the producer layer. # Stateful or offloaded implementations must not opt in without that guarantee. outlives_layer: bool + # The producer builds the next stream itself and writes it at its exit, so + # its exit has no sum left to complete and the previous residual is dropped. + # Only an update applied at the exit can, and it is not a plain add. + writes_stream: bool + # The sum it owes may run as a quantized all-reduce. + quantized_sum: bool def update(self, hidden_states, residual) -> torch.Tensor: """Write hidden_states, the producer contribution, into residual. @@ -70,17 +83,23 @@ class ResidualReadout(Protocol): is_plain_norm: Read is normalization/optional quantization without changing the residual, allowing compatible fused add+norm implementations. reads_before_dp_gather: Preserve this read on source rows before a DP gather. - reads_after_attn_tp_gather: Optional. The read needs every row of the + reads_after_attn_tp_gather: The read needs every row of the attention's, so it runs after any gather over attention TP: a producer that would leave its output on this rank's attention-TP - slice returns it to the attention's rows instead. - completing_fusions: Optional. ReadoutFusion kernels that complete the - sum the input owes with the residual add ahead of this read, tried - before the boundary's own all-reduce or reduce-scatter. - gathering_reads: Optional. Callables ``(hidden_states, residual, - norm)`` that read the input on this rank's attention-TP slice and - gather it over attention TP, returning ``(input, residual)``, or - None when they do not take the batch. + slice returns it to the attention's rows instead. False lets it + read that slice, its result gathered after it. Not derivable from + the rows the stage computes on: a read on the slice followed by a + gather and a read after the gather leave the same compute rows. + completing_fusions: ReadoutFusion kernels that complete the sum the + input owes with the residual add ahead of this read, tried before + the boundary's own all-reduce or reduce-scatter. Empty when the + read offers none: the boundary's own collective followed by + update_and_read is correct for every read. + gathering_reads: Callables ``(hidden_states, residual, norm)`` that + read the input on this rank's attention-TP slice and gather it over + attention TP, returning ``(input, residual)``, or None when they do + not take the batch. Empty when the read offers none: it then runs + on the slice and the boundary gathers what it read. init_residual initializes the stack residual. read consumes an already-written residual; update_and_read first applies the actual @@ -95,6 +114,14 @@ class ResidualReadout(Protocol): is_plain_norm: bool # Preserve the read on the source rows before a DP gather. reads_before_dp_gather: bool + # The read needs every row of the attention's, not this rank's slice. + reads_after_attn_tp_gather: bool + # Kernels that complete the sum the input owes with the residual add + # ahead of this read; empty when the read offers none. + completing_fusions: Tuple + # Reads that run on this rank's attention-TP slice and gather what they + # read; empty when the read offers none. + gathering_reads: Tuple def init_residual(self, hidden_states) -> torch.Tensor: """The residual the layer stack starts from, given its input.""" @@ -138,6 +165,9 @@ class ResidualReadout(Protocol): norm: Consumer normalization module. quant_format: Requested input quantization format, as in read(). post_residual_addition: Optional extra added after update, before norm. + Each read documents where it applies, rejects or does not + apply it. A stream that arrives already written is read with + ``read``, which is not given it. Returns: (compute_input, residual). A fused implementation can preserve FP32 @@ -145,6 +175,31 @@ class ResidualReadout(Protocol): """ +class FinalRead(Protocol): + """The model's read of the layer stack's output, its final norm. + + Called with the output alone on a stream its producer already wrote, else + with the output and the residual, returning ``(normed, residual)``: the + residual add followed by the norm, so a pending update must be a plain add. + + Optional, each read where it is used with the default given: + attn_tp_gather: Its own gather over attention TP of rows the stack's + last stage leaves on this rank's attention-TP slice, returning + None when it does not take the batch. Without it the boundary's + gather runs, which is always correct. + reads_attn_tp_slices: It reads this rank's attention-TP slice of the + rows and gathers what it read itself, so the stack's last FFN + leaves its output on that slice. False by default: the last FFN + returns its output on the attention's rows, which any final read + takes. + gemma_weight: The gamma a deferred MoE finalize normalizes with, when + the model's final norm takes that handoff. + """ + + def __call__(self, hidden_states, residual=None, **read_kwargs): + """The final norm of ``hidden_states``, after adding ``residual``.""" + + class LayerResidualOps(NamedTuple): """The reads and updates a decoder layer declares for its attention and its FFN.""" diff --git a/python/sglang/srt/layers/layer_boundary/residual/add_norm.py b/python/sglang/srt/layers/layer_boundary/residual/add_norm.py index 20589b7713..6bfdd74f77 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/add_norm.py +++ b/python/sglang/srt/layers/layer_boundary/residual/add_norm.py @@ -276,6 +276,8 @@ class PlainAdd: is_plain_add = True applied_at_exit = False outlives_layer = True + writes_stream = False + quantized_sum = True def update(self, hidden_states, residual): hidden_states += residual @@ -297,6 +299,8 @@ class ReplaceAtExit: is_plain_add = False applied_at_exit = True outlives_layer = True + writes_stream = True + quantized_sum = False def update(self, hidden_states, residual): return hidden_states @@ -320,10 +324,17 @@ class NormQuantReadout: """The input is the residual's norm, fused with the quantization the consumer asks for (``quant_format``) and a post-residual addition; a plain add of the previous output runs in the same kernel. An empty batch skips - the norm.""" + the norm. + + ``post_residual_addition`` is added after a plain add and before the norm. + It is not applied by the aiter mxfp4 and fp8-group kernels, after an + update other than a plain add, or by ``read``, which has no add.""" is_plain_norm = True + completing_fusions = () + gathering_reads = () reads_before_dp_gather: bool = False + reads_after_attn_tp_gather = False fp8_input: Optional[Fp8Input] = None def init_residual(self, hidden_states): @@ -355,10 +366,15 @@ class NormQuantReadout: @dataclass(frozen=True) class NormReadout: """The input is the residual's norm; a plain add of the previous output - runs in the same kernel. An empty batch skips the norm.""" + runs in the same kernel. An empty batch skips the norm. It rejects a + ``quant_format`` and a ``post_residual_addition``, except that the latter + is not applied after an update other than a plain add.""" is_plain_norm = True + completing_fusions = () + gathering_reads = () reads_before_dp_gather: bool = False + reads_after_attn_tp_gather = False def init_residual(self, hidden_states): return hidden_states @@ -402,6 +418,8 @@ class UnfusedNormReadout(NormReadout): in models that add their residual themselves. No fused kernel takes it.""" is_plain_norm = False + completing_fusions = () + gathering_reads = () def update_and_read( self, diff --git a/python/sglang/srt/layers/layer_boundary/residual/attn_bank.py b/python/sglang/srt/layers/layer_boundary/residual/attn_bank.py index 3aefe947b2..dbaa02324b 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/attn_bank.py +++ b/python/sglang/srt/layers/layer_boundary/residual/attn_bank.py @@ -195,6 +195,8 @@ class _BankReadout: is_plain_norm = False reads_before_dp_gather = True + completing_fusions = () + gathering_reads = () def __init__(self, state: AttnBankState): self.state = state @@ -217,7 +219,8 @@ class _BankReadout: class _AttnReadout(_BankReadout): """The attention input: the bank aggregation and this layer's input norm. - A write layer snapshots the residual this read forms.""" + A write layer snapshots the residual this read forms. ``read`` rejects a + ``post_residual_addition``; ``update_and_read`` does not apply one.""" @property def reads_after_attn_tp_gather(self): @@ -247,6 +250,8 @@ class _FfnReadout(_BankReadout): """The FFN input: the attention output's add folded into the bank aggregation, and this layer's post-attention norm.""" + reads_after_attn_tp_gather = False + @property def completing_fusions(self) -> Tuple[ReadoutFusion, ...]: state = self.state diff --git a/python/sglang/srt/layers/layer_boundary/residual/gated.py b/python/sglang/srt/layers/layer_boundary/residual/gated.py index 4feef15630..ae176797c3 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/gated.py +++ b/python/sglang/srt/layers/layer_boundary/residual/gated.py @@ -114,9 +114,12 @@ class GatedResidualState: class _AttnReadout: """The normalized, gated mix of the streams, from streams that already hold - the previous layer's output: such a layer takes its input written back.""" + the previous layer's output: such a layer takes its input written back. A + ``post_residual_addition`` is not applied.""" is_plain_norm = False + completing_fusions = () + gathering_reads = () reads_before_dp_gather = False def __init__(self, state: GatedResidualState): @@ -150,6 +153,8 @@ class _AttnUpdate: is_plain_add = False applied_at_exit = False outlives_layer = False + writes_stream = False + quantized_sum = False def __init__(self, state: GatedResidualState): self.state = state @@ -168,7 +173,10 @@ class _FfnReadout: """The attention output's injection and the FFN input's mix.""" is_plain_norm = False + completing_fusions = () + gathering_reads = () reads_before_dp_gather = False + reads_after_attn_tp_gather = False def __init__(self, state: GatedResidualState): self.state = state @@ -198,6 +206,8 @@ class _FfnUpdate: is_plain_add = False applied_at_exit = True outlives_layer = False + writes_stream = False + quantized_sum = False def __init__(self, state: GatedResidualState): self.state = state diff --git a/python/sglang/srt/layers/layer_boundary/residual/ihc.py b/python/sglang/srt/layers/layer_boundary/residual/ihc.py index b4027932e3..ad1c67406c 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/ihc.py +++ b/python/sglang/srt/layers/layer_boundary/residual/ihc.py @@ -101,10 +101,13 @@ class IHCState: class _AttnReadout: """The gated mix of the streams and the input norm, from streams that already hold the previous layer's output: an iHC layer takes its input - written back.""" + written back. A ``post_residual_addition`` is not applied.""" is_plain_norm = False + completing_fusions = () + gathering_reads = () reads_before_dp_gather = False + reads_after_attn_tp_gather = False def __init__(self, state: IHCState): self.state = state @@ -129,6 +132,8 @@ class _AttnUpdate: is_plain_add = False applied_at_exit = False outlives_layer = False + writes_stream = False + quantized_sum = False def __init__(self, state: IHCState): self.state = state @@ -148,7 +153,10 @@ class _FfnReadout: norm, fused in post_pre when the layer provides it.""" is_plain_norm = False + completing_fusions = () + gathering_reads = () reads_before_dp_gather = False + reads_after_attn_tp_gather = False def __init__(self, state: IHCState): self.state = state @@ -176,6 +184,8 @@ class _FfnUpdate: is_plain_add = False applied_at_exit = True outlives_layer = False + writes_stream = False + quantized_sum = False def __init__(self, state: IHCState): self.state = state diff --git a/python/sglang/srt/layers/layer_boundary/residual/mhc.py b/python/sglang/srt/layers/layer_boundary/residual/mhc.py index 6aa41f9d80..18f799ee45 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/mhc.py +++ b/python/sglang/srt/layers/layer_boundary/residual/mhc.py @@ -122,10 +122,14 @@ class MHCState: class _AttnReadout: """hc_pre and the input norm, from streams that already hold the previous - layer's output: an MHC layer takes its input written back.""" + layer's output: an MHC layer takes its input written back. A + ``post_residual_addition`` is not applied.""" is_plain_norm = False + completing_fusions = () + gathering_reads = () reads_before_dp_gather = False + reads_after_attn_tp_gather = False def __init__(self, state: MHCState): self.state = state @@ -149,6 +153,8 @@ class _AttnUpdate: is_plain_add = False applied_at_exit = False outlives_layer = False + writes_stream = False + quantized_sum = False def __init__(self, state: MHCState): self.state = state @@ -168,7 +174,10 @@ class _FfnReadout: in hc_ffn_post_pre when it takes the batch.""" is_plain_norm = False + completing_fusions = () + gathering_reads = () reads_before_dp_gather = False + reads_after_attn_tp_gather = False def __init__(self, state: MHCState): self.state = state @@ -195,6 +204,8 @@ class _FfnUpdate: is_plain_add = False applied_at_exit = True outlives_layer = False + writes_stream = False + quantized_sum = False def __init__(self, state: MHCState): self.state = state diff --git a/python/sglang/srt/layers/layer_boundary/residual/post_norm.py b/python/sglang/srt/layers/layer_boundary/residual/post_norm.py index d46383c990..d10ef75724 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/post_norm.py +++ b/python/sglang/srt/layers/layer_boundary/residual/post_norm.py @@ -26,6 +26,8 @@ class PostNormAdd: is_plain_add = False outlives_layer = True + writes_stream = False + quantized_sum = False def __init__(self, norm, *, applied_at_exit: bool = False): self.norm = norm @@ -45,7 +47,10 @@ class PlainReadout: """The input is the residual itself; the stage's norm is not applied.""" is_plain_norm = False + completing_fusions = () + gathering_reads = () reads_before_dp_gather = False + reads_after_attn_tp_gather = False def init_residual(self, hidden_states): return hidden_states diff --git a/python/sglang/srt/layers/layer_boundary/stage.py b/python/sglang/srt/layers/layer_boundary/stage.py index 53d14fae8e..42f7f3a659 100644 --- a/python/sglang/srt/layers/layer_boundary/stage.py +++ b/python/sglang/srt/layers/layer_boundary/stage.py @@ -35,7 +35,6 @@ from sglang.srt.layers.layer_boundary.ops import sum_output from sglang.srt.layers.layer_boundary.residual.access import buffer, from_pp from sglang.srt.layers.layer_boundary.residual.add_norm import ( PLAIN_ADD, - REPLACE_AT_EXIT, ) from sglang.srt.layers.layer_boundary.residual.batch import stream_of from sglang.srt.layers.layer_boundary.residual.stream import DeclaredSum, ResidualStream @@ -281,11 +280,11 @@ class StageBoundary: """Complete the sum over ``group`` that one part of this FFN's output owes. - For an FFN that builds the next stream itself (REPLACE_AT_EXIT) from + For an FFN that builds the next stream itself (``writes_stream``) from several complete parts, normalizing each before combining them; the stream it hands to its exit is then complete. """ - if self.declaration.update is not REPLACE_AT_EXIT: + if not self.declaration.update.writes_stream: raise RuntimeError( "only an FFN that writes the next stream itself sums its parts" ) diff --git a/python/sglang/srt/models/zaya.py b/python/sglang/srt/models/zaya.py index 2c89f5ce42..5da953b8a6 100644 --- a/python/sglang/srt/models/zaya.py +++ b/python/sglang/srt/models/zaya.py @@ -134,7 +134,10 @@ class _ResidualMergeRead: dtype. The producer's update must be a plain add.""" is_plain_norm = False + completing_fusions = () + gathering_reads = () reads_before_dp_gather = False + reads_after_attn_tp_gather = False def __init__(self, res_scale: Optional[ResidualScaling]): self.res_scale = res_scale diff --git a/test/registered/unit/layer_boundary/test_append_stages.py b/test/registered/unit/layer_boundary/test_append_stages.py index fb1b1feb3c..4c96419639 100644 --- a/test/registered/unit/layer_boundary/test_append_stages.py +++ b/test/registered/unit/layer_boundary/test_append_stages.py @@ -837,6 +837,33 @@ class TestValuesAcrossAPipelineCut(CustomTestCase): self.assert_sums_completed_once(run) +class TestInputScatteredAttentionInput(CustomTestCase): + """On an input-scattered batch an attention without a QKV hook, which + projects the rows it is given, gets every row of its input.""" + + def test_an_attention_without_a_qkv_hook_gets_every_row(self): + parallel = fixture.parallel_of( + attn_dp=1, attn_tp=2, enable_attn_tp_input_scattered=True + ) + with fixture.planning(parallel): + with layer_stack(): + attention, _ = layer() + entry = attention.plan.paths[BatchVariant.INPUT_SCATTERED].entry + rows = torch.ones(2, 4) + with ( + patch.object( + boundary_prepare, + "get_attn_tp_context", + return_value=SimpleNamespace(is_dsa=False), + ), + patch.object( + boundary_prepare, "tp_gather", lambda h, fb: torch.cat([h, h]) + ), + ): + hidden = entry.attn_input_adapter(rows, SimpleNamespace(), None) + self.assertEqual(hidden.shape[0], 4) + + class TestUnpaddedBatches(CustomTestCase): """Without attention DP, --disable-attn-tp-gather lets a batch reach the stages with rows that do not divide over attention TP. Such a batch keeps @@ -996,6 +1023,8 @@ class TestRowsTheConsumerReads(CustomTestCase): is_plain_add = False applied_at_exit = False outlives_layer = True + writes_stream = False + quantized_sum = False # One pipeline rank: an update other than a plain add can't cross one. parallel = fixture.parallel_of(attn_dp=1, attn_tp=2, pp_size=1) @@ -1011,14 +1040,16 @@ class TestRowsTheConsumerReads(CustomTestCase): ): stages = [ stage - for _ in range(2) + for i in range(2) for stage in append_stages( (declare_attn(read=GatheringRead()), fixture.Norm()), ( declare_ffn( sparse=True, next_layer_sparse=True, - update=update, + # The final read adds the last output as a + # plain add. + update=update if i == 0 else PLAIN_ADD, ), fixture.Norm(), ), diff --git a/test/registered/unit/layer_boundary/test_aux_capture_deferred_allreduce.py b/test/registered/unit/layer_boundary/test_aux_capture_deferred_allreduce.py index 8b95c5266d..16eb856e1c 100644 --- a/test/registered/unit/layer_boundary/test_aux_capture_deferred_allreduce.py +++ b/test/registered/unit/layer_boundary/test_aux_capture_deferred_allreduce.py @@ -417,11 +417,14 @@ class TestPipelineResidualReception(CustomTestCase): from dataclasses import replace from sglang.srt.layers.layer_boundary import declare_ffn + from sglang.srt.layers.layer_boundary.residual.add_norm import ( + REPLACE_AT_EXIT, + ) stage = stub_stage(comm_instance, StageKind.ATTENTION) stage.declaration = replace( stage.declaration, - previous=declare_ffn(update=SimpleNamespace(applied_at_exit=True)), + previous=declare_ffn(update=REPLACE_AT_EXIT), ) hidden = stage.from_pp(PPProxyTensors({"hidden_states": streams}), batch) residual = batch.residual_stream diff --git a/test/registered/unit/layer_boundary/test_boundary_edges.py b/test/registered/unit/layer_boundary/test_boundary_edges.py index fdd7fb8448..e4c13fc848 100644 --- a/test/registered/unit/layer_boundary/test_boundary_edges.py +++ b/test/registered/unit/layer_boundary/test_boundary_edges.py @@ -279,7 +279,9 @@ class TestTheProducersUpdateChoosesTheOrder(CustomTestCase): def test_cross_layer_update_requires_a_lifetime_guarantee(self): local = Layout(frozenset()) - stateful_update = SimpleNamespace(is_plain_add=False, applied_at_exit=False) + stateful_update = SimpleNamespace( + is_plain_add=False, applied_at_exit=False, outlives_layer=False + ) edge = EdgeContract( OutputContract(local, update=stateful_update), InputContract(local), @@ -304,6 +306,8 @@ class TestTheProducersUpdateChoosesTheOrder(CustomTestCase): class ProbeRead: reads_before_dp_gather = False + completing_fusions = () + gathering_reads = () """A read that marks what it reads: the input is the residual plus 100. ``is_plain_norm`` says whether it may stand in for a norm.""" diff --git a/test/registered/unit/layer_boundary/test_boundary_output_contracts.py b/test/registered/unit/layer_boundary/test_boundary_output_contracts.py index a5be414e57..60a19b5589 100644 --- a/test/registered/unit/layer_boundary/test_boundary_output_contracts.py +++ b/test/registered/unit/layer_boundary/test_boundary_output_contracts.py @@ -217,11 +217,16 @@ class TestBoundaryIntegrations(unittest.TestCase): def test_an_ffn_that_completes_its_own_sum_owes_none(self): # Its compute completes the sum, so the exit neither sums nor defers it, - # under DP too. - for attn_dp in (1, 2): + # under DP and on an input-scattered batch too. + for attn_dp, scattered in ((1, False), (2, False), (1, True)): + parallel = fixture.parallel_of( + attn_dp=attn_dp, + attn_tp=2, + enable_attn_tp_input_scattered=scattered, + ) with ( - self.subTest(attn_dp=attn_dp), - fixture.planning(fixture.parallel_of(attn_dp=attn_dp, attn_tp=2)), + self.subTest(attn_dp=attn_dp, input_scattered=scattered), + fixture.planning(parallel), ): for sparse in (False, True): stages = { @@ -237,6 +242,10 @@ class TestBoundaryIntegrations(unittest.TestCase): } owed = stages[False].plan.paths[BatchVariant.ORDINARY].output self.assertIsNotNone(owed.group, sparse) + if scattered: + self.assertIn( + BatchVariant.INPUT_SCATTERED, stages[True].plan.paths + ) for variant, path in stages[True].plan.paths.items(): self.assertIsNone(path.output.group, (sparse, variant)) self.assertFalse(path.output.may_defer_to_next) diff --git a/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py b/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py index 3670806aa7..8b22b90f22 100644 --- a/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py +++ b/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py @@ -24,10 +24,18 @@ from sglang.srt.layers import layernorm_sp from sglang.srt.layers.layer_boundary import ( SumGroup, TokenAxis, + append_stages, ) from sglang.srt.layers.layer_boundary import boundary as comm_boundary from sglang.srt.layers.layer_boundary import construction as comm_layer +from sglang.srt.layers.layer_boundary import ( + declare_attn, + declare_ffn, +) from sglang.srt.layers.layer_boundary import exit as comm_exit +from sglang.srt.layers.layer_boundary import ( + layer_stack, +) from sglang.srt.layers.layer_boundary import layout as comm_layout from sglang.srt.layers.layer_boundary import ops as transport_ops from sglang.srt.layers.layer_boundary import prepare as comm_ops @@ -48,6 +56,7 @@ from sglang.srt.layers.layer_boundary.residual import mhc as mhc_module from sglang.srt.layers.layer_boundary.residual.add_norm import ( NORM_READOUT, PLAIN_RESIDUAL_OPS, + REPLACE_AT_EXIT, ) from sglang.srt.layers.layer_boundary.residual.stream import OwedOutput, ResidualStream from sglang.srt.layers.moe import utils as moe_utils @@ -1014,12 +1023,15 @@ class TestInputScatteredAttention(CustomTestCase): patch_communicator("get_parallel", lambda: parallel), patch_communicator( "get_attn_tp_context", - lambda: SimpleNamespace(input_scattered=True), + lambda: SimpleNamespace(input_scattered=True, is_dsa=False), ), patch_communicator( "get_forward", lambda: SimpleNamespace(sp_active=False, attn_input_scattered=False), ), + # This attention has no QKV hook, so it takes every row of its + # input: the slice is gathered after the read. + patch_communicator("tp_gather", lambda h, fb: torch.cat([h, h])), ): hidden, residual = prepare_input( communicator.attn, @@ -1030,7 +1042,7 @@ class TestInputScatteredAttention(CustomTestCase): self.assertEqual(scattered, [4]) # Norm: (2 * (h + r), h + r) on the slice, with h the completed sum. torch.testing.assert_close(residual.residual, torch.full((2, HIDDEN), 5.0)) - torch.testing.assert_close(hidden, torch.full((2, HIDDEN), 10.0)) + torch.testing.assert_close(hidden, torch.full((4, HIDDEN), 10.0)) def test_the_last_layer_completes_its_own_sum(self): parallel = parallel_of( @@ -1955,11 +1967,164 @@ class TestTwoLayers(CustomTestCase): ) +class RecordingGroup(Group): + """A Group that also records which collective each call runs.""" + + def all_reduce(self, x): + state().ops.append("all_reduce") + return super().all_reduce(x) + + def reduce_scatter_tensor(self, output, input): + state().ops.append("reduce_scatter") + return super().reduce_scatter_tensor(output, input) + + def all_gather_into_tensor(self, output, input): + state().ops.append("all_gather") + return super().all_gather_into_tensor(output, input) + + +class TestCompleteOutputsOnInputScatteredBatches(CustomTestCase): + """On an input-scattered batch an FFN that hands on a complete output, one + its compute summed or the next stream it wrote itself, gets no further + sum, at its exit or at the next attention's entry. The next attention + still keeps this rank's slice of the residual and takes every row of its + input, as after an FFN whose output owes its sum.""" + + ATTN_TP = 2 + ROWS = 4 + + def run_ranks(self, output, *, reduction, qkv_hook): + tp = self.ATTN_TP + world = World(tp) + WORLD[0] = world + group = RecordingGroup(world, "tp", range(tp)) + x = torch.arange(1.0, self.ROWS * HIDDEN + 1, dtype=torch.double) + x = x.reshape(self.ROWS, HIDDEN) + # Norm doubles; the attention is 3x and the FFN 5x of its input. + before_ffn = x + 3 * (2 * x) + after_ffn = 5 * (2 * before_ffn) + before_ffn + forward_batch = SimpleNamespace( + forward_mode=SimpleNamespace( + is_context_parallel_extend=lambda: False, + is_decode_or_idle=lambda: False, + is_extend=lambda: True, + ), + dp_padding_mode=SimpleNamespace(is_max_len=lambda: False), + global_dp_buffer_len=self.ROWS, + input_ids=torch.zeros(self.ROWS), + ) + ffn_declaration = { + "summed by its compute": dict(output_complete=True), + "written at its exit": dict(update=REPLACE_AT_EXIT), + "owing its sum": {}, + }[output] + states = [ + SimpleNamespace( + rank=rank, + parallel=parallel_of( + attn_dp=1, + attn_tp=tp, + tp_rank=rank, + attn_tp_rank=rank, + tp_group=group, + attn_tp_group=group, + enable_attn_tp_input_scattered=True, + ), + flags=Flags(), + calls=[], + ops=[], + attn_inputs=[], + ) + for rank in range(tp) + ] + context = SimpleNamespace( + input_scattered=True, + is_dsa=False, + set_attn_inputs=lambda inputs: state().attn_inputs.append(inputs), + ) + + def ffn(hidden, s): + if output == "owing its sum": + return dense_mlp(hidden, s) + if output == "summed by its compute": + return 5 * hidden + # It adds the residual itself and writes the next stream. + return 5 * hidden + before_ffn + + def build(): + hook = {"qkv_latent_func": lambda h, fb: 11 * h} if qkv_hook else {} + with layer_stack(): + attention_1, ffn_1 = append_stages( + (declare_attn(), Norm()), + (declare_ffn(**ffn_declaration), Norm()), + ) + attention_2, _ = append_stages( + (declare_attn(), Norm(), hook), (declare_ffn(), Norm()) + ) + return attention_1, ffn_1, attention_2 + + def forward(rank): + s = states[rank] + attention_1, ffn_1, attention_2 = stages[rank] + hidden, stream = prepare_input( + attention_1, x * WEIGHTS[tp][rank], None, forward_batch + ) + hidden = stream.record( + attention(hidden, s), + comm.PLAIN_ADD, + declared_sum=ffn_1.entry(forward_batch).declared_sum, + ) + hidden, stream = prepare_input(ffn_1, hidden, stream, forward_batch) + with ffn_1.plan.output.ffn_exit( + forward_batch, stream=ResidualStream() + ) as ffn_exit: + hidden = ffn(hidden, s) + computed = len(s.ops) + hidden, stream = finish_exit(ffn_exit, hidden, stream) + hidden, stream = prepare_input(attention_2, hidden, stream, forward_batch) + if qkv_hook: + hidden = s.attn_inputs[-1].fetch_qkv_latent() + return hidden, stream.residual, s.ops[computed:] + + with ( + running(reduce_scatterv=False, use_reduce_scatter=reduction == "rs"), + patch_communicator("get_attn_tp_context", lambda: context), + ): + stages = [] + for s in states: + # Each rank binds its own stages; the stack is not shared + # between threads. + world.local.state = s + stages.append(build()) + results, errors = world.run(states, forward) + for rank, error in enumerate(errors): + if error is not None: + raise AssertionError(f"rank {rank} raised") from error + rows = self.ROWS // tp + for rank, (hidden, residual, ops) in enumerate(results): + want = 2 * after_ffn * (11 if qkv_hook else 1) + torch.testing.assert_close(hidden, want, rtol=0, atol=0) + torch.testing.assert_close( + residual, after_ffn[rank * rows : (rank + 1) * rows], rtol=0, atol=0 + ) + sums = [op for op in ops if op != "all_gather"] + self.assertEqual(len(sums), output == "owing its sum", ops) + + def test_a_complete_output_is_summed_no_further(self): + for output, reduction, qkv_hook in itertools.product( + ("summed by its compute", "written at its exit", "owing its sum"), + ("ar", "rs"), + (False, True), + ): + with self.subTest(output=output, reduction=reduction, qkv_hook=qkv_hook): + self.run_ranks(output, reduction=reduction, qkv_hook=qkv_hook) + + class TestTheFfnInputReduction(CustomTestCase): """With quantized communications a prefill reduces the FFN input quantized for a plain residual; MHC sums its streams in full precision.""" - def test_only_a_plain_residual_reduces_quantized(self): + def test_only_an_update_that_allows_it_reduces_quantized(self): exec_ = SimpleNamespace( comm=SimpleNamespace( enable_quant_communications=True, boundary_reduction="rs+rsv" @@ -1968,9 +2133,11 @@ class TestTheFfnInputReduction(CustomTestCase): batch = SimpleNamespace( forward_mode=SimpleNamespace(is_decode_or_idle=lambda: False) ) - for is_plain_add, reduction in ((True, "quant"), (False, "full")): - with self.subTest(is_plain_add=is_plain_add): - update = SimpleNamespace(is_plain_add=is_plain_add) + for update, reduction in ( + (SimpleNamespace(quantized_sum=True), "quant"), + (SimpleNamespace(quantized_sum=False), "full"), + ): + with self.subTest(update=update): read = SimpleNamespace( update_and_read=lambda update, h, r, norm, **read_kwargs: (h, r) )