From e1f9fce7b33d293233c4a2eefd6e43608c562ba7 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Thu, 15 Oct 2026 02:04:02 -0700 Subject: [PATCH] [Refactor] layer-boundary: Bind residual completion to producer updates (#43605) --- .../srt/layers/layer_boundary/boundary.py | 63 +- .../srt/layers/layer_boundary/construction.py | 12 +- .../srt/layers/layer_boundary/contracts.py | 17 +- .../sglang/srt/layers/layer_boundary/exit.py | 45 +- .../srt/layers/layer_boundary/factories.py | 48 +- .../sglang/srt/layers/layer_boundary/facts.py | 4 + .../srt/layers/layer_boundary/output.py | 6 + .../srt/layers/layer_boundary/prepare.py | 96 ++- .../layer_boundary/residual/__init__.py | 88 ++- .../layer_boundary/residual/add_norm.py | 10 + .../layer_boundary/residual/attn_bank.py | 2 + .../layers/layer_boundary/residual/batch.py | 108 ++- .../layers/layer_boundary/residual/gated.py | 8 + .../srt/layers/layer_boundary/residual/ihc.py | 8 + .../srt/layers/layer_boundary/residual/mhc.py | 8 + .../layer_boundary/residual/post_norm.py | 4 + .../layers/layer_boundary/residual/stream.py | 20 +- .../sglang/srt/layers/layer_boundary/stage.py | 33 +- python/sglang/srt/models/zaya.py | 2 + python/sglang/test/boundary_fixtures.py | 132 ++++ .../unit/layer_boundary/test_append_stages.py | 53 +- .../test_attn_bank_residual_ops.py | 1 + .../layer_boundary/test_boundary_edges.py | 15 +- .../test_declared_decoder_boundary.py | 131 +--- .../test_producer_completion.py | 731 ++++++++++++++++++ .../layer_boundary/test_residual_stream.py | 11 +- .../test_token_order_attention.py | 2 +- 27 files changed, 1445 insertions(+), 213 deletions(-) create mode 100644 test/registered/unit/layer_boundary/test_producer_completion.py diff --git a/python/sglang/srt/layers/layer_boundary/boundary.py b/python/sglang/srt/layers/layer_boundary/boundary.py index 62d66f5061..bca3098c9f 100644 --- a/python/sglang/srt/layers/layer_boundary/boundary.py +++ b/python/sglang/srt/layers/layer_boundary/boundary.py @@ -173,6 +173,13 @@ class _TransformedRead: completing_fusions = () gathering_reads = () + @property + def bind(self): + return self._bind if self.inner.bind is not None else None + + def _bind(self, forward_batch, context): + return _TransformedRead(self.inner.bind(forward_batch, context), self.transform) + @property def reads_before_dp_gather(self): return self.inner.reads_before_dp_gather @@ -341,6 +348,14 @@ def _bind_entry_path( step=entry.step, is_plain_add=is_plain_add, carried_fusions=carried_fusions if plain else (), + carried_read_fusions=tuple( + f + for f in edge.need.read.completing_fusions + if f.carried + and not f.scatters + and (is_plain_add or f.accepts_update is not None) + ), + read=edge.need.read, expected_sum=declared_sum, completed_step=completed_step, written_step=written_step, @@ -367,8 +382,8 @@ def bind_exit( Args: edge: Output contract and destination/residual rows. Deferred updates - must outlive the producer; pipeline handoffs require a plain add - or a residual already written by the producer. + must outlive the producer. Pipeline handoff eligibility is checked + when the stage is constructed. cp_moves: Context-parallel return operations, when the edge needs them. attn_tp_gather: The stage's implementation of a gather over attention TP, tried before the default one. @@ -383,10 +398,6 @@ def bind_exit( raise NotImplementedError( "a deferred update must guarantee its lifetime across layers" ) - if not update.is_plain_add and get_parallel().pp_size > 1: - raise NotImplementedError( - "pipeline boundaries require a plain add or a producer-written residual" - ) if edge.need.layout != edge.residual_to: raise NotImplementedError(f"{edge=}") return _select_exit_move( @@ -398,17 +409,20 @@ def bind_exit( ) -def _read_fusions(read, owes, is_plain_add, *, scatters): - """The kernels a read supplies that complete ``owes`` on the rows this - entry completes it on. They take the residual add themselves, so only a - plain one.""" - if not is_plain_add: - return () - return tuple( - f - for f in read.completing_fusions - if f.completes is owes and f.scatters == scatters +def _read_fusion_args(read, owes, is_plain_add, *, scatters): + """Select callback slots for this group and row placement once. Stateful + readers resolve those slots on the invocation, stateless readers directly + retain their kernels. Non-plain updates require explicit acceptance.""" + indices = tuple( + i + for i, f in enumerate(read.completing_fusions) + if (is_plain_add or f.accepts_update is not None) + and f.completes is owes + and f.scatters == scatters ) + if read.bind is not None: + return {"read_fusion_indices": indices} + return {"read_fusions": tuple(read.completing_fusions[i] for i in indices)} def _select_entry_step( @@ -484,7 +498,7 @@ def _select_entry_step( enters_stack=enters_stack, ) return inner._replace( - step=partial(_move_before_read, move=move, read=inner.step) + step=partial(_move_before_read, move=move, read_step=inner.step) ) if sliced: # Each attention-TP rank takes its own slice: the reduce-scatter @@ -501,7 +515,7 @@ def _select_entry_step( if owes is SumGroup.ATTN_TP: step = partial( _attn_tp_reduce_scatter_update_read, - read_fusions=_read_fusions(read, owes, is_plain_add, scatters=True), + **_read_fusion_args(read, owes, is_plain_add, scatters=True), ) else: step = _attn_tp_slice_update_read @@ -581,7 +595,14 @@ def _select_entry_step( enters_stack=enters_stack, read=read, read_gathers=( - read.gathering_reads if is_plain_add or arrives_written else () + read.gathering_reads + if read.bind is None and (is_plain_add or arrives_written) + else () + ), + read_gather_indices=( + tuple(range(len(read.gathering_reads))) + if read.bind is not None and (is_plain_add or arrives_written) + else () ), ), input_move=( @@ -639,13 +660,13 @@ def _select_entry_step( ): raise NotImplementedError(f"{produced=} {need=}") fused = tuple(f for f in fusions if f.completes is owes) - read_fused = _read_fusions(read, owes, is_plain_add, scatters=False) + read_fused = _read_fusion_args(read, owes, is_plain_add, scatters=False) return EntryStep( partial( _reduce_update_read, gathers_residual=gathers_residual, fusions=tuple(f.run for f in fused), - read_fusions=read_fused, + **read_fused, group=owes, read=read, ), diff --git a/python/sglang/srt/layers/layer_boundary/construction.py b/python/sglang/srt/layers/layer_boundary/construction.py index 0e5b210616..62e851291c 100644 --- a/python/sglang/srt/layers/layer_boundary/construction.py +++ b/python/sglang/srt/layers/layer_boundary/construction.py @@ -50,6 +50,7 @@ from sglang.srt.layers.layer_boundary.prepare import ( _attn_input_default, _attn_input_scattered, ) +from sglang.srt.layers.layer_boundary.residual import takes_pending_update from sglang.srt.layers.layer_boundary.stage import StageBoundary from sglang.srt.runtime_context import ( get_forward, @@ -269,6 +270,7 @@ def _bind_stage( *, final_read=None, capture_preserves_residual=None, + pipeline_handoff=False, **options, ): if incoming.consumer != declaration or outgoing.producer != declaration: @@ -276,7 +278,15 @@ def _bind_stage( if incoming.entries.keys() != outgoing.exits.keys(): raise ValueError("incoming and outgoing batch variants disagree") update = declaration.update - if declaration.terminal and not (update.is_plain_add or update.applied_at_exit): + if pipeline_handoff and not (update.is_plain_add or update.applied_at_exit): + raise NotImplementedError( + "pipeline boundaries require a producer-written residual" + ) + if declaration.terminal and not ( + update.is_plain_add + or update.applied_at_exit + or takes_pending_update(final_read, update) + ): # The final read adds the last output into the residual itself, as a # plain add, before its norm. raise NotImplementedError( diff --git a/python/sglang/srt/layers/layer_boundary/contracts.py b/python/sglang/srt/layers/layer_boundary/contracts.py index 580c07178a..44d74082be 100644 --- a/python/sglang/srt/layers/layer_boundary/contracts.py +++ b/python/sglang/srt/layers/layer_boundary/contracts.py @@ -186,13 +186,14 @@ class FfnInputFusion(msgspec.Struct, frozen=True): class ReadoutFusion(msgspec.Struct, frozen=True): """A kernel a stage's read supplies (its ``completing_fusions``) that - completes the sum its input owes together with the residual add, for a - read that is not the residual's plain norm and so takes no FfnInputFusion. + completes the input's sum and producer update before the consumer read. + With no ``accepts_update`` predicate the update must be plain addition. + Otherwise ``run`` also receives the actual producer's ``update`` keyword. ``run(hidden_states, residual, forward_batch)`` returns the written stream, - the completed sum plus the residual, which the read then reads; or None - when it does not take the batch, before touching its inputs or starting a - collective.""" + or, with ``reads=True``, takes ``norm`` and returns (input, residual). + A carried candidate may receive UnreducedOutput or DeferredFinalize. + Returning None must precede any collective or mutation of input/state.""" # The group whose sum it completes. completes: SumGroup @@ -204,6 +205,12 @@ class ReadoutFusion(msgspec.Struct, frozen=True): # Whether it also does the read: run(hidden_states, residual, # forward_batch, norm) then returns the read's (input, residual). reads: bool = False + # None restricts the existing kernel to a plain addition. An explicit + # predicate validates the actual update before run receives it as a keyword. + accepts_update: Optional[Callable] = None + # Also accepts dynamically carried output; run must decline unsupported + # completion handles without starting communication or changing state. + carried: bool = False class CpMoves(msgspec.Struct, frozen=True): diff --git a/python/sglang/srt/layers/layer_boundary/exit.py b/python/sglang/srt/layers/layer_boundary/exit.py index 0e933f6242..cc6eca90d3 100644 --- a/python/sglang/srt/layers/layer_boundary/exit.py +++ b/python/sglang/srt/layers/layer_boundary/exit.py @@ -49,6 +49,7 @@ from sglang.srt.layers.layer_boundary.ops import ( from sglang.srt.layers.layer_boundary.output import ( UnreducedOutput, ) +from sglang.srt.layers.layer_boundary.residual import bind_update from sglang.srt.layers.layer_boundary.residual.add_norm import ( aiter_ar_fusion_applies, flashinfer_ar_fusion_applies, @@ -116,7 +117,7 @@ def exit_facts(kind, plan, variant, path) -> ExitFacts: ) may_defer_sum = ( tp - and not plan.terminal + and (not plan.terminal or output.may_defer_to_next) and _sum_deferral_allowed(plan, output, scattered=scattered) # Under attention DP the next layer must also run postprocess's # scatter back to this rank's tokens, and nothing more. @@ -199,10 +200,12 @@ class ExitPolicy: forward_batch: ForwardBatch, *, already_reduced: bool = False, + update_context=None, ): """Complete the output of the operation-scheduled producer path, which never defers: the sum it owes, then its move and write-back.""" steps = self.plan.path_for(forward_batch) + update = bind_update(steps.output.update, forward_batch, update_context) if already_reduced and steps.output_move_completes_sum: raise ValueError("already-reduced output cannot use a reduce-scatter move") hidden_states, residual = self._complete_now( @@ -212,6 +215,7 @@ class ExitPolicy: dp_step=None, steps=steps, output_move=steps.output_move, + update=update, owes_sum=not already_reduced and steps.output.group is not None and not steps.output_move_completes_sum, @@ -220,7 +224,7 @@ class ExitPolicy: hidden_states, residual, stream, - steps.output.update, + update, ) @staticmethod @@ -260,7 +264,9 @@ class ExitPolicy: group on a sparse layer, the TP group a dense MLP reduces over.""" return _sum_group(steps.output.group) - def _decide(self, forward_batch: ForwardBatch, steps) -> ExitDecision: + def _decide( + self, forward_batch: ForwardBatch, steps, *, update=None + ) -> ExitDecision: """Decide once, before the FFN runs, what completes its output's sum: the next layer's input, or this layer's postprocess step.""" dp_step = self._dp_reduce_scatter_step(forward_batch, steps) @@ -271,6 +277,7 @@ class ExitPolicy: dp_step=dp_step, steps=steps, output_move=steps.output_move, + update=update, # A reduce-scatter completes the sum on its way back. owes_sum=steps.output.group is not None and not sum_in_reduce_scatter, ) @@ -340,6 +347,7 @@ class ExitPolicy: steps, output_move: Optional[Callable], owes_sum: bool, + update=None, ) -> Tuple[torch.Tensor, torch.Tensor]: """This layer's postprocess, run with the attention-DP step already chosen: the sum the output owes, the move back to where the next layer @@ -364,27 +372,40 @@ class ExitPolicy: residual=residual, forward_batch=forward_batch, ) - update = steps.output.update + if update is None: + update = bind_update(steps.output.update, forward_batch) if residual is not None and (update.applied_at_exit or steps.writes_at_handoff): hidden_states = update.update(hidden_states, residual) residual = None return hidden_states, residual def mixer_exit( - self, forward_batch: ForwardBatch, *, stream: ResidualStream + self, + forward_batch: ForwardBatch, + *, + stream: ResidualStream, + update_context=None, ) -> MixerExit: """Decide once whether this stage's mixer (an attention-like stage) output's sum is completed here or carried on. Call ``finish`` on the result with the mixer's output.""" - return MixerExit(self, forward_batch, stream=stream) + return MixerExit( + self, forward_batch, stream=stream, update_context=update_context + ) def ffn_exit( - self, forward_batch: ForwardBatch, *, stream: ResidualStream + self, + forward_batch: ForwardBatch, + *, + stream: ResidualStream, + update_context=None, ) -> FfnExit: """Decide once how this layer's FFN output reduction completes. Use the result as a context manager around the FFN call when the FFN may hand off its MoE finalize, then call ``finish``.""" - return FfnExit(self, forward_batch, stream=stream) + return FfnExit( + self, forward_batch, stream=stream, update_context=update_context + ) def _defers_sum( self, @@ -485,12 +506,13 @@ class MixerExit: forward_batch: ForwardBatch, *, stream: ResidualStream, + update_context=None, ): steps = boundary.plan.path_for(forward_batch) produced = steps.output self._stream = stream self._forward_batch = forward_batch - self._update = produced.update + self._update = bind_update(produced.update, forward_batch, update_context) self._group = produced.group self._declared_sum = produced.group if produced.always_partial else None self._defers = steps.exit.defers_mixer_sum @@ -552,16 +574,17 @@ class FfnExit: forward_batch: ForwardBatch, *, stream: ResidualStream, + update_context=None, ): self._stream = stream self.boundary = boundary # The same path the entry took: the flags that select it hold for the # whole call, and a stage keeps no state between its entry and exit. steps = boundary.plan.path_for(forward_batch) - completion = boundary._decide(forward_batch, steps) + self._update = bind_update(steps.output.update, forward_batch, update_context) + completion = boundary._decide(forward_batch, steps, update=self._update) self.defer_moe_finalize = completion.defer_moe_finalize self._complete = completion.complete - self._update = steps.output.update # A reduce-scatter that completes the sum on the way back is the # next input's only on an input-scattered batch. self._declared_sum = steps.exit.sum_left_to_next_input diff --git a/python/sglang/srt/layers/layer_boundary/factories.py b/python/sglang/srt/layers/layer_boundary/factories.py index 1d3326b6a4..1456d0ef3c 100644 --- a/python/sglang/srt/layers/layer_boundary/factories.py +++ b/python/sglang/srt/layers/layer_boundary/factories.py @@ -46,6 +46,7 @@ from sglang.srt.layers.layer_boundary.output import OutputTransform from sglang.srt.layers.layer_boundary.residual import ( ResidualReadout, ResidualUpdate, + takes_pending_update, ) from sglang.srt.layers.layer_boundary.residual.add_norm import ( NORM_QUANT_READOUT, @@ -137,6 +138,7 @@ def _resolve_ffn( update=PLAIN_ADD, dense_tp_size=None, output_complete=False, + terminal_accepts_update=False, ): parallel = get_parallel() if dense_tp_size not in (None, 1, parallel.attn_tp_size, parallel.tp_size): @@ -244,7 +246,7 @@ def _resolve_ffn( rows, group=group, may_defer_to_next=may_leave - and not terminal + and (not terminal or terminal_accepts_update) and not update.applied_at_exit and update.outlives_layer and output_transform is None, @@ -372,7 +374,9 @@ def _require(protocol, implementation, facts): def _check_update(update): - _require("ResidualUpdate", update, _UPDATE_FACTS) + _require("ResidualUpdate", update, _UPDATE_FACTS + ("bind",)) + if update.bind is not None and not callable(update.bind): + raise TypeError("ResidualUpdate.bind must be callable or None") name = type(update).__name__ if update.writes_stream and (update.is_plain_add or not update.applied_at_exit): raise ValueError( @@ -386,7 +390,9 @@ def _check_update(update): def _check_read(read): - _require("ResidualReadout", read, _READ_FACTS + _READ_KERNELS) + _require("ResidualReadout", read, _READ_FACTS + _READ_KERNELS + ("bind",)) + if read.bind is not None and not callable(read.bind): + raise TypeError("ResidualReadout.bind must be callable or None") @dataclass(frozen=True) @@ -576,7 +582,7 @@ def _stack_arrival(variant): ) -def _resolve_stage(stage, variant, arrival, following=None): +def _resolve_stage(stage, variant, arrival, following=None, *, final_read=None): """``stage``'s data flow for ``variant``, given what arrives at its entry and the stage after it (None when it ends the stack or a branch).""" axes, attention, local, full = _row_layouts(variant) @@ -591,6 +597,8 @@ def _resolve_stage(stage, variant, arrival, following=None): update=stage.update, dense_tp_size=stage.dense_tp_size, output_complete=stage.output_complete, + terminal_accepts_update=stage.terminal + and takes_pending_update(final_read, stage.update), ) exit_rows = resolve_exit_rows(stage.exit_rows) if exit_rows is ExitRows.ATTENTION or ( @@ -745,7 +753,9 @@ def _exit_edge(stage, flow, entry): ) -def _connect_line(line, origins, *, before=None, after=None, arrivals=None): +def _connect_line( + line, origins, *, before=None, after=None, arrivals=None, final_read=None +): """The connections along a chain of stages, each resolved once per batch variant: into ``line[0]``, between each two, and out of ``line[-1]`` to ``after`` (a stage on the next rank, or None at the end of the stack or a @@ -754,12 +764,33 @@ def _connect_line(line, origins, *, before=None, after=None, arrivals=None): rank's own entry is taken as the stack input's. An error resolving a stage names the append at ``origins[i]`` that declared ``line[i]``, or the one next to the neighbouring rank's stage.""" - variants = tuple(_active_variants()) + # The global dense-TP default may be overridden by every FFN declaration. + # Odd row counts need a separate path only when some connected FFN would + # otherwise slice them over attention TP. Keep neighbours in this decision: + # their row choices still determine what crosses the pipeline boundary. + needs_unpadded = any( + stage is not None + and stage.kind is StageKind.FFN + and _ffn_on_rank_rows(stage.sparse, stage.dense_tp_size) + for stage in (before, *line, after) + ) + variants = tuple( + variant + for variant in _active_variants() + if variant is not BatchVariant.UNPADDED or needs_unpadded + ) def resolve(position, stage, arrivals, following=None): try: return { - v: _resolve_stage(stage, v, arrivals[v], following) for v in variants + v: _resolve_stage( + stage, + v, + arrivals[v], + following, + final_read=final_read if stage.terminal else None, + ) + for v in variants } except Exception as error: _note_origin(error, origins[position]) @@ -1072,6 +1103,7 @@ def _bind_stack(appends, *, previous, following, final_read=None): [append.origin for append, _ in stages], before=chained[0].previous, after=following, + final_read=final_read if following is None else None, ) _bind_line( stages, @@ -1145,6 +1177,8 @@ def _bind_line(stages, chained, connections, bound, sources, *, final_read=None) connections[position], connections[position + 1], final_read=final_read if position == last else None, + pipeline_handoff=position == last + and connections[position + 1].consumer is not None, capture_preserves_residual=( _preserves_residual(boundaries[position + 1]) if consumer is not None diff --git a/python/sglang/srt/layers/layer_boundary/facts.py b/python/sglang/srt/layers/layer_boundary/facts.py index a956f65013..a5779bb41c 100644 --- a/python/sglang/srt/layers/layer_boundary/facts.py +++ b/python/sglang/srt/layers/layer_boundary/facts.py @@ -28,6 +28,8 @@ import msgspec class UpdateFacts(msgspec.Struct, frozen=True): """The capabilities a ResidualUpdate declares (see ResidualUpdate).""" + bind = None + is_plain_add: bool applied_at_exit: bool outlives_layer: bool @@ -52,6 +54,8 @@ class ReadFacts(msgspec.Struct, frozen=True): Binding takes kernels only from the reads of the stages it binds, never from a stage recorded by its facts, so these supply none.""" + bind = None + is_plain_norm: bool reads_before_dp_gather: bool reads_after_attn_tp_gather: bool diff --git a/python/sglang/srt/layers/layer_boundary/output.py b/python/sglang/srt/layers/layer_boundary/output.py index 935e17d75b..08e433b6f0 100644 --- a/python/sglang/srt/layers/layer_boundary/output.py +++ b/python/sglang/srt/layers/layer_boundary/output.py @@ -72,6 +72,12 @@ class DeferredFinalize(msgspec.Struct, frozen=True): there may do that work together with its own; anything else passes it through complete_owed(), which calls ``complete()``.""" + @property + def completion_group(self): + from sglang.srt.layers.layer_boundary.layout import SumGroup + + return SumGroup.MOE_OUTPUT + def complete(self) -> torch.Tensor: """Do the owed work, unfused, and return the complete output.""" raise NotImplementedError diff --git a/python/sglang/srt/layers/layer_boundary/prepare.py b/python/sglang/srt/layers/layer_boundary/prepare.py index 519b1a3c68..3bf9c06cc2 100644 --- a/python/sglang/srt/layers/layer_boundary/prepare.py +++ b/python/sglang/srt/layers/layer_boundary/prepare.py @@ -51,7 +51,11 @@ from sglang.srt.layers.layer_boundary.output import ( UnreducedOutput, complete_owed, ) -from sglang.srt.layers.layer_boundary.residual import ResidualReadout, ResidualUpdate +from sglang.srt.layers.layer_boundary.residual import ( + ResidualReadout, + ResidualUpdate, + bind_read, +) from sglang.srt.layers.layer_boundary.residual.add_norm import NORM_READOUT, PLAIN_ADD from sglang.srt.layers.layer_boundary.residual.stream import DeclaredSum from sglang.srt.model_executor.forward_batch_info import ForwardBatch @@ -77,19 +81,38 @@ from sglang.srt.layers.layer_boundary.ops import ( def _run_read_fusions( - read_fusions, hidden_states, residual, forward_batch, norm, *, read, **call + read_fusions, + hidden_states, + residual, + forward_batch, + norm, + *, + read, + read_fusion_indices: Tuple[int, ...] = (), + update=PLAIN_ADD, + **call, ): """The first of the read's own kernels (ReadoutFusion) that takes the batch: one that completes the sum with the residual add hands the written stream to ``read``; one that also reads returns the read. None when every kernel declines.""" + if read_fusion_indices: + read_fusions = (read.completing_fusions[i] for i in read_fusion_indices) for fused in read_fusions: + args = {} + if fused.accepts_update is None: + if not update.is_plain_add: + continue + else: + if not fused.accepts_update(update): + continue + args["update"] = update if fused.reads: - result = fused.run(hidden_states, residual, forward_batch, norm) + result = fused.run(hidden_states, residual, forward_batch, norm, **args) if result is not None: return result continue - stream = fused.run(hidden_states, residual, forward_batch) + stream = fused.run(hidden_states, residual, forward_batch, **args) if stream is not None: return read.read(stream, norm, **call) return None @@ -105,6 +128,7 @@ def _reduce_update_read( gathers_residual: bool, fusions: Tuple[Callable, ...], read_fusions: Tuple[ReadoutFusion, ...] = (), + read_fusion_indices: Tuple[int, ...] = (), group: SumGroup = SumGroup.ATTN_TP, read: ResidualReadout = NORM_READOUT, update: ResidualUpdate = PLAIN_ADD, @@ -125,6 +149,8 @@ def _reduce_update_read( forward_batch, norm, read=read, + read_fusion_indices=read_fusion_indices, + update=update, quant_format=quant_format, post_residual_addition=post_residual_addition, ) @@ -271,12 +297,15 @@ def _run_entry( step: Callable, is_plain_add: bool, carried_fusions: Tuple[Callable, ...], + carried_read_fusions: Tuple[ReadoutFusion, ...] = (), + read: ResidualReadout = NORM_READOUT, expected_sum: Optional[SumGroup] = None, completed_step: Optional[Callable] = None, pending=None, written_step: Optional[Callable] = None, written: bool = False, update: ResidualUpdate = PLAIN_ADD, + read_context=None, **call, ): """A boundary's half into a stage: complete what the value carries, a sum @@ -287,6 +316,17 @@ def _run_entry( whose declared sum is already completed uses ``completed_step`` instead; a pending declared sum must match ``expected_sum``. ``call`` is what the stage's read takes (the attention's ``quant_format`` and ``post_residual_addition``).""" + bind = read.bind + if bind is not None: + bound = bind_read(read, forward_batch, read_context) + declared = read.completing_fusions + actual = bound.completing_fusions + carried_read_fusions = tuple( + actual[i] + for i, spec in enumerate(declared) + if any(spec is selected for selected in carried_read_fusions) + ) + read = bound if residual is not None and update.is_plain_add != is_plain_add: raise RuntimeError("producer update does not match the boundary's capability") if pending is not None: @@ -299,7 +339,7 @@ def _run_entry( step = completed_step if written and written_step is not None: step = written_step - if not isinstance(hidden_states, torch.Tensor): + if isinstance(hidden_states, (UnreducedOutput, DeferredFinalize)): owed = hidden_states if residual is None: raise RuntimeError(f"{type(owed).__name__} requires residual input") @@ -311,6 +351,30 @@ def _run_entry( if isinstance(owed, UnreducedOutput) and owed.reduce_to_dp_local is not None: hidden_states = complete_owed(owed) else: + read_candidates = tuple( + fused + for fused in carried_read_fusions + if ( + isinstance(owed, DeferredFinalize) + and fused.completes is owed.completion_group + ) + or ( + isinstance(owed, UnreducedOutput) + and _sum_group(fused.completes) is owed.group + ) + ) + result = _run_read_fusions( + read_candidates, + owed, + residual, + forward_batch, + norm, + read=read, + update=update, + **call, + ) + if result is not None: + return result for fused in carried_fusions: result = fused( owed, residual, forward_batch, call.get("post_residual_addition") @@ -321,6 +385,8 @@ def _run_entry( hidden_states = complete_owed(owed) else: hidden_states = owed.partial + if bind is not None: + call["read"] = read return step(hidden_states, residual, forward_batch, norm, update=update, **call) @@ -335,6 +401,7 @@ def _update_read( enters_stack: bool, read: ResidualReadout, read_gathers: Tuple[Callable, ...] = (), + read_gather_indices: Tuple[int, ...] = (), update: ResidualUpdate = PLAIN_ADD, quant_format: str = "", post_residual_addition: Optional[torch.Tensor] = None, @@ -350,6 +417,8 @@ def _update_read( hidden_states, residual = pre_move(hidden_states, residual) if enters: hidden_states, residual = read.init_residual(hidden_states), None + if read_gather_indices: + read_gathers = (read.gathering_reads[i] for i in read_gather_indices) for fused in read_gathers: result = fused(hidden_states, residual, norm) if result is not None: @@ -377,6 +446,7 @@ def _attn_tp_reduce_scatter_update_read( cache=None, scatters_residual: bool, read_fusions: Tuple[ReadoutFusion, ...] = (), + read_fusion_indices: Tuple[int, ...] = (), read: ResidualReadout = NORM_READOUT, update: ResidualUpdate = PLAIN_ADD, ): @@ -385,7 +455,14 @@ def _attn_tp_reduce_scatter_update_read( of the read's ``read_fusions`` that takes the batch does all of this instead.""" result = _run_read_fusions( - read_fusions, hidden_states, residual, forward_batch, norm, read=read + read_fusions, + hidden_states, + residual, + forward_batch, + norm, + read=read, + read_fusion_indices=read_fusion_indices, + update=update, ) if result is not None: return result @@ -474,11 +551,11 @@ def _then_attn_cp_gather( def _move_before_read( - hidden_states, residual, forward_batch, norm, *, move, read, **call + hidden_states, residual, forward_batch, norm, *, move, read_step, **call ): """Complete and place a producer contribution before its residual update.""" hidden_states, residual = move(hidden_states, residual, forward_batch) - return read(hidden_states, residual, forward_batch, norm, **call) + return read_step(hidden_states, residual, forward_batch, norm, **call) def _then_moe_cp_gather( @@ -490,6 +567,7 @@ def _then_moe_cp_gather( cache=None, gather: Callable, update: ResidualUpdate = PLAIN_ADD, + **call, ): """Read each shard, then gather the MoE group's tokens on its CP path. @@ -498,7 +576,7 @@ def _then_moe_cp_gather( ordinary path. The residual stays on this rank's attention rows. """ hidden_states, residual = gather( - hidden_states, residual, forward_batch, norm, update=update, cache=cache + hidden_states, residual, forward_batch, norm, update=update, cache=cache, **call ) hidden_states = moe_cp_gather( hidden_states, diff --git a/python/sglang/srt/layers/layer_boundary/residual/__init__.py b/python/sglang/srt/layers/layer_boundary/residual/__init__.py index adbffd9588..039031d228 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/__init__.py +++ b/python/sglang/srt/layers/layer_boundary/residual/__init__.py @@ -16,7 +16,7 @@ stage reads its input back from it (its read). The producer of a boundary declares the update, the consumer the read; the boundary steps complete sums and move tokens around the two.""" -from typing import NamedTuple, Optional, Protocol, Tuple +from typing import Callable, NamedTuple, Optional, Protocol, Tuple import torch @@ -39,10 +39,18 @@ class ResidualUpdate(Protocol): This is a numerical policy, separate from being a plain add; full precision is always correct. + An optional ``bind(forward_batch, context)`` returns an update owning this call's + dynamic metadata. The stage binds it once before recording a contribution; + every capability must match the declaration. Consumption releases the + invocation update with its contribution. + The shard conversion methods must move any update-associated state together with the residual. Nonlinear updates must not claim is_plain_add. """ + # None explicitly declares a stateless update. Bindings take (batch, context). + bind: Optional[Callable] = None + # A plain add, which one rank may run before the sum it adds into # completes (the DP partial order, a residual that joins the attention # output's sum), and which a fused add + norm kernel may run. @@ -108,6 +116,11 @@ class ResidualReadout(Protocol): snapshot. """ + # bind(batch, context) may replace callback implementations, preserving the + # order and capabilities of completing_fusions and the gathering_reads slots. + # None explicitly declares a stateless consumer. + bind: Optional[Callable] = None + # The input is the residual's norm, in the quantization the call asks for, # and the residual is left as it is: what a fused add + norm kernel computes, # and what may run on the rows a sum completes onto. @@ -175,6 +188,73 @@ class ResidualReadout(Protocol): """ +def bind_read(read, forward_batch, context=None): + """Bind a consumer invocation without changing its declared read capabilities.""" + if read.bind is None: + return read + bound = read.bind(forward_batch, context) + for fact in ( + "is_plain_norm", + "reads_before_dp_gather", + "reads_after_attn_tp_gather", + ): + if getattr(bound, fact) != getattr(read, fact): + raise ValueError("invocation read changed its declared capability") + # Callback slots are selected when the edge is constructed. Binding + # replaces their implementations, not their group or row capabilities. + declared = read.completing_fusions + actual = bound.completing_fusions + if len(actual) != len(declared) or len(bound.gathering_reads) != len( + read.gathering_reads + ): + raise ValueError("invocation read changed its declared callbacks") + for spec, fused in zip(declared, actual): + if (spec.completes, spec.scatters, spec.reads, spec.carried) != ( + fused.completes, + fused.scatters, + fused.reads, + fused.carried, + ) or (spec.accepts_update is None) != (fused.accepts_update is None): + raise ValueError("invocation read changed its declared callbacks") + return bound + + +def bind_update(update, forward_batch, context=None): + """Bind optional per-invocation producer state before recording output. + + ``bind(batch, context)`` must preserve every declared update capability and return + an update owning its invocation context with ``bind = None``. Stateless updates + are unchanged. + """ + if update.bind is None: + return update + actual = update.bind(forward_batch, context) + for fact in ( + "is_plain_add", + "applied_at_exit", + "outlives_layer", + "writes_stream", + "quantized_sum", + ): + if getattr(actual, fact) != getattr(update, fact): + raise ValueError("invocation update changed a declared capability") + if actual.bind is not None: + raise ValueError("invocation update must have bind = None") + return actual + + +def takes_pending_update(read, update): + """Whether a final read explicitly accepts this producer's update.""" + return isinstance(read, PendingUpdateRead) and read.accepts_pending_update(update) + + +class PendingUpdateRead: + """Explicit consumer interface for a terminal pending producer update.""" + + def accepts_pending_update(self, update: ResidualUpdate) -> bool: + raise NotImplementedError + + class FinalRead(Protocol): """The model's read of the layer stack's output, its final norm. @@ -182,6 +262,12 @@ class FinalRead(Protocol): with the output and the residual, returning ``(normed, residual)``: the residual add followed by the norm, so a pending update must be a plain add. + A ResidualReadout can instead explicitly supply + ``accepts_pending_update(update)`` and consume the pending producer update + through residual_batch.read_output(output, batch, read, norm). Construction + checks the declared update; execution checks the actual invocation. This + does not change the ordinary two-input final norm contract. + Optional, each read where it is used with the default given: attn_tp_gather: Its own gather over attention TP of rows the stack's last stage leaves on this rank's attention-TP slice, returning diff --git a/python/sglang/srt/layers/layer_boundary/residual/add_norm.py b/python/sglang/srt/layers/layer_boundary/residual/add_norm.py index 6bfdd74f77..2348145828 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/add_norm.py +++ b/python/sglang/srt/layers/layer_boundary/residual/add_norm.py @@ -273,6 +273,8 @@ def _norm_quant_kernel(quant_format: str): class PlainAdd: """A plain residual: the output is added into it.""" + bind = None + is_plain_add = True applied_at_exit = False outlives_layer = True @@ -296,6 +298,8 @@ class ReplaceAtExit: previous residual is dropped. The stream is complete: the producer has its boundary complete each part's sum first (StageBoundary.sum_part).""" + bind = None + is_plain_add = False applied_at_exit = True outlives_layer = True @@ -330,6 +334,8 @@ class NormQuantReadout: It is not applied by the aiter mxfp4 and fp8-group kernels, after an update other than a plain add, or by ``read``, which has no add.""" + bind = None + is_plain_norm = True completing_fusions = () gathering_reads = () @@ -370,6 +376,8 @@ class NormReadout: ``quant_format`` and a ``post_residual_addition``, except that the latter is not applied after an update other than a plain add.""" + bind = None + is_plain_norm = True completing_fusions = () gathering_reads = () @@ -417,6 +425,8 @@ class UnfusedNormReadout(NormReadout): two steps: the update rounds to the activation dtype before the norm, as in models that add their residual themselves. No fused kernel takes it.""" + bind = None + is_plain_norm = False completing_fusions = () gathering_reads = () diff --git a/python/sglang/srt/layers/layer_boundary/residual/attn_bank.py b/python/sglang/srt/layers/layer_boundary/residual/attn_bank.py index 2caef8bc5f..9257d622f4 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/attn_bank.py +++ b/python/sglang/srt/layers/layer_boundary/residual/attn_bank.py @@ -210,6 +210,8 @@ class _BankReadout: is not the residual's plain norm and no add+norm kernel may claim it. It reads this rank's own rows, which the bank holds, before any DP gather.""" + bind = None + is_plain_norm = False reads_before_dp_gather = True completing_fusions = () diff --git a/python/sglang/srt/layers/layer_boundary/residual/batch.py b/python/sglang/srt/layers/layer_boundary/residual/batch.py index e907621d01..b34d28e582 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/batch.py +++ b/python/sglang/srt/layers/layer_boundary/residual/batch.py @@ -35,6 +35,16 @@ def stream_of(forward_batch): return stream +def clear(forward_batch): + """Release invocation state on completion or an exception.""" + stream = forward_batch.residual_stream + if stream is not None: + if stream.pending is not None: + stream.pending.release() + stream.pending = stream.residual = None + forward_batch.residual_stream = None + + def written_residual(forward_batch): """The residual the last prepare wrote, for a stage whose compute reads it too (e.g. to write the next stream at its exit). Borrowed: do not modify.""" @@ -56,6 +66,17 @@ def take_output(hidden_states, forward_batch): return hidden_states +def _plain_output(hidden_states, forward_batch): + stream = stream_of(forward_batch) + stream.check(hidden_states) + if stream.pending is not None and not stream.pending.update.is_plain_add: + raise NotImplementedError( + "output/residual pair requires a plain residual update; " + "apply the producer update before export" + ) + return stream + + def final_norm( hidden_states, forward_batch, @@ -86,12 +107,12 @@ def final_norm( Normalized tensor. Add and norm remain together to preserve the kernel's accumulation/rounding order; a snapshot is not used as the norm input. """ - hidden_states, residual = stream_of(forward_batch).export( + hidden_states, residual = _plain_output(hidden_states, forward_batch).export( hidden_states, takes_handoff=finalize_norm is not None ) # The terminal consumer now owns the pair. Do not keep layer buffers alive # through logits processing or the next forward on this batch. - forward_batch.residual_stream = None + clear(forward_batch) from sglang.srt.layers.layer_boundary.output import DeferredFinalize if isinstance(hidden_states, DeferredFinalize): @@ -114,6 +135,83 @@ def final_norm( ) +def read_output(hidden_states, forward_batch, read, norm, *, read_context=None): + """Consume the stack's output with an explicitly declared final read. + + This entry accepts producer updates through ``accepts_pending_update``. + Completion candidates see the actual update before any collective; their + fallback completes the owed output once, then applies update_and_read. + A bindable read receives read_context once before consuming the output. + Ordinary model final norms continue to use final_norm(). + """ + from sglang.srt.layers.layer_boundary.layout import _sum_group + from sglang.srt.layers.layer_boundary.output import ( + DeferredFinalize, + UnreducedOutput, + ) + from sglang.srt.layers.layer_boundary.prepare import _run_read_fusions + from sglang.srt.layers.layer_boundary.residual import ( + bind_read, + takes_pending_update, + ) + from sglang.srt.layers.layer_boundary.residual.stream import DeclaredSum + + stream = stream_of(forward_batch) + stream.check(hidden_states) + pending = stream.pending + try: + if pending is not None and not takes_pending_update(read, pending.update): + raise NotImplementedError( + "final read does not accept the pending producer update" + ) + read = bind_read(read, forward_batch, read_context) + if pending is None: + value = stream.residual + if value is None: + value = read.init_residual(hidden_states) + return read.read(value, norm) + if not takes_pending_update(read, pending.update): + raise NotImplementedError( + "final read does not accept the pending producer update" + ) + owed = pending.owed + candidates = tuple( + f + for f in read.completing_fusions + if not f.scatters + and ( + (isinstance(owed, DeclaredSum) and f.completes is owed.group) + or ( + isinstance(owed, UnreducedOutput) + and f.carried + and owed.reduce_to_dp_local is None + and _sum_group(f.completes) is owed.group + ) + or ( + isinstance(owed, DeferredFinalize) + and f.carried + and f.completes is owed.completion_group + ) + ) + ) + result = _run_read_fusions( + candidates, + pending.for_boundary(), + stream.residual, + forward_batch, + norm, + read=read, + update=pending.update, + ) + if result is None: + value = pending.complete() + result = read.update_and_read(pending.update, value, stream.residual, norm) + return result + finally: + # The terminal consumer owns the result, including on an exception. + clear(forward_batch) + + def to_pp(hidden_states, forward_batch): """Export hidden_states and residual as PPProxyTensors. @@ -129,8 +227,10 @@ def to_pp(hidden_states, forward_batch): and is sent as hidden_states alone; the receiver's from_pp reconstructs it as written. """ - hidden_states, residual = stream_of(forward_batch).export(hidden_states) - forward_batch.residual_stream = None + hidden_states, residual = _plain_output(hidden_states, forward_batch).export( + hidden_states + ) + clear(forward_batch) tensors = {"hidden_states": hidden_states} if residual is not None: tensors["residual"] = residual diff --git a/python/sglang/srt/layers/layer_boundary/residual/gated.py b/python/sglang/srt/layers/layer_boundary/residual/gated.py index afa85675fa..0c77d5ba69 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/gated.py +++ b/python/sglang/srt/layers/layer_boundary/residual/gated.py @@ -134,6 +134,8 @@ class _AttnReadout: the previous layer's output: such a layer takes its input written back. A ``post_residual_addition`` is not applied.""" + bind = None + is_plain_norm = False completing_fusions = () gathering_reads = () @@ -167,6 +169,8 @@ class _AttnUpdate: residual its read produced. The injection is not a plain add, so it runs only once the sum it writes in is complete.""" + bind = None + is_plain_add = False applied_at_exit = False outlives_layer = False @@ -189,6 +193,8 @@ class _AttnUpdate: class _FfnReadout: """The attention output's injection and the FFN input's mix.""" + bind = None + is_plain_norm = False completing_fusions = () gathering_reads = () @@ -220,6 +226,8 @@ class _FfnUpdate: """The gated injection of the FFN output, which this layer runs itself. The streams stay widened: the layer stack's terminal read mixes them down.""" + bind = None + is_plain_add = False applied_at_exit = True outlives_layer = False diff --git a/python/sglang/srt/layers/layer_boundary/residual/ihc.py b/python/sglang/srt/layers/layer_boundary/residual/ihc.py index ad1c67406c..90b899a5af 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/ihc.py +++ b/python/sglang/srt/layers/layer_boundary/residual/ihc.py @@ -103,6 +103,8 @@ class _AttnReadout: already hold the previous layer's output: an iHC layer takes its input written back. A ``post_residual_addition`` is not applied.""" + bind = None + is_plain_norm = False completing_fusions = () gathering_reads = () @@ -129,6 +131,8 @@ class _AttnUpdate: It is not a plain add, so it runs only once the sum it writes in is complete.""" + bind = None + is_plain_add = False applied_at_exit = False outlives_layer = False @@ -152,6 +156,8 @@ class _FfnReadout: """The attention output's write-back and the FFN input's gated mix and norm, fused in post_pre when the layer provides it.""" + bind = None + is_plain_norm = False completing_fusions = () gathering_reads = () @@ -181,6 +187,8 @@ class _FfnUpdate: layer stack's terminal read is a learned head rather than a contraction, so it belongs to the final norm.""" + bind = None + is_plain_add = False applied_at_exit = True outlives_layer = False diff --git a/python/sglang/srt/layers/layer_boundary/residual/mhc.py b/python/sglang/srt/layers/layer_boundary/residual/mhc.py index 3ad6206faf..337989fb40 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/mhc.py +++ b/python/sglang/srt/layers/layer_boundary/residual/mhc.py @@ -139,6 +139,8 @@ class _AttnReadout: layer's output: an MHC layer takes its input written back. A ``post_residual_addition`` is not applied.""" + bind = None + is_plain_norm = False completing_fusions = () gathering_reads = () @@ -164,6 +166,8 @@ class _AttnUpdate: """hc_post with the coefficients the attention's read produced. It is not a plain add, so it runs only once the sum it writes in is complete.""" + bind = None + is_plain_add = False applied_at_exit = False outlives_layer = False @@ -187,6 +191,8 @@ class _FfnReadout: """The attention output's hc_post and the FFN input's hc_pre and norm, fused in hc_ffn_post_pre when it takes the batch.""" + bind = None + is_plain_norm = False completing_fusions = () gathering_reads = () @@ -215,6 +221,8 @@ class _FfnUpdate: layer runs itself; the last layer also contracts the streams into the hidden states the layer stack hands on.""" + bind = None + is_plain_add = False applied_at_exit = True outlives_layer = False diff --git a/python/sglang/srt/layers/layer_boundary/residual/post_norm.py b/python/sglang/srt/layers/layer_boundary/residual/post_norm.py index d10ef75724..dbcfebf4bc 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/post_norm.py +++ b/python/sglang/srt/layers/layer_boundary/residual/post_norm.py @@ -24,6 +24,8 @@ class PostNormAdd: """``residual + norm(output)``. It is not a plain add, so it runs only on a complete output. ``applied_at_exit`` makes the stage write it itself.""" + bind = None + is_plain_add = False outlives_layer = True writes_stream = False @@ -46,6 +48,8 @@ class PostNormAdd: class PlainReadout: """The input is the residual itself; the stage's norm is not applied.""" + bind = None + is_plain_norm = False completing_fusions = () gathering_reads = () diff --git a/python/sglang/srt/layers/layer_boundary/residual/stream.py b/python/sglang/srt/layers/layer_boundary/residual/stream.py index 2e0afcfa43..cd7ffab3b3 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/stream.py +++ b/python/sglang/srt/layers/layer_boundary/residual/stream.py @@ -46,7 +46,7 @@ class Contribution(msgspec.Struct): """ value: Optional[torch.Tensor] - update: ResidualUpdate + update: Optional[ResidualUpdate] owed: Union[UnreducedOutput, DeclaredSum, DeferredFinalize, None] = None def for_boundary(self): @@ -66,9 +66,10 @@ class Contribution(msgspec.Struct): return value def release(self): - """Drop the tensors of a contribution its consumer has taken.""" + """Drop a consumed contribution and its per-invocation update state.""" self.value = None self.owed = None + self.update = None class OwedOutput(msgspec.Struct, frozen=True): @@ -134,7 +135,8 @@ class ResidualStream: Args: output: Complete tensor, UnreducedOutput, or producer-specific DeferredFinalize. update: Producer operation that the next boundary must apply. - declared_sum: SumGroup statically owed by a raw tensor, or None. Cannot + declared_sum: The producer's outstanding sum contract, + statically owed by a raw tensor; None when complete. Cannot be combined with another output wrapper's completion contract. Returns: @@ -146,13 +148,23 @@ class ResidualStream: if declared_sum is not None: if not isinstance(output, torch.Tensor): raise RuntimeError("a declared sum cannot carry another completion") - contribution = Contribution(output, update, DeclaredSum(declared_sum)) + owed = ( + declared_sum + if isinstance(declared_sum, DeclaredSum) + else DeclaredSum(declared_sum) + ) + contribution = Contribution(output, update, owed) elif isinstance(output, UnreducedOutput): contribution = Contribution(output.partial, update, output) elif isinstance(output, DeferredFinalize): contribution = Contribution(None, update, output) else: contribution = Contribution(output, update) + bind = update.bind + if bind is not None: + raise RuntimeError( + "bind an invocation update at the stage before recording it" + ) self.pending = contribution return OwedOutput(contribution) if contribution.owed is not None else output diff --git a/python/sglang/srt/layers/layer_boundary/stage.py b/python/sglang/srt/layers/layer_boundary/stage.py index 67c3e7c1eb..bf73903f54 100644 --- a/python/sglang/srt/layers/layer_boundary/stage.py +++ b/python/sglang/srt/layers/layer_boundary/stage.py @@ -32,6 +32,7 @@ from sglang.srt.layers.layer_boundary.contracts import ( ) from sglang.srt.layers.layer_boundary.layout import SumGroup from sglang.srt.layers.layer_boundary.ops import sum_output +from sglang.srt.layers.layer_boundary.residual import bind_update from sglang.srt.layers.layer_boundary.residual.access import buffer, from_pp from sglang.srt.layers.layer_boundary.residual.add_norm import ( PLAIN_ADD, @@ -168,7 +169,7 @@ class StageBoundary: ) return hidden_states - def finish(self, hidden_states, forward_batch): + def finish(self, hidden_states, forward_batch, *, update_context=None): """Hand this stage's output to the following stage: an attention's actual contribution, or an FFN's or a mixer's output with its sum completed or carried on. An FFN whose boundary has fusions (which may @@ -177,16 +178,18 @@ class StageBoundary: produced = self.plan.path_for(forward_batch).output return stream_of(forward_batch).record( hidden_states, - produced.update, + bind_update(produced.update, forward_batch, update_context), declared_sum=produced.group if produced.always_partial else None, ) if self.kind is StageKind.FFN and self.plan.fusions is not None: raise RuntimeError( "an FFN whose boundary has fusions finishes through exit()" ) - return self.exit(forward_batch).finish(hidden_states) + return self.exit(forward_batch, update_context=update_context).finish( + hidden_states + ) - def exit(self, forward_batch): + def exit(self, forward_batch, *, update_context=None): """Select and scope one FFN or single-stage mixer's output decision, for an FFN whose MoE may hand off its finalize (see finish()). @@ -201,10 +204,16 @@ class StageBoundary: using the same decision rather than selecting a second path. """ result = ( - self.plan.output.ffn_exit(forward_batch, stream=stream_of(forward_batch)) + self.plan.output.ffn_exit( + forward_batch, + stream=stream_of(forward_batch), + update_context=update_context, + ) if self.kind is StageKind.FFN else self.plan.output.mixer_exit( - forward_batch, stream=stream_of(forward_batch) + forward_batch, + stream=stream_of(forward_batch), + update_context=update_context, ) ) return result @@ -260,7 +269,14 @@ class StageBoundary: hidden_states = stream.complete(hidden_states) return hidden_states, stream.snapshot(hidden_states) - def complete_now(self, hidden_states, forward_batch, *, already_reduced=False): + def complete_now( + self, + hidden_states, + forward_batch, + *, + already_reduced=False, + update_context=None, + ): """Complete an FFN output outside exit(), including its residual update. already_reduced is for a producer kernel that fused the output sum. @@ -271,6 +287,7 @@ class StageBoundary: stream_of(forward_batch), forward_batch, already_reduced=already_reduced, + update_context=update_context, ) def sum_part(self, hidden_states, forward_batch, group: SumGroup): @@ -347,6 +364,7 @@ class StageBoundary: post_residual_addition: Optional[torch.Tensor] = None, quant_format: str = "", capture: Optional[Callable] = None, + read_context=None, ): # Aux consumers need a materialized output before the input norm. # Complete communication here, preserving the existing add+norm kernel @@ -373,6 +391,7 @@ class StageBoundary: steps, quant_format=quant_format, post_residual_addition=post_residual_addition, + read_context=read_context, ) entry = steps.entry if capture is not None and not capture_before_read: diff --git a/python/sglang/srt/models/zaya.py b/python/sglang/srt/models/zaya.py index fc9c68ac15..f0db52d7ec 100644 --- a/python/sglang/srt/models/zaya.py +++ b/python/sglang/srt/models/zaya.py @@ -133,6 +133,8 @@ class _ResidualMergeRead: sum as the new residual, and the input norm of that sum in the model dtype. The producer's update must be a plain add.""" + bind = None + is_plain_norm = False completing_fusions = () gathering_reads = () diff --git a/python/sglang/test/boundary_fixtures.py b/python/sglang/test/boundary_fixtures.py index cce4292993..b0b6257145 100644 --- a/python/sglang/test/boundary_fixtures.py +++ b/python/sglang/test/boundary_fixtures.py @@ -1,8 +1,11 @@ """Small fixtures around production boundaries; no parallel layout recipes.""" +from contextlib import contextmanager from types import SimpleNamespace from unittest import mock +from unittest.mock import patch +from sglang.srt.layers import layernorm_sp from sglang.srt.layers.layer_boundary import factories from sglang.srt.layers.layer_boundary.construction import StagePlan from sglang.srt.layers.layer_boundary.factories import ( @@ -14,6 +17,8 @@ from sglang.srt.layers.layer_boundary.factories import ( from sglang.srt.layers.layer_boundary.ops import keep_output from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_RESIDUAL_OPS from sglang.srt.layers.layer_boundary.stage import StageBoundary +from sglang.srt.layers.moe import utils as moe_utils +from sglang.test.communicator_patch import patch_communicator def build_stages(*stages, previous=None, terminal=False): @@ -189,3 +194,130 @@ def postprocess_output(boundary, hidden, residual, forward_batch): def identity_input(hidden_states, forward_batch): return hidden_states + + +def parallel_of(*, attn_dp, attn_tp, attn_cp=1, **overrides): + fields = dict( + tp_size=attn_dp * attn_cp * attn_tp, + tp_rank=0, + attn_dp_size=attn_dp, + attn_tp_size=attn_tp, + attn_tp_rank=0, + attn_cp_size=attn_cp, + attn_cp_rank=0, + moe_dense_tp_size=None, + enable_prefill_cp=False, + enable_cp_tp_group_sharing=False, + moe_ep_size=1, + moe_tp_size=attn_dp * attn_cp * attn_tp, + moe_dp_size=1, + dwdp_size=1, + attn_dp_enabled=attn_dp > 1, + enable_attn_tp_input_scattered=False, + disable_attn_tp_gather=False, + tp_group=SimpleNamespace( + name="tp", ranks=list(range(attn_dp * attn_cp * attn_tp)) + ), + attn_tp_group=SimpleNamespace(name="attn_tp", ranks=list(range(attn_tp))), + attn_cp_group=SimpleNamespace(name="attn_cp", ranks=list(range(attn_cp))), + ) + fields.update(overrides) + return SimpleNamespace(**fields) + + +@contextmanager +def planning( + parallel, + *, + sp=False, + a2a=False, + dsa_cp=False, + boundary_reduction="rs+rsv", +): + """What layer planning and communicator construction read, without the + process-wide parallel state. ``parallel`` may be a callable, for a + per-thread parallel state. ``dsa_cp``: the prefill CP is DSA's (MLA's is + the same to the communicator).""" + get_parallel = parallel if callable(parallel) else (lambda: parallel) + + def moe_cp_gathers(): + return get_parallel().attn_cp_size > get_parallel().moe_dp_size + + with ( + patch_communicator("get_parallel", get_parallel), + patch_communicator("is_dsa_enable_prefill_cp", lambda: dsa_cp), + patch_communicator("is_mla_cp_enabled", lambda: False), + patch_communicator( + "get_moe_cp_size", + lambda: get_parallel().attn_cp_size if moe_cp_gathers() else 1, + ), + patch.object(layernorm_sp, "layernorm_sp_enabled", lambda: sp), + patch_communicator( + "get_spec", lambda: SimpleNamespace(speculative_algorithm=None) + ), + patch_communicator( + "get_moe_a2a_backend", + lambda: SimpleNamespace(is_none=lambda: not a2a), + ), + patch_communicator("is_moe_input_scattered_across_dp_ranks", lambda: a2a), + patch_communicator("is_enable_moe_cp_allgather", moe_cp_gathers), + patch_communicator("get_lora", lambda: SimpleNamespace(enable_lora=False)), + # A MoE whose EP and TP sums merge: its output's group is the TP group. + patch_communicator( + "post_experts_reduction_group", lambda: get_parallel().tp_group + ), + # Planning asks whether a dense layer gathers for two-batch overlap. + patch_communicator( + "get_exec", + lambda: SimpleNamespace( + comm=SimpleNamespace( + boundary_reduction=boundary_reduction, + enable_quant_communications=False, + ), + overlap=SimpleNamespace(enable_two_batch_overlap=False), + ), + ), + # What the MoE predicates an exit's decisions read, in their module. + patch.object(moe_utils, "get_parallel", get_parallel), + patch.object( + moe_utils, + "get_moe_a2a_backend", + lambda: SimpleNamespace( + is_none=lambda: not a2a, + is_flashinfer=lambda: False, + is_pplx=lambda: False, + is_flashinfer_megamoe=lambda: False, + ), + ), + patch.object(moe_utils, "get_lora", lambda: SimpleNamespace(enable_lora=False)), + patch.object( + moe_utils, + "get_exec", + lambda: SimpleNamespace( + comm=SimpleNamespace(enable_quant_communications=False) + ), + ), + patch.object( + moe_utils, "post_experts_reduction_group", lambda: get_parallel().tp_group + ), + patch.object( + moe_utils, "should_use_flashinfer_cutlass_moe_fp4_allgather", lambda: False + ), + patch.object( + moe_utils, + "is_dp_attention_enabled", + lambda: get_parallel().attn_dp_size > 1, + ), + ): + yield + + +class Norm: + """add + norm stand-in: (2 * (h + r), h + r); one input: 2 * h.""" + + def __call__(self, x, residual=None, post_residual_addition=None): + assert post_residual_addition is None + if residual is None: + return x * 2 + s = x + residual + return s * 2, s diff --git a/test/registered/unit/layer_boundary/test_append_stages.py b/test/registered/unit/layer_boundary/test_append_stages.py index 205ae3b5d0..1ac8c60e52 100644 --- a/test/registered/unit/layer_boundary/test_append_stages.py +++ b/test/registered/unit/layer_boundary/test_append_stages.py @@ -36,7 +36,7 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import ( NormQuantReadout, ) from sglang.srt.layers.layer_boundary.residual.stream import DeclaredSum -from sglang.srt.model_executor.forward_batch_info import PPProxyTensors +from sglang.srt.model_executor.forward_batch_info import ForwardMode, PPProxyTensors from sglang.srt.utils.common import make_layers from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.communicator_patch import patch_communicator @@ -887,6 +887,52 @@ class TestUnpaddedBatches(CustomTestCase): with layer_stack(): return [s for _ in range(2) for s in layer(sparse=sparse)] + def test_explicit_full_row_ffn_does_not_bind_an_unused_unpadded_path(self): + parallel = fixture.parallel_of( + attn_dp=1, attn_tp=2, disable_attn_tp_gather=True, moe_dense_tp_size=1 + ) + with fixture.planning(parallel): + with layer_stack(): + attention, ffn = append_stages( + (declare_attn(), fixture.Norm()), + (declare_ffn(dense_tp_size=2), fixture.Norm()), + ) + batch = SimpleNamespace( + input_ids=torch.ones(3), + forward_mode=ForwardMode.EXTEND, + attn_cp_metadata=None, + ) + for stage in (attention, ffn): + self.assertEqual(set(stage.plan.paths), {BatchVariant.ORDINARY}) + self.assertIs(stage.plan.variant_for(batch), BatchVariant.ORDINARY) + self.assertTrue(parallel.disable_attn_tp_gather) + + def test_rank_row_pipeline_neighbour_keeps_the_unpadded_variant(self): + parallel = fixture.parallel_of( + attn_dp=1, + attn_tp=2, + pp_size=2, + disable_attn_tp_gather=True, + moe_dense_tp_size=1, + ) + for direction in ("previous_layers", "next_layers"): + with self.subTest(direction=direction), fixture.planning(parallel): + with layer_stack( + **{direction: [lambda: (declare_ffn(dense_tp_size=1),)]} + ): + attention, ffn = append_stages( + (declare_attn(), fixture.Norm()), + (declare_ffn(dense_tp_size=2), fixture.Norm()), + ) + batch = SimpleNamespace( + input_ids=torch.ones(3), + forward_mode=ForwardMode.EXTEND, + attn_cp_metadata=None, + ) + for stage in (attention, ffn): + self.assertIn(BatchVariant.UNPADDED, stage.plan.paths) + self.assertIs(stage.plan.variant_for(batch), BatchVariant.UNPADDED) + def test_an_unpadded_batch_keeps_the_attention_rows(self): for sparse in (True, False): with self.subTest(sparse=sparse): @@ -1032,6 +1078,8 @@ class TestRowsTheConsumerReads(CustomTestCase): class Scaled: """Neither a plain add nor applied at the producer's exit.""" + bind = None + is_plain_add = False applied_at_exit = False outlives_layer = True @@ -1119,6 +1167,9 @@ class TestReadKernels(CustomTestCase): return "boundary", residual class Update: + bind = None + is_plain_add = True + def slice_residual_attn_tp(self, residual): return residual[:2] diff --git a/test/registered/unit/layer_boundary/test_attn_bank_residual_ops.py b/test/registered/unit/layer_boundary/test_attn_bank_residual_ops.py index 1b37db9469..b779b4889b 100644 --- a/test/registered/unit/layer_boundary/test_attn_bank_residual_ops.py +++ b/test/registered/unit/layer_boundary/test_attn_bank_residual_ops.py @@ -420,6 +420,7 @@ class TestAttnBankSpMoeStages(CustomTestCase): residual = torch.randn(4, HIDDEN, generator=torch.Generator().manual_seed(9)) class Update: + bind = None is_plain_add = True def slice_residual_attn_tp(self, residual): diff --git a/test/registered/unit/layer_boundary/test_boundary_edges.py b/test/registered/unit/layer_boundary/test_boundary_edges.py index e4c13fc848..0722ea2737 100644 --- a/test/registered/unit/layer_boundary/test_boundary_edges.py +++ b/test/registered/unit/layer_boundary/test_boundary_edges.py @@ -184,6 +184,8 @@ class _WrittenIn: """An update that is not a plain add: the output is written into the residual only once its sum is complete.""" + bind = None + is_plain_add = False applied_at_exit = False @@ -291,20 +293,9 @@ class TestTheProducersUpdateChoosesTheOrder(CustomTestCase): with self.assertRaisesRegex(NotImplementedError, "lifetime"): bind_exit(edge) - def test_pipeline_cannot_reconstruct_a_non_add_update(self): - local = Layout(frozenset()) - update = SimpleNamespace( - is_plain_add=False, applied_at_exit=False, outlives_layer=True - ) - edge = EdgeContract( - OutputContract(local, update=update), InputContract(local), local, local - ) - with get_parallel().override(pp_size=2): - with self.assertRaisesRegex(NotImplementedError, "pipeline boundaries"): - bind_exit(edge) - class ProbeRead: + bind = None reads_before_dp_gather = False completing_fusions = () gathering_reads = () diff --git a/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py b/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py index 371f3e99ee..c6b26e1d8c 100644 --- a/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py +++ b/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py @@ -20,7 +20,6 @@ from unittest.mock import MagicMock, patch import torch from sglang.srt.layers import layer_boundary as comm -from sglang.srt.layers import layernorm_sp from sglang.srt.layers.layer_boundary import ( SumGroup, TokenAxis, @@ -62,8 +61,11 @@ from sglang.srt.layers.moe import utils as moe_utils from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.runtime_context import get_forward from sglang.test.boundary_fixtures import ( + Norm, finish_exit, make_test_stages, + parallel_of, + planning, postprocess_output, prepare_input, ) @@ -76,133 +78,6 @@ register_cpu_ci(est_time=15, suite="base-a-test-cpu") HIDDEN = 4 -def parallel_of(*, attn_dp, attn_tp, attn_cp=1, **overrides): - fields = dict( - tp_size=attn_dp * attn_cp * attn_tp, - tp_rank=0, - attn_dp_size=attn_dp, - attn_tp_size=attn_tp, - attn_tp_rank=0, - attn_cp_size=attn_cp, - attn_cp_rank=0, - moe_dense_tp_size=None, - enable_prefill_cp=False, - enable_cp_tp_group_sharing=False, - moe_ep_size=1, - moe_tp_size=attn_dp * attn_cp * attn_tp, - moe_dp_size=1, - dwdp_size=1, - attn_dp_enabled=attn_dp > 1, - enable_attn_tp_input_scattered=False, - disable_attn_tp_gather=False, - tp_group=SimpleNamespace( - name="tp", ranks=list(range(attn_dp * attn_cp * attn_tp)) - ), - attn_tp_group=SimpleNamespace(name="attn_tp", ranks=list(range(attn_tp))), - attn_cp_group=SimpleNamespace(name="attn_cp", ranks=list(range(attn_cp))), - ) - fields.update(overrides) - return SimpleNamespace(**fields) - - -@contextmanager -def planning( - parallel, - *, - sp=False, - a2a=False, - dsa_cp=False, - boundary_reduction="rs+rsv", -): - """What layer planning and communicator construction read, without the - process-wide parallel state. ``parallel`` may be a callable, for a - per-thread parallel state. ``dsa_cp``: the prefill CP is DSA's (MLA's is - the same to the communicator).""" - get_parallel = parallel if callable(parallel) else (lambda: parallel) - - def moe_cp_gathers(): - return get_parallel().attn_cp_size > get_parallel().moe_dp_size - - with ( - patch_communicator("get_parallel", get_parallel), - patch_communicator("is_dsa_enable_prefill_cp", lambda: dsa_cp), - patch_communicator("is_mla_cp_enabled", lambda: False), - patch_communicator( - "get_moe_cp_size", - lambda: get_parallel().attn_cp_size if moe_cp_gathers() else 1, - ), - patch.object(layernorm_sp, "layernorm_sp_enabled", lambda: sp), - patch_communicator( - "get_spec", lambda: SimpleNamespace(speculative_algorithm=None) - ), - patch_communicator( - "get_moe_a2a_backend", - lambda: SimpleNamespace(is_none=lambda: not a2a), - ), - patch_communicator("is_moe_input_scattered_across_dp_ranks", lambda: a2a), - patch_communicator("is_enable_moe_cp_allgather", moe_cp_gathers), - patch_communicator("get_lora", lambda: SimpleNamespace(enable_lora=False)), - # A MoE whose EP and TP sums merge: its output's group is the TP group. - patch_communicator( - "post_experts_reduction_group", lambda: get_parallel().tp_group - ), - # Planning asks whether a dense layer gathers for two-batch overlap. - patch_communicator( - "get_exec", - lambda: SimpleNamespace( - comm=SimpleNamespace( - boundary_reduction=boundary_reduction, - enable_quant_communications=False, - ), - overlap=SimpleNamespace(enable_two_batch_overlap=False), - ), - ), - # What the MoE predicates an exit's decisions read, in their module. - patch.object(moe_utils, "get_parallel", get_parallel), - patch.object( - moe_utils, - "get_moe_a2a_backend", - lambda: SimpleNamespace( - is_none=lambda: not a2a, - is_flashinfer=lambda: False, - is_pplx=lambda: False, - is_flashinfer_megamoe=lambda: False, - ), - ), - patch.object(moe_utils, "get_lora", lambda: SimpleNamespace(enable_lora=False)), - patch.object( - moe_utils, - "get_exec", - lambda: SimpleNamespace( - comm=SimpleNamespace(enable_quant_communications=False) - ), - ), - patch.object( - moe_utils, "post_experts_reduction_group", lambda: get_parallel().tp_group - ), - patch.object( - moe_utils, "should_use_flashinfer_cutlass_moe_fp4_allgather", lambda: False - ), - patch.object( - moe_utils, - "is_dp_attention_enabled", - lambda: get_parallel().attn_dp_size > 1, - ), - ): - yield - - -class Norm: - """add + norm stand-in: (2 * (h + r), h + r); one input: 2 * h.""" - - def __call__(self, x, residual=None, post_residual_addition=None): - assert post_residual_addition is None - if residual is None: - return x * 2 - s = x + residual - return s * 2, s - - class FusableNorm(Norm): """A norm with the fused all-reduce + add + norm method.""" diff --git a/test/registered/unit/layer_boundary/test_producer_completion.py b/test/registered/unit/layer_boundary/test_producer_completion.py new file mode 100644 index 0000000000..009d53c23f --- /dev/null +++ b/test/registered/unit/layer_boundary/test_producer_completion.py @@ -0,0 +1,731 @@ +"""Producer updates survive binding, carried completion and terminal handoffs.""" + +import unittest +from collections.abc import Mapping +from dataclasses import fields, is_dataclass +from enum import Enum +from functools import partial +from types import FunctionType, MethodType, SimpleNamespace +from unittest.mock import Mock, patch + +import msgspec +import test_declared_decoder_boundary as fixture +import torch + +from sglang.srt.layers.layer_boundary import ( + ReadoutFusion, + append_stages, + declare_attn, + declare_ffn, + layer_stack, +) +from sglang.srt.layers.layer_boundary.facts import ( + ReadFacts, + TransformFacts, + UpdateFacts, + facts_of, +) +from sglang.srt.layers.layer_boundary.layout import SumGroup +from sglang.srt.layers.layer_boundary.output import ( + DeferredFinalize, + OutputTransform, + UnreducedOutput, +) +from sglang.srt.layers.layer_boundary.residual import PendingUpdateRead +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch +from sglang.srt.layers.layer_boundary.residual.add_norm import ( + PLAIN_ADD, + NormReadout, + PlainAdd, +) +from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream +from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class Nonplain(PlainAdd): + is_plain_add = False + quantized_sum = False + + @staticmethod + def update(value, residual): + return value.square() + residual + + +class TestProducerCompletion(CustomTestCase): + def test_update_context_is_bound_once_for_each_completion_route(self): + class Read(PendingUpdateRead): + bind = None + completing_fusions = () + accepts_pending_update = staticmethod(lambda update: True) + init_residual = staticmethod(lambda value: value) + read = staticmethod(lambda value, norm: value) + update_and_read = staticmethod( + lambda update, value, residual, norm: update.update(value, residual) + ) + + for route in ("finish", "exit", "complete_now", "mixer"): + for at_exit in (False, True): + with self.subTest(route=route, at_exit=at_exit): + bindings = [] + + class Update(Nonplain): + applied_at_exit = at_exit + + def bind(self, batch, context=None): + bindings.append(context) + bound = Nonplain() + bound.applied_at_exit = at_exit + bound.update = lambda value, residual: ( + value * context + residual + ) + return bound + + parallel = fixture.parallel_of(attn_dp=1, attn_tp=1) + declare = declare_attn if route == "mixer" else declare_ffn + with fixture.planning(parallel), layer_stack(final_read=Read()): + (stage,) = append_stages( + (declare(update=Update()), fixture.Norm()) + ) + for context in (3, 7): + batch = SimpleNamespace( + input_ids=torch.ones(2, dtype=torch.long), + forward_mode=ForwardMode.DECODE, + residual_stream=ResidualStream(torch.ones(2, 4)), + ) + with fixture.planning(parallel): + value = torch.full((2, 4), 2.0) + if route == "exit": + result = stage.exit( + batch, update_context=context + ).finish(value) + elif route == "complete_now": + result = stage.complete_now( + value, batch, update_context=context + ) + else: + result = stage.finish( + value, batch, update_context=context + ) + + result = residual_batch.read_output( + result, batch, Read(), None + ) + torch.testing.assert_close( + result, torch.full((2, 4), 2 * context + 1.0) + ) + self.assertEqual(bindings, [3, 7]) + + def test_terminal_pair_export_releases_retained_contribution(self): + """An old owed handle must not retain the update after terminal export.""" + for route in ("pp", "norm", "failing_norm"): + with self.subTest(route=route): + value = torch.ones(2, 4) + update = PlainAdd() + update.context = object() + stream = ResidualStream(value) + batch = SimpleNamespace(residual_stream=stream) + group = SimpleNamespace(all_reduce=lambda value: value * 3) + handle = stream.record(UnreducedOutput(value, group), update) + if route == "pp": + result = residual_batch.to_pp(handle, batch) + torch.testing.assert_close(result["hidden_states"], value * 3) + torch.testing.assert_close(result["residual"], value) + elif route == "norm": + result = residual_batch.final_norm(handle, batch, fixture.Norm()) + torch.testing.assert_close(result, value * 8) + else: + with self.assertRaisesRegex(RuntimeError, "norm failed"): + residual_batch.final_norm( + handle, batch, Mock(side_effect=RuntimeError("norm failed")) + ) + self.assertIsNone(batch.residual_stream) + self.assertIsNone(handle.contribution.update) + self.assertIsNone(handle.contribution.value) + self.assertIsNone(handle.contribution.owed) + self.assertIsNone(stream.pending) + self.assertIsNone(stream.residual) + + def test_terminal_read_binds_each_context_before_read_or_completion(self): + """Terminal reads must not run declaration callbacks with missing state.""" + events = [] + + class Read(NormReadout, PendingUpdateRead): + def __init__(self, route, context=None): + self.route, self.context = route, context + self.completing_fusions = ( + (ReadoutFusion(SumGroup.ATTN_TP, self.complete, reads=True),) + if route == "fused" + else () + ) + + def bind(self, batch, context): + events.append(("bind", context)) + return Read(self.route, batch.context if context is None else context) + + def accepts_pending_update(self, update): + return update.is_plain_add + + def read(self, value, norm, **kwargs): + assert self.context is not None, "unbound terminal read" + events.append("read") + return value + self.context, value + + def update_and_read(self, update, value, residual, norm, **kwargs): + return self.read(update.update(value, residual), norm) + + def complete(self, value, residual, batch, norm): + assert self.context is not None, "unbound terminal fusion" + events.append("complete") + return self.read(value * 3 + residual, norm) + + for route in ("written", "pending", "fused"): + for context in (17, 29): + for explicit_context in (False, True): + with self.subTest( + route=route, context=context, explicit=explicit_context + ): + events.clear() + value = torch.ones(2, 4) + stream = ResidualStream(value) + batch = SimpleNamespace(residual_stream=stream, context=context) + hidden = value + if route != "written": + hidden = stream.record( + value, + PLAIN_ADD, + declared_sum=SumGroup.ATTN_TP + if route == "fused" + else None, + ) + kwargs = {"read_context": context} if explicit_context else {} + result, residual = residual_batch.read_output( + hidden, batch, Read(route), None, **kwargs + ) + expected = {"written": 1, "pending": 2, "fused": 4}[route] + torch.testing.assert_close( + result, torch.full_like(value, expected + context) + ) + torch.testing.assert_close( + residual, torch.full_like(value, expected) + ) + self.assertEqual( + events, + [("bind", context if explicit_context else None)] + + (["complete"] if route == "fused" else []) + + ["read"], + ) + self.assertIsNone(batch.residual_stream) + self.assertIsNone(stream.pending) + + def test_terminal_read_rejects_changed_binding_capabilities_and_cleans_up(self): + """Binding cannot change routing facts or callbacks before terminal execution.""" + for change in ("rows", "callbacks"): + with self.subTest(change=change): + + class Read(NormReadout, PendingUpdateRead): + accepts_pending_update = staticmethod(lambda update: True) + + def bind(self, batch, context): + class Bound(NormReadout): + pass + + bound = Bound(reads_before_dp_gather=change == "rows") + if change == "callbacks": + bound.gathering_reads = (lambda *args: None,) + return bound + + value = torch.ones(2, 4) + stream = ResidualStream(value) + batch = SimpleNamespace(residual_stream=stream) + hidden = stream.record(value, PLAIN_ADD) + contribution = stream.pending + with self.assertRaisesRegex(ValueError, "invocation read changed"): + residual_batch.read_output(hidden, batch, Read(), None) + self.assertIsNone(contribution.update) + self.assertIsNone(batch.residual_stream) + + def test_update_binding_rejects_bindable_results_before_any_exit_route(self): + """Immediate writes and deferred records must reject an unbound result alike.""" + parallel = fixture.parallel_of(attn_dp=1, attn_tp=1) + for at_exit in (False, True): + with self.subTest(at_exit=at_exit): + + class Update(Nonplain): + applied_at_exit = at_exit + update = Mock(side_effect=AssertionError("unbound update ran")) + + def bind(self, batch, context): + return self + + with fixture.planning(parallel), layer_stack(): + stage, _ = append_stages( + (declare_ffn(update=Update()), fixture.Norm()), + (declare_ffn(), fixture.Norm()), + ) + value = torch.ones(2, 4) + batch = SimpleNamespace( + residual_stream=ResidualStream(value), + input_ids=torch.ones(2, dtype=torch.long), + forward_mode=ForwardMode.DECODE, + ) + with ( + fixture.planning(parallel), + self.assertRaisesRegex( + ValueError, "invocation update must have bind = None" + ), + ): + stage.finish(value, batch) + self.assertIsNone(batch.residual_stream.pending) + + def test_pipeline_cut_rejects_deferred_nonplain_but_local_stage_allows_it(self): + parallel = fixture.parallel_of(attn_dp=1, attn_tp=2, pp_size=2) + with fixture.planning(parallel): + with layer_stack(): + append_stages( + (declare_ffn(update=Nonplain()), fixture.Norm()), + (declare_ffn(), fixture.Norm()), + ) + for declare in (declare_attn, declare_ffn): + with self.subTest(stage=declare.__name__): + with self.assertRaisesRegex( + NotImplementedError, "pipeline boundaries" + ): + with layer_stack(next_layers=[lambda: (declare_attn(),)]): + append_stages((declare(update=Nonplain()), fixture.Norm())) + + for a2a, dense in ((True, None), (False, 1)): + parallel = fixture.parallel_of(attn_dp=2, attn_tp=1, pp_size=2) + with ( + self.subTest(a2a=a2a, dense=dense), + fixture.planning(parallel, a2a=a2a), + ): + with self.assertRaisesRegex(NotImplementedError, "pipeline boundaries"): + with layer_stack(next_layers=[lambda: (declare_attn(),)]): + append_stages( + ( + declare_ffn( + update=Nonplain(), sparse=a2a, dense_tp_size=dense + ), + fixture.Norm(), + ) + ) + + def test_pair_export_rejects_nonplain_before_starting_completion(self): + for exporter in (residual_batch.to_pp, residual_batch.final_norm): + with self.subTest(export=exporter.__name__): + group = SimpleNamespace( + all_reduce=Mock(side_effect=AssertionError("sum")) + ) + value = torch.ones(2, 4) + batch = SimpleNamespace(residual_stream=ResidualStream(value)) + output = batch.residual_stream.record( + UnreducedOutput(value, group), Nonplain() + ) + with self.assertRaisesRegex( + NotImplementedError, "plain residual update" + ): + if exporter is residual_batch.final_norm: + exporter(output, batch, fixture.Norm()) + else: + exporter(output, batch) + group.all_reduce.assert_not_called() + self.assertIsNotNone(batch.residual_stream.pending) + + def test_carried_finalize_uses_only_candidates_for_its_completion_group(self): + events = [] + + class Completion(DeferredFinalize, frozen=True): + def complete(self): + events.append("complete") + return torch.full((2, 4), 3.0) + + class Read(NormReadout): + completing_fusions = ( + ReadoutFusion( + SumGroup.ATTN_TP, + lambda *a, **kw: self.fail("wrong completion group"), + carried=True, + ), + ) + + parallel = fixture.parallel_of(attn_dp=1, attn_tp=2, pp_size=1) + with fixture.planning(parallel), layer_stack(): + (stage,) = append_stages((declare_ffn(read=Read()), fixture.Norm())) + batch = SimpleNamespace( + input_ids=torch.ones(2, dtype=torch.long), + forward_mode=ForwardMode.DECODE, + residual_stream=ResidualStream(torch.ones(2, 4)), + ) + hidden = batch.residual_stream.record(Completion(), PLAIN_ADD) + with fixture.planning(parallel): + result = stage.prepare(hidden, batch) + torch.testing.assert_close(result, torch.full((2, 4), 8.0)) + self.assertEqual(events, ["complete"]) + self.assertIsNone(batch.residual_stream.pending) + + def test_neighbour_facts_do_not_retain_layer_state(self): + import test_kimi_k3_stage_kernels as k3_fixture + + from sglang.srt.models import kimi_k3 + + seen = set() + + def walk(value): + if id(value) in seen: + return + seen.add(id(value)) + self.assertNotIsInstance(value, (torch.nn.Module, torch.Tensor)) + if isinstance(value, msgspec.Struct): + for name in value.__struct_fields__: + walk(getattr(value, name)) + elif is_dataclass(value): + for field in fields(value): + walk(getattr(value, field.name)) + elif isinstance(value, Mapping): + for key, item in value.items(): + walk(key) + walk(item) + elif isinstance(value, (tuple, list, set, frozenset)): + for item in value: + walk(item) + elif isinstance(value, partial): + walk(value.func) + walk(value.args) + walk(value.keywords) + elif isinstance(value, MethodType): + walk(value.__self__) + walk(value.__func__) + elif isinstance(value, FunctionType): + walk(value.__defaults__) + walk(value.__kwdefaults__) + for cell in value.__closure__ or (): + walk(cell.cell_contents) + elif not isinstance( + value, (type(None), bool, int, float, str, bytes, Enum) + ): + self.fail(f"unexpected execution object: {type(value)}") + + update = Nonplain() + update.layer = torch.nn.Linear(4, 4) + declaration = declare_ffn( + update=update, output_transform=OutputTransform(update.layer.forward) + ) + facts = facts_of(declaration) + self.assertIsInstance(facts.read, ReadFacts) + self.assertIsInstance(facts.update, UpdateFacts) + self.assertIsInstance(facts.output_transform, TransformFacts) + walk(facts) + # The checker must reject an opaque update even when it is not callable. + with self.assertRaisesRegex(AssertionError, "unexpected execution object"): + walk(update) + + config = SimpleNamespace(**vars(k3_fixture.CONFIG), attn_res_block_size=4) + with ( + fixture.planning(fixture.parallel_of(attn_dp=1, attn_tp=2), a2a=True), + patch.object(kimi_k3, "_shards_moe_rows", return_value=True), + ): + declarations = kimi_k3._stage_facts(config, 2, sp_collective=True) + for declaration in declarations: + walk(facts_of(declaration)) + + def test_pipeline_sender_uses_the_remote_k3_gather(self): + import test_kimi_k3_stage_kernels as k3_fixture + + from sglang.srt.layers.communication import k3_sp_collective + from sglang.srt.layers.layer_boundary import BatchVariant, ops + from sglang.srt.models import kimi_k3 + + config = SimpleNamespace(**vars(k3_fixture.CONFIG), attn_res_block_size=4) + gather = Mock(side_effect=lambda value: torch.cat((value, value + 10))) + with ( + fixture.planning( + fixture.parallel_of(attn_dp=1, attn_tp=2, pp_size=2), a2a=True + ), + patch.object(kimi_k3, "_shards_moe_rows", return_value=True), + patch.object(k3_sp_collective, "all_gather", gather), + layer_stack( + next_layers=[ + lambda: kimi_k3._stage_facts(config, 2, sp_collective=True) + ] + ), + ): + (stage,) = append_stages((declare_ffn(sparse=True), fixture.Norm())) + move = stage.plan.paths[BatchVariant.ORDINARY].output_move + self.assertIs(move.keywords["gather"], gather) + value = torch.ones(2, 4) + with patch.object( + ops, "attn_tp_gather", side_effect=AssertionError("default gather") + ): + output, _ = move( + hidden_states=value, + residual=torch.zeros_like(value), + forward_batch=None, + ) + gather.assert_called_once() + torch.testing.assert_close(output, torch.cat((value, value + 10))) + + def test_invocation_callbacks_complete_selected_sum_and_gather_paths(self): + from sglang.srt.layers.layer_boundary import prepare + from sglang.srt.layers.layer_boundary.boundary import bind_entry + from sglang.srt.layers.layer_boundary.contracts import ( + EdgeContract, + InputContract, + OutputContract, + ) + from sglang.srt.layers.layer_boundary.layout import Layout, TokenAxis + from sglang.srt.layers.layer_boundary.ops import GatheredInput + from sglang.srt.layers.layer_boundary.residual.stream import DeclaredSum + + events = [] + + class Read(NormReadout): + is_plain_norm = False + + def __init__(self, context=None): + self.context = context + self.completing_fusions = tuple( + ReadoutFusion( + group, + partial(self.complete, group, scatters), + scatters=scatters, + reads=True, + carried=not scatters, + ) + for group, scatters in ( + (SumGroup.TP, False), + (SumGroup.ATTN_TP, True), + (SumGroup.ATTN_TP, False), + ) + ) + self.gathering_reads = (self.gather,) + + def bind(self, batch, context): + events.append(("bind", context)) + return Read(context) + + def complete(self, group, scatters, value, residual, batch, norm): + assert self.context is not None, "unbound completion" + events.append(("complete", group, scatters, self.context)) + return torch.full_like(residual, self.context), residual + + def gather(self, value, residual, norm): + assert self.context is not None, "unbound gathering read" + events.append(("gather", self.context)) + return torch.full_like(residual, self.context), residual + + full = Layout(frozenset()) + shard = Layout(frozenset({TokenAxis.ATTN_TP})) + group = object() + with patch.object( + prepare, + "_sum_group", + side_effect=lambda selected: ( + group if selected is SumGroup.ATTN_TP else None + ), + ): + for route in ("sum", "scatter", "gather", "carried"): + source = shard if route == "gather" else full + target = shard if route == "scatter" else full + rows = shard if route in ("gather", "scatter") else full + declared = route in ("sum", "scatter") + edge = EdgeContract( + OutputContract( + source, + group=SumGroup.ATTN_TP if declared else None, + always_partial=declared, + ), + InputContract(target, read=Read()), + rows, + rows, + ) + entry = bind_entry(edge) + for context in (17, 29): + with self.subTest(route=route, context=context): + events.clear() + value = torch.ones(2, 4) + hidden = ( + UnreducedOutput(value, group) + if route == "carried" + else value + ) + result, _ = entry.prepare( + hidden, + value, + SimpleNamespace(), + None, + pending=( + SimpleNamespace(owed=DeclaredSum(SumGroup.ATTN_TP)) + if declared + else None + ), + read_context=context, + ) + if isinstance(result, GatheredInput): + result = result.value + torch.testing.assert_close( + result, torch.full_like(value, context) + ) + callback = ( + ("gather", context) + if route == "gather" + else ( + "complete", + SumGroup.ATTN_TP, + route == "scatter", + context, + ) + ) + self.assertEqual(events, [("bind", context), callback]) + + def test_output_transform_preserves_invocation_binding_and_read_order(self): + from sglang.srt.layers.layer_boundary import prepare + from sglang.srt.layers.layer_boundary.boundary import bind_entry + from sglang.srt.layers.layer_boundary.contracts import ( + EdgeContract, + InputContract, + OutputContract, + ) + from sglang.srt.layers.layer_boundary.layout import Layout + from sglang.srt.layers.layer_boundary.residual.stream import DeclaredSum + + events = [] + batch = SimpleNamespace() + forbidden = Mock(side_effect=AssertionError("fusion bypassed transform")) + + class Read(NormReadout): + completing_fusions = ( + ReadoutFusion(SumGroup.ATTN_TP, forbidden, reads=True, carried=True), + ) + gathering_reads = (forbidden,) + + def __init__(self, context=None): + self.context = context + + def bind(self, forward_batch, context): + self_test.assertIs(forward_batch, batch) + events.append(("bind", context)) + return Read(context) + + def update_and_read(self, update, value, residual, norm, **kwargs): + assert self.context is not None, "unbound fallback read" + events.append(("read", self.context)) + written = update.update(value, residual) + return written + self.context, written + + def reduce(value, forward_batch, **kwargs): + self.assertIs(forward_batch, batch) + events.append("sum") + return value + 3 + + def transform(value): + events.append("transform") + return value.square() + + self_test = self + full = Layout(frozenset()) + with patch.object(prepare, "attn_tp_all_reduce", side_effect=reduce): + for bindable in (True, False): + reader = Read() if bindable else Read(0) + if not bindable: + reader.bind = None + edge = EdgeContract( + OutputContract( + full, + group=SumGroup.ATTN_TP, + always_partial=True, + transform=OutputTransform(transform), + ), + InputContract(full, read=reader), + full, + full, + ) + entry = bind_entry(edge) + for context in (17, 29): + with self.subTest(bindable=bindable, context=context): + events.clear() + value = torch.ones(2, 4) + result, residual = entry.prepare( + value, + value, + batch, + None, + pending=SimpleNamespace(owed=DeclaredSum(SumGroup.ATTN_TP)), + read_context=context, + ) + effective = context if bindable else 0 + torch.testing.assert_close(result, value * (17 + effective)) + torch.testing.assert_close(residual, value * 17) + self.assertEqual( + events, + ([("bind", context)] if bindable else []) + + ["sum", "transform", ("read", effective)], + ) + forbidden.assert_not_called() + + def test_invocation_read_reaches_normal_and_cp_consumers(self): + from sglang.srt.layers.layer_boundary import prepare + from sglang.srt.layers.layer_boundary.boundary import CpMoves, bind_entry + from sglang.srt.layers.layer_boundary.contracts import ( + EdgeContract, + InputContract, + OutputContract, + ) + from sglang.srt.layers.layer_boundary.layout import Layout, TokenAxis + + class Read(NormReadout): + def bind(self, batch, context): + bound = Read() + bound.bind = None + bound.context = context + return bound + + def update_and_read(self, update, value, residual, norm, **kw): + return value + residual + self.context, residual + + full = Layout(frozenset()) + local = Layout(frozenset({TokenAxis.ATTN_CP})) + moves = CpMoves( + gather=prepare._then_moe_cp_gather, take_back=lambda h, r, b: (h[:2], r) + ) + batch = SimpleNamespace( + attn_cp_metadata=SimpleNamespace(per_rank_actual_token=[2, 2]) + ) + for shape in ("local", "gather", "move"): + with self.subTest(shape=shape): + source = full if shape in ("local", "move") else local + residual_rows = local if shape in ("gather", "move") else full + target = full if shape == "gather" else residual_rows + edge = EdgeContract( + OutputContract(source), + InputContract(target, read=Read()), + residual_rows, + residual_rows, + ) + entry = bind_entry(edge, cp_moves=moves) + with ( + patch.object( + prepare, + "moe_cp_gather", + side_effect=lambda h, *a: torch.cat([h, h]), + ), + patch.object(prepare, "get_moe_cp_size", return_value=2), + ): + for context in (3, 7): + value, _ = entry.prepare( + torch.ones(4 if shape == "move" else 2, 4), + torch.ones(2, 4), + batch, + None, + read_context=context, + ) + torch.testing.assert_close( + value, torch.full_like(value, context + 2) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layer_boundary/test_residual_stream.py b/test/registered/unit/layer_boundary/test_residual_stream.py index 95570a9493..64f4db4a70 100644 --- a/test/registered/unit/layer_boundary/test_residual_stream.py +++ b/test/registered/unit/layer_boundary/test_residual_stream.py @@ -45,6 +45,8 @@ class TestResidualStream(CustomTestCase): def test_written_input_does_not_repeat_first_stage_initialization(self): class Read: + completing_fusions = () + bind = None is_plain_norm = False enters = 0 @@ -78,6 +80,7 @@ class TestResidualStream(CustomTestCase): def test_consumer_executes_the_carried_update_not_the_bound_producers(self): class Update: + bind = None is_plain_add = False applied_at_exit = False @@ -90,6 +93,8 @@ class TestResidualStream(CustomTestCase): return value + residual + self.increment class Read: + completing_fusions = () + bind = None is_plain_norm = False def update_and_read(self, update, value, residual, norm, **kwargs): @@ -209,7 +214,7 @@ class TestResidualStream(CustomTestCase): ) def test_snapshot_rejects_stateful_update_without_touching_main_state(self): - update = SimpleNamespace(is_plain_add=False, update=Mock()) + update = SimpleNamespace(is_plain_add=False, update=Mock(), bind=None) stream = ResidualStream(self.residual) hidden = stream.record(UnreducedOutput(self.partial, group=self.group), update) with self.assertRaisesRegex(NotImplementedError, "plain residual update"): @@ -394,6 +399,8 @@ class TestResidualStream(CustomTestCase): def test_prepare_releases_the_consumed_contribution(self): class Read: + completing_fusions = () + bind = None is_plain_norm = True def update_and_read(self, update, value, residual, norm, **kwargs): @@ -467,6 +474,8 @@ class TestBatchStageOwnership(CustomTestCase): from sglang.srt.layers.layer_boundary.residual import batch class Read: + completing_fusions = () + bind = None is_plain_norm = False def init_residual(self, value): diff --git a/test/registered/unit/layer_boundary/test_token_order_attention.py b/test/registered/unit/layer_boundary/test_token_order_attention.py index 6fda661cc7..0d0ea7a471 100644 --- a/test/registered/unit/layer_boundary/test_token_order_attention.py +++ b/test/registered/unit/layer_boundary/test_token_order_attention.py @@ -97,7 +97,7 @@ class TestTokenOrderAttention(CustomTestCase): take_back = step.keywords["move"] self.assertIs(take_back.func, cp_take_back_in_token_order) self.assertIs(take_back.keywords["group"], SumGroup.ATTN_CP) - read = step.keywords["read"] + read = step.keywords["read_step"] self.assertIs(read.func, prepare._then_moe_cp_gather) self.assertIs(read.keywords["gather"].func, prepare._update_read) entry = ffn.plan.paths[CP].entry