diff --git a/docs/docs/developer_guide/layer_boundary.mdx b/docs/docs/developer_guide/layer_boundary.mdx index b18848a040..75491032a1 100644 --- a/docs/docs/developer_guide/layer_boundary.mdx +++ b/docs/docs/developer_guide/layer_boundary.mdx @@ -43,9 +43,8 @@ A declaration states how its compute cooperates with the boundary's reduction de | Contract or state | Meaning | | --- | --- | -| `ProducerReduction.ALWAYS_PARTIAL` | Attention only (the `declare_attn()` default): compute always returns a contribution requiring a sum and publishes it with `finish()`. Build its output projection so that it never reduces (for example `reduce_results=False`). `declare_ffn()` rejects it | -| `ProducerReduction.EXIT_SCOPED` | Compute obeys the exit's reduction decision and publishes through `exit()`: the `declare_ffn()` default, and `declare_attn(reduction=ProducerReduction.EXIT_SCOPED)` for a mixer | -| `ProducerReduction.TAIL_AFTER_SUM` | FFN only: compute adds a replicated component after its internal sum. The boundary never defers that sum to the next layer, but reduce-scatter permissions remain, so compute that skips its sum for `mlp_reduce_scatter` must add the component exactly once. `declare_attn()` rejects it | +| `ProducerReduction.ALWAYS_PARTIAL` | Attention only (the `declare_attn()` default): compute always returns a contribution requiring a sum and publishes it with `finish()`; the next stage's input completes the sum. Build its output projection so that it never reduces (for example `reduce_results=False`) | +| `ProducerReduction.EXIT_SCOPED` | The exit completes the sum or carries it to the next stage's input: every FFN, and `declare_attn(reduction=ProducerReduction.EXIT_SCOPED)` for a mixer | | `OutputContract.may_*` | Construction-time permission to leave the sum to a particular later step | | `Contribution.owed` | Work this specific output still requires | | `OwedOutput` | Model-facing handle preventing ordinary tensor use of that unfinished output | @@ -159,14 +158,13 @@ if hidden_states.shape[0] != 0: hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) -with self.ffn_boundary.exit(forward_batch) as output: - hidden_states = self.mlp(hidden_states, forward_batch=forward_batch) -hidden_states = output.finish(hidden_states) +hidden_states = self.mlp(hidden_states, forward_batch=forward_batch) +hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch) ``` This example uses Qwen3's compute member name `mlp`; boundary construction does not require renaming compute classes or checkpoint parameters. -`prepare()` returns input in the consumer read's format, which can include quantized tensors. An attention with an `ALWAYS_PARTIAL` output contract uses `finish()` to register its contribution. An FFN or single-stage mixer runs its compute under `exit()`, then calls the returned object's `finish()` once. The exit context restores the previous runtime flags even when compute raises; do not call `finish()` on a failed computation. +`prepare()` returns input in the consumer read's format, which can include quantized tensors. An attention with an `ALWAYS_PARTIAL` output contract uses `finish()` to register its contribution, and an FFN or single-stage mixer uses `finish()` to complete or carry on the sum its output owes. An FFN whose boundary has fusions, which may take a MoE finalize handoff, runs its compute under `exit()` instead and calls the returned object's `finish()` once (`finish()` on the boundary rejects it); the exit context restores the previous runtime flags even when compute raises, so do not call `finish()` on a failed computation. Compute never sums a stage's output: build every output projection, MLP and MoE block of a stage with `reduce_results=False`, and leave the sum the output owes to the boundary. The boundary completes it at the exit, or carries it to the next stage's input, where a fused all-reduce + add + norm may take it: @@ -258,7 +256,6 @@ Common errors and their usual cause: | `write the residual update before taking the final output` | `take_output()` ran while a contribution was still pending; fold or norm it first | | `no stage boundary path for the active batch` | The active batch variant (CP, input-scattered, sequence parallel) is not supported by this stage; add a server-argument check that rejects the configuration | | `a prepared branch must enter through branch_input, not prepare` | A `prepared_from` stage was entered with `prepare()` | -| `ALWAYS_PARTIAL is not supported for ffn stages` / `TAIL_AFTER_SUM is not supported for attention stages` | The declaration's `reduction` does not match the stage kind | | `snapshot requires a plain residual update` | A snapshot was taken on a nonlinear (for example MHC) update | ## Boundary internals diff --git a/python/sglang/srt/layers/layer_boundary/boundary.py b/python/sglang/srt/layers/layer_boundary/boundary.py index 3755df5740..cb96d38ae4 100644 --- a/python/sglang/srt/layers/layer_boundary/boundary.py +++ b/python/sglang/srt/layers/layer_boundary/boundary.py @@ -115,9 +115,6 @@ class ExitMove(NamedTuple): # Whether output_move also completes the sum the producer leaves. output_move_completes_sum: bool = False returns_over_dp: bool = False - # When output_move completes the sum: the move of an output compute has - # already reduced onto the same rows. - complete_output_move: Optional[Callable] = None def input_rows(edge: EdgeContract) -> Layout: @@ -599,9 +596,8 @@ def _select_exit_move( ) -> ExitMove: """How the FFN output reaches the rows the layer hands on: by undoing the attention-DP gather (the FFN exit and finish run that step), or else the - move that takes it there, None when there is none to choose here; whether - that move also completes the sum the FFN leaves, and if so the move for an - output compute already reduced.""" + move that takes it there, None when there is none to choose here; and + whether that move also completes the sum the FFN leaves.""" update = produced.update if produced.layout == residual: @@ -634,12 +630,6 @@ def _select_exit_move( update=update, ), output_move_completes_sum=True, - complete_output_move=partial( - residual_slice_output, - sums=False, - gathers_back=to != residual, - update=update, - ), ) if to != residual or not produced.layout.sharded <= residual.sharded: raise NotImplementedError(f"{produced=} {residual=} {to=}") @@ -655,11 +645,7 @@ def _select_exit_move( _sum_group(produced.group), cp_moves.reduce_scatter_group() ): raise NotImplementedError(f"{produced=} {residual=} {to=}") - return ExitMove( - cp_moves.reduce_scatter, - output_move_completes_sum=True, - complete_output_move=cp_moves.take_back, - ) + return ExitMove(cp_moves.reduce_scatter, output_move_completes_sum=True) if returned == {TokenAxis.ATTN_DP, TokenAxis.ATTN_CP}: # This rank's CP shard, from where the DP gather put it. return ExitMove(dp_cp_take_back_output) diff --git a/python/sglang/srt/layers/layer_boundary/construction.py b/python/sglang/srt/layers/layer_boundary/construction.py index 334b64b7fb..ddcaf0a6e8 100644 --- a/python/sglang/srt/layers/layer_boundary/construction.py +++ b/python/sglang/srt/layers/layer_boundary/construction.py @@ -214,7 +214,6 @@ class StagePlan: output_move=out.output_move, output_move_completes_sum=out.output_move_completes_sum, returns_over_dp=out.returns_over_dp, - complete_output_move=out.complete_output_move, ) @property diff --git a/python/sglang/srt/layers/layer_boundary/contracts.py b/python/sglang/srt/layers/layer_boundary/contracts.py index 3f22096b02..025ab6a89c 100644 --- a/python/sglang/srt/layers/layer_boundary/contracts.py +++ b/python/sglang/srt/layers/layer_boundary/contracts.py @@ -27,19 +27,16 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import NORM_READOUT, PLA class ProducerReduction(Enum): - """Describe compute's cooperation with its output boundary. + """Who completes the sum a stage's output owes; compute never does. - ALWAYS_PARTIAL is attention-only: finish() publishes its partial sum. EXIT_SCOPED - follows exit() flags for an FFN or single-stage mixer. TAIL_AFTER_SUM is FFN-only: - compute adds a replicated component after its internal sum, so the - boundary never defers that sum to the next layer. A selected reduce-scatter - still applies, so compute must add the tail only when it completes the sum. + ALWAYS_PARTIAL is attention-only: finish() hands the partial sum to the + next stage's input as its declared sum. EXIT_SCOPED, for an FFN or a + single-stage mixer: the exit completes the sum, or carries it to the next + stage's input. """ ALWAYS_PARTIAL = auto() EXIT_SCOPED = auto() - # A replicated component is added after the sum inside compute. - TAIL_AFTER_SUM = auto() class ExitRows(Enum): @@ -80,30 +77,30 @@ class OutputContract(msgspec.Struct, frozen=True): Fields: layout: Token sharding of the producer contribution. - group: Named sum group, or None when there is no reduction. - always_partial: Compute always returns a partial sum. - may_defer_to_next: Compute can skip reduction under the exit scope - and let the following layer complete it. - may_reduce_scatter: Compute can leave reduction to a fixed-size - reduce-scatter selected by the boundary. - may_reduce_scatterv: Compute can leave reduction to the selected - variable-size attention-DP combine. + group: The group the output owes its sum over, or None when it is + complete. + always_partial: The sum is handed to the next stage's input as its + declared sum. + may_defer_to_next: The exit may carry the sum to the following + stage's input instead of completing it. + may_reduce_scatter: A fixed-size reduce-scatter selected by the + boundary may complete the sum. + may_reduce_scatterv: The selected variable-size attention-DP combine + may complete the sum. update: Producer residual operation; None on an arrival contract that declares capabilities and obtains the actual update from the stream. transform: Optional operation on the contribution before residual update. - These permissions are not evidence that a particular output is partial. - The exit decision and ResidualStream record what that output actually owes. + These are permissions, not a record of a particular output. The exit + decision and ResidualStream record what that output actually owes. """ layout: Layout # None when there is nothing to sum. group: Optional[SumGroup] = None - # The producer never reduces its output, e.g. a row-parallel projection - # built with reduce_results=False. + # The sum is always handed to the next stage's input as a declared sum. always_partial: bool = False - # Otherwise it leaves the sum only when the boundary publishes the flag for - # it: fuse_mlp_allreduce, or mlp_reduce_scatter. + # Otherwise the exit completes it, or may carry it to the next input. may_defer_to_next: bool = False may_reduce_scatter: bool = False # Whether the selected attention-DP combine may complete this sum. @@ -236,8 +233,6 @@ class StagePath(msgspec.Struct, frozen=True): batch by the attention-DP exit path. output_move_completes_sum: Whether that move also reduces the output. returns_over_dp: Whether output uses batch-dependent attention-DP transport. - complete_output_move: The move for an output compute already reduced, - run instead of an output_move that also reduces it. """ entry: EntryPath @@ -247,7 +242,6 @@ class StagePath(msgspec.Struct, frozen=True): output_move_completes_sum: bool = False returns_over_dp: bool = False - complete_output_move: Optional[Callable] = None class StageKind(Enum): diff --git a/python/sglang/srt/layers/layer_boundary/exit.py b/python/sglang/srt/layers/layer_boundary/exit.py index c055f915ef..35585fcd77 100644 --- a/python/sglang/srt/layers/layer_boundary/exit.py +++ b/python/sglang/srt/layers/layer_boundary/exit.py @@ -129,11 +129,11 @@ class ExitPolicy: return stream.record(hidden_states, update, declared_sum=declared_sum) @staticmethod - def _sum_owed_after_skip(steps, skipped_reduction): - # The input-scattered path leaves the sum for the next input's TP - # reduce-scatter. Other skip paths complete it in the output move. + def _sum_left_to_next_input(steps, sum_in_reduce_scatter): + # On an input-scattered batch the next input's TP reduce-scatter + # completes the sum; the other reduce-scatters are this exit's move. if ( - skipped_reduction + sum_in_reduce_scatter and not steps.returns_over_dp and not steps.output_move_completes_sum ): @@ -160,10 +160,9 @@ class ExitPolicy: may_reduce_scatterv=steps.output.may_reduce_scatterv, ) - def _skips_sum_for_reduce_scatter(self, steps, dp_step: Optional[Callable]) -> bool: - """Whether the FFN leaves its sum out because a reduce-scatter completes - it: the attention-DP one ``dp_step`` names, or the CP / input-scattered - one.""" + def _sum_in_reduce_scatter(self, steps, dp_step: Optional[Callable]) -> bool: + """Whether a reduce-scatter completes the FFN output's sum: the + attention-DP one ``dp_step`` names, or the CP / input-scattered one.""" if dp_step is not None: return True if not steps.output.may_reduce_scatter: @@ -178,10 +177,10 @@ class ExitPolicy: return _sum_group(steps.output.group) def _decide(self, forward_batch: ForwardBatch, steps) -> ExitDecision: - """Decide once, before the FFN runs, what it skips and what completes its - output: the next layer's input, or this layer's postprocess step.""" + """Decide once, before the FFN runs, what completes its output's sum: + the next layer's input, or this layer's postprocess step.""" dp_step = self._dp_reduce_scatter_step(forward_batch, steps) - mlp_reduce_scatter = self._skips_sum_for_reduce_scatter(steps, dp_step) + sum_in_reduce_scatter = self._sum_in_reduce_scatter(steps, dp_step) complete_now = partial( self._complete_now, forward_batch=forward_batch, @@ -189,7 +188,7 @@ class ExitPolicy: steps=steps, output_move=steps.output_move, # A reduce-scatter completes the sum on its way back. - owes_sum=steps.output.group is not None and not mlp_reduce_scatter, + owes_sum=steps.output.group is not None and not sum_in_reduce_scatter, ) defer_moe_finalize = ( self.plan.fusions is not None @@ -200,16 +199,18 @@ class ExitPolicy: ): return ExitDecision( defer_moe_finalize=False, - fuse_mlp_allreduce=False, - mlp_reduce_scatter=mlp_reduce_scatter, + sum_in_reduce_scatter=sum_in_reduce_scatter, complete=complete_now, ) # Producers declare remaining work independently of the kernel chosen # by the consumer. Every handoff also carries an unfused completion. - fuse_mlp_allreduce = defer_moe_finalize or self._defers_sum( - forward_batch, steps, mlp_reduce_scatter=mlp_reduce_scatter, dp_step=dp_step + defers = defer_moe_finalize or self._defers_sum( + forward_batch, + steps, + sum_in_reduce_scatter=sum_in_reduce_scatter, + dp_step=dp_step, ) - if fuse_mlp_allreduce: + if defers: group = self.ffn_reduction_group(steps) if steps.returns_over_dp: # Under attention DP the next layer also brings the sum back to @@ -239,8 +240,7 @@ class ExitPolicy: complete = complete_now return ExitDecision( defer_moe_finalize=defer_moe_finalize, - fuse_mlp_allreduce=fuse_mlp_allreduce, - mlp_reduce_scatter=mlp_reduce_scatter, + sum_in_reduce_scatter=sum_in_reduce_scatter, complete=complete, ) @@ -288,15 +288,16 @@ class ExitPolicy: self, forward_batch: ForwardBatch, *, stream: ResidualStream ) -> MixerExit: """Decide once whether this stage's mixer (an attention-like stage) - skips its output all-reduce. Use the result as a context manager around - the mixer, then call ``finish``.""" + output's sum is completed here or carried on. Call ``finish`` on the + result with the mixer's output.""" return MixerExit(self, forward_batch, stream=stream) def ffn_exit( self, forward_batch: ForwardBatch, *, stream: ResidualStream ) -> FfnExit: """Decide once how this layer's FFN output reduction completes. Use the - result as a context manager around the FFN call, then call ``finish``.""" + result as a context manager around the FFN call when the FFN may hand + off its MoE finalize, then call ``finish``.""" return FfnExit(self, forward_batch, stream=stream) def _sum_deferral_allowed(self, steps) -> bool: @@ -331,7 +332,7 @@ class ExitPolicy: forward_batch: ForwardBatch, steps, *, - mlp_reduce_scatter: bool, + sum_in_reduce_scatter: bool, dp_step: Optional[Callable], ) -> bool: """Whether the FFN leaves its output's all-reduce to the next layer's @@ -342,7 +343,7 @@ class ExitPolicy: and not self.plan.terminal and self._sum_deferral_allowed(steps) and _batch_allows_deferred_sum(forward_batch, self.plan) - and not mlp_reduce_scatter + and not sum_in_reduce_scatter # Under attention DP the next layer must also run postprocess's # scatter back to this rank's tokens, and nothing more. and ( @@ -392,9 +393,7 @@ class ExitDecision(msgspec.Struct, frozen=True): Fields: defer_moe_finalize: Compute may return a producer-specific finalize handoff. - fuse_mlp_allreduce: The sum is carried to the next consumer, which - completes it, fused or unfused. - mlp_reduce_scatter: A reduce-scatter on the way back completes the sum. + sum_in_reduce_scatter: A reduce-scatter on the way back completes the sum. complete: Callable(output, residual) returning the completed or wrapped output and its corresponding residual rows. @@ -402,8 +401,7 @@ class ExitDecision(msgspec.Struct, frozen=True): """ defer_moe_finalize: bool - fuse_mlp_allreduce: bool - mlp_reduce_scatter: bool + sum_in_reduce_scatter: bool complete: Callable[[torch.Tensor, torch.Tensor], Tuple] @@ -519,8 +517,8 @@ class FfnExit: self.defer_moe_finalize = completion.defer_moe_finalize self._complete = completion.complete self._update = steps.output.update - self._declared_sum = boundary._sum_owed_after_skip( - steps, completion.mlp_reduce_scatter + self._declared_sum = boundary._sum_left_to_next_input( + steps, completion.sum_in_reduce_scatter ) self._scope = get_forward().scoped(defer_moe_finalize=self.defer_moe_finalize) diff --git a/python/sglang/srt/layers/layer_boundary/factories.py b/python/sglang/srt/layers/layer_boundary/factories.py index 816ca5fd5e..d842c10770 100644 --- a/python/sglang/srt/layers/layer_boundary/factories.py +++ b/python/sglang/srt/layers/layer_boundary/factories.py @@ -82,7 +82,6 @@ def _resolve_ffn( read=NORM_READOUT, update=PLAIN_ADD, dense_tp_size=None, - reduction=ProducerReduction.EXIT_SCOPED, ): parallel = get_parallel() if dense_tp_size not in (None, 1, parallel.tp_size): @@ -169,8 +168,7 @@ def _resolve_ffn( and not terminal and not update.applied_at_exit and update.outlives_layer - and output_transform is None - and reduction is ProducerReduction.EXIT_SCOPED, + and output_transform is None, may_reduce_scatter=use_reduce_scatter and may_scatter, may_reduce_scatterv=use_reduce_scatterv and may_leave, update=update, @@ -203,8 +201,8 @@ class StageDeclaration: residual update. An FFN's exit runs it under an explicit reduction-order contract; an attention's is run by the input of the stage that follows, once the attention's sum is complete. - reduction: Whether compute always leaves a partial sum, obeys the - exit scope, or adds a replicated component after its own sum. + reduction: Whether the next stage's input always completes the sum + (an attention's), or the exit decides (an FFN's or a mixer's). gathers_attn_tp_input: Whether attention gathers TP-sharded input itself. dense_tp_size: Dense FFN compute width: None uses the configured width, 1 means local compute, and the full TP size means TP compute. @@ -282,9 +280,8 @@ def declare_attn( read: Input operation; defaults to normalization with quantization support. update: Operation that adds this stage's output to the residual. terminal: Whether this stage ends the model's layer stack. - reduction: ALWAYS_PARTIAL for an output projection that always skips reduction; - EXIT_SCOPED for a mixer that follows its exit scope's reduction decision. - TAIL_AFTER_SUM is rejected for attention stages. + reduction: ALWAYS_PARTIAL for an attention whose sum the next stage's + input always completes; EXIT_SCOPED for a mixer whose exit decides. gathers_attn_tp_input: Whether compute gathers attention-TP input slices itself. output_transform: Optional operation on the output once its sum is complete, before the residual update (a sandwich norm). The next @@ -294,8 +291,6 @@ def declare_attn( Returns: A StageDeclaration with no norm, tensors or execution plan. """ - if reduction is ProducerReduction.TAIL_AFTER_SUM: - raise ValueError("TAIL_AFTER_SUM is not supported for attention stages") if ( output_transform is not None and reduction is not ProducerReduction.ALWAYS_PARTIAL @@ -325,7 +320,6 @@ def declare_ffn( output_transform=None, next_layer_sparse=False, dense_tp_size=None, - reduction=ProducerReduction.EXIT_SCOPED, exit_rows=None, ): """Declare a dense or MoE FFN independently of its compute module. @@ -342,17 +336,12 @@ def declare_ffn( to derive the TBO exit rows when exit_rows is not supplied. dense_tp_size: Dense compute width: None for configuration, 1 for local compute, or the full TP size. - reduction: How compute cooperates with the exit's reduction decision: - EXIT_SCOPED (default) follows the exit scope; TAIL_AFTER_SUM marks a - replicated tail after the sum. ALWAYS_PARTIAL is rejected for FFN stages. exit_rows: Explicit output-row requirement; otherwise derived from the adjacent FFN kinds and TBO configuration. Returns: A StageDeclaration with no norm, tensors or execution plan. """ - if reduction is ProducerReduction.ALWAYS_PARTIAL: - raise ValueError("ALWAYS_PARTIAL is not supported for ffn stages") return StageDeclaration( StageKind.FFN, read, @@ -363,7 +352,6 @@ def declare_ffn( terminal=terminal, output_transform=output_transform, dense_tp_size=dense_tp_size, - reduction=reduction, exit_rows=exit_rows or tbo_exit_rows(sparse, next_layer_sparse), ) @@ -388,7 +376,6 @@ def _resolve_stage(stage, variant, following=None): read=stage.read, update=stage.update, dense_tp_size=stage.dense_tp_size, - reduction=stage.reduction, ) if resolve_exit_rows(stage.exit_rows) is ExitRows.ATTENTION: returned = attention diff --git a/python/sglang/srt/layers/layer_boundary/ops.py b/python/sglang/srt/layers/layer_boundary/ops.py index 53a411c522..8cd077c60a 100644 --- a/python/sglang/srt/layers/layer_boundary/ops.py +++ b/python/sglang/srt/layers/layer_boundary/ops.py @@ -428,10 +428,9 @@ def moe_cp_take_back_output( ): """Return a MoE output computed on the MoE-CP-gathered rows to this rank's attention rows. - After moe_tensor_model_parallel_all_reduce (which runs unconditionally since - mlp_reduce_scatter=False for this path), all ranks in the moe_cp group hold the - full MoE result for all cp_per_moe token chunks. We simply slice out this rank's - CP-local portion. + Once the exit has completed the MoE output's sum, all ranks in the moe_cp + group hold the full MoE result for all cp_per_moe token chunks. We simply + slice out this rank's CP-local portion. If DP>1, further scatter back to the local DP slice. """ diff --git a/python/sglang/srt/layers/layer_boundary/stage.py b/python/sglang/srt/layers/layer_boundary/stage.py index e8af2e361e..970de6a22e 100644 --- a/python/sglang/srt/layers/layer_boundary/stage.py +++ b/python/sglang/srt/layers/layer_boundary/stage.py @@ -170,20 +170,26 @@ class StageBoundary: return hidden_states def finish(self, hidden_states, forward_batch): - """Hand this attention's actual contribution to the following stage.""" - if self.kind is not StageKind.ATTENTION or not self.plan.finishes_directly: - raise RuntimeError( - "use the stage exit scope for an FFN or single-stage mixer" + """Hand this stage's output to the following stage: an attention's + actual contribution, or an FFN's or a mixer's output with its sum + completed or carried on. An FFN whose boundary has fusions (which may + take a MoE finalize handoff) finishes through exit() instead.""" + if self.kind is StageKind.ATTENTION and self.plan.finishes_directly: + produced = self.plan.produced(forward_batch) + return stream_of(forward_batch).record( + hidden_states, + produced.update, + declared_sum=produced.group if produced.always_partial else None, ) - produced = self.plan.produced(forward_batch) - return stream_of(forward_batch).record( - hidden_states, - produced.update, - declared_sum=produced.group if produced.always_partial else None, - ) + if self.kind is StageKind.FFN and self.plan.fusions is not None: + raise RuntimeError( + "an FFN whose boundary has fusions finishes through exit()" + ) + return self.exit(forward_batch).finish(hidden_states) def exit(self, forward_batch): - """Select and scope one FFN or single-stage mixer's output decision. + """Select and scope one FFN or single-stage mixer's output decision, + for an FFN whose MoE may hand off its finalize (see finish()). Args: forward_batch: Active batch whose stream receives the compute result. diff --git a/python/sglang/srt/layers/linear.py b/python/sglang/srt/layers/linear.py index 1935c9e97f..31ed3f6835 100644 --- a/python/sglang/srt/layers/linear.py +++ b/python/sglang/srt/layers/linear.py @@ -1661,9 +1661,9 @@ class RowParallelLinear(LinearBase): self, input_parallel, output_tensor, bias=bias_ ) - # skip_all_reduce: explicit call-site override. Also honor - # ForwardFlags (fuse_mlp_allreduce / mlp_reduce_scatter) published by - # the decoder — callers should not thread those flags into modules. + # skip_all_reduce: explicit call-site override. Also honor the + # mlp_reduce_scatter ForwardFlag published by the decoder — callers + # should not thread it into modules. if ( ((self.reduce_results and self.tp_size > 1) or self.use_decode_attn_tp) and not skip_all_reduce diff --git a/python/sglang/srt/layers/moe/utils.py b/python/sglang/srt/layers/moe/utils.py index b185aaf00e..eff60f3730 100644 --- a/python/sglang/srt/layers/moe/utils.py +++ b/python/sglang/srt/layers/moe/utils.py @@ -730,12 +730,10 @@ def should_use_dp_reduce_scatterv(): def should_skip_mlp_all_reduce() -> bool: """Whether dense MLP / row-parallel projections should skip their all-reduce. - True when the decoder published ``fuse_mlp_allreduce`` (next residual+LN - absorbs the AR) or ``mlp_reduce_scatter`` (postprocess will reduce-scatter) - on ``get_forward()``. + True when the decoder published ``mlp_reduce_scatter`` (postprocess will + reduce-scatter) on ``get_forward()``. """ - f = get_forward() - return f.fuse_mlp_allreduce or f.mlp_reduce_scatter + return get_forward().mlp_reduce_scatter def post_experts_output_is_complete(*, is_tp_path: bool) -> bool: diff --git a/python/sglang/srt/models/apertus.py b/python/sglang/srt/models/apertus.py index cb1efd5cd2..be1a6b73e7 100644 --- a/python/sglang/srt/models/apertus.py +++ b/python/sglang/srt/models/apertus.py @@ -295,9 +295,8 @@ class ApertusDecoderLayer(nn.Module): # Fully Connected hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp(hidden_states) - return ffn_exit.finish(hidden_states) + hidden_states = self.mlp(hidden_states) + return self.ffn_boundary.finish(hidden_states, forward_batch) class ApertusModel(nn.Module): diff --git a/python/sglang/srt/models/arcee.py b/python/sglang/srt/models/arcee.py index 70ce424c2e..f166732f02 100644 --- a/python/sglang/srt/models/arcee.py +++ b/python/sglang/srt/models/arcee.py @@ -271,9 +271,8 @@ class ArceeDecoderLayer(nn.Module): # Fully Connected hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp(hidden_states) - return ffn_exit.finish(hidden_states) + hidden_states = self.mlp(hidden_states) + return self.ffn_boundary.finish(hidden_states, forward_batch) class ArceeModel(nn.Module): diff --git a/python/sglang/srt/models/bailing_moe.py b/python/sglang/srt/models/bailing_moe.py index c3097375fa..016f35ff51 100644 --- a/python/sglang/srt/models/bailing_moe.py +++ b/python/sglang/srt/models/bailing_moe.py @@ -748,9 +748,8 @@ class BailingMoEBlock(nn.Module): forward_batch=forward_batch, ) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp(hidden_states, forward_batch) - hidden_states = ffn_exit.finish(hidden_states) + hidden_states = self.mlp(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch) return hidden_states diff --git a/python/sglang/srt/models/bailing_moe_linear.py b/python/sglang/srt/models/bailing_moe_linear.py index c68d67fd5c..6cbe96cbdd 100644 --- a/python/sglang/srt/models/bailing_moe_linear.py +++ b/python/sglang/srt/models/bailing_moe_linear.py @@ -883,9 +883,8 @@ class BailingMoELinearDecoderLayer(nn.Module): # logger.warning( # f"===={self.layer_id=}, 3 shape= {hidden_states.shape}, {residual.shape}" # ) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp(hidden_states) - hidden_states = ffn_exit.finish(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch) return hidden_states @staticmethod diff --git a/python/sglang/srt/models/bailing_moe_v3.py b/python/sglang/srt/models/bailing_moe_v3.py index b225e66f26..d5b3ad1f87 100644 --- a/python/sglang/srt/models/bailing_moe_v3.py +++ b/python/sglang/srt/models/bailing_moe_v3.py @@ -1196,17 +1196,16 @@ class BailingMoELinearDecoderLayer(nn.Module): hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - if not ( - is_dense_ffn_fully_dp() - and (not self.is_layer_sparse) - and hidden_states.shape[0] == 0 - ): - hidden_states = self.mlp( - hidden_states, - forward_batch=forward_batch, - ) - hidden_states = ffn_exit.finish(hidden_states) + if not ( + is_dense_ffn_fully_dp() + and (not self.is_layer_sparse) + and hidden_states.shape[0] == 0 + ): + hidden_states = self.mlp( + hidden_states, + forward_batch=forward_batch, + ) + hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch) return hidden_states diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 4264b1934f..e4f509c5ed 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -982,7 +982,6 @@ class DeepseekV2MoE(nn.Module): return dsv2_flashinfer_moe_dual_stream_graph( hidden_states, self.layer_id, - fwd.fuse_mlp_allreduce, fwd.mlp_reduce_scatter, ) elif ( @@ -3535,7 +3534,6 @@ class DeepseekV32ForCausalLM(DeepseekV2ForCausalLM): def dsv2_flashinfer_moe_dual_stream_graph( hidden_states: torch.Tensor, layer_id: int, - fuse_mlp_allreduce: bool, mlp_reduce_scatter: bool, ) -> torch.Tensor: forward_context = get_tc_piecewise_forward_context() @@ -3548,7 +3546,6 @@ def dsv2_flashinfer_moe_dual_stream_graph( # torch.compile. Carry graph-varying control state as scalar operands and # republish it for the nested MoE/linear consumers. with get_forward().scoped( - fuse_mlp_allreduce=fuse_mlp_allreduce, mlp_reduce_scatter=mlp_reduce_scatter, flashinfer_trtllm_bypass=True, lora_batch_layout=LoRABatchLayout.TP_GLOBAL, diff --git a/python/sglang/srt/models/dots3_common/modeling.py b/python/sglang/srt/models/dots3_common/modeling.py index a12a83c54e..4c3540b81b 100644 --- a/python/sglang/srt/models/dots3_common/modeling.py +++ b/python/sglang/srt/models/dots3_common/modeling.py @@ -1464,9 +1464,8 @@ class Dots3DecoderLayer(nn.Module): hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp(hidden_states, forward_batch) - return ffn_exit.finish(hidden_states) + hidden_states = self.mlp(hidden_states, forward_batch) + return self.ffn_boundary.finish(hidden_states, forward_batch) class Dots3Model(nn.Module): diff --git a/python/sglang/srt/models/ernie45_moe_vl.py b/python/sglang/srt/models/ernie45_moe_vl.py index 9261e7240c..17763aa703 100644 --- a/python/sglang/srt/models/ernie45_moe_vl.py +++ b/python/sglang/srt/models/ernie45_moe_vl.py @@ -472,12 +472,11 @@ class Ernie4_5_VLMoeDecoderLayer(nn.Module): # Fully Connected hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - if isinstance(self.mlp, Ernie4_5_VLMoeMoE): - hidden_states = self.mlp(hidden_states, visual_token_mask, **kwargs) - else: - hidden_states = self.mlp(hidden_states) - return ffn_exit.finish(hidden_states) + if isinstance(self.mlp, Ernie4_5_VLMoeMoE): + hidden_states = self.mlp(hidden_states, visual_token_mask, **kwargs) + else: + hidden_states = self.mlp(hidden_states) + return self.ffn_boundary.finish(hidden_states, forward_batch) # only used as text backbone for ernie4.5 vl diff --git a/python/sglang/srt/models/exaone4.py b/python/sglang/srt/models/exaone4.py index 2b013ff4de..230c01d750 100644 --- a/python/sglang/srt/models/exaone4.py +++ b/python/sglang/srt/models/exaone4.py @@ -313,9 +313,8 @@ class Exaone4DecoderLayer(nn.Module): # Fully Connected hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp(hidden_states) - return ffn_exit.finish(hidden_states) + hidden_states = self.mlp(hidden_states) + return self.ffn_boundary.finish(hidden_states, forward_batch) class Exaone4Model(nn.Module): diff --git a/python/sglang/srt/models/exaone_moe.py b/python/sglang/srt/models/exaone_moe.py index 038ab9f510..606bbd6919 100755 --- a/python/sglang/srt/models/exaone_moe.py +++ b/python/sglang/srt/models/exaone_moe.py @@ -533,9 +533,8 @@ class ExaoneMoEDecoderLayer(nn.Module): hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) # Fully Connected - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp(hidden_states, forward_batch) - return ffn_exit.finish(hidden_states) + hidden_states = self.mlp(hidden_states, forward_batch) + return self.ffn_boundary.finish(hidden_states, forward_batch) class ExaoneMoEModel(nn.Module): diff --git a/python/sglang/srt/models/falcon_h1.py b/python/sglang/srt/models/falcon_h1.py index 4921a469d6..03c5b67037 100644 --- a/python/sglang/srt/models/falcon_h1.py +++ b/python/sglang/srt/models/falcon_h1.py @@ -366,9 +366,8 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module): # Fully Connected hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.feed_forward(hidden_states, forward_batch) - return ffn_exit.finish(hidden_states) + hidden_states = self.feed_forward(hidden_states, forward_batch) + return self.ffn_boundary.finish(hidden_states, forward_batch) ALL_DECODER_LAYER_TYPES = { diff --git a/python/sglang/srt/models/gemma4_causal.py b/python/sglang/srt/models/gemma4_causal.py index 1cedbaec1d..38c46f43ad 100644 --- a/python/sglang/srt/models/gemma4_causal.py +++ b/python/sglang/srt/models/gemma4_causal.py @@ -719,11 +719,10 @@ class Gemma4DecoderLayer(nn.Module): hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) residual = residual_batch.written_residual(forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self._next_stream( - hidden_states, residual, per_layer_input, forward_batch - ) - return ffn_exit.finish(hidden_states) + hidden_states = self._next_stream( + hidden_states, residual, per_layer_input, forward_batch + ) + return self.ffn_boundary.finish(hidden_states, forward_batch) def _next_stream( self, diff --git a/python/sglang/srt/models/glm4.py b/python/sglang/srt/models/glm4.py index e73e01995e..e705e46e67 100644 --- a/python/sglang/srt/models/glm4.py +++ b/python/sglang/srt/models/glm4.py @@ -302,9 +302,8 @@ class Glm4DecoderLayer(nn.Module): # Fully Connected hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp(hidden_states) - return ffn_exit.finish(hidden_states) + hidden_states = self.mlp(hidden_states) + return self.ffn_boundary.finish(hidden_states, forward_batch) class Glm4Model(nn.Module): diff --git a/python/sglang/srt/models/glm4_moe.py b/python/sglang/srt/models/glm4_moe.py index 4a1ba3552c..f39c23c4bc 100644 --- a/python/sglang/srt/models/glm4_moe.py +++ b/python/sglang/srt/models/glm4_moe.py @@ -865,9 +865,8 @@ class Glm4MoeDecoderLayer(nn.Module): hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp(hidden_states, forward_batch) - hidden_states = ffn_exit.finish(hidden_states) + hidden_states = self.mlp(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch) return hidden_states diff --git a/python/sglang/srt/models/glm4_moe_lite.py b/python/sglang/srt/models/glm4_moe_lite.py index e9873fdb5d..27d7ef3e4d 100644 --- a/python/sglang/srt/models/glm4_moe_lite.py +++ b/python/sglang/srt/models/glm4_moe_lite.py @@ -556,9 +556,8 @@ class Glm4MoeLiteDecoderLayer(nn.Module): hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp(hidden_states, forward_batch) - hidden_states = ffn_exit.finish(hidden_states) + hidden_states = self.mlp(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch) return hidden_states diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index 23f74a7eb1..ed58952849 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -964,13 +964,13 @@ class Glm5NextDecoderLayer(nn.Module): else: _mlp_ctx = nullcontext() - with self.ffn_boundary.exit(forward_batch) as ffn_exit, _mlp_ctx: + with _mlp_ctx: hidden_states = self.mlp( hidden_states, forward_batch, gemm_output_zero_allocator, ) - hidden_states = ffn_exit.finish(hidden_states) + hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch) return (hidden_states, topk_indices) diff --git a/python/sglang/srt/models/glm_image_vl.py b/python/sglang/srt/models/glm_image_vl.py index 4d6ac85a2b..6b12e961b6 100644 --- a/python/sglang/srt/models/glm_image_vl.py +++ b/python/sglang/srt/models/glm_image_vl.py @@ -947,9 +947,8 @@ class GlmImageTextDecoderLayer(nn.Module): # Fully Connected hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp(hidden_states) - return ffn_exit.finish(hidden_states) + hidden_states = self.mlp(hidden_states) + return self.ffn_boundary.finish(hidden_states, forward_batch) class GlmImageTextModel(nn.Module): diff --git a/python/sglang/srt/models/gpt_oss.py b/python/sglang/srt/models/gpt_oss.py index b22857313a..62d37b61ed 100644 --- a/python/sglang/srt/models/gpt_oss.py +++ b/python/sglang/srt/models/gpt_oss.py @@ -629,9 +629,8 @@ class GptOssDecoderLayer(nn.Module): hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp(hidden_states, forward_batch) - hidden_states = ffn_exit.finish(hidden_states) + hidden_states = self.mlp(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch) return hidden_states diff --git a/python/sglang/srt/models/granitemoehybrid.py b/python/sglang/srt/models/granitemoehybrid.py index 9dc0f9a644..21db5163e7 100644 --- a/python/sglang/srt/models/granitemoehybrid.py +++ b/python/sglang/srt/models/granitemoehybrid.py @@ -155,21 +155,20 @@ class GraniteMoeHybridMambaDecoderLayer(nn.Module): hidden_states = self.attn_boundary.finish(output, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - if self.shared_mlp is None: - if self.block_sparse_moe is not None: - hidden_states = self.block_sparse_moe(hidden_states) - # else: skip + if self.shared_mlp is None: + if self.block_sparse_moe is not None: + hidden_states = self.block_sparse_moe(hidden_states) + # else: skip + else: + # create a copy since block_sparse_moe modifies in-place + if self.block_sparse_moe is not None: + moe_hidden_states = hidden_states.clone() + moe_hidden_states = self.block_sparse_moe(moe_hidden_states) + hidden_states = moe_hidden_states + self.shared_mlp(hidden_states) + del moe_hidden_states else: - # create a copy since block_sparse_moe modifies in-place - if self.block_sparse_moe is not None: - moe_hidden_states = hidden_states.clone() - moe_hidden_states = self.block_sparse_moe(moe_hidden_states) - hidden_states = moe_hidden_states + self.shared_mlp(hidden_states) - del moe_hidden_states - else: - hidden_states = self.shared_mlp(hidden_states) - return ffn_exit.finish(hidden_states) + hidden_states = self.shared_mlp(hidden_states) + return self.ffn_boundary.finish(hidden_states, forward_batch) class GraniteMoeHybridAttention(nn.Module): @@ -339,21 +338,20 @@ class GraniteMoeHybridAttentionDecoderLayer(nn.Module): ) hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - if self.shared_mlp is None: - if self.block_sparse_moe is not None: - hidden_states = self.block_sparse_moe(hidden_states) - # else: skip + if self.shared_mlp is None: + if self.block_sparse_moe is not None: + hidden_states = self.block_sparse_moe(hidden_states) + # else: skip + else: + # create a copy since block_sparse_moe modifies in-place + if self.block_sparse_moe is not None: + moe_hidden_states = hidden_states.clone() + moe_hidden_states = self.block_sparse_moe(moe_hidden_states) + hidden_states = moe_hidden_states + self.shared_mlp(hidden_states) + del moe_hidden_states else: - # create a copy since block_sparse_moe modifies in-place - if self.block_sparse_moe is not None: - moe_hidden_states = hidden_states.clone() - moe_hidden_states = self.block_sparse_moe(moe_hidden_states) - hidden_states = moe_hidden_states + self.shared_mlp(hidden_states) - del moe_hidden_states - else: - hidden_states = self.shared_mlp(hidden_states) - return ffn_exit.finish(hidden_states) + hidden_states = self.shared_mlp(hidden_states) + return self.ffn_boundary.finish(hidden_states, forward_batch) ALL_DECODER_LAYER_TYPES = { diff --git a/python/sglang/srt/models/interns2_mobius.py b/python/sglang/srt/models/interns2_mobius.py index c2a3f40e68..81fb63ce3b 100644 --- a/python/sglang/srt/models/interns2_mobius.py +++ b/python/sglang/srt/models/interns2_mobius.py @@ -429,11 +429,8 @@ class _InternS2MobiusDecoderMixin: ) -> torch.Tensor: hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self._forward_mobius_mlp( - hidden_states, forward_batch, meta_mlp - ) - hidden_states = ffn_exit.finish(hidden_states) + hidden_states = self._forward_mobius_mlp(hidden_states, forward_batch, meta_mlp) + hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch) return hidden_states diff --git a/python/sglang/srt/models/iquest_q1.py b/python/sglang/srt/models/iquest_q1.py index 8bc444f531..0d591c6577 100644 --- a/python/sglang/srt/models/iquest_q1.py +++ b/python/sglang/srt/models/iquest_q1.py @@ -532,9 +532,8 @@ class IQuestQ1DecoderLayer(nn.Module): attn_output = self.self_attn(positions, hidden_states, forward_batch) hidden_states = self.attn_boundary.finish(attn_output, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - mlp_output = self.mlp(hidden_states) - return ffn_exit.finish(mlp_output) + mlp_output = self.mlp(hidden_states) + return self.ffn_boundary.finish(mlp_output, forward_batch) class IQuestQ1Model(nn.Module): diff --git a/python/sglang/srt/models/kimi_linear.py b/python/sglang/srt/models/kimi_linear.py index ae45a3bb41..39691bdbea 100644 --- a/python/sglang/srt/models/kimi_linear.py +++ b/python/sglang/srt/models/kimi_linear.py @@ -652,9 +652,8 @@ class KimiDecoderLayer(nn.Module): # Fully Connected hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp(hidden_states) - return ffn_exit.finish(hidden_states) + hidden_states = self.mlp(hidden_states) + return self.ffn_boundary.finish(hidden_states, forward_batch) class KimiLinearModel(nn.Module): diff --git a/python/sglang/srt/models/laguna.py b/python/sglang/srt/models/laguna.py index 3c334bf939..5c38e95453 100644 --- a/python/sglang/srt/models/laguna.py +++ b/python/sglang/srt/models/laguna.py @@ -104,8 +104,6 @@ class LagunaMLP(nn.Module): ) -> torch.Tensor: gate_up, _ = self.gate_up_proj(x) x = self.act_fn(gate_up) - # RowParallelLinear honors ForwardFlags (fuse_mlp_allreduce / - # mlp_reduce_scatter) published by the decoder via scoped(). x, _ = self.down_proj(x) return x @@ -491,12 +489,11 @@ class LagunaDecoderLayer(nn.Module): hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp( - hidden_states, - forward_batch=forward_batch, - ) - hidden_states = ffn_exit.finish(hidden_states) + hidden_states = self.mlp( + hidden_states, + forward_batch=forward_batch, + ) + hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch) return hidden_states diff --git a/python/sglang/srt/models/llada2.py b/python/sglang/srt/models/llada2.py index ab387cf67c..641921f034 100644 --- a/python/sglang/srt/models/llada2.py +++ b/python/sglang/srt/models/llada2.py @@ -675,9 +675,8 @@ class LLaDA2MoeBlock(nn.Module): forward_batch=forward_batch, ) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp(hidden_states, forward_batch) - hidden_states = ffn_exit.finish(hidden_states) + hidden_states = self.mlp(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch) return hidden_states diff --git a/python/sglang/srt/models/llama.py b/python/sglang/srt/models/llama.py index 8e74c9751e..5ab4729efe 100644 --- a/python/sglang/srt/models/llama.py +++ b/python/sglang/srt/models/llama.py @@ -394,9 +394,8 @@ class LlamaDecoderLayer(nn.Module): # Fully Connected hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp(hidden_states) - return ffn_exit.finish(hidden_states) + hidden_states = self.mlp(hidden_states) + return self.ffn_boundary.finish(hidden_states, forward_batch) class LlamaModel(nn.Module): diff --git a/python/sglang/srt/models/llama4.py b/python/sglang/srt/models/llama4.py index 1d158e8d38..6549923f60 100644 --- a/python/sglang/srt/models/llama4.py +++ b/python/sglang/srt/models/llama4.py @@ -472,9 +472,8 @@ class Llama4DecoderLayer(nn.Module): hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) # Fully Connected - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.feed_forward(hidden_states, forward_batch) - hidden_states = ffn_exit.finish(hidden_states) + hidden_states = self.feed_forward(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch) return hidden_states diff --git a/python/sglang/srt/models/longcat_flash.py b/python/sglang/srt/models/longcat_flash.py index 46e2974a48..b385738767 100644 --- a/python/sglang/srt/models/longcat_flash.py +++ b/python/sglang/srt/models/longcat_flash.py @@ -504,9 +504,8 @@ class LongcatFlashDecoderLayer(nn.Module): hidden_states = self.first_ffn_boundary.branch_input( self.moe_boundary, hidden_states, forward_batch ) - with self.first_ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlps[0](hidden_states) - hidden_states = ffn_exit.finish(hidden_states) + hidden_states = self.mlps[0](hidden_states) + hidden_states = self.first_ffn_boundary.finish(hidden_states, forward_batch) # second_attn_boundary hidden_states = self.second_attn_boundary.prepare(hidden_states, forward_batch) @@ -526,9 +525,8 @@ class LongcatFlashDecoderLayer(nn.Module): # second_mlp hidden_states = self.second_attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.second_ffn_boundary.prepare(hidden_states, forward_batch) - with self.second_ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlps[1](hidden_states) - hidden_states = ffn_exit.finish(hidden_states) + hidden_states = self.mlps[1](hidden_states) + hidden_states = self.second_ffn_boundary.finish(hidden_states, forward_batch) return hidden_states, prev_topk_indices diff --git a/python/sglang/srt/models/longcat_flash_nextn.py b/python/sglang/srt/models/longcat_flash_nextn.py index cb501bb3bc..112d28fbe5 100644 --- a/python/sglang/srt/models/longcat_flash_nextn.py +++ b/python/sglang/srt/models/longcat_flash_nextn.py @@ -191,9 +191,8 @@ class LongcatFlashDenseDecoderLayer(nn.Module): hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp(hidden_states) - hidden_states = ffn_exit.finish(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch) return hidden_states diff --git a/python/sglang/srt/models/mimo_v2.py b/python/sglang/srt/models/mimo_v2.py index 6ad1a23885..47e650259b 100644 --- a/python/sglang/srt/models/mimo_v2.py +++ b/python/sglang/srt/models/mimo_v2.py @@ -898,9 +898,8 @@ class MiMoV2DecoderLayer(nn.Module): hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp(hidden_states, forward_batch) - hidden_states = ffn_exit.finish(hidden_states) + hidden_states = self.mlp(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch) return hidden_states diff --git a/python/sglang/srt/models/mimo_v2_nextn.py b/python/sglang/srt/models/mimo_v2_nextn.py index d702604537..a6a76a2c48 100644 --- a/python/sglang/srt/models/mimo_v2_nextn.py +++ b/python/sglang/srt/models/mimo_v2_nextn.py @@ -155,12 +155,9 @@ class MiMoV2MTPLayer(nn.Module): hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with ( - self.ffn_boundary.exit(forward_batch) as ffn_exit, - get_global_expert_distribution_recorder().disable_this_region(), - ): + with get_global_expert_distribution_recorder().disable_this_region(): hidden_states = self.mlp(hidden_states) - hidden_states = ffn_exit.finish(hidden_states) + hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch) return hidden_states diff --git a/python/sglang/srt/models/minimax_m2.py b/python/sglang/srt/models/minimax_m2.py index b2e1f3f06e..b25d7f89d1 100644 --- a/python/sglang/srt/models/minimax_m2.py +++ b/python/sglang/srt/models/minimax_m2.py @@ -912,9 +912,8 @@ class MiniMaxM2DecoderLayer(nn.Module): hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.block_sparse_moe(hidden_states, forward_batch) - hidden_states = ffn_exit.finish(hidden_states) + hidden_states = self.block_sparse_moe(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch) return hidden_states diff --git a/python/sglang/srt/models/minimax_m3.py b/python/sglang/srt/models/minimax_m3.py index db7de34afd..ec91053004 100644 --- a/python/sglang/srt/models/minimax_m3.py +++ b/python/sglang/srt/models/minimax_m3.py @@ -1374,10 +1374,9 @@ class MiniMaxM3DecoderLayer(nn.Module): hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - if self.is_layer_sparse or hidden_states.shape[0] != 0: - hidden_states = self.mlp(hidden_states, forward_batch=forward_batch) - return ffn_exit.finish(hidden_states) + if self.is_layer_sparse or hidden_states.shape[0] != 0: + hidden_states = self.mlp(hidden_states, forward_batch=forward_batch) + return self.ffn_boundary.finish(hidden_states, forward_batch) class MiniMaxM3Model(nn.Module): diff --git a/python/sglang/srt/models/mixtral.py b/python/sglang/srt/models/mixtral.py index 90b5460c22..271c7634e3 100644 --- a/python/sglang/srt/models/mixtral.py +++ b/python/sglang/srt/models/mixtral.py @@ -264,9 +264,8 @@ class MixtralDecoderLayer(nn.Module): # Fully Connected hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.block_sparse_moe(hidden_states) - return ffn_exit.finish(hidden_states) + hidden_states = self.block_sparse_moe(hidden_states) + return self.ffn_boundary.finish(hidden_states, forward_batch) class MixtralModel(nn.Module): diff --git a/python/sglang/srt/models/moss_vl.py b/python/sglang/srt/models/moss_vl.py index d6f3005049..56c89c889a 100644 --- a/python/sglang/srt/models/moss_vl.py +++ b/python/sglang/srt/models/moss_vl.py @@ -1045,9 +1045,8 @@ class MossVLSelfAttentionDecoderLayer(nn.Module): # MLP hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp(hidden_states) - hidden_states = ffn_exit.finish(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch) return hidden_states diff --git a/python/sglang/srt/models/nanbeige.py b/python/sglang/srt/models/nanbeige.py index c507500d79..f8e522da5a 100644 --- a/python/sglang/srt/models/nanbeige.py +++ b/python/sglang/srt/models/nanbeige.py @@ -262,9 +262,8 @@ class NanbeigeDecoderLayer(nn.Module): # Fully Connected hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp(hidden_states) - return ffn_exit.finish(hidden_states) + hidden_states = self.mlp(hidden_states) + return self.ffn_boundary.finish(hidden_states, forward_batch) class NanbeigeModel(nn.Module): diff --git a/python/sglang/srt/models/nemotron_h.py b/python/sglang/srt/models/nemotron_h.py index f63ceafa0e..6525e6f7d3 100644 --- a/python/sglang/srt/models/nemotron_h.py +++ b/python/sglang/srt/models/nemotron_h.py @@ -384,9 +384,8 @@ class NemotronHMLPLikeDecoderLayer(nn.Module): forward_batch: ForwardBatch, ): hidden_states = self.boundary.prepare(hidden_states, forward_batch) - with self.boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mixer.forward(hidden_states) - return ffn_exit.finish(hidden_states) + hidden_states = self.mixer.forward(hidden_states) + return self.boundary.finish(hidden_states, forward_batch) class NemotronHMLPDecoderLayer(NemotronHMLPLikeDecoderLayer): @@ -467,9 +466,8 @@ class NemotronHAttnLikeDecoderLayer(nn.Module): if forward_batch.forward_mode.is_idle(): return hidden_states - with self.boundary.exit(forward_batch) as mixer_exit: - hidden_states = self._forward_mixer(hidden_states, forward_batch) - return mixer_exit.finish(hidden_states) + hidden_states = self._forward_mixer(hidden_states, forward_batch) + return self.boundary.finish(hidden_states, forward_batch) class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer): diff --git a/python/sglang/srt/models/nemotron_nas.py b/python/sglang/srt/models/nemotron_nas.py index 0f41989d8c..de20b5766c 100644 --- a/python/sglang/srt/models/nemotron_nas.py +++ b/python/sglang/srt/models/nemotron_nas.py @@ -194,9 +194,8 @@ class DeciLMDecoderLayer(nn.Module): # Fully Connected if not self._is_no_op_ffn: hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp(hidden_states) - hidden_states = ffn_exit.finish(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch) return hidden_states diff --git a/python/sglang/srt/models/qwen2.py b/python/sglang/srt/models/qwen2.py index ff0241a528..a5a1c128ce 100644 --- a/python/sglang/srt/models/qwen2.py +++ b/python/sglang/srt/models/qwen2.py @@ -325,9 +325,8 @@ class Qwen2DecoderLayer(nn.Module): # Fully Connected hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp(hidden_states) - return ffn_exit.finish(hidden_states) + hidden_states = self.mlp(hidden_states) + return self.ffn_boundary.finish(hidden_states, forward_batch) class Qwen2Model(nn.Module): diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index 8a39f5a962..9e60ee3216 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -1095,9 +1095,8 @@ class Qwen2MoeDecoderLayer(nn.Module): hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp(hidden_states, forward_batch) - hidden_states = ffn_exit.finish(hidden_states) + hidden_states = self.mlp(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch) return hidden_states diff --git a/python/sglang/srt/models/qwen3.py b/python/sglang/srt/models/qwen3.py index 2adfc5f746..7e1ac5b2f3 100644 --- a/python/sglang/srt/models/qwen3.py +++ b/python/sglang/srt/models/qwen3.py @@ -423,11 +423,10 @@ class Qwen3DecoderLayer(nn.Module): else None ), ) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp(hidden_states, forward_batch=forward_batch) + hidden_states = self.mlp(hidden_states, forward_batch=forward_batch) if _is_npu and get_cmo_stream(): wait_cmo_stream() - hidden_states = ffn_exit.finish(hidden_states) + hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch) return hidden_states diff --git a/python/sglang/srt/models/qwen3_moe.py b/python/sglang/srt/models/qwen3_moe.py index 2c76111952..dc22863527 100644 --- a/python/sglang/srt/models/qwen3_moe.py +++ b/python/sglang/srt/models/qwen3_moe.py @@ -904,9 +904,8 @@ class Qwen3MoeDecoderLayer(nn.Module): hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp(hidden_states, forward_batch) - hidden_states = ffn_exit.finish(hidden_states) + hidden_states = self.mlp(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch) return hidden_states diff --git a/python/sglang/srt/models/qwen3_next.py b/python/sglang/srt/models/qwen3_next.py index a570d0c66b..44b685d45c 100644 --- a/python/sglang/srt/models/qwen3_next.py +++ b/python/sglang/srt/models/qwen3_next.py @@ -479,15 +479,14 @@ def _apply_qwen3_next_mlp( ) -> torch.Tensor: hidden_states = layer.attn_boundary.finish(hidden_states, forward_batch) hidden_states = layer.ffn_boundary.prepare(hidden_states, forward_batch) - with layer.ffn_boundary.exit(forward_batch) as ffn_exit: - if isinstance(layer.mlp, Qwen2MoeSparseMoeBlock): - hidden_states = layer.mlp( - hidden_states, - forward_batch=forward_batch, - ) - else: - hidden_states = layer.mlp(hidden_states) - hidden_states = ffn_exit.finish(hidden_states) + if isinstance(layer.mlp, Qwen2MoeSparseMoeBlock): + hidden_states = layer.mlp( + hidden_states, + forward_batch=forward_batch, + ) + else: + hidden_states = layer.mlp(hidden_states) + hidden_states = layer.ffn_boundary.finish(hidden_states, forward_batch) return hidden_states diff --git a/python/sglang/srt/models/sarvam_moe.py b/python/sglang/srt/models/sarvam_moe.py index 5bbfbcf45f..7f2c871999 100644 --- a/python/sglang/srt/models/sarvam_moe.py +++ b/python/sglang/srt/models/sarvam_moe.py @@ -1080,9 +1080,8 @@ class SarvamMoEMLADecoderLayer(nn.Module): ) hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp(hidden_states, forward_batch) - hidden_states = ffn_exit.finish(hidden_states) + hidden_states = self.mlp(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch) return hidden_states diff --git a/python/sglang/srt/models/sdar.py b/python/sglang/srt/models/sdar.py index 1934110f30..88f32d39f8 100644 --- a/python/sglang/srt/models/sdar.py +++ b/python/sglang/srt/models/sdar.py @@ -325,9 +325,8 @@ class SDARBlock(nn.Module): hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp(hidden_states) - hidden_states = ffn_exit.finish(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch) return hidden_states diff --git a/python/sglang/srt/models/sdar_moe.py b/python/sglang/srt/models/sdar_moe.py index fd6bfff8f7..43ca2b0d2d 100644 --- a/python/sglang/srt/models/sdar_moe.py +++ b/python/sglang/srt/models/sdar_moe.py @@ -384,12 +384,11 @@ class SDARMoeBlock(nn.Module): hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp( - hidden_states, - forward_batch=forward_batch, - ) - hidden_states = ffn_exit.finish(hidden_states) + hidden_states = self.mlp( + hidden_states, + forward_batch=forward_batch, + ) + hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch) return hidden_states diff --git a/python/sglang/srt/models/spark2_5.py b/python/sglang/srt/models/spark2_5.py index bf9ca4ae33..62c6296a85 100644 --- a/python/sglang/srt/models/spark2_5.py +++ b/python/sglang/srt/models/spark2_5.py @@ -296,9 +296,8 @@ class Spark2_5DecoderLayer(nn.Module): ) hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp(hidden_states) - return ffn_exit.finish(hidden_states) + hidden_states = self.mlp(hidden_states) + return self.ffn_boundary.finish(hidden_states, forward_batch) class Spark2_5Model(nn.Module): diff --git a/python/sglang/srt/models/step3_vl.py b/python/sglang/srt/models/step3_vl.py index f3ad5edd12..e752313e16 100644 --- a/python/sglang/srt/models/step3_vl.py +++ b/python/sglang/srt/models/step3_vl.py @@ -417,12 +417,11 @@ class Step3TextDecoderLayer(nn.Module): hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - if self.use_moe: - hidden_states = self.moe_mlp_forward(hidden_states) - else: - hidden_states = self.mlp(hidden_states) - hidden_states = ffn_exit.finish(hidden_states) + if self.use_moe: + hidden_states = self.moe_mlp_forward(hidden_states) + else: + hidden_states = self.mlp(hidden_states) + hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch) return hidden_states diff --git a/python/sglang/srt/models/step3p5.py b/python/sglang/srt/models/step3p5.py index 505194bbd1..3195fa26ec 100644 --- a/python/sglang/srt/models/step3p5.py +++ b/python/sglang/srt/models/step3p5.py @@ -565,16 +565,14 @@ class Step3p5DecoderLayer(nn.Module): hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) if self.use_moe: - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - share_output = self.share_expert(hidden_states) - hidden_states = self.moe( - hidden_states, forward_batch, shared_output=share_output - ) - return ffn_exit.finish(hidden_states) + share_output = self.share_expert(hidden_states) + hidden_states = self.moe( + hidden_states, forward_batch, shared_output=share_output + ) + return self.ffn_boundary.finish(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp(hidden_states) - return ffn_exit.finish(hidden_states) + hidden_states = self.mlp(hidden_states) + return self.ffn_boundary.finish(hidden_states, forward_batch) class Step3p5Model(nn.Module): diff --git a/python/sglang/srt/models/xllm.py b/python/sglang/srt/models/xllm.py index 928e59c9fc..a74487809e 100644 --- a/python/sglang/srt/models/xllm.py +++ b/python/sglang/srt/models/xllm.py @@ -1558,12 +1558,11 @@ class XllmDecoderLayer(nn.Module): hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - if isinstance(self.mlp, XllmMLP): - hidden_states = self.mlp(hidden_states) - else: - hidden_states = self.mlp(hidden_states, forward_batch) - hidden_states = ffn_exit.finish(hidden_states) + if isinstance(self.mlp, XllmMLP): + hidden_states = self.mlp(hidden_states) + else: + hidden_states = self.mlp(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch) return hidden_states diff --git a/python/sglang/srt/models/zaya.py b/python/sglang/srt/models/zaya.py index f6f1111d69..f62993e093 100644 --- a/python/sglang/srt/models/zaya.py +++ b/python/sglang/srt/models/zaya.py @@ -1397,11 +1397,10 @@ class ZayaDecoderMLPLayer(nn.Module): prev_router_hidden_states: Optional[torch.Tensor] = None, ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states, prev_router_hidden_states = self.zaya_block( - hidden_states, prev_router_hidden_states - ) - hidden_states = ffn_exit.finish(hidden_states) + hidden_states, prev_router_hidden_states = self.zaya_block( + hidden_states, prev_router_hidden_states + ) + hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch) return hidden_states, prev_router_hidden_states diff --git a/python/sglang/srt/runtime_context.py b/python/sglang/srt/runtime_context.py index d58c8ccb3d..93ece26669 100644 --- a/python/sglang/srt/runtime_context.py +++ b/python/sglang/srt/runtime_context.py @@ -670,11 +670,9 @@ class ForwardFlags: "is_extend_in_batch": False, # Per-layer MLP collective control (set by decoder via scoped() # around the MLP / MoE / hybrid mixer call). - # fuse_mlp_allreduce: next residual+LN absorbs the post-MLP all-reduce. # mlp_reduce_scatter: postprocess will reduce-scatter (skip MLP AR). # flashinfer_trtllm_bypass: deepseek dual-stream graph topk bypass. # defer_moe_finalize: next layer's fused collective absorbs the finalize. - "fuse_mlp_allreduce": False, "mlp_reduce_scatter": False, "flashinfer_trtllm_bypass": False, "defer_moe_finalize": False, @@ -694,7 +692,6 @@ class ForwardFlags: "attn_input_scattered", "attn_inputs", "is_extend_in_batch", - "fuse_mlp_allreduce", "mlp_reduce_scatter", "flashinfer_trtllm_bypass", "defer_moe_finalize", diff --git a/test/registered/unit/layer_boundary/test_aux_capture_deferred_allreduce.py b/test/registered/unit/layer_boundary/test_aux_capture_deferred_allreduce.py index 2151712501..f4df42625f 100644 --- a/test/registered/unit/layer_boundary/test_aux_capture_deferred_allreduce.py +++ b/test/registered/unit/layer_boundary/test_aux_capture_deferred_allreduce.py @@ -98,8 +98,8 @@ class DeferringLayer(nn.Module): self.layer_communicator.paths[BatchVariant.INPUT_SCATTERED] = None self.layer_communicator.paths[BatchVariant.CONTEXT_PARALLEL] = None self.layer_communicator.output.ffn_reduction_group = lambda forward_batch: GROUP - self.layer_communicator.output._skips_sum_for_reduce_scatter = ( - lambda batch, dp_step: False + self.layer_communicator.output._sum_in_reduce_scatter = lambda batch, dp_step: ( + False ) self.layer_communicator.output._complete_now = lambda hidden, residual, **_: ( all_reduce(hidden), diff --git a/test/registered/unit/layer_boundary/test_boundary_output_contracts.py b/test/registered/unit/layer_boundary/test_boundary_output_contracts.py index b8463e2a37..596c7ec365 100644 --- a/test/registered/unit/layer_boundary/test_boundary_output_contracts.py +++ b/test/registered/unit/layer_boundary/test_boundary_output_contracts.py @@ -1,7 +1,6 @@ """Numerical and interface coverage for boundary/graph integration regressions.""" import unittest -from contextlib import nullcontext from types import SimpleNamespace from unittest.mock import Mock, patch @@ -10,7 +9,6 @@ import torch from sglang.srt.layers.layer_boundary import ( PLAIN_ADD, - ProducerReduction, declare_attn, declare_ffn, ) @@ -200,12 +198,6 @@ class TestBoundaryIntegrations(unittest.TestCase): enabled and (lora or shared), ) - def test_unsupported_producer_contracts_fail_at_declaration(self): - with self.assertRaises(ValueError): - declare_ffn(reduction=ProducerReduction.ALWAYS_PARTIAL) - with self.assertRaises(ValueError): - declare_attn(reduction=ProducerReduction.TAIL_AFTER_SUM) - def test_ffn_writing_the_next_stream_hands_on_a_complete_output(self): # It has each part's sum completed (sum_part) before building the # stream from the parts, so its exit owes no sum and never defers one. @@ -261,14 +253,13 @@ class TestBoundaryIntegrations(unittest.TestCase): capture(hidden) return hidden - output = SimpleNamespace(finish=lambda hidden: hidden) layer = SimpleNamespace( attn_boundary=SimpleNamespace( prepare=prepare, finish=lambda hidden, fb: hidden ), ffn_boundary=SimpleNamespace( prepare=lambda hidden, fb, **kw: hidden, - exit=lambda fb: nullcontext(output), + finish=lambda hidden, fb: hidden, ), self_attn=lambda **kw: kw["hidden_states"], mlp=lambda hidden, **kw: hidden, diff --git a/test/registered/unit/layer_boundary/test_boundary_output_policy.py b/test/registered/unit/layer_boundary/test_boundary_output_policy.py index 0d6bc2a68d..defe9735dd 100644 --- a/test/registered/unit/layer_boundary/test_boundary_output_policy.py +++ b/test/registered/unit/layer_boundary/test_boundary_output_policy.py @@ -213,9 +213,7 @@ class TestBoundaryOutputPolicy(unittest.TestCase): ), ): for skip in (False, True): - with get_forward().scoped( - fuse_mlp_allreduce=False, mlp_reduce_scatter=skip - ): + with get_forward().scoped(mlp_reduce_scatter=skip): self.assertEqual( moe_utils.should_skip_post_experts_all_reduce(is_tp_path=True), skip, diff --git a/test/registered/unit/layer_boundary/test_cutedsl_ar_fusion.py b/test/registered/unit/layer_boundary/test_cutedsl_ar_fusion.py index 8ba1fda633..1d447722ba 100644 --- a/test/registered/unit/layer_boundary/test_cutedsl_ar_fusion.py +++ b/test/registered/unit/layer_boundary/test_cutedsl_ar_fusion.py @@ -481,7 +481,7 @@ def test_dual_stream_op_pins_the_deferral_off_under_a_deferring_caller(): return_value=SimpleNamespace(moe_fusions={0: fusion}), ), ): - out = op.redispatch(cuda_key, torch.zeros(4, 8), 0, True, False) + out = op.redispatch(cuda_key, torch.zeros(4, 8), 0, False) assert get_forward().defer_moe_finalize is True assert fusion.seen_defer is False diff --git a/test/registered/unit/layer_boundary/test_declared_attention_cp.py b/test/registered/unit/layer_boundary/test_declared_attention_cp.py index 6e7ca879f1..6f42cc60f8 100644 --- a/test/registered/unit/layer_boundary/test_declared_attention_cp.py +++ b/test/registered/unit/layer_boundary/test_declared_attention_cp.py @@ -44,7 +44,6 @@ def layernorm(hidden_states, residual=None): class Flags: def __init__(self): - self.fuse_mlp_allreduce = False self.mlp_reduce_scatter = False self.defer_moe_finalize = False self.sp_active = False diff --git a/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py b/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py index 9eed39c8b4..f3c5b0c77a 100644 --- a/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py +++ b/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py @@ -368,7 +368,7 @@ class TestMhcOnTheDeclarations(CustomTestCase): max_len, communicator.ffn.plan.output.plan.path_for(max_len) ) self.assertTrue( - communicator.ffn.plan.output._skips_sum_for_reduce_scatter( + communicator.ffn.plan.output._sum_in_reduce_scatter( communicator.ffn.plan.output.plan.path_for(max_len), step ) ) @@ -1124,7 +1124,7 @@ class TestPrefillCP(CustomTestCase): patch_communicator("_batch_shards_over_cp", lambda fb: shards), ): self.assertIs( - communicator.ffn.plan.output._skips_sum_for_reduce_scatter( + communicator.ffn.plan.output._sum_in_reduce_scatter( communicator.ffn.plan.output.plan.path_for( SimpleNamespace(forward_mode=ForwardMode.DECODE) ), @@ -1590,7 +1590,6 @@ class Flags: """The per-forward flags an FFN exit publishes, one set per rank.""" def __init__(self): - self.fuse_mlp_allreduce = False self.mlp_reduce_scatter = False self.defer_moe_finalize = False self.sp_active = False diff --git a/test/registered/unit/layer_boundary/test_ffn_exit.py b/test/registered/unit/layer_boundary/test_ffn_exit.py index e3e794e1f0..1728c90932 100644 --- a/test/registered/unit/layer_boundary/test_ffn_exit.py +++ b/test/registered/unit/layer_boundary/test_ffn_exit.py @@ -89,9 +89,7 @@ def make_communicator( communicator.output._defers_sum = MagicMock(return_value=fuse) else: communicator.output._defers_sum = MagicMock(return_value=False) - communicator.output._skips_sum_for_reduce_scatter = MagicMock( - return_value=reduce_scatter - ) + communicator.output._sum_in_reduce_scatter = MagicMock(return_value=reduce_scatter) communicator.terminal = False communicator.paths[BatchVariant.SEQUENCE_PARALLEL] = None communicator.paths[BatchVariant.INPUT_SCATTERED] = None @@ -109,10 +107,9 @@ def make_communicator( def published_flags(): - """The skip flags the FFN sees; the boundary completes the sum, so the FFN + """The skip flag the FFN sees; the boundary completes the sum, so the FFN never leaves it out.""" - forward = get_forward() - return forward.fuse_mlp_allreduce, forward.mlp_reduce_scatter + return (get_forward().mlp_reduce_scatter,) class TestFfnExit(CustomTestCase): @@ -150,7 +147,7 @@ class TestFfnExit(CustomTestCase): scatters_to_local_tokens=True, ) seen, (hidden_states, residual) = self.run_exit(communicator) - self.assertEqual(seen, (False, False)) + self.assertEqual(seen, (False,)) self.assertIsInstance(hidden_states, UnreducedOutput) bound = hidden_states.reduce_to_dp_local self.assertIs(bound.func, transport_ops.to_dp_local) @@ -179,7 +176,7 @@ class TestFfnExit(CustomTestCase): fuse=False, reduce_scatter=reduce_scatter ) seen, (hidden_states, residual) = self.run_exit(communicator) - self.assertEqual(seen, (False, False)) + self.assertEqual(seen, (False,)) self.assertNotIsInstance(hidden_states, UnreducedOutput) communicator.output._complete_now.assert_called_once() # A reduce-scatter on the way back completes the sum; otherwise @@ -197,7 +194,7 @@ class TestFfnExit(CustomTestCase): ) seen, (hidden_states, residual) = self.run_exit(communicator) - self.assertEqual(seen, (False, False)) + self.assertEqual(seen, (False,)) communicator.output._defers_sum.assert_not_called() communicator.output._complete_now.assert_called_once() torch.testing.assert_close(hidden_states, self.hidden_states * 2 + 1) @@ -216,7 +213,7 @@ class TestFfnExit(CustomTestCase): decide.return_value = not fuse hidden_states = self.hidden_states * 2 hidden_states, _ = finish_exit(ffn_exit, hidden_states, self.residual) - self.assertEqual(seen, (False, False)) + self.assertEqual(seen, (False,)) self.assertEqual(isinstance(hidden_states, UnreducedOutput), fuse) self.assertEqual(communicator.output._complete_now.called, not fuse) @@ -253,7 +250,7 @@ class TestFfnExit(CustomTestCase): self.forward_batch, stream=ResidualStream() ) as ffn_exit: seen = published_flags() + (get_forward().defer_moe_finalize,) - self.assertEqual(seen, (False, False, True)) + self.assertEqual(seen, (False, True)) self.assertEqual( finish_exit(ffn_exit, handoff, self.residual), (handoff, self.residual) ) @@ -354,8 +351,8 @@ class TestSelectFfnCompletion(CustomTestCase): returns_over_dp=scatters, ) communicator.output._defers_sum = lambda forward_batch, steps, **_: fuse - communicator.output._skips_sum_for_reduce_scatter = ( - lambda forward_batch, dp_step: not fuse + communicator.output._sum_in_reduce_scatter = lambda forward_batch, dp_step: ( + not fuse ) communicator.output._complete_now = lambda h, r, **_: ("now", r) self.group = make_group() diff --git a/test/registered/unit/layer_boundary/test_independent_stage_construction.py b/test/registered/unit/layer_boundary/test_independent_stage_construction.py index aec92ca795..0cb7d2764c 100644 --- a/test/registered/unit/layer_boundary/test_independent_stage_construction.py +++ b/test/registered/unit/layer_boundary/test_independent_stage_construction.py @@ -193,24 +193,16 @@ class TestIndependentStageConstruction(CustomTestCase): ) self.assertEqual(edge.incoming.residual, source_input.residual_to) - def test_a_replicated_tail_stays_in_compute_without_a_fusion_object(self): - for local_tail in (False, True): - with fixture.planning(fixture.parallel_of(attn_dp=1, attn_tp=2)): - boundary = make_ffn_stage( - declaration=declare_ffn( - previous=declare_attn(), - sparse=True, - reduction=ProducerReduction.TAIL_AFTER_SUM - if local_tail - else ProducerReduction.EXIT_SCOPED, - ), - norm=fixture.Norm(), - ) - self.assertIsNone(boundary.plan.fusions) - produced = boundary.plan.paths[BatchVariant.ORDINARY].output - self.assertEqual(produced.may_defer_to_next, not local_tail) - # A local compute tail does not change the existing RS alternative. - self.assertTrue(produced.may_reduce_scatter) + def test_an_ffn_without_a_fusion_object_may_defer_or_reduce_scatter(self): + with fixture.planning(fixture.parallel_of(attn_dp=1, attn_tp=2)): + boundary = make_ffn_stage( + declaration=declare_ffn(previous=declare_attn(), sparse=True), + norm=fixture.Norm(), + ) + self.assertIsNone(boundary.plan.fusions) + produced = boundary.plan.paths[BatchVariant.ORDINARY].output + self.assertTrue(produced.may_defer_to_next) + self.assertTrue(produced.may_reduce_scatter) def test_mixer_successor_comes_from_the_local_sequence(self): for following in ( diff --git a/test/registered/unit/layer_boundary/test_moe_output_reduction.py b/test/registered/unit/layer_boundary/test_moe_output_reduction.py index b2e9c78070..433f511a7d 100644 --- a/test/registered/unit/layer_boundary/test_moe_output_reduction.py +++ b/test/registered/unit/layer_boundary/test_moe_output_reduction.py @@ -89,7 +89,6 @@ class TestReduceMoeOutput(CustomTestCase): def test_a_later_step_owns_the_sum(self): for flags, config in ( - ({"fuse_mlp_allreduce": True}, {}), ({"mlp_reduce_scatter": True}, {}), ({"mlp_reduce_scatter": True}, {"reduce_scatterv": True}), ): @@ -136,7 +135,6 @@ class TestReplicatedMoeOutput(CustomTestCase): def test_a_later_sum_counts_it_once(self): for flags, config in ( - ({"fuse_mlp_allreduce": True}, {}), ({"mlp_reduce_scatter": True}, {}), ({"mlp_reduce_scatter": True}, {"reduce_scatterv": True}), ): @@ -158,7 +156,7 @@ class TestReplicatedMoeOutput(CustomTestCase): def test_a_single_rank_adds_it(self): with ( moe_config(tp_size=1, tp_rank=0), - get_forward().scoped(fuse_mlp_allreduce=True), + get_forward().scoped(mlp_reduce_scatter=True), ): self.assertTrue(should_add_replicated_moe_output()) @@ -243,9 +241,7 @@ class TestModelsWithExplicitDpCompletion(CustomTestCase): ) with ( moe_config(), - get_forward().scoped( - fuse_mlp_allreduce=False, mlp_reduce_scatter=False - ), + get_forward().scoped(mlp_reduce_scatter=False), patch( "sglang.srt.distributed.communication_op.tensor_model_parallel_all_reduce", side_effect=lambda value: trace.append("AR") or value * 2, diff --git a/test/registered/unit/layer_boundary/test_reduction_fusion.py b/test/registered/unit/layer_boundary/test_reduction_fusion.py index 24bf97e0d9..8e6ce236f9 100644 --- a/test/registered/unit/layer_boundary/test_reduction_fusion.py +++ b/test/registered/unit/layer_boundary/test_reduction_fusion.py @@ -334,7 +334,7 @@ class TestFuseMlpAllReduceGate(CustomTestCase): self.assertTrue(self._should_fuse(moe_ep_size=4, moe_tp_size=1)) def test_a_layer_gathered_over_moe_cp_does_not_fuse(self): - # Fusion skips finish_complete_output, which holds the CP scatter; a dense + # Fusion skips the exit's completion, which holds the CP scatter; a dense # layer gathered over the MoE-CP group (moe_dp_size == attn_cp_size) is # not caught by the is_enable_moe_cp_allgather gate. self.assertFalse( @@ -472,7 +472,7 @@ class TestDeferFfnReduction(CustomTestCase): return communicator.output._defers_sum( forward_batch, communicator.output.plan.path_for(forward_batch), - mlp_reduce_scatter=reduce_scatter, + sum_in_reduce_scatter=reduce_scatter, dp_step=step, ) diff --git a/test/registered/unit/models/test_deepseek_v4_amd_tp4.py b/test/registered/unit/models/test_deepseek_v4_amd_tp4.py index a0846df893..46ec3658b7 100644 --- a/test/registered/unit/models/test_deepseek_v4_amd_tp4.py +++ b/test/registered/unit/models/test_deepseek_v4_amd_tp4.py @@ -277,9 +277,7 @@ def test_moe_model_handoff(group, rows, dual, defer, shared_tp1): return hidden, next_pre with ( - get_forward().scoped( - sp_active=False, fuse_mlp_allreduce=False, flashinfer_trtllm_bypass=False - ), + get_forward().scoped(sp_active=False, flashinfer_trtllm_bypass=False), patch( "sglang.srt.models.deepseek_v2.post_experts_all_reduce", side_effect=group.all_reduce, @@ -316,11 +314,11 @@ def test_moe_model_handoff(group, rows, dual, defer, shared_tp1): torch.testing.assert_close(unfused[1], fused[1], atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("flag", ["mlp_reduce_scatter", "fuse_mlp_allreduce"]) +@pytest.mark.parametrize("flag", ["mlp_reduce_scatter"]) @pytest.mark.parametrize("dual", [False, True], ids=["normal", "dual-stream"]) def test_moe_skipped_reduction_keeps_post(group, dual, flag): - """A reduce-scattered or elsewhere-fused MoE output must not reach the fused all-reduce - + mHC post: the pending post stays pending and no collective runs.""" + """A reduce-scattered MoE output must not reach the fused all-reduce + mHC + post: the pending post stays pending and no collective runs.""" from sglang.srt.layers.moe.mhc_post_fusion import ( MhcPostFusion, use_mhc_post_fusion, diff --git a/test/registered/unit/test_runtime_context.py b/test/registered/unit/test_runtime_context.py index 462806429b..9147f84b9b 100644 --- a/test/registered/unit/test_runtime_context.py +++ b/test/registered/unit/test_runtime_context.py @@ -1414,8 +1414,6 @@ class TestForwardFlags(_IsolatedServerArgs): x = x + 1 if fwd.is_extend_in_batch: x = x + 2 - if fwd.fuse_mlp_allreduce: - x = x + 4 if fwd.mlp_reduce_scatter: x = x + 8 if fwd.flashinfer_trtllm_bypass: @@ -1429,11 +1427,10 @@ class TestForwardFlags(_IsolatedServerArgs): self.assertEqual(probe(torch.zeros(())).item(), 2) get_forward().set("is_extend_in_batch", False) with get_forward().scoped( - fuse_mlp_allreduce=True, mlp_reduce_scatter=True, flashinfer_trtllm_bypass=True, ): - self.assertEqual(probe(torch.zeros(())).item(), 28) + self.assertEqual(probe(torch.zeros(())).item(), 24) self.assertEqual(probe(torch.zeros(())).item(), 0) def test_parallel_config_leaves_trace_under_torch_compile(self): @@ -1578,23 +1575,17 @@ class TestForwardFlags(_IsolatedServerArgs): reset_context() fwd = get_forward() - self.assertFalse(fwd.fuse_mlp_allreduce) self.assertFalse(fwd.mlp_reduce_scatter) self.assertFalse(fwd.flashinfer_trtllm_bypass) self.assertFalse(should_skip_mlp_all_reduce()) - with fwd.scoped(fuse_mlp_allreduce=True): - self.assertTrue(fwd.fuse_mlp_allreduce) - self.assertTrue(should_skip_mlp_all_reduce()) - # Fusion alone is enough to skip post-experts AR. - self.assertTrue(should_skip_post_experts_all_reduce(is_tp_path=True)) - self.assertFalse(fwd.fuse_mlp_allreduce) - self.assertFalse(should_skip_mlp_all_reduce()) - with fwd.scoped(mlp_reduce_scatter=True): self.assertTrue(fwd.mlp_reduce_scatter) self.assertTrue(should_skip_mlp_all_reduce()) + # The reduce-scatter alone is enough to skip post-experts AR. + self.assertTrue(should_skip_post_experts_all_reduce(is_tp_path=True)) self.assertFalse(fwd.mlp_reduce_scatter) + self.assertFalse(should_skip_mlp_all_reduce()) with fwd.scoped(flashinfer_trtllm_bypass=True): self.assertTrue(fwd.flashinfer_trtllm_bypass)