From b157d548bb3a648a98f47559c6d7db2e84a80ba6 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:17:48 -0700 Subject: [PATCH] [Refactor] Let stage boundaries complete every stage-output sum (#42301) --- docs/docs/developer_guide/layer_boundary.mdx | 18 ++-- .../layers/layer_boundary/adapters/branch.py | 13 ++- .../sglang/srt/layers/layer_boundary/exit.py | 91 ++++++++-------- .../sglang/srt/layers/layer_boundary/ops.py | 32 ++++++ .../sglang/srt/layers/layer_boundary/stage.py | 36 +++++-- python/sglang/srt/layers/moe/utils.py | 46 ++++++-- python/sglang/srt/model_loader/loader.py | 5 +- python/sglang/srt/models/apertus.py | 6 +- python/sglang/srt/models/arcee.py | 6 +- python/sglang/srt/models/bailing_moe.py | 3 +- .../sglang/srt/models/bailing_moe_linear.py | 5 +- python/sglang/srt/models/bailing_moe_v3.py | 20 +--- python/sglang/srt/models/deepseek_v2.py | 73 ++++++++----- .../srt/models/dots3_common/modeling.py | 65 ++---------- python/sglang/srt/models/exaone4.py | 6 +- python/sglang/srt/models/falcon_h1.py | 1 + python/sglang/srt/models/glm4_moe.py | 4 +- python/sglang/srt/models/glm4_moe_lite.py | 4 +- python/sglang/srt/models/glm5_next.py | 3 + python/sglang/srt/models/gpt_oss.py | 3 - python/sglang/srt/models/interns2_mobius.py | 5 +- python/sglang/srt/models/kimi_linear.py | 10 +- python/sglang/srt/models/laguna.py | 8 +- python/sglang/srt/models/llada2.py | 3 +- python/sglang/srt/models/llama4.py | 4 +- python/sglang/srt/models/longcat_flash.py | 18 +--- .../sglang/srt/models/longcat_flash_nextn.py | 9 +- python/sglang/srt/models/mellum.py | 1 + python/sglang/srt/models/mimo_v2.py | 6 +- python/sglang/srt/models/mimo_v2_nextn.py | 1 + python/sglang/srt/models/minimax_m2.py | 2 - python/sglang/srt/models/minimax_m3.py | 34 +----- python/sglang/srt/models/mixtral.py | 10 +- python/sglang/srt/models/moss_vl.py | 3 + python/sglang/srt/models/nanbeige.py | 6 +- python/sglang/srt/models/nemotron_h.py | 31 ++---- python/sglang/srt/models/qwen2.py | 2 + python/sglang/srt/models/qwen2_moe.py | 16 +-- python/sglang/srt/models/qwen3.py | 1 + python/sglang/srt/models/qwen3_5.py | 10 ++ python/sglang/srt/models/qwen3_moe.py | 7 +- python/sglang/srt/models/qwen3_next.py | 4 + python/sglang/srt/models/qwen4_exp.py | 3 + python/sglang/srt/models/sarvam_moe.py | 5 +- python/sglang/srt/models/sdar.py | 2 +- python/sglang/srt/models/sdar_moe.py | 3 - python/sglang/srt/models/spark2_5.py | 6 +- python/sglang/srt/models/step3_vl.py | 4 +- python/sglang/srt/models/step3p5.py | 3 +- python/sglang/srt/models/xllm.py | 3 +- python/sglang/test/boundary_fixtures.py | 2 +- .../test_boundary_output_contracts.py | 8 +- .../test_boundary_output_policy.py | 92 ++++++++-------- .../test_declared_attention_cp.py | 49 +++++---- .../test_declared_decoder_boundary.py | 35 +++--- .../test_dp_attention_cp_gather.py | 26 +++-- .../unit/layer_boundary/test_ffn_exit.py | 23 ++-- .../test_longcat_flash_shortcut.py | 2 + .../test_nemotron_h_aux_capture.py | 6 +- .../test_nemotron_h_mtp_reduction.py | 19 +++- .../layer_boundary/test_nemotron_h_stages.py | 28 +++-- .../test_stage_producers_leave_sums.py | 100 ++++++++++++++++++ .../test_step3p5_dense_reduce_scatter.py | 91 ---------------- 63 files changed, 617 insertions(+), 524 deletions(-) create mode 100644 test/registered/unit/layer_boundary/test_stage_producers_leave_sums.py delete mode 100644 test/registered/unit/layer_boundary/test_step3p5_dense_reduce_scatter.py diff --git a/docs/docs/developer_guide/layer_boundary.mdx b/docs/docs/developer_guide/layer_boundary.mdx index 1a5b494cd7..b18848a040 100644 --- a/docs/docs/developer_guide/layer_boundary.mdx +++ b/docs/docs/developer_guide/layer_boundary.mdx @@ -142,7 +142,7 @@ first_ffn, second_attn, second_ffn = make_stages( This fragment assumes the source boundary and the three norms already exist. `prepared_from` means the source has already performed its read. Enter through `branch_input(source, hidden_states, forward_batch)`, not `prepare()`, to move that input and fork the residual stream without normalizing twice. The first stage's norm is therefore never applied; LongCat passes the MoE stage's norm there. -The branch adapter also provides `branch_output()` to place a complete contribution on the branch's exit rows and `merge_branch()` to combine it with another branch's output. These methods encode row movement and residual ownership; the model still determines its compute schedule. Branch transport currently supports only ordinary token rows: construction rejects context-parallel, input-scattered, and sequence-parallel variants. See `python/sglang/srt/models/longcat_flash.py` for a complete integration. +The branch adapter also provides `branch_output()` to complete the sum a branch's contribution owes and place it on the branch's exit rows and `merge_branch()` to combine it with another branch's output. These methods encode row movement and residual ownership; the model still determines its compute schedule. Branch transport currently supports only ordinary token rows: construction rejects context-parallel, input-scattered, and sequence-parallel variants. See `python/sglang/srt/models/longcat_flash.py` for a complete integration. ### Execute a layer @@ -166,14 +166,14 @@ hidden_states = output.finish(hidden_states) This example uses Qwen3's compute member name `mlp`; boundary construction does not require renaming compute classes or checkpoint parameters. -`prepare()` returns input in the consumer read's format, which can include quantized tensors. An attention with an `ALWAYS_PARTIAL` output contract uses `finish()` to register its contribution. An FFN or single-stage mixer uses `exit()` to publish flags during compute, then calls the returned object's `finish()` once. The exit context restores the previous runtime flags even when compute raises; do not call `finish()` on a failed computation. +`prepare()` returns input in the consumer read's format, which can include quantized tensors. An attention with an `ALWAYS_PARTIAL` output contract uses `finish()` to register its contribution. An FFN or single-stage mixer runs its compute under `exit()`, then calls the returned object's `finish()` once. The exit context restores the previous runtime flags even when compute raises; do not call `finish()` on a failed computation. -Compute cooperates through the flags the scope publishes on `get_forward()`: +Compute never sums a stage's output: build every output projection, MLP and MoE block of a stage with `reduce_results=False`, and leave the sum the output owes to the boundary. The boundary completes it at the exit, or carries it to the next stage's input, where a fused all-reduce + add + norm may take it: -- `FfnExit` publishes `fuse_mlp_allreduce` (skip the output all-reduce), `mlp_reduce_scatter` (leave the sum to the boundary's reduce-scatter), and `defer_moe_finalize` (a MoE may return a `DeferredFinalize`). -- `MixerExit` publishes `fuse_mlp_allreduce` and exposes the same decision as `skips_reduction`. Before an FFN, the mixer always leaves its attention-TP sum to the FFN's input, including under attention DP. Before another mixer it may defer the sum to that mixer's input, but never under attention DP. +- `FfnExit` decides before compute runs whether the exit completes the sum, a reduce-scatter on the way back completes it, or the next input does. It publishes only `defer_moe_finalize` on `get_forward()`: a MoE may then return a `DeferredFinalize`. +- `MixerExit`: before an FFN, the mixer always leaves its attention-TP sum to the FFN's input, including under attention DP. Before another mixer the exit may carry the sum to that mixer's input, but never under attention DP; otherwise it completes the sum itself. -`RowParallelLinear` and the MoE output reduction honour these flags automatically; a custom reduction reads them from the exit object. Returning a handoff while `defer_moe_finalize` is false fails in `finish()`. +A component every TP rank computes in full (a shared expert replicated with `tp_size=1`, LoRA-B on a partial sum) is added on TP rank 0 only, so that the one sum counts it once. Model loading fails if a module under a stage boundary is built with `reduce_results=True`. Returning a handoff while `defer_moe_finalize` is false fails in `finish()`. A complete tensor after `finish()` can still have a pending **residual update**. An incomplete reduction is exposed to the model as an opaque `OwedOutput`; pass it to the next boundary or a supported access method. Do not inspect it as a tensor, slice it, or unwrap its contribution in model code. @@ -201,7 +201,7 @@ A terminal FFN normally completes its output before the final norm. The exceptio `model_forward_stages()` in `python/sglang/srt/batch_overlap/two_batch_overlap.py` gives each TBO microbatch its own stream: it splits with `ResidualStream.from_handoff()` and merges the results afterwards. Do not share one mutable stream between interleaved microbatches or hold an `exit()` scope open across a TBO yield point. -In the operation-scheduled TBO path, the FFN runs without an `exit()` scope. No skip flags are published, so compute returns a fully reduced output. `boundary.finish_complete_output(hidden_states, forward_batch)` applies any output transform, moves that output back to the layer's exit rows (a plain scatter under attention DP), and leaves nothing owed. +In the operation-scheduled TBO path, the FFN runs without an `exit()` scope and returns its partial output. `boundary.complete_now(hidden_states, forward_batch)` completes the sum it owes, applies any output transform, moves that output back to the layer's exit rows (a plain scatter under attention DP), and leaves nothing owed. ### Auxiliary capture and extra contributions @@ -273,7 +273,7 @@ flowchart TD F["ForwardBatch
runtime facts"] --> B S["ResidualStream
residual and
pending work"] <--> B B --> I["prepare()
complete,
update, read,
move rows"] - I -->|"FFN or mixer"| X["exit()
choose
completion,
publish flags"] + I -->|"FFN or mixer"| X["exit()
choose
completion"] X --> M["Model
computation"] I -->|"ALWAYS_PARTIAL"| M M --> O["finish()
record the
contribution"] @@ -284,7 +284,7 @@ Construction and execution have different lifetimes: 1. **At model initialization**, `declare_attn()` and `declare_ffn()` describe compute behavior. `make_stages()` connects those declarations and binds each stage's own norm and hooks. 2. **During construction**, factories resolve token rows and reduction contracts for supported batch variants. `bind_entry()` binds consumer work; `bind_exit()` binds output transport. `StagePlan` retains those paths. -3. **At each forward**, batch facts select an existing path. An exit can make a batch-dependent completion decision, but it does not rebuild the declarations. The decision publishes compute flags and supplies the matching completion action. +3. **At each forward**, batch facts select an existing path. An exit can make a batch-dependent completion decision, but it does not rebuild the declarations. The decision supplies the matching completion action; the only flag it publishes to compute is `defer_moe_finalize`, for a MoE that may hand off its finalize. 4. **Across stages**, `ForwardBatch.residual_stream` carries residual state and the producer's actual contribution. Stages do not store forward tensors on their reusable plans. The resolved `StageContract` is a lower-level type than `StageDeclaration`: one `InputContract` and one `OutputContract` after topology and batch variant have been applied. An `EdgeContract` combines a producer output, consumer requirement, and the residual's source/destination layouts. diff --git a/python/sglang/srt/layers/layer_boundary/adapters/branch.py b/python/sglang/srt/layers/layer_boundary/adapters/branch.py index 4098d603a7..57d1dbc04f 100644 --- a/python/sglang/srt/layers/layer_boundary/adapters/branch.py +++ b/python/sglang/srt/layers/layer_boundary/adapters/branch.py @@ -19,7 +19,7 @@ from typing import TYPE_CHECKING, Tuple import torch -from sglang.srt.layers.layer_boundary.ops import move_rows +from sglang.srt.layers.layer_boundary.ops import move_rows, sum_output from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.srt.model_executor.forward_batch_info import ForwardBatch @@ -53,9 +53,14 @@ def branch_input( def branch_output( plan: StagePlan, hidden_states: torch.Tensor, forward_batch: ForwardBatch ) -> torch.Tensor: - """This layer's complete FFN output as a branch's contribution, which - adds to the layer's output without writing the residual: moved to the - rows the layer hands on.""" + """This layer's FFN output as a branch's contribution, which adds to the + layer's output without writing the residual: the sum it owes completed, + then moved to the rows the layer hands on.""" + group = plan.path_for(forward_batch).output.group + if group is not None: + hidden_states = sum_output( + hidden_states, group, forward_batch, may_quantize=False + ) rows, _, to = plan.branch_rows(forward_batch) return move_rows(hidden_states, rows, to, forward_batch) diff --git a/python/sglang/srt/layers/layer_boundary/exit.py b/python/sglang/srt/layers/layer_boundary/exit.py index c4321f95db..c055f915ef 100644 --- a/python/sglang/srt/layers/layer_boundary/exit.py +++ b/python/sglang/srt/layers/layer_boundary/exit.py @@ -39,6 +39,7 @@ from sglang.srt.layers.layer_boundary.ops import ( all_reduce_to_dp_local, dp_reduce_scatter, dp_reduce_scatterv, + sum_output, to_dp_local, ) from sglang.srt.layers.layer_boundary.output import ( @@ -94,13 +95,14 @@ class ExitPolicy: def __init__(self, plan): self.plan = plan - def finish_complete_output( + def complete_now( self, hidden_states: torch.Tensor, stream: ResidualStream, forward_batch: ForwardBatch, ): - """Move a complete output from the operation-scheduled producer path.""" + """Complete the output of the operation-scheduled producer path, which + never defers: the sum it owes, then its move and write-back.""" steps = self.plan.path_for(forward_batch) hidden_states, residual = self._complete_now( hidden_states, @@ -108,11 +110,9 @@ class ExitPolicy: forward_batch=forward_batch, dp_step=None, steps=steps, - output_move=( - steps.complete_output_move - if steps.output_move_completes_sum - else steps.output_move - ), + output_move=steps.output_move, + owes_sum=steps.output.group is not None + and not steps.output_move_completes_sum, ) return self._record_output( hidden_states, @@ -188,6 +188,8 @@ class ExitPolicy: dp_step=dp_step, steps=steps, output_move=steps.output_move, + # A reduce-scatter completes the sum on its way back. + owes_sum=steps.output.group is not None and not mlp_reduce_scatter, ) defer_moe_finalize = ( self.plan.fusions is not None @@ -251,10 +253,19 @@ class ExitPolicy: dp_step: Optional[Callable], steps, output_move: Optional[Callable], + owes_sum: bool, ) -> Tuple[torch.Tensor, torch.Tensor]: """This layer's postprocess, run with the attention-DP step already - chosen: the move back to where the next layer reads the FFN output, then - the write-back into the residual for a layer that does it itself.""" + chosen: the sum the output owes, the move back to where the next layer + reads it, then the write-back into the residual for a layer that does + it itself.""" + if owes_sum: + hidden_states = sum_output( + hidden_states, + steps.output.group, + forward_batch, + may_quantize=steps.output.update.is_plain_add, + ) if steps.output.transform is not None: hidden_states = steps.output.transform.apply(hidden_states) if steps.returns_over_dp: @@ -289,7 +300,7 @@ class ExitPolicy: return FfnExit(self, forward_batch, stream=stream) def _sum_deferral_allowed(self, steps) -> bool: - # Under the MoE-CP all-gather the fusion path would skip finish_complete_output + # Under the MoE-CP all-gather the fusion path would skip complete_now # and its MoE-CP scatter, leaving hidden_states longer than the residual. if is_enable_moe_cp_allgather() or steps.output.group is None: return False @@ -377,17 +388,17 @@ def _batch_allows_deferred_sum(forward_batch: ForwardBatch, boundary=None) -> bo class ExitDecision(msgspec.Struct, frozen=True): - """One decision shared by compute flags and the matching output completion. + """One decision for an FFN output, made before compute runs. Fields: defer_moe_finalize: Compute may return a producer-specific finalize handoff. - fuse_mlp_allreduce: Compute skips its all-reduce; completion runs it later - or carries it to the next consumer, whether fused there or unfused. - mlp_reduce_scatter: Compute leaves reduction to the selected scatter path. + fuse_mlp_allreduce: The sum is carried to the next consumer, which + completes it, fused or unfused. + mlp_reduce_scatter: A reduce-scatter on the way back completes the sum. complete: Callable(output, residual) returning the completed or wrapped output and its corresponding residual rows. - Do not independently reselect completion after compute has used these flags. + Do not independently reselect completion after compute has run. """ defer_moe_finalize: bool @@ -405,22 +416,21 @@ def _defer( class MixerExit: - """The scope that publishes a mixer's decision while it runs: inside the - ``with`` block ``fuse_mlp_allreduce`` on ``get_forward()`` tells its - row-parallel output projection to skip the all-reduce. It skips when the - stage's output always leaves its sum (to an FFN stage, which completes it in - its input), or when its declaration permits deferring the sum to the next - attention stage and the batch allows it: TP > 1, no attention DP, and no - input-scattered or MoE-CP all-gather layout. The consumer decides whether - to fuse its completion with the input norm.""" + """A mixer's (an attention-like stage's) output decision. The mixer leaves + its output's sum; the stage either carries it to the next attention stage + (when the declaration permits deferring and the batch allows it: TP > 1, + no attention DP, and no input-scattered or MoE-CP all-gather layout), + hands it on as the declared sum an FFN stage completes in its input, or + completes it here. The consumer decides whether to fuse its completion + with the input norm.""" __slots__ = ( - "skips_reduction", "_defers", - "_scope", + "_group", "_update", "_declared_sum", "_stream", + "_forward_batch", ) def __init__( @@ -433,7 +443,9 @@ class MixerExit: steps = boundary.plan.path_for(forward_batch) produced = steps.output self._stream = stream + self._forward_batch = forward_batch self._update = produced.update + self._group = produced.group self._declared_sum = produced.group if produced.always_partial else None self._defers = ( produced.may_defer_to_next @@ -441,15 +453,12 @@ class MixerExit: and not is_dp_attention_enabled() and boundary._sum_deferral_allowed(steps) ) - self.skips_reduction = produced.always_partial or self._defers - self._scope = get_forward().scoped(fuse_mlp_allreduce=self.skips_reduction) def __enter__(self) -> MixerExit: - self._scope.__enter__() return self def __exit__(self, *exc_info): - return self._scope.__exit__(*exc_info) + return None def finish(self, hidden_states: torch.Tensor): """Hand the actual mixer result to the next boundary through the stream.""" @@ -457,6 +466,13 @@ class MixerExit: hidden_states = UnreducedOutput( hidden_states, group=get_parallel().tp_group ) + elif self._declared_sum is None and self._group is not None: + hidden_states = sum_output( + hidden_states, + self._group, + self._forward_batch, + may_quantize=self._update.is_plain_add, + ) return self._stream.record( hidden_states, self._update, declared_sum=self._declared_sum ) @@ -473,18 +489,15 @@ class FfnExit: Fields: boundary: The owning output boundary. defer_moe_finalize: Whether compute may return a finalize handoff. - fuse_mlp_allreduce: Whether compute leaves its all-reduce to completion. - mlp_reduce_scatter: Whether compute leaves reduction to a scatter path. - The three flags are published on get_forward() only inside the context. - finish(output) uses the same selected action after successful compute. + The FFN leaves its output's sum; finish(output) completes it or carries + it on with the action selected before compute. ``defer_moe_finalize`` is + published on get_forward() only inside the context. """ __slots__ = ( "boundary", "defer_moe_finalize", - "fuse_mlp_allreduce", - "mlp_reduce_scatter", "_complete", "_update", "_declared_sum", @@ -504,18 +517,12 @@ class FfnExit: steps = boundary.plan.path_for(forward_batch) completion = boundary._decide(forward_batch, steps) self.defer_moe_finalize = completion.defer_moe_finalize - self.fuse_mlp_allreduce = completion.fuse_mlp_allreduce - self.mlp_reduce_scatter = completion.mlp_reduce_scatter self._complete = completion.complete self._update = steps.output.update self._declared_sum = boundary._sum_owed_after_skip( steps, completion.mlp_reduce_scatter ) - self._scope = get_forward().scoped( - fuse_mlp_allreduce=self.fuse_mlp_allreduce, - mlp_reduce_scatter=self.mlp_reduce_scatter, - defer_moe_finalize=self.defer_moe_finalize, - ) + self._scope = get_forward().scoped(defer_moe_finalize=self.defer_moe_finalize) def __enter__(self) -> FfnExit: self._scope.__enter__() diff --git a/python/sglang/srt/layers/layer_boundary/ops.py b/python/sglang/srt/layers/layer_boundary/ops.py index 069ebc2808..53a411c522 100644 --- a/python/sglang/srt/layers/layer_boundary/ops.py +++ b/python/sglang/srt/layers/layer_boundary/ops.py @@ -22,6 +22,8 @@ from sglang.srt.distributed import ( GroupCoordinator, attention_tensor_model_parallel_all_reduce, attention_tensor_model_parallel_quant_all_reduce, + tensor_model_parallel_all_reduce, + tensor_model_parallel_quant_all_reduce, ) from sglang.srt.distributed.device_communicators.pynccl_allocator import ( use_symmetric_memory, @@ -47,12 +49,14 @@ from sglang.srt.layers.layer_boundary.adapters.attention import ( ) from sglang.srt.layers.layer_boundary.layout import ( Layout, + SumGroup, TokenAxis, _cp_shard_token_rows, moe_cp_gathered_rows, ) from sglang.srt.layers.layer_boundary.residual import ResidualUpdate from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_ADD +from sglang.srt.layers.moe.utils import sum_post_experts_output from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.runtime_context import get_exec, get_parallel @@ -173,6 +177,34 @@ def attn_tp_all_reduce( return attention_tensor_model_parallel_all_reduce(hidden_states) +def sum_output( + hidden_states: torch.Tensor, + group: SumGroup, + forward_batch: ForwardBatch, + *, + may_quantize: bool, +) -> torch.Tensor: + """Complete the sum an output owes over ``group`` on the rows it is on.""" + parallel = get_parallel() + if group is SumGroup.ATTN_TP: + if parallel.attn_tp_size == 1: + return hidden_states + return attn_tp_all_reduce(hidden_states, forward_batch, may_quantize) + if group is SumGroup.TP: + if parallel.tp_size == 1: + return hidden_states + if ( + may_quantize + and not forward_batch.forward_mode.is_decode_or_idle() + and get_exec().comm.enable_quant_communications + ): + return tensor_model_parallel_quant_all_reduce(hidden_states) + return tensor_model_parallel_all_reduce(hidden_states) + if group is SumGroup.MOE_OUTPUT: + return sum_post_experts_output(hidden_states) + raise ValueError(f"unsupported output sum group: {group!r}") + + def dp_gather( hidden_states: torch.Tensor, forward_batch: ForwardBatch, diff --git a/python/sglang/srt/layers/layer_boundary/stage.py b/python/sglang/srt/layers/layer_boundary/stage.py index fb44b3181b..06cbb8a125 100644 --- a/python/sglang/srt/layers/layer_boundary/stage.py +++ b/python/sglang/srt/layers/layer_boundary/stage.py @@ -187,7 +187,8 @@ class StageBoundary: forward_batch: Active batch whose stream receives the compute result. Returns: - A context manager publishing reduction/finalize flags during compute. + A context manager publishing the producer's finalize choice during + compute. Call its finish(output) exactly once after successful compute, including when compute is skipped for an empty input. It completes or carries work using the same decision rather than selecting a second path. @@ -253,11 +254,11 @@ class StageBoundary: hidden_states = stream.complete(hidden_states) return hidden_states, stream.snapshot(hidden_states) - def finish_complete_output(self, hidden_states, forward_batch): - """Move an FFN output that compute already completed outside exit() - (the operation-scheduled TBO path) onto the rows the layer hands on; - it chooses no reduction step.""" - return self.plan.output.finish_complete_output( + def complete_now(self, hidden_states, forward_batch): + """Complete an FFN output outside exit() (the operation-scheduled TBO + path, which never defers): the sum it owes, then the move onto the + rows the layer hands on.""" + return self.plan.output.complete_now( hidden_states, stream_of(forward_batch), forward_batch ) @@ -379,3 +380,26 @@ class StageBoundary: ), ) return hidden_states, stream + + +def check_stage_producers(model: torch.nn.Module) -> None: + """Reject a model whose stage producers complete their own output sums. + + In a layer built from stage boundaries, the boundary completes every sum a + stage output owes. A submodule there constructed with + ``reduce_results=True`` (a row-parallel projection, a MoE block) would + have its output summed twice. Called once, after the model is built. + """ + offenders = [ + f"{name}.{child_name}" + for name, layer in model.named_modules() + if any(isinstance(v, StageBoundary) for v in vars(layer).values()) + for child_name, child in layer.named_modules() + if child_name and getattr(child, "reduce_results", False) is True + ] + if offenders: + raise ValueError( + "stage boundaries complete their producers' output sums, but these " + f"submodules sum their own output: {', '.join(offenders[:8])}" + + (f" (and {len(offenders) - 8} more)" if len(offenders) > 8 else "") + ) diff --git a/python/sglang/srt/layers/moe/utils.py b/python/sglang/srt/layers/moe/utils.py index 300d09390b..b185aaf00e 100644 --- a/python/sglang/srt/layers/moe/utils.py +++ b/python/sglang/srt/layers/moe/utils.py @@ -797,6 +797,18 @@ def should_add_replicated_moe_output() -> bool: return not (parallel.tp_size > 1 and summed_later and parallel.tp_rank != 0) +def adds_replicated_output_to_partial() -> bool: + """For a MoE block whose stage boundary completes its sum: whether this rank + adds an output every TP rank holds in full, such as a shared expert + replicated with tp_size=1, to its MoE output. While the output still owes a + TP sum, only TP rank 0 adds it, so the sum counts it once.""" + parallel = get_parallel() + owes_sum = parallel.tp_size > 1 and not post_experts_output_is_complete( + is_tp_path=True + ) + return not owes_sum or parallel.tp_rank == 0 + + def can_merge_post_experts_all_reduce() -> bool: """Whether the EP and MoE-TP reductions can collapse into one _TP all-reduce. @@ -818,20 +830,38 @@ def post_experts_all_reduce(hidden_states: torch.Tensor) -> torch.Tensor: sequential ones, which also restores the invariant the fused residual+LN path depends on. """ + parallel = get_parallel() + return _post_experts_sum( + hidden_states, + reduce_ep=parallel.moe_ep_size > 1 + and not should_skip_post_experts_all_reduce(is_tp_path=False), + reduce_tp=parallel.moe_tp_size > 1 + and not should_skip_post_experts_all_reduce(is_tp_path=True), + ) + + +def sum_post_experts_output(hidden_states: torch.Tensor) -> torch.Tensor: + """Complete the sum a MoE output owes over the EP and MoE-TP groups, for the + boundary that owns it; a path the combine already summed is left alone.""" + parallel = get_parallel() + return _post_experts_sum( + hidden_states, + reduce_ep=parallel.moe_ep_size > 1 + and not post_experts_output_is_complete(is_tp_path=False), + reduce_tp=parallel.moe_tp_size > 1 + and not post_experts_output_is_complete(is_tp_path=True), + ) + + +def _post_experts_sum( + hidden_states: torch.Tensor, *, reduce_ep: bool, reduce_tp: bool +) -> torch.Tensor: from sglang.srt.distributed.communication_op import ( moe_expert_parallel_all_reduce, moe_tensor_model_parallel_all_reduce, tensor_model_parallel_all_reduce, ) - parallel = get_parallel() - reduce_ep = parallel.moe_ep_size > 1 and not should_skip_post_experts_all_reduce( - is_tp_path=False - ) - reduce_tp = parallel.moe_tp_size > 1 and not should_skip_post_experts_all_reduce( - is_tp_path=True - ) - if reduce_ep and reduce_tp and can_merge_post_experts_all_reduce(): return tensor_model_parallel_all_reduce(hidden_states) diff --git a/python/sglang/srt/model_loader/loader.py b/python/sglang/srt/model_loader/loader.py index 788ac19b8c..d60efacb38 100644 --- a/python/sglang/srt/model_loader/loader.py +++ b/python/sglang/srt/model_loader/loader.py @@ -83,6 +83,7 @@ from sglang.srt.connector.utils import parse_model_name from sglang.srt.distributed import ( model_parallel_is_initialized, ) +from sglang.srt.layers.layer_boundary.stage import check_stage_producers from sglang.srt.layers.modelopt_utils import QUANT_CFG_CHOICES from sglang.srt.layers.moe.utils import ( install_shared_experts_fusion_decision, @@ -304,7 +305,9 @@ def _initialize_model( if load_config.draft_model_idx is not None: kwargs["draft_model_idx"] = load_config.draft_model_idx - return model_class(**kwargs) + model = model_class(**kwargs) + check_stage_producers(model) + return model def post_load_weights(model: nn.Module) -> None: diff --git a/python/sglang/srt/models/apertus.py b/python/sglang/srt/models/apertus.py index ee12ae8f2d..cb1efd5cd2 100644 --- a/python/sglang/srt/models/apertus.py +++ b/python/sglang/srt/models/apertus.py @@ -259,6 +259,7 @@ class ApertusDecoderLayer(nn.Module): quant_config=quant_config, bias=getattr(config, "mlp_bias", False), prefix=add_prefix("mlp", prefix), + reduce_results=False, ) self.attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.feedforward_layernorm = RMSNorm( @@ -294,8 +295,9 @@ class ApertusDecoderLayer(nn.Module): # Fully Connected hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - hidden_states = self.mlp(hidden_states) - return self.ffn_boundary.finish_complete_output(hidden_states, forward_batch) + with self.ffn_boundary.exit(forward_batch) as ffn_exit: + hidden_states = self.mlp(hidden_states) + return ffn_exit.finish(hidden_states) class ApertusModel(nn.Module): diff --git a/python/sglang/srt/models/arcee.py b/python/sglang/srt/models/arcee.py index 573c6f3ae3..70ce424c2e 100644 --- a/python/sglang/srt/models/arcee.py +++ b/python/sglang/srt/models/arcee.py @@ -235,6 +235,7 @@ class ArceeDecoderLayer(nn.Module): hidden_act=config.hidden_act, quant_config=quant_config, prefix=add_prefix("mlp", prefix), + reduce_results=False, ) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.post_attention_layernorm = RMSNorm( @@ -270,8 +271,9 @@ class ArceeDecoderLayer(nn.Module): # Fully Connected hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - hidden_states = self.mlp(hidden_states) - return self.ffn_boundary.finish_complete_output(hidden_states, forward_batch) + with self.ffn_boundary.exit(forward_batch) as ffn_exit: + hidden_states = self.mlp(hidden_states) + return ffn_exit.finish(hidden_states) class ArceeModel(nn.Module): diff --git a/python/sglang/srt/models/bailing_moe.py b/python/sglang/srt/models/bailing_moe.py index 8f2694a28a..c3097375fa 100644 --- a/python/sglang/srt/models/bailing_moe.py +++ b/python/sglang/srt/models/bailing_moe.py @@ -55,7 +55,6 @@ from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import ( get_deepep_mode, get_moe_a2a_backend, - reduce_moe_output, ) from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE @@ -460,7 +459,6 @@ class BailingMoESparseMoeBlock(nn.Module): if self.num_shared_experts > 0: final_hidden_states = final_hidden_states + shared_output - final_hidden_states = reduce_moe_output(final_hidden_states) return final_hidden_states.view(num_tokens, hidden_size) def forward_deepep( @@ -693,6 +691,7 @@ class BailingMoEBlock(nn.Module): prefix=add_prefix("mlp", prefix), tp_rank=mlp_tp_rank, tp_size=mlp_tp_size, + reduce_results=False, ) self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps) diff --git a/python/sglang/srt/models/bailing_moe_linear.py b/python/sglang/srt/models/bailing_moe_linear.py index 89a7885485..c68d67fd5c 100644 --- a/python/sglang/srt/models/bailing_moe_linear.py +++ b/python/sglang/srt/models/bailing_moe_linear.py @@ -32,7 +32,6 @@ from sglang.srt.layers.linear import ( RowParallelLinear, ) from sglang.srt.layers.logits_processor import LogitsProcessor -from sglang.srt.layers.moe import reduce_moe_output from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.moe.topk import TopK @@ -365,7 +364,6 @@ class BailingMoE(nn.Module): if self.num_shared_experts > 0: final_hidden_states = final_hidden_states + shared_output - final_hidden_states = reduce_moe_output(final_hidden_states) return final_hidden_states @@ -671,6 +669,7 @@ class BailingMoEAttention(nn.Module): bias=config.use_bias, quant_config=quant_config, prefix=f"{prefix}.o_proj", + reduce_results=False, ) if hasattr(config, "rotary_dim"): self.rotary_dim = config.rotary_dim @@ -792,6 +791,7 @@ class BailingMoELinearDecoderLayer(nn.Module): intermediate_size=config.intermediate_size, quant_config=quant_config, prefix=add_prefix("mlp", prefix), + reduce_results=False, ) else: if is_nextn or self.layer_id >= config.first_k_dense_replace: @@ -810,6 +810,7 @@ class BailingMoELinearDecoderLayer(nn.Module): intermediate_size=config.intermediate_size, quant_config=quant_config, prefix=add_prefix("mlp", prefix), + reduce_results=False, ) rms_norm_eps = float(getattr(config, "rms_norm_eps", 1e-5)) self.input_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps) diff --git a/python/sglang/srt/models/bailing_moe_v3.py b/python/sglang/srt/models/bailing_moe_v3.py index b4cecd5621..b225e66f26 100644 --- a/python/sglang/srt/models/bailing_moe_v3.py +++ b/python/sglang/srt/models/bailing_moe_v3.py @@ -19,10 +19,6 @@ from sglang.kernels.ops.quantization.fp8_kernel import ( ) from sglang.srt.configs import KimiLinearConfig from sglang.srt.configs.bailing_hybrid import is_bailing_multi_gate_enabled -from sglang.srt.distributed import ( - moe_expert_parallel_all_reduce, - moe_tensor_model_parallel_all_reduce, -) from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo @@ -47,7 +43,6 @@ from sglang.srt.layers.linear import ( from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import ( get_moe_a2a_backend, - should_skip_post_experts_all_reduce, ) from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE @@ -751,18 +746,6 @@ class BailingMoE(nn.Module): if shared_output is not None: final_hidden_states = final_hidden_states + shared_output - - if self.moe_ep_size > 1 and not should_skip_post_experts_all_reduce( - is_tp_path=False, - ): - final_hidden_states = moe_expert_parallel_all_reduce(final_hidden_states) - - if self.moe_tp_size > 1 and not should_skip_post_experts_all_reduce( - is_tp_path=True, - ): - final_hidden_states = moe_tensor_model_parallel_all_reduce( - final_hidden_states - ) return final_hidden_states def _forward_shared_experts( @@ -978,6 +961,7 @@ class BailingMoEAttention(nn.Module): bias=config.use_bias, quant_config=quant_config, prefix=f"{prefix}.o_proj", + reduce_results=False, ) if hasattr(config, "rotary_dim"): self.rotary_dim = config.rotary_dim @@ -1121,6 +1105,7 @@ class BailingMoELinearDecoderLayer(nn.Module): prefix=prefix, tp_rank=mlp_tp_rank, tp_size=mlp_tp_size, + reduce_results=False, ) else: if is_nextn or self.layer_id >= config.first_k_dense_replace: @@ -1141,6 +1126,7 @@ class BailingMoELinearDecoderLayer(nn.Module): prefix=prefix, tp_rank=mlp_tp_rank, tp_size=mlp_tp_size, + reduce_results=False, ) rms_norm_eps = float(getattr(config, "rms_norm_eps", 1e-5)) self.input_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps) diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 75d1277d12..0145b36e14 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -111,6 +111,7 @@ from sglang.srt.layers.moe.token_dispatcher.base import ( from sglang.srt.layers.moe.topk import BypassedTopKOutput, TopK, TopKOutputFormat from sglang.srt.layers.moe.utils import ( RoutingMethodType, + adds_replicated_output_to_partial, filter_moe_weight_param_global_expert, has_per_rank_fused_shared_slots, is_deepep_class_backend, @@ -118,6 +119,7 @@ from sglang.srt.layers.moe.utils import ( is_shared_experts_fusion_disabled, is_tbo_enabled, should_add_replicated_moe_output, + sum_post_experts_output, ) from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.quantization.fp8 import Fp8Config @@ -275,10 +277,15 @@ class DeepseekV2MLP(nn.Module): tp_rank: Optional[int] = None, tp_size: Optional[int] = None, swiglu_limit: Optional[float] = None, + allow_fused_down: Optional[bool] = None, ) -> None: super().__init__() self.tp_size = tp_size self.swiglu_limit = swiglu_limit + # The fused down path calls deepgemm directly and runs no all-reduce. + self.allow_fused_down = ( + not reduce_results if allow_fused_down is None else allow_fused_down + ) self.gate_up_proj = MergedColumnParallelLinear( hidden_size, @@ -375,7 +382,7 @@ class DeepseekV2MLP(nn.Module): # are fp8 (uint8 storage with weight_scale_inv). if ( self.swiglu_limit is not None - and not self.down_proj.reduce_results + and self.allow_fused_down and getattr(self.down_proj, "weight", None) is not None and self.down_proj.weight.dtype == torch.uint8 and hasattr(self.down_proj, "weight_scale_inv") @@ -592,8 +599,11 @@ class DeepseekV2MoE(nn.Module): is_nextn: bool = False, is_deepseek_v4: bool = False, vl_correction_bias: bool = False, + reduce_results: bool = True, ): super().__init__() + # False when a stage boundary completes this output's sum. + self.reduce_results = reduce_results self.tp_size = get_parallel().tp_size self.moe_ep_size = get_parallel().moe_ep_size self.routed_scaling_factor = config.routed_scaling_factor @@ -1160,7 +1170,7 @@ class DeepseekV2MoE(nn.Module): final_hidden_states, gated_shared_output=shared_output, m=hidden_states.shape[0], - reduce=post_experts_all_reduce, + reduce=sum_post_experts_output, ) all_reduce_done = False @@ -1256,7 +1266,7 @@ class DeepseekV2MoE(nn.Module): self.routed_scaling_factor, ) - if not all_reduce_done: + if not all_reduce_done and self.reduce_results: if ( self.is_deepseek_v4 and self.tp_size > 1 @@ -1274,9 +1284,9 @@ class DeepseekV2MoE(nn.Module): if mhc is not None: mhc.start_stats_before_all_reduce() final_hidden_states = post_experts_all_reduce(final_hidden_states) - # TP1 shared experts are replicated, so add them after all-reduce to - # avoid summing the same shared output once per TP rank. - if self._shared_expert_tp1 and should_add_replicated_moe_output(): + # TP1 shared experts are replicated: add them after this block's + # all-reduce, or on TP rank 0 only when the stage boundary sums. + if self._shared_expert_tp1 and self._adds_replicated_output(): final_hidden_states += shared_output return final_hidden_states @@ -1431,31 +1441,40 @@ class DeepseekV2MoE(nn.Module): self.routed_scaling_factor, ) - if ( - self.is_deepseek_v4 - and self.tp_size > 1 - and not should_skip_post_experts_all_reduce(is_tp_path=True) - ): - from sglang.srt.layers.moe.mhc_post_fusion import current_mhc_post_fusion - - mhc = current_mhc_post_fusion() - if _is_hip and _hip_moe.fused_all_reduce_mhc( - self, mhc, final_hidden_states + if self.reduce_results: + if ( + self.is_deepseek_v4 + and self.tp_size > 1 + and not should_skip_post_experts_all_reduce(is_tp_path=True) ): - return final_hidden_states - if mhc is not None: - mhc.start_stats_before_all_reduce() - final_hidden_states = post_experts_all_reduce(final_hidden_states) - # TP1 shared experts are replicated, so add them after all-reduce to - # avoid summing the same shared output once per TP rank. + from sglang.srt.layers.moe.mhc_post_fusion import ( + current_mhc_post_fusion, + ) + + mhc = current_mhc_post_fusion() + if _is_hip and _hip_moe.fused_all_reduce_mhc( + self, mhc, final_hidden_states + ): + return final_hidden_states + if mhc is not None: + mhc.start_stats_before_all_reduce() + final_hidden_states = post_experts_all_reduce(final_hidden_states) + # TP1 shared experts are replicated: add them after this block's + # all-reduce, or on TP rank 0 only when the stage boundary sums. if ( shared_output is not None and self._shared_expert_tp1 - and should_add_replicated_moe_output() + and self._adds_replicated_output() ): final_hidden_states += shared_output return final_hidden_states + def _adds_replicated_output(self) -> bool: + """Whether this rank adds the replicated TP1 shared-expert output.""" + if self.reduce_results: + return should_add_replicated_moe_output() + return adds_replicated_output_to_partial() + def forward_cpu( self, hidden_states: torch.Tensor, @@ -1507,7 +1526,8 @@ class DeepseekV2MoE(nn.Module): ), # block_size True, # is_vnni ) - final_hidden_states = post_experts_all_reduce(final_hidden_states) + if self.reduce_results: + final_hidden_states = post_experts_all_reduce(final_hidden_states) return final_hidden_states def forward_deepep( @@ -2661,6 +2681,7 @@ class DeepseekV2DecoderLayer(nn.Module): layer_id=self.layer_id, alt_stream=alt_stream, is_nextn=is_nextn, + reduce_results=False, ) else: if is_dense_ffn_fully_dp(): @@ -2676,6 +2697,8 @@ class DeepseekV2DecoderLayer(nn.Module): tp_rank=mlp_tp_rank, tp_size=mlp_tp_size, swiglu_limit=getattr(config, "swiglu_limit", None), + reduce_results=False, + allow_fused_down=False, ) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) @@ -2876,7 +2899,7 @@ class DeepseekV2DecoderLayer(nn.Module): ) def op_comm_postprocess_layer(self, state): - hidden_states = self.ffn_boundary.finish_complete_output( + hidden_states = self.ffn_boundary.complete_now( state.pop("hidden_states_mlp_output"), state.forward_batch ) diff --git a/python/sglang/srt/models/dots3_common/modeling.py b/python/sglang/srt/models/dots3_common/modeling.py index 8e391b83cf..a12a83c54e 100644 --- a/python/sglang/srt/models/dots3_common/modeling.py +++ b/python/sglang/srt/models/dots3_common/modeling.py @@ -45,9 +45,6 @@ from sglang.srt.batch_overlap.two_batch_overlap import ( model_forward_stages, ) from sglang.srt.configs.dots3 import Dots3Config -from sglang.srt.distributed import ( - tensor_model_parallel_all_reduce, -) from sglang.srt.distributed.device_communicators.pynccl_allocator import ( use_symmetric_memory, ) @@ -228,21 +225,13 @@ class Dots3MLP(nn.Module): ) self.act_fn = SiluAndMul() - def forward( - self, - x, - forward_batch=None, - should_allreduce_fusion: bool = False, - use_reduce_scatter: bool = False, - ): + def forward(self, x, forward_batch=None): if (self.tp_size == 1) and x.shape[0] == 0: return x gate_up, _ = self.gate_up_proj(x) x = self.act_fn(gate_up) - x, _ = self.down_proj( - x, skip_all_reduce=should_allreduce_fusion or use_reduce_scatter - ) + x, _ = self.down_proj(x) return x @@ -416,8 +405,6 @@ class Dots3MoE(nn.Module): self, hidden_states: torch.Tensor, forward_batch: Optional[ForwardBatch] = None, - should_allreduce_fusion: bool = False, - use_reduce_scatter: bool = False, ) -> torch.Tensor: if not self._enable_deepep_moe: DUAL_STREAM_TOKEN_THRESHOLD = 1024 @@ -427,26 +414,13 @@ class Dots3MoE(nn.Module): and hidden_states.shape[0] > 0 and hidden_states.shape[0] <= DUAL_STREAM_TOKEN_THRESHOLD ): - return self.forward_normal_dual_stream( - hidden_states, - should_allreduce_fusion, - use_reduce_scatter, - ) + return self.forward_normal_dual_stream(hidden_states) else: - return self.forward_normal( - hidden_states, - should_allreduce_fusion, - use_reduce_scatter, - ) + return self.forward_normal(hidden_states) else: return self.forward_deepep(hidden_states, forward_batch) - def forward_normal_dual_stream( - self, - hidden_states: torch.Tensor, - should_allreduce_fusion: bool = False, - use_reduce_scatter: bool = False, - ) -> torch.Tensor: + def forward_normal_dual_stream(self, hidden_states: torch.Tensor) -> torch.Tensor: current_stream = torch.cuda.current_stream() self.alt_stream.wait_stream(current_stream) @@ -465,21 +439,9 @@ class Dots3MoE(nn.Module): torch.add(final_hidden_states, shared_output, out=final_hidden_states_out) final_hidden_states = final_hidden_states_out sm.tag(final_hidden_states) - if ( - self.tp_size > 1 - and not should_allreduce_fusion - and not use_reduce_scatter - and not should_use_flashinfer_cutlass_moe_fp4_allgather() - ): - final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) return final_hidden_states - def forward_normal( - self, - hidden_states: torch.Tensor, - should_allreduce_fusion: bool = False, - use_reduce_scatter: bool = False, - ) -> torch.Tensor: + def forward_normal(self, hidden_states: torch.Tensor) -> torch.Tensor: if hidden_states.shape[0] > 0: shared_output = self._forward_shared_experts(hidden_states) # router_logits: (num_tokens, n_experts) @@ -496,13 +458,6 @@ class Dots3MoE(nn.Module): torch.add(final_hidden_states, shared_output, out=final_hidden_states_out) final_hidden_states = final_hidden_states_out sm.tag(final_hidden_states) - if ( - self.tp_size > 1 - and not should_allreduce_fusion - and not use_reduce_scatter - and not should_use_flashinfer_cutlass_moe_fp4_allgather() - ): - final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) return final_hidden_states def forward_deepep( @@ -1454,6 +1409,7 @@ class Dots3DecoderLayer(nn.Module): prefix=add_prefix("mlp", prefix), tp_rank=mlp_tp_rank, tp_size=mlp_tp_size, + reduce_results=False, ) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) @@ -1509,12 +1465,7 @@ class Dots3DecoderLayer(nn.Module): hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) with self.ffn_boundary.exit(forward_batch) as ffn_exit: - hidden_states = self.mlp( - hidden_states, - forward_batch, - ffn_exit.fuse_mlp_allreduce, - ffn_exit.mlp_reduce_scatter, - ) + hidden_states = self.mlp(hidden_states, forward_batch) return ffn_exit.finish(hidden_states) diff --git a/python/sglang/srt/models/exaone4.py b/python/sglang/srt/models/exaone4.py index 3f0306fff8..78edb9969a 100644 --- a/python/sglang/srt/models/exaone4.py +++ b/python/sglang/srt/models/exaone4.py @@ -74,6 +74,7 @@ class Exaone4GatedMLP(nn.Module): hidden_size, bias=bias, quant_config=quant_config, + reduce_results=False, prefix=add_prefix("down_proj", prefix), ) if hidden_act != "silu": @@ -312,8 +313,9 @@ class Exaone4DecoderLayer(nn.Module): # Fully Connected hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - hidden_states = self.mlp(hidden_states) - return self.ffn_boundary.finish_complete_output(hidden_states, forward_batch) + with self.ffn_boundary.exit(forward_batch) as ffn_exit: + hidden_states = self.mlp(hidden_states) + return ffn_exit.finish(hidden_states) class Exaone4Model(nn.Module): diff --git a/python/sglang/srt/models/falcon_h1.py b/python/sglang/srt/models/falcon_h1.py index 76fb83516b..4921a469d6 100644 --- a/python/sglang/srt/models/falcon_h1.py +++ b/python/sglang/srt/models/falcon_h1.py @@ -214,6 +214,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module): mlp_multipliers=config.mlp_multipliers, quant_config=quant_config, prefix=add_prefix("mlp", prefix), + reduce_results=False, ) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) diff --git a/python/sglang/srt/models/glm4_moe.py b/python/sglang/srt/models/glm4_moe.py index c23263aa33..4a1ba3552c 100644 --- a/python/sglang/srt/models/glm4_moe.py +++ b/python/sglang/srt/models/glm4_moe.py @@ -59,7 +59,6 @@ from sglang.srt.layers.linear import ( from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import ( get_moe_a2a_backend, - reduce_moe_output, should_use_flashinfer_cutlass_moe_fp4_allgather, ) from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class @@ -607,7 +606,6 @@ class Glm4MoeSparseMoeBlock(nn.Module): current_stream.wait_stream(self.alt_stream) final_hidden_states += shared_output - final_hidden_states = reduce_moe_output(final_hidden_states) return final_hidden_states def forward_normal( @@ -633,7 +631,6 @@ class Glm4MoeSparseMoeBlock(nn.Module): final_hidden_states_out = torch.empty_like(final_hidden_states) torch.add(final_hidden_states, shared_output, out=final_hidden_states_out) final_hidden_states = final_hidden_states_out - final_hidden_states = reduce_moe_output(final_hidden_states) return final_hidden_states def forward_deepep( @@ -768,6 +765,7 @@ class Glm4MoeDecoderLayer(nn.Module): prefix=add_prefix("mlp", prefix), tp_rank=mlp_tp_rank, tp_size=mlp_tp_size, + reduce_results=False, ) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) diff --git a/python/sglang/srt/models/glm4_moe_lite.py b/python/sglang/srt/models/glm4_moe_lite.py index b137ba968e..e9873fdb5d 100644 --- a/python/sglang/srt/models/glm4_moe_lite.py +++ b/python/sglang/srt/models/glm4_moe_lite.py @@ -50,7 +50,6 @@ from sglang.srt.layers.linear import MergedColumnParallelLinear, RowParallelLine from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import ( get_moe_a2a_backend, - reduce_moe_output, should_use_flashinfer_cutlass_moe_fp4_allgather, ) from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class @@ -338,7 +337,6 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module): current_stream.wait_stream(self.alt_stream) final_hidden_states += shared_output - final_hidden_states = reduce_moe_output(final_hidden_states) return final_hidden_states def forward_normal( @@ -364,7 +362,6 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module): final_hidden_states_out = torch.empty_like(final_hidden_states) torch.add(final_hidden_states, shared_output, out=final_hidden_states_out) final_hidden_states = final_hidden_states_out - final_hidden_states = reduce_moe_output(final_hidden_states) return final_hidden_states def forward_deepep( @@ -476,6 +473,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module): prefix=add_prefix("mlp", prefix), tp_rank=mlp_tp_rank, tp_size=mlp_tp_size, + reduce_results=False, ) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index 43cc6dcf27..23f74a7eb1 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -734,6 +734,7 @@ class Glm5NextDecoderLayer(nn.Module): layer_id=self.layer_id, alt_stream=alt_stream, is_nextn=is_nextn, + reduce_results=False, ) else: if is_dense_ffn_fully_dp(): @@ -749,6 +750,8 @@ class Glm5NextDecoderLayer(nn.Module): tp_rank=mlp_tp_rank, tp_size=mlp_tp_size, swiglu_limit=config.swiglu_limit, + reduce_results=False, + allow_fused_down=False, ) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) diff --git a/python/sglang/srt/models/gpt_oss.py b/python/sglang/srt/models/gpt_oss.py index a7e8f0a544..b22857313a 100644 --- a/python/sglang/srt/models/gpt_oss.py +++ b/python/sglang/srt/models/gpt_oss.py @@ -48,7 +48,6 @@ from sglang.srt.layers.linear import ( from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import ( get_moe_a2a_backend, - reduce_moe_output, ) from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE @@ -335,8 +334,6 @@ class GptOssSparseMoeBlock(nn.Module): topk_output = self.topk(router_input, router_logits) final_hidden_states = self.experts(hidden_states, topk_output) - final_hidden_states = reduce_moe_output(final_hidden_states) - # When input was pre-padded, FusedMoE.forward_impl captured the # padded width as `origin_hidden_states_dim` and skipped its own # output-trim contiguous() — so the experts output is still diff --git a/python/sglang/srt/models/interns2_mobius.py b/python/sglang/srt/models/interns2_mobius.py index 371215e059..c2a3f40e68 100644 --- a/python/sglang/srt/models/interns2_mobius.py +++ b/python/sglang/srt/models/interns2_mobius.py @@ -31,9 +31,6 @@ from sglang.srt.layers.linear import ( ReplicatedLinear, RowParallelLinear, ) -from sglang.srt.layers.moe import ( - reduce_moe_output, -) from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.topk import TopK from sglang.srt.layers.moe.utils import ( @@ -422,7 +419,7 @@ class _InternS2MobiusDecoderMixin: routed = _get_mobius_routed_bank(meta_mlp, self.layer_id).forward_routed( hidden_states, forward_batch ) - return reduce_moe_output(routed + shared) + return routed + shared def _forward_after_attention( self, diff --git a/python/sglang/srt/models/kimi_linear.py b/python/sglang/srt/models/kimi_linear.py index 3078c08ae7..ae45a3bb41 100644 --- a/python/sglang/srt/models/kimi_linear.py +++ b/python/sglang/srt/models/kimi_linear.py @@ -12,7 +12,6 @@ from sglang.kernels.ops.attention.fla.fused_norm_gate import FusedRMSNormGated from sglang.srt.configs.kimi_linear import KimiLinearConfig from sglang.srt.distributed import ( divide, - tensor_model_parallel_all_reduce, ) from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.layers.dcp.planner import prepare_decode_context_parallel_metadata @@ -185,9 +184,6 @@ class KimiMoE(nn.Module): if shared_output is not None: final_hidden_states = final_hidden_states + shared_output - - if self.tp_size > 1: - final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) return final_hidden_states.view(num_tokens, hidden_size) @@ -613,6 +609,7 @@ class KimiDecoderLayer(nn.Module): hidden_act=config.hidden_act, quant_config=quant_config, prefix=f"{prefix}.mlp", + reduce_results=False, ) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.post_attention_layernorm = RMSNorm( @@ -655,8 +652,9 @@ class KimiDecoderLayer(nn.Module): # Fully Connected hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - hidden_states = self.mlp(hidden_states) - return self.ffn_boundary.finish_complete_output(hidden_states, forward_batch) + with self.ffn_boundary.exit(forward_batch) as ffn_exit: + hidden_states = self.mlp(hidden_states) + return ffn_exit.finish(hidden_states) class KimiLinearModel(nn.Module): diff --git a/python/sglang/srt/models/laguna.py b/python/sglang/srt/models/laguna.py index c2d7a7b194..3c334bf939 100644 --- a/python/sglang/srt/models/laguna.py +++ b/python/sglang/srt/models/laguna.py @@ -37,11 +37,10 @@ from sglang.srt.layers.linear import ( RowParallelLinear, ) from sglang.srt.layers.logits_processor import LogitsProcessor -from sglang.srt.layers.moe import reduce_moe_output from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.moe.topk import TopK -from sglang.srt.layers.moe.utils import should_add_replicated_moe_output +from sglang.srt.layers.moe.utils import adds_replicated_output_to_partial from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.layers.rotary_embedding import get_rope @@ -226,8 +225,7 @@ class LagunaMoE(nn.Module): else: final = routed_out + shared_out - final = reduce_moe_output(final) - if self._shared_expert_tp1 and should_add_replicated_moe_output(): + if self._shared_expert_tp1 and adds_replicated_output_to_partial(): final = final + shared_out return final @@ -448,7 +446,7 @@ class LagunaDecoderLayer(nn.Module): intermediate_size=config.intermediate_size, hidden_act=config.hidden_act, quant_config=quant_config, - reduce_results=True, + reduce_results=False, prefix=add_prefix("mlp", prefix), ) diff --git a/python/sglang/srt/models/llada2.py b/python/sglang/srt/models/llada2.py index e0b2d9dc3c..ab387cf67c 100644 --- a/python/sglang/srt/models/llada2.py +++ b/python/sglang/srt/models/llada2.py @@ -51,7 +51,6 @@ from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import ( get_deepep_mode, get_moe_a2a_backend, - reduce_moe_output, ) from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE @@ -381,7 +380,6 @@ class LLaDA2MoeSparseMoeBlock(nn.Module): if self.num_shared_experts > 0: final_hidden_states = final_hidden_states + shared_output - final_hidden_states = reduce_moe_output(final_hidden_states) return final_hidden_states.view(num_tokens, hidden_size) def forward_deepep( @@ -628,6 +626,7 @@ class LLaDA2MoeBlock(nn.Module): prefix=add_prefix("mlp", prefix), tp_rank=mlp_tp_rank, tp_size=mlp_tp_size, + reduce_results=False, ) self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps) diff --git a/python/sglang/srt/models/llama4.py b/python/sglang/srt/models/llama4.py index a8f7a19aa1..1d158e8d38 100644 --- a/python/sglang/srt/models/llama4.py +++ b/python/sglang/srt/models/llama4.py @@ -41,7 +41,6 @@ from sglang.srt.layers.linear import ( ReplicatedLinear, RowParallelLinear, ) -from sglang.srt.layers.moe import reduce_moe_output from sglang.srt.layers.moe.fused_moe_triton import FusedMoE from sglang.srt.layers.moe.topk import TopK from sglang.srt.layers.quantization.base_config import QuantizationConfig @@ -148,8 +147,6 @@ class Llama4MoE(nn.Module): out_aD = routed_out + shared_out - out_aD = reduce_moe_output(out_aD) - return out_aD def _forward_core(self, hidden_states, forward_mode: ForwardMode): @@ -418,6 +415,7 @@ class Llama4DecoderLayer(nn.Module): hidden_act="silu", quant_config=quant_config, prefix=add_prefix("feed_forward", prefix), + reduce_results=False, ) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.post_attention_layernorm = RMSNorm( diff --git a/python/sglang/srt/models/longcat_flash.py b/python/sglang/srt/models/longcat_flash.py index 0199ff3052..46e2974a48 100644 --- a/python/sglang/srt/models/longcat_flash.py +++ b/python/sglang/srt/models/longcat_flash.py @@ -44,9 +44,6 @@ from sglang.kernels.ops.gemm.bf16_fp32 import ( from sglang.kernels.ops.moe.ep_moe_kernels import zero_experts_compute_triton from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz from sglang.srt.configs import LongcatFlashConfig -from sglang.srt.distributed import ( - tensor_model_parallel_all_reduce, -) from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers import deep_gemm_wrapper @@ -72,7 +69,6 @@ from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.moe.topk import StandardTopKOutput, TopK from sglang.srt.layers.moe.utils import ( filter_moe_weight_param_global_expert, - should_skip_mlp_all_reduce, ) from sglang.srt.layers.n_gram_embedding import NgramEmbedding from sglang.srt.layers.quantization.base_config import QuantizationConfig @@ -325,18 +321,6 @@ class LongcatFlashMoE(nn.Module): if self.zero_expert_type is not None and hidden_states.shape[0] > 0: final_hidden_states += zero_expert_result.to(final_hidden_states.device) - # LONGCAT_MOE_A2A_SKIP_ALLREDUCE: skip the post-experts TP all-reduce when a - # real EP a2a backend is active -- self.experts (DeepEPMoE) already combined - # expert outputs across EP ranks, so an extra all-reduce double-counts. - from sglang.srt.layers.moe.utils import get_moe_a2a_backend as _lc_gab - - if ( - self.tp_size > 1 - and _lc_gab().is_none() - and not should_skip_mlp_all_reduce() - ): - final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) - return final_hidden_states.view(num_tokens, hidden_dim) def get_moe_weights(self): @@ -414,7 +398,7 @@ class LongcatFlashDecoderLayer(nn.Module): if "mlps" in getattr(config, "disable_quant_module", []) else quant_config ), - reduce_results=True, + reduce_results=False, prefix=add_prefix(f"mlps.{i}", prefix), ) for i in range(2) diff --git a/python/sglang/srt/models/longcat_flash_nextn.py b/python/sglang/srt/models/longcat_flash_nextn.py index 12ee1678ee..cb501bb3bc 100644 --- a/python/sglang/srt/models/longcat_flash_nextn.py +++ b/python/sglang/srt/models/longcat_flash_nextn.py @@ -154,7 +154,7 @@ class LongcatFlashDenseDecoderLayer(nn.Module): intermediate_size=config.intermediate_size, hidden_act=config.hidden_act, quant_config=quant_config, - reduce_results=True, + reduce_results=False, prefix=add_prefix(f"mlps", prefix), ) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) @@ -191,10 +191,9 @@ class LongcatFlashDenseDecoderLayer(nn.Module): hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - hidden_states = self.mlp(hidden_states) - hidden_states = self.ffn_boundary.finish_complete_output( - hidden_states, forward_batch - ) + with self.ffn_boundary.exit(forward_batch) as ffn_exit: + hidden_states = self.mlp(hidden_states) + hidden_states = ffn_exit.finish(hidden_states) return hidden_states diff --git a/python/sglang/srt/models/mellum.py b/python/sglang/srt/models/mellum.py index 4b569f63d1..ee1c076e79 100644 --- a/python/sglang/srt/models/mellum.py +++ b/python/sglang/srt/models/mellum.py @@ -436,6 +436,7 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer): hidden_act=cfg.hidden_act, quant_config=quant_config, prefix=add_prefix("mlp", prefix), + reduce_results=False, ) is_previous_layer_sparse = _is_sparse(layer_id - 1) diff --git a/python/sglang/srt/models/mimo_v2.py b/python/sglang/srt/models/mimo_v2.py index da4a6f963d..6ad1a23885 100644 --- a/python/sglang/srt/models/mimo_v2.py +++ b/python/sglang/srt/models/mimo_v2.py @@ -52,7 +52,6 @@ from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import ( get_moe_a2a_backend, get_moe_runner_backend, - reduce_moe_output, ) from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat @@ -499,8 +498,6 @@ class MiMoV2MoE(nn.Module): final_hidden_states = self.experts(hidden_states, topk_output) - final_hidden_states = reduce_moe_output(final_hidden_states) - return final_hidden_states def forward_deepep( @@ -852,6 +849,7 @@ class MiMoV2DecoderLayer(nn.Module): prefix=add_prefix("mlp", prefix), tp_rank=mlp_tp_rank, tp_size=mlp_tp_size, + reduce_results=False, ) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.layernorm_epsilon) @@ -945,7 +943,7 @@ class MiMoV2DecoderLayer(nn.Module): ) def op_comm_postprocess_layer(self, state): - hidden_states = self.ffn_boundary.finish_complete_output( + hidden_states = self.ffn_boundary.complete_now( state.pop("hidden_states_mlp_output"), state.forward_batch ) diff --git a/python/sglang/srt/models/mimo_v2_nextn.py b/python/sglang/srt/models/mimo_v2_nextn.py index 57ae19213a..d702604537 100644 --- a/python/sglang/srt/models/mimo_v2_nextn.py +++ b/python/sglang/srt/models/mimo_v2_nextn.py @@ -113,6 +113,7 @@ class MiMoV2MTPLayer(nn.Module): prefix=add_prefix("mlp", prefix), tp_rank=mlp_tp_rank, tp_size=mlp_tp_size, + reduce_results=False, ) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.layernorm_epsilon) self.post_attention_layernorm = RMSNorm( diff --git a/python/sglang/srt/models/minimax_m2.py b/python/sglang/srt/models/minimax_m2.py index 9c5d8339ed..b2e1f3f06e 100644 --- a/python/sglang/srt/models/minimax_m2.py +++ b/python/sglang/srt/models/minimax_m2.py @@ -53,7 +53,6 @@ from sglang.srt.layers.linear import ( from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import ( get_moe_a2a_backend, - reduce_moe_output, ) from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE @@ -602,7 +601,6 @@ class MiniMaxM2MoE(nn.Module): topk_output = self.topk.empty_topk_output(hidden_states.device) final_hidden_states = self.experts(hidden_states, topk_output) - final_hidden_states = reduce_moe_output(final_hidden_states) return final_hidden_states.view(num_tokens, hidden_dim) diff --git a/python/sglang/srt/models/minimax_m3.py b/python/sglang/srt/models/minimax_m3.py index 929b9da8e5..db7de34afd 100644 --- a/python/sglang/srt/models/minimax_m3.py +++ b/python/sglang/srt/models/minimax_m3.py @@ -28,9 +28,6 @@ from sglang.srt.configs.model_config import ( get_minimax_sparse_disable_value_layer_ids, get_minimax_sparse_layer_ids, ) -from sglang.srt.distributed import ( - tensor_model_parallel_all_reduce, -) from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo @@ -315,15 +312,10 @@ class MiniMaxM3MLP(nn.Module): self, x, forward_batch: Optional[ForwardBatch] = None, - should_allreduce_fusion: bool = False, - use_reduce_scatter: bool = False, ): gate_up, _ = self.gate_up_proj(x) x = self.act_fn(gate_up) - x, _ = self.down_proj( - x, - skip_all_reduce=should_allreduce_fusion or use_reduce_scatter, - ) + x, _ = self.down_proj(x) return x @@ -432,22 +424,13 @@ class MiniMaxM3MoE(nn.Module): self, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - should_allreduce_fusion: bool = False, - use_reduce_scatter: bool = False, ) -> torch.Tensor: if get_moe_a2a_backend().is_deepep(): return self.forward_deepep(hidden_states, forward_batch) else: - return self.forward_normal( - hidden_states, should_allreduce_fusion, use_reduce_scatter - ) + return self.forward_normal(hidden_states) - def forward_normal( - self, - hidden_states: torch.Tensor, - should_allreduce_fusion: bool = False, - use_reduce_scatter: bool = False, - ) -> torch.Tensor: + def forward_normal(self, hidden_states: torch.Tensor) -> torch.Tensor: if hidden_states.shape[0] > 0: if ( self.alt_stream is not None @@ -470,9 +453,6 @@ class MiniMaxM3MoE(nn.Module): if shared_output is not None: final_hidden_states = final_hidden_states + shared_output - if self.tp_size > 1 and not should_allreduce_fusion and not use_reduce_scatter: - final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) - return final_hidden_states def _forward_router_experts(self, hidden_states: torch.Tensor) -> torch.Tensor: @@ -1325,6 +1305,7 @@ class MiniMaxM3DecoderLayer(nn.Module): intermediate_size=config.dense_intermediate_size, tp_rank=mlp_tp_rank, tp_size=mlp_tp_size, + reduce_results=False, ) self.use_gemma_norm = getattr(config, "use_gemma_norm", False) @@ -1395,12 +1376,7 @@ class MiniMaxM3DecoderLayer(nn.Module): with self.ffn_boundary.exit(forward_batch) as ffn_exit: if self.is_layer_sparse or hidden_states.shape[0] != 0: - hidden_states = self.mlp( - hidden_states, - forward_batch=forward_batch, - should_allreduce_fusion=ffn_exit.fuse_mlp_allreduce, - use_reduce_scatter=ffn_exit.mlp_reduce_scatter, - ) + hidden_states = self.mlp(hidden_states, forward_batch=forward_batch) return ffn_exit.finish(hidden_states) diff --git a/python/sglang/srt/models/mixtral.py b/python/sglang/srt/models/mixtral.py index 2b759e9833..90b5460c22 100644 --- a/python/sglang/srt/models/mixtral.py +++ b/python/sglang/srt/models/mixtral.py @@ -25,9 +25,6 @@ import torch from torch import nn from transformers import MixtralConfig -from sglang.srt.distributed import ( - tensor_model_parallel_all_reduce, -) from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, @@ -118,8 +115,6 @@ class MixtralMoE(nn.Module): router_logits, _ = self.gate(hidden_states) topk_output = self.topk(hidden_states, router_logits) final_hidden_states = self.experts(hidden_states, topk_output) - if self.tp_size > 1: - final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) return final_hidden_states.view(orig_shape) @@ -269,8 +264,9 @@ class MixtralDecoderLayer(nn.Module): # Fully Connected hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - hidden_states = self.block_sparse_moe(hidden_states) - return self.ffn_boundary.finish_complete_output(hidden_states, forward_batch) + with self.ffn_boundary.exit(forward_batch) as ffn_exit: + hidden_states = self.block_sparse_moe(hidden_states) + return ffn_exit.finish(hidden_states) class MixtralModel(nn.Module): diff --git a/python/sglang/srt/models/moss_vl.py b/python/sglang/srt/models/moss_vl.py index 664c15df6c..d6f3005049 100644 --- a/python/sglang/srt/models/moss_vl.py +++ b/python/sglang/srt/models/moss_vl.py @@ -849,6 +849,7 @@ class MossVLTextMLP(nn.Module): hidden_act: str = "silu", quant_config: Optional[QuantizationConfig] = None, prefix: str = "", + reduce_results: bool = True, ): super().__init__() self.gate_up_proj = MergedColumnParallelLinear( @@ -863,6 +864,7 @@ class MossVLTextMLP(nn.Module): hidden_size, bias=False, quant_config=quant_config, + reduce_results=reduce_results, prefix=add_prefix("down_proj", prefix), ) if hidden_act != "silu": @@ -996,6 +998,7 @@ class MossVLSelfAttentionDecoderLayer(nn.Module): hidden_act=config.hidden_act, quant_config=quant_config, prefix=add_prefix("mlp", prefix), + reduce_results=False, ) norm_kwargs = ( dict( diff --git a/python/sglang/srt/models/nanbeige.py b/python/sglang/srt/models/nanbeige.py index 169c5883a2..c507500d79 100644 --- a/python/sglang/srt/models/nanbeige.py +++ b/python/sglang/srt/models/nanbeige.py @@ -77,6 +77,7 @@ class NanbeigeMLP(nn.Module): hidden_size, bias=False, quant_config=quant_config, + reduce_results=False, prefix=add_prefix("down_proj", prefix), ) if hidden_act != "silu": @@ -261,8 +262,9 @@ class NanbeigeDecoderLayer(nn.Module): # Fully Connected hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - hidden_states = self.mlp(hidden_states) - return self.ffn_boundary.finish_complete_output(hidden_states, forward_batch) + with self.ffn_boundary.exit(forward_batch) as ffn_exit: + hidden_states = self.mlp(hidden_states) + return ffn_exit.finish(hidden_states) class NanbeigeModel(nn.Module): diff --git a/python/sglang/srt/models/nemotron_h.py b/python/sglang/srt/models/nemotron_h.py index cfaf75c563..f63ceafa0e 100644 --- a/python/sglang/srt/models/nemotron_h.py +++ b/python/sglang/srt/models/nemotron_h.py @@ -49,7 +49,6 @@ from sglang.srt.layers.moe.topk import TopK from sglang.srt.layers.moe.utils import ( RoutingMethodType, get_moe_a2a_backend, - reduce_moe_output, ) from sglang.srt.layers.quantization import QuantizationConfig from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod @@ -79,7 +78,7 @@ from sglang.srt.model_loader.weight_utils import ( ) from sglang.srt.models.nemotron_h_utils import make_stage_boundary from sglang.srt.models.utils import WeightsMapper -from sglang.srt.runtime_context import get_exec, get_forward, get_parallel +from sglang.srt.runtime_context import get_exec, get_parallel from sglang.srt.utils import ( add_prefix, get_current_device_stream_fast, @@ -365,8 +364,6 @@ class NemotronHMoE(nn.Module): elif shared_output is not None: final_hidden_states += shared_output - final_hidden_states = reduce_moe_output(final_hidden_states) - return final_hidden_states.view(num_tokens, hidden_dim) @@ -420,6 +417,7 @@ class NemotronHMLPDecoderLayer(NemotronHMLPLikeDecoderLayer): quant_config=quant_config, bias=config.mlp_bias, prefix=f"{prefix}.mixer", + reduce_results=False, ) self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon) @@ -470,9 +468,7 @@ class NemotronHAttnLikeDecoderLayer(nn.Module): return hidden_states with self.boundary.exit(forward_batch) as mixer_exit: - hidden_states = self._forward_mixer( - hidden_states, forward_batch, mixer_exit.skips_reduction - ) + hidden_states = self._forward_mixer(hidden_states, forward_batch) return mixer_exit.finish(hidden_states) @@ -496,7 +492,7 @@ class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer): rms_norm_eps=config.layer_norm_epsilon, activation=config.mamba_hidden_act, quant_config=quant_config, - reduce_results=True, + reduce_results=False, prefix=f"{prefix}.mixer", ) @@ -525,19 +521,14 @@ class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer): self, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - skip_reduce: bool, ) -> torch.Tensor: if is_in_breakable_cuda_graph(): output = torch.empty_like(hidden_states) - breakable_nemotron_mamba2_with_output( - hidden_states, output, self.layer_id, skip_reduce - ) + breakable_nemotron_mamba2_with_output(hidden_states, output, self.layer_id) return output if is_in_tc_piecewise_cuda_graph(): output = torch.empty_like(hidden_states) - nemotron_mamba2_with_output( - hidden_states, output, self.layer_id, skip_reduce - ) + nemotron_mamba2_with_output(hidden_states, output, self.layer_id) return output return self._forward_mamba(hidden_states, forward_batch) @@ -593,6 +584,7 @@ class NemotronHAttention(nn.Module): quant_config=quant_config, tp_rank=tp_rank, tp_size=tp_size, + reduce_results=False, use_dp_attention_reduce=is_dp_attention_enabled(), prefix=f"{prefix}.o_proj", ) @@ -645,7 +637,6 @@ class NemotronHAttentionDecoderLayer(NemotronHAttnLikeDecoderLayer): self, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - skip_reduce: bool, ) -> torch.Tensor: return self.mixer.forward( hidden_states=hidden_states, forward_batch=forward_batch @@ -1152,7 +1143,6 @@ def nemotron_mamba2_with_output( hidden_states: torch.Tensor, output: torch.Tensor, layer_id: int, - fuse_mlp_allreduce: bool = False, ) -> None: """Split op for Mamba2 forward in piecewise CUDA graph mode.""" context = get_tc_piecewise_forward_context() @@ -1172,12 +1162,7 @@ def nemotron_mamba2_with_output( if hidden_states.shape[0] != num_actual_tokens: hidden_states = hidden_states[:num_actual_tokens] - # This function is an opaque custom op under torch.compile. The caller's - # ForwardFlags scope is Python control-plane state and is no longer active - # when the compiled graph invokes this implementation. Carry the scalar - # across the graph boundary and republish it for RowParallelLinear. - with get_forward().scoped(fuse_mlp_allreduce=fuse_mlp_allreduce): - ret = mamba_layer._forward_mamba(hidden_states, forward_batch) + ret = mamba_layer._forward_mamba(hidden_states, forward_batch) # Copy result back; output may be larger (padded) so only fill actual tokens output[:num_actual_tokens].view(ret.shape).copy_(ret) diff --git a/python/sglang/srt/models/qwen2.py b/python/sglang/srt/models/qwen2.py index 7cc408f924..6e0016d834 100644 --- a/python/sglang/srt/models/qwen2.py +++ b/python/sglang/srt/models/qwen2.py @@ -72,6 +72,7 @@ class Qwen2MLP(nn.Module): hidden_act: str, quant_config: Optional[QuantizationConfig] = None, prefix: str = "", + reduce_results: bool = True, ) -> None: super().__init__() self.gate_up_proj = MergedColumnParallelLinear( @@ -86,6 +87,7 @@ class Qwen2MLP(nn.Module): hidden_size, bias=False, quant_config=quant_config, + reduce_results=reduce_results, prefix=add_prefix("down_proj", prefix), ) if hidden_act != "silu": diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index 8ce079a33c..8a39f5a962 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -62,10 +62,7 @@ from sglang.srt.layers.linear import ( RowParallelLinear, ) from sglang.srt.layers.logits_processor import LogitsProcessor -from sglang.srt.layers.moe import ( - get_moe_a2a_backend, - reduce_moe_output, -) +from sglang.srt.layers.moe import get_moe_a2a_backend, reduce_moe_output from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton import FusedMoE from sglang.srt.layers.moe.topk import StandardTopKOutput, TopK, TopKOutputChecker @@ -73,6 +70,7 @@ from sglang.srt.layers.moe.utils import ( RoutingMethodType, filter_moe_weight_param_global_expert, is_deepep_class_backend, + sum_post_experts_output, uses_per_rank_fused_shared_slots, ) from sglang.srt.layers.quantization.base_config import QuantizationConfig @@ -266,8 +264,11 @@ class Qwen2MoeSparseMoeBlock(nn.Module): is_nextn: bool = False, support_shared_expert_fusion: bool = False, enable_cuda_shared_expert_fusion: bool = False, + reduce_results: bool = True, ): super().__init__() + # False when a stage boundary completes this output's sum. + self.reduce_results = reduce_results self.tp_size = get_parallel().tp_size self.layer_id = layer_id self.alt_stream = alt_stream @@ -871,7 +872,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module): final_hidden_states, gated_shared_output=shared_output, m=num_tokens, - reduce=reduce_moe_output, + reduce=sum_post_experts_output, ) if shared_output is not None: @@ -884,7 +885,8 @@ class Qwen2MoeSparseMoeBlock(nn.Module): ) else: final_hidden_states += shared_output - final_hidden_states = reduce_moe_output(final_hidden_states) + if self.reduce_results: + final_hidden_states = reduce_moe_output(final_hidden_states) # Debug removed - was causing issues during CUDA graph capture @@ -1035,6 +1037,7 @@ class Qwen2MoeDecoderLayer(nn.Module): quant_config=quant_config, alt_stream=alt_stream, prefix=add_prefix("mlp", prefix), + reduce_results=False, ) else: self.mlp = Qwen2MoeMLP( @@ -1043,6 +1046,7 @@ class Qwen2MoeDecoderLayer(nn.Module): hidden_act=config.hidden_act, quant_config=quant_config, prefix=add_prefix("mlp", prefix), + reduce_results=False, ) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.post_attention_layernorm = RMSNorm( diff --git a/python/sglang/srt/models/qwen3.py b/python/sglang/srt/models/qwen3.py index 97a40f1604..2adfc5f746 100644 --- a/python/sglang/srt/models/qwen3.py +++ b/python/sglang/srt/models/qwen3.py @@ -355,6 +355,7 @@ class Qwen3DecoderLayer(nn.Module): hidden_act=config.hidden_act, quant_config=quant_config, prefix=add_prefix("mlp", prefix), + reduce_results=False, ) norm_kwargs = ( diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index 9e7818525d..44cecb6b5b 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -1041,6 +1041,9 @@ class Qwen3_5GatedDeltaNet(nn.Module): class Qwen3_5LinearDecoderLayer(nn.Module): """Qwen3.5 Decoder Layer with Linear Attention (GatedDeltaNet).""" + # True in a subclass built without stage boundaries. + _ffn_sums_itself = False + def __init__( self, config: Qwen3_5TextConfig, @@ -1073,6 +1076,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module): prefix=add_prefix("mlp", prefix.replace(".linear_attn", "")), is_nextn=is_nextn, support_shared_expert_fusion=not _disable_shared_experts_fusion(), + reduce_results=self._ffn_sums_itself, ) is_layer_sparse = True is_previous_layer_sparse = True @@ -1084,6 +1088,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module): hidden_act=config.hidden_act, quant_config=quant_config, prefix=add_prefix("mlp", prefix.replace(".linear_attn", "")), + reduce_results=self._ffn_sums_itself, ) _maybe_enable_silu_fp4_quant_fusion(self.mlp) is_layer_sparse = False @@ -1169,6 +1174,9 @@ class Qwen3_5LinearDecoderLayer(nn.Module): class Qwen3_5AttentionDecoderLayer(nn.Module): """Qwen3.5 Decoder Layer with Full Attention.""" + # See Qwen3_5LinearDecoderLayer._ffn_sums_itself. + _ffn_sums_itself = False + def __init__( self, config: Qwen3_5TextConfig, @@ -1271,6 +1279,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): hidden_act=config.hidden_act, quant_config=quant_config, prefix=add_prefix("mlp", prefix.replace(".self_attn", "")), + reduce_results=self._ffn_sums_itself, ) is_layer_sparse = False is_previous_layer_sparse = False @@ -1288,6 +1297,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): prefix=add_prefix("mlp", prefix.replace(".self_attn", "")), is_nextn=is_nextn, support_shared_expert_fusion=not _disable_shared_experts_fusion(), + reduce_results=self._ffn_sums_itself, ) is_layer_sparse = True is_previous_layer_sparse = True diff --git a/python/sglang/srt/models/qwen3_moe.py b/python/sglang/srt/models/qwen3_moe.py index 159e74e973..2c76111952 100644 --- a/python/sglang/srt/models/qwen3_moe.py +++ b/python/sglang/srt/models/qwen3_moe.py @@ -45,7 +45,6 @@ from sglang.srt.layers.linear import ( from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import ( get_moe_a2a_backend, - post_experts_all_reduce, ) from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE @@ -342,9 +341,6 @@ class Qwen3MoeSparseMoeBlock(nn.Module): else: topk_output = self.topk.empty_topk_output(hidden_states.device) final_hidden_states = self.experts(hidden_states, topk_output) - - final_hidden_states = post_experts_all_reduce(final_hidden_states) - return final_hidden_states.view(num_tokens, hidden_dim) def forward_deepep( @@ -856,6 +852,7 @@ class Qwen3MoeDecoderLayer(nn.Module): hidden_act=config.hidden_act, quant_config=quant_config, prefix=add_prefix("mlp", prefix), + reduce_results=False, ) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.post_attention_layernorm = RMSNorm( @@ -941,7 +938,7 @@ class Qwen3MoeDecoderLayer(nn.Module): ) def op_comm_postprocess_layer(self, state): - hidden_states = self.ffn_boundary.finish_complete_output( + hidden_states = self.ffn_boundary.complete_now( state.pop("hidden_states_mlp_output"), state.forward_batch ) diff --git a/python/sglang/srt/models/qwen3_next.py b/python/sglang/srt/models/qwen3_next.py index 20ba669bbc..a570d0c66b 100644 --- a/python/sglang/srt/models/qwen3_next.py +++ b/python/sglang/srt/models/qwen3_next.py @@ -524,6 +524,7 @@ class Qwen3HybridLinearDecoderLayer(nn.Module): is_nextn=is_nextn, support_shared_expert_fusion=True, enable_cuda_shared_expert_fusion=True, + reduce_results=False, ) else: self.mlp = Qwen2MoeMLP( @@ -532,6 +533,7 @@ class Qwen3HybridLinearDecoderLayer(nn.Module): hidden_act=config.hidden_act, quant_config=quant_config, prefix=add_prefix("mlp", prefix.replace(".linear_attn", "")), + reduce_results=False, ) self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.post_attention_layernorm = GemmaRMSNorm( @@ -688,6 +690,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module): is_nextn=is_nextn, support_shared_expert_fusion=True, enable_cuda_shared_expert_fusion=True, + reduce_results=False, ) else: self.mlp = Qwen2MoeMLP( @@ -696,6 +699,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module): hidden_act=config.hidden_act, quant_config=quant_config, prefix=add_prefix("mlp", prefix.replace(".self_attn", "")), + reduce_results=False, ) self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.post_attention_layernorm = GemmaRMSNorm( diff --git a/python/sglang/srt/models/qwen4_exp.py b/python/sglang/srt/models/qwen4_exp.py index ee3ddccd74..0894eac174 100644 --- a/python/sglang/srt/models/qwen4_exp.py +++ b/python/sglang/srt/models/qwen4_exp.py @@ -1410,6 +1410,9 @@ class Qwen4ExpPLELayer(nn.Module): class Qwen4ExpLayerExtensionMixin: + # These layers drop the stage boundaries they inherit. + _ffn_sums_itself = True + def _init_qwen4_exp_layer_extensions( self, config: Qwen4ExpTextConfig, diff --git a/python/sglang/srt/models/sarvam_moe.py b/python/sglang/srt/models/sarvam_moe.py index 8358e96d53..cabd911796 100644 --- a/python/sglang/srt/models/sarvam_moe.py +++ b/python/sglang/srt/models/sarvam_moe.py @@ -34,7 +34,6 @@ from sglang.srt.layers.linear import ( RowParallelLinear, ) from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput -from sglang.srt.layers.moe import reduce_moe_output from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.moe.topk import TopK @@ -368,7 +367,6 @@ class SarvamMoESparseMoeBlock(nn.Module): final_hidden_states = final_hidden_states * self.routed_scaling_factor current_stream.wait_stream(self.alt_stream) final_hidden_states = final_hidden_states + shared_out - final_hidden_states = reduce_moe_output(final_hidden_states) return final_hidden_states.view(num_tokens, hidden_dim) def forward_normal( @@ -403,8 +401,6 @@ class SarvamMoESparseMoeBlock(nn.Module): elif self.routed_scaling_factor != 1.0: final_hidden_states = final_hidden_states * self.routed_scaling_factor - final_hidden_states = reduce_moe_output(final_hidden_states) - return final_hidden_states.view(num_tokens, hidden_dim) @@ -1038,6 +1034,7 @@ class SarvamMoEMLADecoderLayer(nn.Module): prefix=add_prefix("mlp", prefix), tp_rank=mlp_tp_rank, tp_size=mlp_tp_size, + reduce_results=False, ) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) diff --git a/python/sglang/srt/models/sdar.py b/python/sglang/srt/models/sdar.py index c51ad528ba..1934110f30 100644 --- a/python/sglang/srt/models/sdar.py +++ b/python/sglang/srt/models/sdar.py @@ -290,7 +290,7 @@ class SDARBlock(nn.Module): self.mlp = SDARMLP( config=config, quant_config=quant_config, - reduce_results=True, + reduce_results=False, prefix=add_prefix("mlp", prefix), ) diff --git a/python/sglang/srt/models/sdar_moe.py b/python/sglang/srt/models/sdar_moe.py index 3af99a83e3..fd6bfff8f7 100644 --- a/python/sglang/srt/models/sdar_moe.py +++ b/python/sglang/srt/models/sdar_moe.py @@ -31,7 +31,6 @@ from sglang.srt.layers.linear import ( from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import ( get_moe_a2a_backend, - reduce_moe_output, ) from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE @@ -147,8 +146,6 @@ class SDARMoeSparseMoeBlock(nn.Module): topk_output = self.topk(hidden_states, router_logits) out = self.experts(hidden_states, topk_output) # (T, H) - out = reduce_moe_output(out) - return out.view(num_tokens, hidden_dim) def forward_deepep(self, hidden_states: torch.Tensor, forward_batch: ForwardBatch): diff --git a/python/sglang/srt/models/spark2_5.py b/python/sglang/srt/models/spark2_5.py index 9460da4ce3..bf9ca4ae33 100644 --- a/python/sglang/srt/models/spark2_5.py +++ b/python/sglang/srt/models/spark2_5.py @@ -264,6 +264,7 @@ class Spark2_5DecoderLayer(nn.Module): intermediate_size=config.intermediate_size, quant_config=quant_config, prefix=add_prefix("mlp", prefix), + reduce_results=False, ) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) @@ -295,8 +296,9 @@ class Spark2_5DecoderLayer(nn.Module): ) hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - hidden_states = self.mlp(hidden_states) - return self.ffn_boundary.finish_complete_output(hidden_states, forward_batch) + with self.ffn_boundary.exit(forward_batch) as ffn_exit: + hidden_states = self.mlp(hidden_states) + return ffn_exit.finish(hidden_states) class Spark2_5Model(nn.Module): diff --git a/python/sglang/srt/models/step3_vl.py b/python/sglang/srt/models/step3_vl.py index ab1ffc4c46..f3ad5edd12 100644 --- a/python/sglang/srt/models/step3_vl.py +++ b/python/sglang/srt/models/step3_vl.py @@ -41,7 +41,6 @@ from sglang.srt.layers.moe import get_moe_a2a_backend from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton import FusedMoE from sglang.srt.layers.moe.topk import TopK -from sglang.srt.layers.moe.utils import reduce_moe_output from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.layers.rotary_embedding import get_rope @@ -346,6 +345,7 @@ class Step3TextDecoderLayer(nn.Module): hidden_act="silu", quant_config=quant_config, prefix=add_prefix("mlp", prefix), + reduce_results=False, ) else: self.use_moe = True @@ -397,7 +397,7 @@ class Step3TextDecoderLayer(nn.Module): hidden_states += self.share_expert(h) else: hidden_states = self.moe(hidden_states) - return reduce_moe_output(hidden_states) + return hidden_states def forward( self, diff --git a/python/sglang/srt/models/step3p5.py b/python/sglang/srt/models/step3p5.py index 73915bdec6..7514ed13be 100644 --- a/python/sglang/srt/models/step3p5.py +++ b/python/sglang/srt/models/step3p5.py @@ -27,7 +27,6 @@ from sglang.srt.layers.linear import ( from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import ( get_moe_a2a_backend, - reduce_moe_output, should_use_flashinfer_cutlass_moe_fp4_allgather, ) from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class @@ -226,7 +225,6 @@ class Step3p5MoEMLP(nn.Module): final_hidden_states = self.experts(hidden_states, topk_output) if shared_output is not None: final_hidden_states += shared_output - final_hidden_states = reduce_moe_output(final_hidden_states) return final_hidden_states.view(num_tokens, hidden_dim) @@ -506,6 +504,7 @@ class Step3p5DecoderLayer(nn.Module): swiglu_limit=swiglu_limit_shared, quant_config=quant_config, prefix=add_prefix("mlp", prefix), + reduce_results=False, ) self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) diff --git a/python/sglang/srt/models/xllm.py b/python/sglang/srt/models/xllm.py index 17ba89b54c..928e59c9fc 100644 --- a/python/sglang/srt/models/xllm.py +++ b/python/sglang/srt/models/xllm.py @@ -58,7 +58,6 @@ from sglang.srt.layers.linear import ( from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import ( get_moe_a2a_backend, - reduce_moe_output, ) from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton import FusedMoE @@ -1059,7 +1058,6 @@ class XllmSparseMoeBlock(nn.Module): if shared_output is not None: final_hidden_states += shared_output - final_hidden_states = reduce_moe_output(final_hidden_states) return final_hidden_states.view(num_tokens, hidden_dim) @@ -1520,6 +1518,7 @@ class XllmDecoderLayer(nn.Module): prefix=add_prefix("mlp", prefix), tp_rank=mlp_tp_rank, tp_size=mlp_tp_size, + reduce_results=False, ) self.input_layernorm = _make_norm(config) diff --git a/python/sglang/test/boundary_fixtures.py b/python/sglang/test/boundary_fixtures.py index f4b9a67c2b..680b1694ab 100644 --- a/python/sglang/test/boundary_fixtures.py +++ b/python/sglang/test/boundary_fixtures.py @@ -152,7 +152,7 @@ def postprocess_output(boundary, hidden, residual, forward_batch): stream = ( residual if isinstance(residual, ResidualStream) else ResidualStream(residual) ) - hidden = boundary.finish_complete_output(hidden, stream, forward_batch) + hidden = boundary.complete_now(hidden, stream, forward_batch) if isinstance(residual, ResidualStream): return hidden, stream return (hidden, None) if stream.pending is None else stream.input(hidden) diff --git a/test/registered/unit/layer_boundary/test_boundary_output_contracts.py b/test/registered/unit/layer_boundary/test_boundary_output_contracts.py index 243bf8512f..931640eb5b 100644 --- a/test/registered/unit/layer_boundary/test_boundary_output_contracts.py +++ b/test/registered/unit/layer_boundary/test_boundary_output_contracts.py @@ -36,20 +36,22 @@ class TestBoundaryIntegrations(unittest.TestCase): parallel = fixture.parallel_of(attn_dp=2, attn_tp=2) rows = fixture.comm.Layout(frozenset()) produced = fixture.comm.OutputContract( - rows, group=SumGroup.ATTN_TP, may_defer_to_next=True + rows, group=SumGroup.ATTN_TP, may_defer_to_next=True, update=PLAIN_ADD ) plan = SimpleNamespace(path_for=lambda _: SimpleNamespace(output=produced)) boundary = exits.ExitPolicy(plan) boundary._sum_deferral_allowed = lambda _: True + summed = Mock(side_effect=lambda value, *args, **kwargs: value * 2) with ( fixture.planning(parallel), patch.object(exits, "is_dp_attention_enabled", return_value=True), + patch.object(exits, "sum_output", summed), ): stream = ResidualStream(torch.zeros(3, 4)) result = exits.MixerExit(boundary, None, stream=stream) - self.assertFalse(result.skips_reduction) value = torch.ones(3, 4) - self.assertIs(result.finish(value), value) + torch.testing.assert_close(result.finish(value), value * 2) + self.assertEqual(summed.call_args.args[1], SumGroup.ATTN_TP) self.assertIsNone(stream.pending.owed) def test_pipeline_preserves_declared_sum_for_one_receiver_completion(self): diff --git a/test/registered/unit/layer_boundary/test_boundary_output_policy.py b/test/registered/unit/layer_boundary/test_boundary_output_policy.py index d00944d79f..0d6bc2a68d 100644 --- a/test/registered/unit/layer_boundary/test_boundary_output_policy.py +++ b/test/registered/unit/layer_boundary/test_boundary_output_policy.py @@ -96,37 +96,24 @@ class TestBoundaryOutputPolicy(unittest.TestCase): reset_context() def test_step3_shared_and_routed_outputs_have_one_completion(self): + """The MoE adds its shared and routed partial outputs and leaves their + one sum to the stage boundary.""" from sglang.srt.models.step3_vl import Step3TextDecoderLayer - parallel = SimpleNamespace(tp_size=2) for fused_shared in (False, True): - for deferred in (False, True): - calls = [] - layer = SimpleNamespace( - num_fused_shared_experts=int(fused_shared), - moe=lambda x: x * (0.75 if fused_shared else 0.25), - share_expert=lambda x: x * 0.5, - ) - with ( - patch.object(moe_utils, "get_parallel", return_value=parallel), - patch.object( - moe_utils, "post_experts_output_is_complete", return_value=False - ), - patch( - "sglang.srt.distributed.communication_op.tensor_model_parallel_all_reduce", - side_effect=lambda x: calls.append("AR") or x * 2, - ), - get_forward().scoped( - fuse_mlp_allreduce=deferred, mlp_reduce_scatter=False - ), - ): - result = Step3TextDecoderLayer.moe_mlp_forward( - layer, torch.ones(2, 4) - ) - self.assertEqual(calls, [] if deferred else ["AR"]) - torch.testing.assert_close( - result, torch.full((2, 4), 0.75 if deferred else 1.5) - ) + calls = [] + layer = SimpleNamespace( + num_fused_shared_experts=int(fused_shared), + moe=lambda x: x * (0.75 if fused_shared else 0.25), + share_expert=lambda x: x * 0.5, + ) + with patch( + "sglang.srt.distributed.communication_op.tensor_model_parallel_all_reduce", + side_effect=lambda x: calls.append("AR") or x * 2, + ): + result = Step3TextDecoderLayer.moe_mlp_forward(layer, torch.ones(2, 4)) + self.assertEqual(calls, []) + torch.testing.assert_close(result, torch.full((2, 4), 0.75)) def test_dp_skip_and_collective_are_selected_together(self): import itertools @@ -147,6 +134,10 @@ class TestBoundaryOutputPolicy(unittest.TestCase): calls.append(step.__name__) return value[:1] + def summed(value, *args, **kwargs): + calls.append("AR") + return value * 2 + selected = ( "dp_reduce_scatterv" if varlen and policy in ("rsv", "rs+rsv") @@ -159,21 +150,25 @@ class TestBoundaryOutputPolicy(unittest.TestCase): patch_communicator("should_use_dp_reduce_scatterv", lambda: varlen), patch_communicator("can_use_dp_reduce_scatter", lambda: can_rs), patch_communicator("to_dp_local", move), + patch_communicator("sum_output", summed), ): with layer.ffn.plan.output.ffn_exit( self.batch(max_len), stream=ResidualStream() ) as output: - self.assertEqual( - get_forward().mlp_reduce_scatter, - selected != "_dp_scatter_step", - ) + self.assertFalse(get_forward().mlp_reduce_scatter) result, _ = finish_exit(output, torch.ones(2, 4), torch.zeros(1, 4)) - self.assertEqual(calls, [selected]) + # A reduce-scatter completes the sum on the way back; a scatter + # follows the exit's own all-reduce. + self.assertEqual( + calls, + ["AR", selected] if selected == "_dp_scatter_step" else [selected], + ) self.assertEqual(result.shape, (1, 4)) - def test_postprocess_only_scatters_an_already_reduced_output(self): - # The operation-scheduled API used by LongCat NextN receives an MLP - # output that was already summed. An enabled RSv must not sum it again. + def test_complete_now_sums_then_only_scatters(self): + # The operation-scheduled path (two-batch overlap) never defers: it + # completes the sum its output owes, then only scatters, whatever the + # reduce-scatter policy. for policy in ("ar", "rs", "rsv", "rs+rsv"): for varlen in (False, True): with self.subTest(policy=policy, varlen=varlen): @@ -191,6 +186,10 @@ class TestBoundaryOutputPolicy(unittest.TestCase): calls.append(step.__name__) return value[:1] + def summed(value, *args, **kwargs): + calls.append("AR") + return value * 2 + with ( fixture.planning(parallel), patch_communicator( @@ -198,11 +197,12 @@ class TestBoundaryOutputPolicy(unittest.TestCase): ), patch_communicator("can_use_dp_reduce_scatter", lambda: True), patch_communicator("to_dp_local", move), + patch_communicator("sum_output", summed), ): - output = layer.ffn.finish_complete_output(hidden, batch) + output = layer.ffn.complete_now(hidden, batch) value, saved_residual = batch.residual_stream.export(output) - self.assertEqual(calls, ["_dp_scatter_step"]) - torch.testing.assert_close(value, hidden[:1]) + self.assertEqual(calls, ["AR", "_dp_scatter_step"]) + torch.testing.assert_close(value, hidden[:1] * 2) self.assertIs(saved_residual, residual) def test_moe_cannot_skip_from_topology_without_boundary_request(self): @@ -221,9 +221,9 @@ class TestBoundaryOutputPolicy(unittest.TestCase): skip, ) - def test_output_transform_stays_before_transport_and_after_producer_ar(self): - # The compute stub obeys the same skip flag as RowParallelLinear. - # Its trace distinguishes AR -> multiply from multiply -> RS. + def test_output_transform_stays_before_transport_and_after_the_sum(self): + # The trace distinguishes AR -> multiply from multiply -> RS; the + # exit runs the all-reduce, the compute never does. for disabled in (False, True): with self.subTest(disabled=disabled): calls = [] @@ -248,19 +248,21 @@ class TestBoundaryOutputPolicy(unittest.TestCase): calls.append("slice") return value[:1] + def summed(value, *args, **kwargs): + calls.append("AR") + return value * 2 + with ( fixture.planning(parallel), patch_communicator("should_use_dp_reduce_scatterv", lambda: False), patch_communicator("can_use_dp_reduce_scatter", lambda: True), patch_communicator("to_dp_local", move), + patch_communicator("sum_output", summed), ): with layer.ffn.plan.output.ffn_exit( self.batch(), stream=ResidualStream() ) as output: value = torch.ones(2, 4) - if not get_forward().mlp_reduce_scatter: - calls.append("AR") - value = value * 2 value, _ = finish_exit(output, value, torch.zeros(1, 4)) self.assertEqual( calls, diff --git a/test/registered/unit/layer_boundary/test_declared_attention_cp.py b/test/registered/unit/layer_boundary/test_declared_attention_cp.py index 83afd4ce81..6e7ca879f1 100644 --- a/test/registered/unit/layer_boundary/test_declared_attention_cp.py +++ b/test/registered/unit/layer_boundary/test_declared_attention_cp.py @@ -188,8 +188,9 @@ class TestAttentionCpBoundary(CustomTestCase): ) def run_ranks(self, use_reduce_scatter): - """Gather on each rank, run an FFN that leaves a partial sum or not, - finish the exit and return what each rank got back and published.""" + """Gather on each rank, run an FFN that leaves its partial sum, finish + the exit and return the ranks the exit summed over, what each rank got + back, and the ranks the reduce-scatter summed over.""" handed = {} def record_gather(cp): @@ -230,10 +231,17 @@ class TestAttentionCpBoundary(CustomTestCase): for cp in range(CP_SIZE): torch.testing.assert_close(gathered[cp], expected_rows, rtol=0, atol=0) - def compute_output(cp, leaves): - return gathered[cp] * PARTIAL_WEIGHTS[cp] if leaves else gathered[cp] + reduced, summed = {}, {} - reduced = {} + def record_sum(cp): + def sum_output(value, *args, **kwargs): + summed[cp] = value.clone() + return value + + return sum_output + + def fill_sum(value, *args, **kwargs): + return sum(summed[r] for r in range(CP_SIZE)) def record_reduce_scatter(cp): def reduce_scatter(output, input_): @@ -248,7 +256,7 @@ class TestAttentionCpBoundary(CustomTestCase): return reduce_scatter - published, back = {}, {} + back = {} for phase in ("record", "fill"): for cp in range(CP_SIZE): reduce_scatter = ( @@ -256,33 +264,38 @@ class TestAttentionCpBoundary(CustomTestCase): if phase == "record" else fill_reduce_scatter(cp) ) - with self.as_rank( - cp, dict(gather=fill_gather, reduce_scatter=reduce_scatter) - ) as rank: + with ( + self.as_rank( + cp, dict(gather=fill_gather, reduce_scatter=reduce_scatter) + ) as rank, + patch_communicator( + "sum_output", record_sum(cp) if phase == "record" else fill_sum + ), + ): communicator = self.build(use_reduce_scatter) with communicator.ffn.plan.output.ffn_exit( self.cp_extend(), stream=ResidualStream() ) as exit_: - published[cp] = rank.flags.mlp_reduce_scatter - output = compute_output(cp, leaves=published[cp]) + self.assertFalse(rank.flags.mlp_reduce_scatter) + output = gathered[cp] * PARTIAL_WEIGHTS[cp] back[cp], _ = finish_exit( exit_, output, ResidualStream(residuals[cp].residual) ) - return published, back, reduced + return summed, back, reduced def test_the_reduce_scatter_completes_the_sum_the_moe_leaves(self): - published, back, reduced = self.run_ranks(use_reduce_scatter=True) - self.assertEqual(published, {0: True, 1: True}, "the MoE leaves its sum") + summed, back, reduced = self.run_ranks(use_reduce_scatter=True) + self.assertEqual(summed, {}, "the exit runs no all-reduce") self.assertEqual(sorted(reduced), [0, 1], "every rank joins the reduce-scatter") for cp in range(CP_SIZE): torch.testing.assert_close( back[cp], self.values[cp] + self.residuals[cp], rtol=0, atol=0 ) - def test_a_complete_output_is_only_taken_back(self): - published, back, reduced = self.run_ranks(use_reduce_scatter=False) - self.assertEqual(published, {0: False, 1: False}, "the MoE sums itself") - self.assertEqual(reduced, {}, "nothing is summed again") + def test_without_a_reduce_scatter_the_exit_sums_then_takes_back(self): + summed, back, reduced = self.run_ranks(use_reduce_scatter=False) + self.assertEqual(sorted(summed), [0, 1], "the exit sums what the MoE leaves") + self.assertEqual(reduced, {}, "the take-back sums nothing") for cp in range(CP_SIZE): torch.testing.assert_close( back[cp], self.values[cp] + self.residuals[cp], rtol=0, atol=0 diff --git a/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py b/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py index 968d1f5c57..8f17a14ef3 100644 --- a/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py +++ b/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py @@ -53,6 +53,7 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import ( from sglang.srt.layers.layer_boundary.residual.stream import OwedOutput, ResidualStream from sglang.srt.layers.moe import utils as moe_utils from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.srt.runtime_context import get_forward from sglang.test.boundary_fixtures import ( finish_exit, make_test_stages, @@ -415,7 +416,9 @@ class TestMhcOnTheDeclarations(CustomTestCase): with communicator.ffn.plan.output.ffn_exit( forward_batch, stream=ResidualStream() ) as ffn_exit: - self.assertTrue(ffn_exit.mlp_reduce_scatter) + # The reduce-scatter on the way back completes the sum; + # the FFN never runs or skips it. + self.assertFalse(get_forward().mlp_reduce_scatter) hidden, residual = finish_exit( ffn_exit, torch.ones(4, 4), torch.full((2, 4), 2.0) ) @@ -1420,7 +1423,11 @@ class TestBranchRows(CustomTestCase): branch_output = branch.branch_output merge_branch = branch.merge_branch - moe = Communicator(branch_rows=lambda fb: (self.local, self.local, self.local)) + moe = Communicator( + branch_rows=lambda fb: (self.local, self.local, self.local), + # The a2a combine already summed the MoE output. + path_for=lambda fb: SimpleNamespace(output=comm.OutputContract(self.local)), + ) dense = Communicator( branch_rows=lambda fb: (self.full, self.attention, self.attention) ) @@ -1642,6 +1649,12 @@ def running(*, reduce_scatterv, a2a=False, use_reduce_scatter=True): "attention_tensor_model_parallel_all_reduce": lambda x: ( state().parallel.attn_tp_group.all_reduce(x) ), + "tensor_model_parallel_all_reduce": lambda x: ( + state().parallel.tp_group.all_reduce(x) + ), + "sum_post_experts_output": lambda x: ( + comm_layout.post_experts_reduction_group().all_reduce(x) + ), "get_global_dp_buffer": lambda g: torch.zeros( state().global_rows, HIDDEN, dtype=torch.double ), @@ -1709,21 +1722,15 @@ def attention(x, state): def dense_mlp(x, state): - """A dense MLP on the TP group: 5 * x, reduced unless a published flag asks - it to leave the sum, as RowParallelLinear does.""" - partial_sum = 5 * x * WEIGHTS[state.parallel.tp_size][state.parallel.tp_rank] - if state.flags.fuse_mlp_allreduce or state.flags.mlp_reduce_scatter: - return partial_sum - return state.parallel.tp_group.all_reduce(partial_sum) + """A dense MLP on the TP group: 5 * x, as this rank's partial sum; the + stage boundary completes it.""" + return 5 * x * WEIGHTS[state.parallel.tp_size][state.parallel.tp_rank] def moe(x, state): - """A MoE block not dispatched per DP shard: 5 * x, reduced over the MoE - output's group unless a published flag asks it to leave the sum.""" - partial_sum = 5 * x * WEIGHTS[state.parallel.tp_size][state.parallel.tp_rank] - if state.flags.fuse_mlp_allreduce or state.flags.mlp_reduce_scatter: - return partial_sum - return comm_layout.post_experts_reduction_group().all_reduce(partial_sum) + """A MoE block not dispatched per DP shard: 5 * x, as this rank's partial + sum over the MoE output's group; the stage boundary completes it.""" + return 5 * x * WEIGHTS[state.parallel.tp_size][state.parallel.tp_rank] def reference(x): diff --git a/test/registered/unit/layer_boundary/test_dp_attention_cp_gather.py b/test/registered/unit/layer_boundary/test_dp_attention_cp_gather.py index 52c18d1144..4f22e585a9 100644 --- a/test/registered/unit/layer_boundary/test_dp_attention_cp_gather.py +++ b/test/registered/unit/layer_boundary/test_dp_attention_cp_gather.py @@ -1,8 +1,8 @@ """The dense-FFN gather and take-back under attention DP x CP x TP, rank by rank. Every rank builds its layer's communicator and runs its prepare_mlp and -finish_complete_output for real on CPU; the TP-group all-reduce is replaced by the -sum of the buffers the ranks hand to it. +complete_now for real on CPU; the TP-group all-reduce is replaced by the sum of +the buffers the ranks hand to it. """ import unittest @@ -22,6 +22,7 @@ from sglang.srt.layers.cp.zigzag import ZigzagCPStrategy from sglang.srt.layers.dp_attention import DpPaddingMode from sglang.srt.layers.layer_boundary import prepare as comm_ops from sglang.srt.layers.layer_boundary.contracts import BatchVariant +from sglang.srt.layers.layer_boundary.layout import SumGroup from sglang.srt.layers.layer_boundary.residual.add_norm import ( NORM_READOUT, PLAIN_RESIDUAL_OPS, @@ -150,8 +151,9 @@ class TestDpCpGather(CustomTestCase): ffn_input=None, sparse=False, ): - """Gather on every rank, check the FFN input and residual, then take back - an FFN output of three times the input and check each rank's rows. + """Gather on every rank, check the FFN input and residual, then complete + and take back an FFN output of three times the input, of which each rank + holds its share, and check each rank's rows. ``norm_rows`` is what ``norm`` does to complete rows; ``ffn_input``, when given, is the step every rank's batch must take.""" # Binary fractions, so the partial sums add back to the value exactly. @@ -340,13 +342,25 @@ class TestDpCpGather(CustomTestCase): hidden_states, residual = gather(rank, lambda x: summed.clone()) torch.testing.assert_close(hidden_states, expected, rtol=0, atol=0) torch.testing.assert_close(residual.residual[:tokens], own, rtol=0, atol=0) - with as_rank(*rank, None) as r: + tp_size = DP_SIZE * CP_SIZE * attn_tp_size + sums = [] + + def boundary_sum(value, group, *args, **kwargs): + # Every rank holds the same share of the same rows. + sums.append(group) + return value * tp_size + + with ( + as_rank(*rank, None) as r, + patch_communicator("sum_output", boundary_sum), + ): back, _ = postprocess_output( r.communicator.ffn.plan.output, - 3 * hidden_states, + 3 * hidden_states / tp_size, residual, r.forward_batch, ) + self.assertEqual(sums, [SumGroup.MOE_OUTPUT if sparse else SumGroup.TP]) expected_back = torch.zeros(group.held_rows[cp], HIDDEN).double() expected_back[:tokens] = 3 * norm_rows(own) torch.testing.assert_close(back, expected_back, rtol=0, atol=0) diff --git a/test/registered/unit/layer_boundary/test_ffn_exit.py b/test/registered/unit/layer_boundary/test_ffn_exit.py index da19337cfe..4a5aa4a3a2 100644 --- a/test/registered/unit/layer_boundary/test_ffn_exit.py +++ b/test/registered/unit/layer_boundary/test_ffn_exit.py @@ -110,6 +110,8 @@ def make_communicator( def published_flags(): + """The skip flags the FFN sees; the boundary completes the sum, so the FFN + never leaves it out.""" forward = get_forward() return forward.fuse_mlp_allreduce, forward.mlp_reduce_scatter @@ -149,7 +151,7 @@ class TestFfnExit(CustomTestCase): scatters_to_local_tokens=True, ) seen, (hidden_states, residual) = self.run_exit(communicator) - self.assertEqual(seen, (False, True)) + self.assertEqual(seen, (False, False)) self.assertIsInstance(hidden_states, UnreducedOutput) bound = hidden_states.reduce_to_dp_local self.assertIs(bound.func, transport_ops.to_dp_local) @@ -178,9 +180,15 @@ class TestFfnExit(CustomTestCase): fuse=False, reduce_scatter=reduce_scatter ) seen, (hidden_states, residual) = self.run_exit(communicator) - self.assertEqual(seen, (False, reduce_scatter)) + self.assertEqual(seen, (False, False)) self.assertNotIsInstance(hidden_states, UnreducedOutput) communicator.output._complete_now.assert_called_once() + # A reduce-scatter on the way back completes the sum; otherwise + # the exit completes it. + self.assertEqual( + communicator.output._complete_now.call_args.kwargs["owes_sum"], + not reduce_scatter, + ) torch.testing.assert_close(hidden_states, self.hidden_states * 2 + 1) self.assertIs(residual, self.residual) @@ -190,7 +198,7 @@ class TestFfnExit(CustomTestCase): ) seen, (hidden_states, residual) = self.run_exit(communicator) - self.assertEqual(seen, (False, True)) + self.assertEqual(seen, (False, False)) communicator.output._defers_sum.assert_not_called() communicator.output._complete_now.assert_called_once() torch.testing.assert_close(hidden_states, self.hidden_states * 2 + 1) @@ -209,7 +217,7 @@ class TestFfnExit(CustomTestCase): decide.return_value = not fuse hidden_states = self.hidden_states * 2 hidden_states, _ = finish_exit(ffn_exit, hidden_states, self.residual) - self.assertEqual(seen, (fuse, False)) + self.assertEqual(seen, (False, False)) self.assertEqual(isinstance(hidden_states, UnreducedOutput), fuse) self.assertEqual(communicator.output._complete_now.called, not fuse) @@ -233,8 +241,9 @@ class TestFfnExit(CustomTestCase): self.assertEqual(published_flags(), before) def test_deferral_implies_fusion_and_passes_the_handoff_through(self): - """A deferring communicator publishes both flags, and a non-tensor - handoff leaves finish() untouched for the next layer's input norm.""" + """A deferring communicator publishes the finalize choice, and a + non-tensor handoff leaves finish() untouched for the next layer's input + norm.""" fusions = types.SimpleNamespace(can_defer_finalize=lambda layer, fb: True) communicator = make_communicator( @@ -245,7 +254,7 @@ class TestFfnExit(CustomTestCase): self.forward_batch, stream=ResidualStream() ) as ffn_exit: seen = published_flags() + (get_forward().defer_moe_finalize,) - self.assertEqual(seen, (True, False, True)) + self.assertEqual(seen, (False, False, True)) self.assertEqual( finish_exit(ffn_exit, handoff, self.residual), (handoff, self.residual) ) diff --git a/test/registered/unit/layer_boundary/test_longcat_flash_shortcut.py b/test/registered/unit/layer_boundary/test_longcat_flash_shortcut.py index 505896abd1..bc972bafbc 100644 --- a/test/registered/unit/layer_boundary/test_longcat_flash_shortcut.py +++ b/test/registered/unit/layer_boundary/test_longcat_flash_shortcut.py @@ -36,6 +36,8 @@ class TestLongcatShortcut(CustomTestCase): prepare_attn=lambda h, r, batch: (h, r), prepare_mlp=lambda h, r, batch: (fork_hidden, fork_residual), branch_rows=lambda batch: (local, local, attention), + # Each rank's own tokens: the MoE output owes no sum. + path_for=lambda batch: SimpleNamespace(output=comm.OutputContract(local)), ) dense_communicator = Communicator( branch_rows=lambda batch: (attention, attention, attention) diff --git a/test/registered/unit/layer_boundary/test_nemotron_h_aux_capture.py b/test/registered/unit/layer_boundary/test_nemotron_h_aux_capture.py index 6a4fea2876..c93f256b81 100644 --- a/test/registered/unit/layer_boundary/test_nemotron_h_aux_capture.py +++ b/test/registered/unit/layer_boundary/test_nemotron_h_aux_capture.py @@ -4,7 +4,6 @@ from types import SimpleNamespace import torch from torch import nn -from sglang.srt.layers.moe.utils import should_skip_mlp_all_reduce from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.models import nemotron_h as model from sglang.srt.runtime_context import get_context, get_flags, get_parallel @@ -28,7 +27,7 @@ class _Norm(nn.Module): class _Mixer(nn.Module): - """Row-parallel stand-in: a TP partial, reduced unless the layer skips it.""" + """Row-parallel stand-in: a TP partial, whose sum the stage boundary completes.""" def __init__(self, scale, tp=1): super().__init__() @@ -36,8 +35,7 @@ class _Mixer(nn.Module): self.tp = tp def forward(self, hidden_states, **kwargs): - partial = hidden_states * (self.scale / self.tp) - return partial if should_skip_mlp_all_reduce() else partial * self.tp + return hidden_states * (self.scale / self.tp) def _build(pattern, tp, capture): diff --git a/test/registered/unit/layer_boundary/test_nemotron_h_mtp_reduction.py b/test/registered/unit/layer_boundary/test_nemotron_h_mtp_reduction.py index 9ad4e97b86..1c00e1a28b 100644 --- a/test/registered/unit/layer_boundary/test_nemotron_h_mtp_reduction.py +++ b/test/registered/unit/layer_boundary/test_nemotron_h_mtp_reduction.py @@ -25,19 +25,32 @@ class _Norm(nn.Module): return x + residual, x + residual +class _Partial(nn.Module): + def __init__(self, tp): + super().__init__() + self.tp = tp + + def forward(self, x, **kwargs): + return x / self.tp + + class TestNemotronMTPReduction(CustomTestCase): def test_attention_partial_is_reduced_once(self): """Under DP attention, the MTP MoE layer sums the attention output over the - attention TP group exactly once before the residual add.""" + attention TP group exactly once before the residual add, and its own + output over the MoE-TP group once at its exit.""" for tp in (1, 2): with self.subTest(tp=tp): reduce = Mock(side_effect=lambda x: x * tp) group = SimpleNamespace(all_reduce=reduce) + moe_reduce = Mock(side_effect=lambda x: x * tp) + moe_group = SimpleNamespace(all_reduce=moe_reduce) with ( get_context().override_server_args(tp_size=tp), get_flags().dp.override(enabled=True), get_parallel().override( attn_tp_group=group, + moe_tp_group=moe_group, launch_world_rank=0, tp_rank=0, tp_size=tp, @@ -66,7 +79,8 @@ class TestNemotronMTPReduction(CustomTestCase): nn.Module.__init__(layer) layer.has_start_projections = False layer.has_end_norm = False - layer.mixer = nn.Identity() + # A MoE stand-in leaving its output's sum to the exit. + layer.mixer = _Partial(tp) layer.norm = _Norm() layer._init_stage_boundary( SimpleNamespace(hybrid_override_pattern="*E"), 1 @@ -93,6 +107,7 @@ class TestNemotronMTPReduction(CustomTestCase): torch.testing.assert_close(hidden, expected) torch.testing.assert_close(batch.residual_stream.residual, expected) self.assertEqual(reduce.call_count, int(tp > 1)) + self.assertEqual(moe_reduce.call_count, int(tp > 1)) if __name__ == "__main__": diff --git a/test/registered/unit/layer_boundary/test_nemotron_h_stages.py b/test/registered/unit/layer_boundary/test_nemotron_h_stages.py index ca8ef7bfc2..44dfbdb40a 100644 --- a/test/registered/unit/layer_boundary/test_nemotron_h_stages.py +++ b/test/registered/unit/layer_boundary/test_nemotron_h_stages.py @@ -7,6 +7,7 @@ from unittest.mock import MagicMock, patch import torch from sglang.srt.layers.layer_boundary import ( + PLAIN_ADD, Layout, MixerExit, OutputContract, @@ -14,6 +15,7 @@ from sglang.srt.layers.layer_boundary import ( TokenAxis, UnreducedOutput, ) +from sglang.srt.layers.layer_boundary import exit as exit_module from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.srt.layers.moe.utils import should_skip_mlp_all_reduce from sglang.srt.models import nemotron_h_utils as utils @@ -166,9 +168,10 @@ class TestStageEdges(CustomTestCase): class TestMixerExit(CustomTestCase): - """A mixer skips its output all-reduce when its output always leaves the sum - (to an FFN stage), and when it may leave it and the fused kernel takes it; - what it hands on says which.""" + """A mixer never runs its output all-reduce. The exit carries the sum to the + next attention stage when its declaration permits deferring and the fused + kernel takes it, hands it on as the declared sum when it always leaves it + (to an FFN stage), and otherwise completes it itself.""" def test_decision_table(self): tp_group = object() @@ -184,6 +187,7 @@ class TestMixerExit(CustomTestCase): group=SumGroup.ATTN_TP if always or may else None, always_partial=always, may_defer_to_next=may, + update=PLAIN_ADD, ) communicator = SimpleNamespace( plan=SimpleNamespace( @@ -192,22 +196,28 @@ class TestMixerExit(CustomTestCase): _sum_deferral_allowed=MagicMock(return_value=movable), ) hidden = torch.ones(2, 4) - with get_parallel().override(tp_group=tp_group, tp_size=2): + summed = MagicMock(side_effect=lambda h, *args, **kwargs: h * 2) + with ( + get_parallel().override(tp_group=tp_group, tp_size=2), + patch.object(exit_module, "sum_output", summed), + ): with MixerExit( communicator, None, stream=ResidualStream() ) as mixer_exit: skipped = should_skip_mlp_all_reduce() output = mixer_exit.finish(hidden) output, _ = mixer_exit._stream.input(output) - self.assertFalse(should_skip_mlp_all_reduce()) - hands_on = may and movable - self.assertEqual(mixer_exit.skips_reduction, always or hands_on) - self.assertEqual(skipped, always or hands_on) - if hands_on: + self.assertFalse(skipped) + if may and movable: self.assertIsInstance(output, UnreducedOutput) self.assertIs(output.group, tp_group) + summed.assert_not_called() + elif may: + self.assertEqual(summed.call_args.args[1], SumGroup.ATTN_TP) + torch.testing.assert_close(output, hidden * 2) else: self.assertIs(output, hidden) + summed.assert_not_called() if __name__ == "__main__": diff --git a/test/registered/unit/layer_boundary/test_stage_producers_leave_sums.py b/test/registered/unit/layer_boundary/test_stage_producers_leave_sums.py new file mode 100644 index 0000000000..20239e7c4b --- /dev/null +++ b/test/registered/unit/layer_boundary/test_stage_producers_leave_sums.py @@ -0,0 +1,100 @@ +"""Stage producers leave their output sums to the stage boundary.""" + +import ast +import unittest +from pathlib import Path + +import torch + +import sglang.srt.models +from sglang.srt.layers.layer_boundary.stage import ( + StageBoundary, + check_stage_producers, +) +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +# Calls that complete a TP / EP sum of a stage output in compute. +_SUMS = { + "reduce_moe_output", + "post_experts_all_reduce", + "tensor_model_parallel_all_reduce", + "moe_expert_parallel_all_reduce", + "moe_tensor_model_parallel_all_reduce", +} +_BUILDS_STAGES = ( + "make_stages(", + "declare_attn(", + "declare_ffn(", + "_build_stages(", + "_init_stage_boundary(", +) + + +def _call_name(node: ast.Call): + func = node.func + return func.attr if isinstance(func, ast.Attribute) else getattr(func, "id", None) + + +def _unguarded_sums(source: str): + """Sum calls not under ``if ... self.reduce_results ...``: such a branch is + a shared class built without stage boundaries by another model.""" + found = [] + + def visit(node, guarded): + if isinstance(node, ast.If) and "reduce_results" in ast.unparse(node.test): + for child in node.body: + visit(child, True) + for child in node.orelse: + visit(child, guarded) + return + if isinstance(node, ast.Call) and not guarded and _call_name(node) in _SUMS: + found.append(f"{node.lineno}: {ast.unparse(node)}") + for child in ast.iter_child_nodes(node): + visit(child, guarded) + + visit(ast.parse(source), False) + return found + + +class TestStageProducersLeaveSums(CustomTestCase): + def test_stage_models_do_not_sum_stage_outputs_in_compute(self): + (models,) = map(Path, sglang.srt.models.__path__) + checked, offenders = 0, {} + for path in sorted(models.rglob("*.py")): + source = path.read_text() + if not any(marker in source for marker in _BUILDS_STAGES): + continue + checked += 1 + if sums := _unguarded_sums(source): + offenders[str(path.relative_to(models))] = sums + self.assertGreater(checked, 30) + self.assertEqual(offenders, {}) + + def test_guard_rejects_a_producer_that_sums_its_own_output(self): + class Producer(torch.nn.Module): + def __init__(self, reduce_results): + super().__init__() + self.reduce_results = reduce_results + + class Layer(torch.nn.Module): + def __init__(self, reduce_results, *, staged=True): + super().__init__() + self.mlp = Producer(reduce_results) + if staged: + self.ffn_boundary = object.__new__(StageBoundary) + + class Model(torch.nn.Module): + def __init__(self, *layers): + super().__init__() + self.layers = torch.nn.ModuleList(layers) + + check_stage_producers(Model(Layer(False), Layer(True, staged=False))) + with self.assertRaisesRegex(ValueError, r"layers\.1\.mlp"): + check_stage_producers(Model(Layer(False), Layer(True))) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layer_boundary/test_step3p5_dense_reduce_scatter.py b/test/registered/unit/layer_boundary/test_step3p5_dense_reduce_scatter.py deleted file mode 100644 index 96c6f85685..0000000000 --- a/test/registered/unit/layer_boundary/test_step3p5_dense_reduce_scatter.py +++ /dev/null @@ -1,91 +0,0 @@ -"""A Step-3.5 dense layer must leave its all-reduce out when postprocess -reduce-scatters its output; otherwise the output is summed twice. The layer's -FFN exit publishes the decision while the MLP runs, then completes it.""" - -import importlib -import unittest -from types import SimpleNamespace -from unittest.mock import MagicMock - -import torch - -from sglang.srt.layers import layer_boundary as comm -from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream -from sglang.srt.runtime_context import get_forward -from sglang.test.boundary_fixtures import stub_plan -from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.test_utils import CustomTestCase - -register_cpu_ci(est_time=10, suite="base-a-test-cpu") - - -def dense_layer(*, reduce_scatter): - step3p5 = importlib.import_module("sglang.srt.models.step3p5") - layer = step3p5.Step3p5DecoderLayer.__new__(step3p5.Step3p5DecoderLayer) - torch.nn.Module.__init__(layer) - seen = {} - - def mlp(hidden_states): - # RowParallelLinear skips its all-reduce when either flag is published. - seen["mlp_reduce_scatter"] = get_forward().mlp_reduce_scatter - seen["fuse_mlp_allreduce"] = get_forward().fuse_mlp_allreduce - return hidden_states - - complete = MagicMock(side_effect=lambda h, r: (h, r)) - plan = stub_plan() - plan.path_for = lambda fb: SimpleNamespace( - output=comm.OutputContract(comm.Layout(frozenset())), - returns_over_dp=False, - output_move_completes_sum=False, - ) - plan.output._decide = lambda fb, steps: comm.ExitDecision( - defer_moe_finalize=False, - fuse_mlp_allreduce=False, - mlp_reduce_scatter=reduce_scatter, - complete=complete, - ) - - def stage_exit(fb): - return plan.output.ffn_exit(fb, stream=fb.residual_stream) - - object.__setattr__( - layer, - "attn_boundary", - SimpleNamespace( - prepare=lambda h, fb, **kwargs: h, - finish=lambda h, fb: h, - ), - ) - object.__setattr__( - layer, - "ffn_boundary", - SimpleNamespace( - prepare=lambda h, fb: h, - exit=stage_exit, - ), - ) - object.__setattr__(layer, "use_moe", False) - object.__setattr__(layer, "mlp", mlp) - object.__setattr__(layer, "self_attn", lambda **kwargs: kwargs["hidden_states"]) - return layer, seen, complete - - -class TestStep3p5DenseReduceScatter(CustomTestCase): - def test_dense_mlp_sees_the_postprocess_reduce_scatter(self): - for reduce_scatter in (False, True): - with self.subTest(reduce_scatter=reduce_scatter): - layer, seen, complete = dense_layer(reduce_scatter=reduce_scatter) - layer.forward( - positions=None, - hidden_states=torch.ones(2, 4), - forward_batch=SimpleNamespace( - residual_stream=ResidualStream(torch.ones(2, 4)) - ), - ) - self.assertEqual(seen["mlp_reduce_scatter"], reduce_scatter) - self.assertFalse(seen["fuse_mlp_allreduce"]) - complete.assert_called_once() - - -if __name__ == "__main__": - unittest.main()