From b157d548bb3a648a98f47559c6d7db2e84a80ba6 Mon Sep 17 00:00:00 2001
From: Cheng Wan <54331508+ch-wan@users.noreply.github.com>
Date: Sat, 3 Oct 2026 02:17:48 -0700
Subject: [PATCH] [Refactor] Let stage boundaries complete every stage-output
sum (#42301)
---
docs/docs/developer_guide/layer_boundary.mdx | 18 ++--
.../layers/layer_boundary/adapters/branch.py | 13 ++-
.../sglang/srt/layers/layer_boundary/exit.py | 91 ++++++++--------
.../sglang/srt/layers/layer_boundary/ops.py | 32 ++++++
.../sglang/srt/layers/layer_boundary/stage.py | 36 +++++--
python/sglang/srt/layers/moe/utils.py | 46 ++++++--
python/sglang/srt/model_loader/loader.py | 5 +-
python/sglang/srt/models/apertus.py | 6 +-
python/sglang/srt/models/arcee.py | 6 +-
python/sglang/srt/models/bailing_moe.py | 3 +-
.../sglang/srt/models/bailing_moe_linear.py | 5 +-
python/sglang/srt/models/bailing_moe_v3.py | 20 +---
python/sglang/srt/models/deepseek_v2.py | 73 ++++++++-----
.../srt/models/dots3_common/modeling.py | 65 ++----------
python/sglang/srt/models/exaone4.py | 6 +-
python/sglang/srt/models/falcon_h1.py | 1 +
python/sglang/srt/models/glm4_moe.py | 4 +-
python/sglang/srt/models/glm4_moe_lite.py | 4 +-
python/sglang/srt/models/glm5_next.py | 3 +
python/sglang/srt/models/gpt_oss.py | 3 -
python/sglang/srt/models/interns2_mobius.py | 5 +-
python/sglang/srt/models/kimi_linear.py | 10 +-
python/sglang/srt/models/laguna.py | 8 +-
python/sglang/srt/models/llada2.py | 3 +-
python/sglang/srt/models/llama4.py | 4 +-
python/sglang/srt/models/longcat_flash.py | 18 +---
.../sglang/srt/models/longcat_flash_nextn.py | 9 +-
python/sglang/srt/models/mellum.py | 1 +
python/sglang/srt/models/mimo_v2.py | 6 +-
python/sglang/srt/models/mimo_v2_nextn.py | 1 +
python/sglang/srt/models/minimax_m2.py | 2 -
python/sglang/srt/models/minimax_m3.py | 34 +-----
python/sglang/srt/models/mixtral.py | 10 +-
python/sglang/srt/models/moss_vl.py | 3 +
python/sglang/srt/models/nanbeige.py | 6 +-
python/sglang/srt/models/nemotron_h.py | 31 ++----
python/sglang/srt/models/qwen2.py | 2 +
python/sglang/srt/models/qwen2_moe.py | 16 +--
python/sglang/srt/models/qwen3.py | 1 +
python/sglang/srt/models/qwen3_5.py | 10 ++
python/sglang/srt/models/qwen3_moe.py | 7 +-
python/sglang/srt/models/qwen3_next.py | 4 +
python/sglang/srt/models/qwen4_exp.py | 3 +
python/sglang/srt/models/sarvam_moe.py | 5 +-
python/sglang/srt/models/sdar.py | 2 +-
python/sglang/srt/models/sdar_moe.py | 3 -
python/sglang/srt/models/spark2_5.py | 6 +-
python/sglang/srt/models/step3_vl.py | 4 +-
python/sglang/srt/models/step3p5.py | 3 +-
python/sglang/srt/models/xllm.py | 3 +-
python/sglang/test/boundary_fixtures.py | 2 +-
.../test_boundary_output_contracts.py | 8 +-
.../test_boundary_output_policy.py | 92 ++++++++--------
.../test_declared_attention_cp.py | 49 +++++----
.../test_declared_decoder_boundary.py | 35 +++---
.../test_dp_attention_cp_gather.py | 26 +++--
.../unit/layer_boundary/test_ffn_exit.py | 23 ++--
.../test_longcat_flash_shortcut.py | 2 +
.../test_nemotron_h_aux_capture.py | 6 +-
.../test_nemotron_h_mtp_reduction.py | 19 +++-
.../layer_boundary/test_nemotron_h_stages.py | 28 +++--
.../test_stage_producers_leave_sums.py | 100 ++++++++++++++++++
.../test_step3p5_dense_reduce_scatter.py | 91 ----------------
63 files changed, 617 insertions(+), 524 deletions(-)
create mode 100644 test/registered/unit/layer_boundary/test_stage_producers_leave_sums.py
delete mode 100644 test/registered/unit/layer_boundary/test_step3p5_dense_reduce_scatter.py
diff --git a/docs/docs/developer_guide/layer_boundary.mdx b/docs/docs/developer_guide/layer_boundary.mdx
index 1a5b494cd7..b18848a040 100644
--- a/docs/docs/developer_guide/layer_boundary.mdx
+++ b/docs/docs/developer_guide/layer_boundary.mdx
@@ -142,7 +142,7 @@ first_ffn, second_attn, second_ffn = make_stages(
This fragment assumes the source boundary and the three norms already exist. `prepared_from` means the source has already performed its read. Enter through `branch_input(source, hidden_states, forward_batch)`, not `prepare()`, to move that input and fork the residual stream without normalizing twice. The first stage's norm is therefore never applied; LongCat passes the MoE stage's norm there.
-The branch adapter also provides `branch_output()` to place a complete contribution on the branch's exit rows and `merge_branch()` to combine it with another branch's output. These methods encode row movement and residual ownership; the model still determines its compute schedule. Branch transport currently supports only ordinary token rows: construction rejects context-parallel, input-scattered, and sequence-parallel variants. See `python/sglang/srt/models/longcat_flash.py` for a complete integration.
+The branch adapter also provides `branch_output()` to complete the sum a branch's contribution owes and place it on the branch's exit rows and `merge_branch()` to combine it with another branch's output. These methods encode row movement and residual ownership; the model still determines its compute schedule. Branch transport currently supports only ordinary token rows: construction rejects context-parallel, input-scattered, and sequence-parallel variants. See `python/sglang/srt/models/longcat_flash.py` for a complete integration.
### Execute a layer
@@ -166,14 +166,14 @@ hidden_states = output.finish(hidden_states)
This example uses Qwen3's compute member name `mlp`; boundary construction does not require renaming compute classes or checkpoint parameters.
-`prepare()` returns input in the consumer read's format, which can include quantized tensors. An attention with an `ALWAYS_PARTIAL` output contract uses `finish()` to register its contribution. An FFN or single-stage mixer uses `exit()` to publish flags during compute, then calls the returned object's `finish()` once. The exit context restores the previous runtime flags even when compute raises; do not call `finish()` on a failed computation.
+`prepare()` returns input in the consumer read's format, which can include quantized tensors. An attention with an `ALWAYS_PARTIAL` output contract uses `finish()` to register its contribution. An FFN or single-stage mixer runs its compute under `exit()`, then calls the returned object's `finish()` once. The exit context restores the previous runtime flags even when compute raises; do not call `finish()` on a failed computation.
-Compute cooperates through the flags the scope publishes on `get_forward()`:
+Compute never sums a stage's output: build every output projection, MLP and MoE block of a stage with `reduce_results=False`, and leave the sum the output owes to the boundary. The boundary completes it at the exit, or carries it to the next stage's input, where a fused all-reduce + add + norm may take it:
-- `FfnExit` publishes `fuse_mlp_allreduce` (skip the output all-reduce), `mlp_reduce_scatter` (leave the sum to the boundary's reduce-scatter), and `defer_moe_finalize` (a MoE may return a `DeferredFinalize`).
-- `MixerExit` publishes `fuse_mlp_allreduce` and exposes the same decision as `skips_reduction`. Before an FFN, the mixer always leaves its attention-TP sum to the FFN's input, including under attention DP. Before another mixer it may defer the sum to that mixer's input, but never under attention DP.
+- `FfnExit` decides before compute runs whether the exit completes the sum, a reduce-scatter on the way back completes it, or the next input does. It publishes only `defer_moe_finalize` on `get_forward()`: a MoE may then return a `DeferredFinalize`.
+- `MixerExit`: before an FFN, the mixer always leaves its attention-TP sum to the FFN's input, including under attention DP. Before another mixer the exit may carry the sum to that mixer's input, but never under attention DP; otherwise it completes the sum itself.
-`RowParallelLinear` and the MoE output reduction honour these flags automatically; a custom reduction reads them from the exit object. Returning a handoff while `defer_moe_finalize` is false fails in `finish()`.
+A component every TP rank computes in full (a shared expert replicated with `tp_size=1`, LoRA-B on a partial sum) is added on TP rank 0 only, so that the one sum counts it once. Model loading fails if a module under a stage boundary is built with `reduce_results=True`. Returning a handoff while `defer_moe_finalize` is false fails in `finish()`.
A complete tensor after `finish()` can still have a pending **residual update**. An incomplete reduction is exposed to the model as an opaque `OwedOutput`; pass it to the next boundary or a supported access method. Do not inspect it as a tensor, slice it, or unwrap its contribution in model code.
@@ -201,7 +201,7 @@ A terminal FFN normally completes its output before the final norm. The exceptio
`model_forward_stages()` in `python/sglang/srt/batch_overlap/two_batch_overlap.py` gives each TBO microbatch its own stream: it splits with `ResidualStream.from_handoff()` and merges the results afterwards. Do not share one mutable stream between interleaved microbatches or hold an `exit()` scope open across a TBO yield point.
-In the operation-scheduled TBO path, the FFN runs without an `exit()` scope. No skip flags are published, so compute returns a fully reduced output. `boundary.finish_complete_output(hidden_states, forward_batch)` applies any output transform, moves that output back to the layer's exit rows (a plain scatter under attention DP), and leaves nothing owed.
+In the operation-scheduled TBO path, the FFN runs without an `exit()` scope and returns its partial output. `boundary.complete_now(hidden_states, forward_batch)` completes the sum it owes, applies any output transform, moves that output back to the layer's exit rows (a plain scatter under attention DP), and leaves nothing owed.
### Auxiliary capture and extra contributions
@@ -273,7 +273,7 @@ flowchart TD
F["ForwardBatch
runtime facts"] --> B
S["ResidualStream
residual and
pending work"] <--> B
B --> I["prepare()
complete,
update, read,
move rows"]
- I -->|"FFN or mixer"| X["exit()
choose
completion,
publish flags"]
+ I -->|"FFN or mixer"| X["exit()
choose
completion"]
X --> M["Model
computation"]
I -->|"ALWAYS_PARTIAL"| M
M --> O["finish()
record the
contribution"]
@@ -284,7 +284,7 @@ Construction and execution have different lifetimes:
1. **At model initialization**, `declare_attn()` and `declare_ffn()` describe compute behavior. `make_stages()` connects those declarations and binds each stage's own norm and hooks.
2. **During construction**, factories resolve token rows and reduction contracts for supported batch variants. `bind_entry()` binds consumer work; `bind_exit()` binds output transport. `StagePlan` retains those paths.
-3. **At each forward**, batch facts select an existing path. An exit can make a batch-dependent completion decision, but it does not rebuild the declarations. The decision publishes compute flags and supplies the matching completion action.
+3. **At each forward**, batch facts select an existing path. An exit can make a batch-dependent completion decision, but it does not rebuild the declarations. The decision supplies the matching completion action; the only flag it publishes to compute is `defer_moe_finalize`, for a MoE that may hand off its finalize.
4. **Across stages**, `ForwardBatch.residual_stream` carries residual state and the producer's actual contribution. Stages do not store forward tensors on their reusable plans.
The resolved `StageContract` is a lower-level type than `StageDeclaration`: one `InputContract` and one `OutputContract` after topology and batch variant have been applied. An `EdgeContract` combines a producer output, consumer requirement, and the residual's source/destination layouts.
diff --git a/python/sglang/srt/layers/layer_boundary/adapters/branch.py b/python/sglang/srt/layers/layer_boundary/adapters/branch.py
index 4098d603a7..57d1dbc04f 100644
--- a/python/sglang/srt/layers/layer_boundary/adapters/branch.py
+++ b/python/sglang/srt/layers/layer_boundary/adapters/branch.py
@@ -19,7 +19,7 @@ from typing import TYPE_CHECKING, Tuple
import torch
-from sglang.srt.layers.layer_boundary.ops import move_rows
+from sglang.srt.layers.layer_boundary.ops import move_rows, sum_output
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
@@ -53,9 +53,14 @@ def branch_input(
def branch_output(
plan: StagePlan, hidden_states: torch.Tensor, forward_batch: ForwardBatch
) -> torch.Tensor:
- """This layer's complete FFN output as a branch's contribution, which
- adds to the layer's output without writing the residual: moved to the
- rows the layer hands on."""
+ """This layer's FFN output as a branch's contribution, which adds to the
+ layer's output without writing the residual: the sum it owes completed,
+ then moved to the rows the layer hands on."""
+ group = plan.path_for(forward_batch).output.group
+ if group is not None:
+ hidden_states = sum_output(
+ hidden_states, group, forward_batch, may_quantize=False
+ )
rows, _, to = plan.branch_rows(forward_batch)
return move_rows(hidden_states, rows, to, forward_batch)
diff --git a/python/sglang/srt/layers/layer_boundary/exit.py b/python/sglang/srt/layers/layer_boundary/exit.py
index c4321f95db..c055f915ef 100644
--- a/python/sglang/srt/layers/layer_boundary/exit.py
+++ b/python/sglang/srt/layers/layer_boundary/exit.py
@@ -39,6 +39,7 @@ from sglang.srt.layers.layer_boundary.ops import (
all_reduce_to_dp_local,
dp_reduce_scatter,
dp_reduce_scatterv,
+ sum_output,
to_dp_local,
)
from sglang.srt.layers.layer_boundary.output import (
@@ -94,13 +95,14 @@ class ExitPolicy:
def __init__(self, plan):
self.plan = plan
- def finish_complete_output(
+ def complete_now(
self,
hidden_states: torch.Tensor,
stream: ResidualStream,
forward_batch: ForwardBatch,
):
- """Move a complete output from the operation-scheduled producer path."""
+ """Complete the output of the operation-scheduled producer path, which
+ never defers: the sum it owes, then its move and write-back."""
steps = self.plan.path_for(forward_batch)
hidden_states, residual = self._complete_now(
hidden_states,
@@ -108,11 +110,9 @@ class ExitPolicy:
forward_batch=forward_batch,
dp_step=None,
steps=steps,
- output_move=(
- steps.complete_output_move
- if steps.output_move_completes_sum
- else steps.output_move
- ),
+ output_move=steps.output_move,
+ owes_sum=steps.output.group is not None
+ and not steps.output_move_completes_sum,
)
return self._record_output(
hidden_states,
@@ -188,6 +188,8 @@ class ExitPolicy:
dp_step=dp_step,
steps=steps,
output_move=steps.output_move,
+ # A reduce-scatter completes the sum on its way back.
+ owes_sum=steps.output.group is not None and not mlp_reduce_scatter,
)
defer_moe_finalize = (
self.plan.fusions is not None
@@ -251,10 +253,19 @@ class ExitPolicy:
dp_step: Optional[Callable],
steps,
output_move: Optional[Callable],
+ owes_sum: bool,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""This layer's postprocess, run with the attention-DP step already
- chosen: the move back to where the next layer reads the FFN output, then
- the write-back into the residual for a layer that does it itself."""
+ chosen: the sum the output owes, the move back to where the next layer
+ reads it, then the write-back into the residual for a layer that does
+ it itself."""
+ if owes_sum:
+ hidden_states = sum_output(
+ hidden_states,
+ steps.output.group,
+ forward_batch,
+ may_quantize=steps.output.update.is_plain_add,
+ )
if steps.output.transform is not None:
hidden_states = steps.output.transform.apply(hidden_states)
if steps.returns_over_dp:
@@ -289,7 +300,7 @@ class ExitPolicy:
return FfnExit(self, forward_batch, stream=stream)
def _sum_deferral_allowed(self, steps) -> bool:
- # Under the MoE-CP all-gather the fusion path would skip finish_complete_output
+ # Under the MoE-CP all-gather the fusion path would skip complete_now
# and its MoE-CP scatter, leaving hidden_states longer than the residual.
if is_enable_moe_cp_allgather() or steps.output.group is None:
return False
@@ -377,17 +388,17 @@ def _batch_allows_deferred_sum(forward_batch: ForwardBatch, boundary=None) -> bo
class ExitDecision(msgspec.Struct, frozen=True):
- """One decision shared by compute flags and the matching output completion.
+ """One decision for an FFN output, made before compute runs.
Fields:
defer_moe_finalize: Compute may return a producer-specific finalize handoff.
- fuse_mlp_allreduce: Compute skips its all-reduce; completion runs it later
- or carries it to the next consumer, whether fused there or unfused.
- mlp_reduce_scatter: Compute leaves reduction to the selected scatter path.
+ fuse_mlp_allreduce: The sum is carried to the next consumer, which
+ completes it, fused or unfused.
+ mlp_reduce_scatter: A reduce-scatter on the way back completes the sum.
complete: Callable(output, residual) returning the completed or wrapped
output and its corresponding residual rows.
- Do not independently reselect completion after compute has used these flags.
+ Do not independently reselect completion after compute has run.
"""
defer_moe_finalize: bool
@@ -405,22 +416,21 @@ def _defer(
class MixerExit:
- """The scope that publishes a mixer's decision while it runs: inside the
- ``with`` block ``fuse_mlp_allreduce`` on ``get_forward()`` tells its
- row-parallel output projection to skip the all-reduce. It skips when the
- stage's output always leaves its sum (to an FFN stage, which completes it in
- its input), or when its declaration permits deferring the sum to the next
- attention stage and the batch allows it: TP > 1, no attention DP, and no
- input-scattered or MoE-CP all-gather layout. The consumer decides whether
- to fuse its completion with the input norm."""
+ """A mixer's (an attention-like stage's) output decision. The mixer leaves
+ its output's sum; the stage either carries it to the next attention stage
+ (when the declaration permits deferring and the batch allows it: TP > 1,
+ no attention DP, and no input-scattered or MoE-CP all-gather layout),
+ hands it on as the declared sum an FFN stage completes in its input, or
+ completes it here. The consumer decides whether to fuse its completion
+ with the input norm."""
__slots__ = (
- "skips_reduction",
"_defers",
- "_scope",
+ "_group",
"_update",
"_declared_sum",
"_stream",
+ "_forward_batch",
)
def __init__(
@@ -433,7 +443,9 @@ class MixerExit:
steps = boundary.plan.path_for(forward_batch)
produced = steps.output
self._stream = stream
+ self._forward_batch = forward_batch
self._update = produced.update
+ self._group = produced.group
self._declared_sum = produced.group if produced.always_partial else None
self._defers = (
produced.may_defer_to_next
@@ -441,15 +453,12 @@ class MixerExit:
and not is_dp_attention_enabled()
and boundary._sum_deferral_allowed(steps)
)
- self.skips_reduction = produced.always_partial or self._defers
- self._scope = get_forward().scoped(fuse_mlp_allreduce=self.skips_reduction)
def __enter__(self) -> MixerExit:
- self._scope.__enter__()
return self
def __exit__(self, *exc_info):
- return self._scope.__exit__(*exc_info)
+ return None
def finish(self, hidden_states: torch.Tensor):
"""Hand the actual mixer result to the next boundary through the stream."""
@@ -457,6 +466,13 @@ class MixerExit:
hidden_states = UnreducedOutput(
hidden_states, group=get_parallel().tp_group
)
+ elif self._declared_sum is None and self._group is not None:
+ hidden_states = sum_output(
+ hidden_states,
+ self._group,
+ self._forward_batch,
+ may_quantize=self._update.is_plain_add,
+ )
return self._stream.record(
hidden_states, self._update, declared_sum=self._declared_sum
)
@@ -473,18 +489,15 @@ class FfnExit:
Fields:
boundary: The owning output boundary.
defer_moe_finalize: Whether compute may return a finalize handoff.
- fuse_mlp_allreduce: Whether compute leaves its all-reduce to completion.
- mlp_reduce_scatter: Whether compute leaves reduction to a scatter path.
- The three flags are published on get_forward() only inside the context.
- finish(output) uses the same selected action after successful compute.
+ The FFN leaves its output's sum; finish(output) completes it or carries
+ it on with the action selected before compute. ``defer_moe_finalize`` is
+ published on get_forward() only inside the context.
"""
__slots__ = (
"boundary",
"defer_moe_finalize",
- "fuse_mlp_allreduce",
- "mlp_reduce_scatter",
"_complete",
"_update",
"_declared_sum",
@@ -504,18 +517,12 @@ class FfnExit:
steps = boundary.plan.path_for(forward_batch)
completion = boundary._decide(forward_batch, steps)
self.defer_moe_finalize = completion.defer_moe_finalize
- self.fuse_mlp_allreduce = completion.fuse_mlp_allreduce
- self.mlp_reduce_scatter = completion.mlp_reduce_scatter
self._complete = completion.complete
self._update = steps.output.update
self._declared_sum = boundary._sum_owed_after_skip(
steps, completion.mlp_reduce_scatter
)
- self._scope = get_forward().scoped(
- fuse_mlp_allreduce=self.fuse_mlp_allreduce,
- mlp_reduce_scatter=self.mlp_reduce_scatter,
- defer_moe_finalize=self.defer_moe_finalize,
- )
+ self._scope = get_forward().scoped(defer_moe_finalize=self.defer_moe_finalize)
def __enter__(self) -> FfnExit:
self._scope.__enter__()
diff --git a/python/sglang/srt/layers/layer_boundary/ops.py b/python/sglang/srt/layers/layer_boundary/ops.py
index 069ebc2808..53a411c522 100644
--- a/python/sglang/srt/layers/layer_boundary/ops.py
+++ b/python/sglang/srt/layers/layer_boundary/ops.py
@@ -22,6 +22,8 @@ from sglang.srt.distributed import (
GroupCoordinator,
attention_tensor_model_parallel_all_reduce,
attention_tensor_model_parallel_quant_all_reduce,
+ tensor_model_parallel_all_reduce,
+ tensor_model_parallel_quant_all_reduce,
)
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
use_symmetric_memory,
@@ -47,12 +49,14 @@ from sglang.srt.layers.layer_boundary.adapters.attention import (
)
from sglang.srt.layers.layer_boundary.layout import (
Layout,
+ SumGroup,
TokenAxis,
_cp_shard_token_rows,
moe_cp_gathered_rows,
)
from sglang.srt.layers.layer_boundary.residual import ResidualUpdate
from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_ADD
+from sglang.srt.layers.moe.utils import sum_post_experts_output
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.runtime_context import get_exec, get_parallel
@@ -173,6 +177,34 @@ def attn_tp_all_reduce(
return attention_tensor_model_parallel_all_reduce(hidden_states)
+def sum_output(
+ hidden_states: torch.Tensor,
+ group: SumGroup,
+ forward_batch: ForwardBatch,
+ *,
+ may_quantize: bool,
+) -> torch.Tensor:
+ """Complete the sum an output owes over ``group`` on the rows it is on."""
+ parallel = get_parallel()
+ if group is SumGroup.ATTN_TP:
+ if parallel.attn_tp_size == 1:
+ return hidden_states
+ return attn_tp_all_reduce(hidden_states, forward_batch, may_quantize)
+ if group is SumGroup.TP:
+ if parallel.tp_size == 1:
+ return hidden_states
+ if (
+ may_quantize
+ and not forward_batch.forward_mode.is_decode_or_idle()
+ and get_exec().comm.enable_quant_communications
+ ):
+ return tensor_model_parallel_quant_all_reduce(hidden_states)
+ return tensor_model_parallel_all_reduce(hidden_states)
+ if group is SumGroup.MOE_OUTPUT:
+ return sum_post_experts_output(hidden_states)
+ raise ValueError(f"unsupported output sum group: {group!r}")
+
+
def dp_gather(
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
diff --git a/python/sglang/srt/layers/layer_boundary/stage.py b/python/sglang/srt/layers/layer_boundary/stage.py
index fb44b3181b..06cbb8a125 100644
--- a/python/sglang/srt/layers/layer_boundary/stage.py
+++ b/python/sglang/srt/layers/layer_boundary/stage.py
@@ -187,7 +187,8 @@ class StageBoundary:
forward_batch: Active batch whose stream receives the compute result.
Returns:
- A context manager publishing reduction/finalize flags during compute.
+ A context manager publishing the producer's finalize choice during
+ compute.
Call its finish(output) exactly once after successful compute, including
when compute is skipped for an empty input. It completes or carries work
using the same decision rather than selecting a second path.
@@ -253,11 +254,11 @@ class StageBoundary:
hidden_states = stream.complete(hidden_states)
return hidden_states, stream.snapshot(hidden_states)
- def finish_complete_output(self, hidden_states, forward_batch):
- """Move an FFN output that compute already completed outside exit()
- (the operation-scheduled TBO path) onto the rows the layer hands on;
- it chooses no reduction step."""
- return self.plan.output.finish_complete_output(
+ def complete_now(self, hidden_states, forward_batch):
+ """Complete an FFN output outside exit() (the operation-scheduled TBO
+ path, which never defers): the sum it owes, then the move onto the
+ rows the layer hands on."""
+ return self.plan.output.complete_now(
hidden_states, stream_of(forward_batch), forward_batch
)
@@ -379,3 +380,26 @@ class StageBoundary:
),
)
return hidden_states, stream
+
+
+def check_stage_producers(model: torch.nn.Module) -> None:
+ """Reject a model whose stage producers complete their own output sums.
+
+ In a layer built from stage boundaries, the boundary completes every sum a
+ stage output owes. A submodule there constructed with
+ ``reduce_results=True`` (a row-parallel projection, a MoE block) would
+ have its output summed twice. Called once, after the model is built.
+ """
+ offenders = [
+ f"{name}.{child_name}"
+ for name, layer in model.named_modules()
+ if any(isinstance(v, StageBoundary) for v in vars(layer).values())
+ for child_name, child in layer.named_modules()
+ if child_name and getattr(child, "reduce_results", False) is True
+ ]
+ if offenders:
+ raise ValueError(
+ "stage boundaries complete their producers' output sums, but these "
+ f"submodules sum their own output: {', '.join(offenders[:8])}"
+ + (f" (and {len(offenders) - 8} more)" if len(offenders) > 8 else "")
+ )
diff --git a/python/sglang/srt/layers/moe/utils.py b/python/sglang/srt/layers/moe/utils.py
index 300d09390b..b185aaf00e 100644
--- a/python/sglang/srt/layers/moe/utils.py
+++ b/python/sglang/srt/layers/moe/utils.py
@@ -797,6 +797,18 @@ def should_add_replicated_moe_output() -> bool:
return not (parallel.tp_size > 1 and summed_later and parallel.tp_rank != 0)
+def adds_replicated_output_to_partial() -> bool:
+ """For a MoE block whose stage boundary completes its sum: whether this rank
+ adds an output every TP rank holds in full, such as a shared expert
+ replicated with tp_size=1, to its MoE output. While the output still owes a
+ TP sum, only TP rank 0 adds it, so the sum counts it once."""
+ parallel = get_parallel()
+ owes_sum = parallel.tp_size > 1 and not post_experts_output_is_complete(
+ is_tp_path=True
+ )
+ return not owes_sum or parallel.tp_rank == 0
+
+
def can_merge_post_experts_all_reduce() -> bool:
"""Whether the EP and MoE-TP reductions can collapse into one _TP all-reduce.
@@ -818,20 +830,38 @@ def post_experts_all_reduce(hidden_states: torch.Tensor) -> torch.Tensor:
sequential ones, which also restores the invariant the fused residual+LN path
depends on.
"""
+ parallel = get_parallel()
+ return _post_experts_sum(
+ hidden_states,
+ reduce_ep=parallel.moe_ep_size > 1
+ and not should_skip_post_experts_all_reduce(is_tp_path=False),
+ reduce_tp=parallel.moe_tp_size > 1
+ and not should_skip_post_experts_all_reduce(is_tp_path=True),
+ )
+
+
+def sum_post_experts_output(hidden_states: torch.Tensor) -> torch.Tensor:
+ """Complete the sum a MoE output owes over the EP and MoE-TP groups, for the
+ boundary that owns it; a path the combine already summed is left alone."""
+ parallel = get_parallel()
+ return _post_experts_sum(
+ hidden_states,
+ reduce_ep=parallel.moe_ep_size > 1
+ and not post_experts_output_is_complete(is_tp_path=False),
+ reduce_tp=parallel.moe_tp_size > 1
+ and not post_experts_output_is_complete(is_tp_path=True),
+ )
+
+
+def _post_experts_sum(
+ hidden_states: torch.Tensor, *, reduce_ep: bool, reduce_tp: bool
+) -> torch.Tensor:
from sglang.srt.distributed.communication_op import (
moe_expert_parallel_all_reduce,
moe_tensor_model_parallel_all_reduce,
tensor_model_parallel_all_reduce,
)
- parallel = get_parallel()
- reduce_ep = parallel.moe_ep_size > 1 and not should_skip_post_experts_all_reduce(
- is_tp_path=False
- )
- reduce_tp = parallel.moe_tp_size > 1 and not should_skip_post_experts_all_reduce(
- is_tp_path=True
- )
-
if reduce_ep and reduce_tp and can_merge_post_experts_all_reduce():
return tensor_model_parallel_all_reduce(hidden_states)
diff --git a/python/sglang/srt/model_loader/loader.py b/python/sglang/srt/model_loader/loader.py
index 788ac19b8c..d60efacb38 100644
--- a/python/sglang/srt/model_loader/loader.py
+++ b/python/sglang/srt/model_loader/loader.py
@@ -83,6 +83,7 @@ from sglang.srt.connector.utils import parse_model_name
from sglang.srt.distributed import (
model_parallel_is_initialized,
)
+from sglang.srt.layers.layer_boundary.stage import check_stage_producers
from sglang.srt.layers.modelopt_utils import QUANT_CFG_CHOICES
from sglang.srt.layers.moe.utils import (
install_shared_experts_fusion_decision,
@@ -304,7 +305,9 @@ def _initialize_model(
if load_config.draft_model_idx is not None:
kwargs["draft_model_idx"] = load_config.draft_model_idx
- return model_class(**kwargs)
+ model = model_class(**kwargs)
+ check_stage_producers(model)
+ return model
def post_load_weights(model: nn.Module) -> None:
diff --git a/python/sglang/srt/models/apertus.py b/python/sglang/srt/models/apertus.py
index ee12ae8f2d..cb1efd5cd2 100644
--- a/python/sglang/srt/models/apertus.py
+++ b/python/sglang/srt/models/apertus.py
@@ -259,6 +259,7 @@ class ApertusDecoderLayer(nn.Module):
quant_config=quant_config,
bias=getattr(config, "mlp_bias", False),
prefix=add_prefix("mlp", prefix),
+ reduce_results=False,
)
self.attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.feedforward_layernorm = RMSNorm(
@@ -294,8 +295,9 @@ class ApertusDecoderLayer(nn.Module):
# Fully Connected
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
- hidden_states = self.mlp(hidden_states)
- return self.ffn_boundary.finish_complete_output(hidden_states, forward_batch)
+ with self.ffn_boundary.exit(forward_batch) as ffn_exit:
+ hidden_states = self.mlp(hidden_states)
+ return ffn_exit.finish(hidden_states)
class ApertusModel(nn.Module):
diff --git a/python/sglang/srt/models/arcee.py b/python/sglang/srt/models/arcee.py
index 573c6f3ae3..70ce424c2e 100644
--- a/python/sglang/srt/models/arcee.py
+++ b/python/sglang/srt/models/arcee.py
@@ -235,6 +235,7 @@ class ArceeDecoderLayer(nn.Module):
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
+ reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNorm(
@@ -270,8 +271,9 @@ class ArceeDecoderLayer(nn.Module):
# Fully Connected
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
- hidden_states = self.mlp(hidden_states)
- return self.ffn_boundary.finish_complete_output(hidden_states, forward_batch)
+ with self.ffn_boundary.exit(forward_batch) as ffn_exit:
+ hidden_states = self.mlp(hidden_states)
+ return ffn_exit.finish(hidden_states)
class ArceeModel(nn.Module):
diff --git a/python/sglang/srt/models/bailing_moe.py b/python/sglang/srt/models/bailing_moe.py
index 8f2694a28a..c3097375fa 100644
--- a/python/sglang/srt/models/bailing_moe.py
+++ b/python/sglang/srt/models/bailing_moe.py
@@ -55,7 +55,6 @@ from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_deepep_mode,
get_moe_a2a_backend,
- reduce_moe_output,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
@@ -460,7 +459,6 @@ class BailingMoESparseMoeBlock(nn.Module):
if self.num_shared_experts > 0:
final_hidden_states = final_hidden_states + shared_output
- final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_size)
def forward_deepep(
@@ -693,6 +691,7 @@ class BailingMoEBlock(nn.Module):
prefix=add_prefix("mlp", prefix),
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
+ reduce_results=False,
)
self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps)
diff --git a/python/sglang/srt/models/bailing_moe_linear.py b/python/sglang/srt/models/bailing_moe_linear.py
index 89a7885485..c68d67fd5c 100644
--- a/python/sglang/srt/models/bailing_moe_linear.py
+++ b/python/sglang/srt/models/bailing_moe_linear.py
@@ -32,7 +32,6 @@ from sglang.srt.layers.linear import (
RowParallelLinear,
)
from sglang.srt.layers.logits_processor import LogitsProcessor
-from sglang.srt.layers.moe import reduce_moe_output
from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.moe.topk import TopK
@@ -365,7 +364,6 @@ class BailingMoE(nn.Module):
if self.num_shared_experts > 0:
final_hidden_states = final_hidden_states + shared_output
- final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states
@@ -671,6 +669,7 @@ class BailingMoEAttention(nn.Module):
bias=config.use_bias,
quant_config=quant_config,
prefix=f"{prefix}.o_proj",
+ reduce_results=False,
)
if hasattr(config, "rotary_dim"):
self.rotary_dim = config.rotary_dim
@@ -792,6 +791,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
intermediate_size=config.intermediate_size,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
+ reduce_results=False,
)
else:
if is_nextn or self.layer_id >= config.first_k_dense_replace:
@@ -810,6 +810,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
intermediate_size=config.intermediate_size,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
+ reduce_results=False,
)
rms_norm_eps = float(getattr(config, "rms_norm_eps", 1e-5))
self.input_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
diff --git a/python/sglang/srt/models/bailing_moe_v3.py b/python/sglang/srt/models/bailing_moe_v3.py
index b4cecd5621..b225e66f26 100644
--- a/python/sglang/srt/models/bailing_moe_v3.py
+++ b/python/sglang/srt/models/bailing_moe_v3.py
@@ -19,10 +19,6 @@ from sglang.kernels.ops.quantization.fp8_kernel import (
)
from sglang.srt.configs import KimiLinearConfig
from sglang.srt.configs.bailing_hybrid import is_bailing_multi_gate_enabled
-from sglang.srt.distributed import (
- moe_expert_parallel_all_reduce,
- moe_tensor_model_parallel_all_reduce,
-)
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
@@ -47,7 +43,6 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
- should_skip_post_experts_all_reduce,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
@@ -751,18 +746,6 @@ class BailingMoE(nn.Module):
if shared_output is not None:
final_hidden_states = final_hidden_states + shared_output
-
- if self.moe_ep_size > 1 and not should_skip_post_experts_all_reduce(
- is_tp_path=False,
- ):
- final_hidden_states = moe_expert_parallel_all_reduce(final_hidden_states)
-
- if self.moe_tp_size > 1 and not should_skip_post_experts_all_reduce(
- is_tp_path=True,
- ):
- final_hidden_states = moe_tensor_model_parallel_all_reduce(
- final_hidden_states
- )
return final_hidden_states
def _forward_shared_experts(
@@ -978,6 +961,7 @@ class BailingMoEAttention(nn.Module):
bias=config.use_bias,
quant_config=quant_config,
prefix=f"{prefix}.o_proj",
+ reduce_results=False,
)
if hasattr(config, "rotary_dim"):
self.rotary_dim = config.rotary_dim
@@ -1121,6 +1105,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
prefix=prefix,
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
+ reduce_results=False,
)
else:
if is_nextn or self.layer_id >= config.first_k_dense_replace:
@@ -1141,6 +1126,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
prefix=prefix,
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
+ reduce_results=False,
)
rms_norm_eps = float(getattr(config, "rms_norm_eps", 1e-5))
self.input_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py
index 75d1277d12..0145b36e14 100644
--- a/python/sglang/srt/models/deepseek_v2.py
+++ b/python/sglang/srt/models/deepseek_v2.py
@@ -111,6 +111,7 @@ from sglang.srt.layers.moe.token_dispatcher.base import (
from sglang.srt.layers.moe.topk import BypassedTopKOutput, TopK, TopKOutputFormat
from sglang.srt.layers.moe.utils import (
RoutingMethodType,
+ adds_replicated_output_to_partial,
filter_moe_weight_param_global_expert,
has_per_rank_fused_shared_slots,
is_deepep_class_backend,
@@ -118,6 +119,7 @@ from sglang.srt.layers.moe.utils import (
is_shared_experts_fusion_disabled,
is_tbo_enabled,
should_add_replicated_moe_output,
+ sum_post_experts_output,
)
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.quantization.fp8 import Fp8Config
@@ -275,10 +277,15 @@ class DeepseekV2MLP(nn.Module):
tp_rank: Optional[int] = None,
tp_size: Optional[int] = None,
swiglu_limit: Optional[float] = None,
+ allow_fused_down: Optional[bool] = None,
) -> None:
super().__init__()
self.tp_size = tp_size
self.swiglu_limit = swiglu_limit
+ # The fused down path calls deepgemm directly and runs no all-reduce.
+ self.allow_fused_down = (
+ not reduce_results if allow_fused_down is None else allow_fused_down
+ )
self.gate_up_proj = MergedColumnParallelLinear(
hidden_size,
@@ -375,7 +382,7 @@ class DeepseekV2MLP(nn.Module):
# are fp8 (uint8 storage with weight_scale_inv).
if (
self.swiglu_limit is not None
- and not self.down_proj.reduce_results
+ and self.allow_fused_down
and getattr(self.down_proj, "weight", None) is not None
and self.down_proj.weight.dtype == torch.uint8
and hasattr(self.down_proj, "weight_scale_inv")
@@ -592,8 +599,11 @@ class DeepseekV2MoE(nn.Module):
is_nextn: bool = False,
is_deepseek_v4: bool = False,
vl_correction_bias: bool = False,
+ reduce_results: bool = True,
):
super().__init__()
+ # False when a stage boundary completes this output's sum.
+ self.reduce_results = reduce_results
self.tp_size = get_parallel().tp_size
self.moe_ep_size = get_parallel().moe_ep_size
self.routed_scaling_factor = config.routed_scaling_factor
@@ -1160,7 +1170,7 @@ class DeepseekV2MoE(nn.Module):
final_hidden_states,
gated_shared_output=shared_output,
m=hidden_states.shape[0],
- reduce=post_experts_all_reduce,
+ reduce=sum_post_experts_output,
)
all_reduce_done = False
@@ -1256,7 +1266,7 @@ class DeepseekV2MoE(nn.Module):
self.routed_scaling_factor,
)
- if not all_reduce_done:
+ if not all_reduce_done and self.reduce_results:
if (
self.is_deepseek_v4
and self.tp_size > 1
@@ -1274,9 +1284,9 @@ class DeepseekV2MoE(nn.Module):
if mhc is not None:
mhc.start_stats_before_all_reduce()
final_hidden_states = post_experts_all_reduce(final_hidden_states)
- # TP1 shared experts are replicated, so add them after all-reduce to
- # avoid summing the same shared output once per TP rank.
- if self._shared_expert_tp1 and should_add_replicated_moe_output():
+ # TP1 shared experts are replicated: add them after this block's
+ # all-reduce, or on TP rank 0 only when the stage boundary sums.
+ if self._shared_expert_tp1 and self._adds_replicated_output():
final_hidden_states += shared_output
return final_hidden_states
@@ -1431,31 +1441,40 @@ class DeepseekV2MoE(nn.Module):
self.routed_scaling_factor,
)
- if (
- self.is_deepseek_v4
- and self.tp_size > 1
- and not should_skip_post_experts_all_reduce(is_tp_path=True)
- ):
- from sglang.srt.layers.moe.mhc_post_fusion import current_mhc_post_fusion
-
- mhc = current_mhc_post_fusion()
- if _is_hip and _hip_moe.fused_all_reduce_mhc(
- self, mhc, final_hidden_states
+ if self.reduce_results:
+ if (
+ self.is_deepseek_v4
+ and self.tp_size > 1
+ and not should_skip_post_experts_all_reduce(is_tp_path=True)
):
- return final_hidden_states
- if mhc is not None:
- mhc.start_stats_before_all_reduce()
- final_hidden_states = post_experts_all_reduce(final_hidden_states)
- # TP1 shared experts are replicated, so add them after all-reduce to
- # avoid summing the same shared output once per TP rank.
+ from sglang.srt.layers.moe.mhc_post_fusion import (
+ current_mhc_post_fusion,
+ )
+
+ mhc = current_mhc_post_fusion()
+ if _is_hip and _hip_moe.fused_all_reduce_mhc(
+ self, mhc, final_hidden_states
+ ):
+ return final_hidden_states
+ if mhc is not None:
+ mhc.start_stats_before_all_reduce()
+ final_hidden_states = post_experts_all_reduce(final_hidden_states)
+ # TP1 shared experts are replicated: add them after this block's
+ # all-reduce, or on TP rank 0 only when the stage boundary sums.
if (
shared_output is not None
and self._shared_expert_tp1
- and should_add_replicated_moe_output()
+ and self._adds_replicated_output()
):
final_hidden_states += shared_output
return final_hidden_states
+ def _adds_replicated_output(self) -> bool:
+ """Whether this rank adds the replicated TP1 shared-expert output."""
+ if self.reduce_results:
+ return should_add_replicated_moe_output()
+ return adds_replicated_output_to_partial()
+
def forward_cpu(
self,
hidden_states: torch.Tensor,
@@ -1507,7 +1526,8 @@ class DeepseekV2MoE(nn.Module):
), # block_size
True, # is_vnni
)
- final_hidden_states = post_experts_all_reduce(final_hidden_states)
+ if self.reduce_results:
+ final_hidden_states = post_experts_all_reduce(final_hidden_states)
return final_hidden_states
def forward_deepep(
@@ -2661,6 +2681,7 @@ class DeepseekV2DecoderLayer(nn.Module):
layer_id=self.layer_id,
alt_stream=alt_stream,
is_nextn=is_nextn,
+ reduce_results=False,
)
else:
if is_dense_ffn_fully_dp():
@@ -2676,6 +2697,8 @@ class DeepseekV2DecoderLayer(nn.Module):
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
swiglu_limit=getattr(config, "swiglu_limit", None),
+ reduce_results=False,
+ allow_fused_down=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -2876,7 +2899,7 @@ class DeepseekV2DecoderLayer(nn.Module):
)
def op_comm_postprocess_layer(self, state):
- hidden_states = self.ffn_boundary.finish_complete_output(
+ hidden_states = self.ffn_boundary.complete_now(
state.pop("hidden_states_mlp_output"), state.forward_batch
)
diff --git a/python/sglang/srt/models/dots3_common/modeling.py b/python/sglang/srt/models/dots3_common/modeling.py
index 8e391b83cf..a12a83c54e 100644
--- a/python/sglang/srt/models/dots3_common/modeling.py
+++ b/python/sglang/srt/models/dots3_common/modeling.py
@@ -45,9 +45,6 @@ from sglang.srt.batch_overlap.two_batch_overlap import (
model_forward_stages,
)
from sglang.srt.configs.dots3 import Dots3Config
-from sglang.srt.distributed import (
- tensor_model_parallel_all_reduce,
-)
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
use_symmetric_memory,
)
@@ -228,21 +225,13 @@ class Dots3MLP(nn.Module):
)
self.act_fn = SiluAndMul()
- def forward(
- self,
- x,
- forward_batch=None,
- should_allreduce_fusion: bool = False,
- use_reduce_scatter: bool = False,
- ):
+ def forward(self, x, forward_batch=None):
if (self.tp_size == 1) and x.shape[0] == 0:
return x
gate_up, _ = self.gate_up_proj(x)
x = self.act_fn(gate_up)
- x, _ = self.down_proj(
- x, skip_all_reduce=should_allreduce_fusion or use_reduce_scatter
- )
+ x, _ = self.down_proj(x)
return x
@@ -416,8 +405,6 @@ class Dots3MoE(nn.Module):
self,
hidden_states: torch.Tensor,
forward_batch: Optional[ForwardBatch] = None,
- should_allreduce_fusion: bool = False,
- use_reduce_scatter: bool = False,
) -> torch.Tensor:
if not self._enable_deepep_moe:
DUAL_STREAM_TOKEN_THRESHOLD = 1024
@@ -427,26 +414,13 @@ class Dots3MoE(nn.Module):
and hidden_states.shape[0] > 0
and hidden_states.shape[0] <= DUAL_STREAM_TOKEN_THRESHOLD
):
- return self.forward_normal_dual_stream(
- hidden_states,
- should_allreduce_fusion,
- use_reduce_scatter,
- )
+ return self.forward_normal_dual_stream(hidden_states)
else:
- return self.forward_normal(
- hidden_states,
- should_allreduce_fusion,
- use_reduce_scatter,
- )
+ return self.forward_normal(hidden_states)
else:
return self.forward_deepep(hidden_states, forward_batch)
- def forward_normal_dual_stream(
- self,
- hidden_states: torch.Tensor,
- should_allreduce_fusion: bool = False,
- use_reduce_scatter: bool = False,
- ) -> torch.Tensor:
+ def forward_normal_dual_stream(self, hidden_states: torch.Tensor) -> torch.Tensor:
current_stream = torch.cuda.current_stream()
self.alt_stream.wait_stream(current_stream)
@@ -465,21 +439,9 @@ class Dots3MoE(nn.Module):
torch.add(final_hidden_states, shared_output, out=final_hidden_states_out)
final_hidden_states = final_hidden_states_out
sm.tag(final_hidden_states)
- if (
- self.tp_size > 1
- and not should_allreduce_fusion
- and not use_reduce_scatter
- and not should_use_flashinfer_cutlass_moe_fp4_allgather()
- ):
- final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
return final_hidden_states
- def forward_normal(
- self,
- hidden_states: torch.Tensor,
- should_allreduce_fusion: bool = False,
- use_reduce_scatter: bool = False,
- ) -> torch.Tensor:
+ def forward_normal(self, hidden_states: torch.Tensor) -> torch.Tensor:
if hidden_states.shape[0] > 0:
shared_output = self._forward_shared_experts(hidden_states)
# router_logits: (num_tokens, n_experts)
@@ -496,13 +458,6 @@ class Dots3MoE(nn.Module):
torch.add(final_hidden_states, shared_output, out=final_hidden_states_out)
final_hidden_states = final_hidden_states_out
sm.tag(final_hidden_states)
- if (
- self.tp_size > 1
- and not should_allreduce_fusion
- and not use_reduce_scatter
- and not should_use_flashinfer_cutlass_moe_fp4_allgather()
- ):
- final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
return final_hidden_states
def forward_deepep(
@@ -1454,6 +1409,7 @@ class Dots3DecoderLayer(nn.Module):
prefix=add_prefix("mlp", prefix),
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
+ reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -1509,12 +1465,7 @@ class Dots3DecoderLayer(nn.Module):
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
- hidden_states = self.mlp(
- hidden_states,
- forward_batch,
- ffn_exit.fuse_mlp_allreduce,
- ffn_exit.mlp_reduce_scatter,
- )
+ hidden_states = self.mlp(hidden_states, forward_batch)
return ffn_exit.finish(hidden_states)
diff --git a/python/sglang/srt/models/exaone4.py b/python/sglang/srt/models/exaone4.py
index 3f0306fff8..78edb9969a 100644
--- a/python/sglang/srt/models/exaone4.py
+++ b/python/sglang/srt/models/exaone4.py
@@ -74,6 +74,7 @@ class Exaone4GatedMLP(nn.Module):
hidden_size,
bias=bias,
quant_config=quant_config,
+ reduce_results=False,
prefix=add_prefix("down_proj", prefix),
)
if hidden_act != "silu":
@@ -312,8 +313,9 @@ class Exaone4DecoderLayer(nn.Module):
# Fully Connected
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
- hidden_states = self.mlp(hidden_states)
- return self.ffn_boundary.finish_complete_output(hidden_states, forward_batch)
+ with self.ffn_boundary.exit(forward_batch) as ffn_exit:
+ hidden_states = self.mlp(hidden_states)
+ return ffn_exit.finish(hidden_states)
class Exaone4Model(nn.Module):
diff --git a/python/sglang/srt/models/falcon_h1.py b/python/sglang/srt/models/falcon_h1.py
index 76fb83516b..4921a469d6 100644
--- a/python/sglang/srt/models/falcon_h1.py
+++ b/python/sglang/srt/models/falcon_h1.py
@@ -214,6 +214,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
mlp_multipliers=config.mlp_multipliers,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
+ reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
diff --git a/python/sglang/srt/models/glm4_moe.py b/python/sglang/srt/models/glm4_moe.py
index c23263aa33..4a1ba3552c 100644
--- a/python/sglang/srt/models/glm4_moe.py
+++ b/python/sglang/srt/models/glm4_moe.py
@@ -59,7 +59,6 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
- reduce_moe_output,
should_use_flashinfer_cutlass_moe_fp4_allgather,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
@@ -607,7 +606,6 @@ class Glm4MoeSparseMoeBlock(nn.Module):
current_stream.wait_stream(self.alt_stream)
final_hidden_states += shared_output
- final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states
def forward_normal(
@@ -633,7 +631,6 @@ class Glm4MoeSparseMoeBlock(nn.Module):
final_hidden_states_out = torch.empty_like(final_hidden_states)
torch.add(final_hidden_states, shared_output, out=final_hidden_states_out)
final_hidden_states = final_hidden_states_out
- final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states
def forward_deepep(
@@ -768,6 +765,7 @@ class Glm4MoeDecoderLayer(nn.Module):
prefix=add_prefix("mlp", prefix),
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
+ reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
diff --git a/python/sglang/srt/models/glm4_moe_lite.py b/python/sglang/srt/models/glm4_moe_lite.py
index b137ba968e..e9873fdb5d 100644
--- a/python/sglang/srt/models/glm4_moe_lite.py
+++ b/python/sglang/srt/models/glm4_moe_lite.py
@@ -50,7 +50,6 @@ from sglang.srt.layers.linear import MergedColumnParallelLinear, RowParallelLine
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
- reduce_moe_output,
should_use_flashinfer_cutlass_moe_fp4_allgather,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
@@ -338,7 +337,6 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module):
current_stream.wait_stream(self.alt_stream)
final_hidden_states += shared_output
- final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states
def forward_normal(
@@ -364,7 +362,6 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module):
final_hidden_states_out = torch.empty_like(final_hidden_states)
torch.add(final_hidden_states, shared_output, out=final_hidden_states_out)
final_hidden_states = final_hidden_states_out
- final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states
def forward_deepep(
@@ -476,6 +473,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
prefix=add_prefix("mlp", prefix),
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
+ reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py
index 43cc6dcf27..23f74a7eb1 100644
--- a/python/sglang/srt/models/glm5_next.py
+++ b/python/sglang/srt/models/glm5_next.py
@@ -734,6 +734,7 @@ class Glm5NextDecoderLayer(nn.Module):
layer_id=self.layer_id,
alt_stream=alt_stream,
is_nextn=is_nextn,
+ reduce_results=False,
)
else:
if is_dense_ffn_fully_dp():
@@ -749,6 +750,8 @@ class Glm5NextDecoderLayer(nn.Module):
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
swiglu_limit=config.swiglu_limit,
+ reduce_results=False,
+ allow_fused_down=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
diff --git a/python/sglang/srt/models/gpt_oss.py b/python/sglang/srt/models/gpt_oss.py
index a7e8f0a544..b22857313a 100644
--- a/python/sglang/srt/models/gpt_oss.py
+++ b/python/sglang/srt/models/gpt_oss.py
@@ -48,7 +48,6 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
- reduce_moe_output,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
@@ -335,8 +334,6 @@ class GptOssSparseMoeBlock(nn.Module):
topk_output = self.topk(router_input, router_logits)
final_hidden_states = self.experts(hidden_states, topk_output)
- final_hidden_states = reduce_moe_output(final_hidden_states)
-
# When input was pre-padded, FusedMoE.forward_impl captured the
# padded width as `origin_hidden_states_dim` and skipped its own
# output-trim contiguous() — so the experts output is still
diff --git a/python/sglang/srt/models/interns2_mobius.py b/python/sglang/srt/models/interns2_mobius.py
index 371215e059..c2a3f40e68 100644
--- a/python/sglang/srt/models/interns2_mobius.py
+++ b/python/sglang/srt/models/interns2_mobius.py
@@ -31,9 +31,6 @@ from sglang.srt.layers.linear import (
ReplicatedLinear,
RowParallelLinear,
)
-from sglang.srt.layers.moe import (
- reduce_moe_output,
-)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.topk import TopK
from sglang.srt.layers.moe.utils import (
@@ -422,7 +419,7 @@ class _InternS2MobiusDecoderMixin:
routed = _get_mobius_routed_bank(meta_mlp, self.layer_id).forward_routed(
hidden_states, forward_batch
)
- return reduce_moe_output(routed + shared)
+ return routed + shared
def _forward_after_attention(
self,
diff --git a/python/sglang/srt/models/kimi_linear.py b/python/sglang/srt/models/kimi_linear.py
index 3078c08ae7..ae45a3bb41 100644
--- a/python/sglang/srt/models/kimi_linear.py
+++ b/python/sglang/srt/models/kimi_linear.py
@@ -12,7 +12,6 @@ from sglang.kernels.ops.attention.fla.fused_norm_gate import FusedRMSNormGated
from sglang.srt.configs.kimi_linear import KimiLinearConfig
from sglang.srt.distributed import (
divide,
- tensor_model_parallel_all_reduce,
)
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers.dcp.planner import prepare_decode_context_parallel_metadata
@@ -185,9 +184,6 @@ class KimiMoE(nn.Module):
if shared_output is not None:
final_hidden_states = final_hidden_states + shared_output
-
- if self.tp_size > 1:
- final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_size)
@@ -613,6 +609,7 @@ class KimiDecoderLayer(nn.Module):
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=f"{prefix}.mlp",
+ reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNorm(
@@ -655,8 +652,9 @@ class KimiDecoderLayer(nn.Module):
# Fully Connected
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
- hidden_states = self.mlp(hidden_states)
- return self.ffn_boundary.finish_complete_output(hidden_states, forward_batch)
+ with self.ffn_boundary.exit(forward_batch) as ffn_exit:
+ hidden_states = self.mlp(hidden_states)
+ return ffn_exit.finish(hidden_states)
class KimiLinearModel(nn.Module):
diff --git a/python/sglang/srt/models/laguna.py b/python/sglang/srt/models/laguna.py
index c2d7a7b194..3c334bf939 100644
--- a/python/sglang/srt/models/laguna.py
+++ b/python/sglang/srt/models/laguna.py
@@ -37,11 +37,10 @@ from sglang.srt.layers.linear import (
RowParallelLinear,
)
from sglang.srt.layers.logits_processor import LogitsProcessor
-from sglang.srt.layers.moe import reduce_moe_output
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.moe.topk import TopK
-from sglang.srt.layers.moe.utils import should_add_replicated_moe_output
+from sglang.srt.layers.moe.utils import adds_replicated_output_to_partial
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.layers.rotary_embedding import get_rope
@@ -226,8 +225,7 @@ class LagunaMoE(nn.Module):
else:
final = routed_out + shared_out
- final = reduce_moe_output(final)
- if self._shared_expert_tp1 and should_add_replicated_moe_output():
+ if self._shared_expert_tp1 and adds_replicated_output_to_partial():
final = final + shared_out
return final
@@ -448,7 +446,7 @@ class LagunaDecoderLayer(nn.Module):
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
- reduce_results=True,
+ reduce_results=False,
prefix=add_prefix("mlp", prefix),
)
diff --git a/python/sglang/srt/models/llada2.py b/python/sglang/srt/models/llada2.py
index e0b2d9dc3c..ab387cf67c 100644
--- a/python/sglang/srt/models/llada2.py
+++ b/python/sglang/srt/models/llada2.py
@@ -51,7 +51,6 @@ from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_deepep_mode,
get_moe_a2a_backend,
- reduce_moe_output,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
@@ -381,7 +380,6 @@ class LLaDA2MoeSparseMoeBlock(nn.Module):
if self.num_shared_experts > 0:
final_hidden_states = final_hidden_states + shared_output
- final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_size)
def forward_deepep(
@@ -628,6 +626,7 @@ class LLaDA2MoeBlock(nn.Module):
prefix=add_prefix("mlp", prefix),
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
+ reduce_results=False,
)
self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps)
diff --git a/python/sglang/srt/models/llama4.py b/python/sglang/srt/models/llama4.py
index a8f7a19aa1..1d158e8d38 100644
--- a/python/sglang/srt/models/llama4.py
+++ b/python/sglang/srt/models/llama4.py
@@ -41,7 +41,6 @@ from sglang.srt.layers.linear import (
ReplicatedLinear,
RowParallelLinear,
)
-from sglang.srt.layers.moe import reduce_moe_output
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
from sglang.srt.layers.moe.topk import TopK
from sglang.srt.layers.quantization.base_config import QuantizationConfig
@@ -148,8 +147,6 @@ class Llama4MoE(nn.Module):
out_aD = routed_out + shared_out
- out_aD = reduce_moe_output(out_aD)
-
return out_aD
def _forward_core(self, hidden_states, forward_mode: ForwardMode):
@@ -418,6 +415,7 @@ class Llama4DecoderLayer(nn.Module):
hidden_act="silu",
quant_config=quant_config,
prefix=add_prefix("feed_forward", prefix),
+ reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNorm(
diff --git a/python/sglang/srt/models/longcat_flash.py b/python/sglang/srt/models/longcat_flash.py
index 0199ff3052..46e2974a48 100644
--- a/python/sglang/srt/models/longcat_flash.py
+++ b/python/sglang/srt/models/longcat_flash.py
@@ -44,9 +44,6 @@ from sglang.kernels.ops.gemm.bf16_fp32 import (
from sglang.kernels.ops.moe.ep_moe_kernels import zero_experts_compute_triton
from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
from sglang.srt.configs import LongcatFlashConfig
-from sglang.srt.distributed import (
- tensor_model_parallel_all_reduce,
-)
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers import deep_gemm_wrapper
@@ -72,7 +69,6 @@ from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.moe.topk import StandardTopKOutput, TopK
from sglang.srt.layers.moe.utils import (
filter_moe_weight_param_global_expert,
- should_skip_mlp_all_reduce,
)
from sglang.srt.layers.n_gram_embedding import NgramEmbedding
from sglang.srt.layers.quantization.base_config import QuantizationConfig
@@ -325,18 +321,6 @@ class LongcatFlashMoE(nn.Module):
if self.zero_expert_type is not None and hidden_states.shape[0] > 0:
final_hidden_states += zero_expert_result.to(final_hidden_states.device)
- # LONGCAT_MOE_A2A_SKIP_ALLREDUCE: skip the post-experts TP all-reduce when a
- # real EP a2a backend is active -- self.experts (DeepEPMoE) already combined
- # expert outputs across EP ranks, so an extra all-reduce double-counts.
- from sglang.srt.layers.moe.utils import get_moe_a2a_backend as _lc_gab
-
- if (
- self.tp_size > 1
- and _lc_gab().is_none()
- and not should_skip_mlp_all_reduce()
- ):
- final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
-
return final_hidden_states.view(num_tokens, hidden_dim)
def get_moe_weights(self):
@@ -414,7 +398,7 @@ class LongcatFlashDecoderLayer(nn.Module):
if "mlps" in getattr(config, "disable_quant_module", [])
else quant_config
),
- reduce_results=True,
+ reduce_results=False,
prefix=add_prefix(f"mlps.{i}", prefix),
)
for i in range(2)
diff --git a/python/sglang/srt/models/longcat_flash_nextn.py b/python/sglang/srt/models/longcat_flash_nextn.py
index 12ee1678ee..cb501bb3bc 100644
--- a/python/sglang/srt/models/longcat_flash_nextn.py
+++ b/python/sglang/srt/models/longcat_flash_nextn.py
@@ -154,7 +154,7 @@ class LongcatFlashDenseDecoderLayer(nn.Module):
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
- reduce_results=True,
+ reduce_results=False,
prefix=add_prefix(f"mlps", prefix),
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -191,10 +191,9 @@ class LongcatFlashDenseDecoderLayer(nn.Module):
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
- hidden_states = self.mlp(hidden_states)
- hidden_states = self.ffn_boundary.finish_complete_output(
- hidden_states, forward_batch
- )
+ with self.ffn_boundary.exit(forward_batch) as ffn_exit:
+ hidden_states = self.mlp(hidden_states)
+ hidden_states = ffn_exit.finish(hidden_states)
return hidden_states
diff --git a/python/sglang/srt/models/mellum.py b/python/sglang/srt/models/mellum.py
index 4b569f63d1..ee1c076e79 100644
--- a/python/sglang/srt/models/mellum.py
+++ b/python/sglang/srt/models/mellum.py
@@ -436,6 +436,7 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer):
hidden_act=cfg.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
+ reduce_results=False,
)
is_previous_layer_sparse = _is_sparse(layer_id - 1)
diff --git a/python/sglang/srt/models/mimo_v2.py b/python/sglang/srt/models/mimo_v2.py
index da4a6f963d..6ad1a23885 100644
--- a/python/sglang/srt/models/mimo_v2.py
+++ b/python/sglang/srt/models/mimo_v2.py
@@ -52,7 +52,6 @@ from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
get_moe_runner_backend,
- reduce_moe_output,
)
from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class
from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat
@@ -499,8 +498,6 @@ class MiMoV2MoE(nn.Module):
final_hidden_states = self.experts(hidden_states, topk_output)
- final_hidden_states = reduce_moe_output(final_hidden_states)
-
return final_hidden_states
def forward_deepep(
@@ -852,6 +849,7 @@ class MiMoV2DecoderLayer(nn.Module):
prefix=add_prefix("mlp", prefix),
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
+ reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.layernorm_epsilon)
@@ -945,7 +943,7 @@ class MiMoV2DecoderLayer(nn.Module):
)
def op_comm_postprocess_layer(self, state):
- hidden_states = self.ffn_boundary.finish_complete_output(
+ hidden_states = self.ffn_boundary.complete_now(
state.pop("hidden_states_mlp_output"), state.forward_batch
)
diff --git a/python/sglang/srt/models/mimo_v2_nextn.py b/python/sglang/srt/models/mimo_v2_nextn.py
index 57ae19213a..d702604537 100644
--- a/python/sglang/srt/models/mimo_v2_nextn.py
+++ b/python/sglang/srt/models/mimo_v2_nextn.py
@@ -113,6 +113,7 @@ class MiMoV2MTPLayer(nn.Module):
prefix=add_prefix("mlp", prefix),
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
+ reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.layernorm_epsilon)
self.post_attention_layernorm = RMSNorm(
diff --git a/python/sglang/srt/models/minimax_m2.py b/python/sglang/srt/models/minimax_m2.py
index 9c5d8339ed..b2e1f3f06e 100644
--- a/python/sglang/srt/models/minimax_m2.py
+++ b/python/sglang/srt/models/minimax_m2.py
@@ -53,7 +53,6 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
- reduce_moe_output,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
@@ -602,7 +601,6 @@ class MiniMaxM2MoE(nn.Module):
topk_output = self.topk.empty_topk_output(hidden_states.device)
final_hidden_states = self.experts(hidden_states, topk_output)
- final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_dim)
diff --git a/python/sglang/srt/models/minimax_m3.py b/python/sglang/srt/models/minimax_m3.py
index 929b9da8e5..db7de34afd 100644
--- a/python/sglang/srt/models/minimax_m3.py
+++ b/python/sglang/srt/models/minimax_m3.py
@@ -28,9 +28,6 @@ from sglang.srt.configs.model_config import (
get_minimax_sparse_disable_value_layer_ids,
get_minimax_sparse_layer_ids,
)
-from sglang.srt.distributed import (
- tensor_model_parallel_all_reduce,
-)
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
@@ -315,15 +312,10 @@ class MiniMaxM3MLP(nn.Module):
self,
x,
forward_batch: Optional[ForwardBatch] = None,
- should_allreduce_fusion: bool = False,
- use_reduce_scatter: bool = False,
):
gate_up, _ = self.gate_up_proj(x)
x = self.act_fn(gate_up)
- x, _ = self.down_proj(
- x,
- skip_all_reduce=should_allreduce_fusion or use_reduce_scatter,
- )
+ x, _ = self.down_proj(x)
return x
@@ -432,22 +424,13 @@ class MiniMaxM3MoE(nn.Module):
self,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
- should_allreduce_fusion: bool = False,
- use_reduce_scatter: bool = False,
) -> torch.Tensor:
if get_moe_a2a_backend().is_deepep():
return self.forward_deepep(hidden_states, forward_batch)
else:
- return self.forward_normal(
- hidden_states, should_allreduce_fusion, use_reduce_scatter
- )
+ return self.forward_normal(hidden_states)
- def forward_normal(
- self,
- hidden_states: torch.Tensor,
- should_allreduce_fusion: bool = False,
- use_reduce_scatter: bool = False,
- ) -> torch.Tensor:
+ def forward_normal(self, hidden_states: torch.Tensor) -> torch.Tensor:
if hidden_states.shape[0] > 0:
if (
self.alt_stream is not None
@@ -470,9 +453,6 @@ class MiniMaxM3MoE(nn.Module):
if shared_output is not None:
final_hidden_states = final_hidden_states + shared_output
- if self.tp_size > 1 and not should_allreduce_fusion and not use_reduce_scatter:
- final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
-
return final_hidden_states
def _forward_router_experts(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -1325,6 +1305,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
intermediate_size=config.dense_intermediate_size,
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
+ reduce_results=False,
)
self.use_gemma_norm = getattr(config, "use_gemma_norm", False)
@@ -1395,12 +1376,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
if self.is_layer_sparse or hidden_states.shape[0] != 0:
- hidden_states = self.mlp(
- hidden_states,
- forward_batch=forward_batch,
- should_allreduce_fusion=ffn_exit.fuse_mlp_allreduce,
- use_reduce_scatter=ffn_exit.mlp_reduce_scatter,
- )
+ hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
return ffn_exit.finish(hidden_states)
diff --git a/python/sglang/srt/models/mixtral.py b/python/sglang/srt/models/mixtral.py
index 2b759e9833..90b5460c22 100644
--- a/python/sglang/srt/models/mixtral.py
+++ b/python/sglang/srt/models/mixtral.py
@@ -25,9 +25,6 @@ import torch
from torch import nn
from transformers import MixtralConfig
-from sglang.srt.distributed import (
- tensor_model_parallel_all_reduce,
-)
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
@@ -118,8 +115,6 @@ class MixtralMoE(nn.Module):
router_logits, _ = self.gate(hidden_states)
topk_output = self.topk(hidden_states, router_logits)
final_hidden_states = self.experts(hidden_states, topk_output)
- if self.tp_size > 1:
- final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
return final_hidden_states.view(orig_shape)
@@ -269,8 +264,9 @@ class MixtralDecoderLayer(nn.Module):
# Fully Connected
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
- hidden_states = self.block_sparse_moe(hidden_states)
- return self.ffn_boundary.finish_complete_output(hidden_states, forward_batch)
+ with self.ffn_boundary.exit(forward_batch) as ffn_exit:
+ hidden_states = self.block_sparse_moe(hidden_states)
+ return ffn_exit.finish(hidden_states)
class MixtralModel(nn.Module):
diff --git a/python/sglang/srt/models/moss_vl.py b/python/sglang/srt/models/moss_vl.py
index 664c15df6c..d6f3005049 100644
--- a/python/sglang/srt/models/moss_vl.py
+++ b/python/sglang/srt/models/moss_vl.py
@@ -849,6 +849,7 @@ class MossVLTextMLP(nn.Module):
hidden_act: str = "silu",
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
+ reduce_results: bool = True,
):
super().__init__()
self.gate_up_proj = MergedColumnParallelLinear(
@@ -863,6 +864,7 @@ class MossVLTextMLP(nn.Module):
hidden_size,
bias=False,
quant_config=quant_config,
+ reduce_results=reduce_results,
prefix=add_prefix("down_proj", prefix),
)
if hidden_act != "silu":
@@ -996,6 +998,7 @@ class MossVLSelfAttentionDecoderLayer(nn.Module):
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
+ reduce_results=False,
)
norm_kwargs = (
dict(
diff --git a/python/sglang/srt/models/nanbeige.py b/python/sglang/srt/models/nanbeige.py
index 169c5883a2..c507500d79 100644
--- a/python/sglang/srt/models/nanbeige.py
+++ b/python/sglang/srt/models/nanbeige.py
@@ -77,6 +77,7 @@ class NanbeigeMLP(nn.Module):
hidden_size,
bias=False,
quant_config=quant_config,
+ reduce_results=False,
prefix=add_prefix("down_proj", prefix),
)
if hidden_act != "silu":
@@ -261,8 +262,9 @@ class NanbeigeDecoderLayer(nn.Module):
# Fully Connected
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
- hidden_states = self.mlp(hidden_states)
- return self.ffn_boundary.finish_complete_output(hidden_states, forward_batch)
+ with self.ffn_boundary.exit(forward_batch) as ffn_exit:
+ hidden_states = self.mlp(hidden_states)
+ return ffn_exit.finish(hidden_states)
class NanbeigeModel(nn.Module):
diff --git a/python/sglang/srt/models/nemotron_h.py b/python/sglang/srt/models/nemotron_h.py
index cfaf75c563..f63ceafa0e 100644
--- a/python/sglang/srt/models/nemotron_h.py
+++ b/python/sglang/srt/models/nemotron_h.py
@@ -49,7 +49,6 @@ from sglang.srt.layers.moe.topk import TopK
from sglang.srt.layers.moe.utils import (
RoutingMethodType,
get_moe_a2a_backend,
- reduce_moe_output,
)
from sglang.srt.layers.quantization import QuantizationConfig
from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod
@@ -79,7 +78,7 @@ from sglang.srt.model_loader.weight_utils import (
)
from sglang.srt.models.nemotron_h_utils import make_stage_boundary
from sglang.srt.models.utils import WeightsMapper
-from sglang.srt.runtime_context import get_exec, get_forward, get_parallel
+from sglang.srt.runtime_context import get_exec, get_parallel
from sglang.srt.utils import (
add_prefix,
get_current_device_stream_fast,
@@ -365,8 +364,6 @@ class NemotronHMoE(nn.Module):
elif shared_output is not None:
final_hidden_states += shared_output
- final_hidden_states = reduce_moe_output(final_hidden_states)
-
return final_hidden_states.view(num_tokens, hidden_dim)
@@ -420,6 +417,7 @@ class NemotronHMLPDecoderLayer(NemotronHMLPLikeDecoderLayer):
quant_config=quant_config,
bias=config.mlp_bias,
prefix=f"{prefix}.mixer",
+ reduce_results=False,
)
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
@@ -470,9 +468,7 @@ class NemotronHAttnLikeDecoderLayer(nn.Module):
return hidden_states
with self.boundary.exit(forward_batch) as mixer_exit:
- hidden_states = self._forward_mixer(
- hidden_states, forward_batch, mixer_exit.skips_reduction
- )
+ hidden_states = self._forward_mixer(hidden_states, forward_batch)
return mixer_exit.finish(hidden_states)
@@ -496,7 +492,7 @@ class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer):
rms_norm_eps=config.layer_norm_epsilon,
activation=config.mamba_hidden_act,
quant_config=quant_config,
- reduce_results=True,
+ reduce_results=False,
prefix=f"{prefix}.mixer",
)
@@ -525,19 +521,14 @@ class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer):
self,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
- skip_reduce: bool,
) -> torch.Tensor:
if is_in_breakable_cuda_graph():
output = torch.empty_like(hidden_states)
- breakable_nemotron_mamba2_with_output(
- hidden_states, output, self.layer_id, skip_reduce
- )
+ breakable_nemotron_mamba2_with_output(hidden_states, output, self.layer_id)
return output
if is_in_tc_piecewise_cuda_graph():
output = torch.empty_like(hidden_states)
- nemotron_mamba2_with_output(
- hidden_states, output, self.layer_id, skip_reduce
- )
+ nemotron_mamba2_with_output(hidden_states, output, self.layer_id)
return output
return self._forward_mamba(hidden_states, forward_batch)
@@ -593,6 +584,7 @@ class NemotronHAttention(nn.Module):
quant_config=quant_config,
tp_rank=tp_rank,
tp_size=tp_size,
+ reduce_results=False,
use_dp_attention_reduce=is_dp_attention_enabled(),
prefix=f"{prefix}.o_proj",
)
@@ -645,7 +637,6 @@ class NemotronHAttentionDecoderLayer(NemotronHAttnLikeDecoderLayer):
self,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
- skip_reduce: bool,
) -> torch.Tensor:
return self.mixer.forward(
hidden_states=hidden_states, forward_batch=forward_batch
@@ -1152,7 +1143,6 @@ def nemotron_mamba2_with_output(
hidden_states: torch.Tensor,
output: torch.Tensor,
layer_id: int,
- fuse_mlp_allreduce: bool = False,
) -> None:
"""Split op for Mamba2 forward in piecewise CUDA graph mode."""
context = get_tc_piecewise_forward_context()
@@ -1172,12 +1162,7 @@ def nemotron_mamba2_with_output(
if hidden_states.shape[0] != num_actual_tokens:
hidden_states = hidden_states[:num_actual_tokens]
- # This function is an opaque custom op under torch.compile. The caller's
- # ForwardFlags scope is Python control-plane state and is no longer active
- # when the compiled graph invokes this implementation. Carry the scalar
- # across the graph boundary and republish it for RowParallelLinear.
- with get_forward().scoped(fuse_mlp_allreduce=fuse_mlp_allreduce):
- ret = mamba_layer._forward_mamba(hidden_states, forward_batch)
+ ret = mamba_layer._forward_mamba(hidden_states, forward_batch)
# Copy result back; output may be larger (padded) so only fill actual tokens
output[:num_actual_tokens].view(ret.shape).copy_(ret)
diff --git a/python/sglang/srt/models/qwen2.py b/python/sglang/srt/models/qwen2.py
index 7cc408f924..6e0016d834 100644
--- a/python/sglang/srt/models/qwen2.py
+++ b/python/sglang/srt/models/qwen2.py
@@ -72,6 +72,7 @@ class Qwen2MLP(nn.Module):
hidden_act: str,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
+ reduce_results: bool = True,
) -> None:
super().__init__()
self.gate_up_proj = MergedColumnParallelLinear(
@@ -86,6 +87,7 @@ class Qwen2MLP(nn.Module):
hidden_size,
bias=False,
quant_config=quant_config,
+ reduce_results=reduce_results,
prefix=add_prefix("down_proj", prefix),
)
if hidden_act != "silu":
diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py
index 8ce079a33c..8a39f5a962 100644
--- a/python/sglang/srt/models/qwen2_moe.py
+++ b/python/sglang/srt/models/qwen2_moe.py
@@ -62,10 +62,7 @@ from sglang.srt.layers.linear import (
RowParallelLinear,
)
from sglang.srt.layers.logits_processor import LogitsProcessor
-from sglang.srt.layers.moe import (
- get_moe_a2a_backend,
- reduce_moe_output,
-)
+from sglang.srt.layers.moe import get_moe_a2a_backend, reduce_moe_output
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
from sglang.srt.layers.moe.topk import StandardTopKOutput, TopK, TopKOutputChecker
@@ -73,6 +70,7 @@ from sglang.srt.layers.moe.utils import (
RoutingMethodType,
filter_moe_weight_param_global_expert,
is_deepep_class_backend,
+ sum_post_experts_output,
uses_per_rank_fused_shared_slots,
)
from sglang.srt.layers.quantization.base_config import QuantizationConfig
@@ -266,8 +264,11 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
is_nextn: bool = False,
support_shared_expert_fusion: bool = False,
enable_cuda_shared_expert_fusion: bool = False,
+ reduce_results: bool = True,
):
super().__init__()
+ # False when a stage boundary completes this output's sum.
+ self.reduce_results = reduce_results
self.tp_size = get_parallel().tp_size
self.layer_id = layer_id
self.alt_stream = alt_stream
@@ -871,7 +872,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
final_hidden_states,
gated_shared_output=shared_output,
m=num_tokens,
- reduce=reduce_moe_output,
+ reduce=sum_post_experts_output,
)
if shared_output is not None:
@@ -884,7 +885,8 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
)
else:
final_hidden_states += shared_output
- final_hidden_states = reduce_moe_output(final_hidden_states)
+ if self.reduce_results:
+ final_hidden_states = reduce_moe_output(final_hidden_states)
# Debug removed - was causing issues during CUDA graph capture
@@ -1035,6 +1037,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
quant_config=quant_config,
alt_stream=alt_stream,
prefix=add_prefix("mlp", prefix),
+ reduce_results=False,
)
else:
self.mlp = Qwen2MoeMLP(
@@ -1043,6 +1046,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
+ reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNorm(
diff --git a/python/sglang/srt/models/qwen3.py b/python/sglang/srt/models/qwen3.py
index 97a40f1604..2adfc5f746 100644
--- a/python/sglang/srt/models/qwen3.py
+++ b/python/sglang/srt/models/qwen3.py
@@ -355,6 +355,7 @@ class Qwen3DecoderLayer(nn.Module):
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
+ reduce_results=False,
)
norm_kwargs = (
diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py
index 9e7818525d..44cecb6b5b 100644
--- a/python/sglang/srt/models/qwen3_5.py
+++ b/python/sglang/srt/models/qwen3_5.py
@@ -1041,6 +1041,9 @@ class Qwen3_5GatedDeltaNet(nn.Module):
class Qwen3_5LinearDecoderLayer(nn.Module):
"""Qwen3.5 Decoder Layer with Linear Attention (GatedDeltaNet)."""
+ # True in a subclass built without stage boundaries.
+ _ffn_sums_itself = False
+
def __init__(
self,
config: Qwen3_5TextConfig,
@@ -1073,6 +1076,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
prefix=add_prefix("mlp", prefix.replace(".linear_attn", "")),
is_nextn=is_nextn,
support_shared_expert_fusion=not _disable_shared_experts_fusion(),
+ reduce_results=self._ffn_sums_itself,
)
is_layer_sparse = True
is_previous_layer_sparse = True
@@ -1084,6 +1088,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix.replace(".linear_attn", "")),
+ reduce_results=self._ffn_sums_itself,
)
_maybe_enable_silu_fp4_quant_fusion(self.mlp)
is_layer_sparse = False
@@ -1169,6 +1174,9 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
class Qwen3_5AttentionDecoderLayer(nn.Module):
"""Qwen3.5 Decoder Layer with Full Attention."""
+ # See Qwen3_5LinearDecoderLayer._ffn_sums_itself.
+ _ffn_sums_itself = False
+
def __init__(
self,
config: Qwen3_5TextConfig,
@@ -1271,6 +1279,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix.replace(".self_attn", "")),
+ reduce_results=self._ffn_sums_itself,
)
is_layer_sparse = False
is_previous_layer_sparse = False
@@ -1288,6 +1297,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
prefix=add_prefix("mlp", prefix.replace(".self_attn", "")),
is_nextn=is_nextn,
support_shared_expert_fusion=not _disable_shared_experts_fusion(),
+ reduce_results=self._ffn_sums_itself,
)
is_layer_sparse = True
is_previous_layer_sparse = True
diff --git a/python/sglang/srt/models/qwen3_moe.py b/python/sglang/srt/models/qwen3_moe.py
index 159e74e973..2c76111952 100644
--- a/python/sglang/srt/models/qwen3_moe.py
+++ b/python/sglang/srt/models/qwen3_moe.py
@@ -45,7 +45,6 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
- post_experts_all_reduce,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
@@ -342,9 +341,6 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
final_hidden_states = self.experts(hidden_states, topk_output)
-
- final_hidden_states = post_experts_all_reduce(final_hidden_states)
-
return final_hidden_states.view(num_tokens, hidden_dim)
def forward_deepep(
@@ -856,6 +852,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
+ reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNorm(
@@ -941,7 +938,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
)
def op_comm_postprocess_layer(self, state):
- hidden_states = self.ffn_boundary.finish_complete_output(
+ hidden_states = self.ffn_boundary.complete_now(
state.pop("hidden_states_mlp_output"), state.forward_batch
)
diff --git a/python/sglang/srt/models/qwen3_next.py b/python/sglang/srt/models/qwen3_next.py
index 20ba669bbc..a570d0c66b 100644
--- a/python/sglang/srt/models/qwen3_next.py
+++ b/python/sglang/srt/models/qwen3_next.py
@@ -524,6 +524,7 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
is_nextn=is_nextn,
support_shared_expert_fusion=True,
enable_cuda_shared_expert_fusion=True,
+ reduce_results=False,
)
else:
self.mlp = Qwen2MoeMLP(
@@ -532,6 +533,7 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix.replace(".linear_attn", "")),
+ reduce_results=False,
)
self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = GemmaRMSNorm(
@@ -688,6 +690,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
is_nextn=is_nextn,
support_shared_expert_fusion=True,
enable_cuda_shared_expert_fusion=True,
+ reduce_results=False,
)
else:
self.mlp = Qwen2MoeMLP(
@@ -696,6 +699,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix.replace(".self_attn", "")),
+ reduce_results=False,
)
self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = GemmaRMSNorm(
diff --git a/python/sglang/srt/models/qwen4_exp.py b/python/sglang/srt/models/qwen4_exp.py
index ee3ddccd74..0894eac174 100644
--- a/python/sglang/srt/models/qwen4_exp.py
+++ b/python/sglang/srt/models/qwen4_exp.py
@@ -1410,6 +1410,9 @@ class Qwen4ExpPLELayer(nn.Module):
class Qwen4ExpLayerExtensionMixin:
+ # These layers drop the stage boundaries they inherit.
+ _ffn_sums_itself = True
+
def _init_qwen4_exp_layer_extensions(
self,
config: Qwen4ExpTextConfig,
diff --git a/python/sglang/srt/models/sarvam_moe.py b/python/sglang/srt/models/sarvam_moe.py
index 8358e96d53..cabd911796 100644
--- a/python/sglang/srt/models/sarvam_moe.py
+++ b/python/sglang/srt/models/sarvam_moe.py
@@ -34,7 +34,6 @@ from sglang.srt.layers.linear import (
RowParallelLinear,
)
from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput
-from sglang.srt.layers.moe import reduce_moe_output
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.moe.topk import TopK
@@ -368,7 +367,6 @@ class SarvamMoESparseMoeBlock(nn.Module):
final_hidden_states = final_hidden_states * self.routed_scaling_factor
current_stream.wait_stream(self.alt_stream)
final_hidden_states = final_hidden_states + shared_out
- final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_dim)
def forward_normal(
@@ -403,8 +401,6 @@ class SarvamMoESparseMoeBlock(nn.Module):
elif self.routed_scaling_factor != 1.0:
final_hidden_states = final_hidden_states * self.routed_scaling_factor
- final_hidden_states = reduce_moe_output(final_hidden_states)
-
return final_hidden_states.view(num_tokens, hidden_dim)
@@ -1038,6 +1034,7 @@ class SarvamMoEMLADecoderLayer(nn.Module):
prefix=add_prefix("mlp", prefix),
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
+ reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
diff --git a/python/sglang/srt/models/sdar.py b/python/sglang/srt/models/sdar.py
index c51ad528ba..1934110f30 100644
--- a/python/sglang/srt/models/sdar.py
+++ b/python/sglang/srt/models/sdar.py
@@ -290,7 +290,7 @@ class SDARBlock(nn.Module):
self.mlp = SDARMLP(
config=config,
quant_config=quant_config,
- reduce_results=True,
+ reduce_results=False,
prefix=add_prefix("mlp", prefix),
)
diff --git a/python/sglang/srt/models/sdar_moe.py b/python/sglang/srt/models/sdar_moe.py
index 3af99a83e3..fd6bfff8f7 100644
--- a/python/sglang/srt/models/sdar_moe.py
+++ b/python/sglang/srt/models/sdar_moe.py
@@ -31,7 +31,6 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
- reduce_moe_output,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
@@ -147,8 +146,6 @@ class SDARMoeSparseMoeBlock(nn.Module):
topk_output = self.topk(hidden_states, router_logits)
out = self.experts(hidden_states, topk_output) # (T, H)
- out = reduce_moe_output(out)
-
return out.view(num_tokens, hidden_dim)
def forward_deepep(self, hidden_states: torch.Tensor, forward_batch: ForwardBatch):
diff --git a/python/sglang/srt/models/spark2_5.py b/python/sglang/srt/models/spark2_5.py
index 9460da4ce3..bf9ca4ae33 100644
--- a/python/sglang/srt/models/spark2_5.py
+++ b/python/sglang/srt/models/spark2_5.py
@@ -264,6 +264,7 @@ class Spark2_5DecoderLayer(nn.Module):
intermediate_size=config.intermediate_size,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
+ reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -295,8 +296,9 @@ class Spark2_5DecoderLayer(nn.Module):
)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
- hidden_states = self.mlp(hidden_states)
- return self.ffn_boundary.finish_complete_output(hidden_states, forward_batch)
+ with self.ffn_boundary.exit(forward_batch) as ffn_exit:
+ hidden_states = self.mlp(hidden_states)
+ return ffn_exit.finish(hidden_states)
class Spark2_5Model(nn.Module):
diff --git a/python/sglang/srt/models/step3_vl.py b/python/sglang/srt/models/step3_vl.py
index ab1ffc4c46..f3ad5edd12 100644
--- a/python/sglang/srt/models/step3_vl.py
+++ b/python/sglang/srt/models/step3_vl.py
@@ -41,7 +41,6 @@ from sglang.srt.layers.moe import get_moe_a2a_backend
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
from sglang.srt.layers.moe.topk import TopK
-from sglang.srt.layers.moe.utils import reduce_moe_output
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.layers.rotary_embedding import get_rope
@@ -346,6 +345,7 @@ class Step3TextDecoderLayer(nn.Module):
hidden_act="silu",
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
+ reduce_results=False,
)
else:
self.use_moe = True
@@ -397,7 +397,7 @@ class Step3TextDecoderLayer(nn.Module):
hidden_states += self.share_expert(h)
else:
hidden_states = self.moe(hidden_states)
- return reduce_moe_output(hidden_states)
+ return hidden_states
def forward(
self,
diff --git a/python/sglang/srt/models/step3p5.py b/python/sglang/srt/models/step3p5.py
index 73915bdec6..7514ed13be 100644
--- a/python/sglang/srt/models/step3p5.py
+++ b/python/sglang/srt/models/step3p5.py
@@ -27,7 +27,6 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
- reduce_moe_output,
should_use_flashinfer_cutlass_moe_fp4_allgather,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
@@ -226,7 +225,6 @@ class Step3p5MoEMLP(nn.Module):
final_hidden_states = self.experts(hidden_states, topk_output)
if shared_output is not None:
final_hidden_states += shared_output
- final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_dim)
@@ -506,6 +504,7 @@ class Step3p5DecoderLayer(nn.Module):
swiglu_limit=swiglu_limit_shared,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
+ reduce_results=False,
)
self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
diff --git a/python/sglang/srt/models/xllm.py b/python/sglang/srt/models/xllm.py
index 17ba89b54c..928e59c9fc 100644
--- a/python/sglang/srt/models/xllm.py
+++ b/python/sglang/srt/models/xllm.py
@@ -58,7 +58,6 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
- reduce_moe_output,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
@@ -1059,7 +1058,6 @@ class XllmSparseMoeBlock(nn.Module):
if shared_output is not None:
final_hidden_states += shared_output
- final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_dim)
@@ -1520,6 +1518,7 @@ class XllmDecoderLayer(nn.Module):
prefix=add_prefix("mlp", prefix),
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
+ reduce_results=False,
)
self.input_layernorm = _make_norm(config)
diff --git a/python/sglang/test/boundary_fixtures.py b/python/sglang/test/boundary_fixtures.py
index f4b9a67c2b..680b1694ab 100644
--- a/python/sglang/test/boundary_fixtures.py
+++ b/python/sglang/test/boundary_fixtures.py
@@ -152,7 +152,7 @@ def postprocess_output(boundary, hidden, residual, forward_batch):
stream = (
residual if isinstance(residual, ResidualStream) else ResidualStream(residual)
)
- hidden = boundary.finish_complete_output(hidden, stream, forward_batch)
+ hidden = boundary.complete_now(hidden, stream, forward_batch)
if isinstance(residual, ResidualStream):
return hidden, stream
return (hidden, None) if stream.pending is None else stream.input(hidden)
diff --git a/test/registered/unit/layer_boundary/test_boundary_output_contracts.py b/test/registered/unit/layer_boundary/test_boundary_output_contracts.py
index 243bf8512f..931640eb5b 100644
--- a/test/registered/unit/layer_boundary/test_boundary_output_contracts.py
+++ b/test/registered/unit/layer_boundary/test_boundary_output_contracts.py
@@ -36,20 +36,22 @@ class TestBoundaryIntegrations(unittest.TestCase):
parallel = fixture.parallel_of(attn_dp=2, attn_tp=2)
rows = fixture.comm.Layout(frozenset())
produced = fixture.comm.OutputContract(
- rows, group=SumGroup.ATTN_TP, may_defer_to_next=True
+ rows, group=SumGroup.ATTN_TP, may_defer_to_next=True, update=PLAIN_ADD
)
plan = SimpleNamespace(path_for=lambda _: SimpleNamespace(output=produced))
boundary = exits.ExitPolicy(plan)
boundary._sum_deferral_allowed = lambda _: True
+ summed = Mock(side_effect=lambda value, *args, **kwargs: value * 2)
with (
fixture.planning(parallel),
patch.object(exits, "is_dp_attention_enabled", return_value=True),
+ patch.object(exits, "sum_output", summed),
):
stream = ResidualStream(torch.zeros(3, 4))
result = exits.MixerExit(boundary, None, stream=stream)
- self.assertFalse(result.skips_reduction)
value = torch.ones(3, 4)
- self.assertIs(result.finish(value), value)
+ torch.testing.assert_close(result.finish(value), value * 2)
+ self.assertEqual(summed.call_args.args[1], SumGroup.ATTN_TP)
self.assertIsNone(stream.pending.owed)
def test_pipeline_preserves_declared_sum_for_one_receiver_completion(self):
diff --git a/test/registered/unit/layer_boundary/test_boundary_output_policy.py b/test/registered/unit/layer_boundary/test_boundary_output_policy.py
index d00944d79f..0d6bc2a68d 100644
--- a/test/registered/unit/layer_boundary/test_boundary_output_policy.py
+++ b/test/registered/unit/layer_boundary/test_boundary_output_policy.py
@@ -96,37 +96,24 @@ class TestBoundaryOutputPolicy(unittest.TestCase):
reset_context()
def test_step3_shared_and_routed_outputs_have_one_completion(self):
+ """The MoE adds its shared and routed partial outputs and leaves their
+ one sum to the stage boundary."""
from sglang.srt.models.step3_vl import Step3TextDecoderLayer
- parallel = SimpleNamespace(tp_size=2)
for fused_shared in (False, True):
- for deferred in (False, True):
- calls = []
- layer = SimpleNamespace(
- num_fused_shared_experts=int(fused_shared),
- moe=lambda x: x * (0.75 if fused_shared else 0.25),
- share_expert=lambda x: x * 0.5,
- )
- with (
- patch.object(moe_utils, "get_parallel", return_value=parallel),
- patch.object(
- moe_utils, "post_experts_output_is_complete", return_value=False
- ),
- patch(
- "sglang.srt.distributed.communication_op.tensor_model_parallel_all_reduce",
- side_effect=lambda x: calls.append("AR") or x * 2,
- ),
- get_forward().scoped(
- fuse_mlp_allreduce=deferred, mlp_reduce_scatter=False
- ),
- ):
- result = Step3TextDecoderLayer.moe_mlp_forward(
- layer, torch.ones(2, 4)
- )
- self.assertEqual(calls, [] if deferred else ["AR"])
- torch.testing.assert_close(
- result, torch.full((2, 4), 0.75 if deferred else 1.5)
- )
+ calls = []
+ layer = SimpleNamespace(
+ num_fused_shared_experts=int(fused_shared),
+ moe=lambda x: x * (0.75 if fused_shared else 0.25),
+ share_expert=lambda x: x * 0.5,
+ )
+ with patch(
+ "sglang.srt.distributed.communication_op.tensor_model_parallel_all_reduce",
+ side_effect=lambda x: calls.append("AR") or x * 2,
+ ):
+ result = Step3TextDecoderLayer.moe_mlp_forward(layer, torch.ones(2, 4))
+ self.assertEqual(calls, [])
+ torch.testing.assert_close(result, torch.full((2, 4), 0.75))
def test_dp_skip_and_collective_are_selected_together(self):
import itertools
@@ -147,6 +134,10 @@ class TestBoundaryOutputPolicy(unittest.TestCase):
calls.append(step.__name__)
return value[:1]
+ def summed(value, *args, **kwargs):
+ calls.append("AR")
+ return value * 2
+
selected = (
"dp_reduce_scatterv"
if varlen and policy in ("rsv", "rs+rsv")
@@ -159,21 +150,25 @@ class TestBoundaryOutputPolicy(unittest.TestCase):
patch_communicator("should_use_dp_reduce_scatterv", lambda: varlen),
patch_communicator("can_use_dp_reduce_scatter", lambda: can_rs),
patch_communicator("to_dp_local", move),
+ patch_communicator("sum_output", summed),
):
with layer.ffn.plan.output.ffn_exit(
self.batch(max_len), stream=ResidualStream()
) as output:
- self.assertEqual(
- get_forward().mlp_reduce_scatter,
- selected != "_dp_scatter_step",
- )
+ self.assertFalse(get_forward().mlp_reduce_scatter)
result, _ = finish_exit(output, torch.ones(2, 4), torch.zeros(1, 4))
- self.assertEqual(calls, [selected])
+ # A reduce-scatter completes the sum on the way back; a scatter
+ # follows the exit's own all-reduce.
+ self.assertEqual(
+ calls,
+ ["AR", selected] if selected == "_dp_scatter_step" else [selected],
+ )
self.assertEqual(result.shape, (1, 4))
- def test_postprocess_only_scatters_an_already_reduced_output(self):
- # The operation-scheduled API used by LongCat NextN receives an MLP
- # output that was already summed. An enabled RSv must not sum it again.
+ def test_complete_now_sums_then_only_scatters(self):
+ # The operation-scheduled path (two-batch overlap) never defers: it
+ # completes the sum its output owes, then only scatters, whatever the
+ # reduce-scatter policy.
for policy in ("ar", "rs", "rsv", "rs+rsv"):
for varlen in (False, True):
with self.subTest(policy=policy, varlen=varlen):
@@ -191,6 +186,10 @@ class TestBoundaryOutputPolicy(unittest.TestCase):
calls.append(step.__name__)
return value[:1]
+ def summed(value, *args, **kwargs):
+ calls.append("AR")
+ return value * 2
+
with (
fixture.planning(parallel),
patch_communicator(
@@ -198,11 +197,12 @@ class TestBoundaryOutputPolicy(unittest.TestCase):
),
patch_communicator("can_use_dp_reduce_scatter", lambda: True),
patch_communicator("to_dp_local", move),
+ patch_communicator("sum_output", summed),
):
- output = layer.ffn.finish_complete_output(hidden, batch)
+ output = layer.ffn.complete_now(hidden, batch)
value, saved_residual = batch.residual_stream.export(output)
- self.assertEqual(calls, ["_dp_scatter_step"])
- torch.testing.assert_close(value, hidden[:1])
+ self.assertEqual(calls, ["AR", "_dp_scatter_step"])
+ torch.testing.assert_close(value, hidden[:1] * 2)
self.assertIs(saved_residual, residual)
def test_moe_cannot_skip_from_topology_without_boundary_request(self):
@@ -221,9 +221,9 @@ class TestBoundaryOutputPolicy(unittest.TestCase):
skip,
)
- def test_output_transform_stays_before_transport_and_after_producer_ar(self):
- # The compute stub obeys the same skip flag as RowParallelLinear.
- # Its trace distinguishes AR -> multiply from multiply -> RS.
+ def test_output_transform_stays_before_transport_and_after_the_sum(self):
+ # The trace distinguishes AR -> multiply from multiply -> RS; the
+ # exit runs the all-reduce, the compute never does.
for disabled in (False, True):
with self.subTest(disabled=disabled):
calls = []
@@ -248,19 +248,21 @@ class TestBoundaryOutputPolicy(unittest.TestCase):
calls.append("slice")
return value[:1]
+ def summed(value, *args, **kwargs):
+ calls.append("AR")
+ return value * 2
+
with (
fixture.planning(parallel),
patch_communicator("should_use_dp_reduce_scatterv", lambda: False),
patch_communicator("can_use_dp_reduce_scatter", lambda: True),
patch_communicator("to_dp_local", move),
+ patch_communicator("sum_output", summed),
):
with layer.ffn.plan.output.ffn_exit(
self.batch(), stream=ResidualStream()
) as output:
value = torch.ones(2, 4)
- if not get_forward().mlp_reduce_scatter:
- calls.append("AR")
- value = value * 2
value, _ = finish_exit(output, value, torch.zeros(1, 4))
self.assertEqual(
calls,
diff --git a/test/registered/unit/layer_boundary/test_declared_attention_cp.py b/test/registered/unit/layer_boundary/test_declared_attention_cp.py
index 83afd4ce81..6e7ca879f1 100644
--- a/test/registered/unit/layer_boundary/test_declared_attention_cp.py
+++ b/test/registered/unit/layer_boundary/test_declared_attention_cp.py
@@ -188,8 +188,9 @@ class TestAttentionCpBoundary(CustomTestCase):
)
def run_ranks(self, use_reduce_scatter):
- """Gather on each rank, run an FFN that leaves a partial sum or not,
- finish the exit and return what each rank got back and published."""
+ """Gather on each rank, run an FFN that leaves its partial sum, finish
+ the exit and return the ranks the exit summed over, what each rank got
+ back, and the ranks the reduce-scatter summed over."""
handed = {}
def record_gather(cp):
@@ -230,10 +231,17 @@ class TestAttentionCpBoundary(CustomTestCase):
for cp in range(CP_SIZE):
torch.testing.assert_close(gathered[cp], expected_rows, rtol=0, atol=0)
- def compute_output(cp, leaves):
- return gathered[cp] * PARTIAL_WEIGHTS[cp] if leaves else gathered[cp]
+ reduced, summed = {}, {}
- reduced = {}
+ def record_sum(cp):
+ def sum_output(value, *args, **kwargs):
+ summed[cp] = value.clone()
+ return value
+
+ return sum_output
+
+ def fill_sum(value, *args, **kwargs):
+ return sum(summed[r] for r in range(CP_SIZE))
def record_reduce_scatter(cp):
def reduce_scatter(output, input_):
@@ -248,7 +256,7 @@ class TestAttentionCpBoundary(CustomTestCase):
return reduce_scatter
- published, back = {}, {}
+ back = {}
for phase in ("record", "fill"):
for cp in range(CP_SIZE):
reduce_scatter = (
@@ -256,33 +264,38 @@ class TestAttentionCpBoundary(CustomTestCase):
if phase == "record"
else fill_reduce_scatter(cp)
)
- with self.as_rank(
- cp, dict(gather=fill_gather, reduce_scatter=reduce_scatter)
- ) as rank:
+ with (
+ self.as_rank(
+ cp, dict(gather=fill_gather, reduce_scatter=reduce_scatter)
+ ) as rank,
+ patch_communicator(
+ "sum_output", record_sum(cp) if phase == "record" else fill_sum
+ ),
+ ):
communicator = self.build(use_reduce_scatter)
with communicator.ffn.plan.output.ffn_exit(
self.cp_extend(), stream=ResidualStream()
) as exit_:
- published[cp] = rank.flags.mlp_reduce_scatter
- output = compute_output(cp, leaves=published[cp])
+ self.assertFalse(rank.flags.mlp_reduce_scatter)
+ output = gathered[cp] * PARTIAL_WEIGHTS[cp]
back[cp], _ = finish_exit(
exit_, output, ResidualStream(residuals[cp].residual)
)
- return published, back, reduced
+ return summed, back, reduced
def test_the_reduce_scatter_completes_the_sum_the_moe_leaves(self):
- published, back, reduced = self.run_ranks(use_reduce_scatter=True)
- self.assertEqual(published, {0: True, 1: True}, "the MoE leaves its sum")
+ summed, back, reduced = self.run_ranks(use_reduce_scatter=True)
+ self.assertEqual(summed, {}, "the exit runs no all-reduce")
self.assertEqual(sorted(reduced), [0, 1], "every rank joins the reduce-scatter")
for cp in range(CP_SIZE):
torch.testing.assert_close(
back[cp], self.values[cp] + self.residuals[cp], rtol=0, atol=0
)
- def test_a_complete_output_is_only_taken_back(self):
- published, back, reduced = self.run_ranks(use_reduce_scatter=False)
- self.assertEqual(published, {0: False, 1: False}, "the MoE sums itself")
- self.assertEqual(reduced, {}, "nothing is summed again")
+ def test_without_a_reduce_scatter_the_exit_sums_then_takes_back(self):
+ summed, back, reduced = self.run_ranks(use_reduce_scatter=False)
+ self.assertEqual(sorted(summed), [0, 1], "the exit sums what the MoE leaves")
+ self.assertEqual(reduced, {}, "the take-back sums nothing")
for cp in range(CP_SIZE):
torch.testing.assert_close(
back[cp], self.values[cp] + self.residuals[cp], rtol=0, atol=0
diff --git a/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py b/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py
index 968d1f5c57..8f17a14ef3 100644
--- a/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py
+++ b/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py
@@ -53,6 +53,7 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import (
from sglang.srt.layers.layer_boundary.residual.stream import OwedOutput, ResidualStream
from sglang.srt.layers.moe import utils as moe_utils
from sglang.srt.model_executor.forward_batch_info import ForwardMode
+from sglang.srt.runtime_context import get_forward
from sglang.test.boundary_fixtures import (
finish_exit,
make_test_stages,
@@ -415,7 +416,9 @@ class TestMhcOnTheDeclarations(CustomTestCase):
with communicator.ffn.plan.output.ffn_exit(
forward_batch, stream=ResidualStream()
) as ffn_exit:
- self.assertTrue(ffn_exit.mlp_reduce_scatter)
+ # The reduce-scatter on the way back completes the sum;
+ # the FFN never runs or skips it.
+ self.assertFalse(get_forward().mlp_reduce_scatter)
hidden, residual = finish_exit(
ffn_exit, torch.ones(4, 4), torch.full((2, 4), 2.0)
)
@@ -1420,7 +1423,11 @@ class TestBranchRows(CustomTestCase):
branch_output = branch.branch_output
merge_branch = branch.merge_branch
- moe = Communicator(branch_rows=lambda fb: (self.local, self.local, self.local))
+ moe = Communicator(
+ branch_rows=lambda fb: (self.local, self.local, self.local),
+ # The a2a combine already summed the MoE output.
+ path_for=lambda fb: SimpleNamespace(output=comm.OutputContract(self.local)),
+ )
dense = Communicator(
branch_rows=lambda fb: (self.full, self.attention, self.attention)
)
@@ -1642,6 +1649,12 @@ def running(*, reduce_scatterv, a2a=False, use_reduce_scatter=True):
"attention_tensor_model_parallel_all_reduce": lambda x: (
state().parallel.attn_tp_group.all_reduce(x)
),
+ "tensor_model_parallel_all_reduce": lambda x: (
+ state().parallel.tp_group.all_reduce(x)
+ ),
+ "sum_post_experts_output": lambda x: (
+ comm_layout.post_experts_reduction_group().all_reduce(x)
+ ),
"get_global_dp_buffer": lambda g: torch.zeros(
state().global_rows, HIDDEN, dtype=torch.double
),
@@ -1709,21 +1722,15 @@ def attention(x, state):
def dense_mlp(x, state):
- """A dense MLP on the TP group: 5 * x, reduced unless a published flag asks
- it to leave the sum, as RowParallelLinear does."""
- partial_sum = 5 * x * WEIGHTS[state.parallel.tp_size][state.parallel.tp_rank]
- if state.flags.fuse_mlp_allreduce or state.flags.mlp_reduce_scatter:
- return partial_sum
- return state.parallel.tp_group.all_reduce(partial_sum)
+ """A dense MLP on the TP group: 5 * x, as this rank's partial sum; the
+ stage boundary completes it."""
+ return 5 * x * WEIGHTS[state.parallel.tp_size][state.parallel.tp_rank]
def moe(x, state):
- """A MoE block not dispatched per DP shard: 5 * x, reduced over the MoE
- output's group unless a published flag asks it to leave the sum."""
- partial_sum = 5 * x * WEIGHTS[state.parallel.tp_size][state.parallel.tp_rank]
- if state.flags.fuse_mlp_allreduce or state.flags.mlp_reduce_scatter:
- return partial_sum
- return comm_layout.post_experts_reduction_group().all_reduce(partial_sum)
+ """A MoE block not dispatched per DP shard: 5 * x, as this rank's partial
+ sum over the MoE output's group; the stage boundary completes it."""
+ return 5 * x * WEIGHTS[state.parallel.tp_size][state.parallel.tp_rank]
def reference(x):
diff --git a/test/registered/unit/layer_boundary/test_dp_attention_cp_gather.py b/test/registered/unit/layer_boundary/test_dp_attention_cp_gather.py
index 52c18d1144..4f22e585a9 100644
--- a/test/registered/unit/layer_boundary/test_dp_attention_cp_gather.py
+++ b/test/registered/unit/layer_boundary/test_dp_attention_cp_gather.py
@@ -1,8 +1,8 @@
"""The dense-FFN gather and take-back under attention DP x CP x TP, rank by rank.
Every rank builds its layer's communicator and runs its prepare_mlp and
-finish_complete_output for real on CPU; the TP-group all-reduce is replaced by the
-sum of the buffers the ranks hand to it.
+complete_now for real on CPU; the TP-group all-reduce is replaced by the sum of
+the buffers the ranks hand to it.
"""
import unittest
@@ -22,6 +22,7 @@ from sglang.srt.layers.cp.zigzag import ZigzagCPStrategy
from sglang.srt.layers.dp_attention import DpPaddingMode
from sglang.srt.layers.layer_boundary import prepare as comm_ops
from sglang.srt.layers.layer_boundary.contracts import BatchVariant
+from sglang.srt.layers.layer_boundary.layout import SumGroup
from sglang.srt.layers.layer_boundary.residual.add_norm import (
NORM_READOUT,
PLAIN_RESIDUAL_OPS,
@@ -150,8 +151,9 @@ class TestDpCpGather(CustomTestCase):
ffn_input=None,
sparse=False,
):
- """Gather on every rank, check the FFN input and residual, then take back
- an FFN output of three times the input and check each rank's rows.
+ """Gather on every rank, check the FFN input and residual, then complete
+ and take back an FFN output of three times the input, of which each rank
+ holds its share, and check each rank's rows.
``norm_rows`` is what ``norm`` does to complete rows; ``ffn_input``, when
given, is the step every rank's batch must take."""
# Binary fractions, so the partial sums add back to the value exactly.
@@ -340,13 +342,25 @@ class TestDpCpGather(CustomTestCase):
hidden_states, residual = gather(rank, lambda x: summed.clone())
torch.testing.assert_close(hidden_states, expected, rtol=0, atol=0)
torch.testing.assert_close(residual.residual[:tokens], own, rtol=0, atol=0)
- with as_rank(*rank, None) as r:
+ tp_size = DP_SIZE * CP_SIZE * attn_tp_size
+ sums = []
+
+ def boundary_sum(value, group, *args, **kwargs):
+ # Every rank holds the same share of the same rows.
+ sums.append(group)
+ return value * tp_size
+
+ with (
+ as_rank(*rank, None) as r,
+ patch_communicator("sum_output", boundary_sum),
+ ):
back, _ = postprocess_output(
r.communicator.ffn.plan.output,
- 3 * hidden_states,
+ 3 * hidden_states / tp_size,
residual,
r.forward_batch,
)
+ self.assertEqual(sums, [SumGroup.MOE_OUTPUT if sparse else SumGroup.TP])
expected_back = torch.zeros(group.held_rows[cp], HIDDEN).double()
expected_back[:tokens] = 3 * norm_rows(own)
torch.testing.assert_close(back, expected_back, rtol=0, atol=0)
diff --git a/test/registered/unit/layer_boundary/test_ffn_exit.py b/test/registered/unit/layer_boundary/test_ffn_exit.py
index da19337cfe..4a5aa4a3a2 100644
--- a/test/registered/unit/layer_boundary/test_ffn_exit.py
+++ b/test/registered/unit/layer_boundary/test_ffn_exit.py
@@ -110,6 +110,8 @@ def make_communicator(
def published_flags():
+ """The skip flags the FFN sees; the boundary completes the sum, so the FFN
+ never leaves it out."""
forward = get_forward()
return forward.fuse_mlp_allreduce, forward.mlp_reduce_scatter
@@ -149,7 +151,7 @@ class TestFfnExit(CustomTestCase):
scatters_to_local_tokens=True,
)
seen, (hidden_states, residual) = self.run_exit(communicator)
- self.assertEqual(seen, (False, True))
+ self.assertEqual(seen, (False, False))
self.assertIsInstance(hidden_states, UnreducedOutput)
bound = hidden_states.reduce_to_dp_local
self.assertIs(bound.func, transport_ops.to_dp_local)
@@ -178,9 +180,15 @@ class TestFfnExit(CustomTestCase):
fuse=False, reduce_scatter=reduce_scatter
)
seen, (hidden_states, residual) = self.run_exit(communicator)
- self.assertEqual(seen, (False, reduce_scatter))
+ self.assertEqual(seen, (False, False))
self.assertNotIsInstance(hidden_states, UnreducedOutput)
communicator.output._complete_now.assert_called_once()
+ # A reduce-scatter on the way back completes the sum; otherwise
+ # the exit completes it.
+ self.assertEqual(
+ communicator.output._complete_now.call_args.kwargs["owes_sum"],
+ not reduce_scatter,
+ )
torch.testing.assert_close(hidden_states, self.hidden_states * 2 + 1)
self.assertIs(residual, self.residual)
@@ -190,7 +198,7 @@ class TestFfnExit(CustomTestCase):
)
seen, (hidden_states, residual) = self.run_exit(communicator)
- self.assertEqual(seen, (False, True))
+ self.assertEqual(seen, (False, False))
communicator.output._defers_sum.assert_not_called()
communicator.output._complete_now.assert_called_once()
torch.testing.assert_close(hidden_states, self.hidden_states * 2 + 1)
@@ -209,7 +217,7 @@ class TestFfnExit(CustomTestCase):
decide.return_value = not fuse
hidden_states = self.hidden_states * 2
hidden_states, _ = finish_exit(ffn_exit, hidden_states, self.residual)
- self.assertEqual(seen, (fuse, False))
+ self.assertEqual(seen, (False, False))
self.assertEqual(isinstance(hidden_states, UnreducedOutput), fuse)
self.assertEqual(communicator.output._complete_now.called, not fuse)
@@ -233,8 +241,9 @@ class TestFfnExit(CustomTestCase):
self.assertEqual(published_flags(), before)
def test_deferral_implies_fusion_and_passes_the_handoff_through(self):
- """A deferring communicator publishes both flags, and a non-tensor
- handoff leaves finish() untouched for the next layer's input norm."""
+ """A deferring communicator publishes the finalize choice, and a
+ non-tensor handoff leaves finish() untouched for the next layer's input
+ norm."""
fusions = types.SimpleNamespace(can_defer_finalize=lambda layer, fb: True)
communicator = make_communicator(
@@ -245,7 +254,7 @@ class TestFfnExit(CustomTestCase):
self.forward_batch, stream=ResidualStream()
) as ffn_exit:
seen = published_flags() + (get_forward().defer_moe_finalize,)
- self.assertEqual(seen, (True, False, True))
+ self.assertEqual(seen, (False, False, True))
self.assertEqual(
finish_exit(ffn_exit, handoff, self.residual), (handoff, self.residual)
)
diff --git a/test/registered/unit/layer_boundary/test_longcat_flash_shortcut.py b/test/registered/unit/layer_boundary/test_longcat_flash_shortcut.py
index 505896abd1..bc972bafbc 100644
--- a/test/registered/unit/layer_boundary/test_longcat_flash_shortcut.py
+++ b/test/registered/unit/layer_boundary/test_longcat_flash_shortcut.py
@@ -36,6 +36,8 @@ class TestLongcatShortcut(CustomTestCase):
prepare_attn=lambda h, r, batch: (h, r),
prepare_mlp=lambda h, r, batch: (fork_hidden, fork_residual),
branch_rows=lambda batch: (local, local, attention),
+ # Each rank's own tokens: the MoE output owes no sum.
+ path_for=lambda batch: SimpleNamespace(output=comm.OutputContract(local)),
)
dense_communicator = Communicator(
branch_rows=lambda batch: (attention, attention, attention)
diff --git a/test/registered/unit/layer_boundary/test_nemotron_h_aux_capture.py b/test/registered/unit/layer_boundary/test_nemotron_h_aux_capture.py
index 6a4fea2876..c93f256b81 100644
--- a/test/registered/unit/layer_boundary/test_nemotron_h_aux_capture.py
+++ b/test/registered/unit/layer_boundary/test_nemotron_h_aux_capture.py
@@ -4,7 +4,6 @@ from types import SimpleNamespace
import torch
from torch import nn
-from sglang.srt.layers.moe.utils import should_skip_mlp_all_reduce
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.models import nemotron_h as model
from sglang.srt.runtime_context import get_context, get_flags, get_parallel
@@ -28,7 +27,7 @@ class _Norm(nn.Module):
class _Mixer(nn.Module):
- """Row-parallel stand-in: a TP partial, reduced unless the layer skips it."""
+ """Row-parallel stand-in: a TP partial, whose sum the stage boundary completes."""
def __init__(self, scale, tp=1):
super().__init__()
@@ -36,8 +35,7 @@ class _Mixer(nn.Module):
self.tp = tp
def forward(self, hidden_states, **kwargs):
- partial = hidden_states * (self.scale / self.tp)
- return partial if should_skip_mlp_all_reduce() else partial * self.tp
+ return hidden_states * (self.scale / self.tp)
def _build(pattern, tp, capture):
diff --git a/test/registered/unit/layer_boundary/test_nemotron_h_mtp_reduction.py b/test/registered/unit/layer_boundary/test_nemotron_h_mtp_reduction.py
index 9ad4e97b86..1c00e1a28b 100644
--- a/test/registered/unit/layer_boundary/test_nemotron_h_mtp_reduction.py
+++ b/test/registered/unit/layer_boundary/test_nemotron_h_mtp_reduction.py
@@ -25,19 +25,32 @@ class _Norm(nn.Module):
return x + residual, x + residual
+class _Partial(nn.Module):
+ def __init__(self, tp):
+ super().__init__()
+ self.tp = tp
+
+ def forward(self, x, **kwargs):
+ return x / self.tp
+
+
class TestNemotronMTPReduction(CustomTestCase):
def test_attention_partial_is_reduced_once(self):
"""Under DP attention, the MTP MoE layer sums the attention output over the
- attention TP group exactly once before the residual add."""
+ attention TP group exactly once before the residual add, and its own
+ output over the MoE-TP group once at its exit."""
for tp in (1, 2):
with self.subTest(tp=tp):
reduce = Mock(side_effect=lambda x: x * tp)
group = SimpleNamespace(all_reduce=reduce)
+ moe_reduce = Mock(side_effect=lambda x: x * tp)
+ moe_group = SimpleNamespace(all_reduce=moe_reduce)
with (
get_context().override_server_args(tp_size=tp),
get_flags().dp.override(enabled=True),
get_parallel().override(
attn_tp_group=group,
+ moe_tp_group=moe_group,
launch_world_rank=0,
tp_rank=0,
tp_size=tp,
@@ -66,7 +79,8 @@ class TestNemotronMTPReduction(CustomTestCase):
nn.Module.__init__(layer)
layer.has_start_projections = False
layer.has_end_norm = False
- layer.mixer = nn.Identity()
+ # A MoE stand-in leaving its output's sum to the exit.
+ layer.mixer = _Partial(tp)
layer.norm = _Norm()
layer._init_stage_boundary(
SimpleNamespace(hybrid_override_pattern="*E"), 1
@@ -93,6 +107,7 @@ class TestNemotronMTPReduction(CustomTestCase):
torch.testing.assert_close(hidden, expected)
torch.testing.assert_close(batch.residual_stream.residual, expected)
self.assertEqual(reduce.call_count, int(tp > 1))
+ self.assertEqual(moe_reduce.call_count, int(tp > 1))
if __name__ == "__main__":
diff --git a/test/registered/unit/layer_boundary/test_nemotron_h_stages.py b/test/registered/unit/layer_boundary/test_nemotron_h_stages.py
index ca8ef7bfc2..44dfbdb40a 100644
--- a/test/registered/unit/layer_boundary/test_nemotron_h_stages.py
+++ b/test/registered/unit/layer_boundary/test_nemotron_h_stages.py
@@ -7,6 +7,7 @@ from unittest.mock import MagicMock, patch
import torch
from sglang.srt.layers.layer_boundary import (
+ PLAIN_ADD,
Layout,
MixerExit,
OutputContract,
@@ -14,6 +15,7 @@ from sglang.srt.layers.layer_boundary import (
TokenAxis,
UnreducedOutput,
)
+from sglang.srt.layers.layer_boundary import exit as exit_module
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
from sglang.srt.layers.moe.utils import should_skip_mlp_all_reduce
from sglang.srt.models import nemotron_h_utils as utils
@@ -166,9 +168,10 @@ class TestStageEdges(CustomTestCase):
class TestMixerExit(CustomTestCase):
- """A mixer skips its output all-reduce when its output always leaves the sum
- (to an FFN stage), and when it may leave it and the fused kernel takes it;
- what it hands on says which."""
+ """A mixer never runs its output all-reduce. The exit carries the sum to the
+ next attention stage when its declaration permits deferring and the fused
+ kernel takes it, hands it on as the declared sum when it always leaves it
+ (to an FFN stage), and otherwise completes it itself."""
def test_decision_table(self):
tp_group = object()
@@ -184,6 +187,7 @@ class TestMixerExit(CustomTestCase):
group=SumGroup.ATTN_TP if always or may else None,
always_partial=always,
may_defer_to_next=may,
+ update=PLAIN_ADD,
)
communicator = SimpleNamespace(
plan=SimpleNamespace(
@@ -192,22 +196,28 @@ class TestMixerExit(CustomTestCase):
_sum_deferral_allowed=MagicMock(return_value=movable),
)
hidden = torch.ones(2, 4)
- with get_parallel().override(tp_group=tp_group, tp_size=2):
+ summed = MagicMock(side_effect=lambda h, *args, **kwargs: h * 2)
+ with (
+ get_parallel().override(tp_group=tp_group, tp_size=2),
+ patch.object(exit_module, "sum_output", summed),
+ ):
with MixerExit(
communicator, None, stream=ResidualStream()
) as mixer_exit:
skipped = should_skip_mlp_all_reduce()
output = mixer_exit.finish(hidden)
output, _ = mixer_exit._stream.input(output)
- self.assertFalse(should_skip_mlp_all_reduce())
- hands_on = may and movable
- self.assertEqual(mixer_exit.skips_reduction, always or hands_on)
- self.assertEqual(skipped, always or hands_on)
- if hands_on:
+ self.assertFalse(skipped)
+ if may and movable:
self.assertIsInstance(output, UnreducedOutput)
self.assertIs(output.group, tp_group)
+ summed.assert_not_called()
+ elif may:
+ self.assertEqual(summed.call_args.args[1], SumGroup.ATTN_TP)
+ torch.testing.assert_close(output, hidden * 2)
else:
self.assertIs(output, hidden)
+ summed.assert_not_called()
if __name__ == "__main__":
diff --git a/test/registered/unit/layer_boundary/test_stage_producers_leave_sums.py b/test/registered/unit/layer_boundary/test_stage_producers_leave_sums.py
new file mode 100644
index 0000000000..20239e7c4b
--- /dev/null
+++ b/test/registered/unit/layer_boundary/test_stage_producers_leave_sums.py
@@ -0,0 +1,100 @@
+"""Stage producers leave their output sums to the stage boundary."""
+
+import ast
+import unittest
+from pathlib import Path
+
+import torch
+
+import sglang.srt.models
+from sglang.srt.layers.layer_boundary.stage import (
+ StageBoundary,
+ check_stage_producers,
+)
+from sglang.test.ci.ci_register import register_cpu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_cpu_ci(est_time=5, suite="base-a-test-cpu")
+
+# Calls that complete a TP / EP sum of a stage output in compute.
+_SUMS = {
+ "reduce_moe_output",
+ "post_experts_all_reduce",
+ "tensor_model_parallel_all_reduce",
+ "moe_expert_parallel_all_reduce",
+ "moe_tensor_model_parallel_all_reduce",
+}
+_BUILDS_STAGES = (
+ "make_stages(",
+ "declare_attn(",
+ "declare_ffn(",
+ "_build_stages(",
+ "_init_stage_boundary(",
+)
+
+
+def _call_name(node: ast.Call):
+ func = node.func
+ return func.attr if isinstance(func, ast.Attribute) else getattr(func, "id", None)
+
+
+def _unguarded_sums(source: str):
+ """Sum calls not under ``if ... self.reduce_results ...``: such a branch is
+ a shared class built without stage boundaries by another model."""
+ found = []
+
+ def visit(node, guarded):
+ if isinstance(node, ast.If) and "reduce_results" in ast.unparse(node.test):
+ for child in node.body:
+ visit(child, True)
+ for child in node.orelse:
+ visit(child, guarded)
+ return
+ if isinstance(node, ast.Call) and not guarded and _call_name(node) in _SUMS:
+ found.append(f"{node.lineno}: {ast.unparse(node)}")
+ for child in ast.iter_child_nodes(node):
+ visit(child, guarded)
+
+ visit(ast.parse(source), False)
+ return found
+
+
+class TestStageProducersLeaveSums(CustomTestCase):
+ def test_stage_models_do_not_sum_stage_outputs_in_compute(self):
+ (models,) = map(Path, sglang.srt.models.__path__)
+ checked, offenders = 0, {}
+ for path in sorted(models.rglob("*.py")):
+ source = path.read_text()
+ if not any(marker in source for marker in _BUILDS_STAGES):
+ continue
+ checked += 1
+ if sums := _unguarded_sums(source):
+ offenders[str(path.relative_to(models))] = sums
+ self.assertGreater(checked, 30)
+ self.assertEqual(offenders, {})
+
+ def test_guard_rejects_a_producer_that_sums_its_own_output(self):
+ class Producer(torch.nn.Module):
+ def __init__(self, reduce_results):
+ super().__init__()
+ self.reduce_results = reduce_results
+
+ class Layer(torch.nn.Module):
+ def __init__(self, reduce_results, *, staged=True):
+ super().__init__()
+ self.mlp = Producer(reduce_results)
+ if staged:
+ self.ffn_boundary = object.__new__(StageBoundary)
+
+ class Model(torch.nn.Module):
+ def __init__(self, *layers):
+ super().__init__()
+ self.layers = torch.nn.ModuleList(layers)
+
+ check_stage_producers(Model(Layer(False), Layer(True, staged=False)))
+ with self.assertRaisesRegex(ValueError, r"layers\.1\.mlp"):
+ check_stage_producers(Model(Layer(False), Layer(True)))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/registered/unit/layer_boundary/test_step3p5_dense_reduce_scatter.py b/test/registered/unit/layer_boundary/test_step3p5_dense_reduce_scatter.py
deleted file mode 100644
index 96c6f85685..0000000000
--- a/test/registered/unit/layer_boundary/test_step3p5_dense_reduce_scatter.py
+++ /dev/null
@@ -1,91 +0,0 @@
-"""A Step-3.5 dense layer must leave its all-reduce out when postprocess
-reduce-scatters its output; otherwise the output is summed twice. The layer's
-FFN exit publishes the decision while the MLP runs, then completes it."""
-
-import importlib
-import unittest
-from types import SimpleNamespace
-from unittest.mock import MagicMock
-
-import torch
-
-from sglang.srt.layers import layer_boundary as comm
-from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
-from sglang.srt.runtime_context import get_forward
-from sglang.test.boundary_fixtures import stub_plan
-from sglang.test.ci.ci_register import register_cpu_ci
-from sglang.test.test_utils import CustomTestCase
-
-register_cpu_ci(est_time=10, suite="base-a-test-cpu")
-
-
-def dense_layer(*, reduce_scatter):
- step3p5 = importlib.import_module("sglang.srt.models.step3p5")
- layer = step3p5.Step3p5DecoderLayer.__new__(step3p5.Step3p5DecoderLayer)
- torch.nn.Module.__init__(layer)
- seen = {}
-
- def mlp(hidden_states):
- # RowParallelLinear skips its all-reduce when either flag is published.
- seen["mlp_reduce_scatter"] = get_forward().mlp_reduce_scatter
- seen["fuse_mlp_allreduce"] = get_forward().fuse_mlp_allreduce
- return hidden_states
-
- complete = MagicMock(side_effect=lambda h, r: (h, r))
- plan = stub_plan()
- plan.path_for = lambda fb: SimpleNamespace(
- output=comm.OutputContract(comm.Layout(frozenset())),
- returns_over_dp=False,
- output_move_completes_sum=False,
- )
- plan.output._decide = lambda fb, steps: comm.ExitDecision(
- defer_moe_finalize=False,
- fuse_mlp_allreduce=False,
- mlp_reduce_scatter=reduce_scatter,
- complete=complete,
- )
-
- def stage_exit(fb):
- return plan.output.ffn_exit(fb, stream=fb.residual_stream)
-
- object.__setattr__(
- layer,
- "attn_boundary",
- SimpleNamespace(
- prepare=lambda h, fb, **kwargs: h,
- finish=lambda h, fb: h,
- ),
- )
- object.__setattr__(
- layer,
- "ffn_boundary",
- SimpleNamespace(
- prepare=lambda h, fb: h,
- exit=stage_exit,
- ),
- )
- object.__setattr__(layer, "use_moe", False)
- object.__setattr__(layer, "mlp", mlp)
- object.__setattr__(layer, "self_attn", lambda **kwargs: kwargs["hidden_states"])
- return layer, seen, complete
-
-
-class TestStep3p5DenseReduceScatter(CustomTestCase):
- def test_dense_mlp_sees_the_postprocess_reduce_scatter(self):
- for reduce_scatter in (False, True):
- with self.subTest(reduce_scatter=reduce_scatter):
- layer, seen, complete = dense_layer(reduce_scatter=reduce_scatter)
- layer.forward(
- positions=None,
- hidden_states=torch.ones(2, 4),
- forward_batch=SimpleNamespace(
- residual_stream=ResidualStream(torch.ones(2, 4))
- ),
- )
- self.assertEqual(seen["mlp_reduce_scatter"], reduce_scatter)
- self.assertFalse(seen["fuse_mlp_allreduce"])
- complete.assert_called_once()
-
-
-if __name__ == "__main__":
- unittest.main()