[Refactor] Let stage boundaries complete every stage-output sum (#42301)

This commit is contained in:
Cheng Wan
2026-10-03 02:17:48 -07:00
committed by GitHub
parent d5a8d76bf5
commit b157d548bb
63 changed files with 617 additions and 524 deletions
@@ -19,7 +19,7 @@ from typing import TYPE_CHECKING, Tuple
import torch
from sglang.srt.layers.layer_boundary.ops import move_rows
from sglang.srt.layers.layer_boundary.ops import move_rows, sum_output
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
@@ -53,9 +53,14 @@ def branch_input(
def branch_output(
plan: StagePlan, hidden_states: torch.Tensor, forward_batch: ForwardBatch
) -> torch.Tensor:
"""This layer's complete FFN output as a branch's contribution, which
adds to the layer's output without writing the residual: moved to the
rows the layer hands on."""
"""This layer's FFN output as a branch's contribution, which adds to the
layer's output without writing the residual: the sum it owes completed,
then moved to the rows the layer hands on."""
group = plan.path_for(forward_batch).output.group
if group is not None:
hidden_states = sum_output(
hidden_states, group, forward_batch, may_quantize=False
)
rows, _, to = plan.branch_rows(forward_batch)
return move_rows(hidden_states, rows, to, forward_batch)
+49 -42
View File
@@ -39,6 +39,7 @@ from sglang.srt.layers.layer_boundary.ops import (
all_reduce_to_dp_local,
dp_reduce_scatter,
dp_reduce_scatterv,
sum_output,
to_dp_local,
)
from sglang.srt.layers.layer_boundary.output import (
@@ -94,13 +95,14 @@ class ExitPolicy:
def __init__(self, plan):
self.plan = plan
def finish_complete_output(
def complete_now(
self,
hidden_states: torch.Tensor,
stream: ResidualStream,
forward_batch: ForwardBatch,
):
"""Move a complete output from the operation-scheduled producer path."""
"""Complete the output of the operation-scheduled producer path, which
never defers: the sum it owes, then its move and write-back."""
steps = self.plan.path_for(forward_batch)
hidden_states, residual = self._complete_now(
hidden_states,
@@ -108,11 +110,9 @@ class ExitPolicy:
forward_batch=forward_batch,
dp_step=None,
steps=steps,
output_move=(
steps.complete_output_move
if steps.output_move_completes_sum
else steps.output_move
),
output_move=steps.output_move,
owes_sum=steps.output.group is not None
and not steps.output_move_completes_sum,
)
return self._record_output(
hidden_states,
@@ -188,6 +188,8 @@ class ExitPolicy:
dp_step=dp_step,
steps=steps,
output_move=steps.output_move,
# A reduce-scatter completes the sum on its way back.
owes_sum=steps.output.group is not None and not mlp_reduce_scatter,
)
defer_moe_finalize = (
self.plan.fusions is not None
@@ -251,10 +253,19 @@ class ExitPolicy:
dp_step: Optional[Callable],
steps,
output_move: Optional[Callable],
owes_sum: bool,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""This layer's postprocess, run with the attention-DP step already
chosen: the move back to where the next layer reads the FFN output, then
the write-back into the residual for a layer that does it itself."""
chosen: the sum the output owes, the move back to where the next layer
reads it, then the write-back into the residual for a layer that does
it itself."""
if owes_sum:
hidden_states = sum_output(
hidden_states,
steps.output.group,
forward_batch,
may_quantize=steps.output.update.is_plain_add,
)
if steps.output.transform is not None:
hidden_states = steps.output.transform.apply(hidden_states)
if steps.returns_over_dp:
@@ -289,7 +300,7 @@ class ExitPolicy:
return FfnExit(self, forward_batch, stream=stream)
def _sum_deferral_allowed(self, steps) -> bool:
# Under the MoE-CP all-gather the fusion path would skip finish_complete_output
# Under the MoE-CP all-gather the fusion path would skip complete_now
# and its MoE-CP scatter, leaving hidden_states longer than the residual.
if is_enable_moe_cp_allgather() or steps.output.group is None:
return False
@@ -377,17 +388,17 @@ def _batch_allows_deferred_sum(forward_batch: ForwardBatch, boundary=None) -> bo
class ExitDecision(msgspec.Struct, frozen=True):
"""One decision shared by compute flags and the matching output completion.
"""One decision for an FFN output, made before compute runs.
Fields:
defer_moe_finalize: Compute may return a producer-specific finalize handoff.
fuse_mlp_allreduce: Compute skips its all-reduce; completion runs it later
or carries it to the next consumer, whether fused there or unfused.
mlp_reduce_scatter: Compute leaves reduction to the selected scatter path.
fuse_mlp_allreduce: The sum is carried to the next consumer, which
completes it, fused or unfused.
mlp_reduce_scatter: A reduce-scatter on the way back completes the sum.
complete: Callable(output, residual) returning the completed or wrapped
output and its corresponding residual rows.
Do not independently reselect completion after compute has used these flags.
Do not independently reselect completion after compute has run.
"""
defer_moe_finalize: bool
@@ -405,22 +416,21 @@ def _defer(
class MixerExit:
"""The scope that publishes a mixer's decision while it runs: inside the
``with`` block ``fuse_mlp_allreduce`` on ``get_forward()`` tells its
row-parallel output projection to skip the all-reduce. It skips when the
stage's output always leaves its sum (to an FFN stage, which completes it in
its input), or when its declaration permits deferring the sum to the next
attention stage and the batch allows it: TP > 1, no attention DP, and no
input-scattered or MoE-CP all-gather layout. The consumer decides whether
to fuse its completion with the input norm."""
"""A mixer's (an attention-like stage's) output decision. The mixer leaves
its output's sum; the stage either carries it to the next attention stage
(when the declaration permits deferring and the batch allows it: TP > 1,
no attention DP, and no input-scattered or MoE-CP all-gather layout),
hands it on as the declared sum an FFN stage completes in its input, or
completes it here. The consumer decides whether to fuse its completion
with the input norm."""
__slots__ = (
"skips_reduction",
"_defers",
"_scope",
"_group",
"_update",
"_declared_sum",
"_stream",
"_forward_batch",
)
def __init__(
@@ -433,7 +443,9 @@ class MixerExit:
steps = boundary.plan.path_for(forward_batch)
produced = steps.output
self._stream = stream
self._forward_batch = forward_batch
self._update = produced.update
self._group = produced.group
self._declared_sum = produced.group if produced.always_partial else None
self._defers = (
produced.may_defer_to_next
@@ -441,15 +453,12 @@ class MixerExit:
and not is_dp_attention_enabled()
and boundary._sum_deferral_allowed(steps)
)
self.skips_reduction = produced.always_partial or self._defers
self._scope = get_forward().scoped(fuse_mlp_allreduce=self.skips_reduction)
def __enter__(self) -> MixerExit:
self._scope.__enter__()
return self
def __exit__(self, *exc_info):
return self._scope.__exit__(*exc_info)
return None
def finish(self, hidden_states: torch.Tensor):
"""Hand the actual mixer result to the next boundary through the stream."""
@@ -457,6 +466,13 @@ class MixerExit:
hidden_states = UnreducedOutput(
hidden_states, group=get_parallel().tp_group
)
elif self._declared_sum is None and self._group is not None:
hidden_states = sum_output(
hidden_states,
self._group,
self._forward_batch,
may_quantize=self._update.is_plain_add,
)
return self._stream.record(
hidden_states, self._update, declared_sum=self._declared_sum
)
@@ -473,18 +489,15 @@ class FfnExit:
Fields:
boundary: The owning output boundary.
defer_moe_finalize: Whether compute may return a finalize handoff.
fuse_mlp_allreduce: Whether compute leaves its all-reduce to completion.
mlp_reduce_scatter: Whether compute leaves reduction to a scatter path.
The three flags are published on get_forward() only inside the context.
finish(output) uses the same selected action after successful compute.
The FFN leaves its output's sum; finish(output) completes it or carries
it on with the action selected before compute. ``defer_moe_finalize`` is
published on get_forward() only inside the context.
"""
__slots__ = (
"boundary",
"defer_moe_finalize",
"fuse_mlp_allreduce",
"mlp_reduce_scatter",
"_complete",
"_update",
"_declared_sum",
@@ -504,18 +517,12 @@ class FfnExit:
steps = boundary.plan.path_for(forward_batch)
completion = boundary._decide(forward_batch, steps)
self.defer_moe_finalize = completion.defer_moe_finalize
self.fuse_mlp_allreduce = completion.fuse_mlp_allreduce
self.mlp_reduce_scatter = completion.mlp_reduce_scatter
self._complete = completion.complete
self._update = steps.output.update
self._declared_sum = boundary._sum_owed_after_skip(
steps, completion.mlp_reduce_scatter
)
self._scope = get_forward().scoped(
fuse_mlp_allreduce=self.fuse_mlp_allreduce,
mlp_reduce_scatter=self.mlp_reduce_scatter,
defer_moe_finalize=self.defer_moe_finalize,
)
self._scope = get_forward().scoped(defer_moe_finalize=self.defer_moe_finalize)
def __enter__(self) -> FfnExit:
self._scope.__enter__()
@@ -22,6 +22,8 @@ from sglang.srt.distributed import (
GroupCoordinator,
attention_tensor_model_parallel_all_reduce,
attention_tensor_model_parallel_quant_all_reduce,
tensor_model_parallel_all_reduce,
tensor_model_parallel_quant_all_reduce,
)
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
use_symmetric_memory,
@@ -47,12 +49,14 @@ from sglang.srt.layers.layer_boundary.adapters.attention import (
)
from sglang.srt.layers.layer_boundary.layout import (
Layout,
SumGroup,
TokenAxis,
_cp_shard_token_rows,
moe_cp_gathered_rows,
)
from sglang.srt.layers.layer_boundary.residual import ResidualUpdate
from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_ADD
from sglang.srt.layers.moe.utils import sum_post_experts_output
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.runtime_context import get_exec, get_parallel
@@ -173,6 +177,34 @@ def attn_tp_all_reduce(
return attention_tensor_model_parallel_all_reduce(hidden_states)
def sum_output(
hidden_states: torch.Tensor,
group: SumGroup,
forward_batch: ForwardBatch,
*,
may_quantize: bool,
) -> torch.Tensor:
"""Complete the sum an output owes over ``group`` on the rows it is on."""
parallel = get_parallel()
if group is SumGroup.ATTN_TP:
if parallel.attn_tp_size == 1:
return hidden_states
return attn_tp_all_reduce(hidden_states, forward_batch, may_quantize)
if group is SumGroup.TP:
if parallel.tp_size == 1:
return hidden_states
if (
may_quantize
and not forward_batch.forward_mode.is_decode_or_idle()
and get_exec().comm.enable_quant_communications
):
return tensor_model_parallel_quant_all_reduce(hidden_states)
return tensor_model_parallel_all_reduce(hidden_states)
if group is SumGroup.MOE_OUTPUT:
return sum_post_experts_output(hidden_states)
raise ValueError(f"unsupported output sum group: {group!r}")
def dp_gather(
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
@@ -187,7 +187,8 @@ class StageBoundary:
forward_batch: Active batch whose stream receives the compute result.
Returns:
A context manager publishing reduction/finalize flags during compute.
A context manager publishing the producer's finalize choice during
compute.
Call its finish(output) exactly once after successful compute, including
when compute is skipped for an empty input. It completes or carries work
using the same decision rather than selecting a second path.
@@ -253,11 +254,11 @@ class StageBoundary:
hidden_states = stream.complete(hidden_states)
return hidden_states, stream.snapshot(hidden_states)
def finish_complete_output(self, hidden_states, forward_batch):
"""Move an FFN output that compute already completed outside exit()
(the operation-scheduled TBO path) onto the rows the layer hands on;
it chooses no reduction step."""
return self.plan.output.finish_complete_output(
def complete_now(self, hidden_states, forward_batch):
"""Complete an FFN output outside exit() (the operation-scheduled TBO
path, which never defers): the sum it owes, then the move onto the
rows the layer hands on."""
return self.plan.output.complete_now(
hidden_states, stream_of(forward_batch), forward_batch
)
@@ -379,3 +380,26 @@ class StageBoundary:
),
)
return hidden_states, stream
def check_stage_producers(model: torch.nn.Module) -> None:
"""Reject a model whose stage producers complete their own output sums.
In a layer built from stage boundaries, the boundary completes every sum a
stage output owes. A submodule there constructed with
``reduce_results=True`` (a row-parallel projection, a MoE block) would
have its output summed twice. Called once, after the model is built.
"""
offenders = [
f"{name}.{child_name}"
for name, layer in model.named_modules()
if any(isinstance(v, StageBoundary) for v in vars(layer).values())
for child_name, child in layer.named_modules()
if child_name and getattr(child, "reduce_results", False) is True
]
if offenders:
raise ValueError(
"stage boundaries complete their producers' output sums, but these "
f"submodules sum their own output: {', '.join(offenders[:8])}"
+ (f" (and {len(offenders) - 8} more)" if len(offenders) > 8 else "")
)
+38 -8
View File
@@ -797,6 +797,18 @@ def should_add_replicated_moe_output() -> bool:
return not (parallel.tp_size > 1 and summed_later and parallel.tp_rank != 0)
def adds_replicated_output_to_partial() -> bool:
"""For a MoE block whose stage boundary completes its sum: whether this rank
adds an output every TP rank holds in full, such as a shared expert
replicated with tp_size=1, to its MoE output. While the output still owes a
TP sum, only TP rank 0 adds it, so the sum counts it once."""
parallel = get_parallel()
owes_sum = parallel.tp_size > 1 and not post_experts_output_is_complete(
is_tp_path=True
)
return not owes_sum or parallel.tp_rank == 0
def can_merge_post_experts_all_reduce() -> bool:
"""Whether the EP and MoE-TP reductions can collapse into one _TP all-reduce.
@@ -818,20 +830,38 @@ def post_experts_all_reduce(hidden_states: torch.Tensor) -> torch.Tensor:
sequential ones, which also restores the invariant the fused residual+LN path
depends on.
"""
parallel = get_parallel()
return _post_experts_sum(
hidden_states,
reduce_ep=parallel.moe_ep_size > 1
and not should_skip_post_experts_all_reduce(is_tp_path=False),
reduce_tp=parallel.moe_tp_size > 1
and not should_skip_post_experts_all_reduce(is_tp_path=True),
)
def sum_post_experts_output(hidden_states: torch.Tensor) -> torch.Tensor:
"""Complete the sum a MoE output owes over the EP and MoE-TP groups, for the
boundary that owns it; a path the combine already summed is left alone."""
parallel = get_parallel()
return _post_experts_sum(
hidden_states,
reduce_ep=parallel.moe_ep_size > 1
and not post_experts_output_is_complete(is_tp_path=False),
reduce_tp=parallel.moe_tp_size > 1
and not post_experts_output_is_complete(is_tp_path=True),
)
def _post_experts_sum(
hidden_states: torch.Tensor, *, reduce_ep: bool, reduce_tp: bool
) -> torch.Tensor:
from sglang.srt.distributed.communication_op import (
moe_expert_parallel_all_reduce,
moe_tensor_model_parallel_all_reduce,
tensor_model_parallel_all_reduce,
)
parallel = get_parallel()
reduce_ep = parallel.moe_ep_size > 1 and not should_skip_post_experts_all_reduce(
is_tp_path=False
)
reduce_tp = parallel.moe_tp_size > 1 and not should_skip_post_experts_all_reduce(
is_tp_path=True
)
if reduce_ep and reduce_tp and can_merge_post_experts_all_reduce():
return tensor_model_parallel_all_reduce(hidden_states)
+4 -1
View File
@@ -83,6 +83,7 @@ from sglang.srt.connector.utils import parse_model_name
from sglang.srt.distributed import (
model_parallel_is_initialized,
)
from sglang.srt.layers.layer_boundary.stage import check_stage_producers
from sglang.srt.layers.modelopt_utils import QUANT_CFG_CHOICES
from sglang.srt.layers.moe.utils import (
install_shared_experts_fusion_decision,
@@ -304,7 +305,9 @@ def _initialize_model(
if load_config.draft_model_idx is not None:
kwargs["draft_model_idx"] = load_config.draft_model_idx
return model_class(**kwargs)
model = model_class(**kwargs)
check_stage_producers(model)
return model
def post_load_weights(model: nn.Module) -> None:
+4 -2
View File
@@ -259,6 +259,7 @@ class ApertusDecoderLayer(nn.Module):
quant_config=quant_config,
bias=getattr(config, "mlp_bias", False),
prefix=add_prefix("mlp", prefix),
reduce_results=False,
)
self.attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.feedforward_layernorm = RMSNorm(
@@ -294,8 +295,9 @@ class ApertusDecoderLayer(nn.Module):
# Fully Connected
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
hidden_states = self.mlp(hidden_states)
return self.ffn_boundary.finish_complete_output(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
return ffn_exit.finish(hidden_states)
class ApertusModel(nn.Module):
+4 -2
View File
@@ -235,6 +235,7 @@ class ArceeDecoderLayer(nn.Module):
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNorm(
@@ -270,8 +271,9 @@ class ArceeDecoderLayer(nn.Module):
# Fully Connected
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
hidden_states = self.mlp(hidden_states)
return self.ffn_boundary.finish_complete_output(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
return ffn_exit.finish(hidden_states)
class ArceeModel(nn.Module):
+1 -2
View File
@@ -55,7 +55,6 @@ from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_deepep_mode,
get_moe_a2a_backend,
reduce_moe_output,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
@@ -460,7 +459,6 @@ class BailingMoESparseMoeBlock(nn.Module):
if self.num_shared_experts > 0:
final_hidden_states = final_hidden_states + shared_output
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_size)
def forward_deepep(
@@ -693,6 +691,7 @@ class BailingMoEBlock(nn.Module):
prefix=add_prefix("mlp", prefix),
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
reduce_results=False,
)
self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps)
@@ -32,7 +32,6 @@ from sglang.srt.layers.linear import (
RowParallelLinear,
)
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import reduce_moe_output
from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.moe.topk import TopK
@@ -365,7 +364,6 @@ class BailingMoE(nn.Module):
if self.num_shared_experts > 0:
final_hidden_states = final_hidden_states + shared_output
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states
@@ -671,6 +669,7 @@ class BailingMoEAttention(nn.Module):
bias=config.use_bias,
quant_config=quant_config,
prefix=f"{prefix}.o_proj",
reduce_results=False,
)
if hasattr(config, "rotary_dim"):
self.rotary_dim = config.rotary_dim
@@ -792,6 +791,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
intermediate_size=config.intermediate_size,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
reduce_results=False,
)
else:
if is_nextn or self.layer_id >= config.first_k_dense_replace:
@@ -810,6 +810,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
intermediate_size=config.intermediate_size,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
reduce_results=False,
)
rms_norm_eps = float(getattr(config, "rms_norm_eps", 1e-5))
self.input_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
+3 -17
View File
@@ -19,10 +19,6 @@ from sglang.kernels.ops.quantization.fp8_kernel import (
)
from sglang.srt.configs import KimiLinearConfig
from sglang.srt.configs.bailing_hybrid import is_bailing_multi_gate_enabled
from sglang.srt.distributed import (
moe_expert_parallel_all_reduce,
moe_tensor_model_parallel_all_reduce,
)
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
@@ -47,7 +43,6 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
should_skip_post_experts_all_reduce,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
@@ -751,18 +746,6 @@ class BailingMoE(nn.Module):
if shared_output is not None:
final_hidden_states = final_hidden_states + shared_output
if self.moe_ep_size > 1 and not should_skip_post_experts_all_reduce(
is_tp_path=False,
):
final_hidden_states = moe_expert_parallel_all_reduce(final_hidden_states)
if self.moe_tp_size > 1 and not should_skip_post_experts_all_reduce(
is_tp_path=True,
):
final_hidden_states = moe_tensor_model_parallel_all_reduce(
final_hidden_states
)
return final_hidden_states
def _forward_shared_experts(
@@ -978,6 +961,7 @@ class BailingMoEAttention(nn.Module):
bias=config.use_bias,
quant_config=quant_config,
prefix=f"{prefix}.o_proj",
reduce_results=False,
)
if hasattr(config, "rotary_dim"):
self.rotary_dim = config.rotary_dim
@@ -1121,6 +1105,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
prefix=prefix,
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
reduce_results=False,
)
else:
if is_nextn or self.layer_id >= config.first_k_dense_replace:
@@ -1141,6 +1126,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
prefix=prefix,
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
reduce_results=False,
)
rms_norm_eps = float(getattr(config, "rms_norm_eps", 1e-5))
self.input_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
+48 -25
View File
@@ -111,6 +111,7 @@ from sglang.srt.layers.moe.token_dispatcher.base import (
from sglang.srt.layers.moe.topk import BypassedTopKOutput, TopK, TopKOutputFormat
from sglang.srt.layers.moe.utils import (
RoutingMethodType,
adds_replicated_output_to_partial,
filter_moe_weight_param_global_expert,
has_per_rank_fused_shared_slots,
is_deepep_class_backend,
@@ -118,6 +119,7 @@ from sglang.srt.layers.moe.utils import (
is_shared_experts_fusion_disabled,
is_tbo_enabled,
should_add_replicated_moe_output,
sum_post_experts_output,
)
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.quantization.fp8 import Fp8Config
@@ -275,10 +277,15 @@ class DeepseekV2MLP(nn.Module):
tp_rank: Optional[int] = None,
tp_size: Optional[int] = None,
swiglu_limit: Optional[float] = None,
allow_fused_down: Optional[bool] = None,
) -> None:
super().__init__()
self.tp_size = tp_size
self.swiglu_limit = swiglu_limit
# The fused down path calls deepgemm directly and runs no all-reduce.
self.allow_fused_down = (
not reduce_results if allow_fused_down is None else allow_fused_down
)
self.gate_up_proj = MergedColumnParallelLinear(
hidden_size,
@@ -375,7 +382,7 @@ class DeepseekV2MLP(nn.Module):
# are fp8 (uint8 storage with weight_scale_inv).
if (
self.swiglu_limit is not None
and not self.down_proj.reduce_results
and self.allow_fused_down
and getattr(self.down_proj, "weight", None) is not None
and self.down_proj.weight.dtype == torch.uint8
and hasattr(self.down_proj, "weight_scale_inv")
@@ -592,8 +599,11 @@ class DeepseekV2MoE(nn.Module):
is_nextn: bool = False,
is_deepseek_v4: bool = False,
vl_correction_bias: bool = False,
reduce_results: bool = True,
):
super().__init__()
# False when a stage boundary completes this output's sum.
self.reduce_results = reduce_results
self.tp_size = get_parallel().tp_size
self.moe_ep_size = get_parallel().moe_ep_size
self.routed_scaling_factor = config.routed_scaling_factor
@@ -1160,7 +1170,7 @@ class DeepseekV2MoE(nn.Module):
final_hidden_states,
gated_shared_output=shared_output,
m=hidden_states.shape[0],
reduce=post_experts_all_reduce,
reduce=sum_post_experts_output,
)
all_reduce_done = False
@@ -1256,7 +1266,7 @@ class DeepseekV2MoE(nn.Module):
self.routed_scaling_factor,
)
if not all_reduce_done:
if not all_reduce_done and self.reduce_results:
if (
self.is_deepseek_v4
and self.tp_size > 1
@@ -1274,9 +1284,9 @@ class DeepseekV2MoE(nn.Module):
if mhc is not None:
mhc.start_stats_before_all_reduce()
final_hidden_states = post_experts_all_reduce(final_hidden_states)
# TP1 shared experts are replicated, so add them after all-reduce to
# avoid summing the same shared output once per TP rank.
if self._shared_expert_tp1 and should_add_replicated_moe_output():
# TP1 shared experts are replicated: add them after this block's
# all-reduce, or on TP rank 0 only when the stage boundary sums.
if self._shared_expert_tp1 and self._adds_replicated_output():
final_hidden_states += shared_output
return final_hidden_states
@@ -1431,31 +1441,40 @@ class DeepseekV2MoE(nn.Module):
self.routed_scaling_factor,
)
if (
self.is_deepseek_v4
and self.tp_size > 1
and not should_skip_post_experts_all_reduce(is_tp_path=True)
):
from sglang.srt.layers.moe.mhc_post_fusion import current_mhc_post_fusion
mhc = current_mhc_post_fusion()
if _is_hip and _hip_moe.fused_all_reduce_mhc(
self, mhc, final_hidden_states
if self.reduce_results:
if (
self.is_deepseek_v4
and self.tp_size > 1
and not should_skip_post_experts_all_reduce(is_tp_path=True)
):
return final_hidden_states
if mhc is not None:
mhc.start_stats_before_all_reduce()
final_hidden_states = post_experts_all_reduce(final_hidden_states)
# TP1 shared experts are replicated, so add them after all-reduce to
# avoid summing the same shared output once per TP rank.
from sglang.srt.layers.moe.mhc_post_fusion import (
current_mhc_post_fusion,
)
mhc = current_mhc_post_fusion()
if _is_hip and _hip_moe.fused_all_reduce_mhc(
self, mhc, final_hidden_states
):
return final_hidden_states
if mhc is not None:
mhc.start_stats_before_all_reduce()
final_hidden_states = post_experts_all_reduce(final_hidden_states)
# TP1 shared experts are replicated: add them after this block's
# all-reduce, or on TP rank 0 only when the stage boundary sums.
if (
shared_output is not None
and self._shared_expert_tp1
and should_add_replicated_moe_output()
and self._adds_replicated_output()
):
final_hidden_states += shared_output
return final_hidden_states
def _adds_replicated_output(self) -> bool:
"""Whether this rank adds the replicated TP1 shared-expert output."""
if self.reduce_results:
return should_add_replicated_moe_output()
return adds_replicated_output_to_partial()
def forward_cpu(
self,
hidden_states: torch.Tensor,
@@ -1507,7 +1526,8 @@ class DeepseekV2MoE(nn.Module):
), # block_size
True, # is_vnni
)
final_hidden_states = post_experts_all_reduce(final_hidden_states)
if self.reduce_results:
final_hidden_states = post_experts_all_reduce(final_hidden_states)
return final_hidden_states
def forward_deepep(
@@ -2661,6 +2681,7 @@ class DeepseekV2DecoderLayer(nn.Module):
layer_id=self.layer_id,
alt_stream=alt_stream,
is_nextn=is_nextn,
reduce_results=False,
)
else:
if is_dense_ffn_fully_dp():
@@ -2676,6 +2697,8 @@ class DeepseekV2DecoderLayer(nn.Module):
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
swiglu_limit=getattr(config, "swiglu_limit", None),
reduce_results=False,
allow_fused_down=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -2876,7 +2899,7 @@ class DeepseekV2DecoderLayer(nn.Module):
)
def op_comm_postprocess_layer(self, state):
hidden_states = self.ffn_boundary.finish_complete_output(
hidden_states = self.ffn_boundary.complete_now(
state.pop("hidden_states_mlp_output"), state.forward_batch
)
@@ -45,9 +45,6 @@ from sglang.srt.batch_overlap.two_batch_overlap import (
model_forward_stages,
)
from sglang.srt.configs.dots3 import Dots3Config
from sglang.srt.distributed import (
tensor_model_parallel_all_reduce,
)
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
use_symmetric_memory,
)
@@ -228,21 +225,13 @@ class Dots3MLP(nn.Module):
)
self.act_fn = SiluAndMul()
def forward(
self,
x,
forward_batch=None,
should_allreduce_fusion: bool = False,
use_reduce_scatter: bool = False,
):
def forward(self, x, forward_batch=None):
if (self.tp_size == 1) and x.shape[0] == 0:
return x
gate_up, _ = self.gate_up_proj(x)
x = self.act_fn(gate_up)
x, _ = self.down_proj(
x, skip_all_reduce=should_allreduce_fusion or use_reduce_scatter
)
x, _ = self.down_proj(x)
return x
@@ -416,8 +405,6 @@ class Dots3MoE(nn.Module):
self,
hidden_states: torch.Tensor,
forward_batch: Optional[ForwardBatch] = None,
should_allreduce_fusion: bool = False,
use_reduce_scatter: bool = False,
) -> torch.Tensor:
if not self._enable_deepep_moe:
DUAL_STREAM_TOKEN_THRESHOLD = 1024
@@ -427,26 +414,13 @@ class Dots3MoE(nn.Module):
and hidden_states.shape[0] > 0
and hidden_states.shape[0] <= DUAL_STREAM_TOKEN_THRESHOLD
):
return self.forward_normal_dual_stream(
hidden_states,
should_allreduce_fusion,
use_reduce_scatter,
)
return self.forward_normal_dual_stream(hidden_states)
else:
return self.forward_normal(
hidden_states,
should_allreduce_fusion,
use_reduce_scatter,
)
return self.forward_normal(hidden_states)
else:
return self.forward_deepep(hidden_states, forward_batch)
def forward_normal_dual_stream(
self,
hidden_states: torch.Tensor,
should_allreduce_fusion: bool = False,
use_reduce_scatter: bool = False,
) -> torch.Tensor:
def forward_normal_dual_stream(self, hidden_states: torch.Tensor) -> torch.Tensor:
current_stream = torch.cuda.current_stream()
self.alt_stream.wait_stream(current_stream)
@@ -465,21 +439,9 @@ class Dots3MoE(nn.Module):
torch.add(final_hidden_states, shared_output, out=final_hidden_states_out)
final_hidden_states = final_hidden_states_out
sm.tag(final_hidden_states)
if (
self.tp_size > 1
and not should_allreduce_fusion
and not use_reduce_scatter
and not should_use_flashinfer_cutlass_moe_fp4_allgather()
):
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
return final_hidden_states
def forward_normal(
self,
hidden_states: torch.Tensor,
should_allreduce_fusion: bool = False,
use_reduce_scatter: bool = False,
) -> torch.Tensor:
def forward_normal(self, hidden_states: torch.Tensor) -> torch.Tensor:
if hidden_states.shape[0] > 0:
shared_output = self._forward_shared_experts(hidden_states)
# router_logits: (num_tokens, n_experts)
@@ -496,13 +458,6 @@ class Dots3MoE(nn.Module):
torch.add(final_hidden_states, shared_output, out=final_hidden_states_out)
final_hidden_states = final_hidden_states_out
sm.tag(final_hidden_states)
if (
self.tp_size > 1
and not should_allreduce_fusion
and not use_reduce_scatter
and not should_use_flashinfer_cutlass_moe_fp4_allgather()
):
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
return final_hidden_states
def forward_deepep(
@@ -1454,6 +1409,7 @@ class Dots3DecoderLayer(nn.Module):
prefix=add_prefix("mlp", prefix),
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -1509,12 +1465,7 @@ class Dots3DecoderLayer(nn.Module):
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(
hidden_states,
forward_batch,
ffn_exit.fuse_mlp_allreduce,
ffn_exit.mlp_reduce_scatter,
)
hidden_states = self.mlp(hidden_states, forward_batch)
return ffn_exit.finish(hidden_states)
+4 -2
View File
@@ -74,6 +74,7 @@ class Exaone4GatedMLP(nn.Module):
hidden_size,
bias=bias,
quant_config=quant_config,
reduce_results=False,
prefix=add_prefix("down_proj", prefix),
)
if hidden_act != "silu":
@@ -312,8 +313,9 @@ class Exaone4DecoderLayer(nn.Module):
# Fully Connected
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
hidden_states = self.mlp(hidden_states)
return self.ffn_boundary.finish_complete_output(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
return ffn_exit.finish(hidden_states)
class Exaone4Model(nn.Module):
+1
View File
@@ -214,6 +214,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
mlp_multipliers=config.mlp_multipliers,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+1 -3
View File
@@ -59,7 +59,6 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
reduce_moe_output,
should_use_flashinfer_cutlass_moe_fp4_allgather,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
@@ -607,7 +606,6 @@ class Glm4MoeSparseMoeBlock(nn.Module):
current_stream.wait_stream(self.alt_stream)
final_hidden_states += shared_output
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states
def forward_normal(
@@ -633,7 +631,6 @@ class Glm4MoeSparseMoeBlock(nn.Module):
final_hidden_states_out = torch.empty_like(final_hidden_states)
torch.add(final_hidden_states, shared_output, out=final_hidden_states_out)
final_hidden_states = final_hidden_states_out
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states
def forward_deepep(
@@ -768,6 +765,7 @@ class Glm4MoeDecoderLayer(nn.Module):
prefix=add_prefix("mlp", prefix),
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+1 -3
View File
@@ -50,7 +50,6 @@ from sglang.srt.layers.linear import MergedColumnParallelLinear, RowParallelLine
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
reduce_moe_output,
should_use_flashinfer_cutlass_moe_fp4_allgather,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
@@ -338,7 +337,6 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module):
current_stream.wait_stream(self.alt_stream)
final_hidden_states += shared_output
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states
def forward_normal(
@@ -364,7 +362,6 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module):
final_hidden_states_out = torch.empty_like(final_hidden_states)
torch.add(final_hidden_states, shared_output, out=final_hidden_states_out)
final_hidden_states = final_hidden_states_out
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states
def forward_deepep(
@@ -476,6 +473,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
prefix=add_prefix("mlp", prefix),
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+3
View File
@@ -734,6 +734,7 @@ class Glm5NextDecoderLayer(nn.Module):
layer_id=self.layer_id,
alt_stream=alt_stream,
is_nextn=is_nextn,
reduce_results=False,
)
else:
if is_dense_ffn_fully_dp():
@@ -749,6 +750,8 @@ class Glm5NextDecoderLayer(nn.Module):
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
swiglu_limit=config.swiglu_limit,
reduce_results=False,
allow_fused_down=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
-3
View File
@@ -48,7 +48,6 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
reduce_moe_output,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
@@ -335,8 +334,6 @@ class GptOssSparseMoeBlock(nn.Module):
topk_output = self.topk(router_input, router_logits)
final_hidden_states = self.experts(hidden_states, topk_output)
final_hidden_states = reduce_moe_output(final_hidden_states)
# When input was pre-padded, FusedMoE.forward_impl captured the
# padded width as `origin_hidden_states_dim` and skipped its own
# output-trim contiguous() — so the experts output is still
+1 -4
View File
@@ -31,9 +31,6 @@ from sglang.srt.layers.linear import (
ReplicatedLinear,
RowParallelLinear,
)
from sglang.srt.layers.moe import (
reduce_moe_output,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.topk import TopK
from sglang.srt.layers.moe.utils import (
@@ -422,7 +419,7 @@ class _InternS2MobiusDecoderMixin:
routed = _get_mobius_routed_bank(meta_mlp, self.layer_id).forward_routed(
hidden_states, forward_batch
)
return reduce_moe_output(routed + shared)
return routed + shared
def _forward_after_attention(
self,
+4 -6
View File
@@ -12,7 +12,6 @@ from sglang.kernels.ops.attention.fla.fused_norm_gate import FusedRMSNormGated
from sglang.srt.configs.kimi_linear import KimiLinearConfig
from sglang.srt.distributed import (
divide,
tensor_model_parallel_all_reduce,
)
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers.dcp.planner import prepare_decode_context_parallel_metadata
@@ -185,9 +184,6 @@ class KimiMoE(nn.Module):
if shared_output is not None:
final_hidden_states = final_hidden_states + shared_output
if self.tp_size > 1:
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_size)
@@ -613,6 +609,7 @@ class KimiDecoderLayer(nn.Module):
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=f"{prefix}.mlp",
reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNorm(
@@ -655,8 +652,9 @@ class KimiDecoderLayer(nn.Module):
# Fully Connected
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
hidden_states = self.mlp(hidden_states)
return self.ffn_boundary.finish_complete_output(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
return ffn_exit.finish(hidden_states)
class KimiLinearModel(nn.Module):
+3 -5
View File
@@ -37,11 +37,10 @@ from sglang.srt.layers.linear import (
RowParallelLinear,
)
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import reduce_moe_output
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.moe.topk import TopK
from sglang.srt.layers.moe.utils import should_add_replicated_moe_output
from sglang.srt.layers.moe.utils import adds_replicated_output_to_partial
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.layers.rotary_embedding import get_rope
@@ -226,8 +225,7 @@ class LagunaMoE(nn.Module):
else:
final = routed_out + shared_out
final = reduce_moe_output(final)
if self._shared_expert_tp1 and should_add_replicated_moe_output():
if self._shared_expert_tp1 and adds_replicated_output_to_partial():
final = final + shared_out
return final
@@ -448,7 +446,7 @@ class LagunaDecoderLayer(nn.Module):
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
reduce_results=True,
reduce_results=False,
prefix=add_prefix("mlp", prefix),
)
+1 -2
View File
@@ -51,7 +51,6 @@ from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_deepep_mode,
get_moe_a2a_backend,
reduce_moe_output,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
@@ -381,7 +380,6 @@ class LLaDA2MoeSparseMoeBlock(nn.Module):
if self.num_shared_experts > 0:
final_hidden_states = final_hidden_states + shared_output
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_size)
def forward_deepep(
@@ -628,6 +626,7 @@ class LLaDA2MoeBlock(nn.Module):
prefix=add_prefix("mlp", prefix),
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
reduce_results=False,
)
self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps)
+1 -3
View File
@@ -41,7 +41,6 @@ from sglang.srt.layers.linear import (
ReplicatedLinear,
RowParallelLinear,
)
from sglang.srt.layers.moe import reduce_moe_output
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
from sglang.srt.layers.moe.topk import TopK
from sglang.srt.layers.quantization.base_config import QuantizationConfig
@@ -148,8 +147,6 @@ class Llama4MoE(nn.Module):
out_aD = routed_out + shared_out
out_aD = reduce_moe_output(out_aD)
return out_aD
def _forward_core(self, hidden_states, forward_mode: ForwardMode):
@@ -418,6 +415,7 @@ class Llama4DecoderLayer(nn.Module):
hidden_act="silu",
quant_config=quant_config,
prefix=add_prefix("feed_forward", prefix),
reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNorm(
+1 -17
View File
@@ -44,9 +44,6 @@ from sglang.kernels.ops.gemm.bf16_fp32 import (
from sglang.kernels.ops.moe.ep_moe_kernels import zero_experts_compute_triton
from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
from sglang.srt.configs import LongcatFlashConfig
from sglang.srt.distributed import (
tensor_model_parallel_all_reduce,
)
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers import deep_gemm_wrapper
@@ -72,7 +69,6 @@ from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.moe.topk import StandardTopKOutput, TopK
from sglang.srt.layers.moe.utils import (
filter_moe_weight_param_global_expert,
should_skip_mlp_all_reduce,
)
from sglang.srt.layers.n_gram_embedding import NgramEmbedding
from sglang.srt.layers.quantization.base_config import QuantizationConfig
@@ -325,18 +321,6 @@ class LongcatFlashMoE(nn.Module):
if self.zero_expert_type is not None and hidden_states.shape[0] > 0:
final_hidden_states += zero_expert_result.to(final_hidden_states.device)
# LONGCAT_MOE_A2A_SKIP_ALLREDUCE: skip the post-experts TP all-reduce when a
# real EP a2a backend is active -- self.experts (DeepEPMoE) already combined
# expert outputs across EP ranks, so an extra all-reduce double-counts.
from sglang.srt.layers.moe.utils import get_moe_a2a_backend as _lc_gab
if (
self.tp_size > 1
and _lc_gab().is_none()
and not should_skip_mlp_all_reduce()
):
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_dim)
def get_moe_weights(self):
@@ -414,7 +398,7 @@ class LongcatFlashDecoderLayer(nn.Module):
if "mlps" in getattr(config, "disable_quant_module", [])
else quant_config
),
reduce_results=True,
reduce_results=False,
prefix=add_prefix(f"mlps.{i}", prefix),
)
for i in range(2)
@@ -154,7 +154,7 @@ class LongcatFlashDenseDecoderLayer(nn.Module):
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
reduce_results=True,
reduce_results=False,
prefix=add_prefix(f"mlps", prefix),
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -191,10 +191,9 @@ class LongcatFlashDenseDecoderLayer(nn.Module):
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
hidden_states = self.mlp(hidden_states)
hidden_states = self.ffn_boundary.finish_complete_output(
hidden_states, forward_batch
)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states
+1
View File
@@ -436,6 +436,7 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer):
hidden_act=cfg.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
reduce_results=False,
)
is_previous_layer_sparse = _is_sparse(layer_id - 1)
+2 -4
View File
@@ -52,7 +52,6 @@ from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
get_moe_runner_backend,
reduce_moe_output,
)
from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class
from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat
@@ -499,8 +498,6 @@ class MiMoV2MoE(nn.Module):
final_hidden_states = self.experts(hidden_states, topk_output)
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states
def forward_deepep(
@@ -852,6 +849,7 @@ class MiMoV2DecoderLayer(nn.Module):
prefix=add_prefix("mlp", prefix),
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.layernorm_epsilon)
@@ -945,7 +943,7 @@ class MiMoV2DecoderLayer(nn.Module):
)
def op_comm_postprocess_layer(self, state):
hidden_states = self.ffn_boundary.finish_complete_output(
hidden_states = self.ffn_boundary.complete_now(
state.pop("hidden_states_mlp_output"), state.forward_batch
)
@@ -113,6 +113,7 @@ class MiMoV2MTPLayer(nn.Module):
prefix=add_prefix("mlp", prefix),
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.layernorm_epsilon)
self.post_attention_layernorm = RMSNorm(
-2
View File
@@ -53,7 +53,6 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
reduce_moe_output,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
@@ -602,7 +601,6 @@ class MiniMaxM2MoE(nn.Module):
topk_output = self.topk.empty_topk_output(hidden_states.device)
final_hidden_states = self.experts(hidden_states, topk_output)
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_dim)
+5 -29
View File
@@ -28,9 +28,6 @@ from sglang.srt.configs.model_config import (
get_minimax_sparse_disable_value_layer_ids,
get_minimax_sparse_layer_ids,
)
from sglang.srt.distributed import (
tensor_model_parallel_all_reduce,
)
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
@@ -315,15 +312,10 @@ class MiniMaxM3MLP(nn.Module):
self,
x,
forward_batch: Optional[ForwardBatch] = None,
should_allreduce_fusion: bool = False,
use_reduce_scatter: bool = False,
):
gate_up, _ = self.gate_up_proj(x)
x = self.act_fn(gate_up)
x, _ = self.down_proj(
x,
skip_all_reduce=should_allreduce_fusion or use_reduce_scatter,
)
x, _ = self.down_proj(x)
return x
@@ -432,22 +424,13 @@ class MiniMaxM3MoE(nn.Module):
self,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
should_allreduce_fusion: bool = False,
use_reduce_scatter: bool = False,
) -> torch.Tensor:
if get_moe_a2a_backend().is_deepep():
return self.forward_deepep(hidden_states, forward_batch)
else:
return self.forward_normal(
hidden_states, should_allreduce_fusion, use_reduce_scatter
)
return self.forward_normal(hidden_states)
def forward_normal(
self,
hidden_states: torch.Tensor,
should_allreduce_fusion: bool = False,
use_reduce_scatter: bool = False,
) -> torch.Tensor:
def forward_normal(self, hidden_states: torch.Tensor) -> torch.Tensor:
if hidden_states.shape[0] > 0:
if (
self.alt_stream is not None
@@ -470,9 +453,6 @@ class MiniMaxM3MoE(nn.Module):
if shared_output is not None:
final_hidden_states = final_hidden_states + shared_output
if self.tp_size > 1 and not should_allreduce_fusion and not use_reduce_scatter:
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
return final_hidden_states
def _forward_router_experts(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -1325,6 +1305,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
intermediate_size=config.dense_intermediate_size,
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
reduce_results=False,
)
self.use_gemma_norm = getattr(config, "use_gemma_norm", False)
@@ -1395,12 +1376,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
if self.is_layer_sparse or hidden_states.shape[0] != 0:
hidden_states = self.mlp(
hidden_states,
forward_batch=forward_batch,
should_allreduce_fusion=ffn_exit.fuse_mlp_allreduce,
use_reduce_scatter=ffn_exit.mlp_reduce_scatter,
)
hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
return ffn_exit.finish(hidden_states)
+3 -7
View File
@@ -25,9 +25,6 @@ import torch
from torch import nn
from transformers import MixtralConfig
from sglang.srt.distributed import (
tensor_model_parallel_all_reduce,
)
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
@@ -118,8 +115,6 @@ class MixtralMoE(nn.Module):
router_logits, _ = self.gate(hidden_states)
topk_output = self.topk(hidden_states, router_logits)
final_hidden_states = self.experts(hidden_states, topk_output)
if self.tp_size > 1:
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
return final_hidden_states.view(orig_shape)
@@ -269,8 +264,9 @@ class MixtralDecoderLayer(nn.Module):
# Fully Connected
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
hidden_states = self.block_sparse_moe(hidden_states)
return self.ffn_boundary.finish_complete_output(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.block_sparse_moe(hidden_states)
return ffn_exit.finish(hidden_states)
class MixtralModel(nn.Module):
+3
View File
@@ -849,6 +849,7 @@ class MossVLTextMLP(nn.Module):
hidden_act: str = "silu",
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
reduce_results: bool = True,
):
super().__init__()
self.gate_up_proj = MergedColumnParallelLinear(
@@ -863,6 +864,7 @@ class MossVLTextMLP(nn.Module):
hidden_size,
bias=False,
quant_config=quant_config,
reduce_results=reduce_results,
prefix=add_prefix("down_proj", prefix),
)
if hidden_act != "silu":
@@ -996,6 +998,7 @@ class MossVLSelfAttentionDecoderLayer(nn.Module):
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
reduce_results=False,
)
norm_kwargs = (
dict(
+4 -2
View File
@@ -77,6 +77,7 @@ class NanbeigeMLP(nn.Module):
hidden_size,
bias=False,
quant_config=quant_config,
reduce_results=False,
prefix=add_prefix("down_proj", prefix),
)
if hidden_act != "silu":
@@ -261,8 +262,9 @@ class NanbeigeDecoderLayer(nn.Module):
# Fully Connected
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
hidden_states = self.mlp(hidden_states)
return self.ffn_boundary.finish_complete_output(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
return ffn_exit.finish(hidden_states)
class NanbeigeModel(nn.Module):
+8 -23
View File
@@ -49,7 +49,6 @@ from sglang.srt.layers.moe.topk import TopK
from sglang.srt.layers.moe.utils import (
RoutingMethodType,
get_moe_a2a_backend,
reduce_moe_output,
)
from sglang.srt.layers.quantization import QuantizationConfig
from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod
@@ -79,7 +78,7 @@ from sglang.srt.model_loader.weight_utils import (
)
from sglang.srt.models.nemotron_h_utils import make_stage_boundary
from sglang.srt.models.utils import WeightsMapper
from sglang.srt.runtime_context import get_exec, get_forward, get_parallel
from sglang.srt.runtime_context import get_exec, get_parallel
from sglang.srt.utils import (
add_prefix,
get_current_device_stream_fast,
@@ -365,8 +364,6 @@ class NemotronHMoE(nn.Module):
elif shared_output is not None:
final_hidden_states += shared_output
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_dim)
@@ -420,6 +417,7 @@ class NemotronHMLPDecoderLayer(NemotronHMLPLikeDecoderLayer):
quant_config=quant_config,
bias=config.mlp_bias,
prefix=f"{prefix}.mixer",
reduce_results=False,
)
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
@@ -470,9 +468,7 @@ class NemotronHAttnLikeDecoderLayer(nn.Module):
return hidden_states
with self.boundary.exit(forward_batch) as mixer_exit:
hidden_states = self._forward_mixer(
hidden_states, forward_batch, mixer_exit.skips_reduction
)
hidden_states = self._forward_mixer(hidden_states, forward_batch)
return mixer_exit.finish(hidden_states)
@@ -496,7 +492,7 @@ class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer):
rms_norm_eps=config.layer_norm_epsilon,
activation=config.mamba_hidden_act,
quant_config=quant_config,
reduce_results=True,
reduce_results=False,
prefix=f"{prefix}.mixer",
)
@@ -525,19 +521,14 @@ class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer):
self,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
skip_reduce: bool,
) -> torch.Tensor:
if is_in_breakable_cuda_graph():
output = torch.empty_like(hidden_states)
breakable_nemotron_mamba2_with_output(
hidden_states, output, self.layer_id, skip_reduce
)
breakable_nemotron_mamba2_with_output(hidden_states, output, self.layer_id)
return output
if is_in_tc_piecewise_cuda_graph():
output = torch.empty_like(hidden_states)
nemotron_mamba2_with_output(
hidden_states, output, self.layer_id, skip_reduce
)
nemotron_mamba2_with_output(hidden_states, output, self.layer_id)
return output
return self._forward_mamba(hidden_states, forward_batch)
@@ -593,6 +584,7 @@ class NemotronHAttention(nn.Module):
quant_config=quant_config,
tp_rank=tp_rank,
tp_size=tp_size,
reduce_results=False,
use_dp_attention_reduce=is_dp_attention_enabled(),
prefix=f"{prefix}.o_proj",
)
@@ -645,7 +637,6 @@ class NemotronHAttentionDecoderLayer(NemotronHAttnLikeDecoderLayer):
self,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
skip_reduce: bool,
) -> torch.Tensor:
return self.mixer.forward(
hidden_states=hidden_states, forward_batch=forward_batch
@@ -1152,7 +1143,6 @@ def nemotron_mamba2_with_output(
hidden_states: torch.Tensor,
output: torch.Tensor,
layer_id: int,
fuse_mlp_allreduce: bool = False,
) -> None:
"""Split op for Mamba2 forward in piecewise CUDA graph mode."""
context = get_tc_piecewise_forward_context()
@@ -1172,12 +1162,7 @@ def nemotron_mamba2_with_output(
if hidden_states.shape[0] != num_actual_tokens:
hidden_states = hidden_states[:num_actual_tokens]
# This function is an opaque custom op under torch.compile. The caller's
# ForwardFlags scope is Python control-plane state and is no longer active
# when the compiled graph invokes this implementation. Carry the scalar
# across the graph boundary and republish it for RowParallelLinear.
with get_forward().scoped(fuse_mlp_allreduce=fuse_mlp_allreduce):
ret = mamba_layer._forward_mamba(hidden_states, forward_batch)
ret = mamba_layer._forward_mamba(hidden_states, forward_batch)
# Copy result back; output may be larger (padded) so only fill actual tokens
output[:num_actual_tokens].view(ret.shape).copy_(ret)
+2
View File
@@ -72,6 +72,7 @@ class Qwen2MLP(nn.Module):
hidden_act: str,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
reduce_results: bool = True,
) -> None:
super().__init__()
self.gate_up_proj = MergedColumnParallelLinear(
@@ -86,6 +87,7 @@ class Qwen2MLP(nn.Module):
hidden_size,
bias=False,
quant_config=quant_config,
reduce_results=reduce_results,
prefix=add_prefix("down_proj", prefix),
)
if hidden_act != "silu":
+10 -6
View File
@@ -62,10 +62,7 @@ from sglang.srt.layers.linear import (
RowParallelLinear,
)
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
reduce_moe_output,
)
from sglang.srt.layers.moe import get_moe_a2a_backend, reduce_moe_output
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
from sglang.srt.layers.moe.topk import StandardTopKOutput, TopK, TopKOutputChecker
@@ -73,6 +70,7 @@ from sglang.srt.layers.moe.utils import (
RoutingMethodType,
filter_moe_weight_param_global_expert,
is_deepep_class_backend,
sum_post_experts_output,
uses_per_rank_fused_shared_slots,
)
from sglang.srt.layers.quantization.base_config import QuantizationConfig
@@ -266,8 +264,11 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
is_nextn: bool = False,
support_shared_expert_fusion: bool = False,
enable_cuda_shared_expert_fusion: bool = False,
reduce_results: bool = True,
):
super().__init__()
# False when a stage boundary completes this output's sum.
self.reduce_results = reduce_results
self.tp_size = get_parallel().tp_size
self.layer_id = layer_id
self.alt_stream = alt_stream
@@ -871,7 +872,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
final_hidden_states,
gated_shared_output=shared_output,
m=num_tokens,
reduce=reduce_moe_output,
reduce=sum_post_experts_output,
)
if shared_output is not None:
@@ -884,7 +885,8 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
)
else:
final_hidden_states += shared_output
final_hidden_states = reduce_moe_output(final_hidden_states)
if self.reduce_results:
final_hidden_states = reduce_moe_output(final_hidden_states)
# Debug removed - was causing issues during CUDA graph capture
@@ -1035,6 +1037,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
quant_config=quant_config,
alt_stream=alt_stream,
prefix=add_prefix("mlp", prefix),
reduce_results=False,
)
else:
self.mlp = Qwen2MoeMLP(
@@ -1043,6 +1046,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNorm(
+1
View File
@@ -355,6 +355,7 @@ class Qwen3DecoderLayer(nn.Module):
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
reduce_results=False,
)
norm_kwargs = (
+10
View File
@@ -1041,6 +1041,9 @@ class Qwen3_5GatedDeltaNet(nn.Module):
class Qwen3_5LinearDecoderLayer(nn.Module):
"""Qwen3.5 Decoder Layer with Linear Attention (GatedDeltaNet)."""
# True in a subclass built without stage boundaries.
_ffn_sums_itself = False
def __init__(
self,
config: Qwen3_5TextConfig,
@@ -1073,6 +1076,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
prefix=add_prefix("mlp", prefix.replace(".linear_attn", "")),
is_nextn=is_nextn,
support_shared_expert_fusion=not _disable_shared_experts_fusion(),
reduce_results=self._ffn_sums_itself,
)
is_layer_sparse = True
is_previous_layer_sparse = True
@@ -1084,6 +1088,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix.replace(".linear_attn", "")),
reduce_results=self._ffn_sums_itself,
)
_maybe_enable_silu_fp4_quant_fusion(self.mlp)
is_layer_sparse = False
@@ -1169,6 +1174,9 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
class Qwen3_5AttentionDecoderLayer(nn.Module):
"""Qwen3.5 Decoder Layer with Full Attention."""
# See Qwen3_5LinearDecoderLayer._ffn_sums_itself.
_ffn_sums_itself = False
def __init__(
self,
config: Qwen3_5TextConfig,
@@ -1271,6 +1279,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix.replace(".self_attn", "")),
reduce_results=self._ffn_sums_itself,
)
is_layer_sparse = False
is_previous_layer_sparse = False
@@ -1288,6 +1297,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
prefix=add_prefix("mlp", prefix.replace(".self_attn", "")),
is_nextn=is_nextn,
support_shared_expert_fusion=not _disable_shared_experts_fusion(),
reduce_results=self._ffn_sums_itself,
)
is_layer_sparse = True
is_previous_layer_sparse = True
+2 -5
View File
@@ -45,7 +45,6 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
post_experts_all_reduce,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
@@ -342,9 +341,6 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
final_hidden_states = self.experts(hidden_states, topk_output)
final_hidden_states = post_experts_all_reduce(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_dim)
def forward_deepep(
@@ -856,6 +852,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNorm(
@@ -941,7 +938,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
)
def op_comm_postprocess_layer(self, state):
hidden_states = self.ffn_boundary.finish_complete_output(
hidden_states = self.ffn_boundary.complete_now(
state.pop("hidden_states_mlp_output"), state.forward_batch
)
+4
View File
@@ -524,6 +524,7 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
is_nextn=is_nextn,
support_shared_expert_fusion=True,
enable_cuda_shared_expert_fusion=True,
reduce_results=False,
)
else:
self.mlp = Qwen2MoeMLP(
@@ -532,6 +533,7 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix.replace(".linear_attn", "")),
reduce_results=False,
)
self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = GemmaRMSNorm(
@@ -688,6 +690,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
is_nextn=is_nextn,
support_shared_expert_fusion=True,
enable_cuda_shared_expert_fusion=True,
reduce_results=False,
)
else:
self.mlp = Qwen2MoeMLP(
@@ -696,6 +699,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix.replace(".self_attn", "")),
reduce_results=False,
)
self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = GemmaRMSNorm(
+3
View File
@@ -1410,6 +1410,9 @@ class Qwen4ExpPLELayer(nn.Module):
class Qwen4ExpLayerExtensionMixin:
# These layers drop the stage boundaries they inherit.
_ffn_sums_itself = True
def _init_qwen4_exp_layer_extensions(
self,
config: Qwen4ExpTextConfig,
+1 -4
View File
@@ -34,7 +34,6 @@ from sglang.srt.layers.linear import (
RowParallelLinear,
)
from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput
from sglang.srt.layers.moe import reduce_moe_output
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.moe.topk import TopK
@@ -368,7 +367,6 @@ class SarvamMoESparseMoeBlock(nn.Module):
final_hidden_states = final_hidden_states * self.routed_scaling_factor
current_stream.wait_stream(self.alt_stream)
final_hidden_states = final_hidden_states + shared_out
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_dim)
def forward_normal(
@@ -403,8 +401,6 @@ class SarvamMoESparseMoeBlock(nn.Module):
elif self.routed_scaling_factor != 1.0:
final_hidden_states = final_hidden_states * self.routed_scaling_factor
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_dim)
@@ -1038,6 +1034,7 @@ class SarvamMoEMLADecoderLayer(nn.Module):
prefix=add_prefix("mlp", prefix),
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+1 -1
View File
@@ -290,7 +290,7 @@ class SDARBlock(nn.Module):
self.mlp = SDARMLP(
config=config,
quant_config=quant_config,
reduce_results=True,
reduce_results=False,
prefix=add_prefix("mlp", prefix),
)
-3
View File
@@ -31,7 +31,6 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
reduce_moe_output,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
@@ -147,8 +146,6 @@ class SDARMoeSparseMoeBlock(nn.Module):
topk_output = self.topk(hidden_states, router_logits)
out = self.experts(hidden_states, topk_output) # (T, H)
out = reduce_moe_output(out)
return out.view(num_tokens, hidden_dim)
def forward_deepep(self, hidden_states: torch.Tensor, forward_batch: ForwardBatch):
+4 -2
View File
@@ -264,6 +264,7 @@ class Spark2_5DecoderLayer(nn.Module):
intermediate_size=config.intermediate_size,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -295,8 +296,9 @@ class Spark2_5DecoderLayer(nn.Module):
)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
hidden_states = self.mlp(hidden_states)
return self.ffn_boundary.finish_complete_output(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
return ffn_exit.finish(hidden_states)
class Spark2_5Model(nn.Module):
+2 -2
View File
@@ -41,7 +41,6 @@ from sglang.srt.layers.moe import get_moe_a2a_backend
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
from sglang.srt.layers.moe.topk import TopK
from sglang.srt.layers.moe.utils import reduce_moe_output
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.layers.rotary_embedding import get_rope
@@ -346,6 +345,7 @@ class Step3TextDecoderLayer(nn.Module):
hidden_act="silu",
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
reduce_results=False,
)
else:
self.use_moe = True
@@ -397,7 +397,7 @@ class Step3TextDecoderLayer(nn.Module):
hidden_states += self.share_expert(h)
else:
hidden_states = self.moe(hidden_states)
return reduce_moe_output(hidden_states)
return hidden_states
def forward(
self,
+1 -2
View File
@@ -27,7 +27,6 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
reduce_moe_output,
should_use_flashinfer_cutlass_moe_fp4_allgather,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
@@ -226,7 +225,6 @@ class Step3p5MoEMLP(nn.Module):
final_hidden_states = self.experts(hidden_states, topk_output)
if shared_output is not None:
final_hidden_states += shared_output
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_dim)
@@ -506,6 +504,7 @@ class Step3p5DecoderLayer(nn.Module):
swiglu_limit=swiglu_limit_shared,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
reduce_results=False,
)
self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+1 -2
View File
@@ -58,7 +58,6 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
reduce_moe_output,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
@@ -1059,7 +1058,6 @@ class XllmSparseMoeBlock(nn.Module):
if shared_output is not None:
final_hidden_states += shared_output
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_dim)
@@ -1520,6 +1518,7 @@ class XllmDecoderLayer(nn.Module):
prefix=add_prefix("mlp", prefix),
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
reduce_results=False,
)
self.input_layernorm = _make_norm(config)
+1 -1
View File
@@ -152,7 +152,7 @@ def postprocess_output(boundary, hidden, residual, forward_batch):
stream = (
residual if isinstance(residual, ResidualStream) else ResidualStream(residual)
)
hidden = boundary.finish_complete_output(hidden, stream, forward_batch)
hidden = boundary.complete_now(hidden, stream, forward_batch)
if isinstance(residual, ResidualStream):
return hidden, stream
return (hidden, None) if stream.pending is None else stream.input(hidden)