[Fix] moe: Count routed_scaling_factor once when top-k folds it (#44901)

This commit is contained in:
Cheng Wan
2026-10-19 16:54:27 -07:00
committed by GitHub
parent 28448a86d4
commit 3a059de94c
19 changed files with 560 additions and 188 deletions
@@ -288,6 +288,7 @@ class HashTopK(nn.Module):
top_k=self.topk,
num_fused_shared_experts=num_fused_shared_experts,
routed_scaling_factor=self.routed_scaling_factor,
apply_routed_scaling_factor_on_output=self.apply_routed_scaling_factor_on_output,
),
)
else:
+1 -1
View File
@@ -306,7 +306,7 @@ def _run_mega_routed(
activation_clamp=moe.experts.moe_runner_config.swiglu_limit,
routed_scaling_factor=(
1.0
if moe.experts.should_fuse_routed_scaling_factor_in_topk
if moe.routed_scaling_factor_fused_in_topk
else float(moe.routed_scaling_factor)
),
num_shared_experts=(
@@ -373,8 +373,6 @@ def _run_mega_routed(
output = mega.forward(x_in, topk_weights, topk_ids, slice_output=False)[:num_tokens]
from sglang.srt.models.deepseek_common.utils import _use_aiter
if not (moe.experts.should_fuse_routed_scaling_factor_in_topk or _use_aiter):
if not moe.routed_scaling_factor_fused_in_topk:
output.mul_(moe.routed_scaling_factor)
return output
+54 -64
View File
@@ -249,7 +249,7 @@ if _is_musa:
@dataclass
class TopKConfig:
top_k: int
top_k: int # Ordinary output width: routed experts plus shared TopK slots.
use_grouped_topk: bool = False
topk_group: Optional[int] = None
num_expert_group: Optional[int] = None
@@ -260,6 +260,8 @@ class TopKConfig:
torch_native: bool = False
routed_scaling_factor: Optional[float] = None
apply_routed_scaling_factor_on_output: bool = False
# Absolute appended weight for global AITER slots; a multiplier on the
# router's shared weights on the other standard path.
fused_shared_experts_scaling_factor: Optional[float] = None
output_format: Optional[TopKOutputFormat] = None
scoring_func: str = "softmax"
@@ -585,9 +587,9 @@ class TopK(BaseFusedOp):
per-layer EPLB state are keyed by it.
--num_fused_shared_experts: num of shared experts, can be activate both in TP or EP mode.
--routed_scaling_factor: the scaling factor for routed experts in topk_weights.
--fused_shared_experts_scaling_factor: scaling factor applied to the fused shared experts'
topk weight (models pass 1/ep_size under standard EP, where the per-rank shared-expert
outputs are all-reduced).
--fused_shared_experts_scaling_factor: absolute appended weight on the global AITER
path; otherwise multiplies the router's existing shared weight under standard EP.
Models include 1/ep_size when the per-rank shared-expert outputs are all-reduced.
"""
def __init__(
@@ -1655,10 +1657,6 @@ def biased_grouped_topk_impl(
dtype=topk_ids.dtype,
device=topk_ids.device,
)
if routed_scaling_factor is not None:
topk_weights[:, -1] = (
topk_weights[:, :-1].sum(dim=-1) / routed_scaling_factor
)
if renormalize:
# fp32 like the reference gate (the epsilon is not representable in
@@ -1670,8 +1668,14 @@ def biased_grouped_topk_impl(
else topk_weights[:, :-1].sum(dim=-1, keepdim=True, dtype=torch.float32)
)
topk_weights = topk_weights / (topk_weights_sum + _RENORMALIZE_SUM_EPSILON)
if apply_routed_scaling_factor_on_output:
topk_weights *= routed_scaling_factor
if apply_routed_scaling_factor_on_output:
topk_weights *= routed_scaling_factor
if num_fused_shared_experts:
topk_weights[:, -1] = fused_shared_expert_weight(
routed_scaling_factor=routed_scaling_factor,
fused_in_topk=bool(apply_routed_scaling_factor_on_output),
)
topk_weights, topk_ids = topk_weights.to(torch.float32), topk_ids.to(torch.int32)
@@ -1814,9 +1818,7 @@ def biased_grouped_topk_gpu(
# The kernel wants the total width; select_experts passes a routed-only
# topk on the aiter path only (`num_routed_topk if _use_aiter else top_k`).
#
# True, not the caller's flag: an aiter runner skips the post-MoE multiply,
# so the routed weights must carry routed_scaling_factor and the shared
# slot must be 1.0. That flag describes the runner, not this kernel.
# AITER folds this route even when no shared slots are requested.
return jit_grouped_gate(
gating_output,
correction_bias.to(dtype=torch.float32),
@@ -1828,9 +1830,7 @@ def biased_grouped_topk_gpu(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
),
apply_routed_scaling_factor_on_output=(
True
if (_use_aiter and num_fused_shared_experts > 0)
else bool(apply_routed_scaling_factor_on_output)
_use_aiter or bool(apply_routed_scaling_factor_on_output)
),
num_expert_group=num_expert_group,
topk_group=topk_group,
@@ -2208,6 +2208,31 @@ else:
fused_topk_native = fused_topk_torch_native
def is_routed_scaling_factor_fused_in_topk(topk_config: TopKConfig) -> bool:
"""Whether routing already applied the scale that the MoE epilogue must skip."""
if topk_config.apply_routed_scaling_factor_on_output:
return True
if not _use_aiter:
return False
# AITER always folds biased grouped and ungrouped sqrtsoftplus routing.
if topk_config.use_grouped_topk:
return topk_config.correction_bias is not None
return (
topk_config.scoring_func == "sqrtsoftplus"
and topk_config.custom_routing_function is None
)
def fused_shared_expert_weight(
*, routed_scaling_factor: Optional[float], fused_in_topk: bool
) -> float:
"""Shared-slot weight that cancels any later routed-output scaling."""
if fused_in_topk or not routed_scaling_factor:
return 1.0
# The routed output, shared column included, is scaled by the factor later.
return 1.0 / routed_scaling_factor
def remap_topk_for_per_rank_shared_slots(
topk_ids: torch.Tensor,
topk_weights: torch.Tensor,
@@ -2215,16 +2240,7 @@ def remap_topk_for_per_rank_shared_slots(
num_physical_routed_experts: int,
topk_config: TopKConfig,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Remap TopK IDs to a per-rank shared-slot layout.
DeepEP and MegaMoE dispatch need each rank's shared expert at a unique ID
so tokens route to the correct rank. The layout is ordered by rank:
[rank0 routed..., rank0 shared, rank1 routed..., rank1 shared, ...].
Routed IDs: e -> e + (e // num_local_routed) * num_fused_shared_experts
Shared IDs: ep_rank * num_local_experts + num_local_routed
Shared weight: 1.0 on the aiter path, else 1/routed_scaling_factor (see below).
"""
"""Map routed and shared columns into rank-major expert blocks."""
if topk_ids.shape[0] == 0:
return topk_ids, topk_weights
@@ -2253,34 +2269,10 @@ def remap_topk_for_per_rank_shared_slots(
+ torch.arange(num_fused_shared_experts, device=topk_ids.device)
)
# Override the fused shared expert's weight so its net contribution is 1.0x.
#
# The correct value depends on whether routed_scaling_factor is applied to
# the MoE output AFTER the experts run, or already folded into the routed
# topk weights BEFORE dispatch:
#
# * Post-MoE scaling path (default): DeepseekV2MoE.forward_deepep later
# multiplies the whole MoE output by routed_scaling_factor, so the shared
# weight must be 1/routed_scaling_factor for (1/rsf) * rsf = 1.0.
# * aiter (HIP) path: aiter_biased_grouped_topk folds routed_scaling_factor
# into each routed topk weight, and forward_deepep SKIPS the post-MoE
# multiply for _use_aiter (see its `not (... or _use_aiter)` guard). The
# shared weight must therefore be 1.0 -- applying 1/rsf here would
# under-weight the always-on shared expert by routed_scaling_factor and
# corrupt every MoE layer.
#
# NOTE: forward_deepep also skips the post-MoE multiply for the non-aiter
# families where routed_scaling_factor is pre-folded in topk
# (should_fuse_routed_scaling_factor_in_topk / apply_routed_scaling_factor_on_output:
# ModelOpt NVFP4, cutlass/trtllm-routed fp8), so those would likewise need a
# 1.0 shared weight. This fix is deliberately scoped to the aiter path (the
# one validated on AMD MI355X); those other backends are left at their
# existing behavior and can be addressed by their maintainers.
routed_scaling_factor = topk_config.routed_scaling_factor
if _use_aiter:
topk_weights[:, -num_fused_shared_experts:] = 1.0
elif routed_scaling_factor is not None and routed_scaling_factor != 0:
topk_weights[:, -num_fused_shared_experts:] = 1.0 / routed_scaling_factor
topk_weights[:, -num_fused_shared_experts:] = fused_shared_expert_weight(
routed_scaling_factor=topk_config.routed_scaling_factor,
fused_in_topk=is_routed_scaling_factor_fused_in_topk(topk_config),
)
return topk_ids, topk_weights
@@ -2450,14 +2442,6 @@ def _post_process_topk_ids(
# fused_append_shared_experts() + eager per-rank shared-slot remap pair,
# collapsing ~6 launch-bound elementwise kernels/layer (div_floor / add /
# arange / fill / copy) into the one append kernel that already runs.
#
# Shared weight is 1.0 here because this branch is aiter-only:
# aiter_biased_grouped_topk folds routed_scaling_factor into the routed
# weights and forward_deepep skips the post-MoE multiply for _use_aiter,
# so the always-on shared expert must contribute 1.0x. (The eager
# per-rank shared-slot remap instead sets shared weight to
# 1/routed_scaling_factor to compensate a post-MoE scale that the aiter
# path does not apply; see PR #28237.)
num_physical_routed_experts = (
expert_location_dispatch_info.num_physical_experts
if expert_location_dispatch_info is not None
@@ -2478,7 +2462,10 @@ def _post_process_topk_ids(
topk_ids,
topk_weights,
num_fused_shared_experts,
1.0, # shared-expert weight on the aiter path
fused_shared_expert_weight(
routed_scaling_factor=topk_config.routed_scaling_factor,
fused_in_topk=is_routed_scaling_factor_fused_in_topk(topk_config),
),
shared_id_base,
num_local_routed,
num_token_non_padded=(
@@ -2656,7 +2643,10 @@ def select_experts(
topk_group=topk_group or 1,
num_fused_shared_experts=num_fused_shared_experts,
routed_scaling_factor=routed_scaling_factor,
apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output,
# Per-token bias bypasses AITER but must retain its folding policy.
apply_routed_scaling_factor_on_output=is_routed_scaling_factor_fused_in_topk(
topk_config
),
)
elif use_grouped_topk:
assert topk_group is not None
+2 -4
View File
@@ -77,16 +77,14 @@ class WaterfillBalancer:
world_size: int,
rank: int,
layer_id: int,
routed_scaling_factor: float = 1.0,
shared_weight: float,
):
self.num_routed_experts = num_routed_experts
self.world_size = world_size
self.rank = rank
self.layer_id = layer_id
self.old_experts_per_rank = num_routed_experts // world_size
self.shared_weight = (
1.0 / routed_scaling_factor if routed_scaling_factor != 0 else 1.0
)
self.shared_weight = shared_weight
self._counts_buf: Optional[Tensor] = None
self.use_static_waterfill = not envs.SGLANG_DISABLE_STATIC_WATERFILL.get()
@@ -12,7 +12,11 @@ from sglang.srt.eplb.lplb_solver import (
set_global_lplb_solver,
)
from sglang.srt.layers.moe.hash_topk import HashTopK
from sglang.srt.layers.moe.topk import TopK
from sglang.srt.layers.moe.topk import (
TopK,
fused_shared_expert_weight,
is_routed_scaling_factor_fused_in_topk,
)
from sglang.srt.runtime_context import get_exec, get_parallel
from sglang.srt.utils import get_bool_env_var, is_hip, log_info_on_rank0
@@ -52,15 +56,18 @@ def prepare_moe_topk(*, model, model_config: ModelConfig) -> None:
)
if isinstance(module, TopK):
routed_scaling_factor = module.topk_config.routed_scaling_factor
fused_in_topk = is_routed_scaling_factor_fused_in_topk(module.topk_config)
else:
routed_scaling_factor = module.routed_scaling_factor
fused_in_topk = module.apply_routed_scaling_factor_on_output
module.waterfill_balancer = balancer_cls(
num_routed_experts=num_physical_routed_experts,
world_size=parallel.moe_ep_size,
rank=parallel.moe_ep_rank,
layer_id=module.layer_id,
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
shared_weight=fused_shared_expert_weight(
routed_scaling_factor=routed_scaling_factor,
fused_in_topk=fused_in_topk,
),
)
num_prepared += 1
+3 -3
View File
@@ -47,7 +47,7 @@ from sglang.srt.layers.moe import (
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.moe.topk import TopK
from sglang.srt.layers.moe.topk import TopK, is_routed_scaling_factor_fused_in_topk
from sglang.srt.layers.moe.utils import is_shared_experts_fusion_disabled
from sglang.srt.layers.multi_gate import (
create_multi_gate_mm_indices,
@@ -604,11 +604,11 @@ class BailingMoE(nn.Module):
# Whether to apply routed_scaling_factor at model layer.
# For A2A MoE paths (e.g., DeepEP), the runner/post_permute does not apply it,
# so apply it here unless the selected runner fuses it into TopK weights.
# so apply it here unless top-k already folded it into the routed weights.
# This scales only routed expert output; shared experts are added unscaled.
self._apply_routed_scaling_factor_on_output = (
self._enable_a2a_moe
and not self.experts.should_fuse_routed_scaling_factor_in_topk
and not is_routed_scaling_factor_fused_in_topk(self.topk.topk_config)
and self.routed_scaling_factor is not None
and self.routed_scaling_factor != 1.0
)
+65 -34
View File
@@ -118,7 +118,13 @@ from sglang.srt.layers.moe.token_dispatcher.base import (
CombineInput,
DispatchOutput,
)
from sglang.srt.layers.moe.topk import BypassedTopKOutput, TopK, TopKOutputFormat
from sglang.srt.layers.moe.topk import (
BypassedTopKOutput,
TopK,
TopKOutputFormat,
fused_shared_expert_weight,
is_routed_scaling_factor_fused_in_topk,
)
from sglang.srt.layers.moe.utils import (
RoutingMethodType,
adds_replicated_output_to_partial,
@@ -625,6 +631,23 @@ class MergedMoEOutput(MoEOutput):
return self.routed
def _scale_routed_and_add_shared(
routed: torch.Tensor,
shared: Optional[torch.Tensor],
*,
routed_scaling_factor: float,
fused_in_topk: bool,
) -> torch.Tensor:
"""Apply any outstanding routed scale before adding the shared output."""
if shared is not None:
if fused_in_topk:
return shared.add_(routed)
return shared.add_(routed, alpha=routed_scaling_factor)
if not fused_in_topk:
routed *= routed_scaling_factor
return routed
class DeepseekV2MoE(nn.Module):
def __init__(
self,
@@ -748,6 +771,10 @@ class DeepseekV2MoE(nn.Module):
apply_routed_scaling_factor_on_output=self.experts.should_fuse_routed_scaling_factor_in_topk,
layer_id=self.layer_id,
)
# Hash routing folds the factor only when its flag asks for it.
self.routed_scaling_factor_fused_in_topk = (
self.topk.apply_routed_scaling_factor_on_output
)
else:
# Default: grouped noaux_tc top-k. Covers V3/V3.2/GLM-5/Glm4MoeLite.
topk_kwargs = dict(
@@ -785,6 +812,8 @@ class DeepseekV2MoE(nn.Module):
getattr(config, "model_type", None) == "deepseek_v41"
),
is_fp4_experts=getattr(quant_config, "is_fp4_experts", False),
# aiter's topk_gating folds the factor regardless; the flag makes
# the other routes (e.g. the vision routing fallback) fold too.
apply_routed_scaling_factor_on_output=(
True
if _use_aiter
@@ -792,6 +821,27 @@ class DeepseekV2MoE(nn.Module):
),
)
self.topk = TopK(**topk_kwargs)
self.routed_scaling_factor_fused_in_topk = (
is_routed_scaling_factor_fused_in_topk(self.topk.topk_config)
)
if (
_use_aiter
and self.num_fused_shared_experts > 0
and not _uses_per_rank_shared_slots
and not self.routed_scaling_factor_fused_in_topk
):
# Compensate for both post-MoE scaling and EP replication.
self.topk.topk_config.fused_shared_experts_scaling_factor = (
fused_shared_expert_weight(
routed_scaling_factor=self.routed_scaling_factor,
fused_in_topk=self.routed_scaling_factor_fused_in_topk,
)
* (
fused_shared_experts_scaling_factor
if fused_shared_experts_scaling_factor is not None
else 1.0
)
)
self.shared_experts_is_int8 = False
self.shared_experts_is_fp8 = False
@@ -1191,7 +1241,7 @@ class DeepseekV2MoE(nn.Module):
if (
not _is_cuda
and not _is_musa
and not _use_aiter
and not (_use_aiter and self.routed_scaling_factor_fused_in_topk)
or isinstance(self.experts.quant_method, KTEPWrapperMethod)
):
final_hidden_states *= self.routed_scaling_factor
@@ -1383,10 +1433,9 @@ class DeepseekV2MoE(nn.Module):
not _is_cuda
and not _is_musa
and not _is_xpu
and not _use_aiter
and not (_use_aiter and self.routed_scaling_factor_fused_in_topk)
or isinstance(self.experts.quant_method, KTEPWrapperMethod)
):
# fused in biased_grouped_topk so we can skip here
final_hidden_states *= self.routed_scaling_factor
if (
@@ -1705,22 +1754,12 @@ class DeepseekV2MoE(nn.Module):
):
torch.cuda.current_stream().wait_event(shared_event)
if shared_output is not None:
x = shared_output
# aiter moe call will handle routed_scaling_factor in the function
# so add _use_aiter condition to eliminate to use self.routed_scaling_factor in add_ call
if self.experts.should_fuse_routed_scaling_factor_in_topk or _use_aiter:
x.add_(final_hidden_states)
else:
x.add_(final_hidden_states, alpha=self.routed_scaling_factor)
final_hidden_states = x
else:
if not (
self.experts.should_fuse_routed_scaling_factor_in_topk or _use_aiter
):
final_hidden_states *= self.routed_scaling_factor
return final_hidden_states
return _scale_routed_and_add_shared(
final_hidden_states,
shared_output,
routed_scaling_factor=self.routed_scaling_factor,
fused_in_topk=self.routed_scaling_factor_fused_in_topk,
)
def _forward_shared_experts(
self,
@@ -1987,20 +2026,12 @@ class DeepseekV2MoE(nn.Module):
num_tokens = state.pop("num_tokens")
final_hidden_states = final_hidden_states[:num_tokens]
if (shared_output := state.pop("shared_output")) is not None:
x = shared_output
if _use_aiter:
x.add_(final_hidden_states)
else:
x.add_(final_hidden_states, alpha=self.routed_scaling_factor)
final_hidden_states = x
elif _use_aiter:
# fused in aiter_biased_grouped_topk so we can skip here
pass
else:
final_hidden_states *= self.routed_scaling_factor
state.hidden_states_mlp_output = final_hidden_states
state.hidden_states_mlp_output = _scale_routed_and_add_shared(
final_hidden_states,
state.pop("shared_output"),
routed_scaling_factor=self.routed_scaling_factor,
fused_in_topk=self.routed_scaling_factor_fused_in_topk,
)
class DeepseekV2AttentionMLA(
@@ -79,7 +79,7 @@ from sglang.srt.layers.moe import (
)
from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.moe.topk import TopK
from sglang.srt.layers.moe.topk import TopK, is_routed_scaling_factor_fused_in_topk
from sglang.srt.layers.moe.utils import is_shared_experts_fusion_disabled
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.quantization.fp8_utils import (
@@ -484,14 +484,15 @@ class Dots3MoE(nn.Module):
topk_output=topk_output,
)
fused_in_topk = is_routed_scaling_factor_fused_in_topk(self.topk.topk_config)
if shared_output is not None:
x = shared_output
if self.experts.should_fuse_routed_scaling_factor_in_topk:
if fused_in_topk:
x.add_(final_hidden_states)
else:
x.add_(final_hidden_states, alpha=self.routed_scaling_factor)
final_hidden_states = x
elif not self.experts.should_fuse_routed_scaling_factor_in_topk:
elif not fused_in_topk:
final_hidden_states *= self.routed_scaling_factor
return final_hidden_states
+5 -5
View File
@@ -65,7 +65,7 @@ from sglang.srt.layers.moe import (
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.moe.kt_ep_wrapper import KTEPWrapperMethod
from sglang.srt.layers.moe.topk import TopK
from sglang.srt.layers.moe.topk import TopK, is_routed_scaling_factor_fused_in_topk
from sglang.srt.layers.moe.utils import (
RoutingMethodType,
filter_moe_weight_param_global_expert,
@@ -667,16 +667,16 @@ class Glm4MoeSparseMoeBlock(nn.Module):
if enable_npu_dual_stream:
wait_share_stream()
fused_in_topk = is_routed_scaling_factor_fused_in_topk(self.topk.topk_config)
if shared_output is not None:
x = shared_output
if self.experts.should_fuse_routed_scaling_factor_in_topk:
if fused_in_topk:
x.add_(final_hidden_states)
else:
x.add_(final_hidden_states, alpha=self.routed_scaling_factor)
final_hidden_states = x
else:
if not self.experts.should_fuse_routed_scaling_factor_in_topk:
final_hidden_states *= self.routed_scaling_factor
elif not fused_in_topk:
final_hidden_states *= self.routed_scaling_factor
return final_hidden_states
+9 -5
View File
@@ -59,7 +59,11 @@ from sglang.srt.layers.moe import (
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.moe.kt_ep_wrapper import KTEPWrapperMethod
from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat
from sglang.srt.layers.moe.topk import (
TopK,
TopKOutputFormat,
is_routed_scaling_factor_fused_in_topk,
)
from sglang.srt.layers.moe.utils import (
filter_moe_weight_param_global_expert,
is_shared_experts_fusion_disabled,
@@ -388,16 +392,16 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module):
topk_output=topk_output,
)
fused_in_topk = is_routed_scaling_factor_fused_in_topk(self.topk.topk_config)
if shared_output is not None:
x = shared_output
if self.experts.should_fuse_routed_scaling_factor_in_topk:
if fused_in_topk:
x.add_(final_hidden_states)
else:
x.add_(final_hidden_states, alpha=self.routed_scaling_factor)
final_hidden_states = x
else:
if not self.experts.should_fuse_routed_scaling_factor_in_topk:
final_hidden_states *= self.routed_scaling_factor
elif not fused_in_topk:
final_hidden_states *= self.routed_scaling_factor
return final_hidden_states
@@ -149,12 +149,6 @@ def _finish_vision_topk(
expert_location_dispatch_info=expert_location_dispatch_info,
padded_rows_masked=padded_rows_masked,
)
if (
config.num_fused_shared_experts
and config.apply_routed_scaling_factor_on_output
):
# MegaMoE applies no post-expert scale when it is already in top-k.
weights[:, -config.num_fused_shared_experts :] = 1.0
if recorder_ids is not None:
get_global_expert_distribution_recorder().on_select_experts(
topk_ids=recorder_ids