[Fix] moe: Count routed_scaling_factor once when top-k folds it (#44901)
This commit is contained in:
@@ -288,6 +288,7 @@ class HashTopK(nn.Module):
|
||||
top_k=self.topk,
|
||||
num_fused_shared_experts=num_fused_shared_experts,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
apply_routed_scaling_factor_on_output=self.apply_routed_scaling_factor_on_output,
|
||||
),
|
||||
)
|
||||
else:
|
||||
|
||||
@@ -306,7 +306,7 @@ def _run_mega_routed(
|
||||
activation_clamp=moe.experts.moe_runner_config.swiglu_limit,
|
||||
routed_scaling_factor=(
|
||||
1.0
|
||||
if moe.experts.should_fuse_routed_scaling_factor_in_topk
|
||||
if moe.routed_scaling_factor_fused_in_topk
|
||||
else float(moe.routed_scaling_factor)
|
||||
),
|
||||
num_shared_experts=(
|
||||
|
||||
@@ -373,8 +373,6 @@ def _run_mega_routed(
|
||||
|
||||
output = mega.forward(x_in, topk_weights, topk_ids, slice_output=False)[:num_tokens]
|
||||
|
||||
from sglang.srt.models.deepseek_common.utils import _use_aiter
|
||||
|
||||
if not (moe.experts.should_fuse_routed_scaling_factor_in_topk or _use_aiter):
|
||||
if not moe.routed_scaling_factor_fused_in_topk:
|
||||
output.mul_(moe.routed_scaling_factor)
|
||||
return output
|
||||
|
||||
@@ -249,7 +249,7 @@ if _is_musa:
|
||||
|
||||
@dataclass
|
||||
class TopKConfig:
|
||||
top_k: int
|
||||
top_k: int # Ordinary output width: routed experts plus shared TopK slots.
|
||||
use_grouped_topk: bool = False
|
||||
topk_group: Optional[int] = None
|
||||
num_expert_group: Optional[int] = None
|
||||
@@ -260,6 +260,8 @@ class TopKConfig:
|
||||
torch_native: bool = False
|
||||
routed_scaling_factor: Optional[float] = None
|
||||
apply_routed_scaling_factor_on_output: bool = False
|
||||
# Absolute appended weight for global AITER slots; a multiplier on the
|
||||
# router's shared weights on the other standard path.
|
||||
fused_shared_experts_scaling_factor: Optional[float] = None
|
||||
output_format: Optional[TopKOutputFormat] = None
|
||||
scoring_func: str = "softmax"
|
||||
@@ -585,9 +587,9 @@ class TopK(BaseFusedOp):
|
||||
per-layer EPLB state are keyed by it.
|
||||
--num_fused_shared_experts: num of shared experts, can be activate both in TP or EP mode.
|
||||
--routed_scaling_factor: the scaling factor for routed experts in topk_weights.
|
||||
--fused_shared_experts_scaling_factor: scaling factor applied to the fused shared experts'
|
||||
topk weight (models pass 1/ep_size under standard EP, where the per-rank shared-expert
|
||||
outputs are all-reduced).
|
||||
--fused_shared_experts_scaling_factor: absolute appended weight on the global AITER
|
||||
path; otherwise multiplies the router's existing shared weight under standard EP.
|
||||
Models include 1/ep_size when the per-rank shared-expert outputs are all-reduced.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -1655,10 +1657,6 @@ def biased_grouped_topk_impl(
|
||||
dtype=topk_ids.dtype,
|
||||
device=topk_ids.device,
|
||||
)
|
||||
if routed_scaling_factor is not None:
|
||||
topk_weights[:, -1] = (
|
||||
topk_weights[:, :-1].sum(dim=-1) / routed_scaling_factor
|
||||
)
|
||||
|
||||
if renormalize:
|
||||
# fp32 like the reference gate (the epsilon is not representable in
|
||||
@@ -1670,8 +1668,14 @@ def biased_grouped_topk_impl(
|
||||
else topk_weights[:, :-1].sum(dim=-1, keepdim=True, dtype=torch.float32)
|
||||
)
|
||||
topk_weights = topk_weights / (topk_weights_sum + _RENORMALIZE_SUM_EPSILON)
|
||||
if apply_routed_scaling_factor_on_output:
|
||||
topk_weights *= routed_scaling_factor
|
||||
if apply_routed_scaling_factor_on_output:
|
||||
topk_weights *= routed_scaling_factor
|
||||
|
||||
if num_fused_shared_experts:
|
||||
topk_weights[:, -1] = fused_shared_expert_weight(
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
fused_in_topk=bool(apply_routed_scaling_factor_on_output),
|
||||
)
|
||||
|
||||
topk_weights, topk_ids = topk_weights.to(torch.float32), topk_ids.to(torch.int32)
|
||||
|
||||
@@ -1814,9 +1818,7 @@ def biased_grouped_topk_gpu(
|
||||
# The kernel wants the total width; select_experts passes a routed-only
|
||||
# topk on the aiter path only (`num_routed_topk if _use_aiter else top_k`).
|
||||
#
|
||||
# True, not the caller's flag: an aiter runner skips the post-MoE multiply,
|
||||
# so the routed weights must carry routed_scaling_factor and the shared
|
||||
# slot must be 1.0. That flag describes the runner, not this kernel.
|
||||
# AITER folds this route even when no shared slots are requested.
|
||||
return jit_grouped_gate(
|
||||
gating_output,
|
||||
correction_bias.to(dtype=torch.float32),
|
||||
@@ -1828,9 +1830,7 @@ def biased_grouped_topk_gpu(
|
||||
routed_scaling_factor if routed_scaling_factor is not None else 1.0
|
||||
),
|
||||
apply_routed_scaling_factor_on_output=(
|
||||
True
|
||||
if (_use_aiter and num_fused_shared_experts > 0)
|
||||
else bool(apply_routed_scaling_factor_on_output)
|
||||
_use_aiter or bool(apply_routed_scaling_factor_on_output)
|
||||
),
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
@@ -2208,6 +2208,31 @@ else:
|
||||
fused_topk_native = fused_topk_torch_native
|
||||
|
||||
|
||||
def is_routed_scaling_factor_fused_in_topk(topk_config: TopKConfig) -> bool:
|
||||
"""Whether routing already applied the scale that the MoE epilogue must skip."""
|
||||
if topk_config.apply_routed_scaling_factor_on_output:
|
||||
return True
|
||||
if not _use_aiter:
|
||||
return False
|
||||
# AITER always folds biased grouped and ungrouped sqrtsoftplus routing.
|
||||
if topk_config.use_grouped_topk:
|
||||
return topk_config.correction_bias is not None
|
||||
return (
|
||||
topk_config.scoring_func == "sqrtsoftplus"
|
||||
and topk_config.custom_routing_function is None
|
||||
)
|
||||
|
||||
|
||||
def fused_shared_expert_weight(
|
||||
*, routed_scaling_factor: Optional[float], fused_in_topk: bool
|
||||
) -> float:
|
||||
"""Shared-slot weight that cancels any later routed-output scaling."""
|
||||
if fused_in_topk or not routed_scaling_factor:
|
||||
return 1.0
|
||||
# The routed output, shared column included, is scaled by the factor later.
|
||||
return 1.0 / routed_scaling_factor
|
||||
|
||||
|
||||
def remap_topk_for_per_rank_shared_slots(
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
@@ -2215,16 +2240,7 @@ def remap_topk_for_per_rank_shared_slots(
|
||||
num_physical_routed_experts: int,
|
||||
topk_config: TopKConfig,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Remap TopK IDs to a per-rank shared-slot layout.
|
||||
|
||||
DeepEP and MegaMoE dispatch need each rank's shared expert at a unique ID
|
||||
so tokens route to the correct rank. The layout is ordered by rank:
|
||||
[rank0 routed..., rank0 shared, rank1 routed..., rank1 shared, ...].
|
||||
|
||||
Routed IDs: e -> e + (e // num_local_routed) * num_fused_shared_experts
|
||||
Shared IDs: ep_rank * num_local_experts + num_local_routed
|
||||
Shared weight: 1.0 on the aiter path, else 1/routed_scaling_factor (see below).
|
||||
"""
|
||||
"""Map routed and shared columns into rank-major expert blocks."""
|
||||
if topk_ids.shape[0] == 0:
|
||||
return topk_ids, topk_weights
|
||||
|
||||
@@ -2253,34 +2269,10 @@ def remap_topk_for_per_rank_shared_slots(
|
||||
+ torch.arange(num_fused_shared_experts, device=topk_ids.device)
|
||||
)
|
||||
|
||||
# Override the fused shared expert's weight so its net contribution is 1.0x.
|
||||
#
|
||||
# The correct value depends on whether routed_scaling_factor is applied to
|
||||
# the MoE output AFTER the experts run, or already folded into the routed
|
||||
# topk weights BEFORE dispatch:
|
||||
#
|
||||
# * Post-MoE scaling path (default): DeepseekV2MoE.forward_deepep later
|
||||
# multiplies the whole MoE output by routed_scaling_factor, so the shared
|
||||
# weight must be 1/routed_scaling_factor for (1/rsf) * rsf = 1.0.
|
||||
# * aiter (HIP) path: aiter_biased_grouped_topk folds routed_scaling_factor
|
||||
# into each routed topk weight, and forward_deepep SKIPS the post-MoE
|
||||
# multiply for _use_aiter (see its `not (... or _use_aiter)` guard). The
|
||||
# shared weight must therefore be 1.0 -- applying 1/rsf here would
|
||||
# under-weight the always-on shared expert by routed_scaling_factor and
|
||||
# corrupt every MoE layer.
|
||||
#
|
||||
# NOTE: forward_deepep also skips the post-MoE multiply for the non-aiter
|
||||
# families where routed_scaling_factor is pre-folded in topk
|
||||
# (should_fuse_routed_scaling_factor_in_topk / apply_routed_scaling_factor_on_output:
|
||||
# ModelOpt NVFP4, cutlass/trtllm-routed fp8), so those would likewise need a
|
||||
# 1.0 shared weight. This fix is deliberately scoped to the aiter path (the
|
||||
# one validated on AMD MI355X); those other backends are left at their
|
||||
# existing behavior and can be addressed by their maintainers.
|
||||
routed_scaling_factor = topk_config.routed_scaling_factor
|
||||
if _use_aiter:
|
||||
topk_weights[:, -num_fused_shared_experts:] = 1.0
|
||||
elif routed_scaling_factor is not None and routed_scaling_factor != 0:
|
||||
topk_weights[:, -num_fused_shared_experts:] = 1.0 / routed_scaling_factor
|
||||
topk_weights[:, -num_fused_shared_experts:] = fused_shared_expert_weight(
|
||||
routed_scaling_factor=topk_config.routed_scaling_factor,
|
||||
fused_in_topk=is_routed_scaling_factor_fused_in_topk(topk_config),
|
||||
)
|
||||
|
||||
return topk_ids, topk_weights
|
||||
|
||||
@@ -2450,14 +2442,6 @@ def _post_process_topk_ids(
|
||||
# fused_append_shared_experts() + eager per-rank shared-slot remap pair,
|
||||
# collapsing ~6 launch-bound elementwise kernels/layer (div_floor / add /
|
||||
# arange / fill / copy) into the one append kernel that already runs.
|
||||
#
|
||||
# Shared weight is 1.0 here because this branch is aiter-only:
|
||||
# aiter_biased_grouped_topk folds routed_scaling_factor into the routed
|
||||
# weights and forward_deepep skips the post-MoE multiply for _use_aiter,
|
||||
# so the always-on shared expert must contribute 1.0x. (The eager
|
||||
# per-rank shared-slot remap instead sets shared weight to
|
||||
# 1/routed_scaling_factor to compensate a post-MoE scale that the aiter
|
||||
# path does not apply; see PR #28237.)
|
||||
num_physical_routed_experts = (
|
||||
expert_location_dispatch_info.num_physical_experts
|
||||
if expert_location_dispatch_info is not None
|
||||
@@ -2478,7 +2462,10 @@ def _post_process_topk_ids(
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
num_fused_shared_experts,
|
||||
1.0, # shared-expert weight on the aiter path
|
||||
fused_shared_expert_weight(
|
||||
routed_scaling_factor=topk_config.routed_scaling_factor,
|
||||
fused_in_topk=is_routed_scaling_factor_fused_in_topk(topk_config),
|
||||
),
|
||||
shared_id_base,
|
||||
num_local_routed,
|
||||
num_token_non_padded=(
|
||||
@@ -2656,7 +2643,10 @@ def select_experts(
|
||||
topk_group=topk_group or 1,
|
||||
num_fused_shared_experts=num_fused_shared_experts,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output,
|
||||
# Per-token bias bypasses AITER but must retain its folding policy.
|
||||
apply_routed_scaling_factor_on_output=is_routed_scaling_factor_fused_in_topk(
|
||||
topk_config
|
||||
),
|
||||
)
|
||||
elif use_grouped_topk:
|
||||
assert topk_group is not None
|
||||
|
||||
@@ -77,16 +77,14 @@ class WaterfillBalancer:
|
||||
world_size: int,
|
||||
rank: int,
|
||||
layer_id: int,
|
||||
routed_scaling_factor: float = 1.0,
|
||||
shared_weight: float,
|
||||
):
|
||||
self.num_routed_experts = num_routed_experts
|
||||
self.world_size = world_size
|
||||
self.rank = rank
|
||||
self.layer_id = layer_id
|
||||
self.old_experts_per_rank = num_routed_experts // world_size
|
||||
self.shared_weight = (
|
||||
1.0 / routed_scaling_factor if routed_scaling_factor != 0 else 1.0
|
||||
)
|
||||
self.shared_weight = shared_weight
|
||||
self._counts_buf: Optional[Tensor] = None
|
||||
self.use_static_waterfill = not envs.SGLANG_DISABLE_STATIC_WATERFILL.get()
|
||||
|
||||
|
||||
@@ -12,7 +12,11 @@ from sglang.srt.eplb.lplb_solver import (
|
||||
set_global_lplb_solver,
|
||||
)
|
||||
from sglang.srt.layers.moe.hash_topk import HashTopK
|
||||
from sglang.srt.layers.moe.topk import TopK
|
||||
from sglang.srt.layers.moe.topk import (
|
||||
TopK,
|
||||
fused_shared_expert_weight,
|
||||
is_routed_scaling_factor_fused_in_topk,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel
|
||||
from sglang.srt.utils import get_bool_env_var, is_hip, log_info_on_rank0
|
||||
|
||||
@@ -52,15 +56,18 @@ def prepare_moe_topk(*, model, model_config: ModelConfig) -> None:
|
||||
)
|
||||
if isinstance(module, TopK):
|
||||
routed_scaling_factor = module.topk_config.routed_scaling_factor
|
||||
fused_in_topk = is_routed_scaling_factor_fused_in_topk(module.topk_config)
|
||||
else:
|
||||
routed_scaling_factor = module.routed_scaling_factor
|
||||
fused_in_topk = module.apply_routed_scaling_factor_on_output
|
||||
module.waterfill_balancer = balancer_cls(
|
||||
num_routed_experts=num_physical_routed_experts,
|
||||
world_size=parallel.moe_ep_size,
|
||||
rank=parallel.moe_ep_rank,
|
||||
layer_id=module.layer_id,
|
||||
routed_scaling_factor=(
|
||||
routed_scaling_factor if routed_scaling_factor is not None else 1.0
|
||||
shared_weight=fused_shared_expert_weight(
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
fused_in_topk=fused_in_topk,
|
||||
),
|
||||
)
|
||||
num_prepared += 1
|
||||
|
||||
@@ -47,7 +47,7 @@ from sglang.srt.layers.moe import (
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.moe.topk import TopK
|
||||
from sglang.srt.layers.moe.topk import TopK, is_routed_scaling_factor_fused_in_topk
|
||||
from sglang.srt.layers.moe.utils import is_shared_experts_fusion_disabled
|
||||
from sglang.srt.layers.multi_gate import (
|
||||
create_multi_gate_mm_indices,
|
||||
@@ -604,11 +604,11 @@ class BailingMoE(nn.Module):
|
||||
|
||||
# Whether to apply routed_scaling_factor at model layer.
|
||||
# For A2A MoE paths (e.g., DeepEP), the runner/post_permute does not apply it,
|
||||
# so apply it here unless the selected runner fuses it into TopK weights.
|
||||
# so apply it here unless top-k already folded it into the routed weights.
|
||||
# This scales only routed expert output; shared experts are added unscaled.
|
||||
self._apply_routed_scaling_factor_on_output = (
|
||||
self._enable_a2a_moe
|
||||
and not self.experts.should_fuse_routed_scaling_factor_in_topk
|
||||
and not is_routed_scaling_factor_fused_in_topk(self.topk.topk_config)
|
||||
and self.routed_scaling_factor is not None
|
||||
and self.routed_scaling_factor != 1.0
|
||||
)
|
||||
|
||||
@@ -118,7 +118,13 @@ from sglang.srt.layers.moe.token_dispatcher.base import (
|
||||
CombineInput,
|
||||
DispatchOutput,
|
||||
)
|
||||
from sglang.srt.layers.moe.topk import BypassedTopKOutput, TopK, TopKOutputFormat
|
||||
from sglang.srt.layers.moe.topk import (
|
||||
BypassedTopKOutput,
|
||||
TopK,
|
||||
TopKOutputFormat,
|
||||
fused_shared_expert_weight,
|
||||
is_routed_scaling_factor_fused_in_topk,
|
||||
)
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
RoutingMethodType,
|
||||
adds_replicated_output_to_partial,
|
||||
@@ -625,6 +631,23 @@ class MergedMoEOutput(MoEOutput):
|
||||
return self.routed
|
||||
|
||||
|
||||
def _scale_routed_and_add_shared(
|
||||
routed: torch.Tensor,
|
||||
shared: Optional[torch.Tensor],
|
||||
*,
|
||||
routed_scaling_factor: float,
|
||||
fused_in_topk: bool,
|
||||
) -> torch.Tensor:
|
||||
"""Apply any outstanding routed scale before adding the shared output."""
|
||||
if shared is not None:
|
||||
if fused_in_topk:
|
||||
return shared.add_(routed)
|
||||
return shared.add_(routed, alpha=routed_scaling_factor)
|
||||
if not fused_in_topk:
|
||||
routed *= routed_scaling_factor
|
||||
return routed
|
||||
|
||||
|
||||
class DeepseekV2MoE(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -748,6 +771,10 @@ class DeepseekV2MoE(nn.Module):
|
||||
apply_routed_scaling_factor_on_output=self.experts.should_fuse_routed_scaling_factor_in_topk,
|
||||
layer_id=self.layer_id,
|
||||
)
|
||||
# Hash routing folds the factor only when its flag asks for it.
|
||||
self.routed_scaling_factor_fused_in_topk = (
|
||||
self.topk.apply_routed_scaling_factor_on_output
|
||||
)
|
||||
else:
|
||||
# Default: grouped noaux_tc top-k. Covers V3/V3.2/GLM-5/Glm4MoeLite.
|
||||
topk_kwargs = dict(
|
||||
@@ -785,6 +812,8 @@ class DeepseekV2MoE(nn.Module):
|
||||
getattr(config, "model_type", None) == "deepseek_v41"
|
||||
),
|
||||
is_fp4_experts=getattr(quant_config, "is_fp4_experts", False),
|
||||
# aiter's topk_gating folds the factor regardless; the flag makes
|
||||
# the other routes (e.g. the vision routing fallback) fold too.
|
||||
apply_routed_scaling_factor_on_output=(
|
||||
True
|
||||
if _use_aiter
|
||||
@@ -792,6 +821,27 @@ class DeepseekV2MoE(nn.Module):
|
||||
),
|
||||
)
|
||||
self.topk = TopK(**topk_kwargs)
|
||||
self.routed_scaling_factor_fused_in_topk = (
|
||||
is_routed_scaling_factor_fused_in_topk(self.topk.topk_config)
|
||||
)
|
||||
if (
|
||||
_use_aiter
|
||||
and self.num_fused_shared_experts > 0
|
||||
and not _uses_per_rank_shared_slots
|
||||
and not self.routed_scaling_factor_fused_in_topk
|
||||
):
|
||||
# Compensate for both post-MoE scaling and EP replication.
|
||||
self.topk.topk_config.fused_shared_experts_scaling_factor = (
|
||||
fused_shared_expert_weight(
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
fused_in_topk=self.routed_scaling_factor_fused_in_topk,
|
||||
)
|
||||
* (
|
||||
fused_shared_experts_scaling_factor
|
||||
if fused_shared_experts_scaling_factor is not None
|
||||
else 1.0
|
||||
)
|
||||
)
|
||||
|
||||
self.shared_experts_is_int8 = False
|
||||
self.shared_experts_is_fp8 = False
|
||||
@@ -1191,7 +1241,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
if (
|
||||
not _is_cuda
|
||||
and not _is_musa
|
||||
and not _use_aiter
|
||||
and not (_use_aiter and self.routed_scaling_factor_fused_in_topk)
|
||||
or isinstance(self.experts.quant_method, KTEPWrapperMethod)
|
||||
):
|
||||
final_hidden_states *= self.routed_scaling_factor
|
||||
@@ -1383,10 +1433,9 @@ class DeepseekV2MoE(nn.Module):
|
||||
not _is_cuda
|
||||
and not _is_musa
|
||||
and not _is_xpu
|
||||
and not _use_aiter
|
||||
and not (_use_aiter and self.routed_scaling_factor_fused_in_topk)
|
||||
or isinstance(self.experts.quant_method, KTEPWrapperMethod)
|
||||
):
|
||||
# fused in biased_grouped_topk so we can skip here
|
||||
final_hidden_states *= self.routed_scaling_factor
|
||||
|
||||
if (
|
||||
@@ -1705,22 +1754,12 @@ class DeepseekV2MoE(nn.Module):
|
||||
):
|
||||
torch.cuda.current_stream().wait_event(shared_event)
|
||||
|
||||
if shared_output is not None:
|
||||
x = shared_output
|
||||
# aiter moe call will handle routed_scaling_factor in the function
|
||||
# so add _use_aiter condition to eliminate to use self.routed_scaling_factor in add_ call
|
||||
if self.experts.should_fuse_routed_scaling_factor_in_topk or _use_aiter:
|
||||
x.add_(final_hidden_states)
|
||||
else:
|
||||
x.add_(final_hidden_states, alpha=self.routed_scaling_factor)
|
||||
final_hidden_states = x
|
||||
else:
|
||||
if not (
|
||||
self.experts.should_fuse_routed_scaling_factor_in_topk or _use_aiter
|
||||
):
|
||||
final_hidden_states *= self.routed_scaling_factor
|
||||
|
||||
return final_hidden_states
|
||||
return _scale_routed_and_add_shared(
|
||||
final_hidden_states,
|
||||
shared_output,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
fused_in_topk=self.routed_scaling_factor_fused_in_topk,
|
||||
)
|
||||
|
||||
def _forward_shared_experts(
|
||||
self,
|
||||
@@ -1987,20 +2026,12 @@ class DeepseekV2MoE(nn.Module):
|
||||
num_tokens = state.pop("num_tokens")
|
||||
final_hidden_states = final_hidden_states[:num_tokens]
|
||||
|
||||
if (shared_output := state.pop("shared_output")) is not None:
|
||||
x = shared_output
|
||||
if _use_aiter:
|
||||
x.add_(final_hidden_states)
|
||||
else:
|
||||
x.add_(final_hidden_states, alpha=self.routed_scaling_factor)
|
||||
final_hidden_states = x
|
||||
elif _use_aiter:
|
||||
# fused in aiter_biased_grouped_topk so we can skip here
|
||||
pass
|
||||
else:
|
||||
final_hidden_states *= self.routed_scaling_factor
|
||||
|
||||
state.hidden_states_mlp_output = final_hidden_states
|
||||
state.hidden_states_mlp_output = _scale_routed_and_add_shared(
|
||||
final_hidden_states,
|
||||
state.pop("shared_output"),
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
fused_in_topk=self.routed_scaling_factor_fused_in_topk,
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV2AttentionMLA(
|
||||
|
||||
@@ -79,7 +79,7 @@ from sglang.srt.layers.moe import (
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.moe.topk import TopK
|
||||
from sglang.srt.layers.moe.topk import TopK, is_routed_scaling_factor_fused_in_topk
|
||||
from sglang.srt.layers.moe.utils import is_shared_experts_fusion_disabled
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.quantization.fp8_utils import (
|
||||
@@ -484,14 +484,15 @@ class Dots3MoE(nn.Module):
|
||||
topk_output=topk_output,
|
||||
)
|
||||
|
||||
fused_in_topk = is_routed_scaling_factor_fused_in_topk(self.topk.topk_config)
|
||||
if shared_output is not None:
|
||||
x = shared_output
|
||||
if self.experts.should_fuse_routed_scaling_factor_in_topk:
|
||||
if fused_in_topk:
|
||||
x.add_(final_hidden_states)
|
||||
else:
|
||||
x.add_(final_hidden_states, alpha=self.routed_scaling_factor)
|
||||
final_hidden_states = x
|
||||
elif not self.experts.should_fuse_routed_scaling_factor_in_topk:
|
||||
elif not fused_in_topk:
|
||||
final_hidden_states *= self.routed_scaling_factor
|
||||
|
||||
return final_hidden_states
|
||||
|
||||
@@ -65,7 +65,7 @@ from sglang.srt.layers.moe import (
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.moe.kt_ep_wrapper import KTEPWrapperMethod
|
||||
from sglang.srt.layers.moe.topk import TopK
|
||||
from sglang.srt.layers.moe.topk import TopK, is_routed_scaling_factor_fused_in_topk
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
RoutingMethodType,
|
||||
filter_moe_weight_param_global_expert,
|
||||
@@ -667,16 +667,16 @@ class Glm4MoeSparseMoeBlock(nn.Module):
|
||||
if enable_npu_dual_stream:
|
||||
wait_share_stream()
|
||||
|
||||
fused_in_topk = is_routed_scaling_factor_fused_in_topk(self.topk.topk_config)
|
||||
if shared_output is not None:
|
||||
x = shared_output
|
||||
if self.experts.should_fuse_routed_scaling_factor_in_topk:
|
||||
if fused_in_topk:
|
||||
x.add_(final_hidden_states)
|
||||
else:
|
||||
x.add_(final_hidden_states, alpha=self.routed_scaling_factor)
|
||||
final_hidden_states = x
|
||||
else:
|
||||
if not self.experts.should_fuse_routed_scaling_factor_in_topk:
|
||||
final_hidden_states *= self.routed_scaling_factor
|
||||
elif not fused_in_topk:
|
||||
final_hidden_states *= self.routed_scaling_factor
|
||||
|
||||
return final_hidden_states
|
||||
|
||||
|
||||
@@ -59,7 +59,11 @@ from sglang.srt.layers.moe import (
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.moe.kt_ep_wrapper import KTEPWrapperMethod
|
||||
from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat
|
||||
from sglang.srt.layers.moe.topk import (
|
||||
TopK,
|
||||
TopKOutputFormat,
|
||||
is_routed_scaling_factor_fused_in_topk,
|
||||
)
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
filter_moe_weight_param_global_expert,
|
||||
is_shared_experts_fusion_disabled,
|
||||
@@ -388,16 +392,16 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module):
|
||||
topk_output=topk_output,
|
||||
)
|
||||
|
||||
fused_in_topk = is_routed_scaling_factor_fused_in_topk(self.topk.topk_config)
|
||||
if shared_output is not None:
|
||||
x = shared_output
|
||||
if self.experts.should_fuse_routed_scaling_factor_in_topk:
|
||||
if fused_in_topk:
|
||||
x.add_(final_hidden_states)
|
||||
else:
|
||||
x.add_(final_hidden_states, alpha=self.routed_scaling_factor)
|
||||
final_hidden_states = x
|
||||
else:
|
||||
if not self.experts.should_fuse_routed_scaling_factor_in_topk:
|
||||
final_hidden_states *= self.routed_scaling_factor
|
||||
elif not fused_in_topk:
|
||||
final_hidden_states *= self.routed_scaling_factor
|
||||
|
||||
return final_hidden_states
|
||||
|
||||
|
||||
@@ -149,12 +149,6 @@ def _finish_vision_topk(
|
||||
expert_location_dispatch_info=expert_location_dispatch_info,
|
||||
padded_rows_masked=padded_rows_masked,
|
||||
)
|
||||
if (
|
||||
config.num_fused_shared_experts
|
||||
and config.apply_routed_scaling_factor_on_output
|
||||
):
|
||||
# MegaMoE applies no post-expert scale when it is already in top-k.
|
||||
weights[:, -config.num_fused_shared_experts :] = 1.0
|
||||
if recorder_ids is not None:
|
||||
get_global_expert_distribution_recorder().on_select_experts(
|
||||
topk_ids=recorder_ids
|
||||
|
||||
Reference in New Issue
Block a user