From 3a059de94c1642b35441cb16cd365b07fbae6990 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Mon, 19 Oct 2026 16:54:27 -0700 Subject: [PATCH] [Fix] moe: Count routed_scaling_factor once when top-k folds it (#44901) --- python/sglang/srt/layers/moe/hash_topk.py | 1 + python/sglang/srt/layers/moe/mega_moe.py | 2 +- .../sglang/srt/layers/moe/mega_moe_flydsl.py | 4 +- python/sglang/srt/layers/moe/topk.py | 118 +++--- python/sglang/srt/layers/moe/waterfill.py | 6 +- .../model_runner_components/moe_ep_setup.py | 13 +- python/sglang/srt/models/bailing_moe_v3.py | 6 +- python/sglang/srt/models/deepseek_v2.py | 99 +++-- .../srt/models/dots3_common/modeling.py | 7 +- python/sglang/srt/models/glm4_moe.py | 10 +- python/sglang/srt/models/glm4_moe_lite.py | 14 +- .../sglang/srt/multimodal/dsv41/vl_routing.py | 6 - .../kernels/ops/moe/test_jit_grouped_topk.py | 30 ++ ...used_append_remap_per_rank_shared_slots.py | 78 ++-- test/registered/moe/test_hash_topk.py | 7 +- .../moe/test_fused_shared_expert_scaling.py | 343 +++++++++++++++++- .../moe/test_mega_moe_decode_capacity.py | 1 + .../layers/moe/test_mega_moe_deepgemm_api.py | 2 + .../unit/models/test_deepseek_v4_amd_tp4.py | 1 + 19 files changed, 560 insertions(+), 188 deletions(-) diff --git a/python/sglang/srt/layers/moe/hash_topk.py b/python/sglang/srt/layers/moe/hash_topk.py index d32c600ff2..3a037fecbb 100644 --- a/python/sglang/srt/layers/moe/hash_topk.py +++ b/python/sglang/srt/layers/moe/hash_topk.py @@ -288,6 +288,7 @@ class HashTopK(nn.Module): top_k=self.topk, num_fused_shared_experts=num_fused_shared_experts, routed_scaling_factor=self.routed_scaling_factor, + apply_routed_scaling_factor_on_output=self.apply_routed_scaling_factor_on_output, ), ) else: diff --git a/python/sglang/srt/layers/moe/mega_moe.py b/python/sglang/srt/layers/moe/mega_moe.py index cedb344db0..75daa18295 100644 --- a/python/sglang/srt/layers/moe/mega_moe.py +++ b/python/sglang/srt/layers/moe/mega_moe.py @@ -306,7 +306,7 @@ def _run_mega_routed( activation_clamp=moe.experts.moe_runner_config.swiglu_limit, routed_scaling_factor=( 1.0 - if moe.experts.should_fuse_routed_scaling_factor_in_topk + if moe.routed_scaling_factor_fused_in_topk else float(moe.routed_scaling_factor) ), num_shared_experts=( diff --git a/python/sglang/srt/layers/moe/mega_moe_flydsl.py b/python/sglang/srt/layers/moe/mega_moe_flydsl.py index 96f37d4510..9ac73e2652 100644 --- a/python/sglang/srt/layers/moe/mega_moe_flydsl.py +++ b/python/sglang/srt/layers/moe/mega_moe_flydsl.py @@ -373,8 +373,6 @@ def _run_mega_routed( output = mega.forward(x_in, topk_weights, topk_ids, slice_output=False)[:num_tokens] - from sglang.srt.models.deepseek_common.utils import _use_aiter - - if not (moe.experts.should_fuse_routed_scaling_factor_in_topk or _use_aiter): + if not moe.routed_scaling_factor_fused_in_topk: output.mul_(moe.routed_scaling_factor) return output diff --git a/python/sglang/srt/layers/moe/topk.py b/python/sglang/srt/layers/moe/topk.py index 8d90c269ff..17c2c2049b 100644 --- a/python/sglang/srt/layers/moe/topk.py +++ b/python/sglang/srt/layers/moe/topk.py @@ -249,7 +249,7 @@ if _is_musa: @dataclass class TopKConfig: - top_k: int + top_k: int # Ordinary output width: routed experts plus shared TopK slots. use_grouped_topk: bool = False topk_group: Optional[int] = None num_expert_group: Optional[int] = None @@ -260,6 +260,8 @@ class TopKConfig: torch_native: bool = False routed_scaling_factor: Optional[float] = None apply_routed_scaling_factor_on_output: bool = False + # Absolute appended weight for global AITER slots; a multiplier on the + # router's shared weights on the other standard path. fused_shared_experts_scaling_factor: Optional[float] = None output_format: Optional[TopKOutputFormat] = None scoring_func: str = "softmax" @@ -585,9 +587,9 @@ class TopK(BaseFusedOp): per-layer EPLB state are keyed by it. --num_fused_shared_experts: num of shared experts, can be activate both in TP or EP mode. --routed_scaling_factor: the scaling factor for routed experts in topk_weights. - --fused_shared_experts_scaling_factor: scaling factor applied to the fused shared experts' - topk weight (models pass 1/ep_size under standard EP, where the per-rank shared-expert - outputs are all-reduced). + --fused_shared_experts_scaling_factor: absolute appended weight on the global AITER + path; otherwise multiplies the router's existing shared weight under standard EP. + Models include 1/ep_size when the per-rank shared-expert outputs are all-reduced. """ def __init__( @@ -1655,10 +1657,6 @@ def biased_grouped_topk_impl( dtype=topk_ids.dtype, device=topk_ids.device, ) - if routed_scaling_factor is not None: - topk_weights[:, -1] = ( - topk_weights[:, :-1].sum(dim=-1) / routed_scaling_factor - ) if renormalize: # fp32 like the reference gate (the epsilon is not representable in @@ -1670,8 +1668,14 @@ def biased_grouped_topk_impl( else topk_weights[:, :-1].sum(dim=-1, keepdim=True, dtype=torch.float32) ) topk_weights = topk_weights / (topk_weights_sum + _RENORMALIZE_SUM_EPSILON) - if apply_routed_scaling_factor_on_output: - topk_weights *= routed_scaling_factor + if apply_routed_scaling_factor_on_output: + topk_weights *= routed_scaling_factor + + if num_fused_shared_experts: + topk_weights[:, -1] = fused_shared_expert_weight( + routed_scaling_factor=routed_scaling_factor, + fused_in_topk=bool(apply_routed_scaling_factor_on_output), + ) topk_weights, topk_ids = topk_weights.to(torch.float32), topk_ids.to(torch.int32) @@ -1814,9 +1818,7 @@ def biased_grouped_topk_gpu( # The kernel wants the total width; select_experts passes a routed-only # topk on the aiter path only (`num_routed_topk if _use_aiter else top_k`). # - # True, not the caller's flag: an aiter runner skips the post-MoE multiply, - # so the routed weights must carry routed_scaling_factor and the shared - # slot must be 1.0. That flag describes the runner, not this kernel. + # AITER folds this route even when no shared slots are requested. return jit_grouped_gate( gating_output, correction_bias.to(dtype=torch.float32), @@ -1828,9 +1830,7 @@ def biased_grouped_topk_gpu( routed_scaling_factor if routed_scaling_factor is not None else 1.0 ), apply_routed_scaling_factor_on_output=( - True - if (_use_aiter and num_fused_shared_experts > 0) - else bool(apply_routed_scaling_factor_on_output) + _use_aiter or bool(apply_routed_scaling_factor_on_output) ), num_expert_group=num_expert_group, topk_group=topk_group, @@ -2208,6 +2208,31 @@ else: fused_topk_native = fused_topk_torch_native +def is_routed_scaling_factor_fused_in_topk(topk_config: TopKConfig) -> bool: + """Whether routing already applied the scale that the MoE epilogue must skip.""" + if topk_config.apply_routed_scaling_factor_on_output: + return True + if not _use_aiter: + return False + # AITER always folds biased grouped and ungrouped sqrtsoftplus routing. + if topk_config.use_grouped_topk: + return topk_config.correction_bias is not None + return ( + topk_config.scoring_func == "sqrtsoftplus" + and topk_config.custom_routing_function is None + ) + + +def fused_shared_expert_weight( + *, routed_scaling_factor: Optional[float], fused_in_topk: bool +) -> float: + """Shared-slot weight that cancels any later routed-output scaling.""" + if fused_in_topk or not routed_scaling_factor: + return 1.0 + # The routed output, shared column included, is scaled by the factor later. + return 1.0 / routed_scaling_factor + + def remap_topk_for_per_rank_shared_slots( topk_ids: torch.Tensor, topk_weights: torch.Tensor, @@ -2215,16 +2240,7 @@ def remap_topk_for_per_rank_shared_slots( num_physical_routed_experts: int, topk_config: TopKConfig, ) -> tuple[torch.Tensor, torch.Tensor]: - """Remap TopK IDs to a per-rank shared-slot layout. - - DeepEP and MegaMoE dispatch need each rank's shared expert at a unique ID - so tokens route to the correct rank. The layout is ordered by rank: - [rank0 routed..., rank0 shared, rank1 routed..., rank1 shared, ...]. - - Routed IDs: e -> e + (e // num_local_routed) * num_fused_shared_experts - Shared IDs: ep_rank * num_local_experts + num_local_routed - Shared weight: 1.0 on the aiter path, else 1/routed_scaling_factor (see below). - """ + """Map routed and shared columns into rank-major expert blocks.""" if topk_ids.shape[0] == 0: return topk_ids, topk_weights @@ -2253,34 +2269,10 @@ def remap_topk_for_per_rank_shared_slots( + torch.arange(num_fused_shared_experts, device=topk_ids.device) ) - # Override the fused shared expert's weight so its net contribution is 1.0x. - # - # The correct value depends on whether routed_scaling_factor is applied to - # the MoE output AFTER the experts run, or already folded into the routed - # topk weights BEFORE dispatch: - # - # * Post-MoE scaling path (default): DeepseekV2MoE.forward_deepep later - # multiplies the whole MoE output by routed_scaling_factor, so the shared - # weight must be 1/routed_scaling_factor for (1/rsf) * rsf = 1.0. - # * aiter (HIP) path: aiter_biased_grouped_topk folds routed_scaling_factor - # into each routed topk weight, and forward_deepep SKIPS the post-MoE - # multiply for _use_aiter (see its `not (... or _use_aiter)` guard). The - # shared weight must therefore be 1.0 -- applying 1/rsf here would - # under-weight the always-on shared expert by routed_scaling_factor and - # corrupt every MoE layer. - # - # NOTE: forward_deepep also skips the post-MoE multiply for the non-aiter - # families where routed_scaling_factor is pre-folded in topk - # (should_fuse_routed_scaling_factor_in_topk / apply_routed_scaling_factor_on_output: - # ModelOpt NVFP4, cutlass/trtllm-routed fp8), so those would likewise need a - # 1.0 shared weight. This fix is deliberately scoped to the aiter path (the - # one validated on AMD MI355X); those other backends are left at their - # existing behavior and can be addressed by their maintainers. - routed_scaling_factor = topk_config.routed_scaling_factor - if _use_aiter: - topk_weights[:, -num_fused_shared_experts:] = 1.0 - elif routed_scaling_factor is not None and routed_scaling_factor != 0: - topk_weights[:, -num_fused_shared_experts:] = 1.0 / routed_scaling_factor + topk_weights[:, -num_fused_shared_experts:] = fused_shared_expert_weight( + routed_scaling_factor=topk_config.routed_scaling_factor, + fused_in_topk=is_routed_scaling_factor_fused_in_topk(topk_config), + ) return topk_ids, topk_weights @@ -2450,14 +2442,6 @@ def _post_process_topk_ids( # fused_append_shared_experts() + eager per-rank shared-slot remap pair, # collapsing ~6 launch-bound elementwise kernels/layer (div_floor / add / # arange / fill / copy) into the one append kernel that already runs. - # - # Shared weight is 1.0 here because this branch is aiter-only: - # aiter_biased_grouped_topk folds routed_scaling_factor into the routed - # weights and forward_deepep skips the post-MoE multiply for _use_aiter, - # so the always-on shared expert must contribute 1.0x. (The eager - # per-rank shared-slot remap instead sets shared weight to - # 1/routed_scaling_factor to compensate a post-MoE scale that the aiter - # path does not apply; see PR #28237.) num_physical_routed_experts = ( expert_location_dispatch_info.num_physical_experts if expert_location_dispatch_info is not None @@ -2478,7 +2462,10 @@ def _post_process_topk_ids( topk_ids, topk_weights, num_fused_shared_experts, - 1.0, # shared-expert weight on the aiter path + fused_shared_expert_weight( + routed_scaling_factor=topk_config.routed_scaling_factor, + fused_in_topk=is_routed_scaling_factor_fused_in_topk(topk_config), + ), shared_id_base, num_local_routed, num_token_non_padded=( @@ -2656,7 +2643,10 @@ def select_experts( topk_group=topk_group or 1, num_fused_shared_experts=num_fused_shared_experts, routed_scaling_factor=routed_scaling_factor, - apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output, + # Per-token bias bypasses AITER but must retain its folding policy. + apply_routed_scaling_factor_on_output=is_routed_scaling_factor_fused_in_topk( + topk_config + ), ) elif use_grouped_topk: assert topk_group is not None diff --git a/python/sglang/srt/layers/moe/waterfill.py b/python/sglang/srt/layers/moe/waterfill.py index aced3aa7bb..24e2d7c4e2 100644 --- a/python/sglang/srt/layers/moe/waterfill.py +++ b/python/sglang/srt/layers/moe/waterfill.py @@ -77,16 +77,14 @@ class WaterfillBalancer: world_size: int, rank: int, layer_id: int, - routed_scaling_factor: float = 1.0, + shared_weight: float, ): self.num_routed_experts = num_routed_experts self.world_size = world_size self.rank = rank self.layer_id = layer_id self.old_experts_per_rank = num_routed_experts // world_size - self.shared_weight = ( - 1.0 / routed_scaling_factor if routed_scaling_factor != 0 else 1.0 - ) + self.shared_weight = shared_weight self._counts_buf: Optional[Tensor] = None self.use_static_waterfill = not envs.SGLANG_DISABLE_STATIC_WATERFILL.get() diff --git a/python/sglang/srt/model_executor/model_runner_components/moe_ep_setup.py b/python/sglang/srt/model_executor/model_runner_components/moe_ep_setup.py index 9f6a047f46..73087bbb1b 100644 --- a/python/sglang/srt/model_executor/model_runner_components/moe_ep_setup.py +++ b/python/sglang/srt/model_executor/model_runner_components/moe_ep_setup.py @@ -12,7 +12,11 @@ from sglang.srt.eplb.lplb_solver import ( set_global_lplb_solver, ) from sglang.srt.layers.moe.hash_topk import HashTopK -from sglang.srt.layers.moe.topk import TopK +from sglang.srt.layers.moe.topk import ( + TopK, + fused_shared_expert_weight, + is_routed_scaling_factor_fused_in_topk, +) from sglang.srt.runtime_context import get_exec, get_parallel from sglang.srt.utils import get_bool_env_var, is_hip, log_info_on_rank0 @@ -52,15 +56,18 @@ def prepare_moe_topk(*, model, model_config: ModelConfig) -> None: ) if isinstance(module, TopK): routed_scaling_factor = module.topk_config.routed_scaling_factor + fused_in_topk = is_routed_scaling_factor_fused_in_topk(module.topk_config) else: routed_scaling_factor = module.routed_scaling_factor + fused_in_topk = module.apply_routed_scaling_factor_on_output module.waterfill_balancer = balancer_cls( num_routed_experts=num_physical_routed_experts, world_size=parallel.moe_ep_size, rank=parallel.moe_ep_rank, layer_id=module.layer_id, - routed_scaling_factor=( - routed_scaling_factor if routed_scaling_factor is not None else 1.0 + shared_weight=fused_shared_expert_weight( + routed_scaling_factor=routed_scaling_factor, + fused_in_topk=fused_in_topk, ), ) num_prepared += 1 diff --git a/python/sglang/srt/models/bailing_moe_v3.py b/python/sglang/srt/models/bailing_moe_v3.py index 73d4292ed6..9f0989aaa9 100644 --- a/python/sglang/srt/models/bailing_moe_v3.py +++ b/python/sglang/srt/models/bailing_moe_v3.py @@ -47,7 +47,7 @@ from sglang.srt.layers.moe import ( ) from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE -from sglang.srt.layers.moe.topk import TopK +from sglang.srt.layers.moe.topk import TopK, is_routed_scaling_factor_fused_in_topk from sglang.srt.layers.moe.utils import is_shared_experts_fusion_disabled from sglang.srt.layers.multi_gate import ( create_multi_gate_mm_indices, @@ -604,11 +604,11 @@ class BailingMoE(nn.Module): # Whether to apply routed_scaling_factor at model layer. # For A2A MoE paths (e.g., DeepEP), the runner/post_permute does not apply it, - # so apply it here unless the selected runner fuses it into TopK weights. + # so apply it here unless top-k already folded it into the routed weights. # This scales only routed expert output; shared experts are added unscaled. self._apply_routed_scaling_factor_on_output = ( self._enable_a2a_moe - and not self.experts.should_fuse_routed_scaling_factor_in_topk + and not is_routed_scaling_factor_fused_in_topk(self.topk.topk_config) and self.routed_scaling_factor is not None and self.routed_scaling_factor != 1.0 ) diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 4c9ec84f60..bbde545b8d 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -118,7 +118,13 @@ from sglang.srt.layers.moe.token_dispatcher.base import ( CombineInput, DispatchOutput, ) -from sglang.srt.layers.moe.topk import BypassedTopKOutput, TopK, TopKOutputFormat +from sglang.srt.layers.moe.topk import ( + BypassedTopKOutput, + TopK, + TopKOutputFormat, + fused_shared_expert_weight, + is_routed_scaling_factor_fused_in_topk, +) from sglang.srt.layers.moe.utils import ( RoutingMethodType, adds_replicated_output_to_partial, @@ -625,6 +631,23 @@ class MergedMoEOutput(MoEOutput): return self.routed +def _scale_routed_and_add_shared( + routed: torch.Tensor, + shared: Optional[torch.Tensor], + *, + routed_scaling_factor: float, + fused_in_topk: bool, +) -> torch.Tensor: + """Apply any outstanding routed scale before adding the shared output.""" + if shared is not None: + if fused_in_topk: + return shared.add_(routed) + return shared.add_(routed, alpha=routed_scaling_factor) + if not fused_in_topk: + routed *= routed_scaling_factor + return routed + + class DeepseekV2MoE(nn.Module): def __init__( self, @@ -748,6 +771,10 @@ class DeepseekV2MoE(nn.Module): apply_routed_scaling_factor_on_output=self.experts.should_fuse_routed_scaling_factor_in_topk, layer_id=self.layer_id, ) + # Hash routing folds the factor only when its flag asks for it. + self.routed_scaling_factor_fused_in_topk = ( + self.topk.apply_routed_scaling_factor_on_output + ) else: # Default: grouped noaux_tc top-k. Covers V3/V3.2/GLM-5/Glm4MoeLite. topk_kwargs = dict( @@ -785,6 +812,8 @@ class DeepseekV2MoE(nn.Module): getattr(config, "model_type", None) == "deepseek_v41" ), is_fp4_experts=getattr(quant_config, "is_fp4_experts", False), + # aiter's topk_gating folds the factor regardless; the flag makes + # the other routes (e.g. the vision routing fallback) fold too. apply_routed_scaling_factor_on_output=( True if _use_aiter @@ -792,6 +821,27 @@ class DeepseekV2MoE(nn.Module): ), ) self.topk = TopK(**topk_kwargs) + self.routed_scaling_factor_fused_in_topk = ( + is_routed_scaling_factor_fused_in_topk(self.topk.topk_config) + ) + if ( + _use_aiter + and self.num_fused_shared_experts > 0 + and not _uses_per_rank_shared_slots + and not self.routed_scaling_factor_fused_in_topk + ): + # Compensate for both post-MoE scaling and EP replication. + self.topk.topk_config.fused_shared_experts_scaling_factor = ( + fused_shared_expert_weight( + routed_scaling_factor=self.routed_scaling_factor, + fused_in_topk=self.routed_scaling_factor_fused_in_topk, + ) + * ( + fused_shared_experts_scaling_factor + if fused_shared_experts_scaling_factor is not None + else 1.0 + ) + ) self.shared_experts_is_int8 = False self.shared_experts_is_fp8 = False @@ -1191,7 +1241,7 @@ class DeepseekV2MoE(nn.Module): if ( not _is_cuda and not _is_musa - and not _use_aiter + and not (_use_aiter and self.routed_scaling_factor_fused_in_topk) or isinstance(self.experts.quant_method, KTEPWrapperMethod) ): final_hidden_states *= self.routed_scaling_factor @@ -1383,10 +1433,9 @@ class DeepseekV2MoE(nn.Module): not _is_cuda and not _is_musa and not _is_xpu - and not _use_aiter + and not (_use_aiter and self.routed_scaling_factor_fused_in_topk) or isinstance(self.experts.quant_method, KTEPWrapperMethod) ): - # fused in biased_grouped_topk so we can skip here final_hidden_states *= self.routed_scaling_factor if ( @@ -1705,22 +1754,12 @@ class DeepseekV2MoE(nn.Module): ): torch.cuda.current_stream().wait_event(shared_event) - if shared_output is not None: - x = shared_output - # aiter moe call will handle routed_scaling_factor in the function - # so add _use_aiter condition to eliminate to use self.routed_scaling_factor in add_ call - if self.experts.should_fuse_routed_scaling_factor_in_topk or _use_aiter: - x.add_(final_hidden_states) - else: - x.add_(final_hidden_states, alpha=self.routed_scaling_factor) - final_hidden_states = x - else: - if not ( - self.experts.should_fuse_routed_scaling_factor_in_topk or _use_aiter - ): - final_hidden_states *= self.routed_scaling_factor - - return final_hidden_states + return _scale_routed_and_add_shared( + final_hidden_states, + shared_output, + routed_scaling_factor=self.routed_scaling_factor, + fused_in_topk=self.routed_scaling_factor_fused_in_topk, + ) def _forward_shared_experts( self, @@ -1987,20 +2026,12 @@ class DeepseekV2MoE(nn.Module): num_tokens = state.pop("num_tokens") final_hidden_states = final_hidden_states[:num_tokens] - if (shared_output := state.pop("shared_output")) is not None: - x = shared_output - if _use_aiter: - x.add_(final_hidden_states) - else: - x.add_(final_hidden_states, alpha=self.routed_scaling_factor) - final_hidden_states = x - elif _use_aiter: - # fused in aiter_biased_grouped_topk so we can skip here - pass - else: - final_hidden_states *= self.routed_scaling_factor - - state.hidden_states_mlp_output = final_hidden_states + state.hidden_states_mlp_output = _scale_routed_and_add_shared( + final_hidden_states, + state.pop("shared_output"), + routed_scaling_factor=self.routed_scaling_factor, + fused_in_topk=self.routed_scaling_factor_fused_in_topk, + ) class DeepseekV2AttentionMLA( diff --git a/python/sglang/srt/models/dots3_common/modeling.py b/python/sglang/srt/models/dots3_common/modeling.py index db2cf3fe12..31b5e59f24 100644 --- a/python/sglang/srt/models/dots3_common/modeling.py +++ b/python/sglang/srt/models/dots3_common/modeling.py @@ -79,7 +79,7 @@ from sglang.srt.layers.moe import ( ) from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE -from sglang.srt.layers.moe.topk import TopK +from sglang.srt.layers.moe.topk import TopK, is_routed_scaling_factor_fused_in_topk from sglang.srt.layers.moe.utils import is_shared_experts_fusion_disabled from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.quantization.fp8_utils import ( @@ -484,14 +484,15 @@ class Dots3MoE(nn.Module): topk_output=topk_output, ) + fused_in_topk = is_routed_scaling_factor_fused_in_topk(self.topk.topk_config) if shared_output is not None: x = shared_output - if self.experts.should_fuse_routed_scaling_factor_in_topk: + if fused_in_topk: x.add_(final_hidden_states) else: x.add_(final_hidden_states, alpha=self.routed_scaling_factor) final_hidden_states = x - elif not self.experts.should_fuse_routed_scaling_factor_in_topk: + elif not fused_in_topk: final_hidden_states *= self.routed_scaling_factor return final_hidden_states diff --git a/python/sglang/srt/models/glm4_moe.py b/python/sglang/srt/models/glm4_moe.py index fc8ebbb43e..2b6ff441a0 100644 --- a/python/sglang/srt/models/glm4_moe.py +++ b/python/sglang/srt/models/glm4_moe.py @@ -65,7 +65,7 @@ from sglang.srt.layers.moe import ( from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.moe.kt_ep_wrapper import KTEPWrapperMethod -from sglang.srt.layers.moe.topk import TopK +from sglang.srt.layers.moe.topk import TopK, is_routed_scaling_factor_fused_in_topk from sglang.srt.layers.moe.utils import ( RoutingMethodType, filter_moe_weight_param_global_expert, @@ -667,16 +667,16 @@ class Glm4MoeSparseMoeBlock(nn.Module): if enable_npu_dual_stream: wait_share_stream() + fused_in_topk = is_routed_scaling_factor_fused_in_topk(self.topk.topk_config) if shared_output is not None: x = shared_output - if self.experts.should_fuse_routed_scaling_factor_in_topk: + if fused_in_topk: x.add_(final_hidden_states) else: x.add_(final_hidden_states, alpha=self.routed_scaling_factor) final_hidden_states = x - else: - if not self.experts.should_fuse_routed_scaling_factor_in_topk: - final_hidden_states *= self.routed_scaling_factor + elif not fused_in_topk: + final_hidden_states *= self.routed_scaling_factor return final_hidden_states diff --git a/python/sglang/srt/models/glm4_moe_lite.py b/python/sglang/srt/models/glm4_moe_lite.py index 01e196281a..7fcf314333 100644 --- a/python/sglang/srt/models/glm4_moe_lite.py +++ b/python/sglang/srt/models/glm4_moe_lite.py @@ -59,7 +59,11 @@ from sglang.srt.layers.moe import ( from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.moe.kt_ep_wrapper import KTEPWrapperMethod -from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat +from sglang.srt.layers.moe.topk import ( + TopK, + TopKOutputFormat, + is_routed_scaling_factor_fused_in_topk, +) from sglang.srt.layers.moe.utils import ( filter_moe_weight_param_global_expert, is_shared_experts_fusion_disabled, @@ -388,16 +392,16 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module): topk_output=topk_output, ) + fused_in_topk = is_routed_scaling_factor_fused_in_topk(self.topk.topk_config) if shared_output is not None: x = shared_output - if self.experts.should_fuse_routed_scaling_factor_in_topk: + if fused_in_topk: x.add_(final_hidden_states) else: x.add_(final_hidden_states, alpha=self.routed_scaling_factor) final_hidden_states = x - else: - if not self.experts.should_fuse_routed_scaling_factor_in_topk: - final_hidden_states *= self.routed_scaling_factor + elif not fused_in_topk: + final_hidden_states *= self.routed_scaling_factor return final_hidden_states diff --git a/python/sglang/srt/multimodal/dsv41/vl_routing.py b/python/sglang/srt/multimodal/dsv41/vl_routing.py index b03978d84e..ddd54f63d7 100644 --- a/python/sglang/srt/multimodal/dsv41/vl_routing.py +++ b/python/sglang/srt/multimodal/dsv41/vl_routing.py @@ -149,12 +149,6 @@ def _finish_vision_topk( expert_location_dispatch_info=expert_location_dispatch_info, padded_rows_masked=padded_rows_masked, ) - if ( - config.num_fused_shared_experts - and config.apply_routed_scaling_factor_on_output - ): - # MegaMoE applies no post-expert scale when it is already in top-k. - weights[:, -config.num_fused_shared_experts :] = 1.0 if recorder_ids is not None: get_global_expert_distribution_recorder().on_select_experts( topk_ids=recorder_ids diff --git a/test/registered/kernels/ops/moe/test_jit_grouped_topk.py b/test/registered/kernels/ops/moe/test_jit_grouped_topk.py index 08bb8866c0..e548ce6e93 100644 --- a/test/registered/kernels/ops/moe/test_jit_grouped_topk.py +++ b/test/registered/kernels/ops/moe/test_jit_grouped_topk.py @@ -182,5 +182,35 @@ def test_router_is_asked_for_the_total_width(monkeypatch, use_aiter): ) +@pytest.mark.parametrize("use_aiter", [True, False]) +def test_routed_weights_carry_the_factor_where_later_steps_skip_it( + monkeypatch, use_aiter +): + """AITER routing must fold the scale even without shared slots.""" + from sglang.srt.layers.moe import topk as topk_mod + + monkeypatch.setenv("SGLANG_OPT_USE_JIT_KERNEL_GROUPED_TOPK", "1") + monkeypatch.setattr(topk_mod, "_use_aiter", use_aiter, raising=False) + + hidden, bias, logits = _inputs(48) + weights, _ = topk_mod.biased_grouped_topk_gpu( + hidden_states=hidden, + gating_output=logits, + correction_bias=bias, + topk=TOPK_ROUTED, + renormalize=True, + num_expert_group=8, + topk_group=1, + num_fused_shared_experts=0, + routed_scaling_factor=SCALE, + apply_routed_scaling_factor_on_output=False, + ) + + want = SCALE if use_aiter else 1.0 + torch.testing.assert_close( + weights.sum(-1), torch.full_like(weights[:, 0], want), rtol=1e-5, atol=1e-5 + ) + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/moe/test_fused_append_remap_per_rank_shared_slots.py b/test/registered/moe/test_fused_append_remap_per_rank_shared_slots.py index b5090f5a14..3fef553852 100644 --- a/test/registered/moe/test_fused_append_remap_per_rank_shared_slots.py +++ b/test/registered/moe/test_fused_append_remap_per_rank_shared_slots.py @@ -17,7 +17,6 @@ from sglang.kernels.ops.moe.fused_moe_triton_kernels import ( ) from sglang.srt.layers.moe.topk import ( TopKConfig, - _use_aiter, remap_topk_for_per_rank_shared_slots, ) from sglang.srt.runtime_context import get_parallel @@ -178,53 +177,52 @@ class TestFusedAppendRemapPerRankSharedSlots(CustomTestCase): self.assertLess(max(routed_set), ep_size * num_local_experts) def test_equivalence_with_eager_append_then_remap(self): - """Fused kernel == append shared experts + per-rank shared-slot remap. - - The eager remap overwrites the shared weight: 1.0 on the aiter/HIP path - (routed_scaling_factor is pre-folded into the routed topk weights), else - 1/routed_scaling_factor. The fused kernel is invoked with that same value - so the two paths stay bit-identical (ids match regardless of scaling). - """ + """Fused and eager append agree for folding and nonfolding routes.""" rsf = 2.5 - scale_factor = 1.0 if _use_aiter else 1.0 / rsf - for m, k, npr, ep_size, ep_rank, s in self.CASES: - with self.subTest(m=m, k=k, npr=npr, ep_rank=ep_rank, s=s): - shared_id_base, num_local_routed = self._shared_id_base( - npr, ep_size, ep_rank, s - ) - topk_ids, topk_weights = self._make_inputs(m, k, npr) + for folded, scale_factor in ((False, 1.0 / rsf), (True, 1.0)): + for m, k, npr, ep_size, ep_rank, s in self.CASES: + with self.subTest( + m=m, k=k, npr=npr, ep_rank=ep_rank, s=s, folded=folded + ): + shared_id_base, num_local_routed = self._shared_id_base( + npr, ep_size, ep_rank, s + ) + topk_ids, topk_weights = self._make_inputs(m, k, npr) - fused_ids, fused_w = fused_append_remap_shared_experts_deepep( - topk_ids.clone(), - topk_weights.clone(), - s, - scale_factor, - shared_id_base, - num_local_routed, - ) - - with get_parallel().override(moe_ep_size=ep_size, moe_ep_rank=ep_rank): - eager_ids, eager_w = fused_append_shared_experts( + fused_ids, fused_w = fused_append_remap_shared_experts_deepep( topk_ids.clone(), topk_weights.clone(), s, scale_factor, - npr, # shared-expert base id (overwritten by the remap) - ) - eager_ids, eager_w = remap_topk_for_per_rank_shared_slots( - eager_ids, - eager_w, - s, - npr, - TopKConfig( - top_k=k, - num_fused_shared_experts=s, - routed_scaling_factor=rsf, - ), + shared_id_base, + num_local_routed, ) - self.assertTrue(torch.equal(fused_ids, eager_ids)) - self.assertTrue(torch.allclose(fused_w, eager_w)) + with get_parallel().override( + moe_ep_size=ep_size, moe_ep_rank=ep_rank + ): + eager_ids, eager_w = fused_append_shared_experts( + topk_ids.clone(), + topk_weights.clone(), + s, + scale_factor, + npr, # shared-expert base id (overwritten by the remap) + ) + eager_ids, eager_w = remap_topk_for_per_rank_shared_slots( + eager_ids, + eager_w, + s, + npr, + TopKConfig( + top_k=k, + num_fused_shared_experts=s, + routed_scaling_factor=rsf, + apply_routed_scaling_factor_on_output=folded, + ), + ) + + self.assertTrue(torch.equal(fused_ids, eager_ids)) + self.assertTrue(torch.allclose(fused_w, eager_w)) def test_shared_weight_is_one_on_aiter_path(self): """On the aiter path the always-on shared expert must contribute 1.0x.""" diff --git a/test/registered/moe/test_hash_topk.py b/test/registered/moe/test_hash_topk.py index a4546ca0d5..96c46d4d7b 100644 --- a/test/registered/moe/test_hash_topk.py +++ b/test/registered/moe/test_hash_topk.py @@ -6,6 +6,7 @@ import torch from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.moe import hash_topk as hash_topk_module +from sglang.srt.layers.moe import topk as topk_module from sglang.srt.layers.moe.hash_topk import HashTopK from sglang.srt.layers.moe.topk import ( StandardTopKOutput, @@ -23,7 +24,10 @@ def _set_dummy_server_args(): set_global_server_args_for_scheduler(ServerArgs(model_path="dummy")) -def test_hash_topk_remaps_per_rank_fused_shared_slots(monkeypatch): +@pytest.mark.parametrize("use_aiter", [False, True]) +def test_hash_topk_remaps_per_rank_fused_shared_slots(monkeypatch, use_aiter): + # Hash routing leaves scaling to the MoE epilogue, including on AITER. + monkeypatch.setattr(topk_module, "_use_aiter", use_aiter) monkeypatch.setattr(hash_topk_module, "_is_xpu", False) monkeypatch.setattr( hash_topk_module, "has_per_rank_fused_shared_slots", lambda *_args: True @@ -131,6 +135,7 @@ def test_deepep_empty_forward_does_not_append_shared_slot_twice(): experts=FakeExperts(), alt_stream=None, routed_scaling_factor=1.0, + routed_scaling_factor_fused_in_topk=True, ) hidden_states = torch.empty((0, 4), dtype=torch.float32) diff --git a/test/registered/unit/layers/moe/test_fused_shared_expert_scaling.py b/test/registered/unit/layers/moe/test_fused_shared_expert_scaling.py index 3faad1deb1..c48016400b 100644 --- a/test/registered/unit/layers/moe/test_fused_shared_expert_scaling.py +++ b/test/registered/unit/layers/moe/test_fused_shared_expert_scaling.py @@ -1,21 +1,8 @@ -"""Unit tests for fused shared-expert weight scaling. - -These tests pin the fused shared expert's topk weight contract on three paths: - - * aiter (HIP) per-rank-slot path: routed_scaling_factor is folded into the - routed weights and the post-MoE multiply is skipped, so the shared weight - must be 1.0 for a net 1.0x contribution. - * post-MoE scaling per-rank-slot path (default): the whole MoE output is - multiplied by routed_scaling_factor afterward, so the shared weight must - be 1/rsf. - * standard EP path (no per-rank slots): every rank computes the fused shared - expert and the outputs are all-reduced, so the model-supplied 1/ep_size - factor must be applied to the shared weight. -""" +"""Routed and shared contributions must each receive their scale exactly once.""" from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=11, suite="base-a-test-cpu") +register_cpu_ci(est_time=15, suite="base-a-test-cpu") import unittest from contextlib import nullcontext @@ -24,8 +11,15 @@ from unittest.mock import patch import torch +from sglang.srt.batch_overlap.operations import _StateDict +from sglang.srt.layers.moe import hash_topk as hash_topk_module from sglang.srt.layers.moe import topk as topk_module from sglang.srt.layers.moe.topk import TopK, TopKConfig +from sglang.srt.layers.moe.utils import MoeA2ABackend +from sglang.srt.model_executor.model_runner_components import moe_ep_setup +from sglang.srt.models import deepseek_v2 as deepseek_v2_module +from sglang.srt.models.deepseek_v2 import DeepseekV2MoE +from sglang.srt.runtime_context import get_context, get_flags, get_parallel from sglang.test.test_utils import CustomTestCase @@ -35,8 +29,12 @@ class TestFusedSharedExpertScaling(CustomTestCase): EP_SIZE = 8 EP_RANK = 0 ROUTED_SCALING_FACTOR = 2.5 + # DeepSeek-V3-style grouped routing with a correction bias, the route that + # aiter folds rsf into whatever the flag says. + ROUTING = {"use_grouped_topk": True, "correction_bias": torch.zeros(256)} + UNBIASED_ROUTING = {"use_grouped_topk": True, "correction_bias": None} - def _run_remap(self, *, use_aiter): + def _run_remap(self, *, use_aiter, folded_in_topk=False): # Layout: [routed, routed, routed, shared]; the trailing column is the # fused shared expert. The shared weight starts at a sentinel to prove # the function overwrites it. @@ -48,6 +46,8 @@ class TestFusedSharedExpertScaling(CustomTestCase): top_k=4, num_fused_shared_experts=1, routed_scaling_factor=self.ROUTED_SCALING_FACTOR, + apply_routed_scaling_factor_on_output=folded_in_topk, + **self.ROUTING, ) with ( @@ -84,6 +84,317 @@ class TestFusedSharedExpertScaling(CustomTestCase): shared_weight = self._run_remap(use_aiter=False) self.assertAlmostEqual(shared_weight, 1.0 / self.ROUTED_SCALING_FACTOR) + def _waterfill_shared_weight(self, *, use_aiter, folded_in_topk): + # Waterfill takes the shared column out of TopK and appends it itself. + topk = TopK.__new__(TopK) + torch.nn.Module.__init__(topk) + topk.layer_id = 0 + topk.enable_waterfill = True + topk.waterfill_balancer = None + topk.topk_config = TopKConfig( + top_k=8, + routed_scaling_factor=self.ROUTED_SCALING_FACTOR, + apply_routed_scaling_factor_on_output=folded_in_topk, + **self.ROUTING, + ) + model_config = SimpleNamespace( + hf_config=SimpleNamespace(n_routed_experts=self.NUM_PHYSICAL_ROUTED) + ) + with ( + patch.object(topk_module, "_use_aiter", use_aiter), + patch.object( + moe_ep_setup, + "get_exec", + return_value=SimpleNamespace( + moe=SimpleNamespace(ep_num_redundant_experts=0) + ), + ), + get_parallel().override(moe_ep_size=self.EP_SIZE, moe_ep_rank=self.EP_RANK), + ): + moe_ep_setup.prepare_moe_topk( + model=torch.nn.ModuleList([topk]), model_config=model_config + ) + return topk.waterfill_balancer.shared_weight + + def test_waterfill_shared_weight_matches_the_per_rank_slot(self): + # Both append paths must compensate for the same routing scale. + rsf = self.ROUTED_SCALING_FACTOR + cases = ( + # (use_aiter, folded_in_topk, routing, shared weight for a net 1.0x) + (False, False, self.ROUTING, 1.0 / rsf), + (False, True, self.ROUTING, 1.0), + (True, False, self.ROUTING, 1.0), + (True, False, self.UNBIASED_ROUTING, 1.0 / rsf), + ) + for use_aiter, folded_in_topk, routing, expected in cases: + with ( + self.subTest( + use_aiter=use_aiter, + folded_in_topk=folded_in_topk, + bias=routing["correction_bias"] is not None, + ), + patch.object(self, "ROUTING", routing), + ): + waterfill = self._waterfill_shared_weight( + use_aiter=use_aiter, folded_in_topk=folded_in_topk + ) + per_rank_slot = self._run_remap( + use_aiter=use_aiter, folded_in_topk=folded_in_topk + ) + self.assertAlmostEqual(waterfill, expected) + self.assertAlmostEqual(per_rank_slot, expected) + + def test_two_batch_overlap_epilogue_applies_factor_once(self): + """The overlap epilogue must apply the routed scale exactly once.""" + torch.manual_seed(0) + routed = torch.randn(4, 6) + shared = torch.randn(4, 6) + for folded_in_topk in (False, True): + for has_shared in (False, True): + with self.subTest(folded_in_topk=folded_in_topk, has_shared=has_shared): + moe = SimpleNamespace( + routed_scaling_factor=self.ROUTED_SCALING_FACTOR, + routed_scaling_factor_fused_in_topk=folded_in_topk, + ) + state = _StateDict() + state.hidden_states_after_combine = routed.clone() + state.shared_output = shared.clone() if has_shared else None + with get_flags().moe.override(a2a_backend=MoeA2ABackend.DEEPEP): + DeepseekV2MoE.op_output(moe, state) + + expected = routed * ( + 1.0 if folded_in_topk else self.ROUTED_SCALING_FACTOR + ) + if has_shared: + expected = expected + shared + torch.testing.assert_close(state.hidden_states_mlp_output, expected) + + def _build_deepseek_moe( + self, + *, + folded_in_topk, + use_aiter, + correction_bias=True, + deepseek_v4=False, + hash_layer=False, + fused_shared=0, + ep_size=1, + ): + config = SimpleNamespace( + routed_scaling_factor=self.ROUTED_SCALING_FACTOR, + n_shared_experts=fused_shared or None, + n_routed_experts=16, + num_experts_per_tok=2, + hidden_act="silu", + hidden_size=8, + moe_intermediate_size=8, + norm_topk_prob=True, + n_group=4, + topk_group=2, + scoring_func="sqrtsoftplus" if deepseek_v4 else "sigmoid", + num_hash_layers=1 if hash_layer else 0, + vocab_size=4, + ) + gate = SimpleNamespace( + e_score_correction_bias=torch.zeros(16) if correction_bias else None + ) + experts = SimpleNamespace( + should_fuse_routed_scaling_factor_in_topk=folded_in_topk + ) + exec_config = SimpleNamespace( + moe=SimpleNamespace(enable_waterfill=False, ep_num_redundant_experts=0) + ) + module = deepseek_v2_module + with ( + patch.object(module, "_use_aiter", use_aiter), + patch.object(topk_module, "_use_aiter", use_aiter), + patch.object( + module, + "get_parallel", + return_value=SimpleNamespace(tp_size=1, moe_ep_size=ep_size), + ), + patch.object(module, "get_exec", return_value=exec_config), + patch.object(topk_module, "get_exec", return_value=exec_config), + patch.object(hash_topk_module, "get_exec", return_value=exec_config), + patch.object( + module, + "is_shared_experts_fusion_disabled", + return_value=not fused_shared, + ), + patch.object(module, "MoEGate", return_value=gate), + patch.object( + module, "get_moe_impl_class", return_value=lambda **kwargs: experts + ), + patch.object( + module, + "SboFlags", + SimpleNamespace(fuse_shared_experts_inside_sbo=lambda: False), + ), + ): + return module.DeepseekV2MoE( + config=config, layer_id=0, is_deepseek_v4=deepseek_v4 + ) + + def test_deepseek_moe_takes_the_fold_from_the_router_it_builds(self): + """The MoE epilogue must follow the constructed router's folding policy.""" + cases = ( + # (folded_in_topk, use_aiter, route, routed weights carry rsf) + (False, False, {}, False), + (True, False, {}, True), + (False, True, {}, True), + (False, True, {"correction_bias": False}, False), + (False, True, {"deepseek_v4": True}, True), + (False, True, {"deepseek_v4": True, "hash_layer": True}, False), + (True, False, {"deepseek_v4": True, "hash_layer": True}, True), + ) + for folded_in_topk, use_aiter, route, expected in cases: + with self.subTest( + folded_in_topk=folded_in_topk, use_aiter=use_aiter, **route + ): + moe = self._build_deepseek_moe( + folded_in_topk=folded_in_topk, use_aiter=use_aiter, **route + ) + self.assertEqual(moe.routed_scaling_factor_fused_in_topk, expected) + + def test_dynamic_bias_preserves_the_final_routed_scale(self): + """Per-token bias must preserve the final scale of the static AITER route.""" + hidden = torch.ones(2, 4) + logits = torch.zeros(2, 16) + rsf = self.ROUTED_SCALING_FACTOR + + def aiter_route( + logits, bias, weights, ids, groups, selected, renormalize, scale + ): + weights.fill_((1.0 / weights.shape[1] if renormalize else 0.5) * scale) + ids.copy_(torch.arange(ids.shape[1]).expand_as(ids)) + + with ( + get_context().override_server_args(), + patch.object(topk_module, "_use_aiter", True), + patch.object(topk_module, "_is_cuda", False), + patch.object(topk_module, "_is_hip", False), + patch.object( + topk_module, "biased_grouped_topk", topk_module.biased_grouped_topk_gpu + ), + patch.object( + topk_module, "aiter_biased_grouped_topk", aiter_route, create=True + ), + ): + for renormalize in (False, True): + config = TopKConfig( + top_k=4, + use_grouped_topk=True, + num_expert_group=4, + topk_group=2, + renormalize=renormalize, + correction_bias=torch.zeros(16), + routed_scaling_factor=rsf, + scoring_func="sigmoid", + allow_routed_experts_capture=False, + ) + for dynamic in (False, True): + with self.subTest(renormalize=renormalize, dynamic=dynamic): + output = topk_module.select_experts( + hidden, + logits, + config, + dynamic_expert_bias=torch.zeros_like(logits) + if dynamic + else None, + ) + routed = output.topk_weights.sum(dim=-1, keepdim=True) + final = deepseek_v2_module._scale_routed_and_add_shared( + routed, + None, + routed_scaling_factor=rsf, + fused_in_topk=topk_module.is_routed_scaling_factor_fused_in_topk( + config + ), + ) + expected = torch.full_like( + final, rsf if renormalize else 2.0 * rsf + ) + torch.testing.assert_close(final, expected) + + def test_per_token_bias_keeps_the_shared_coefficient_independent(self): + """Dynamic routing must not scale shared output by the selected-score sum.""" + logits = torch.zeros(2, 16) + rsf = self.ROUTED_SCALING_FACTOR + for renormalize in (False, True): + for folded in (False, True): + with self.subTest(renormalize=renormalize, folded=folded): + weights, _ = topk_module.biased_grouped_topk_impl( + logits, + logits, + torch.zeros_like(logits), + topk=5, + renormalize=renormalize, + num_expert_group=4, + topk_group=2, + num_fused_shared_experts=1, + routed_scaling_factor=rsf, + apply_routed_scaling_factor_on_output=folded, + ) + shared = weights[:, -1] * (1.0 if folded else rsf) + torch.testing.assert_close(shared, torch.ones_like(shared)) + + def test_unbiased_aiter_global_shared_output_is_scaled_once(self): + """Restoring the routed scale must retain one shared contribution, including EP.""" + from sglang.kernels.ops.moe import fused_moe_triton_kernels + + def append(ids, weights, count, scale, base, **kwargs): + return ( + torch.cat((ids, ids.new_full((ids.shape[0], count), base)), dim=-1), + torch.cat( + (weights, weights.new_full((weights.shape[0], count), scale)), + dim=-1, + ), + ) + + with ( + get_context().override_server_args(), + patch.object(topk_module, "_use_aiter", True), + patch.object(topk_module, "_is_cuda", False), + patch.object(topk_module, "_is_hip", False), + patch.object(topk_module, "grouped_topk", topk_module.grouped_topk_gpu), + patch.object( + fused_moe_triton_kernels, "fused_append_shared_experts", append + ), + ): + for ep_size in (1, 4): + with self.subTest(ep_size=ep_size): + moe = self._build_deepseek_moe( + folded_in_topk=False, + use_aiter=True, + correction_bias=False, + fused_shared=1, + ep_size=ep_size, + ) + moe.topk.topk_config.allow_routed_experts_capture = False + output = topk_module.select_experts( + torch.ones(1, 8), + torch.zeros(1, 16), + moe.topk.topk_config, + ) + # Orthogonal expert outputs expose routed and shared coefficients. + combined = torch.stack( + ( + output.topk_weights[:, :-1].sum(dim=-1), + output.topk_weights[:, -1], + ), + dim=-1, + ) + final = deepseek_v2_module._scale_routed_and_add_shared( + combined, + None, + routed_scaling_factor=self.ROUTED_SCALING_FACTOR, + fused_in_topk=moe.routed_scaling_factor_fused_in_topk, + ) + torch.testing.assert_close( + final, + torch.tensor([[self.ROUTED_SCALING_FACTOR, 1.0 / ep_size]]), + ) + def _run_post_process_standard_path(self, *, scaling_factor): topk_ids = torch.tensor([[5, 40, 100, 256]], dtype=torch.int32) topk_weights = torch.tensor([[1.0, 0.5, 0.25, 1.0]], dtype=torch.float32) diff --git a/test/registered/unit/layers/moe/test_mega_moe_decode_capacity.py b/test/registered/unit/layers/moe/test_mega_moe_decode_capacity.py index aa6db48fcd..fb20cc786f 100644 --- a/test/registered/unit/layers/moe/test_mega_moe_decode_capacity.py +++ b/test/registered/unit/layers/moe/test_mega_moe_decode_capacity.py @@ -116,6 +116,7 @@ def runtime(monkeypatch): is_hash=False, layer_id=0, routed_scaling_factor=1.0, + routed_scaling_factor_fused_in_topk=True, ) forward_batch = ( None diff --git a/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py b/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py index 5965ea59ba..df1fc41c0d 100644 --- a/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py +++ b/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py @@ -234,6 +234,7 @@ class TestDeepGemmMegaMoeApi(CustomTestCase): num_fused_shared_experts=0, layer_id=0, routed_scaling_factor=1.0, + routed_scaling_factor_fused_in_topk=True, mega_shared_l1_weights=None, mega_shared_l2_weights=None, ) @@ -463,6 +464,7 @@ class TestDeepGemmMegaMoeApi(CustomTestCase): num_fused_shared_experts=fused_shared, layer_id=0, routed_scaling_factor=2.0, + routed_scaling_factor_fused_in_topk=scale_in_topk, mega_shared_l1_weights=None, mega_shared_l2_weights=None, ) diff --git a/test/registered/unit/models/test_deepseek_v4_amd_tp4.py b/test/registered/unit/models/test_deepseek_v4_amd_tp4.py index a717ebb2d1..9f6ce379fc 100644 --- a/test/registered/unit/models/test_deepseek_v4_amd_tp4.py +++ b/test/registered/unit/models/test_deepseek_v4_amd_tp4.py @@ -235,6 +235,7 @@ def _moe_stub(rank, *, dual, shared_tp1): _fuse_shared_experts_inside_sbo=False, num_fused_shared_experts=0, routed_scaling_factor=1.0, + routed_scaling_factor_fused_in_topk=True, experts=experts, gate=gate, alt_stream=torch.cuda.Stream(),