diff --git a/python/sglang/srt/hardware_backend/npu/moe/topk.py b/python/sglang/srt/hardware_backend/npu/moe/topk.py index 50b90aeb8a..117096bb50 100644 --- a/python/sglang/srt/hardware_backend/npu/moe/topk.py +++ b/python/sglang/srt/hardware_backend/npu/moe/topk.py @@ -61,7 +61,7 @@ def fused_topk_npu( if renormalize: topk_weights = l1_norm( topk_weights - if topk_config.num_fused_shared_experts == 0 + if topk_config.num_shared_topk_slots == 0 else topk_weights[:, :-1] ) topk_weights = topk_weights.to(torch.float32) diff --git a/python/sglang/srt/layers/moe/hash_topk.py b/python/sglang/srt/layers/moe/hash_topk.py index 7de6d941a4..852cf223c5 100644 --- a/python/sglang/srt/layers/moe/hash_topk.py +++ b/python/sglang/srt/layers/moe/hash_topk.py @@ -22,8 +22,11 @@ from sglang.srt.layers.moe.topk import ( _zero_topk_weights_padded_region, remap_topk_for_per_rank_shared_slots, ) -from sglang.srt.layers.moe.utils import has_per_rank_fused_shared_slots -from sglang.srt.runtime_context import get_exec +from sglang.srt.layers.moe.utils import ( + has_per_rank_fused_shared_slots, + shared_topk_slots, + waterfill_routes_shared_experts, +) from sglang.srt.utils import is_hip, is_npu, is_xpu logger = logging.getLogger(__name__) @@ -48,23 +51,20 @@ class HashTopK(nn.Module): super().__init__() self.layer_id = layer_id - self.enable_waterfill = ( - num_fused_shared_experts > 0 and get_exec().moe.enable_waterfill + self.enable_waterfill = waterfill_routes_shared_experts( + num_fused_shared_experts ) self.waterfill_balancer = None - - if self.enable_waterfill: - # Waterfill appends the shared expert after EPLB maps routed IDs. - topk -= num_fused_shared_experts - num_fused_shared_experts = 0 + num_routed_experts_per_token = topk - num_fused_shared_experts self.num_experts = num_experts - self.topk = topk + # Trailing columns the hash router fills with fused shared experts. + self.num_shared_topk_slots = shared_topk_slots(num_fused_shared_experts) + self.topk = num_routed_experts_per_token + self.num_shared_topk_slots self.routed_scaling_factor = routed_scaling_factor - self.num_fused_shared_experts = num_fused_shared_experts self.score_func = scoring_func self.tid2eid = nn.Parameter( - torch.empty(vocab_size, topk - num_fused_shared_experts, dtype=torch.int32), + torch.empty(vocab_size, num_routed_experts_per_token, dtype=torch.int32), requires_grad=False, ) self._init_default_tid2eid() @@ -72,7 +72,7 @@ class HashTopK(nn.Module): self.apply_routed_scaling_factor_on_output = ( apply_routed_scaling_factor_on_output ) - if apply_routed_scaling_factor_on_output and num_fused_shared_experts > 0: + if apply_routed_scaling_factor_on_output and self.num_shared_topk_slots > 0: raise NotImplementedError( "HashTopK + apply_routed_scaling_factor_on_output is not supported " "with fused shared experts; pass --disable-shared-experts-fusion." @@ -98,7 +98,7 @@ class HashTopK(nn.Module): def empty_topk_output( self, device: torch.device, *, layer_id: Optional[int] = None ): - topk = self.topk - self.num_fused_shared_experts + topk = self.topk - self.num_shared_topk_slots if layer_id is not None: from sglang.srt.eplb.lplb_solver import get_global_lplb_solver @@ -111,8 +111,8 @@ class HashTopK(nn.Module): topk_ids = torch.full((0, topk), -1, dtype=torch.int32, device=device) router_logits = torch.empty((0, topk), dtype=torch.float32, device=device) topk_output = StandardTopKOutput(topk_weights, topk_ids, router_logits) - if has_per_rank_fused_shared_slots(self.num_fused_shared_experts): - n = self.num_fused_shared_experts + if has_per_rank_fused_shared_slots(self.num_shared_topk_slots): + n = self.num_shared_topk_slots topk_output = topk_output._replace( topk_ids=topk_output.topk_ids.new_empty( (0, topk_output.topk_ids.shape[-1] + n) @@ -153,7 +153,7 @@ class HashTopK(nn.Module): (num_token, self.topk), dtype=scores.dtype, device=scores.device ) - if self.num_fused_shared_experts == 1: + if self.num_shared_topk_slots == 1: topk_ids[:, :-1] = self.tid2eid[input_ids] topk_weights[:, :-1] = scores.gather(1, topk_ids[:, :-1]) @@ -165,7 +165,7 @@ class HashTopK(nn.Module): topk_ids[:, -1] = torch.randint( low=self.num_experts, - high=self.num_experts + self.num_fused_shared_experts, + high=self.num_experts + self.num_shared_topk_slots, size=(num_token,), dtype=topk_ids.dtype, device=topk_ids.device, @@ -195,7 +195,7 @@ class HashTopK(nn.Module): num_tokens = router_logits.size(0) topk_routed = self.tid2eid.size(1) - topk_fused = topk_routed + self.num_fused_shared_experts + topk_fused = topk_routed + self.num_shared_topk_slots topk_ids = torch.empty( (num_tokens, topk_fused), dtype=torch.int32, device=router_logits.device ) @@ -238,7 +238,7 @@ class HashTopK(nn.Module): router_logits=router_logits, input_ids=input_ids, tid2eid=self.tid2eid, - num_fused_shared_experts=self.num_fused_shared_experts, + num_fused_shared_experts=self.num_shared_topk_slots, routed_scaling_factor=self.routed_scaling_factor, scoring_func=self.score_func, ) @@ -250,7 +250,7 @@ class HashTopK(nn.Module): if self.apply_routed_scaling_factor_on_output: topk_weights = topk_weights * self.routed_scaling_factor - num_fused_shared_experts = self.num_fused_shared_experts + num_shared_topk_slots = self.num_shared_topk_slots log2phy_prob = None if ( expert_location_dispatch_info is not None @@ -265,10 +265,10 @@ class HashTopK(nn.Module): log2phy_prob = lplb_solver.solve(topk_ids) recorder_topk_ids = None - if has_per_rank_fused_shared_slots(num_fused_shared_experts): + if has_per_rank_fused_shared_slots(num_shared_topk_slots): # The trailing columns are placeholders; the remap appends the real ones. routed_cols = topk_ids_logical_to_physical( - topk_ids[:, :-num_fused_shared_experts], + topk_ids[:, :-num_shared_topk_slots], expert_location_dispatch_info, log2phy_prob, ) @@ -281,12 +281,12 @@ class HashTopK(nn.Module): ) topk_ids, topk_weights = remap_topk_for_per_rank_shared_slots( routed_cols, - topk_weights[:, :-num_fused_shared_experts], - num_fused_shared_experts, + topk_weights[:, :-num_shared_topk_slots], + num_shared_topk_slots, num_physical_routed_experts, TopKConfig( top_k=self.topk, - num_fused_shared_experts=num_fused_shared_experts, + num_shared_topk_slots=num_shared_topk_slots, routed_scaling_factor=self.routed_scaling_factor, apply_routed_scaling_factor_on_output=self.apply_routed_scaling_factor_on_output, ), diff --git a/python/sglang/srt/layers/moe/topk.py b/python/sglang/srt/layers/moe/topk.py index 00628c62bd..d10fe60434 100644 --- a/python/sglang/srt/layers/moe/topk.py +++ b/python/sglang/srt/layers/moe/topk.py @@ -104,7 +104,11 @@ from sglang.srt.layers.moe import ( get_moe_runner_backend, is_moe_input_scattered_across_dp_ranks, ) -from sglang.srt.layers.moe.utils import has_per_rank_fused_shared_slots +from sglang.srt.layers.moe.utils import ( + has_per_rank_fused_shared_slots, + shared_topk_slots, + waterfill_routes_shared_experts, +) from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer from sglang.srt.utils import ( cpu_has_amx_support, @@ -160,7 +164,7 @@ def _use_rocm_triton_softmax_topk( gating_output: torch.Tensor, topk: int, correction_bias: Optional[torch.Tensor], - num_fused_shared_experts: int, + num_shared_topk_slots: int, packed_out: Optional[torch.Tensor], ) -> bool: """Use the lower-latency Triton router for Qwen3.5 decode-sized rows.""" @@ -174,7 +178,7 @@ def _use_rocm_triton_softmax_topk( and gating_output.dtype == torch.bfloat16 and topk == 10 and correction_bias is None - and num_fused_shared_experts == 0 + and num_shared_topk_slots == 0 and packed_out is None ) @@ -254,7 +258,9 @@ class TopKConfig: topk_group: Optional[int] = None num_expert_group: Optional[int] = None renormalize: bool = True - num_fused_shared_experts: int = 0 + # Trailing columns of top_k that the router fills with fused shared experts. + # Zero when Waterfill appends the shared expert after routing. + num_shared_topk_slots: int = 0 custom_routing_function: Optional[Callable] = None correction_bias: Optional[torch.Tensor] = None torch_native: bool = False @@ -585,7 +591,9 @@ class TopK(BaseFusedOp): --top_k: The all number of top experts selected per token, including the fused shared expert(s). --layer_id: global index of the owning layer. Required: routed-experts capture and per-layer EPLB state are keyed by it. - --num_fused_shared_experts: num of shared experts, can be activate both in TP or EP mode. + --num_fused_shared_experts: shared experts the experts layer runs as extra expert slots. + Their trailing top-k columns are written by the router, or by Waterfill when it is + enabled (``TopKConfig.num_shared_topk_slots`` then excludes them). --routed_scaling_factor: the scaling factor for routed experts in topk_weights. --fused_shared_experts_scaling_factor: absolute appended weight on the global AITER path; otherwise multiplies the router's existing shared weight under standard EP. @@ -624,33 +632,32 @@ class TopK(BaseFusedOp): self.layer_id = layer_id - self.enable_waterfill = ( - num_fused_shared_experts > 0 and get_exec().moe.enable_waterfill + self.enable_waterfill = waterfill_routes_shared_experts( + num_fused_shared_experts ) - self.waterfill_balancer = None - if self.enable_waterfill: - # TODO(ch-wan): Refactor shared-expert fusion and routed TopK fusion. - top_k -= num_fused_shared_experts - num_fused_shared_experts = 0 - output_format = TopKOutputFormat.STANDARD + num_shared_slots = shared_topk_slots(num_fused_shared_experts) + num_routed_experts_per_token = top_k - num_fused_shared_experts # Under the flashinfer_mxfp4 backend, fp4-expert ckpts take STANDARD # (consumes topk_ids/weights); otherwise BYPASSED. No-op on other backends. self.is_fp4_experts = is_fp4_experts self.topk_config = TopKConfig( - top_k=top_k, + top_k=num_routed_experts_per_token + num_shared_slots, use_grouped_topk=use_grouped_topk, renormalize=renormalize, topk_group=topk_group, num_expert_group=num_expert_group, - num_fused_shared_experts=num_fused_shared_experts, + num_shared_topk_slots=num_shared_slots, custom_routing_function=custom_routing_function, correction_bias=correction_bias, routed_scaling_factor=routed_scaling_factor, apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output, fused_shared_experts_scaling_factor=fused_shared_experts_scaling_factor, - output_format=output_format, + # Waterfill expands a standard top-k output. + output_format=( + TopKOutputFormat.STANDARD if self.enable_waterfill else output_format + ), scoring_func=scoring_func, sqrtsoftplus_log1p=sqrtsoftplus_log1p, fused_gate_packed_ids=fused_gate_packed_ids, @@ -869,7 +876,7 @@ class TopK(BaseFusedOp): device=device, ) ) - topk = self.topk_config.top_k - self.topk_config.num_fused_shared_experts + topk = self.topk_config.top_k - self.topk_config.num_shared_topk_slots with use_symmetric_memory( get_parallel().tp_group, disabled=not is_allocation_symmetric() ): @@ -878,8 +885,8 @@ class TopK(BaseFusedOp): # FIXME: router_logits should be of size (0, num_experts) router_logits = torch.empty((0, topk), dtype=torch.float32, device=device) topk_output = StandardTopKOutput(topk_weights, topk_ids, router_logits) - if has_per_rank_fused_shared_slots(self.topk_config.num_fused_shared_experts): - n = self.topk_config.num_fused_shared_experts + if has_per_rank_fused_shared_slots(self.topk_config.num_shared_topk_slots): + n = self.topk_config.num_shared_topk_slots topk_output = topk_output._replace( topk_ids=topk_output.topk_ids.new_empty( (0, topk_output.topk_ids.shape[-1] + n) @@ -1720,7 +1727,7 @@ def _fused_gate_masks_padded_rows(scoring_func: str) -> bool: def _fused_gate_emits_packed_ids( scoring_func: str, - num_fused_shared_experts: int, + num_shared_topk_slots: int, expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo], routing_overridden: bool, enabled: bool, @@ -1732,7 +1739,7 @@ def _fused_gate_emits_packed_ids( and _fused_gate_masks_padded_rows(scoring_func) and get_moe_runner_backend().is_flashinfer_mxfp4() and expert_location_dispatch_info is None - and num_fused_shared_experts == 0 + and num_shared_topk_slots == 0 and not routing_overridden ) @@ -2310,28 +2317,26 @@ def _post_process_topk_ids( expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None, padded_rows_masked: bool = False, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - num_fused_shared_experts = topk_config.num_fused_shared_experts - use_per_rank_shared_slots = has_per_rank_fused_shared_slots( - num_fused_shared_experts - ) + num_shared_topk_slots = topk_config.num_shared_topk_slots + use_per_rank_shared_slots = has_per_rank_fused_shared_slots(num_shared_topk_slots) fused_shared_experts_scaling_factor = ( topk_config.fused_shared_experts_scaling_factor ) # False when select_experts asked the router for routed experts only. router_wrote_shared_cols = ( - num_fused_shared_experts > 0 and topk_ids.shape[-1] == topk_config.top_k + num_shared_topk_slots > 0 and topk_ids.shape[-1] == topk_config.top_k ) # The router already wrote the aiter shared columns (see _aiter_append below). _gate_appended = ( router_wrote_shared_cols and _use_aiter and not use_per_rank_shared_slots ) routed_topk_ids = ( - topk_ids[:, :-num_fused_shared_experts] if _gate_appended else topk_ids + topk_ids[:, :-num_shared_topk_slots] if _gate_appended else topk_ids ) capture_routed_experts_if_allowed(topk_config, layer_id, routed_topk_ids) recorder_topk_ids = None _aiter_append = ( - num_fused_shared_experts > 0 + num_shared_topk_slots > 0 and _use_aiter and topk_ids.shape[-1] < topk_config.top_k ) @@ -2360,8 +2365,8 @@ def _post_process_topk_ids( elif router_wrote_shared_cols: # Shared IDs are outside EPLB's routed-expert table for both global # and per-rank layouts, so remap only routed columns. - shared_cols = topk_ids[:, -num_fused_shared_experts:] - routed_cols = topk_ids[:, :-num_fused_shared_experts] + shared_cols = topk_ids[:, -num_shared_topk_slots:] + routed_cols = topk_ids[:, :-num_shared_topk_slots] routed_cols = _biased_grouped_topk_postprocess( routed_cols, expert_location_dispatch_info, num_token_non_padded ) @@ -2438,7 +2443,7 @@ def _post_process_topk_ids( if recorder_topk_ids is None and not recorder_was_fused: recorder_topk_ids = ( - topk_ids[:, :-num_fused_shared_experts] if _gate_appended else topk_ids + topk_ids[:, :-num_shared_topk_slots] if _gate_appended else topk_ids ) if _aiter_append and use_per_rank_shared_slots: @@ -2455,7 +2460,7 @@ def _post_process_topk_ids( ep_size = get_parallel().moe_ep_size ep_rank = get_parallel().moe_ep_rank num_local_routed = num_physical_routed_experts // ep_size - num_local_experts = num_local_routed + num_fused_shared_experts + num_local_experts = num_local_routed + num_shared_topk_slots shared_id_base = ep_rank * num_local_experts + num_local_routed # Lazy import to avoid circular-import issues @@ -2466,7 +2471,7 @@ def _post_process_topk_ids( topk_ids, topk_weights = fused_append_remap_shared_experts_deepep( topk_ids, topk_weights, - num_fused_shared_experts, + num_shared_topk_slots, fused_shared_expert_weight( routed_scaling_factor=topk_config.routed_scaling_factor, fused_in_topk=is_routed_scaling_factor_fused_in_topk(topk_config), @@ -2493,7 +2498,7 @@ def _post_process_topk_ids( topk_ids, topk_weights = fused_append_shared_experts( topk_ids, topk_weights, - num_fused_shared_experts, + num_shared_topk_slots, scale_factor, N, # base id for shared experts num_token_non_padded=( @@ -2511,26 +2516,22 @@ def _post_process_topk_ids( ) if router_wrote_shared_cols: # Placeholders; the remap appends the real shared columns. - topk_ids = topk_ids[:, :-num_fused_shared_experts] - topk_weights = topk_weights[:, :-num_fused_shared_experts] + topk_ids = topk_ids[:, :-num_shared_topk_slots] + topk_weights = topk_weights[:, :-num_shared_topk_slots] topk_ids, topk_weights = remap_topk_for_per_rank_shared_slots( topk_ids, topk_weights, - num_fused_shared_experts, + num_shared_topk_slots, num_physical_routed_experts, topk_config, ) if _is_cuda: # The remap shifts a padded row's -1 and gives it a live shared slot. _mask_topk_ids_padded_region(topk_ids, num_token_non_padded) - elif ( - num_fused_shared_experts > 0 and fused_shared_experts_scaling_factor is not None - ): + elif num_shared_topk_slots > 0 and fused_shared_experts_scaling_factor is not None: # Standard EP all-reduces the per-rank shared-expert outputs; without the # supplied 1/ep_size factor the shared contribution is summed ep_size times. - topk_weights[:, -num_fused_shared_experts:] *= ( - fused_shared_experts_scaling_factor - ) + topk_weights[:, -num_shared_topk_slots:] *= fused_shared_experts_scaling_factor if _is_hip and not _skip_hip_pad_mask and not _fold_pad_weights_into_append: # Shared-expert append/remap can introduce non-zero weights after the @@ -2557,7 +2558,7 @@ def select_experts( topk_group = topk_config.topk_group num_expert_group = topk_config.num_expert_group renormalize = topk_config.renormalize - num_fused_shared_experts = topk_config.num_fused_shared_experts + num_shared_topk_slots = topk_config.num_shared_topk_slots custom_routing_function = topk_config.custom_routing_function correction_bias = ( topk_config.correction_bias @@ -2594,8 +2595,8 @@ def select_experts( and not torch_native and expert_location_dispatch_info is None and ( - num_fused_shared_experts == 0 - or has_per_rank_fused_shared_slots(num_fused_shared_experts) + num_shared_topk_slots == 0 + or has_per_rank_fused_shared_slots(num_shared_topk_slots) or not is_expert_remapping_enabled() ) ): @@ -2624,15 +2625,15 @@ def select_experts( ): correction_bias = topk_config.correction_bias_for_dtype(router_logits.dtype) - num_routed_topk = top_k - num_fused_shared_experts + num_routed_topk = top_k - num_shared_topk_slots # On aiter and on the per-rank shared-slot path, _post_process_topk_ids appends # the shared columns, so the router selects and renormalizes routed experts only. - if _use_aiter or has_per_rank_fused_shared_slots(num_fused_shared_experts): + if _use_aiter or has_per_rank_fused_shared_slots(num_shared_topk_slots): top_k_for_gate = num_routed_topk - num_fused_shared_experts_for_gate = 0 + num_shared_topk_slots_for_gate = 0 else: top_k_for_gate = top_k - num_fused_shared_experts_for_gate = num_fused_shared_experts + num_shared_topk_slots_for_gate = num_shared_topk_slots if dynamic_expert_bias is not None: if scoring_func != "sigmoid": raise ValueError( @@ -2646,7 +2647,7 @@ def select_experts( renormalize=renormalize, num_expert_group=num_expert_group or 1, topk_group=topk_group or 1, - num_fused_shared_experts=num_fused_shared_experts, + num_fused_shared_experts=num_shared_topk_slots, routed_scaling_factor=routed_scaling_factor, # Per-token bias bypasses AITER but must retain its folding policy. apply_routed_scaling_factor_on_output=is_routed_scaling_factor_fused_in_topk( @@ -2664,7 +2665,7 @@ def select_experts( renormalize=renormalize, num_expert_group=num_expert_group, topk_group=topk_group, - num_fused_shared_experts=num_fused_shared_experts_for_gate, + num_fused_shared_experts=num_shared_topk_slots_for_gate, routed_scaling_factor=routed_scaling_factor, apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output, scoring_func=scoring_func, @@ -2678,7 +2679,7 @@ def select_experts( renormalize=renormalize, num_expert_group=num_expert_group, topk_group=topk_group, - num_fused_shared_experts=num_fused_shared_experts_for_gate, + num_fused_shared_experts=num_shared_topk_slots_for_gate, routed_scaling_factor=routed_scaling_factor, apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output, ) @@ -2710,7 +2711,7 @@ def select_experts( _packed_kwargs = {} if _fused_gate_emits_packed_ids( scoring_func, - num_fused_shared_experts, + num_shared_topk_slots, expert_location_dispatch_info, routing_overridden, topk_config.fused_gate_packed_ids, @@ -2727,11 +2728,11 @@ def select_experts( _packed_kwargs["router_logits_partials"] = router_logits_partials if ( _use_aiter - and num_fused_shared_experts > 0 - and not has_per_rank_fused_shared_slots(num_fused_shared_experts) + and num_shared_topk_slots > 0 + and not has_per_rank_fused_shared_slots(num_shared_topk_slots) ): # the ROCm decode gate writes the aiter shared columns itself - _packed_kwargs["num_shared_append"] = num_fused_shared_experts + _packed_kwargs["num_shared_append"] = num_shared_topk_slots topk_weights, topk_ids = _biased_topk( hidden_states=hidden_states, gating_output=router_logits, @@ -2739,7 +2740,7 @@ def select_experts( topk=top_k_for_gate, renormalize=renormalize, scoring_func=scoring_func, - num_fused_shared_experts=num_fused_shared_experts_for_gate, + num_fused_shared_experts=num_shared_topk_slots_for_gate, routed_scaling_factor=routed_scaling_factor, num_token_non_padded=num_token_non_padded, expert_location_dispatch_info=expert_location_dispatch_info, @@ -2774,7 +2775,7 @@ def select_experts( and scoring_func == "softmax" and correction_bias is None and expert_location_dispatch_info is None - and num_fused_shared_experts == 0 + and num_shared_topk_slots == 0 and not routing_overridden ): num_experts = router_logits.shape[-1] @@ -2803,7 +2804,7 @@ def select_experts( renormalize=renormalize, correction_bias=correction_bias, scoring_func=scoring_func, - num_fused_shared_experts=num_fused_shared_experts_for_gate, + num_fused_shared_experts=num_shared_topk_slots_for_gate, routed_scaling_factor=routed_scaling_factor, apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output, **_fused_topk_kwargs, @@ -2904,7 +2905,7 @@ def precomputed_topk_postprocess_is_noop( """ return ( _is_cuda - and topk_config.num_fused_shared_experts == 0 + and topk_config.num_shared_topk_slots == 0 and num_token_non_padded is None and expert_location_dispatch_info is None and not envs.SGLANG_SIMULATE_UNIFORM_EXPERTS.get() diff --git a/python/sglang/srt/layers/moe/utils.py b/python/sglang/srt/layers/moe/utils.py index cd46f23647..ee9e798058 100644 --- a/python/sglang/srt/layers/moe/utils.py +++ b/python/sglang/srt/layers/moe/utils.py @@ -694,9 +694,27 @@ def uses_per_rank_fused_shared_slots() -> bool: return is_deepep_class_backend() or get_moe_a2a_backend().is_megamoe() -def has_per_rank_fused_shared_slots(num_fused_shared_experts: int) -> bool: - """Check whether this layer has fused shared experts in per-rank slots.""" - return num_fused_shared_experts > 0 and uses_per_rank_fused_shared_slots() +def has_per_rank_fused_shared_slots(num_shared: int) -> bool: + """Whether the backend gives each rank its own fused shared-expert slots.""" + return num_shared > 0 and uses_per_rank_fused_shared_slots() + + +def waterfill_routes_shared_experts(num_fused_shared_experts: int) -> bool: + """Whether Waterfill, not the top-k router, writes the fused shared experts' column. + + Waterfill appends that column after the EPLB remap and may place it on another + rank, so the router it follows emits the routed columns only.""" + return num_fused_shared_experts > 0 and get_exec().moe.enable_waterfill + + +def shared_topk_slots(num_fused_shared_experts: int) -> int: + """How many trailing top-k columns the router fills with fused shared experts. + + ``num_fused_shared_experts`` counts the shared experts the experts layer runs as + extra expert slots; the router writes their columns unless Waterfill does.""" + if waterfill_routes_shared_experts(num_fused_shared_experts): + return 0 + return num_fused_shared_experts def is_flashinfer_cutedsl_v1_path() -> bool: diff --git a/python/sglang/srt/layers/quantization/mxfp4_flashinfer_trtllm_moe.py b/python/sglang/srt/layers/quantization/mxfp4_flashinfer_trtllm_moe.py index 491cd585cd..d100e78a0b 100644 --- a/python/sglang/srt/layers/quantization/mxfp4_flashinfer_trtllm_moe.py +++ b/python/sglang/srt/layers/quantization/mxfp4_flashinfer_trtllm_moe.py @@ -126,7 +126,7 @@ class Mxfp4FlashinferTrtllmMoEMethod: and config.scoring_func == "sigmoid" and config.correction_bias is not None and config.custom_routing_function is None - and config.num_fused_shared_experts == 0 + and config.num_shared_topk_slots == 0 and not config.torch_native ) self._bypassed_routing_method = ( diff --git a/python/sglang/srt/multimodal/dsv41/vl_routing.py b/python/sglang/srt/multimodal/dsv41/vl_routing.py index ddd54f63d7..747b030093 100644 --- a/python/sglang/srt/multimodal/dsv41/vl_routing.py +++ b/python/sglang/srt/multimodal/dsv41/vl_routing.py @@ -33,11 +33,11 @@ def vision_topk( expert_location_dispatch_info=None, ): config = moe.topk.topk_config - num_fused_shared_experts = config.num_fused_shared_experts + num_shared_topk_slots = config.num_shared_topk_slots use_mega_moe = get_moe_a2a_backend().is_megamoe() - if num_fused_shared_experts and not use_mega_moe: + if num_shared_topk_slots and not use_mega_moe: # This path bypasses _post_process_topk_ids, which appends the per-rank slots. - assert not has_per_rank_fused_shared_slots(num_fused_shared_experts), ( + assert not has_per_rank_fused_shared_slots(num_shared_topk_slots), ( "VL routing does not support per-rank fused shared slots" ) if is_cuda(): @@ -48,7 +48,7 @@ def vision_topk( # rescaled below would rewrite weights after the router. packed_topk = None if ( - num_fused_shared_experts == 0 + num_shared_topk_slots == 0 and not use_mega_moe and get_moe_runner_backend().is_flashinfer_mxfp4() ): @@ -61,7 +61,7 @@ def vision_topk( moe.gate.e_score_correction_bias, topk=config.top_k, scoring_func="sqrtsoftplus", - num_fused_shared_experts=num_fused_shared_experts, + num_fused_shared_experts=num_shared_topk_slots, bias_alt=moe.gate.e_score_correction_bias_vl, input_ids=input_ids, bias_alt_token_id=moe.config.image_token_id, @@ -95,19 +95,19 @@ def vision_topk( moe.gate.e_score_correction_bias, ) # The shared slots appended below use the same layout as biased_grouped_topk_gpu. - topk_routed = config.top_k - num_fused_shared_experts + topk_routed = config.top_k - num_shared_topk_slots indices = (scores + bias).topk(topk_routed, dim=-1).indices weights = scores.gather(-1, indices) routed_sum = weights.sum(-1, keepdim=True, dtype=torch.float32) - if num_fused_shared_experts: + if num_shared_topk_slots: shared_ids = logits.shape[-1] + torch.arange( - num_fused_shared_experts, device=indices.device, dtype=indices.dtype + num_shared_topk_slots, device=indices.device, dtype=indices.dtype ) indices = torch.cat( [indices, shared_ids.expand(indices.shape[0], -1)], dim=-1, ) - weights = F.pad(weights, (0, num_fused_shared_experts)) + weights = F.pad(weights, (0, num_shared_topk_slots)) weights[:, topk_routed:] = routed_sum / config.routed_scaling_factor if config.renormalize and config.top_k > 1: weights = weights / (routed_sum + _RENORMALIZE_SUM_EPSILON) @@ -156,13 +156,13 @@ def _finish_vision_topk( else: weights = _scale_fused_shared_weights( weights, - config.num_fused_shared_experts, + config.num_shared_topk_slots, config.fused_shared_experts_scaling_factor, ) # Shared-slot remapping can overwrite padded weights; mask last so idle # and partially padded DP ranks never dispatch a phantom shared expert. if num_token_non_padded is not None and ( - not padded_rows_masked or (use_mega_moe and config.num_fused_shared_experts) + not padded_rows_masked or (use_mega_moe and config.num_shared_topk_slots) ): _mask_topk_ids_padded_region(indices, num_token_non_padded) _zero_topk_weights_padded_region(weights, num_token_non_padded) diff --git a/test/manual/minimax_m3/test_npu_topk.py b/test/manual/minimax_m3/test_npu_topk.py index 06c8ba1efc..6028e65edb 100644 --- a/test/manual/minimax_m3/test_npu_topk.py +++ b/test/manual/minimax_m3/test_npu_topk.py @@ -92,7 +92,7 @@ def _make_topk_config( num_expert_group=None, renormalize=renormalize, scoring_func="sigmoid", - num_fused_shared_experts=0, + num_shared_topk_slots=0, routed_scaling_factor=routed_scaling_factor, apply_routed_scaling_factor_on_output=True, ) diff --git a/test/registered/kernels/ops/moe/test_jit_grouped_topk.py b/test/registered/kernels/ops/moe/test_jit_grouped_topk.py index e548ce6e93..d48a7015ea 100644 --- a/test/registered/kernels/ops/moe/test_jit_grouped_topk.py +++ b/test/registered/kernels/ops/moe/test_jit_grouped_topk.py @@ -37,7 +37,7 @@ def _jit_routed(monkeypatch, logits, hidden, bias, groups): renormalize=True, use_grouped_topk=True, num_expert_group=groups, - num_fused_shared_experts=SHARED, + num_shared_topk_slots=SHARED, topk_group=1, scoring_func="sigmoid", correction_bias=bias, diff --git a/test/registered/kernels/ops/moe/test_rocm_router_gate.py b/test/registered/kernels/ops/moe/test_rocm_router_gate.py index ea89ae2fdc..37e2f7483e 100644 --- a/test/registered/kernels/ops/moe/test_rocm_router_gate.py +++ b/test/registered/kernels/ops/moe/test_rocm_router_gate.py @@ -135,7 +135,7 @@ class TestRocmRouterGate(CustomTestCase): config = TopKConfig( top_k=TOPK + 1, renormalize=True, - num_fused_shared_experts=1, + num_shared_topk_slots=1, correction_bias=self.bias_bf16, routed_scaling_factor=1.5, fused_shared_experts_scaling_factor=scaling, diff --git a/test/registered/kernels/ops/moe/test_round_robin_ep_balance.py b/test/registered/kernels/ops/moe/test_round_robin_ep_balance.py index 08060747a3..9c224929ba 100644 --- a/test/registered/kernels/ops/moe/test_round_robin_ep_balance.py +++ b/test/registered/kernels/ops/moe/test_round_robin_ep_balance.py @@ -227,7 +227,7 @@ def _select( router_logits=torch.zeros((num_tokens, NUM_EXPERTS), device="cuda"), topk_config=TopKConfig( top_k=top_k, - num_fused_shared_experts=num_fused_shared_experts, + num_shared_topk_slots=num_fused_shared_experts, custom_routing_function=_fixed_router, ), layer_id=0, diff --git a/test/registered/moe/test_fused_append_remap_per_rank_shared_slots.py b/test/registered/moe/test_fused_append_remap_per_rank_shared_slots.py index 79a0700d38..b817243680 100644 --- a/test/registered/moe/test_fused_append_remap_per_rank_shared_slots.py +++ b/test/registered/moe/test_fused_append_remap_per_rank_shared_slots.py @@ -210,7 +210,7 @@ class TestFusedAppendRemapPerRankSharedSlots(CustomTestCase): npr, TopKConfig( top_k=k, - num_fused_shared_experts=s, + num_shared_topk_slots=s, routed_scaling_factor=rsf, apply_routed_scaling_factor_on_output=folded, ), @@ -314,7 +314,7 @@ class TestSelectExpertsPerRankSharedSlots(CustomTestCase): use_grouped_topk=num_expert_group is not None, num_expert_group=num_expert_group, topk_group=topk_group, - num_fused_shared_experts=num_fused_shared_experts, + num_shared_topk_slots=num_fused_shared_experts, correction_bias=bias, scoring_func=scoring_func, routed_scaling_factor=2.5, @@ -388,7 +388,7 @@ class TestSelectExpertsPerRankSharedSlots(CustomTestCase): num_expert_group=8, topk_group=4, renormalize=False, - num_fused_shared_experts=1, + num_shared_topk_slots=1, correction_bias=torch.zeros(256, device=get_device()), scoring_func="sigmoid", routed_scaling_factor=rsf, diff --git a/test/registered/moe/test_topk_padded_region.py b/test/registered/moe/test_topk_padded_region.py index b6fb401c4f..f53af8b063 100644 --- a/test/registered/moe/test_topk_padded_region.py +++ b/test/registered/moe/test_topk_padded_region.py @@ -163,7 +163,7 @@ class TestPostProcessPaddedMaskingHip(CustomTestCase): topk_ids = torch.randint(0, 64, (n, k), device=self.DEVICE, dtype=torch.int32) router_logits = torch.rand((n, 64), device=self.DEVICE, dtype=torch.float32) pad = torch.tensor(n_valid, device=self.DEVICE, dtype=torch.int32) - cfg = TopKConfig(top_k=k, num_fused_shared_experts=0) + cfg = TopKConfig(top_k=k, num_shared_topk_slots=0) _, out_weights, _ = _post_process_topk_ids( topk_ids, topk_weights, @@ -197,7 +197,7 @@ class TestPostProcessPaddedMaskingHip(CustomTestCase): topk_ids = torch.randint(0, 64, (n, k), device=self.DEVICE, dtype=torch.int32) router_logits = torch.rand((n, 64), device=self.DEVICE) pad = torch.tensor(n_valid, device=self.DEVICE, dtype=torch.int32) - cfg = TopKConfig(top_k=k + 1, num_fused_shared_experts=1) + cfg = TopKConfig(top_k=k + 1, num_shared_topk_slots=1) orig_skip = topk_mod._skip_hip_pad_mask orig_aiter = topk_mod._use_aiter topk_mod._skip_hip_pad_mask = True @@ -301,7 +301,7 @@ class TestSelectExpertsPerRankSharedSlotPadMask(CustomTestCase): use_grouped_topk=num_expert_group is not None, num_expert_group=num_expert_group, topk_group=4 if num_expert_group else None, - num_fused_shared_experts=1, + num_shared_topk_slots=1, correction_bias=bias.to(self.DEVICE), scoring_func=scoring_func, routed_scaling_factor=2.5, diff --git a/test/registered/unit/eplb/test_waterfill_eplb.py b/test/registered/unit/eplb/test_waterfill_eplb.py index 86c17d8ec2..16048ee4bc 100644 --- a/test/registered/unit/eplb/test_waterfill_eplb.py +++ b/test/registered/unit/eplb/test_waterfill_eplb.py @@ -12,7 +12,9 @@ import torch from torch import nn from sglang.srt.layers.moe import topk as topk_module -from sglang.srt.layers.moe.topk import TopKConfig +from sglang.srt.layers.moe import utils as moe_utils +from sglang.srt.layers.moe.hash_topk import HashTopK +from sglang.srt.layers.moe.topk import TopK, TopKConfig, TopKOutputFormat from sglang.srt.models.deepseek_v2 import DeepseekV2MoE from sglang.srt.runtime_context import get_parallel from sglang.test.test_utils import CustomTestCase @@ -61,7 +63,7 @@ class TestWaterfillEPLB(CustomTestCase): topk_weights = torch.ones_like(topk_ids, dtype=torch.float32) topk_config = TopKConfig( top_k=4, - num_fused_shared_experts=1, + num_shared_topk_slots=1, routed_scaling_factor=1.0, ) dispatch_info = SimpleNamespace( @@ -104,7 +106,7 @@ class TestWaterfillEPLB(CustomTestCase): topk_weights = torch.ones_like(topk_ids, dtype=torch.float32) topk_config = TopKConfig( top_k=4, - num_fused_shared_experts=1, + num_shared_topk_slots=1, routed_scaling_factor=1.0, ) dispatch_info = SimpleNamespace( @@ -147,7 +149,7 @@ class TestWaterfillEPLB(CustomTestCase): topk_weights = torch.ones_like(topk_ids, dtype=torch.float32) topk_config = TopKConfig( top_k=4, - num_fused_shared_experts=1, + num_shared_topk_slots=1, routed_scaling_factor=1.0, ) dispatch_info = SimpleNamespace( @@ -183,6 +185,49 @@ class TestWaterfillEPLB(CustomTestCase): self.assertTrue(torch.equal(processed_ids, torch.tensor([[1, 34, 201, 256]]))) self.assertTrue(torch.equal(recorder_ids, torch.tensor([[1, 34, 201]]))) + def test_the_shared_expert_column_has_exactly_one_writer(self): + """Shared slots must be written exactly once, by routing or Waterfill.""" + for enable_waterfill in (False, True): + exec_config = SimpleNamespace( + moe=SimpleNamespace(enable_waterfill=enable_waterfill) + ) + with ( + self.subTest(enable_waterfill=enable_waterfill), + patch.object(moe_utils, "get_exec", return_value=exec_config), + ): + topk = TopK(top_k=9, layer_id=0, num_fused_shared_experts=1) + hash_topk = HashTopK( + topk=9, + num_experts=256, + num_fused_shared_experts=1, + vocab_size=4, + layer_id=0, + ) + unfused_topk = TopK(top_k=8, layer_id=0) + + for layer, router_top_k, router_shared_columns in ( + ( + topk, + topk.topk_config.top_k, + topk.topk_config.num_shared_topk_slots, + ), + (hash_topk, hash_topk.topk, hash_topk.num_shared_topk_slots), + ): + self.assertEqual(layer.enable_waterfill, enable_waterfill) + self.assertEqual( + router_shared_columns + int(layer.enable_waterfill), 1 + ) + self.assertEqual(router_top_k - router_shared_columns, 8) + self.assertEqual(hash_topk.tid2eid.shape[1], 8) + if enable_waterfill: + self.assertEqual( + topk.topk_config.output_format, TopKOutputFormat.STANDARD + ) + + self.assertFalse(unfused_topk.enable_waterfill) + self.assertEqual(unfused_topk.topk_config.top_k, 8) + self.assertEqual(unfused_topk.topk_config.num_shared_topk_slots, 0) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/layers/moe/test_fused_shared_expert_scaling.py b/test/registered/unit/layers/moe/test_fused_shared_expert_scaling.py index d120bd8b8a..96e21e863d 100644 --- a/test/registered/unit/layers/moe/test_fused_shared_expert_scaling.py +++ b/test/registered/unit/layers/moe/test_fused_shared_expert_scaling.py @@ -12,8 +12,8 @@ from unittest.mock import patch import torch from sglang.srt.batch_overlap.operations import _StateDict -from sglang.srt.layers.moe import hash_topk as hash_topk_module from sglang.srt.layers.moe import topk as topk_module +from sglang.srt.layers.moe import utils as moe_utils from sglang.srt.layers.moe.topk import TopK, TopKConfig from sglang.srt.layers.moe.utils import MoeA2ABackend from sglang.srt.model_executor.model_runner_components import moe_ep_setup @@ -42,7 +42,7 @@ class TestFusedSharedExpertScaling(CustomTestCase): topk_config = TopKConfig( top_k=4, - num_fused_shared_experts=1, + num_shared_topk_slots=1, routed_scaling_factor=self.ROUTED_SCALING_FACTOR, apply_routed_scaling_factor_on_output=folded_in_topk, **self.ROUTING, @@ -213,7 +213,8 @@ class TestFusedSharedExpertScaling(CustomTestCase): ), patch.object(module, "get_exec", return_value=exec_config), patch.object(topk_module, "get_exec", return_value=exec_config), - patch.object(hash_topk_module, "get_exec", return_value=exec_config), + # TopK and HashTopK ask layers/moe/utils.py whether Waterfill is on. + patch.object(moe_utils, "get_exec", return_value=exec_config), patch.object( module, "is_shared_experts_fusion_disabled", @@ -398,7 +399,7 @@ class TestFusedSharedExpertScaling(CustomTestCase): topk_weights = torch.tensor([[1.0, 0.5, 0.25, 1.0]], dtype=torch.float32) topk_config = TopKConfig( top_k=4, - num_fused_shared_experts=1, + num_shared_topk_slots=1, fused_shared_experts_scaling_factor=scaling_factor, allow_routed_experts_capture=False, ) @@ -443,7 +444,7 @@ class TestFusedSharedExpertScaling(CustomTestCase): topk_weights = torch.tensor([[1.0, 0.5, 0.25]], dtype=torch.float32) topk_config = TopKConfig( top_k=4, - num_fused_shared_experts=1, + num_shared_topk_slots=1, routed_scaling_factor=self.ROUTED_SCALING_FACTOR, ) with ( @@ -471,7 +472,7 @@ class TestFusedSharedExpertScaling(CustomTestCase): def _route_with_per_rank_slots(self, *, num_fused_shared_experts, logits, bias): topk_config = TopKConfig( top_k=8 + num_fused_shared_experts, - num_fused_shared_experts=num_fused_shared_experts, + num_shared_topk_slots=num_fused_shared_experts, renormalize=True, correction_bias=bias, torch_native=True, @@ -528,7 +529,7 @@ class TestFusedSharedExpertScaling(CustomTestCase): here would make the AITER postprocessor append shared ID 288 twice.""" topk = TopK.__new__(TopK) torch.nn.Module.__init__(topk) - topk.topk_config = TopKConfig(top_k=9, num_fused_shared_experts=1) + topk.topk_config = TopKConfig(top_k=9, num_shared_topk_slots=1) topk.enable_waterfill = False topk.waterfill_balancer = None diff --git a/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py b/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py index df1fc41c0d..c224bf10e9 100644 --- a/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py +++ b/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py @@ -454,7 +454,7 @@ class TestDeepGemmMegaMoeApi(CustomTestCase): topk=SimpleNamespace( topk_config=TopKConfig( top_k=2 + fused_shared, - num_fused_shared_experts=fused_shared, + num_shared_topk_slots=fused_shared, renormalize=True, routed_scaling_factor=2.0, apply_routed_scaling_factor_on_output=scale_in_topk,