[Refactor] moe: Split the two meanings of num_fused_shared_experts (#44912)

This commit is contained in:
Cheng Wan
2026-10-19 16:57:15 -07:00
committed by GitHub
parent 48ea486928
commit 964ab2b280
15 changed files with 190 additions and 125 deletions
@@ -61,7 +61,7 @@ def fused_topk_npu(
if renormalize:
topk_weights = l1_norm(
topk_weights
if topk_config.num_fused_shared_experts == 0
if topk_config.num_shared_topk_slots == 0
else topk_weights[:, :-1]
)
topk_weights = topk_weights.to(torch.float32)
+26 -26
View File
@@ -22,8 +22,11 @@ from sglang.srt.layers.moe.topk import (
_zero_topk_weights_padded_region,
remap_topk_for_per_rank_shared_slots,
)
from sglang.srt.layers.moe.utils import has_per_rank_fused_shared_slots
from sglang.srt.runtime_context import get_exec
from sglang.srt.layers.moe.utils import (
has_per_rank_fused_shared_slots,
shared_topk_slots,
waterfill_routes_shared_experts,
)
from sglang.srt.utils import is_hip, is_npu, is_xpu
logger = logging.getLogger(__name__)
@@ -48,23 +51,20 @@ class HashTopK(nn.Module):
super().__init__()
self.layer_id = layer_id
self.enable_waterfill = (
num_fused_shared_experts > 0 and get_exec().moe.enable_waterfill
self.enable_waterfill = waterfill_routes_shared_experts(
num_fused_shared_experts
)
self.waterfill_balancer = None
if self.enable_waterfill:
# Waterfill appends the shared expert after EPLB maps routed IDs.
topk -= num_fused_shared_experts
num_fused_shared_experts = 0
num_routed_experts_per_token = topk - num_fused_shared_experts
self.num_experts = num_experts
self.topk = topk
# Trailing columns the hash router fills with fused shared experts.
self.num_shared_topk_slots = shared_topk_slots(num_fused_shared_experts)
self.topk = num_routed_experts_per_token + self.num_shared_topk_slots
self.routed_scaling_factor = routed_scaling_factor
self.num_fused_shared_experts = num_fused_shared_experts
self.score_func = scoring_func
self.tid2eid = nn.Parameter(
torch.empty(vocab_size, topk - num_fused_shared_experts, dtype=torch.int32),
torch.empty(vocab_size, num_routed_experts_per_token, dtype=torch.int32),
requires_grad=False,
)
self._init_default_tid2eid()
@@ -72,7 +72,7 @@ class HashTopK(nn.Module):
self.apply_routed_scaling_factor_on_output = (
apply_routed_scaling_factor_on_output
)
if apply_routed_scaling_factor_on_output and num_fused_shared_experts > 0:
if apply_routed_scaling_factor_on_output and self.num_shared_topk_slots > 0:
raise NotImplementedError(
"HashTopK + apply_routed_scaling_factor_on_output is not supported "
"with fused shared experts; pass --disable-shared-experts-fusion."
@@ -98,7 +98,7 @@ class HashTopK(nn.Module):
def empty_topk_output(
self, device: torch.device, *, layer_id: Optional[int] = None
):
topk = self.topk - self.num_fused_shared_experts
topk = self.topk - self.num_shared_topk_slots
if layer_id is not None:
from sglang.srt.eplb.lplb_solver import get_global_lplb_solver
@@ -111,8 +111,8 @@ class HashTopK(nn.Module):
topk_ids = torch.full((0, topk), -1, dtype=torch.int32, device=device)
router_logits = torch.empty((0, topk), dtype=torch.float32, device=device)
topk_output = StandardTopKOutput(topk_weights, topk_ids, router_logits)
if has_per_rank_fused_shared_slots(self.num_fused_shared_experts):
n = self.num_fused_shared_experts
if has_per_rank_fused_shared_slots(self.num_shared_topk_slots):
n = self.num_shared_topk_slots
topk_output = topk_output._replace(
topk_ids=topk_output.topk_ids.new_empty(
(0, topk_output.topk_ids.shape[-1] + n)
@@ -153,7 +153,7 @@ class HashTopK(nn.Module):
(num_token, self.topk), dtype=scores.dtype, device=scores.device
)
if self.num_fused_shared_experts == 1:
if self.num_shared_topk_slots == 1:
topk_ids[:, :-1] = self.tid2eid[input_ids]
topk_weights[:, :-1] = scores.gather(1, topk_ids[:, :-1])
@@ -165,7 +165,7 @@ class HashTopK(nn.Module):
topk_ids[:, -1] = torch.randint(
low=self.num_experts,
high=self.num_experts + self.num_fused_shared_experts,
high=self.num_experts + self.num_shared_topk_slots,
size=(num_token,),
dtype=topk_ids.dtype,
device=topk_ids.device,
@@ -195,7 +195,7 @@ class HashTopK(nn.Module):
num_tokens = router_logits.size(0)
topk_routed = self.tid2eid.size(1)
topk_fused = topk_routed + self.num_fused_shared_experts
topk_fused = topk_routed + self.num_shared_topk_slots
topk_ids = torch.empty(
(num_tokens, topk_fused), dtype=torch.int32, device=router_logits.device
)
@@ -238,7 +238,7 @@ class HashTopK(nn.Module):
router_logits=router_logits,
input_ids=input_ids,
tid2eid=self.tid2eid,
num_fused_shared_experts=self.num_fused_shared_experts,
num_fused_shared_experts=self.num_shared_topk_slots,
routed_scaling_factor=self.routed_scaling_factor,
scoring_func=self.score_func,
)
@@ -250,7 +250,7 @@ class HashTopK(nn.Module):
if self.apply_routed_scaling_factor_on_output:
topk_weights = topk_weights * self.routed_scaling_factor
num_fused_shared_experts = self.num_fused_shared_experts
num_shared_topk_slots = self.num_shared_topk_slots
log2phy_prob = None
if (
expert_location_dispatch_info is not None
@@ -265,10 +265,10 @@ class HashTopK(nn.Module):
log2phy_prob = lplb_solver.solve(topk_ids)
recorder_topk_ids = None
if has_per_rank_fused_shared_slots(num_fused_shared_experts):
if has_per_rank_fused_shared_slots(num_shared_topk_slots):
# The trailing columns are placeholders; the remap appends the real ones.
routed_cols = topk_ids_logical_to_physical(
topk_ids[:, :-num_fused_shared_experts],
topk_ids[:, :-num_shared_topk_slots],
expert_location_dispatch_info,
log2phy_prob,
)
@@ -281,12 +281,12 @@ class HashTopK(nn.Module):
)
topk_ids, topk_weights = remap_topk_for_per_rank_shared_slots(
routed_cols,
topk_weights[:, :-num_fused_shared_experts],
num_fused_shared_experts,
topk_weights[:, :-num_shared_topk_slots],
num_shared_topk_slots,
num_physical_routed_experts,
TopKConfig(
top_k=self.topk,
num_fused_shared_experts=num_fused_shared_experts,
num_shared_topk_slots=num_shared_topk_slots,
routed_scaling_factor=self.routed_scaling_factor,
apply_routed_scaling_factor_on_output=self.apply_routed_scaling_factor_on_output,
),
+62 -61
View File
@@ -104,7 +104,11 @@ from sglang.srt.layers.moe import (
get_moe_runner_backend,
is_moe_input_scattered_across_dp_ranks,
)
from sglang.srt.layers.moe.utils import has_per_rank_fused_shared_slots
from sglang.srt.layers.moe.utils import (
has_per_rank_fused_shared_slots,
shared_topk_slots,
waterfill_routes_shared_experts,
)
from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer
from sglang.srt.utils import (
cpu_has_amx_support,
@@ -160,7 +164,7 @@ def _use_rocm_triton_softmax_topk(
gating_output: torch.Tensor,
topk: int,
correction_bias: Optional[torch.Tensor],
num_fused_shared_experts: int,
num_shared_topk_slots: int,
packed_out: Optional[torch.Tensor],
) -> bool:
"""Use the lower-latency Triton router for Qwen3.5 decode-sized rows."""
@@ -174,7 +178,7 @@ def _use_rocm_triton_softmax_topk(
and gating_output.dtype == torch.bfloat16
and topk == 10
and correction_bias is None
and num_fused_shared_experts == 0
and num_shared_topk_slots == 0
and packed_out is None
)
@@ -254,7 +258,9 @@ class TopKConfig:
topk_group: Optional[int] = None
num_expert_group: Optional[int] = None
renormalize: bool = True
num_fused_shared_experts: int = 0
# Trailing columns of top_k that the router fills with fused shared experts.
# Zero when Waterfill appends the shared expert after routing.
num_shared_topk_slots: int = 0
custom_routing_function: Optional[Callable] = None
correction_bias: Optional[torch.Tensor] = None
torch_native: bool = False
@@ -585,7 +591,9 @@ class TopK(BaseFusedOp):
--top_k: The all number of top experts selected per token, including the fused shared expert(s).
--layer_id: global index of the owning layer. Required: routed-experts capture and
per-layer EPLB state are keyed by it.
--num_fused_shared_experts: num of shared experts, can be activate both in TP or EP mode.
--num_fused_shared_experts: shared experts the experts layer runs as extra expert slots.
Their trailing top-k columns are written by the router, or by Waterfill when it is
enabled (``TopKConfig.num_shared_topk_slots`` then excludes them).
--routed_scaling_factor: the scaling factor for routed experts in topk_weights.
--fused_shared_experts_scaling_factor: absolute appended weight on the global AITER
path; otherwise multiplies the router's existing shared weight under standard EP.
@@ -624,33 +632,32 @@ class TopK(BaseFusedOp):
self.layer_id = layer_id
self.enable_waterfill = (
num_fused_shared_experts > 0 and get_exec().moe.enable_waterfill
self.enable_waterfill = waterfill_routes_shared_experts(
num_fused_shared_experts
)
self.waterfill_balancer = None
if self.enable_waterfill:
# TODO(ch-wan): Refactor shared-expert fusion and routed TopK fusion.
top_k -= num_fused_shared_experts
num_fused_shared_experts = 0
output_format = TopKOutputFormat.STANDARD
num_shared_slots = shared_topk_slots(num_fused_shared_experts)
num_routed_experts_per_token = top_k - num_fused_shared_experts
# Under the flashinfer_mxfp4 backend, fp4-expert ckpts take STANDARD
# (consumes topk_ids/weights); otherwise BYPASSED. No-op on other backends.
self.is_fp4_experts = is_fp4_experts
self.topk_config = TopKConfig(
top_k=top_k,
top_k=num_routed_experts_per_token + num_shared_slots,
use_grouped_topk=use_grouped_topk,
renormalize=renormalize,
topk_group=topk_group,
num_expert_group=num_expert_group,
num_fused_shared_experts=num_fused_shared_experts,
num_shared_topk_slots=num_shared_slots,
custom_routing_function=custom_routing_function,
correction_bias=correction_bias,
routed_scaling_factor=routed_scaling_factor,
apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output,
fused_shared_experts_scaling_factor=fused_shared_experts_scaling_factor,
output_format=output_format,
# Waterfill expands a standard top-k output.
output_format=(
TopKOutputFormat.STANDARD if self.enable_waterfill else output_format
),
scoring_func=scoring_func,
sqrtsoftplus_log1p=sqrtsoftplus_log1p,
fused_gate_packed_ids=fused_gate_packed_ids,
@@ -869,7 +876,7 @@ class TopK(BaseFusedOp):
device=device,
)
)
topk = self.topk_config.top_k - self.topk_config.num_fused_shared_experts
topk = self.topk_config.top_k - self.topk_config.num_shared_topk_slots
with use_symmetric_memory(
get_parallel().tp_group, disabled=not is_allocation_symmetric()
):
@@ -878,8 +885,8 @@ class TopK(BaseFusedOp):
# FIXME: router_logits should be of size (0, num_experts)
router_logits = torch.empty((0, topk), dtype=torch.float32, device=device)
topk_output = StandardTopKOutput(topk_weights, topk_ids, router_logits)
if has_per_rank_fused_shared_slots(self.topk_config.num_fused_shared_experts):
n = self.topk_config.num_fused_shared_experts
if has_per_rank_fused_shared_slots(self.topk_config.num_shared_topk_slots):
n = self.topk_config.num_shared_topk_slots
topk_output = topk_output._replace(
topk_ids=topk_output.topk_ids.new_empty(
(0, topk_output.topk_ids.shape[-1] + n)
@@ -1720,7 +1727,7 @@ def _fused_gate_masks_padded_rows(scoring_func: str) -> bool:
def _fused_gate_emits_packed_ids(
scoring_func: str,
num_fused_shared_experts: int,
num_shared_topk_slots: int,
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo],
routing_overridden: bool,
enabled: bool,
@@ -1732,7 +1739,7 @@ def _fused_gate_emits_packed_ids(
and _fused_gate_masks_padded_rows(scoring_func)
and get_moe_runner_backend().is_flashinfer_mxfp4()
and expert_location_dispatch_info is None
and num_fused_shared_experts == 0
and num_shared_topk_slots == 0
and not routing_overridden
)
@@ -2310,28 +2317,26 @@ def _post_process_topk_ids(
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None,
padded_rows_masked: bool = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
num_fused_shared_experts = topk_config.num_fused_shared_experts
use_per_rank_shared_slots = has_per_rank_fused_shared_slots(
num_fused_shared_experts
)
num_shared_topk_slots = topk_config.num_shared_topk_slots
use_per_rank_shared_slots = has_per_rank_fused_shared_slots(num_shared_topk_slots)
fused_shared_experts_scaling_factor = (
topk_config.fused_shared_experts_scaling_factor
)
# False when select_experts asked the router for routed experts only.
router_wrote_shared_cols = (
num_fused_shared_experts > 0 and topk_ids.shape[-1] == topk_config.top_k
num_shared_topk_slots > 0 and topk_ids.shape[-1] == topk_config.top_k
)
# The router already wrote the aiter shared columns (see _aiter_append below).
_gate_appended = (
router_wrote_shared_cols and _use_aiter and not use_per_rank_shared_slots
)
routed_topk_ids = (
topk_ids[:, :-num_fused_shared_experts] if _gate_appended else topk_ids
topk_ids[:, :-num_shared_topk_slots] if _gate_appended else topk_ids
)
capture_routed_experts_if_allowed(topk_config, layer_id, routed_topk_ids)
recorder_topk_ids = None
_aiter_append = (
num_fused_shared_experts > 0
num_shared_topk_slots > 0
and _use_aiter
and topk_ids.shape[-1] < topk_config.top_k
)
@@ -2360,8 +2365,8 @@ def _post_process_topk_ids(
elif router_wrote_shared_cols:
# Shared IDs are outside EPLB's routed-expert table for both global
# and per-rank layouts, so remap only routed columns.
shared_cols = topk_ids[:, -num_fused_shared_experts:]
routed_cols = topk_ids[:, :-num_fused_shared_experts]
shared_cols = topk_ids[:, -num_shared_topk_slots:]
routed_cols = topk_ids[:, :-num_shared_topk_slots]
routed_cols = _biased_grouped_topk_postprocess(
routed_cols, expert_location_dispatch_info, num_token_non_padded
)
@@ -2438,7 +2443,7 @@ def _post_process_topk_ids(
if recorder_topk_ids is None and not recorder_was_fused:
recorder_topk_ids = (
topk_ids[:, :-num_fused_shared_experts] if _gate_appended else topk_ids
topk_ids[:, :-num_shared_topk_slots] if _gate_appended else topk_ids
)
if _aiter_append and use_per_rank_shared_slots:
@@ -2455,7 +2460,7 @@ def _post_process_topk_ids(
ep_size = get_parallel().moe_ep_size
ep_rank = get_parallel().moe_ep_rank
num_local_routed = num_physical_routed_experts // ep_size
num_local_experts = num_local_routed + num_fused_shared_experts
num_local_experts = num_local_routed + num_shared_topk_slots
shared_id_base = ep_rank * num_local_experts + num_local_routed
# Lazy import to avoid circular-import issues
@@ -2466,7 +2471,7 @@ def _post_process_topk_ids(
topk_ids, topk_weights = fused_append_remap_shared_experts_deepep(
topk_ids,
topk_weights,
num_fused_shared_experts,
num_shared_topk_slots,
fused_shared_expert_weight(
routed_scaling_factor=topk_config.routed_scaling_factor,
fused_in_topk=is_routed_scaling_factor_fused_in_topk(topk_config),
@@ -2493,7 +2498,7 @@ def _post_process_topk_ids(
topk_ids, topk_weights = fused_append_shared_experts(
topk_ids,
topk_weights,
num_fused_shared_experts,
num_shared_topk_slots,
scale_factor,
N, # base id for shared experts
num_token_non_padded=(
@@ -2511,26 +2516,22 @@ def _post_process_topk_ids(
)
if router_wrote_shared_cols:
# Placeholders; the remap appends the real shared columns.
topk_ids = topk_ids[:, :-num_fused_shared_experts]
topk_weights = topk_weights[:, :-num_fused_shared_experts]
topk_ids = topk_ids[:, :-num_shared_topk_slots]
topk_weights = topk_weights[:, :-num_shared_topk_slots]
topk_ids, topk_weights = remap_topk_for_per_rank_shared_slots(
topk_ids,
topk_weights,
num_fused_shared_experts,
num_shared_topk_slots,
num_physical_routed_experts,
topk_config,
)
if _is_cuda:
# The remap shifts a padded row's -1 and gives it a live shared slot.
_mask_topk_ids_padded_region(topk_ids, num_token_non_padded)
elif (
num_fused_shared_experts > 0 and fused_shared_experts_scaling_factor is not None
):
elif num_shared_topk_slots > 0 and fused_shared_experts_scaling_factor is not None:
# Standard EP all-reduces the per-rank shared-expert outputs; without the
# supplied 1/ep_size factor the shared contribution is summed ep_size times.
topk_weights[:, -num_fused_shared_experts:] *= (
fused_shared_experts_scaling_factor
)
topk_weights[:, -num_shared_topk_slots:] *= fused_shared_experts_scaling_factor
if _is_hip and not _skip_hip_pad_mask and not _fold_pad_weights_into_append:
# Shared-expert append/remap can introduce non-zero weights after the
@@ -2557,7 +2558,7 @@ def select_experts(
topk_group = topk_config.topk_group
num_expert_group = topk_config.num_expert_group
renormalize = topk_config.renormalize
num_fused_shared_experts = topk_config.num_fused_shared_experts
num_shared_topk_slots = topk_config.num_shared_topk_slots
custom_routing_function = topk_config.custom_routing_function
correction_bias = (
topk_config.correction_bias
@@ -2594,8 +2595,8 @@ def select_experts(
and not torch_native
and expert_location_dispatch_info is None
and (
num_fused_shared_experts == 0
or has_per_rank_fused_shared_slots(num_fused_shared_experts)
num_shared_topk_slots == 0
or has_per_rank_fused_shared_slots(num_shared_topk_slots)
or not is_expert_remapping_enabled()
)
):
@@ -2624,15 +2625,15 @@ def select_experts(
):
correction_bias = topk_config.correction_bias_for_dtype(router_logits.dtype)
num_routed_topk = top_k - num_fused_shared_experts
num_routed_topk = top_k - num_shared_topk_slots
# On aiter and on the per-rank shared-slot path, _post_process_topk_ids appends
# the shared columns, so the router selects and renormalizes routed experts only.
if _use_aiter or has_per_rank_fused_shared_slots(num_fused_shared_experts):
if _use_aiter or has_per_rank_fused_shared_slots(num_shared_topk_slots):
top_k_for_gate = num_routed_topk
num_fused_shared_experts_for_gate = 0
num_shared_topk_slots_for_gate = 0
else:
top_k_for_gate = top_k
num_fused_shared_experts_for_gate = num_fused_shared_experts
num_shared_topk_slots_for_gate = num_shared_topk_slots
if dynamic_expert_bias is not None:
if scoring_func != "sigmoid":
raise ValueError(
@@ -2646,7 +2647,7 @@ def select_experts(
renormalize=renormalize,
num_expert_group=num_expert_group or 1,
topk_group=topk_group or 1,
num_fused_shared_experts=num_fused_shared_experts,
num_fused_shared_experts=num_shared_topk_slots,
routed_scaling_factor=routed_scaling_factor,
# Per-token bias bypasses AITER but must retain its folding policy.
apply_routed_scaling_factor_on_output=is_routed_scaling_factor_fused_in_topk(
@@ -2664,7 +2665,7 @@ def select_experts(
renormalize=renormalize,
num_expert_group=num_expert_group,
topk_group=topk_group,
num_fused_shared_experts=num_fused_shared_experts_for_gate,
num_fused_shared_experts=num_shared_topk_slots_for_gate,
routed_scaling_factor=routed_scaling_factor,
apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output,
scoring_func=scoring_func,
@@ -2678,7 +2679,7 @@ def select_experts(
renormalize=renormalize,
num_expert_group=num_expert_group,
topk_group=topk_group,
num_fused_shared_experts=num_fused_shared_experts_for_gate,
num_fused_shared_experts=num_shared_topk_slots_for_gate,
routed_scaling_factor=routed_scaling_factor,
apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output,
)
@@ -2710,7 +2711,7 @@ def select_experts(
_packed_kwargs = {}
if _fused_gate_emits_packed_ids(
scoring_func,
num_fused_shared_experts,
num_shared_topk_slots,
expert_location_dispatch_info,
routing_overridden,
topk_config.fused_gate_packed_ids,
@@ -2727,11 +2728,11 @@ def select_experts(
_packed_kwargs["router_logits_partials"] = router_logits_partials
if (
_use_aiter
and num_fused_shared_experts > 0
and not has_per_rank_fused_shared_slots(num_fused_shared_experts)
and num_shared_topk_slots > 0
and not has_per_rank_fused_shared_slots(num_shared_topk_slots)
):
# the ROCm decode gate writes the aiter shared columns itself
_packed_kwargs["num_shared_append"] = num_fused_shared_experts
_packed_kwargs["num_shared_append"] = num_shared_topk_slots
topk_weights, topk_ids = _biased_topk(
hidden_states=hidden_states,
gating_output=router_logits,
@@ -2739,7 +2740,7 @@ def select_experts(
topk=top_k_for_gate,
renormalize=renormalize,
scoring_func=scoring_func,
num_fused_shared_experts=num_fused_shared_experts_for_gate,
num_fused_shared_experts=num_shared_topk_slots_for_gate,
routed_scaling_factor=routed_scaling_factor,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=expert_location_dispatch_info,
@@ -2774,7 +2775,7 @@ def select_experts(
and scoring_func == "softmax"
and correction_bias is None
and expert_location_dispatch_info is None
and num_fused_shared_experts == 0
and num_shared_topk_slots == 0
and not routing_overridden
):
num_experts = router_logits.shape[-1]
@@ -2803,7 +2804,7 @@ def select_experts(
renormalize=renormalize,
correction_bias=correction_bias,
scoring_func=scoring_func,
num_fused_shared_experts=num_fused_shared_experts_for_gate,
num_fused_shared_experts=num_shared_topk_slots_for_gate,
routed_scaling_factor=routed_scaling_factor,
apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output,
**_fused_topk_kwargs,
@@ -2904,7 +2905,7 @@ def precomputed_topk_postprocess_is_noop(
"""
return (
_is_cuda
and topk_config.num_fused_shared_experts == 0
and topk_config.num_shared_topk_slots == 0
and num_token_non_padded is None
and expert_location_dispatch_info is None
and not envs.SGLANG_SIMULATE_UNIFORM_EXPERTS.get()
+21 -3
View File
@@ -694,9 +694,27 @@ def uses_per_rank_fused_shared_slots() -> bool:
return is_deepep_class_backend() or get_moe_a2a_backend().is_megamoe()
def has_per_rank_fused_shared_slots(num_fused_shared_experts: int) -> bool:
"""Check whether this layer has fused shared experts in per-rank slots."""
return num_fused_shared_experts > 0 and uses_per_rank_fused_shared_slots()
def has_per_rank_fused_shared_slots(num_shared: int) -> bool:
"""Whether the backend gives each rank its own fused shared-expert slots."""
return num_shared > 0 and uses_per_rank_fused_shared_slots()
def waterfill_routes_shared_experts(num_fused_shared_experts: int) -> bool:
"""Whether Waterfill, not the top-k router, writes the fused shared experts' column.
Waterfill appends that column after the EPLB remap and may place it on another
rank, so the router it follows emits the routed columns only."""
return num_fused_shared_experts > 0 and get_exec().moe.enable_waterfill
def shared_topk_slots(num_fused_shared_experts: int) -> int:
"""How many trailing top-k columns the router fills with fused shared experts.
``num_fused_shared_experts`` counts the shared experts the experts layer runs as
extra expert slots; the router writes their columns unless Waterfill does."""
if waterfill_routes_shared_experts(num_fused_shared_experts):
return 0
return num_fused_shared_experts
def is_flashinfer_cutedsl_v1_path() -> bool:
@@ -126,7 +126,7 @@ class Mxfp4FlashinferTrtllmMoEMethod:
and config.scoring_func == "sigmoid"
and config.correction_bias is not None
and config.custom_routing_function is None
and config.num_fused_shared_experts == 0
and config.num_shared_topk_slots == 0
and not config.torch_native
)
self._bypassed_routing_method = (
@@ -33,11 +33,11 @@ def vision_topk(
expert_location_dispatch_info=None,
):
config = moe.topk.topk_config
num_fused_shared_experts = config.num_fused_shared_experts
num_shared_topk_slots = config.num_shared_topk_slots
use_mega_moe = get_moe_a2a_backend().is_megamoe()
if num_fused_shared_experts and not use_mega_moe:
if num_shared_topk_slots and not use_mega_moe:
# This path bypasses _post_process_topk_ids, which appends the per-rank slots.
assert not has_per_rank_fused_shared_slots(num_fused_shared_experts), (
assert not has_per_rank_fused_shared_slots(num_shared_topk_slots), (
"VL routing does not support per-rank fused shared slots"
)
if is_cuda():
@@ -48,7 +48,7 @@ def vision_topk(
# rescaled below would rewrite weights after the router.
packed_topk = None
if (
num_fused_shared_experts == 0
num_shared_topk_slots == 0
and not use_mega_moe
and get_moe_runner_backend().is_flashinfer_mxfp4()
):
@@ -61,7 +61,7 @@ def vision_topk(
moe.gate.e_score_correction_bias,
topk=config.top_k,
scoring_func="sqrtsoftplus",
num_fused_shared_experts=num_fused_shared_experts,
num_fused_shared_experts=num_shared_topk_slots,
bias_alt=moe.gate.e_score_correction_bias_vl,
input_ids=input_ids,
bias_alt_token_id=moe.config.image_token_id,
@@ -95,19 +95,19 @@ def vision_topk(
moe.gate.e_score_correction_bias,
)
# The shared slots appended below use the same layout as biased_grouped_topk_gpu.
topk_routed = config.top_k - num_fused_shared_experts
topk_routed = config.top_k - num_shared_topk_slots
indices = (scores + bias).topk(topk_routed, dim=-1).indices
weights = scores.gather(-1, indices)
routed_sum = weights.sum(-1, keepdim=True, dtype=torch.float32)
if num_fused_shared_experts:
if num_shared_topk_slots:
shared_ids = logits.shape[-1] + torch.arange(
num_fused_shared_experts, device=indices.device, dtype=indices.dtype
num_shared_topk_slots, device=indices.device, dtype=indices.dtype
)
indices = torch.cat(
[indices, shared_ids.expand(indices.shape[0], -1)],
dim=-1,
)
weights = F.pad(weights, (0, num_fused_shared_experts))
weights = F.pad(weights, (0, num_shared_topk_slots))
weights[:, topk_routed:] = routed_sum / config.routed_scaling_factor
if config.renormalize and config.top_k > 1:
weights = weights / (routed_sum + _RENORMALIZE_SUM_EPSILON)
@@ -156,13 +156,13 @@ def _finish_vision_topk(
else:
weights = _scale_fused_shared_weights(
weights,
config.num_fused_shared_experts,
config.num_shared_topk_slots,
config.fused_shared_experts_scaling_factor,
)
# Shared-slot remapping can overwrite padded weights; mask last so idle
# and partially padded DP ranks never dispatch a phantom shared expert.
if num_token_non_padded is not None and (
not padded_rows_masked or (use_mega_moe and config.num_fused_shared_experts)
not padded_rows_masked or (use_mega_moe and config.num_shared_topk_slots)
):
_mask_topk_ids_padded_region(indices, num_token_non_padded)
_zero_topk_weights_padded_region(weights, num_token_non_padded)