[Refactor] moe: Split the two meanings of num_fused_shared_experts (#44912)
This commit is contained in:
@@ -61,7 +61,7 @@ def fused_topk_npu(
|
||||
if renormalize:
|
||||
topk_weights = l1_norm(
|
||||
topk_weights
|
||||
if topk_config.num_fused_shared_experts == 0
|
||||
if topk_config.num_shared_topk_slots == 0
|
||||
else topk_weights[:, :-1]
|
||||
)
|
||||
topk_weights = topk_weights.to(torch.float32)
|
||||
|
||||
@@ -22,8 +22,11 @@ from sglang.srt.layers.moe.topk import (
|
||||
_zero_topk_weights_padded_region,
|
||||
remap_topk_for_per_rank_shared_slots,
|
||||
)
|
||||
from sglang.srt.layers.moe.utils import has_per_rank_fused_shared_slots
|
||||
from sglang.srt.runtime_context import get_exec
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
has_per_rank_fused_shared_slots,
|
||||
shared_topk_slots,
|
||||
waterfill_routes_shared_experts,
|
||||
)
|
||||
from sglang.srt.utils import is_hip, is_npu, is_xpu
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -48,23 +51,20 @@ class HashTopK(nn.Module):
|
||||
super().__init__()
|
||||
self.layer_id = layer_id
|
||||
|
||||
self.enable_waterfill = (
|
||||
num_fused_shared_experts > 0 and get_exec().moe.enable_waterfill
|
||||
self.enable_waterfill = waterfill_routes_shared_experts(
|
||||
num_fused_shared_experts
|
||||
)
|
||||
self.waterfill_balancer = None
|
||||
|
||||
if self.enable_waterfill:
|
||||
# Waterfill appends the shared expert after EPLB maps routed IDs.
|
||||
topk -= num_fused_shared_experts
|
||||
num_fused_shared_experts = 0
|
||||
num_routed_experts_per_token = topk - num_fused_shared_experts
|
||||
|
||||
self.num_experts = num_experts
|
||||
self.topk = topk
|
||||
# Trailing columns the hash router fills with fused shared experts.
|
||||
self.num_shared_topk_slots = shared_topk_slots(num_fused_shared_experts)
|
||||
self.topk = num_routed_experts_per_token + self.num_shared_topk_slots
|
||||
self.routed_scaling_factor = routed_scaling_factor
|
||||
self.num_fused_shared_experts = num_fused_shared_experts
|
||||
self.score_func = scoring_func
|
||||
self.tid2eid = nn.Parameter(
|
||||
torch.empty(vocab_size, topk - num_fused_shared_experts, dtype=torch.int32),
|
||||
torch.empty(vocab_size, num_routed_experts_per_token, dtype=torch.int32),
|
||||
requires_grad=False,
|
||||
)
|
||||
self._init_default_tid2eid()
|
||||
@@ -72,7 +72,7 @@ class HashTopK(nn.Module):
|
||||
self.apply_routed_scaling_factor_on_output = (
|
||||
apply_routed_scaling_factor_on_output
|
||||
)
|
||||
if apply_routed_scaling_factor_on_output and num_fused_shared_experts > 0:
|
||||
if apply_routed_scaling_factor_on_output and self.num_shared_topk_slots > 0:
|
||||
raise NotImplementedError(
|
||||
"HashTopK + apply_routed_scaling_factor_on_output is not supported "
|
||||
"with fused shared experts; pass --disable-shared-experts-fusion."
|
||||
@@ -98,7 +98,7 @@ class HashTopK(nn.Module):
|
||||
def empty_topk_output(
|
||||
self, device: torch.device, *, layer_id: Optional[int] = None
|
||||
):
|
||||
topk = self.topk - self.num_fused_shared_experts
|
||||
topk = self.topk - self.num_shared_topk_slots
|
||||
if layer_id is not None:
|
||||
from sglang.srt.eplb.lplb_solver import get_global_lplb_solver
|
||||
|
||||
@@ -111,8 +111,8 @@ class HashTopK(nn.Module):
|
||||
topk_ids = torch.full((0, topk), -1, dtype=torch.int32, device=device)
|
||||
router_logits = torch.empty((0, topk), dtype=torch.float32, device=device)
|
||||
topk_output = StandardTopKOutput(topk_weights, topk_ids, router_logits)
|
||||
if has_per_rank_fused_shared_slots(self.num_fused_shared_experts):
|
||||
n = self.num_fused_shared_experts
|
||||
if has_per_rank_fused_shared_slots(self.num_shared_topk_slots):
|
||||
n = self.num_shared_topk_slots
|
||||
topk_output = topk_output._replace(
|
||||
topk_ids=topk_output.topk_ids.new_empty(
|
||||
(0, topk_output.topk_ids.shape[-1] + n)
|
||||
@@ -153,7 +153,7 @@ class HashTopK(nn.Module):
|
||||
(num_token, self.topk), dtype=scores.dtype, device=scores.device
|
||||
)
|
||||
|
||||
if self.num_fused_shared_experts == 1:
|
||||
if self.num_shared_topk_slots == 1:
|
||||
topk_ids[:, :-1] = self.tid2eid[input_ids]
|
||||
topk_weights[:, :-1] = scores.gather(1, topk_ids[:, :-1])
|
||||
|
||||
@@ -165,7 +165,7 @@ class HashTopK(nn.Module):
|
||||
|
||||
topk_ids[:, -1] = torch.randint(
|
||||
low=self.num_experts,
|
||||
high=self.num_experts + self.num_fused_shared_experts,
|
||||
high=self.num_experts + self.num_shared_topk_slots,
|
||||
size=(num_token,),
|
||||
dtype=topk_ids.dtype,
|
||||
device=topk_ids.device,
|
||||
@@ -195,7 +195,7 @@ class HashTopK(nn.Module):
|
||||
|
||||
num_tokens = router_logits.size(0)
|
||||
topk_routed = self.tid2eid.size(1)
|
||||
topk_fused = topk_routed + self.num_fused_shared_experts
|
||||
topk_fused = topk_routed + self.num_shared_topk_slots
|
||||
topk_ids = torch.empty(
|
||||
(num_tokens, topk_fused), dtype=torch.int32, device=router_logits.device
|
||||
)
|
||||
@@ -238,7 +238,7 @@ class HashTopK(nn.Module):
|
||||
router_logits=router_logits,
|
||||
input_ids=input_ids,
|
||||
tid2eid=self.tid2eid,
|
||||
num_fused_shared_experts=self.num_fused_shared_experts,
|
||||
num_fused_shared_experts=self.num_shared_topk_slots,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
scoring_func=self.score_func,
|
||||
)
|
||||
@@ -250,7 +250,7 @@ class HashTopK(nn.Module):
|
||||
if self.apply_routed_scaling_factor_on_output:
|
||||
topk_weights = topk_weights * self.routed_scaling_factor
|
||||
|
||||
num_fused_shared_experts = self.num_fused_shared_experts
|
||||
num_shared_topk_slots = self.num_shared_topk_slots
|
||||
log2phy_prob = None
|
||||
if (
|
||||
expert_location_dispatch_info is not None
|
||||
@@ -265,10 +265,10 @@ class HashTopK(nn.Module):
|
||||
log2phy_prob = lplb_solver.solve(topk_ids)
|
||||
|
||||
recorder_topk_ids = None
|
||||
if has_per_rank_fused_shared_slots(num_fused_shared_experts):
|
||||
if has_per_rank_fused_shared_slots(num_shared_topk_slots):
|
||||
# The trailing columns are placeholders; the remap appends the real ones.
|
||||
routed_cols = topk_ids_logical_to_physical(
|
||||
topk_ids[:, :-num_fused_shared_experts],
|
||||
topk_ids[:, :-num_shared_topk_slots],
|
||||
expert_location_dispatch_info,
|
||||
log2phy_prob,
|
||||
)
|
||||
@@ -281,12 +281,12 @@ class HashTopK(nn.Module):
|
||||
)
|
||||
topk_ids, topk_weights = remap_topk_for_per_rank_shared_slots(
|
||||
routed_cols,
|
||||
topk_weights[:, :-num_fused_shared_experts],
|
||||
num_fused_shared_experts,
|
||||
topk_weights[:, :-num_shared_topk_slots],
|
||||
num_shared_topk_slots,
|
||||
num_physical_routed_experts,
|
||||
TopKConfig(
|
||||
top_k=self.topk,
|
||||
num_fused_shared_experts=num_fused_shared_experts,
|
||||
num_shared_topk_slots=num_shared_topk_slots,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
apply_routed_scaling_factor_on_output=self.apply_routed_scaling_factor_on_output,
|
||||
),
|
||||
|
||||
@@ -104,7 +104,11 @@ from sglang.srt.layers.moe import (
|
||||
get_moe_runner_backend,
|
||||
is_moe_input_scattered_across_dp_ranks,
|
||||
)
|
||||
from sglang.srt.layers.moe.utils import has_per_rank_fused_shared_slots
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
has_per_rank_fused_shared_slots,
|
||||
shared_topk_slots,
|
||||
waterfill_routes_shared_experts,
|
||||
)
|
||||
from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer
|
||||
from sglang.srt.utils import (
|
||||
cpu_has_amx_support,
|
||||
@@ -160,7 +164,7 @@ def _use_rocm_triton_softmax_topk(
|
||||
gating_output: torch.Tensor,
|
||||
topk: int,
|
||||
correction_bias: Optional[torch.Tensor],
|
||||
num_fused_shared_experts: int,
|
||||
num_shared_topk_slots: int,
|
||||
packed_out: Optional[torch.Tensor],
|
||||
) -> bool:
|
||||
"""Use the lower-latency Triton router for Qwen3.5 decode-sized rows."""
|
||||
@@ -174,7 +178,7 @@ def _use_rocm_triton_softmax_topk(
|
||||
and gating_output.dtype == torch.bfloat16
|
||||
and topk == 10
|
||||
and correction_bias is None
|
||||
and num_fused_shared_experts == 0
|
||||
and num_shared_topk_slots == 0
|
||||
and packed_out is None
|
||||
)
|
||||
|
||||
@@ -254,7 +258,9 @@ class TopKConfig:
|
||||
topk_group: Optional[int] = None
|
||||
num_expert_group: Optional[int] = None
|
||||
renormalize: bool = True
|
||||
num_fused_shared_experts: int = 0
|
||||
# Trailing columns of top_k that the router fills with fused shared experts.
|
||||
# Zero when Waterfill appends the shared expert after routing.
|
||||
num_shared_topk_slots: int = 0
|
||||
custom_routing_function: Optional[Callable] = None
|
||||
correction_bias: Optional[torch.Tensor] = None
|
||||
torch_native: bool = False
|
||||
@@ -585,7 +591,9 @@ class TopK(BaseFusedOp):
|
||||
--top_k: The all number of top experts selected per token, including the fused shared expert(s).
|
||||
--layer_id: global index of the owning layer. Required: routed-experts capture and
|
||||
per-layer EPLB state are keyed by it.
|
||||
--num_fused_shared_experts: num of shared experts, can be activate both in TP or EP mode.
|
||||
--num_fused_shared_experts: shared experts the experts layer runs as extra expert slots.
|
||||
Their trailing top-k columns are written by the router, or by Waterfill when it is
|
||||
enabled (``TopKConfig.num_shared_topk_slots`` then excludes them).
|
||||
--routed_scaling_factor: the scaling factor for routed experts in topk_weights.
|
||||
--fused_shared_experts_scaling_factor: absolute appended weight on the global AITER
|
||||
path; otherwise multiplies the router's existing shared weight under standard EP.
|
||||
@@ -624,33 +632,32 @@ class TopK(BaseFusedOp):
|
||||
|
||||
self.layer_id = layer_id
|
||||
|
||||
self.enable_waterfill = (
|
||||
num_fused_shared_experts > 0 and get_exec().moe.enable_waterfill
|
||||
self.enable_waterfill = waterfill_routes_shared_experts(
|
||||
num_fused_shared_experts
|
||||
)
|
||||
|
||||
self.waterfill_balancer = None
|
||||
if self.enable_waterfill:
|
||||
# TODO(ch-wan): Refactor shared-expert fusion and routed TopK fusion.
|
||||
top_k -= num_fused_shared_experts
|
||||
num_fused_shared_experts = 0
|
||||
output_format = TopKOutputFormat.STANDARD
|
||||
num_shared_slots = shared_topk_slots(num_fused_shared_experts)
|
||||
num_routed_experts_per_token = top_k - num_fused_shared_experts
|
||||
|
||||
# Under the flashinfer_mxfp4 backend, fp4-expert ckpts take STANDARD
|
||||
# (consumes topk_ids/weights); otherwise BYPASSED. No-op on other backends.
|
||||
self.is_fp4_experts = is_fp4_experts
|
||||
self.topk_config = TopKConfig(
|
||||
top_k=top_k,
|
||||
top_k=num_routed_experts_per_token + num_shared_slots,
|
||||
use_grouped_topk=use_grouped_topk,
|
||||
renormalize=renormalize,
|
||||
topk_group=topk_group,
|
||||
num_expert_group=num_expert_group,
|
||||
num_fused_shared_experts=num_fused_shared_experts,
|
||||
num_shared_topk_slots=num_shared_slots,
|
||||
custom_routing_function=custom_routing_function,
|
||||
correction_bias=correction_bias,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output,
|
||||
fused_shared_experts_scaling_factor=fused_shared_experts_scaling_factor,
|
||||
output_format=output_format,
|
||||
# Waterfill expands a standard top-k output.
|
||||
output_format=(
|
||||
TopKOutputFormat.STANDARD if self.enable_waterfill else output_format
|
||||
),
|
||||
scoring_func=scoring_func,
|
||||
sqrtsoftplus_log1p=sqrtsoftplus_log1p,
|
||||
fused_gate_packed_ids=fused_gate_packed_ids,
|
||||
@@ -869,7 +876,7 @@ class TopK(BaseFusedOp):
|
||||
device=device,
|
||||
)
|
||||
)
|
||||
topk = self.topk_config.top_k - self.topk_config.num_fused_shared_experts
|
||||
topk = self.topk_config.top_k - self.topk_config.num_shared_topk_slots
|
||||
with use_symmetric_memory(
|
||||
get_parallel().tp_group, disabled=not is_allocation_symmetric()
|
||||
):
|
||||
@@ -878,8 +885,8 @@ class TopK(BaseFusedOp):
|
||||
# FIXME: router_logits should be of size (0, num_experts)
|
||||
router_logits = torch.empty((0, topk), dtype=torch.float32, device=device)
|
||||
topk_output = StandardTopKOutput(topk_weights, topk_ids, router_logits)
|
||||
if has_per_rank_fused_shared_slots(self.topk_config.num_fused_shared_experts):
|
||||
n = self.topk_config.num_fused_shared_experts
|
||||
if has_per_rank_fused_shared_slots(self.topk_config.num_shared_topk_slots):
|
||||
n = self.topk_config.num_shared_topk_slots
|
||||
topk_output = topk_output._replace(
|
||||
topk_ids=topk_output.topk_ids.new_empty(
|
||||
(0, topk_output.topk_ids.shape[-1] + n)
|
||||
@@ -1720,7 +1727,7 @@ def _fused_gate_masks_padded_rows(scoring_func: str) -> bool:
|
||||
|
||||
def _fused_gate_emits_packed_ids(
|
||||
scoring_func: str,
|
||||
num_fused_shared_experts: int,
|
||||
num_shared_topk_slots: int,
|
||||
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo],
|
||||
routing_overridden: bool,
|
||||
enabled: bool,
|
||||
@@ -1732,7 +1739,7 @@ def _fused_gate_emits_packed_ids(
|
||||
and _fused_gate_masks_padded_rows(scoring_func)
|
||||
and get_moe_runner_backend().is_flashinfer_mxfp4()
|
||||
and expert_location_dispatch_info is None
|
||||
and num_fused_shared_experts == 0
|
||||
and num_shared_topk_slots == 0
|
||||
and not routing_overridden
|
||||
)
|
||||
|
||||
@@ -2310,28 +2317,26 @@ def _post_process_topk_ids(
|
||||
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None,
|
||||
padded_rows_masked: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
num_fused_shared_experts = topk_config.num_fused_shared_experts
|
||||
use_per_rank_shared_slots = has_per_rank_fused_shared_slots(
|
||||
num_fused_shared_experts
|
||||
)
|
||||
num_shared_topk_slots = topk_config.num_shared_topk_slots
|
||||
use_per_rank_shared_slots = has_per_rank_fused_shared_slots(num_shared_topk_slots)
|
||||
fused_shared_experts_scaling_factor = (
|
||||
topk_config.fused_shared_experts_scaling_factor
|
||||
)
|
||||
# False when select_experts asked the router for routed experts only.
|
||||
router_wrote_shared_cols = (
|
||||
num_fused_shared_experts > 0 and topk_ids.shape[-1] == topk_config.top_k
|
||||
num_shared_topk_slots > 0 and topk_ids.shape[-1] == topk_config.top_k
|
||||
)
|
||||
# The router already wrote the aiter shared columns (see _aiter_append below).
|
||||
_gate_appended = (
|
||||
router_wrote_shared_cols and _use_aiter and not use_per_rank_shared_slots
|
||||
)
|
||||
routed_topk_ids = (
|
||||
topk_ids[:, :-num_fused_shared_experts] if _gate_appended else topk_ids
|
||||
topk_ids[:, :-num_shared_topk_slots] if _gate_appended else topk_ids
|
||||
)
|
||||
capture_routed_experts_if_allowed(topk_config, layer_id, routed_topk_ids)
|
||||
recorder_topk_ids = None
|
||||
_aiter_append = (
|
||||
num_fused_shared_experts > 0
|
||||
num_shared_topk_slots > 0
|
||||
and _use_aiter
|
||||
and topk_ids.shape[-1] < topk_config.top_k
|
||||
)
|
||||
@@ -2360,8 +2365,8 @@ def _post_process_topk_ids(
|
||||
elif router_wrote_shared_cols:
|
||||
# Shared IDs are outside EPLB's routed-expert table for both global
|
||||
# and per-rank layouts, so remap only routed columns.
|
||||
shared_cols = topk_ids[:, -num_fused_shared_experts:]
|
||||
routed_cols = topk_ids[:, :-num_fused_shared_experts]
|
||||
shared_cols = topk_ids[:, -num_shared_topk_slots:]
|
||||
routed_cols = topk_ids[:, :-num_shared_topk_slots]
|
||||
routed_cols = _biased_grouped_topk_postprocess(
|
||||
routed_cols, expert_location_dispatch_info, num_token_non_padded
|
||||
)
|
||||
@@ -2438,7 +2443,7 @@ def _post_process_topk_ids(
|
||||
|
||||
if recorder_topk_ids is None and not recorder_was_fused:
|
||||
recorder_topk_ids = (
|
||||
topk_ids[:, :-num_fused_shared_experts] if _gate_appended else topk_ids
|
||||
topk_ids[:, :-num_shared_topk_slots] if _gate_appended else topk_ids
|
||||
)
|
||||
|
||||
if _aiter_append and use_per_rank_shared_slots:
|
||||
@@ -2455,7 +2460,7 @@ def _post_process_topk_ids(
|
||||
ep_size = get_parallel().moe_ep_size
|
||||
ep_rank = get_parallel().moe_ep_rank
|
||||
num_local_routed = num_physical_routed_experts // ep_size
|
||||
num_local_experts = num_local_routed + num_fused_shared_experts
|
||||
num_local_experts = num_local_routed + num_shared_topk_slots
|
||||
shared_id_base = ep_rank * num_local_experts + num_local_routed
|
||||
|
||||
# Lazy import to avoid circular-import issues
|
||||
@@ -2466,7 +2471,7 @@ def _post_process_topk_ids(
|
||||
topk_ids, topk_weights = fused_append_remap_shared_experts_deepep(
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
num_fused_shared_experts,
|
||||
num_shared_topk_slots,
|
||||
fused_shared_expert_weight(
|
||||
routed_scaling_factor=topk_config.routed_scaling_factor,
|
||||
fused_in_topk=is_routed_scaling_factor_fused_in_topk(topk_config),
|
||||
@@ -2493,7 +2498,7 @@ def _post_process_topk_ids(
|
||||
topk_ids, topk_weights = fused_append_shared_experts(
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
num_fused_shared_experts,
|
||||
num_shared_topk_slots,
|
||||
scale_factor,
|
||||
N, # base id for shared experts
|
||||
num_token_non_padded=(
|
||||
@@ -2511,26 +2516,22 @@ def _post_process_topk_ids(
|
||||
)
|
||||
if router_wrote_shared_cols:
|
||||
# Placeholders; the remap appends the real shared columns.
|
||||
topk_ids = topk_ids[:, :-num_fused_shared_experts]
|
||||
topk_weights = topk_weights[:, :-num_fused_shared_experts]
|
||||
topk_ids = topk_ids[:, :-num_shared_topk_slots]
|
||||
topk_weights = topk_weights[:, :-num_shared_topk_slots]
|
||||
topk_ids, topk_weights = remap_topk_for_per_rank_shared_slots(
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
num_fused_shared_experts,
|
||||
num_shared_topk_slots,
|
||||
num_physical_routed_experts,
|
||||
topk_config,
|
||||
)
|
||||
if _is_cuda:
|
||||
# The remap shifts a padded row's -1 and gives it a live shared slot.
|
||||
_mask_topk_ids_padded_region(topk_ids, num_token_non_padded)
|
||||
elif (
|
||||
num_fused_shared_experts > 0 and fused_shared_experts_scaling_factor is not None
|
||||
):
|
||||
elif num_shared_topk_slots > 0 and fused_shared_experts_scaling_factor is not None:
|
||||
# Standard EP all-reduces the per-rank shared-expert outputs; without the
|
||||
# supplied 1/ep_size factor the shared contribution is summed ep_size times.
|
||||
topk_weights[:, -num_fused_shared_experts:] *= (
|
||||
fused_shared_experts_scaling_factor
|
||||
)
|
||||
topk_weights[:, -num_shared_topk_slots:] *= fused_shared_experts_scaling_factor
|
||||
|
||||
if _is_hip and not _skip_hip_pad_mask and not _fold_pad_weights_into_append:
|
||||
# Shared-expert append/remap can introduce non-zero weights after the
|
||||
@@ -2557,7 +2558,7 @@ def select_experts(
|
||||
topk_group = topk_config.topk_group
|
||||
num_expert_group = topk_config.num_expert_group
|
||||
renormalize = topk_config.renormalize
|
||||
num_fused_shared_experts = topk_config.num_fused_shared_experts
|
||||
num_shared_topk_slots = topk_config.num_shared_topk_slots
|
||||
custom_routing_function = topk_config.custom_routing_function
|
||||
correction_bias = (
|
||||
topk_config.correction_bias
|
||||
@@ -2594,8 +2595,8 @@ def select_experts(
|
||||
and not torch_native
|
||||
and expert_location_dispatch_info is None
|
||||
and (
|
||||
num_fused_shared_experts == 0
|
||||
or has_per_rank_fused_shared_slots(num_fused_shared_experts)
|
||||
num_shared_topk_slots == 0
|
||||
or has_per_rank_fused_shared_slots(num_shared_topk_slots)
|
||||
or not is_expert_remapping_enabled()
|
||||
)
|
||||
):
|
||||
@@ -2624,15 +2625,15 @@ def select_experts(
|
||||
):
|
||||
correction_bias = topk_config.correction_bias_for_dtype(router_logits.dtype)
|
||||
|
||||
num_routed_topk = top_k - num_fused_shared_experts
|
||||
num_routed_topk = top_k - num_shared_topk_slots
|
||||
# On aiter and on the per-rank shared-slot path, _post_process_topk_ids appends
|
||||
# the shared columns, so the router selects and renormalizes routed experts only.
|
||||
if _use_aiter or has_per_rank_fused_shared_slots(num_fused_shared_experts):
|
||||
if _use_aiter or has_per_rank_fused_shared_slots(num_shared_topk_slots):
|
||||
top_k_for_gate = num_routed_topk
|
||||
num_fused_shared_experts_for_gate = 0
|
||||
num_shared_topk_slots_for_gate = 0
|
||||
else:
|
||||
top_k_for_gate = top_k
|
||||
num_fused_shared_experts_for_gate = num_fused_shared_experts
|
||||
num_shared_topk_slots_for_gate = num_shared_topk_slots
|
||||
if dynamic_expert_bias is not None:
|
||||
if scoring_func != "sigmoid":
|
||||
raise ValueError(
|
||||
@@ -2646,7 +2647,7 @@ def select_experts(
|
||||
renormalize=renormalize,
|
||||
num_expert_group=num_expert_group or 1,
|
||||
topk_group=topk_group or 1,
|
||||
num_fused_shared_experts=num_fused_shared_experts,
|
||||
num_fused_shared_experts=num_shared_topk_slots,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
# Per-token bias bypasses AITER but must retain its folding policy.
|
||||
apply_routed_scaling_factor_on_output=is_routed_scaling_factor_fused_in_topk(
|
||||
@@ -2664,7 +2665,7 @@ def select_experts(
|
||||
renormalize=renormalize,
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
num_fused_shared_experts=num_fused_shared_experts_for_gate,
|
||||
num_fused_shared_experts=num_shared_topk_slots_for_gate,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output,
|
||||
scoring_func=scoring_func,
|
||||
@@ -2678,7 +2679,7 @@ def select_experts(
|
||||
renormalize=renormalize,
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
num_fused_shared_experts=num_fused_shared_experts_for_gate,
|
||||
num_fused_shared_experts=num_shared_topk_slots_for_gate,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output,
|
||||
)
|
||||
@@ -2710,7 +2711,7 @@ def select_experts(
|
||||
_packed_kwargs = {}
|
||||
if _fused_gate_emits_packed_ids(
|
||||
scoring_func,
|
||||
num_fused_shared_experts,
|
||||
num_shared_topk_slots,
|
||||
expert_location_dispatch_info,
|
||||
routing_overridden,
|
||||
topk_config.fused_gate_packed_ids,
|
||||
@@ -2727,11 +2728,11 @@ def select_experts(
|
||||
_packed_kwargs["router_logits_partials"] = router_logits_partials
|
||||
if (
|
||||
_use_aiter
|
||||
and num_fused_shared_experts > 0
|
||||
and not has_per_rank_fused_shared_slots(num_fused_shared_experts)
|
||||
and num_shared_topk_slots > 0
|
||||
and not has_per_rank_fused_shared_slots(num_shared_topk_slots)
|
||||
):
|
||||
# the ROCm decode gate writes the aiter shared columns itself
|
||||
_packed_kwargs["num_shared_append"] = num_fused_shared_experts
|
||||
_packed_kwargs["num_shared_append"] = num_shared_topk_slots
|
||||
topk_weights, topk_ids = _biased_topk(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
@@ -2739,7 +2740,7 @@ def select_experts(
|
||||
topk=top_k_for_gate,
|
||||
renormalize=renormalize,
|
||||
scoring_func=scoring_func,
|
||||
num_fused_shared_experts=num_fused_shared_experts_for_gate,
|
||||
num_fused_shared_experts=num_shared_topk_slots_for_gate,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
expert_location_dispatch_info=expert_location_dispatch_info,
|
||||
@@ -2774,7 +2775,7 @@ def select_experts(
|
||||
and scoring_func == "softmax"
|
||||
and correction_bias is None
|
||||
and expert_location_dispatch_info is None
|
||||
and num_fused_shared_experts == 0
|
||||
and num_shared_topk_slots == 0
|
||||
and not routing_overridden
|
||||
):
|
||||
num_experts = router_logits.shape[-1]
|
||||
@@ -2803,7 +2804,7 @@ def select_experts(
|
||||
renormalize=renormalize,
|
||||
correction_bias=correction_bias,
|
||||
scoring_func=scoring_func,
|
||||
num_fused_shared_experts=num_fused_shared_experts_for_gate,
|
||||
num_fused_shared_experts=num_shared_topk_slots_for_gate,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output,
|
||||
**_fused_topk_kwargs,
|
||||
@@ -2904,7 +2905,7 @@ def precomputed_topk_postprocess_is_noop(
|
||||
"""
|
||||
return (
|
||||
_is_cuda
|
||||
and topk_config.num_fused_shared_experts == 0
|
||||
and topk_config.num_shared_topk_slots == 0
|
||||
and num_token_non_padded is None
|
||||
and expert_location_dispatch_info is None
|
||||
and not envs.SGLANG_SIMULATE_UNIFORM_EXPERTS.get()
|
||||
|
||||
@@ -694,9 +694,27 @@ def uses_per_rank_fused_shared_slots() -> bool:
|
||||
return is_deepep_class_backend() or get_moe_a2a_backend().is_megamoe()
|
||||
|
||||
|
||||
def has_per_rank_fused_shared_slots(num_fused_shared_experts: int) -> bool:
|
||||
"""Check whether this layer has fused shared experts in per-rank slots."""
|
||||
return num_fused_shared_experts > 0 and uses_per_rank_fused_shared_slots()
|
||||
def has_per_rank_fused_shared_slots(num_shared: int) -> bool:
|
||||
"""Whether the backend gives each rank its own fused shared-expert slots."""
|
||||
return num_shared > 0 and uses_per_rank_fused_shared_slots()
|
||||
|
||||
|
||||
def waterfill_routes_shared_experts(num_fused_shared_experts: int) -> bool:
|
||||
"""Whether Waterfill, not the top-k router, writes the fused shared experts' column.
|
||||
|
||||
Waterfill appends that column after the EPLB remap and may place it on another
|
||||
rank, so the router it follows emits the routed columns only."""
|
||||
return num_fused_shared_experts > 0 and get_exec().moe.enable_waterfill
|
||||
|
||||
|
||||
def shared_topk_slots(num_fused_shared_experts: int) -> int:
|
||||
"""How many trailing top-k columns the router fills with fused shared experts.
|
||||
|
||||
``num_fused_shared_experts`` counts the shared experts the experts layer runs as
|
||||
extra expert slots; the router writes their columns unless Waterfill does."""
|
||||
if waterfill_routes_shared_experts(num_fused_shared_experts):
|
||||
return 0
|
||||
return num_fused_shared_experts
|
||||
|
||||
|
||||
def is_flashinfer_cutedsl_v1_path() -> bool:
|
||||
|
||||
@@ -126,7 +126,7 @@ class Mxfp4FlashinferTrtllmMoEMethod:
|
||||
and config.scoring_func == "sigmoid"
|
||||
and config.correction_bias is not None
|
||||
and config.custom_routing_function is None
|
||||
and config.num_fused_shared_experts == 0
|
||||
and config.num_shared_topk_slots == 0
|
||||
and not config.torch_native
|
||||
)
|
||||
self._bypassed_routing_method = (
|
||||
|
||||
@@ -33,11 +33,11 @@ def vision_topk(
|
||||
expert_location_dispatch_info=None,
|
||||
):
|
||||
config = moe.topk.topk_config
|
||||
num_fused_shared_experts = config.num_fused_shared_experts
|
||||
num_shared_topk_slots = config.num_shared_topk_slots
|
||||
use_mega_moe = get_moe_a2a_backend().is_megamoe()
|
||||
if num_fused_shared_experts and not use_mega_moe:
|
||||
if num_shared_topk_slots and not use_mega_moe:
|
||||
# This path bypasses _post_process_topk_ids, which appends the per-rank slots.
|
||||
assert not has_per_rank_fused_shared_slots(num_fused_shared_experts), (
|
||||
assert not has_per_rank_fused_shared_slots(num_shared_topk_slots), (
|
||||
"VL routing does not support per-rank fused shared slots"
|
||||
)
|
||||
if is_cuda():
|
||||
@@ -48,7 +48,7 @@ def vision_topk(
|
||||
# rescaled below would rewrite weights after the router.
|
||||
packed_topk = None
|
||||
if (
|
||||
num_fused_shared_experts == 0
|
||||
num_shared_topk_slots == 0
|
||||
and not use_mega_moe
|
||||
and get_moe_runner_backend().is_flashinfer_mxfp4()
|
||||
):
|
||||
@@ -61,7 +61,7 @@ def vision_topk(
|
||||
moe.gate.e_score_correction_bias,
|
||||
topk=config.top_k,
|
||||
scoring_func="sqrtsoftplus",
|
||||
num_fused_shared_experts=num_fused_shared_experts,
|
||||
num_fused_shared_experts=num_shared_topk_slots,
|
||||
bias_alt=moe.gate.e_score_correction_bias_vl,
|
||||
input_ids=input_ids,
|
||||
bias_alt_token_id=moe.config.image_token_id,
|
||||
@@ -95,19 +95,19 @@ def vision_topk(
|
||||
moe.gate.e_score_correction_bias,
|
||||
)
|
||||
# The shared slots appended below use the same layout as biased_grouped_topk_gpu.
|
||||
topk_routed = config.top_k - num_fused_shared_experts
|
||||
topk_routed = config.top_k - num_shared_topk_slots
|
||||
indices = (scores + bias).topk(topk_routed, dim=-1).indices
|
||||
weights = scores.gather(-1, indices)
|
||||
routed_sum = weights.sum(-1, keepdim=True, dtype=torch.float32)
|
||||
if num_fused_shared_experts:
|
||||
if num_shared_topk_slots:
|
||||
shared_ids = logits.shape[-1] + torch.arange(
|
||||
num_fused_shared_experts, device=indices.device, dtype=indices.dtype
|
||||
num_shared_topk_slots, device=indices.device, dtype=indices.dtype
|
||||
)
|
||||
indices = torch.cat(
|
||||
[indices, shared_ids.expand(indices.shape[0], -1)],
|
||||
dim=-1,
|
||||
)
|
||||
weights = F.pad(weights, (0, num_fused_shared_experts))
|
||||
weights = F.pad(weights, (0, num_shared_topk_slots))
|
||||
weights[:, topk_routed:] = routed_sum / config.routed_scaling_factor
|
||||
if config.renormalize and config.top_k > 1:
|
||||
weights = weights / (routed_sum + _RENORMALIZE_SUM_EPSILON)
|
||||
@@ -156,13 +156,13 @@ def _finish_vision_topk(
|
||||
else:
|
||||
weights = _scale_fused_shared_weights(
|
||||
weights,
|
||||
config.num_fused_shared_experts,
|
||||
config.num_shared_topk_slots,
|
||||
config.fused_shared_experts_scaling_factor,
|
||||
)
|
||||
# Shared-slot remapping can overwrite padded weights; mask last so idle
|
||||
# and partially padded DP ranks never dispatch a phantom shared expert.
|
||||
if num_token_non_padded is not None and (
|
||||
not padded_rows_masked or (use_mega_moe and config.num_fused_shared_experts)
|
||||
not padded_rows_masked or (use_mega_moe and config.num_shared_topk_slots)
|
||||
):
|
||||
_mask_topk_ids_padded_region(indices, num_token_non_padded)
|
||||
_zero_topk_weights_padded_region(weights, num_token_non_padded)
|
||||
|
||||
Reference in New Issue
Block a user