[Fix] moe: Renormalize per-rank shared-slot routing over routed experts only (#44904)

This commit is contained in:
Cheng Wan
2026-10-19 16:56:21 -07:00
committed by GitHub
parent 3a059de94c
commit 48ea486928
11 changed files with 486 additions and 127 deletions
+6 -6
View File
@@ -266,12 +266,12 @@ class HashTopK(nn.Module):
recorder_topk_ids = None
if has_per_rank_fused_shared_slots(num_fused_shared_experts):
shared_cols = topk_ids[:, -num_fused_shared_experts:]
routed_cols = topk_ids[:, :-num_fused_shared_experts]
# The trailing columns are placeholders; the remap appends the real ones.
routed_cols = topk_ids_logical_to_physical(
routed_cols, expert_location_dispatch_info, log2phy_prob
topk_ids[:, :-num_fused_shared_experts],
expert_location_dispatch_info,
log2phy_prob,
)
topk_ids = torch.cat([routed_cols, shared_cols], dim=-1)
recorder_topk_ids = routed_cols
num_physical_routed_experts = (
@@ -280,8 +280,8 @@ class HashTopK(nn.Module):
else self.num_experts
)
topk_ids, topk_weights = remap_topk_for_per_rank_shared_slots(
topk_ids,
topk_weights,
routed_cols,
topk_weights[:, :-num_fused_shared_experts],
num_fused_shared_experts,
num_physical_routed_experts,
TopKConfig(
+63 -58
View File
@@ -198,7 +198,7 @@ if _is_cuda:
scaling_factor: float,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
renormalize: bool,
launch_with_pdl: bool,
) -> None:
_fused_topk_deepseek(
gating_output,
@@ -209,7 +209,7 @@ if _is_cuda:
scaling_factor,
topk_weights,
topk_ids,
renormalize,
launch_with_pdl,
)
except ImportError:
@@ -1815,8 +1815,8 @@ def biased_grouped_topk_gpu(
moe_fused_gate as jit_grouped_gate,
)
# The kernel wants the total width; select_experts passes a routed-only
# topk on the aiter path only (`num_routed_topk if _use_aiter else top_k`).
# The kernel wants the total width; off aiter, topk already counts the
# num_fused_shared_experts slots it writes.
#
# AITER folds this route even when no shared slots are requested.
return jit_grouped_gate(
@@ -1838,6 +1838,8 @@ def biased_grouped_topk_gpu(
if (
_is_cuda
and fused_topk_deepseek is not None
# flashinfer always renormalizes the selected weights
and renormalize
and is_power_of_two(num_experts)
# flashinfer constraints (applied to routed experts only)
and topk_routed <= 8
@@ -1873,21 +1875,21 @@ def biased_grouped_topk_gpu(
scaling_factor,
topk_weights,
topk_ids,
True,
launch_with_pdl=True,
)
if num_fused_shared_experts > 0:
# Append shared expert columns: ID = num_experts (first shared slot),
# weight = sum(routed) / scaling_factor (matching biased_grouped_topk_impl).
# For DeepEP/MegaMOE per-rank shared-slot layout, post-process remaps
# this placeholder ID and overwrites the shared weight for the active scaling path.
# Append the global shared columns: ID = num_experts (first shared slot).
# On CUDA the routed weights carry the factor exactly when asked to.
topk_ids = F.pad(topk_ids, (0, num_fused_shared_experts), value=num_experts)
topk_weights = F.pad(topk_weights, (0, num_fused_shared_experts))
if routed_scaling_factor is not None:
topk_weights[:, topk_routed:] = (
topk_weights[:, :topk_routed].sum(dim=-1, keepdim=True)
/ routed_scaling_factor
)
topk_weights = F.pad(
topk_weights,
(0, num_fused_shared_experts),
value=fused_shared_expert_weight(
routed_scaling_factor=routed_scaling_factor,
fused_in_topk=bool(apply_routed_scaling_factor_on_output),
),
)
return topk_weights, topk_ids
@@ -2240,10 +2242,7 @@ def remap_topk_for_per_rank_shared_slots(
num_physical_routed_experts: int,
topk_config: TopKConfig,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Map routed and shared columns into rank-major expert blocks."""
if topk_ids.shape[0] == 0:
return topk_ids, topk_weights
"""Append shared columns to routed-only inputs in rank-major expert blocks."""
ep_size = get_parallel().moe_ep_size
ep_rank = get_parallel().moe_ep_rank
# Static EPLB may add redundant physical experts. At this point routed
@@ -2251,31 +2250,36 @@ def remap_topk_for_per_rank_shared_slots(
# per-rank shared-slot layout must use the physical routed count.
num_local_routed = num_physical_routed_experts // ep_size
num_local_experts = num_local_routed + num_fused_shared_experts
num_tokens = topk_ids.shape[0]
# Remap routed IDs: insert gaps for shared expert slots (single fused op).
# Each rank r == e // num_local_routed is preceded by r shared-slot blocks of
# width num_fused_shared_experts, so shift by (e // num_local_routed) * S --
# a single-slot shift (S == 1) would let routed ids collide with an earlier
# rank's shared slots once S > 1.
routed = topk_ids[:, :-num_fused_shared_experts]
topk_ids[:, :-num_fused_shared_experts] = (
routed + (routed // num_local_routed) * num_fused_shared_experts
)
routed_ids = topk_ids + (topk_ids // num_local_routed) * num_fused_shared_experts
# Set shared expert IDs to route to home rank (vectorized)
topk_ids[:, -num_fused_shared_experts:] = (
# Shared expert IDs route to the home rank.
shared_ids = (
ep_rank * num_local_experts
+ num_local_routed
+ torch.arange(num_fused_shared_experts, device=topk_ids.device)
+ torch.arange(
num_fused_shared_experts, dtype=topk_ids.dtype, device=topk_ids.device
)
).expand(num_tokens, -1)
shared_weights = topk_weights.new_full(
(num_tokens, num_fused_shared_experts),
fused_shared_expert_weight(
routed_scaling_factor=topk_config.routed_scaling_factor,
fused_in_topk=is_routed_scaling_factor_fused_in_topk(topk_config),
),
)
topk_weights[:, -num_fused_shared_experts:] = fused_shared_expert_weight(
routed_scaling_factor=topk_config.routed_scaling_factor,
fused_in_topk=is_routed_scaling_factor_fused_in_topk(topk_config),
return (
torch.cat([routed_ids, shared_ids], dim=-1),
torch.cat([topk_weights, shared_weights], dim=-1),
)
return topk_ids, topk_weights
def capture_routed_experts_if_allowed(
topk_config: TopKConfig,
@@ -2313,12 +2317,13 @@ def _post_process_topk_ids(
fused_shared_experts_scaling_factor = (
topk_config.fused_shared_experts_scaling_factor
)
# False when select_experts asked the router for routed experts only.
router_wrote_shared_cols = (
num_fused_shared_experts > 0 and topk_ids.shape[-1] == topk_config.top_k
)
# The router already wrote the aiter shared columns (see _aiter_append below).
_gate_appended = (
num_fused_shared_experts > 0
and _use_aiter
and not use_per_rank_shared_slots
and topk_ids.shape[-1] == topk_config.top_k
router_wrote_shared_cols and _use_aiter and not use_per_rank_shared_slots
)
routed_topk_ids = (
topk_ids[:, :-num_fused_shared_experts] if _gate_appended else topk_ids
@@ -2352,7 +2357,7 @@ def _post_process_topk_ids(
topk_ids, expert_location_dispatch_info, log2phy_prob
)
_mask_topk_ids_padded_region(topk_ids, num_token_non_padded)
elif num_fused_shared_experts > 0:
elif router_wrote_shared_cols:
# Shared IDs are outside EPLB's routed-expert table for both global
# and per-rank layouts, so remap only routed columns.
shared_cols = topk_ids[:, -num_fused_shared_experts:]
@@ -2504,6 +2509,10 @@ def _post_process_topk_ids(
if expert_location_dispatch_info is not None
else router_logits.shape[1]
)
if router_wrote_shared_cols:
# Placeholders; the remap appends the real shared columns.
topk_ids = topk_ids[:, :-num_fused_shared_experts]
topk_weights = topk_weights[:, :-num_fused_shared_experts]
topk_ids, topk_weights = remap_topk_for_per_rank_shared_slots(
topk_ids,
topk_weights,
@@ -2511,6 +2520,9 @@ def _post_process_topk_ids(
num_physical_routed_experts,
topk_config,
)
if _is_cuda:
# The remap shifts a padded row's -1 and gives it a live shared slot.
_mask_topk_ids_padded_region(topk_ids, num_token_non_padded)
elif (
num_fused_shared_experts > 0 and fused_shared_experts_scaling_factor is not None
):
@@ -2612,22 +2624,15 @@ def select_experts(
):
correction_bias = topk_config.correction_bias_for_dtype(router_logits.dtype)
# DeepSeek V2/V3/R1 series models use grouped_top_k
# remove num_fused_shared_experts from grouped_topk/biased_grouped_topk
num_routed_topk = top_k - num_fused_shared_experts
# On the per-rank shared-slot path the shared expert is appended afterwards by
# fused_append_remap_shared_experts_deepep, so the gate must not emit a shared
# marker as well. Doing both costs one routed expert (the gate is asked for
# K_routed = top_k - num_fused_shared_experts and then spends one of those
# slots on the marker) and places that marker at id num_experts, which the
# DeepEP remap shifts one past the end of the expert space -- 384 -> 392 for
# 384 routed experts on EP8, where the valid ids are 0..391.
# aiter appends the shared expert in _post_process_topk_ids, so the gate must not.
num_fused_shared_experts_for_gate = (
0
if (has_per_rank_fused_shared_slots(num_fused_shared_experts) or _use_aiter)
else num_fused_shared_experts
)
# On aiter and on the per-rank shared-slot path, _post_process_topk_ids appends
# the shared columns, so the router selects and renormalizes routed experts only.
if _use_aiter or has_per_rank_fused_shared_slots(num_fused_shared_experts):
top_k_for_gate = num_routed_topk
num_fused_shared_experts_for_gate = 0
else:
top_k_for_gate = top_k
num_fused_shared_experts_for_gate = num_fused_shared_experts
if dynamic_expert_bias is not None:
if scoring_func != "sigmoid":
raise ValueError(
@@ -2655,7 +2660,7 @@ def select_experts(
topk_weights, topk_ids = grouped_topk(
hidden_states=hidden_states,
gating_output=router_logits,
topk=num_routed_topk if _use_aiter else top_k,
topk=top_k_for_gate,
renormalize=renormalize,
num_expert_group=num_expert_group,
topk_group=topk_group,
@@ -2669,7 +2674,7 @@ def select_experts(
hidden_states=hidden_states,
gating_output=router_logits,
correction_bias=correction_bias,
topk=num_routed_topk if _use_aiter else top_k,
topk=top_k_for_gate,
renormalize=renormalize,
num_expert_group=num_expert_group,
topk_group=topk_group,
@@ -2686,7 +2691,7 @@ def select_experts(
topk_weights, topk_ids = fused_topk_native(
hidden_states=hidden_states,
gating_output=router_logits,
topk=num_routed_topk if _use_aiter else top_k,
topk=top_k_for_gate,
renormalize=renormalize,
correction_bias=correction_bias,
scoring_func=scoring_func,
@@ -2731,7 +2736,7 @@ def select_experts(
hidden_states=hidden_states,
gating_output=router_logits,
correction_bias=correction_bias,
topk=num_routed_topk if _use_aiter else top_k,
topk=top_k_for_gate,
renormalize=renormalize,
scoring_func=scoring_func,
num_fused_shared_experts=num_fused_shared_experts_for_gate,
@@ -2751,7 +2756,7 @@ def select_experts(
topk_weights, topk_ids = fused_topk_softmax_torch_raw_logits(
hidden_states=hidden_states,
gating_output=router_logits,
topk=num_routed_topk if _use_aiter else top_k,
topk=top_k_for_gate,
renormalize=renormalize,
)
else:
@@ -2794,7 +2799,7 @@ def select_experts(
topk_weights, topk_ids = fused_topk(
hidden_states=hidden_states,
gating_output=router_logits,
topk=num_routed_topk if _use_aiter else top_k,
topk=top_k_for_gate,
renormalize=renormalize,
correction_bias=correction_bias,
scoring_func=scoring_func,
@@ -2812,7 +2817,7 @@ def select_experts(
topk_weights, topk_ids = custom_routing_function(
hidden_states=hidden_states,
gating_output=router_logits,
topk=num_routed_topk if _use_aiter else top_k,
topk=top_k_for_gate,
renormalize=renormalize,
)
@@ -3,6 +3,7 @@ from typing import Any, Optional
import numpy as np
import pybase64
import torch
import torch.nn.functional as F
from sglang.srt.configs.model_config import ModelConfig
from sglang.srt.layers.dp_attention import (
@@ -106,6 +107,18 @@ class RoutedExpertsCapturer(BaseTopkCapturer):
)
def capture(self, layer_id: int, topk_indices: torch.Tensor):
device_width = self.device_cache.buffer.shape[2]
if topk_indices.shape[1] != device_width:
# Routes whose shared columns are appended after capture pass routed ids only.
if topk_indices.shape[1] != device_width - self.num_fused_shared_experts:
raise ValueError(
f"Captured top-k width {topk_indices.shape[1]} is neither the "
f"device width {device_width} nor its routed part "
f"{device_width - self.num_fused_shared_experts}"
)
topk_indices = F.pad(
topk_indices, (0, self.num_fused_shared_experts), value=-1
)
if _is_scattered_a2a_backend():
local_topk = topk_indices
topk_indices = self.gather_buffer[