[Fix] moe: Renormalize per-rank shared-slot routing over routed experts only (#44904)
This commit is contained in:
@@ -1398,8 +1398,8 @@ def _fused_append_remap_shared_experts_deepep_kernel(
|
||||
):
|
||||
"""Append shared experts AND apply the DeepEP interleaved remap in one pass.
|
||||
|
||||
Equivalent to fused_append_shared_experts() immediately followed by
|
||||
topk._remap_topk_for_deepep(), but the remap math runs on the rows already
|
||||
Equivalent to topk.remap_topk_for_per_rank_shared_slots() on the same
|
||||
routed columns, but the remap math runs on the rows already
|
||||
loaded into registers, so it costs a few ALU ops instead of ~6 extra eager
|
||||
kernel launches (div_floor / add / arange / fill / copy) per MoE layer.
|
||||
|
||||
@@ -1464,7 +1464,7 @@ def fused_append_remap_shared_experts_deepep(
|
||||
):
|
||||
"""Fused append + DeepEP remap (see kernel docstring).
|
||||
|
||||
Replaces the fused_append_shared_experts() + _remap_topk_for_deepep() pair on
|
||||
Fused form of topk.remap_topk_for_per_rank_shared_slots() for
|
||||
the aiter/DeepEP-class path. Host computes the scalar remap params so the
|
||||
kernel stays branch-free.
|
||||
"""
|
||||
|
||||
@@ -284,15 +284,6 @@ def _router_triton_kernel(
|
||||
:, None
|
||||
] # [BLOCK_M, 1]
|
||||
|
||||
# Fill fused-shared-expert slots: weight = routed_sum / routed_scaling_factor,
|
||||
# id = num_experts + (slot - K_ROUTED).
|
||||
if K_ROUTED < K:
|
||||
is_shared = (offs_k[None, :] >= K_ROUTED) & mask_k_total[None, :]
|
||||
shared_weight = routed_sum / routed_scaling_factor # [BLOCK_M, 1]
|
||||
shared_idx = (N + (offs_k - K_ROUTED)).to(tl.int32)[None, :] # [1, BLOCK_K]
|
||||
selected_vals = tl.where(is_shared, shared_weight, selected_vals)
|
||||
selected_idx = tl.where(is_shared, shared_idx, selected_idx)
|
||||
|
||||
if USE_PDL:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
@@ -304,6 +295,13 @@ def _router_triton_kernel(
|
||||
selected_vals = selected_vals / norm
|
||||
if APPLY_SCALE:
|
||||
selected_vals = selected_vals * routed_scaling_factor
|
||||
if K_ROUTED < K:
|
||||
# The shared expert's contribution is independent of routed normalization.
|
||||
is_shared = (offs_k[None, :] >= K_ROUTED) & mask_k_total[None, :]
|
||||
shared_weight = 1.0 if APPLY_SCALE else 1.0 / routed_scaling_factor
|
||||
shared_idx = (N + (offs_k - K_ROUTED)).to(tl.int32)[None, :]
|
||||
selected_vals = tl.where(is_shared, shared_weight, selected_vals)
|
||||
selected_idx = tl.where(is_shared, shared_idx, selected_idx)
|
||||
if HAS_PADDING:
|
||||
selected_vals = tl.where(live_m[:, None], selected_vals, 0.0)
|
||||
selected_idx = tl.where(live_m[:, None], selected_idx, -1)
|
||||
|
||||
@@ -266,12 +266,12 @@ class HashTopK(nn.Module):
|
||||
|
||||
recorder_topk_ids = None
|
||||
if has_per_rank_fused_shared_slots(num_fused_shared_experts):
|
||||
shared_cols = topk_ids[:, -num_fused_shared_experts:]
|
||||
routed_cols = topk_ids[:, :-num_fused_shared_experts]
|
||||
# The trailing columns are placeholders; the remap appends the real ones.
|
||||
routed_cols = topk_ids_logical_to_physical(
|
||||
routed_cols, expert_location_dispatch_info, log2phy_prob
|
||||
topk_ids[:, :-num_fused_shared_experts],
|
||||
expert_location_dispatch_info,
|
||||
log2phy_prob,
|
||||
)
|
||||
topk_ids = torch.cat([routed_cols, shared_cols], dim=-1)
|
||||
recorder_topk_ids = routed_cols
|
||||
|
||||
num_physical_routed_experts = (
|
||||
@@ -280,8 +280,8 @@ class HashTopK(nn.Module):
|
||||
else self.num_experts
|
||||
)
|
||||
topk_ids, topk_weights = remap_topk_for_per_rank_shared_slots(
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
routed_cols,
|
||||
topk_weights[:, :-num_fused_shared_experts],
|
||||
num_fused_shared_experts,
|
||||
num_physical_routed_experts,
|
||||
TopKConfig(
|
||||
|
||||
@@ -198,7 +198,7 @@ if _is_cuda:
|
||||
scaling_factor: float,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
renormalize: bool,
|
||||
launch_with_pdl: bool,
|
||||
) -> None:
|
||||
_fused_topk_deepseek(
|
||||
gating_output,
|
||||
@@ -209,7 +209,7 @@ if _is_cuda:
|
||||
scaling_factor,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
renormalize,
|
||||
launch_with_pdl,
|
||||
)
|
||||
|
||||
except ImportError:
|
||||
@@ -1815,8 +1815,8 @@ def biased_grouped_topk_gpu(
|
||||
moe_fused_gate as jit_grouped_gate,
|
||||
)
|
||||
|
||||
# The kernel wants the total width; select_experts passes a routed-only
|
||||
# topk on the aiter path only (`num_routed_topk if _use_aiter else top_k`).
|
||||
# The kernel wants the total width; off aiter, topk already counts the
|
||||
# num_fused_shared_experts slots it writes.
|
||||
#
|
||||
# AITER folds this route even when no shared slots are requested.
|
||||
return jit_grouped_gate(
|
||||
@@ -1838,6 +1838,8 @@ def biased_grouped_topk_gpu(
|
||||
if (
|
||||
_is_cuda
|
||||
and fused_topk_deepseek is not None
|
||||
# flashinfer always renormalizes the selected weights
|
||||
and renormalize
|
||||
and is_power_of_two(num_experts)
|
||||
# flashinfer constraints (applied to routed experts only)
|
||||
and topk_routed <= 8
|
||||
@@ -1873,21 +1875,21 @@ def biased_grouped_topk_gpu(
|
||||
scaling_factor,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
True,
|
||||
launch_with_pdl=True,
|
||||
)
|
||||
|
||||
if num_fused_shared_experts > 0:
|
||||
# Append shared expert columns: ID = num_experts (first shared slot),
|
||||
# weight = sum(routed) / scaling_factor (matching biased_grouped_topk_impl).
|
||||
# For DeepEP/MegaMOE per-rank shared-slot layout, post-process remaps
|
||||
# this placeholder ID and overwrites the shared weight for the active scaling path.
|
||||
# Append the global shared columns: ID = num_experts (first shared slot).
|
||||
# On CUDA the routed weights carry the factor exactly when asked to.
|
||||
topk_ids = F.pad(topk_ids, (0, num_fused_shared_experts), value=num_experts)
|
||||
topk_weights = F.pad(topk_weights, (0, num_fused_shared_experts))
|
||||
if routed_scaling_factor is not None:
|
||||
topk_weights[:, topk_routed:] = (
|
||||
topk_weights[:, :topk_routed].sum(dim=-1, keepdim=True)
|
||||
/ routed_scaling_factor
|
||||
)
|
||||
topk_weights = F.pad(
|
||||
topk_weights,
|
||||
(0, num_fused_shared_experts),
|
||||
value=fused_shared_expert_weight(
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
fused_in_topk=bool(apply_routed_scaling_factor_on_output),
|
||||
),
|
||||
)
|
||||
|
||||
return topk_weights, topk_ids
|
||||
|
||||
@@ -2240,10 +2242,7 @@ def remap_topk_for_per_rank_shared_slots(
|
||||
num_physical_routed_experts: int,
|
||||
topk_config: TopKConfig,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Map routed and shared columns into rank-major expert blocks."""
|
||||
if topk_ids.shape[0] == 0:
|
||||
return topk_ids, topk_weights
|
||||
|
||||
"""Append shared columns to routed-only inputs in rank-major expert blocks."""
|
||||
ep_size = get_parallel().moe_ep_size
|
||||
ep_rank = get_parallel().moe_ep_rank
|
||||
# Static EPLB may add redundant physical experts. At this point routed
|
||||
@@ -2251,31 +2250,36 @@ def remap_topk_for_per_rank_shared_slots(
|
||||
# per-rank shared-slot layout must use the physical routed count.
|
||||
num_local_routed = num_physical_routed_experts // ep_size
|
||||
num_local_experts = num_local_routed + num_fused_shared_experts
|
||||
num_tokens = topk_ids.shape[0]
|
||||
|
||||
# Remap routed IDs: insert gaps for shared expert slots (single fused op).
|
||||
# Each rank r == e // num_local_routed is preceded by r shared-slot blocks of
|
||||
# width num_fused_shared_experts, so shift by (e // num_local_routed) * S --
|
||||
# a single-slot shift (S == 1) would let routed ids collide with an earlier
|
||||
# rank's shared slots once S > 1.
|
||||
routed = topk_ids[:, :-num_fused_shared_experts]
|
||||
topk_ids[:, :-num_fused_shared_experts] = (
|
||||
routed + (routed // num_local_routed) * num_fused_shared_experts
|
||||
)
|
||||
routed_ids = topk_ids + (topk_ids // num_local_routed) * num_fused_shared_experts
|
||||
|
||||
# Set shared expert IDs to route to home rank (vectorized)
|
||||
topk_ids[:, -num_fused_shared_experts:] = (
|
||||
# Shared expert IDs route to the home rank.
|
||||
shared_ids = (
|
||||
ep_rank * num_local_experts
|
||||
+ num_local_routed
|
||||
+ torch.arange(num_fused_shared_experts, device=topk_ids.device)
|
||||
+ torch.arange(
|
||||
num_fused_shared_experts, dtype=topk_ids.dtype, device=topk_ids.device
|
||||
)
|
||||
).expand(num_tokens, -1)
|
||||
shared_weights = topk_weights.new_full(
|
||||
(num_tokens, num_fused_shared_experts),
|
||||
fused_shared_expert_weight(
|
||||
routed_scaling_factor=topk_config.routed_scaling_factor,
|
||||
fused_in_topk=is_routed_scaling_factor_fused_in_topk(topk_config),
|
||||
),
|
||||
)
|
||||
|
||||
topk_weights[:, -num_fused_shared_experts:] = fused_shared_expert_weight(
|
||||
routed_scaling_factor=topk_config.routed_scaling_factor,
|
||||
fused_in_topk=is_routed_scaling_factor_fused_in_topk(topk_config),
|
||||
return (
|
||||
torch.cat([routed_ids, shared_ids], dim=-1),
|
||||
torch.cat([topk_weights, shared_weights], dim=-1),
|
||||
)
|
||||
|
||||
return topk_ids, topk_weights
|
||||
|
||||
|
||||
def capture_routed_experts_if_allowed(
|
||||
topk_config: TopKConfig,
|
||||
@@ -2313,12 +2317,13 @@ def _post_process_topk_ids(
|
||||
fused_shared_experts_scaling_factor = (
|
||||
topk_config.fused_shared_experts_scaling_factor
|
||||
)
|
||||
# False when select_experts asked the router for routed experts only.
|
||||
router_wrote_shared_cols = (
|
||||
num_fused_shared_experts > 0 and topk_ids.shape[-1] == topk_config.top_k
|
||||
)
|
||||
# The router already wrote the aiter shared columns (see _aiter_append below).
|
||||
_gate_appended = (
|
||||
num_fused_shared_experts > 0
|
||||
and _use_aiter
|
||||
and not use_per_rank_shared_slots
|
||||
and topk_ids.shape[-1] == topk_config.top_k
|
||||
router_wrote_shared_cols and _use_aiter and not use_per_rank_shared_slots
|
||||
)
|
||||
routed_topk_ids = (
|
||||
topk_ids[:, :-num_fused_shared_experts] if _gate_appended else topk_ids
|
||||
@@ -2352,7 +2357,7 @@ def _post_process_topk_ids(
|
||||
topk_ids, expert_location_dispatch_info, log2phy_prob
|
||||
)
|
||||
_mask_topk_ids_padded_region(topk_ids, num_token_non_padded)
|
||||
elif num_fused_shared_experts > 0:
|
||||
elif router_wrote_shared_cols:
|
||||
# Shared IDs are outside EPLB's routed-expert table for both global
|
||||
# and per-rank layouts, so remap only routed columns.
|
||||
shared_cols = topk_ids[:, -num_fused_shared_experts:]
|
||||
@@ -2504,6 +2509,10 @@ def _post_process_topk_ids(
|
||||
if expert_location_dispatch_info is not None
|
||||
else router_logits.shape[1]
|
||||
)
|
||||
if router_wrote_shared_cols:
|
||||
# Placeholders; the remap appends the real shared columns.
|
||||
topk_ids = topk_ids[:, :-num_fused_shared_experts]
|
||||
topk_weights = topk_weights[:, :-num_fused_shared_experts]
|
||||
topk_ids, topk_weights = remap_topk_for_per_rank_shared_slots(
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
@@ -2511,6 +2520,9 @@ def _post_process_topk_ids(
|
||||
num_physical_routed_experts,
|
||||
topk_config,
|
||||
)
|
||||
if _is_cuda:
|
||||
# The remap shifts a padded row's -1 and gives it a live shared slot.
|
||||
_mask_topk_ids_padded_region(topk_ids, num_token_non_padded)
|
||||
elif (
|
||||
num_fused_shared_experts > 0 and fused_shared_experts_scaling_factor is not None
|
||||
):
|
||||
@@ -2612,22 +2624,15 @@ def select_experts(
|
||||
):
|
||||
correction_bias = topk_config.correction_bias_for_dtype(router_logits.dtype)
|
||||
|
||||
# DeepSeek V2/V3/R1 series models use grouped_top_k
|
||||
# remove num_fused_shared_experts from grouped_topk/biased_grouped_topk
|
||||
num_routed_topk = top_k - num_fused_shared_experts
|
||||
# On the per-rank shared-slot path the shared expert is appended afterwards by
|
||||
# fused_append_remap_shared_experts_deepep, so the gate must not emit a shared
|
||||
# marker as well. Doing both costs one routed expert (the gate is asked for
|
||||
# K_routed = top_k - num_fused_shared_experts and then spends one of those
|
||||
# slots on the marker) and places that marker at id num_experts, which the
|
||||
# DeepEP remap shifts one past the end of the expert space -- 384 -> 392 for
|
||||
# 384 routed experts on EP8, where the valid ids are 0..391.
|
||||
# aiter appends the shared expert in _post_process_topk_ids, so the gate must not.
|
||||
num_fused_shared_experts_for_gate = (
|
||||
0
|
||||
if (has_per_rank_fused_shared_slots(num_fused_shared_experts) or _use_aiter)
|
||||
else num_fused_shared_experts
|
||||
)
|
||||
# On aiter and on the per-rank shared-slot path, _post_process_topk_ids appends
|
||||
# the shared columns, so the router selects and renormalizes routed experts only.
|
||||
if _use_aiter or has_per_rank_fused_shared_slots(num_fused_shared_experts):
|
||||
top_k_for_gate = num_routed_topk
|
||||
num_fused_shared_experts_for_gate = 0
|
||||
else:
|
||||
top_k_for_gate = top_k
|
||||
num_fused_shared_experts_for_gate = num_fused_shared_experts
|
||||
if dynamic_expert_bias is not None:
|
||||
if scoring_func != "sigmoid":
|
||||
raise ValueError(
|
||||
@@ -2655,7 +2660,7 @@ def select_experts(
|
||||
topk_weights, topk_ids = grouped_topk(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
topk=num_routed_topk if _use_aiter else top_k,
|
||||
topk=top_k_for_gate,
|
||||
renormalize=renormalize,
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
@@ -2669,7 +2674,7 @@ def select_experts(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
correction_bias=correction_bias,
|
||||
topk=num_routed_topk if _use_aiter else top_k,
|
||||
topk=top_k_for_gate,
|
||||
renormalize=renormalize,
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
@@ -2686,7 +2691,7 @@ def select_experts(
|
||||
topk_weights, topk_ids = fused_topk_native(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
topk=num_routed_topk if _use_aiter else top_k,
|
||||
topk=top_k_for_gate,
|
||||
renormalize=renormalize,
|
||||
correction_bias=correction_bias,
|
||||
scoring_func=scoring_func,
|
||||
@@ -2731,7 +2736,7 @@ def select_experts(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
correction_bias=correction_bias,
|
||||
topk=num_routed_topk if _use_aiter else top_k,
|
||||
topk=top_k_for_gate,
|
||||
renormalize=renormalize,
|
||||
scoring_func=scoring_func,
|
||||
num_fused_shared_experts=num_fused_shared_experts_for_gate,
|
||||
@@ -2751,7 +2756,7 @@ def select_experts(
|
||||
topk_weights, topk_ids = fused_topk_softmax_torch_raw_logits(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
topk=num_routed_topk if _use_aiter else top_k,
|
||||
topk=top_k_for_gate,
|
||||
renormalize=renormalize,
|
||||
)
|
||||
else:
|
||||
@@ -2794,7 +2799,7 @@ def select_experts(
|
||||
topk_weights, topk_ids = fused_topk(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
topk=num_routed_topk if _use_aiter else top_k,
|
||||
topk=top_k_for_gate,
|
||||
renormalize=renormalize,
|
||||
correction_bias=correction_bias,
|
||||
scoring_func=scoring_func,
|
||||
@@ -2812,7 +2817,7 @@ def select_experts(
|
||||
topk_weights, topk_ids = custom_routing_function(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
topk=num_routed_topk if _use_aiter else top_k,
|
||||
topk=top_k_for_gate,
|
||||
renormalize=renormalize,
|
||||
)
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ from typing import Any, Optional
|
||||
import numpy as np
|
||||
import pybase64
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from sglang.srt.configs.model_config import ModelConfig
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -106,6 +107,18 @@ class RoutedExpertsCapturer(BaseTopkCapturer):
|
||||
)
|
||||
|
||||
def capture(self, layer_id: int, topk_indices: torch.Tensor):
|
||||
device_width = self.device_cache.buffer.shape[2]
|
||||
if topk_indices.shape[1] != device_width:
|
||||
# Routes whose shared columns are appended after capture pass routed ids only.
|
||||
if topk_indices.shape[1] != device_width - self.num_fused_shared_experts:
|
||||
raise ValueError(
|
||||
f"Captured top-k width {topk_indices.shape[1]} is neither the "
|
||||
f"device width {device_width} nor its routed part "
|
||||
f"{device_width - self.num_fused_shared_experts}"
|
||||
)
|
||||
topk_indices = F.pad(
|
||||
topk_indices, (0, self.num_fused_shared_experts), value=-1
|
||||
)
|
||||
if _is_scattered_a2a_backend():
|
||||
local_topk = topk_indices
|
||||
topk_indices = self.gather_buffer[
|
||||
|
||||
Reference in New Issue
Block a user