[Fix] moe: Renormalize per-rank shared-slot routing over routed experts only (#44904)

This commit is contained in:
Cheng Wan
2026-10-19 16:56:21 -07:00
committed by GitHub
parent 3a059de94c
commit 48ea486928
11 changed files with 486 additions and 127 deletions
@@ -1398,8 +1398,8 @@ def _fused_append_remap_shared_experts_deepep_kernel(
):
"""Append shared experts AND apply the DeepEP interleaved remap in one pass.
Equivalent to fused_append_shared_experts() immediately followed by
topk._remap_topk_for_deepep(), but the remap math runs on the rows already
Equivalent to topk.remap_topk_for_per_rank_shared_slots() on the same
routed columns, but the remap math runs on the rows already
loaded into registers, so it costs a few ALU ops instead of ~6 extra eager
kernel launches (div_floor / add / arange / fill / copy) per MoE layer.
@@ -1464,7 +1464,7 @@ def fused_append_remap_shared_experts_deepep(
):
"""Fused append + DeepEP remap (see kernel docstring).
Replaces the fused_append_shared_experts() + _remap_topk_for_deepep() pair on
Fused form of topk.remap_topk_for_per_rank_shared_slots() for
the aiter/DeepEP-class path. Host computes the scalar remap params so the
kernel stays branch-free.
"""
@@ -284,15 +284,6 @@ def _router_triton_kernel(
:, None
] # [BLOCK_M, 1]
# Fill fused-shared-expert slots: weight = routed_sum / routed_scaling_factor,
# id = num_experts + (slot - K_ROUTED).
if K_ROUTED < K:
is_shared = (offs_k[None, :] >= K_ROUTED) & mask_k_total[None, :]
shared_weight = routed_sum / routed_scaling_factor # [BLOCK_M, 1]
shared_idx = (N + (offs_k - K_ROUTED)).to(tl.int32)[None, :] # [1, BLOCK_K]
selected_vals = tl.where(is_shared, shared_weight, selected_vals)
selected_idx = tl.where(is_shared, shared_idx, selected_idx)
if USE_PDL:
tl.extra.cuda.gdc_launch_dependents()
@@ -304,6 +295,13 @@ def _router_triton_kernel(
selected_vals = selected_vals / norm
if APPLY_SCALE:
selected_vals = selected_vals * routed_scaling_factor
if K_ROUTED < K:
# The shared expert's contribution is independent of routed normalization.
is_shared = (offs_k[None, :] >= K_ROUTED) & mask_k_total[None, :]
shared_weight = 1.0 if APPLY_SCALE else 1.0 / routed_scaling_factor
shared_idx = (N + (offs_k - K_ROUTED)).to(tl.int32)[None, :]
selected_vals = tl.where(is_shared, shared_weight, selected_vals)
selected_idx = tl.where(is_shared, shared_idx, selected_idx)
if HAS_PADDING:
selected_vals = tl.where(live_m[:, None], selected_vals, 0.0)
selected_idx = tl.where(live_m[:, None], selected_idx, -1)