[Fix] moe: Renormalize per-rank shared-slot routing over routed experts only (#44904)
This commit is contained in:
@@ -1398,8 +1398,8 @@ def _fused_append_remap_shared_experts_deepep_kernel(
|
||||
):
|
||||
"""Append shared experts AND apply the DeepEP interleaved remap in one pass.
|
||||
|
||||
Equivalent to fused_append_shared_experts() immediately followed by
|
||||
topk._remap_topk_for_deepep(), but the remap math runs on the rows already
|
||||
Equivalent to topk.remap_topk_for_per_rank_shared_slots() on the same
|
||||
routed columns, but the remap math runs on the rows already
|
||||
loaded into registers, so it costs a few ALU ops instead of ~6 extra eager
|
||||
kernel launches (div_floor / add / arange / fill / copy) per MoE layer.
|
||||
|
||||
@@ -1464,7 +1464,7 @@ def fused_append_remap_shared_experts_deepep(
|
||||
):
|
||||
"""Fused append + DeepEP remap (see kernel docstring).
|
||||
|
||||
Replaces the fused_append_shared_experts() + _remap_topk_for_deepep() pair on
|
||||
Fused form of topk.remap_topk_for_per_rank_shared_slots() for
|
||||
the aiter/DeepEP-class path. Host computes the scalar remap params so the
|
||||
kernel stays branch-free.
|
||||
"""
|
||||
|
||||
@@ -284,15 +284,6 @@ def _router_triton_kernel(
|
||||
:, None
|
||||
] # [BLOCK_M, 1]
|
||||
|
||||
# Fill fused-shared-expert slots: weight = routed_sum / routed_scaling_factor,
|
||||
# id = num_experts + (slot - K_ROUTED).
|
||||
if K_ROUTED < K:
|
||||
is_shared = (offs_k[None, :] >= K_ROUTED) & mask_k_total[None, :]
|
||||
shared_weight = routed_sum / routed_scaling_factor # [BLOCK_M, 1]
|
||||
shared_idx = (N + (offs_k - K_ROUTED)).to(tl.int32)[None, :] # [1, BLOCK_K]
|
||||
selected_vals = tl.where(is_shared, shared_weight, selected_vals)
|
||||
selected_idx = tl.where(is_shared, shared_idx, selected_idx)
|
||||
|
||||
if USE_PDL:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
@@ -304,6 +295,13 @@ def _router_triton_kernel(
|
||||
selected_vals = selected_vals / norm
|
||||
if APPLY_SCALE:
|
||||
selected_vals = selected_vals * routed_scaling_factor
|
||||
if K_ROUTED < K:
|
||||
# The shared expert's contribution is independent of routed normalization.
|
||||
is_shared = (offs_k[None, :] >= K_ROUTED) & mask_k_total[None, :]
|
||||
shared_weight = 1.0 if APPLY_SCALE else 1.0 / routed_scaling_factor
|
||||
shared_idx = (N + (offs_k - K_ROUTED)).to(tl.int32)[None, :]
|
||||
selected_vals = tl.where(is_shared, shared_weight, selected_vals)
|
||||
selected_idx = tl.where(is_shared, shared_idx, selected_idx)
|
||||
if HAS_PADDING:
|
||||
selected_vals = tl.where(live_m[:, None], selected_vals, 0.0)
|
||||
selected_idx = tl.where(live_m[:, None], selected_idx, -1)
|
||||
|
||||
@@ -266,12 +266,12 @@ class HashTopK(nn.Module):
|
||||
|
||||
recorder_topk_ids = None
|
||||
if has_per_rank_fused_shared_slots(num_fused_shared_experts):
|
||||
shared_cols = topk_ids[:, -num_fused_shared_experts:]
|
||||
routed_cols = topk_ids[:, :-num_fused_shared_experts]
|
||||
# The trailing columns are placeholders; the remap appends the real ones.
|
||||
routed_cols = topk_ids_logical_to_physical(
|
||||
routed_cols, expert_location_dispatch_info, log2phy_prob
|
||||
topk_ids[:, :-num_fused_shared_experts],
|
||||
expert_location_dispatch_info,
|
||||
log2phy_prob,
|
||||
)
|
||||
topk_ids = torch.cat([routed_cols, shared_cols], dim=-1)
|
||||
recorder_topk_ids = routed_cols
|
||||
|
||||
num_physical_routed_experts = (
|
||||
@@ -280,8 +280,8 @@ class HashTopK(nn.Module):
|
||||
else self.num_experts
|
||||
)
|
||||
topk_ids, topk_weights = remap_topk_for_per_rank_shared_slots(
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
routed_cols,
|
||||
topk_weights[:, :-num_fused_shared_experts],
|
||||
num_fused_shared_experts,
|
||||
num_physical_routed_experts,
|
||||
TopKConfig(
|
||||
|
||||
@@ -198,7 +198,7 @@ if _is_cuda:
|
||||
scaling_factor: float,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
renormalize: bool,
|
||||
launch_with_pdl: bool,
|
||||
) -> None:
|
||||
_fused_topk_deepseek(
|
||||
gating_output,
|
||||
@@ -209,7 +209,7 @@ if _is_cuda:
|
||||
scaling_factor,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
renormalize,
|
||||
launch_with_pdl,
|
||||
)
|
||||
|
||||
except ImportError:
|
||||
@@ -1815,8 +1815,8 @@ def biased_grouped_topk_gpu(
|
||||
moe_fused_gate as jit_grouped_gate,
|
||||
)
|
||||
|
||||
# The kernel wants the total width; select_experts passes a routed-only
|
||||
# topk on the aiter path only (`num_routed_topk if _use_aiter else top_k`).
|
||||
# The kernel wants the total width; off aiter, topk already counts the
|
||||
# num_fused_shared_experts slots it writes.
|
||||
#
|
||||
# AITER folds this route even when no shared slots are requested.
|
||||
return jit_grouped_gate(
|
||||
@@ -1838,6 +1838,8 @@ def biased_grouped_topk_gpu(
|
||||
if (
|
||||
_is_cuda
|
||||
and fused_topk_deepseek is not None
|
||||
# flashinfer always renormalizes the selected weights
|
||||
and renormalize
|
||||
and is_power_of_two(num_experts)
|
||||
# flashinfer constraints (applied to routed experts only)
|
||||
and topk_routed <= 8
|
||||
@@ -1873,21 +1875,21 @@ def biased_grouped_topk_gpu(
|
||||
scaling_factor,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
True,
|
||||
launch_with_pdl=True,
|
||||
)
|
||||
|
||||
if num_fused_shared_experts > 0:
|
||||
# Append shared expert columns: ID = num_experts (first shared slot),
|
||||
# weight = sum(routed) / scaling_factor (matching biased_grouped_topk_impl).
|
||||
# For DeepEP/MegaMOE per-rank shared-slot layout, post-process remaps
|
||||
# this placeholder ID and overwrites the shared weight for the active scaling path.
|
||||
# Append the global shared columns: ID = num_experts (first shared slot).
|
||||
# On CUDA the routed weights carry the factor exactly when asked to.
|
||||
topk_ids = F.pad(topk_ids, (0, num_fused_shared_experts), value=num_experts)
|
||||
topk_weights = F.pad(topk_weights, (0, num_fused_shared_experts))
|
||||
if routed_scaling_factor is not None:
|
||||
topk_weights[:, topk_routed:] = (
|
||||
topk_weights[:, :topk_routed].sum(dim=-1, keepdim=True)
|
||||
/ routed_scaling_factor
|
||||
)
|
||||
topk_weights = F.pad(
|
||||
topk_weights,
|
||||
(0, num_fused_shared_experts),
|
||||
value=fused_shared_expert_weight(
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
fused_in_topk=bool(apply_routed_scaling_factor_on_output),
|
||||
),
|
||||
)
|
||||
|
||||
return topk_weights, topk_ids
|
||||
|
||||
@@ -2240,10 +2242,7 @@ def remap_topk_for_per_rank_shared_slots(
|
||||
num_physical_routed_experts: int,
|
||||
topk_config: TopKConfig,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Map routed and shared columns into rank-major expert blocks."""
|
||||
if topk_ids.shape[0] == 0:
|
||||
return topk_ids, topk_weights
|
||||
|
||||
"""Append shared columns to routed-only inputs in rank-major expert blocks."""
|
||||
ep_size = get_parallel().moe_ep_size
|
||||
ep_rank = get_parallel().moe_ep_rank
|
||||
# Static EPLB may add redundant physical experts. At this point routed
|
||||
@@ -2251,31 +2250,36 @@ def remap_topk_for_per_rank_shared_slots(
|
||||
# per-rank shared-slot layout must use the physical routed count.
|
||||
num_local_routed = num_physical_routed_experts // ep_size
|
||||
num_local_experts = num_local_routed + num_fused_shared_experts
|
||||
num_tokens = topk_ids.shape[0]
|
||||
|
||||
# Remap routed IDs: insert gaps for shared expert slots (single fused op).
|
||||
# Each rank r == e // num_local_routed is preceded by r shared-slot blocks of
|
||||
# width num_fused_shared_experts, so shift by (e // num_local_routed) * S --
|
||||
# a single-slot shift (S == 1) would let routed ids collide with an earlier
|
||||
# rank's shared slots once S > 1.
|
||||
routed = topk_ids[:, :-num_fused_shared_experts]
|
||||
topk_ids[:, :-num_fused_shared_experts] = (
|
||||
routed + (routed // num_local_routed) * num_fused_shared_experts
|
||||
)
|
||||
routed_ids = topk_ids + (topk_ids // num_local_routed) * num_fused_shared_experts
|
||||
|
||||
# Set shared expert IDs to route to home rank (vectorized)
|
||||
topk_ids[:, -num_fused_shared_experts:] = (
|
||||
# Shared expert IDs route to the home rank.
|
||||
shared_ids = (
|
||||
ep_rank * num_local_experts
|
||||
+ num_local_routed
|
||||
+ torch.arange(num_fused_shared_experts, device=topk_ids.device)
|
||||
+ torch.arange(
|
||||
num_fused_shared_experts, dtype=topk_ids.dtype, device=topk_ids.device
|
||||
)
|
||||
).expand(num_tokens, -1)
|
||||
shared_weights = topk_weights.new_full(
|
||||
(num_tokens, num_fused_shared_experts),
|
||||
fused_shared_expert_weight(
|
||||
routed_scaling_factor=topk_config.routed_scaling_factor,
|
||||
fused_in_topk=is_routed_scaling_factor_fused_in_topk(topk_config),
|
||||
),
|
||||
)
|
||||
|
||||
topk_weights[:, -num_fused_shared_experts:] = fused_shared_expert_weight(
|
||||
routed_scaling_factor=topk_config.routed_scaling_factor,
|
||||
fused_in_topk=is_routed_scaling_factor_fused_in_topk(topk_config),
|
||||
return (
|
||||
torch.cat([routed_ids, shared_ids], dim=-1),
|
||||
torch.cat([topk_weights, shared_weights], dim=-1),
|
||||
)
|
||||
|
||||
return topk_ids, topk_weights
|
||||
|
||||
|
||||
def capture_routed_experts_if_allowed(
|
||||
topk_config: TopKConfig,
|
||||
@@ -2313,12 +2317,13 @@ def _post_process_topk_ids(
|
||||
fused_shared_experts_scaling_factor = (
|
||||
topk_config.fused_shared_experts_scaling_factor
|
||||
)
|
||||
# False when select_experts asked the router for routed experts only.
|
||||
router_wrote_shared_cols = (
|
||||
num_fused_shared_experts > 0 and topk_ids.shape[-1] == topk_config.top_k
|
||||
)
|
||||
# The router already wrote the aiter shared columns (see _aiter_append below).
|
||||
_gate_appended = (
|
||||
num_fused_shared_experts > 0
|
||||
and _use_aiter
|
||||
and not use_per_rank_shared_slots
|
||||
and topk_ids.shape[-1] == topk_config.top_k
|
||||
router_wrote_shared_cols and _use_aiter and not use_per_rank_shared_slots
|
||||
)
|
||||
routed_topk_ids = (
|
||||
topk_ids[:, :-num_fused_shared_experts] if _gate_appended else topk_ids
|
||||
@@ -2352,7 +2357,7 @@ def _post_process_topk_ids(
|
||||
topk_ids, expert_location_dispatch_info, log2phy_prob
|
||||
)
|
||||
_mask_topk_ids_padded_region(topk_ids, num_token_non_padded)
|
||||
elif num_fused_shared_experts > 0:
|
||||
elif router_wrote_shared_cols:
|
||||
# Shared IDs are outside EPLB's routed-expert table for both global
|
||||
# and per-rank layouts, so remap only routed columns.
|
||||
shared_cols = topk_ids[:, -num_fused_shared_experts:]
|
||||
@@ -2504,6 +2509,10 @@ def _post_process_topk_ids(
|
||||
if expert_location_dispatch_info is not None
|
||||
else router_logits.shape[1]
|
||||
)
|
||||
if router_wrote_shared_cols:
|
||||
# Placeholders; the remap appends the real shared columns.
|
||||
topk_ids = topk_ids[:, :-num_fused_shared_experts]
|
||||
topk_weights = topk_weights[:, :-num_fused_shared_experts]
|
||||
topk_ids, topk_weights = remap_topk_for_per_rank_shared_slots(
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
@@ -2511,6 +2520,9 @@ def _post_process_topk_ids(
|
||||
num_physical_routed_experts,
|
||||
topk_config,
|
||||
)
|
||||
if _is_cuda:
|
||||
# The remap shifts a padded row's -1 and gives it a live shared slot.
|
||||
_mask_topk_ids_padded_region(topk_ids, num_token_non_padded)
|
||||
elif (
|
||||
num_fused_shared_experts > 0 and fused_shared_experts_scaling_factor is not None
|
||||
):
|
||||
@@ -2612,22 +2624,15 @@ def select_experts(
|
||||
):
|
||||
correction_bias = topk_config.correction_bias_for_dtype(router_logits.dtype)
|
||||
|
||||
# DeepSeek V2/V3/R1 series models use grouped_top_k
|
||||
# remove num_fused_shared_experts from grouped_topk/biased_grouped_topk
|
||||
num_routed_topk = top_k - num_fused_shared_experts
|
||||
# On the per-rank shared-slot path the shared expert is appended afterwards by
|
||||
# fused_append_remap_shared_experts_deepep, so the gate must not emit a shared
|
||||
# marker as well. Doing both costs one routed expert (the gate is asked for
|
||||
# K_routed = top_k - num_fused_shared_experts and then spends one of those
|
||||
# slots on the marker) and places that marker at id num_experts, which the
|
||||
# DeepEP remap shifts one past the end of the expert space -- 384 -> 392 for
|
||||
# 384 routed experts on EP8, where the valid ids are 0..391.
|
||||
# aiter appends the shared expert in _post_process_topk_ids, so the gate must not.
|
||||
num_fused_shared_experts_for_gate = (
|
||||
0
|
||||
if (has_per_rank_fused_shared_slots(num_fused_shared_experts) or _use_aiter)
|
||||
else num_fused_shared_experts
|
||||
)
|
||||
# On aiter and on the per-rank shared-slot path, _post_process_topk_ids appends
|
||||
# the shared columns, so the router selects and renormalizes routed experts only.
|
||||
if _use_aiter or has_per_rank_fused_shared_slots(num_fused_shared_experts):
|
||||
top_k_for_gate = num_routed_topk
|
||||
num_fused_shared_experts_for_gate = 0
|
||||
else:
|
||||
top_k_for_gate = top_k
|
||||
num_fused_shared_experts_for_gate = num_fused_shared_experts
|
||||
if dynamic_expert_bias is not None:
|
||||
if scoring_func != "sigmoid":
|
||||
raise ValueError(
|
||||
@@ -2655,7 +2660,7 @@ def select_experts(
|
||||
topk_weights, topk_ids = grouped_topk(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
topk=num_routed_topk if _use_aiter else top_k,
|
||||
topk=top_k_for_gate,
|
||||
renormalize=renormalize,
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
@@ -2669,7 +2674,7 @@ def select_experts(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
correction_bias=correction_bias,
|
||||
topk=num_routed_topk if _use_aiter else top_k,
|
||||
topk=top_k_for_gate,
|
||||
renormalize=renormalize,
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
@@ -2686,7 +2691,7 @@ def select_experts(
|
||||
topk_weights, topk_ids = fused_topk_native(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
topk=num_routed_topk if _use_aiter else top_k,
|
||||
topk=top_k_for_gate,
|
||||
renormalize=renormalize,
|
||||
correction_bias=correction_bias,
|
||||
scoring_func=scoring_func,
|
||||
@@ -2731,7 +2736,7 @@ def select_experts(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
correction_bias=correction_bias,
|
||||
topk=num_routed_topk if _use_aiter else top_k,
|
||||
topk=top_k_for_gate,
|
||||
renormalize=renormalize,
|
||||
scoring_func=scoring_func,
|
||||
num_fused_shared_experts=num_fused_shared_experts_for_gate,
|
||||
@@ -2751,7 +2756,7 @@ def select_experts(
|
||||
topk_weights, topk_ids = fused_topk_softmax_torch_raw_logits(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
topk=num_routed_topk if _use_aiter else top_k,
|
||||
topk=top_k_for_gate,
|
||||
renormalize=renormalize,
|
||||
)
|
||||
else:
|
||||
@@ -2794,7 +2799,7 @@ def select_experts(
|
||||
topk_weights, topk_ids = fused_topk(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
topk=num_routed_topk if _use_aiter else top_k,
|
||||
topk=top_k_for_gate,
|
||||
renormalize=renormalize,
|
||||
correction_bias=correction_bias,
|
||||
scoring_func=scoring_func,
|
||||
@@ -2812,7 +2817,7 @@ def select_experts(
|
||||
topk_weights, topk_ids = custom_routing_function(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
topk=num_routed_topk if _use_aiter else top_k,
|
||||
topk=top_k_for_gate,
|
||||
renormalize=renormalize,
|
||||
)
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ from typing import Any, Optional
|
||||
import numpy as np
|
||||
import pybase64
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from sglang.srt.configs.model_config import ModelConfig
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -106,6 +107,18 @@ class RoutedExpertsCapturer(BaseTopkCapturer):
|
||||
)
|
||||
|
||||
def capture(self, layer_id: int, topk_indices: torch.Tensor):
|
||||
device_width = self.device_cache.buffer.shape[2]
|
||||
if topk_indices.shape[1] != device_width:
|
||||
# Routes whose shared columns are appended after capture pass routed ids only.
|
||||
if topk_indices.shape[1] != device_width - self.num_fused_shared_experts:
|
||||
raise ValueError(
|
||||
f"Captured top-k width {topk_indices.shape[1]} is neither the "
|
||||
f"device width {device_width} nor its routed part "
|
||||
f"{device_width - self.num_fused_shared_experts}"
|
||||
)
|
||||
topk_indices = F.pad(
|
||||
topk_indices, (0, self.num_fused_shared_experts), value=-1
|
||||
)
|
||||
if _is_scattered_a2a_backend():
|
||||
local_topk = topk_indices
|
||||
topk_indices = self.gather_buffer[
|
||||
|
||||
@@ -1,16 +1,4 @@
|
||||
"""Correctness tests for the Triton :func:`moe_fused_gate` router.
|
||||
|
||||
For the ungrouped case (``num_expert_group == 1``) we validate it two ways:
|
||||
|
||||
* against an explicit, definition-based torch reference (documents the math),
|
||||
at small shapes and at model routing shapes, and
|
||||
* against the production ``biased_grouped_topk_impl`` for the sigmoid / no-shared
|
||||
path that the kernel actually replaces in ``topk.py``.
|
||||
|
||||
Comparisons are order-independent: weights are scattered back to a dense
|
||||
``[M, num_experts + num_shared]`` layout so the per-row column order (and any
|
||||
tie-break choice) does not matter.
|
||||
"""
|
||||
"""Triton routing must match independent math and supported production routes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -79,20 +67,24 @@ def _reference_gate(
|
||||
work.scatter_(1, winner.long().unsqueeze(1), float("-inf"))
|
||||
|
||||
routed_sum = routed_wgt.sum(dim=1, keepdim=True)
|
||||
if renormalize:
|
||||
norm = torch.where(routed_sum > 0.0, routed_sum, torch.ones_like(routed_sum))
|
||||
routed_wgt = routed_wgt / norm
|
||||
if apply_routed_scaling_factor_on_output:
|
||||
routed_wgt = routed_wgt * routed_scaling_factor
|
||||
weights = torch.empty(bs, topk, dtype=torch.float32, device=scores.device)
|
||||
indices = torch.empty(bs, topk, dtype=torch.int32, device=scores.device)
|
||||
weights[:, :num_routed] = routed_wgt
|
||||
indices[:, :num_routed] = routed_idx
|
||||
if num_fused_shared_experts > 0:
|
||||
weights[:, num_routed:] = routed_sum / routed_scaling_factor
|
||||
weights[:, num_routed:] = (
|
||||
1.0
|
||||
if apply_routed_scaling_factor_on_output
|
||||
else 1.0 / routed_scaling_factor
|
||||
)
|
||||
for j in range(num_fused_shared_experts):
|
||||
indices[:, num_routed + j] = num_experts + j
|
||||
|
||||
if renormalize:
|
||||
norm = torch.where(routed_sum > 0.0, routed_sum, torch.ones_like(routed_sum))
|
||||
weights = weights / norm
|
||||
if apply_routed_scaling_factor_on_output:
|
||||
weights = weights * routed_scaling_factor
|
||||
return weights, indices
|
||||
|
||||
|
||||
@@ -203,15 +195,14 @@ def test_moe_fused_gate_matches_reference_at_model_shapes(
|
||||
|
||||
@pytest.mark.parametrize("num_experts,topk", [(256, 6), (384, 8), (512, 8)])
|
||||
@pytest.mark.parametrize("apply_scale", [True, False])
|
||||
@pytest.mark.parametrize("renormalize", [True, False])
|
||||
def test_moe_fused_gate_matches_production_impl(
|
||||
num_experts: int, topk: int, apply_scale: bool
|
||||
num_experts: int, topk: int, apply_scale: bool, renormalize: bool
|
||||
) -> None:
|
||||
"""Match production ``biased_grouped_topk_impl`` on the path it replaces.
|
||||
|
||||
The kernel supersedes the ungrouped sigmoid CUDA path in ``topk.py``; that
|
||||
reference hardcodes sigmoid and no fused shared expert, and the production
|
||||
path always renormalizes (the impl only applies the scaling factor when
|
||||
``renormalize`` is set), so we compare on the renormalized path.
|
||||
Compare ungrouped sigmoid routing without shared slots, with normalization
|
||||
and explicit routed scaling enabled independently.
|
||||
"""
|
||||
from sglang.srt.layers.moe.topk import biased_grouped_topk_impl
|
||||
|
||||
@@ -224,7 +215,7 @@ def test_moe_fused_gate_matches_production_impl(
|
||||
bias,
|
||||
topk=topk,
|
||||
scoring_func="sigmoid",
|
||||
renormalize=True,
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=scale,
|
||||
apply_routed_scaling_factor_on_output=apply_scale,
|
||||
)
|
||||
@@ -234,7 +225,7 @@ def test_moe_fused_gate_matches_production_impl(
|
||||
scores,
|
||||
bias,
|
||||
topk,
|
||||
True,
|
||||
renormalize,
|
||||
num_expert_group=1,
|
||||
topk_group=1,
|
||||
routed_scaling_factor=scale,
|
||||
|
||||
@@ -1,26 +1,28 @@
|
||||
"""Unit tests for fused append + per-rank shared-slot remap.
|
||||
|
||||
Covers ``fused_append_remap_shared_experts_deepep``, which collapses
|
||||
``fused_append_shared_experts()`` followed by
|
||||
``remap_topk_for_per_rank_shared_slots()`` into a
|
||||
single Triton launch on the per-rank shared-slot path. The kernel is GPU-only
|
||||
(Triton), so these tests are skipped when no accelerator is present.
|
||||
Covers ``fused_append_remap_shared_experts_deepep``, the single Triton launch
|
||||
form of ``remap_topk_for_per_rank_shared_slots()`` on the per-rank shared-slot
|
||||
path, and ``select_experts`` on that path with the real routers. The kernels are
|
||||
GPU-only (Triton), so these tests are skipped when no accelerator is present.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.moe.fused_moe_triton_kernels import (
|
||||
fused_append_remap_shared_experts_deepep,
|
||||
fused_append_shared_experts,
|
||||
)
|
||||
from sglang.srt.layers.moe import topk as topk_module
|
||||
from sglang.srt.layers.moe.topk import (
|
||||
TopKConfig,
|
||||
biased_grouped_topk_impl,
|
||||
remap_topk_for_per_rank_shared_slots,
|
||||
select_experts,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import get_device
|
||||
from sglang.srt.runtime_context import get_context, get_parallel
|
||||
from sglang.srt.utils import get_device, is_cuda
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
@@ -176,7 +178,7 @@ class TestFusedAppendRemapPerRankSharedSlots(CustomTestCase):
|
||||
self.assertEqual(len(routed_set), npr)
|
||||
self.assertLess(max(routed_set), ep_size * num_local_experts)
|
||||
|
||||
def test_equivalence_with_eager_append_then_remap(self):
|
||||
def test_equivalence_with_eager_remap(self):
|
||||
"""Fused and eager append agree for folding and nonfolding routes."""
|
||||
rsf = 2.5
|
||||
for folded, scale_factor in ((False, 1.0 / rsf), (True, 1.0)):
|
||||
@@ -201,17 +203,10 @@ class TestFusedAppendRemapPerRankSharedSlots(CustomTestCase):
|
||||
with get_parallel().override(
|
||||
moe_ep_size=ep_size, moe_ep_rank=ep_rank
|
||||
):
|
||||
eager_ids, eager_w = fused_append_shared_experts(
|
||||
eager_ids, eager_w = remap_topk_for_per_rank_shared_slots(
|
||||
topk_ids.clone(),
|
||||
topk_weights.clone(),
|
||||
s,
|
||||
scale_factor,
|
||||
npr, # shared-expert base id (overwritten by the remap)
|
||||
)
|
||||
eager_ids, eager_w = remap_topk_for_per_rank_shared_slots(
|
||||
eager_ids,
|
||||
eager_w,
|
||||
s,
|
||||
npr,
|
||||
TopKConfig(
|
||||
top_k=k,
|
||||
@@ -294,5 +289,160 @@ class TestFusedAppendRemapPerRankSharedSlots(CustomTestCase):
|
||||
self.assertTrue(torch.equal(got_w, topk_weights))
|
||||
|
||||
|
||||
@unittest.skipUnless(is_cuda(), "covers the CUDA routers")
|
||||
class TestSelectExpertsPerRankSharedSlots(CustomTestCase):
|
||||
"""Shared slots must not alter routed selection or normalization."""
|
||||
|
||||
NUM_TOKENS, TOP_K, EP_SIZE, EP_RANK = 32, 8, 8, 3
|
||||
# (num_experts, num_expert_group, topk_group, scoring_func, biased); each row
|
||||
# reaches a different router.
|
||||
ROUTERS = [
|
||||
(256, 8, 4, "sigmoid", True), # DeepSeek-V3 grouped routing
|
||||
(384, 8, 4, "sigmoid", True), # grouped, outside FlashInfer's shapes
|
||||
(64, 8, 3, "softmax", False), # grouped, no correction bias
|
||||
(256, None, None, "sqrtsoftplus", True), # DeepSeek-V4 routing
|
||||
(128, None, None, "softmax", False),
|
||||
]
|
||||
|
||||
def _route(
|
||||
self, *, num_fused_shared_experts, logits, bias, router, renormalize=True
|
||||
):
|
||||
_, num_expert_group, topk_group, scoring_func, _ = router
|
||||
topk_config = TopKConfig(
|
||||
top_k=self.TOP_K + num_fused_shared_experts,
|
||||
renormalize=renormalize,
|
||||
use_grouped_topk=num_expert_group is not None,
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
num_fused_shared_experts=num_fused_shared_experts,
|
||||
correction_bias=bias,
|
||||
scoring_func=scoring_func,
|
||||
routed_scaling_factor=2.5,
|
||||
allow_routed_experts_capture=False,
|
||||
)
|
||||
with (
|
||||
patch.object(
|
||||
topk_module,
|
||||
"has_per_rank_fused_shared_slots",
|
||||
side_effect=lambda num_fused: num_fused > 0,
|
||||
),
|
||||
get_parallel().override(moe_ep_size=self.EP_SIZE, moe_ep_rank=self.EP_RANK),
|
||||
):
|
||||
out = select_experts(
|
||||
torch.empty(logits.shape[0], 16, device=logits.device),
|
||||
logits,
|
||||
topk_config,
|
||||
)
|
||||
return out.topk_ids.long(), out.topk_weights
|
||||
|
||||
def test_routes_like_no_fusion(self):
|
||||
k = self.TOP_K
|
||||
for router in self.ROUTERS:
|
||||
num_experts, _, _, _, biased = router
|
||||
with self.subTest(router=router):
|
||||
g = torch.Generator(device="cpu").manual_seed(num_experts)
|
||||
logits = torch.randn(self.NUM_TOKENS, num_experts, generator=g)
|
||||
bias = 0.1 * torch.randn(num_experts, generator=g) if biased else None
|
||||
logits = logits.to(get_device())
|
||||
bias = bias.to(get_device()) if biased else None
|
||||
ref_ids, ref_weights = self._route(
|
||||
num_fused_shared_experts=0, logits=logits, bias=bias, router=router
|
||||
)
|
||||
ids, weights = self._route(
|
||||
num_fused_shared_experts=1, logits=logits, bias=bias, router=router
|
||||
)
|
||||
|
||||
self.assertEqual(ids.shape, (self.NUM_TOKENS, k + 1))
|
||||
torch.testing.assert_close(
|
||||
weights[:, :k].sum(-1), torch.ones_like(weights[:, 0])
|
||||
)
|
||||
# Rank r's routed block starts at r * (num_local_routed + 1).
|
||||
num_local_routed = num_experts // self.EP_SIZE
|
||||
expected_ids = ref_ids + ref_ids // num_local_routed
|
||||
order, ref_order = ids[:, :k].argsort(-1), expected_ids.argsort(-1)
|
||||
self.assertTrue(
|
||||
torch.equal(
|
||||
ids[:, :k].gather(-1, order), expected_ids.gather(-1, ref_order)
|
||||
)
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
weights[:, :k].gather(-1, order), ref_weights.gather(-1, ref_order)
|
||||
)
|
||||
home_shared_id = (
|
||||
self.EP_RANK * (num_local_routed + 1) + num_local_routed
|
||||
)
|
||||
self.assertTrue(torch.all(ids[:, k] == home_shared_id))
|
||||
|
||||
def test_unnormalized_global_shared_contribution_is_one(self):
|
||||
"""Raw routed scores must not multiply the global shared contribution."""
|
||||
logits = torch.zeros(4, 256, device=get_device())
|
||||
rsf = 2.5
|
||||
with get_context().override_server_args(
|
||||
moe_a2a_backend="none", moe_runner_backend="triton"
|
||||
):
|
||||
for folded in (False, True):
|
||||
with self.subTest(folded=folded):
|
||||
config = TopKConfig(
|
||||
top_k=9,
|
||||
use_grouped_topk=True,
|
||||
num_expert_group=8,
|
||||
topk_group=4,
|
||||
renormalize=False,
|
||||
num_fused_shared_experts=1,
|
||||
correction_bias=torch.zeros(256, device=get_device()),
|
||||
scoring_func="sigmoid",
|
||||
routed_scaling_factor=rsf,
|
||||
apply_routed_scaling_factor_on_output=folded,
|
||||
allow_routed_experts_capture=False,
|
||||
)
|
||||
output = select_experts(logits, logits, config)
|
||||
torch.testing.assert_close(
|
||||
output.topk_weights[:, :8],
|
||||
torch.full_like(
|
||||
output.topk_weights[:, :8], 0.5 * (rsf if folded else 1.0)
|
||||
),
|
||||
)
|
||||
shared = output.topk_weights[:, -1] * (1.0 if folded else rsf)
|
||||
torch.testing.assert_close(shared, torch.ones_like(shared))
|
||||
|
||||
def test_unnormalized_weights_stay_raw(self):
|
||||
"""Unnormalized routing must preserve raw selected sigmoid scores."""
|
||||
k, num_experts = self.TOP_K, 256
|
||||
router = (num_experts, 8, 4, "sigmoid", True)
|
||||
g = torch.Generator(device="cpu").manual_seed(0)
|
||||
logits = torch.randn(self.NUM_TOKENS, num_experts, generator=g).to(get_device())
|
||||
bias = (0.1 * torch.randn(num_experts, generator=g)).to(get_device())
|
||||
_, ref_ids = biased_grouped_topk_impl(
|
||||
logits,
|
||||
logits,
|
||||
bias,
|
||||
topk=k,
|
||||
renormalize=False,
|
||||
num_expert_group=8,
|
||||
topk_group=4,
|
||||
)
|
||||
num_local_routed = num_experts // self.EP_SIZE
|
||||
for num_fused_shared_experts in (0, 1):
|
||||
with self.subTest(num_fused_shared_experts=num_fused_shared_experts):
|
||||
ids, weights = self._route(
|
||||
num_fused_shared_experts=num_fused_shared_experts,
|
||||
logits=logits,
|
||||
bias=bias,
|
||||
router=router,
|
||||
renormalize=False,
|
||||
)
|
||||
routed_ids = ids[:, :k]
|
||||
if num_fused_shared_experts:
|
||||
routed_ids = routed_ids - routed_ids // (num_local_routed + 1)
|
||||
self.assertTrue(
|
||||
torch.equal(
|
||||
routed_ids.sort(-1).values, ref_ids.long().sort(-1).values
|
||||
)
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
weights[:, :k], logits.sigmoid().gather(-1, routed_ids)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
@@ -11,6 +12,7 @@ from sglang.srt.layers.moe.topk import (
|
||||
_post_process_topk_ids,
|
||||
_zero_topk_weights_padded_region,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import is_hip
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
@@ -275,5 +277,57 @@ class TestSelectExpertsCustomRoutingPadMask(CustomTestCase):
|
||||
self.assertTrue(torch.all(out.topk_ids[:n_valid] < num_experts))
|
||||
|
||||
|
||||
@unittest.skipUnless(torch.cuda.is_available(), "padded-region masking needs a GPU")
|
||||
@unittest.skipIf(_IS_HIP, "HIP marks padded rows with zero weights instead")
|
||||
class TestSelectExpertsPerRankSharedSlotPadMask(CustomTestCase):
|
||||
"""With per-rank fused shared slots, padded rows must leave select_experts
|
||||
fully -1, the appended shared column included. Masking only before the
|
||||
append left them routed ids of -1 - S and a live home-rank shared slot."""
|
||||
|
||||
DEVICE = "cuda"
|
||||
|
||||
def test_padded_tail_masked_after_shared_append(self):
|
||||
num_tokens, num_experts, top_k, n_valid = 12, 256, 8, 9
|
||||
# EP8 rank 3: 32 routed slots per rank, then the rank's shared slot.
|
||||
home_shared_id = 3 * 33 + 32
|
||||
# Grouped sigmoid is masked after routing, sqrtsoftplus by its router.
|
||||
for scoring_func, num_expert_group in (("sigmoid", 8), ("sqrtsoftplus", None)):
|
||||
with self.subTest(scoring_func=scoring_func):
|
||||
g = torch.Generator(device="cpu").manual_seed(0)
|
||||
logits = torch.randn(num_tokens, num_experts, generator=g)
|
||||
bias = 0.1 * torch.randn(num_experts, generator=g)
|
||||
topk_config = TopKConfig(
|
||||
top_k=top_k + 1,
|
||||
use_grouped_topk=num_expert_group is not None,
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=4 if num_expert_group else None,
|
||||
num_fused_shared_experts=1,
|
||||
correction_bias=bias.to(self.DEVICE),
|
||||
scoring_func=scoring_func,
|
||||
routed_scaling_factor=2.5,
|
||||
allow_routed_experts_capture=False,
|
||||
)
|
||||
with (
|
||||
patch.object(
|
||||
topk_mod,
|
||||
"has_per_rank_fused_shared_slots",
|
||||
side_effect=lambda num_fused: num_fused > 0,
|
||||
),
|
||||
get_parallel().override(moe_ep_size=8, moe_ep_rank=3),
|
||||
):
|
||||
out = topk_mod.select_experts(
|
||||
hidden_states=torch.empty(num_tokens, 16, device=self.DEVICE),
|
||||
router_logits=logits.to(self.DEVICE),
|
||||
topk_config=topk_config,
|
||||
layer_id=0,
|
||||
num_token_non_padded=torch.tensor(
|
||||
n_valid, device=self.DEVICE, dtype=torch.int32
|
||||
),
|
||||
)
|
||||
self.assertTrue(torch.all(out.topk_ids[n_valid:] == -1))
|
||||
self.assertTrue(torch.all(out.topk_ids[:n_valid] >= 0))
|
||||
self.assertTrue(torch.all(out.topk_ids[:n_valid, -1] == home_shared_id))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -98,6 +98,50 @@ class TestWaterfillEPLB(CustomTestCase):
|
||||
self.assertTrue(torch.equal(processed_ids, torch.tensor([[0, 34, 270, 271]])))
|
||||
self.assertTrue(torch.equal(recorder_ids, torch.tensor([[0, 33, 263]])))
|
||||
|
||||
def test_eplb_remaps_every_column_of_routed_only_per_rank_ids(self):
|
||||
"""EPLB must remap all routed columns before shared-slot append and capture."""
|
||||
topk_ids = torch.tensor([[0, 33, 262]], dtype=torch.int32)
|
||||
topk_weights = torch.ones_like(topk_ids, dtype=torch.float32)
|
||||
topk_config = TopKConfig(
|
||||
top_k=4,
|
||||
num_fused_shared_experts=1,
|
||||
routed_scaling_factor=1.0,
|
||||
)
|
||||
dispatch_info = SimpleNamespace(
|
||||
num_physical_experts=264, ep_dispatch_algorithm="static"
|
||||
)
|
||||
|
||||
def fake_eplb_postprocess(
|
||||
ids, expert_location_dispatch_info, num_token_non_padded
|
||||
):
|
||||
return ids + 1
|
||||
|
||||
with (
|
||||
patch.object(topk_module, "_is_cuda", True),
|
||||
patch.object(topk_module, "_use_aiter", False),
|
||||
patch.object(
|
||||
topk_module, "has_per_rank_fused_shared_slots", return_value=True
|
||||
),
|
||||
get_parallel().override(moe_ep_size=8, moe_ep_rank=7),
|
||||
patch.object(
|
||||
topk_module,
|
||||
"_biased_grouped_topk_postprocess",
|
||||
side_effect=fake_eplb_postprocess,
|
||||
),
|
||||
):
|
||||
processed_ids, _, recorder_ids = topk_module._post_process_topk_ids(
|
||||
topk_ids=topk_ids.clone(),
|
||||
topk_weights=topk_weights.clone(),
|
||||
topk_config=topk_config,
|
||||
router_logits=torch.empty((1, 256)),
|
||||
layer_id=0,
|
||||
expert_location_dispatch_info=dispatch_info,
|
||||
)
|
||||
|
||||
# Physical ids 1, 34, 263 in the EP8 per-rank layout; 271 is rank 7's shared slot.
|
||||
self.assertTrue(torch.equal(processed_ids, torch.tensor([[1, 35, 270, 271]])))
|
||||
self.assertTrue(torch.equal(recorder_ids, torch.tensor([[1, 34, 263]])))
|
||||
|
||||
def test_topk_recorder_ids_exclude_global_fused_shared_slot(self):
|
||||
topk_ids = torch.tensor([[0, 33, 200, 256]], dtype=torch.int32)
|
||||
topk_weights = torch.ones_like(topk_ids, dtype=torch.float32)
|
||||
|
||||
@@ -35,12 +35,10 @@ class TestFusedSharedExpertScaling(CustomTestCase):
|
||||
UNBIASED_ROUTING = {"use_grouped_topk": True, "correction_bias": None}
|
||||
|
||||
def _run_remap(self, *, use_aiter, folded_in_topk=False):
|
||||
# Layout: [routed, routed, routed, shared]; the trailing column is the
|
||||
# fused shared expert. The shared weight starts at a sentinel to prove
|
||||
# the function overwrites it.
|
||||
topk_ids = torch.tensor([[5, 40, 100, 999]], dtype=torch.int32)
|
||||
# The remap appends the fused shared column after the routed ones.
|
||||
topk_ids = torch.tensor([[5, 40, 100]], dtype=torch.int32)
|
||||
routed_weights = torch.tensor([1.0, 0.5, 0.25], dtype=torch.float32)
|
||||
topk_weights = torch.tensor([[1.0, 0.5, 0.25, -123.0]], dtype=torch.float32)
|
||||
topk_weights = routed_weights.unsqueeze(0)
|
||||
|
||||
topk_config = TopKConfig(
|
||||
top_k=4,
|
||||
@@ -441,8 +439,8 @@ class TestFusedSharedExpertScaling(CustomTestCase):
|
||||
def test_shared_expert_ids_route_to_home_rank(self):
|
||||
# Sanity: the shared slot id is placed at this rank's interleaved
|
||||
# position (ep_rank * num_local_experts + num_local_routed).
|
||||
topk_ids = torch.tensor([[5, 40, 100, 999]], dtype=torch.int32)
|
||||
topk_weights = torch.tensor([[1.0, 0.5, 0.25, 0.0]], dtype=torch.float32)
|
||||
topk_ids = torch.tensor([[5, 40, 100]], dtype=torch.int32)
|
||||
topk_weights = torch.tensor([[1.0, 0.5, 0.25]], dtype=torch.float32)
|
||||
topk_config = TopKConfig(
|
||||
top_k=4,
|
||||
num_fused_shared_experts=1,
|
||||
@@ -470,6 +468,61 @@ class TestFusedSharedExpertScaling(CustomTestCase):
|
||||
expected_shared_id = self.EP_RANK * num_local_experts + num_local_routed
|
||||
self.assertEqual(out_ids[0, -1].item(), expected_shared_id)
|
||||
|
||||
def _route_with_per_rank_slots(self, *, num_fused_shared_experts, logits, bias):
|
||||
topk_config = TopKConfig(
|
||||
top_k=8 + num_fused_shared_experts,
|
||||
num_fused_shared_experts=num_fused_shared_experts,
|
||||
renormalize=True,
|
||||
correction_bias=bias,
|
||||
torch_native=True,
|
||||
scoring_func="sigmoid",
|
||||
routed_scaling_factor=self.ROUTED_SCALING_FACTOR,
|
||||
allow_routed_experts_capture=False,
|
||||
)
|
||||
with (
|
||||
patch.object(topk_module, "_use_aiter", False),
|
||||
patch.object(
|
||||
topk_module,
|
||||
"has_per_rank_fused_shared_slots",
|
||||
side_effect=lambda num_fused: num_fused > 0,
|
||||
),
|
||||
get_parallel().override(moe_ep_size=self.EP_SIZE, moe_ep_rank=self.EP_RANK),
|
||||
):
|
||||
out = topk_module.select_experts(
|
||||
torch.empty(logits.shape[0], 1), logits, topk_config
|
||||
)
|
||||
return out.topk_ids, out.topk_weights
|
||||
|
||||
def test_per_rank_slot_routes_like_no_fusion(self):
|
||||
"""With per-rank shared slots the router must pick and renormalize the
|
||||
routed experts alone. Asked for top_k with the shared slot counted, it
|
||||
renormalized over one routed expert too many, which the shared column
|
||||
then overwrote, so the kept routed weights summed to about 0.9."""
|
||||
torch.manual_seed(0)
|
||||
logits = torch.randn(16, self.NUM_PHYSICAL_ROUTED)
|
||||
bias = 0.1 * torch.randn(self.NUM_PHYSICAL_ROUTED)
|
||||
ref_ids, ref_weights = self._route_with_per_rank_slots(
|
||||
num_fused_shared_experts=0, logits=logits, bias=bias
|
||||
)
|
||||
ids, weights = self._route_with_per_rank_slots(
|
||||
num_fused_shared_experts=1, logits=logits, bias=bias
|
||||
)
|
||||
|
||||
self.assertEqual(ids.shape, (16, 9))
|
||||
torch.testing.assert_close(weights[:, :8].sum(-1), torch.ones(16))
|
||||
# Rank r's routed block starts at r * (num_local_routed + 1).
|
||||
num_local_routed = self.NUM_PHYSICAL_ROUTED // self.EP_SIZE
|
||||
expected_ids = ref_ids + ref_ids // num_local_routed
|
||||
order, ref_order = ids[:, :8].argsort(-1), expected_ids.argsort(-1)
|
||||
self.assertTrue(
|
||||
torch.equal(
|
||||
ids[:, :8].gather(-1, order), expected_ids.gather(-1, ref_order)
|
||||
)
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
weights[:, :8].gather(-1, order), ref_weights.gather(-1, ref_order)
|
||||
)
|
||||
|
||||
def test_glm53_idle_rank_preserves_routed_topk_contract(self):
|
||||
"""An idle rank must return the pre-append top-8 shape; returning top-9
|
||||
here would make the AITER postprocessor append shared ID 288 twice."""
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""DeepEP-family backend recognition in RoutedExpertsCapturer."""
|
||||
"""RoutedExpertsCapturer: DeepEP-family backend recognition and capture width."""
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
@@ -8,6 +8,7 @@ import torch
|
||||
|
||||
from sglang.srt.layers.moe.utils import MoeA2ABackend
|
||||
from sglang.srt.state_capturer import routed_experts as re_mod
|
||||
from sglang.srt.state_capturer.base import BaseDeviceCache
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
@@ -67,5 +68,55 @@ class TestGetLocalSliceBackendBranch(CustomTestCase):
|
||||
self.assertTrue(torch.equal(out, buf[6:10, :, : self.K]))
|
||||
|
||||
|
||||
class TestCaptureRoutedIdsOnly(CustomTestCase):
|
||||
TOPK_SIZE = 3
|
||||
|
||||
def _capturer(self):
|
||||
cap = object.__new__(re_mod.RoutedExpertsCapturer)
|
||||
cap.num_fused_shared_experts = 1
|
||||
cap.device_cache = BaseDeviceCache(
|
||||
max_batch_size=8,
|
||||
num_layers=2,
|
||||
topk_size=self.TOPK_SIZE + 1,
|
||||
device="cpu",
|
||||
name="routed_experts",
|
||||
)
|
||||
cap.gather_buffer = torch.empty((8, self.TOPK_SIZE + 1), dtype=torch.int32)
|
||||
return cap
|
||||
|
||||
def _capture(self, cap, ids, backend):
|
||||
with (
|
||||
mock.patch.object(
|
||||
re_mod, "get_moe_a2a_backend", return_value=MoeA2ABackend(backend)
|
||||
),
|
||||
mock.patch.object(
|
||||
re_mod,
|
||||
"attn_tp_all_gather_into_tensor",
|
||||
side_effect=lambda out, inp: out.copy_(inp),
|
||||
),
|
||||
mock.patch.object(
|
||||
re_mod, "get_parallel", return_value=SimpleNamespace(attn_tp_size=1)
|
||||
),
|
||||
):
|
||||
cap.capture(layer_id=1, topk_indices=ids)
|
||||
|
||||
def test_routed_only_ids_fill_the_routed_columns(self):
|
||||
"""Routed-only capture must fill the routed columns, including scattered a2a."""
|
||||
ids = torch.arange(4 * self.TOPK_SIZE, dtype=torch.int32).view(4, -1)
|
||||
for backend in ("none", "deepep"):
|
||||
with self.subTest(backend=backend):
|
||||
cap = self._capturer()
|
||||
self._capture(cap, ids, backend)
|
||||
self.assertTrue(
|
||||
torch.equal(cap.device_cache.buffer[:4, 1, : self.TOPK_SIZE], ids)
|
||||
)
|
||||
|
||||
def test_other_widths_are_rejected(self):
|
||||
"""Only the routed-only width may be padded; any other width is a caller bug."""
|
||||
ids = torch.zeros((4, self.TOPK_SIZE - 1), dtype=torch.int32)
|
||||
with self.assertRaises(ValueError):
|
||||
self._capture(self._capturer(), ids, "none")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user