[AMD][V4.1][*/N] Fix shared-expert fusion accuracy and speed up MoE routing on ROCm (#42011)

Co-authored-by: wunhuang <wunhuang@amd.com>
This commit is contained in:
kk
2026-10-01 18:19:06 -07:00
committed by GitHub
co-authored by wunhuang
parent 8acd36b56b
commit 29f6d408c0
7 changed files with 248 additions and 14 deletions
@@ -224,11 +224,13 @@ def _router_gate_kernel(
stride_pm,
stride_om,
routed_scaling_factor,
shared_id_base,
SPLIT_K: tl.constexpr,
WRITE_LOGITS: tl.constexpr,
HAS_BIAS: tl.constexpr,
RENORM: tl.constexpr,
TOPK: tl.constexpr,
NUM_SHARED: tl.constexpr,
):
row = tl.program_id(0)
weights, ids = _gate_row(
@@ -247,7 +249,12 @@ def _router_gate_kernel(
TOPK,
)
lane = tl.arange(0, 64)
out_mask = lane < TOPK
if NUM_SHARED > 0:
# fused_append_shared_experts: shared slot s holds id shared_id_base + s, weight 1.0
is_shared = lane >= TOPK
weights = tl.where(is_shared, 1.0, weights)
ids = tl.where(is_shared, shared_id_base + lane - TOPK, ids)
out_mask = lane < TOPK + NUM_SHARED
tl.store(weights_ptr + row * stride_om + lane, weights, mask=out_mask)
tl.store(ids_ptr + row * stride_om + lane, ids, mask=out_mask)
@@ -260,12 +267,15 @@ def rocm_router_gate(
routed_scaling_factor: Optional[float],
*,
partials: Optional[torch.Tensor] = None,
num_shared: int = 0,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""aiter topk_gating(..., score_func="sqrtsoftplus") for 384 experts: fp32 weights and int32
ids [M, topk]; with partials their fixed-order sum is gated and written into
gating_output."""
gating_output. num_shared > 0 appends fused_append_shared_experts(..., scale_factor=1.0,
N=384) columns: ids and weights become [M, topk + num_shared]."""
M, num_experts = gating_output.shape
assert num_experts == _GATE_NUM_EXPERTS and 0 < topk <= _MAX_TOPK
assert 0 <= num_shared <= _WARP - topk
assert gating_output.stride(1) == 1
if partials is not None:
assert (
@@ -282,8 +292,9 @@ def rocm_router_gate(
assert (
correction_bias.shape == (num_experts,) and correction_bias.is_contiguous()
)
weights = torch.empty((M, topk), dtype=torch.float32, device=gating_output.device)
ids = torch.empty((M, topk), dtype=torch.int32, device=gating_output.device)
width = topk + num_shared
weights = torch.empty((M, width), dtype=torch.float32, device=gating_output.device)
ids = torch.empty((M, width), dtype=torch.int32, device=gating_output.device)
if M == 0:
return weights, ids
_router_gate_kernel[(M,)](
@@ -297,11 +308,13 @@ def rocm_router_gate(
stride_pm,
weights.stride(0),
float(1.0 if routed_scaling_factor is None else routed_scaling_factor),
num_experts,
SPLIT_K=split_k,
WRITE_LOGITS=split_k > 0,
HAS_BIAS=correction_bias is not None,
RENORM=bool(renormalize),
TOPK=topk,
NUM_SHARED=num_shared,
num_warps=1,
)
return weights, ids
+4
View File
@@ -887,6 +887,10 @@ class Envs:
# (matches `gate_mode="separated"`, the layout used by gptoss_fp4 tuned
# configs and by Mxfp4MoEMethod's post-fix weight shuffle).
SGLANG_USE_AITER_MOE_GU_ITLV = EnvBool(True)
# aiter opus moe_sorting dispatch policy (0 auto, 1 oneshot, 2 multi-phase). Auto picks
# oneshot below ~24 tokens, which on gfx950 costs 11-16 us vs 6-7 us for multi-phase at
# E=385 / 129; outputs are identical and multi-phase is never slower up to 16384 tokens.
SGLANG_AITER_MOE_SORTING_DISPATCH_POLICY = EnvInt(2)
# Fold `silu(gate) * up` into the triton MoE up-GEMM epilogue. W13 rows are
# permuted in place at load so gate/up land in adjacent columns of the same
# output tile, which removes intermediate_cache1 and the standalone
@@ -12,6 +12,7 @@ import torch
_SMALLM_MOE_ON = os.environ.get("SGLANG_ROCM_SMALLM_MOE", "1") != "0"
from sglang.srt.environ import envs
from sglang.srt.layers.moe.moe_runner.base import (
MoeQuantInfo,
MoeRunnerConfig,
@@ -43,6 +44,8 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
_MOE_SORTING_DISPATCH_POLICY = envs.SGLANG_AITER_MOE_SORTING_DISPATCH_POLICY.get()
class AiterQuantType(str, Enum):
NONE = "No"
@@ -327,6 +330,7 @@ class AiterRunnerCore(MoeRunnerCore):
extra["swiglu_limit"] = quant_info.swiglu_limit
if self.config.no_combine:
extra["no_combine"] = True
extra["moe_sorting_dispatch_policy"] = _MOE_SORTING_DISPATCH_POLICY
# gfx950 small-M MXFP4 kernel (on by default, SGLANG_ROCM_SMALLM_MOE=0 disables): same layouts as aiter, bf16 activations.
if _SMALLM_MOE_ON and quant_info.w13_weight.element_size() == 1:
+34 -7
View File
@@ -1441,8 +1441,12 @@ def biased_topk_jit_kernel_impl(
packed_out: Optional[torch.Tensor] = None,
sqrtsoftplus_log1p: bool = False,
router_logits_partials: Optional[torch.Tensor] = None,
num_shared_append: int = 0,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""num_shared_append is only workable when router_logits_partials is not None
and it is only used by rocm_router_gate"""
assert hidden_states.shape[0] == gating_output.shape[0], "Number of tokens mismatch"
assert num_shared_append == 0 or router_logits_partials is not None
if _use_aiter and scoring_func == "sqrtsoftplus" and num_fused_shared_experts == 0:
assert packed_out is None, "aiter topk_gating cannot emit packed ids"
@@ -1455,6 +1459,7 @@ def biased_topk_jit_kernel_impl(
renormalize,
routed_scaling_factor,
partials=router_logits_partials,
num_shared=num_shared_append,
)
from aiter import topk_gating
@@ -2274,7 +2279,17 @@ def _post_process_topk_ids(
fused_shared_experts_scaling_factor = (
topk_config.fused_shared_experts_scaling_factor
)
capture_routed_experts_if_allowed(topk_config, layer_id, topk_ids)
# The router already wrote the aiter shared columns (see _aiter_append below).
_gate_appended = (
num_fused_shared_experts > 0
and _use_aiter
and not use_per_rank_shared_slots
and topk_ids.shape[-1] == topk_config.top_k
)
routed_topk_ids = (
topk_ids[:, :-num_fused_shared_experts] if _gate_appended else topk_ids
)
capture_routed_experts_if_allowed(topk_config, layer_id, routed_topk_ids)
recorder_topk_ids = None
_fold_pad_into_append = False
recorder_was_fused = False
@@ -2378,13 +2393,17 @@ def _post_process_topk_ids(
# second zeroing here would be redundant (zeroing is idempotent).
if recorder_topk_ids is None and not recorder_was_fused:
recorder_topk_ids = topk_ids
recorder_topk_ids = (
topk_ids[:, :-num_fused_shared_experts] if _gate_appended else topk_ids
)
_aiter_append = num_fused_shared_experts > 0 and _use_aiter
if _aiter_append and envs.SGLANG_OPT_USE_JIT_KERNEL_GROUPED_TOPK.get():
# That router emits the shared slots itself; appending again would write
# the shared id twice and evict a real routed expert.
_aiter_append = topk_ids.shape[-1] < topk_config.top_k
# The JIT grouped router and the ROCm decode gate emit the shared slots themselves;
# appending again would write the shared id twice and evict a real routed expert.
_aiter_append = (
num_fused_shared_experts > 0
and _use_aiter
and topk_ids.shape[-1] < topk_config.top_k
)
if _aiter_append and use_per_rank_shared_slots:
# Fused path: append shared experts AND apply the per-rank shared-slot
@@ -2536,6 +2555,7 @@ def select_experts(
and (
num_fused_shared_experts == 0
or has_per_rank_fused_shared_slots(num_fused_shared_experts)
or not _eplb_remap_enabled()
)
):
# only the aiter sqrtsoftplus gate takes the partials; every other route reads router_logits
@@ -2668,6 +2688,13 @@ def select_experts(
_packed_kwargs["sqrtsoftplus_log1p"] = True
if router_logits_partials is not None:
_packed_kwargs["router_logits_partials"] = router_logits_partials
if (
_use_aiter
and num_fused_shared_experts > 0
and not has_per_rank_fused_shared_slots(num_fused_shared_experts)
):
# the ROCm decode gate writes the aiter shared columns itself
_packed_kwargs["num_shared_append"] = num_fused_shared_experts
topk_weights, topk_ids = _biased_topk(
hidden_states=hidden_states,
gating_output=router_logits,
@@ -1729,6 +1729,10 @@ def quantize_block_fp8_weight_to_mxfp4(
weight_block_size: List[int],
mxfp4_block_size: int = 32,
) -> Tuple[torch.Tensor, torch.Tensor]:
if _is_hip:
return _quantize_block_fp8_weight_to_mxfp4_rne(
fp8_weight, fp8_scale, weight_block_size, mxfp4_block_size
)
fp8_weight_dequant = block_quant_dequant(
fp8_weight,
fp8_scale.to(torch.float32),
@@ -1747,6 +1751,44 @@ def quantize_block_fp8_weight_to_mxfp4(
return fp4_weight, fp4_scale.contiguous().view(torch.float8_e8m0fnu)
def _quantize_block_fp8_weight_to_mxfp4_rne(
fp8_weight: torch.Tensor,
fp8_scale: torch.Tensor,
weight_block_size: List[int],
mxfp4_block_size: int,
) -> Tuple[torch.Tensor, torch.Tensor]:
fp8_weight_dequant = block_quant_dequant(
fp8_weight,
fp8_scale.to(torch.float32),
weight_block_size,
torch.float32,
)
*lead, k = fp8_weight_dequant.shape
x = fp8_weight_dequant.reshape(-1, mxfp4_block_size)
# Scale as in aiter dynamic_mxfp4_quant: round amax's mantissa at 1.75, then
# floor(log2) - 2 (e2m1 emax), saturating the few elements above 6 * scale.
amax = x.abs().amax(dim=-1, keepdim=True).clamp_min(torch.finfo(torch.float32).tiny)
amax = ((amax.view(torch.int32) + 0x200000) & 0x7F800000).view(torch.float32)
exp = (torch.floor(torch.log2(amax)) - 2).clamp(-127, 127)
y = (x.abs() / torch.exp2(exp)).clamp(max=6.0)
# Round to nearest, ties to even mantissa. The FP8 source puts many elements
# exactly on e2m1 midpoints; ties toward zero shrink each matrix by ~5%.
mag = torch.where(
y < 2.0,
torch.round(y * 2.0),
torch.where(y < 4.0, torch.round(y) + 2.0, torch.round(y * 0.5) + 4.0),
).to(torch.uint8)
codes = mag | ((x < 0) & (mag != 0)).to(torch.uint8) << 3
codes = codes.reshape(*lead, k)
fp4_weight = (codes[..., 0::2] | (codes[..., 1::2] << 4)).contiguous()
fp4_scale = (exp + 127).to(torch.uint8).reshape(*lead, k // mxfp4_block_size)
return fp4_weight.view(torch.int8), fp4_scale.contiguous().view(
torch.float8_e8m0fnu
)
def requant_weight_ue8m0_inplace(weight, weight_scale_inv, weight_block_size):
assert isinstance(weight, torch.nn.Parameter)
assert isinstance(weight_scale_inv, torch.nn.Parameter)