MoE: small-batch sorting path with fused mxfp8 quantisation (#36559)

Co-authored-by: mikevin920 <mikevin920@yahoo.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: Kevin Mi <mikevin920@gmail.com>
Co-authored-by: Kevin Mi <kevin.mi@radixark.ai>
Co-authored-by: Kevin Mi <45493463+kevin-mii@users.noreply.github.com>
Co-authored-by: YC Yen-Ching Tseng <yctseng@amd.com>
Co-authored-by: Liangsheng Yin <hnyls2002@gmail.com>
This commit is contained in:
Chunan Zeng
2026-09-24 13:18:18 -07:00
committed by GitHub
co-authored by mikevin920 Claude Fable 5 Kevin Mi Kevin Mi Kevin Mi YC Yen-Ching Tseng Liangsheng Yin
parent 0b53305f48
commit 36f59982fa
5 changed files with 649 additions and 19 deletions
@@ -0,0 +1,557 @@
# SPDX-License-Identifier: Apache-2.0
"""Single-launch MoE sorting (+ stage1 mxfp8 activation quant) for decode-sized
batches on the aiter MoE runner, installed as runtime patches on ``aiter.fused_moe``.
Output layouts match the aiter kernels bit-for-bit:
sorted_ids[i] = (topk_slot << 24) | token (padding: (topk << 24) | M)
sorted_weights = pair weight (padding: 0)
sorted_expert_ids[b] = expert of block b
num_valid_ids = [num_blocks * block_size, M]
quant a1 = per-token fp8 rows; e8m0 RoundUp scale byte per
(sorted_row, group) at aiter's mx_scale_shuffle_idx address
"""
from __future__ import annotations
import functools
import logging
from contextvars import ContextVar
import torch
import triton
import triton.language as tl
from sglang.srt.utils import is_gfx95_supported
logger = logging.getLogger(__name__)
# P <= 64: one sort CTA does the whole P x P rank compare
# per-M ints stay unspecialized so prefill sizes reuse the graph-capture compiles
@triton.jit(do_not_specialize=["M", "moe_buf_numel", "num_buf"])
def _moe_sorting_small_kernel(
topk_ids_ptr, # [M, topk] i32
topk_weights_ptr, # [M, topk] fp32
sorted_ids_ptr, # [max_padded] i32
sorted_weights_ptr, # [max_padded] fp32
sorted_expert_ids_ptr, # [max_blocks] i32
num_valid_ids_ptr, # [2] i32
moe_buf_ptr,
moe_buf_numel,
qx_ptr, # [M, N_COLS] activations to mx-quantize (EMIT_MX only)
qout_ptr, # [M, N_COLS] fp8 out
qscale_ptr, # swizzled e8m0 bytes, one per (sorted_row, group)
M,
TOPK: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
P_POW2: tl.constexpr, # >= M * topk
PAD_POW2: tl.constexpr, # >= (M * topk) * BLOCK_SIZE (worst-case padded len)
BUF_BLOCK: tl.constexpr,
num_buf, # buf-zero CTAs occupy pids [1, num_buf]
EMIT_MX: tl.constexpr, # also emit the mxfp8 quant of qx (group_size 32)
N_COLS: tl.constexpr,
QCHUNK: tl.constexpr, # columns per quant iteration (multiple of 32)
SCALEN_PAD: tl.constexpr, # ceil(N_COLS/32 / 8) * 8
):
pid = tl.program_id(0)
if pid > 0 and pid <= num_buf:
offs = (pid - 1) * BUF_BLOCK + tl.arange(0, BUF_BLOCK)
tl.store(
moe_buf_ptr + offs,
tl.zeros((BUF_BLOCK,), moe_buf_ptr.dtype.element_ty),
mask=offs < moe_buf_numel,
)
return
P = M * TOPK
offs_p = tl.arange(0, P_POW2)
# the quant CTAs recompute the P x P sort math instead of waiting on pid 0
mask_p = offs_p < P
# sentinel expert keeps inactive lanes out of every "smaller expert" count
e = tl.load(topk_ids_ptr + offs_p, mask=mask_p, other=0x7FFFFFFF)
w = tl.load(topk_weights_ptr + offs_p, mask=mask_p, other=0.0)
token = offs_p // TOPK
slot = offs_p % TOPK
# Stable rank within the pair's expert, and per-expert pair count.
same = e[:, None] == e[None, :]
rank = tl.sum(tl.where(same & (offs_p[None, :] < offs_p[:, None]), 1, 0), axis=1)
cnt = tl.sum(tl.where(same & mask_p[None, :], 1, 0), axis=1)
blocks_of_e = (cnt + BLOCK_SIZE - 1) // BLOCK_SIZE
is_leader = (rank == 0) & mask_p
# leaders carry their expert's block count
smaller = e[None, :] < e[:, None]
blocks_before = tl.sum(
tl.where(smaller & is_leader[None, :], blocks_of_e[None, :], 0), axis=1
)
dest = blocks_before * BLOCK_SIZE + rank
if EMIT_MX and pid > num_buf:
# quant CTA: one (pair, column-chunk) slice, mirroring fused_dynamic_mxfp8_quant_moe_sort
q_id = pid - num_buf - 1
CHUNKS: tl.constexpr = N_COLS // QCHUNK
p = q_id // CHUNKS
c0 = (q_id % CHUNKS) * QCHUNK
if p < P:
offs_q = tl.arange(0, QCHUNK)
offs_g = tl.arange(0, QCHUNK // 32)
token_p = tl.sum(tl.where(offs_p == p, token, 0), axis=0)
dest_p = tl.sum(tl.where(offs_p == p, dest, 0), axis=0)
base_sw = (
(dest_p // 32) * (SCALEN_PAD * 32)
+ (dest_p % 16) * 4
+ (dest_p % 32) // 16
)
x = tl.load(qx_ptr + token_p * N_COLS + c0 + offs_q).to(tl.float32)
x2 = tl.reshape(x, (QCHUNK // 32, 32))
amax = tl.maximum(tl.max(tl.abs(x2), axis=1), 1e-10)
sf = amax * (1.0 / 448.0)
bits = sf.to(tl.int32, bitcast=True)
exp = (bits >> 23) & 0xFF
exp = tl.where((bits & 0x7FFFFF) != 0, exp + 1, exp)
scale = (exp << 23).to(tl.float32, bitcast=True)
if p % TOPK == 0:
# one fp8 out row per token (pairs are token-major)
q = tl.clamp(x2 / scale[:, None], -448.0, 448.0)
tl.store(
qout_ptr + token_p * N_COLS + c0 + offs_q,
tl.reshape(q, (QCHUNK,)).to(qout_ptr.dtype.element_ty),
)
y = c0 // 32 + offs_g
sw = base_sw + (y // 8) * 256 + (y % 4) * 64 + ((y % 8) // 4) * 2
tl.store(qscale_ptr + sw, exp.to(tl.uint8))
return
total_blocks = tl.sum(tl.where(is_leader, blocks_of_e, 0), axis=0)
num_valid = total_blocks * BLOCK_SIZE
# pid 0: pad the whole used region first, then scatter the real pairs over it
offs_pad = tl.arange(0, PAD_POW2)
pad_mask = offs_pad < num_valid
pad_val = (TOPK << 24) | M
tl.store(
sorted_ids_ptr + offs_pad,
tl.full((PAD_POW2,), 0, tl.int32) + pad_val,
mask=pad_mask,
)
tl.store(
sorted_weights_ptr + offs_pad, tl.zeros((PAD_POW2,), tl.float32), mask=pad_mask
)
tl.debug_barrier()
tl.store(sorted_ids_ptr + dest, (slot << 24) | token, mask=mask_p)
tl.store(sorted_weights_ptr + dest, w, mask=mask_p)
# leaders write their expert into each of their blocks: at most 2 with P <= 2 * BLOCK_SIZE
for j in tl.static_range(2):
bm = is_leader & (j < blocks_of_e)
tl.store(sorted_expert_ids_ptr + blocks_before + j, e, mask=bm)
tl.store(
num_valid_ids_ptr + tl.arange(0, 2),
tl.where(tl.arange(0, 2) == 0, num_valid, M),
)
@triton.jit
def _expert_chunk_blocks(e, offs_chunk, BLOCK_SIZE: tl.constexpr):
cnt = tl.sum((e[:, None] == offs_chunk[None, :]).to(tl.int32), axis=0)
return cnt, (cnt + BLOCK_SIZE - 1) // BLOCK_SIZE
# per-M ints stay unspecialized so prefill sizes reuse the graph-capture compiles
@triton.jit(do_not_specialize=["M", "moe_buf_numel", "num_buf"])
def _moe_sorting_small_kernel_distributed(
topk_ids_ptr, # [M, topk] i32
topk_weights_ptr, # [M, topk] fp32
sorted_ids_ptr, # [max_padded] i32
sorted_weights_ptr, # [max_padded] fp32
sorted_expert_ids_ptr, # [max_blocks] i32
num_valid_ids_ptr, # [2] i32
moe_buf_ptr,
moe_buf_numel,
qx_ptr, # [M, N_COLS] activations to mx-quantize (EMIT_MX only)
qout_ptr, # [M, N_COLS] fp8 out
qscale_ptr, # swizzled e8m0 bytes, one per (sorted_row, group)
M,
num_experts,
TOPK: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
P_POW2: tl.constexpr, # >= M * topk
P_CHUNK: tl.constexpr, # tile width for pairwise/expert-space loops
E_CEIL: tl.constexpr, # num_experts rounded up to P_CHUNK
PAD_POW2: tl.constexpr, # >= (M * topk) * BLOCK_SIZE (worst-case padded len)
BUF_BLOCK: tl.constexpr,
num_buf, # buf-zero CTAs occupy pids [1, num_buf]
EMIT_MX: tl.constexpr, # also emit the mxfp8 quant of qx (group_size 32)
N_COLS: tl.constexpr,
QCHUNK: tl.constexpr, # columns per quant iteration (multiple of 32)
SCALEN_PAD: tl.constexpr, # ceil(N_COLS/32 / 8) * 8
):
pid = tl.program_id(0)
P = M * TOPK
offs_p = tl.arange(0, P_POW2)
mask_p = offs_p < P
# sentinel expert keeps inactive lanes out of every "smaller expert" count
e = tl.load(topk_ids_ptr + offs_p, mask=mask_p, other=0x7FFFFFFF)
# one writer per byte: expert-chunk CTAs, then num_buf zero-fill CTAs, then one CTA per pair
NUM_ECHUNK: tl.constexpr = E_CEIL // P_CHUNK
if pid < NUM_ECHUNK:
# expert-chunk CTA: expert-id table, block-tail padding and (last chunk) num_valid
x0 = pid * P_CHUNK
offs_x = x0 + tl.arange(0, P_CHUNK)
cnt_x, blocks_x = _expert_chunk_blocks(e, offs_x, BLOCK_SIZE)
# a runtime `if` inside static_range mis-lowers, so accumulate under a uniform mask
blocks_before_chunk = tl.zeros((), tl.int32)
for y0 in tl.static_range(0, E_CEIL, P_CHUNK):
offs_y = y0 + tl.arange(0, P_CHUNK)
cnt_y, blocks_y = _expert_chunk_blocks(e, offs_y, BLOCK_SIZE)
blocks_before_chunk += tl.sum(tl.where(offs_y < x0, blocks_y, 0), axis=0)
# exclusive prefix within the chunk
prefix_x = tl.sum(
tl.where(offs_x[None, :] < offs_x[:, None], blocks_x[None, :], 0), axis=1
)
bb_x = blocks_before_chunk + prefix_x
MAX_BLOCKS_PER_EXPERT: tl.constexpr = (P_POW2 + BLOCK_SIZE - 1) // BLOCK_SIZE
for j in tl.static_range(MAX_BLOCKS_PER_EXPERT):
bm = (offs_x < num_experts) & (j < blocks_x)
tl.store(sorted_expert_ids_ptr + bb_x + j, offs_x, mask=bm)
pad_val = (TOPK << 24) | M
for j in tl.static_range(MAX_BLOCKS_PER_EXPERT * BLOCK_SIZE):
pm = (j >= cnt_x) & (j < blocks_x * BLOCK_SIZE)
tl.store(sorted_ids_ptr + bb_x * BLOCK_SIZE + j, pad_val, mask=pm)
tl.store(sorted_weights_ptr + bb_x * BLOCK_SIZE + j, 0.0, mask=pm)
if pid == NUM_ECHUNK - 1:
num_valid = (blocks_before_chunk + tl.sum(blocks_x, axis=0)) * BLOCK_SIZE
tl.store(
num_valid_ids_ptr + tl.arange(0, 2),
tl.where(tl.arange(0, 2) == 0, num_valid, M),
)
return
if pid < NUM_ECHUNK + num_buf:
offs = (pid - NUM_ECHUNK) * BUF_BLOCK + tl.arange(0, BUF_BLOCK)
tl.store(
moe_buf_ptr + offs,
tl.zeros((BUF_BLOCK,), moe_buf_ptr.dtype.element_ty),
mask=offs < moe_buf_numel,
)
return
# per-pair CTA: dest = blocks of smaller experts * BLOCK_SIZE + stable rank in its expert
p = pid - NUM_ECHUNK - num_buf
if p >= P:
return
e_p = tl.sum(tl.where(offs_p == p, e, 0), axis=0)
rank_p = tl.sum(tl.where((offs_p < p) & (e == e_p), 1, 0), axis=0)
bb_p = tl.zeros((), tl.int32)
for x0 in tl.static_range(0, E_CEIL, P_CHUNK):
offs_x = x0 + tl.arange(0, P_CHUNK)
cnt_x, blocks_x = _expert_chunk_blocks(e, offs_x, BLOCK_SIZE)
bb_p += tl.sum(tl.where(offs_x < e_p, blocks_x, 0), axis=0)
dest_p = bb_p * BLOCK_SIZE + rank_p
token_p2 = p // TOPK
slot_p = p % TOPK
w_p = tl.load(topk_weights_ptr + p)
tl.store(sorted_ids_ptr + dest_p, (slot_p << 24) | token_p2)
tl.store(sorted_weights_ptr + dest_p, w_p)
if EMIT_MX:
# mirrors fused_dynamic_mxfp8_quant_moe_sort: e8m0 RoundUp scale, swizzled address
token_p = p // TOPK
offs_q = tl.arange(0, QCHUNK)
offs_g = tl.arange(0, QCHUNK // 32)
base_sw = (
(dest_p // 32) * (SCALEN_PAD * 32) + (dest_p % 16) * 4 + (dest_p % 32) // 16
)
for cc in tl.static_range(N_COLS // QCHUNK):
c0 = cc * QCHUNK
x = tl.load(qx_ptr + token_p * N_COLS + c0 + offs_q).to(tl.float32)
x2 = tl.reshape(x, (QCHUNK // 32, 32))
amax = tl.maximum(tl.max(tl.abs(x2), axis=1), 1e-10)
sf = amax * (1.0 / 448.0)
bits = sf.to(tl.int32, bitcast=True)
exp = (bits >> 23) & 0xFF
exp = tl.where((bits & 0x7FFFFF) != 0, exp + 1, exp)
scale = (exp << 23).to(tl.float32, bitcast=True)
if p % TOPK == 0:
# one fp8 out row per token (pairs are token-major)
q = tl.clamp(x2 / scale[:, None], -448.0, 448.0)
tl.store(
qout_ptr + token_p * N_COLS + c0 + offs_q,
tl.reshape(q, (QCHUNK,)).to(qout_ptr.dtype.element_ty),
)
y = c0 // 32 + offs_g
sw = base_sw + (y // 8) * 256 + (y % 4) * 64 + ((y % 8) // 4) * 2
tl.store(qscale_ptr + sw, exp.to(tl.uint8))
def _small_sort_supported(topk_ids, block_size, expert_mask, num_local_tokens):
m, topk = topk_ids.shape
return (
expert_mask is None
and num_local_tokens is None
and m * topk <= 256
and topk < 128
and topk_ids.dtype == torch.int32
and topk_ids.is_contiguous()
)
def _run_small_sort(
topk_ids,
topk_weights,
sorted_ids,
sorted_weights,
sorted_expert_ids,
num_valid_ids,
moe_buf,
block_size,
mx_quant_input,
num_experts,
):
m, topk = topk_ids.shape
p = m * topk
buf_block = 4096
emit_mx = mx_quant_input is not None
if emit_mx:
n_cols = mx_quant_input.shape[-1]
scalen_pad = ((n_cols // 32 + 7) // 8) * 8
max_padded = sorted_ids.shape[0]
qout = torch.empty(m, n_cols, dtype=torch.float8_e4m3fn, device=topk_ids.device)
qscale = torch.empty(
((max_padded + 31) // 32) * 32,
scalen_pad,
dtype=torch.uint8,
device=topk_ids.device,
)
else:
n_cols, scalen_pad = 32, 8
qout = qscale = moe_buf # unused placeholder pointers
num_buf = triton.cdiv(max(moe_buf.numel(), 1), buf_block)
if p <= 64 and p <= 2 * block_size:
# compact variant: one sort CTA does the P x P rank compare
num_quant = (p * (n_cols // min(2048, n_cols))) if emit_mx else 0
grid = (1 + num_buf + num_quant,)
_moe_sorting_small_kernel[grid](
topk_ids,
topk_weights,
sorted_ids,
sorted_weights,
sorted_expert_ids,
num_valid_ids,
moe_buf,
moe_buf.numel(),
mx_quant_input if emit_mx else moe_buf,
qout,
qscale,
m,
TOPK=topk,
BLOCK_SIZE=block_size,
P_POW2=triton.next_power_of_2(p),
PAD_POW2=triton.next_power_of_2(p * block_size),
BUF_BLOCK=buf_block,
num_buf=num_buf,
EMIT_MX=emit_mx,
N_COLS=n_cols,
QCHUNK=min(2048, n_cols),
SCALEN_PAD=scalen_pad,
num_warps=4,
)
if emit_mx:
return qout, qscale.view(torch.float8_e8m0fnu)
return None
# distributed variant: 64-wide expert/pair tiles, one writer per byte
p_chunk = 64
num_echunk = (num_experts + p_chunk - 1) // p_chunk
num_pair = p
grid = (num_echunk + num_buf + num_pair,)
_moe_sorting_small_kernel_distributed[grid](
topk_ids,
topk_weights,
sorted_ids,
sorted_weights,
sorted_expert_ids,
num_valid_ids,
moe_buf,
moe_buf.numel(),
mx_quant_input if emit_mx else moe_buf,
qout,
qscale,
m,
num_experts,
TOPK=topk,
BLOCK_SIZE=block_size,
P_POW2=triton.next_power_of_2(p),
P_CHUNK=p_chunk,
E_CEIL=(num_experts + p_chunk - 1) // p_chunk * p_chunk,
PAD_POW2=triton.next_power_of_2(p * block_size),
BUF_BLOCK=buf_block,
num_buf=num_buf,
EMIT_MX=emit_mx,
N_COLS=n_cols,
QCHUNK=min(2048, n_cols),
SCALEN_PAD=scalen_pad,
num_warps=8,
)
if emit_mx:
return qout, qscale.view(torch.float8_e8m0fnu)
return None
# aiter calls _moe_sorting_impl from inside fused_moe without passing hidden_states through
_pending_quant_input: ContextVar[torch.Tensor | None] = ContextVar(
"aiter_pending_quant_input", default=None
)
# the sort-time quant, handed on to the patched fused_dynamic_mxfp8_quant_moe_sort
_emitted_quant: ContextVar[tuple[torch.Tensor, torch.Tensor] | None] = ContextVar(
"aiter_emitted_quant", default=None
)
@functools.cache
def apply_aiter_small_moe_sort_patch() -> None:
"""Patch a stock aiter so decode-sized MoE sorting (+ stage1 mxfp8 quant) is one launch."""
if not is_gfx95_supported():
return
try:
import aiter.fused_moe as fm
from aiter import dtypes
orig_fused_moe = fm.fused_moe
orig_sorting_impl = fm._moe_sorting_impl
orig_mx_quant = fm.fused_dynamic_mxfp8_quant_moe_sort
except (ImportError, AttributeError) as exc:
logger.info("aiter small-batch MoE sorting patch not applied: %s", exc)
return
@functools.wraps(orig_fused_moe)
def fused_moe_wrapper(
hidden_states, w1, w2, topk_weight, topk_ids, *args, **kwargs
):
quant_type = kwargs.get("quant_type", fm.QuantType.No)
emit = (
quant_type == fm.QuantType.per_1x32
and w1.dtype in (dtypes.fp4x2, dtypes.fp8)
and hidden_states.dtype in (torch.bfloat16, torch.float16)
and hidden_states.is_contiguous()
and hidden_states.shape[-1] % 2048 == 0
and topk_ids.numel() <= 256
)
input_token = _pending_quant_input.set(hidden_states if emit else None)
emitted_token = _emitted_quant.set(None)
try:
return orig_fused_moe(
hidden_states, w1, w2, topk_weight, topk_ids, *args, **kwargs
)
finally:
_emitted_quant.reset(emitted_token)
_pending_quant_input.reset(input_token)
@functools.wraps(orig_sorting_impl)
def sorting_impl_wrapper(
topk_ids,
topk_weights,
num_experts,
model_dim,
moebuf_dtype,
block_size,
expert_mask,
num_local_tokens,
dispatch_policy,
use_opus,
return_local_topk_ids=False,
accumulate=True,
output_aux=False,
**orig_kwargs,
):
# newer aiter passes a caller-owned moe_buf as output=; leave that to aiter
if (
not output_aux
and not return_local_topk_ids
and orig_kwargs.get("output") is None
and _small_sort_supported(
topk_ids, int(block_size), expert_mask, num_local_tokens
)
):
device = topk_ids.device
M, topk = topk_ids.shape
max_num_tokens_padded = int(
topk_ids.numel() + num_experts * block_size - topk
)
max_num_m_blocks = int(
(max_num_tokens_padded + block_size - 1) // block_size
)
sorted_ids = torch.empty(
max_num_tokens_padded, dtype=dtypes.i32, device=device
)
sorted_weights = torch.empty(
max_num_tokens_padded, dtype=dtypes.fp32, device=device
)
sorted_expert_ids = torch.empty(
max_num_m_blocks, dtype=dtypes.i32, device=device
)
num_valid_ids = torch.empty(2, dtype=dtypes.i32, device=device)
if accumulate:
moe_buf = torch.empty((M, model_dim), dtype=moebuf_dtype, device=device)
else:
moe_buf = torch.empty((0, 0), dtype=moebuf_dtype, device=device)
quant_ret = _run_small_sort(
topk_ids,
topk_weights,
sorted_ids,
sorted_weights,
sorted_expert_ids,
num_valid_ids,
moe_buf,
int(block_size),
_pending_quant_input.get(),
int(num_experts),
)
if quant_ret is not None:
_emitted_quant.set(quant_ret)
return (
sorted_ids,
sorted_weights,
sorted_expert_ids,
num_valid_ids,
moe_buf,
)
return orig_sorting_impl(
topk_ids,
topk_weights,
num_experts,
model_dim,
moebuf_dtype,
block_size,
expert_mask,
num_local_tokens,
dispatch_policy,
use_opus,
return_local_topk_ids=return_local_topk_ids,
accumulate=accumulate,
output_aux=output_aux,
**orig_kwargs,
)
@functools.wraps(orig_mx_quant)
def mx_quant_wrapper(input, sorted_ids, *args, **kwargs):
pre = _emitted_quant.get()
if pre is not None and pre[0].shape == input.shape:
_emitted_quant.set(None)
return pre
return orig_mx_quant(input, sorted_ids, *args, **kwargs)
fm.fused_moe = fused_moe_wrapper
fm._moe_sorting_impl = sorting_impl_wrapper
fm.fused_dynamic_mxfp8_quant_moe_sort = mx_quant_wrapper
logger.info("aiter small-batch MoE sorting patch applied")
@@ -258,6 +258,7 @@ def fused_moe_mxfp8_native(
limit: Optional[float],
no_combine: bool = False,
expert_map: Optional[torch.Tensor] = None,
sanitize_topk_ids: bool = True,
) -> torch.Tensor:
# Lazy import: the jit_kernel package pulls in Triton at first use; importing
# at call time avoids any import-time cycle with the moe runner package.
@@ -279,9 +280,12 @@ def fused_moe_mxfp8_native(
topk_ids.masked_fill_(
~valid_global | (topk_ids < 0) | (topk_ids >= local_num_experts), -1
)
else:
elif sanitize_topk_ids:
topk_ids = topk_ids.to(torch.int32, copy=True)
topk_ids.masked_fill_((topk_ids < 0) | (topk_ids >= local_num_experts), -1)
else:
# May alias the caller's tensor; everything below reads topk_ids only.
topk_ids = topk_ids.to(torch.int32)
block_m = 64
sorted_ids, expert_ids, num_post = moe_align_block_size(
@@ -380,6 +384,7 @@ def fused_experts_mxfp8(
swiglu_limit: Optional[float] = None,
gate_up_interleaved: bool = True,
expert_map: Optional[torch.Tensor] = None,
sanitize_topk_ids: bool = True,
) -> torch.Tensor:
"""Native MXFP8 MoE entry (CDNA4 ``dot_scaled``).
@@ -426,6 +431,7 @@ def fused_experts_mxfp8(
limit=limit,
no_combine=no_combine,
expert_map=expert_map,
sanitize_topk_ids=sanitize_topk_ids,
)
if no_combine:
@@ -23,7 +23,7 @@ from sglang.srt.layers.moe.moe_runner.base import (
)
from sglang.srt.layers.moe.utils import MoeRunnerBackend
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import get_bool_env_var, get_int_env_var
from sglang.srt.utils import get_bool_env_var, get_int_env_var, is_gfx95_supported
if TYPE_CHECKING:
from sglang.srt.layers.moe.token_dispatcher.base import CombineInput
@@ -106,10 +106,32 @@ _AITER_ACTIVATIONS = {
}
def _aiter_activation(activation: str):
# aiter's ActivationType.Swiglu is SwiGLU-OAI with alpha and the (up + beta) bias baked in
_AITER_SWIGLU_OAI_ALPHA = 1.702
_AITER_SWIGLU_OAI_BETA = 1.0
def aiter_swiglu_oai_limit(config: MoeRunnerConfig) -> Optional[float]:
"""Return the clamp limit for matching SwiGLU-OAI configs on gfx95."""
if not is_gfx95_supported():
return None
if config.activation != "silu" or not config.is_gated:
return None
if config.gemm1_alpha != _AITER_SWIGLU_OAI_ALPHA:
return None
if config.gemm1_beta not in (None, _AITER_SWIGLU_OAI_BETA):
return None
if config.gemm1_clamp_limit is None:
return None
return float(config.gemm1_clamp_limit)
def _aiter_activation(config: MoeRunnerConfig):
from aiter import ActivationType
return getattr(ActivationType, _AITER_ACTIVATIONS.get(activation, "Gelu"))
if aiter_swiglu_oai_limit(config) is not None:
return ActivationType.Swiglu
return getattr(ActivationType, _AITER_ACTIVATIONS.get(config.activation, "Gelu"))
def _aiter_quant_type(quant_type: AiterQuantType):
@@ -223,6 +245,14 @@ def _mori_decode_recv_bound(recv_rows: int, topk: int) -> int:
class AiterRunnerCore(MoeRunnerCore):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
from sglang.kernels.ops.moe.moe_sorting_small import (
apply_aiter_small_moe_sort_patch,
)
apply_aiter_small_moe_sort_patch()
def run(
self,
runner_input: AiterRunnerInput,
@@ -258,6 +288,7 @@ class AiterRunnerCore(MoeRunnerCore):
else quant_info.a13_scale
)
is_gfx95 = is_gfx95_supported()
extra: dict = {}
if quant_info.fused_moe_kwargs:
extra.update(quant_info.fused_moe_kwargs)
@@ -273,6 +304,8 @@ class AiterRunnerCore(MoeRunnerCore):
extra["beta"] = float(self.config.gemm1_alpha)
if self.config.gemm1_clamp_limit is not None:
extra["linear_beta"] = float(self.config.gemm1_clamp_limit)
elif is_gfx95 and quant_info.swiglu_limit > 0 and "gate_mode" in extra:
extra["swiglu_limit"] = quant_info.swiglu_limit
elif quant_info.swiglu_limit > 0:
# GateMode is only needed for the gpt-oss MXFP4 swiglu_limit path.
# Import lazily so models that don't use it (e.g. DeepSeek-V3 fp8,
@@ -280,16 +313,14 @@ class AiterRunnerCore(MoeRunnerCore):
# lives elsewhere / is absent.
from aiter.ops.flydsl.moe_common import GateMode
# Default (INTERLEAVE) preserves the pre-fix behavior for paths
# that prepare weights in the gate/up-interleaved layout. Set
# `SGLANG_USE_AITER_MOE_GU_ITLV=0` to switch to SEPARATED, which
# matches the layout produced by `Mxfp4MoEMethod` (gpt-oss
# MXFP4) and the gptoss_fp4 tuned FlyDSL kernels.
# a gate_mode from fused_moe_kwargs wins; else gfx95 honors the weight layout,
# and SGLANG_USE_AITER_MOE_GU_ITLV=0 selects SEPARATED (gpt-oss MXFP4 layout)
extra.setdefault(
"gate_mode",
(
GateMode.INTERLEAVE.value
if envs.SGLANG_USE_AITER_MOE_GU_ITLV.get()
if (not is_gfx95 or self.config.gate_up_interleaved)
and envs.SGLANG_USE_AITER_MOE_GU_ITLV.get()
else GateMode.SEPARATED.value
),
)
@@ -330,6 +361,11 @@ class AiterRunnerCore(MoeRunnerCore):
return AiterRunnerOutput(hidden_states=out)
except _smallm.SmallMMoeUnavailable:
pass
activation = extra.pop("activation", None) if is_gfx95 else None
if activation is None:
activation = _aiter_activation(self.config)
output = fused_moe(
hidden_states=runner_input.hidden_states,
w1=quant_info.w13_weight,
@@ -337,7 +373,7 @@ class AiterRunnerCore(MoeRunnerCore):
topk_weight=runner_input.topk_weights,
topk_ids=runner_input.topk_ids,
quant_type=_aiter_quant_type(runner_input.quant_type),
activation=_aiter_activation(self.config.activation),
activation=activation,
w1_scale=quant_info.w13_scale,
w2_scale=quant_info.w2_scale,
a1_scale=a1_scale,
@@ -72,6 +72,11 @@ class TritonMoeQuantInfo(MoeQuantInfo):
fuse_swiglu_interleaved: bool = False
def _topk_ids_may_be_nonlocal(config: MoeRunnerConfig) -> bool:
# only expert parallelism can route a token to an expert this rank does not hold
return config.num_experts is None or config.num_experts != config.num_local_experts
class TritonRunnerCore(MoeRunnerCore):
def __init__(self, config: MoeRunnerConfig):
super().__init__(config)
@@ -108,6 +113,7 @@ class TritonRunnerCore(MoeRunnerCore):
gemm1_limit=self.config.gemm1_clamp_limit,
swiglu_limit=self.config.swiglu_limit,
gate_up_interleaved=self.config.gate_up_interleaved,
sanitize_topk_ids=_topk_ids_may_be_nonlocal(self.config),
)
return TritonRunnerOutput(hidden_states=out)
@@ -208,6 +214,7 @@ def fused_experts_none_to_triton(
gemm1_limit=runner_config.gemm1_clamp_limit,
swiglu_limit=runner_config.swiglu_limit,
gate_up_interleaved=runner_config.gate_up_interleaved,
sanitize_topk_ids=_topk_ids_may_be_nonlocal(runner_config),
)
else:
if quant_info.use_mxfp8 and is_cuda():
@@ -42,6 +42,7 @@ from sglang.srt.utils import (
get_bool_env_var,
is_cpu,
is_cuda,
is_gfx95_supported,
is_hip,
is_npu,
is_xpu,
@@ -947,10 +948,32 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp):
param.data = param.data.reshape(expected_shape)
def _aiter_ck_moe_supported(self, layer) -> bool:
def _aiter_ck_moe_unsupported_reason(self, layer) -> Optional[str]:
# aiter CK fused-MoE requires intermediate_size_per_partition to be 128-aligned
# (GemmSpec=Default; otherwise CK raises "not support this GEMM problem").
return layer.intermediate_size_per_partition % 128 == 0
if layer.intermediate_size_per_partition % 128 != 0:
return (
"intermediate_size_per_partition="
f"{layer.intermediate_size_per_partition} is not 128-aligned"
)
if not is_gfx95_supported():
return None
cfg = layer.moe_runner_config
wants_swiglu_oai = (
cfg.gemm1_alpha is not None or cfg.gemm1_clamp_limit is not None
)
if wants_swiglu_oai:
return (
f"activation={cfg.activation} with gemm1_alpha={cfg.gemm1_alpha} / "
f"gemm1_clamp_limit={cfg.gemm1_clamp_limit} (SwiGLU-OAI) is not "
"implemented by the aiter CK bf16 fused-MoE kernels"
)
return None
def _aiter_ck_moe_supported(self, layer) -> bool:
return self._aiter_ck_moe_unsupported_reason(layer) is None
def create_moe_runner(
self, layer: torch.nn.Module, moe_runner_config: MoeRunnerConfig
@@ -979,7 +1002,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp):
backend = MoeRunnerBackend.TRITON
self.runner = MoeRunner(backend, moe_runner_config)
# aiter CK fused-MoE only supports 128-aligned shapes; otherwise use triton.
# aiter CK fused-MoE only supports some shapes / activations; else use triton.
self._aiter_runner: Optional[MoeRunner] = None
if (
_use_aiter
@@ -989,20 +1012,19 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp):
)
and get_moe_a2a_backend().supports_aiter()
):
if self._aiter_ck_moe_supported(layer):
reason = self._aiter_ck_moe_unsupported_reason(layer)
if reason is None:
self._aiter_runner = MoeRunner(
MoeRunnerBackend.AITER, moe_runner_config
)
elif get_moe_runner_backend().is_aiter():
raise ValueError(
"moe_runner_backend=aiter is not supported for "
f"intermediate_size_per_partition={layer.intermediate_size_per_partition}; "
f"moe_runner_backend=aiter is not supported: {reason}; "
"use --moe-runner-backend triton."
)
else:
logger.warning_once(
"aiter CK fused-MoE does not support "
f"intermediate_size_per_partition={layer.intermediate_size_per_partition}; "
f"aiter CK fused-MoE does not support this layer: {reason}; "
"using triton MoE runner."
)
@@ -1106,12 +1128,14 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp):
if self._aiter_runner is not None:
from sglang.srt.layers.moe.moe_runner.aiter import (
AiterMoeQuantInfo,
aiter_swiglu_oai_limit,
)
quant_info = AiterMoeQuantInfo(
w13_weight=layer.w13_weight,
w2_weight=layer.w2_weight,
expert_mask=layer.dispatcher.expert_mask_gpu,
swiglu_limit=aiter_swiglu_oai_limit(self.moe_runner_config) or 0.0,
)
return self._aiter_runner.run(dispatch_output, quant_info)