diff --git a/python/sglang/kernels/ops/moe/moe_sorting_small.py b/python/sglang/kernels/ops/moe/moe_sorting_small.py new file mode 100644 index 0000000000..0fed1c6231 --- /dev/null +++ b/python/sglang/kernels/ops/moe/moe_sorting_small.py @@ -0,0 +1,557 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Single-launch MoE sorting (+ stage1 mxfp8 activation quant) for decode-sized +batches on the aiter MoE runner, installed as runtime patches on ``aiter.fused_moe``. + +Output layouts match the aiter kernels bit-for-bit: + sorted_ids[i] = (topk_slot << 24) | token (padding: (topk << 24) | M) + sorted_weights = pair weight (padding: 0) + sorted_expert_ids[b] = expert of block b + num_valid_ids = [num_blocks * block_size, M] + quant a1 = per-token fp8 rows; e8m0 RoundUp scale byte per + (sorted_row, group) at aiter's mx_scale_shuffle_idx address +""" + +from __future__ import annotations + +import functools +import logging +from contextvars import ContextVar + +import torch +import triton +import triton.language as tl + +from sglang.srt.utils import is_gfx95_supported + +logger = logging.getLogger(__name__) + + +# P <= 64: one sort CTA does the whole P x P rank compare +# per-M ints stay unspecialized so prefill sizes reuse the graph-capture compiles +@triton.jit(do_not_specialize=["M", "moe_buf_numel", "num_buf"]) +def _moe_sorting_small_kernel( + topk_ids_ptr, # [M, topk] i32 + topk_weights_ptr, # [M, topk] fp32 + sorted_ids_ptr, # [max_padded] i32 + sorted_weights_ptr, # [max_padded] fp32 + sorted_expert_ids_ptr, # [max_blocks] i32 + num_valid_ids_ptr, # [2] i32 + moe_buf_ptr, + moe_buf_numel, + qx_ptr, # [M, N_COLS] activations to mx-quantize (EMIT_MX only) + qout_ptr, # [M, N_COLS] fp8 out + qscale_ptr, # swizzled e8m0 bytes, one per (sorted_row, group) + M, + TOPK: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + P_POW2: tl.constexpr, # >= M * topk + PAD_POW2: tl.constexpr, # >= (M * topk) * BLOCK_SIZE (worst-case padded len) + BUF_BLOCK: tl.constexpr, + num_buf, # buf-zero CTAs occupy pids [1, num_buf] + EMIT_MX: tl.constexpr, # also emit the mxfp8 quant of qx (group_size 32) + N_COLS: tl.constexpr, + QCHUNK: tl.constexpr, # columns per quant iteration (multiple of 32) + SCALEN_PAD: tl.constexpr, # ceil(N_COLS/32 / 8) * 8 +): + pid = tl.program_id(0) + if pid > 0 and pid <= num_buf: + offs = (pid - 1) * BUF_BLOCK + tl.arange(0, BUF_BLOCK) + tl.store( + moe_buf_ptr + offs, + tl.zeros((BUF_BLOCK,), moe_buf_ptr.dtype.element_ty), + mask=offs < moe_buf_numel, + ) + return + + P = M * TOPK + offs_p = tl.arange(0, P_POW2) + # the quant CTAs recompute the P x P sort math instead of waiting on pid 0 + mask_p = offs_p < P + # sentinel expert keeps inactive lanes out of every "smaller expert" count + e = tl.load(topk_ids_ptr + offs_p, mask=mask_p, other=0x7FFFFFFF) + w = tl.load(topk_weights_ptr + offs_p, mask=mask_p, other=0.0) + token = offs_p // TOPK + slot = offs_p % TOPK + + # Stable rank within the pair's expert, and per-expert pair count. + same = e[:, None] == e[None, :] + rank = tl.sum(tl.where(same & (offs_p[None, :] < offs_p[:, None]), 1, 0), axis=1) + cnt = tl.sum(tl.where(same & mask_p[None, :], 1, 0), axis=1) + blocks_of_e = (cnt + BLOCK_SIZE - 1) // BLOCK_SIZE + is_leader = (rank == 0) & mask_p + + # leaders carry their expert's block count + smaller = e[None, :] < e[:, None] + blocks_before = tl.sum( + tl.where(smaller & is_leader[None, :], blocks_of_e[None, :], 0), axis=1 + ) + dest = blocks_before * BLOCK_SIZE + rank + + if EMIT_MX and pid > num_buf: + # quant CTA: one (pair, column-chunk) slice, mirroring fused_dynamic_mxfp8_quant_moe_sort + q_id = pid - num_buf - 1 + CHUNKS: tl.constexpr = N_COLS // QCHUNK + p = q_id // CHUNKS + c0 = (q_id % CHUNKS) * QCHUNK + + if p < P: + offs_q = tl.arange(0, QCHUNK) + offs_g = tl.arange(0, QCHUNK // 32) + token_p = tl.sum(tl.where(offs_p == p, token, 0), axis=0) + dest_p = tl.sum(tl.where(offs_p == p, dest, 0), axis=0) + base_sw = ( + (dest_p // 32) * (SCALEN_PAD * 32) + + (dest_p % 16) * 4 + + (dest_p % 32) // 16 + ) + x = tl.load(qx_ptr + token_p * N_COLS + c0 + offs_q).to(tl.float32) + x2 = tl.reshape(x, (QCHUNK // 32, 32)) + amax = tl.maximum(tl.max(tl.abs(x2), axis=1), 1e-10) + sf = amax * (1.0 / 448.0) + bits = sf.to(tl.int32, bitcast=True) + exp = (bits >> 23) & 0xFF + exp = tl.where((bits & 0x7FFFFF) != 0, exp + 1, exp) + scale = (exp << 23).to(tl.float32, bitcast=True) + if p % TOPK == 0: + # one fp8 out row per token (pairs are token-major) + q = tl.clamp(x2 / scale[:, None], -448.0, 448.0) + tl.store( + qout_ptr + token_p * N_COLS + c0 + offs_q, + tl.reshape(q, (QCHUNK,)).to(qout_ptr.dtype.element_ty), + ) + y = c0 // 32 + offs_g + sw = base_sw + (y // 8) * 256 + (y % 4) * 64 + ((y % 8) // 4) * 2 + tl.store(qscale_ptr + sw, exp.to(tl.uint8)) + return + + total_blocks = tl.sum(tl.where(is_leader, blocks_of_e, 0), axis=0) + num_valid = total_blocks * BLOCK_SIZE + + # pid 0: pad the whole used region first, then scatter the real pairs over it + offs_pad = tl.arange(0, PAD_POW2) + pad_mask = offs_pad < num_valid + pad_val = (TOPK << 24) | M + tl.store( + sorted_ids_ptr + offs_pad, + tl.full((PAD_POW2,), 0, tl.int32) + pad_val, + mask=pad_mask, + ) + tl.store( + sorted_weights_ptr + offs_pad, tl.zeros((PAD_POW2,), tl.float32), mask=pad_mask + ) + tl.debug_barrier() + tl.store(sorted_ids_ptr + dest, (slot << 24) | token, mask=mask_p) + tl.store(sorted_weights_ptr + dest, w, mask=mask_p) + + # leaders write their expert into each of their blocks: at most 2 with P <= 2 * BLOCK_SIZE + for j in tl.static_range(2): + bm = is_leader & (j < blocks_of_e) + tl.store(sorted_expert_ids_ptr + blocks_before + j, e, mask=bm) + + tl.store( + num_valid_ids_ptr + tl.arange(0, 2), + tl.where(tl.arange(0, 2) == 0, num_valid, M), + ) + + +@triton.jit +def _expert_chunk_blocks(e, offs_chunk, BLOCK_SIZE: tl.constexpr): + cnt = tl.sum((e[:, None] == offs_chunk[None, :]).to(tl.int32), axis=0) + return cnt, (cnt + BLOCK_SIZE - 1) // BLOCK_SIZE + + +# per-M ints stay unspecialized so prefill sizes reuse the graph-capture compiles +@triton.jit(do_not_specialize=["M", "moe_buf_numel", "num_buf"]) +def _moe_sorting_small_kernel_distributed( + topk_ids_ptr, # [M, topk] i32 + topk_weights_ptr, # [M, topk] fp32 + sorted_ids_ptr, # [max_padded] i32 + sorted_weights_ptr, # [max_padded] fp32 + sorted_expert_ids_ptr, # [max_blocks] i32 + num_valid_ids_ptr, # [2] i32 + moe_buf_ptr, + moe_buf_numel, + qx_ptr, # [M, N_COLS] activations to mx-quantize (EMIT_MX only) + qout_ptr, # [M, N_COLS] fp8 out + qscale_ptr, # swizzled e8m0 bytes, one per (sorted_row, group) + M, + num_experts, + TOPK: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + P_POW2: tl.constexpr, # >= M * topk + P_CHUNK: tl.constexpr, # tile width for pairwise/expert-space loops + E_CEIL: tl.constexpr, # num_experts rounded up to P_CHUNK + PAD_POW2: tl.constexpr, # >= (M * topk) * BLOCK_SIZE (worst-case padded len) + BUF_BLOCK: tl.constexpr, + num_buf, # buf-zero CTAs occupy pids [1, num_buf] + EMIT_MX: tl.constexpr, # also emit the mxfp8 quant of qx (group_size 32) + N_COLS: tl.constexpr, + QCHUNK: tl.constexpr, # columns per quant iteration (multiple of 32) + SCALEN_PAD: tl.constexpr, # ceil(N_COLS/32 / 8) * 8 +): + pid = tl.program_id(0) + P = M * TOPK + offs_p = tl.arange(0, P_POW2) + mask_p = offs_p < P + # sentinel expert keeps inactive lanes out of every "smaller expert" count + e = tl.load(topk_ids_ptr + offs_p, mask=mask_p, other=0x7FFFFFFF) + + # one writer per byte: expert-chunk CTAs, then num_buf zero-fill CTAs, then one CTA per pair + NUM_ECHUNK: tl.constexpr = E_CEIL // P_CHUNK + + if pid < NUM_ECHUNK: + # expert-chunk CTA: expert-id table, block-tail padding and (last chunk) num_valid + x0 = pid * P_CHUNK + offs_x = x0 + tl.arange(0, P_CHUNK) + cnt_x, blocks_x = _expert_chunk_blocks(e, offs_x, BLOCK_SIZE) + # a runtime `if` inside static_range mis-lowers, so accumulate under a uniform mask + blocks_before_chunk = tl.zeros((), tl.int32) + for y0 in tl.static_range(0, E_CEIL, P_CHUNK): + offs_y = y0 + tl.arange(0, P_CHUNK) + cnt_y, blocks_y = _expert_chunk_blocks(e, offs_y, BLOCK_SIZE) + blocks_before_chunk += tl.sum(tl.where(offs_y < x0, blocks_y, 0), axis=0) + # exclusive prefix within the chunk + prefix_x = tl.sum( + tl.where(offs_x[None, :] < offs_x[:, None], blocks_x[None, :], 0), axis=1 + ) + bb_x = blocks_before_chunk + prefix_x + + MAX_BLOCKS_PER_EXPERT: tl.constexpr = (P_POW2 + BLOCK_SIZE - 1) // BLOCK_SIZE + for j in tl.static_range(MAX_BLOCKS_PER_EXPERT): + bm = (offs_x < num_experts) & (j < blocks_x) + tl.store(sorted_expert_ids_ptr + bb_x + j, offs_x, mask=bm) + + pad_val = (TOPK << 24) | M + for j in tl.static_range(MAX_BLOCKS_PER_EXPERT * BLOCK_SIZE): + pm = (j >= cnt_x) & (j < blocks_x * BLOCK_SIZE) + tl.store(sorted_ids_ptr + bb_x * BLOCK_SIZE + j, pad_val, mask=pm) + tl.store(sorted_weights_ptr + bb_x * BLOCK_SIZE + j, 0.0, mask=pm) + + if pid == NUM_ECHUNK - 1: + num_valid = (blocks_before_chunk + tl.sum(blocks_x, axis=0)) * BLOCK_SIZE + tl.store( + num_valid_ids_ptr + tl.arange(0, 2), + tl.where(tl.arange(0, 2) == 0, num_valid, M), + ) + return + + if pid < NUM_ECHUNK + num_buf: + offs = (pid - NUM_ECHUNK) * BUF_BLOCK + tl.arange(0, BUF_BLOCK) + tl.store( + moe_buf_ptr + offs, + tl.zeros((BUF_BLOCK,), moe_buf_ptr.dtype.element_ty), + mask=offs < moe_buf_numel, + ) + return + + # per-pair CTA: dest = blocks of smaller experts * BLOCK_SIZE + stable rank in its expert + p = pid - NUM_ECHUNK - num_buf + if p >= P: + return + + e_p = tl.sum(tl.where(offs_p == p, e, 0), axis=0) + rank_p = tl.sum(tl.where((offs_p < p) & (e == e_p), 1, 0), axis=0) + bb_p = tl.zeros((), tl.int32) + for x0 in tl.static_range(0, E_CEIL, P_CHUNK): + offs_x = x0 + tl.arange(0, P_CHUNK) + cnt_x, blocks_x = _expert_chunk_blocks(e, offs_x, BLOCK_SIZE) + bb_p += tl.sum(tl.where(offs_x < e_p, blocks_x, 0), axis=0) + dest_p = bb_p * BLOCK_SIZE + rank_p + + token_p2 = p // TOPK + slot_p = p % TOPK + w_p = tl.load(topk_weights_ptr + p) + tl.store(sorted_ids_ptr + dest_p, (slot_p << 24) | token_p2) + tl.store(sorted_weights_ptr + dest_p, w_p) + + if EMIT_MX: + # mirrors fused_dynamic_mxfp8_quant_moe_sort: e8m0 RoundUp scale, swizzled address + token_p = p // TOPK + offs_q = tl.arange(0, QCHUNK) + offs_g = tl.arange(0, QCHUNK // 32) + base_sw = ( + (dest_p // 32) * (SCALEN_PAD * 32) + (dest_p % 16) * 4 + (dest_p % 32) // 16 + ) + for cc in tl.static_range(N_COLS // QCHUNK): + c0 = cc * QCHUNK + x = tl.load(qx_ptr + token_p * N_COLS + c0 + offs_q).to(tl.float32) + x2 = tl.reshape(x, (QCHUNK // 32, 32)) + amax = tl.maximum(tl.max(tl.abs(x2), axis=1), 1e-10) + sf = amax * (1.0 / 448.0) + bits = sf.to(tl.int32, bitcast=True) + exp = (bits >> 23) & 0xFF + exp = tl.where((bits & 0x7FFFFF) != 0, exp + 1, exp) + scale = (exp << 23).to(tl.float32, bitcast=True) + if p % TOPK == 0: + # one fp8 out row per token (pairs are token-major) + q = tl.clamp(x2 / scale[:, None], -448.0, 448.0) + tl.store( + qout_ptr + token_p * N_COLS + c0 + offs_q, + tl.reshape(q, (QCHUNK,)).to(qout_ptr.dtype.element_ty), + ) + y = c0 // 32 + offs_g + sw = base_sw + (y // 8) * 256 + (y % 4) * 64 + ((y % 8) // 4) * 2 + tl.store(qscale_ptr + sw, exp.to(tl.uint8)) + + +def _small_sort_supported(topk_ids, block_size, expert_mask, num_local_tokens): + m, topk = topk_ids.shape + return ( + expert_mask is None + and num_local_tokens is None + and m * topk <= 256 + and topk < 128 + and topk_ids.dtype == torch.int32 + and topk_ids.is_contiguous() + ) + + +def _run_small_sort( + topk_ids, + topk_weights, + sorted_ids, + sorted_weights, + sorted_expert_ids, + num_valid_ids, + moe_buf, + block_size, + mx_quant_input, + num_experts, +): + m, topk = topk_ids.shape + p = m * topk + buf_block = 4096 + emit_mx = mx_quant_input is not None + if emit_mx: + n_cols = mx_quant_input.shape[-1] + scalen_pad = ((n_cols // 32 + 7) // 8) * 8 + max_padded = sorted_ids.shape[0] + qout = torch.empty(m, n_cols, dtype=torch.float8_e4m3fn, device=topk_ids.device) + qscale = torch.empty( + ((max_padded + 31) // 32) * 32, + scalen_pad, + dtype=torch.uint8, + device=topk_ids.device, + ) + else: + n_cols, scalen_pad = 32, 8 + qout = qscale = moe_buf # unused placeholder pointers + num_buf = triton.cdiv(max(moe_buf.numel(), 1), buf_block) + if p <= 64 and p <= 2 * block_size: + # compact variant: one sort CTA does the P x P rank compare + num_quant = (p * (n_cols // min(2048, n_cols))) if emit_mx else 0 + grid = (1 + num_buf + num_quant,) + _moe_sorting_small_kernel[grid]( + topk_ids, + topk_weights, + sorted_ids, + sorted_weights, + sorted_expert_ids, + num_valid_ids, + moe_buf, + moe_buf.numel(), + mx_quant_input if emit_mx else moe_buf, + qout, + qscale, + m, + TOPK=topk, + BLOCK_SIZE=block_size, + P_POW2=triton.next_power_of_2(p), + PAD_POW2=triton.next_power_of_2(p * block_size), + BUF_BLOCK=buf_block, + num_buf=num_buf, + EMIT_MX=emit_mx, + N_COLS=n_cols, + QCHUNK=min(2048, n_cols), + SCALEN_PAD=scalen_pad, + num_warps=4, + ) + if emit_mx: + return qout, qscale.view(torch.float8_e8m0fnu) + return None + # distributed variant: 64-wide expert/pair tiles, one writer per byte + p_chunk = 64 + num_echunk = (num_experts + p_chunk - 1) // p_chunk + num_pair = p + grid = (num_echunk + num_buf + num_pair,) + _moe_sorting_small_kernel_distributed[grid]( + topk_ids, + topk_weights, + sorted_ids, + sorted_weights, + sorted_expert_ids, + num_valid_ids, + moe_buf, + moe_buf.numel(), + mx_quant_input if emit_mx else moe_buf, + qout, + qscale, + m, + num_experts, + TOPK=topk, + BLOCK_SIZE=block_size, + P_POW2=triton.next_power_of_2(p), + P_CHUNK=p_chunk, + E_CEIL=(num_experts + p_chunk - 1) // p_chunk * p_chunk, + PAD_POW2=triton.next_power_of_2(p * block_size), + BUF_BLOCK=buf_block, + num_buf=num_buf, + EMIT_MX=emit_mx, + N_COLS=n_cols, + QCHUNK=min(2048, n_cols), + SCALEN_PAD=scalen_pad, + num_warps=8, + ) + if emit_mx: + return qout, qscale.view(torch.float8_e8m0fnu) + return None + + +# aiter calls _moe_sorting_impl from inside fused_moe without passing hidden_states through +_pending_quant_input: ContextVar[torch.Tensor | None] = ContextVar( + "aiter_pending_quant_input", default=None +) +# the sort-time quant, handed on to the patched fused_dynamic_mxfp8_quant_moe_sort +_emitted_quant: ContextVar[tuple[torch.Tensor, torch.Tensor] | None] = ContextVar( + "aiter_emitted_quant", default=None +) + + +@functools.cache +def apply_aiter_small_moe_sort_patch() -> None: + """Patch a stock aiter so decode-sized MoE sorting (+ stage1 mxfp8 quant) is one launch.""" + if not is_gfx95_supported(): + return + + try: + import aiter.fused_moe as fm + from aiter import dtypes + + orig_fused_moe = fm.fused_moe + orig_sorting_impl = fm._moe_sorting_impl + orig_mx_quant = fm.fused_dynamic_mxfp8_quant_moe_sort + except (ImportError, AttributeError) as exc: + logger.info("aiter small-batch MoE sorting patch not applied: %s", exc) + return + + @functools.wraps(orig_fused_moe) + def fused_moe_wrapper( + hidden_states, w1, w2, topk_weight, topk_ids, *args, **kwargs + ): + quant_type = kwargs.get("quant_type", fm.QuantType.No) + emit = ( + quant_type == fm.QuantType.per_1x32 + and w1.dtype in (dtypes.fp4x2, dtypes.fp8) + and hidden_states.dtype in (torch.bfloat16, torch.float16) + and hidden_states.is_contiguous() + and hidden_states.shape[-1] % 2048 == 0 + and topk_ids.numel() <= 256 + ) + input_token = _pending_quant_input.set(hidden_states if emit else None) + emitted_token = _emitted_quant.set(None) + try: + return orig_fused_moe( + hidden_states, w1, w2, topk_weight, topk_ids, *args, **kwargs + ) + finally: + _emitted_quant.reset(emitted_token) + _pending_quant_input.reset(input_token) + + @functools.wraps(orig_sorting_impl) + def sorting_impl_wrapper( + topk_ids, + topk_weights, + num_experts, + model_dim, + moebuf_dtype, + block_size, + expert_mask, + num_local_tokens, + dispatch_policy, + use_opus, + return_local_topk_ids=False, + accumulate=True, + output_aux=False, + **orig_kwargs, + ): + # newer aiter passes a caller-owned moe_buf as output=; leave that to aiter + if ( + not output_aux + and not return_local_topk_ids + and orig_kwargs.get("output") is None + and _small_sort_supported( + topk_ids, int(block_size), expert_mask, num_local_tokens + ) + ): + device = topk_ids.device + M, topk = topk_ids.shape + max_num_tokens_padded = int( + topk_ids.numel() + num_experts * block_size - topk + ) + max_num_m_blocks = int( + (max_num_tokens_padded + block_size - 1) // block_size + ) + sorted_ids = torch.empty( + max_num_tokens_padded, dtype=dtypes.i32, device=device + ) + sorted_weights = torch.empty( + max_num_tokens_padded, dtype=dtypes.fp32, device=device + ) + sorted_expert_ids = torch.empty( + max_num_m_blocks, dtype=dtypes.i32, device=device + ) + num_valid_ids = torch.empty(2, dtype=dtypes.i32, device=device) + if accumulate: + moe_buf = torch.empty((M, model_dim), dtype=moebuf_dtype, device=device) + else: + moe_buf = torch.empty((0, 0), dtype=moebuf_dtype, device=device) + quant_ret = _run_small_sort( + topk_ids, + topk_weights, + sorted_ids, + sorted_weights, + sorted_expert_ids, + num_valid_ids, + moe_buf, + int(block_size), + _pending_quant_input.get(), + int(num_experts), + ) + if quant_ret is not None: + _emitted_quant.set(quant_ret) + return ( + sorted_ids, + sorted_weights, + sorted_expert_ids, + num_valid_ids, + moe_buf, + ) + return orig_sorting_impl( + topk_ids, + topk_weights, + num_experts, + model_dim, + moebuf_dtype, + block_size, + expert_mask, + num_local_tokens, + dispatch_policy, + use_opus, + return_local_topk_ids=return_local_topk_ids, + accumulate=accumulate, + output_aux=output_aux, + **orig_kwargs, + ) + + @functools.wraps(orig_mx_quant) + def mx_quant_wrapper(input, sorted_ids, *args, **kwargs): + pre = _emitted_quant.get() + if pre is not None and pre[0].shape == input.shape: + _emitted_quant.set(None) + return pre + return orig_mx_quant(input, sorted_ids, *args, **kwargs) + + fm.fused_moe = fused_moe_wrapper + fm._moe_sorting_impl = sorting_impl_wrapper + fm.fused_dynamic_mxfp8_quant_moe_sort = mx_quant_wrapper + logger.info("aiter small-batch MoE sorting patch applied") diff --git a/python/sglang/kernels/ops/moe/mxfp8_moe_amd_gfx95.py b/python/sglang/kernels/ops/moe/mxfp8_moe_amd_gfx95.py index 53800bf3f5..6ff509d3e9 100644 --- a/python/sglang/kernels/ops/moe/mxfp8_moe_amd_gfx95.py +++ b/python/sglang/kernels/ops/moe/mxfp8_moe_amd_gfx95.py @@ -258,6 +258,7 @@ def fused_moe_mxfp8_native( limit: Optional[float], no_combine: bool = False, expert_map: Optional[torch.Tensor] = None, + sanitize_topk_ids: bool = True, ) -> torch.Tensor: # Lazy import: the jit_kernel package pulls in Triton at first use; importing # at call time avoids any import-time cycle with the moe runner package. @@ -279,9 +280,12 @@ def fused_moe_mxfp8_native( topk_ids.masked_fill_( ~valid_global | (topk_ids < 0) | (topk_ids >= local_num_experts), -1 ) - else: + elif sanitize_topk_ids: topk_ids = topk_ids.to(torch.int32, copy=True) topk_ids.masked_fill_((topk_ids < 0) | (topk_ids >= local_num_experts), -1) + else: + # May alias the caller's tensor; everything below reads topk_ids only. + topk_ids = topk_ids.to(torch.int32) block_m = 64 sorted_ids, expert_ids, num_post = moe_align_block_size( @@ -380,6 +384,7 @@ def fused_experts_mxfp8( swiglu_limit: Optional[float] = None, gate_up_interleaved: bool = True, expert_map: Optional[torch.Tensor] = None, + sanitize_topk_ids: bool = True, ) -> torch.Tensor: """Native MXFP8 MoE entry (CDNA4 ``dot_scaled``). @@ -426,6 +431,7 @@ def fused_experts_mxfp8( limit=limit, no_combine=no_combine, expert_map=expert_map, + sanitize_topk_ids=sanitize_topk_ids, ) if no_combine: diff --git a/python/sglang/srt/layers/moe/moe_runner/aiter.py b/python/sglang/srt/layers/moe/moe_runner/aiter.py index b4fe80a996..44ec32f574 100644 --- a/python/sglang/srt/layers/moe/moe_runner/aiter.py +++ b/python/sglang/srt/layers/moe/moe_runner/aiter.py @@ -23,7 +23,7 @@ from sglang.srt.layers.moe.moe_runner.base import ( ) from sglang.srt.layers.moe.utils import MoeRunnerBackend from sglang.srt.runtime_context import get_parallel -from sglang.srt.utils import get_bool_env_var, get_int_env_var +from sglang.srt.utils import get_bool_env_var, get_int_env_var, is_gfx95_supported if TYPE_CHECKING: from sglang.srt.layers.moe.token_dispatcher.base import CombineInput @@ -106,10 +106,32 @@ _AITER_ACTIVATIONS = { } -def _aiter_activation(activation: str): +# aiter's ActivationType.Swiglu is SwiGLU-OAI with alpha and the (up + beta) bias baked in +_AITER_SWIGLU_OAI_ALPHA = 1.702 +_AITER_SWIGLU_OAI_BETA = 1.0 + + +def aiter_swiglu_oai_limit(config: MoeRunnerConfig) -> Optional[float]: + """Return the clamp limit for matching SwiGLU-OAI configs on gfx95.""" + if not is_gfx95_supported(): + return None + if config.activation != "silu" or not config.is_gated: + return None + if config.gemm1_alpha != _AITER_SWIGLU_OAI_ALPHA: + return None + if config.gemm1_beta not in (None, _AITER_SWIGLU_OAI_BETA): + return None + if config.gemm1_clamp_limit is None: + return None + return float(config.gemm1_clamp_limit) + + +def _aiter_activation(config: MoeRunnerConfig): from aiter import ActivationType - return getattr(ActivationType, _AITER_ACTIVATIONS.get(activation, "Gelu")) + if aiter_swiglu_oai_limit(config) is not None: + return ActivationType.Swiglu + return getattr(ActivationType, _AITER_ACTIVATIONS.get(config.activation, "Gelu")) def _aiter_quant_type(quant_type: AiterQuantType): @@ -223,6 +245,14 @@ def _mori_decode_recv_bound(recv_rows: int, topk: int) -> int: class AiterRunnerCore(MoeRunnerCore): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + from sglang.kernels.ops.moe.moe_sorting_small import ( + apply_aiter_small_moe_sort_patch, + ) + + apply_aiter_small_moe_sort_patch() + def run( self, runner_input: AiterRunnerInput, @@ -258,6 +288,7 @@ class AiterRunnerCore(MoeRunnerCore): else quant_info.a13_scale ) + is_gfx95 = is_gfx95_supported() extra: dict = {} if quant_info.fused_moe_kwargs: extra.update(quant_info.fused_moe_kwargs) @@ -273,6 +304,8 @@ class AiterRunnerCore(MoeRunnerCore): extra["beta"] = float(self.config.gemm1_alpha) if self.config.gemm1_clamp_limit is not None: extra["linear_beta"] = float(self.config.gemm1_clamp_limit) + elif is_gfx95 and quant_info.swiglu_limit > 0 and "gate_mode" in extra: + extra["swiglu_limit"] = quant_info.swiglu_limit elif quant_info.swiglu_limit > 0: # GateMode is only needed for the gpt-oss MXFP4 swiglu_limit path. # Import lazily so models that don't use it (e.g. DeepSeek-V3 fp8, @@ -280,16 +313,14 @@ class AiterRunnerCore(MoeRunnerCore): # lives elsewhere / is absent. from aiter.ops.flydsl.moe_common import GateMode - # Default (INTERLEAVE) preserves the pre-fix behavior for paths - # that prepare weights in the gate/up-interleaved layout. Set - # `SGLANG_USE_AITER_MOE_GU_ITLV=0` to switch to SEPARATED, which - # matches the layout produced by `Mxfp4MoEMethod` (gpt-oss - # MXFP4) and the gptoss_fp4 tuned FlyDSL kernels. + # a gate_mode from fused_moe_kwargs wins; else gfx95 honors the weight layout, + # and SGLANG_USE_AITER_MOE_GU_ITLV=0 selects SEPARATED (gpt-oss MXFP4 layout) extra.setdefault( "gate_mode", ( GateMode.INTERLEAVE.value - if envs.SGLANG_USE_AITER_MOE_GU_ITLV.get() + if (not is_gfx95 or self.config.gate_up_interleaved) + and envs.SGLANG_USE_AITER_MOE_GU_ITLV.get() else GateMode.SEPARATED.value ), ) @@ -330,6 +361,11 @@ class AiterRunnerCore(MoeRunnerCore): return AiterRunnerOutput(hidden_states=out) except _smallm.SmallMMoeUnavailable: pass + + activation = extra.pop("activation", None) if is_gfx95 else None + if activation is None: + activation = _aiter_activation(self.config) + output = fused_moe( hidden_states=runner_input.hidden_states, w1=quant_info.w13_weight, @@ -337,7 +373,7 @@ class AiterRunnerCore(MoeRunnerCore): topk_weight=runner_input.topk_weights, topk_ids=runner_input.topk_ids, quant_type=_aiter_quant_type(runner_input.quant_type), - activation=_aiter_activation(self.config.activation), + activation=activation, w1_scale=quant_info.w13_scale, w2_scale=quant_info.w2_scale, a1_scale=a1_scale, diff --git a/python/sglang/srt/layers/moe/moe_runner/triton.py b/python/sglang/srt/layers/moe/moe_runner/triton.py index 961b581d23..5939aaf181 100644 --- a/python/sglang/srt/layers/moe/moe_runner/triton.py +++ b/python/sglang/srt/layers/moe/moe_runner/triton.py @@ -72,6 +72,11 @@ class TritonMoeQuantInfo(MoeQuantInfo): fuse_swiglu_interleaved: bool = False +def _topk_ids_may_be_nonlocal(config: MoeRunnerConfig) -> bool: + # only expert parallelism can route a token to an expert this rank does not hold + return config.num_experts is None or config.num_experts != config.num_local_experts + + class TritonRunnerCore(MoeRunnerCore): def __init__(self, config: MoeRunnerConfig): super().__init__(config) @@ -108,6 +113,7 @@ class TritonRunnerCore(MoeRunnerCore): gemm1_limit=self.config.gemm1_clamp_limit, swiglu_limit=self.config.swiglu_limit, gate_up_interleaved=self.config.gate_up_interleaved, + sanitize_topk_ids=_topk_ids_may_be_nonlocal(self.config), ) return TritonRunnerOutput(hidden_states=out) @@ -208,6 +214,7 @@ def fused_experts_none_to_triton( gemm1_limit=runner_config.gemm1_clamp_limit, swiglu_limit=runner_config.swiglu_limit, gate_up_interleaved=runner_config.gate_up_interleaved, + sanitize_topk_ids=_topk_ids_may_be_nonlocal(runner_config), ) else: if quant_info.use_mxfp8 and is_cuda(): diff --git a/python/sglang/srt/layers/quantization/unquant.py b/python/sglang/srt/layers/quantization/unquant.py index 0ca43541d6..8edd7e7240 100644 --- a/python/sglang/srt/layers/quantization/unquant.py +++ b/python/sglang/srt/layers/quantization/unquant.py @@ -42,6 +42,7 @@ from sglang.srt.utils import ( get_bool_env_var, is_cpu, is_cuda, + is_gfx95_supported, is_hip, is_npu, is_xpu, @@ -947,10 +948,32 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp): param.data = param.data.reshape(expected_shape) - def _aiter_ck_moe_supported(self, layer) -> bool: + def _aiter_ck_moe_unsupported_reason(self, layer) -> Optional[str]: # aiter CK fused-MoE requires intermediate_size_per_partition to be 128-aligned # (GemmSpec=Default; otherwise CK raises "not support this GEMM problem"). - return layer.intermediate_size_per_partition % 128 == 0 + if layer.intermediate_size_per_partition % 128 != 0: + return ( + "intermediate_size_per_partition=" + f"{layer.intermediate_size_per_partition} is not 128-aligned" + ) + + if not is_gfx95_supported(): + return None + + cfg = layer.moe_runner_config + wants_swiglu_oai = ( + cfg.gemm1_alpha is not None or cfg.gemm1_clamp_limit is not None + ) + if wants_swiglu_oai: + return ( + f"activation={cfg.activation} with gemm1_alpha={cfg.gemm1_alpha} / " + f"gemm1_clamp_limit={cfg.gemm1_clamp_limit} (SwiGLU-OAI) is not " + "implemented by the aiter CK bf16 fused-MoE kernels" + ) + return None + + def _aiter_ck_moe_supported(self, layer) -> bool: + return self._aiter_ck_moe_unsupported_reason(layer) is None def create_moe_runner( self, layer: torch.nn.Module, moe_runner_config: MoeRunnerConfig @@ -979,7 +1002,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp): backend = MoeRunnerBackend.TRITON self.runner = MoeRunner(backend, moe_runner_config) - # aiter CK fused-MoE only supports 128-aligned shapes; otherwise use triton. + # aiter CK fused-MoE only supports some shapes / activations; else use triton. self._aiter_runner: Optional[MoeRunner] = None if ( _use_aiter @@ -989,20 +1012,19 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp): ) and get_moe_a2a_backend().supports_aiter() ): - if self._aiter_ck_moe_supported(layer): + reason = self._aiter_ck_moe_unsupported_reason(layer) + if reason is None: self._aiter_runner = MoeRunner( MoeRunnerBackend.AITER, moe_runner_config ) elif get_moe_runner_backend().is_aiter(): raise ValueError( - "moe_runner_backend=aiter is not supported for " - f"intermediate_size_per_partition={layer.intermediate_size_per_partition}; " + f"moe_runner_backend=aiter is not supported: {reason}; " "use --moe-runner-backend triton." ) else: logger.warning_once( - "aiter CK fused-MoE does not support " - f"intermediate_size_per_partition={layer.intermediate_size_per_partition}; " + f"aiter CK fused-MoE does not support this layer: {reason}; " "using triton MoE runner." ) @@ -1106,12 +1128,14 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp): if self._aiter_runner is not None: from sglang.srt.layers.moe.moe_runner.aiter import ( AiterMoeQuantInfo, + aiter_swiglu_oai_limit, ) quant_info = AiterMoeQuantInfo( w13_weight=layer.w13_weight, w2_weight=layer.w2_weight, expert_mask=layer.dispatcher.expert_mask_gpu, + swiglu_limit=aiter_swiglu_oai_limit(self.moe_runner_config) or 0.0, ) return self._aiter_runner.run(dispatch_output, quant_info)