MiniMax-M3: run the sparse prefill main attention through AITER Gluon paged attention (#36546)

Co-authored-by: Kevin Mi <mikevin920@yahoo.com>
Co-authored-by: Alex Sun <alex.s@amd.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: Kevin Mi <45493463+kevin-mii@users.noreply.github.com>
Co-authored-by: Kevin Mi <mikevin920@gmail.com>
Co-authored-by: YC Yen-Ching Tseng <yctseng@amd.com>
Co-authored-by: Kevin Mi <kevin.mi@radixark.ai>
This commit is contained in:
Chunan Zeng
2026-09-23 14:49:10 -07:00
committed by GitHub
co-authored by Kevin Mi Alex Sun Claude Opus 5 Kevin Mi Kevin Mi YC Yen-Ching Tseng Kevin Mi
parent f766397f2e
commit 4fa2c9c1de
6 changed files with 966 additions and 122 deletions
@@ -6,6 +6,8 @@ import torch
import triton
import triton.language as tl
from sglang.srt.utils import is_gfx95_supported
from ..common.utils import (
_bitonic_merge,
_sort_ids_ascending,
@@ -16,6 +18,9 @@ from ..common.utils import (
unit_scale,
)
# gfx950 MI350X: per-page wins at <=2 pages/block, loses at >=8
_MAX_PER_PAGE_SLOT_UNROLL = 2
@triton.heuristics(
{
@@ -439,6 +444,142 @@ def _topk_index_kernel(
tl.store(ti_ptrs, topk_idx.to(ti_ptrs.dtype.element_ty), mask=topk_mask)
@triton.heuristics(
{"BLOCK_SIZE_KD": lambda args: triton.next_power_of_2(args["qk_head_dim"])}
)
@triton.autotune(
configs=[
triton.Config({"BLOCK_SIZE_Q": 64}, num_warps=4, num_stages=2),
triton.Config({"BLOCK_SIZE_Q": 64}, num_warps=4, num_stages=3),
triton.Config({"BLOCK_SIZE_Q": 64}, num_warps=8, num_stages=2),
triton.Config({"BLOCK_SIZE_Q": 128}, num_warps=8, num_stages=2),
triton.Config({"BLOCK_SIZE_Q": 128}, num_warps=8, num_stages=3),
],
key=["qk_head_dim", "block_size"],
)
@triton.jit
def _index_block_score_only_kernel(
q_ptr, # Q: [total_q, h, d]
k_cache_ptr, # K paged: [max_slots, kh, d]
score_ptr, # Score: [h, total_q, max_seqblock]
req_to_token_ptr, # [max_reqs, max_kv_len]
cu_seqlens,
seq_lens,
prefix_lens,
slot_ids,
max_slots,
num_heads,
gqa_group_size,
qk_head_dim,
sm_scale,
stride_q_n,
stride_q_h,
stride_q_d,
stride_k_s,
stride_k_h,
stride_k_d,
stride_s_h,
stride_s_q,
stride_s_k,
stride_r2t_b,
BLOCK_SIZE_Q: tl.constexpr,
block_size: tl.constexpr, # sparse K block size (== 128)
page_size: tl.constexpr, # paged-cache page size; block_size % page_size == 0
PER_PAGE_SLOTS: tl.constexpr, # derive slots from one base slot per page
BLOCK_SIZE_KD: tl.constexpr,
IS_FP8: tl.constexpr,
):
"""Causal max index score per query and KV block, without the index-value output.
With PER_PAGE_SLOTS, each token's slot is derived as base_slot + in-page offset
from one req_to_token load per page, relying on the allocator's contiguous pages.
"""
sm_scale_log2e = sm_scale * 1.4426950409
pid_q, pid_bh = tl.program_id(0), tl.program_id(1)
pid_b = pid_bh // num_heads
pid_h = pid_bh % num_heads
pid_kh = pid_h // gqa_group_size
seq_start = tl.load(cu_seqlens + pid_b)
q_len = tl.load(cu_seqlens + pid_b + 1) - seq_start
seq_len = tl.load(seq_lens + pid_b)
prefix_len = tl.load(prefix_lens + pid_b)
if BLOCK_SIZE_Q * pid_q >= q_len:
return
sid = (tl.load(slot_ids + pid_b).to(tl.int64) + max_slots) % max_slots
q_ptrs = tl.make_block_ptr(
base=q_ptr + seq_start * stride_q_n + pid_h * stride_q_h,
shape=(q_len, qk_head_dim),
strides=(stride_q_n, stride_q_d),
offsets=(pid_q * BLOCK_SIZE_Q, 0),
block_shape=(BLOCK_SIZE_Q, BLOCK_SIZE_KD),
order=(1, 0),
)
q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero")
off_q = tl.arange(0, BLOCK_SIZE_Q) + pid_q * BLOCK_SIZE_Q + prefix_len
off_k = tl.arange(0, block_size)
off_kd = tl.arange(0, BLOCK_SIZE_KD)
q_row = pid_q * BLOCK_SIZE_Q + tl.arange(0, BLOCK_SIZE_Q)
q_store_mask = q_row < q_len
pages_per_block: tl.constexpr = block_size // page_size
# Within a sparse block, each token's page index and in-page offset are fixed.
page_of = off_k // page_size # [block_size] which physical page (0..pages-1)
in_page = off_k % page_size # [block_size] offset inside that page
hi = min(seq_len, prefix_len + (pid_q + 1) * BLOCK_SIZE_Q)
for i in tl.range(0, hi, block_size):
blk = i // block_size
pos = i + off_k
pos_mask = pos < seq_len
# One base-slot load per page instead of one per token.
if PER_PAGE_SLOTS:
slots = tl.zeros([block_size], dtype=tl.int64)
for p in tl.static_range(0, pages_per_block):
page_tok0 = i + p * page_size
base_slot = tl.load(
req_to_token_ptr + sid * stride_r2t_b + page_tok0,
mask=page_tok0 < seq_len,
other=0,
).to(tl.int64)
base_slot = (base_slot + max_slots) % max_slots
slots = tl.where(page_of == p, base_slot + in_page, slots)
else:
slots = tl.load(
req_to_token_ptr + sid * stride_r2t_b + pos,
mask=pos_mask,
other=0,
).to(tl.int64)
slots = (slots + max_slots) % max_slots
# head_dim (128) is a power of 2 == BLOCK_SIZE_KD, so the dim mask is
# always true -> only mask the K (token) dimension.
k = tl.load(
k_cache_ptr
+ slots[None, :] * stride_k_s
+ pid_kh * stride_k_h
+ off_kd[:, None] * stride_k_d,
mask=pos_mask[None, :],
other=0.0,
)
if IS_FP8:
# Widening dequant for an fp8 index-K cache with bf16 Q; exact
# no-op cast when Q is fp8 too. Compiled out for bf16 K.
k = k.to(q.dtype)
qk = tl.dot(q, k) * sm_scale_log2e
# single fused causal + K-boundary mask
qk = tl.where(
(off_q[:, None] >= pos[None, :]) & pos_mask[None, :], qk, float("-inf")
)
score = tl.max(qk, axis=1) # [BLOCK_SIZE_Q]
s_ptrs = (
score_ptr
+ pid_h * stride_s_h
+ (seq_start + q_row) * stride_s_q
+ blk * stride_s_k
)
tl.store(s_ptrs, score, mask=q_store_mask)
@torch.no_grad()
def flash_prefill_with_topk_index(
q: torch.Tensor,
@@ -467,6 +608,7 @@ def flash_prefill_with_topk_index(
q_scale: Optional[float] = None,
k_scale: Optional[float] = None,
v_scale: Optional[float] = None,
page_size: int = 1,
):
assert score_type in (
"max",
@@ -525,51 +667,94 @@ def flash_prefill_with_topk_index(
def grid(META):
return (triton.cdiv(max_seqlen_q, META["BLOCK_SIZE_Q"]), batch_size * num_heads)
_flash_attn_fwd_with_block_score_kernel[grid](
q,
k_cache,
v_cache,
sink,
o,
score,
req_to_token,
cu_seqlens,
seq_lens,
prefix_lens,
slot_ids,
max_slots,
num_heads,
gqa_group_size,
qk_head_dim,
v_head_dim,
block_size_k,
sm_scale,
k_scale,
v_scale,
False,
1,
q.stride(0),
q.stride(1),
q.stride(2),
k_cache.stride(0),
k_cache.stride(1),
k_cache.stride(2),
v_cache.stride(0) if v_cache is not None else 0,
v_cache.stride(1) if v_cache is not None else 0,
v_cache.stride(2) if v_cache is not None else 0,
sink.stride(0) if sink is not None else 0,
sink.stride(1) if sink is not None else 0,
o.stride(0) if o is not None else 0,
o.stride(1) if o is not None else 0,
o.stride(2) if o is not None else 0,
score.stride(0),
score.stride(1),
score.stride(2),
req_to_token.stride(0),
SCORE_TYPE=score_type,
DISABLE_INDEX_VALUE=disable_index_value,
IS_FP8=is_fp8,
)
if (
is_gfx95_supported()
and disable_index_value
and score_type == "max"
and sink is None
and q_scale in (None, 1.0)
and k_scale == 1.0
and qk_head_dim == 128
and block_size_k == 128
and page_size > 0
and block_size_k % page_size == 0
):
# Source layers do not use idx_o, so run the score-only kernel.
_index_block_score_only_kernel[grid](
q,
k_cache,
score,
req_to_token,
cu_seqlens,
seq_lens,
prefix_lens,
slot_ids,
max_slots,
num_heads,
gqa_group_size,
qk_head_dim,
sm_scale,
q.stride(0),
q.stride(1),
q.stride(2),
k_cache.stride(0),
k_cache.stride(1),
k_cache.stride(2),
score.stride(0),
score.stride(1),
score.stride(2),
req_to_token.stride(0),
block_size=block_size_k,
page_size=page_size,
PER_PAGE_SLOTS=(block_size_k // page_size <= _MAX_PER_PAGE_SLOT_UNROLL),
IS_FP8=is_fp8,
)
else:
_flash_attn_fwd_with_block_score_kernel[grid](
q,
k_cache,
v_cache,
sink,
o,
score,
req_to_token,
cu_seqlens,
seq_lens,
prefix_lens,
slot_ids,
max_slots,
num_heads,
gqa_group_size,
qk_head_dim,
v_head_dim,
block_size_k,
sm_scale,
k_scale,
v_scale,
False,
1,
q.stride(0),
q.stride(1),
q.stride(2),
k_cache.stride(0),
k_cache.stride(1),
k_cache.stride(2),
v_cache.stride(0) if v_cache is not None else 0,
v_cache.stride(1) if v_cache is not None else 0,
v_cache.stride(2) if v_cache is not None else 0,
sink.stride(0) if sink is not None else 0,
sink.stride(1) if sink is not None else 0,
o.stride(0) if o is not None else 0,
o.stride(1) if o is not None else 0,
o.stride(2) if o is not None else 0,
score.stride(0),
score.stride(1),
score.stride(2),
req_to_token.stride(0),
SCORE_TYPE=score_type,
DISABLE_INDEX_VALUE=disable_index_value,
IS_FP8=is_fp8,
)
# topk extraction kernel
topk_idx = torch.full(
@@ -6,6 +6,8 @@ import torch
import triton
import triton.language as tl
from sglang.srt.utils import is_gfx95_supported
from ..common.utils import (
check_sparse_kv_fp8,
get_cu_seqblocks,
@@ -15,6 +17,11 @@ from ..common.utils import (
)
def _sparse_subk(block_size_k: int) -> int:
"""Sub-tile only the measured 128-token block; keep other widths unchanged."""
return 64 if is_gfx95_supported() and block_size_k == 128 else 0
@triton.heuristics(
{
"BLOCK_SIZE_KD": lambda args: triton.next_power_of_2(args["qk_head_dim"]),
@@ -32,12 +39,30 @@ from ..common.utils import (
}
)
@triton.autotune(
# Configs that fail to compile on the target arch are skipped, so widening
# the num_warps x num_stages grid only adds candidates, never a bad kernel.
# Keep the additional configs on gfx95; gfx942 retains the original set.
configs=[
triton.Config({}, num_warps=nw, num_stages=ns)
for nw in (2, 4, 8)
for ns in (2, 3, 4)
# CDNA sub-tiled MFMA configs; NVIDIA's Triton rejects these launch kwargs.
*(
[
triton.Config(
{"matrix_instr_nonkdim": 16, "kpack": 2},
num_warps=1,
num_stages=1,
),
triton.Config(
{"matrix_instr_nonkdim": 16, "kpack": 2},
num_warps=2,
num_stages=1,
),
]
if is_gfx95_supported()
else []
),
*[
triton.Config({}, num_warps=nw, num_stages=ns)
for nw in (2, 4, 8)
for ns in (2, 3, 4)
],
],
key=[
"BLOCK_SIZE_Q",
@@ -104,6 +129,7 @@ def _gqa_share_sparse_fwd_kernel(
BLOCK_SIZE_H: tl.constexpr,
BLOCK_SIZE_T: tl.constexpr,
BLOCK_SIZE_QH: tl.constexpr,
SUB_K: tl.constexpr, # CDNA KV sub-tile width (0 = disabled)
# has sink
HAS_SINK: tl.constexpr,
USE_TMA: tl.constexpr,
@@ -190,77 +216,145 @@ def _gqa_share_sparse_fwd_kernel(
acc_o = tl.full((BLOCK_SIZE_QH, BLOCK_SIZE_VD), 0, dtype=tl.float32)
q = tl.reshape(q, BLOCK_SIZE_QH, BLOCK_SIZE_KD)
# sparse attention
for i in range(real_topk):
# get current block start index (absolute K position)
c = tl.load(t_ptr_j).to(tl.int32) * BLOCK_SIZE_K
t_ptr_j = t_ptr_j + stride_tk
# paged load K via req_to_token: pos -> slot -> k_cache
pos = c + off_n
pos_mask = pos < seq_len
slots = tl.load(
req_to_token_ptr + sid * stride_r2t_b + pos,
mask=pos_mask,
other=0,
).to(tl.int64)
if HAS_LOC_MAPPING:
if SUB_K > 0:
# Split each BLOCK_SIZE_K-token KV block into SUB_K-token sub-tiles so
# each QK/PV MFMA is right-sized. Numerically equivalent to the dense
# path (flash-softmax reassociation).
NUM_SUB: tl.constexpr = BLOCK_SIZE_K // SUB_K
for i in range(real_topk):
c = tl.load(t_ptr_j).to(tl.int32) * BLOCK_SIZE_K
t_ptr_j = t_ptr_j + stride_tk
for sub_i in range(NUM_SUB):
off_sub = tl.arange(0, SUB_K) + sub_i * SUB_K
pos_s = c + off_sub
pos_mask_s = pos_s < seq_len
slots_s = tl.load(
req_to_token_ptr + sid * stride_r2t_b + pos_s,
mask=pos_mask_s,
other=0,
).to(tl.int64)
if HAS_LOC_MAPPING:
slots_s = tl.load(
loc_mapping_ptr + slots_s,
mask=pos_mask_s,
other=0,
).to(tl.int64)
slots_s = (slots_s + max_slots) % max_slots
k_s = tl.load(
k_cache_ptr
+ slots_s[None, :] * stride_ks
+ pid_kh * stride_kh
+ off_kd[:, None] * stride_kd,
mask=kd_mask[:, None] & pos_mask_s[None, :],
other=0.0,
)
if IS_FP8:
k_s = k_s.to(q.dtype)
off_q_s = (
tl.arange(0, BLOCK_SIZE_Q)[:, None]
+ pid_q_j * BLOCK_SIZE_Q
+ prefix_len
- off_sub[None, :]
)
qk_s = tl.zeros(
(BLOCK_SIZE_Q, BLOCK_SIZE_H, SUB_K), dtype=tl.float32
)
qk_s += tl.where(off_q_s[:, None, :] >= c, 0, float("-inf"))
qk_s = tl.reshape(qk_s, BLOCK_SIZE_QH, SUB_K)
qk_s += tl.dot(q, k_s) * (sm_scale_log2e * k_scale)
qk_s += tl.where(pos_mask_s[None, :], 0, float("-inf"))
m_ij = tl.maximum(m_i, tl.max(qk_s, axis=1))
p_s = tl.exp2(qk_s - m_ij[:, None])
l_ij = tl.sum(p_s, axis=1)
acc_o = acc_o * tl.exp2(m_i - m_ij)[:, None]
v_s = tl.load(
v_cache_ptr
+ slots_s[:, None] * stride_vs
+ pid_kh * stride_vh
+ off_vd[None, :] * stride_vd,
mask=pos_mask_s[:, None] & vd_mask[None, :],
other=0.0,
)
if IS_FP8:
v_s = v_s.to(q.dtype)
p_s = p_s.to(v_s.dtype)
acc_o += tl.dot(p_s, v_s) * v_scale
m_i = m_ij
lse_i = m_ij + tl.log2(tl.exp2(lse_i - m_ij) + l_ij)
else:
for i in range(real_topk):
# get current block start index (absolute K position)
c = tl.load(t_ptr_j).to(tl.int32) * BLOCK_SIZE_K
t_ptr_j = t_ptr_j + stride_tk
# paged load K via req_to_token: pos -> slot -> k_cache
pos = c + off_n
pos_mask = pos < seq_len
slots = tl.load(
loc_mapping_ptr + slots,
req_to_token_ptr + sid * stride_r2t_b + pos,
mask=pos_mask,
other=0,
).to(tl.int64)
slots = (slots + max_slots) % max_slots # safety against negative
# k shape: [BLOCK_SIZE_KD, BLOCK_SIZE_K] (transposed for tl.dot)
k = tl.load(
k_cache_ptr
+ slots[None, :] * stride_ks
+ pid_kh * stride_kh
+ off_kd[:, None] * stride_kd,
mask=kd_mask[:, None] & pos_mask[None, :],
other=0.0,
)
if IS_FP8:
# fp8 main K cache: widening cast with bf16/fp16 Q (unit-scaled
# cache -> exact inverse dequant; k_scale covers calibrated
# caches), no-op with fp8 Q (fp8 attn-GEMM mode) so tl.dot runs
# fp8x8. Compiled out when the cache is bf16.
k = k.to(q.dtype)
# compute qk
qk = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_K), dtype=tl.float32)
# causal mask
qk += tl.where(off_q_k[:, None, :] >= c, 0, float("-inf"))
qk = tl.reshape(qk, BLOCK_SIZE_QH, BLOCK_SIZE_K)
# [BLOCK_SIZE_QH, qk_head_dim] @ [qk_head_dim, BLOCK_SIZE_K]
# -> [BLOCK_SIZE_QH, BLOCK_SIZE_K]
qk += tl.dot(q, k) * (sm_scale_log2e * k_scale)
# K boundary mask: positions beyond seq_len contribute -inf
qk += tl.where(pos_mask[None, :], 0, float("-inf"))
# compute m_ij and l_ij
m_ij = tl.maximum(m_i, tl.max(qk, axis=1))
p = tl.exp2(qk - m_ij[:, None])
l_ij = tl.sum(p, axis=1)
# scale acc_o
acc_o_scale = tl.exp2(m_i - m_ij)
acc_o = acc_o * acc_o_scale[:, None]
# paged load V
v = tl.load(
v_cache_ptr
+ slots[:, None] * stride_vs
+ pid_kh * stride_vh
+ off_vd[None, :] * stride_vd,
mask=pos_mask[:, None] & vd_mask[None, :],
other=0.0,
)
if IS_FP8:
# Cast V to the compute dtype: widening with bf16/fp16 Q (so
# `p.to(v.dtype)` keeps P in the compute dtype), no-op with fp8
# Q where P is quantized to e4m3 for the fp8 PV MMA — the same
# accuracy contract as fmha_sm100's fp8 kernel.
v = v.to(q.dtype)
p = p.to(v.dtype)
acc_o += tl.dot(p, v) * v_scale
# update statistics
m_i = m_ij
lse_i = m_ij + tl.log2(tl.exp2(lse_i - m_ij) + l_ij)
if HAS_LOC_MAPPING:
slots = tl.load(
loc_mapping_ptr + slots,
mask=pos_mask,
other=0,
).to(tl.int64)
slots = (slots + max_slots) % max_slots # safety against negative
# k shape: [BLOCK_SIZE_KD, BLOCK_SIZE_K] (transposed for tl.dot)
k = tl.load(
k_cache_ptr
+ slots[None, :] * stride_ks
+ pid_kh * stride_kh
+ off_kd[:, None] * stride_kd,
mask=kd_mask[:, None] & pos_mask[None, :],
other=0.0,
)
if IS_FP8:
# fp8 main K cache: widening cast with bf16/fp16 Q (unit-scaled
# cache -> exact inverse dequant; k_scale covers calibrated
# caches), no-op with fp8 Q (fp8 attn-GEMM mode) so tl.dot runs
# fp8x8. Compiled out when the cache is bf16.
k = k.to(q.dtype)
# compute qk
qk = tl.zeros(
(BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_K), dtype=tl.float32
)
# causal mask
qk += tl.where(off_q_k[:, None, :] >= c, 0, float("-inf"))
qk = tl.reshape(qk, BLOCK_SIZE_QH, BLOCK_SIZE_K)
# [BLOCK_SIZE_QH, qk_head_dim] @ [qk_head_dim, BLOCK_SIZE_K]
# -> [BLOCK_SIZE_QH, BLOCK_SIZE_K]
qk += tl.dot(q, k) * (sm_scale_log2e * k_scale)
# K boundary mask: positions beyond seq_len contribute -inf
qk += tl.where(pos_mask[None, :], 0, float("-inf"))
# compute m_ij and l_ij
m_ij = tl.maximum(m_i, tl.max(qk, axis=1))
p = tl.exp2(qk - m_ij[:, None])
l_ij = tl.sum(p, axis=1)
# scale acc_o
acc_o_scale = tl.exp2(m_i - m_ij)
acc_o = acc_o * acc_o_scale[:, None]
# paged load V
v = tl.load(
v_cache_ptr
+ slots[:, None] * stride_vs
+ pid_kh * stride_vh
+ off_vd[None, :] * stride_vd,
mask=pos_mask[:, None] & vd_mask[None, :],
other=0.0,
)
if IS_FP8:
# Cast V to the compute dtype: widening with bf16/fp16 Q (so
# `p.to(v.dtype)` keeps P in the compute dtype), no-op with fp8
# Q where P is quantized to e4m3 for the fp8 PV MMA — the same
# accuracy contract as fmha_sm100's fp8 kernel.
v = v.to(q.dtype)
p = p.to(v.dtype)
acc_o += tl.dot(p, v) * v_scale
# update statistics
m_i = m_ij
lse_i = m_ij + tl.log2(tl.exp2(lse_i - m_ij) + l_ij)
# final scale
acc_o = acc_o * tl.exp2(m_i - lse_i)[:, None]
# save output
@@ -385,6 +479,7 @@ def flash_prefill_with_gqa_share_sparse(
o.stride(2),
req_to_token.stride(0),
BLOCK_SIZE_Q=BLOCK_SIZE_Q,
SUB_K=_sparse_subk(BLOCK_SIZE_K),
BLOCK_SIZE_K=BLOCK_SIZE_K,
USE_TMA=use_tma,
IS_FP8=is_fp8,
+3
View File
@@ -1698,6 +1698,9 @@ class Envs:
# 2 is the accuracy-safe default: higher values reuse staler selections
# in the skip layers.
SGLANG_MINIMAX_M3_INDEX_TOPK_FREQ = EnvInt(2)
# Run the sparse prefill main attention through AITER's Gluon paged attention
# instead of the Triton kernel. Unsupported cases fall back to Triton.
SGLANG_OPT_USE_MINIMAX_GLUON_PREFILL = EnvBool(True)
# MiniMax M3 NPU prefill MAIN-attention: route the sparse main attention through
# the native Ascend FA op `torch.ops.npu.npu_fused_infer_attention_score` (FIA)
# with a per-query CUSTOM block_table
@@ -1570,6 +1570,7 @@ class MiniMaxSparseAttnBackend(AttentionBackend):
disable_index_value=disable_value,
use_msa=self.use_msa,
seqlens_cpu=forward_batch.extend_seq_lens_cpu,
seq_lens_cpu=forward_batch.seq_lens_cpu,
cu_seqblocks_q=cu_seqblocks_q,
max_seqblock_q=max_seqblock_q,
all_seqblock_q=all_seqblock_q,
@@ -1579,6 +1580,7 @@ class MiniMaxSparseAttnBackend(AttentionBackend):
idx_q_scale=layer.idx_q_scale_float,
idx_k_scale=layer.idx_k_scale_float,
idx_v_scale=layer.idx_v_scale_float,
page_size=self.page_size,
cached_topk_idx=cached_topk_idx,
return_topk_idx=want_topk,
loc_mapping=self._loc_mapping,
@@ -0,0 +1,492 @@
# Copyright 2025 SGLang Team
# Ported from
# https://github.com/alexsun07/sglang/blob/191624ff8a6d8e9fc31b9363d2138c16db66f124/python/sglang/srt/layers/attention/minimax_sparse_ops/atom_prefill.py
"""MiniMax-M3 sparse prefill using AITER's Gluon paged attention.
The main KV pool remains NHD ``[max_slots, 1, head_dim]``. Each request's
context is gathered into persistent SHUFFLE 5D scratch pages:
- key scratch: [num_pages, 1, head_dim // x, GLUON_PAGE_SIZE, x]
- value scratch: [num_pages, 1, GLUON_PAGE_SIZE // x, head_dim, x] (transposed)
with ``x = 16 // dtype.itemsize``. Each request occupies a contiguous,
position-ordered page range rounded up to whole sparse blocks.
"""
from __future__ import annotations
from itertools import accumulate
from typing import Optional
import torch
import triton
import triton.language as tl
from sglang.srt.layers.attention.aiter_utils import (
get_recommended_splits,
pa_decode_gluon,
)
SPARSE_BLOCK_SIZE = 128
# The Gluon PS kernel takes kv_block_size 16 or 64; 64 halves the block-table width.
GLUON_PAGE_SIZE = 64
PAGES_PER_BLOCK = SPARSE_BLOCK_SIZE // GLUON_PAGE_SIZE
HEAD_DIM = 128
# Scratch grows in 1024-page steps (16 KiB/page/buffer -> 16 MiB granularity).
_SCRATCH_GROW_PAGES = 1024
_PAGE_ELEMS = HEAD_DIM * GLUON_PAGE_SIZE
# 512 MiB per buffer at bf16 (~2.1M context tokens per forward); past it the caller
# falls back to the Triton kernel, which reads the pool in place.
_MAX_SCRATCH_PAGES = (512 * 1024 * 1024) // (_PAGE_ELEMS * 2)
class GluonPrefillUnavailableError(RuntimeError):
"""Raised when the scratch cannot hold this forward; the caller falls back to Triton."""
@triton.jit
def _gather_nhd_to_shuffle_kernel(
k_pool_ptr, # [max_slots, 1, HEAD_DIM] (NHD main pool, this layer)
v_pool_ptr, # [max_slots, 1, HEAD_DIM]
k_out_ptr, # [num_pages, 1, HEAD_DIM // X, PAGE, X] contiguous
v_out_ptr, # [num_pages, 1, PAGE // X, HEAD_DIM, X] contiguous
req_to_token_ptr, # [max_reqs, max_kv_len]
req_pool_indices_ptr, # [batch]
seq_lens_ptr, # [batch] total K length (prefix + current chunk)
page_req_ptr, # [total_pages] -> batch index owning this page
page_pos_ptr, # [total_pages] -> page index within the request
stride_k_slot,
stride_v_slot,
stride_r2t_b,
HEAD_DIM_C: tl.constexpr,
PAGE: tl.constexpr,
X: tl.constexpr,
):
pid = tl.program_id(0)
req = tl.load(page_req_ptr + pid)
ppos = tl.load(page_pos_ptr + pid)
pool_id = tl.load(req_pool_indices_ptr + req).to(tl.int64)
seq_len = tl.load(seq_lens_ptr + req)
off_t = tl.arange(0, PAGE)
tok = ppos * PAGE + off_t
tmask = tok < seq_len
slots = tl.load(
req_to_token_ptr + pool_id * stride_r2t_b + tok, mask=tmask, other=0
).to(tl.int64)
off_d = tl.arange(0, HEAD_DIM_C)
# Coalesced NHD reads: each token's head_dim vector is contiguous.
k_tile = tl.load(
k_pool_ptr + slots[:, None] * stride_k_slot + off_d[None, :],
mask=tmask[:, None],
other=0.0,
)
v_tile = tl.load(
v_pool_ptr + slots[:, None] * stride_v_slot + off_d[None, :],
mask=tmask[:, None],
other=0.0,
)
page_base = pid.to(tl.int64) * (HEAD_DIM_C * PAGE)
# K shuffle page [1, D//X, PAGE, X]: elem(d, t) at (d//X)*(PAGE*X) + t*X + d%X
k_off = (
(off_d[None, :] // X) * (PAGE * X) + off_t[:, None] * X + (off_d[None, :] % X)
)
# V transposed page [1, PAGE//X, D, X]: elem(d, t) at (t//X)*(D*X) + d*X + t%X
v_off = (
(off_t[:, None] // X) * (HEAD_DIM_C * X)
+ off_d[None, :] * X
+ (off_t[:, None] % X)
)
# Unmasked: tiles are zero past seq_len, so no previous layer's data survives.
tl.store(k_out_ptr + page_base + k_off, k_tile)
tl.store(v_out_ptr + page_base + v_off, v_tile)
# Grow-only (device, dtype) -> (k_flat, v_flat); layers run in sequence and share it.
_SCRATCH: dict = {}
def _get_scratch(
device: torch.device, dtype: torch.dtype, total_pages: int
) -> tuple[torch.Tensor, torch.Tensor]:
key = (device, dtype)
entry = _SCRATCH.get(key)
needed = total_pages * _PAGE_ELEMS
if entry is None or entry[0].numel() < needed:
cap_pages = (
(total_pages + _SCRATCH_GROW_PAGES - 1) // _SCRATCH_GROW_PAGES
) * _SCRATCH_GROW_PAGES
# Grows after the KV pool claimed its budget; report OOM as unsupported so
# the caller falls back to the scratch-free Triton kernel.
try:
entry = (
torch.empty(cap_pages * _PAGE_ELEMS, dtype=dtype, device=device),
torch.empty(cap_pages * _PAGE_ELEMS, dtype=dtype, device=device),
)
except torch.cuda.OutOfMemoryError as exc:
raise GluonPrefillUnavailableError(
f"gluon prefill scratch of {cap_pages} pages "
f"({cap_pages * _PAGE_ELEMS * dtype.itemsize * 2 >> 20} MiB) "
"does not fit; lower --mem-fraction-static"
) from exc
_SCRATCH[key] = entry
return entry
def _gather_context_to_shuffle(
k_cache: torch.Tensor,
v_cache: torch.Tensor,
req_to_token: torch.Tensor,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
page_req: torch.Tensor,
page_pos: torch.Tensor,
total_pages: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Gather this batch's context tokens (NHD pool slots) into SHUFFLE 5D pages."""
x = 16 // k_cache.dtype.itemsize # 8 for bf16/fp16
k_flat, v_flat = _get_scratch(k_cache.device, k_cache.dtype, total_pages)
_gather_nhd_to_shuffle_kernel[(total_pages,)](
k_cache,
v_cache,
k_flat,
v_flat,
req_to_token,
req_pool_indices,
seq_lens,
page_req,
page_pos,
k_cache.stride(0),
v_cache.stride(0),
req_to_token.stride(0),
HEAD_DIM_C=HEAD_DIM,
PAGE=GLUON_PAGE_SIZE,
X=x,
)
k5 = k_flat[: total_pages * _PAGE_ELEMS].view(
total_pages, 1, HEAD_DIM // x, GLUON_PAGE_SIZE, x
)
v5 = v_flat[: total_pages * _PAGE_ELEMS].view(
total_pages, 1, GLUON_PAGE_SIZE // x, HEAD_DIM, x
)
return k5, v5
@triton.jit
def _build_gluon_sparse_bt_prefill_kernel(
topk_ptr, # [Hkv=1, total_q, topk] int32
req_id_ptr, # [total_q] int32
abs_pos_ptr, # [total_q] int32
page_start_ptr, # [batch] int32: first scratch page of each request
sparse_bt_ptr, # [total_q, topk * pages_per_block] int32
sparse_ctx_ptr, # [total_q] int32
max_topk,
stride_topk_n,
stride_topk_t,
stride_sparse_bt_n,
sparse_block_size: tl.constexpr,
pages_per_block: tl.constexpr,
BLOCK_SIZE_T: tl.constexpr,
):
pid_n = tl.program_id(0)
req_id = tl.load(req_id_ptr + pid_n)
base_page = tl.load(page_start_ptr + req_id)
abs_pos = tl.load(abs_pos_ptr + pid_n)
causal_len = abs_pos + 1
self_blk = abs_pos // sparse_block_size
topk_row = topk_ptr + pid_n * stride_topk_n
out_row = sparse_bt_ptr + pid_n * stride_sparse_bt_n
off_t = tl.arange(0, BLOCK_SIZE_T)
blk = tl.load(topk_row + off_t * stride_topk_t, mask=off_t < max_topk, other=-1)
# Causal: a query at abs_pos may attend its own block and all earlier ones.
valid = (blk >= 0) & (blk <= self_blk)
is_tail = valid & (blk == self_blk)
is_full = valid & (blk < self_blk)
n_full = tl.sum(is_full.to(tl.int32), axis=0)
n_valid = tl.sum(valid.to(tl.int32), axis=0)
# Full blocks first, the partial causal tail last, so the walk matches sparse_ctx.
earlier_full = tl.cumsum(is_full.to(tl.int32), axis=0) - is_full.to(tl.int32)
sparse_slot = tl.where(is_full, earlier_full, n_full)
# Scratch pages for each request are contiguous and position-ordered.
dst_base = sparse_slot * pages_per_block
for j in tl.static_range(0, pages_per_block):
phys_page = base_page + blk * pages_per_block + j
tl.store(out_row + dst_base + j, phys_page, mask=valid)
n_used = n_valid * pages_per_block
off_w = tl.arange(0, BLOCK_SIZE_T * pages_per_block)
# BLOCK_SIZE_T is next_power_of_2(topk), so off_w overshoots the row whenever
# topk is not a power of two -- bound the tail fill by the real row width.
tl.store(
out_row + off_w,
tl.zeros_like(off_w),
mask=(off_w >= n_used) & (off_w < max_topk * pages_per_block),
)
# Effective KV length the kernel will walk across the packed pages.
tail_tokens = causal_len - self_blk * sparse_block_size
has_tail = tl.sum(is_tail.to(tl.int32), axis=0) > 0
ctx_len = n_full * sparse_block_size + tl.where(has_tail, tail_tokens, 0)
ctx_len = tl.where(
has_tail, ctx_len, tl.minimum(n_valid * sparse_block_size, causal_len)
)
tl.store(sparse_ctx_ptr + pid_n, ctx_len)
_META_CACHE: Optional[tuple] = None
_BT_OUT_CACHE: Optional[tuple] = None
def _build_gluon_prefill_meta(
cu_seqlens: torch.Tensor,
prefix_lens: torch.Tensor,
seq_lens: torch.Tensor,
seq_lens_cpu: torch.Tensor,
total_q: int,
):
global _META_CACHE, _BT_OUT_CACHE
if _META_CACHE is not None:
(k_cu, k_prefix, k_seq, k_total_q), cached = _META_CACHE
if (
k_cu is cu_seqlens
and k_prefix is prefix_lens
and k_seq is seq_lens
and k_total_q == total_q
):
return cached
device = cu_seqlens.device
pos = torch.arange(total_q, dtype=torch.int32, device=device)
req_id = torch.searchsorted(cu_seqlens[1:].contiguous(), pos, right=True).to(
torch.int32
)
abs_pos = (prefix_lens[req_id] + (pos - cu_seqlens[req_id])).to(torch.int32)
# Host-side page layout: each request's context span rounds up to whole
# sparse blocks so every emitted page id stays inside its own span.
pages_per_req = [
((kv_len + SPARSE_BLOCK_SIZE - 1) // SPARSE_BLOCK_SIZE) * PAGES_PER_BLOCK
for kv_len in seq_lens_cpu.tolist()
]
total_pages = sum(pages_per_req)
page_start = torch.tensor(
list(accumulate(pages_per_req, initial=0))[:-1],
dtype=torch.int32,
device=device,
)
repeats = torch.tensor(pages_per_req, dtype=torch.int64, device=device)
page_req = torch.repeat_interleave(
torch.arange(len(pages_per_req), dtype=torch.int32, device=device),
repeats,
output_size=total_pages,
)
page_pos = (
torch.arange(total_pages, dtype=torch.int32, device=device)
- page_start[page_req.long()]
).to(torch.int32)
meta = (req_id, abs_pos, page_start, page_req, page_pos, total_pages)
_BT_OUT_CACHE = None # new forward: no block table can be valid anymore
_META_CACHE = ((cu_seqlens, prefix_lens, seq_lens, total_q), meta)
return meta
def _build_gluon_sparse_bt_prefill(
topk_idx: torch.Tensor,
req_id: torch.Tensor,
abs_pos: torch.Tensor,
page_start: torch.Tensor,
sparse_block_size: int,
) -> tuple[torch.Tensor, torch.Tensor]:
# topk_idx: [num_kv_heads=1, total_q, topk]
total_q = topk_idx.shape[1]
topk = topk_idx.shape[2]
global _BT_OUT_CACHE
if _BT_OUT_CACHE is not None:
(k_topk_idx, k_total_q, k_topk, k_block_size), out_cached = _BT_OUT_CACHE
if (
k_topk_idx is topk_idx
and k_total_q == total_q
and k_topk == topk
and k_block_size == sparse_block_size
):
return out_cached
sparse_bt = torch.empty(
(total_q, topk * PAGES_PER_BLOCK),
dtype=torch.int32,
device=topk_idx.device,
)
sparse_ctx = torch.empty((total_q,), dtype=torch.int32, device=topk_idx.device)
_build_gluon_sparse_bt_prefill_kernel[(total_q,)](
topk_idx,
req_id,
abs_pos,
page_start,
sparse_bt,
sparse_ctx,
topk,
topk_idx.stride(1),
topk_idx.stride(2),
sparse_bt.stride(0),
sparse_block_size=sparse_block_size,
pages_per_block=PAGES_PER_BLOCK,
BLOCK_SIZE_T=triton.next_power_of_2(topk),
)
_BT_OUT_CACHE = (
(topk_idx, total_q, topk, sparse_block_size),
(sparse_bt, sparse_ctx),
)
return sparse_bt, sparse_ctx
def _unit_or_none(scale) -> bool:
return scale is None or scale == 1.0
def can_use_gluon_prefill(
q: torch.Tensor,
k_cache: torch.Tensor,
v_cache: torch.Tensor,
sink: Optional[torch.Tensor],
block_size_k: int,
seq_lens_cpu: Optional[torch.Tensor],
q_scale: Optional[float] = None,
k_scale: Optional[float] = None,
v_scale: Optional[float] = None,
) -> bool:
"""Cheap static gate; the scratch size is checked per forward."""
if (
pa_decode_gluon is None
or get_recommended_splits is None
or sink is not None
or seq_lens_cpu is None
or block_size_k != SPARSE_BLOCK_SIZE
):
return False
# The gather requires a contiguous, single-head NHD pool. Calibrated and
# unsupported cache formats stay on the Triton path.
return (
q.dim() == 3
and q.shape[-1] == HEAD_DIM
and k_cache.dim() == 3
and v_cache.dim() == 3
and k_cache.shape[1] == 1
and v_cache.shape[1] == 1
and k_cache.shape[2] == HEAD_DIM
and v_cache.shape[2] == HEAD_DIM
and q.dtype in (torch.bfloat16, torch.float16)
and k_cache.dtype == q.dtype
and v_cache.dtype == q.dtype
and k_cache.stride(2) == 1
and v_cache.stride(2) == 1
and _unit_or_none(q_scale)
and _unit_or_none(k_scale)
and _unit_or_none(v_scale)
)
@torch.no_grad()
def gluon_sparse_prefill(
q: torch.Tensor,
k_cache: torch.Tensor,
v_cache: torch.Tensor,
topk_idx: torch.Tensor,
req_to_token: torch.Tensor,
req_pool_indices: torch.Tensor,
cu_seqlens: torch.Tensor,
seq_lens: torch.Tensor,
prefix_lens: torch.Tensor,
seq_lens_cpu: torch.Tensor,
block_size_k: int,
sm_scale: Optional[float] = None,
) -> torch.Tensor:
"""Run Gluon sparse prefill over the NHD KV pool via the SHUFFLE scratch.
Raises GluonPrefillUnavailableError when the scratch cannot hold the context.
"""
if sm_scale is None:
sm_scale = q.shape[-1] ** -0.5
q = q.contiguous()
total_q, num_q_heads, head_dim = q.shape
if total_q == 0:
return torch.empty_like(q)
assert topk_idx.shape[0] == 1 and topk_idx.shape[1] == total_q, (
f"gluon prefill expects topk_idx [1, total_q, topk], got {tuple(topk_idx.shape)}"
)
batch = cu_seqlens.shape[0] - 1
assert seq_lens_cpu.numel() == batch and seq_lens.shape[0] == batch, (
f"gluon prefill batch mismatch: cu_seqlens batch {batch}, "
f"seq_lens {tuple(seq_lens.shape)}, seq_lens_cpu {seq_lens_cpu.numel()}"
)
req_id, abs_pos, page_start, page_req, page_pos, total_pages = (
_build_gluon_prefill_meta(
cu_seqlens, prefix_lens, seq_lens, seq_lens_cpu, total_q
)
)
if total_pages > _MAX_SCRATCH_PAGES:
raise GluonPrefillUnavailableError(
f"gluon prefill context span too large for scratch: {total_pages} pages "
f"> cap {_MAX_SCRATCH_PAGES}"
)
# Gather the current layer's prefix and current chunk into SHUFFLE 5D views.
k_cache, v_cache = _gather_context_to_shuffle(
k_cache,
v_cache,
req_to_token,
req_pool_indices,
seq_lens,
page_req,
page_pos,
total_pages,
)
sparse_bt, sparse_ctx = _build_gluon_sparse_bt_prefill(
topk_idx, req_id, abs_pos, page_start, block_size_k
)
out = torch.empty_like(q)
num_seqs = total_q
ctx_part = 256
max_part_num = get_recommended_splits(num_seqs, 1)
intermediate_shape = (num_seqs, 1, max_part_num, num_q_heads)
exp_sums = torch.empty(intermediate_shape, dtype=torch.float32, device=q.device)
max_logits = torch.empty_like(exp_sums)
temporary_output = torch.empty(
(*intermediate_shape, head_dim), dtype=q.dtype, device=q.device
)
pa_decode_gluon(
output=out,
query=q,
key_cache=k_cache,
value_cache=v_cache,
context_lengths=sparse_ctx,
block_tables=sparse_bt,
softmax_scale=sm_scale,
query_length=1,
max_context_partition_num=max_part_num,
context_partition_size=ctx_part,
compute_type=q.dtype,
key_scale=None,
value_scale=None,
exp_sums=exp_sums,
max_logits=max_logits,
temporary_output=temporary_output,
sinks=None,
sliding_window=0,
ps=True,
)
return out
@@ -19,9 +19,14 @@ from sglang.kernels.ops.attention.minimax_sparse.prefill.flash_with_topk_idx imp
from sglang.kernels.ops.attention.minimax_sparse.prefill.topk_sparse import (
flash_prefill_with_gqa_share_sparse,
)
from sglang.srt.environ import envs
from sglang.srt.utils import is_gfx95_supported, is_hip
_use_aiter_gfx95 = envs.SGLANG_USE_AITER.get() and is_hip() and is_gfx95_supported()
logger = logging.getLogger(__name__)
_msa_fallback_warned = False
_gluon_fallback_warned = False
def _warn_msa_fallback(err: Exception) -> None:
@@ -35,6 +40,18 @@ def _warn_msa_fallback(err: Exception) -> None:
_msa_fallback_warned = True
def _warn_gluon_fallback(msg: str) -> None:
global _gluon_fallback_warned
if _gluon_fallback_warned:
return
logger.warning(
"MiniMax Gluon sparse prefill is unavailable (%s); falling back to Triton "
"sparse attention.",
msg,
)
_gluon_fallback_warned = True
def minimax_sparse_prefill(
q: torch.Tensor, # [total_extend_tokens, num_q_heads, qk_head_dim]
k_cache: torch.Tensor, # [max_slots, num_kv_heads, head_dim] (paged main)
@@ -76,6 +93,8 @@ def minimax_sparse_prefill(
cached_topk_idx: Optional[torch.Tensor] = None,
return_topk_idx: bool = False,
loc_mapping: Optional[torch.Tensor] = None,
page_size: int = 1,
seq_lens_cpu: Optional[torch.Tensor] = None,
):
"""Run MiniMax-M3 sparse prefill.
@@ -91,6 +110,10 @@ def minimax_sparse_prefill(
kernels. Supplying them avoids recomputing the same block layout twice.
``seqlens_cpu`` (host copy of ``torch.diff(cu_seqlens)``) is forwarded to
``get_cu_seqblocks`` to avoid a per-layer device sync when it recomputes.
``seq_lens_cpu`` (host copy of ``seq_lens``, i.e. prefix + current chunk
per request) is only consumed by the env-gated Gluon prefill path for
sync-free scratch-page sizing; ``None`` disables that path.
"""
if cu_seqblocks_q is None or max_seqblock_q is None or all_seqblock_q is None:
cu_seqblocks_q, max_seqblock_q, all_seqblock_q, _, _, _ = get_cu_seqblocks(
@@ -129,6 +152,7 @@ def minimax_sparse_prefill(
cu_seqblocks_q=cu_seqblocks_q,
max_seqblock_q=max_seqblock_q,
all_seqblock_q=all_seqblock_q,
page_size=page_size,
q_scale=idx_q_scale,
k_scale=idx_k_scale,
v_scale=idx_v_scale,
@@ -144,10 +168,53 @@ def minimax_sparse_prefill(
# Reduced top-k cached by the caller for subsequent skip layers.
reduced_topk_idx = topk_idx
# Step 3: Sparse attention using topk index (main head). The MSA path only
# replaces this step; the indexer above is unchanged. MSA has no attn-sink
# input, so keep the Triton path when sink is present.
if use_msa and sink is None and loc_mapping is None:
# Step 3: Sparse attention using topk index (main head). The Gluon and
# MSA paths only replace this step; the indexer above is unchanged. MSA has
# no attn-sink input, so keep the Triton path when sink is present.
o = None
if (
_use_aiter_gfx95
and envs.SGLANG_OPT_USE_MINIMAX_GLUON_PREFILL.get()
# the scratch gather reads pool slots without the HiSparse remap
and loc_mapping is None
):
from .gluon_prefill import (
GluonPrefillUnavailableError,
can_use_gluon_prefill,
gluon_sparse_prefill,
)
if can_use_gluon_prefill(
q,
k_cache,
v_cache,
sink,
block_size_k,
seq_lens_cpu,
q_scale,
k_scale,
v_scale,
):
try:
o = gluon_sparse_prefill(
q=q,
k_cache=k_cache,
v_cache=v_cache,
topk_idx=topk_idx,
req_to_token=req_to_token,
req_pool_indices=slot_ids,
cu_seqlens=cu_seqlens,
seq_lens=seq_lens,
prefix_lens=prefix_lens,
seq_lens_cpu=seq_lens_cpu,
block_size_k=block_size_k,
sm_scale=sm_scale,
)
except GluonPrefillUnavailableError as err:
_warn_gluon_fallback(str(err))
else:
_warn_gluon_fallback("unsupported batch/cache layout or dtype")
if o is None and use_msa and sink is None and loc_mapping is None:
from .msa import MSAUnavailableError, msa_sparse_prefill_main
try:
@@ -191,7 +258,7 @@ def minimax_sparse_prefill(
v_scale=v_scale,
loc_mapping=loc_mapping,
)
else:
elif o is None:
o = flash_prefill_with_gqa_share_sparse(
q=q,
k_cache=k_cache,