MiniMax-M3: run the sparse prefill main attention through AITER Gluon paged attention (#36546)
Co-authored-by: Kevin Mi <mikevin920@yahoo.com> Co-authored-by: Alex Sun <alex.s@amd.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: Kevin Mi <45493463+kevin-mii@users.noreply.github.com> Co-authored-by: Kevin Mi <mikevin920@gmail.com> Co-authored-by: YC Yen-Ching Tseng <yctseng@amd.com> Co-authored-by: Kevin Mi <kevin.mi@radixark.ai>
This commit is contained in:
co-authored by
Kevin Mi
Alex Sun
Claude Opus 5
Kevin Mi
Kevin Mi
YC Yen-Ching Tseng
Kevin Mi
parent
f766397f2e
commit
4fa2c9c1de
+230
-45
@@ -6,6 +6,8 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.srt.utils import is_gfx95_supported
|
||||
|
||||
from ..common.utils import (
|
||||
_bitonic_merge,
|
||||
_sort_ids_ascending,
|
||||
@@ -16,6 +18,9 @@ from ..common.utils import (
|
||||
unit_scale,
|
||||
)
|
||||
|
||||
# gfx950 MI350X: per-page wins at <=2 pages/block, loses at >=8
|
||||
_MAX_PER_PAGE_SLOT_UNROLL = 2
|
||||
|
||||
|
||||
@triton.heuristics(
|
||||
{
|
||||
@@ -439,6 +444,142 @@ def _topk_index_kernel(
|
||||
tl.store(ti_ptrs, topk_idx.to(ti_ptrs.dtype.element_ty), mask=topk_mask)
|
||||
|
||||
|
||||
@triton.heuristics(
|
||||
{"BLOCK_SIZE_KD": lambda args: triton.next_power_of_2(args["qk_head_dim"])}
|
||||
)
|
||||
@triton.autotune(
|
||||
configs=[
|
||||
triton.Config({"BLOCK_SIZE_Q": 64}, num_warps=4, num_stages=2),
|
||||
triton.Config({"BLOCK_SIZE_Q": 64}, num_warps=4, num_stages=3),
|
||||
triton.Config({"BLOCK_SIZE_Q": 64}, num_warps=8, num_stages=2),
|
||||
triton.Config({"BLOCK_SIZE_Q": 128}, num_warps=8, num_stages=2),
|
||||
triton.Config({"BLOCK_SIZE_Q": 128}, num_warps=8, num_stages=3),
|
||||
],
|
||||
key=["qk_head_dim", "block_size"],
|
||||
)
|
||||
@triton.jit
|
||||
def _index_block_score_only_kernel(
|
||||
q_ptr, # Q: [total_q, h, d]
|
||||
k_cache_ptr, # K paged: [max_slots, kh, d]
|
||||
score_ptr, # Score: [h, total_q, max_seqblock]
|
||||
req_to_token_ptr, # [max_reqs, max_kv_len]
|
||||
cu_seqlens,
|
||||
seq_lens,
|
||||
prefix_lens,
|
||||
slot_ids,
|
||||
max_slots,
|
||||
num_heads,
|
||||
gqa_group_size,
|
||||
qk_head_dim,
|
||||
sm_scale,
|
||||
stride_q_n,
|
||||
stride_q_h,
|
||||
stride_q_d,
|
||||
stride_k_s,
|
||||
stride_k_h,
|
||||
stride_k_d,
|
||||
stride_s_h,
|
||||
stride_s_q,
|
||||
stride_s_k,
|
||||
stride_r2t_b,
|
||||
BLOCK_SIZE_Q: tl.constexpr,
|
||||
block_size: tl.constexpr, # sparse K block size (== 128)
|
||||
page_size: tl.constexpr, # paged-cache page size; block_size % page_size == 0
|
||||
PER_PAGE_SLOTS: tl.constexpr, # derive slots from one base slot per page
|
||||
BLOCK_SIZE_KD: tl.constexpr,
|
||||
IS_FP8: tl.constexpr,
|
||||
):
|
||||
"""Causal max index score per query and KV block, without the index-value output.
|
||||
|
||||
With PER_PAGE_SLOTS, each token's slot is derived as base_slot + in-page offset
|
||||
from one req_to_token load per page, relying on the allocator's contiguous pages.
|
||||
"""
|
||||
sm_scale_log2e = sm_scale * 1.4426950409
|
||||
pid_q, pid_bh = tl.program_id(0), tl.program_id(1)
|
||||
pid_b = pid_bh // num_heads
|
||||
pid_h = pid_bh % num_heads
|
||||
pid_kh = pid_h // gqa_group_size
|
||||
seq_start = tl.load(cu_seqlens + pid_b)
|
||||
q_len = tl.load(cu_seqlens + pid_b + 1) - seq_start
|
||||
seq_len = tl.load(seq_lens + pid_b)
|
||||
prefix_len = tl.load(prefix_lens + pid_b)
|
||||
if BLOCK_SIZE_Q * pid_q >= q_len:
|
||||
return
|
||||
sid = (tl.load(slot_ids + pid_b).to(tl.int64) + max_slots) % max_slots
|
||||
|
||||
q_ptrs = tl.make_block_ptr(
|
||||
base=q_ptr + seq_start * stride_q_n + pid_h * stride_q_h,
|
||||
shape=(q_len, qk_head_dim),
|
||||
strides=(stride_q_n, stride_q_d),
|
||||
offsets=(pid_q * BLOCK_SIZE_Q, 0),
|
||||
block_shape=(BLOCK_SIZE_Q, BLOCK_SIZE_KD),
|
||||
order=(1, 0),
|
||||
)
|
||||
q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero")
|
||||
|
||||
off_q = tl.arange(0, BLOCK_SIZE_Q) + pid_q * BLOCK_SIZE_Q + prefix_len
|
||||
off_k = tl.arange(0, block_size)
|
||||
off_kd = tl.arange(0, BLOCK_SIZE_KD)
|
||||
q_row = pid_q * BLOCK_SIZE_Q + tl.arange(0, BLOCK_SIZE_Q)
|
||||
q_store_mask = q_row < q_len
|
||||
pages_per_block: tl.constexpr = block_size // page_size
|
||||
# Within a sparse block, each token's page index and in-page offset are fixed.
|
||||
page_of = off_k // page_size # [block_size] which physical page (0..pages-1)
|
||||
in_page = off_k % page_size # [block_size] offset inside that page
|
||||
|
||||
hi = min(seq_len, prefix_len + (pid_q + 1) * BLOCK_SIZE_Q)
|
||||
for i in tl.range(0, hi, block_size):
|
||||
blk = i // block_size
|
||||
pos = i + off_k
|
||||
pos_mask = pos < seq_len
|
||||
# One base-slot load per page instead of one per token.
|
||||
if PER_PAGE_SLOTS:
|
||||
slots = tl.zeros([block_size], dtype=tl.int64)
|
||||
for p in tl.static_range(0, pages_per_block):
|
||||
page_tok0 = i + p * page_size
|
||||
base_slot = tl.load(
|
||||
req_to_token_ptr + sid * stride_r2t_b + page_tok0,
|
||||
mask=page_tok0 < seq_len,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
base_slot = (base_slot + max_slots) % max_slots
|
||||
slots = tl.where(page_of == p, base_slot + in_page, slots)
|
||||
else:
|
||||
slots = tl.load(
|
||||
req_to_token_ptr + sid * stride_r2t_b + pos,
|
||||
mask=pos_mask,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
slots = (slots + max_slots) % max_slots
|
||||
# head_dim (128) is a power of 2 == BLOCK_SIZE_KD, so the dim mask is
|
||||
# always true -> only mask the K (token) dimension.
|
||||
k = tl.load(
|
||||
k_cache_ptr
|
||||
+ slots[None, :] * stride_k_s
|
||||
+ pid_kh * stride_k_h
|
||||
+ off_kd[:, None] * stride_k_d,
|
||||
mask=pos_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
if IS_FP8:
|
||||
# Widening dequant for an fp8 index-K cache with bf16 Q; exact
|
||||
# no-op cast when Q is fp8 too. Compiled out for bf16 K.
|
||||
k = k.to(q.dtype)
|
||||
qk = tl.dot(q, k) * sm_scale_log2e
|
||||
# single fused causal + K-boundary mask
|
||||
qk = tl.where(
|
||||
(off_q[:, None] >= pos[None, :]) & pos_mask[None, :], qk, float("-inf")
|
||||
)
|
||||
score = tl.max(qk, axis=1) # [BLOCK_SIZE_Q]
|
||||
s_ptrs = (
|
||||
score_ptr
|
||||
+ pid_h * stride_s_h
|
||||
+ (seq_start + q_row) * stride_s_q
|
||||
+ blk * stride_s_k
|
||||
)
|
||||
tl.store(s_ptrs, score, mask=q_store_mask)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def flash_prefill_with_topk_index(
|
||||
q: torch.Tensor,
|
||||
@@ -467,6 +608,7 @@ def flash_prefill_with_topk_index(
|
||||
q_scale: Optional[float] = None,
|
||||
k_scale: Optional[float] = None,
|
||||
v_scale: Optional[float] = None,
|
||||
page_size: int = 1,
|
||||
):
|
||||
assert score_type in (
|
||||
"max",
|
||||
@@ -525,51 +667,94 @@ def flash_prefill_with_topk_index(
|
||||
def grid(META):
|
||||
return (triton.cdiv(max_seqlen_q, META["BLOCK_SIZE_Q"]), batch_size * num_heads)
|
||||
|
||||
_flash_attn_fwd_with_block_score_kernel[grid](
|
||||
q,
|
||||
k_cache,
|
||||
v_cache,
|
||||
sink,
|
||||
o,
|
||||
score,
|
||||
req_to_token,
|
||||
cu_seqlens,
|
||||
seq_lens,
|
||||
prefix_lens,
|
||||
slot_ids,
|
||||
max_slots,
|
||||
num_heads,
|
||||
gqa_group_size,
|
||||
qk_head_dim,
|
||||
v_head_dim,
|
||||
block_size_k,
|
||||
sm_scale,
|
||||
k_scale,
|
||||
v_scale,
|
||||
False,
|
||||
1,
|
||||
q.stride(0),
|
||||
q.stride(1),
|
||||
q.stride(2),
|
||||
k_cache.stride(0),
|
||||
k_cache.stride(1),
|
||||
k_cache.stride(2),
|
||||
v_cache.stride(0) if v_cache is not None else 0,
|
||||
v_cache.stride(1) if v_cache is not None else 0,
|
||||
v_cache.stride(2) if v_cache is not None else 0,
|
||||
sink.stride(0) if sink is not None else 0,
|
||||
sink.stride(1) if sink is not None else 0,
|
||||
o.stride(0) if o is not None else 0,
|
||||
o.stride(1) if o is not None else 0,
|
||||
o.stride(2) if o is not None else 0,
|
||||
score.stride(0),
|
||||
score.stride(1),
|
||||
score.stride(2),
|
||||
req_to_token.stride(0),
|
||||
SCORE_TYPE=score_type,
|
||||
DISABLE_INDEX_VALUE=disable_index_value,
|
||||
IS_FP8=is_fp8,
|
||||
)
|
||||
if (
|
||||
is_gfx95_supported()
|
||||
and disable_index_value
|
||||
and score_type == "max"
|
||||
and sink is None
|
||||
and q_scale in (None, 1.0)
|
||||
and k_scale == 1.0
|
||||
and qk_head_dim == 128
|
||||
and block_size_k == 128
|
||||
and page_size > 0
|
||||
and block_size_k % page_size == 0
|
||||
):
|
||||
# Source layers do not use idx_o, so run the score-only kernel.
|
||||
_index_block_score_only_kernel[grid](
|
||||
q,
|
||||
k_cache,
|
||||
score,
|
||||
req_to_token,
|
||||
cu_seqlens,
|
||||
seq_lens,
|
||||
prefix_lens,
|
||||
slot_ids,
|
||||
max_slots,
|
||||
num_heads,
|
||||
gqa_group_size,
|
||||
qk_head_dim,
|
||||
sm_scale,
|
||||
q.stride(0),
|
||||
q.stride(1),
|
||||
q.stride(2),
|
||||
k_cache.stride(0),
|
||||
k_cache.stride(1),
|
||||
k_cache.stride(2),
|
||||
score.stride(0),
|
||||
score.stride(1),
|
||||
score.stride(2),
|
||||
req_to_token.stride(0),
|
||||
block_size=block_size_k,
|
||||
page_size=page_size,
|
||||
PER_PAGE_SLOTS=(block_size_k // page_size <= _MAX_PER_PAGE_SLOT_UNROLL),
|
||||
IS_FP8=is_fp8,
|
||||
)
|
||||
else:
|
||||
_flash_attn_fwd_with_block_score_kernel[grid](
|
||||
q,
|
||||
k_cache,
|
||||
v_cache,
|
||||
sink,
|
||||
o,
|
||||
score,
|
||||
req_to_token,
|
||||
cu_seqlens,
|
||||
seq_lens,
|
||||
prefix_lens,
|
||||
slot_ids,
|
||||
max_slots,
|
||||
num_heads,
|
||||
gqa_group_size,
|
||||
qk_head_dim,
|
||||
v_head_dim,
|
||||
block_size_k,
|
||||
sm_scale,
|
||||
k_scale,
|
||||
v_scale,
|
||||
False,
|
||||
1,
|
||||
q.stride(0),
|
||||
q.stride(1),
|
||||
q.stride(2),
|
||||
k_cache.stride(0),
|
||||
k_cache.stride(1),
|
||||
k_cache.stride(2),
|
||||
v_cache.stride(0) if v_cache is not None else 0,
|
||||
v_cache.stride(1) if v_cache is not None else 0,
|
||||
v_cache.stride(2) if v_cache is not None else 0,
|
||||
sink.stride(0) if sink is not None else 0,
|
||||
sink.stride(1) if sink is not None else 0,
|
||||
o.stride(0) if o is not None else 0,
|
||||
o.stride(1) if o is not None else 0,
|
||||
o.stride(2) if o is not None else 0,
|
||||
score.stride(0),
|
||||
score.stride(1),
|
||||
score.stride(2),
|
||||
req_to_token.stride(0),
|
||||
SCORE_TYPE=score_type,
|
||||
DISABLE_INDEX_VALUE=disable_index_value,
|
||||
IS_FP8=is_fp8,
|
||||
)
|
||||
|
||||
# topk extraction kernel
|
||||
topk_idx = torch.full(
|
||||
|
||||
@@ -6,6 +6,8 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.srt.utils import is_gfx95_supported
|
||||
|
||||
from ..common.utils import (
|
||||
check_sparse_kv_fp8,
|
||||
get_cu_seqblocks,
|
||||
@@ -15,6 +17,11 @@ from ..common.utils import (
|
||||
)
|
||||
|
||||
|
||||
def _sparse_subk(block_size_k: int) -> int:
|
||||
"""Sub-tile only the measured 128-token block; keep other widths unchanged."""
|
||||
return 64 if is_gfx95_supported() and block_size_k == 128 else 0
|
||||
|
||||
|
||||
@triton.heuristics(
|
||||
{
|
||||
"BLOCK_SIZE_KD": lambda args: triton.next_power_of_2(args["qk_head_dim"]),
|
||||
@@ -32,12 +39,30 @@ from ..common.utils import (
|
||||
}
|
||||
)
|
||||
@triton.autotune(
|
||||
# Configs that fail to compile on the target arch are skipped, so widening
|
||||
# the num_warps x num_stages grid only adds candidates, never a bad kernel.
|
||||
# Keep the additional configs on gfx95; gfx942 retains the original set.
|
||||
configs=[
|
||||
triton.Config({}, num_warps=nw, num_stages=ns)
|
||||
for nw in (2, 4, 8)
|
||||
for ns in (2, 3, 4)
|
||||
# CDNA sub-tiled MFMA configs; NVIDIA's Triton rejects these launch kwargs.
|
||||
*(
|
||||
[
|
||||
triton.Config(
|
||||
{"matrix_instr_nonkdim": 16, "kpack": 2},
|
||||
num_warps=1,
|
||||
num_stages=1,
|
||||
),
|
||||
triton.Config(
|
||||
{"matrix_instr_nonkdim": 16, "kpack": 2},
|
||||
num_warps=2,
|
||||
num_stages=1,
|
||||
),
|
||||
]
|
||||
if is_gfx95_supported()
|
||||
else []
|
||||
),
|
||||
*[
|
||||
triton.Config({}, num_warps=nw, num_stages=ns)
|
||||
for nw in (2, 4, 8)
|
||||
for ns in (2, 3, 4)
|
||||
],
|
||||
],
|
||||
key=[
|
||||
"BLOCK_SIZE_Q",
|
||||
@@ -104,6 +129,7 @@ def _gqa_share_sparse_fwd_kernel(
|
||||
BLOCK_SIZE_H: tl.constexpr,
|
||||
BLOCK_SIZE_T: tl.constexpr,
|
||||
BLOCK_SIZE_QH: tl.constexpr,
|
||||
SUB_K: tl.constexpr, # CDNA KV sub-tile width (0 = disabled)
|
||||
# has sink
|
||||
HAS_SINK: tl.constexpr,
|
||||
USE_TMA: tl.constexpr,
|
||||
@@ -190,77 +216,145 @@ def _gqa_share_sparse_fwd_kernel(
|
||||
acc_o = tl.full((BLOCK_SIZE_QH, BLOCK_SIZE_VD), 0, dtype=tl.float32)
|
||||
q = tl.reshape(q, BLOCK_SIZE_QH, BLOCK_SIZE_KD)
|
||||
# sparse attention
|
||||
for i in range(real_topk):
|
||||
# get current block start index (absolute K position)
|
||||
c = tl.load(t_ptr_j).to(tl.int32) * BLOCK_SIZE_K
|
||||
t_ptr_j = t_ptr_j + stride_tk
|
||||
# paged load K via req_to_token: pos -> slot -> k_cache
|
||||
pos = c + off_n
|
||||
pos_mask = pos < seq_len
|
||||
slots = tl.load(
|
||||
req_to_token_ptr + sid * stride_r2t_b + pos,
|
||||
mask=pos_mask,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
if HAS_LOC_MAPPING:
|
||||
if SUB_K > 0:
|
||||
# Split each BLOCK_SIZE_K-token KV block into SUB_K-token sub-tiles so
|
||||
# each QK/PV MFMA is right-sized. Numerically equivalent to the dense
|
||||
# path (flash-softmax reassociation).
|
||||
NUM_SUB: tl.constexpr = BLOCK_SIZE_K // SUB_K
|
||||
for i in range(real_topk):
|
||||
c = tl.load(t_ptr_j).to(tl.int32) * BLOCK_SIZE_K
|
||||
t_ptr_j = t_ptr_j + stride_tk
|
||||
for sub_i in range(NUM_SUB):
|
||||
off_sub = tl.arange(0, SUB_K) + sub_i * SUB_K
|
||||
pos_s = c + off_sub
|
||||
pos_mask_s = pos_s < seq_len
|
||||
slots_s = tl.load(
|
||||
req_to_token_ptr + sid * stride_r2t_b + pos_s,
|
||||
mask=pos_mask_s,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
if HAS_LOC_MAPPING:
|
||||
slots_s = tl.load(
|
||||
loc_mapping_ptr + slots_s,
|
||||
mask=pos_mask_s,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
slots_s = (slots_s + max_slots) % max_slots
|
||||
k_s = tl.load(
|
||||
k_cache_ptr
|
||||
+ slots_s[None, :] * stride_ks
|
||||
+ pid_kh * stride_kh
|
||||
+ off_kd[:, None] * stride_kd,
|
||||
mask=kd_mask[:, None] & pos_mask_s[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
if IS_FP8:
|
||||
k_s = k_s.to(q.dtype)
|
||||
off_q_s = (
|
||||
tl.arange(0, BLOCK_SIZE_Q)[:, None]
|
||||
+ pid_q_j * BLOCK_SIZE_Q
|
||||
+ prefix_len
|
||||
- off_sub[None, :]
|
||||
)
|
||||
qk_s = tl.zeros(
|
||||
(BLOCK_SIZE_Q, BLOCK_SIZE_H, SUB_K), dtype=tl.float32
|
||||
)
|
||||
qk_s += tl.where(off_q_s[:, None, :] >= c, 0, float("-inf"))
|
||||
qk_s = tl.reshape(qk_s, BLOCK_SIZE_QH, SUB_K)
|
||||
qk_s += tl.dot(q, k_s) * (sm_scale_log2e * k_scale)
|
||||
qk_s += tl.where(pos_mask_s[None, :], 0, float("-inf"))
|
||||
m_ij = tl.maximum(m_i, tl.max(qk_s, axis=1))
|
||||
p_s = tl.exp2(qk_s - m_ij[:, None])
|
||||
l_ij = tl.sum(p_s, axis=1)
|
||||
acc_o = acc_o * tl.exp2(m_i - m_ij)[:, None]
|
||||
v_s = tl.load(
|
||||
v_cache_ptr
|
||||
+ slots_s[:, None] * stride_vs
|
||||
+ pid_kh * stride_vh
|
||||
+ off_vd[None, :] * stride_vd,
|
||||
mask=pos_mask_s[:, None] & vd_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
if IS_FP8:
|
||||
v_s = v_s.to(q.dtype)
|
||||
p_s = p_s.to(v_s.dtype)
|
||||
acc_o += tl.dot(p_s, v_s) * v_scale
|
||||
m_i = m_ij
|
||||
lse_i = m_ij + tl.log2(tl.exp2(lse_i - m_ij) + l_ij)
|
||||
else:
|
||||
for i in range(real_topk):
|
||||
# get current block start index (absolute K position)
|
||||
c = tl.load(t_ptr_j).to(tl.int32) * BLOCK_SIZE_K
|
||||
t_ptr_j = t_ptr_j + stride_tk
|
||||
# paged load K via req_to_token: pos -> slot -> k_cache
|
||||
pos = c + off_n
|
||||
pos_mask = pos < seq_len
|
||||
slots = tl.load(
|
||||
loc_mapping_ptr + slots,
|
||||
req_to_token_ptr + sid * stride_r2t_b + pos,
|
||||
mask=pos_mask,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
slots = (slots + max_slots) % max_slots # safety against negative
|
||||
# k shape: [BLOCK_SIZE_KD, BLOCK_SIZE_K] (transposed for tl.dot)
|
||||
k = tl.load(
|
||||
k_cache_ptr
|
||||
+ slots[None, :] * stride_ks
|
||||
+ pid_kh * stride_kh
|
||||
+ off_kd[:, None] * stride_kd,
|
||||
mask=kd_mask[:, None] & pos_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
if IS_FP8:
|
||||
# fp8 main K cache: widening cast with bf16/fp16 Q (unit-scaled
|
||||
# cache -> exact inverse dequant; k_scale covers calibrated
|
||||
# caches), no-op with fp8 Q (fp8 attn-GEMM mode) so tl.dot runs
|
||||
# fp8x8. Compiled out when the cache is bf16.
|
||||
k = k.to(q.dtype)
|
||||
# compute qk
|
||||
qk = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_K), dtype=tl.float32)
|
||||
# causal mask
|
||||
qk += tl.where(off_q_k[:, None, :] >= c, 0, float("-inf"))
|
||||
qk = tl.reshape(qk, BLOCK_SIZE_QH, BLOCK_SIZE_K)
|
||||
# [BLOCK_SIZE_QH, qk_head_dim] @ [qk_head_dim, BLOCK_SIZE_K]
|
||||
# -> [BLOCK_SIZE_QH, BLOCK_SIZE_K]
|
||||
qk += tl.dot(q, k) * (sm_scale_log2e * k_scale)
|
||||
# K boundary mask: positions beyond seq_len contribute -inf
|
||||
qk += tl.where(pos_mask[None, :], 0, float("-inf"))
|
||||
# compute m_ij and l_ij
|
||||
m_ij = tl.maximum(m_i, tl.max(qk, axis=1))
|
||||
p = tl.exp2(qk - m_ij[:, None])
|
||||
l_ij = tl.sum(p, axis=1)
|
||||
# scale acc_o
|
||||
acc_o_scale = tl.exp2(m_i - m_ij)
|
||||
acc_o = acc_o * acc_o_scale[:, None]
|
||||
# paged load V
|
||||
v = tl.load(
|
||||
v_cache_ptr
|
||||
+ slots[:, None] * stride_vs
|
||||
+ pid_kh * stride_vh
|
||||
+ off_vd[None, :] * stride_vd,
|
||||
mask=pos_mask[:, None] & vd_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
if IS_FP8:
|
||||
# Cast V to the compute dtype: widening with bf16/fp16 Q (so
|
||||
# `p.to(v.dtype)` keeps P in the compute dtype), no-op with fp8
|
||||
# Q where P is quantized to e4m3 for the fp8 PV MMA — the same
|
||||
# accuracy contract as fmha_sm100's fp8 kernel.
|
||||
v = v.to(q.dtype)
|
||||
p = p.to(v.dtype)
|
||||
acc_o += tl.dot(p, v) * v_scale
|
||||
# update statistics
|
||||
m_i = m_ij
|
||||
lse_i = m_ij + tl.log2(tl.exp2(lse_i - m_ij) + l_ij)
|
||||
if HAS_LOC_MAPPING:
|
||||
slots = tl.load(
|
||||
loc_mapping_ptr + slots,
|
||||
mask=pos_mask,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
slots = (slots + max_slots) % max_slots # safety against negative
|
||||
# k shape: [BLOCK_SIZE_KD, BLOCK_SIZE_K] (transposed for tl.dot)
|
||||
k = tl.load(
|
||||
k_cache_ptr
|
||||
+ slots[None, :] * stride_ks
|
||||
+ pid_kh * stride_kh
|
||||
+ off_kd[:, None] * stride_kd,
|
||||
mask=kd_mask[:, None] & pos_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
if IS_FP8:
|
||||
# fp8 main K cache: widening cast with bf16/fp16 Q (unit-scaled
|
||||
# cache -> exact inverse dequant; k_scale covers calibrated
|
||||
# caches), no-op with fp8 Q (fp8 attn-GEMM mode) so tl.dot runs
|
||||
# fp8x8. Compiled out when the cache is bf16.
|
||||
k = k.to(q.dtype)
|
||||
# compute qk
|
||||
qk = tl.zeros(
|
||||
(BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_K), dtype=tl.float32
|
||||
)
|
||||
# causal mask
|
||||
qk += tl.where(off_q_k[:, None, :] >= c, 0, float("-inf"))
|
||||
qk = tl.reshape(qk, BLOCK_SIZE_QH, BLOCK_SIZE_K)
|
||||
# [BLOCK_SIZE_QH, qk_head_dim] @ [qk_head_dim, BLOCK_SIZE_K]
|
||||
# -> [BLOCK_SIZE_QH, BLOCK_SIZE_K]
|
||||
qk += tl.dot(q, k) * (sm_scale_log2e * k_scale)
|
||||
# K boundary mask: positions beyond seq_len contribute -inf
|
||||
qk += tl.where(pos_mask[None, :], 0, float("-inf"))
|
||||
# compute m_ij and l_ij
|
||||
m_ij = tl.maximum(m_i, tl.max(qk, axis=1))
|
||||
p = tl.exp2(qk - m_ij[:, None])
|
||||
l_ij = tl.sum(p, axis=1)
|
||||
# scale acc_o
|
||||
acc_o_scale = tl.exp2(m_i - m_ij)
|
||||
acc_o = acc_o * acc_o_scale[:, None]
|
||||
# paged load V
|
||||
v = tl.load(
|
||||
v_cache_ptr
|
||||
+ slots[:, None] * stride_vs
|
||||
+ pid_kh * stride_vh
|
||||
+ off_vd[None, :] * stride_vd,
|
||||
mask=pos_mask[:, None] & vd_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
if IS_FP8:
|
||||
# Cast V to the compute dtype: widening with bf16/fp16 Q (so
|
||||
# `p.to(v.dtype)` keeps P in the compute dtype), no-op with fp8
|
||||
# Q where P is quantized to e4m3 for the fp8 PV MMA — the same
|
||||
# accuracy contract as fmha_sm100's fp8 kernel.
|
||||
v = v.to(q.dtype)
|
||||
p = p.to(v.dtype)
|
||||
acc_o += tl.dot(p, v) * v_scale
|
||||
# update statistics
|
||||
m_i = m_ij
|
||||
lse_i = m_ij + tl.log2(tl.exp2(lse_i - m_ij) + l_ij)
|
||||
# final scale
|
||||
acc_o = acc_o * tl.exp2(m_i - lse_i)[:, None]
|
||||
# save output
|
||||
@@ -385,6 +479,7 @@ def flash_prefill_with_gqa_share_sparse(
|
||||
o.stride(2),
|
||||
req_to_token.stride(0),
|
||||
BLOCK_SIZE_Q=BLOCK_SIZE_Q,
|
||||
SUB_K=_sparse_subk(BLOCK_SIZE_K),
|
||||
BLOCK_SIZE_K=BLOCK_SIZE_K,
|
||||
USE_TMA=use_tma,
|
||||
IS_FP8=is_fp8,
|
||||
|
||||
@@ -1698,6 +1698,9 @@ class Envs:
|
||||
# 2 is the accuracy-safe default: higher values reuse staler selections
|
||||
# in the skip layers.
|
||||
SGLANG_MINIMAX_M3_INDEX_TOPK_FREQ = EnvInt(2)
|
||||
# Run the sparse prefill main attention through AITER's Gluon paged attention
|
||||
# instead of the Triton kernel. Unsupported cases fall back to Triton.
|
||||
SGLANG_OPT_USE_MINIMAX_GLUON_PREFILL = EnvBool(True)
|
||||
# MiniMax M3 NPU prefill MAIN-attention: route the sparse main attention through
|
||||
# the native Ascend FA op `torch.ops.npu.npu_fused_infer_attention_score` (FIA)
|
||||
# with a per-query CUSTOM block_table
|
||||
|
||||
@@ -1570,6 +1570,7 @@ class MiniMaxSparseAttnBackend(AttentionBackend):
|
||||
disable_index_value=disable_value,
|
||||
use_msa=self.use_msa,
|
||||
seqlens_cpu=forward_batch.extend_seq_lens_cpu,
|
||||
seq_lens_cpu=forward_batch.seq_lens_cpu,
|
||||
cu_seqblocks_q=cu_seqblocks_q,
|
||||
max_seqblock_q=max_seqblock_q,
|
||||
all_seqblock_q=all_seqblock_q,
|
||||
@@ -1579,6 +1580,7 @@ class MiniMaxSparseAttnBackend(AttentionBackend):
|
||||
idx_q_scale=layer.idx_q_scale_float,
|
||||
idx_k_scale=layer.idx_k_scale_float,
|
||||
idx_v_scale=layer.idx_v_scale_float,
|
||||
page_size=self.page_size,
|
||||
cached_topk_idx=cached_topk_idx,
|
||||
return_topk_idx=want_topk,
|
||||
loc_mapping=self._loc_mapping,
|
||||
|
||||
@@ -0,0 +1,492 @@
|
||||
# Copyright 2025 SGLang Team
|
||||
# Ported from
|
||||
# https://github.com/alexsun07/sglang/blob/191624ff8a6d8e9fc31b9363d2138c16db66f124/python/sglang/srt/layers/attention/minimax_sparse_ops/atom_prefill.py
|
||||
"""MiniMax-M3 sparse prefill using AITER's Gluon paged attention.
|
||||
|
||||
The main KV pool remains NHD ``[max_slots, 1, head_dim]``. Each request's
|
||||
context is gathered into persistent SHUFFLE 5D scratch pages:
|
||||
|
||||
- key scratch: [num_pages, 1, head_dim // x, GLUON_PAGE_SIZE, x]
|
||||
- value scratch: [num_pages, 1, GLUON_PAGE_SIZE // x, head_dim, x] (transposed)
|
||||
|
||||
with ``x = 16 // dtype.itemsize``. Each request occupies a contiguous,
|
||||
position-ordered page range rounded up to whole sparse blocks.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from itertools import accumulate
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.srt.layers.attention.aiter_utils import (
|
||||
get_recommended_splits,
|
||||
pa_decode_gluon,
|
||||
)
|
||||
|
||||
SPARSE_BLOCK_SIZE = 128
|
||||
# The Gluon PS kernel takes kv_block_size 16 or 64; 64 halves the block-table width.
|
||||
GLUON_PAGE_SIZE = 64
|
||||
PAGES_PER_BLOCK = SPARSE_BLOCK_SIZE // GLUON_PAGE_SIZE
|
||||
HEAD_DIM = 128
|
||||
# Scratch grows in 1024-page steps (16 KiB/page/buffer -> 16 MiB granularity).
|
||||
_SCRATCH_GROW_PAGES = 1024
|
||||
_PAGE_ELEMS = HEAD_DIM * GLUON_PAGE_SIZE
|
||||
|
||||
# 512 MiB per buffer at bf16 (~2.1M context tokens per forward); past it the caller
|
||||
# falls back to the Triton kernel, which reads the pool in place.
|
||||
_MAX_SCRATCH_PAGES = (512 * 1024 * 1024) // (_PAGE_ELEMS * 2)
|
||||
|
||||
|
||||
class GluonPrefillUnavailableError(RuntimeError):
|
||||
"""Raised when the scratch cannot hold this forward; the caller falls back to Triton."""
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _gather_nhd_to_shuffle_kernel(
|
||||
k_pool_ptr, # [max_slots, 1, HEAD_DIM] (NHD main pool, this layer)
|
||||
v_pool_ptr, # [max_slots, 1, HEAD_DIM]
|
||||
k_out_ptr, # [num_pages, 1, HEAD_DIM // X, PAGE, X] contiguous
|
||||
v_out_ptr, # [num_pages, 1, PAGE // X, HEAD_DIM, X] contiguous
|
||||
req_to_token_ptr, # [max_reqs, max_kv_len]
|
||||
req_pool_indices_ptr, # [batch]
|
||||
seq_lens_ptr, # [batch] total K length (prefix + current chunk)
|
||||
page_req_ptr, # [total_pages] -> batch index owning this page
|
||||
page_pos_ptr, # [total_pages] -> page index within the request
|
||||
stride_k_slot,
|
||||
stride_v_slot,
|
||||
stride_r2t_b,
|
||||
HEAD_DIM_C: tl.constexpr,
|
||||
PAGE: tl.constexpr,
|
||||
X: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
req = tl.load(page_req_ptr + pid)
|
||||
ppos = tl.load(page_pos_ptr + pid)
|
||||
pool_id = tl.load(req_pool_indices_ptr + req).to(tl.int64)
|
||||
seq_len = tl.load(seq_lens_ptr + req)
|
||||
|
||||
off_t = tl.arange(0, PAGE)
|
||||
tok = ppos * PAGE + off_t
|
||||
tmask = tok < seq_len
|
||||
slots = tl.load(
|
||||
req_to_token_ptr + pool_id * stride_r2t_b + tok, mask=tmask, other=0
|
||||
).to(tl.int64)
|
||||
|
||||
off_d = tl.arange(0, HEAD_DIM_C)
|
||||
# Coalesced NHD reads: each token's head_dim vector is contiguous.
|
||||
k_tile = tl.load(
|
||||
k_pool_ptr + slots[:, None] * stride_k_slot + off_d[None, :],
|
||||
mask=tmask[:, None],
|
||||
other=0.0,
|
||||
)
|
||||
v_tile = tl.load(
|
||||
v_pool_ptr + slots[:, None] * stride_v_slot + off_d[None, :],
|
||||
mask=tmask[:, None],
|
||||
other=0.0,
|
||||
)
|
||||
|
||||
page_base = pid.to(tl.int64) * (HEAD_DIM_C * PAGE)
|
||||
# K shuffle page [1, D//X, PAGE, X]: elem(d, t) at (d//X)*(PAGE*X) + t*X + d%X
|
||||
k_off = (
|
||||
(off_d[None, :] // X) * (PAGE * X) + off_t[:, None] * X + (off_d[None, :] % X)
|
||||
)
|
||||
# V transposed page [1, PAGE//X, D, X]: elem(d, t) at (t//X)*(D*X) + d*X + t%X
|
||||
v_off = (
|
||||
(off_t[:, None] // X) * (HEAD_DIM_C * X)
|
||||
+ off_d[None, :] * X
|
||||
+ (off_t[:, None] % X)
|
||||
)
|
||||
# Unmasked: tiles are zero past seq_len, so no previous layer's data survives.
|
||||
tl.store(k_out_ptr + page_base + k_off, k_tile)
|
||||
tl.store(v_out_ptr + page_base + v_off, v_tile)
|
||||
|
||||
|
||||
# Grow-only (device, dtype) -> (k_flat, v_flat); layers run in sequence and share it.
|
||||
_SCRATCH: dict = {}
|
||||
|
||||
|
||||
def _get_scratch(
|
||||
device: torch.device, dtype: torch.dtype, total_pages: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
key = (device, dtype)
|
||||
entry = _SCRATCH.get(key)
|
||||
needed = total_pages * _PAGE_ELEMS
|
||||
if entry is None or entry[0].numel() < needed:
|
||||
cap_pages = (
|
||||
(total_pages + _SCRATCH_GROW_PAGES - 1) // _SCRATCH_GROW_PAGES
|
||||
) * _SCRATCH_GROW_PAGES
|
||||
# Grows after the KV pool claimed its budget; report OOM as unsupported so
|
||||
# the caller falls back to the scratch-free Triton kernel.
|
||||
try:
|
||||
entry = (
|
||||
torch.empty(cap_pages * _PAGE_ELEMS, dtype=dtype, device=device),
|
||||
torch.empty(cap_pages * _PAGE_ELEMS, dtype=dtype, device=device),
|
||||
)
|
||||
except torch.cuda.OutOfMemoryError as exc:
|
||||
raise GluonPrefillUnavailableError(
|
||||
f"gluon prefill scratch of {cap_pages} pages "
|
||||
f"({cap_pages * _PAGE_ELEMS * dtype.itemsize * 2 >> 20} MiB) "
|
||||
"does not fit; lower --mem-fraction-static"
|
||||
) from exc
|
||||
_SCRATCH[key] = entry
|
||||
return entry
|
||||
|
||||
|
||||
def _gather_context_to_shuffle(
|
||||
k_cache: torch.Tensor,
|
||||
v_cache: torch.Tensor,
|
||||
req_to_token: torch.Tensor,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
page_req: torch.Tensor,
|
||||
page_pos: torch.Tensor,
|
||||
total_pages: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Gather this batch's context tokens (NHD pool slots) into SHUFFLE 5D pages."""
|
||||
x = 16 // k_cache.dtype.itemsize # 8 for bf16/fp16
|
||||
k_flat, v_flat = _get_scratch(k_cache.device, k_cache.dtype, total_pages)
|
||||
_gather_nhd_to_shuffle_kernel[(total_pages,)](
|
||||
k_cache,
|
||||
v_cache,
|
||||
k_flat,
|
||||
v_flat,
|
||||
req_to_token,
|
||||
req_pool_indices,
|
||||
seq_lens,
|
||||
page_req,
|
||||
page_pos,
|
||||
k_cache.stride(0),
|
||||
v_cache.stride(0),
|
||||
req_to_token.stride(0),
|
||||
HEAD_DIM_C=HEAD_DIM,
|
||||
PAGE=GLUON_PAGE_SIZE,
|
||||
X=x,
|
||||
)
|
||||
k5 = k_flat[: total_pages * _PAGE_ELEMS].view(
|
||||
total_pages, 1, HEAD_DIM // x, GLUON_PAGE_SIZE, x
|
||||
)
|
||||
v5 = v_flat[: total_pages * _PAGE_ELEMS].view(
|
||||
total_pages, 1, GLUON_PAGE_SIZE // x, HEAD_DIM, x
|
||||
)
|
||||
return k5, v5
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _build_gluon_sparse_bt_prefill_kernel(
|
||||
topk_ptr, # [Hkv=1, total_q, topk] int32
|
||||
req_id_ptr, # [total_q] int32
|
||||
abs_pos_ptr, # [total_q] int32
|
||||
page_start_ptr, # [batch] int32: first scratch page of each request
|
||||
sparse_bt_ptr, # [total_q, topk * pages_per_block] int32
|
||||
sparse_ctx_ptr, # [total_q] int32
|
||||
max_topk,
|
||||
stride_topk_n,
|
||||
stride_topk_t,
|
||||
stride_sparse_bt_n,
|
||||
sparse_block_size: tl.constexpr,
|
||||
pages_per_block: tl.constexpr,
|
||||
BLOCK_SIZE_T: tl.constexpr,
|
||||
):
|
||||
pid_n = tl.program_id(0)
|
||||
|
||||
req_id = tl.load(req_id_ptr + pid_n)
|
||||
base_page = tl.load(page_start_ptr + req_id)
|
||||
abs_pos = tl.load(abs_pos_ptr + pid_n)
|
||||
causal_len = abs_pos + 1
|
||||
self_blk = abs_pos // sparse_block_size
|
||||
|
||||
topk_row = topk_ptr + pid_n * stride_topk_n
|
||||
out_row = sparse_bt_ptr + pid_n * stride_sparse_bt_n
|
||||
off_t = tl.arange(0, BLOCK_SIZE_T)
|
||||
blk = tl.load(topk_row + off_t * stride_topk_t, mask=off_t < max_topk, other=-1)
|
||||
|
||||
# Causal: a query at abs_pos may attend its own block and all earlier ones.
|
||||
valid = (blk >= 0) & (blk <= self_blk)
|
||||
is_tail = valid & (blk == self_blk)
|
||||
is_full = valid & (blk < self_blk)
|
||||
n_full = tl.sum(is_full.to(tl.int32), axis=0)
|
||||
n_valid = tl.sum(valid.to(tl.int32), axis=0)
|
||||
# Full blocks first, the partial causal tail last, so the walk matches sparse_ctx.
|
||||
earlier_full = tl.cumsum(is_full.to(tl.int32), axis=0) - is_full.to(tl.int32)
|
||||
sparse_slot = tl.where(is_full, earlier_full, n_full)
|
||||
|
||||
# Scratch pages for each request are contiguous and position-ordered.
|
||||
dst_base = sparse_slot * pages_per_block
|
||||
for j in tl.static_range(0, pages_per_block):
|
||||
phys_page = base_page + blk * pages_per_block + j
|
||||
tl.store(out_row + dst_base + j, phys_page, mask=valid)
|
||||
|
||||
n_used = n_valid * pages_per_block
|
||||
off_w = tl.arange(0, BLOCK_SIZE_T * pages_per_block)
|
||||
# BLOCK_SIZE_T is next_power_of_2(topk), so off_w overshoots the row whenever
|
||||
# topk is not a power of two -- bound the tail fill by the real row width.
|
||||
tl.store(
|
||||
out_row + off_w,
|
||||
tl.zeros_like(off_w),
|
||||
mask=(off_w >= n_used) & (off_w < max_topk * pages_per_block),
|
||||
)
|
||||
|
||||
# Effective KV length the kernel will walk across the packed pages.
|
||||
tail_tokens = causal_len - self_blk * sparse_block_size
|
||||
has_tail = tl.sum(is_tail.to(tl.int32), axis=0) > 0
|
||||
ctx_len = n_full * sparse_block_size + tl.where(has_tail, tail_tokens, 0)
|
||||
ctx_len = tl.where(
|
||||
has_tail, ctx_len, tl.minimum(n_valid * sparse_block_size, causal_len)
|
||||
)
|
||||
tl.store(sparse_ctx_ptr + pid_n, ctx_len)
|
||||
|
||||
|
||||
_META_CACHE: Optional[tuple] = None
|
||||
|
||||
_BT_OUT_CACHE: Optional[tuple] = None
|
||||
|
||||
|
||||
def _build_gluon_prefill_meta(
|
||||
cu_seqlens: torch.Tensor,
|
||||
prefix_lens: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
seq_lens_cpu: torch.Tensor,
|
||||
total_q: int,
|
||||
):
|
||||
global _META_CACHE, _BT_OUT_CACHE
|
||||
if _META_CACHE is not None:
|
||||
(k_cu, k_prefix, k_seq, k_total_q), cached = _META_CACHE
|
||||
if (
|
||||
k_cu is cu_seqlens
|
||||
and k_prefix is prefix_lens
|
||||
and k_seq is seq_lens
|
||||
and k_total_q == total_q
|
||||
):
|
||||
return cached
|
||||
|
||||
device = cu_seqlens.device
|
||||
pos = torch.arange(total_q, dtype=torch.int32, device=device)
|
||||
req_id = torch.searchsorted(cu_seqlens[1:].contiguous(), pos, right=True).to(
|
||||
torch.int32
|
||||
)
|
||||
abs_pos = (prefix_lens[req_id] + (pos - cu_seqlens[req_id])).to(torch.int32)
|
||||
|
||||
# Host-side page layout: each request's context span rounds up to whole
|
||||
# sparse blocks so every emitted page id stays inside its own span.
|
||||
pages_per_req = [
|
||||
((kv_len + SPARSE_BLOCK_SIZE - 1) // SPARSE_BLOCK_SIZE) * PAGES_PER_BLOCK
|
||||
for kv_len in seq_lens_cpu.tolist()
|
||||
]
|
||||
total_pages = sum(pages_per_req)
|
||||
page_start = torch.tensor(
|
||||
list(accumulate(pages_per_req, initial=0))[:-1],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
repeats = torch.tensor(pages_per_req, dtype=torch.int64, device=device)
|
||||
page_req = torch.repeat_interleave(
|
||||
torch.arange(len(pages_per_req), dtype=torch.int32, device=device),
|
||||
repeats,
|
||||
output_size=total_pages,
|
||||
)
|
||||
page_pos = (
|
||||
torch.arange(total_pages, dtype=torch.int32, device=device)
|
||||
- page_start[page_req.long()]
|
||||
).to(torch.int32)
|
||||
|
||||
meta = (req_id, abs_pos, page_start, page_req, page_pos, total_pages)
|
||||
_BT_OUT_CACHE = None # new forward: no block table can be valid anymore
|
||||
_META_CACHE = ((cu_seqlens, prefix_lens, seq_lens, total_q), meta)
|
||||
return meta
|
||||
|
||||
|
||||
def _build_gluon_sparse_bt_prefill(
|
||||
topk_idx: torch.Tensor,
|
||||
req_id: torch.Tensor,
|
||||
abs_pos: torch.Tensor,
|
||||
page_start: torch.Tensor,
|
||||
sparse_block_size: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# topk_idx: [num_kv_heads=1, total_q, topk]
|
||||
total_q = topk_idx.shape[1]
|
||||
topk = topk_idx.shape[2]
|
||||
|
||||
global _BT_OUT_CACHE
|
||||
if _BT_OUT_CACHE is not None:
|
||||
(k_topk_idx, k_total_q, k_topk, k_block_size), out_cached = _BT_OUT_CACHE
|
||||
if (
|
||||
k_topk_idx is topk_idx
|
||||
and k_total_q == total_q
|
||||
and k_topk == topk
|
||||
and k_block_size == sparse_block_size
|
||||
):
|
||||
return out_cached
|
||||
|
||||
sparse_bt = torch.empty(
|
||||
(total_q, topk * PAGES_PER_BLOCK),
|
||||
dtype=torch.int32,
|
||||
device=topk_idx.device,
|
||||
)
|
||||
sparse_ctx = torch.empty((total_q,), dtype=torch.int32, device=topk_idx.device)
|
||||
_build_gluon_sparse_bt_prefill_kernel[(total_q,)](
|
||||
topk_idx,
|
||||
req_id,
|
||||
abs_pos,
|
||||
page_start,
|
||||
sparse_bt,
|
||||
sparse_ctx,
|
||||
topk,
|
||||
topk_idx.stride(1),
|
||||
topk_idx.stride(2),
|
||||
sparse_bt.stride(0),
|
||||
sparse_block_size=sparse_block_size,
|
||||
pages_per_block=PAGES_PER_BLOCK,
|
||||
BLOCK_SIZE_T=triton.next_power_of_2(topk),
|
||||
)
|
||||
_BT_OUT_CACHE = (
|
||||
(topk_idx, total_q, topk, sparse_block_size),
|
||||
(sparse_bt, sparse_ctx),
|
||||
)
|
||||
return sparse_bt, sparse_ctx
|
||||
|
||||
|
||||
def _unit_or_none(scale) -> bool:
|
||||
return scale is None or scale == 1.0
|
||||
|
||||
|
||||
def can_use_gluon_prefill(
|
||||
q: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
v_cache: torch.Tensor,
|
||||
sink: Optional[torch.Tensor],
|
||||
block_size_k: int,
|
||||
seq_lens_cpu: Optional[torch.Tensor],
|
||||
q_scale: Optional[float] = None,
|
||||
k_scale: Optional[float] = None,
|
||||
v_scale: Optional[float] = None,
|
||||
) -> bool:
|
||||
"""Cheap static gate; the scratch size is checked per forward."""
|
||||
if (
|
||||
pa_decode_gluon is None
|
||||
or get_recommended_splits is None
|
||||
or sink is not None
|
||||
or seq_lens_cpu is None
|
||||
or block_size_k != SPARSE_BLOCK_SIZE
|
||||
):
|
||||
return False
|
||||
# The gather requires a contiguous, single-head NHD pool. Calibrated and
|
||||
# unsupported cache formats stay on the Triton path.
|
||||
return (
|
||||
q.dim() == 3
|
||||
and q.shape[-1] == HEAD_DIM
|
||||
and k_cache.dim() == 3
|
||||
and v_cache.dim() == 3
|
||||
and k_cache.shape[1] == 1
|
||||
and v_cache.shape[1] == 1
|
||||
and k_cache.shape[2] == HEAD_DIM
|
||||
and v_cache.shape[2] == HEAD_DIM
|
||||
and q.dtype in (torch.bfloat16, torch.float16)
|
||||
and k_cache.dtype == q.dtype
|
||||
and v_cache.dtype == q.dtype
|
||||
and k_cache.stride(2) == 1
|
||||
and v_cache.stride(2) == 1
|
||||
and _unit_or_none(q_scale)
|
||||
and _unit_or_none(k_scale)
|
||||
and _unit_or_none(v_scale)
|
||||
)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def gluon_sparse_prefill(
|
||||
q: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
v_cache: torch.Tensor,
|
||||
topk_idx: torch.Tensor,
|
||||
req_to_token: torch.Tensor,
|
||||
req_pool_indices: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
prefix_lens: torch.Tensor,
|
||||
seq_lens_cpu: torch.Tensor,
|
||||
block_size_k: int,
|
||||
sm_scale: Optional[float] = None,
|
||||
) -> torch.Tensor:
|
||||
"""Run Gluon sparse prefill over the NHD KV pool via the SHUFFLE scratch.
|
||||
|
||||
Raises GluonPrefillUnavailableError when the scratch cannot hold the context.
|
||||
"""
|
||||
if sm_scale is None:
|
||||
sm_scale = q.shape[-1] ** -0.5
|
||||
|
||||
q = q.contiguous()
|
||||
total_q, num_q_heads, head_dim = q.shape
|
||||
if total_q == 0:
|
||||
return torch.empty_like(q)
|
||||
assert topk_idx.shape[0] == 1 and topk_idx.shape[1] == total_q, (
|
||||
f"gluon prefill expects topk_idx [1, total_q, topk], got {tuple(topk_idx.shape)}"
|
||||
)
|
||||
batch = cu_seqlens.shape[0] - 1
|
||||
assert seq_lens_cpu.numel() == batch and seq_lens.shape[0] == batch, (
|
||||
f"gluon prefill batch mismatch: cu_seqlens batch {batch}, "
|
||||
f"seq_lens {tuple(seq_lens.shape)}, seq_lens_cpu {seq_lens_cpu.numel()}"
|
||||
)
|
||||
|
||||
req_id, abs_pos, page_start, page_req, page_pos, total_pages = (
|
||||
_build_gluon_prefill_meta(
|
||||
cu_seqlens, prefix_lens, seq_lens, seq_lens_cpu, total_q
|
||||
)
|
||||
)
|
||||
if total_pages > _MAX_SCRATCH_PAGES:
|
||||
raise GluonPrefillUnavailableError(
|
||||
f"gluon prefill context span too large for scratch: {total_pages} pages "
|
||||
f"> cap {_MAX_SCRATCH_PAGES}"
|
||||
)
|
||||
|
||||
# Gather the current layer's prefix and current chunk into SHUFFLE 5D views.
|
||||
k_cache, v_cache = _gather_context_to_shuffle(
|
||||
k_cache,
|
||||
v_cache,
|
||||
req_to_token,
|
||||
req_pool_indices,
|
||||
seq_lens,
|
||||
page_req,
|
||||
page_pos,
|
||||
total_pages,
|
||||
)
|
||||
|
||||
sparse_bt, sparse_ctx = _build_gluon_sparse_bt_prefill(
|
||||
topk_idx, req_id, abs_pos, page_start, block_size_k
|
||||
)
|
||||
|
||||
out = torch.empty_like(q)
|
||||
num_seqs = total_q
|
||||
ctx_part = 256
|
||||
max_part_num = get_recommended_splits(num_seqs, 1)
|
||||
intermediate_shape = (num_seqs, 1, max_part_num, num_q_heads)
|
||||
exp_sums = torch.empty(intermediate_shape, dtype=torch.float32, device=q.device)
|
||||
max_logits = torch.empty_like(exp_sums)
|
||||
temporary_output = torch.empty(
|
||||
(*intermediate_shape, head_dim), dtype=q.dtype, device=q.device
|
||||
)
|
||||
|
||||
pa_decode_gluon(
|
||||
output=out,
|
||||
query=q,
|
||||
key_cache=k_cache,
|
||||
value_cache=v_cache,
|
||||
context_lengths=sparse_ctx,
|
||||
block_tables=sparse_bt,
|
||||
softmax_scale=sm_scale,
|
||||
query_length=1,
|
||||
max_context_partition_num=max_part_num,
|
||||
context_partition_size=ctx_part,
|
||||
compute_type=q.dtype,
|
||||
key_scale=None,
|
||||
value_scale=None,
|
||||
exp_sums=exp_sums,
|
||||
max_logits=max_logits,
|
||||
temporary_output=temporary_output,
|
||||
sinks=None,
|
||||
sliding_window=0,
|
||||
ps=True,
|
||||
)
|
||||
return out
|
||||
@@ -19,9 +19,14 @@ from sglang.kernels.ops.attention.minimax_sparse.prefill.flash_with_topk_idx imp
|
||||
from sglang.kernels.ops.attention.minimax_sparse.prefill.topk_sparse import (
|
||||
flash_prefill_with_gqa_share_sparse,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.utils import is_gfx95_supported, is_hip
|
||||
|
||||
_use_aiter_gfx95 = envs.SGLANG_USE_AITER.get() and is_hip() and is_gfx95_supported()
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_msa_fallback_warned = False
|
||||
_gluon_fallback_warned = False
|
||||
|
||||
|
||||
def _warn_msa_fallback(err: Exception) -> None:
|
||||
@@ -35,6 +40,18 @@ def _warn_msa_fallback(err: Exception) -> None:
|
||||
_msa_fallback_warned = True
|
||||
|
||||
|
||||
def _warn_gluon_fallback(msg: str) -> None:
|
||||
global _gluon_fallback_warned
|
||||
if _gluon_fallback_warned:
|
||||
return
|
||||
logger.warning(
|
||||
"MiniMax Gluon sparse prefill is unavailable (%s); falling back to Triton "
|
||||
"sparse attention.",
|
||||
msg,
|
||||
)
|
||||
_gluon_fallback_warned = True
|
||||
|
||||
|
||||
def minimax_sparse_prefill(
|
||||
q: torch.Tensor, # [total_extend_tokens, num_q_heads, qk_head_dim]
|
||||
k_cache: torch.Tensor, # [max_slots, num_kv_heads, head_dim] (paged main)
|
||||
@@ -76,6 +93,8 @@ def minimax_sparse_prefill(
|
||||
cached_topk_idx: Optional[torch.Tensor] = None,
|
||||
return_topk_idx: bool = False,
|
||||
loc_mapping: Optional[torch.Tensor] = None,
|
||||
page_size: int = 1,
|
||||
seq_lens_cpu: Optional[torch.Tensor] = None,
|
||||
):
|
||||
"""Run MiniMax-M3 sparse prefill.
|
||||
|
||||
@@ -91,6 +110,10 @@ def minimax_sparse_prefill(
|
||||
kernels. Supplying them avoids recomputing the same block layout twice.
|
||||
``seqlens_cpu`` (host copy of ``torch.diff(cu_seqlens)``) is forwarded to
|
||||
``get_cu_seqblocks`` to avoid a per-layer device sync when it recomputes.
|
||||
|
||||
``seq_lens_cpu`` (host copy of ``seq_lens``, i.e. prefix + current chunk
|
||||
per request) is only consumed by the env-gated Gluon prefill path for
|
||||
sync-free scratch-page sizing; ``None`` disables that path.
|
||||
"""
|
||||
if cu_seqblocks_q is None or max_seqblock_q is None or all_seqblock_q is None:
|
||||
cu_seqblocks_q, max_seqblock_q, all_seqblock_q, _, _, _ = get_cu_seqblocks(
|
||||
@@ -129,6 +152,7 @@ def minimax_sparse_prefill(
|
||||
cu_seqblocks_q=cu_seqblocks_q,
|
||||
max_seqblock_q=max_seqblock_q,
|
||||
all_seqblock_q=all_seqblock_q,
|
||||
page_size=page_size,
|
||||
q_scale=idx_q_scale,
|
||||
k_scale=idx_k_scale,
|
||||
v_scale=idx_v_scale,
|
||||
@@ -144,10 +168,53 @@ def minimax_sparse_prefill(
|
||||
|
||||
# Reduced top-k cached by the caller for subsequent skip layers.
|
||||
reduced_topk_idx = topk_idx
|
||||
# Step 3: Sparse attention using topk index (main head). The MSA path only
|
||||
# replaces this step; the indexer above is unchanged. MSA has no attn-sink
|
||||
# input, so keep the Triton path when sink is present.
|
||||
if use_msa and sink is None and loc_mapping is None:
|
||||
# Step 3: Sparse attention using topk index (main head). The Gluon and
|
||||
# MSA paths only replace this step; the indexer above is unchanged. MSA has
|
||||
# no attn-sink input, so keep the Triton path when sink is present.
|
||||
o = None
|
||||
if (
|
||||
_use_aiter_gfx95
|
||||
and envs.SGLANG_OPT_USE_MINIMAX_GLUON_PREFILL.get()
|
||||
# the scratch gather reads pool slots without the HiSparse remap
|
||||
and loc_mapping is None
|
||||
):
|
||||
from .gluon_prefill import (
|
||||
GluonPrefillUnavailableError,
|
||||
can_use_gluon_prefill,
|
||||
gluon_sparse_prefill,
|
||||
)
|
||||
|
||||
if can_use_gluon_prefill(
|
||||
q,
|
||||
k_cache,
|
||||
v_cache,
|
||||
sink,
|
||||
block_size_k,
|
||||
seq_lens_cpu,
|
||||
q_scale,
|
||||
k_scale,
|
||||
v_scale,
|
||||
):
|
||||
try:
|
||||
o = gluon_sparse_prefill(
|
||||
q=q,
|
||||
k_cache=k_cache,
|
||||
v_cache=v_cache,
|
||||
topk_idx=topk_idx,
|
||||
req_to_token=req_to_token,
|
||||
req_pool_indices=slot_ids,
|
||||
cu_seqlens=cu_seqlens,
|
||||
seq_lens=seq_lens,
|
||||
prefix_lens=prefix_lens,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
block_size_k=block_size_k,
|
||||
sm_scale=sm_scale,
|
||||
)
|
||||
except GluonPrefillUnavailableError as err:
|
||||
_warn_gluon_fallback(str(err))
|
||||
else:
|
||||
_warn_gluon_fallback("unsupported batch/cache layout or dtype")
|
||||
if o is None and use_msa and sink is None and loc_mapping is None:
|
||||
from .msa import MSAUnavailableError, msa_sparse_prefill_main
|
||||
|
||||
try:
|
||||
@@ -191,7 +258,7 @@ def minimax_sparse_prefill(
|
||||
v_scale=v_scale,
|
||||
loc_mapping=loc_mapping,
|
||||
)
|
||||
else:
|
||||
elif o is None:
|
||||
o = flash_prefill_with_gqa_share_sparse(
|
||||
q=q,
|
||||
k_cache=k_cache,
|
||||
|
||||
Reference in New Issue
Block a user