diff --git a/python/sglang/kernels/ops/attention/minimax_sparse/prefill/flash_with_topk_idx.py b/python/sglang/kernels/ops/attention/minimax_sparse/prefill/flash_with_topk_idx.py index 932a08e0e7..04329ef419 100644 --- a/python/sglang/kernels/ops/attention/minimax_sparse/prefill/flash_with_topk_idx.py +++ b/python/sglang/kernels/ops/attention/minimax_sparse/prefill/flash_with_topk_idx.py @@ -6,6 +6,8 @@ import torch import triton import triton.language as tl +from sglang.srt.utils import is_gfx95_supported + from ..common.utils import ( _bitonic_merge, _sort_ids_ascending, @@ -16,6 +18,9 @@ from ..common.utils import ( unit_scale, ) +# gfx950 MI350X: per-page wins at <=2 pages/block, loses at >=8 +_MAX_PER_PAGE_SLOT_UNROLL = 2 + @triton.heuristics( { @@ -439,6 +444,142 @@ def _topk_index_kernel( tl.store(ti_ptrs, topk_idx.to(ti_ptrs.dtype.element_ty), mask=topk_mask) +@triton.heuristics( + {"BLOCK_SIZE_KD": lambda args: triton.next_power_of_2(args["qk_head_dim"])} +) +@triton.autotune( + configs=[ + triton.Config({"BLOCK_SIZE_Q": 64}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_SIZE_Q": 64}, num_warps=4, num_stages=3), + triton.Config({"BLOCK_SIZE_Q": 64}, num_warps=8, num_stages=2), + triton.Config({"BLOCK_SIZE_Q": 128}, num_warps=8, num_stages=2), + triton.Config({"BLOCK_SIZE_Q": 128}, num_warps=8, num_stages=3), + ], + key=["qk_head_dim", "block_size"], +) +@triton.jit +def _index_block_score_only_kernel( + q_ptr, # Q: [total_q, h, d] + k_cache_ptr, # K paged: [max_slots, kh, d] + score_ptr, # Score: [h, total_q, max_seqblock] + req_to_token_ptr, # [max_reqs, max_kv_len] + cu_seqlens, + seq_lens, + prefix_lens, + slot_ids, + max_slots, + num_heads, + gqa_group_size, + qk_head_dim, + sm_scale, + stride_q_n, + stride_q_h, + stride_q_d, + stride_k_s, + stride_k_h, + stride_k_d, + stride_s_h, + stride_s_q, + stride_s_k, + stride_r2t_b, + BLOCK_SIZE_Q: tl.constexpr, + block_size: tl.constexpr, # sparse K block size (== 128) + page_size: tl.constexpr, # paged-cache page size; block_size % page_size == 0 + PER_PAGE_SLOTS: tl.constexpr, # derive slots from one base slot per page + BLOCK_SIZE_KD: tl.constexpr, + IS_FP8: tl.constexpr, +): + """Causal max index score per query and KV block, without the index-value output. + + With PER_PAGE_SLOTS, each token's slot is derived as base_slot + in-page offset + from one req_to_token load per page, relying on the allocator's contiguous pages. + """ + sm_scale_log2e = sm_scale * 1.4426950409 + pid_q, pid_bh = tl.program_id(0), tl.program_id(1) + pid_b = pid_bh // num_heads + pid_h = pid_bh % num_heads + pid_kh = pid_h // gqa_group_size + seq_start = tl.load(cu_seqlens + pid_b) + q_len = tl.load(cu_seqlens + pid_b + 1) - seq_start + seq_len = tl.load(seq_lens + pid_b) + prefix_len = tl.load(prefix_lens + pid_b) + if BLOCK_SIZE_Q * pid_q >= q_len: + return + sid = (tl.load(slot_ids + pid_b).to(tl.int64) + max_slots) % max_slots + + q_ptrs = tl.make_block_ptr( + base=q_ptr + seq_start * stride_q_n + pid_h * stride_q_h, + shape=(q_len, qk_head_dim), + strides=(stride_q_n, stride_q_d), + offsets=(pid_q * BLOCK_SIZE_Q, 0), + block_shape=(BLOCK_SIZE_Q, BLOCK_SIZE_KD), + order=(1, 0), + ) + q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero") + + off_q = tl.arange(0, BLOCK_SIZE_Q) + pid_q * BLOCK_SIZE_Q + prefix_len + off_k = tl.arange(0, block_size) + off_kd = tl.arange(0, BLOCK_SIZE_KD) + q_row = pid_q * BLOCK_SIZE_Q + tl.arange(0, BLOCK_SIZE_Q) + q_store_mask = q_row < q_len + pages_per_block: tl.constexpr = block_size // page_size + # Within a sparse block, each token's page index and in-page offset are fixed. + page_of = off_k // page_size # [block_size] which physical page (0..pages-1) + in_page = off_k % page_size # [block_size] offset inside that page + + hi = min(seq_len, prefix_len + (pid_q + 1) * BLOCK_SIZE_Q) + for i in tl.range(0, hi, block_size): + blk = i // block_size + pos = i + off_k + pos_mask = pos < seq_len + # One base-slot load per page instead of one per token. + if PER_PAGE_SLOTS: + slots = tl.zeros([block_size], dtype=tl.int64) + for p in tl.static_range(0, pages_per_block): + page_tok0 = i + p * page_size + base_slot = tl.load( + req_to_token_ptr + sid * stride_r2t_b + page_tok0, + mask=page_tok0 < seq_len, + other=0, + ).to(tl.int64) + base_slot = (base_slot + max_slots) % max_slots + slots = tl.where(page_of == p, base_slot + in_page, slots) + else: + slots = tl.load( + req_to_token_ptr + sid * stride_r2t_b + pos, + mask=pos_mask, + other=0, + ).to(tl.int64) + slots = (slots + max_slots) % max_slots + # head_dim (128) is a power of 2 == BLOCK_SIZE_KD, so the dim mask is + # always true -> only mask the K (token) dimension. + k = tl.load( + k_cache_ptr + + slots[None, :] * stride_k_s + + pid_kh * stride_k_h + + off_kd[:, None] * stride_k_d, + mask=pos_mask[None, :], + other=0.0, + ) + if IS_FP8: + # Widening dequant for an fp8 index-K cache with bf16 Q; exact + # no-op cast when Q is fp8 too. Compiled out for bf16 K. + k = k.to(q.dtype) + qk = tl.dot(q, k) * sm_scale_log2e + # single fused causal + K-boundary mask + qk = tl.where( + (off_q[:, None] >= pos[None, :]) & pos_mask[None, :], qk, float("-inf") + ) + score = tl.max(qk, axis=1) # [BLOCK_SIZE_Q] + s_ptrs = ( + score_ptr + + pid_h * stride_s_h + + (seq_start + q_row) * stride_s_q + + blk * stride_s_k + ) + tl.store(s_ptrs, score, mask=q_store_mask) + + @torch.no_grad() def flash_prefill_with_topk_index( q: torch.Tensor, @@ -467,6 +608,7 @@ def flash_prefill_with_topk_index( q_scale: Optional[float] = None, k_scale: Optional[float] = None, v_scale: Optional[float] = None, + page_size: int = 1, ): assert score_type in ( "max", @@ -525,51 +667,94 @@ def flash_prefill_with_topk_index( def grid(META): return (triton.cdiv(max_seqlen_q, META["BLOCK_SIZE_Q"]), batch_size * num_heads) - _flash_attn_fwd_with_block_score_kernel[grid]( - q, - k_cache, - v_cache, - sink, - o, - score, - req_to_token, - cu_seqlens, - seq_lens, - prefix_lens, - slot_ids, - max_slots, - num_heads, - gqa_group_size, - qk_head_dim, - v_head_dim, - block_size_k, - sm_scale, - k_scale, - v_scale, - False, - 1, - q.stride(0), - q.stride(1), - q.stride(2), - k_cache.stride(0), - k_cache.stride(1), - k_cache.stride(2), - v_cache.stride(0) if v_cache is not None else 0, - v_cache.stride(1) if v_cache is not None else 0, - v_cache.stride(2) if v_cache is not None else 0, - sink.stride(0) if sink is not None else 0, - sink.stride(1) if sink is not None else 0, - o.stride(0) if o is not None else 0, - o.stride(1) if o is not None else 0, - o.stride(2) if o is not None else 0, - score.stride(0), - score.stride(1), - score.stride(2), - req_to_token.stride(0), - SCORE_TYPE=score_type, - DISABLE_INDEX_VALUE=disable_index_value, - IS_FP8=is_fp8, - ) + if ( + is_gfx95_supported() + and disable_index_value + and score_type == "max" + and sink is None + and q_scale in (None, 1.0) + and k_scale == 1.0 + and qk_head_dim == 128 + and block_size_k == 128 + and page_size > 0 + and block_size_k % page_size == 0 + ): + # Source layers do not use idx_o, so run the score-only kernel. + _index_block_score_only_kernel[grid]( + q, + k_cache, + score, + req_to_token, + cu_seqlens, + seq_lens, + prefix_lens, + slot_ids, + max_slots, + num_heads, + gqa_group_size, + qk_head_dim, + sm_scale, + q.stride(0), + q.stride(1), + q.stride(2), + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + score.stride(0), + score.stride(1), + score.stride(2), + req_to_token.stride(0), + block_size=block_size_k, + page_size=page_size, + PER_PAGE_SLOTS=(block_size_k // page_size <= _MAX_PER_PAGE_SLOT_UNROLL), + IS_FP8=is_fp8, + ) + else: + _flash_attn_fwd_with_block_score_kernel[grid]( + q, + k_cache, + v_cache, + sink, + o, + score, + req_to_token, + cu_seqlens, + seq_lens, + prefix_lens, + slot_ids, + max_slots, + num_heads, + gqa_group_size, + qk_head_dim, + v_head_dim, + block_size_k, + sm_scale, + k_scale, + v_scale, + False, + 1, + q.stride(0), + q.stride(1), + q.stride(2), + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + v_cache.stride(0) if v_cache is not None else 0, + v_cache.stride(1) if v_cache is not None else 0, + v_cache.stride(2) if v_cache is not None else 0, + sink.stride(0) if sink is not None else 0, + sink.stride(1) if sink is not None else 0, + o.stride(0) if o is not None else 0, + o.stride(1) if o is not None else 0, + o.stride(2) if o is not None else 0, + score.stride(0), + score.stride(1), + score.stride(2), + req_to_token.stride(0), + SCORE_TYPE=score_type, + DISABLE_INDEX_VALUE=disable_index_value, + IS_FP8=is_fp8, + ) # topk extraction kernel topk_idx = torch.full( diff --git a/python/sglang/kernels/ops/attention/minimax_sparse/prefill/topk_sparse.py b/python/sglang/kernels/ops/attention/minimax_sparse/prefill/topk_sparse.py index 5abc81e02a..ddaca7372e 100644 --- a/python/sglang/kernels/ops/attention/minimax_sparse/prefill/topk_sparse.py +++ b/python/sglang/kernels/ops/attention/minimax_sparse/prefill/topk_sparse.py @@ -6,6 +6,8 @@ import torch import triton import triton.language as tl +from sglang.srt.utils import is_gfx95_supported + from ..common.utils import ( check_sparse_kv_fp8, get_cu_seqblocks, @@ -15,6 +17,11 @@ from ..common.utils import ( ) +def _sparse_subk(block_size_k: int) -> int: + """Sub-tile only the measured 128-token block; keep other widths unchanged.""" + return 64 if is_gfx95_supported() and block_size_k == 128 else 0 + + @triton.heuristics( { "BLOCK_SIZE_KD": lambda args: triton.next_power_of_2(args["qk_head_dim"]), @@ -32,12 +39,30 @@ from ..common.utils import ( } ) @triton.autotune( - # Configs that fail to compile on the target arch are skipped, so widening - # the num_warps x num_stages grid only adds candidates, never a bad kernel. + # Keep the additional configs on gfx95; gfx942 retains the original set. configs=[ - triton.Config({}, num_warps=nw, num_stages=ns) - for nw in (2, 4, 8) - for ns in (2, 3, 4) + # CDNA sub-tiled MFMA configs; NVIDIA's Triton rejects these launch kwargs. + *( + [ + triton.Config( + {"matrix_instr_nonkdim": 16, "kpack": 2}, + num_warps=1, + num_stages=1, + ), + triton.Config( + {"matrix_instr_nonkdim": 16, "kpack": 2}, + num_warps=2, + num_stages=1, + ), + ] + if is_gfx95_supported() + else [] + ), + *[ + triton.Config({}, num_warps=nw, num_stages=ns) + for nw in (2, 4, 8) + for ns in (2, 3, 4) + ], ], key=[ "BLOCK_SIZE_Q", @@ -104,6 +129,7 @@ def _gqa_share_sparse_fwd_kernel( BLOCK_SIZE_H: tl.constexpr, BLOCK_SIZE_T: tl.constexpr, BLOCK_SIZE_QH: tl.constexpr, + SUB_K: tl.constexpr, # CDNA KV sub-tile width (0 = disabled) # has sink HAS_SINK: tl.constexpr, USE_TMA: tl.constexpr, @@ -190,77 +216,145 @@ def _gqa_share_sparse_fwd_kernel( acc_o = tl.full((BLOCK_SIZE_QH, BLOCK_SIZE_VD), 0, dtype=tl.float32) q = tl.reshape(q, BLOCK_SIZE_QH, BLOCK_SIZE_KD) # sparse attention - for i in range(real_topk): - # get current block start index (absolute K position) - c = tl.load(t_ptr_j).to(tl.int32) * BLOCK_SIZE_K - t_ptr_j = t_ptr_j + stride_tk - # paged load K via req_to_token: pos -> slot -> k_cache - pos = c + off_n - pos_mask = pos < seq_len - slots = tl.load( - req_to_token_ptr + sid * stride_r2t_b + pos, - mask=pos_mask, - other=0, - ).to(tl.int64) - if HAS_LOC_MAPPING: + if SUB_K > 0: + # Split each BLOCK_SIZE_K-token KV block into SUB_K-token sub-tiles so + # each QK/PV MFMA is right-sized. Numerically equivalent to the dense + # path (flash-softmax reassociation). + NUM_SUB: tl.constexpr = BLOCK_SIZE_K // SUB_K + for i in range(real_topk): + c = tl.load(t_ptr_j).to(tl.int32) * BLOCK_SIZE_K + t_ptr_j = t_ptr_j + stride_tk + for sub_i in range(NUM_SUB): + off_sub = tl.arange(0, SUB_K) + sub_i * SUB_K + pos_s = c + off_sub + pos_mask_s = pos_s < seq_len + slots_s = tl.load( + req_to_token_ptr + sid * stride_r2t_b + pos_s, + mask=pos_mask_s, + other=0, + ).to(tl.int64) + if HAS_LOC_MAPPING: + slots_s = tl.load( + loc_mapping_ptr + slots_s, + mask=pos_mask_s, + other=0, + ).to(tl.int64) + slots_s = (slots_s + max_slots) % max_slots + k_s = tl.load( + k_cache_ptr + + slots_s[None, :] * stride_ks + + pid_kh * stride_kh + + off_kd[:, None] * stride_kd, + mask=kd_mask[:, None] & pos_mask_s[None, :], + other=0.0, + ) + if IS_FP8: + k_s = k_s.to(q.dtype) + off_q_s = ( + tl.arange(0, BLOCK_SIZE_Q)[:, None] + + pid_q_j * BLOCK_SIZE_Q + + prefix_len + - off_sub[None, :] + ) + qk_s = tl.zeros( + (BLOCK_SIZE_Q, BLOCK_SIZE_H, SUB_K), dtype=tl.float32 + ) + qk_s += tl.where(off_q_s[:, None, :] >= c, 0, float("-inf")) + qk_s = tl.reshape(qk_s, BLOCK_SIZE_QH, SUB_K) + qk_s += tl.dot(q, k_s) * (sm_scale_log2e * k_scale) + qk_s += tl.where(pos_mask_s[None, :], 0, float("-inf")) + m_ij = tl.maximum(m_i, tl.max(qk_s, axis=1)) + p_s = tl.exp2(qk_s - m_ij[:, None]) + l_ij = tl.sum(p_s, axis=1) + acc_o = acc_o * tl.exp2(m_i - m_ij)[:, None] + v_s = tl.load( + v_cache_ptr + + slots_s[:, None] * stride_vs + + pid_kh * stride_vh + + off_vd[None, :] * stride_vd, + mask=pos_mask_s[:, None] & vd_mask[None, :], + other=0.0, + ) + if IS_FP8: + v_s = v_s.to(q.dtype) + p_s = p_s.to(v_s.dtype) + acc_o += tl.dot(p_s, v_s) * v_scale + m_i = m_ij + lse_i = m_ij + tl.log2(tl.exp2(lse_i - m_ij) + l_ij) + else: + for i in range(real_topk): + # get current block start index (absolute K position) + c = tl.load(t_ptr_j).to(tl.int32) * BLOCK_SIZE_K + t_ptr_j = t_ptr_j + stride_tk + # paged load K via req_to_token: pos -> slot -> k_cache + pos = c + off_n + pos_mask = pos < seq_len slots = tl.load( - loc_mapping_ptr + slots, + req_to_token_ptr + sid * stride_r2t_b + pos, mask=pos_mask, other=0, ).to(tl.int64) - slots = (slots + max_slots) % max_slots # safety against negative - # k shape: [BLOCK_SIZE_KD, BLOCK_SIZE_K] (transposed for tl.dot) - k = tl.load( - k_cache_ptr - + slots[None, :] * stride_ks - + pid_kh * stride_kh - + off_kd[:, None] * stride_kd, - mask=kd_mask[:, None] & pos_mask[None, :], - other=0.0, - ) - if IS_FP8: - # fp8 main K cache: widening cast with bf16/fp16 Q (unit-scaled - # cache -> exact inverse dequant; k_scale covers calibrated - # caches), no-op with fp8 Q (fp8 attn-GEMM mode) so tl.dot runs - # fp8x8. Compiled out when the cache is bf16. - k = k.to(q.dtype) - # compute qk - qk = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_K), dtype=tl.float32) - # causal mask - qk += tl.where(off_q_k[:, None, :] >= c, 0, float("-inf")) - qk = tl.reshape(qk, BLOCK_SIZE_QH, BLOCK_SIZE_K) - # [BLOCK_SIZE_QH, qk_head_dim] @ [qk_head_dim, BLOCK_SIZE_K] - # -> [BLOCK_SIZE_QH, BLOCK_SIZE_K] - qk += tl.dot(q, k) * (sm_scale_log2e * k_scale) - # K boundary mask: positions beyond seq_len contribute -inf - qk += tl.where(pos_mask[None, :], 0, float("-inf")) - # compute m_ij and l_ij - m_ij = tl.maximum(m_i, tl.max(qk, axis=1)) - p = tl.exp2(qk - m_ij[:, None]) - l_ij = tl.sum(p, axis=1) - # scale acc_o - acc_o_scale = tl.exp2(m_i - m_ij) - acc_o = acc_o * acc_o_scale[:, None] - # paged load V - v = tl.load( - v_cache_ptr - + slots[:, None] * stride_vs - + pid_kh * stride_vh - + off_vd[None, :] * stride_vd, - mask=pos_mask[:, None] & vd_mask[None, :], - other=0.0, - ) - if IS_FP8: - # Cast V to the compute dtype: widening with bf16/fp16 Q (so - # `p.to(v.dtype)` keeps P in the compute dtype), no-op with fp8 - # Q where P is quantized to e4m3 for the fp8 PV MMA — the same - # accuracy contract as fmha_sm100's fp8 kernel. - v = v.to(q.dtype) - p = p.to(v.dtype) - acc_o += tl.dot(p, v) * v_scale - # update statistics - m_i = m_ij - lse_i = m_ij + tl.log2(tl.exp2(lse_i - m_ij) + l_ij) + if HAS_LOC_MAPPING: + slots = tl.load( + loc_mapping_ptr + slots, + mask=pos_mask, + other=0, + ).to(tl.int64) + slots = (slots + max_slots) % max_slots # safety against negative + # k shape: [BLOCK_SIZE_KD, BLOCK_SIZE_K] (transposed for tl.dot) + k = tl.load( + k_cache_ptr + + slots[None, :] * stride_ks + + pid_kh * stride_kh + + off_kd[:, None] * stride_kd, + mask=kd_mask[:, None] & pos_mask[None, :], + other=0.0, + ) + if IS_FP8: + # fp8 main K cache: widening cast with bf16/fp16 Q (unit-scaled + # cache -> exact inverse dequant; k_scale covers calibrated + # caches), no-op with fp8 Q (fp8 attn-GEMM mode) so tl.dot runs + # fp8x8. Compiled out when the cache is bf16. + k = k.to(q.dtype) + # compute qk + qk = tl.zeros( + (BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_K), dtype=tl.float32 + ) + # causal mask + qk += tl.where(off_q_k[:, None, :] >= c, 0, float("-inf")) + qk = tl.reshape(qk, BLOCK_SIZE_QH, BLOCK_SIZE_K) + # [BLOCK_SIZE_QH, qk_head_dim] @ [qk_head_dim, BLOCK_SIZE_K] + # -> [BLOCK_SIZE_QH, BLOCK_SIZE_K] + qk += tl.dot(q, k) * (sm_scale_log2e * k_scale) + # K boundary mask: positions beyond seq_len contribute -inf + qk += tl.where(pos_mask[None, :], 0, float("-inf")) + # compute m_ij and l_ij + m_ij = tl.maximum(m_i, tl.max(qk, axis=1)) + p = tl.exp2(qk - m_ij[:, None]) + l_ij = tl.sum(p, axis=1) + # scale acc_o + acc_o_scale = tl.exp2(m_i - m_ij) + acc_o = acc_o * acc_o_scale[:, None] + # paged load V + v = tl.load( + v_cache_ptr + + slots[:, None] * stride_vs + + pid_kh * stride_vh + + off_vd[None, :] * stride_vd, + mask=pos_mask[:, None] & vd_mask[None, :], + other=0.0, + ) + if IS_FP8: + # Cast V to the compute dtype: widening with bf16/fp16 Q (so + # `p.to(v.dtype)` keeps P in the compute dtype), no-op with fp8 + # Q where P is quantized to e4m3 for the fp8 PV MMA — the same + # accuracy contract as fmha_sm100's fp8 kernel. + v = v.to(q.dtype) + p = p.to(v.dtype) + acc_o += tl.dot(p, v) * v_scale + # update statistics + m_i = m_ij + lse_i = m_ij + tl.log2(tl.exp2(lse_i - m_ij) + l_ij) # final scale acc_o = acc_o * tl.exp2(m_i - lse_i)[:, None] # save output @@ -385,6 +479,7 @@ def flash_prefill_with_gqa_share_sparse( o.stride(2), req_to_token.stride(0), BLOCK_SIZE_Q=BLOCK_SIZE_Q, + SUB_K=_sparse_subk(BLOCK_SIZE_K), BLOCK_SIZE_K=BLOCK_SIZE_K, USE_TMA=use_tma, IS_FP8=is_fp8, diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 25a5cb72a9..1012624d16 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1698,6 +1698,9 @@ class Envs: # 2 is the accuracy-safe default: higher values reuse staler selections # in the skip layers. SGLANG_MINIMAX_M3_INDEX_TOPK_FREQ = EnvInt(2) + # Run the sparse prefill main attention through AITER's Gluon paged attention + # instead of the Triton kernel. Unsupported cases fall back to Triton. + SGLANG_OPT_USE_MINIMAX_GLUON_PREFILL = EnvBool(True) # MiniMax M3 NPU prefill MAIN-attention: route the sparse main attention through # the native Ascend FA op `torch.ops.npu.npu_fused_infer_attention_score` (FIA) # with a per-query CUSTOM block_table diff --git a/python/sglang/srt/layers/attention/minimax_sparse_backend.py b/python/sglang/srt/layers/attention/minimax_sparse_backend.py index 11171a61f4..c3745ab17a 100644 --- a/python/sglang/srt/layers/attention/minimax_sparse_backend.py +++ b/python/sglang/srt/layers/attention/minimax_sparse_backend.py @@ -1570,6 +1570,7 @@ class MiniMaxSparseAttnBackend(AttentionBackend): disable_index_value=disable_value, use_msa=self.use_msa, seqlens_cpu=forward_batch.extend_seq_lens_cpu, + seq_lens_cpu=forward_batch.seq_lens_cpu, cu_seqblocks_q=cu_seqblocks_q, max_seqblock_q=max_seqblock_q, all_seqblock_q=all_seqblock_q, @@ -1579,6 +1580,7 @@ class MiniMaxSparseAttnBackend(AttentionBackend): idx_q_scale=layer.idx_q_scale_float, idx_k_scale=layer.idx_k_scale_float, idx_v_scale=layer.idx_v_scale_float, + page_size=self.page_size, cached_topk_idx=cached_topk_idx, return_topk_idx=want_topk, loc_mapping=self._loc_mapping, diff --git a/python/sglang/srt/layers/attention/minimax_sparse_ops/gluon_prefill.py b/python/sglang/srt/layers/attention/minimax_sparse_ops/gluon_prefill.py new file mode 100644 index 0000000000..7d67bbc91f --- /dev/null +++ b/python/sglang/srt/layers/attention/minimax_sparse_ops/gluon_prefill.py @@ -0,0 +1,492 @@ +# Copyright 2025 SGLang Team +# Ported from +# https://github.com/alexsun07/sglang/blob/191624ff8a6d8e9fc31b9363d2138c16db66f124/python/sglang/srt/layers/attention/minimax_sparse_ops/atom_prefill.py +"""MiniMax-M3 sparse prefill using AITER's Gluon paged attention. + +The main KV pool remains NHD ``[max_slots, 1, head_dim]``. Each request's +context is gathered into persistent SHUFFLE 5D scratch pages: + +- key scratch: [num_pages, 1, head_dim // x, GLUON_PAGE_SIZE, x] +- value scratch: [num_pages, 1, GLUON_PAGE_SIZE // x, head_dim, x] (transposed) + +with ``x = 16 // dtype.itemsize``. Each request occupies a contiguous, +position-ordered page range rounded up to whole sparse blocks. +""" + +from __future__ import annotations + +from itertools import accumulate +from typing import Optional + +import torch +import triton +import triton.language as tl + +from sglang.srt.layers.attention.aiter_utils import ( + get_recommended_splits, + pa_decode_gluon, +) + +SPARSE_BLOCK_SIZE = 128 +# The Gluon PS kernel takes kv_block_size 16 or 64; 64 halves the block-table width. +GLUON_PAGE_SIZE = 64 +PAGES_PER_BLOCK = SPARSE_BLOCK_SIZE // GLUON_PAGE_SIZE +HEAD_DIM = 128 +# Scratch grows in 1024-page steps (16 KiB/page/buffer -> 16 MiB granularity). +_SCRATCH_GROW_PAGES = 1024 +_PAGE_ELEMS = HEAD_DIM * GLUON_PAGE_SIZE + +# 512 MiB per buffer at bf16 (~2.1M context tokens per forward); past it the caller +# falls back to the Triton kernel, which reads the pool in place. +_MAX_SCRATCH_PAGES = (512 * 1024 * 1024) // (_PAGE_ELEMS * 2) + + +class GluonPrefillUnavailableError(RuntimeError): + """Raised when the scratch cannot hold this forward; the caller falls back to Triton.""" + + +@triton.jit +def _gather_nhd_to_shuffle_kernel( + k_pool_ptr, # [max_slots, 1, HEAD_DIM] (NHD main pool, this layer) + v_pool_ptr, # [max_slots, 1, HEAD_DIM] + k_out_ptr, # [num_pages, 1, HEAD_DIM // X, PAGE, X] contiguous + v_out_ptr, # [num_pages, 1, PAGE // X, HEAD_DIM, X] contiguous + req_to_token_ptr, # [max_reqs, max_kv_len] + req_pool_indices_ptr, # [batch] + seq_lens_ptr, # [batch] total K length (prefix + current chunk) + page_req_ptr, # [total_pages] -> batch index owning this page + page_pos_ptr, # [total_pages] -> page index within the request + stride_k_slot, + stride_v_slot, + stride_r2t_b, + HEAD_DIM_C: tl.constexpr, + PAGE: tl.constexpr, + X: tl.constexpr, +): + pid = tl.program_id(0) + req = tl.load(page_req_ptr + pid) + ppos = tl.load(page_pos_ptr + pid) + pool_id = tl.load(req_pool_indices_ptr + req).to(tl.int64) + seq_len = tl.load(seq_lens_ptr + req) + + off_t = tl.arange(0, PAGE) + tok = ppos * PAGE + off_t + tmask = tok < seq_len + slots = tl.load( + req_to_token_ptr + pool_id * stride_r2t_b + tok, mask=tmask, other=0 + ).to(tl.int64) + + off_d = tl.arange(0, HEAD_DIM_C) + # Coalesced NHD reads: each token's head_dim vector is contiguous. + k_tile = tl.load( + k_pool_ptr + slots[:, None] * stride_k_slot + off_d[None, :], + mask=tmask[:, None], + other=0.0, + ) + v_tile = tl.load( + v_pool_ptr + slots[:, None] * stride_v_slot + off_d[None, :], + mask=tmask[:, None], + other=0.0, + ) + + page_base = pid.to(tl.int64) * (HEAD_DIM_C * PAGE) + # K shuffle page [1, D//X, PAGE, X]: elem(d, t) at (d//X)*(PAGE*X) + t*X + d%X + k_off = ( + (off_d[None, :] // X) * (PAGE * X) + off_t[:, None] * X + (off_d[None, :] % X) + ) + # V transposed page [1, PAGE//X, D, X]: elem(d, t) at (t//X)*(D*X) + d*X + t%X + v_off = ( + (off_t[:, None] // X) * (HEAD_DIM_C * X) + + off_d[None, :] * X + + (off_t[:, None] % X) + ) + # Unmasked: tiles are zero past seq_len, so no previous layer's data survives. + tl.store(k_out_ptr + page_base + k_off, k_tile) + tl.store(v_out_ptr + page_base + v_off, v_tile) + + +# Grow-only (device, dtype) -> (k_flat, v_flat); layers run in sequence and share it. +_SCRATCH: dict = {} + + +def _get_scratch( + device: torch.device, dtype: torch.dtype, total_pages: int +) -> tuple[torch.Tensor, torch.Tensor]: + key = (device, dtype) + entry = _SCRATCH.get(key) + needed = total_pages * _PAGE_ELEMS + if entry is None or entry[0].numel() < needed: + cap_pages = ( + (total_pages + _SCRATCH_GROW_PAGES - 1) // _SCRATCH_GROW_PAGES + ) * _SCRATCH_GROW_PAGES + # Grows after the KV pool claimed its budget; report OOM as unsupported so + # the caller falls back to the scratch-free Triton kernel. + try: + entry = ( + torch.empty(cap_pages * _PAGE_ELEMS, dtype=dtype, device=device), + torch.empty(cap_pages * _PAGE_ELEMS, dtype=dtype, device=device), + ) + except torch.cuda.OutOfMemoryError as exc: + raise GluonPrefillUnavailableError( + f"gluon prefill scratch of {cap_pages} pages " + f"({cap_pages * _PAGE_ELEMS * dtype.itemsize * 2 >> 20} MiB) " + "does not fit; lower --mem-fraction-static" + ) from exc + _SCRATCH[key] = entry + return entry + + +def _gather_context_to_shuffle( + k_cache: torch.Tensor, + v_cache: torch.Tensor, + req_to_token: torch.Tensor, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + page_req: torch.Tensor, + page_pos: torch.Tensor, + total_pages: int, +) -> tuple[torch.Tensor, torch.Tensor]: + """Gather this batch's context tokens (NHD pool slots) into SHUFFLE 5D pages.""" + x = 16 // k_cache.dtype.itemsize # 8 for bf16/fp16 + k_flat, v_flat = _get_scratch(k_cache.device, k_cache.dtype, total_pages) + _gather_nhd_to_shuffle_kernel[(total_pages,)]( + k_cache, + v_cache, + k_flat, + v_flat, + req_to_token, + req_pool_indices, + seq_lens, + page_req, + page_pos, + k_cache.stride(0), + v_cache.stride(0), + req_to_token.stride(0), + HEAD_DIM_C=HEAD_DIM, + PAGE=GLUON_PAGE_SIZE, + X=x, + ) + k5 = k_flat[: total_pages * _PAGE_ELEMS].view( + total_pages, 1, HEAD_DIM // x, GLUON_PAGE_SIZE, x + ) + v5 = v_flat[: total_pages * _PAGE_ELEMS].view( + total_pages, 1, GLUON_PAGE_SIZE // x, HEAD_DIM, x + ) + return k5, v5 + + +@triton.jit +def _build_gluon_sparse_bt_prefill_kernel( + topk_ptr, # [Hkv=1, total_q, topk] int32 + req_id_ptr, # [total_q] int32 + abs_pos_ptr, # [total_q] int32 + page_start_ptr, # [batch] int32: first scratch page of each request + sparse_bt_ptr, # [total_q, topk * pages_per_block] int32 + sparse_ctx_ptr, # [total_q] int32 + max_topk, + stride_topk_n, + stride_topk_t, + stride_sparse_bt_n, + sparse_block_size: tl.constexpr, + pages_per_block: tl.constexpr, + BLOCK_SIZE_T: tl.constexpr, +): + pid_n = tl.program_id(0) + + req_id = tl.load(req_id_ptr + pid_n) + base_page = tl.load(page_start_ptr + req_id) + abs_pos = tl.load(abs_pos_ptr + pid_n) + causal_len = abs_pos + 1 + self_blk = abs_pos // sparse_block_size + + topk_row = topk_ptr + pid_n * stride_topk_n + out_row = sparse_bt_ptr + pid_n * stride_sparse_bt_n + off_t = tl.arange(0, BLOCK_SIZE_T) + blk = tl.load(topk_row + off_t * stride_topk_t, mask=off_t < max_topk, other=-1) + + # Causal: a query at abs_pos may attend its own block and all earlier ones. + valid = (blk >= 0) & (blk <= self_blk) + is_tail = valid & (blk == self_blk) + is_full = valid & (blk < self_blk) + n_full = tl.sum(is_full.to(tl.int32), axis=0) + n_valid = tl.sum(valid.to(tl.int32), axis=0) + # Full blocks first, the partial causal tail last, so the walk matches sparse_ctx. + earlier_full = tl.cumsum(is_full.to(tl.int32), axis=0) - is_full.to(tl.int32) + sparse_slot = tl.where(is_full, earlier_full, n_full) + + # Scratch pages for each request are contiguous and position-ordered. + dst_base = sparse_slot * pages_per_block + for j in tl.static_range(0, pages_per_block): + phys_page = base_page + blk * pages_per_block + j + tl.store(out_row + dst_base + j, phys_page, mask=valid) + + n_used = n_valid * pages_per_block + off_w = tl.arange(0, BLOCK_SIZE_T * pages_per_block) + # BLOCK_SIZE_T is next_power_of_2(topk), so off_w overshoots the row whenever + # topk is not a power of two -- bound the tail fill by the real row width. + tl.store( + out_row + off_w, + tl.zeros_like(off_w), + mask=(off_w >= n_used) & (off_w < max_topk * pages_per_block), + ) + + # Effective KV length the kernel will walk across the packed pages. + tail_tokens = causal_len - self_blk * sparse_block_size + has_tail = tl.sum(is_tail.to(tl.int32), axis=0) > 0 + ctx_len = n_full * sparse_block_size + tl.where(has_tail, tail_tokens, 0) + ctx_len = tl.where( + has_tail, ctx_len, tl.minimum(n_valid * sparse_block_size, causal_len) + ) + tl.store(sparse_ctx_ptr + pid_n, ctx_len) + + +_META_CACHE: Optional[tuple] = None + +_BT_OUT_CACHE: Optional[tuple] = None + + +def _build_gluon_prefill_meta( + cu_seqlens: torch.Tensor, + prefix_lens: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_cpu: torch.Tensor, + total_q: int, +): + global _META_CACHE, _BT_OUT_CACHE + if _META_CACHE is not None: + (k_cu, k_prefix, k_seq, k_total_q), cached = _META_CACHE + if ( + k_cu is cu_seqlens + and k_prefix is prefix_lens + and k_seq is seq_lens + and k_total_q == total_q + ): + return cached + + device = cu_seqlens.device + pos = torch.arange(total_q, dtype=torch.int32, device=device) + req_id = torch.searchsorted(cu_seqlens[1:].contiguous(), pos, right=True).to( + torch.int32 + ) + abs_pos = (prefix_lens[req_id] + (pos - cu_seqlens[req_id])).to(torch.int32) + + # Host-side page layout: each request's context span rounds up to whole + # sparse blocks so every emitted page id stays inside its own span. + pages_per_req = [ + ((kv_len + SPARSE_BLOCK_SIZE - 1) // SPARSE_BLOCK_SIZE) * PAGES_PER_BLOCK + for kv_len in seq_lens_cpu.tolist() + ] + total_pages = sum(pages_per_req) + page_start = torch.tensor( + list(accumulate(pages_per_req, initial=0))[:-1], + dtype=torch.int32, + device=device, + ) + repeats = torch.tensor(pages_per_req, dtype=torch.int64, device=device) + page_req = torch.repeat_interleave( + torch.arange(len(pages_per_req), dtype=torch.int32, device=device), + repeats, + output_size=total_pages, + ) + page_pos = ( + torch.arange(total_pages, dtype=torch.int32, device=device) + - page_start[page_req.long()] + ).to(torch.int32) + + meta = (req_id, abs_pos, page_start, page_req, page_pos, total_pages) + _BT_OUT_CACHE = None # new forward: no block table can be valid anymore + _META_CACHE = ((cu_seqlens, prefix_lens, seq_lens, total_q), meta) + return meta + + +def _build_gluon_sparse_bt_prefill( + topk_idx: torch.Tensor, + req_id: torch.Tensor, + abs_pos: torch.Tensor, + page_start: torch.Tensor, + sparse_block_size: int, +) -> tuple[torch.Tensor, torch.Tensor]: + # topk_idx: [num_kv_heads=1, total_q, topk] + total_q = topk_idx.shape[1] + topk = topk_idx.shape[2] + + global _BT_OUT_CACHE + if _BT_OUT_CACHE is not None: + (k_topk_idx, k_total_q, k_topk, k_block_size), out_cached = _BT_OUT_CACHE + if ( + k_topk_idx is topk_idx + and k_total_q == total_q + and k_topk == topk + and k_block_size == sparse_block_size + ): + return out_cached + + sparse_bt = torch.empty( + (total_q, topk * PAGES_PER_BLOCK), + dtype=torch.int32, + device=topk_idx.device, + ) + sparse_ctx = torch.empty((total_q,), dtype=torch.int32, device=topk_idx.device) + _build_gluon_sparse_bt_prefill_kernel[(total_q,)]( + topk_idx, + req_id, + abs_pos, + page_start, + sparse_bt, + sparse_ctx, + topk, + topk_idx.stride(1), + topk_idx.stride(2), + sparse_bt.stride(0), + sparse_block_size=sparse_block_size, + pages_per_block=PAGES_PER_BLOCK, + BLOCK_SIZE_T=triton.next_power_of_2(topk), + ) + _BT_OUT_CACHE = ( + (topk_idx, total_q, topk, sparse_block_size), + (sparse_bt, sparse_ctx), + ) + return sparse_bt, sparse_ctx + + +def _unit_or_none(scale) -> bool: + return scale is None or scale == 1.0 + + +def can_use_gluon_prefill( + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + sink: Optional[torch.Tensor], + block_size_k: int, + seq_lens_cpu: Optional[torch.Tensor], + q_scale: Optional[float] = None, + k_scale: Optional[float] = None, + v_scale: Optional[float] = None, +) -> bool: + """Cheap static gate; the scratch size is checked per forward.""" + if ( + pa_decode_gluon is None + or get_recommended_splits is None + or sink is not None + or seq_lens_cpu is None + or block_size_k != SPARSE_BLOCK_SIZE + ): + return False + # The gather requires a contiguous, single-head NHD pool. Calibrated and + # unsupported cache formats stay on the Triton path. + return ( + q.dim() == 3 + and q.shape[-1] == HEAD_DIM + and k_cache.dim() == 3 + and v_cache.dim() == 3 + and k_cache.shape[1] == 1 + and v_cache.shape[1] == 1 + and k_cache.shape[2] == HEAD_DIM + and v_cache.shape[2] == HEAD_DIM + and q.dtype in (torch.bfloat16, torch.float16) + and k_cache.dtype == q.dtype + and v_cache.dtype == q.dtype + and k_cache.stride(2) == 1 + and v_cache.stride(2) == 1 + and _unit_or_none(q_scale) + and _unit_or_none(k_scale) + and _unit_or_none(v_scale) + ) + + +@torch.no_grad() +def gluon_sparse_prefill( + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + topk_idx: torch.Tensor, + req_to_token: torch.Tensor, + req_pool_indices: torch.Tensor, + cu_seqlens: torch.Tensor, + seq_lens: torch.Tensor, + prefix_lens: torch.Tensor, + seq_lens_cpu: torch.Tensor, + block_size_k: int, + sm_scale: Optional[float] = None, +) -> torch.Tensor: + """Run Gluon sparse prefill over the NHD KV pool via the SHUFFLE scratch. + + Raises GluonPrefillUnavailableError when the scratch cannot hold the context. + """ + if sm_scale is None: + sm_scale = q.shape[-1] ** -0.5 + + q = q.contiguous() + total_q, num_q_heads, head_dim = q.shape + if total_q == 0: + return torch.empty_like(q) + assert topk_idx.shape[0] == 1 and topk_idx.shape[1] == total_q, ( + f"gluon prefill expects topk_idx [1, total_q, topk], got {tuple(topk_idx.shape)}" + ) + batch = cu_seqlens.shape[0] - 1 + assert seq_lens_cpu.numel() == batch and seq_lens.shape[0] == batch, ( + f"gluon prefill batch mismatch: cu_seqlens batch {batch}, " + f"seq_lens {tuple(seq_lens.shape)}, seq_lens_cpu {seq_lens_cpu.numel()}" + ) + + req_id, abs_pos, page_start, page_req, page_pos, total_pages = ( + _build_gluon_prefill_meta( + cu_seqlens, prefix_lens, seq_lens, seq_lens_cpu, total_q + ) + ) + if total_pages > _MAX_SCRATCH_PAGES: + raise GluonPrefillUnavailableError( + f"gluon prefill context span too large for scratch: {total_pages} pages " + f"> cap {_MAX_SCRATCH_PAGES}" + ) + + # Gather the current layer's prefix and current chunk into SHUFFLE 5D views. + k_cache, v_cache = _gather_context_to_shuffle( + k_cache, + v_cache, + req_to_token, + req_pool_indices, + seq_lens, + page_req, + page_pos, + total_pages, + ) + + sparse_bt, sparse_ctx = _build_gluon_sparse_bt_prefill( + topk_idx, req_id, abs_pos, page_start, block_size_k + ) + + out = torch.empty_like(q) + num_seqs = total_q + ctx_part = 256 + max_part_num = get_recommended_splits(num_seqs, 1) + intermediate_shape = (num_seqs, 1, max_part_num, num_q_heads) + exp_sums = torch.empty(intermediate_shape, dtype=torch.float32, device=q.device) + max_logits = torch.empty_like(exp_sums) + temporary_output = torch.empty( + (*intermediate_shape, head_dim), dtype=q.dtype, device=q.device + ) + + pa_decode_gluon( + output=out, + query=q, + key_cache=k_cache, + value_cache=v_cache, + context_lengths=sparse_ctx, + block_tables=sparse_bt, + softmax_scale=sm_scale, + query_length=1, + max_context_partition_num=max_part_num, + context_partition_size=ctx_part, + compute_type=q.dtype, + key_scale=None, + value_scale=None, + exp_sums=exp_sums, + max_logits=max_logits, + temporary_output=temporary_output, + sinks=None, + sliding_window=0, + ps=True, + ) + return out diff --git a/python/sglang/srt/layers/attention/minimax_sparse_ops/minimax_sparse.py b/python/sglang/srt/layers/attention/minimax_sparse_ops/minimax_sparse.py index a9d0c0b757..a0eb9e99c3 100644 --- a/python/sglang/srt/layers/attention/minimax_sparse_ops/minimax_sparse.py +++ b/python/sglang/srt/layers/attention/minimax_sparse_ops/minimax_sparse.py @@ -19,9 +19,14 @@ from sglang.kernels.ops.attention.minimax_sparse.prefill.flash_with_topk_idx imp from sglang.kernels.ops.attention.minimax_sparse.prefill.topk_sparse import ( flash_prefill_with_gqa_share_sparse, ) +from sglang.srt.environ import envs +from sglang.srt.utils import is_gfx95_supported, is_hip + +_use_aiter_gfx95 = envs.SGLANG_USE_AITER.get() and is_hip() and is_gfx95_supported() logger = logging.getLogger(__name__) _msa_fallback_warned = False +_gluon_fallback_warned = False def _warn_msa_fallback(err: Exception) -> None: @@ -35,6 +40,18 @@ def _warn_msa_fallback(err: Exception) -> None: _msa_fallback_warned = True +def _warn_gluon_fallback(msg: str) -> None: + global _gluon_fallback_warned + if _gluon_fallback_warned: + return + logger.warning( + "MiniMax Gluon sparse prefill is unavailable (%s); falling back to Triton " + "sparse attention.", + msg, + ) + _gluon_fallback_warned = True + + def minimax_sparse_prefill( q: torch.Tensor, # [total_extend_tokens, num_q_heads, qk_head_dim] k_cache: torch.Tensor, # [max_slots, num_kv_heads, head_dim] (paged main) @@ -76,6 +93,8 @@ def minimax_sparse_prefill( cached_topk_idx: Optional[torch.Tensor] = None, return_topk_idx: bool = False, loc_mapping: Optional[torch.Tensor] = None, + page_size: int = 1, + seq_lens_cpu: Optional[torch.Tensor] = None, ): """Run MiniMax-M3 sparse prefill. @@ -91,6 +110,10 @@ def minimax_sparse_prefill( kernels. Supplying them avoids recomputing the same block layout twice. ``seqlens_cpu`` (host copy of ``torch.diff(cu_seqlens)``) is forwarded to ``get_cu_seqblocks`` to avoid a per-layer device sync when it recomputes. + + ``seq_lens_cpu`` (host copy of ``seq_lens``, i.e. prefix + current chunk + per request) is only consumed by the env-gated Gluon prefill path for + sync-free scratch-page sizing; ``None`` disables that path. """ if cu_seqblocks_q is None or max_seqblock_q is None or all_seqblock_q is None: cu_seqblocks_q, max_seqblock_q, all_seqblock_q, _, _, _ = get_cu_seqblocks( @@ -129,6 +152,7 @@ def minimax_sparse_prefill( cu_seqblocks_q=cu_seqblocks_q, max_seqblock_q=max_seqblock_q, all_seqblock_q=all_seqblock_q, + page_size=page_size, q_scale=idx_q_scale, k_scale=idx_k_scale, v_scale=idx_v_scale, @@ -144,10 +168,53 @@ def minimax_sparse_prefill( # Reduced top-k cached by the caller for subsequent skip layers. reduced_topk_idx = topk_idx - # Step 3: Sparse attention using topk index (main head). The MSA path only - # replaces this step; the indexer above is unchanged. MSA has no attn-sink - # input, so keep the Triton path when sink is present. - if use_msa and sink is None and loc_mapping is None: + # Step 3: Sparse attention using topk index (main head). The Gluon and + # MSA paths only replace this step; the indexer above is unchanged. MSA has + # no attn-sink input, so keep the Triton path when sink is present. + o = None + if ( + _use_aiter_gfx95 + and envs.SGLANG_OPT_USE_MINIMAX_GLUON_PREFILL.get() + # the scratch gather reads pool slots without the HiSparse remap + and loc_mapping is None + ): + from .gluon_prefill import ( + GluonPrefillUnavailableError, + can_use_gluon_prefill, + gluon_sparse_prefill, + ) + + if can_use_gluon_prefill( + q, + k_cache, + v_cache, + sink, + block_size_k, + seq_lens_cpu, + q_scale, + k_scale, + v_scale, + ): + try: + o = gluon_sparse_prefill( + q=q, + k_cache=k_cache, + v_cache=v_cache, + topk_idx=topk_idx, + req_to_token=req_to_token, + req_pool_indices=slot_ids, + cu_seqlens=cu_seqlens, + seq_lens=seq_lens, + prefix_lens=prefix_lens, + seq_lens_cpu=seq_lens_cpu, + block_size_k=block_size_k, + sm_scale=sm_scale, + ) + except GluonPrefillUnavailableError as err: + _warn_gluon_fallback(str(err)) + else: + _warn_gluon_fallback("unsupported batch/cache layout or dtype") + if o is None and use_msa and sink is None and loc_mapping is None: from .msa import MSAUnavailableError, msa_sparse_prefill_main try: @@ -191,7 +258,7 @@ def minimax_sparse_prefill( v_scale=v_scale, loc_mapping=loc_mapping, ) - else: + elif o is None: o = flash_prefill_with_gqa_share_sparse( q=q, k_cache=k_cache,