[MiniMax-M3] Instantiate the decode block top-k in register buckets (#38615)

Co-authored-by: Hao Phan <htphan@nvidia.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
This commit is contained in:
Thanhhao
2026-10-08 19:33:17 +08:00
committed by GitHub
co-authored by Hao Phan Claude Opus 5 Po-Han Huang
parent 45abea0269
commit 6e6ccb26f2
5 changed files with 155 additions and 47 deletions
@@ -42,10 +42,11 @@ struct TopKTraitImpl {
static constexpr uint32_t kCTASize = kNumThreads;
static constexpr uint32_t kNumWarps = kCTASize / device::kWarpThreads;
// Must match _MAX_NUM_BLOCKS in ops/attention/minimax_decode_topk.py.
#ifdef USE_ROCM
static constexpr uint32_t kMaxNumBlocks = 16384; // block topk
#else
static constexpr uint32_t kMaxNumBlocks = 4096; // block topk
#ifndef USE_ROCM
// Smallest register bucket for the multi-element radix path; the launchers
// instantiate larger ones on demand up to kMaxNumBlocks.
static constexpr uint32_t kMinNumBlocksBucket = 4096;
#endif
static constexpr uint32_t kSmallThreshold = 8 * kNumWarps;
static constexpr uint32_t kRadixBits = 8;
@@ -301,6 +302,9 @@ struct TopKTraitImpl {
}
#endif // USE_ROCM
// kMaxBlocks is the CUDA register bucket; ROCm sizes its slots from
// num_blocks at run time instead.
template <uint32_t kMaxBlocks = kMaxNumBlocks>
SGL_DEVICE static void forward(
const float* __restrict__ scores,
const uint32_t num_blocks,
@@ -455,15 +459,16 @@ struct TopKTraitImpl {
if (write_pos < topk) topk_out[write_pos] = tx;
} else {
// num_blocks in (kCTASize, kMaxNumBlocks]: each thread caches its (up to
// num_blocks in (kCTASize, kMaxBlocks]: each thread caches its (up to
// kIters) slice of the row in registers -- read from global exactly ONCE --
// then runs the same 4-pass radix select as the single-element path looped
// over those slots. Liveness is a uint32_t bitmask (bit i = slot i still in
// the running set), so there is no per-element flag array; selection is an
// in-loop scatter, so there is no per-element position array. Nothing is
// cached in shared memory beyond the histogram.
constexpr uint32_t kIters = kMaxNumBlocks / kCTASize;
constexpr uint32_t kIters = kMaxBlocks / kCTASize;
static_assert(kIters <= 32, "active liveness is packed into a uint32_t");
static_assert(kMaxBlocks % kCTASize == 0, "bucket must be a multiple of kCTASize");
uint32_t key[kIters];
uint32_t active = 0;
#pragma unroll
@@ -529,6 +534,15 @@ struct TopKTraitImpl {
using TopKTrait = TopKTraitImpl<512>;
#ifdef USE_ROCM
using TopKTraitWide = TopKTraitImpl<1024>;
#else
// Smallest instantiated bucket covering max_seqblock, so a short row is not
// charged the register footprint of the largest one.
inline uint32_t topk_bucket_for(int max_seqblock) {
const auto n = static_cast<uint32_t>(max_seqblock);
if (n <= TopKTrait::kMinNumBlocksBucket) return TopKTrait::kMinNumBlocksBucket; // 4096
if (n <= 2 * TopKTrait::kMinNumBlocksBucket) return 2 * TopKTrait::kMinNumBlocksBucket; // 8192
return TopKTrait::kMaxNumBlocks; // 16384
}
#endif
// -------------------------------------------------------------------------
@@ -541,7 +555,7 @@ using TopKTraitWide = TopKTraitImpl<1024>;
// ascending), [k_eff:topk) = -1. Ascending order is a hard requirement of the
// MSA fmha_sm100 consumer (kv_block_indexes must be strictly ascending; its
// sorted-order early-exit otherwise mis-masks the partial last block).
template <typename SeqLenT, bool kUsePDL, typename TopKTrait>
template <typename SeqLenT, bool kUsePDL, typename TopKTrait, uint32_t kMaxBlocks = TopKTrait::kMaxNumBlocks>
__global__ void minimax_decode_topk_block_kernel(
const float* __restrict__ score,
const SeqLenT* __restrict__ seq_lens,
@@ -573,7 +587,8 @@ __global__ void minimax_decode_topk_block_kernel(
const float* __restrict__ row = score + (static_cast<int64_t>(h) * batch + b) * max_seqblock;
__shared__ typename TopKTrait::Smem smem;
__shared__ int32_t s_topk[TopKTrait::kMaxTopK];
TopKTrait::forward(row, static_cast<uint32_t>(num_blocks), s_topk, static_cast<uint32_t>(topk), &smem);
TopKTrait::template forward<kMaxBlocks>(
row, static_cast<uint32_t>(num_blocks), s_topk, static_cast<uint32_t>(topk), &smem);
__syncthreads(); // s_topk fully written before the sort reads it
// Emit ascending: num_blocks > topk here, so all topk slots hold distinct
@@ -636,7 +651,7 @@ __global__ void minimax_decode_topk_block_kernel(
// [num_pages, nkv, page_size, D] reshaped to [num_pages*nkv, 1, page_size, D] (a
// free view when the cache is contiguous HND). num_heads == 1 (h == 0) reproduces
// the single-kv-head TP>=4 behavior (page index == base_page).
template <typename SeqLenT, bool kUsePDL, typename TopKTrait>
template <typename SeqLenT, bool kUsePDL, typename TopKTrait, uint32_t kMaxBlocks = TopKTrait::kMaxNumBlocks>
__global__ void minimax_decode_topk_page_table_kernel(
const float* __restrict__ score,
const SeqLenT* __restrict__ seq_lens,
@@ -686,7 +701,8 @@ __global__ void minimax_decode_topk_page_table_kernel(
const float* __restrict__ row = score + (static_cast<int64_t>(h) * batch + b) * max_seqblock; // head-major score
__shared__ typename TopKTrait::Smem smem;
__shared__ int32_t s_topk[TopKTrait::kMaxTopK];
TopKTrait::forward(row, static_cast<uint32_t>(num_blocks), s_topk, static_cast<uint32_t>(topk), &smem);
TopKTrait::template forward<kMaxBlocks>(
row, static_cast<uint32_t>(num_blocks), s_topk, static_cast<uint32_t>(topk), &smem);
__syncthreads(); // s_topk fully written before the transform reads it
// Sort the selected block ids ascending (k_eff <= kMaxTopK is tiny) so the
@@ -786,17 +802,34 @@ void minimax_decode_topk(
return;
}
#endif
LaunchKernel(grid, TopKTrait::kCTASize, device, 0)
.enable_pdl(kUsePDL)(
minimax_decode_topk_block_kernel<SeqLenT, kUsePDL, TopKTrait>,
static_cast<const float*>(score.data_ptr()),
static_cast<const SeqLenT*>(seq_lens.data_ptr()),
static_cast<int32_t*>(topk_idx.data_ptr()),
batch,
num_heads,
max_seqblock,
static_cast<int>(block_size),
topk_i);
auto launch = [&](auto kernel) {
LaunchKernel(grid, TopKTrait::kCTASize, device, 0)
.enable_pdl(kUsePDL)(
kernel,
static_cast<const float*>(score.data_ptr()),
static_cast<const SeqLenT*>(seq_lens.data_ptr()),
static_cast<int32_t*>(topk_idx.data_ptr()),
batch,
num_heads,
max_seqblock,
static_cast<int>(block_size),
topk_i);
};
#ifdef USE_ROCM
launch(minimax_decode_topk_block_kernel<SeqLenT, kUsePDL, TopKTrait>);
#else
switch (topk_bucket_for(max_seqblock)) {
case TopKTrait::kMinNumBlocksBucket:
launch(minimax_decode_topk_block_kernel<SeqLenT, kUsePDL, TopKTrait, TopKTrait::kMinNumBlocksBucket>);
break;
case 2 * TopKTrait::kMinNumBlocksBucket:
launch(minimax_decode_topk_block_kernel<SeqLenT, kUsePDL, TopKTrait, 2 * TopKTrait::kMinNumBlocksBucket>);
break;
default:
launch(minimax_decode_topk_block_kernel<SeqLenT, kUsePDL, TopKTrait, TopKTrait::kMaxNumBlocks>);
break;
}
#endif
}
// Page-table variant: emit the per-(batch, kv-head) paged page table consumed by
@@ -891,24 +924,41 @@ void minimax_decode_topk_page_table(
return;
}
#endif
LaunchKernel(grid, TopKTrait::kCTASize, device, 0)
.enable_pdl(kUsePDL)(
minimax_decode_topk_page_table_kernel<SeqLenT, kUsePDL, TopKTrait>,
static_cast<const float*>(score.data_ptr()),
static_cast<const SeqLenT*>(seq_lens.data_ptr()),
static_cast<const int32_t*>(req_to_token.data_ptr()),
static_cast<const int64_t*>(slot_ids.data_ptr()),
static_cast<int32_t*>(page_table.data_ptr()),
static_cast<int32_t*>(seq_lens_out.data_ptr()),
batch,
num_heads,
max_seqblock,
static_cast<int>(block_size),
static_cast<int>(topk),
static_cast<int>(page_size),
r2t_stride,
max_kv_len,
max_sparse_pages);
auto launch = [&](auto kernel) {
LaunchKernel(grid, TopKTrait::kCTASize, device, 0)
.enable_pdl(kUsePDL)(
kernel,
static_cast<const float*>(score.data_ptr()),
static_cast<const SeqLenT*>(seq_lens.data_ptr()),
static_cast<const int32_t*>(req_to_token.data_ptr()),
static_cast<const int64_t*>(slot_ids.data_ptr()),
static_cast<int32_t*>(page_table.data_ptr()),
static_cast<int32_t*>(seq_lens_out.data_ptr()),
batch,
num_heads,
max_seqblock,
static_cast<int>(block_size),
static_cast<int>(topk),
static_cast<int>(page_size),
r2t_stride,
max_kv_len,
max_sparse_pages);
};
#ifdef USE_ROCM
launch(minimax_decode_topk_page_table_kernel<SeqLenT, kUsePDL, TopKTrait>);
#else
switch (topk_bucket_for(max_seqblock)) {
case TopKTrait::kMinNumBlocksBucket:
launch(minimax_decode_topk_page_table_kernel<SeqLenT, kUsePDL, TopKTrait, TopKTrait::kMinNumBlocksBucket>);
break;
case 2 * TopKTrait::kMinNumBlocksBucket:
launch(minimax_decode_topk_page_table_kernel<SeqLenT, kUsePDL, TopKTrait, 2 * TopKTrait::kMinNumBlocksBucket>);
break;
default:
launch(minimax_decode_topk_page_table_kernel<SeqLenT, kUsePDL, TopKTrait, TopKTrait::kMaxNumBlocks>);
break;
}
#endif
}
} // namespace sglang
@@ -30,7 +30,7 @@ if TYPE_CHECKING:
from tvm_ffi.module import Module
# Must match TopKTrait::kMaxNumBlocks in minimax_decode_topk.cuh.
_MAX_NUM_BLOCKS = 16384 if torch.version.hip else 4096
_MAX_NUM_BLOCKS = 16384
@cache_once
@@ -18,8 +18,9 @@ from ..common.utils import (
unit_scale,
)
# Must match _MAX_NUM_BLOCKS in ops/attention/minimax_decode_topk.py.
_JIT_TOPK_MAX_NUM_BLOCKS = 16384 if torch.version.hip else 4096
# Largest row the single-stage top-k takes: kMaxNumBlocks of sgl-kernel-xpu's own
# kernel on XPU, else _MAX_NUM_BLOCKS in ops/attention/minimax_decode_topk.py.
_JIT_TOPK_MAX_NUM_BLOCKS = 4096 if is_xpu() else 16384
def _prune_decode_configs(configs, named_args, **kwargs):
@@ -9,7 +9,10 @@ the consumer ``_gqa_share_sparse_decode_kernel`` contract.
import pytest
import torch
from sglang.kernels.ops.attention.minimax_decode_topk import minimax_decode_topk
from sglang.kernels.ops.attention.minimax_decode_topk import (
_jit_module,
minimax_decode_topk,
)
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
register_cuda_ci(est_time=40, stage="base-b-kernel-unit", runner_config="1-gpu-large")
@@ -64,9 +67,7 @@ def _check_contract(out, seq_lens, block_size, topk, S):
@pytest.mark.parametrize("H", [1, 2])
@pytest.mark.parametrize("B", [1, 5, 32])
@pytest.mark.parametrize("topk", [16, 32, 64])
@pytest.mark.parametrize(
"max_ctx", [4096, 131072, 524288] + ([2097152] if torch.version.hip else [])
)
@pytest.mark.parametrize("max_ctx", [4096, 131072, 524288, 2097152])
def test_decode_topk_distinct(dtype_sl, H, B, topk, max_ctx):
torch.manual_seed(1234)
block_size = 128
@@ -131,6 +132,53 @@ def test_decode_topk_small_num_blocks(seq_len):
assert out[0, 0, :nb].tolist() == list(range(nb))
@pytest.mark.parametrize("S", [4096, 4097, 8192, 8193, 16384])
def test_decode_topk_register_buckets(S):
"""Cover both sides of every register-bucket boundary.
On CUDA the radix path is instantiated per bucket (4096 / 8192 / 16384 rows),
so a row one past a bucket selects the next instantiation. The other cases only
reach the 4096 and 16384 buckets, and only at their exact capacities.
"""
block_size, topk = 128, 16
H, B = 1, 1
torch.manual_seed(S)
score = torch.randn(H, B, S, dtype=torch.float32, device="cuda")
seq_lens = torch.tensor([S * block_size], device="cuda", dtype=torch.int32)
out = minimax_decode_topk(score, seq_lens, block_size, topk)
_check_contract(out, seq_lens, block_size, topk, S)
# Tie-robust: compare the selected scores, not the indices.
for got, want in zip(
_selected_scores_sorted(score, out),
_selected_scores_sorted(score, _ref(score, seq_lens, block_size, topk)),
):
torch.testing.assert_close(got, want)
sel = out[0, 0]
sel = sel[sel >= 0]
assert torch.equal(sel, torch.sort(sel).values), f"not ascending: {sel}"
def test_decode_topk_rejects_over_cap():
"""Above the largest bucket both the wrapper and the kernel must raise.
The CUDA launcher sends every row past 8192 blocks to the 16384 bucket, so the
kernel's own cap check is what keeps a direct caller from silently selecting
from the first 16384 blocks; the wrapper asserts the same bound first.
"""
block_size, topk = 128, 16
S = 16385
score = torch.randn(1, 1, S, dtype=torch.float32, device="cuda")
seq_lens = torch.tensor([S * block_size], device="cuda", dtype=torch.int32)
with pytest.raises(AssertionError, match="exceeds kMaxNumBlocks"):
minimax_decode_topk(score, seq_lens, block_size, topk)
out = torch.empty(1, 1, topk, dtype=torch.int32, device="cuda")
with pytest.raises(RuntimeError, match="exceeds kMaxNumBlocks"):
_jit_module(seq_lens.dtype).minimax_decode_topk(
score, seq_lens, out, block_size, topk
)
def test_decode_topk_out_param():
block_size = 128
H, B, S, topk = 1, 4, 1024, 16
@@ -57,7 +57,7 @@ def _effective_kv_from_selection(ti, seq_lens, block):
(16, 8000),
(2, 40000), # num_blocks=313 -> medium radix path
(1, 90000), # num_blocks=704 -> large compaction path
(1, 480000), # num_blocks=3750 -> large path near kMaxNumBlocks
(1, 480000), # num_blocks=3750 -> large path near the 4096 bucket
],
)
@pytest.mark.parametrize("nqh", [8, 16])
@@ -119,7 +119,16 @@ def test_fused_page_table_matches_custom(bs, seq_len, nqh):
assert cos > 0.999, f"cos={cos}"
@pytest.mark.parametrize("seq_len", [300, 5000, 90000])
@pytest.mark.parametrize(
"seq_len",
[
300,
5000,
90000,
600000, # num_blocks=4688 -> 8192 bucket
1048577, # num_blocks=8193 -> 16384 bucket
],
)
@pytest.mark.parametrize("bs", [1, 3])
@pytest.mark.parametrize("nkv", [2, 4])
def test_dp_flattened_page_table(nkv, bs, seq_len):