[MiniMax-M3] Instantiate the decode block top-k in register buckets (#38615)
Co-authored-by: Hao Phan <htphan@nvidia.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
This commit is contained in:
co-authored by
Hao Phan
Claude Opus 5
Po-Han Huang
parent
45abea0269
commit
6e6ccb26f2
@@ -42,10 +42,11 @@ struct TopKTraitImpl {
|
||||
static constexpr uint32_t kCTASize = kNumThreads;
|
||||
static constexpr uint32_t kNumWarps = kCTASize / device::kWarpThreads;
|
||||
// Must match _MAX_NUM_BLOCKS in ops/attention/minimax_decode_topk.py.
|
||||
#ifdef USE_ROCM
|
||||
static constexpr uint32_t kMaxNumBlocks = 16384; // block topk
|
||||
#else
|
||||
static constexpr uint32_t kMaxNumBlocks = 4096; // block topk
|
||||
#ifndef USE_ROCM
|
||||
// Smallest register bucket for the multi-element radix path; the launchers
|
||||
// instantiate larger ones on demand up to kMaxNumBlocks.
|
||||
static constexpr uint32_t kMinNumBlocksBucket = 4096;
|
||||
#endif
|
||||
static constexpr uint32_t kSmallThreshold = 8 * kNumWarps;
|
||||
static constexpr uint32_t kRadixBits = 8;
|
||||
@@ -301,6 +302,9 @@ struct TopKTraitImpl {
|
||||
}
|
||||
#endif // USE_ROCM
|
||||
|
||||
// kMaxBlocks is the CUDA register bucket; ROCm sizes its slots from
|
||||
// num_blocks at run time instead.
|
||||
template <uint32_t kMaxBlocks = kMaxNumBlocks>
|
||||
SGL_DEVICE static void forward(
|
||||
const float* __restrict__ scores,
|
||||
const uint32_t num_blocks,
|
||||
@@ -455,15 +459,16 @@ struct TopKTraitImpl {
|
||||
|
||||
if (write_pos < topk) topk_out[write_pos] = tx;
|
||||
} else {
|
||||
// num_blocks in (kCTASize, kMaxNumBlocks]: each thread caches its (up to
|
||||
// num_blocks in (kCTASize, kMaxBlocks]: each thread caches its (up to
|
||||
// kIters) slice of the row in registers -- read from global exactly ONCE --
|
||||
// then runs the same 4-pass radix select as the single-element path looped
|
||||
// over those slots. Liveness is a uint32_t bitmask (bit i = slot i still in
|
||||
// the running set), so there is no per-element flag array; selection is an
|
||||
// in-loop scatter, so there is no per-element position array. Nothing is
|
||||
// cached in shared memory beyond the histogram.
|
||||
constexpr uint32_t kIters = kMaxNumBlocks / kCTASize;
|
||||
constexpr uint32_t kIters = kMaxBlocks / kCTASize;
|
||||
static_assert(kIters <= 32, "active liveness is packed into a uint32_t");
|
||||
static_assert(kMaxBlocks % kCTASize == 0, "bucket must be a multiple of kCTASize");
|
||||
uint32_t key[kIters];
|
||||
uint32_t active = 0;
|
||||
#pragma unroll
|
||||
@@ -529,6 +534,15 @@ struct TopKTraitImpl {
|
||||
using TopKTrait = TopKTraitImpl<512>;
|
||||
#ifdef USE_ROCM
|
||||
using TopKTraitWide = TopKTraitImpl<1024>;
|
||||
#else
|
||||
// Smallest instantiated bucket covering max_seqblock, so a short row is not
|
||||
// charged the register footprint of the largest one.
|
||||
inline uint32_t topk_bucket_for(int max_seqblock) {
|
||||
const auto n = static_cast<uint32_t>(max_seqblock);
|
||||
if (n <= TopKTrait::kMinNumBlocksBucket) return TopKTrait::kMinNumBlocksBucket; // 4096
|
||||
if (n <= 2 * TopKTrait::kMinNumBlocksBucket) return 2 * TopKTrait::kMinNumBlocksBucket; // 8192
|
||||
return TopKTrait::kMaxNumBlocks; // 16384
|
||||
}
|
||||
#endif
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
@@ -541,7 +555,7 @@ using TopKTraitWide = TopKTraitImpl<1024>;
|
||||
// ascending), [k_eff:topk) = -1. Ascending order is a hard requirement of the
|
||||
// MSA fmha_sm100 consumer (kv_block_indexes must be strictly ascending; its
|
||||
// sorted-order early-exit otherwise mis-masks the partial last block).
|
||||
template <typename SeqLenT, bool kUsePDL, typename TopKTrait>
|
||||
template <typename SeqLenT, bool kUsePDL, typename TopKTrait, uint32_t kMaxBlocks = TopKTrait::kMaxNumBlocks>
|
||||
__global__ void minimax_decode_topk_block_kernel(
|
||||
const float* __restrict__ score,
|
||||
const SeqLenT* __restrict__ seq_lens,
|
||||
@@ -573,7 +587,8 @@ __global__ void minimax_decode_topk_block_kernel(
|
||||
const float* __restrict__ row = score + (static_cast<int64_t>(h) * batch + b) * max_seqblock;
|
||||
__shared__ typename TopKTrait::Smem smem;
|
||||
__shared__ int32_t s_topk[TopKTrait::kMaxTopK];
|
||||
TopKTrait::forward(row, static_cast<uint32_t>(num_blocks), s_topk, static_cast<uint32_t>(topk), &smem);
|
||||
TopKTrait::template forward<kMaxBlocks>(
|
||||
row, static_cast<uint32_t>(num_blocks), s_topk, static_cast<uint32_t>(topk), &smem);
|
||||
__syncthreads(); // s_topk fully written before the sort reads it
|
||||
|
||||
// Emit ascending: num_blocks > topk here, so all topk slots hold distinct
|
||||
@@ -636,7 +651,7 @@ __global__ void minimax_decode_topk_block_kernel(
|
||||
// [num_pages, nkv, page_size, D] reshaped to [num_pages*nkv, 1, page_size, D] (a
|
||||
// free view when the cache is contiguous HND). num_heads == 1 (h == 0) reproduces
|
||||
// the single-kv-head TP>=4 behavior (page index == base_page).
|
||||
template <typename SeqLenT, bool kUsePDL, typename TopKTrait>
|
||||
template <typename SeqLenT, bool kUsePDL, typename TopKTrait, uint32_t kMaxBlocks = TopKTrait::kMaxNumBlocks>
|
||||
__global__ void minimax_decode_topk_page_table_kernel(
|
||||
const float* __restrict__ score,
|
||||
const SeqLenT* __restrict__ seq_lens,
|
||||
@@ -686,7 +701,8 @@ __global__ void minimax_decode_topk_page_table_kernel(
|
||||
const float* __restrict__ row = score + (static_cast<int64_t>(h) * batch + b) * max_seqblock; // head-major score
|
||||
__shared__ typename TopKTrait::Smem smem;
|
||||
__shared__ int32_t s_topk[TopKTrait::kMaxTopK];
|
||||
TopKTrait::forward(row, static_cast<uint32_t>(num_blocks), s_topk, static_cast<uint32_t>(topk), &smem);
|
||||
TopKTrait::template forward<kMaxBlocks>(
|
||||
row, static_cast<uint32_t>(num_blocks), s_topk, static_cast<uint32_t>(topk), &smem);
|
||||
__syncthreads(); // s_topk fully written before the transform reads it
|
||||
|
||||
// Sort the selected block ids ascending (k_eff <= kMaxTopK is tiny) so the
|
||||
@@ -786,17 +802,34 @@ void minimax_decode_topk(
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
LaunchKernel(grid, TopKTrait::kCTASize, device, 0)
|
||||
.enable_pdl(kUsePDL)(
|
||||
minimax_decode_topk_block_kernel<SeqLenT, kUsePDL, TopKTrait>,
|
||||
static_cast<const float*>(score.data_ptr()),
|
||||
static_cast<const SeqLenT*>(seq_lens.data_ptr()),
|
||||
static_cast<int32_t*>(topk_idx.data_ptr()),
|
||||
batch,
|
||||
num_heads,
|
||||
max_seqblock,
|
||||
static_cast<int>(block_size),
|
||||
topk_i);
|
||||
auto launch = [&](auto kernel) {
|
||||
LaunchKernel(grid, TopKTrait::kCTASize, device, 0)
|
||||
.enable_pdl(kUsePDL)(
|
||||
kernel,
|
||||
static_cast<const float*>(score.data_ptr()),
|
||||
static_cast<const SeqLenT*>(seq_lens.data_ptr()),
|
||||
static_cast<int32_t*>(topk_idx.data_ptr()),
|
||||
batch,
|
||||
num_heads,
|
||||
max_seqblock,
|
||||
static_cast<int>(block_size),
|
||||
topk_i);
|
||||
};
|
||||
#ifdef USE_ROCM
|
||||
launch(minimax_decode_topk_block_kernel<SeqLenT, kUsePDL, TopKTrait>);
|
||||
#else
|
||||
switch (topk_bucket_for(max_seqblock)) {
|
||||
case TopKTrait::kMinNumBlocksBucket:
|
||||
launch(minimax_decode_topk_block_kernel<SeqLenT, kUsePDL, TopKTrait, TopKTrait::kMinNumBlocksBucket>);
|
||||
break;
|
||||
case 2 * TopKTrait::kMinNumBlocksBucket:
|
||||
launch(minimax_decode_topk_block_kernel<SeqLenT, kUsePDL, TopKTrait, 2 * TopKTrait::kMinNumBlocksBucket>);
|
||||
break;
|
||||
default:
|
||||
launch(minimax_decode_topk_block_kernel<SeqLenT, kUsePDL, TopKTrait, TopKTrait::kMaxNumBlocks>);
|
||||
break;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
// Page-table variant: emit the per-(batch, kv-head) paged page table consumed by
|
||||
@@ -891,24 +924,41 @@ void minimax_decode_topk_page_table(
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
LaunchKernel(grid, TopKTrait::kCTASize, device, 0)
|
||||
.enable_pdl(kUsePDL)(
|
||||
minimax_decode_topk_page_table_kernel<SeqLenT, kUsePDL, TopKTrait>,
|
||||
static_cast<const float*>(score.data_ptr()),
|
||||
static_cast<const SeqLenT*>(seq_lens.data_ptr()),
|
||||
static_cast<const int32_t*>(req_to_token.data_ptr()),
|
||||
static_cast<const int64_t*>(slot_ids.data_ptr()),
|
||||
static_cast<int32_t*>(page_table.data_ptr()),
|
||||
static_cast<int32_t*>(seq_lens_out.data_ptr()),
|
||||
batch,
|
||||
num_heads,
|
||||
max_seqblock,
|
||||
static_cast<int>(block_size),
|
||||
static_cast<int>(topk),
|
||||
static_cast<int>(page_size),
|
||||
r2t_stride,
|
||||
max_kv_len,
|
||||
max_sparse_pages);
|
||||
auto launch = [&](auto kernel) {
|
||||
LaunchKernel(grid, TopKTrait::kCTASize, device, 0)
|
||||
.enable_pdl(kUsePDL)(
|
||||
kernel,
|
||||
static_cast<const float*>(score.data_ptr()),
|
||||
static_cast<const SeqLenT*>(seq_lens.data_ptr()),
|
||||
static_cast<const int32_t*>(req_to_token.data_ptr()),
|
||||
static_cast<const int64_t*>(slot_ids.data_ptr()),
|
||||
static_cast<int32_t*>(page_table.data_ptr()),
|
||||
static_cast<int32_t*>(seq_lens_out.data_ptr()),
|
||||
batch,
|
||||
num_heads,
|
||||
max_seqblock,
|
||||
static_cast<int>(block_size),
|
||||
static_cast<int>(topk),
|
||||
static_cast<int>(page_size),
|
||||
r2t_stride,
|
||||
max_kv_len,
|
||||
max_sparse_pages);
|
||||
};
|
||||
#ifdef USE_ROCM
|
||||
launch(minimax_decode_topk_page_table_kernel<SeqLenT, kUsePDL, TopKTrait>);
|
||||
#else
|
||||
switch (topk_bucket_for(max_seqblock)) {
|
||||
case TopKTrait::kMinNumBlocksBucket:
|
||||
launch(minimax_decode_topk_page_table_kernel<SeqLenT, kUsePDL, TopKTrait, TopKTrait::kMinNumBlocksBucket>);
|
||||
break;
|
||||
case 2 * TopKTrait::kMinNumBlocksBucket:
|
||||
launch(minimax_decode_topk_page_table_kernel<SeqLenT, kUsePDL, TopKTrait, 2 * TopKTrait::kMinNumBlocksBucket>);
|
||||
break;
|
||||
default:
|
||||
launch(minimax_decode_topk_page_table_kernel<SeqLenT, kUsePDL, TopKTrait, TopKTrait::kMaxNumBlocks>);
|
||||
break;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace sglang
|
||||
|
||||
@@ -30,7 +30,7 @@ if TYPE_CHECKING:
|
||||
from tvm_ffi.module import Module
|
||||
|
||||
# Must match TopKTrait::kMaxNumBlocks in minimax_decode_topk.cuh.
|
||||
_MAX_NUM_BLOCKS = 16384 if torch.version.hip else 4096
|
||||
_MAX_NUM_BLOCKS = 16384
|
||||
|
||||
|
||||
@cache_once
|
||||
|
||||
@@ -18,8 +18,9 @@ from ..common.utils import (
|
||||
unit_scale,
|
||||
)
|
||||
|
||||
# Must match _MAX_NUM_BLOCKS in ops/attention/minimax_decode_topk.py.
|
||||
_JIT_TOPK_MAX_NUM_BLOCKS = 16384 if torch.version.hip else 4096
|
||||
# Largest row the single-stage top-k takes: kMaxNumBlocks of sgl-kernel-xpu's own
|
||||
# kernel on XPU, else _MAX_NUM_BLOCKS in ops/attention/minimax_decode_topk.py.
|
||||
_JIT_TOPK_MAX_NUM_BLOCKS = 4096 if is_xpu() else 16384
|
||||
|
||||
|
||||
def _prune_decode_configs(configs, named_args, **kwargs):
|
||||
|
||||
@@ -9,7 +9,10 @@ the consumer ``_gqa_share_sparse_decode_kernel`` contract.
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.attention.minimax_decode_topk import minimax_decode_topk
|
||||
from sglang.kernels.ops.attention.minimax_decode_topk import (
|
||||
_jit_module,
|
||||
minimax_decode_topk,
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=40, stage="base-b-kernel-unit", runner_config="1-gpu-large")
|
||||
@@ -64,9 +67,7 @@ def _check_contract(out, seq_lens, block_size, topk, S):
|
||||
@pytest.mark.parametrize("H", [1, 2])
|
||||
@pytest.mark.parametrize("B", [1, 5, 32])
|
||||
@pytest.mark.parametrize("topk", [16, 32, 64])
|
||||
@pytest.mark.parametrize(
|
||||
"max_ctx", [4096, 131072, 524288] + ([2097152] if torch.version.hip else [])
|
||||
)
|
||||
@pytest.mark.parametrize("max_ctx", [4096, 131072, 524288, 2097152])
|
||||
def test_decode_topk_distinct(dtype_sl, H, B, topk, max_ctx):
|
||||
torch.manual_seed(1234)
|
||||
block_size = 128
|
||||
@@ -131,6 +132,53 @@ def test_decode_topk_small_num_blocks(seq_len):
|
||||
assert out[0, 0, :nb].tolist() == list(range(nb))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("S", [4096, 4097, 8192, 8193, 16384])
|
||||
def test_decode_topk_register_buckets(S):
|
||||
"""Cover both sides of every register-bucket boundary.
|
||||
|
||||
On CUDA the radix path is instantiated per bucket (4096 / 8192 / 16384 rows),
|
||||
so a row one past a bucket selects the next instantiation. The other cases only
|
||||
reach the 4096 and 16384 buckets, and only at their exact capacities.
|
||||
"""
|
||||
block_size, topk = 128, 16
|
||||
H, B = 1, 1
|
||||
torch.manual_seed(S)
|
||||
score = torch.randn(H, B, S, dtype=torch.float32, device="cuda")
|
||||
seq_lens = torch.tensor([S * block_size], device="cuda", dtype=torch.int32)
|
||||
|
||||
out = minimax_decode_topk(score, seq_lens, block_size, topk)
|
||||
_check_contract(out, seq_lens, block_size, topk, S)
|
||||
# Tie-robust: compare the selected scores, not the indices.
|
||||
for got, want in zip(
|
||||
_selected_scores_sorted(score, out),
|
||||
_selected_scores_sorted(score, _ref(score, seq_lens, block_size, topk)),
|
||||
):
|
||||
torch.testing.assert_close(got, want)
|
||||
sel = out[0, 0]
|
||||
sel = sel[sel >= 0]
|
||||
assert torch.equal(sel, torch.sort(sel).values), f"not ascending: {sel}"
|
||||
|
||||
|
||||
def test_decode_topk_rejects_over_cap():
|
||||
"""Above the largest bucket both the wrapper and the kernel must raise.
|
||||
|
||||
The CUDA launcher sends every row past 8192 blocks to the 16384 bucket, so the
|
||||
kernel's own cap check is what keeps a direct caller from silently selecting
|
||||
from the first 16384 blocks; the wrapper asserts the same bound first.
|
||||
"""
|
||||
block_size, topk = 128, 16
|
||||
S = 16385
|
||||
score = torch.randn(1, 1, S, dtype=torch.float32, device="cuda")
|
||||
seq_lens = torch.tensor([S * block_size], device="cuda", dtype=torch.int32)
|
||||
with pytest.raises(AssertionError, match="exceeds kMaxNumBlocks"):
|
||||
minimax_decode_topk(score, seq_lens, block_size, topk)
|
||||
out = torch.empty(1, 1, topk, dtype=torch.int32, device="cuda")
|
||||
with pytest.raises(RuntimeError, match="exceeds kMaxNumBlocks"):
|
||||
_jit_module(seq_lens.dtype).minimax_decode_topk(
|
||||
score, seq_lens, out, block_size, topk
|
||||
)
|
||||
|
||||
|
||||
def test_decode_topk_out_param():
|
||||
block_size = 128
|
||||
H, B, S, topk = 1, 4, 1024, 16
|
||||
|
||||
@@ -57,7 +57,7 @@ def _effective_kv_from_selection(ti, seq_lens, block):
|
||||
(16, 8000),
|
||||
(2, 40000), # num_blocks=313 -> medium radix path
|
||||
(1, 90000), # num_blocks=704 -> large compaction path
|
||||
(1, 480000), # num_blocks=3750 -> large path near kMaxNumBlocks
|
||||
(1, 480000), # num_blocks=3750 -> large path near the 4096 bucket
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("nqh", [8, 16])
|
||||
@@ -119,7 +119,16 @@ def test_fused_page_table_matches_custom(bs, seq_len, nqh):
|
||||
assert cos > 0.999, f"cos={cos}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("seq_len", [300, 5000, 90000])
|
||||
@pytest.mark.parametrize(
|
||||
"seq_len",
|
||||
[
|
||||
300,
|
||||
5000,
|
||||
90000,
|
||||
600000, # num_blocks=4688 -> 8192 bucket
|
||||
1048577, # num_blocks=8193 -> 16384 bucket
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("bs", [1, 3])
|
||||
@pytest.mark.parametrize("nkv", [2, 4])
|
||||
def test_dp_flattened_page_table(nkv, bs, seq_len):
|
||||
|
||||
Reference in New Issue
Block a user