diff --git a/python/sglang/kernels/jit/csrc/minimax/minimax_decode_topk.cuh b/python/sglang/kernels/jit/csrc/minimax/minimax_decode_topk.cuh index f5210ee143..24932b20ba 100644 --- a/python/sglang/kernels/jit/csrc/minimax/minimax_decode_topk.cuh +++ b/python/sglang/kernels/jit/csrc/minimax/minimax_decode_topk.cuh @@ -42,10 +42,11 @@ struct TopKTraitImpl { static constexpr uint32_t kCTASize = kNumThreads; static constexpr uint32_t kNumWarps = kCTASize / device::kWarpThreads; // Must match _MAX_NUM_BLOCKS in ops/attention/minimax_decode_topk.py. -#ifdef USE_ROCM static constexpr uint32_t kMaxNumBlocks = 16384; // block topk -#else - static constexpr uint32_t kMaxNumBlocks = 4096; // block topk +#ifndef USE_ROCM + // Smallest register bucket for the multi-element radix path; the launchers + // instantiate larger ones on demand up to kMaxNumBlocks. + static constexpr uint32_t kMinNumBlocksBucket = 4096; #endif static constexpr uint32_t kSmallThreshold = 8 * kNumWarps; static constexpr uint32_t kRadixBits = 8; @@ -301,6 +302,9 @@ struct TopKTraitImpl { } #endif // USE_ROCM + // kMaxBlocks is the CUDA register bucket; ROCm sizes its slots from + // num_blocks at run time instead. + template SGL_DEVICE static void forward( const float* __restrict__ scores, const uint32_t num_blocks, @@ -455,15 +459,16 @@ struct TopKTraitImpl { if (write_pos < topk) topk_out[write_pos] = tx; } else { - // num_blocks in (kCTASize, kMaxNumBlocks]: each thread caches its (up to + // num_blocks in (kCTASize, kMaxBlocks]: each thread caches its (up to // kIters) slice of the row in registers -- read from global exactly ONCE -- // then runs the same 4-pass radix select as the single-element path looped // over those slots. Liveness is a uint32_t bitmask (bit i = slot i still in // the running set), so there is no per-element flag array; selection is an // in-loop scatter, so there is no per-element position array. Nothing is // cached in shared memory beyond the histogram. - constexpr uint32_t kIters = kMaxNumBlocks / kCTASize; + constexpr uint32_t kIters = kMaxBlocks / kCTASize; static_assert(kIters <= 32, "active liveness is packed into a uint32_t"); + static_assert(kMaxBlocks % kCTASize == 0, "bucket must be a multiple of kCTASize"); uint32_t key[kIters]; uint32_t active = 0; #pragma unroll @@ -529,6 +534,15 @@ struct TopKTraitImpl { using TopKTrait = TopKTraitImpl<512>; #ifdef USE_ROCM using TopKTraitWide = TopKTraitImpl<1024>; +#else +// Smallest instantiated bucket covering max_seqblock, so a short row is not +// charged the register footprint of the largest one. +inline uint32_t topk_bucket_for(int max_seqblock) { + const auto n = static_cast(max_seqblock); + if (n <= TopKTrait::kMinNumBlocksBucket) return TopKTrait::kMinNumBlocksBucket; // 4096 + if (n <= 2 * TopKTrait::kMinNumBlocksBucket) return 2 * TopKTrait::kMinNumBlocksBucket; // 8192 + return TopKTrait::kMaxNumBlocks; // 16384 +} #endif // ------------------------------------------------------------------------- @@ -541,7 +555,7 @@ using TopKTraitWide = TopKTraitImpl<1024>; // ascending), [k_eff:topk) = -1. Ascending order is a hard requirement of the // MSA fmha_sm100 consumer (kv_block_indexes must be strictly ascending; its // sorted-order early-exit otherwise mis-masks the partial last block). -template +template __global__ void minimax_decode_topk_block_kernel( const float* __restrict__ score, const SeqLenT* __restrict__ seq_lens, @@ -573,7 +587,8 @@ __global__ void minimax_decode_topk_block_kernel( const float* __restrict__ row = score + (static_cast(h) * batch + b) * max_seqblock; __shared__ typename TopKTrait::Smem smem; __shared__ int32_t s_topk[TopKTrait::kMaxTopK]; - TopKTrait::forward(row, static_cast(num_blocks), s_topk, static_cast(topk), &smem); + TopKTrait::template forward( + row, static_cast(num_blocks), s_topk, static_cast(topk), &smem); __syncthreads(); // s_topk fully written before the sort reads it // Emit ascending: num_blocks > topk here, so all topk slots hold distinct @@ -636,7 +651,7 @@ __global__ void minimax_decode_topk_block_kernel( // [num_pages, nkv, page_size, D] reshaped to [num_pages*nkv, 1, page_size, D] (a // free view when the cache is contiguous HND). num_heads == 1 (h == 0) reproduces // the single-kv-head TP>=4 behavior (page index == base_page). -template +template __global__ void minimax_decode_topk_page_table_kernel( const float* __restrict__ score, const SeqLenT* __restrict__ seq_lens, @@ -686,7 +701,8 @@ __global__ void minimax_decode_topk_page_table_kernel( const float* __restrict__ row = score + (static_cast(h) * batch + b) * max_seqblock; // head-major score __shared__ typename TopKTrait::Smem smem; __shared__ int32_t s_topk[TopKTrait::kMaxTopK]; - TopKTrait::forward(row, static_cast(num_blocks), s_topk, static_cast(topk), &smem); + TopKTrait::template forward( + row, static_cast(num_blocks), s_topk, static_cast(topk), &smem); __syncthreads(); // s_topk fully written before the transform reads it // Sort the selected block ids ascending (k_eff <= kMaxTopK is tiny) so the @@ -786,17 +802,34 @@ void minimax_decode_topk( return; } #endif - LaunchKernel(grid, TopKTrait::kCTASize, device, 0) - .enable_pdl(kUsePDL)( - minimax_decode_topk_block_kernel, - static_cast(score.data_ptr()), - static_cast(seq_lens.data_ptr()), - static_cast(topk_idx.data_ptr()), - batch, - num_heads, - max_seqblock, - static_cast(block_size), - topk_i); + auto launch = [&](auto kernel) { + LaunchKernel(grid, TopKTrait::kCTASize, device, 0) + .enable_pdl(kUsePDL)( + kernel, + static_cast(score.data_ptr()), + static_cast(seq_lens.data_ptr()), + static_cast(topk_idx.data_ptr()), + batch, + num_heads, + max_seqblock, + static_cast(block_size), + topk_i); + }; +#ifdef USE_ROCM + launch(minimax_decode_topk_block_kernel); +#else + switch (topk_bucket_for(max_seqblock)) { + case TopKTrait::kMinNumBlocksBucket: + launch(minimax_decode_topk_block_kernel); + break; + case 2 * TopKTrait::kMinNumBlocksBucket: + launch(minimax_decode_topk_block_kernel); + break; + default: + launch(minimax_decode_topk_block_kernel); + break; + } +#endif } // Page-table variant: emit the per-(batch, kv-head) paged page table consumed by @@ -891,24 +924,41 @@ void minimax_decode_topk_page_table( return; } #endif - LaunchKernel(grid, TopKTrait::kCTASize, device, 0) - .enable_pdl(kUsePDL)( - minimax_decode_topk_page_table_kernel, - static_cast(score.data_ptr()), - static_cast(seq_lens.data_ptr()), - static_cast(req_to_token.data_ptr()), - static_cast(slot_ids.data_ptr()), - static_cast(page_table.data_ptr()), - static_cast(seq_lens_out.data_ptr()), - batch, - num_heads, - max_seqblock, - static_cast(block_size), - static_cast(topk), - static_cast(page_size), - r2t_stride, - max_kv_len, - max_sparse_pages); + auto launch = [&](auto kernel) { + LaunchKernel(grid, TopKTrait::kCTASize, device, 0) + .enable_pdl(kUsePDL)( + kernel, + static_cast(score.data_ptr()), + static_cast(seq_lens.data_ptr()), + static_cast(req_to_token.data_ptr()), + static_cast(slot_ids.data_ptr()), + static_cast(page_table.data_ptr()), + static_cast(seq_lens_out.data_ptr()), + batch, + num_heads, + max_seqblock, + static_cast(block_size), + static_cast(topk), + static_cast(page_size), + r2t_stride, + max_kv_len, + max_sparse_pages); + }; +#ifdef USE_ROCM + launch(minimax_decode_topk_page_table_kernel); +#else + switch (topk_bucket_for(max_seqblock)) { + case TopKTrait::kMinNumBlocksBucket: + launch(minimax_decode_topk_page_table_kernel); + break; + case 2 * TopKTrait::kMinNumBlocksBucket: + launch(minimax_decode_topk_page_table_kernel); + break; + default: + launch(minimax_decode_topk_page_table_kernel); + break; + } +#endif } } // namespace sglang diff --git a/python/sglang/kernels/ops/attention/minimax_decode_topk.py b/python/sglang/kernels/ops/attention/minimax_decode_topk.py index 6a895eb47b..7800f0c2e2 100644 --- a/python/sglang/kernels/ops/attention/minimax_decode_topk.py +++ b/python/sglang/kernels/ops/attention/minimax_decode_topk.py @@ -30,7 +30,7 @@ if TYPE_CHECKING: from tvm_ffi.module import Module # Must match TopKTrait::kMaxNumBlocks in minimax_decode_topk.cuh. -_MAX_NUM_BLOCKS = 16384 if torch.version.hip else 4096 +_MAX_NUM_BLOCKS = 16384 @cache_once diff --git a/python/sglang/kernels/ops/attention/minimax_sparse/decode/flash_with_topk_idx.py b/python/sglang/kernels/ops/attention/minimax_sparse/decode/flash_with_topk_idx.py index 471cace6c4..c2fa2a8fdc 100644 --- a/python/sglang/kernels/ops/attention/minimax_sparse/decode/flash_with_topk_idx.py +++ b/python/sglang/kernels/ops/attention/minimax_sparse/decode/flash_with_topk_idx.py @@ -18,8 +18,9 @@ from ..common.utils import ( unit_scale, ) -# Must match _MAX_NUM_BLOCKS in ops/attention/minimax_decode_topk.py. -_JIT_TOPK_MAX_NUM_BLOCKS = 16384 if torch.version.hip else 4096 +# Largest row the single-stage top-k takes: kMaxNumBlocks of sgl-kernel-xpu's own +# kernel on XPU, else _MAX_NUM_BLOCKS in ops/attention/minimax_decode_topk.py. +_JIT_TOPK_MAX_NUM_BLOCKS = 4096 if is_xpu() else 16384 def _prune_decode_configs(configs, named_args, **kwargs): diff --git a/test/registered/kernels/ops/attention/test_minimax_decode_topk.py b/test/registered/kernels/ops/attention/test_minimax_decode_topk.py index aeaf103f08..5f464180bd 100644 --- a/test/registered/kernels/ops/attention/test_minimax_decode_topk.py +++ b/test/registered/kernels/ops/attention/test_minimax_decode_topk.py @@ -9,7 +9,10 @@ the consumer ``_gqa_share_sparse_decode_kernel`` contract. import pytest import torch -from sglang.kernels.ops.attention.minimax_decode_topk import minimax_decode_topk +from sglang.kernels.ops.attention.minimax_decode_topk import ( + _jit_module, + minimax_decode_topk, +) from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci register_cuda_ci(est_time=40, stage="base-b-kernel-unit", runner_config="1-gpu-large") @@ -64,9 +67,7 @@ def _check_contract(out, seq_lens, block_size, topk, S): @pytest.mark.parametrize("H", [1, 2]) @pytest.mark.parametrize("B", [1, 5, 32]) @pytest.mark.parametrize("topk", [16, 32, 64]) -@pytest.mark.parametrize( - "max_ctx", [4096, 131072, 524288] + ([2097152] if torch.version.hip else []) -) +@pytest.mark.parametrize("max_ctx", [4096, 131072, 524288, 2097152]) def test_decode_topk_distinct(dtype_sl, H, B, topk, max_ctx): torch.manual_seed(1234) block_size = 128 @@ -131,6 +132,53 @@ def test_decode_topk_small_num_blocks(seq_len): assert out[0, 0, :nb].tolist() == list(range(nb)) +@pytest.mark.parametrize("S", [4096, 4097, 8192, 8193, 16384]) +def test_decode_topk_register_buckets(S): + """Cover both sides of every register-bucket boundary. + + On CUDA the radix path is instantiated per bucket (4096 / 8192 / 16384 rows), + so a row one past a bucket selects the next instantiation. The other cases only + reach the 4096 and 16384 buckets, and only at their exact capacities. + """ + block_size, topk = 128, 16 + H, B = 1, 1 + torch.manual_seed(S) + score = torch.randn(H, B, S, dtype=torch.float32, device="cuda") + seq_lens = torch.tensor([S * block_size], device="cuda", dtype=torch.int32) + + out = minimax_decode_topk(score, seq_lens, block_size, topk) + _check_contract(out, seq_lens, block_size, topk, S) + # Tie-robust: compare the selected scores, not the indices. + for got, want in zip( + _selected_scores_sorted(score, out), + _selected_scores_sorted(score, _ref(score, seq_lens, block_size, topk)), + ): + torch.testing.assert_close(got, want) + sel = out[0, 0] + sel = sel[sel >= 0] + assert torch.equal(sel, torch.sort(sel).values), f"not ascending: {sel}" + + +def test_decode_topk_rejects_over_cap(): + """Above the largest bucket both the wrapper and the kernel must raise. + + The CUDA launcher sends every row past 8192 blocks to the 16384 bucket, so the + kernel's own cap check is what keeps a direct caller from silently selecting + from the first 16384 blocks; the wrapper asserts the same bound first. + """ + block_size, topk = 128, 16 + S = 16385 + score = torch.randn(1, 1, S, dtype=torch.float32, device="cuda") + seq_lens = torch.tensor([S * block_size], device="cuda", dtype=torch.int32) + with pytest.raises(AssertionError, match="exceeds kMaxNumBlocks"): + minimax_decode_topk(score, seq_lens, block_size, topk) + out = torch.empty(1, 1, topk, dtype=torch.int32, device="cuda") + with pytest.raises(RuntimeError, match="exceeds kMaxNumBlocks"): + _jit_module(seq_lens.dtype).minimax_decode_topk( + score, seq_lens, out, block_size, topk + ) + + def test_decode_topk_out_param(): block_size = 128 H, B, S, topk = 1, 4, 1024, 16 diff --git a/test/registered/kernels/ops/attention/test_minimax_decode_topk_page_table.py b/test/registered/kernels/ops/attention/test_minimax_decode_topk_page_table.py index 23200ab49b..35c1be4f04 100644 --- a/test/registered/kernels/ops/attention/test_minimax_decode_topk_page_table.py +++ b/test/registered/kernels/ops/attention/test_minimax_decode_topk_page_table.py @@ -57,7 +57,7 @@ def _effective_kv_from_selection(ti, seq_lens, block): (16, 8000), (2, 40000), # num_blocks=313 -> medium radix path (1, 90000), # num_blocks=704 -> large compaction path - (1, 480000), # num_blocks=3750 -> large path near kMaxNumBlocks + (1, 480000), # num_blocks=3750 -> large path near the 4096 bucket ], ) @pytest.mark.parametrize("nqh", [8, 16]) @@ -119,7 +119,16 @@ def test_fused_page_table_matches_custom(bs, seq_len, nqh): assert cos > 0.999, f"cos={cos}" -@pytest.mark.parametrize("seq_len", [300, 5000, 90000]) +@pytest.mark.parametrize( + "seq_len", + [ + 300, + 5000, + 90000, + 600000, # num_blocks=4688 -> 8192 bucket + 1048577, # num_blocks=8193 -> 16384 bucket + ], +) @pytest.mark.parametrize("bs", [1, 3]) @pytest.mark.parametrize("nkv", [2, 4]) def test_dp_flattened_page_table(nkv, bs, seq_len):