From c5bf3aafbc229e887eb6a6f01045b8f90a7dee13 Mon Sep 17 00:00:00 2001 From: ormandj Date: Thu, 15 Oct 2026 08:44:58 -0500 Subject: [PATCH] Fix overflow in legacy and v1 DSA top-k selectors (#37625) Co-authored-by: Benjamin Oldenburg --- .../aot/csrc/elementwise/deepseek_v4_topk.cu | 168 +------- .../kernels/aot/csrc/elementwise/topk.cu | 187 +-------- .../sgl_kernel/dsa/legacy_radix_topk.cuh | 239 ++++++++++++ python/sglang/kernels/aot/tests/test_topk.py | 151 +++++++- .../kernels/jit/csrc/deepseek_v4/topk_v1.cuh | 176 +-------- .../sgl_kernel/dsa/legacy_radix_topk.cuh | 238 ++++++++++++ .../test_dsa_kpool_topk_transform.py | 365 ++++++++++++++++++ .../kernels/ops/attention/test_topk_v1.py | 197 ++++++++++ 8 files changed, 1204 insertions(+), 517 deletions(-) create mode 100644 python/sglang/kernels/aot/include/sgl_kernel/dsa/legacy_radix_topk.cuh create mode 100644 python/sglang/kernels/jit/include/sgl_kernel/dsa/legacy_radix_topk.cuh create mode 100644 test/registered/kernels/ops/attention/test_dsa_kpool_topk_transform.py create mode 100644 test/registered/kernels/ops/attention/test_topk_v1.py diff --git a/python/sglang/kernels/aot/csrc/elementwise/deepseek_v4_topk.cu b/python/sglang/kernels/aot/csrc/elementwise/deepseek_v4_topk.cu index 94048a6055..e9ea5d3210 100644 --- a/python/sglang/kernels/aot/csrc/elementwise/deepseek_v4_topk.cu +++ b/python/sglang/kernels/aot/csrc/elementwise/deepseek_v4_topk.cu @@ -24,6 +24,7 @@ limitations under the License. #include #include #include +#include namespace { @@ -62,18 +63,6 @@ struct TopKParams { bool sort_output; }; -__device__ __forceinline__ uint8_t convert_to_uint8(float x) { - __half h = __float2half_rn(x); - uint16_t bits = __half_as_ushort(h); - uint16_t key = (bits & 0x8000) ? static_cast(~bits) : static_cast(bits | 0x8000); - return static_cast(key >> 8); -} - -__device__ __forceinline__ uint32_t convert_to_uint32(float x) { - uint32_t bits = __float_as_uint(x); - return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u); -} - __device__ __forceinline__ int32_t page_to_slot(const int32_t* __restrict__ page_table, uint32_t i, uint32_t page_bits) { const uint32_t mask = (1u << page_bits) - 1u; @@ -104,154 +93,10 @@ __device__ void naive_paged_transform( __device__ void radix_topk(const float* __restrict__ input, int32_t* __restrict__ output, uint32_t length, uint32_t topk) { - constexpr uint32_t RADIX = 256; - constexpr uint32_t BLOCK_SIZE = kBlockSize; - constexpr uint32_t SMEM_INPUT_SIZE = kSMEM / (2 * sizeof(int32_t)); - - alignas(128) __shared__ uint32_t _s_histogram_buf[2][RADIX + 32]; - alignas(128) __shared__ uint32_t s_counter; - alignas(128) __shared__ uint32_t s_threshold_bin_id; - alignas(128) __shared__ uint32_t s_num_input[2]; - alignas(128) __shared__ int32_t s_last_remain; - - extern __shared__ uint32_t s_input_idx[][SMEM_INPUT_SIZE]; - - const uint32_t tx = threadIdx.x; - uint32_t remain_topk = topk; - auto& s_histogram = _s_histogram_buf[0]; - - const auto run_cumsum = [&] { -#pragma unroll 8 - for (int32_t i = 0; i < 8; ++i) { - static_assert(1 << 8 == RADIX); - if (tx < RADIX) { - const auto j = 1 << i; - const auto k = i & 1; - auto value = _s_histogram_buf[k][tx]; - if (tx + j < RADIX) { - value += _s_histogram_buf[k][tx + j]; - } - _s_histogram_buf[k ^ 1][tx] = value; - } - __syncthreads(); - } - }; - - // stage 1: 8bit coarse histogram - if (tx < RADIX + 1) s_histogram[tx] = 0; + ::sglang::device::legacy_radix_topk:: + select(kBlockSize), static_cast(kMaxTopK), static_cast(kSMEM / (2 * sizeof(int32_t)))>( + input, output, 0, static_cast(length), static_cast(topk)); __syncthreads(); - for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) { - const auto bin = convert_to_uint8(input[idx]); - ::atomicAdd(&s_histogram[bin], 1); - } - __syncthreads(); - run_cumsum(); - if (tx < RADIX && s_histogram[tx] > remain_topk && s_histogram[tx + 1] <= remain_topk) { - s_threshold_bin_id = tx; - s_num_input[0] = 0; - s_counter = 0; - } - __syncthreads(); - - { - const auto threshold_bin = s_threshold_bin_id; - remain_topk -= s_histogram[threshold_bin + 1]; - if (remain_topk == 0) { - for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) { - const uint32_t bin = convert_to_uint8(input[idx]); - if (bin > threshold_bin) { - const auto pos = ::atomicAdd(&s_counter, 1); - output[pos] = static_cast(idx); - } - } - __syncthreads(); - return; - } - __syncthreads(); - if (tx < RADIX + 1) s_histogram[tx] = 0; - __syncthreads(); - - for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) { - const float raw_input = input[idx]; - const uint32_t bin = convert_to_uint8(raw_input); - if (bin > threshold_bin) { - const auto pos = ::atomicAdd(&s_counter, 1); - output[pos] = static_cast(idx); - } else if (bin == threshold_bin) { - const auto pos = ::atomicAdd(&s_num_input[0], 1); - if (C10_LIKELY(pos < SMEM_INPUT_SIZE)) { - s_input_idx[0][pos] = idx; - const auto bin32 = convert_to_uint32(raw_input); - const auto sub_bin = (bin32 >> 24) & 0xFF; - ::atomicAdd(&s_histogram[sub_bin], 1); - } - } - } - __syncthreads(); - } - - // stage 2: refine with 8bit radix passes -#pragma unroll 4 - for (int round = 0; round < 4; ++round) { - const auto r_idx = round % 2; - - const auto raw_num_input = s_num_input[r_idx]; - const auto num_input = raw_num_input < SMEM_INPUT_SIZE ? raw_num_input : SMEM_INPUT_SIZE; - - run_cumsum(); - if (tx < RADIX && s_histogram[tx] > remain_topk && s_histogram[tx + 1] <= remain_topk) { - s_threshold_bin_id = tx; - s_num_input[r_idx ^ 1] = 0; - s_last_remain = static_cast(remain_topk - s_histogram[tx + 1]); - } - __syncthreads(); - - const auto threshold_bin = s_threshold_bin_id; - remain_topk -= s_histogram[threshold_bin + 1]; - - if (remain_topk == 0) { - for (uint32_t i = tx; i < num_input; i += BLOCK_SIZE) { - const auto idx = s_input_idx[r_idx][i]; - const auto offset = 24 - round * 8; - const auto bin = (convert_to_uint32(input[idx]) >> offset) & 0xFF; - if (bin > threshold_bin) { - const auto pos = ::atomicAdd(&s_counter, 1); - output[pos] = static_cast(idx); - } - } - __syncthreads(); - break; - } - __syncthreads(); - if (tx < RADIX + 1) s_histogram[tx] = 0; - __syncthreads(); - for (uint32_t i = tx; i < num_input; i += BLOCK_SIZE) { - const auto idx = s_input_idx[r_idx][i]; - const auto raw_input = input[idx]; - const auto offset = 24 - round * 8; - const auto bin = (convert_to_uint32(raw_input) >> offset) & 0xFF; - if (bin > threshold_bin) { - const auto pos = ::atomicAdd(&s_counter, 1); - output[pos] = static_cast(idx); - } else if (bin == threshold_bin) { - if (round == 3) { - const auto pos = ::atomicAdd(&s_last_remain, -1); - if (pos > 0) { - output[topk - pos] = static_cast(idx); - } - } else { - const auto pos = ::atomicAdd(&s_num_input[r_idx ^ 1], 1); - if (C10_LIKELY(pos < SMEM_INPUT_SIZE)) { - s_input_idx[r_idx ^ 1][pos] = idx; - const auto bin32 = convert_to_uint32(raw_input); - const auto sub_bin = (bin32 >> (offset - 8)) & 0xFF; - ::atomicAdd(&s_histogram[sub_bin], 1); - } - } - } - } - __syncthreads(); - } } #ifdef USE_ROCM @@ -366,7 +211,8 @@ __device__ __forceinline__ uint32_t next_pow2_at_least_64(uint32_t x) { __global__ __launch_bounds__(kBlockSize) void deepseek_v4_topk_transform_kernel(const TopKParams params) { const auto bid = blockIdx.x; - const auto seq_len = params.seq_lens[bid]; + const auto raw_seq_len = params.seq_lens[bid]; + const auto seq_len = raw_seq_len < 0 ? 0 : raw_seq_len; const auto topk = params.topk; const auto score_ptr = params.scores + bid * params.score_stride; const auto page_ptr = params.page_table + bid * params.page_table_stride; @@ -433,7 +279,7 @@ __global__ __launch_bounds__(kBlockSize) void deepseek_v4_topk_transform_kernel( for (uint32_t i = threadIdx.x; i < topk; i += kBlockSize) { const auto raw = s_topk_indices[i]; - indices_ptr[i] = page_to_slot(page_ptr, static_cast(raw), params.page_bits); + indices_ptr[i] = raw < 0 ? -1 : page_to_slot(page_ptr, static_cast(raw), params.page_bits); if (raw_indices_ptr != nullptr) { raw_indices_ptr[i] = raw; } diff --git a/python/sglang/kernels/aot/csrc/elementwise/topk.cu b/python/sglang/kernels/aot/csrc/elementwise/topk.cu index 04a2b73be5..51f7664d26 100644 --- a/python/sglang/kernels/aot/csrc/elementwise/topk.cu +++ b/python/sglang/kernels/aot/csrc/elementwise/topk.cu @@ -17,6 +17,7 @@ #include #include #include +#include namespace { @@ -75,180 +76,10 @@ __device__ void naive_topk_transform_ragged( } } -__device__ __forceinline__ auto convert_to_uint8(float x) -> uint8_t { - __half h = __float2half_rn(x); - uint16_t bits = __half_as_ushort(h); - uint16_t key = (bits & 0x8000) ? static_cast(~bits) : static_cast(bits | 0x8000); - return static_cast(key >> 8); -} - -__device__ __forceinline__ auto convert_to_uint32(float x) -> uint32_t { - uint32_t bits = __float_as_uint(x); - return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u); -} - __device__ void fast_topk_cuda_tl(const float* __restrict__ input, int* __restrict__ index, int row_start, int length) { - // An optimized topk kernel copied from tilelang kernel - // We assume length > TopK here, or it will crash - int topk = TopK; - constexpr auto BLOCK_SIZE = 1024; - constexpr auto RADIX = 256; - constexpr auto SMEM_INPUT_SIZE = kSmem / (2 * sizeof(int)); - - alignas(128) __shared__ int s_histogram_buf[2][RADIX + 128]; - alignas(128) __shared__ int s_counter; - alignas(128) __shared__ int s_threshold_bin_id; - alignas(128) __shared__ int s_num_input[2]; - - auto& s_histogram = s_histogram_buf[0]; - // allocate for two rounds - extern __shared__ int s_input_idx[][SMEM_INPUT_SIZE]; - - const int tx = threadIdx.x; - - // stage 1: 8bit coarse histogram - if (tx < RADIX + 1) s_histogram[tx] = 0; + ::sglang::device::legacy_radix_topk::select(kSmem / (2 * sizeof(int)))>( + input, index, row_start, length, TopK); __syncthreads(); - - for (int idx = tx; idx < length; idx += BLOCK_SIZE) { - const auto bin = convert_to_uint8(input[idx + row_start]); - ::atomicAdd(&s_histogram[bin], 1); - } - __syncthreads(); - - const auto run_cumsum = [&] { -#pragma unroll 8 - for (int i = 0; i < 8; ++i) { - static_assert(1 << 8 == RADIX); - if (C10_LIKELY(tx < RADIX)) { - const auto j = 1 << i; - const auto k = i & 1; - auto value = s_histogram_buf[k][tx]; - if (tx < RADIX - j) { - value += s_histogram_buf[k][tx + j]; - } - s_histogram_buf[k ^ 1][tx] = value; - } - __syncthreads(); - } - }; - - run_cumsum(); - if (tx < RADIX && s_histogram[tx] > topk && s_histogram[tx + 1] <= topk) { - s_threshold_bin_id = tx; - s_num_input[0] = 0; - s_counter = 0; - } - __syncthreads(); - - const auto threshold_bin = s_threshold_bin_id; - topk -= s_histogram[threshold_bin + 1]; - - if (topk == 0) { - for (int idx = tx; idx < length; idx += BLOCK_SIZE) { - const auto bin = static_cast(convert_to_uint8(input[idx + row_start])); - if (bin > threshold_bin) { - const auto pos = ::atomicAdd(&s_counter, 1); - index[pos] = idx; - } - } - __syncthreads(); - return; - } else { - __syncthreads(); - if (tx < RADIX + 1) { - s_histogram[tx] = 0; - } - __syncthreads(); - - for (int idx = tx; idx < length; idx += BLOCK_SIZE) { - const auto raw_input = input[idx + row_start]; - const auto bin = static_cast(convert_to_uint8(raw_input)); - if (bin > threshold_bin) { - const auto pos = ::atomicAdd(&s_counter, 1); - index[pos] = idx; - } else if (bin == threshold_bin) { - const auto pos = ::atomicAdd(&s_num_input[0], 1); - /// NOTE: (dark) fuse the histogram computation here - if (C10_LIKELY(pos < SMEM_INPUT_SIZE)) { - s_input_idx[0][pos] = idx; - const auto bin = convert_to_uint32(raw_input); - const auto sub_bin = (bin >> 24) & 0xFF; - ::atomicAdd(&s_histogram[sub_bin], 1); - } - } - } - __syncthreads(); - } - - // stage 2: refine with 8bit radix passes -#pragma unroll 4 - for (int round = 0; round < 4; ++round) { - __shared__ int s_last_remain; - const auto r_idx = round % 2; - - // clip here to prevent overflow - const auto _raw_num_input = s_num_input[r_idx]; - const auto num_input = (_raw_num_input < int(SMEM_INPUT_SIZE)) ? _raw_num_input : int(SMEM_INPUT_SIZE); - - run_cumsum(); - if (tx < RADIX && s_histogram[tx] > topk && s_histogram[tx + 1] <= topk) { - s_threshold_bin_id = tx; - s_num_input[r_idx ^ 1] = 0; - s_last_remain = topk - s_histogram[tx + 1]; - } - __syncthreads(); - - const auto threshold_bin = s_threshold_bin_id; - topk -= s_histogram[threshold_bin + 1]; - - if (topk == 0) { - for (int i = tx; i < num_input; i += BLOCK_SIZE) { - const auto idx = s_input_idx[r_idx][i]; - const auto offset = 24 - round * 8; - const auto bin = (convert_to_uint32(input[idx + row_start]) >> offset) & 0xFF; - if (bin > threshold_bin) { - const auto pos = ::atomicAdd(&s_counter, 1); - index[pos] = idx; - } - } - __syncthreads(); - break; - } else { - __syncthreads(); - if (tx < RADIX + 1) { - s_histogram[tx] = 0; - } - __syncthreads(); - for (int i = tx; i < num_input; i += BLOCK_SIZE) { - const auto idx = s_input_idx[r_idx][i]; - const auto raw_input = input[idx + row_start]; - const auto offset = 24 - round * 8; - const auto bin = (convert_to_uint32(raw_input) >> offset) & 0xFF; - if (bin > threshold_bin) { - const auto pos = ::atomicAdd(&s_counter, 1); - index[pos] = idx; - } else if (bin == threshold_bin) { - if (round == 3) { - const auto pos = ::atomicAdd(&s_last_remain, -1); - if (pos > 0) { - index[TopK - pos] = idx; - } - } else { - const auto pos = ::atomicAdd(&s_num_input[r_idx ^ 1], 1); - if (C10_LIKELY(pos < SMEM_INPUT_SIZE)) { - /// NOTE: (dark) fuse the histogram computation here - s_input_idx[r_idx ^ 1][pos] = idx; - const auto bin = convert_to_uint32(raw_input); - const auto sub_bin = (bin >> (offset - 8)) & 0xFF; - ::atomicAdd(&s_histogram[sub_bin], 1); - } - } - } - } - __syncthreads(); - } - } } __global__ __launch_bounds__(kThreadsPerBlock) // topk @@ -290,10 +121,10 @@ __global__ __launch_bounds__(kThreadsPerBlock) // decode static_assert(TopK / kThreadsPerBlock == 2); const auto idx_0 = tid; const auto pos_0 = s_indices[idx_0]; - dst_page_entry[idx_0] = src_page_entry[pos_0]; + dst_page_entry[idx_0] = pos_0 < 0 ? -1 : src_page_entry[pos_0]; const auto idx_1 = tid + kThreadsPerBlock; const auto pos_1 = s_indices[idx_1]; - dst_page_entry[idx_1] = src_page_entry[pos_1]; + dst_page_entry[idx_1] = pos_1 < 0 ? -1 : src_page_entry[pos_1]; } } @@ -342,10 +173,10 @@ __global__ __launch_bounds__(kThreadsPerBlock) // prefill static_assert(TopK / kThreadsPerBlock == 2); const auto idx_0 = tid; const auto pos_0 = s_indices[idx_0]; - dst_page_entry[idx_0] = src_page_entry[pos_0]; + dst_page_entry[idx_0] = pos_0 < 0 ? -1 : src_page_entry[pos_0]; const auto idx_1 = tid + kThreadsPerBlock; const auto pos_1 = s_indices[idx_1]; - dst_page_entry[idx_1] = src_page_entry[pos_1]; + dst_page_entry[idx_1] = pos_1 < 0 ? -1 : src_page_entry[pos_1]; } } @@ -373,10 +204,10 @@ __global__ __launch_bounds__(kThreadsPerBlock) // prefill, ragged kv static_assert(TopK / kThreadsPerBlock == 2); const auto idx_0 = tid; const auto pos_0 = s_indices[idx_0]; - dst_indices_entry[idx_0] = pos_0 + offset; + dst_indices_entry[idx_0] = pos_0 < 0 ? -1 : pos_0 + offset; const auto idx_1 = tid + kThreadsPerBlock; const auto pos_1 = s_indices[idx_1]; - dst_indices_entry[idx_1] = pos_1 + offset; + dst_indices_entry[idx_1] = pos_1 < 0 ? -1 : pos_1 + offset; } } diff --git a/python/sglang/kernels/aot/include/sgl_kernel/dsa/legacy_radix_topk.cuh b/python/sglang/kernels/aot/include/sgl_kernel/dsa/legacy_radix_topk.cuh new file mode 100644 index 0000000000..a9b15eec64 --- /dev/null +++ b/python/sglang/kernels/aot/include/sgl_kernel/dsa/legacy_radix_topk.cuh @@ -0,0 +1,239 @@ +#pragma once + +#include + +#include +#include + +namespace sglang::device::legacy_radix_topk { + +inline constexpr int kRadix = 256; + +// Coarse bin: high byte of the ordered FP16 value, as in the original selectors. +__device__ __forceinline__ uint8_t coarse_key(float x) { + const __half h = __float2half_rn(x); + const uint16_t bits = __half_as_ushort(h); + const uint16_t key = (bits & 0x8000u) ? static_cast(~bits) : static_cast(bits | 0x8000u); + return static_cast(key >> 8); +} + +__device__ __forceinline__ uint32_t exact_key(float x) { + const uint32_t bits = __float_as_uint(x); + return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u); +} + +// Exact radix top-k over finite scores. A threshold bin that fits the stash +// keeps the original two-stash refinement; a larger one is refined by rescanning +// the input. Unfilled output slots stay -1. +template +__device__ __forceinline__ void +select(const float* __restrict__ input, int32_t* __restrict__ output, int row_start, int length, int topk) { + static_assert(BlockSize >= kRadix + 1, "the block must initialize the complete radix histogram"); + static_assert(OutputCapacity <= StashEntries, "top-k output must fit in one shared-memory stash"); + + alignas(128) __shared__ int histogram_buf[2][kRadix + 1]; + alignas(128) __shared__ int counter; + alignas(128) __shared__ int threshold_bin_id; + alignas(128) __shared__ int num_input[2]; + + extern __shared__ int dynamic_smem[]; + auto* input_idx = reinterpret_cast(dynamic_smem); + auto& histogram = histogram_buf[0]; + const int tx = threadIdx.x; + + for (int i = tx; i < topk; i += BlockSize) + output[i] = -1; + if (tx < kRadix + 1) histogram[tx] = 0; + if (tx == 0) { + counter = 0; + threshold_bin_id = -1; + num_input[0] = 0; + num_input[1] = 0; + } + __syncthreads(); + + for (int idx = tx; idx < length; idx += BlockSize) { + atomicAdd(&histogram[coarse_key(input[row_start + idx])], 1); + } + __syncthreads(); + + const auto run_cumsum = [&] { +#pragma unroll 8 + for (int i = 0; i < 8; ++i) { + if (tx < kRadix) { + const int offset = 1 << i; + const int src = i & 1; + int value = histogram_buf[src][tx]; + if (tx < kRadix - offset) value += histogram_buf[src][tx + offset]; + histogram_buf[src ^ 1][tx] = value; + } + __syncthreads(); + } + }; + + run_cumsum(); + if (tx < kRadix && histogram[tx] > topk && histogram[tx + 1] <= topk) { + threshold_bin_id = tx; + } + __syncthreads(); + + const int coarse_threshold = threshold_bin_id; + if (coarse_threshold < 0) return; + const int coarse_above = histogram[coarse_threshold + 1]; + int remaining = topk - coarse_above; + + if (remaining == 0) { + for (int idx = tx; idx < length; idx += BlockSize) { + if (coarse_key(input[row_start + idx]) > coarse_threshold) { + const int pos = atomicAdd(&counter, 1); + if (pos < topk) output[pos] = idx; + } + } + __syncthreads(); + return; + } + + const int coarse_population = histogram[coarse_threshold] - histogram[coarse_threshold + 1]; + __syncthreads(); + if (coarse_population >= StashEntries) { + // Find the exact cutoff one key byte per pass. Pass L counts byte L and + // emits what byte L-1 placed strictly above the cutoff; pass 0 emits the + // values above the coarse bin and pass 4 the last strict values and ties. + uint32_t prefix = 0; + int last_threshold = -1; + int strict_count = 0; +#pragma unroll 1 + for (int level = 0; level <= 4; ++level) { + const uint32_t mask = level == 0 ? 0u : (~0u << (32 - level * 8)); + const uint32_t last_mask = level <= 1 ? 0u : (~0u << (40 - level * 8)); + const int last_shift = 32 - level * 8; + if (level < 4) { + if (tx < kRadix + 1) histogram[tx] = 0; + if (tx == 0) threshold_bin_id = -1; + } + __syncthreads(); + + for (int idx = tx; idx < length; idx += BlockSize) { + const float value = input[row_start + idx]; + const int bin = coarse_key(value); + if (bin != coarse_threshold) { + if (level == 0 && bin > coarse_threshold) { + const int pos = atomicAdd(&counter, 1); + if (pos < topk) output[pos] = idx; + } + continue; + } + const uint32_t key = exact_key(value); + if (level > 0 && (key & last_mask) == (prefix & last_mask) && + static_cast((key >> last_shift) & 0xFFu) > last_threshold) { + const int pos = atomicAdd(&num_input[0], 1); + if (pos < strict_count) output[coarse_above + pos] = idx; + } else if ((key & mask) == prefix) { + if (level < 4) { + atomicAdd(&histogram[(key >> (24 - level * 8)) & 0xFFu], 1); + } else { + const int pos = atomicAdd(&num_input[1], 1); + if (pos < remaining) output[coarse_above + strict_count + pos] = idx; + } + } + } + __syncthreads(); + if (level == 4) return; + + run_cumsum(); + if (tx < kRadix && histogram[tx] >= remaining && histogram[tx + 1] < remaining) { + threshold_bin_id = tx; + } + __syncthreads(); + + const int threshold = threshold_bin_id; + if (threshold < 0) return; + const int above = histogram[threshold + 1]; + strict_count += above; + remaining -= above; + prefix |= static_cast(threshold) << (24 - level * 8); + last_threshold = threshold; + // All threads must read the histogram before the next pass clears it. + __syncthreads(); + } + return; + } + + // The threshold bin fits the stash: original two-stash refinement. + if (tx < kRadix + 1) histogram[tx] = 0; + __syncthreads(); + for (int idx = tx; idx < length; idx += BlockSize) { + const float value = input[row_start + idx]; + const int bin = coarse_key(value); + if (bin > coarse_threshold) { + const int pos = atomicAdd(&counter, 1); + if (pos < topk) output[pos] = idx; + } else if (bin == coarse_threshold) { + const int pos = atomicAdd(&num_input[0], 1); + if (pos < StashEntries) { + input_idx[0][pos] = idx; + atomicAdd(&histogram[(exact_key(value) >> 24) & 0xFFu], 1); + } + } + } + __syncthreads(); + +#pragma unroll 4 + for (int round = 0; round < 4; ++round) { + const int current = round & 1; + const int next = current ^ 1; + const int raw_num_input = num_input[current]; + const int current_count = raw_num_input < StashEntries ? raw_num_input : StashEntries; + + if (tx == 0) { + threshold_bin_id = -1; + num_input[next] = 0; + } + run_cumsum(); + if (tx < kRadix && histogram[tx] > remaining && histogram[tx + 1] <= remaining) { + threshold_bin_id = tx; + } + __syncthreads(); + + const int threshold = threshold_bin_id; + if (threshold < 0) return; + const int above = histogram[threshold + 1]; + const int shift = 24 - round * 8; + + __syncthreads(); + if (round < 3 && tx < kRadix + 1) histogram[tx] = 0; + __syncthreads(); + for (int i = tx; i < current_count; i += BlockSize) { + const int idx = input_idx[current][i]; + const float value = input[row_start + idx]; + const int bin = static_cast((exact_key(value) >> shift) & 0xFFu); + if (bin > threshold) { + const int pos = atomicAdd(&counter, 1); + if (pos < topk) output[pos] = idx; + } else if (bin == threshold && round < 3) { + const int pos = atomicAdd(&num_input[next], 1); + if (pos < StashEntries) { + input_idx[next][pos] = idx; + atomicAdd(&histogram[(exact_key(value) >> (shift - 8)) & 0xFFu], 1); + } + } + } + __syncthreads(); + + remaining -= above; + if (remaining == 0) return; + if (round == 3) { + for (int i = tx; i < current_count; i += BlockSize) { + const int idx = input_idx[current][i]; + if (static_cast(exact_key(input[row_start + idx]) & 0xFFu) == threshold) { + const int pos = atomicAdd(&counter, 1); + if (pos < topk) output[pos] = idx; + } + } + __syncthreads(); + return; + } + } +} + +} // namespace sglang::device::legacy_radix_topk diff --git a/python/sglang/kernels/aot/tests/test_topk.py b/python/sglang/kernels/aot/tests/test_topk.py index 4de40917af..73d53614f1 100644 --- a/python/sglang/kernels/aot/tests/test_topk.py +++ b/python/sglang/kernels/aot/tests/test_topk.py @@ -1,5 +1,5 @@ import sys -from typing import Any, Optional +from typing import Optional import pytest import torch @@ -67,6 +67,13 @@ def _ref_torch_transform_ragged_impl( MAX_SEQ_LEN = 131072 +def _tight_cluster(batch: int, width: int) -> torch.Tensor: + """Finite scores that all land in one fp16 coarse radix bin.""" + base = torch.arange(width, dtype=torch.float32, device="cuda") + values = 1.0 + base * (1e-3 / width) + return values.expand(batch, -1).contiguous() + + def assert_equal( score: torch.Tensor, indices_ref: torch.Tensor, @@ -75,29 +82,39 @@ def assert_equal( k: int, seq_len: int, topk_indices_offset: Optional[torch.Tensor] = None, - max_permit_error: int = 0, + row_starts: Optional[torch.Tensor] = None, ): indices_our_cpu = indices_our.cpu().tolist() indices_ref_cpu = indices_ref.cpu().tolist() wrong_values = 0 for i in range(bs): + offset = topk_indices_offset[i].item() if topk_indices_offset is not None else 0 + row_start = row_starts[i].item() if row_starts is not None else 0 indices_ref_set_i = set(indices_ref_cpu[i]) indices_our_set_i = set(indices_our_cpu[i]) + assert len(indices_our_cpu[i]) == k + assert len(indices_our_set_i) == k, "top-k output indices must be distinct" + assert all(0 <= idx - offset < seq_len for idx in indices_our_cpu[i]), ( + "top-k output index is outside the selected window" + ) more = indices_our_set_i - indices_ref_set_i less = indices_ref_set_i - indices_our_set_i - offset = topk_indices_offset[i].item() if topk_indices_offset is not None else 0 if len(more) > 0 or len(less) > 0: # check whether more values are the same with less values # if so, either one is acceptable, since their values are the same - more_values = sorted(score[i, idx - offset].item() for idx in more) - less_values = sorted(score[i, idx - offset].item() for idx in less) + more_values = sorted( + score[i, row_start + idx - offset].item() for idx in more + ) + less_values = sorted( + score[i, row_start + idx - offset].item() for idx in less + ) if more_values != less_values: wrong_values += len(more) print( f"{bs=}, {k=}, {seq_len=}, {i=}, {more=}, {less=} failed, with {more_values=}, {less_values=}" ) - assert wrong_values <= max_permit_error, f"{wrong_values=}, {max_permit_error=}" + assert wrong_values == 0, f"{wrong_values=}" @pytest.mark.parametrize("bs", [1, 132, 256, 4096]) @@ -125,8 +142,7 @@ def test_topk_kernel(bs: int, k: int, seq_len: int, has_row_starts: bool) -> Non indices_ref = torch.sort(indices_ref, dim=-1).values indices_our = torch.sort(indices_our, dim=-1).values - # Tests can pass with max_permit_error=3, set to 5 for safety - assert_equal(score, indices_ref, indices_our, bs, k, seq_len, max_permit_error=5) + assert_equal(score, indices_ref, indices_our, bs, k, seq_len, row_starts=row_starts) @pytest.mark.parametrize("bs", [1, 132, 256, 4096]) @@ -189,7 +205,7 @@ def test_topk_transform_kernel(bs: int, k: int, seq_len: int, mode: str) -> None bs, k, seq_len, - max_permit_error=5, + row_starts=row_starts, ) @@ -245,10 +261,71 @@ def test_topk_transform_ragged_kernel( k, seq_len, topk_indices_offset, - max_permit_error=5, + row_starts, ) +@torch.inference_mode() +def test_legacy_topk_oversized_coarse_bin() -> None: + """The shared legacy selector must not clip an oversized threshold bin.""" + torch.manual_seed(20260903) + batch, seq_len, topk, row_start = 2, 7001, 2048, 3 + scores = torch.full( + (batch, seq_len + row_start), -1000.0, dtype=torch.float32, device="cuda" + ) + scores[:, row_start:] = _tight_cluster(batch, seq_len) + lengths = torch.full((batch,), seq_len, dtype=torch.int32, device="cuda") + row_starts = torch.full((batch,), row_start, dtype=torch.int32, device="cuda") + reference = _ref_torch_impl(scores, seq_len, topk, row_starts=row_starts) + + direct = fast_topk_v2(scores, lengths, topk, row_starts=row_starts) + assert_equal(scores[:, row_start:], reference, direct, batch, topk, seq_len) + + offsets = torch.tensor([17, 4099], dtype=torch.int32, device="cuda") + ragged = fast_topk_transform_ragged_fused( + score=scores, + lengths=lengths, + topk_indices_offset=offsets, + topk=topk, + row_starts=row_starts, + ) + assert_equal( + scores[:, row_start:], + reference + offsets[:, None], + ragged, + batch, + topk, + seq_len, + offsets, + ) + + page_table = torch.arange(seq_len, dtype=torch.int32, device="cuda").expand( + batch, -1 + ) + cu_seqlens_q = torch.arange(batch + 1, dtype=torch.int32, device="cuda") + paged = fast_topk_transform_fused( + score=scores, + lengths=lengths, + page_table_size_1=page_table, + cu_seqlens_q=cu_seqlens_q, + topk=topk, + row_starts=row_starts, + ) + assert_equal(scores[:, row_start:], reference, paged, batch, topk, seq_len) + + # With one query per row and no row starts, this interface selects the + # separate decode kernel rather than the prefill kernel above. + decode_scores = scores[:, row_start:].contiguous() + decode = fast_topk_transform_fused( + score=decode_scores, + lengths=lengths, + page_table_size_1=page_table, + cu_seqlens_q=cu_seqlens_q, + topk=topk, + ) + assert_equal(decode_scores, reference, decode, batch, topk, seq_len) + + @pytest.mark.skipif( torch.version.hip is None, reason="deepseek_v4_topk_transform_512 is only built on ROCm", @@ -499,5 +576,59 @@ def test_topk_variants_are_exact_on_variable_windows( assert_exact_rows(score, mapped_raw, lengths, k) +@pytest.mark.skipif( + torch.version.hip is None, + reason="deepseek_v4_topk_transform_512 is only built on ROCm", +) +@torch.inference_mode() +def test_deepseek_v4_topk_oversized_coarse_bin_and_idle_row() -> None: + """Oversized bins select exactly; a negative DP-idle length emits padding.""" + from sgl_kernel import deepseek_v4_topk_transform_512 + + torch.manual_seed(20260903) + active_batch, batch, c4_len, topk, page_size = 2, 3, 7001, 1024, 64 + scores = _tight_cluster(batch, c4_len) + seq_lens = torch.tensor( + [c4_len] * active_batch + [-1], dtype=torch.int32, device="cuda" + ) + num_pages = (c4_len + page_size - 1) // page_size + page_table = ( + torch.arange(num_pages, dtype=torch.int32, device="cuda") + .expand(batch, -1) + .contiguous() + ) + page_indices = torch.full((batch, topk), -2, dtype=torch.int32, device="cuda") + raw_indices = torch.full_like(page_indices, -2) + + deepseek_v4_topk_transform_512( + scores, + seq_lens, + page_table, + page_indices, + page_size, + raw_indices, + ) + + reference = torch.topk(scores[:active_batch], topk, dim=-1, sorted=False).indices + assert_equal( + scores[:active_batch], + reference, + raw_indices[:active_batch], + active_batch, + topk, + c4_len, + ) + assert_equal( + scores[:active_batch], + reference, + page_indices[:active_batch], + active_batch, + topk, + c4_len, + ) + assert bool((raw_indices[-1] == -1).all()) + assert bool((page_indices[-1] == -1).all()) + + if __name__ == "__main__": sys.exit(pytest.main([__file__])) diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/topk_v1.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/topk_v1.cuh index da3bbaaa4a..df9cf58cee 100644 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/topk_v1.cuh +++ b/python/sglang/kernels/jit/csrc/deepseek_v4/topk_v1.cuh @@ -4,6 +4,8 @@ #include #include +#include + #include #include @@ -39,18 +41,6 @@ struct TopKParams { uint32_t topk; }; -SGL_DEVICE uint8_t convert_to_uint8(float x) { - __half h = __float2half_rn(x); - uint16_t bits = __half_as_ushort(h); - uint16_t key = (bits & 0x8000) ? static_cast(~bits) : static_cast(bits | 0x8000); - return static_cast(key >> 8); -} - -SGL_DEVICE uint32_t convert_to_uint32(float x) { - uint32_t bits = __float_as_uint(x); - return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u); -} - SGL_DEVICE int32_t page_to_indices(const int32_t* __restrict__ page_table, uint32_t i, uint32_t page_bits) { const uint32_t mask = (1u << page_bits) - 1u; return (page_table[i >> page_bits] << page_bits) | (i & mask); @@ -81,160 +71,9 @@ SGL_DEVICE void naive_transform( [[maybe_unused]] SGL_DEVICE void radix_topk(const float* __restrict__ input, int32_t* __restrict__ output, const uint32_t length, const uint32_t topk) { - constexpr uint32_t RADIX = 256; - constexpr uint32_t BLOCK_SIZE = kTopKBlockSize; - constexpr uint32_t SMEM_INPUT_SIZE = kSMEM / (2 * sizeof(int32_t)); - - alignas(128) __shared__ uint32_t _s_histogram_buf[2][RADIX + 32]; - alignas(128) __shared__ uint32_t s_counter; - alignas(128) __shared__ uint32_t s_threshold_bin_id; - alignas(128) __shared__ uint32_t s_num_input[2]; - alignas(128) __shared__ int32_t s_last_remain; - - extern __shared__ uint32_t s_input_idx[][kSMEM / (2 * sizeof(int32_t))]; - - const uint32_t tx = threadIdx.x; - uint32_t remain_topk = topk; - auto& s_histogram = _s_histogram_buf[0]; - - const auto run_cumsum = [&] { -#pragma unroll 8 - for (int32_t i = 0; i < 8; ++i) { - static_assert(1 << 8 == RADIX); - if (tx < RADIX) { - const auto j = 1 << i; - const auto k = i & 1; - auto value = _s_histogram_buf[k][tx]; - if (tx + j < RADIX) { - value += _s_histogram_buf[k][tx + j]; - } - _s_histogram_buf[k ^ 1][tx] = value; - } - __syncthreads(); - } - }; - - // stage 1: 8bit coarse histogram - if (tx < RADIX + 1) s_histogram[tx] = 0; - __syncthreads(); - for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) { - const auto bin = convert_to_uint8(input[idx]); - ::atomicAdd(&s_histogram[bin], 1); - } - __syncthreads(); - run_cumsum(); - if (tx < RADIX && s_histogram[tx] > remain_topk && s_histogram[tx + 1] <= remain_topk) { - s_threshold_bin_id = tx; - s_num_input[0] = 0; - s_counter = 0; - } - __syncthreads(); - - const auto threshold_bin = s_threshold_bin_id; - remain_topk -= s_histogram[threshold_bin + 1]; - if (remain_topk == 0) { - for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) { - const uint32_t bin = convert_to_uint8(input[idx]); - if (bin > threshold_bin) { - const auto pos = ::atomicAdd(&s_counter, 1); - output[pos] = idx; - } - } - __syncthreads(); - return; - } else { - __syncthreads(); - if (tx < RADIX + 1) { - s_histogram[tx] = 0; - } - __syncthreads(); - - for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) { - const float raw_input = input[idx]; - const uint32_t bin = convert_to_uint8(raw_input); - if (bin > threshold_bin) { - const auto pos = ::atomicAdd(&s_counter, 1); - output[pos] = idx; - } else if (bin == threshold_bin) { - const auto pos = ::atomicAdd(&s_num_input[0], 1); - if (pos < SMEM_INPUT_SIZE) { - [[likely]] s_input_idx[0][pos] = idx; - const auto bin = convert_to_uint32(raw_input); - const auto sub_bin = (bin >> 24) & 0xFF; - ::atomicAdd(&s_histogram[sub_bin], 1); - } - } - } - __syncthreads(); - } - - // stage 2: refine with 8bit radix passes -#pragma unroll 4 - for (int round = 0; round < 4; ++round) { - const auto r_idx = round % 2; - - // clip here to prevent overflow - const auto raw_num_input = s_num_input[r_idx]; - const auto num_input = raw_num_input < SMEM_INPUT_SIZE ? raw_num_input : SMEM_INPUT_SIZE; - - run_cumsum(); - if (tx < RADIX && s_histogram[tx] > remain_topk && s_histogram[tx + 1] <= remain_topk) { - s_threshold_bin_id = tx; - s_num_input[r_idx ^ 1] = 0; - s_last_remain = remain_topk - s_histogram[tx + 1]; - } - __syncthreads(); - - const auto threshold_bin = s_threshold_bin_id; - remain_topk -= s_histogram[threshold_bin + 1]; - - if (remain_topk == 0) { - for (uint32_t i = tx; i < num_input; i += BLOCK_SIZE) { - const auto idx = s_input_idx[r_idx][i]; - const auto offset = 24 - round * 8; - const auto bin = (convert_to_uint32(input[idx]) >> offset) & 0xFF; - if (bin > threshold_bin) { - const auto pos = ::atomicAdd(&s_counter, 1); - output[pos] = idx; - } - } - __syncthreads(); - break; - } else { - __syncthreads(); - if (tx < RADIX + 1) { - s_histogram[tx] = 0; - } - __syncthreads(); - for (uint32_t i = tx; i < num_input; i += BLOCK_SIZE) { - const auto idx = s_input_idx[r_idx][i]; - const auto raw_input = input[idx]; - const auto offset = 24 - round * 8; - const auto bin = (convert_to_uint32(raw_input) >> offset) & 0xFF; - if (bin > threshold_bin) { - const auto pos = ::atomicAdd(&s_counter, 1); - output[pos] = idx; - } else if (bin == threshold_bin) { - if (round == 3) { - const auto pos = ::atomicAdd(&s_last_remain, -1); - if (pos > 0) { - output[topk - pos] = idx; - } - } else { - const auto pos = ::atomicAdd(&s_num_input[r_idx ^ 1], 1); - if (pos < SMEM_INPUT_SIZE) { - /// NOTE: (dark) fuse the histogram computation here - [[likely]] s_input_idx[r_idx ^ 1][pos] = idx; - const auto bin = convert_to_uint32(raw_input); - const auto sub_bin = (bin >> (offset - 8)) & 0xFF; - ::atomicAdd(&s_histogram[sub_bin], 1); - } - } - } - } - __syncthreads(); - } - } + ::sglang::device::legacy_radix_topk:: + select(kSMEM / (2 * sizeof(int32_t)))>( + input, output, 0, static_cast(length), static_cast(topk)); } template @@ -261,9 +100,10 @@ __global__ void topk_transform_kernel(const __grid_constant__ TopKParams params) radix_topk(score_ptr, s_topk_indices, seq_len, topk); const auto tx = threadIdx.x; if (tx < topk) { - indices_ptr[tx] = page_to_indices(page_ptr, s_topk_indices[tx], page_bits); + const auto raw = s_topk_indices[tx]; + indices_ptr[tx] = raw < 0 ? -1 : page_to_indices(page_ptr, raw, page_bits); if (raw_indices_ptr != nullptr) { - raw_indices_ptr[tx] = s_topk_indices[tx]; + raw_indices_ptr[tx] = raw; } } } diff --git a/python/sglang/kernels/jit/include/sgl_kernel/dsa/legacy_radix_topk.cuh b/python/sglang/kernels/jit/include/sgl_kernel/dsa/legacy_radix_topk.cuh new file mode 100644 index 0000000000..70cbd2f127 --- /dev/null +++ b/python/sglang/kernels/jit/include/sgl_kernel/dsa/legacy_radix_topk.cuh @@ -0,0 +1,238 @@ +#pragma once + +#include + +#include + +namespace sglang::device::legacy_radix_topk { + +inline constexpr int kRadix = 256; + +// Coarse bin: high byte of the ordered FP16 value, as in the original selectors. +SGL_DEVICE uint8_t coarse_key(float x) { + const __half h = __float2half_rn(x); + const uint16_t bits = __half_as_ushort(h); + const uint16_t key = (bits & 0x8000u) ? static_cast(~bits) : static_cast(bits | 0x8000u); + return static_cast(key >> 8); +} + +SGL_DEVICE uint32_t exact_key(float x) { + const uint32_t bits = __float_as_uint(x); + return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u); +} + +// Exact radix top-k over finite scores. A threshold bin that fits the stash +// keeps the original two-stash refinement; a larger one is refined by rescanning +// the input. Unfilled output slots stay -1. +template +SGL_DEVICE void +select(const float* __restrict__ input, int32_t* __restrict__ output, int row_start, int length, int topk) { + static_assert(BlockSize >= kRadix + 1, "the block must initialize the complete radix histogram"); + static_assert(OutputCapacity <= StashEntries, "top-k output must fit in one shared-memory stash"); + + alignas(128) __shared__ int histogram_buf[2][kRadix + 1]; + alignas(128) __shared__ int counter; + alignas(128) __shared__ int threshold_bin_id; + alignas(128) __shared__ int num_input[2]; + + extern __shared__ int dynamic_smem[]; + auto* input_idx = reinterpret_cast(dynamic_smem); + auto& histogram = histogram_buf[0]; + const int tx = threadIdx.x; + + for (int i = tx; i < topk; i += BlockSize) + output[i] = -1; + if (tx < kRadix + 1) histogram[tx] = 0; + if (tx == 0) { + counter = 0; + threshold_bin_id = -1; + num_input[0] = 0; + num_input[1] = 0; + } + __syncthreads(); + + for (int idx = tx; idx < length; idx += BlockSize) { + atomicAdd(&histogram[coarse_key(input[row_start + idx])], 1); + } + __syncthreads(); + + const auto run_cumsum = [&] { +#pragma unroll 8 + for (int i = 0; i < 8; ++i) { + if (tx < kRadix) { + const int offset = 1 << i; + const int src = i & 1; + int value = histogram_buf[src][tx]; + if (tx < kRadix - offset) value += histogram_buf[src][tx + offset]; + histogram_buf[src ^ 1][tx] = value; + } + __syncthreads(); + } + }; + + run_cumsum(); + if (tx < kRadix && histogram[tx] > topk && histogram[tx + 1] <= topk) { + threshold_bin_id = tx; + } + __syncthreads(); + + const int coarse_threshold = threshold_bin_id; + if (coarse_threshold < 0) return; + const int coarse_above = histogram[coarse_threshold + 1]; + int remaining = topk - coarse_above; + + if (remaining == 0) { + for (int idx = tx; idx < length; idx += BlockSize) { + if (coarse_key(input[row_start + idx]) > coarse_threshold) { + const int pos = atomicAdd(&counter, 1); + if (pos < topk) output[pos] = idx; + } + } + __syncthreads(); + return; + } + + const int coarse_population = histogram[coarse_threshold] - histogram[coarse_threshold + 1]; + __syncthreads(); + if (coarse_population >= StashEntries) { + // Find the exact cutoff one key byte per pass. Pass L counts byte L and + // emits what byte L-1 placed strictly above the cutoff; pass 0 emits the + // values above the coarse bin and pass 4 the last strict values and ties. + uint32_t prefix = 0; + int last_threshold = -1; + int strict_count = 0; +#pragma unroll 1 + for (int level = 0; level <= 4; ++level) { + const uint32_t mask = level == 0 ? 0u : (~0u << (32 - level * 8)); + const uint32_t last_mask = level <= 1 ? 0u : (~0u << (40 - level * 8)); + const int last_shift = 32 - level * 8; + if (level < 4) { + if (tx < kRadix + 1) histogram[tx] = 0; + if (tx == 0) threshold_bin_id = -1; + } + __syncthreads(); + + for (int idx = tx; idx < length; idx += BlockSize) { + const float value = input[row_start + idx]; + const int bin = coarse_key(value); + if (bin != coarse_threshold) { + if (level == 0 && bin > coarse_threshold) { + const int pos = atomicAdd(&counter, 1); + if (pos < topk) output[pos] = idx; + } + continue; + } + const uint32_t key = exact_key(value); + if (level > 0 && (key & last_mask) == (prefix & last_mask) && + static_cast((key >> last_shift) & 0xFFu) > last_threshold) { + const int pos = atomicAdd(&num_input[0], 1); + if (pos < strict_count) output[coarse_above + pos] = idx; + } else if ((key & mask) == prefix) { + if (level < 4) { + atomicAdd(&histogram[(key >> (24 - level * 8)) & 0xFFu], 1); + } else { + const int pos = atomicAdd(&num_input[1], 1); + if (pos < remaining) output[coarse_above + strict_count + pos] = idx; + } + } + } + __syncthreads(); + if (level == 4) return; + + run_cumsum(); + if (tx < kRadix && histogram[tx] >= remaining && histogram[tx + 1] < remaining) { + threshold_bin_id = tx; + } + __syncthreads(); + + const int threshold = threshold_bin_id; + if (threshold < 0) return; + const int above = histogram[threshold + 1]; + strict_count += above; + remaining -= above; + prefix |= static_cast(threshold) << (24 - level * 8); + last_threshold = threshold; + // All threads must read the histogram before the next pass clears it. + __syncthreads(); + } + return; + } + + // The threshold bin fits the stash: original two-stash refinement. + if (tx < kRadix + 1) histogram[tx] = 0; + __syncthreads(); + for (int idx = tx; idx < length; idx += BlockSize) { + const float value = input[row_start + idx]; + const int bin = coarse_key(value); + if (bin > coarse_threshold) { + const int pos = atomicAdd(&counter, 1); + if (pos < topk) output[pos] = idx; + } else if (bin == coarse_threshold) { + const int pos = atomicAdd(&num_input[0], 1); + if (pos < StashEntries) { + input_idx[0][pos] = idx; + atomicAdd(&histogram[(exact_key(value) >> 24) & 0xFFu], 1); + } + } + } + __syncthreads(); + +#pragma unroll 4 + for (int round = 0; round < 4; ++round) { + const int current = round & 1; + const int next = current ^ 1; + const int raw_num_input = num_input[current]; + const int current_count = raw_num_input < StashEntries ? raw_num_input : StashEntries; + + if (tx == 0) { + threshold_bin_id = -1; + num_input[next] = 0; + } + run_cumsum(); + if (tx < kRadix && histogram[tx] > remaining && histogram[tx + 1] <= remaining) { + threshold_bin_id = tx; + } + __syncthreads(); + + const int threshold = threshold_bin_id; + if (threshold < 0) return; + const int above = histogram[threshold + 1]; + const int shift = 24 - round * 8; + + __syncthreads(); + if (round < 3 && tx < kRadix + 1) histogram[tx] = 0; + __syncthreads(); + for (int i = tx; i < current_count; i += BlockSize) { + const int idx = input_idx[current][i]; + const float value = input[row_start + idx]; + const int bin = static_cast((exact_key(value) >> shift) & 0xFFu); + if (bin > threshold) { + const int pos = atomicAdd(&counter, 1); + if (pos < topk) output[pos] = idx; + } else if (bin == threshold && round < 3) { + const int pos = atomicAdd(&num_input[next], 1); + if (pos < StashEntries) { + input_idx[next][pos] = idx; + atomicAdd(&histogram[(exact_key(value) >> (shift - 8)) & 0xFFu], 1); + } + } + } + __syncthreads(); + + remaining -= above; + if (remaining == 0) return; + if (round == 3) { + for (int i = tx; i < current_count; i += BlockSize) { + const int idx = input_idx[current][i]; + if (static_cast(exact_key(input[row_start + idx]) & 0xFFu) == threshold) { + const int pos = atomicAdd(&counter, 1); + if (pos < topk) output[pos] = idx; + } + } + __syncthreads(); + return; + } + } +} + +} // namespace sglang::device::legacy_radix_topk diff --git a/test/registered/kernels/ops/attention/test_dsa_kpool_topk_transform.py b/test/registered/kernels/ops/attention/test_dsa_kpool_topk_transform.py new file mode 100644 index 0000000000..100bb1dcc4 --- /dev/null +++ b/test/registered/kernels/ops/attention/test_dsa_kpool_topk_transform.py @@ -0,0 +1,365 @@ +"""GPU regressions for the DSA kpool pooled radix top-k transform. + +The kernel (``kernels/jit/csrc/dsa/kpool_topk_transform.cuh``) selects the +top ``group_topk`` pool groups of a score row with a two-stage radix select. +Stage-1 threshold bins that fit a 4096-entry shared-memory stash use it; +larger bins are resolved by an exact rescan. These tests +build rows on both sides of that boundary (tight clusters, all-equal rows, +clusters that only separate at a deeper key byte), the stage-1 direct-fill +boundary, and the minimal ``length == K + 1`` row, and compare the selected +groups against ``torch.topk``. + +Comparison is by value multiset: when the K-th value is tied, any subset of +the tied groups is a valid selection, so the sorted selected values must equal +the sorted reference values while the group ids must be distinct and in +range. + +Scores within a row are finite by contract (the kernel's NaN ordering is +unspecified), so no row here contains NaN; ``+inf`` is used only as padding +beyond ``lengths``, where the kernel must not read. +""" + +import unittest +from typing import Optional + +import torch + +from sglang.kernels.ops.attention.dsa.kpool_topk_transform import ( + fast_kpool_topk_transform_fused, +) +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=180, stage="base-b-kernel-unit", runner_config="1-gpu-large") +register_amd_ci(est_time=180, stage="jit-kernel-unit", runner_config="amd") + +# kSmem / (2 * sizeof(int)) in kpool_topk_transform.cuh: entries per stash round. +STASH_ENTRIES = 4096 +POOL_SIZE = 4 +# 512 is the instantiation used by GLM-5.3-Flash (token top-k 2048, pool 4); +# 256 is the module default. +GROUP_TOPKS = (512, 256) + + +def _floats_from_keys(keys: torch.Tensor) -> torch.Tensor: + """Inverse of the kernel's monotone key for positive floats. + + The kernel maps a float ``x`` to ``bits | 0x80000000`` when ``x >= +0``, + so a key with the top bit set corresponds to the float whose raw bits are + the low 31 bits of the key. + """ + assert bool(((keys >> 31) & 1).all()), "positive-float keys have bit 31 set" + bits = (keys & 0x7FFFFFFF).to(torch.int32) + return bits.view(torch.float32) + + +def _coarse_keys(values: torch.Tensor) -> torch.Tensor: + """Mirror the kernel's high-byte ordered-FP16 key on the CPU.""" + bits = values.to(torch.float16).view(torch.int16).to(torch.int32) & 0xFFFF + ordered = torch.where( + (bits & 0x8000) != 0, + (~bits) & 0xFFFF, + bits | 0x8000, + ) + return ordered >> 8 + + +@unittest.skipUnless(torch.cuda.is_available(), "Test requires CUDA") +class TestDsaKpoolTopkTransform(CustomTestCase): + def setUp(self): + super().setUp() + self.device = torch.device("cuda") + self.generator = torch.Generator(device="cpu").manual_seed(0) + + def tearDown(self): + torch.cuda.empty_cache() + super().tearDown() + + def _rand(self, *shape) -> torch.Tensor: + return torch.rand(*shape, generator=self.generator, dtype=torch.float32) + + def _tight_cluster(self, length: int, *, base: float = 1.0) -> torch.Tensor: + # Strictly increasing (modulo FP32 rounding), with the best candidates + # deliberately last. All values share the fp16 coarse bin, so an + # arrival-order stash prefix is deterministically wrong. + return base + torch.arange(length, dtype=torch.float32) * (1e-3 / length) + + def _byte_depth_clusters(self, depth: int, per_side: int) -> torch.Tensor: + """Two clusters of ``per_side`` keys that first differ at key bit ``depth``. + + Both share key byte 24 (0xBF, float 0.5..1.0), so stage 1 puts all + ``2 * per_side`` entries in one bin. The cluster with bit ``depth`` set + holds every true top-K member; bits below ``depth`` are random. + """ + base = 0xBF000000 + noise_bits = depth + noise = torch.randint( + 0, 1 << noise_bits, (2 * per_side,), generator=self.generator + ) + keys = torch.full((2 * per_side,), base, dtype=torch.int64) + noise + keys[:per_side] += 1 << depth + perm = torch.randperm(2 * per_side, generator=self.generator) + return _floats_from_keys(keys[perm]) + + def _run( + self, + rows: list, + group_topk: int, + page_table_bias: Optional[int] = None, + topk_offset: Optional[int] = None, + row_start: int = 0, + ): + batch = len(rows) + stride = row_start + max(row.numel() for row in rows) + # Pad with +inf so any read past ``lengths`` would corrupt the selection. + score = torch.full((batch, stride), float("inf"), dtype=torch.float32) + lengths = torch.empty(batch, dtype=torch.int32) + for i, row in enumerate(rows): + score[i, row_start : row_start + row.numel()] = row + lengths[i] = row.numel() + score = score.to(self.device) + lengths = lengths.to(self.device) + row_starts = ( + torch.full((batch,), row_start, dtype=torch.int32, device=self.device) + if row_start + else None + ) + + page_table = None + topk_indices_offset = None + if page_table_bias is not None: + page_table = ( + torch.arange(stride * POOL_SIZE, dtype=torch.int32) + .unsqueeze(0) + .repeat(batch, 1) + + page_table_bias + ).to(self.device) + if topk_offset is not None: + topk_indices_offset = torch.full( + (batch,), topk_offset, dtype=torch.int32, device=self.device + ) + + out = fast_kpool_topk_transform_fused( + score=score, + lengths=lengths, + kpool=POOL_SIZE, + topk=group_topk * POOL_SIZE, + page_table=page_table, + topk_indices_offset=topk_indices_offset, + row_starts=row_starts, + ) + self.assertEqual(out.shape, (batch, group_topk * POOL_SIZE)) + return out.cpu() + + def _selected_groups(self, out_row: torch.Tensor, group_topk: int, length: int): + """Decode expanded token columns back to group ids and check the layout.""" + self.assertTrue( + bool((out_row >= 0).all()), "no pad columns expected when length > K" + ) + tokens = out_row.view(group_topk, POOL_SIZE).to(torch.int64) + group_ids = tokens[:, 0] // POOL_SIZE + expected_tokens = group_ids.unsqueeze(1) * POOL_SIZE + torch.arange(POOL_SIZE) + self.assertTrue( + torch.equal(tokens, expected_tokens), + "each selected group must expand to its pool_size consecutive tokens", + ) + self.assertLess(int(group_ids.max()), length) + self.assertEqual( + torch.unique(group_ids).numel(), group_topk, "group ids must be distinct" + ) + return group_ids + + def _check_row(self, out_row: torch.Tensor, row: torch.Tensor, group_topk: int): + group_ids = self._selected_groups(out_row, group_topk, row.numel()) + selected = row[group_ids].sort().values + reference = torch.topk(row, group_topk).values.sort().values + self.assertTrue( + torch.equal(selected, reference), + "selected values must match torch.topk as a multiset", + ) + + def _check_rows(self, rows: list, group_topk: int): + out = self._run(rows, group_topk) + for i, row in enumerate(rows): + with self.subTest(row=i, length=row.numel(), group_topk=group_topk): + self._check_row(out[i], row, group_topk) + + def test_stage1_bin_exceeds_stash(self): + # 4097 and 17802 candidates in one stage-1 bin, both beyond the 4096-entry + # stash. A clipped stash would keep the first 4096 by atomic arrival + # order and disagree with torch.topk on such rows. + for group_topk in GROUP_TOPKS: + rows = [ + self._tight_cluster(STASH_ENTRIES + 1), + self._tight_cluster(17802), + ] + self._check_rows(rows, group_topk) + + def test_oversized_bin_with_strictly_higher_values(self): + # Exercise the output offset between values emitted above the coarse + # threshold and values selected by the exact overflow rescan. + for group_topk in GROUP_TOPKS: + high_count = 37 + high = 2.0 + torch.arange(high_count, dtype=torch.float32) * 1e-3 + threshold = self._tight_cluster(STASH_ENTRIES + 5311) + low = torch.zeros(257, dtype=torch.float32) + self._check_rows([torch.cat([high, threshold, low])], group_topk) + + def test_stride_larger_than_length(self): + # Row length far below the score stride, as with a padded score buffer. + for group_topk in GROUP_TOPKS: + long_row = self._tight_cluster(17802) + short_row = self._rand(group_topk + 1) + out = self._run([long_row, short_row], group_topk) + self._check_row(out[0], long_row, group_topk) + self._check_row(out[1], short_row, group_topk) + + def test_nonzero_row_start_with_oversized_bin(self): + for group_topk in GROUP_TOPKS: + row = self._tight_cluster(STASH_ENTRIES + 5311) + out = self._run([row], group_topk, row_start=3) + self._check_row(out[0], row, group_topk) + + def test_all_equal_rows_longer_than_stash(self): + # Every key is identical, so the final scan must clip the exact ties + # without duplicating indices or leaving an output slot unfilled. + for group_topk in GROUP_TOPKS: + rows = [ + torch.full((STASH_ENTRIES + 1,), 0.75), + torch.full((STASH_ENTRIES + 1,), 0.0), + torch.full((STASH_ENTRIES + 1,), -0.0), + ] + self._check_rows(rows, group_topk) + + def test_clusters_separating_at_each_key_byte(self): + # 4097 vs 4097 keys that first differ at key bit 16, 8 or 0: the + # stage-1 bin overflows and the rescan must walk to the byte where the + # clusters separate. + for group_topk in GROUP_TOPKS: + rows = [ + self._byte_depth_clusters(depth, STASH_ENTRIES + 1) + for depth in (16, 8, 0) + ] + self._check_rows(rows, group_topk) + + def test_stage1_exact_fill_boundary(self): + # Exactly K distinct values above a 65536-way tie: stage 1 finds the + # threshold bin with nothing left to refine and fills all K slots directly. + for group_topk in GROUP_TOPKS: + top = torch.linspace(2.0, 3.0, group_topk) + tie = torch.full((65536,), 1.0) + row = torch.cat([top, tie]) + row = row[torch.randperm(row.numel(), generator=self.generator)] + self._check_rows([row], group_topk) + + def test_stash_capacity_boundary(self): + # The exact-capacity case brackets the stash boundary. Different + # input orders protect against an arrival-order subset being selected; + # the mixed-bin row also covers the output offset after values above the + # exact-capacity threshold bin have already been emitted. + row = self._tight_cluster(STASH_ENTRIES) + self.assertEqual(torch.unique(_coarse_keys(row)).numel(), 1) + negative_row = self._tight_cluster(STASH_ENTRIES, base=-0.9) + self.assertEqual(torch.unique(_coarse_keys(negative_row)).numel(), 1) + high = 2.0 + torch.arange(37, dtype=torch.float32) * 1e-3 + low = torch.full((257,), -2.0) + mixed_bin_row = torch.cat([high, row, low]) + threshold_key = _coarse_keys(row)[0] + self.assertEqual( + int((_coarse_keys(mixed_bin_row) == threshold_key).sum()), + STASH_ENTRIES, + ) + rows = [ + row, + row.flip(0), + row[torch.randperm(row.numel(), generator=self.generator)], + negative_row, + mixed_bin_row, + ] + below_capacity = self._tight_cluster(STASH_ENTRIES - 1) + self.assertEqual(torch.unique(_coarse_keys(below_capacity)).numel(), 1) + for group_topk in GROUP_TOPKS: + self._check_rows(rows, group_topk) + self._check_rows([below_capacity], group_topk) + + def test_minimal_radix_length(self): + # length == K + 1 is the shortest row that takes the radix path. + for group_topk in GROUP_TOPKS: + self._check_rows([self._rand(group_topk + 1)], group_topk) + + def test_random_rows_batch(self): + for group_topk in GROUP_TOPKS: + rows = [self._rand(length) for length in (3000, 20000)] + self._check_rows(rows, group_topk) + + def test_page_table_and_offset_expansion(self): + group_topk = 512 + row = self._tight_cluster(STASH_ENTRIES + 5311) + + bias = 1000 + out = self._run([row], group_topk, page_table_bias=bias) + self._check_row(out[0] - bias, row, group_topk) + + offset = 777 + out = self._run([row], group_topk, topk_offset=offset) + self._check_row(out[0] - offset, row, group_topk) + + def test_tail_append_with_page_table_row_remap(self): + """Tail layout and absolute page-table row selection work together.""" + group_topk = 512 + rows = [self._tight_cluster(STASH_ENTRIES + 5311), self._rand(500)] + batch = len(rows) + stride = max(row.numel() for row in rows) + score = torch.full((batch, stride), float("inf"), dtype=torch.float32) + lengths = torch.tensor([row.numel() for row in rows], dtype=torch.int32) + for i, row in enumerate(rows): + score[i, : row.numel()] = row + + pool_size = POOL_SIZE + topk = group_topk * pool_size + max_tokens = stride * pool_size + pool_size - 1 + page_biases = torch.tensor([10000, 20000, 30000], dtype=torch.int32) + page_table = torch.arange(max_tokens, dtype=torch.int32).expand(3, -1) + page_table = page_table + page_biases[:, None] + page_table_row_index = torch.tensor([2, 0], dtype=torch.int32) + tail_counts = (3, 1) + seq_lens = torch.tensor( + [rows[i].numel() * pool_size + tail_counts[i] for i in range(batch)], + dtype=torch.int32, + ) + + out = fast_kpool_topk_transform_fused( + score=score.cuda(), + lengths=lengths.cuda(), + kpool=pool_size, + topk=topk, + page_table=page_table.cuda(), + seq_lens=seq_lens.cuda(), + page_table_row_index=page_table_row_index.cuda(), + ).cpu() + + for i, row in enumerate(rows): + bias = int(page_biases[page_table_row_index[i]]) + history_len = min(row.numel() * pool_size, topk) + if row.numel() > group_topk: + self._check_row(out[i, :history_len] - bias, row, group_topk) + else: + self.assertTrue( + torch.equal( + out[i, :history_len] - bias, + torch.arange(history_len, dtype=torch.int32), + ) + ) + + tail = out[i, history_len : history_len + tail_counts[i]] - bias + expected_tail = torch.arange( + row.numel() * pool_size, + row.numel() * pool_size + tail_counts[i], + dtype=torch.int32, + ) + self.assertTrue(torch.equal(tail, expected_tail)) + self.assertTrue(bool((out[i, history_len + tail_counts[i] :] == -1).all())) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/attention/test_topk_v1.py b/test/registered/kernels/ops/attention/test_topk_v1.py new file mode 100644 index 0000000000..f9fe40695d --- /dev/null +++ b/test/registered/kernels/ops/attention/test_topk_v1.py @@ -0,0 +1,197 @@ +"""Correctness tests for the DeepSeek-V4 JIT top-k transform v1. + +``topk_transform_paged`` dispatches to ``deepseek_v4/topk_v1.cuh`` on CUDA. Its +radix select first bins scores by the high byte of their ordered FP16 value and +stashes the threshold bin in shared memory (8192 entries). These tests put the +threshold bin on both sides of that capacity, with strictly higher values +emitted before it, ties at the cutoff, and clusters that only separate in one +of the lower three FP32 key bytes. They cover runtime ``topk`` values, the +``seq_len <= topk`` path, score rows padded past ``seq_lens``, a noncontiguous +score view, and both the page-table mapped and raw outputs. + +Selections are compared with ``torch.topk`` as value multisets: any subset of +values tied at the cutoff is a valid selection, but indices must be distinct +and in range, and the mapped output must be the page-table transform of the +raw output. +""" + +from __future__ import annotations + +import sys + +import pytest +import torch + +from sglang.kernels.ops.attention.dsv4.topk import topk_transform_paged +from sglang.srt.utils import is_hip +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=60, stage="base-b-kernel-unit", runner_config="1-gpu-large") + +pytestmark = pytest.mark.skipif( + not torch.cuda.is_available() or is_hip(), + reason="topk_transform_paged uses the JIT v1 kernel on CUDA only", +) + +# kSMEM / (2 * sizeof(int32_t)) in topk_v1.cuh: entries per stash round. +STASH_ENTRIES = 8192 +PAGE_SIZE = 64 +TOPKS = (512, 1024, 300) + + +def _generator(seed: int = 0) -> torch.Generator: + return torch.Generator(device="cpu").manual_seed(seed) + + +def _coarse_keys(values: torch.Tensor) -> torch.Tensor: + """Mirror the kernel's high-byte ordered-FP16 key on the CPU.""" + bits = values.to(torch.float16).view(torch.int16).to(torch.int32) & 0xFFFF + ordered = torch.where((bits & 0x8000) != 0, (~bits) & 0xFFFF, bits | 0x8000) + return ordered >> 8 + + +def _tight_cluster(length: int, base: float = 1.0) -> torch.Tensor: + # Increasing values in one coarse bin, best candidates last, so an + # arrival-order stash prefix is wrong. + return base + torch.arange(length, dtype=torch.float32) * (1e-3 / length) + + +def _byte_depth_clusters(depth: int, per_side: int, gen) -> torch.Tensor: + # Two clusters sharing the coarse bin whose keys first differ at bit + # ``depth``; the cluster with that bit set holds the true top-k. + keys = torch.full((2 * per_side,), 0x3F000000, dtype=torch.int64) + keys += torch.randint(0, 1 << depth, (2 * per_side,), generator=gen) + keys[:per_side] += 1 << depth + keys = keys[torch.randperm(2 * per_side, generator=gen)] + return keys.to(torch.int32).view(torch.float32) + + +def _run_and_check( + rows: list[torch.Tensor], topk: int, stride_pad: int = 0, view_pad: int = 0 +): + gen = _generator(topk) + batch = len(rows) + stride = max(row.numel() for row in rows) + stride_pad + # Pad with +inf so any read past ``seq_lens`` would corrupt the selection. + # ``view_pad`` passes a column slice whose row stride exceeds its width. + storage = torch.full( + (batch, stride + view_pad), float("inf"), dtype=torch.float32, device="cuda" + ) + scores = torch.full((batch, stride), float("inf"), dtype=torch.float32) + for i, row in enumerate(rows): + scores[i, : row.numel()] = row + seq_lens = torch.tensor([row.numel() for row in rows], dtype=torch.int32) + num_pages = (stride + PAGE_SIZE - 1) // PAGE_SIZE + page_table = torch.stack( + [torch.randperm(4 * num_pages, generator=gen)[:num_pages] for _ in range(batch)] + ).to(torch.int32) + + out_page = torch.full((batch, topk), -7, dtype=torch.int32, device="cuda") + out_raw = torch.full((batch, topk), -7, dtype=torch.int32, device="cuda") + scores_view = storage[:, :stride] + scores_view.copy_(scores) + assert scores_view.is_contiguous() == (view_pad == 0) + topk_transform_paged( + scores_view, + seq_lens.cuda(), + page_table.cuda(), + out_page, + PAGE_SIZE, + out_raw, + ) + out_page = out_page.cpu().to(torch.int64) + out_raw = out_raw.cpu().to(torch.int64) + + for i, row in enumerate(rows): + length = row.numel() + raw = out_raw[i] + if length <= topk: + expected = torch.full((topk,), -1, dtype=torch.int64) + expected[:length] = torch.arange(length) + assert torch.equal(raw, expected), f"row {i}: trivial raw indices" + else: + assert bool((raw >= 0).all()) and bool((raw < length).all()), f"row {i}" + assert torch.unique(raw).numel() == topk, f"row {i}: duplicate index" + selected = row[raw].sort().values + reference = torch.topk(row, topk).values.sort().values + assert torch.equal(selected, reference), f"row {i}: wrong selection" + valid = raw >= 0 + mapped = torch.full_like(raw, -1) + mapped[valid] = ( + page_table[i, raw[valid] // PAGE_SIZE].to(torch.int64) * PAGE_SIZE + + raw[valid] % PAGE_SIZE + ) + assert torch.equal(out_page[i], mapped), f"row {i}: page mapping" + + +@pytest.mark.parametrize("topk", TOPKS) +def test_threshold_bin_around_stash_capacity(topk): + gen = _generator(1) + below = _tight_cluster(STASH_ENTRIES - 1) + exact = _tight_cluster(STASH_ENTRIES) + above = _tight_cluster(STASH_ENTRIES + 1) + for row in (below, exact, above): + assert torch.unique(_coarse_keys(row)).numel() == 1 + rows = [ + below, + exact, + exact.flip(0), + exact[torch.randperm(exact.numel(), generator=gen)], + _tight_cluster(STASH_ENTRIES, base=-0.9), + above, + _tight_cluster(3 * STASH_ENTRIES + 17), + ] + _run_and_check(rows, topk) + + +@pytest.mark.parametrize("topk", TOPKS) +def test_oversized_bin_after_strictly_higher_values(topk): + gen = _generator(2) + high = 2.0 + torch.arange(37, dtype=torch.float32) * 1e-3 + low = torch.full((257,), -2.0) + row = torch.cat([high, _tight_cluster(STASH_ENTRIES + 5311), low]) + exact_bin_row = torch.cat([high, _tight_cluster(STASH_ENTRIES), low]) + rows = [row, row[torch.randperm(row.numel(), generator=gen)], exact_bin_row] + _run_and_check(rows, topk) + + +@pytest.mark.parametrize("topk", TOPKS) +def test_cutoff_ties(topk): + gen = _generator(3) + top = torch.linspace(2.0, 3.0, topk) + tie_row = torch.cat([top, torch.full((65536,), 1.0)]) + rows = [ + torch.full((STASH_ENTRIES + 1,), 0.75), + torch.full((STASH_ENTRIES + 1,), 0.0), + torch.full((STASH_ENTRIES + 1,), -0.0), + torch.full((topk + 1,), 0.5), + tie_row[torch.randperm(tie_row.numel(), generator=gen)], + torch.cat([top[: topk // 2], torch.full((2 * STASH_ENTRIES,), 1.0)]), + ] + _run_and_check(rows, topk) + + +@pytest.mark.parametrize("topk", TOPKS) +def test_clusters_separating_at_lower_key_bytes(topk): + gen = _generator(4) + rows = [_byte_depth_clusters(depth, STASH_ENTRIES + 1, gen) for depth in (16, 8, 0)] + _run_and_check(rows, topk) + + +@pytest.mark.parametrize("topk", TOPKS) +def test_short_and_random_rows_with_padded_and_strided_scores(topk): + gen = _generator(5) + rows = [ + torch.rand(1, generator=gen), + torch.rand(topk - 1, generator=gen), + torch.rand(topk, generator=gen), + torch.rand(topk + 1, generator=gen), + torch.rand(3000, generator=gen), + torch.randn(40000, generator=gen), + ] + _run_and_check(rows, topk, stride_pad=PAGE_SIZE + 3) + _run_and_check(rows, topk, view_pad=PAGE_SIZE + 5) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"]))