Fix overflow in legacy and v1 DSA top-k selectors (#37625)

Co-authored-by: Benjamin Oldenburg <benjamin.oldenburg@ordis.co.th>
This commit is contained in:
ormandj
2026-10-15 21:44:58 +08:00
committed by GitHub
co-authored by Benjamin Oldenburg
parent 2f5407da28
commit c5bf3aafbc
8 changed files with 1204 additions and 517 deletions
@@ -24,6 +24,7 @@ limitations under the License.
#include <cstddef>
#include <cstdint>
#include <optional>
#include <sgl_kernel/dsa/legacy_radix_topk.cuh>
namespace {
@@ -62,18 +63,6 @@ struct TopKParams {
bool sort_output;
};
__device__ __forceinline__ uint8_t convert_to_uint8(float x) {
__half h = __float2half_rn(x);
uint16_t bits = __half_as_ushort(h);
uint16_t key = (bits & 0x8000) ? static_cast<uint16_t>(~bits) : static_cast<uint16_t>(bits | 0x8000);
return static_cast<uint8_t>(key >> 8);
}
__device__ __forceinline__ uint32_t convert_to_uint32(float x) {
uint32_t bits = __float_as_uint(x);
return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u);
}
__device__ __forceinline__ int32_t
page_to_slot(const int32_t* __restrict__ page_table, uint32_t i, uint32_t page_bits) {
const uint32_t mask = (1u << page_bits) - 1u;
@@ -104,154 +93,10 @@ __device__ void naive_paged_transform(
__device__ void
radix_topk(const float* __restrict__ input, int32_t* __restrict__ output, uint32_t length, uint32_t topk) {
constexpr uint32_t RADIX = 256;
constexpr uint32_t BLOCK_SIZE = kBlockSize;
constexpr uint32_t SMEM_INPUT_SIZE = kSMEM / (2 * sizeof(int32_t));
alignas(128) __shared__ uint32_t _s_histogram_buf[2][RADIX + 32];
alignas(128) __shared__ uint32_t s_counter;
alignas(128) __shared__ uint32_t s_threshold_bin_id;
alignas(128) __shared__ uint32_t s_num_input[2];
alignas(128) __shared__ int32_t s_last_remain;
extern __shared__ uint32_t s_input_idx[][SMEM_INPUT_SIZE];
const uint32_t tx = threadIdx.x;
uint32_t remain_topk = topk;
auto& s_histogram = _s_histogram_buf[0];
const auto run_cumsum = [&] {
#pragma unroll 8
for (int32_t i = 0; i < 8; ++i) {
static_assert(1 << 8 == RADIX);
if (tx < RADIX) {
const auto j = 1 << i;
const auto k = i & 1;
auto value = _s_histogram_buf[k][tx];
if (tx + j < RADIX) {
value += _s_histogram_buf[k][tx + j];
}
_s_histogram_buf[k ^ 1][tx] = value;
}
__syncthreads();
}
};
// stage 1: 8bit coarse histogram
if (tx < RADIX + 1) s_histogram[tx] = 0;
::sglang::device::legacy_radix_topk::
select<static_cast<int>(kBlockSize), static_cast<int>(kMaxTopK), static_cast<int>(kSMEM / (2 * sizeof(int32_t)))>(
input, output, 0, static_cast<int>(length), static_cast<int>(topk));
__syncthreads();
for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) {
const auto bin = convert_to_uint8(input[idx]);
::atomicAdd(&s_histogram[bin], 1);
}
__syncthreads();
run_cumsum();
if (tx < RADIX && s_histogram[tx] > remain_topk && s_histogram[tx + 1] <= remain_topk) {
s_threshold_bin_id = tx;
s_num_input[0] = 0;
s_counter = 0;
}
__syncthreads();
{
const auto threshold_bin = s_threshold_bin_id;
remain_topk -= s_histogram[threshold_bin + 1];
if (remain_topk == 0) {
for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) {
const uint32_t bin = convert_to_uint8(input[idx]);
if (bin > threshold_bin) {
const auto pos = ::atomicAdd(&s_counter, 1);
output[pos] = static_cast<int32_t>(idx);
}
}
__syncthreads();
return;
}
__syncthreads();
if (tx < RADIX + 1) s_histogram[tx] = 0;
__syncthreads();
for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) {
const float raw_input = input[idx];
const uint32_t bin = convert_to_uint8(raw_input);
if (bin > threshold_bin) {
const auto pos = ::atomicAdd(&s_counter, 1);
output[pos] = static_cast<int32_t>(idx);
} else if (bin == threshold_bin) {
const auto pos = ::atomicAdd(&s_num_input[0], 1);
if (C10_LIKELY(pos < SMEM_INPUT_SIZE)) {
s_input_idx[0][pos] = idx;
const auto bin32 = convert_to_uint32(raw_input);
const auto sub_bin = (bin32 >> 24) & 0xFF;
::atomicAdd(&s_histogram[sub_bin], 1);
}
}
}
__syncthreads();
}
// stage 2: refine with 8bit radix passes
#pragma unroll 4
for (int round = 0; round < 4; ++round) {
const auto r_idx = round % 2;
const auto raw_num_input = s_num_input[r_idx];
const auto num_input = raw_num_input < SMEM_INPUT_SIZE ? raw_num_input : SMEM_INPUT_SIZE;
run_cumsum();
if (tx < RADIX && s_histogram[tx] > remain_topk && s_histogram[tx + 1] <= remain_topk) {
s_threshold_bin_id = tx;
s_num_input[r_idx ^ 1] = 0;
s_last_remain = static_cast<int32_t>(remain_topk - s_histogram[tx + 1]);
}
__syncthreads();
const auto threshold_bin = s_threshold_bin_id;
remain_topk -= s_histogram[threshold_bin + 1];
if (remain_topk == 0) {
for (uint32_t i = tx; i < num_input; i += BLOCK_SIZE) {
const auto idx = s_input_idx[r_idx][i];
const auto offset = 24 - round * 8;
const auto bin = (convert_to_uint32(input[idx]) >> offset) & 0xFF;
if (bin > threshold_bin) {
const auto pos = ::atomicAdd(&s_counter, 1);
output[pos] = static_cast<int32_t>(idx);
}
}
__syncthreads();
break;
}
__syncthreads();
if (tx < RADIX + 1) s_histogram[tx] = 0;
__syncthreads();
for (uint32_t i = tx; i < num_input; i += BLOCK_SIZE) {
const auto idx = s_input_idx[r_idx][i];
const auto raw_input = input[idx];
const auto offset = 24 - round * 8;
const auto bin = (convert_to_uint32(raw_input) >> offset) & 0xFF;
if (bin > threshold_bin) {
const auto pos = ::atomicAdd(&s_counter, 1);
output[pos] = static_cast<int32_t>(idx);
} else if (bin == threshold_bin) {
if (round == 3) {
const auto pos = ::atomicAdd(&s_last_remain, -1);
if (pos > 0) {
output[topk - pos] = static_cast<int32_t>(idx);
}
} else {
const auto pos = ::atomicAdd(&s_num_input[r_idx ^ 1], 1);
if (C10_LIKELY(pos < SMEM_INPUT_SIZE)) {
s_input_idx[r_idx ^ 1][pos] = idx;
const auto bin32 = convert_to_uint32(raw_input);
const auto sub_bin = (bin32 >> (offset - 8)) & 0xFF;
::atomicAdd(&s_histogram[sub_bin], 1);
}
}
}
}
__syncthreads();
}
}
#ifdef USE_ROCM
@@ -366,7 +211,8 @@ __device__ __forceinline__ uint32_t next_pow2_at_least_64(uint32_t x) {
__global__ __launch_bounds__(kBlockSize) void deepseek_v4_topk_transform_kernel(const TopKParams params) {
const auto bid = blockIdx.x;
const auto seq_len = params.seq_lens[bid];
const auto raw_seq_len = params.seq_lens[bid];
const auto seq_len = raw_seq_len < 0 ? 0 : raw_seq_len;
const auto topk = params.topk;
const auto score_ptr = params.scores + bid * params.score_stride;
const auto page_ptr = params.page_table + bid * params.page_table_stride;
@@ -433,7 +279,7 @@ __global__ __launch_bounds__(kBlockSize) void deepseek_v4_topk_transform_kernel(
for (uint32_t i = threadIdx.x; i < topk; i += kBlockSize) {
const auto raw = s_topk_indices[i];
indices_ptr[i] = page_to_slot(page_ptr, static_cast<uint32_t>(raw), params.page_bits);
indices_ptr[i] = raw < 0 ? -1 : page_to_slot(page_ptr, static_cast<uint32_t>(raw), params.page_bits);
if (raw_indices_ptr != nullptr) {
raw_indices_ptr[i] = raw;
}
@@ -17,6 +17,7 @@
#include <cstddef>
#include <cstdint>
#include <optional>
#include <sgl_kernel/dsa/legacy_radix_topk.cuh>
namespace {
@@ -75,180 +76,10 @@ __device__ void naive_topk_transform_ragged(
}
}
__device__ __forceinline__ auto convert_to_uint8(float x) -> uint8_t {
__half h = __float2half_rn(x);
uint16_t bits = __half_as_ushort(h);
uint16_t key = (bits & 0x8000) ? static_cast<uint16_t>(~bits) : static_cast<uint16_t>(bits | 0x8000);
return static_cast<uint8_t>(key >> 8);
}
__device__ __forceinline__ auto convert_to_uint32(float x) -> uint32_t {
uint32_t bits = __float_as_uint(x);
return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u);
}
__device__ void fast_topk_cuda_tl(const float* __restrict__ input, int* __restrict__ index, int row_start, int length) {
// An optimized topk kernel copied from tilelang kernel
// We assume length > TopK here, or it will crash
int topk = TopK;
constexpr auto BLOCK_SIZE = 1024;
constexpr auto RADIX = 256;
constexpr auto SMEM_INPUT_SIZE = kSmem / (2 * sizeof(int));
alignas(128) __shared__ int s_histogram_buf[2][RADIX + 128];
alignas(128) __shared__ int s_counter;
alignas(128) __shared__ int s_threshold_bin_id;
alignas(128) __shared__ int s_num_input[2];
auto& s_histogram = s_histogram_buf[0];
// allocate for two rounds
extern __shared__ int s_input_idx[][SMEM_INPUT_SIZE];
const int tx = threadIdx.x;
// stage 1: 8bit coarse histogram
if (tx < RADIX + 1) s_histogram[tx] = 0;
::sglang::device::legacy_radix_topk::select<kThreadsPerBlock, TopK, static_cast<int>(kSmem / (2 * sizeof(int)))>(
input, index, row_start, length, TopK);
__syncthreads();
for (int idx = tx; idx < length; idx += BLOCK_SIZE) {
const auto bin = convert_to_uint8(input[idx + row_start]);
::atomicAdd(&s_histogram[bin], 1);
}
__syncthreads();
const auto run_cumsum = [&] {
#pragma unroll 8
for (int i = 0; i < 8; ++i) {
static_assert(1 << 8 == RADIX);
if (C10_LIKELY(tx < RADIX)) {
const auto j = 1 << i;
const auto k = i & 1;
auto value = s_histogram_buf[k][tx];
if (tx < RADIX - j) {
value += s_histogram_buf[k][tx + j];
}
s_histogram_buf[k ^ 1][tx] = value;
}
__syncthreads();
}
};
run_cumsum();
if (tx < RADIX && s_histogram[tx] > topk && s_histogram[tx + 1] <= topk) {
s_threshold_bin_id = tx;
s_num_input[0] = 0;
s_counter = 0;
}
__syncthreads();
const auto threshold_bin = s_threshold_bin_id;
topk -= s_histogram[threshold_bin + 1];
if (topk == 0) {
for (int idx = tx; idx < length; idx += BLOCK_SIZE) {
const auto bin = static_cast<int>(convert_to_uint8(input[idx + row_start]));
if (bin > threshold_bin) {
const auto pos = ::atomicAdd(&s_counter, 1);
index[pos] = idx;
}
}
__syncthreads();
return;
} else {
__syncthreads();
if (tx < RADIX + 1) {
s_histogram[tx] = 0;
}
__syncthreads();
for (int idx = tx; idx < length; idx += BLOCK_SIZE) {
const auto raw_input = input[idx + row_start];
const auto bin = static_cast<int>(convert_to_uint8(raw_input));
if (bin > threshold_bin) {
const auto pos = ::atomicAdd(&s_counter, 1);
index[pos] = idx;
} else if (bin == threshold_bin) {
const auto pos = ::atomicAdd(&s_num_input[0], 1);
/// NOTE: (dark) fuse the histogram computation here
if (C10_LIKELY(pos < SMEM_INPUT_SIZE)) {
s_input_idx[0][pos] = idx;
const auto bin = convert_to_uint32(raw_input);
const auto sub_bin = (bin >> 24) & 0xFF;
::atomicAdd(&s_histogram[sub_bin], 1);
}
}
}
__syncthreads();
}
// stage 2: refine with 8bit radix passes
#pragma unroll 4
for (int round = 0; round < 4; ++round) {
__shared__ int s_last_remain;
const auto r_idx = round % 2;
// clip here to prevent overflow
const auto _raw_num_input = s_num_input[r_idx];
const auto num_input = (_raw_num_input < int(SMEM_INPUT_SIZE)) ? _raw_num_input : int(SMEM_INPUT_SIZE);
run_cumsum();
if (tx < RADIX && s_histogram[tx] > topk && s_histogram[tx + 1] <= topk) {
s_threshold_bin_id = tx;
s_num_input[r_idx ^ 1] = 0;
s_last_remain = topk - s_histogram[tx + 1];
}
__syncthreads();
const auto threshold_bin = s_threshold_bin_id;
topk -= s_histogram[threshold_bin + 1];
if (topk == 0) {
for (int i = tx; i < num_input; i += BLOCK_SIZE) {
const auto idx = s_input_idx[r_idx][i];
const auto offset = 24 - round * 8;
const auto bin = (convert_to_uint32(input[idx + row_start]) >> offset) & 0xFF;
if (bin > threshold_bin) {
const auto pos = ::atomicAdd(&s_counter, 1);
index[pos] = idx;
}
}
__syncthreads();
break;
} else {
__syncthreads();
if (tx < RADIX + 1) {
s_histogram[tx] = 0;
}
__syncthreads();
for (int i = tx; i < num_input; i += BLOCK_SIZE) {
const auto idx = s_input_idx[r_idx][i];
const auto raw_input = input[idx + row_start];
const auto offset = 24 - round * 8;
const auto bin = (convert_to_uint32(raw_input) >> offset) & 0xFF;
if (bin > threshold_bin) {
const auto pos = ::atomicAdd(&s_counter, 1);
index[pos] = idx;
} else if (bin == threshold_bin) {
if (round == 3) {
const auto pos = ::atomicAdd(&s_last_remain, -1);
if (pos > 0) {
index[TopK - pos] = idx;
}
} else {
const auto pos = ::atomicAdd(&s_num_input[r_idx ^ 1], 1);
if (C10_LIKELY(pos < SMEM_INPUT_SIZE)) {
/// NOTE: (dark) fuse the histogram computation here
s_input_idx[r_idx ^ 1][pos] = idx;
const auto bin = convert_to_uint32(raw_input);
const auto sub_bin = (bin >> (offset - 8)) & 0xFF;
::atomicAdd(&s_histogram[sub_bin], 1);
}
}
}
}
__syncthreads();
}
}
}
__global__ __launch_bounds__(kThreadsPerBlock) // topk
@@ -290,10 +121,10 @@ __global__ __launch_bounds__(kThreadsPerBlock) // decode
static_assert(TopK / kThreadsPerBlock == 2);
const auto idx_0 = tid;
const auto pos_0 = s_indices[idx_0];
dst_page_entry[idx_0] = src_page_entry[pos_0];
dst_page_entry[idx_0] = pos_0 < 0 ? -1 : src_page_entry[pos_0];
const auto idx_1 = tid + kThreadsPerBlock;
const auto pos_1 = s_indices[idx_1];
dst_page_entry[idx_1] = src_page_entry[pos_1];
dst_page_entry[idx_1] = pos_1 < 0 ? -1 : src_page_entry[pos_1];
}
}
@@ -342,10 +173,10 @@ __global__ __launch_bounds__(kThreadsPerBlock) // prefill
static_assert(TopK / kThreadsPerBlock == 2);
const auto idx_0 = tid;
const auto pos_0 = s_indices[idx_0];
dst_page_entry[idx_0] = src_page_entry[pos_0];
dst_page_entry[idx_0] = pos_0 < 0 ? -1 : src_page_entry[pos_0];
const auto idx_1 = tid + kThreadsPerBlock;
const auto pos_1 = s_indices[idx_1];
dst_page_entry[idx_1] = src_page_entry[pos_1];
dst_page_entry[idx_1] = pos_1 < 0 ? -1 : src_page_entry[pos_1];
}
}
@@ -373,10 +204,10 @@ __global__ __launch_bounds__(kThreadsPerBlock) // prefill, ragged kv
static_assert(TopK / kThreadsPerBlock == 2);
const auto idx_0 = tid;
const auto pos_0 = s_indices[idx_0];
dst_indices_entry[idx_0] = pos_0 + offset;
dst_indices_entry[idx_0] = pos_0 < 0 ? -1 : pos_0 + offset;
const auto idx_1 = tid + kThreadsPerBlock;
const auto pos_1 = s_indices[idx_1];
dst_indices_entry[idx_1] = pos_1 + offset;
dst_indices_entry[idx_1] = pos_1 < 0 ? -1 : pos_1 + offset;
}
}
@@ -0,0 +1,239 @@
#pragma once
#include <cuda_fp16.h>
#include <cstddef>
#include <cstdint>
namespace sglang::device::legacy_radix_topk {
inline constexpr int kRadix = 256;
// Coarse bin: high byte of the ordered FP16 value, as in the original selectors.
__device__ __forceinline__ uint8_t coarse_key(float x) {
const __half h = __float2half_rn(x);
const uint16_t bits = __half_as_ushort(h);
const uint16_t key = (bits & 0x8000u) ? static_cast<uint16_t>(~bits) : static_cast<uint16_t>(bits | 0x8000u);
return static_cast<uint8_t>(key >> 8);
}
__device__ __forceinline__ uint32_t exact_key(float x) {
const uint32_t bits = __float_as_uint(x);
return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u);
}
// Exact radix top-k over finite scores. A threshold bin that fits the stash
// keeps the original two-stash refinement; a larger one is refined by rescanning
// the input. Unfilled output slots stay -1.
template <int BlockSize, int OutputCapacity, int StashEntries>
__device__ __forceinline__ void
select(const float* __restrict__ input, int32_t* __restrict__ output, int row_start, int length, int topk) {
static_assert(BlockSize >= kRadix + 1, "the block must initialize the complete radix histogram");
static_assert(OutputCapacity <= StashEntries, "top-k output must fit in one shared-memory stash");
alignas(128) __shared__ int histogram_buf[2][kRadix + 1];
alignas(128) __shared__ int counter;
alignas(128) __shared__ int threshold_bin_id;
alignas(128) __shared__ int num_input[2];
extern __shared__ int dynamic_smem[];
auto* input_idx = reinterpret_cast<int (*)[StashEntries]>(dynamic_smem);
auto& histogram = histogram_buf[0];
const int tx = threadIdx.x;
for (int i = tx; i < topk; i += BlockSize)
output[i] = -1;
if (tx < kRadix + 1) histogram[tx] = 0;
if (tx == 0) {
counter = 0;
threshold_bin_id = -1;
num_input[0] = 0;
num_input[1] = 0;
}
__syncthreads();
for (int idx = tx; idx < length; idx += BlockSize) {
atomicAdd(&histogram[coarse_key(input[row_start + idx])], 1);
}
__syncthreads();
const auto run_cumsum = [&] {
#pragma unroll 8
for (int i = 0; i < 8; ++i) {
if (tx < kRadix) {
const int offset = 1 << i;
const int src = i & 1;
int value = histogram_buf[src][tx];
if (tx < kRadix - offset) value += histogram_buf[src][tx + offset];
histogram_buf[src ^ 1][tx] = value;
}
__syncthreads();
}
};
run_cumsum();
if (tx < kRadix && histogram[tx] > topk && histogram[tx + 1] <= topk) {
threshold_bin_id = tx;
}
__syncthreads();
const int coarse_threshold = threshold_bin_id;
if (coarse_threshold < 0) return;
const int coarse_above = histogram[coarse_threshold + 1];
int remaining = topk - coarse_above;
if (remaining == 0) {
for (int idx = tx; idx < length; idx += BlockSize) {
if (coarse_key(input[row_start + idx]) > coarse_threshold) {
const int pos = atomicAdd(&counter, 1);
if (pos < topk) output[pos] = idx;
}
}
__syncthreads();
return;
}
const int coarse_population = histogram[coarse_threshold] - histogram[coarse_threshold + 1];
__syncthreads();
if (coarse_population >= StashEntries) {
// Find the exact cutoff one key byte per pass. Pass L counts byte L and
// emits what byte L-1 placed strictly above the cutoff; pass 0 emits the
// values above the coarse bin and pass 4 the last strict values and ties.
uint32_t prefix = 0;
int last_threshold = -1;
int strict_count = 0;
#pragma unroll 1
for (int level = 0; level <= 4; ++level) {
const uint32_t mask = level == 0 ? 0u : (~0u << (32 - level * 8));
const uint32_t last_mask = level <= 1 ? 0u : (~0u << (40 - level * 8));
const int last_shift = 32 - level * 8;
if (level < 4) {
if (tx < kRadix + 1) histogram[tx] = 0;
if (tx == 0) threshold_bin_id = -1;
}
__syncthreads();
for (int idx = tx; idx < length; idx += BlockSize) {
const float value = input[row_start + idx];
const int bin = coarse_key(value);
if (bin != coarse_threshold) {
if (level == 0 && bin > coarse_threshold) {
const int pos = atomicAdd(&counter, 1);
if (pos < topk) output[pos] = idx;
}
continue;
}
const uint32_t key = exact_key(value);
if (level > 0 && (key & last_mask) == (prefix & last_mask) &&
static_cast<int>((key >> last_shift) & 0xFFu) > last_threshold) {
const int pos = atomicAdd(&num_input[0], 1);
if (pos < strict_count) output[coarse_above + pos] = idx;
} else if ((key & mask) == prefix) {
if (level < 4) {
atomicAdd(&histogram[(key >> (24 - level * 8)) & 0xFFu], 1);
} else {
const int pos = atomicAdd(&num_input[1], 1);
if (pos < remaining) output[coarse_above + strict_count + pos] = idx;
}
}
}
__syncthreads();
if (level == 4) return;
run_cumsum();
if (tx < kRadix && histogram[tx] >= remaining && histogram[tx + 1] < remaining) {
threshold_bin_id = tx;
}
__syncthreads();
const int threshold = threshold_bin_id;
if (threshold < 0) return;
const int above = histogram[threshold + 1];
strict_count += above;
remaining -= above;
prefix |= static_cast<uint32_t>(threshold) << (24 - level * 8);
last_threshold = threshold;
// All threads must read the histogram before the next pass clears it.
__syncthreads();
}
return;
}
// The threshold bin fits the stash: original two-stash refinement.
if (tx < kRadix + 1) histogram[tx] = 0;
__syncthreads();
for (int idx = tx; idx < length; idx += BlockSize) {
const float value = input[row_start + idx];
const int bin = coarse_key(value);
if (bin > coarse_threshold) {
const int pos = atomicAdd(&counter, 1);
if (pos < topk) output[pos] = idx;
} else if (bin == coarse_threshold) {
const int pos = atomicAdd(&num_input[0], 1);
if (pos < StashEntries) {
input_idx[0][pos] = idx;
atomicAdd(&histogram[(exact_key(value) >> 24) & 0xFFu], 1);
}
}
}
__syncthreads();
#pragma unroll 4
for (int round = 0; round < 4; ++round) {
const int current = round & 1;
const int next = current ^ 1;
const int raw_num_input = num_input[current];
const int current_count = raw_num_input < StashEntries ? raw_num_input : StashEntries;
if (tx == 0) {
threshold_bin_id = -1;
num_input[next] = 0;
}
run_cumsum();
if (tx < kRadix && histogram[tx] > remaining && histogram[tx + 1] <= remaining) {
threshold_bin_id = tx;
}
__syncthreads();
const int threshold = threshold_bin_id;
if (threshold < 0) return;
const int above = histogram[threshold + 1];
const int shift = 24 - round * 8;
__syncthreads();
if (round < 3 && tx < kRadix + 1) histogram[tx] = 0;
__syncthreads();
for (int i = tx; i < current_count; i += BlockSize) {
const int idx = input_idx[current][i];
const float value = input[row_start + idx];
const int bin = static_cast<int>((exact_key(value) >> shift) & 0xFFu);
if (bin > threshold) {
const int pos = atomicAdd(&counter, 1);
if (pos < topk) output[pos] = idx;
} else if (bin == threshold && round < 3) {
const int pos = atomicAdd(&num_input[next], 1);
if (pos < StashEntries) {
input_idx[next][pos] = idx;
atomicAdd(&histogram[(exact_key(value) >> (shift - 8)) & 0xFFu], 1);
}
}
}
__syncthreads();
remaining -= above;
if (remaining == 0) return;
if (round == 3) {
for (int i = tx; i < current_count; i += BlockSize) {
const int idx = input_idx[current][i];
if (static_cast<int>(exact_key(input[row_start + idx]) & 0xFFu) == threshold) {
const int pos = atomicAdd(&counter, 1);
if (pos < topk) output[pos] = idx;
}
}
__syncthreads();
return;
}
}
}
} // namespace sglang::device::legacy_radix_topk
+141 -10
View File
@@ -1,5 +1,5 @@
import sys
from typing import Any, Optional
from typing import Optional
import pytest
import torch
@@ -67,6 +67,13 @@ def _ref_torch_transform_ragged_impl(
MAX_SEQ_LEN = 131072
def _tight_cluster(batch: int, width: int) -> torch.Tensor:
"""Finite scores that all land in one fp16 coarse radix bin."""
base = torch.arange(width, dtype=torch.float32, device="cuda")
values = 1.0 + base * (1e-3 / width)
return values.expand(batch, -1).contiguous()
def assert_equal(
score: torch.Tensor,
indices_ref: torch.Tensor,
@@ -75,29 +82,39 @@ def assert_equal(
k: int,
seq_len: int,
topk_indices_offset: Optional[torch.Tensor] = None,
max_permit_error: int = 0,
row_starts: Optional[torch.Tensor] = None,
):
indices_our_cpu = indices_our.cpu().tolist()
indices_ref_cpu = indices_ref.cpu().tolist()
wrong_values = 0
for i in range(bs):
offset = topk_indices_offset[i].item() if topk_indices_offset is not None else 0
row_start = row_starts[i].item() if row_starts is not None else 0
indices_ref_set_i = set(indices_ref_cpu[i])
indices_our_set_i = set(indices_our_cpu[i])
assert len(indices_our_cpu[i]) == k
assert len(indices_our_set_i) == k, "top-k output indices must be distinct"
assert all(0 <= idx - offset < seq_len for idx in indices_our_cpu[i]), (
"top-k output index is outside the selected window"
)
more = indices_our_set_i - indices_ref_set_i
less = indices_ref_set_i - indices_our_set_i
offset = topk_indices_offset[i].item() if topk_indices_offset is not None else 0
if len(more) > 0 or len(less) > 0:
# check whether more values are the same with less values
# if so, either one is acceptable, since their values are the same
more_values = sorted(score[i, idx - offset].item() for idx in more)
less_values = sorted(score[i, idx - offset].item() for idx in less)
more_values = sorted(
score[i, row_start + idx - offset].item() for idx in more
)
less_values = sorted(
score[i, row_start + idx - offset].item() for idx in less
)
if more_values != less_values:
wrong_values += len(more)
print(
f"{bs=}, {k=}, {seq_len=}, {i=}, {more=}, {less=} failed, with {more_values=}, {less_values=}"
)
assert wrong_values <= max_permit_error, f"{wrong_values=}, {max_permit_error=}"
assert wrong_values == 0, f"{wrong_values=}"
@pytest.mark.parametrize("bs", [1, 132, 256, 4096])
@@ -125,8 +142,7 @@ def test_topk_kernel(bs: int, k: int, seq_len: int, has_row_starts: bool) -> Non
indices_ref = torch.sort(indices_ref, dim=-1).values
indices_our = torch.sort(indices_our, dim=-1).values
# Tests can pass with max_permit_error=3, set to 5 for safety
assert_equal(score, indices_ref, indices_our, bs, k, seq_len, max_permit_error=5)
assert_equal(score, indices_ref, indices_our, bs, k, seq_len, row_starts=row_starts)
@pytest.mark.parametrize("bs", [1, 132, 256, 4096])
@@ -189,7 +205,7 @@ def test_topk_transform_kernel(bs: int, k: int, seq_len: int, mode: str) -> None
bs,
k,
seq_len,
max_permit_error=5,
row_starts=row_starts,
)
@@ -245,10 +261,71 @@ def test_topk_transform_ragged_kernel(
k,
seq_len,
topk_indices_offset,
max_permit_error=5,
row_starts,
)
@torch.inference_mode()
def test_legacy_topk_oversized_coarse_bin() -> None:
"""The shared legacy selector must not clip an oversized threshold bin."""
torch.manual_seed(20260903)
batch, seq_len, topk, row_start = 2, 7001, 2048, 3
scores = torch.full(
(batch, seq_len + row_start), -1000.0, dtype=torch.float32, device="cuda"
)
scores[:, row_start:] = _tight_cluster(batch, seq_len)
lengths = torch.full((batch,), seq_len, dtype=torch.int32, device="cuda")
row_starts = torch.full((batch,), row_start, dtype=torch.int32, device="cuda")
reference = _ref_torch_impl(scores, seq_len, topk, row_starts=row_starts)
direct = fast_topk_v2(scores, lengths, topk, row_starts=row_starts)
assert_equal(scores[:, row_start:], reference, direct, batch, topk, seq_len)
offsets = torch.tensor([17, 4099], dtype=torch.int32, device="cuda")
ragged = fast_topk_transform_ragged_fused(
score=scores,
lengths=lengths,
topk_indices_offset=offsets,
topk=topk,
row_starts=row_starts,
)
assert_equal(
scores[:, row_start:],
reference + offsets[:, None],
ragged,
batch,
topk,
seq_len,
offsets,
)
page_table = torch.arange(seq_len, dtype=torch.int32, device="cuda").expand(
batch, -1
)
cu_seqlens_q = torch.arange(batch + 1, dtype=torch.int32, device="cuda")
paged = fast_topk_transform_fused(
score=scores,
lengths=lengths,
page_table_size_1=page_table,
cu_seqlens_q=cu_seqlens_q,
topk=topk,
row_starts=row_starts,
)
assert_equal(scores[:, row_start:], reference, paged, batch, topk, seq_len)
# With one query per row and no row starts, this interface selects the
# separate decode kernel rather than the prefill kernel above.
decode_scores = scores[:, row_start:].contiguous()
decode = fast_topk_transform_fused(
score=decode_scores,
lengths=lengths,
page_table_size_1=page_table,
cu_seqlens_q=cu_seqlens_q,
topk=topk,
)
assert_equal(decode_scores, reference, decode, batch, topk, seq_len)
@pytest.mark.skipif(
torch.version.hip is None,
reason="deepseek_v4_topk_transform_512 is only built on ROCm",
@@ -499,5 +576,59 @@ def test_topk_variants_are_exact_on_variable_windows(
assert_exact_rows(score, mapped_raw, lengths, k)
@pytest.mark.skipif(
torch.version.hip is None,
reason="deepseek_v4_topk_transform_512 is only built on ROCm",
)
@torch.inference_mode()
def test_deepseek_v4_topk_oversized_coarse_bin_and_idle_row() -> None:
"""Oversized bins select exactly; a negative DP-idle length emits padding."""
from sgl_kernel import deepseek_v4_topk_transform_512
torch.manual_seed(20260903)
active_batch, batch, c4_len, topk, page_size = 2, 3, 7001, 1024, 64
scores = _tight_cluster(batch, c4_len)
seq_lens = torch.tensor(
[c4_len] * active_batch + [-1], dtype=torch.int32, device="cuda"
)
num_pages = (c4_len + page_size - 1) // page_size
page_table = (
torch.arange(num_pages, dtype=torch.int32, device="cuda")
.expand(batch, -1)
.contiguous()
)
page_indices = torch.full((batch, topk), -2, dtype=torch.int32, device="cuda")
raw_indices = torch.full_like(page_indices, -2)
deepseek_v4_topk_transform_512(
scores,
seq_lens,
page_table,
page_indices,
page_size,
raw_indices,
)
reference = torch.topk(scores[:active_batch], topk, dim=-1, sorted=False).indices
assert_equal(
scores[:active_batch],
reference,
raw_indices[:active_batch],
active_batch,
topk,
c4_len,
)
assert_equal(
scores[:active_batch],
reference,
page_indices[:active_batch],
active_batch,
topk,
c4_len,
)
assert bool((raw_indices[-1] == -1).all())
assert bool((page_indices[-1] == -1).all())
if __name__ == "__main__":
sys.exit(pytest.main([__file__]))
@@ -4,6 +4,8 @@
#include <sgl_kernel/runtime.cuh>
#include <sgl_kernel/utils.cuh>
#include <sgl_kernel/dsa/legacy_radix_topk.cuh>
#include <dlpack/dlpack.h>
#include <tvm/ffi/container/tensor.h>
@@ -39,18 +41,6 @@ struct TopKParams {
uint32_t topk;
};
SGL_DEVICE uint8_t convert_to_uint8(float x) {
__half h = __float2half_rn(x);
uint16_t bits = __half_as_ushort(h);
uint16_t key = (bits & 0x8000) ? static_cast<uint16_t>(~bits) : static_cast<uint16_t>(bits | 0x8000);
return static_cast<uint8_t>(key >> 8);
}
SGL_DEVICE uint32_t convert_to_uint32(float x) {
uint32_t bits = __float_as_uint(x);
return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u);
}
SGL_DEVICE int32_t page_to_indices(const int32_t* __restrict__ page_table, uint32_t i, uint32_t page_bits) {
const uint32_t mask = (1u << page_bits) - 1u;
return (page_table[i >> page_bits] << page_bits) | (i & mask);
@@ -81,160 +71,9 @@ SGL_DEVICE void naive_transform(
[[maybe_unused]]
SGL_DEVICE void
radix_topk(const float* __restrict__ input, int32_t* __restrict__ output, const uint32_t length, const uint32_t topk) {
constexpr uint32_t RADIX = 256;
constexpr uint32_t BLOCK_SIZE = kTopKBlockSize;
constexpr uint32_t SMEM_INPUT_SIZE = kSMEM / (2 * sizeof(int32_t));
alignas(128) __shared__ uint32_t _s_histogram_buf[2][RADIX + 32];
alignas(128) __shared__ uint32_t s_counter;
alignas(128) __shared__ uint32_t s_threshold_bin_id;
alignas(128) __shared__ uint32_t s_num_input[2];
alignas(128) __shared__ int32_t s_last_remain;
extern __shared__ uint32_t s_input_idx[][kSMEM / (2 * sizeof(int32_t))];
const uint32_t tx = threadIdx.x;
uint32_t remain_topk = topk;
auto& s_histogram = _s_histogram_buf[0];
const auto run_cumsum = [&] {
#pragma unroll 8
for (int32_t i = 0; i < 8; ++i) {
static_assert(1 << 8 == RADIX);
if (tx < RADIX) {
const auto j = 1 << i;
const auto k = i & 1;
auto value = _s_histogram_buf[k][tx];
if (tx + j < RADIX) {
value += _s_histogram_buf[k][tx + j];
}
_s_histogram_buf[k ^ 1][tx] = value;
}
__syncthreads();
}
};
// stage 1: 8bit coarse histogram
if (tx < RADIX + 1) s_histogram[tx] = 0;
__syncthreads();
for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) {
const auto bin = convert_to_uint8(input[idx]);
::atomicAdd(&s_histogram[bin], 1);
}
__syncthreads();
run_cumsum();
if (tx < RADIX && s_histogram[tx] > remain_topk && s_histogram[tx + 1] <= remain_topk) {
s_threshold_bin_id = tx;
s_num_input[0] = 0;
s_counter = 0;
}
__syncthreads();
const auto threshold_bin = s_threshold_bin_id;
remain_topk -= s_histogram[threshold_bin + 1];
if (remain_topk == 0) {
for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) {
const uint32_t bin = convert_to_uint8(input[idx]);
if (bin > threshold_bin) {
const auto pos = ::atomicAdd(&s_counter, 1);
output[pos] = idx;
}
}
__syncthreads();
return;
} else {
__syncthreads();
if (tx < RADIX + 1) {
s_histogram[tx] = 0;
}
__syncthreads();
for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) {
const float raw_input = input[idx];
const uint32_t bin = convert_to_uint8(raw_input);
if (bin > threshold_bin) {
const auto pos = ::atomicAdd(&s_counter, 1);
output[pos] = idx;
} else if (bin == threshold_bin) {
const auto pos = ::atomicAdd(&s_num_input[0], 1);
if (pos < SMEM_INPUT_SIZE) {
[[likely]] s_input_idx[0][pos] = idx;
const auto bin = convert_to_uint32(raw_input);
const auto sub_bin = (bin >> 24) & 0xFF;
::atomicAdd(&s_histogram[sub_bin], 1);
}
}
}
__syncthreads();
}
// stage 2: refine with 8bit radix passes
#pragma unroll 4
for (int round = 0; round < 4; ++round) {
const auto r_idx = round % 2;
// clip here to prevent overflow
const auto raw_num_input = s_num_input[r_idx];
const auto num_input = raw_num_input < SMEM_INPUT_SIZE ? raw_num_input : SMEM_INPUT_SIZE;
run_cumsum();
if (tx < RADIX && s_histogram[tx] > remain_topk && s_histogram[tx + 1] <= remain_topk) {
s_threshold_bin_id = tx;
s_num_input[r_idx ^ 1] = 0;
s_last_remain = remain_topk - s_histogram[tx + 1];
}
__syncthreads();
const auto threshold_bin = s_threshold_bin_id;
remain_topk -= s_histogram[threshold_bin + 1];
if (remain_topk == 0) {
for (uint32_t i = tx; i < num_input; i += BLOCK_SIZE) {
const auto idx = s_input_idx[r_idx][i];
const auto offset = 24 - round * 8;
const auto bin = (convert_to_uint32(input[idx]) >> offset) & 0xFF;
if (bin > threshold_bin) {
const auto pos = ::atomicAdd(&s_counter, 1);
output[pos] = idx;
}
}
__syncthreads();
break;
} else {
__syncthreads();
if (tx < RADIX + 1) {
s_histogram[tx] = 0;
}
__syncthreads();
for (uint32_t i = tx; i < num_input; i += BLOCK_SIZE) {
const auto idx = s_input_idx[r_idx][i];
const auto raw_input = input[idx];
const auto offset = 24 - round * 8;
const auto bin = (convert_to_uint32(raw_input) >> offset) & 0xFF;
if (bin > threshold_bin) {
const auto pos = ::atomicAdd(&s_counter, 1);
output[pos] = idx;
} else if (bin == threshold_bin) {
if (round == 3) {
const auto pos = ::atomicAdd(&s_last_remain, -1);
if (pos > 0) {
output[topk - pos] = idx;
}
} else {
const auto pos = ::atomicAdd(&s_num_input[r_idx ^ 1], 1);
if (pos < SMEM_INPUT_SIZE) {
/// NOTE: (dark) fuse the histogram computation here
[[likely]] s_input_idx[r_idx ^ 1][pos] = idx;
const auto bin = convert_to_uint32(raw_input);
const auto sub_bin = (bin >> (offset - 8)) & 0xFF;
::atomicAdd(&s_histogram[sub_bin], 1);
}
}
}
}
__syncthreads();
}
}
::sglang::device::legacy_radix_topk::
select<kTopKBlockSize, kMaxTopK, static_cast<int>(kSMEM / (2 * sizeof(int32_t)))>(
input, output, 0, static_cast<int>(length), static_cast<int>(topk));
}
template <bool kUsePDL>
@@ -261,9 +100,10 @@ __global__ void topk_transform_kernel(const __grid_constant__ TopKParams params)
radix_topk(score_ptr, s_topk_indices, seq_len, topk);
const auto tx = threadIdx.x;
if (tx < topk) {
indices_ptr[tx] = page_to_indices(page_ptr, s_topk_indices[tx], page_bits);
const auto raw = s_topk_indices[tx];
indices_ptr[tx] = raw < 0 ? -1 : page_to_indices(page_ptr, raw, page_bits);
if (raw_indices_ptr != nullptr) {
raw_indices_ptr[tx] = s_topk_indices[tx];
raw_indices_ptr[tx] = raw;
}
}
}
@@ -0,0 +1,238 @@
#pragma once
#include <sgl_kernel/utils.cuh>
#include <cstdint>
namespace sglang::device::legacy_radix_topk {
inline constexpr int kRadix = 256;
// Coarse bin: high byte of the ordered FP16 value, as in the original selectors.
SGL_DEVICE uint8_t coarse_key(float x) {
const __half h = __float2half_rn(x);
const uint16_t bits = __half_as_ushort(h);
const uint16_t key = (bits & 0x8000u) ? static_cast<uint16_t>(~bits) : static_cast<uint16_t>(bits | 0x8000u);
return static_cast<uint8_t>(key >> 8);
}
SGL_DEVICE uint32_t exact_key(float x) {
const uint32_t bits = __float_as_uint(x);
return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u);
}
// Exact radix top-k over finite scores. A threshold bin that fits the stash
// keeps the original two-stash refinement; a larger one is refined by rescanning
// the input. Unfilled output slots stay -1.
template <int BlockSize, int OutputCapacity, int StashEntries>
SGL_DEVICE void
select(const float* __restrict__ input, int32_t* __restrict__ output, int row_start, int length, int topk) {
static_assert(BlockSize >= kRadix + 1, "the block must initialize the complete radix histogram");
static_assert(OutputCapacity <= StashEntries, "top-k output must fit in one shared-memory stash");
alignas(128) __shared__ int histogram_buf[2][kRadix + 1];
alignas(128) __shared__ int counter;
alignas(128) __shared__ int threshold_bin_id;
alignas(128) __shared__ int num_input[2];
extern __shared__ int dynamic_smem[];
auto* input_idx = reinterpret_cast<int (*)[StashEntries]>(dynamic_smem);
auto& histogram = histogram_buf[0];
const int tx = threadIdx.x;
for (int i = tx; i < topk; i += BlockSize)
output[i] = -1;
if (tx < kRadix + 1) histogram[tx] = 0;
if (tx == 0) {
counter = 0;
threshold_bin_id = -1;
num_input[0] = 0;
num_input[1] = 0;
}
__syncthreads();
for (int idx = tx; idx < length; idx += BlockSize) {
atomicAdd(&histogram[coarse_key(input[row_start + idx])], 1);
}
__syncthreads();
const auto run_cumsum = [&] {
#pragma unroll 8
for (int i = 0; i < 8; ++i) {
if (tx < kRadix) {
const int offset = 1 << i;
const int src = i & 1;
int value = histogram_buf[src][tx];
if (tx < kRadix - offset) value += histogram_buf[src][tx + offset];
histogram_buf[src ^ 1][tx] = value;
}
__syncthreads();
}
};
run_cumsum();
if (tx < kRadix && histogram[tx] > topk && histogram[tx + 1] <= topk) {
threshold_bin_id = tx;
}
__syncthreads();
const int coarse_threshold = threshold_bin_id;
if (coarse_threshold < 0) return;
const int coarse_above = histogram[coarse_threshold + 1];
int remaining = topk - coarse_above;
if (remaining == 0) {
for (int idx = tx; idx < length; idx += BlockSize) {
if (coarse_key(input[row_start + idx]) > coarse_threshold) {
const int pos = atomicAdd(&counter, 1);
if (pos < topk) output[pos] = idx;
}
}
__syncthreads();
return;
}
const int coarse_population = histogram[coarse_threshold] - histogram[coarse_threshold + 1];
__syncthreads();
if (coarse_population >= StashEntries) {
// Find the exact cutoff one key byte per pass. Pass L counts byte L and
// emits what byte L-1 placed strictly above the cutoff; pass 0 emits the
// values above the coarse bin and pass 4 the last strict values and ties.
uint32_t prefix = 0;
int last_threshold = -1;
int strict_count = 0;
#pragma unroll 1
for (int level = 0; level <= 4; ++level) {
const uint32_t mask = level == 0 ? 0u : (~0u << (32 - level * 8));
const uint32_t last_mask = level <= 1 ? 0u : (~0u << (40 - level * 8));
const int last_shift = 32 - level * 8;
if (level < 4) {
if (tx < kRadix + 1) histogram[tx] = 0;
if (tx == 0) threshold_bin_id = -1;
}
__syncthreads();
for (int idx = tx; idx < length; idx += BlockSize) {
const float value = input[row_start + idx];
const int bin = coarse_key(value);
if (bin != coarse_threshold) {
if (level == 0 && bin > coarse_threshold) {
const int pos = atomicAdd(&counter, 1);
if (pos < topk) output[pos] = idx;
}
continue;
}
const uint32_t key = exact_key(value);
if (level > 0 && (key & last_mask) == (prefix & last_mask) &&
static_cast<int>((key >> last_shift) & 0xFFu) > last_threshold) {
const int pos = atomicAdd(&num_input[0], 1);
if (pos < strict_count) output[coarse_above + pos] = idx;
} else if ((key & mask) == prefix) {
if (level < 4) {
atomicAdd(&histogram[(key >> (24 - level * 8)) & 0xFFu], 1);
} else {
const int pos = atomicAdd(&num_input[1], 1);
if (pos < remaining) output[coarse_above + strict_count + pos] = idx;
}
}
}
__syncthreads();
if (level == 4) return;
run_cumsum();
if (tx < kRadix && histogram[tx] >= remaining && histogram[tx + 1] < remaining) {
threshold_bin_id = tx;
}
__syncthreads();
const int threshold = threshold_bin_id;
if (threshold < 0) return;
const int above = histogram[threshold + 1];
strict_count += above;
remaining -= above;
prefix |= static_cast<uint32_t>(threshold) << (24 - level * 8);
last_threshold = threshold;
// All threads must read the histogram before the next pass clears it.
__syncthreads();
}
return;
}
// The threshold bin fits the stash: original two-stash refinement.
if (tx < kRadix + 1) histogram[tx] = 0;
__syncthreads();
for (int idx = tx; idx < length; idx += BlockSize) {
const float value = input[row_start + idx];
const int bin = coarse_key(value);
if (bin > coarse_threshold) {
const int pos = atomicAdd(&counter, 1);
if (pos < topk) output[pos] = idx;
} else if (bin == coarse_threshold) {
const int pos = atomicAdd(&num_input[0], 1);
if (pos < StashEntries) {
input_idx[0][pos] = idx;
atomicAdd(&histogram[(exact_key(value) >> 24) & 0xFFu], 1);
}
}
}
__syncthreads();
#pragma unroll 4
for (int round = 0; round < 4; ++round) {
const int current = round & 1;
const int next = current ^ 1;
const int raw_num_input = num_input[current];
const int current_count = raw_num_input < StashEntries ? raw_num_input : StashEntries;
if (tx == 0) {
threshold_bin_id = -1;
num_input[next] = 0;
}
run_cumsum();
if (tx < kRadix && histogram[tx] > remaining && histogram[tx + 1] <= remaining) {
threshold_bin_id = tx;
}
__syncthreads();
const int threshold = threshold_bin_id;
if (threshold < 0) return;
const int above = histogram[threshold + 1];
const int shift = 24 - round * 8;
__syncthreads();
if (round < 3 && tx < kRadix + 1) histogram[tx] = 0;
__syncthreads();
for (int i = tx; i < current_count; i += BlockSize) {
const int idx = input_idx[current][i];
const float value = input[row_start + idx];
const int bin = static_cast<int>((exact_key(value) >> shift) & 0xFFu);
if (bin > threshold) {
const int pos = atomicAdd(&counter, 1);
if (pos < topk) output[pos] = idx;
} else if (bin == threshold && round < 3) {
const int pos = atomicAdd(&num_input[next], 1);
if (pos < StashEntries) {
input_idx[next][pos] = idx;
atomicAdd(&histogram[(exact_key(value) >> (shift - 8)) & 0xFFu], 1);
}
}
}
__syncthreads();
remaining -= above;
if (remaining == 0) return;
if (round == 3) {
for (int i = tx; i < current_count; i += BlockSize) {
const int idx = input_idx[current][i];
if (static_cast<int>(exact_key(input[row_start + idx]) & 0xFFu) == threshold) {
const int pos = atomicAdd(&counter, 1);
if (pos < topk) output[pos] = idx;
}
}
__syncthreads();
return;
}
}
}
} // namespace sglang::device::legacy_radix_topk
@@ -0,0 +1,365 @@
"""GPU regressions for the DSA kpool pooled radix top-k transform.
The kernel (``kernels/jit/csrc/dsa/kpool_topk_transform.cuh``) selects the
top ``group_topk`` pool groups of a score row with a two-stage radix select.
Stage-1 threshold bins that fit a 4096-entry shared-memory stash use it;
larger bins are resolved by an exact rescan. These tests
build rows on both sides of that boundary (tight clusters, all-equal rows,
clusters that only separate at a deeper key byte), the stage-1 direct-fill
boundary, and the minimal ``length == K + 1`` row, and compare the selected
groups against ``torch.topk``.
Comparison is by value multiset: when the K-th value is tied, any subset of
the tied groups is a valid selection, so the sorted selected values must equal
the sorted reference values while the group ids must be distinct and in
range.
Scores within a row are finite by contract (the kernel's NaN ordering is
unspecified), so no row here contains NaN; ``+inf`` is used only as padding
beyond ``lengths``, where the kernel must not read.
"""
import unittest
from typing import Optional
import torch
from sglang.kernels.ops.attention.dsa.kpool_topk_transform import (
fast_kpool_topk_transform_fused,
)
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=180, stage="base-b-kernel-unit", runner_config="1-gpu-large")
register_amd_ci(est_time=180, stage="jit-kernel-unit", runner_config="amd")
# kSmem / (2 * sizeof(int)) in kpool_topk_transform.cuh: entries per stash round.
STASH_ENTRIES = 4096
POOL_SIZE = 4
# 512 is the instantiation used by GLM-5.3-Flash (token top-k 2048, pool 4);
# 256 is the module default.
GROUP_TOPKS = (512, 256)
def _floats_from_keys(keys: torch.Tensor) -> torch.Tensor:
"""Inverse of the kernel's monotone key for positive floats.
The kernel maps a float ``x`` to ``bits | 0x80000000`` when ``x >= +0``,
so a key with the top bit set corresponds to the float whose raw bits are
the low 31 bits of the key.
"""
assert bool(((keys >> 31) & 1).all()), "positive-float keys have bit 31 set"
bits = (keys & 0x7FFFFFFF).to(torch.int32)
return bits.view(torch.float32)
def _coarse_keys(values: torch.Tensor) -> torch.Tensor:
"""Mirror the kernel's high-byte ordered-FP16 key on the CPU."""
bits = values.to(torch.float16).view(torch.int16).to(torch.int32) & 0xFFFF
ordered = torch.where(
(bits & 0x8000) != 0,
(~bits) & 0xFFFF,
bits | 0x8000,
)
return ordered >> 8
@unittest.skipUnless(torch.cuda.is_available(), "Test requires CUDA")
class TestDsaKpoolTopkTransform(CustomTestCase):
def setUp(self):
super().setUp()
self.device = torch.device("cuda")
self.generator = torch.Generator(device="cpu").manual_seed(0)
def tearDown(self):
torch.cuda.empty_cache()
super().tearDown()
def _rand(self, *shape) -> torch.Tensor:
return torch.rand(*shape, generator=self.generator, dtype=torch.float32)
def _tight_cluster(self, length: int, *, base: float = 1.0) -> torch.Tensor:
# Strictly increasing (modulo FP32 rounding), with the best candidates
# deliberately last. All values share the fp16 coarse bin, so an
# arrival-order stash prefix is deterministically wrong.
return base + torch.arange(length, dtype=torch.float32) * (1e-3 / length)
def _byte_depth_clusters(self, depth: int, per_side: int) -> torch.Tensor:
"""Two clusters of ``per_side`` keys that first differ at key bit ``depth``.
Both share key byte 24 (0xBF, float 0.5..1.0), so stage 1 puts all
``2 * per_side`` entries in one bin. The cluster with bit ``depth`` set
holds every true top-K member; bits below ``depth`` are random.
"""
base = 0xBF000000
noise_bits = depth
noise = torch.randint(
0, 1 << noise_bits, (2 * per_side,), generator=self.generator
)
keys = torch.full((2 * per_side,), base, dtype=torch.int64) + noise
keys[:per_side] += 1 << depth
perm = torch.randperm(2 * per_side, generator=self.generator)
return _floats_from_keys(keys[perm])
def _run(
self,
rows: list,
group_topk: int,
page_table_bias: Optional[int] = None,
topk_offset: Optional[int] = None,
row_start: int = 0,
):
batch = len(rows)
stride = row_start + max(row.numel() for row in rows)
# Pad with +inf so any read past ``lengths`` would corrupt the selection.
score = torch.full((batch, stride), float("inf"), dtype=torch.float32)
lengths = torch.empty(batch, dtype=torch.int32)
for i, row in enumerate(rows):
score[i, row_start : row_start + row.numel()] = row
lengths[i] = row.numel()
score = score.to(self.device)
lengths = lengths.to(self.device)
row_starts = (
torch.full((batch,), row_start, dtype=torch.int32, device=self.device)
if row_start
else None
)
page_table = None
topk_indices_offset = None
if page_table_bias is not None:
page_table = (
torch.arange(stride * POOL_SIZE, dtype=torch.int32)
.unsqueeze(0)
.repeat(batch, 1)
+ page_table_bias
).to(self.device)
if topk_offset is not None:
topk_indices_offset = torch.full(
(batch,), topk_offset, dtype=torch.int32, device=self.device
)
out = fast_kpool_topk_transform_fused(
score=score,
lengths=lengths,
kpool=POOL_SIZE,
topk=group_topk * POOL_SIZE,
page_table=page_table,
topk_indices_offset=topk_indices_offset,
row_starts=row_starts,
)
self.assertEqual(out.shape, (batch, group_topk * POOL_SIZE))
return out.cpu()
def _selected_groups(self, out_row: torch.Tensor, group_topk: int, length: int):
"""Decode expanded token columns back to group ids and check the layout."""
self.assertTrue(
bool((out_row >= 0).all()), "no pad columns expected when length > K"
)
tokens = out_row.view(group_topk, POOL_SIZE).to(torch.int64)
group_ids = tokens[:, 0] // POOL_SIZE
expected_tokens = group_ids.unsqueeze(1) * POOL_SIZE + torch.arange(POOL_SIZE)
self.assertTrue(
torch.equal(tokens, expected_tokens),
"each selected group must expand to its pool_size consecutive tokens",
)
self.assertLess(int(group_ids.max()), length)
self.assertEqual(
torch.unique(group_ids).numel(), group_topk, "group ids must be distinct"
)
return group_ids
def _check_row(self, out_row: torch.Tensor, row: torch.Tensor, group_topk: int):
group_ids = self._selected_groups(out_row, group_topk, row.numel())
selected = row[group_ids].sort().values
reference = torch.topk(row, group_topk).values.sort().values
self.assertTrue(
torch.equal(selected, reference),
"selected values must match torch.topk as a multiset",
)
def _check_rows(self, rows: list, group_topk: int):
out = self._run(rows, group_topk)
for i, row in enumerate(rows):
with self.subTest(row=i, length=row.numel(), group_topk=group_topk):
self._check_row(out[i], row, group_topk)
def test_stage1_bin_exceeds_stash(self):
# 4097 and 17802 candidates in one stage-1 bin, both beyond the 4096-entry
# stash. A clipped stash would keep the first 4096 by atomic arrival
# order and disagree with torch.topk on such rows.
for group_topk in GROUP_TOPKS:
rows = [
self._tight_cluster(STASH_ENTRIES + 1),
self._tight_cluster(17802),
]
self._check_rows(rows, group_topk)
def test_oversized_bin_with_strictly_higher_values(self):
# Exercise the output offset between values emitted above the coarse
# threshold and values selected by the exact overflow rescan.
for group_topk in GROUP_TOPKS:
high_count = 37
high = 2.0 + torch.arange(high_count, dtype=torch.float32) * 1e-3
threshold = self._tight_cluster(STASH_ENTRIES + 5311)
low = torch.zeros(257, dtype=torch.float32)
self._check_rows([torch.cat([high, threshold, low])], group_topk)
def test_stride_larger_than_length(self):
# Row length far below the score stride, as with a padded score buffer.
for group_topk in GROUP_TOPKS:
long_row = self._tight_cluster(17802)
short_row = self._rand(group_topk + 1)
out = self._run([long_row, short_row], group_topk)
self._check_row(out[0], long_row, group_topk)
self._check_row(out[1], short_row, group_topk)
def test_nonzero_row_start_with_oversized_bin(self):
for group_topk in GROUP_TOPKS:
row = self._tight_cluster(STASH_ENTRIES + 5311)
out = self._run([row], group_topk, row_start=3)
self._check_row(out[0], row, group_topk)
def test_all_equal_rows_longer_than_stash(self):
# Every key is identical, so the final scan must clip the exact ties
# without duplicating indices or leaving an output slot unfilled.
for group_topk in GROUP_TOPKS:
rows = [
torch.full((STASH_ENTRIES + 1,), 0.75),
torch.full((STASH_ENTRIES + 1,), 0.0),
torch.full((STASH_ENTRIES + 1,), -0.0),
]
self._check_rows(rows, group_topk)
def test_clusters_separating_at_each_key_byte(self):
# 4097 vs 4097 keys that first differ at key bit 16, 8 or 0: the
# stage-1 bin overflows and the rescan must walk to the byte where the
# clusters separate.
for group_topk in GROUP_TOPKS:
rows = [
self._byte_depth_clusters(depth, STASH_ENTRIES + 1)
for depth in (16, 8, 0)
]
self._check_rows(rows, group_topk)
def test_stage1_exact_fill_boundary(self):
# Exactly K distinct values above a 65536-way tie: stage 1 finds the
# threshold bin with nothing left to refine and fills all K slots directly.
for group_topk in GROUP_TOPKS:
top = torch.linspace(2.0, 3.0, group_topk)
tie = torch.full((65536,), 1.0)
row = torch.cat([top, tie])
row = row[torch.randperm(row.numel(), generator=self.generator)]
self._check_rows([row], group_topk)
def test_stash_capacity_boundary(self):
# The exact-capacity case brackets the stash boundary. Different
# input orders protect against an arrival-order subset being selected;
# the mixed-bin row also covers the output offset after values above the
# exact-capacity threshold bin have already been emitted.
row = self._tight_cluster(STASH_ENTRIES)
self.assertEqual(torch.unique(_coarse_keys(row)).numel(), 1)
negative_row = self._tight_cluster(STASH_ENTRIES, base=-0.9)
self.assertEqual(torch.unique(_coarse_keys(negative_row)).numel(), 1)
high = 2.0 + torch.arange(37, dtype=torch.float32) * 1e-3
low = torch.full((257,), -2.0)
mixed_bin_row = torch.cat([high, row, low])
threshold_key = _coarse_keys(row)[0]
self.assertEqual(
int((_coarse_keys(mixed_bin_row) == threshold_key).sum()),
STASH_ENTRIES,
)
rows = [
row,
row.flip(0),
row[torch.randperm(row.numel(), generator=self.generator)],
negative_row,
mixed_bin_row,
]
below_capacity = self._tight_cluster(STASH_ENTRIES - 1)
self.assertEqual(torch.unique(_coarse_keys(below_capacity)).numel(), 1)
for group_topk in GROUP_TOPKS:
self._check_rows(rows, group_topk)
self._check_rows([below_capacity], group_topk)
def test_minimal_radix_length(self):
# length == K + 1 is the shortest row that takes the radix path.
for group_topk in GROUP_TOPKS:
self._check_rows([self._rand(group_topk + 1)], group_topk)
def test_random_rows_batch(self):
for group_topk in GROUP_TOPKS:
rows = [self._rand(length) for length in (3000, 20000)]
self._check_rows(rows, group_topk)
def test_page_table_and_offset_expansion(self):
group_topk = 512
row = self._tight_cluster(STASH_ENTRIES + 5311)
bias = 1000
out = self._run([row], group_topk, page_table_bias=bias)
self._check_row(out[0] - bias, row, group_topk)
offset = 777
out = self._run([row], group_topk, topk_offset=offset)
self._check_row(out[0] - offset, row, group_topk)
def test_tail_append_with_page_table_row_remap(self):
"""Tail layout and absolute page-table row selection work together."""
group_topk = 512
rows = [self._tight_cluster(STASH_ENTRIES + 5311), self._rand(500)]
batch = len(rows)
stride = max(row.numel() for row in rows)
score = torch.full((batch, stride), float("inf"), dtype=torch.float32)
lengths = torch.tensor([row.numel() for row in rows], dtype=torch.int32)
for i, row in enumerate(rows):
score[i, : row.numel()] = row
pool_size = POOL_SIZE
topk = group_topk * pool_size
max_tokens = stride * pool_size + pool_size - 1
page_biases = torch.tensor([10000, 20000, 30000], dtype=torch.int32)
page_table = torch.arange(max_tokens, dtype=torch.int32).expand(3, -1)
page_table = page_table + page_biases[:, None]
page_table_row_index = torch.tensor([2, 0], dtype=torch.int32)
tail_counts = (3, 1)
seq_lens = torch.tensor(
[rows[i].numel() * pool_size + tail_counts[i] for i in range(batch)],
dtype=torch.int32,
)
out = fast_kpool_topk_transform_fused(
score=score.cuda(),
lengths=lengths.cuda(),
kpool=pool_size,
topk=topk,
page_table=page_table.cuda(),
seq_lens=seq_lens.cuda(),
page_table_row_index=page_table_row_index.cuda(),
).cpu()
for i, row in enumerate(rows):
bias = int(page_biases[page_table_row_index[i]])
history_len = min(row.numel() * pool_size, topk)
if row.numel() > group_topk:
self._check_row(out[i, :history_len] - bias, row, group_topk)
else:
self.assertTrue(
torch.equal(
out[i, :history_len] - bias,
torch.arange(history_len, dtype=torch.int32),
)
)
tail = out[i, history_len : history_len + tail_counts[i]] - bias
expected_tail = torch.arange(
row.numel() * pool_size,
row.numel() * pool_size + tail_counts[i],
dtype=torch.int32,
)
self.assertTrue(torch.equal(tail, expected_tail))
self.assertTrue(bool((out[i, history_len + tail_counts[i] :] == -1).all()))
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,197 @@
"""Correctness tests for the DeepSeek-V4 JIT top-k transform v1.
``topk_transform_paged`` dispatches to ``deepseek_v4/topk_v1.cuh`` on CUDA. Its
radix select first bins scores by the high byte of their ordered FP16 value and
stashes the threshold bin in shared memory (8192 entries). These tests put the
threshold bin on both sides of that capacity, with strictly higher values
emitted before it, ties at the cutoff, and clusters that only separate in one
of the lower three FP32 key bytes. They cover runtime ``topk`` values, the
``seq_len <= topk`` path, score rows padded past ``seq_lens``, a noncontiguous
score view, and both the page-table mapped and raw outputs.
Selections are compared with ``torch.topk`` as value multisets: any subset of
values tied at the cutoff is a valid selection, but indices must be distinct
and in range, and the mapped output must be the page-table transform of the
raw output.
"""
from __future__ import annotations
import sys
import pytest
import torch
from sglang.kernels.ops.attention.dsv4.topk import topk_transform_paged
from sglang.srt.utils import is_hip
from sglang.test.ci.ci_register import register_cuda_ci
register_cuda_ci(est_time=60, stage="base-b-kernel-unit", runner_config="1-gpu-large")
pytestmark = pytest.mark.skipif(
not torch.cuda.is_available() or is_hip(),
reason="topk_transform_paged uses the JIT v1 kernel on CUDA only",
)
# kSMEM / (2 * sizeof(int32_t)) in topk_v1.cuh: entries per stash round.
STASH_ENTRIES = 8192
PAGE_SIZE = 64
TOPKS = (512, 1024, 300)
def _generator(seed: int = 0) -> torch.Generator:
return torch.Generator(device="cpu").manual_seed(seed)
def _coarse_keys(values: torch.Tensor) -> torch.Tensor:
"""Mirror the kernel's high-byte ordered-FP16 key on the CPU."""
bits = values.to(torch.float16).view(torch.int16).to(torch.int32) & 0xFFFF
ordered = torch.where((bits & 0x8000) != 0, (~bits) & 0xFFFF, bits | 0x8000)
return ordered >> 8
def _tight_cluster(length: int, base: float = 1.0) -> torch.Tensor:
# Increasing values in one coarse bin, best candidates last, so an
# arrival-order stash prefix is wrong.
return base + torch.arange(length, dtype=torch.float32) * (1e-3 / length)
def _byte_depth_clusters(depth: int, per_side: int, gen) -> torch.Tensor:
# Two clusters sharing the coarse bin whose keys first differ at bit
# ``depth``; the cluster with that bit set holds the true top-k.
keys = torch.full((2 * per_side,), 0x3F000000, dtype=torch.int64)
keys += torch.randint(0, 1 << depth, (2 * per_side,), generator=gen)
keys[:per_side] += 1 << depth
keys = keys[torch.randperm(2 * per_side, generator=gen)]
return keys.to(torch.int32).view(torch.float32)
def _run_and_check(
rows: list[torch.Tensor], topk: int, stride_pad: int = 0, view_pad: int = 0
):
gen = _generator(topk)
batch = len(rows)
stride = max(row.numel() for row in rows) + stride_pad
# Pad with +inf so any read past ``seq_lens`` would corrupt the selection.
# ``view_pad`` passes a column slice whose row stride exceeds its width.
storage = torch.full(
(batch, stride + view_pad), float("inf"), dtype=torch.float32, device="cuda"
)
scores = torch.full((batch, stride), float("inf"), dtype=torch.float32)
for i, row in enumerate(rows):
scores[i, : row.numel()] = row
seq_lens = torch.tensor([row.numel() for row in rows], dtype=torch.int32)
num_pages = (stride + PAGE_SIZE - 1) // PAGE_SIZE
page_table = torch.stack(
[torch.randperm(4 * num_pages, generator=gen)[:num_pages] for _ in range(batch)]
).to(torch.int32)
out_page = torch.full((batch, topk), -7, dtype=torch.int32, device="cuda")
out_raw = torch.full((batch, topk), -7, dtype=torch.int32, device="cuda")
scores_view = storage[:, :stride]
scores_view.copy_(scores)
assert scores_view.is_contiguous() == (view_pad == 0)
topk_transform_paged(
scores_view,
seq_lens.cuda(),
page_table.cuda(),
out_page,
PAGE_SIZE,
out_raw,
)
out_page = out_page.cpu().to(torch.int64)
out_raw = out_raw.cpu().to(torch.int64)
for i, row in enumerate(rows):
length = row.numel()
raw = out_raw[i]
if length <= topk:
expected = torch.full((topk,), -1, dtype=torch.int64)
expected[:length] = torch.arange(length)
assert torch.equal(raw, expected), f"row {i}: trivial raw indices"
else:
assert bool((raw >= 0).all()) and bool((raw < length).all()), f"row {i}"
assert torch.unique(raw).numel() == topk, f"row {i}: duplicate index"
selected = row[raw].sort().values
reference = torch.topk(row, topk).values.sort().values
assert torch.equal(selected, reference), f"row {i}: wrong selection"
valid = raw >= 0
mapped = torch.full_like(raw, -1)
mapped[valid] = (
page_table[i, raw[valid] // PAGE_SIZE].to(torch.int64) * PAGE_SIZE
+ raw[valid] % PAGE_SIZE
)
assert torch.equal(out_page[i], mapped), f"row {i}: page mapping"
@pytest.mark.parametrize("topk", TOPKS)
def test_threshold_bin_around_stash_capacity(topk):
gen = _generator(1)
below = _tight_cluster(STASH_ENTRIES - 1)
exact = _tight_cluster(STASH_ENTRIES)
above = _tight_cluster(STASH_ENTRIES + 1)
for row in (below, exact, above):
assert torch.unique(_coarse_keys(row)).numel() == 1
rows = [
below,
exact,
exact.flip(0),
exact[torch.randperm(exact.numel(), generator=gen)],
_tight_cluster(STASH_ENTRIES, base=-0.9),
above,
_tight_cluster(3 * STASH_ENTRIES + 17),
]
_run_and_check(rows, topk)
@pytest.mark.parametrize("topk", TOPKS)
def test_oversized_bin_after_strictly_higher_values(topk):
gen = _generator(2)
high = 2.0 + torch.arange(37, dtype=torch.float32) * 1e-3
low = torch.full((257,), -2.0)
row = torch.cat([high, _tight_cluster(STASH_ENTRIES + 5311), low])
exact_bin_row = torch.cat([high, _tight_cluster(STASH_ENTRIES), low])
rows = [row, row[torch.randperm(row.numel(), generator=gen)], exact_bin_row]
_run_and_check(rows, topk)
@pytest.mark.parametrize("topk", TOPKS)
def test_cutoff_ties(topk):
gen = _generator(3)
top = torch.linspace(2.0, 3.0, topk)
tie_row = torch.cat([top, torch.full((65536,), 1.0)])
rows = [
torch.full((STASH_ENTRIES + 1,), 0.75),
torch.full((STASH_ENTRIES + 1,), 0.0),
torch.full((STASH_ENTRIES + 1,), -0.0),
torch.full((topk + 1,), 0.5),
tie_row[torch.randperm(tie_row.numel(), generator=gen)],
torch.cat([top[: topk // 2], torch.full((2 * STASH_ENTRIES,), 1.0)]),
]
_run_and_check(rows, topk)
@pytest.mark.parametrize("topk", TOPKS)
def test_clusters_separating_at_lower_key_bytes(topk):
gen = _generator(4)
rows = [_byte_depth_clusters(depth, STASH_ENTRIES + 1, gen) for depth in (16, 8, 0)]
_run_and_check(rows, topk)
@pytest.mark.parametrize("topk", TOPKS)
def test_short_and_random_rows_with_padded_and_strided_scores(topk):
gen = _generator(5)
rows = [
torch.rand(1, generator=gen),
torch.rand(topk - 1, generator=gen),
torch.rand(topk, generator=gen),
torch.rand(topk + 1, generator=gen),
torch.rand(3000, generator=gen),
torch.randn(40000, generator=gen),
]
_run_and_check(rows, topk, stride_pad=PAGE_SIZE + 3)
_run_and_check(rows, topk, view_pad=PAGE_SIZE + 5)
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-v"]))