Fix overflow in legacy and v1 DSA top-k selectors (#37625)

Co-authored-by: Benjamin Oldenburg <benjamin.oldenburg@ordis.co.th>
This commit is contained in:
ormandj
2026-10-15 21:44:58 +08:00
committed by GitHub
co-authored by Benjamin Oldenburg
parent 2f5407da28
commit c5bf3aafbc
8 changed files with 1204 additions and 517 deletions
@@ -24,6 +24,7 @@ limitations under the License.
#include <cstddef>
#include <cstdint>
#include <optional>
#include <sgl_kernel/dsa/legacy_radix_topk.cuh>
namespace {
@@ -62,18 +63,6 @@ struct TopKParams {
bool sort_output;
};
__device__ __forceinline__ uint8_t convert_to_uint8(float x) {
__half h = __float2half_rn(x);
uint16_t bits = __half_as_ushort(h);
uint16_t key = (bits & 0x8000) ? static_cast<uint16_t>(~bits) : static_cast<uint16_t>(bits | 0x8000);
return static_cast<uint8_t>(key >> 8);
}
__device__ __forceinline__ uint32_t convert_to_uint32(float x) {
uint32_t bits = __float_as_uint(x);
return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u);
}
__device__ __forceinline__ int32_t
page_to_slot(const int32_t* __restrict__ page_table, uint32_t i, uint32_t page_bits) {
const uint32_t mask = (1u << page_bits) - 1u;
@@ -104,154 +93,10 @@ __device__ void naive_paged_transform(
__device__ void
radix_topk(const float* __restrict__ input, int32_t* __restrict__ output, uint32_t length, uint32_t topk) {
constexpr uint32_t RADIX = 256;
constexpr uint32_t BLOCK_SIZE = kBlockSize;
constexpr uint32_t SMEM_INPUT_SIZE = kSMEM / (2 * sizeof(int32_t));
alignas(128) __shared__ uint32_t _s_histogram_buf[2][RADIX + 32];
alignas(128) __shared__ uint32_t s_counter;
alignas(128) __shared__ uint32_t s_threshold_bin_id;
alignas(128) __shared__ uint32_t s_num_input[2];
alignas(128) __shared__ int32_t s_last_remain;
extern __shared__ uint32_t s_input_idx[][SMEM_INPUT_SIZE];
const uint32_t tx = threadIdx.x;
uint32_t remain_topk = topk;
auto& s_histogram = _s_histogram_buf[0];
const auto run_cumsum = [&] {
#pragma unroll 8
for (int32_t i = 0; i < 8; ++i) {
static_assert(1 << 8 == RADIX);
if (tx < RADIX) {
const auto j = 1 << i;
const auto k = i & 1;
auto value = _s_histogram_buf[k][tx];
if (tx + j < RADIX) {
value += _s_histogram_buf[k][tx + j];
}
_s_histogram_buf[k ^ 1][tx] = value;
}
__syncthreads();
}
};
// stage 1: 8bit coarse histogram
if (tx < RADIX + 1) s_histogram[tx] = 0;
::sglang::device::legacy_radix_topk::
select<static_cast<int>(kBlockSize), static_cast<int>(kMaxTopK), static_cast<int>(kSMEM / (2 * sizeof(int32_t)))>(
input, output, 0, static_cast<int>(length), static_cast<int>(topk));
__syncthreads();
for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) {
const auto bin = convert_to_uint8(input[idx]);
::atomicAdd(&s_histogram[bin], 1);
}
__syncthreads();
run_cumsum();
if (tx < RADIX && s_histogram[tx] > remain_topk && s_histogram[tx + 1] <= remain_topk) {
s_threshold_bin_id = tx;
s_num_input[0] = 0;
s_counter = 0;
}
__syncthreads();
{
const auto threshold_bin = s_threshold_bin_id;
remain_topk -= s_histogram[threshold_bin + 1];
if (remain_topk == 0) {
for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) {
const uint32_t bin = convert_to_uint8(input[idx]);
if (bin > threshold_bin) {
const auto pos = ::atomicAdd(&s_counter, 1);
output[pos] = static_cast<int32_t>(idx);
}
}
__syncthreads();
return;
}
__syncthreads();
if (tx < RADIX + 1) s_histogram[tx] = 0;
__syncthreads();
for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) {
const float raw_input = input[idx];
const uint32_t bin = convert_to_uint8(raw_input);
if (bin > threshold_bin) {
const auto pos = ::atomicAdd(&s_counter, 1);
output[pos] = static_cast<int32_t>(idx);
} else if (bin == threshold_bin) {
const auto pos = ::atomicAdd(&s_num_input[0], 1);
if (C10_LIKELY(pos < SMEM_INPUT_SIZE)) {
s_input_idx[0][pos] = idx;
const auto bin32 = convert_to_uint32(raw_input);
const auto sub_bin = (bin32 >> 24) & 0xFF;
::atomicAdd(&s_histogram[sub_bin], 1);
}
}
}
__syncthreads();
}
// stage 2: refine with 8bit radix passes
#pragma unroll 4
for (int round = 0; round < 4; ++round) {
const auto r_idx = round % 2;
const auto raw_num_input = s_num_input[r_idx];
const auto num_input = raw_num_input < SMEM_INPUT_SIZE ? raw_num_input : SMEM_INPUT_SIZE;
run_cumsum();
if (tx < RADIX && s_histogram[tx] > remain_topk && s_histogram[tx + 1] <= remain_topk) {
s_threshold_bin_id = tx;
s_num_input[r_idx ^ 1] = 0;
s_last_remain = static_cast<int32_t>(remain_topk - s_histogram[tx + 1]);
}
__syncthreads();
const auto threshold_bin = s_threshold_bin_id;
remain_topk -= s_histogram[threshold_bin + 1];
if (remain_topk == 0) {
for (uint32_t i = tx; i < num_input; i += BLOCK_SIZE) {
const auto idx = s_input_idx[r_idx][i];
const auto offset = 24 - round * 8;
const auto bin = (convert_to_uint32(input[idx]) >> offset) & 0xFF;
if (bin > threshold_bin) {
const auto pos = ::atomicAdd(&s_counter, 1);
output[pos] = static_cast<int32_t>(idx);
}
}
__syncthreads();
break;
}
__syncthreads();
if (tx < RADIX + 1) s_histogram[tx] = 0;
__syncthreads();
for (uint32_t i = tx; i < num_input; i += BLOCK_SIZE) {
const auto idx = s_input_idx[r_idx][i];
const auto raw_input = input[idx];
const auto offset = 24 - round * 8;
const auto bin = (convert_to_uint32(raw_input) >> offset) & 0xFF;
if (bin > threshold_bin) {
const auto pos = ::atomicAdd(&s_counter, 1);
output[pos] = static_cast<int32_t>(idx);
} else if (bin == threshold_bin) {
if (round == 3) {
const auto pos = ::atomicAdd(&s_last_remain, -1);
if (pos > 0) {
output[topk - pos] = static_cast<int32_t>(idx);
}
} else {
const auto pos = ::atomicAdd(&s_num_input[r_idx ^ 1], 1);
if (C10_LIKELY(pos < SMEM_INPUT_SIZE)) {
s_input_idx[r_idx ^ 1][pos] = idx;
const auto bin32 = convert_to_uint32(raw_input);
const auto sub_bin = (bin32 >> (offset - 8)) & 0xFF;
::atomicAdd(&s_histogram[sub_bin], 1);
}
}
}
}
__syncthreads();
}
}
#ifdef USE_ROCM
@@ -366,7 +211,8 @@ __device__ __forceinline__ uint32_t next_pow2_at_least_64(uint32_t x) {
__global__ __launch_bounds__(kBlockSize) void deepseek_v4_topk_transform_kernel(const TopKParams params) {
const auto bid = blockIdx.x;
const auto seq_len = params.seq_lens[bid];
const auto raw_seq_len = params.seq_lens[bid];
const auto seq_len = raw_seq_len < 0 ? 0 : raw_seq_len;
const auto topk = params.topk;
const auto score_ptr = params.scores + bid * params.score_stride;
const auto page_ptr = params.page_table + bid * params.page_table_stride;
@@ -433,7 +279,7 @@ __global__ __launch_bounds__(kBlockSize) void deepseek_v4_topk_transform_kernel(
for (uint32_t i = threadIdx.x; i < topk; i += kBlockSize) {
const auto raw = s_topk_indices[i];
indices_ptr[i] = page_to_slot(page_ptr, static_cast<uint32_t>(raw), params.page_bits);
indices_ptr[i] = raw < 0 ? -1 : page_to_slot(page_ptr, static_cast<uint32_t>(raw), params.page_bits);
if (raw_indices_ptr != nullptr) {
raw_indices_ptr[i] = raw;
}
@@ -17,6 +17,7 @@
#include <cstddef>
#include <cstdint>
#include <optional>
#include <sgl_kernel/dsa/legacy_radix_topk.cuh>
namespace {
@@ -75,180 +76,10 @@ __device__ void naive_topk_transform_ragged(
}
}
__device__ __forceinline__ auto convert_to_uint8(float x) -> uint8_t {
__half h = __float2half_rn(x);
uint16_t bits = __half_as_ushort(h);
uint16_t key = (bits & 0x8000) ? static_cast<uint16_t>(~bits) : static_cast<uint16_t>(bits | 0x8000);
return static_cast<uint8_t>(key >> 8);
}
__device__ __forceinline__ auto convert_to_uint32(float x) -> uint32_t {
uint32_t bits = __float_as_uint(x);
return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u);
}
__device__ void fast_topk_cuda_tl(const float* __restrict__ input, int* __restrict__ index, int row_start, int length) {
// An optimized topk kernel copied from tilelang kernel
// We assume length > TopK here, or it will crash
int topk = TopK;
constexpr auto BLOCK_SIZE = 1024;
constexpr auto RADIX = 256;
constexpr auto SMEM_INPUT_SIZE = kSmem / (2 * sizeof(int));
alignas(128) __shared__ int s_histogram_buf[2][RADIX + 128];
alignas(128) __shared__ int s_counter;
alignas(128) __shared__ int s_threshold_bin_id;
alignas(128) __shared__ int s_num_input[2];
auto& s_histogram = s_histogram_buf[0];
// allocate for two rounds
extern __shared__ int s_input_idx[][SMEM_INPUT_SIZE];
const int tx = threadIdx.x;
// stage 1: 8bit coarse histogram
if (tx < RADIX + 1) s_histogram[tx] = 0;
::sglang::device::legacy_radix_topk::select<kThreadsPerBlock, TopK, static_cast<int>(kSmem / (2 * sizeof(int)))>(
input, index, row_start, length, TopK);
__syncthreads();
for (int idx = tx; idx < length; idx += BLOCK_SIZE) {
const auto bin = convert_to_uint8(input[idx + row_start]);
::atomicAdd(&s_histogram[bin], 1);
}
__syncthreads();
const auto run_cumsum = [&] {
#pragma unroll 8
for (int i = 0; i < 8; ++i) {
static_assert(1 << 8 == RADIX);
if (C10_LIKELY(tx < RADIX)) {
const auto j = 1 << i;
const auto k = i & 1;
auto value = s_histogram_buf[k][tx];
if (tx < RADIX - j) {
value += s_histogram_buf[k][tx + j];
}
s_histogram_buf[k ^ 1][tx] = value;
}
__syncthreads();
}
};
run_cumsum();
if (tx < RADIX && s_histogram[tx] > topk && s_histogram[tx + 1] <= topk) {
s_threshold_bin_id = tx;
s_num_input[0] = 0;
s_counter = 0;
}
__syncthreads();
const auto threshold_bin = s_threshold_bin_id;
topk -= s_histogram[threshold_bin + 1];
if (topk == 0) {
for (int idx = tx; idx < length; idx += BLOCK_SIZE) {
const auto bin = static_cast<int>(convert_to_uint8(input[idx + row_start]));
if (bin > threshold_bin) {
const auto pos = ::atomicAdd(&s_counter, 1);
index[pos] = idx;
}
}
__syncthreads();
return;
} else {
__syncthreads();
if (tx < RADIX + 1) {
s_histogram[tx] = 0;
}
__syncthreads();
for (int idx = tx; idx < length; idx += BLOCK_SIZE) {
const auto raw_input = input[idx + row_start];
const auto bin = static_cast<int>(convert_to_uint8(raw_input));
if (bin > threshold_bin) {
const auto pos = ::atomicAdd(&s_counter, 1);
index[pos] = idx;
} else if (bin == threshold_bin) {
const auto pos = ::atomicAdd(&s_num_input[0], 1);
/// NOTE: (dark) fuse the histogram computation here
if (C10_LIKELY(pos < SMEM_INPUT_SIZE)) {
s_input_idx[0][pos] = idx;
const auto bin = convert_to_uint32(raw_input);
const auto sub_bin = (bin >> 24) & 0xFF;
::atomicAdd(&s_histogram[sub_bin], 1);
}
}
}
__syncthreads();
}
// stage 2: refine with 8bit radix passes
#pragma unroll 4
for (int round = 0; round < 4; ++round) {
__shared__ int s_last_remain;
const auto r_idx = round % 2;
// clip here to prevent overflow
const auto _raw_num_input = s_num_input[r_idx];
const auto num_input = (_raw_num_input < int(SMEM_INPUT_SIZE)) ? _raw_num_input : int(SMEM_INPUT_SIZE);
run_cumsum();
if (tx < RADIX && s_histogram[tx] > topk && s_histogram[tx + 1] <= topk) {
s_threshold_bin_id = tx;
s_num_input[r_idx ^ 1] = 0;
s_last_remain = topk - s_histogram[tx + 1];
}
__syncthreads();
const auto threshold_bin = s_threshold_bin_id;
topk -= s_histogram[threshold_bin + 1];
if (topk == 0) {
for (int i = tx; i < num_input; i += BLOCK_SIZE) {
const auto idx = s_input_idx[r_idx][i];
const auto offset = 24 - round * 8;
const auto bin = (convert_to_uint32(input[idx + row_start]) >> offset) & 0xFF;
if (bin > threshold_bin) {
const auto pos = ::atomicAdd(&s_counter, 1);
index[pos] = idx;
}
}
__syncthreads();
break;
} else {
__syncthreads();
if (tx < RADIX + 1) {
s_histogram[tx] = 0;
}
__syncthreads();
for (int i = tx; i < num_input; i += BLOCK_SIZE) {
const auto idx = s_input_idx[r_idx][i];
const auto raw_input = input[idx + row_start];
const auto offset = 24 - round * 8;
const auto bin = (convert_to_uint32(raw_input) >> offset) & 0xFF;
if (bin > threshold_bin) {
const auto pos = ::atomicAdd(&s_counter, 1);
index[pos] = idx;
} else if (bin == threshold_bin) {
if (round == 3) {
const auto pos = ::atomicAdd(&s_last_remain, -1);
if (pos > 0) {
index[TopK - pos] = idx;
}
} else {
const auto pos = ::atomicAdd(&s_num_input[r_idx ^ 1], 1);
if (C10_LIKELY(pos < SMEM_INPUT_SIZE)) {
/// NOTE: (dark) fuse the histogram computation here
s_input_idx[r_idx ^ 1][pos] = idx;
const auto bin = convert_to_uint32(raw_input);
const auto sub_bin = (bin >> (offset - 8)) & 0xFF;
::atomicAdd(&s_histogram[sub_bin], 1);
}
}
}
}
__syncthreads();
}
}
}
__global__ __launch_bounds__(kThreadsPerBlock) // topk
@@ -290,10 +121,10 @@ __global__ __launch_bounds__(kThreadsPerBlock) // decode
static_assert(TopK / kThreadsPerBlock == 2);
const auto idx_0 = tid;
const auto pos_0 = s_indices[idx_0];
dst_page_entry[idx_0] = src_page_entry[pos_0];
dst_page_entry[idx_0] = pos_0 < 0 ? -1 : src_page_entry[pos_0];
const auto idx_1 = tid + kThreadsPerBlock;
const auto pos_1 = s_indices[idx_1];
dst_page_entry[idx_1] = src_page_entry[pos_1];
dst_page_entry[idx_1] = pos_1 < 0 ? -1 : src_page_entry[pos_1];
}
}
@@ -342,10 +173,10 @@ __global__ __launch_bounds__(kThreadsPerBlock) // prefill
static_assert(TopK / kThreadsPerBlock == 2);
const auto idx_0 = tid;
const auto pos_0 = s_indices[idx_0];
dst_page_entry[idx_0] = src_page_entry[pos_0];
dst_page_entry[idx_0] = pos_0 < 0 ? -1 : src_page_entry[pos_0];
const auto idx_1 = tid + kThreadsPerBlock;
const auto pos_1 = s_indices[idx_1];
dst_page_entry[idx_1] = src_page_entry[pos_1];
dst_page_entry[idx_1] = pos_1 < 0 ? -1 : src_page_entry[pos_1];
}
}
@@ -373,10 +204,10 @@ __global__ __launch_bounds__(kThreadsPerBlock) // prefill, ragged kv
static_assert(TopK / kThreadsPerBlock == 2);
const auto idx_0 = tid;
const auto pos_0 = s_indices[idx_0];
dst_indices_entry[idx_0] = pos_0 + offset;
dst_indices_entry[idx_0] = pos_0 < 0 ? -1 : pos_0 + offset;
const auto idx_1 = tid + kThreadsPerBlock;
const auto pos_1 = s_indices[idx_1];
dst_indices_entry[idx_1] = pos_1 + offset;
dst_indices_entry[idx_1] = pos_1 < 0 ? -1 : pos_1 + offset;
}
}
@@ -0,0 +1,239 @@
#pragma once
#include <cuda_fp16.h>
#include <cstddef>
#include <cstdint>
namespace sglang::device::legacy_radix_topk {
inline constexpr int kRadix = 256;
// Coarse bin: high byte of the ordered FP16 value, as in the original selectors.
__device__ __forceinline__ uint8_t coarse_key(float x) {
const __half h = __float2half_rn(x);
const uint16_t bits = __half_as_ushort(h);
const uint16_t key = (bits & 0x8000u) ? static_cast<uint16_t>(~bits) : static_cast<uint16_t>(bits | 0x8000u);
return static_cast<uint8_t>(key >> 8);
}
__device__ __forceinline__ uint32_t exact_key(float x) {
const uint32_t bits = __float_as_uint(x);
return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u);
}
// Exact radix top-k over finite scores. A threshold bin that fits the stash
// keeps the original two-stash refinement; a larger one is refined by rescanning
// the input. Unfilled output slots stay -1.
template <int BlockSize, int OutputCapacity, int StashEntries>
__device__ __forceinline__ void
select(const float* __restrict__ input, int32_t* __restrict__ output, int row_start, int length, int topk) {
static_assert(BlockSize >= kRadix + 1, "the block must initialize the complete radix histogram");
static_assert(OutputCapacity <= StashEntries, "top-k output must fit in one shared-memory stash");
alignas(128) __shared__ int histogram_buf[2][kRadix + 1];
alignas(128) __shared__ int counter;
alignas(128) __shared__ int threshold_bin_id;
alignas(128) __shared__ int num_input[2];
extern __shared__ int dynamic_smem[];
auto* input_idx = reinterpret_cast<int (*)[StashEntries]>(dynamic_smem);
auto& histogram = histogram_buf[0];
const int tx = threadIdx.x;
for (int i = tx; i < topk; i += BlockSize)
output[i] = -1;
if (tx < kRadix + 1) histogram[tx] = 0;
if (tx == 0) {
counter = 0;
threshold_bin_id = -1;
num_input[0] = 0;
num_input[1] = 0;
}
__syncthreads();
for (int idx = tx; idx < length; idx += BlockSize) {
atomicAdd(&histogram[coarse_key(input[row_start + idx])], 1);
}
__syncthreads();
const auto run_cumsum = [&] {
#pragma unroll 8
for (int i = 0; i < 8; ++i) {
if (tx < kRadix) {
const int offset = 1 << i;
const int src = i & 1;
int value = histogram_buf[src][tx];
if (tx < kRadix - offset) value += histogram_buf[src][tx + offset];
histogram_buf[src ^ 1][tx] = value;
}
__syncthreads();
}
};
run_cumsum();
if (tx < kRadix && histogram[tx] > topk && histogram[tx + 1] <= topk) {
threshold_bin_id = tx;
}
__syncthreads();
const int coarse_threshold = threshold_bin_id;
if (coarse_threshold < 0) return;
const int coarse_above = histogram[coarse_threshold + 1];
int remaining = topk - coarse_above;
if (remaining == 0) {
for (int idx = tx; idx < length; idx += BlockSize) {
if (coarse_key(input[row_start + idx]) > coarse_threshold) {
const int pos = atomicAdd(&counter, 1);
if (pos < topk) output[pos] = idx;
}
}
__syncthreads();
return;
}
const int coarse_population = histogram[coarse_threshold] - histogram[coarse_threshold + 1];
__syncthreads();
if (coarse_population >= StashEntries) {
// Find the exact cutoff one key byte per pass. Pass L counts byte L and
// emits what byte L-1 placed strictly above the cutoff; pass 0 emits the
// values above the coarse bin and pass 4 the last strict values and ties.
uint32_t prefix = 0;
int last_threshold = -1;
int strict_count = 0;
#pragma unroll 1
for (int level = 0; level <= 4; ++level) {
const uint32_t mask = level == 0 ? 0u : (~0u << (32 - level * 8));
const uint32_t last_mask = level <= 1 ? 0u : (~0u << (40 - level * 8));
const int last_shift = 32 - level * 8;
if (level < 4) {
if (tx < kRadix + 1) histogram[tx] = 0;
if (tx == 0) threshold_bin_id = -1;
}
__syncthreads();
for (int idx = tx; idx < length; idx += BlockSize) {
const float value = input[row_start + idx];
const int bin = coarse_key(value);
if (bin != coarse_threshold) {
if (level == 0 && bin > coarse_threshold) {
const int pos = atomicAdd(&counter, 1);
if (pos < topk) output[pos] = idx;
}
continue;
}
const uint32_t key = exact_key(value);
if (level > 0 && (key & last_mask) == (prefix & last_mask) &&
static_cast<int>((key >> last_shift) & 0xFFu) > last_threshold) {
const int pos = atomicAdd(&num_input[0], 1);
if (pos < strict_count) output[coarse_above + pos] = idx;
} else if ((key & mask) == prefix) {
if (level < 4) {
atomicAdd(&histogram[(key >> (24 - level * 8)) & 0xFFu], 1);
} else {
const int pos = atomicAdd(&num_input[1], 1);
if (pos < remaining) output[coarse_above + strict_count + pos] = idx;
}
}
}
__syncthreads();
if (level == 4) return;
run_cumsum();
if (tx < kRadix && histogram[tx] >= remaining && histogram[tx + 1] < remaining) {
threshold_bin_id = tx;
}
__syncthreads();
const int threshold = threshold_bin_id;
if (threshold < 0) return;
const int above = histogram[threshold + 1];
strict_count += above;
remaining -= above;
prefix |= static_cast<uint32_t>(threshold) << (24 - level * 8);
last_threshold = threshold;
// All threads must read the histogram before the next pass clears it.
__syncthreads();
}
return;
}
// The threshold bin fits the stash: original two-stash refinement.
if (tx < kRadix + 1) histogram[tx] = 0;
__syncthreads();
for (int idx = tx; idx < length; idx += BlockSize) {
const float value = input[row_start + idx];
const int bin = coarse_key(value);
if (bin > coarse_threshold) {
const int pos = atomicAdd(&counter, 1);
if (pos < topk) output[pos] = idx;
} else if (bin == coarse_threshold) {
const int pos = atomicAdd(&num_input[0], 1);
if (pos < StashEntries) {
input_idx[0][pos] = idx;
atomicAdd(&histogram[(exact_key(value) >> 24) & 0xFFu], 1);
}
}
}
__syncthreads();
#pragma unroll 4
for (int round = 0; round < 4; ++round) {
const int current = round & 1;
const int next = current ^ 1;
const int raw_num_input = num_input[current];
const int current_count = raw_num_input < StashEntries ? raw_num_input : StashEntries;
if (tx == 0) {
threshold_bin_id = -1;
num_input[next] = 0;
}
run_cumsum();
if (tx < kRadix && histogram[tx] > remaining && histogram[tx + 1] <= remaining) {
threshold_bin_id = tx;
}
__syncthreads();
const int threshold = threshold_bin_id;
if (threshold < 0) return;
const int above = histogram[threshold + 1];
const int shift = 24 - round * 8;
__syncthreads();
if (round < 3 && tx < kRadix + 1) histogram[tx] = 0;
__syncthreads();
for (int i = tx; i < current_count; i += BlockSize) {
const int idx = input_idx[current][i];
const float value = input[row_start + idx];
const int bin = static_cast<int>((exact_key(value) >> shift) & 0xFFu);
if (bin > threshold) {
const int pos = atomicAdd(&counter, 1);
if (pos < topk) output[pos] = idx;
} else if (bin == threshold && round < 3) {
const int pos = atomicAdd(&num_input[next], 1);
if (pos < StashEntries) {
input_idx[next][pos] = idx;
atomicAdd(&histogram[(exact_key(value) >> (shift - 8)) & 0xFFu], 1);
}
}
}
__syncthreads();
remaining -= above;
if (remaining == 0) return;
if (round == 3) {
for (int i = tx; i < current_count; i += BlockSize) {
const int idx = input_idx[current][i];
if (static_cast<int>(exact_key(input[row_start + idx]) & 0xFFu) == threshold) {
const int pos = atomicAdd(&counter, 1);
if (pos < topk) output[pos] = idx;
}
}
__syncthreads();
return;
}
}
}
} // namespace sglang::device::legacy_radix_topk
+141 -10
View File
@@ -1,5 +1,5 @@
import sys
from typing import Any, Optional
from typing import Optional
import pytest
import torch
@@ -67,6 +67,13 @@ def _ref_torch_transform_ragged_impl(
MAX_SEQ_LEN = 131072
def _tight_cluster(batch: int, width: int) -> torch.Tensor:
"""Finite scores that all land in one fp16 coarse radix bin."""
base = torch.arange(width, dtype=torch.float32, device="cuda")
values = 1.0 + base * (1e-3 / width)
return values.expand(batch, -1).contiguous()
def assert_equal(
score: torch.Tensor,
indices_ref: torch.Tensor,
@@ -75,29 +82,39 @@ def assert_equal(
k: int,
seq_len: int,
topk_indices_offset: Optional[torch.Tensor] = None,
max_permit_error: int = 0,
row_starts: Optional[torch.Tensor] = None,
):
indices_our_cpu = indices_our.cpu().tolist()
indices_ref_cpu = indices_ref.cpu().tolist()
wrong_values = 0
for i in range(bs):
offset = topk_indices_offset[i].item() if topk_indices_offset is not None else 0
row_start = row_starts[i].item() if row_starts is not None else 0
indices_ref_set_i = set(indices_ref_cpu[i])
indices_our_set_i = set(indices_our_cpu[i])
assert len(indices_our_cpu[i]) == k
assert len(indices_our_set_i) == k, "top-k output indices must be distinct"
assert all(0 <= idx - offset < seq_len for idx in indices_our_cpu[i]), (
"top-k output index is outside the selected window"
)
more = indices_our_set_i - indices_ref_set_i
less = indices_ref_set_i - indices_our_set_i
offset = topk_indices_offset[i].item() if topk_indices_offset is not None else 0
if len(more) > 0 or len(less) > 0:
# check whether more values are the same with less values
# if so, either one is acceptable, since their values are the same
more_values = sorted(score[i, idx - offset].item() for idx in more)
less_values = sorted(score[i, idx - offset].item() for idx in less)
more_values = sorted(
score[i, row_start + idx - offset].item() for idx in more
)
less_values = sorted(
score[i, row_start + idx - offset].item() for idx in less
)
if more_values != less_values:
wrong_values += len(more)
print(
f"{bs=}, {k=}, {seq_len=}, {i=}, {more=}, {less=} failed, with {more_values=}, {less_values=}"
)
assert wrong_values <= max_permit_error, f"{wrong_values=}, {max_permit_error=}"
assert wrong_values == 0, f"{wrong_values=}"
@pytest.mark.parametrize("bs", [1, 132, 256, 4096])
@@ -125,8 +142,7 @@ def test_topk_kernel(bs: int, k: int, seq_len: int, has_row_starts: bool) -> Non
indices_ref = torch.sort(indices_ref, dim=-1).values
indices_our = torch.sort(indices_our, dim=-1).values
# Tests can pass with max_permit_error=3, set to 5 for safety
assert_equal(score, indices_ref, indices_our, bs, k, seq_len, max_permit_error=5)
assert_equal(score, indices_ref, indices_our, bs, k, seq_len, row_starts=row_starts)
@pytest.mark.parametrize("bs", [1, 132, 256, 4096])
@@ -189,7 +205,7 @@ def test_topk_transform_kernel(bs: int, k: int, seq_len: int, mode: str) -> None
bs,
k,
seq_len,
max_permit_error=5,
row_starts=row_starts,
)
@@ -245,10 +261,71 @@ def test_topk_transform_ragged_kernel(
k,
seq_len,
topk_indices_offset,
max_permit_error=5,
row_starts,
)
@torch.inference_mode()
def test_legacy_topk_oversized_coarse_bin() -> None:
"""The shared legacy selector must not clip an oversized threshold bin."""
torch.manual_seed(20260903)
batch, seq_len, topk, row_start = 2, 7001, 2048, 3
scores = torch.full(
(batch, seq_len + row_start), -1000.0, dtype=torch.float32, device="cuda"
)
scores[:, row_start:] = _tight_cluster(batch, seq_len)
lengths = torch.full((batch,), seq_len, dtype=torch.int32, device="cuda")
row_starts = torch.full((batch,), row_start, dtype=torch.int32, device="cuda")
reference = _ref_torch_impl(scores, seq_len, topk, row_starts=row_starts)
direct = fast_topk_v2(scores, lengths, topk, row_starts=row_starts)
assert_equal(scores[:, row_start:], reference, direct, batch, topk, seq_len)
offsets = torch.tensor([17, 4099], dtype=torch.int32, device="cuda")
ragged = fast_topk_transform_ragged_fused(
score=scores,
lengths=lengths,
topk_indices_offset=offsets,
topk=topk,
row_starts=row_starts,
)
assert_equal(
scores[:, row_start:],
reference + offsets[:, None],
ragged,
batch,
topk,
seq_len,
offsets,
)
page_table = torch.arange(seq_len, dtype=torch.int32, device="cuda").expand(
batch, -1
)
cu_seqlens_q = torch.arange(batch + 1, dtype=torch.int32, device="cuda")
paged = fast_topk_transform_fused(
score=scores,
lengths=lengths,
page_table_size_1=page_table,
cu_seqlens_q=cu_seqlens_q,
topk=topk,
row_starts=row_starts,
)
assert_equal(scores[:, row_start:], reference, paged, batch, topk, seq_len)
# With one query per row and no row starts, this interface selects the
# separate decode kernel rather than the prefill kernel above.
decode_scores = scores[:, row_start:].contiguous()
decode = fast_topk_transform_fused(
score=decode_scores,
lengths=lengths,
page_table_size_1=page_table,
cu_seqlens_q=cu_seqlens_q,
topk=topk,
)
assert_equal(decode_scores, reference, decode, batch, topk, seq_len)
@pytest.mark.skipif(
torch.version.hip is None,
reason="deepseek_v4_topk_transform_512 is only built on ROCm",
@@ -499,5 +576,59 @@ def test_topk_variants_are_exact_on_variable_windows(
assert_exact_rows(score, mapped_raw, lengths, k)
@pytest.mark.skipif(
torch.version.hip is None,
reason="deepseek_v4_topk_transform_512 is only built on ROCm",
)
@torch.inference_mode()
def test_deepseek_v4_topk_oversized_coarse_bin_and_idle_row() -> None:
"""Oversized bins select exactly; a negative DP-idle length emits padding."""
from sgl_kernel import deepseek_v4_topk_transform_512
torch.manual_seed(20260903)
active_batch, batch, c4_len, topk, page_size = 2, 3, 7001, 1024, 64
scores = _tight_cluster(batch, c4_len)
seq_lens = torch.tensor(
[c4_len] * active_batch + [-1], dtype=torch.int32, device="cuda"
)
num_pages = (c4_len + page_size - 1) // page_size
page_table = (
torch.arange(num_pages, dtype=torch.int32, device="cuda")
.expand(batch, -1)
.contiguous()
)
page_indices = torch.full((batch, topk), -2, dtype=torch.int32, device="cuda")
raw_indices = torch.full_like(page_indices, -2)
deepseek_v4_topk_transform_512(
scores,
seq_lens,
page_table,
page_indices,
page_size,
raw_indices,
)
reference = torch.topk(scores[:active_batch], topk, dim=-1, sorted=False).indices
assert_equal(
scores[:active_batch],
reference,
raw_indices[:active_batch],
active_batch,
topk,
c4_len,
)
assert_equal(
scores[:active_batch],
reference,
page_indices[:active_batch],
active_batch,
topk,
c4_len,
)
assert bool((raw_indices[-1] == -1).all())
assert bool((page_indices[-1] == -1).all())
if __name__ == "__main__":
sys.exit(pytest.main([__file__]))
@@ -4,6 +4,8 @@
#include <sgl_kernel/runtime.cuh>
#include <sgl_kernel/utils.cuh>
#include <sgl_kernel/dsa/legacy_radix_topk.cuh>
#include <dlpack/dlpack.h>
#include <tvm/ffi/container/tensor.h>
@@ -39,18 +41,6 @@ struct TopKParams {
uint32_t topk;
};
SGL_DEVICE uint8_t convert_to_uint8(float x) {
__half h = __float2half_rn(x);
uint16_t bits = __half_as_ushort(h);
uint16_t key = (bits & 0x8000) ? static_cast<uint16_t>(~bits) : static_cast<uint16_t>(bits | 0x8000);
return static_cast<uint8_t>(key >> 8);
}
SGL_DEVICE uint32_t convert_to_uint32(float x) {
uint32_t bits = __float_as_uint(x);
return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u);
}
SGL_DEVICE int32_t page_to_indices(const int32_t* __restrict__ page_table, uint32_t i, uint32_t page_bits) {
const uint32_t mask = (1u << page_bits) - 1u;
return (page_table[i >> page_bits] << page_bits) | (i & mask);
@@ -81,160 +71,9 @@ SGL_DEVICE void naive_transform(
[[maybe_unused]]
SGL_DEVICE void
radix_topk(const float* __restrict__ input, int32_t* __restrict__ output, const uint32_t length, const uint32_t topk) {
constexpr uint32_t RADIX = 256;
constexpr uint32_t BLOCK_SIZE = kTopKBlockSize;
constexpr uint32_t SMEM_INPUT_SIZE = kSMEM / (2 * sizeof(int32_t));
alignas(128) __shared__ uint32_t _s_histogram_buf[2][RADIX + 32];
alignas(128) __shared__ uint32_t s_counter;
alignas(128) __shared__ uint32_t s_threshold_bin_id;
alignas(128) __shared__ uint32_t s_num_input[2];
alignas(128) __shared__ int32_t s_last_remain;
extern __shared__ uint32_t s_input_idx[][kSMEM / (2 * sizeof(int32_t))];
const uint32_t tx = threadIdx.x;
uint32_t remain_topk = topk;
auto& s_histogram = _s_histogram_buf[0];
const auto run_cumsum = [&] {
#pragma unroll 8
for (int32_t i = 0; i < 8; ++i) {
static_assert(1 << 8 == RADIX);
if (tx < RADIX) {
const auto j = 1 << i;
const auto k = i & 1;
auto value = _s_histogram_buf[k][tx];
if (tx + j < RADIX) {
value += _s_histogram_buf[k][tx + j];
}
_s_histogram_buf[k ^ 1][tx] = value;
}
__syncthreads();
}
};
// stage 1: 8bit coarse histogram
if (tx < RADIX + 1) s_histogram[tx] = 0;
__syncthreads();
for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) {
const auto bin = convert_to_uint8(input[idx]);
::atomicAdd(&s_histogram[bin], 1);
}
__syncthreads();
run_cumsum();
if (tx < RADIX && s_histogram[tx] > remain_topk && s_histogram[tx + 1] <= remain_topk) {
s_threshold_bin_id = tx;
s_num_input[0] = 0;
s_counter = 0;
}
__syncthreads();
const auto threshold_bin = s_threshold_bin_id;
remain_topk -= s_histogram[threshold_bin + 1];
if (remain_topk == 0) {
for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) {
const uint32_t bin = convert_to_uint8(input[idx]);
if (bin > threshold_bin) {
const auto pos = ::atomicAdd(&s_counter, 1);
output[pos] = idx;
}
}
__syncthreads();
return;
} else {
__syncthreads();
if (tx < RADIX + 1) {
s_histogram[tx] = 0;
}
__syncthreads();
for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) {
const float raw_input = input[idx];
const uint32_t bin = convert_to_uint8(raw_input);
if (bin > threshold_bin) {
const auto pos = ::atomicAdd(&s_counter, 1);
output[pos] = idx;
} else if (bin == threshold_bin) {
const auto pos = ::atomicAdd(&s_num_input[0], 1);
if (pos < SMEM_INPUT_SIZE) {
[[likely]] s_input_idx[0][pos] = idx;
const auto bin = convert_to_uint32(raw_input);
const auto sub_bin = (bin >> 24) & 0xFF;
::atomicAdd(&s_histogram[sub_bin], 1);
}
}
}
__syncthreads();
}
// stage 2: refine with 8bit radix passes
#pragma unroll 4
for (int round = 0; round < 4; ++round) {
const auto r_idx = round % 2;
// clip here to prevent overflow
const auto raw_num_input = s_num_input[r_idx];
const auto num_input = raw_num_input < SMEM_INPUT_SIZE ? raw_num_input : SMEM_INPUT_SIZE;
run_cumsum();
if (tx < RADIX && s_histogram[tx] > remain_topk && s_histogram[tx + 1] <= remain_topk) {
s_threshold_bin_id = tx;
s_num_input[r_idx ^ 1] = 0;
s_last_remain = remain_topk - s_histogram[tx + 1];
}
__syncthreads();
const auto threshold_bin = s_threshold_bin_id;
remain_topk -= s_histogram[threshold_bin + 1];
if (remain_topk == 0) {
for (uint32_t i = tx; i < num_input; i += BLOCK_SIZE) {
const auto idx = s_input_idx[r_idx][i];
const auto offset = 24 - round * 8;
const auto bin = (convert_to_uint32(input[idx]) >> offset) & 0xFF;
if (bin > threshold_bin) {
const auto pos = ::atomicAdd(&s_counter, 1);
output[pos] = idx;
}
}
__syncthreads();
break;
} else {
__syncthreads();
if (tx < RADIX + 1) {
s_histogram[tx] = 0;
}
__syncthreads();
for (uint32_t i = tx; i < num_input; i += BLOCK_SIZE) {
const auto idx = s_input_idx[r_idx][i];
const auto raw_input = input[idx];
const auto offset = 24 - round * 8;
const auto bin = (convert_to_uint32(raw_input) >> offset) & 0xFF;
if (bin > threshold_bin) {
const auto pos = ::atomicAdd(&s_counter, 1);
output[pos] = idx;
} else if (bin == threshold_bin) {
if (round == 3) {
const auto pos = ::atomicAdd(&s_last_remain, -1);
if (pos > 0) {
output[topk - pos] = idx;
}
} else {
const auto pos = ::atomicAdd(&s_num_input[r_idx ^ 1], 1);
if (pos < SMEM_INPUT_SIZE) {
/// NOTE: (dark) fuse the histogram computation here
[[likely]] s_input_idx[r_idx ^ 1][pos] = idx;
const auto bin = convert_to_uint32(raw_input);
const auto sub_bin = (bin >> (offset - 8)) & 0xFF;
::atomicAdd(&s_histogram[sub_bin], 1);
}
}
}
}
__syncthreads();
}
}
::sglang::device::legacy_radix_topk::
select<kTopKBlockSize, kMaxTopK, static_cast<int>(kSMEM / (2 * sizeof(int32_t)))>(
input, output, 0, static_cast<int>(length), static_cast<int>(topk));
}
template <bool kUsePDL>
@@ -261,9 +100,10 @@ __global__ void topk_transform_kernel(const __grid_constant__ TopKParams params)
radix_topk(score_ptr, s_topk_indices, seq_len, topk);
const auto tx = threadIdx.x;
if (tx < topk) {
indices_ptr[tx] = page_to_indices(page_ptr, s_topk_indices[tx], page_bits);
const auto raw = s_topk_indices[tx];
indices_ptr[tx] = raw < 0 ? -1 : page_to_indices(page_ptr, raw, page_bits);
if (raw_indices_ptr != nullptr) {
raw_indices_ptr[tx] = s_topk_indices[tx];
raw_indices_ptr[tx] = raw;
}
}
}
@@ -0,0 +1,238 @@
#pragma once
#include <sgl_kernel/utils.cuh>
#include <cstdint>
namespace sglang::device::legacy_radix_topk {
inline constexpr int kRadix = 256;
// Coarse bin: high byte of the ordered FP16 value, as in the original selectors.
SGL_DEVICE uint8_t coarse_key(float x) {
const __half h = __float2half_rn(x);
const uint16_t bits = __half_as_ushort(h);
const uint16_t key = (bits & 0x8000u) ? static_cast<uint16_t>(~bits) : static_cast<uint16_t>(bits | 0x8000u);
return static_cast<uint8_t>(key >> 8);
}
SGL_DEVICE uint32_t exact_key(float x) {
const uint32_t bits = __float_as_uint(x);
return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u);
}
// Exact radix top-k over finite scores. A threshold bin that fits the stash
// keeps the original two-stash refinement; a larger one is refined by rescanning
// the input. Unfilled output slots stay -1.
template <int BlockSize, int OutputCapacity, int StashEntries>
SGL_DEVICE void
select(const float* __restrict__ input, int32_t* __restrict__ output, int row_start, int length, int topk) {
static_assert(BlockSize >= kRadix + 1, "the block must initialize the complete radix histogram");
static_assert(OutputCapacity <= StashEntries, "top-k output must fit in one shared-memory stash");
alignas(128) __shared__ int histogram_buf[2][kRadix + 1];
alignas(128) __shared__ int counter;
alignas(128) __shared__ int threshold_bin_id;
alignas(128) __shared__ int num_input[2];
extern __shared__ int dynamic_smem[];
auto* input_idx = reinterpret_cast<int (*)[StashEntries]>(dynamic_smem);
auto& histogram = histogram_buf[0];
const int tx = threadIdx.x;
for (int i = tx; i < topk; i += BlockSize)
output[i] = -1;
if (tx < kRadix + 1) histogram[tx] = 0;
if (tx == 0) {
counter = 0;
threshold_bin_id = -1;
num_input[0] = 0;
num_input[1] = 0;
}
__syncthreads();
for (int idx = tx; idx < length; idx += BlockSize) {
atomicAdd(&histogram[coarse_key(input[row_start + idx])], 1);
}
__syncthreads();
const auto run_cumsum = [&] {
#pragma unroll 8
for (int i = 0; i < 8; ++i) {
if (tx < kRadix) {
const int offset = 1 << i;
const int src = i & 1;
int value = histogram_buf[src][tx];
if (tx < kRadix - offset) value += histogram_buf[src][tx + offset];
histogram_buf[src ^ 1][tx] = value;
}
__syncthreads();
}
};
run_cumsum();
if (tx < kRadix && histogram[tx] > topk && histogram[tx + 1] <= topk) {
threshold_bin_id = tx;
}
__syncthreads();
const int coarse_threshold = threshold_bin_id;
if (coarse_threshold < 0) return;
const int coarse_above = histogram[coarse_threshold + 1];
int remaining = topk - coarse_above;
if (remaining == 0) {
for (int idx = tx; idx < length; idx += BlockSize) {
if (coarse_key(input[row_start + idx]) > coarse_threshold) {
const int pos = atomicAdd(&counter, 1);
if (pos < topk) output[pos] = idx;
}
}
__syncthreads();
return;
}
const int coarse_population = histogram[coarse_threshold] - histogram[coarse_threshold + 1];
__syncthreads();
if (coarse_population >= StashEntries) {
// Find the exact cutoff one key byte per pass. Pass L counts byte L and
// emits what byte L-1 placed strictly above the cutoff; pass 0 emits the
// values above the coarse bin and pass 4 the last strict values and ties.
uint32_t prefix = 0;
int last_threshold = -1;
int strict_count = 0;
#pragma unroll 1
for (int level = 0; level <= 4; ++level) {
const uint32_t mask = level == 0 ? 0u : (~0u << (32 - level * 8));
const uint32_t last_mask = level <= 1 ? 0u : (~0u << (40 - level * 8));
const int last_shift = 32 - level * 8;
if (level < 4) {
if (tx < kRadix + 1) histogram[tx] = 0;
if (tx == 0) threshold_bin_id = -1;
}
__syncthreads();
for (int idx = tx; idx < length; idx += BlockSize) {
const float value = input[row_start + idx];
const int bin = coarse_key(value);
if (bin != coarse_threshold) {
if (level == 0 && bin > coarse_threshold) {
const int pos = atomicAdd(&counter, 1);
if (pos < topk) output[pos] = idx;
}
continue;
}
const uint32_t key = exact_key(value);
if (level > 0 && (key & last_mask) == (prefix & last_mask) &&
static_cast<int>((key >> last_shift) & 0xFFu) > last_threshold) {
const int pos = atomicAdd(&num_input[0], 1);
if (pos < strict_count) output[coarse_above + pos] = idx;
} else if ((key & mask) == prefix) {
if (level < 4) {
atomicAdd(&histogram[(key >> (24 - level * 8)) & 0xFFu], 1);
} else {
const int pos = atomicAdd(&num_input[1], 1);
if (pos < remaining) output[coarse_above + strict_count + pos] = idx;
}
}
}
__syncthreads();
if (level == 4) return;
run_cumsum();
if (tx < kRadix && histogram[tx] >= remaining && histogram[tx + 1] < remaining) {
threshold_bin_id = tx;
}
__syncthreads();
const int threshold = threshold_bin_id;
if (threshold < 0) return;
const int above = histogram[threshold + 1];
strict_count += above;
remaining -= above;
prefix |= static_cast<uint32_t>(threshold) << (24 - level * 8);
last_threshold = threshold;
// All threads must read the histogram before the next pass clears it.
__syncthreads();
}
return;
}
// The threshold bin fits the stash: original two-stash refinement.
if (tx < kRadix + 1) histogram[tx] = 0;
__syncthreads();
for (int idx = tx; idx < length; idx += BlockSize) {
const float value = input[row_start + idx];
const int bin = coarse_key(value);
if (bin > coarse_threshold) {
const int pos = atomicAdd(&counter, 1);
if (pos < topk) output[pos] = idx;
} else if (bin == coarse_threshold) {
const int pos = atomicAdd(&num_input[0], 1);
if (pos < StashEntries) {
input_idx[0][pos] = idx;
atomicAdd(&histogram[(exact_key(value) >> 24) & 0xFFu], 1);
}
}
}
__syncthreads();
#pragma unroll 4
for (int round = 0; round < 4; ++round) {
const int current = round & 1;
const int next = current ^ 1;
const int raw_num_input = num_input[current];
const int current_count = raw_num_input < StashEntries ? raw_num_input : StashEntries;
if (tx == 0) {
threshold_bin_id = -1;
num_input[next] = 0;
}
run_cumsum();
if (tx < kRadix && histogram[tx] > remaining && histogram[tx + 1] <= remaining) {
threshold_bin_id = tx;
}
__syncthreads();
const int threshold = threshold_bin_id;
if (threshold < 0) return;
const int above = histogram[threshold + 1];
const int shift = 24 - round * 8;
__syncthreads();
if (round < 3 && tx < kRadix + 1) histogram[tx] = 0;
__syncthreads();
for (int i = tx; i < current_count; i += BlockSize) {
const int idx = input_idx[current][i];
const float value = input[row_start + idx];
const int bin = static_cast<int>((exact_key(value) >> shift) & 0xFFu);
if (bin > threshold) {
const int pos = atomicAdd(&counter, 1);
if (pos < topk) output[pos] = idx;
} else if (bin == threshold && round < 3) {
const int pos = atomicAdd(&num_input[next], 1);
if (pos < StashEntries) {
input_idx[next][pos] = idx;
atomicAdd(&histogram[(exact_key(value) >> (shift - 8)) & 0xFFu], 1);
}
}
}
__syncthreads();
remaining -= above;
if (remaining == 0) return;
if (round == 3) {
for (int i = tx; i < current_count; i += BlockSize) {
const int idx = input_idx[current][i];
if (static_cast<int>(exact_key(input[row_start + idx]) & 0xFFu) == threshold) {
const int pos = atomicAdd(&counter, 1);
if (pos < topk) output[pos] = idx;
}
}
__syncthreads();
return;
}
}
}
} // namespace sglang::device::legacy_radix_topk