Fix overflow in legacy and v1 DSA top-k selectors (#37625)
Co-authored-by: Benjamin Oldenburg <benjamin.oldenburg@ordis.co.th>
This commit is contained in:
co-authored by
Benjamin Oldenburg
parent
2f5407da28
commit
c5bf3aafbc
@@ -24,6 +24,7 @@ limitations under the License.
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <sgl_kernel/dsa/legacy_radix_topk.cuh>
|
||||
|
||||
namespace {
|
||||
|
||||
@@ -62,18 +63,6 @@ struct TopKParams {
|
||||
bool sort_output;
|
||||
};
|
||||
|
||||
__device__ __forceinline__ uint8_t convert_to_uint8(float x) {
|
||||
__half h = __float2half_rn(x);
|
||||
uint16_t bits = __half_as_ushort(h);
|
||||
uint16_t key = (bits & 0x8000) ? static_cast<uint16_t>(~bits) : static_cast<uint16_t>(bits | 0x8000);
|
||||
return static_cast<uint8_t>(key >> 8);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ uint32_t convert_to_uint32(float x) {
|
||||
uint32_t bits = __float_as_uint(x);
|
||||
return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ int32_t
|
||||
page_to_slot(const int32_t* __restrict__ page_table, uint32_t i, uint32_t page_bits) {
|
||||
const uint32_t mask = (1u << page_bits) - 1u;
|
||||
@@ -104,154 +93,10 @@ __device__ void naive_paged_transform(
|
||||
|
||||
__device__ void
|
||||
radix_topk(const float* __restrict__ input, int32_t* __restrict__ output, uint32_t length, uint32_t topk) {
|
||||
constexpr uint32_t RADIX = 256;
|
||||
constexpr uint32_t BLOCK_SIZE = kBlockSize;
|
||||
constexpr uint32_t SMEM_INPUT_SIZE = kSMEM / (2 * sizeof(int32_t));
|
||||
|
||||
alignas(128) __shared__ uint32_t _s_histogram_buf[2][RADIX + 32];
|
||||
alignas(128) __shared__ uint32_t s_counter;
|
||||
alignas(128) __shared__ uint32_t s_threshold_bin_id;
|
||||
alignas(128) __shared__ uint32_t s_num_input[2];
|
||||
alignas(128) __shared__ int32_t s_last_remain;
|
||||
|
||||
extern __shared__ uint32_t s_input_idx[][SMEM_INPUT_SIZE];
|
||||
|
||||
const uint32_t tx = threadIdx.x;
|
||||
uint32_t remain_topk = topk;
|
||||
auto& s_histogram = _s_histogram_buf[0];
|
||||
|
||||
const auto run_cumsum = [&] {
|
||||
#pragma unroll 8
|
||||
for (int32_t i = 0; i < 8; ++i) {
|
||||
static_assert(1 << 8 == RADIX);
|
||||
if (tx < RADIX) {
|
||||
const auto j = 1 << i;
|
||||
const auto k = i & 1;
|
||||
auto value = _s_histogram_buf[k][tx];
|
||||
if (tx + j < RADIX) {
|
||||
value += _s_histogram_buf[k][tx + j];
|
||||
}
|
||||
_s_histogram_buf[k ^ 1][tx] = value;
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
};
|
||||
|
||||
// stage 1: 8bit coarse histogram
|
||||
if (tx < RADIX + 1) s_histogram[tx] = 0;
|
||||
::sglang::device::legacy_radix_topk::
|
||||
select<static_cast<int>(kBlockSize), static_cast<int>(kMaxTopK), static_cast<int>(kSMEM / (2 * sizeof(int32_t)))>(
|
||||
input, output, 0, static_cast<int>(length), static_cast<int>(topk));
|
||||
__syncthreads();
|
||||
for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) {
|
||||
const auto bin = convert_to_uint8(input[idx]);
|
||||
::atomicAdd(&s_histogram[bin], 1);
|
||||
}
|
||||
__syncthreads();
|
||||
run_cumsum();
|
||||
if (tx < RADIX && s_histogram[tx] > remain_topk && s_histogram[tx + 1] <= remain_topk) {
|
||||
s_threshold_bin_id = tx;
|
||||
s_num_input[0] = 0;
|
||||
s_counter = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
{
|
||||
const auto threshold_bin = s_threshold_bin_id;
|
||||
remain_topk -= s_histogram[threshold_bin + 1];
|
||||
if (remain_topk == 0) {
|
||||
for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) {
|
||||
const uint32_t bin = convert_to_uint8(input[idx]);
|
||||
if (bin > threshold_bin) {
|
||||
const auto pos = ::atomicAdd(&s_counter, 1);
|
||||
output[pos] = static_cast<int32_t>(idx);
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
return;
|
||||
}
|
||||
__syncthreads();
|
||||
if (tx < RADIX + 1) s_histogram[tx] = 0;
|
||||
__syncthreads();
|
||||
|
||||
for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) {
|
||||
const float raw_input = input[idx];
|
||||
const uint32_t bin = convert_to_uint8(raw_input);
|
||||
if (bin > threshold_bin) {
|
||||
const auto pos = ::atomicAdd(&s_counter, 1);
|
||||
output[pos] = static_cast<int32_t>(idx);
|
||||
} else if (bin == threshold_bin) {
|
||||
const auto pos = ::atomicAdd(&s_num_input[0], 1);
|
||||
if (C10_LIKELY(pos < SMEM_INPUT_SIZE)) {
|
||||
s_input_idx[0][pos] = idx;
|
||||
const auto bin32 = convert_to_uint32(raw_input);
|
||||
const auto sub_bin = (bin32 >> 24) & 0xFF;
|
||||
::atomicAdd(&s_histogram[sub_bin], 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
// stage 2: refine with 8bit radix passes
|
||||
#pragma unroll 4
|
||||
for (int round = 0; round < 4; ++round) {
|
||||
const auto r_idx = round % 2;
|
||||
|
||||
const auto raw_num_input = s_num_input[r_idx];
|
||||
const auto num_input = raw_num_input < SMEM_INPUT_SIZE ? raw_num_input : SMEM_INPUT_SIZE;
|
||||
|
||||
run_cumsum();
|
||||
if (tx < RADIX && s_histogram[tx] > remain_topk && s_histogram[tx + 1] <= remain_topk) {
|
||||
s_threshold_bin_id = tx;
|
||||
s_num_input[r_idx ^ 1] = 0;
|
||||
s_last_remain = static_cast<int32_t>(remain_topk - s_histogram[tx + 1]);
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const auto threshold_bin = s_threshold_bin_id;
|
||||
remain_topk -= s_histogram[threshold_bin + 1];
|
||||
|
||||
if (remain_topk == 0) {
|
||||
for (uint32_t i = tx; i < num_input; i += BLOCK_SIZE) {
|
||||
const auto idx = s_input_idx[r_idx][i];
|
||||
const auto offset = 24 - round * 8;
|
||||
const auto bin = (convert_to_uint32(input[idx]) >> offset) & 0xFF;
|
||||
if (bin > threshold_bin) {
|
||||
const auto pos = ::atomicAdd(&s_counter, 1);
|
||||
output[pos] = static_cast<int32_t>(idx);
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
break;
|
||||
}
|
||||
__syncthreads();
|
||||
if (tx < RADIX + 1) s_histogram[tx] = 0;
|
||||
__syncthreads();
|
||||
for (uint32_t i = tx; i < num_input; i += BLOCK_SIZE) {
|
||||
const auto idx = s_input_idx[r_idx][i];
|
||||
const auto raw_input = input[idx];
|
||||
const auto offset = 24 - round * 8;
|
||||
const auto bin = (convert_to_uint32(raw_input) >> offset) & 0xFF;
|
||||
if (bin > threshold_bin) {
|
||||
const auto pos = ::atomicAdd(&s_counter, 1);
|
||||
output[pos] = static_cast<int32_t>(idx);
|
||||
} else if (bin == threshold_bin) {
|
||||
if (round == 3) {
|
||||
const auto pos = ::atomicAdd(&s_last_remain, -1);
|
||||
if (pos > 0) {
|
||||
output[topk - pos] = static_cast<int32_t>(idx);
|
||||
}
|
||||
} else {
|
||||
const auto pos = ::atomicAdd(&s_num_input[r_idx ^ 1], 1);
|
||||
if (C10_LIKELY(pos < SMEM_INPUT_SIZE)) {
|
||||
s_input_idx[r_idx ^ 1][pos] = idx;
|
||||
const auto bin32 = convert_to_uint32(raw_input);
|
||||
const auto sub_bin = (bin32 >> (offset - 8)) & 0xFF;
|
||||
::atomicAdd(&s_histogram[sub_bin], 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef USE_ROCM
|
||||
@@ -366,7 +211,8 @@ __device__ __forceinline__ uint32_t next_pow2_at_least_64(uint32_t x) {
|
||||
|
||||
__global__ __launch_bounds__(kBlockSize) void deepseek_v4_topk_transform_kernel(const TopKParams params) {
|
||||
const auto bid = blockIdx.x;
|
||||
const auto seq_len = params.seq_lens[bid];
|
||||
const auto raw_seq_len = params.seq_lens[bid];
|
||||
const auto seq_len = raw_seq_len < 0 ? 0 : raw_seq_len;
|
||||
const auto topk = params.topk;
|
||||
const auto score_ptr = params.scores + bid * params.score_stride;
|
||||
const auto page_ptr = params.page_table + bid * params.page_table_stride;
|
||||
@@ -433,7 +279,7 @@ __global__ __launch_bounds__(kBlockSize) void deepseek_v4_topk_transform_kernel(
|
||||
|
||||
for (uint32_t i = threadIdx.x; i < topk; i += kBlockSize) {
|
||||
const auto raw = s_topk_indices[i];
|
||||
indices_ptr[i] = page_to_slot(page_ptr, static_cast<uint32_t>(raw), params.page_bits);
|
||||
indices_ptr[i] = raw < 0 ? -1 : page_to_slot(page_ptr, static_cast<uint32_t>(raw), params.page_bits);
|
||||
if (raw_indices_ptr != nullptr) {
|
||||
raw_indices_ptr[i] = raw;
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <sgl_kernel/dsa/legacy_radix_topk.cuh>
|
||||
|
||||
namespace {
|
||||
|
||||
@@ -75,180 +76,10 @@ __device__ void naive_topk_transform_ragged(
|
||||
}
|
||||
}
|
||||
|
||||
__device__ __forceinline__ auto convert_to_uint8(float x) -> uint8_t {
|
||||
__half h = __float2half_rn(x);
|
||||
uint16_t bits = __half_as_ushort(h);
|
||||
uint16_t key = (bits & 0x8000) ? static_cast<uint16_t>(~bits) : static_cast<uint16_t>(bits | 0x8000);
|
||||
return static_cast<uint8_t>(key >> 8);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ auto convert_to_uint32(float x) -> uint32_t {
|
||||
uint32_t bits = __float_as_uint(x);
|
||||
return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u);
|
||||
}
|
||||
|
||||
__device__ void fast_topk_cuda_tl(const float* __restrict__ input, int* __restrict__ index, int row_start, int length) {
|
||||
// An optimized topk kernel copied from tilelang kernel
|
||||
// We assume length > TopK here, or it will crash
|
||||
int topk = TopK;
|
||||
constexpr auto BLOCK_SIZE = 1024;
|
||||
constexpr auto RADIX = 256;
|
||||
constexpr auto SMEM_INPUT_SIZE = kSmem / (2 * sizeof(int));
|
||||
|
||||
alignas(128) __shared__ int s_histogram_buf[2][RADIX + 128];
|
||||
alignas(128) __shared__ int s_counter;
|
||||
alignas(128) __shared__ int s_threshold_bin_id;
|
||||
alignas(128) __shared__ int s_num_input[2];
|
||||
|
||||
auto& s_histogram = s_histogram_buf[0];
|
||||
// allocate for two rounds
|
||||
extern __shared__ int s_input_idx[][SMEM_INPUT_SIZE];
|
||||
|
||||
const int tx = threadIdx.x;
|
||||
|
||||
// stage 1: 8bit coarse histogram
|
||||
if (tx < RADIX + 1) s_histogram[tx] = 0;
|
||||
::sglang::device::legacy_radix_topk::select<kThreadsPerBlock, TopK, static_cast<int>(kSmem / (2 * sizeof(int)))>(
|
||||
input, index, row_start, length, TopK);
|
||||
__syncthreads();
|
||||
|
||||
for (int idx = tx; idx < length; idx += BLOCK_SIZE) {
|
||||
const auto bin = convert_to_uint8(input[idx + row_start]);
|
||||
::atomicAdd(&s_histogram[bin], 1);
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const auto run_cumsum = [&] {
|
||||
#pragma unroll 8
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
static_assert(1 << 8 == RADIX);
|
||||
if (C10_LIKELY(tx < RADIX)) {
|
||||
const auto j = 1 << i;
|
||||
const auto k = i & 1;
|
||||
auto value = s_histogram_buf[k][tx];
|
||||
if (tx < RADIX - j) {
|
||||
value += s_histogram_buf[k][tx + j];
|
||||
}
|
||||
s_histogram_buf[k ^ 1][tx] = value;
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
};
|
||||
|
||||
run_cumsum();
|
||||
if (tx < RADIX && s_histogram[tx] > topk && s_histogram[tx + 1] <= topk) {
|
||||
s_threshold_bin_id = tx;
|
||||
s_num_input[0] = 0;
|
||||
s_counter = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const auto threshold_bin = s_threshold_bin_id;
|
||||
topk -= s_histogram[threshold_bin + 1];
|
||||
|
||||
if (topk == 0) {
|
||||
for (int idx = tx; idx < length; idx += BLOCK_SIZE) {
|
||||
const auto bin = static_cast<int>(convert_to_uint8(input[idx + row_start]));
|
||||
if (bin > threshold_bin) {
|
||||
const auto pos = ::atomicAdd(&s_counter, 1);
|
||||
index[pos] = idx;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
return;
|
||||
} else {
|
||||
__syncthreads();
|
||||
if (tx < RADIX + 1) {
|
||||
s_histogram[tx] = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
for (int idx = tx; idx < length; idx += BLOCK_SIZE) {
|
||||
const auto raw_input = input[idx + row_start];
|
||||
const auto bin = static_cast<int>(convert_to_uint8(raw_input));
|
||||
if (bin > threshold_bin) {
|
||||
const auto pos = ::atomicAdd(&s_counter, 1);
|
||||
index[pos] = idx;
|
||||
} else if (bin == threshold_bin) {
|
||||
const auto pos = ::atomicAdd(&s_num_input[0], 1);
|
||||
/// NOTE: (dark) fuse the histogram computation here
|
||||
if (C10_LIKELY(pos < SMEM_INPUT_SIZE)) {
|
||||
s_input_idx[0][pos] = idx;
|
||||
const auto bin = convert_to_uint32(raw_input);
|
||||
const auto sub_bin = (bin >> 24) & 0xFF;
|
||||
::atomicAdd(&s_histogram[sub_bin], 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
// stage 2: refine with 8bit radix passes
|
||||
#pragma unroll 4
|
||||
for (int round = 0; round < 4; ++round) {
|
||||
__shared__ int s_last_remain;
|
||||
const auto r_idx = round % 2;
|
||||
|
||||
// clip here to prevent overflow
|
||||
const auto _raw_num_input = s_num_input[r_idx];
|
||||
const auto num_input = (_raw_num_input < int(SMEM_INPUT_SIZE)) ? _raw_num_input : int(SMEM_INPUT_SIZE);
|
||||
|
||||
run_cumsum();
|
||||
if (tx < RADIX && s_histogram[tx] > topk && s_histogram[tx + 1] <= topk) {
|
||||
s_threshold_bin_id = tx;
|
||||
s_num_input[r_idx ^ 1] = 0;
|
||||
s_last_remain = topk - s_histogram[tx + 1];
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const auto threshold_bin = s_threshold_bin_id;
|
||||
topk -= s_histogram[threshold_bin + 1];
|
||||
|
||||
if (topk == 0) {
|
||||
for (int i = tx; i < num_input; i += BLOCK_SIZE) {
|
||||
const auto idx = s_input_idx[r_idx][i];
|
||||
const auto offset = 24 - round * 8;
|
||||
const auto bin = (convert_to_uint32(input[idx + row_start]) >> offset) & 0xFF;
|
||||
if (bin > threshold_bin) {
|
||||
const auto pos = ::atomicAdd(&s_counter, 1);
|
||||
index[pos] = idx;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
break;
|
||||
} else {
|
||||
__syncthreads();
|
||||
if (tx < RADIX + 1) {
|
||||
s_histogram[tx] = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
for (int i = tx; i < num_input; i += BLOCK_SIZE) {
|
||||
const auto idx = s_input_idx[r_idx][i];
|
||||
const auto raw_input = input[idx + row_start];
|
||||
const auto offset = 24 - round * 8;
|
||||
const auto bin = (convert_to_uint32(raw_input) >> offset) & 0xFF;
|
||||
if (bin > threshold_bin) {
|
||||
const auto pos = ::atomicAdd(&s_counter, 1);
|
||||
index[pos] = idx;
|
||||
} else if (bin == threshold_bin) {
|
||||
if (round == 3) {
|
||||
const auto pos = ::atomicAdd(&s_last_remain, -1);
|
||||
if (pos > 0) {
|
||||
index[TopK - pos] = idx;
|
||||
}
|
||||
} else {
|
||||
const auto pos = ::atomicAdd(&s_num_input[r_idx ^ 1], 1);
|
||||
if (C10_LIKELY(pos < SMEM_INPUT_SIZE)) {
|
||||
/// NOTE: (dark) fuse the histogram computation here
|
||||
s_input_idx[r_idx ^ 1][pos] = idx;
|
||||
const auto bin = convert_to_uint32(raw_input);
|
||||
const auto sub_bin = (bin >> (offset - 8)) & 0xFF;
|
||||
::atomicAdd(&s_histogram[sub_bin], 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
__global__ __launch_bounds__(kThreadsPerBlock) // topk
|
||||
@@ -290,10 +121,10 @@ __global__ __launch_bounds__(kThreadsPerBlock) // decode
|
||||
static_assert(TopK / kThreadsPerBlock == 2);
|
||||
const auto idx_0 = tid;
|
||||
const auto pos_0 = s_indices[idx_0];
|
||||
dst_page_entry[idx_0] = src_page_entry[pos_0];
|
||||
dst_page_entry[idx_0] = pos_0 < 0 ? -1 : src_page_entry[pos_0];
|
||||
const auto idx_1 = tid + kThreadsPerBlock;
|
||||
const auto pos_1 = s_indices[idx_1];
|
||||
dst_page_entry[idx_1] = src_page_entry[pos_1];
|
||||
dst_page_entry[idx_1] = pos_1 < 0 ? -1 : src_page_entry[pos_1];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -342,10 +173,10 @@ __global__ __launch_bounds__(kThreadsPerBlock) // prefill
|
||||
static_assert(TopK / kThreadsPerBlock == 2);
|
||||
const auto idx_0 = tid;
|
||||
const auto pos_0 = s_indices[idx_0];
|
||||
dst_page_entry[idx_0] = src_page_entry[pos_0];
|
||||
dst_page_entry[idx_0] = pos_0 < 0 ? -1 : src_page_entry[pos_0];
|
||||
const auto idx_1 = tid + kThreadsPerBlock;
|
||||
const auto pos_1 = s_indices[idx_1];
|
||||
dst_page_entry[idx_1] = src_page_entry[pos_1];
|
||||
dst_page_entry[idx_1] = pos_1 < 0 ? -1 : src_page_entry[pos_1];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -373,10 +204,10 @@ __global__ __launch_bounds__(kThreadsPerBlock) // prefill, ragged kv
|
||||
static_assert(TopK / kThreadsPerBlock == 2);
|
||||
const auto idx_0 = tid;
|
||||
const auto pos_0 = s_indices[idx_0];
|
||||
dst_indices_entry[idx_0] = pos_0 + offset;
|
||||
dst_indices_entry[idx_0] = pos_0 < 0 ? -1 : pos_0 + offset;
|
||||
const auto idx_1 = tid + kThreadsPerBlock;
|
||||
const auto pos_1 = s_indices[idx_1];
|
||||
dst_indices_entry[idx_1] = pos_1 + offset;
|
||||
dst_indices_entry[idx_1] = pos_1 < 0 ? -1 : pos_1 + offset;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,239 @@
|
||||
#pragma once
|
||||
|
||||
#include <cuda_fp16.h>
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
|
||||
namespace sglang::device::legacy_radix_topk {
|
||||
|
||||
inline constexpr int kRadix = 256;
|
||||
|
||||
// Coarse bin: high byte of the ordered FP16 value, as in the original selectors.
|
||||
__device__ __forceinline__ uint8_t coarse_key(float x) {
|
||||
const __half h = __float2half_rn(x);
|
||||
const uint16_t bits = __half_as_ushort(h);
|
||||
const uint16_t key = (bits & 0x8000u) ? static_cast<uint16_t>(~bits) : static_cast<uint16_t>(bits | 0x8000u);
|
||||
return static_cast<uint8_t>(key >> 8);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ uint32_t exact_key(float x) {
|
||||
const uint32_t bits = __float_as_uint(x);
|
||||
return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u);
|
||||
}
|
||||
|
||||
// Exact radix top-k over finite scores. A threshold bin that fits the stash
|
||||
// keeps the original two-stash refinement; a larger one is refined by rescanning
|
||||
// the input. Unfilled output slots stay -1.
|
||||
template <int BlockSize, int OutputCapacity, int StashEntries>
|
||||
__device__ __forceinline__ void
|
||||
select(const float* __restrict__ input, int32_t* __restrict__ output, int row_start, int length, int topk) {
|
||||
static_assert(BlockSize >= kRadix + 1, "the block must initialize the complete radix histogram");
|
||||
static_assert(OutputCapacity <= StashEntries, "top-k output must fit in one shared-memory stash");
|
||||
|
||||
alignas(128) __shared__ int histogram_buf[2][kRadix + 1];
|
||||
alignas(128) __shared__ int counter;
|
||||
alignas(128) __shared__ int threshold_bin_id;
|
||||
alignas(128) __shared__ int num_input[2];
|
||||
|
||||
extern __shared__ int dynamic_smem[];
|
||||
auto* input_idx = reinterpret_cast<int (*)[StashEntries]>(dynamic_smem);
|
||||
auto& histogram = histogram_buf[0];
|
||||
const int tx = threadIdx.x;
|
||||
|
||||
for (int i = tx; i < topk; i += BlockSize)
|
||||
output[i] = -1;
|
||||
if (tx < kRadix + 1) histogram[tx] = 0;
|
||||
if (tx == 0) {
|
||||
counter = 0;
|
||||
threshold_bin_id = -1;
|
||||
num_input[0] = 0;
|
||||
num_input[1] = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
for (int idx = tx; idx < length; idx += BlockSize) {
|
||||
atomicAdd(&histogram[coarse_key(input[row_start + idx])], 1);
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const auto run_cumsum = [&] {
|
||||
#pragma unroll 8
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
if (tx < kRadix) {
|
||||
const int offset = 1 << i;
|
||||
const int src = i & 1;
|
||||
int value = histogram_buf[src][tx];
|
||||
if (tx < kRadix - offset) value += histogram_buf[src][tx + offset];
|
||||
histogram_buf[src ^ 1][tx] = value;
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
};
|
||||
|
||||
run_cumsum();
|
||||
if (tx < kRadix && histogram[tx] > topk && histogram[tx + 1] <= topk) {
|
||||
threshold_bin_id = tx;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const int coarse_threshold = threshold_bin_id;
|
||||
if (coarse_threshold < 0) return;
|
||||
const int coarse_above = histogram[coarse_threshold + 1];
|
||||
int remaining = topk - coarse_above;
|
||||
|
||||
if (remaining == 0) {
|
||||
for (int idx = tx; idx < length; idx += BlockSize) {
|
||||
if (coarse_key(input[row_start + idx]) > coarse_threshold) {
|
||||
const int pos = atomicAdd(&counter, 1);
|
||||
if (pos < topk) output[pos] = idx;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
return;
|
||||
}
|
||||
|
||||
const int coarse_population = histogram[coarse_threshold] - histogram[coarse_threshold + 1];
|
||||
__syncthreads();
|
||||
if (coarse_population >= StashEntries) {
|
||||
// Find the exact cutoff one key byte per pass. Pass L counts byte L and
|
||||
// emits what byte L-1 placed strictly above the cutoff; pass 0 emits the
|
||||
// values above the coarse bin and pass 4 the last strict values and ties.
|
||||
uint32_t prefix = 0;
|
||||
int last_threshold = -1;
|
||||
int strict_count = 0;
|
||||
#pragma unroll 1
|
||||
for (int level = 0; level <= 4; ++level) {
|
||||
const uint32_t mask = level == 0 ? 0u : (~0u << (32 - level * 8));
|
||||
const uint32_t last_mask = level <= 1 ? 0u : (~0u << (40 - level * 8));
|
||||
const int last_shift = 32 - level * 8;
|
||||
if (level < 4) {
|
||||
if (tx < kRadix + 1) histogram[tx] = 0;
|
||||
if (tx == 0) threshold_bin_id = -1;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
for (int idx = tx; idx < length; idx += BlockSize) {
|
||||
const float value = input[row_start + idx];
|
||||
const int bin = coarse_key(value);
|
||||
if (bin != coarse_threshold) {
|
||||
if (level == 0 && bin > coarse_threshold) {
|
||||
const int pos = atomicAdd(&counter, 1);
|
||||
if (pos < topk) output[pos] = idx;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const uint32_t key = exact_key(value);
|
||||
if (level > 0 && (key & last_mask) == (prefix & last_mask) &&
|
||||
static_cast<int>((key >> last_shift) & 0xFFu) > last_threshold) {
|
||||
const int pos = atomicAdd(&num_input[0], 1);
|
||||
if (pos < strict_count) output[coarse_above + pos] = idx;
|
||||
} else if ((key & mask) == prefix) {
|
||||
if (level < 4) {
|
||||
atomicAdd(&histogram[(key >> (24 - level * 8)) & 0xFFu], 1);
|
||||
} else {
|
||||
const int pos = atomicAdd(&num_input[1], 1);
|
||||
if (pos < remaining) output[coarse_above + strict_count + pos] = idx;
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
if (level == 4) return;
|
||||
|
||||
run_cumsum();
|
||||
if (tx < kRadix && histogram[tx] >= remaining && histogram[tx + 1] < remaining) {
|
||||
threshold_bin_id = tx;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const int threshold = threshold_bin_id;
|
||||
if (threshold < 0) return;
|
||||
const int above = histogram[threshold + 1];
|
||||
strict_count += above;
|
||||
remaining -= above;
|
||||
prefix |= static_cast<uint32_t>(threshold) << (24 - level * 8);
|
||||
last_threshold = threshold;
|
||||
// All threads must read the histogram before the next pass clears it.
|
||||
__syncthreads();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// The threshold bin fits the stash: original two-stash refinement.
|
||||
if (tx < kRadix + 1) histogram[tx] = 0;
|
||||
__syncthreads();
|
||||
for (int idx = tx; idx < length; idx += BlockSize) {
|
||||
const float value = input[row_start + idx];
|
||||
const int bin = coarse_key(value);
|
||||
if (bin > coarse_threshold) {
|
||||
const int pos = atomicAdd(&counter, 1);
|
||||
if (pos < topk) output[pos] = idx;
|
||||
} else if (bin == coarse_threshold) {
|
||||
const int pos = atomicAdd(&num_input[0], 1);
|
||||
if (pos < StashEntries) {
|
||||
input_idx[0][pos] = idx;
|
||||
atomicAdd(&histogram[(exact_key(value) >> 24) & 0xFFu], 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
#pragma unroll 4
|
||||
for (int round = 0; round < 4; ++round) {
|
||||
const int current = round & 1;
|
||||
const int next = current ^ 1;
|
||||
const int raw_num_input = num_input[current];
|
||||
const int current_count = raw_num_input < StashEntries ? raw_num_input : StashEntries;
|
||||
|
||||
if (tx == 0) {
|
||||
threshold_bin_id = -1;
|
||||
num_input[next] = 0;
|
||||
}
|
||||
run_cumsum();
|
||||
if (tx < kRadix && histogram[tx] > remaining && histogram[tx + 1] <= remaining) {
|
||||
threshold_bin_id = tx;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const int threshold = threshold_bin_id;
|
||||
if (threshold < 0) return;
|
||||
const int above = histogram[threshold + 1];
|
||||
const int shift = 24 - round * 8;
|
||||
|
||||
__syncthreads();
|
||||
if (round < 3 && tx < kRadix + 1) histogram[tx] = 0;
|
||||
__syncthreads();
|
||||
for (int i = tx; i < current_count; i += BlockSize) {
|
||||
const int idx = input_idx[current][i];
|
||||
const float value = input[row_start + idx];
|
||||
const int bin = static_cast<int>((exact_key(value) >> shift) & 0xFFu);
|
||||
if (bin > threshold) {
|
||||
const int pos = atomicAdd(&counter, 1);
|
||||
if (pos < topk) output[pos] = idx;
|
||||
} else if (bin == threshold && round < 3) {
|
||||
const int pos = atomicAdd(&num_input[next], 1);
|
||||
if (pos < StashEntries) {
|
||||
input_idx[next][pos] = idx;
|
||||
atomicAdd(&histogram[(exact_key(value) >> (shift - 8)) & 0xFFu], 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
remaining -= above;
|
||||
if (remaining == 0) return;
|
||||
if (round == 3) {
|
||||
for (int i = tx; i < current_count; i += BlockSize) {
|
||||
const int idx = input_idx[current][i];
|
||||
if (static_cast<int>(exact_key(input[row_start + idx]) & 0xFFu) == threshold) {
|
||||
const int pos = atomicAdd(&counter, 1);
|
||||
if (pos < topk) output[pos] = idx;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace sglang::device::legacy_radix_topk
|
||||
@@ -1,5 +1,5 @@
|
||||
import sys
|
||||
from typing import Any, Optional
|
||||
from typing import Optional
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -67,6 +67,13 @@ def _ref_torch_transform_ragged_impl(
|
||||
MAX_SEQ_LEN = 131072
|
||||
|
||||
|
||||
def _tight_cluster(batch: int, width: int) -> torch.Tensor:
|
||||
"""Finite scores that all land in one fp16 coarse radix bin."""
|
||||
base = torch.arange(width, dtype=torch.float32, device="cuda")
|
||||
values = 1.0 + base * (1e-3 / width)
|
||||
return values.expand(batch, -1).contiguous()
|
||||
|
||||
|
||||
def assert_equal(
|
||||
score: torch.Tensor,
|
||||
indices_ref: torch.Tensor,
|
||||
@@ -75,29 +82,39 @@ def assert_equal(
|
||||
k: int,
|
||||
seq_len: int,
|
||||
topk_indices_offset: Optional[torch.Tensor] = None,
|
||||
max_permit_error: int = 0,
|
||||
row_starts: Optional[torch.Tensor] = None,
|
||||
):
|
||||
indices_our_cpu = indices_our.cpu().tolist()
|
||||
indices_ref_cpu = indices_ref.cpu().tolist()
|
||||
|
||||
wrong_values = 0
|
||||
for i in range(bs):
|
||||
offset = topk_indices_offset[i].item() if topk_indices_offset is not None else 0
|
||||
row_start = row_starts[i].item() if row_starts is not None else 0
|
||||
indices_ref_set_i = set(indices_ref_cpu[i])
|
||||
indices_our_set_i = set(indices_our_cpu[i])
|
||||
assert len(indices_our_cpu[i]) == k
|
||||
assert len(indices_our_set_i) == k, "top-k output indices must be distinct"
|
||||
assert all(0 <= idx - offset < seq_len for idx in indices_our_cpu[i]), (
|
||||
"top-k output index is outside the selected window"
|
||||
)
|
||||
more = indices_our_set_i - indices_ref_set_i
|
||||
less = indices_ref_set_i - indices_our_set_i
|
||||
offset = topk_indices_offset[i].item() if topk_indices_offset is not None else 0
|
||||
if len(more) > 0 or len(less) > 0:
|
||||
# check whether more values are the same with less values
|
||||
# if so, either one is acceptable, since their values are the same
|
||||
more_values = sorted(score[i, idx - offset].item() for idx in more)
|
||||
less_values = sorted(score[i, idx - offset].item() for idx in less)
|
||||
more_values = sorted(
|
||||
score[i, row_start + idx - offset].item() for idx in more
|
||||
)
|
||||
less_values = sorted(
|
||||
score[i, row_start + idx - offset].item() for idx in less
|
||||
)
|
||||
if more_values != less_values:
|
||||
wrong_values += len(more)
|
||||
print(
|
||||
f"{bs=}, {k=}, {seq_len=}, {i=}, {more=}, {less=} failed, with {more_values=}, {less_values=}"
|
||||
)
|
||||
assert wrong_values <= max_permit_error, f"{wrong_values=}, {max_permit_error=}"
|
||||
assert wrong_values == 0, f"{wrong_values=}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bs", [1, 132, 256, 4096])
|
||||
@@ -125,8 +142,7 @@ def test_topk_kernel(bs: int, k: int, seq_len: int, has_row_starts: bool) -> Non
|
||||
indices_ref = torch.sort(indices_ref, dim=-1).values
|
||||
indices_our = torch.sort(indices_our, dim=-1).values
|
||||
|
||||
# Tests can pass with max_permit_error=3, set to 5 for safety
|
||||
assert_equal(score, indices_ref, indices_our, bs, k, seq_len, max_permit_error=5)
|
||||
assert_equal(score, indices_ref, indices_our, bs, k, seq_len, row_starts=row_starts)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bs", [1, 132, 256, 4096])
|
||||
@@ -189,7 +205,7 @@ def test_topk_transform_kernel(bs: int, k: int, seq_len: int, mode: str) -> None
|
||||
bs,
|
||||
k,
|
||||
seq_len,
|
||||
max_permit_error=5,
|
||||
row_starts=row_starts,
|
||||
)
|
||||
|
||||
|
||||
@@ -245,10 +261,71 @@ def test_topk_transform_ragged_kernel(
|
||||
k,
|
||||
seq_len,
|
||||
topk_indices_offset,
|
||||
max_permit_error=5,
|
||||
row_starts,
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_legacy_topk_oversized_coarse_bin() -> None:
|
||||
"""The shared legacy selector must not clip an oversized threshold bin."""
|
||||
torch.manual_seed(20260903)
|
||||
batch, seq_len, topk, row_start = 2, 7001, 2048, 3
|
||||
scores = torch.full(
|
||||
(batch, seq_len + row_start), -1000.0, dtype=torch.float32, device="cuda"
|
||||
)
|
||||
scores[:, row_start:] = _tight_cluster(batch, seq_len)
|
||||
lengths = torch.full((batch,), seq_len, dtype=torch.int32, device="cuda")
|
||||
row_starts = torch.full((batch,), row_start, dtype=torch.int32, device="cuda")
|
||||
reference = _ref_torch_impl(scores, seq_len, topk, row_starts=row_starts)
|
||||
|
||||
direct = fast_topk_v2(scores, lengths, topk, row_starts=row_starts)
|
||||
assert_equal(scores[:, row_start:], reference, direct, batch, topk, seq_len)
|
||||
|
||||
offsets = torch.tensor([17, 4099], dtype=torch.int32, device="cuda")
|
||||
ragged = fast_topk_transform_ragged_fused(
|
||||
score=scores,
|
||||
lengths=lengths,
|
||||
topk_indices_offset=offsets,
|
||||
topk=topk,
|
||||
row_starts=row_starts,
|
||||
)
|
||||
assert_equal(
|
||||
scores[:, row_start:],
|
||||
reference + offsets[:, None],
|
||||
ragged,
|
||||
batch,
|
||||
topk,
|
||||
seq_len,
|
||||
offsets,
|
||||
)
|
||||
|
||||
page_table = torch.arange(seq_len, dtype=torch.int32, device="cuda").expand(
|
||||
batch, -1
|
||||
)
|
||||
cu_seqlens_q = torch.arange(batch + 1, dtype=torch.int32, device="cuda")
|
||||
paged = fast_topk_transform_fused(
|
||||
score=scores,
|
||||
lengths=lengths,
|
||||
page_table_size_1=page_table,
|
||||
cu_seqlens_q=cu_seqlens_q,
|
||||
topk=topk,
|
||||
row_starts=row_starts,
|
||||
)
|
||||
assert_equal(scores[:, row_start:], reference, paged, batch, topk, seq_len)
|
||||
|
||||
# With one query per row and no row starts, this interface selects the
|
||||
# separate decode kernel rather than the prefill kernel above.
|
||||
decode_scores = scores[:, row_start:].contiguous()
|
||||
decode = fast_topk_transform_fused(
|
||||
score=decode_scores,
|
||||
lengths=lengths,
|
||||
page_table_size_1=page_table,
|
||||
cu_seqlens_q=cu_seqlens_q,
|
||||
topk=topk,
|
||||
)
|
||||
assert_equal(decode_scores, reference, decode, batch, topk, seq_len)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.version.hip is None,
|
||||
reason="deepseek_v4_topk_transform_512 is only built on ROCm",
|
||||
@@ -499,5 +576,59 @@ def test_topk_variants_are_exact_on_variable_windows(
|
||||
assert_exact_rows(score, mapped_raw, lengths, k)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.version.hip is None,
|
||||
reason="deepseek_v4_topk_transform_512 is only built on ROCm",
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_deepseek_v4_topk_oversized_coarse_bin_and_idle_row() -> None:
|
||||
"""Oversized bins select exactly; a negative DP-idle length emits padding."""
|
||||
from sgl_kernel import deepseek_v4_topk_transform_512
|
||||
|
||||
torch.manual_seed(20260903)
|
||||
active_batch, batch, c4_len, topk, page_size = 2, 3, 7001, 1024, 64
|
||||
scores = _tight_cluster(batch, c4_len)
|
||||
seq_lens = torch.tensor(
|
||||
[c4_len] * active_batch + [-1], dtype=torch.int32, device="cuda"
|
||||
)
|
||||
num_pages = (c4_len + page_size - 1) // page_size
|
||||
page_table = (
|
||||
torch.arange(num_pages, dtype=torch.int32, device="cuda")
|
||||
.expand(batch, -1)
|
||||
.contiguous()
|
||||
)
|
||||
page_indices = torch.full((batch, topk), -2, dtype=torch.int32, device="cuda")
|
||||
raw_indices = torch.full_like(page_indices, -2)
|
||||
|
||||
deepseek_v4_topk_transform_512(
|
||||
scores,
|
||||
seq_lens,
|
||||
page_table,
|
||||
page_indices,
|
||||
page_size,
|
||||
raw_indices,
|
||||
)
|
||||
|
||||
reference = torch.topk(scores[:active_batch], topk, dim=-1, sorted=False).indices
|
||||
assert_equal(
|
||||
scores[:active_batch],
|
||||
reference,
|
||||
raw_indices[:active_batch],
|
||||
active_batch,
|
||||
topk,
|
||||
c4_len,
|
||||
)
|
||||
assert_equal(
|
||||
scores[:active_batch],
|
||||
reference,
|
||||
page_indices[:active_batch],
|
||||
active_batch,
|
||||
topk,
|
||||
c4_len,
|
||||
)
|
||||
assert bool((raw_indices[-1] == -1).all())
|
||||
assert bool((page_indices[-1] == -1).all())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__]))
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
#include <sgl_kernel/runtime.cuh>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
|
||||
#include <sgl_kernel/dsa/legacy_radix_topk.cuh>
|
||||
|
||||
#include <dlpack/dlpack.h>
|
||||
#include <tvm/ffi/container/tensor.h>
|
||||
|
||||
@@ -39,18 +41,6 @@ struct TopKParams {
|
||||
uint32_t topk;
|
||||
};
|
||||
|
||||
SGL_DEVICE uint8_t convert_to_uint8(float x) {
|
||||
__half h = __float2half_rn(x);
|
||||
uint16_t bits = __half_as_ushort(h);
|
||||
uint16_t key = (bits & 0x8000) ? static_cast<uint16_t>(~bits) : static_cast<uint16_t>(bits | 0x8000);
|
||||
return static_cast<uint8_t>(key >> 8);
|
||||
}
|
||||
|
||||
SGL_DEVICE uint32_t convert_to_uint32(float x) {
|
||||
uint32_t bits = __float_as_uint(x);
|
||||
return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u);
|
||||
}
|
||||
|
||||
SGL_DEVICE int32_t page_to_indices(const int32_t* __restrict__ page_table, uint32_t i, uint32_t page_bits) {
|
||||
const uint32_t mask = (1u << page_bits) - 1u;
|
||||
return (page_table[i >> page_bits] << page_bits) | (i & mask);
|
||||
@@ -81,160 +71,9 @@ SGL_DEVICE void naive_transform(
|
||||
[[maybe_unused]]
|
||||
SGL_DEVICE void
|
||||
radix_topk(const float* __restrict__ input, int32_t* __restrict__ output, const uint32_t length, const uint32_t topk) {
|
||||
constexpr uint32_t RADIX = 256;
|
||||
constexpr uint32_t BLOCK_SIZE = kTopKBlockSize;
|
||||
constexpr uint32_t SMEM_INPUT_SIZE = kSMEM / (2 * sizeof(int32_t));
|
||||
|
||||
alignas(128) __shared__ uint32_t _s_histogram_buf[2][RADIX + 32];
|
||||
alignas(128) __shared__ uint32_t s_counter;
|
||||
alignas(128) __shared__ uint32_t s_threshold_bin_id;
|
||||
alignas(128) __shared__ uint32_t s_num_input[2];
|
||||
alignas(128) __shared__ int32_t s_last_remain;
|
||||
|
||||
extern __shared__ uint32_t s_input_idx[][kSMEM / (2 * sizeof(int32_t))];
|
||||
|
||||
const uint32_t tx = threadIdx.x;
|
||||
uint32_t remain_topk = topk;
|
||||
auto& s_histogram = _s_histogram_buf[0];
|
||||
|
||||
const auto run_cumsum = [&] {
|
||||
#pragma unroll 8
|
||||
for (int32_t i = 0; i < 8; ++i) {
|
||||
static_assert(1 << 8 == RADIX);
|
||||
if (tx < RADIX) {
|
||||
const auto j = 1 << i;
|
||||
const auto k = i & 1;
|
||||
auto value = _s_histogram_buf[k][tx];
|
||||
if (tx + j < RADIX) {
|
||||
value += _s_histogram_buf[k][tx + j];
|
||||
}
|
||||
_s_histogram_buf[k ^ 1][tx] = value;
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
};
|
||||
|
||||
// stage 1: 8bit coarse histogram
|
||||
if (tx < RADIX + 1) s_histogram[tx] = 0;
|
||||
__syncthreads();
|
||||
for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) {
|
||||
const auto bin = convert_to_uint8(input[idx]);
|
||||
::atomicAdd(&s_histogram[bin], 1);
|
||||
}
|
||||
__syncthreads();
|
||||
run_cumsum();
|
||||
if (tx < RADIX && s_histogram[tx] > remain_topk && s_histogram[tx + 1] <= remain_topk) {
|
||||
s_threshold_bin_id = tx;
|
||||
s_num_input[0] = 0;
|
||||
s_counter = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const auto threshold_bin = s_threshold_bin_id;
|
||||
remain_topk -= s_histogram[threshold_bin + 1];
|
||||
if (remain_topk == 0) {
|
||||
for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) {
|
||||
const uint32_t bin = convert_to_uint8(input[idx]);
|
||||
if (bin > threshold_bin) {
|
||||
const auto pos = ::atomicAdd(&s_counter, 1);
|
||||
output[pos] = idx;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
return;
|
||||
} else {
|
||||
__syncthreads();
|
||||
if (tx < RADIX + 1) {
|
||||
s_histogram[tx] = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) {
|
||||
const float raw_input = input[idx];
|
||||
const uint32_t bin = convert_to_uint8(raw_input);
|
||||
if (bin > threshold_bin) {
|
||||
const auto pos = ::atomicAdd(&s_counter, 1);
|
||||
output[pos] = idx;
|
||||
} else if (bin == threshold_bin) {
|
||||
const auto pos = ::atomicAdd(&s_num_input[0], 1);
|
||||
if (pos < SMEM_INPUT_SIZE) {
|
||||
[[likely]] s_input_idx[0][pos] = idx;
|
||||
const auto bin = convert_to_uint32(raw_input);
|
||||
const auto sub_bin = (bin >> 24) & 0xFF;
|
||||
::atomicAdd(&s_histogram[sub_bin], 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
// stage 2: refine with 8bit radix passes
|
||||
#pragma unroll 4
|
||||
for (int round = 0; round < 4; ++round) {
|
||||
const auto r_idx = round % 2;
|
||||
|
||||
// clip here to prevent overflow
|
||||
const auto raw_num_input = s_num_input[r_idx];
|
||||
const auto num_input = raw_num_input < SMEM_INPUT_SIZE ? raw_num_input : SMEM_INPUT_SIZE;
|
||||
|
||||
run_cumsum();
|
||||
if (tx < RADIX && s_histogram[tx] > remain_topk && s_histogram[tx + 1] <= remain_topk) {
|
||||
s_threshold_bin_id = tx;
|
||||
s_num_input[r_idx ^ 1] = 0;
|
||||
s_last_remain = remain_topk - s_histogram[tx + 1];
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const auto threshold_bin = s_threshold_bin_id;
|
||||
remain_topk -= s_histogram[threshold_bin + 1];
|
||||
|
||||
if (remain_topk == 0) {
|
||||
for (uint32_t i = tx; i < num_input; i += BLOCK_SIZE) {
|
||||
const auto idx = s_input_idx[r_idx][i];
|
||||
const auto offset = 24 - round * 8;
|
||||
const auto bin = (convert_to_uint32(input[idx]) >> offset) & 0xFF;
|
||||
if (bin > threshold_bin) {
|
||||
const auto pos = ::atomicAdd(&s_counter, 1);
|
||||
output[pos] = idx;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
break;
|
||||
} else {
|
||||
__syncthreads();
|
||||
if (tx < RADIX + 1) {
|
||||
s_histogram[tx] = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
for (uint32_t i = tx; i < num_input; i += BLOCK_SIZE) {
|
||||
const auto idx = s_input_idx[r_idx][i];
|
||||
const auto raw_input = input[idx];
|
||||
const auto offset = 24 - round * 8;
|
||||
const auto bin = (convert_to_uint32(raw_input) >> offset) & 0xFF;
|
||||
if (bin > threshold_bin) {
|
||||
const auto pos = ::atomicAdd(&s_counter, 1);
|
||||
output[pos] = idx;
|
||||
} else if (bin == threshold_bin) {
|
||||
if (round == 3) {
|
||||
const auto pos = ::atomicAdd(&s_last_remain, -1);
|
||||
if (pos > 0) {
|
||||
output[topk - pos] = idx;
|
||||
}
|
||||
} else {
|
||||
const auto pos = ::atomicAdd(&s_num_input[r_idx ^ 1], 1);
|
||||
if (pos < SMEM_INPUT_SIZE) {
|
||||
/// NOTE: (dark) fuse the histogram computation here
|
||||
[[likely]] s_input_idx[r_idx ^ 1][pos] = idx;
|
||||
const auto bin = convert_to_uint32(raw_input);
|
||||
const auto sub_bin = (bin >> (offset - 8)) & 0xFF;
|
||||
::atomicAdd(&s_histogram[sub_bin], 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
}
|
||||
::sglang::device::legacy_radix_topk::
|
||||
select<kTopKBlockSize, kMaxTopK, static_cast<int>(kSMEM / (2 * sizeof(int32_t)))>(
|
||||
input, output, 0, static_cast<int>(length), static_cast<int>(topk));
|
||||
}
|
||||
|
||||
template <bool kUsePDL>
|
||||
@@ -261,9 +100,10 @@ __global__ void topk_transform_kernel(const __grid_constant__ TopKParams params)
|
||||
radix_topk(score_ptr, s_topk_indices, seq_len, topk);
|
||||
const auto tx = threadIdx.x;
|
||||
if (tx < topk) {
|
||||
indices_ptr[tx] = page_to_indices(page_ptr, s_topk_indices[tx], page_bits);
|
||||
const auto raw = s_topk_indices[tx];
|
||||
indices_ptr[tx] = raw < 0 ? -1 : page_to_indices(page_ptr, raw, page_bits);
|
||||
if (raw_indices_ptr != nullptr) {
|
||||
raw_indices_ptr[tx] = s_topk_indices[tx];
|
||||
raw_indices_ptr[tx] = raw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
#pragma once
|
||||
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
namespace sglang::device::legacy_radix_topk {
|
||||
|
||||
inline constexpr int kRadix = 256;
|
||||
|
||||
// Coarse bin: high byte of the ordered FP16 value, as in the original selectors.
|
||||
SGL_DEVICE uint8_t coarse_key(float x) {
|
||||
const __half h = __float2half_rn(x);
|
||||
const uint16_t bits = __half_as_ushort(h);
|
||||
const uint16_t key = (bits & 0x8000u) ? static_cast<uint16_t>(~bits) : static_cast<uint16_t>(bits | 0x8000u);
|
||||
return static_cast<uint8_t>(key >> 8);
|
||||
}
|
||||
|
||||
SGL_DEVICE uint32_t exact_key(float x) {
|
||||
const uint32_t bits = __float_as_uint(x);
|
||||
return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u);
|
||||
}
|
||||
|
||||
// Exact radix top-k over finite scores. A threshold bin that fits the stash
|
||||
// keeps the original two-stash refinement; a larger one is refined by rescanning
|
||||
// the input. Unfilled output slots stay -1.
|
||||
template <int BlockSize, int OutputCapacity, int StashEntries>
|
||||
SGL_DEVICE void
|
||||
select(const float* __restrict__ input, int32_t* __restrict__ output, int row_start, int length, int topk) {
|
||||
static_assert(BlockSize >= kRadix + 1, "the block must initialize the complete radix histogram");
|
||||
static_assert(OutputCapacity <= StashEntries, "top-k output must fit in one shared-memory stash");
|
||||
|
||||
alignas(128) __shared__ int histogram_buf[2][kRadix + 1];
|
||||
alignas(128) __shared__ int counter;
|
||||
alignas(128) __shared__ int threshold_bin_id;
|
||||
alignas(128) __shared__ int num_input[2];
|
||||
|
||||
extern __shared__ int dynamic_smem[];
|
||||
auto* input_idx = reinterpret_cast<int (*)[StashEntries]>(dynamic_smem);
|
||||
auto& histogram = histogram_buf[0];
|
||||
const int tx = threadIdx.x;
|
||||
|
||||
for (int i = tx; i < topk; i += BlockSize)
|
||||
output[i] = -1;
|
||||
if (tx < kRadix + 1) histogram[tx] = 0;
|
||||
if (tx == 0) {
|
||||
counter = 0;
|
||||
threshold_bin_id = -1;
|
||||
num_input[0] = 0;
|
||||
num_input[1] = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
for (int idx = tx; idx < length; idx += BlockSize) {
|
||||
atomicAdd(&histogram[coarse_key(input[row_start + idx])], 1);
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const auto run_cumsum = [&] {
|
||||
#pragma unroll 8
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
if (tx < kRadix) {
|
||||
const int offset = 1 << i;
|
||||
const int src = i & 1;
|
||||
int value = histogram_buf[src][tx];
|
||||
if (tx < kRadix - offset) value += histogram_buf[src][tx + offset];
|
||||
histogram_buf[src ^ 1][tx] = value;
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
};
|
||||
|
||||
run_cumsum();
|
||||
if (tx < kRadix && histogram[tx] > topk && histogram[tx + 1] <= topk) {
|
||||
threshold_bin_id = tx;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const int coarse_threshold = threshold_bin_id;
|
||||
if (coarse_threshold < 0) return;
|
||||
const int coarse_above = histogram[coarse_threshold + 1];
|
||||
int remaining = topk - coarse_above;
|
||||
|
||||
if (remaining == 0) {
|
||||
for (int idx = tx; idx < length; idx += BlockSize) {
|
||||
if (coarse_key(input[row_start + idx]) > coarse_threshold) {
|
||||
const int pos = atomicAdd(&counter, 1);
|
||||
if (pos < topk) output[pos] = idx;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
return;
|
||||
}
|
||||
|
||||
const int coarse_population = histogram[coarse_threshold] - histogram[coarse_threshold + 1];
|
||||
__syncthreads();
|
||||
if (coarse_population >= StashEntries) {
|
||||
// Find the exact cutoff one key byte per pass. Pass L counts byte L and
|
||||
// emits what byte L-1 placed strictly above the cutoff; pass 0 emits the
|
||||
// values above the coarse bin and pass 4 the last strict values and ties.
|
||||
uint32_t prefix = 0;
|
||||
int last_threshold = -1;
|
||||
int strict_count = 0;
|
||||
#pragma unroll 1
|
||||
for (int level = 0; level <= 4; ++level) {
|
||||
const uint32_t mask = level == 0 ? 0u : (~0u << (32 - level * 8));
|
||||
const uint32_t last_mask = level <= 1 ? 0u : (~0u << (40 - level * 8));
|
||||
const int last_shift = 32 - level * 8;
|
||||
if (level < 4) {
|
||||
if (tx < kRadix + 1) histogram[tx] = 0;
|
||||
if (tx == 0) threshold_bin_id = -1;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
for (int idx = tx; idx < length; idx += BlockSize) {
|
||||
const float value = input[row_start + idx];
|
||||
const int bin = coarse_key(value);
|
||||
if (bin != coarse_threshold) {
|
||||
if (level == 0 && bin > coarse_threshold) {
|
||||
const int pos = atomicAdd(&counter, 1);
|
||||
if (pos < topk) output[pos] = idx;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const uint32_t key = exact_key(value);
|
||||
if (level > 0 && (key & last_mask) == (prefix & last_mask) &&
|
||||
static_cast<int>((key >> last_shift) & 0xFFu) > last_threshold) {
|
||||
const int pos = atomicAdd(&num_input[0], 1);
|
||||
if (pos < strict_count) output[coarse_above + pos] = idx;
|
||||
} else if ((key & mask) == prefix) {
|
||||
if (level < 4) {
|
||||
atomicAdd(&histogram[(key >> (24 - level * 8)) & 0xFFu], 1);
|
||||
} else {
|
||||
const int pos = atomicAdd(&num_input[1], 1);
|
||||
if (pos < remaining) output[coarse_above + strict_count + pos] = idx;
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
if (level == 4) return;
|
||||
|
||||
run_cumsum();
|
||||
if (tx < kRadix && histogram[tx] >= remaining && histogram[tx + 1] < remaining) {
|
||||
threshold_bin_id = tx;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const int threshold = threshold_bin_id;
|
||||
if (threshold < 0) return;
|
||||
const int above = histogram[threshold + 1];
|
||||
strict_count += above;
|
||||
remaining -= above;
|
||||
prefix |= static_cast<uint32_t>(threshold) << (24 - level * 8);
|
||||
last_threshold = threshold;
|
||||
// All threads must read the histogram before the next pass clears it.
|
||||
__syncthreads();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// The threshold bin fits the stash: original two-stash refinement.
|
||||
if (tx < kRadix + 1) histogram[tx] = 0;
|
||||
__syncthreads();
|
||||
for (int idx = tx; idx < length; idx += BlockSize) {
|
||||
const float value = input[row_start + idx];
|
||||
const int bin = coarse_key(value);
|
||||
if (bin > coarse_threshold) {
|
||||
const int pos = atomicAdd(&counter, 1);
|
||||
if (pos < topk) output[pos] = idx;
|
||||
} else if (bin == coarse_threshold) {
|
||||
const int pos = atomicAdd(&num_input[0], 1);
|
||||
if (pos < StashEntries) {
|
||||
input_idx[0][pos] = idx;
|
||||
atomicAdd(&histogram[(exact_key(value) >> 24) & 0xFFu], 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
#pragma unroll 4
|
||||
for (int round = 0; round < 4; ++round) {
|
||||
const int current = round & 1;
|
||||
const int next = current ^ 1;
|
||||
const int raw_num_input = num_input[current];
|
||||
const int current_count = raw_num_input < StashEntries ? raw_num_input : StashEntries;
|
||||
|
||||
if (tx == 0) {
|
||||
threshold_bin_id = -1;
|
||||
num_input[next] = 0;
|
||||
}
|
||||
run_cumsum();
|
||||
if (tx < kRadix && histogram[tx] > remaining && histogram[tx + 1] <= remaining) {
|
||||
threshold_bin_id = tx;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const int threshold = threshold_bin_id;
|
||||
if (threshold < 0) return;
|
||||
const int above = histogram[threshold + 1];
|
||||
const int shift = 24 - round * 8;
|
||||
|
||||
__syncthreads();
|
||||
if (round < 3 && tx < kRadix + 1) histogram[tx] = 0;
|
||||
__syncthreads();
|
||||
for (int i = tx; i < current_count; i += BlockSize) {
|
||||
const int idx = input_idx[current][i];
|
||||
const float value = input[row_start + idx];
|
||||
const int bin = static_cast<int>((exact_key(value) >> shift) & 0xFFu);
|
||||
if (bin > threshold) {
|
||||
const int pos = atomicAdd(&counter, 1);
|
||||
if (pos < topk) output[pos] = idx;
|
||||
} else if (bin == threshold && round < 3) {
|
||||
const int pos = atomicAdd(&num_input[next], 1);
|
||||
if (pos < StashEntries) {
|
||||
input_idx[next][pos] = idx;
|
||||
atomicAdd(&histogram[(exact_key(value) >> (shift - 8)) & 0xFFu], 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
remaining -= above;
|
||||
if (remaining == 0) return;
|
||||
if (round == 3) {
|
||||
for (int i = tx; i < current_count; i += BlockSize) {
|
||||
const int idx = input_idx[current][i];
|
||||
if (static_cast<int>(exact_key(input[row_start + idx]) & 0xFFu) == threshold) {
|
||||
const int pos = atomicAdd(&counter, 1);
|
||||
if (pos < topk) output[pos] = idx;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace sglang::device::legacy_radix_topk
|
||||
Reference in New Issue
Block a user