Fix overflow in legacy and v1 DSA top-k selectors (#37625)
Co-authored-by: Benjamin Oldenburg <benjamin.oldenburg@ordis.co.th>
This commit is contained in:
co-authored by
Benjamin Oldenburg
parent
2f5407da28
commit
c5bf3aafbc
@@ -24,6 +24,7 @@ limitations under the License.
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <sgl_kernel/dsa/legacy_radix_topk.cuh>
|
||||
|
||||
namespace {
|
||||
|
||||
@@ -62,18 +63,6 @@ struct TopKParams {
|
||||
bool sort_output;
|
||||
};
|
||||
|
||||
__device__ __forceinline__ uint8_t convert_to_uint8(float x) {
|
||||
__half h = __float2half_rn(x);
|
||||
uint16_t bits = __half_as_ushort(h);
|
||||
uint16_t key = (bits & 0x8000) ? static_cast<uint16_t>(~bits) : static_cast<uint16_t>(bits | 0x8000);
|
||||
return static_cast<uint8_t>(key >> 8);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ uint32_t convert_to_uint32(float x) {
|
||||
uint32_t bits = __float_as_uint(x);
|
||||
return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ int32_t
|
||||
page_to_slot(const int32_t* __restrict__ page_table, uint32_t i, uint32_t page_bits) {
|
||||
const uint32_t mask = (1u << page_bits) - 1u;
|
||||
@@ -104,154 +93,10 @@ __device__ void naive_paged_transform(
|
||||
|
||||
__device__ void
|
||||
radix_topk(const float* __restrict__ input, int32_t* __restrict__ output, uint32_t length, uint32_t topk) {
|
||||
constexpr uint32_t RADIX = 256;
|
||||
constexpr uint32_t BLOCK_SIZE = kBlockSize;
|
||||
constexpr uint32_t SMEM_INPUT_SIZE = kSMEM / (2 * sizeof(int32_t));
|
||||
|
||||
alignas(128) __shared__ uint32_t _s_histogram_buf[2][RADIX + 32];
|
||||
alignas(128) __shared__ uint32_t s_counter;
|
||||
alignas(128) __shared__ uint32_t s_threshold_bin_id;
|
||||
alignas(128) __shared__ uint32_t s_num_input[2];
|
||||
alignas(128) __shared__ int32_t s_last_remain;
|
||||
|
||||
extern __shared__ uint32_t s_input_idx[][SMEM_INPUT_SIZE];
|
||||
|
||||
const uint32_t tx = threadIdx.x;
|
||||
uint32_t remain_topk = topk;
|
||||
auto& s_histogram = _s_histogram_buf[0];
|
||||
|
||||
const auto run_cumsum = [&] {
|
||||
#pragma unroll 8
|
||||
for (int32_t i = 0; i < 8; ++i) {
|
||||
static_assert(1 << 8 == RADIX);
|
||||
if (tx < RADIX) {
|
||||
const auto j = 1 << i;
|
||||
const auto k = i & 1;
|
||||
auto value = _s_histogram_buf[k][tx];
|
||||
if (tx + j < RADIX) {
|
||||
value += _s_histogram_buf[k][tx + j];
|
||||
}
|
||||
_s_histogram_buf[k ^ 1][tx] = value;
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
};
|
||||
|
||||
// stage 1: 8bit coarse histogram
|
||||
if (tx < RADIX + 1) s_histogram[tx] = 0;
|
||||
::sglang::device::legacy_radix_topk::
|
||||
select<static_cast<int>(kBlockSize), static_cast<int>(kMaxTopK), static_cast<int>(kSMEM / (2 * sizeof(int32_t)))>(
|
||||
input, output, 0, static_cast<int>(length), static_cast<int>(topk));
|
||||
__syncthreads();
|
||||
for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) {
|
||||
const auto bin = convert_to_uint8(input[idx]);
|
||||
::atomicAdd(&s_histogram[bin], 1);
|
||||
}
|
||||
__syncthreads();
|
||||
run_cumsum();
|
||||
if (tx < RADIX && s_histogram[tx] > remain_topk && s_histogram[tx + 1] <= remain_topk) {
|
||||
s_threshold_bin_id = tx;
|
||||
s_num_input[0] = 0;
|
||||
s_counter = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
{
|
||||
const auto threshold_bin = s_threshold_bin_id;
|
||||
remain_topk -= s_histogram[threshold_bin + 1];
|
||||
if (remain_topk == 0) {
|
||||
for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) {
|
||||
const uint32_t bin = convert_to_uint8(input[idx]);
|
||||
if (bin > threshold_bin) {
|
||||
const auto pos = ::atomicAdd(&s_counter, 1);
|
||||
output[pos] = static_cast<int32_t>(idx);
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
return;
|
||||
}
|
||||
__syncthreads();
|
||||
if (tx < RADIX + 1) s_histogram[tx] = 0;
|
||||
__syncthreads();
|
||||
|
||||
for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) {
|
||||
const float raw_input = input[idx];
|
||||
const uint32_t bin = convert_to_uint8(raw_input);
|
||||
if (bin > threshold_bin) {
|
||||
const auto pos = ::atomicAdd(&s_counter, 1);
|
||||
output[pos] = static_cast<int32_t>(idx);
|
||||
} else if (bin == threshold_bin) {
|
||||
const auto pos = ::atomicAdd(&s_num_input[0], 1);
|
||||
if (C10_LIKELY(pos < SMEM_INPUT_SIZE)) {
|
||||
s_input_idx[0][pos] = idx;
|
||||
const auto bin32 = convert_to_uint32(raw_input);
|
||||
const auto sub_bin = (bin32 >> 24) & 0xFF;
|
||||
::atomicAdd(&s_histogram[sub_bin], 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
// stage 2: refine with 8bit radix passes
|
||||
#pragma unroll 4
|
||||
for (int round = 0; round < 4; ++round) {
|
||||
const auto r_idx = round % 2;
|
||||
|
||||
const auto raw_num_input = s_num_input[r_idx];
|
||||
const auto num_input = raw_num_input < SMEM_INPUT_SIZE ? raw_num_input : SMEM_INPUT_SIZE;
|
||||
|
||||
run_cumsum();
|
||||
if (tx < RADIX && s_histogram[tx] > remain_topk && s_histogram[tx + 1] <= remain_topk) {
|
||||
s_threshold_bin_id = tx;
|
||||
s_num_input[r_idx ^ 1] = 0;
|
||||
s_last_remain = static_cast<int32_t>(remain_topk - s_histogram[tx + 1]);
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const auto threshold_bin = s_threshold_bin_id;
|
||||
remain_topk -= s_histogram[threshold_bin + 1];
|
||||
|
||||
if (remain_topk == 0) {
|
||||
for (uint32_t i = tx; i < num_input; i += BLOCK_SIZE) {
|
||||
const auto idx = s_input_idx[r_idx][i];
|
||||
const auto offset = 24 - round * 8;
|
||||
const auto bin = (convert_to_uint32(input[idx]) >> offset) & 0xFF;
|
||||
if (bin > threshold_bin) {
|
||||
const auto pos = ::atomicAdd(&s_counter, 1);
|
||||
output[pos] = static_cast<int32_t>(idx);
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
break;
|
||||
}
|
||||
__syncthreads();
|
||||
if (tx < RADIX + 1) s_histogram[tx] = 0;
|
||||
__syncthreads();
|
||||
for (uint32_t i = tx; i < num_input; i += BLOCK_SIZE) {
|
||||
const auto idx = s_input_idx[r_idx][i];
|
||||
const auto raw_input = input[idx];
|
||||
const auto offset = 24 - round * 8;
|
||||
const auto bin = (convert_to_uint32(raw_input) >> offset) & 0xFF;
|
||||
if (bin > threshold_bin) {
|
||||
const auto pos = ::atomicAdd(&s_counter, 1);
|
||||
output[pos] = static_cast<int32_t>(idx);
|
||||
} else if (bin == threshold_bin) {
|
||||
if (round == 3) {
|
||||
const auto pos = ::atomicAdd(&s_last_remain, -1);
|
||||
if (pos > 0) {
|
||||
output[topk - pos] = static_cast<int32_t>(idx);
|
||||
}
|
||||
} else {
|
||||
const auto pos = ::atomicAdd(&s_num_input[r_idx ^ 1], 1);
|
||||
if (C10_LIKELY(pos < SMEM_INPUT_SIZE)) {
|
||||
s_input_idx[r_idx ^ 1][pos] = idx;
|
||||
const auto bin32 = convert_to_uint32(raw_input);
|
||||
const auto sub_bin = (bin32 >> (offset - 8)) & 0xFF;
|
||||
::atomicAdd(&s_histogram[sub_bin], 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef USE_ROCM
|
||||
@@ -366,7 +211,8 @@ __device__ __forceinline__ uint32_t next_pow2_at_least_64(uint32_t x) {
|
||||
|
||||
__global__ __launch_bounds__(kBlockSize) void deepseek_v4_topk_transform_kernel(const TopKParams params) {
|
||||
const auto bid = blockIdx.x;
|
||||
const auto seq_len = params.seq_lens[bid];
|
||||
const auto raw_seq_len = params.seq_lens[bid];
|
||||
const auto seq_len = raw_seq_len < 0 ? 0 : raw_seq_len;
|
||||
const auto topk = params.topk;
|
||||
const auto score_ptr = params.scores + bid * params.score_stride;
|
||||
const auto page_ptr = params.page_table + bid * params.page_table_stride;
|
||||
@@ -433,7 +279,7 @@ __global__ __launch_bounds__(kBlockSize) void deepseek_v4_topk_transform_kernel(
|
||||
|
||||
for (uint32_t i = threadIdx.x; i < topk; i += kBlockSize) {
|
||||
const auto raw = s_topk_indices[i];
|
||||
indices_ptr[i] = page_to_slot(page_ptr, static_cast<uint32_t>(raw), params.page_bits);
|
||||
indices_ptr[i] = raw < 0 ? -1 : page_to_slot(page_ptr, static_cast<uint32_t>(raw), params.page_bits);
|
||||
if (raw_indices_ptr != nullptr) {
|
||||
raw_indices_ptr[i] = raw;
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <sgl_kernel/dsa/legacy_radix_topk.cuh>
|
||||
|
||||
namespace {
|
||||
|
||||
@@ -75,180 +76,10 @@ __device__ void naive_topk_transform_ragged(
|
||||
}
|
||||
}
|
||||
|
||||
__device__ __forceinline__ auto convert_to_uint8(float x) -> uint8_t {
|
||||
__half h = __float2half_rn(x);
|
||||
uint16_t bits = __half_as_ushort(h);
|
||||
uint16_t key = (bits & 0x8000) ? static_cast<uint16_t>(~bits) : static_cast<uint16_t>(bits | 0x8000);
|
||||
return static_cast<uint8_t>(key >> 8);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ auto convert_to_uint32(float x) -> uint32_t {
|
||||
uint32_t bits = __float_as_uint(x);
|
||||
return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u);
|
||||
}
|
||||
|
||||
__device__ void fast_topk_cuda_tl(const float* __restrict__ input, int* __restrict__ index, int row_start, int length) {
|
||||
// An optimized topk kernel copied from tilelang kernel
|
||||
// We assume length > TopK here, or it will crash
|
||||
int topk = TopK;
|
||||
constexpr auto BLOCK_SIZE = 1024;
|
||||
constexpr auto RADIX = 256;
|
||||
constexpr auto SMEM_INPUT_SIZE = kSmem / (2 * sizeof(int));
|
||||
|
||||
alignas(128) __shared__ int s_histogram_buf[2][RADIX + 128];
|
||||
alignas(128) __shared__ int s_counter;
|
||||
alignas(128) __shared__ int s_threshold_bin_id;
|
||||
alignas(128) __shared__ int s_num_input[2];
|
||||
|
||||
auto& s_histogram = s_histogram_buf[0];
|
||||
// allocate for two rounds
|
||||
extern __shared__ int s_input_idx[][SMEM_INPUT_SIZE];
|
||||
|
||||
const int tx = threadIdx.x;
|
||||
|
||||
// stage 1: 8bit coarse histogram
|
||||
if (tx < RADIX + 1) s_histogram[tx] = 0;
|
||||
::sglang::device::legacy_radix_topk::select<kThreadsPerBlock, TopK, static_cast<int>(kSmem / (2 * sizeof(int)))>(
|
||||
input, index, row_start, length, TopK);
|
||||
__syncthreads();
|
||||
|
||||
for (int idx = tx; idx < length; idx += BLOCK_SIZE) {
|
||||
const auto bin = convert_to_uint8(input[idx + row_start]);
|
||||
::atomicAdd(&s_histogram[bin], 1);
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const auto run_cumsum = [&] {
|
||||
#pragma unroll 8
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
static_assert(1 << 8 == RADIX);
|
||||
if (C10_LIKELY(tx < RADIX)) {
|
||||
const auto j = 1 << i;
|
||||
const auto k = i & 1;
|
||||
auto value = s_histogram_buf[k][tx];
|
||||
if (tx < RADIX - j) {
|
||||
value += s_histogram_buf[k][tx + j];
|
||||
}
|
||||
s_histogram_buf[k ^ 1][tx] = value;
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
};
|
||||
|
||||
run_cumsum();
|
||||
if (tx < RADIX && s_histogram[tx] > topk && s_histogram[tx + 1] <= topk) {
|
||||
s_threshold_bin_id = tx;
|
||||
s_num_input[0] = 0;
|
||||
s_counter = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const auto threshold_bin = s_threshold_bin_id;
|
||||
topk -= s_histogram[threshold_bin + 1];
|
||||
|
||||
if (topk == 0) {
|
||||
for (int idx = tx; idx < length; idx += BLOCK_SIZE) {
|
||||
const auto bin = static_cast<int>(convert_to_uint8(input[idx + row_start]));
|
||||
if (bin > threshold_bin) {
|
||||
const auto pos = ::atomicAdd(&s_counter, 1);
|
||||
index[pos] = idx;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
return;
|
||||
} else {
|
||||
__syncthreads();
|
||||
if (tx < RADIX + 1) {
|
||||
s_histogram[tx] = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
for (int idx = tx; idx < length; idx += BLOCK_SIZE) {
|
||||
const auto raw_input = input[idx + row_start];
|
||||
const auto bin = static_cast<int>(convert_to_uint8(raw_input));
|
||||
if (bin > threshold_bin) {
|
||||
const auto pos = ::atomicAdd(&s_counter, 1);
|
||||
index[pos] = idx;
|
||||
} else if (bin == threshold_bin) {
|
||||
const auto pos = ::atomicAdd(&s_num_input[0], 1);
|
||||
/// NOTE: (dark) fuse the histogram computation here
|
||||
if (C10_LIKELY(pos < SMEM_INPUT_SIZE)) {
|
||||
s_input_idx[0][pos] = idx;
|
||||
const auto bin = convert_to_uint32(raw_input);
|
||||
const auto sub_bin = (bin >> 24) & 0xFF;
|
||||
::atomicAdd(&s_histogram[sub_bin], 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
// stage 2: refine with 8bit radix passes
|
||||
#pragma unroll 4
|
||||
for (int round = 0; round < 4; ++round) {
|
||||
__shared__ int s_last_remain;
|
||||
const auto r_idx = round % 2;
|
||||
|
||||
// clip here to prevent overflow
|
||||
const auto _raw_num_input = s_num_input[r_idx];
|
||||
const auto num_input = (_raw_num_input < int(SMEM_INPUT_SIZE)) ? _raw_num_input : int(SMEM_INPUT_SIZE);
|
||||
|
||||
run_cumsum();
|
||||
if (tx < RADIX && s_histogram[tx] > topk && s_histogram[tx + 1] <= topk) {
|
||||
s_threshold_bin_id = tx;
|
||||
s_num_input[r_idx ^ 1] = 0;
|
||||
s_last_remain = topk - s_histogram[tx + 1];
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const auto threshold_bin = s_threshold_bin_id;
|
||||
topk -= s_histogram[threshold_bin + 1];
|
||||
|
||||
if (topk == 0) {
|
||||
for (int i = tx; i < num_input; i += BLOCK_SIZE) {
|
||||
const auto idx = s_input_idx[r_idx][i];
|
||||
const auto offset = 24 - round * 8;
|
||||
const auto bin = (convert_to_uint32(input[idx + row_start]) >> offset) & 0xFF;
|
||||
if (bin > threshold_bin) {
|
||||
const auto pos = ::atomicAdd(&s_counter, 1);
|
||||
index[pos] = idx;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
break;
|
||||
} else {
|
||||
__syncthreads();
|
||||
if (tx < RADIX + 1) {
|
||||
s_histogram[tx] = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
for (int i = tx; i < num_input; i += BLOCK_SIZE) {
|
||||
const auto idx = s_input_idx[r_idx][i];
|
||||
const auto raw_input = input[idx + row_start];
|
||||
const auto offset = 24 - round * 8;
|
||||
const auto bin = (convert_to_uint32(raw_input) >> offset) & 0xFF;
|
||||
if (bin > threshold_bin) {
|
||||
const auto pos = ::atomicAdd(&s_counter, 1);
|
||||
index[pos] = idx;
|
||||
} else if (bin == threshold_bin) {
|
||||
if (round == 3) {
|
||||
const auto pos = ::atomicAdd(&s_last_remain, -1);
|
||||
if (pos > 0) {
|
||||
index[TopK - pos] = idx;
|
||||
}
|
||||
} else {
|
||||
const auto pos = ::atomicAdd(&s_num_input[r_idx ^ 1], 1);
|
||||
if (C10_LIKELY(pos < SMEM_INPUT_SIZE)) {
|
||||
/// NOTE: (dark) fuse the histogram computation here
|
||||
s_input_idx[r_idx ^ 1][pos] = idx;
|
||||
const auto bin = convert_to_uint32(raw_input);
|
||||
const auto sub_bin = (bin >> (offset - 8)) & 0xFF;
|
||||
::atomicAdd(&s_histogram[sub_bin], 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
__global__ __launch_bounds__(kThreadsPerBlock) // topk
|
||||
@@ -290,10 +121,10 @@ __global__ __launch_bounds__(kThreadsPerBlock) // decode
|
||||
static_assert(TopK / kThreadsPerBlock == 2);
|
||||
const auto idx_0 = tid;
|
||||
const auto pos_0 = s_indices[idx_0];
|
||||
dst_page_entry[idx_0] = src_page_entry[pos_0];
|
||||
dst_page_entry[idx_0] = pos_0 < 0 ? -1 : src_page_entry[pos_0];
|
||||
const auto idx_1 = tid + kThreadsPerBlock;
|
||||
const auto pos_1 = s_indices[idx_1];
|
||||
dst_page_entry[idx_1] = src_page_entry[pos_1];
|
||||
dst_page_entry[idx_1] = pos_1 < 0 ? -1 : src_page_entry[pos_1];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -342,10 +173,10 @@ __global__ __launch_bounds__(kThreadsPerBlock) // prefill
|
||||
static_assert(TopK / kThreadsPerBlock == 2);
|
||||
const auto idx_0 = tid;
|
||||
const auto pos_0 = s_indices[idx_0];
|
||||
dst_page_entry[idx_0] = src_page_entry[pos_0];
|
||||
dst_page_entry[idx_0] = pos_0 < 0 ? -1 : src_page_entry[pos_0];
|
||||
const auto idx_1 = tid + kThreadsPerBlock;
|
||||
const auto pos_1 = s_indices[idx_1];
|
||||
dst_page_entry[idx_1] = src_page_entry[pos_1];
|
||||
dst_page_entry[idx_1] = pos_1 < 0 ? -1 : src_page_entry[pos_1];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -373,10 +204,10 @@ __global__ __launch_bounds__(kThreadsPerBlock) // prefill, ragged kv
|
||||
static_assert(TopK / kThreadsPerBlock == 2);
|
||||
const auto idx_0 = tid;
|
||||
const auto pos_0 = s_indices[idx_0];
|
||||
dst_indices_entry[idx_0] = pos_0 + offset;
|
||||
dst_indices_entry[idx_0] = pos_0 < 0 ? -1 : pos_0 + offset;
|
||||
const auto idx_1 = tid + kThreadsPerBlock;
|
||||
const auto pos_1 = s_indices[idx_1];
|
||||
dst_indices_entry[idx_1] = pos_1 + offset;
|
||||
dst_indices_entry[idx_1] = pos_1 < 0 ? -1 : pos_1 + offset;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,239 @@
|
||||
#pragma once
|
||||
|
||||
#include <cuda_fp16.h>
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
|
||||
namespace sglang::device::legacy_radix_topk {
|
||||
|
||||
inline constexpr int kRadix = 256;
|
||||
|
||||
// Coarse bin: high byte of the ordered FP16 value, as in the original selectors.
|
||||
__device__ __forceinline__ uint8_t coarse_key(float x) {
|
||||
const __half h = __float2half_rn(x);
|
||||
const uint16_t bits = __half_as_ushort(h);
|
||||
const uint16_t key = (bits & 0x8000u) ? static_cast<uint16_t>(~bits) : static_cast<uint16_t>(bits | 0x8000u);
|
||||
return static_cast<uint8_t>(key >> 8);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ uint32_t exact_key(float x) {
|
||||
const uint32_t bits = __float_as_uint(x);
|
||||
return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u);
|
||||
}
|
||||
|
||||
// Exact radix top-k over finite scores. A threshold bin that fits the stash
|
||||
// keeps the original two-stash refinement; a larger one is refined by rescanning
|
||||
// the input. Unfilled output slots stay -1.
|
||||
template <int BlockSize, int OutputCapacity, int StashEntries>
|
||||
__device__ __forceinline__ void
|
||||
select(const float* __restrict__ input, int32_t* __restrict__ output, int row_start, int length, int topk) {
|
||||
static_assert(BlockSize >= kRadix + 1, "the block must initialize the complete radix histogram");
|
||||
static_assert(OutputCapacity <= StashEntries, "top-k output must fit in one shared-memory stash");
|
||||
|
||||
alignas(128) __shared__ int histogram_buf[2][kRadix + 1];
|
||||
alignas(128) __shared__ int counter;
|
||||
alignas(128) __shared__ int threshold_bin_id;
|
||||
alignas(128) __shared__ int num_input[2];
|
||||
|
||||
extern __shared__ int dynamic_smem[];
|
||||
auto* input_idx = reinterpret_cast<int (*)[StashEntries]>(dynamic_smem);
|
||||
auto& histogram = histogram_buf[0];
|
||||
const int tx = threadIdx.x;
|
||||
|
||||
for (int i = tx; i < topk; i += BlockSize)
|
||||
output[i] = -1;
|
||||
if (tx < kRadix + 1) histogram[tx] = 0;
|
||||
if (tx == 0) {
|
||||
counter = 0;
|
||||
threshold_bin_id = -1;
|
||||
num_input[0] = 0;
|
||||
num_input[1] = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
for (int idx = tx; idx < length; idx += BlockSize) {
|
||||
atomicAdd(&histogram[coarse_key(input[row_start + idx])], 1);
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const auto run_cumsum = [&] {
|
||||
#pragma unroll 8
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
if (tx < kRadix) {
|
||||
const int offset = 1 << i;
|
||||
const int src = i & 1;
|
||||
int value = histogram_buf[src][tx];
|
||||
if (tx < kRadix - offset) value += histogram_buf[src][tx + offset];
|
||||
histogram_buf[src ^ 1][tx] = value;
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
};
|
||||
|
||||
run_cumsum();
|
||||
if (tx < kRadix && histogram[tx] > topk && histogram[tx + 1] <= topk) {
|
||||
threshold_bin_id = tx;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const int coarse_threshold = threshold_bin_id;
|
||||
if (coarse_threshold < 0) return;
|
||||
const int coarse_above = histogram[coarse_threshold + 1];
|
||||
int remaining = topk - coarse_above;
|
||||
|
||||
if (remaining == 0) {
|
||||
for (int idx = tx; idx < length; idx += BlockSize) {
|
||||
if (coarse_key(input[row_start + idx]) > coarse_threshold) {
|
||||
const int pos = atomicAdd(&counter, 1);
|
||||
if (pos < topk) output[pos] = idx;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
return;
|
||||
}
|
||||
|
||||
const int coarse_population = histogram[coarse_threshold] - histogram[coarse_threshold + 1];
|
||||
__syncthreads();
|
||||
if (coarse_population >= StashEntries) {
|
||||
// Find the exact cutoff one key byte per pass. Pass L counts byte L and
|
||||
// emits what byte L-1 placed strictly above the cutoff; pass 0 emits the
|
||||
// values above the coarse bin and pass 4 the last strict values and ties.
|
||||
uint32_t prefix = 0;
|
||||
int last_threshold = -1;
|
||||
int strict_count = 0;
|
||||
#pragma unroll 1
|
||||
for (int level = 0; level <= 4; ++level) {
|
||||
const uint32_t mask = level == 0 ? 0u : (~0u << (32 - level * 8));
|
||||
const uint32_t last_mask = level <= 1 ? 0u : (~0u << (40 - level * 8));
|
||||
const int last_shift = 32 - level * 8;
|
||||
if (level < 4) {
|
||||
if (tx < kRadix + 1) histogram[tx] = 0;
|
||||
if (tx == 0) threshold_bin_id = -1;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
for (int idx = tx; idx < length; idx += BlockSize) {
|
||||
const float value = input[row_start + idx];
|
||||
const int bin = coarse_key(value);
|
||||
if (bin != coarse_threshold) {
|
||||
if (level == 0 && bin > coarse_threshold) {
|
||||
const int pos = atomicAdd(&counter, 1);
|
||||
if (pos < topk) output[pos] = idx;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const uint32_t key = exact_key(value);
|
||||
if (level > 0 && (key & last_mask) == (prefix & last_mask) &&
|
||||
static_cast<int>((key >> last_shift) & 0xFFu) > last_threshold) {
|
||||
const int pos = atomicAdd(&num_input[0], 1);
|
||||
if (pos < strict_count) output[coarse_above + pos] = idx;
|
||||
} else if ((key & mask) == prefix) {
|
||||
if (level < 4) {
|
||||
atomicAdd(&histogram[(key >> (24 - level * 8)) & 0xFFu], 1);
|
||||
} else {
|
||||
const int pos = atomicAdd(&num_input[1], 1);
|
||||
if (pos < remaining) output[coarse_above + strict_count + pos] = idx;
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
if (level == 4) return;
|
||||
|
||||
run_cumsum();
|
||||
if (tx < kRadix && histogram[tx] >= remaining && histogram[tx + 1] < remaining) {
|
||||
threshold_bin_id = tx;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const int threshold = threshold_bin_id;
|
||||
if (threshold < 0) return;
|
||||
const int above = histogram[threshold + 1];
|
||||
strict_count += above;
|
||||
remaining -= above;
|
||||
prefix |= static_cast<uint32_t>(threshold) << (24 - level * 8);
|
||||
last_threshold = threshold;
|
||||
// All threads must read the histogram before the next pass clears it.
|
||||
__syncthreads();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// The threshold bin fits the stash: original two-stash refinement.
|
||||
if (tx < kRadix + 1) histogram[tx] = 0;
|
||||
__syncthreads();
|
||||
for (int idx = tx; idx < length; idx += BlockSize) {
|
||||
const float value = input[row_start + idx];
|
||||
const int bin = coarse_key(value);
|
||||
if (bin > coarse_threshold) {
|
||||
const int pos = atomicAdd(&counter, 1);
|
||||
if (pos < topk) output[pos] = idx;
|
||||
} else if (bin == coarse_threshold) {
|
||||
const int pos = atomicAdd(&num_input[0], 1);
|
||||
if (pos < StashEntries) {
|
||||
input_idx[0][pos] = idx;
|
||||
atomicAdd(&histogram[(exact_key(value) >> 24) & 0xFFu], 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
#pragma unroll 4
|
||||
for (int round = 0; round < 4; ++round) {
|
||||
const int current = round & 1;
|
||||
const int next = current ^ 1;
|
||||
const int raw_num_input = num_input[current];
|
||||
const int current_count = raw_num_input < StashEntries ? raw_num_input : StashEntries;
|
||||
|
||||
if (tx == 0) {
|
||||
threshold_bin_id = -1;
|
||||
num_input[next] = 0;
|
||||
}
|
||||
run_cumsum();
|
||||
if (tx < kRadix && histogram[tx] > remaining && histogram[tx + 1] <= remaining) {
|
||||
threshold_bin_id = tx;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const int threshold = threshold_bin_id;
|
||||
if (threshold < 0) return;
|
||||
const int above = histogram[threshold + 1];
|
||||
const int shift = 24 - round * 8;
|
||||
|
||||
__syncthreads();
|
||||
if (round < 3 && tx < kRadix + 1) histogram[tx] = 0;
|
||||
__syncthreads();
|
||||
for (int i = tx; i < current_count; i += BlockSize) {
|
||||
const int idx = input_idx[current][i];
|
||||
const float value = input[row_start + idx];
|
||||
const int bin = static_cast<int>((exact_key(value) >> shift) & 0xFFu);
|
||||
if (bin > threshold) {
|
||||
const int pos = atomicAdd(&counter, 1);
|
||||
if (pos < topk) output[pos] = idx;
|
||||
} else if (bin == threshold && round < 3) {
|
||||
const int pos = atomicAdd(&num_input[next], 1);
|
||||
if (pos < StashEntries) {
|
||||
input_idx[next][pos] = idx;
|
||||
atomicAdd(&histogram[(exact_key(value) >> (shift - 8)) & 0xFFu], 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
remaining -= above;
|
||||
if (remaining == 0) return;
|
||||
if (round == 3) {
|
||||
for (int i = tx; i < current_count; i += BlockSize) {
|
||||
const int idx = input_idx[current][i];
|
||||
if (static_cast<int>(exact_key(input[row_start + idx]) & 0xFFu) == threshold) {
|
||||
const int pos = atomicAdd(&counter, 1);
|
||||
if (pos < topk) output[pos] = idx;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace sglang::device::legacy_radix_topk
|
||||
@@ -1,5 +1,5 @@
|
||||
import sys
|
||||
from typing import Any, Optional
|
||||
from typing import Optional
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -67,6 +67,13 @@ def _ref_torch_transform_ragged_impl(
|
||||
MAX_SEQ_LEN = 131072
|
||||
|
||||
|
||||
def _tight_cluster(batch: int, width: int) -> torch.Tensor:
|
||||
"""Finite scores that all land in one fp16 coarse radix bin."""
|
||||
base = torch.arange(width, dtype=torch.float32, device="cuda")
|
||||
values = 1.0 + base * (1e-3 / width)
|
||||
return values.expand(batch, -1).contiguous()
|
||||
|
||||
|
||||
def assert_equal(
|
||||
score: torch.Tensor,
|
||||
indices_ref: torch.Tensor,
|
||||
@@ -75,29 +82,39 @@ def assert_equal(
|
||||
k: int,
|
||||
seq_len: int,
|
||||
topk_indices_offset: Optional[torch.Tensor] = None,
|
||||
max_permit_error: int = 0,
|
||||
row_starts: Optional[torch.Tensor] = None,
|
||||
):
|
||||
indices_our_cpu = indices_our.cpu().tolist()
|
||||
indices_ref_cpu = indices_ref.cpu().tolist()
|
||||
|
||||
wrong_values = 0
|
||||
for i in range(bs):
|
||||
offset = topk_indices_offset[i].item() if topk_indices_offset is not None else 0
|
||||
row_start = row_starts[i].item() if row_starts is not None else 0
|
||||
indices_ref_set_i = set(indices_ref_cpu[i])
|
||||
indices_our_set_i = set(indices_our_cpu[i])
|
||||
assert len(indices_our_cpu[i]) == k
|
||||
assert len(indices_our_set_i) == k, "top-k output indices must be distinct"
|
||||
assert all(0 <= idx - offset < seq_len for idx in indices_our_cpu[i]), (
|
||||
"top-k output index is outside the selected window"
|
||||
)
|
||||
more = indices_our_set_i - indices_ref_set_i
|
||||
less = indices_ref_set_i - indices_our_set_i
|
||||
offset = topk_indices_offset[i].item() if topk_indices_offset is not None else 0
|
||||
if len(more) > 0 or len(less) > 0:
|
||||
# check whether more values are the same with less values
|
||||
# if so, either one is acceptable, since their values are the same
|
||||
more_values = sorted(score[i, idx - offset].item() for idx in more)
|
||||
less_values = sorted(score[i, idx - offset].item() for idx in less)
|
||||
more_values = sorted(
|
||||
score[i, row_start + idx - offset].item() for idx in more
|
||||
)
|
||||
less_values = sorted(
|
||||
score[i, row_start + idx - offset].item() for idx in less
|
||||
)
|
||||
if more_values != less_values:
|
||||
wrong_values += len(more)
|
||||
print(
|
||||
f"{bs=}, {k=}, {seq_len=}, {i=}, {more=}, {less=} failed, with {more_values=}, {less_values=}"
|
||||
)
|
||||
assert wrong_values <= max_permit_error, f"{wrong_values=}, {max_permit_error=}"
|
||||
assert wrong_values == 0, f"{wrong_values=}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bs", [1, 132, 256, 4096])
|
||||
@@ -125,8 +142,7 @@ def test_topk_kernel(bs: int, k: int, seq_len: int, has_row_starts: bool) -> Non
|
||||
indices_ref = torch.sort(indices_ref, dim=-1).values
|
||||
indices_our = torch.sort(indices_our, dim=-1).values
|
||||
|
||||
# Tests can pass with max_permit_error=3, set to 5 for safety
|
||||
assert_equal(score, indices_ref, indices_our, bs, k, seq_len, max_permit_error=5)
|
||||
assert_equal(score, indices_ref, indices_our, bs, k, seq_len, row_starts=row_starts)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bs", [1, 132, 256, 4096])
|
||||
@@ -189,7 +205,7 @@ def test_topk_transform_kernel(bs: int, k: int, seq_len: int, mode: str) -> None
|
||||
bs,
|
||||
k,
|
||||
seq_len,
|
||||
max_permit_error=5,
|
||||
row_starts=row_starts,
|
||||
)
|
||||
|
||||
|
||||
@@ -245,10 +261,71 @@ def test_topk_transform_ragged_kernel(
|
||||
k,
|
||||
seq_len,
|
||||
topk_indices_offset,
|
||||
max_permit_error=5,
|
||||
row_starts,
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_legacy_topk_oversized_coarse_bin() -> None:
|
||||
"""The shared legacy selector must not clip an oversized threshold bin."""
|
||||
torch.manual_seed(20260903)
|
||||
batch, seq_len, topk, row_start = 2, 7001, 2048, 3
|
||||
scores = torch.full(
|
||||
(batch, seq_len + row_start), -1000.0, dtype=torch.float32, device="cuda"
|
||||
)
|
||||
scores[:, row_start:] = _tight_cluster(batch, seq_len)
|
||||
lengths = torch.full((batch,), seq_len, dtype=torch.int32, device="cuda")
|
||||
row_starts = torch.full((batch,), row_start, dtype=torch.int32, device="cuda")
|
||||
reference = _ref_torch_impl(scores, seq_len, topk, row_starts=row_starts)
|
||||
|
||||
direct = fast_topk_v2(scores, lengths, topk, row_starts=row_starts)
|
||||
assert_equal(scores[:, row_start:], reference, direct, batch, topk, seq_len)
|
||||
|
||||
offsets = torch.tensor([17, 4099], dtype=torch.int32, device="cuda")
|
||||
ragged = fast_topk_transform_ragged_fused(
|
||||
score=scores,
|
||||
lengths=lengths,
|
||||
topk_indices_offset=offsets,
|
||||
topk=topk,
|
||||
row_starts=row_starts,
|
||||
)
|
||||
assert_equal(
|
||||
scores[:, row_start:],
|
||||
reference + offsets[:, None],
|
||||
ragged,
|
||||
batch,
|
||||
topk,
|
||||
seq_len,
|
||||
offsets,
|
||||
)
|
||||
|
||||
page_table = torch.arange(seq_len, dtype=torch.int32, device="cuda").expand(
|
||||
batch, -1
|
||||
)
|
||||
cu_seqlens_q = torch.arange(batch + 1, dtype=torch.int32, device="cuda")
|
||||
paged = fast_topk_transform_fused(
|
||||
score=scores,
|
||||
lengths=lengths,
|
||||
page_table_size_1=page_table,
|
||||
cu_seqlens_q=cu_seqlens_q,
|
||||
topk=topk,
|
||||
row_starts=row_starts,
|
||||
)
|
||||
assert_equal(scores[:, row_start:], reference, paged, batch, topk, seq_len)
|
||||
|
||||
# With one query per row and no row starts, this interface selects the
|
||||
# separate decode kernel rather than the prefill kernel above.
|
||||
decode_scores = scores[:, row_start:].contiguous()
|
||||
decode = fast_topk_transform_fused(
|
||||
score=decode_scores,
|
||||
lengths=lengths,
|
||||
page_table_size_1=page_table,
|
||||
cu_seqlens_q=cu_seqlens_q,
|
||||
topk=topk,
|
||||
)
|
||||
assert_equal(decode_scores, reference, decode, batch, topk, seq_len)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.version.hip is None,
|
||||
reason="deepseek_v4_topk_transform_512 is only built on ROCm",
|
||||
@@ -499,5 +576,59 @@ def test_topk_variants_are_exact_on_variable_windows(
|
||||
assert_exact_rows(score, mapped_raw, lengths, k)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.version.hip is None,
|
||||
reason="deepseek_v4_topk_transform_512 is only built on ROCm",
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_deepseek_v4_topk_oversized_coarse_bin_and_idle_row() -> None:
|
||||
"""Oversized bins select exactly; a negative DP-idle length emits padding."""
|
||||
from sgl_kernel import deepseek_v4_topk_transform_512
|
||||
|
||||
torch.manual_seed(20260903)
|
||||
active_batch, batch, c4_len, topk, page_size = 2, 3, 7001, 1024, 64
|
||||
scores = _tight_cluster(batch, c4_len)
|
||||
seq_lens = torch.tensor(
|
||||
[c4_len] * active_batch + [-1], dtype=torch.int32, device="cuda"
|
||||
)
|
||||
num_pages = (c4_len + page_size - 1) // page_size
|
||||
page_table = (
|
||||
torch.arange(num_pages, dtype=torch.int32, device="cuda")
|
||||
.expand(batch, -1)
|
||||
.contiguous()
|
||||
)
|
||||
page_indices = torch.full((batch, topk), -2, dtype=torch.int32, device="cuda")
|
||||
raw_indices = torch.full_like(page_indices, -2)
|
||||
|
||||
deepseek_v4_topk_transform_512(
|
||||
scores,
|
||||
seq_lens,
|
||||
page_table,
|
||||
page_indices,
|
||||
page_size,
|
||||
raw_indices,
|
||||
)
|
||||
|
||||
reference = torch.topk(scores[:active_batch], topk, dim=-1, sorted=False).indices
|
||||
assert_equal(
|
||||
scores[:active_batch],
|
||||
reference,
|
||||
raw_indices[:active_batch],
|
||||
active_batch,
|
||||
topk,
|
||||
c4_len,
|
||||
)
|
||||
assert_equal(
|
||||
scores[:active_batch],
|
||||
reference,
|
||||
page_indices[:active_batch],
|
||||
active_batch,
|
||||
topk,
|
||||
c4_len,
|
||||
)
|
||||
assert bool((raw_indices[-1] == -1).all())
|
||||
assert bool((page_indices[-1] == -1).all())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__]))
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
#include <sgl_kernel/runtime.cuh>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
|
||||
#include <sgl_kernel/dsa/legacy_radix_topk.cuh>
|
||||
|
||||
#include <dlpack/dlpack.h>
|
||||
#include <tvm/ffi/container/tensor.h>
|
||||
|
||||
@@ -39,18 +41,6 @@ struct TopKParams {
|
||||
uint32_t topk;
|
||||
};
|
||||
|
||||
SGL_DEVICE uint8_t convert_to_uint8(float x) {
|
||||
__half h = __float2half_rn(x);
|
||||
uint16_t bits = __half_as_ushort(h);
|
||||
uint16_t key = (bits & 0x8000) ? static_cast<uint16_t>(~bits) : static_cast<uint16_t>(bits | 0x8000);
|
||||
return static_cast<uint8_t>(key >> 8);
|
||||
}
|
||||
|
||||
SGL_DEVICE uint32_t convert_to_uint32(float x) {
|
||||
uint32_t bits = __float_as_uint(x);
|
||||
return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u);
|
||||
}
|
||||
|
||||
SGL_DEVICE int32_t page_to_indices(const int32_t* __restrict__ page_table, uint32_t i, uint32_t page_bits) {
|
||||
const uint32_t mask = (1u << page_bits) - 1u;
|
||||
return (page_table[i >> page_bits] << page_bits) | (i & mask);
|
||||
@@ -81,160 +71,9 @@ SGL_DEVICE void naive_transform(
|
||||
[[maybe_unused]]
|
||||
SGL_DEVICE void
|
||||
radix_topk(const float* __restrict__ input, int32_t* __restrict__ output, const uint32_t length, const uint32_t topk) {
|
||||
constexpr uint32_t RADIX = 256;
|
||||
constexpr uint32_t BLOCK_SIZE = kTopKBlockSize;
|
||||
constexpr uint32_t SMEM_INPUT_SIZE = kSMEM / (2 * sizeof(int32_t));
|
||||
|
||||
alignas(128) __shared__ uint32_t _s_histogram_buf[2][RADIX + 32];
|
||||
alignas(128) __shared__ uint32_t s_counter;
|
||||
alignas(128) __shared__ uint32_t s_threshold_bin_id;
|
||||
alignas(128) __shared__ uint32_t s_num_input[2];
|
||||
alignas(128) __shared__ int32_t s_last_remain;
|
||||
|
||||
extern __shared__ uint32_t s_input_idx[][kSMEM / (2 * sizeof(int32_t))];
|
||||
|
||||
const uint32_t tx = threadIdx.x;
|
||||
uint32_t remain_topk = topk;
|
||||
auto& s_histogram = _s_histogram_buf[0];
|
||||
|
||||
const auto run_cumsum = [&] {
|
||||
#pragma unroll 8
|
||||
for (int32_t i = 0; i < 8; ++i) {
|
||||
static_assert(1 << 8 == RADIX);
|
||||
if (tx < RADIX) {
|
||||
const auto j = 1 << i;
|
||||
const auto k = i & 1;
|
||||
auto value = _s_histogram_buf[k][tx];
|
||||
if (tx + j < RADIX) {
|
||||
value += _s_histogram_buf[k][tx + j];
|
||||
}
|
||||
_s_histogram_buf[k ^ 1][tx] = value;
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
};
|
||||
|
||||
// stage 1: 8bit coarse histogram
|
||||
if (tx < RADIX + 1) s_histogram[tx] = 0;
|
||||
__syncthreads();
|
||||
for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) {
|
||||
const auto bin = convert_to_uint8(input[idx]);
|
||||
::atomicAdd(&s_histogram[bin], 1);
|
||||
}
|
||||
__syncthreads();
|
||||
run_cumsum();
|
||||
if (tx < RADIX && s_histogram[tx] > remain_topk && s_histogram[tx + 1] <= remain_topk) {
|
||||
s_threshold_bin_id = tx;
|
||||
s_num_input[0] = 0;
|
||||
s_counter = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const auto threshold_bin = s_threshold_bin_id;
|
||||
remain_topk -= s_histogram[threshold_bin + 1];
|
||||
if (remain_topk == 0) {
|
||||
for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) {
|
||||
const uint32_t bin = convert_to_uint8(input[idx]);
|
||||
if (bin > threshold_bin) {
|
||||
const auto pos = ::atomicAdd(&s_counter, 1);
|
||||
output[pos] = idx;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
return;
|
||||
} else {
|
||||
__syncthreads();
|
||||
if (tx < RADIX + 1) {
|
||||
s_histogram[tx] = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) {
|
||||
const float raw_input = input[idx];
|
||||
const uint32_t bin = convert_to_uint8(raw_input);
|
||||
if (bin > threshold_bin) {
|
||||
const auto pos = ::atomicAdd(&s_counter, 1);
|
||||
output[pos] = idx;
|
||||
} else if (bin == threshold_bin) {
|
||||
const auto pos = ::atomicAdd(&s_num_input[0], 1);
|
||||
if (pos < SMEM_INPUT_SIZE) {
|
||||
[[likely]] s_input_idx[0][pos] = idx;
|
||||
const auto bin = convert_to_uint32(raw_input);
|
||||
const auto sub_bin = (bin >> 24) & 0xFF;
|
||||
::atomicAdd(&s_histogram[sub_bin], 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
// stage 2: refine with 8bit radix passes
|
||||
#pragma unroll 4
|
||||
for (int round = 0; round < 4; ++round) {
|
||||
const auto r_idx = round % 2;
|
||||
|
||||
// clip here to prevent overflow
|
||||
const auto raw_num_input = s_num_input[r_idx];
|
||||
const auto num_input = raw_num_input < SMEM_INPUT_SIZE ? raw_num_input : SMEM_INPUT_SIZE;
|
||||
|
||||
run_cumsum();
|
||||
if (tx < RADIX && s_histogram[tx] > remain_topk && s_histogram[tx + 1] <= remain_topk) {
|
||||
s_threshold_bin_id = tx;
|
||||
s_num_input[r_idx ^ 1] = 0;
|
||||
s_last_remain = remain_topk - s_histogram[tx + 1];
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const auto threshold_bin = s_threshold_bin_id;
|
||||
remain_topk -= s_histogram[threshold_bin + 1];
|
||||
|
||||
if (remain_topk == 0) {
|
||||
for (uint32_t i = tx; i < num_input; i += BLOCK_SIZE) {
|
||||
const auto idx = s_input_idx[r_idx][i];
|
||||
const auto offset = 24 - round * 8;
|
||||
const auto bin = (convert_to_uint32(input[idx]) >> offset) & 0xFF;
|
||||
if (bin > threshold_bin) {
|
||||
const auto pos = ::atomicAdd(&s_counter, 1);
|
||||
output[pos] = idx;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
break;
|
||||
} else {
|
||||
__syncthreads();
|
||||
if (tx < RADIX + 1) {
|
||||
s_histogram[tx] = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
for (uint32_t i = tx; i < num_input; i += BLOCK_SIZE) {
|
||||
const auto idx = s_input_idx[r_idx][i];
|
||||
const auto raw_input = input[idx];
|
||||
const auto offset = 24 - round * 8;
|
||||
const auto bin = (convert_to_uint32(raw_input) >> offset) & 0xFF;
|
||||
if (bin > threshold_bin) {
|
||||
const auto pos = ::atomicAdd(&s_counter, 1);
|
||||
output[pos] = idx;
|
||||
} else if (bin == threshold_bin) {
|
||||
if (round == 3) {
|
||||
const auto pos = ::atomicAdd(&s_last_remain, -1);
|
||||
if (pos > 0) {
|
||||
output[topk - pos] = idx;
|
||||
}
|
||||
} else {
|
||||
const auto pos = ::atomicAdd(&s_num_input[r_idx ^ 1], 1);
|
||||
if (pos < SMEM_INPUT_SIZE) {
|
||||
/// NOTE: (dark) fuse the histogram computation here
|
||||
[[likely]] s_input_idx[r_idx ^ 1][pos] = idx;
|
||||
const auto bin = convert_to_uint32(raw_input);
|
||||
const auto sub_bin = (bin >> (offset - 8)) & 0xFF;
|
||||
::atomicAdd(&s_histogram[sub_bin], 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
}
|
||||
::sglang::device::legacy_radix_topk::
|
||||
select<kTopKBlockSize, kMaxTopK, static_cast<int>(kSMEM / (2 * sizeof(int32_t)))>(
|
||||
input, output, 0, static_cast<int>(length), static_cast<int>(topk));
|
||||
}
|
||||
|
||||
template <bool kUsePDL>
|
||||
@@ -261,9 +100,10 @@ __global__ void topk_transform_kernel(const __grid_constant__ TopKParams params)
|
||||
radix_topk(score_ptr, s_topk_indices, seq_len, topk);
|
||||
const auto tx = threadIdx.x;
|
||||
if (tx < topk) {
|
||||
indices_ptr[tx] = page_to_indices(page_ptr, s_topk_indices[tx], page_bits);
|
||||
const auto raw = s_topk_indices[tx];
|
||||
indices_ptr[tx] = raw < 0 ? -1 : page_to_indices(page_ptr, raw, page_bits);
|
||||
if (raw_indices_ptr != nullptr) {
|
||||
raw_indices_ptr[tx] = s_topk_indices[tx];
|
||||
raw_indices_ptr[tx] = raw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
#pragma once
|
||||
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
namespace sglang::device::legacy_radix_topk {
|
||||
|
||||
inline constexpr int kRadix = 256;
|
||||
|
||||
// Coarse bin: high byte of the ordered FP16 value, as in the original selectors.
|
||||
SGL_DEVICE uint8_t coarse_key(float x) {
|
||||
const __half h = __float2half_rn(x);
|
||||
const uint16_t bits = __half_as_ushort(h);
|
||||
const uint16_t key = (bits & 0x8000u) ? static_cast<uint16_t>(~bits) : static_cast<uint16_t>(bits | 0x8000u);
|
||||
return static_cast<uint8_t>(key >> 8);
|
||||
}
|
||||
|
||||
SGL_DEVICE uint32_t exact_key(float x) {
|
||||
const uint32_t bits = __float_as_uint(x);
|
||||
return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u);
|
||||
}
|
||||
|
||||
// Exact radix top-k over finite scores. A threshold bin that fits the stash
|
||||
// keeps the original two-stash refinement; a larger one is refined by rescanning
|
||||
// the input. Unfilled output slots stay -1.
|
||||
template <int BlockSize, int OutputCapacity, int StashEntries>
|
||||
SGL_DEVICE void
|
||||
select(const float* __restrict__ input, int32_t* __restrict__ output, int row_start, int length, int topk) {
|
||||
static_assert(BlockSize >= kRadix + 1, "the block must initialize the complete radix histogram");
|
||||
static_assert(OutputCapacity <= StashEntries, "top-k output must fit in one shared-memory stash");
|
||||
|
||||
alignas(128) __shared__ int histogram_buf[2][kRadix + 1];
|
||||
alignas(128) __shared__ int counter;
|
||||
alignas(128) __shared__ int threshold_bin_id;
|
||||
alignas(128) __shared__ int num_input[2];
|
||||
|
||||
extern __shared__ int dynamic_smem[];
|
||||
auto* input_idx = reinterpret_cast<int (*)[StashEntries]>(dynamic_smem);
|
||||
auto& histogram = histogram_buf[0];
|
||||
const int tx = threadIdx.x;
|
||||
|
||||
for (int i = tx; i < topk; i += BlockSize)
|
||||
output[i] = -1;
|
||||
if (tx < kRadix + 1) histogram[tx] = 0;
|
||||
if (tx == 0) {
|
||||
counter = 0;
|
||||
threshold_bin_id = -1;
|
||||
num_input[0] = 0;
|
||||
num_input[1] = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
for (int idx = tx; idx < length; idx += BlockSize) {
|
||||
atomicAdd(&histogram[coarse_key(input[row_start + idx])], 1);
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const auto run_cumsum = [&] {
|
||||
#pragma unroll 8
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
if (tx < kRadix) {
|
||||
const int offset = 1 << i;
|
||||
const int src = i & 1;
|
||||
int value = histogram_buf[src][tx];
|
||||
if (tx < kRadix - offset) value += histogram_buf[src][tx + offset];
|
||||
histogram_buf[src ^ 1][tx] = value;
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
};
|
||||
|
||||
run_cumsum();
|
||||
if (tx < kRadix && histogram[tx] > topk && histogram[tx + 1] <= topk) {
|
||||
threshold_bin_id = tx;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const int coarse_threshold = threshold_bin_id;
|
||||
if (coarse_threshold < 0) return;
|
||||
const int coarse_above = histogram[coarse_threshold + 1];
|
||||
int remaining = topk - coarse_above;
|
||||
|
||||
if (remaining == 0) {
|
||||
for (int idx = tx; idx < length; idx += BlockSize) {
|
||||
if (coarse_key(input[row_start + idx]) > coarse_threshold) {
|
||||
const int pos = atomicAdd(&counter, 1);
|
||||
if (pos < topk) output[pos] = idx;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
return;
|
||||
}
|
||||
|
||||
const int coarse_population = histogram[coarse_threshold] - histogram[coarse_threshold + 1];
|
||||
__syncthreads();
|
||||
if (coarse_population >= StashEntries) {
|
||||
// Find the exact cutoff one key byte per pass. Pass L counts byte L and
|
||||
// emits what byte L-1 placed strictly above the cutoff; pass 0 emits the
|
||||
// values above the coarse bin and pass 4 the last strict values and ties.
|
||||
uint32_t prefix = 0;
|
||||
int last_threshold = -1;
|
||||
int strict_count = 0;
|
||||
#pragma unroll 1
|
||||
for (int level = 0; level <= 4; ++level) {
|
||||
const uint32_t mask = level == 0 ? 0u : (~0u << (32 - level * 8));
|
||||
const uint32_t last_mask = level <= 1 ? 0u : (~0u << (40 - level * 8));
|
||||
const int last_shift = 32 - level * 8;
|
||||
if (level < 4) {
|
||||
if (tx < kRadix + 1) histogram[tx] = 0;
|
||||
if (tx == 0) threshold_bin_id = -1;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
for (int idx = tx; idx < length; idx += BlockSize) {
|
||||
const float value = input[row_start + idx];
|
||||
const int bin = coarse_key(value);
|
||||
if (bin != coarse_threshold) {
|
||||
if (level == 0 && bin > coarse_threshold) {
|
||||
const int pos = atomicAdd(&counter, 1);
|
||||
if (pos < topk) output[pos] = idx;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const uint32_t key = exact_key(value);
|
||||
if (level > 0 && (key & last_mask) == (prefix & last_mask) &&
|
||||
static_cast<int>((key >> last_shift) & 0xFFu) > last_threshold) {
|
||||
const int pos = atomicAdd(&num_input[0], 1);
|
||||
if (pos < strict_count) output[coarse_above + pos] = idx;
|
||||
} else if ((key & mask) == prefix) {
|
||||
if (level < 4) {
|
||||
atomicAdd(&histogram[(key >> (24 - level * 8)) & 0xFFu], 1);
|
||||
} else {
|
||||
const int pos = atomicAdd(&num_input[1], 1);
|
||||
if (pos < remaining) output[coarse_above + strict_count + pos] = idx;
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
if (level == 4) return;
|
||||
|
||||
run_cumsum();
|
||||
if (tx < kRadix && histogram[tx] >= remaining && histogram[tx + 1] < remaining) {
|
||||
threshold_bin_id = tx;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const int threshold = threshold_bin_id;
|
||||
if (threshold < 0) return;
|
||||
const int above = histogram[threshold + 1];
|
||||
strict_count += above;
|
||||
remaining -= above;
|
||||
prefix |= static_cast<uint32_t>(threshold) << (24 - level * 8);
|
||||
last_threshold = threshold;
|
||||
// All threads must read the histogram before the next pass clears it.
|
||||
__syncthreads();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// The threshold bin fits the stash: original two-stash refinement.
|
||||
if (tx < kRadix + 1) histogram[tx] = 0;
|
||||
__syncthreads();
|
||||
for (int idx = tx; idx < length; idx += BlockSize) {
|
||||
const float value = input[row_start + idx];
|
||||
const int bin = coarse_key(value);
|
||||
if (bin > coarse_threshold) {
|
||||
const int pos = atomicAdd(&counter, 1);
|
||||
if (pos < topk) output[pos] = idx;
|
||||
} else if (bin == coarse_threshold) {
|
||||
const int pos = atomicAdd(&num_input[0], 1);
|
||||
if (pos < StashEntries) {
|
||||
input_idx[0][pos] = idx;
|
||||
atomicAdd(&histogram[(exact_key(value) >> 24) & 0xFFu], 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
#pragma unroll 4
|
||||
for (int round = 0; round < 4; ++round) {
|
||||
const int current = round & 1;
|
||||
const int next = current ^ 1;
|
||||
const int raw_num_input = num_input[current];
|
||||
const int current_count = raw_num_input < StashEntries ? raw_num_input : StashEntries;
|
||||
|
||||
if (tx == 0) {
|
||||
threshold_bin_id = -1;
|
||||
num_input[next] = 0;
|
||||
}
|
||||
run_cumsum();
|
||||
if (tx < kRadix && histogram[tx] > remaining && histogram[tx + 1] <= remaining) {
|
||||
threshold_bin_id = tx;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const int threshold = threshold_bin_id;
|
||||
if (threshold < 0) return;
|
||||
const int above = histogram[threshold + 1];
|
||||
const int shift = 24 - round * 8;
|
||||
|
||||
__syncthreads();
|
||||
if (round < 3 && tx < kRadix + 1) histogram[tx] = 0;
|
||||
__syncthreads();
|
||||
for (int i = tx; i < current_count; i += BlockSize) {
|
||||
const int idx = input_idx[current][i];
|
||||
const float value = input[row_start + idx];
|
||||
const int bin = static_cast<int>((exact_key(value) >> shift) & 0xFFu);
|
||||
if (bin > threshold) {
|
||||
const int pos = atomicAdd(&counter, 1);
|
||||
if (pos < topk) output[pos] = idx;
|
||||
} else if (bin == threshold && round < 3) {
|
||||
const int pos = atomicAdd(&num_input[next], 1);
|
||||
if (pos < StashEntries) {
|
||||
input_idx[next][pos] = idx;
|
||||
atomicAdd(&histogram[(exact_key(value) >> (shift - 8)) & 0xFFu], 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
remaining -= above;
|
||||
if (remaining == 0) return;
|
||||
if (round == 3) {
|
||||
for (int i = tx; i < current_count; i += BlockSize) {
|
||||
const int idx = input_idx[current][i];
|
||||
if (static_cast<int>(exact_key(input[row_start + idx]) & 0xFFu) == threshold) {
|
||||
const int pos = atomicAdd(&counter, 1);
|
||||
if (pos < topk) output[pos] = idx;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace sglang::device::legacy_radix_topk
|
||||
@@ -0,0 +1,365 @@
|
||||
"""GPU regressions for the DSA kpool pooled radix top-k transform.
|
||||
|
||||
The kernel (``kernels/jit/csrc/dsa/kpool_topk_transform.cuh``) selects the
|
||||
top ``group_topk`` pool groups of a score row with a two-stage radix select.
|
||||
Stage-1 threshold bins that fit a 4096-entry shared-memory stash use it;
|
||||
larger bins are resolved by an exact rescan. These tests
|
||||
build rows on both sides of that boundary (tight clusters, all-equal rows,
|
||||
clusters that only separate at a deeper key byte), the stage-1 direct-fill
|
||||
boundary, and the minimal ``length == K + 1`` row, and compare the selected
|
||||
groups against ``torch.topk``.
|
||||
|
||||
Comparison is by value multiset: when the K-th value is tied, any subset of
|
||||
the tied groups is a valid selection, so the sorted selected values must equal
|
||||
the sorted reference values while the group ids must be distinct and in
|
||||
range.
|
||||
|
||||
Scores within a row are finite by contract (the kernel's NaN ordering is
|
||||
unspecified), so no row here contains NaN; ``+inf`` is used only as padding
|
||||
beyond ``lengths``, where the kernel must not read.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.attention.dsa.kpool_topk_transform import (
|
||||
fast_kpool_topk_transform_fused,
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(est_time=180, stage="base-b-kernel-unit", runner_config="1-gpu-large")
|
||||
register_amd_ci(est_time=180, stage="jit-kernel-unit", runner_config="amd")
|
||||
|
||||
# kSmem / (2 * sizeof(int)) in kpool_topk_transform.cuh: entries per stash round.
|
||||
STASH_ENTRIES = 4096
|
||||
POOL_SIZE = 4
|
||||
# 512 is the instantiation used by GLM-5.3-Flash (token top-k 2048, pool 4);
|
||||
# 256 is the module default.
|
||||
GROUP_TOPKS = (512, 256)
|
||||
|
||||
|
||||
def _floats_from_keys(keys: torch.Tensor) -> torch.Tensor:
|
||||
"""Inverse of the kernel's monotone key for positive floats.
|
||||
|
||||
The kernel maps a float ``x`` to ``bits | 0x80000000`` when ``x >= +0``,
|
||||
so a key with the top bit set corresponds to the float whose raw bits are
|
||||
the low 31 bits of the key.
|
||||
"""
|
||||
assert bool(((keys >> 31) & 1).all()), "positive-float keys have bit 31 set"
|
||||
bits = (keys & 0x7FFFFFFF).to(torch.int32)
|
||||
return bits.view(torch.float32)
|
||||
|
||||
|
||||
def _coarse_keys(values: torch.Tensor) -> torch.Tensor:
|
||||
"""Mirror the kernel's high-byte ordered-FP16 key on the CPU."""
|
||||
bits = values.to(torch.float16).view(torch.int16).to(torch.int32) & 0xFFFF
|
||||
ordered = torch.where(
|
||||
(bits & 0x8000) != 0,
|
||||
(~bits) & 0xFFFF,
|
||||
bits | 0x8000,
|
||||
)
|
||||
return ordered >> 8
|
||||
|
||||
|
||||
@unittest.skipUnless(torch.cuda.is_available(), "Test requires CUDA")
|
||||
class TestDsaKpoolTopkTransform(CustomTestCase):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.device = torch.device("cuda")
|
||||
self.generator = torch.Generator(device="cpu").manual_seed(0)
|
||||
|
||||
def tearDown(self):
|
||||
torch.cuda.empty_cache()
|
||||
super().tearDown()
|
||||
|
||||
def _rand(self, *shape) -> torch.Tensor:
|
||||
return torch.rand(*shape, generator=self.generator, dtype=torch.float32)
|
||||
|
||||
def _tight_cluster(self, length: int, *, base: float = 1.0) -> torch.Tensor:
|
||||
# Strictly increasing (modulo FP32 rounding), with the best candidates
|
||||
# deliberately last. All values share the fp16 coarse bin, so an
|
||||
# arrival-order stash prefix is deterministically wrong.
|
||||
return base + torch.arange(length, dtype=torch.float32) * (1e-3 / length)
|
||||
|
||||
def _byte_depth_clusters(self, depth: int, per_side: int) -> torch.Tensor:
|
||||
"""Two clusters of ``per_side`` keys that first differ at key bit ``depth``.
|
||||
|
||||
Both share key byte 24 (0xBF, float 0.5..1.0), so stage 1 puts all
|
||||
``2 * per_side`` entries in one bin. The cluster with bit ``depth`` set
|
||||
holds every true top-K member; bits below ``depth`` are random.
|
||||
"""
|
||||
base = 0xBF000000
|
||||
noise_bits = depth
|
||||
noise = torch.randint(
|
||||
0, 1 << noise_bits, (2 * per_side,), generator=self.generator
|
||||
)
|
||||
keys = torch.full((2 * per_side,), base, dtype=torch.int64) + noise
|
||||
keys[:per_side] += 1 << depth
|
||||
perm = torch.randperm(2 * per_side, generator=self.generator)
|
||||
return _floats_from_keys(keys[perm])
|
||||
|
||||
def _run(
|
||||
self,
|
||||
rows: list,
|
||||
group_topk: int,
|
||||
page_table_bias: Optional[int] = None,
|
||||
topk_offset: Optional[int] = None,
|
||||
row_start: int = 0,
|
||||
):
|
||||
batch = len(rows)
|
||||
stride = row_start + max(row.numel() for row in rows)
|
||||
# Pad with +inf so any read past ``lengths`` would corrupt the selection.
|
||||
score = torch.full((batch, stride), float("inf"), dtype=torch.float32)
|
||||
lengths = torch.empty(batch, dtype=torch.int32)
|
||||
for i, row in enumerate(rows):
|
||||
score[i, row_start : row_start + row.numel()] = row
|
||||
lengths[i] = row.numel()
|
||||
score = score.to(self.device)
|
||||
lengths = lengths.to(self.device)
|
||||
row_starts = (
|
||||
torch.full((batch,), row_start, dtype=torch.int32, device=self.device)
|
||||
if row_start
|
||||
else None
|
||||
)
|
||||
|
||||
page_table = None
|
||||
topk_indices_offset = None
|
||||
if page_table_bias is not None:
|
||||
page_table = (
|
||||
torch.arange(stride * POOL_SIZE, dtype=torch.int32)
|
||||
.unsqueeze(0)
|
||||
.repeat(batch, 1)
|
||||
+ page_table_bias
|
||||
).to(self.device)
|
||||
if topk_offset is not None:
|
||||
topk_indices_offset = torch.full(
|
||||
(batch,), topk_offset, dtype=torch.int32, device=self.device
|
||||
)
|
||||
|
||||
out = fast_kpool_topk_transform_fused(
|
||||
score=score,
|
||||
lengths=lengths,
|
||||
kpool=POOL_SIZE,
|
||||
topk=group_topk * POOL_SIZE,
|
||||
page_table=page_table,
|
||||
topk_indices_offset=topk_indices_offset,
|
||||
row_starts=row_starts,
|
||||
)
|
||||
self.assertEqual(out.shape, (batch, group_topk * POOL_SIZE))
|
||||
return out.cpu()
|
||||
|
||||
def _selected_groups(self, out_row: torch.Tensor, group_topk: int, length: int):
|
||||
"""Decode expanded token columns back to group ids and check the layout."""
|
||||
self.assertTrue(
|
||||
bool((out_row >= 0).all()), "no pad columns expected when length > K"
|
||||
)
|
||||
tokens = out_row.view(group_topk, POOL_SIZE).to(torch.int64)
|
||||
group_ids = tokens[:, 0] // POOL_SIZE
|
||||
expected_tokens = group_ids.unsqueeze(1) * POOL_SIZE + torch.arange(POOL_SIZE)
|
||||
self.assertTrue(
|
||||
torch.equal(tokens, expected_tokens),
|
||||
"each selected group must expand to its pool_size consecutive tokens",
|
||||
)
|
||||
self.assertLess(int(group_ids.max()), length)
|
||||
self.assertEqual(
|
||||
torch.unique(group_ids).numel(), group_topk, "group ids must be distinct"
|
||||
)
|
||||
return group_ids
|
||||
|
||||
def _check_row(self, out_row: torch.Tensor, row: torch.Tensor, group_topk: int):
|
||||
group_ids = self._selected_groups(out_row, group_topk, row.numel())
|
||||
selected = row[group_ids].sort().values
|
||||
reference = torch.topk(row, group_topk).values.sort().values
|
||||
self.assertTrue(
|
||||
torch.equal(selected, reference),
|
||||
"selected values must match torch.topk as a multiset",
|
||||
)
|
||||
|
||||
def _check_rows(self, rows: list, group_topk: int):
|
||||
out = self._run(rows, group_topk)
|
||||
for i, row in enumerate(rows):
|
||||
with self.subTest(row=i, length=row.numel(), group_topk=group_topk):
|
||||
self._check_row(out[i], row, group_topk)
|
||||
|
||||
def test_stage1_bin_exceeds_stash(self):
|
||||
# 4097 and 17802 candidates in one stage-1 bin, both beyond the 4096-entry
|
||||
# stash. A clipped stash would keep the first 4096 by atomic arrival
|
||||
# order and disagree with torch.topk on such rows.
|
||||
for group_topk in GROUP_TOPKS:
|
||||
rows = [
|
||||
self._tight_cluster(STASH_ENTRIES + 1),
|
||||
self._tight_cluster(17802),
|
||||
]
|
||||
self._check_rows(rows, group_topk)
|
||||
|
||||
def test_oversized_bin_with_strictly_higher_values(self):
|
||||
# Exercise the output offset between values emitted above the coarse
|
||||
# threshold and values selected by the exact overflow rescan.
|
||||
for group_topk in GROUP_TOPKS:
|
||||
high_count = 37
|
||||
high = 2.0 + torch.arange(high_count, dtype=torch.float32) * 1e-3
|
||||
threshold = self._tight_cluster(STASH_ENTRIES + 5311)
|
||||
low = torch.zeros(257, dtype=torch.float32)
|
||||
self._check_rows([torch.cat([high, threshold, low])], group_topk)
|
||||
|
||||
def test_stride_larger_than_length(self):
|
||||
# Row length far below the score stride, as with a padded score buffer.
|
||||
for group_topk in GROUP_TOPKS:
|
||||
long_row = self._tight_cluster(17802)
|
||||
short_row = self._rand(group_topk + 1)
|
||||
out = self._run([long_row, short_row], group_topk)
|
||||
self._check_row(out[0], long_row, group_topk)
|
||||
self._check_row(out[1], short_row, group_topk)
|
||||
|
||||
def test_nonzero_row_start_with_oversized_bin(self):
|
||||
for group_topk in GROUP_TOPKS:
|
||||
row = self._tight_cluster(STASH_ENTRIES + 5311)
|
||||
out = self._run([row], group_topk, row_start=3)
|
||||
self._check_row(out[0], row, group_topk)
|
||||
|
||||
def test_all_equal_rows_longer_than_stash(self):
|
||||
# Every key is identical, so the final scan must clip the exact ties
|
||||
# without duplicating indices or leaving an output slot unfilled.
|
||||
for group_topk in GROUP_TOPKS:
|
||||
rows = [
|
||||
torch.full((STASH_ENTRIES + 1,), 0.75),
|
||||
torch.full((STASH_ENTRIES + 1,), 0.0),
|
||||
torch.full((STASH_ENTRIES + 1,), -0.0),
|
||||
]
|
||||
self._check_rows(rows, group_topk)
|
||||
|
||||
def test_clusters_separating_at_each_key_byte(self):
|
||||
# 4097 vs 4097 keys that first differ at key bit 16, 8 or 0: the
|
||||
# stage-1 bin overflows and the rescan must walk to the byte where the
|
||||
# clusters separate.
|
||||
for group_topk in GROUP_TOPKS:
|
||||
rows = [
|
||||
self._byte_depth_clusters(depth, STASH_ENTRIES + 1)
|
||||
for depth in (16, 8, 0)
|
||||
]
|
||||
self._check_rows(rows, group_topk)
|
||||
|
||||
def test_stage1_exact_fill_boundary(self):
|
||||
# Exactly K distinct values above a 65536-way tie: stage 1 finds the
|
||||
# threshold bin with nothing left to refine and fills all K slots directly.
|
||||
for group_topk in GROUP_TOPKS:
|
||||
top = torch.linspace(2.0, 3.0, group_topk)
|
||||
tie = torch.full((65536,), 1.0)
|
||||
row = torch.cat([top, tie])
|
||||
row = row[torch.randperm(row.numel(), generator=self.generator)]
|
||||
self._check_rows([row], group_topk)
|
||||
|
||||
def test_stash_capacity_boundary(self):
|
||||
# The exact-capacity case brackets the stash boundary. Different
|
||||
# input orders protect against an arrival-order subset being selected;
|
||||
# the mixed-bin row also covers the output offset after values above the
|
||||
# exact-capacity threshold bin have already been emitted.
|
||||
row = self._tight_cluster(STASH_ENTRIES)
|
||||
self.assertEqual(torch.unique(_coarse_keys(row)).numel(), 1)
|
||||
negative_row = self._tight_cluster(STASH_ENTRIES, base=-0.9)
|
||||
self.assertEqual(torch.unique(_coarse_keys(negative_row)).numel(), 1)
|
||||
high = 2.0 + torch.arange(37, dtype=torch.float32) * 1e-3
|
||||
low = torch.full((257,), -2.0)
|
||||
mixed_bin_row = torch.cat([high, row, low])
|
||||
threshold_key = _coarse_keys(row)[0]
|
||||
self.assertEqual(
|
||||
int((_coarse_keys(mixed_bin_row) == threshold_key).sum()),
|
||||
STASH_ENTRIES,
|
||||
)
|
||||
rows = [
|
||||
row,
|
||||
row.flip(0),
|
||||
row[torch.randperm(row.numel(), generator=self.generator)],
|
||||
negative_row,
|
||||
mixed_bin_row,
|
||||
]
|
||||
below_capacity = self._tight_cluster(STASH_ENTRIES - 1)
|
||||
self.assertEqual(torch.unique(_coarse_keys(below_capacity)).numel(), 1)
|
||||
for group_topk in GROUP_TOPKS:
|
||||
self._check_rows(rows, group_topk)
|
||||
self._check_rows([below_capacity], group_topk)
|
||||
|
||||
def test_minimal_radix_length(self):
|
||||
# length == K + 1 is the shortest row that takes the radix path.
|
||||
for group_topk in GROUP_TOPKS:
|
||||
self._check_rows([self._rand(group_topk + 1)], group_topk)
|
||||
|
||||
def test_random_rows_batch(self):
|
||||
for group_topk in GROUP_TOPKS:
|
||||
rows = [self._rand(length) for length in (3000, 20000)]
|
||||
self._check_rows(rows, group_topk)
|
||||
|
||||
def test_page_table_and_offset_expansion(self):
|
||||
group_topk = 512
|
||||
row = self._tight_cluster(STASH_ENTRIES + 5311)
|
||||
|
||||
bias = 1000
|
||||
out = self._run([row], group_topk, page_table_bias=bias)
|
||||
self._check_row(out[0] - bias, row, group_topk)
|
||||
|
||||
offset = 777
|
||||
out = self._run([row], group_topk, topk_offset=offset)
|
||||
self._check_row(out[0] - offset, row, group_topk)
|
||||
|
||||
def test_tail_append_with_page_table_row_remap(self):
|
||||
"""Tail layout and absolute page-table row selection work together."""
|
||||
group_topk = 512
|
||||
rows = [self._tight_cluster(STASH_ENTRIES + 5311), self._rand(500)]
|
||||
batch = len(rows)
|
||||
stride = max(row.numel() for row in rows)
|
||||
score = torch.full((batch, stride), float("inf"), dtype=torch.float32)
|
||||
lengths = torch.tensor([row.numel() for row in rows], dtype=torch.int32)
|
||||
for i, row in enumerate(rows):
|
||||
score[i, : row.numel()] = row
|
||||
|
||||
pool_size = POOL_SIZE
|
||||
topk = group_topk * pool_size
|
||||
max_tokens = stride * pool_size + pool_size - 1
|
||||
page_biases = torch.tensor([10000, 20000, 30000], dtype=torch.int32)
|
||||
page_table = torch.arange(max_tokens, dtype=torch.int32).expand(3, -1)
|
||||
page_table = page_table + page_biases[:, None]
|
||||
page_table_row_index = torch.tensor([2, 0], dtype=torch.int32)
|
||||
tail_counts = (3, 1)
|
||||
seq_lens = torch.tensor(
|
||||
[rows[i].numel() * pool_size + tail_counts[i] for i in range(batch)],
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
out = fast_kpool_topk_transform_fused(
|
||||
score=score.cuda(),
|
||||
lengths=lengths.cuda(),
|
||||
kpool=pool_size,
|
||||
topk=topk,
|
||||
page_table=page_table.cuda(),
|
||||
seq_lens=seq_lens.cuda(),
|
||||
page_table_row_index=page_table_row_index.cuda(),
|
||||
).cpu()
|
||||
|
||||
for i, row in enumerate(rows):
|
||||
bias = int(page_biases[page_table_row_index[i]])
|
||||
history_len = min(row.numel() * pool_size, topk)
|
||||
if row.numel() > group_topk:
|
||||
self._check_row(out[i, :history_len] - bias, row, group_topk)
|
||||
else:
|
||||
self.assertTrue(
|
||||
torch.equal(
|
||||
out[i, :history_len] - bias,
|
||||
torch.arange(history_len, dtype=torch.int32),
|
||||
)
|
||||
)
|
||||
|
||||
tail = out[i, history_len : history_len + tail_counts[i]] - bias
|
||||
expected_tail = torch.arange(
|
||||
row.numel() * pool_size,
|
||||
row.numel() * pool_size + tail_counts[i],
|
||||
dtype=torch.int32,
|
||||
)
|
||||
self.assertTrue(torch.equal(tail, expected_tail))
|
||||
self.assertTrue(bool((out[i, history_len + tail_counts[i] :] == -1).all()))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,197 @@
|
||||
"""Correctness tests for the DeepSeek-V4 JIT top-k transform v1.
|
||||
|
||||
``topk_transform_paged`` dispatches to ``deepseek_v4/topk_v1.cuh`` on CUDA. Its
|
||||
radix select first bins scores by the high byte of their ordered FP16 value and
|
||||
stashes the threshold bin in shared memory (8192 entries). These tests put the
|
||||
threshold bin on both sides of that capacity, with strictly higher values
|
||||
emitted before it, ties at the cutoff, and clusters that only separate in one
|
||||
of the lower three FP32 key bytes. They cover runtime ``topk`` values, the
|
||||
``seq_len <= topk`` path, score rows padded past ``seq_lens``, a noncontiguous
|
||||
score view, and both the page-table mapped and raw outputs.
|
||||
|
||||
Selections are compared with ``torch.topk`` as value multisets: any subset of
|
||||
values tied at the cutoff is a valid selection, but indices must be distinct
|
||||
and in range, and the mapped output must be the page-table transform of the
|
||||
raw output.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.attention.dsv4.topk import topk_transform_paged
|
||||
from sglang.srt.utils import is_hip
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=60, stage="base-b-kernel-unit", runner_config="1-gpu-large")
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not torch.cuda.is_available() or is_hip(),
|
||||
reason="topk_transform_paged uses the JIT v1 kernel on CUDA only",
|
||||
)
|
||||
|
||||
# kSMEM / (2 * sizeof(int32_t)) in topk_v1.cuh: entries per stash round.
|
||||
STASH_ENTRIES = 8192
|
||||
PAGE_SIZE = 64
|
||||
TOPKS = (512, 1024, 300)
|
||||
|
||||
|
||||
def _generator(seed: int = 0) -> torch.Generator:
|
||||
return torch.Generator(device="cpu").manual_seed(seed)
|
||||
|
||||
|
||||
def _coarse_keys(values: torch.Tensor) -> torch.Tensor:
|
||||
"""Mirror the kernel's high-byte ordered-FP16 key on the CPU."""
|
||||
bits = values.to(torch.float16).view(torch.int16).to(torch.int32) & 0xFFFF
|
||||
ordered = torch.where((bits & 0x8000) != 0, (~bits) & 0xFFFF, bits | 0x8000)
|
||||
return ordered >> 8
|
||||
|
||||
|
||||
def _tight_cluster(length: int, base: float = 1.0) -> torch.Tensor:
|
||||
# Increasing values in one coarse bin, best candidates last, so an
|
||||
# arrival-order stash prefix is wrong.
|
||||
return base + torch.arange(length, dtype=torch.float32) * (1e-3 / length)
|
||||
|
||||
|
||||
def _byte_depth_clusters(depth: int, per_side: int, gen) -> torch.Tensor:
|
||||
# Two clusters sharing the coarse bin whose keys first differ at bit
|
||||
# ``depth``; the cluster with that bit set holds the true top-k.
|
||||
keys = torch.full((2 * per_side,), 0x3F000000, dtype=torch.int64)
|
||||
keys += torch.randint(0, 1 << depth, (2 * per_side,), generator=gen)
|
||||
keys[:per_side] += 1 << depth
|
||||
keys = keys[torch.randperm(2 * per_side, generator=gen)]
|
||||
return keys.to(torch.int32).view(torch.float32)
|
||||
|
||||
|
||||
def _run_and_check(
|
||||
rows: list[torch.Tensor], topk: int, stride_pad: int = 0, view_pad: int = 0
|
||||
):
|
||||
gen = _generator(topk)
|
||||
batch = len(rows)
|
||||
stride = max(row.numel() for row in rows) + stride_pad
|
||||
# Pad with +inf so any read past ``seq_lens`` would corrupt the selection.
|
||||
# ``view_pad`` passes a column slice whose row stride exceeds its width.
|
||||
storage = torch.full(
|
||||
(batch, stride + view_pad), float("inf"), dtype=torch.float32, device="cuda"
|
||||
)
|
||||
scores = torch.full((batch, stride), float("inf"), dtype=torch.float32)
|
||||
for i, row in enumerate(rows):
|
||||
scores[i, : row.numel()] = row
|
||||
seq_lens = torch.tensor([row.numel() for row in rows], dtype=torch.int32)
|
||||
num_pages = (stride + PAGE_SIZE - 1) // PAGE_SIZE
|
||||
page_table = torch.stack(
|
||||
[torch.randperm(4 * num_pages, generator=gen)[:num_pages] for _ in range(batch)]
|
||||
).to(torch.int32)
|
||||
|
||||
out_page = torch.full((batch, topk), -7, dtype=torch.int32, device="cuda")
|
||||
out_raw = torch.full((batch, topk), -7, dtype=torch.int32, device="cuda")
|
||||
scores_view = storage[:, :stride]
|
||||
scores_view.copy_(scores)
|
||||
assert scores_view.is_contiguous() == (view_pad == 0)
|
||||
topk_transform_paged(
|
||||
scores_view,
|
||||
seq_lens.cuda(),
|
||||
page_table.cuda(),
|
||||
out_page,
|
||||
PAGE_SIZE,
|
||||
out_raw,
|
||||
)
|
||||
out_page = out_page.cpu().to(torch.int64)
|
||||
out_raw = out_raw.cpu().to(torch.int64)
|
||||
|
||||
for i, row in enumerate(rows):
|
||||
length = row.numel()
|
||||
raw = out_raw[i]
|
||||
if length <= topk:
|
||||
expected = torch.full((topk,), -1, dtype=torch.int64)
|
||||
expected[:length] = torch.arange(length)
|
||||
assert torch.equal(raw, expected), f"row {i}: trivial raw indices"
|
||||
else:
|
||||
assert bool((raw >= 0).all()) and bool((raw < length).all()), f"row {i}"
|
||||
assert torch.unique(raw).numel() == topk, f"row {i}: duplicate index"
|
||||
selected = row[raw].sort().values
|
||||
reference = torch.topk(row, topk).values.sort().values
|
||||
assert torch.equal(selected, reference), f"row {i}: wrong selection"
|
||||
valid = raw >= 0
|
||||
mapped = torch.full_like(raw, -1)
|
||||
mapped[valid] = (
|
||||
page_table[i, raw[valid] // PAGE_SIZE].to(torch.int64) * PAGE_SIZE
|
||||
+ raw[valid] % PAGE_SIZE
|
||||
)
|
||||
assert torch.equal(out_page[i], mapped), f"row {i}: page mapping"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("topk", TOPKS)
|
||||
def test_threshold_bin_around_stash_capacity(topk):
|
||||
gen = _generator(1)
|
||||
below = _tight_cluster(STASH_ENTRIES - 1)
|
||||
exact = _tight_cluster(STASH_ENTRIES)
|
||||
above = _tight_cluster(STASH_ENTRIES + 1)
|
||||
for row in (below, exact, above):
|
||||
assert torch.unique(_coarse_keys(row)).numel() == 1
|
||||
rows = [
|
||||
below,
|
||||
exact,
|
||||
exact.flip(0),
|
||||
exact[torch.randperm(exact.numel(), generator=gen)],
|
||||
_tight_cluster(STASH_ENTRIES, base=-0.9),
|
||||
above,
|
||||
_tight_cluster(3 * STASH_ENTRIES + 17),
|
||||
]
|
||||
_run_and_check(rows, topk)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("topk", TOPKS)
|
||||
def test_oversized_bin_after_strictly_higher_values(topk):
|
||||
gen = _generator(2)
|
||||
high = 2.0 + torch.arange(37, dtype=torch.float32) * 1e-3
|
||||
low = torch.full((257,), -2.0)
|
||||
row = torch.cat([high, _tight_cluster(STASH_ENTRIES + 5311), low])
|
||||
exact_bin_row = torch.cat([high, _tight_cluster(STASH_ENTRIES), low])
|
||||
rows = [row, row[torch.randperm(row.numel(), generator=gen)], exact_bin_row]
|
||||
_run_and_check(rows, topk)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("topk", TOPKS)
|
||||
def test_cutoff_ties(topk):
|
||||
gen = _generator(3)
|
||||
top = torch.linspace(2.0, 3.0, topk)
|
||||
tie_row = torch.cat([top, torch.full((65536,), 1.0)])
|
||||
rows = [
|
||||
torch.full((STASH_ENTRIES + 1,), 0.75),
|
||||
torch.full((STASH_ENTRIES + 1,), 0.0),
|
||||
torch.full((STASH_ENTRIES + 1,), -0.0),
|
||||
torch.full((topk + 1,), 0.5),
|
||||
tie_row[torch.randperm(tie_row.numel(), generator=gen)],
|
||||
torch.cat([top[: topk // 2], torch.full((2 * STASH_ENTRIES,), 1.0)]),
|
||||
]
|
||||
_run_and_check(rows, topk)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("topk", TOPKS)
|
||||
def test_clusters_separating_at_lower_key_bytes(topk):
|
||||
gen = _generator(4)
|
||||
rows = [_byte_depth_clusters(depth, STASH_ENTRIES + 1, gen) for depth in (16, 8, 0)]
|
||||
_run_and_check(rows, topk)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("topk", TOPKS)
|
||||
def test_short_and_random_rows_with_padded_and_strided_scores(topk):
|
||||
gen = _generator(5)
|
||||
rows = [
|
||||
torch.rand(1, generator=gen),
|
||||
torch.rand(topk - 1, generator=gen),
|
||||
torch.rand(topk, generator=gen),
|
||||
torch.rand(topk + 1, generator=gen),
|
||||
torch.rand(3000, generator=gen),
|
||||
torch.randn(40000, generator=gen),
|
||||
]
|
||||
_run_and_check(rows, topk, stride_pad=PAGE_SIZE + 3)
|
||||
_run_and_check(rows, topk, view_pad=PAGE_SIZE + 5)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__, "-v"]))
|
||||
Reference in New Issue
Block a user