[Perf] dsa: Route the k-pool indexer top-k through the top-k v2 kernel (#43775)

Co-authored-by: Mohammad Angkad <mohammad.angkad@radixark.ai>
This commit is contained in:
Mohammad Miadh Angkad
2026-10-19 09:49:39 +08:00
committed by GitHub
co-authored by Mohammad Angkad
parent df078cc729
commit 377cd867fd
12 changed files with 862 additions and 158 deletions
@@ -37,6 +37,7 @@ enum class TopKMode {
INDICES, ///< raw selected indices into `out`; `page_table` unused
PAGE_TABLE, ///< page-table-transformed indices into `out`
DUAL_OUTPUT, ///< page-table-transformed indices into `out` and raw indices into `raw_indices`
KPOOL,
};
using Register2 = impl::TopKRegister<2>; // <= 8192, register-resident, 1 read
@@ -85,11 +86,18 @@ struct PageTransform {
const int32_t* __restrict__ page_table;
uint32_t page_bits;
int32_t* __restrict__ raw_out; // the row's raw output, written in DUAL_OUTPUT only
int32_t offset = 0;
uint32_t pool_bits = 0;
uint32_t tail_count = 0;
uint32_t out_cols = 0;
SGL_DEVICE int32_t page_to_indices(uint32_t i) const {
const uint32_t mask = (1u << page_bits) - 1u;
return (page_table[i >> page_bits] << page_bits) | (i & mask);
}
SGL_DEVICE int32_t kpool_token(uint32_t i) const {
return page_table != nullptr ? page_to_indices(i) : static_cast<int32_t>(i) + offset;
}
};
struct TopKPagedParams {
@@ -105,6 +113,11 @@ struct TopKPagedParams {
uint32_t page_bits;
uint32_t static_cluster_floor; // only used in small batch variant
uint32_t batch_size;
int64_t out_stride;
const int32_t* __restrict__ token_seq_lens;
const int32_t* __restrict__ out_offsets;
uint32_t pool_bits;
uint32_t out_cols;
SGL_DEVICE const GlobalMetadata& global() const {
return *reinterpret_cast<const GlobalMetadata*>(metadata);
@@ -116,19 +129,28 @@ struct TopKPagedParams {
return metadata[1 + i];
}
SGL_DEVICE int32_t* get_output_ptr(uint32_t batch_id) const {
return page_indices + batch_id * static_cast<int64_t>(topk);
return page_indices + batch_id * out_stride;
}
template <TopKMode kMode>
SGL_DEVICE PageTransform get_transform(uint32_t batch_id) const {
return {
auto transform = PageTransform{
page_table == nullptr ? nullptr : page_table + batch_id * page_table_stride,
page_bits,
raw_indices == nullptr ? nullptr : raw_indices + batch_id * static_cast<int64_t>(topk)};
if constexpr (kMode == TopKMode::KPOOL) {
const uint32_t pool_mask = (1u << pool_bits) - 1u;
transform.offset = out_offsets == nullptr ? 0 : out_offsets[batch_id];
transform.pool_bits = pool_bits;
transform.tail_count =
token_seq_lens == nullptr ? 0u : static_cast<uint32_t>(token_seq_lens[batch_id]) & pool_mask;
transform.out_cols = out_cols;
}
return transform;
}
SGL_DEVICE TopKProblem problem(uint32_t batch_id, uint32_t seq_len) const {
const auto k = static_cast<int64_t>(topk);
return TopKProblem{
.in = scores + batch_id * score_stride,
.out = page_indices + batch_id * k,
.out = page_indices + batch_id * out_stride,
.topk = topk,
.seq_len = seq_len,
};
@@ -161,10 +183,34 @@ SGL_DEVICE void for_each_item(uint32_t topk, const F& f) {
}
}
template <typename F>
SGL_DEVICE void
kpool_expand(uint32_t num_pools, uint32_t topk, int32_t* out, const PageTransform& t, const F& selected) {
const auto pool_bits = t.pool_bits;
const uint32_t pool_mask = (1u << pool_bits) - 1u;
const uint32_t history = min(num_pools, topk) << pool_bits;
const uint32_t tail_end = history + t.tail_count;
const uint32_t tail_base = num_pools << pool_bits;
for (uint32_t col = threadIdx.x; col < t.out_cols; col += kBlockSize) {
int32_t value = -1;
if (col < history) {
const int32_t pool_id = selected(col >> pool_bits);
if (pool_id >= 0) value = t.kpool_token((static_cast<uint32_t>(pool_id) << pool_bits) | (col & pool_mask));
} else if (col < tail_end) {
value = t.kpool_token(tail_base + (col - history));
}
out[col] = value;
}
}
template <bool kPDL, TopKMode kMode>
SGL_DEVICE void trivial_transform(const TopKProblem& problem, const PageTransform& transform) {
device::PDLWaitPrimary<kPDL>();
device::PDLTriggerSecondary<kPDL>();
if constexpr (kMode == TopKMode::KPOOL) {
return kpool_expand(
problem.seq_len, problem.topk, problem.out, transform, [](uint32_t r) { return static_cast<int32_t>(r); });
}
for_each_item(problem.topk, [&](uint32_t tx, uint32_t) {
if constexpr (kMode == TopKMode::INDICES) {
problem.out[tx] = tx < problem.seq_len ? static_cast<int32_t>(tx) : -1;
@@ -181,6 +227,10 @@ template <TopKMode kMode>
SGL_DEVICE void paged_transform(const TopKProblem& problem, int32_t* out, const PageTransform& transform) {
static_assert(kMode != TopKMode::INDICES, "paged_transform requires page-table output");
static_assert(kMaxTopK % kBlockSize == 0);
if constexpr (kMode == TopKMode::KPOOL) {
const auto staged = problem.out;
return kpool_expand(problem.seq_len, problem.topk, out, transform, [staged](uint32_t r) { return staged[r]; });
}
constexpr uint32_t kNumElems = kMaxTopK / kBlockSize;
int32_t indices[kNumElems];
for_each_item(problem.topk, [&](uint32_t tx, uint32_t i) {
@@ -272,6 +322,109 @@ TOPK_KERNEL void topk_ragged_kernel(const __grid_constant__ TopKRaggedParams par
// PDL trigger secondary at the end the block typically has no use, so ignore it
}
struct TopKRaggedKpoolParams {
float* __restrict__ scores;
const int32_t* __restrict__ pool_lens;
const int32_t* __restrict__ row_starts;
const int32_t* __restrict__ token_seq_lens;
const int32_t* __restrict__ page_table;
const int32_t* __restrict__ page_table_row_index;
const int32_t* __restrict__ out_offsets;
int32_t* __restrict__ out;
int64_t score_stride;
int64_t page_table_stride;
int64_t out_stride;
uint32_t topk;
uint32_t page_bits;
uint32_t pool_bits;
uint32_t out_cols;
SGL_DEVICE PageTransform get_transform(uint32_t bx) const {
const auto table_row = page_table_row_index == nullptr ? bx : static_cast<uint32_t>(page_table_row_index[bx]);
const uint32_t pool_mask = (1u << pool_bits) - 1u;
auto transform = PageTransform{
page_table == nullptr ? nullptr : page_table + static_cast<int64_t>(table_row) * page_table_stride,
page_bits,
nullptr};
transform.offset = out_offsets == nullptr ? 0 : out_offsets[bx];
transform.pool_bits = pool_bits;
transform.tail_count = token_seq_lens == nullptr ? 0u : static_cast<uint32_t>(token_seq_lens[bx]) & pool_mask;
transform.out_cols = out_cols;
return transform;
}
};
using RaggedSmem = impl::MaxSmem<Register2::Smem, Register4::Smem, Streaming::Smem>;
// Writes `score` in place: the <= 3 columns ahead of an unaligned window are masked, as in topk_ragged_kernel.
template <bool kPDL, TopKMode kMode>
SGL_DEVICE void ragged_select_transform(
float* score,
const uint32_t row_start,
const uint32_t seq_len,
const uint32_t topk,
int32_t* out,
const PageTransform& transform,
int32_t* s_topk_indices,
RaggedSmem* smem) {
constexpr uint32_t kVecSize = impl::TopKStreaming::kVecSize;
auto problem = TopKProblem{
.in = score + row_start,
.out = out,
.topk = topk,
.seq_len = seq_len,
};
if (seq_len <= topk) {
return trivial_transform<kPDL, kMode>(problem, transform);
}
const auto rem = row_start % kVecSize;
if (rem != 0) {
// The mask has to land after the indexer has retired
// Otherwise it may be accidentally overwritten by DG upstream
device::PDLWaitPrimary<kPDL>();
static_assert(kVecSize <= kBlockSize, "not enough threads ");
if (const auto tx = threadIdx.x; tx < rem) {
score[row_start - rem + tx] = impl::padding_value();
}
}
using device::topk::broadcast;
problem.in -= rem;
problem.out = s_topk_indices; // write into stage buffer in smem first
problem.seq_len = seq_len + rem;
problem.bias = broadcast(-static_cast<int32_t>(rem));
problem.input_start = broadcast(rem);
if (problem.seq_len <= Register2::kMaxSeqLen) {
Register2::forward<kPDL>(problem, smem);
} else if (problem.seq_len <= Register4::kMaxSeqLen) {
Register4::forward<kPDL>(problem, smem);
} else {
Streaming::forward<kPDL>(problem, smem);
}
device::PDLTriggerSecondary<kPDL>();
__syncthreads();
problem.seq_len = seq_len;
paged_transform<kMode>(problem, out, transform);
}
template <bool kPDL>
TOPK_KERNEL void topk_ragged_kpool_kernel(const __grid_constant__ TopKRaggedKpoolParams params) {
device::enable_smem_spilling();
__shared__ RaggedSmem smem;
__shared__ int32_t s_topk_indices[kMaxTopK];
const auto bx = blockIdx.x;
ragged_select_transform<kPDL, TopKMode::KPOOL>(
params.scores + bx * params.score_stride,
params.row_starts == nullptr ? 0u : static_cast<uint32_t>(params.row_starts[bx]),
static_cast<uint32_t>(params.pool_lens[bx]),
params.topk,
params.out + bx * params.out_stride,
params.get_transform(bx),
s_topk_indices,
&smem);
}
#ifdef USE_ROCM
// Only the ROCm DSA prefill emits this layout today, so CUDA/XPU builds stay
// unchanged. Nothing below is AMD-specific; the guard can be dropped later.
@@ -315,57 +468,18 @@ struct TopKPackedParams {
template <bool kPDL>
TOPK_KERNEL void topk_packed_kernel(const __grid_constant__ TopKPackedParams params) {
device::enable_smem_spilling();
constexpr uint32_t kVecSize = impl::TopKStreaming::kVecSize;
__shared__ impl::MaxSmem<Register2::Smem, Register4::Smem, Streaming::Smem> smem;
__shared__ RaggedSmem smem;
__shared__ int32_t s_topk_indices[kMaxTopK];
const auto bx = blockIdx.x;
const auto seq_len = static_cast<uint32_t>(params.seq_lens[bx]);
const auto row_start = static_cast<uint32_t>(params.row_starts[bx]);
const auto topk = params.topk;
const auto transform = params.get_transform(bx);
const auto out = params.page_indices + bx * static_cast<int64_t>(topk);
const auto score = params.scores + bx * params.score_stride;
auto problem = TopKProblem{
.in = score + row_start,
.out = out,
.topk = topk,
.seq_len = seq_len,
};
if (seq_len <= topk) {
return trivial_transform<kPDL, TopKMode::PAGE_TABLE>(problem, transform);
}
// Round the window down to a `kVecSize` boundary and mask the <= 3 columns
// that pulls in, with the same bias / input_start as `topk_ragged_kernel`.
const auto rem = row_start % kVecSize;
if (rem != 0) {
// The mask has to land after the indexer has retired
// Otherwise it may be accidentally overwritten by DG upstream
device::PDLWaitPrimary<kPDL>();
static_assert(kVecSize <= kBlockSize, "not enough threads ");
if (const auto tx = threadIdx.x; tx < rem) {
score[row_start - rem + tx] = impl::padding_value();
}
}
using device::topk::broadcast;
problem.in -= rem;
problem.out = s_topk_indices; // write into stage buffer in smem first
problem.seq_len = seq_len + rem;
problem.bias = broadcast(-static_cast<int32_t>(rem));
problem.input_start = broadcast(rem);
if (problem.seq_len <= Register2::kMaxSeqLen) {
Register2::forward<kPDL>(problem, &smem);
} else if (problem.seq_len <= Register4::kMaxSeqLen) {
Register4::forward<kPDL>(problem, &smem);
} else {
Streaming::forward<kPDL>(problem, &smem);
}
device::PDLTriggerSecondary<kPDL>();
__syncthreads();
paged_transform<TopKMode::PAGE_TABLE>(problem, out, transform);
ragged_select_transform<kPDL, TopKMode::PAGE_TABLE>(
params.scores + bx * params.score_stride,
static_cast<uint32_t>(params.row_starts[bx]),
static_cast<uint32_t>(params.seq_lens[bx]),
params.topk,
params.page_indices + bx * static_cast<int64_t>(params.topk),
params.get_transform(bx),
s_topk_indices,
&smem);
}
#endif // USE_ROCM
@@ -395,7 +509,7 @@ TOPK_KERNEL void topk_main_kernel(const __grid_constant__ TopKPagedParams params
const auto bx = blockIdx.x;
auto problem = params.problem(bx);
if (problem.seq_len <= problem.topk) {
return trivial_transform<kPDLEarly, kMode>(problem, params.get_transform(bx));
return trivial_transform<kPDLEarly, kMode>(problem, params.template get_transform<kMode>(bx));
}
if constexpr (kNeedStaging) {
problem.out = s_topk_indices; // write into stage buffer in smem first
@@ -422,7 +536,14 @@ TOPK_KERNEL void topk_main_kernel(const __grid_constant__ TopKPagedParams params
device::PDLWaitPrimary<kPDLFinal>();
problem.out = params.get_output_ptr(bx); // in-place transform
device::PDLTriggerSecondary<kPDL>();
return paged_transform<kMode>(problem, problem.out, params.get_transform(bx));
if constexpr (kMode == TopKMode::KPOOL) {
for_each_item(problem.topk, [&](uint32_t tx, uint32_t) { s_topk_indices[tx] = problem.out[tx]; });
__syncthreads();
const auto out = problem.out;
problem.out = s_topk_indices;
return paged_transform<kMode>(problem, out, params.template get_transform<kMode>(bx));
}
return paged_transform<kMode>(problem, problem.out, params.template get_transform<kMode>(bx));
} else {
return device::PDLTriggerSecondary<kPDL>();
}
@@ -432,7 +553,7 @@ TOPK_KERNEL void topk_main_kernel(const __grid_constant__ TopKPagedParams params
device::PDLTriggerSecondary<kPDL>();
if constexpr (kNeedStaging) {
__syncthreads();
paged_transform<kMode>(problem, params.get_output_ptr(bx), params.get_transform(bx));
paged_transform<kMode>(problem, params.get_output_ptr(bx), params.template get_transform<kMode>(bx));
}
}
@@ -493,7 +614,7 @@ CLUSTER_TOPK_KERNEL void topk_small_batch_cluster_kernel(const __grid_constant__
const auto worker_rank = bx % kClusterSize;
if (problem.seq_len <= problem.topk) {
if (by != worker_rank) return;
return trivial_transform<kPDL, kMode>(problem, params.get_transform(bx));
return trivial_transform<kPDL, kMode>(problem, params.template get_transform<kMode>(bx));
}
if constexpr (kNeedStaging) {
@@ -517,7 +638,7 @@ CLUSTER_TOPK_KERNEL void topk_small_batch_cluster_kernel(const __grid_constant__
cluster.sync();
if (by != 0) return;
problem.out = s_topk_indices;
return paged_transform<kMode>(problem, params.get_output_ptr(bx), params.get_transform(bx));
return paged_transform<kMode>(problem, params.get_output_ptr(bx), params.template get_transform<kMode>(bx));
} else {
return device::PDLTriggerSecondary<kPDL>();
}
@@ -526,7 +647,7 @@ CLUSTER_TOPK_KERNEL void topk_small_batch_cluster_kernel(const __grid_constant__
device::PDLTriggerSecondary<kPDL>();
if constexpr (kNeedStaging) {
__syncthreads();
paged_transform<kMode>(problem, params.get_output_ptr(bx), params.get_transform(bx));
paged_transform<kMode>(problem, params.get_output_ptr(bx), params.template get_transform<kMode>(bx));
}
}
@@ -869,7 +990,7 @@ TOPK_KERNEL void topk_split_select(const __grid_constant__ TopKPagedParams param
if (rank != 0) return;
__shared__ int32_t s_topk_indices[kNeedStaging ? kMaxTopK : 1];
if (problem.seq_len <= problem.topk) {
return trivial_transform<false, kMode>(problem, params.get_transform(row));
return trivial_transform<false, kMode>(problem, params.template get_transform<kMode>(row));
}
if constexpr (kNeedStaging) problem.out = s_topk_indices;
if (problem.seq_len <= kReg4MaxSeqLen) {
@@ -879,7 +1000,7 @@ TOPK_KERNEL void topk_split_select(const __grid_constant__ TopKPagedParams param
}
if constexpr (kNeedStaging) {
__syncthreads();
paged_transform<kMode>(problem, params.get_output_ptr(row), params.get_transform(row));
paged_transform<kMode>(problem, params.get_output_ptr(row), params.template get_transform<kMode>(row));
}
return;
}
@@ -899,7 +1020,7 @@ TOPK_KERNEL void topk_split_select(const __grid_constant__ TopKPagedParams param
// problem.out is already the destination, and paged_transform reads every
// slot into registers before writing any, so transforming in place is safe.
__syncthreads();
paged_transform<kMode>(problem, problem.out, params.get_transform(row));
paged_transform<kMode>(problem, problem.out, params.template get_transform<kMode>(row));
}
}
@@ -1016,6 +1137,89 @@ struct TopKKernel {
#endif
}
static uint32_t get_static_cluster_floor(uint32_t batch_size) {
// NOTE: 15 is exactly 0.5 wave which saturate all cluster-8 SMs on Hopper/Blackwell
if constexpr (SGL_ARCH_BLACKWELL_OR_GREATER) {
return batch_size <= 15 ? 24576 : 30720;
} else if constexpr (SGL_ARCH_HOPPER_OR_GREATER) {
return batch_size <= 15 ? 32768 : 65536;
} else {
return UINT_MAX;
}
}
template <TopKMode kMode>
static void launch(const TopKPagedParams& params, const uint32_t max_seq_len, const DLDevice device) {
using namespace host;
const auto batch_size = params.batch_size;
#if SUPPORT_CLUSTER
const bool use_cluster = (max_seq_len > params.static_cluster_floor) && (batch_size <= kClusterMaxBatch);
if (use_cluster) {
if constexpr (kMaxCluster16BatchSize > 0) {
if (batch_size <= kMaxCluster16BatchSize) {
constexpr uint32_t kClusterSize = 16;
// Widths above 8 are non-portable; the launch is rejected without this.
const auto kernel = topk_small_batch_cluster_kernel<kUsePDL, kMode, kClusterSize, 1>;
const auto kernel_ptr = reinterpret_cast<const void*>(kernel);
runtime::init_per_device(device, [kernel_ptr] {
CHECK_CUDA(::cudaFuncSetAttribute(kernel_ptr, ::cudaFuncAttributeNonPortableClusterSizeAllowed, 1));
});
return LaunchKernel({batch_size, kClusterSize}, kBlockSize, device)
.config({.use_pdl = kUsePDL, .cluster_dim = dim3{1, kClusterSize}})
.launch(kernel, params);
}
}
if constexpr (kNumPersistentClusters > 0) {
if (batch_size <= kNumPersistentClusters) {
constexpr uint32_t kClusterSize = 8;
return LaunchKernel({batch_size, kClusterSize}, kBlockSize, device)
.config({.use_pdl = kUsePDL, .cluster_dim = dim3{1, kClusterSize}})
.launch(topk_small_batch_cluster_kernel<kUsePDL, kMode, kClusterSize, 2>, params);
} else {
constexpr uint32_t kClusterSize = 8;
const uint32_t num_clusters = std::min(batch_size, kNumPersistentClusters);
LaunchKernel({num_clusters, kClusterSize}, kBlockSize, device)
.config({.use_pdl = kUsePDL, .cluster_dim = dim3{1, kClusterSize}})
.launch(topk_persistent_cluster_kernel<kUsePDL, kClusterSize>, params);
LaunchKernel(batch_size, kBlockSize, device)
.config({.use_pdl = kUsePDL})
.launch(topk_main_kernel<kUsePDL, /*kLevel=*/3, kMode>, params);
return void();
}
}
}
#elif defined(USE_ROCM)
// Split dispatch. One block per row leaves a long row latency bound on one
// CU however idle the rest is; split_plan decides where a second launch pays.
// PDL stays off: the hist -> select kernel boundary is the barrier.
if constexpr (kMode != TopKMode::KPOOL) {
if (const auto [split, split_ws] = split_plan(batch_size, max_seq_len, device); split >= kSplitMin) {
LaunchKernel({batch_size, split}, kBlockSize, device)
.config({.use_pdl = false})
.launch(topk_split_hist, params, split_ws);
LaunchKernel({batch_size, split}, kBlockSize, device)
.config({.use_pdl = false})
.launch(topk_split_select<kMode>, params, split_ws);
return;
}
}
#endif
if (max_seq_len <= kReg2MaxSeqLen) {
LaunchKernel(batch_size, kBlockSize, device)
.config({.use_pdl = kUsePDL})
.launch(topk_main_kernel<kUsePDL, /*kLevel=*/0, kMode>, params);
} else if (max_seq_len <= kReg4MaxSeqLen) {
LaunchKernel(batch_size, kBlockSize, device)
.config({.use_pdl = kUsePDL})
.launch(topk_main_kernel<kUsePDL, /*kLevel=*/1, kMode>, params);
} else {
LaunchKernel(batch_size, kBlockSize, device)
.config({.use_pdl = kUsePDL})
.launch(topk_main_kernel<kUsePDL, /*kLevel=*/2, kMode>, params);
}
}
static void transform_paged(
const tvm::ffi::TensorView scores,
const tvm::ffi::TensorView seq_lens,
@@ -1085,17 +1289,6 @@ struct TopKKernel {
const auto max_seq_len = static_cast<uint32_t>(L.unwrap());
const auto device = device_.unwrap();
constexpr auto get_static_cluster_floor = [](uint32_t batch_size) -> uint32_t {
// NOTE: 15 is exactly 0.5 wave which saturate all cluster-8 SMs on Hopper/Blackwell
if constexpr (SGL_ARCH_BLACKWELL_OR_GREATER) {
return batch_size <= 15 ? 24576 : 30720;
} else if constexpr (SGL_ARCH_HOPPER_OR_GREATER) {
return batch_size <= 15 ? 32768 : 65536;
} else {
return UINT_MAX;
}
};
const auto params = TopKPagedParams{
.scores = static_cast<const float*>(scores.data_ptr()),
.seq_lens = static_cast<const int32_t*>(seq_lens.data_ptr()),
@@ -1111,6 +1304,11 @@ struct TopKKernel {
.static_cluster_floor = get_static_cluster_floor(batch_size),
// used for persistent cluster kernel and main kernel
.batch_size = batch_size,
.out_stride = static_cast<int64_t>(topk),
.token_seq_lens = nullptr,
.out_offsets = nullptr,
.pool_bits = 0,
.out_cols = topk,
};
const auto dispatch = [&]<typename F>(F&& f) {
@@ -1128,72 +1326,116 @@ struct TopKKernel {
Panic("Invalid mode, this path should be unreachable");
}
};
dispatch([&]<TopKMode kMode>() {
#if SUPPORT_CLUSTER
const bool use_cluster = (max_seq_len > params.static_cluster_floor) && (batch_size <= kClusterMaxBatch);
if (use_cluster) {
if constexpr (kMaxCluster16BatchSize > 0) {
if (batch_size <= kMaxCluster16BatchSize) {
constexpr uint32_t kClusterSize = 16;
// Widths above 8 are non-portable; the launch is rejected without this.
const auto kernel = topk_small_batch_cluster_kernel<kUsePDL, kMode, kClusterSize, 1>;
const auto kernel_ptr = reinterpret_cast<const void*>(kernel);
runtime::init_per_device(device, [kernel_ptr] {
CHECK_CUDA(::cudaFuncSetAttribute(kernel_ptr, ::cudaFuncAttributeNonPortableClusterSizeAllowed, 1));
});
return LaunchKernel({batch_size, kClusterSize}, kBlockSize, device)
.config({.use_pdl = kUsePDL, .cluster_dim = dim3{1, kClusterSize}})
.launch(kernel, params);
}
}
dispatch([&]<TopKMode kMode>() { launch<kMode>(params, max_seq_len, device); });
}
if constexpr (kNumPersistentClusters > 0) {
if (batch_size <= kNumPersistentClusters) {
constexpr uint32_t kClusterSize = 8;
return LaunchKernel({batch_size, kClusterSize}, kBlockSize, device)
.config({.use_pdl = kUsePDL, .cluster_dim = dim3{1, kClusterSize}})
.launch(topk_small_batch_cluster_kernel<kUsePDL, kMode, kClusterSize, 2>, params);
} else {
constexpr uint32_t kClusterSize = 8;
const uint32_t num_clusters = std::min(batch_size, kNumPersistentClusters);
LaunchKernel({num_clusters, kClusterSize}, kBlockSize, device)
.config({.use_pdl = kUsePDL, .cluster_dim = dim3{1, kClusterSize}})
.launch(topk_persistent_cluster_kernel<kUsePDL, kClusterSize>, params);
LaunchKernel(batch_size, kBlockSize, device)
.config({.use_pdl = kUsePDL})
.launch(topk_main_kernel<kUsePDL, /*kLevel=*/3, kMode>, params);
return void();
}
}
}
#elif defined(USE_ROCM)
// Split dispatch. One block per row leaves a long row latency bound on one
// CU however idle the rest is; split_plan decides where a second launch pays.
// PDL stays off: the hist -> select kernel boundary is the barrier.
if (const auto [split, split_ws] = split_plan(batch_size, max_seq_len, device); split >= kSplitMin) {
LaunchKernel({batch_size, split}, kBlockSize, device)
.config({.use_pdl = false})
.launch(topk_split_hist, params, split_ws);
LaunchKernel({batch_size, split}, kBlockSize, device)
.config({.use_pdl = false})
.launch(topk_split_select<kMode>, params, split_ws);
return;
}
#endif
if (max_seq_len <= kReg2MaxSeqLen) {
LaunchKernel(batch_size, kBlockSize, device)
.config({.use_pdl = kUsePDL})
.launch(topk_main_kernel<kUsePDL, /*kLevel=*/0, kMode>, params);
} else if (max_seq_len <= kReg4MaxSeqLen) {
LaunchKernel(batch_size, kBlockSize, device)
.config({.use_pdl = kUsePDL})
.launch(topk_main_kernel<kUsePDL, /*kLevel=*/1, kMode>, params);
} else {
LaunchKernel(batch_size, kBlockSize, device)
.config({.use_pdl = kUsePDL})
.launch(topk_main_kernel<kUsePDL, /*kLevel=*/2, kMode>, params);
}
});
/**
* \brief GLM k-pool variant of `transform_paged`: top-k over per-pool scores,
* fused with pool expansion, tail append and the output transform.
*
* Row `b` selects the `topk` best of `scores[b, :pool_lens[b]]`, where
* `topk = (out.size(1) - tail_cols) / pool_size` and `tail_cols` is
* `pool_size - 1` when `token_seq_lens` is given, else 0. Each selected pool `g`
* expands to tokens `g * pool_size + [0, pool_size)`; then the
* `token_seq_lens[b] % pool_size` tokens of the open tail pool follow. Tokens
* map through `page_table` (page size `page_size`) when given, else get
* `out_offsets[b]` added (0 when absent); the rest of the row is `-1`.
*
* \param scores 16-byte-aligned float32 rows; width and row stride must be multiples of 4.
* \param pool_lens NON-NEGATIVE; `metadata` must be `plan` over these values.
* \param metadata Read before the PDL wait, so it must be written before the kernel
* that produces `scores` is launched, not between that kernel and this one.
*/
static void transform_kpool(
const tvm::ffi::TensorView scores,
const tvm::ffi::TensorView pool_lens,
const tvm::ffi::Optional<tvm::ffi::TensorView> token_seq_lens,
const tvm::ffi::Optional<tvm::ffi::TensorView> page_table,
const tvm::ffi::Optional<tvm::ffi::TensorView> out_offsets,
const tvm::ffi::TensorView out,
const uint32_t page_size,
const uint32_t pool_size,
const tvm::ffi::TensorView metadata) {
using namespace host;
auto B = SymbolicSize{"batch_size"};
auto L = SymbolicSize{"max_seq_len"};
auto S = SymbolicSize{"score_stride"};
auto C = SymbolicSize{"out_cols"};
auto O = SymbolicSize{"out_stride"};
auto device_ = SymbolicDevice{};
device_.set_options<kDLGPU>();
TensorMatcher({B, L}) // score
.with_strides({S, 1})
.with_dtype<float>()
.with_device(device_)
.ensure_alignment(16)
.verify(scores);
TensorMatcher({B}) // pool_lens
.with_dtype<int32_t>()
.with_device(device_)
.verify(pool_lens);
TensorMatcher({B, C}) // out
.with_strides({O, 1})
.with_dtype<int32_t>()
.with_device(device_)
.verify(out);
TensorMatcher({-1, 2}) // metadata
.with_dtype<int32_t>()
.with_device(device_)
.verify(metadata);
const int32_t* token_seq_lens_ptr = nullptr;
if (token_seq_lens.has_value()) {
TensorMatcher({B}).with_dtype<int32_t>().with_device(device_).verify(token_seq_lens.value());
token_seq_lens_ptr = static_cast<const int32_t*>(token_seq_lens.value().data_ptr());
}
const int32_t* page_table_ptr = nullptr;
int64_t page_table_stride = 0;
if (page_table.has_value()) {
TensorMatcher({B, -1}).with_strides({-1, 1}).with_dtype<int32_t>().with_device(device_).verify(
page_table.value());
page_table_ptr = static_cast<const int32_t*>(page_table.value().data_ptr());
page_table_stride = page_table.value().stride(0);
}
const int32_t* out_offsets_ptr = nullptr;
if (out_offsets.has_value()) {
RuntimeCheck(!page_table.has_value(), "page_table and out_offsets are mutually exclusive");
TensorMatcher({B}).with_dtype<int32_t>().with_device(device_).verify(out_offsets.value());
out_offsets_ptr = static_cast<const int32_t*>(out_offsets.value().data_ptr());
}
RuntimeCheck(std::has_single_bit(page_size), "page_size must be power of 2");
RuntimeCheck(pool_size > 1 && std::has_single_bit(pool_size), "pool_size must be a power of 2 > 1");
RuntimeCheck(S.unwrap() % 4 == 0, "score_stride must be a multiple of 4 (16-byte vectorized load)");
CHECK_HOST(L.unwrap() % 4 == 0) << "score width must be a multiple of 4 (16-byte vectorized load)";
RuntimeCheck(metadata.size(0) == B.unwrap() + 1, "invalid metadata shape");
const auto out_cols = static_cast<uint32_t>(C.unwrap());
const auto tail_cols = token_seq_lens.has_value() ? pool_size - 1 : 0u;
RuntimeCheck(out_cols > tail_cols && (out_cols - tail_cols) % pool_size == 0, "invalid out width ", out_cols);
const auto topk = (out_cols - tail_cols) / pool_size;
RuntimeCheck(topk > 0 && topk <= kMaxTopK, "pool-level topk must be in (0, 2048]");
const auto batch_size = static_cast<uint32_t>(B.unwrap());
const auto params = TopKPagedParams{
.scores = static_cast<const float*>(scores.data_ptr()),
.seq_lens = static_cast<const int32_t*>(pool_lens.data_ptr()),
.page_table = page_table_ptr,
.page_indices = static_cast<int32_t*>(out.data_ptr()),
.raw_indices = nullptr,
.metadata = static_cast<const PlanItem*>(metadata.data_ptr()),
.score_stride = S.unwrap(),
.page_table_stride = page_table_stride,
.topk = topk,
.page_bits = static_cast<uint32_t>(std::countr_zero(page_size)),
.static_cluster_floor = get_static_cluster_floor(batch_size),
.batch_size = batch_size,
.out_stride = O.unwrap(),
.token_seq_lens = token_seq_lens_ptr,
.out_offsets = out_offsets_ptr,
.pool_bits = static_cast<uint32_t>(std::countr_zero(pool_size)),
.out_cols = out_cols,
};
launch<TopKMode::KPOOL>(params, static_cast<uint32_t>(L.unwrap()), device_.unwrap());
}
/**
@@ -1267,6 +1509,90 @@ struct TopKKernel {
.launch(topk_ragged_kernel<kUsePDL>, params);
}
/**
* \brief Ragged (prefill) variant of `transform_kpool`: per-row window, no plan.
*
* Row `b` selects among `scores[b, ks : ks + pool_lens[b]]` (`ks = row_starts[b]`,
* 0 when absent), with the same expansion, tail and output transform as
* `transform_kpool`. `page_table_row_index[b]` picks the page-table row when
* several score rows share one; without it `page_table` needs one row per score row.
*
* `scores` is written in place: the <= 3 columns ahead of an unaligned window
* are masked, as in `transform_ragged`. The buffer must have no other consumer.
*/
static void transform_kpool_ragged(
const tvm::ffi::TensorView scores,
const tvm::ffi::TensorView pool_lens,
const tvm::ffi::Optional<tvm::ffi::TensorView> row_starts,
const tvm::ffi::Optional<tvm::ffi::TensorView> token_seq_lens,
const tvm::ffi::Optional<tvm::ffi::TensorView> page_table,
const tvm::ffi::Optional<tvm::ffi::TensorView> page_table_row_index,
const tvm::ffi::Optional<tvm::ffi::TensorView> out_offsets,
const tvm::ffi::TensorView out,
const uint32_t page_size,
const uint32_t pool_size) {
using namespace host;
auto B = SymbolicSize{"batch_size"};
auto S = SymbolicSize{"score_stride"};
auto C = SymbolicSize{"out_cols"};
auto O = SymbolicSize{"out_stride"};
auto device_ = SymbolicDevice{};
device_.set_options<kDLGPU>();
TensorMatcher({B, -1}).with_strides({S, 1}).with_dtype<float>().with_device(device_).ensure_alignment(16).verify(
scores);
TensorMatcher({B}).with_dtype<int32_t>().with_device(device_).verify(pool_lens);
TensorMatcher({B, C}).with_strides({O, 1}).with_dtype<int32_t>().with_device(device_).verify(out);
const auto optional_rows = [&](const tvm::ffi::Optional<tvm::ffi::TensorView>& t) -> const int32_t* {
if (!t.has_value()) return nullptr;
TensorMatcher({B}).with_dtype<int32_t>().with_device(device_).verify(t.value());
return static_cast<const int32_t*>(t.value().data_ptr());
};
const int32_t* page_table_ptr = nullptr;
int64_t page_table_stride = 0;
if (page_table.has_value()) {
TensorMatcher({-1, -1}).with_strides({-1, 1}).with_dtype<int32_t>().with_device(device_).verify(
page_table.value());
if (!page_table_row_index.has_value()) {
RuntimeCheck(page_table.value().size(0) == B.unwrap(), "page_table needs one row per score row");
}
page_table_ptr = static_cast<const int32_t*>(page_table.value().data_ptr());
page_table_stride = page_table.value().stride(0);
}
RuntimeCheck(!page_table_row_index.has_value() || page_table.has_value(), "page_table_row_index needs page_table");
RuntimeCheck(!(page_table.has_value() && out_offsets.has_value()), "page_table and out_offsets are exclusive");
RuntimeCheck(std::has_single_bit(page_size), "page_size must be power of 2");
RuntimeCheck(pool_size > 1 && std::has_single_bit(pool_size), "pool_size must be a power of 2 > 1");
RuntimeCheck(S.unwrap() % 4 == 0, "score_stride must be a multiple of 4 (16-byte vectorized load)");
const auto out_cols = static_cast<uint32_t>(C.unwrap());
const auto tail_cols = token_seq_lens.has_value() ? pool_size - 1 : 0u;
RuntimeCheck(out_cols > tail_cols && (out_cols - tail_cols) % pool_size == 0, "invalid out width ", out_cols);
const auto topk = (out_cols - tail_cols) / pool_size;
RuntimeCheck(topk > 0 && topk <= kMaxTopK, "pool-level topk must be in (0, 2048]");
const auto params = TopKRaggedKpoolParams{
.scores = static_cast<float*>(scores.data_ptr()),
.pool_lens = static_cast<const int32_t*>(pool_lens.data_ptr()),
.row_starts = optional_rows(row_starts),
.token_seq_lens = optional_rows(token_seq_lens),
.page_table = page_table_ptr,
.page_table_row_index = optional_rows(page_table_row_index),
.out_offsets = optional_rows(out_offsets),
.out = static_cast<int32_t*>(out.data_ptr()),
.score_stride = S.unwrap(),
.page_table_stride = page_table_stride,
.out_stride = O.unwrap(),
.topk = topk,
.page_bits = static_cast<uint32_t>(std::countr_zero(page_size)),
.pool_bits = static_cast<uint32_t>(std::countr_zero(pool_size)),
.out_cols = out_cols,
};
LaunchKernel(static_cast<uint32_t>(B.unwrap()), kBlockSize, device_.unwrap())
.config({.use_pdl = kUsePDL})
.launch(topk_ragged_kpool_kernel<kUsePDL>, params);
}
#ifdef USE_ROCM // see the packed kernel above
/**
* \brief Packed (DSA extend prefill) variant of `transform_paged`: per-row
@@ -304,6 +304,22 @@ register_kernel(
capabilities=CUDA_ONLY,
)
)
register_kernel(
KernelSpec(
op="attention.topk_transform_kpool_v2",
backend=KernelBackend.JIT,
target="sglang.kernels.ops.attention.dsv4.topk:topk_transform_kpool_v2",
capabilities=frozenset({CapabilityRequirement.CUDA}),
)
)
register_kernel(
KernelSpec(
op="attention.topk_transform_kpool_ragged_v2",
backend=KernelBackend.JIT,
target="sglang.kernels.ops.attention.dsv4.topk:topk_transform_kpool_ragged_v2",
capabilities=frozenset({CapabilityRequirement.CUDA}),
)
)
register_kernel(
KernelSpec(
op="attention.fast_topk",
@@ -25,6 +25,8 @@ from .elementwise import (
from .fp8_wo_a import sglang_per_token_group_quant_fp8_dsv4_wo_a
from .topk import (
plan_topk_v2,
topk_transform_kpool_ragged_v2,
topk_transform_kpool_v2,
topk_transform_paged,
topk_transform_paged_v2,
topk_transform_ragged_v2,
@@ -50,6 +52,8 @@ __all__ = [
"get_paged_mqa_logits_metadata",
"triton_create_paged_compress_data",
"topk_transform_paged",
"topk_transform_kpool_v2",
"topk_transform_kpool_ragged_v2",
"topk_transform_paged_v2",
"topk_transform_ragged_v2",
"plan_topk_v2",
@@ -54,6 +54,8 @@ def _jit_topk_v2_module():
("topk_transform_paged", f"{kernel}::transform_paged"),
("topk_transform_ragged", f"{kernel}::transform_ragged"),
("topk_plan", f"{kernel}::plan"),
("topk_transform_kpool", f"{kernel}::transform_kpool"),
("topk_transform_kpool_ragged", f"{kernel}::transform_kpool_ragged"),
]
if is_hip_runtime():
# transform_packed only exists under USE_ROCM, see topk_v2.cuh
@@ -153,7 +155,12 @@ def topk_transform_paged_torch(
_PLAN_METADATA_INTS_PER_BATCH = 2
def plan_topk_v2(seq_lens: torch.Tensor, static_threshold: int = -1) -> torch.Tensor:
def plan_topk_v2(
seq_lens: torch.Tensor,
static_threshold: int = -1,
*,
out: Optional[torch.Tensor] = None,
) -> torch.Tensor:
"""
Preprocess the per-batch routing plan for :func:`topk_transform_paged_v2`.
NOTE: every entry of ``seq_lens`` must be NON-NEGATIVE.
@@ -164,7 +171,11 @@ def plan_topk_v2(seq_lens: torch.Tensor, static_threshold: int = -1) -> torch.Te
"""
module = _jit_topk_v2_module()
bs = seq_lens.shape[0]
metadata = seq_lens.new_empty(bs + 1, _PLAN_METADATA_INTS_PER_BATCH)
metadata = (
seq_lens.new_empty(bs + 1, _PLAN_METADATA_INTS_PER_BATCH)
if out is None
else out
)
module.topk_plan(seq_lens, metadata, static_threshold)
return metadata
@@ -281,6 +292,70 @@ def topk_transform_paged_v2(
)
def topk_transform_kpool_v2(
scores: torch.Tensor,
pool_lens: torch.Tensor,
out: torch.Tensor,
pool_size: int,
metadata: torch.Tensor,
*,
token_seq_lens: Optional[torch.Tensor] = None,
page_table: Optional[torch.Tensor] = None,
page_size: int = 1,
out_offsets: Optional[torch.Tensor] = None,
) -> None:
"""GLM k-pool top-k over per-pool scores, fused with pool expansion, the
``token_seq_lens % pool_size`` tail and the page-table or offset transform.
``metadata`` is :func:`plan_topk_v2` over ``pool_lens``. The kernel reads it
before its PDL wait, so plan before launching the kernel that writes ``scores``.
"""
_jit_topk_v2_module().topk_transform_kpool(
scores,
pool_lens,
token_seq_lens,
page_table,
out_offsets,
out,
page_size,
pool_size,
metadata,
)
def topk_transform_kpool_ragged_v2(
scores: torch.Tensor,
pool_lens: torch.Tensor,
out: torch.Tensor,
pool_size: int,
*,
row_starts: Optional[torch.Tensor] = None,
token_seq_lens: Optional[torch.Tensor] = None,
page_table: Optional[torch.Tensor] = None,
page_table_row_index: Optional[torch.Tensor] = None,
page_size: int = 1,
out_offsets: Optional[torch.Tensor] = None,
) -> None:
"""Ragged (prefill) :func:`topk_transform_kpool_v2`: row ``i`` selects among
``scores[i, ks : ks + pool_lens[i]]`` (``ks = row_starts[i]``, 0 when omitted);
no plan is needed.
NOTE: ``scores`` is written in place, as in :func:`topk_transform_ragged_v2`.
"""
_jit_topk_v2_module().topk_transform_kpool_ragged(
scores,
pool_lens,
row_starts,
token_seq_lens,
page_table,
page_table_row_index,
out_offsets,
out,
page_size,
pool_size,
)
def topk_transform_packed_v2(
scores: torch.Tensor,
seq_lens: torch.Tensor,
@@ -75,6 +75,14 @@ def _should_fuse_kpool_topk(metadata: BaseIndexerMetadata) -> bool:
)
def _topk_v2_plan_for(
*, owned_plan: Optional[torch.Tensor], pool_lens: torch.Tensor
) -> Optional[torch.Tensor]:
if owned_plan is not None and owned_plan.shape[0] == pool_lens.shape[0] + 1:
return owned_plan
return None
def _slice_rows(
*, tensor: Optional[torch.Tensor], rows: slice
) -> Optional[torch.Tensor]:
@@ -763,6 +771,7 @@ class IndexerKPool(MultiPlatformOp):
row_starts: Optional[torch.Tensor] = None,
out_rows: Optional[int] = None,
page_table_row_index: Optional[torch.Tensor] = None,
topk_v2_plan: Optional[torch.Tensor] = None,
) -> torch.Tensor:
from sglang.srt.layers.attention.dsa.kpool_fp8_index import (
topk_from_pooled_history_logits,
@@ -791,6 +800,7 @@ class IndexerKPool(MultiPlatformOp):
row_starts=row_starts,
out_rows=out_rows,
page_table_row_index=page_table_row_index,
topk_v2_plan=topk_v2_plan,
)
def _get_kpool_decode_metadata(
@@ -800,7 +810,13 @@ class IndexerKPool(MultiPlatformOp):
seqlens_32: torch.Tensor,
blocksize: int,
build_schedule_metadata: bool = True,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
) -> Tuple[
torch.Tensor,
torch.Tensor,
torch.Tensor,
Optional[torch.Tensor],
Optional[torch.Tensor],
]:
from sglang.srt.layers.attention.dsa.kpool_fp8_index import (
build_pooled_page_table_64,
)
@@ -828,9 +844,13 @@ class IndexerKPool(MultiPlatformOp):
pool_context_lens,
pool_block_tables,
pool_schedule_metadata,
_topk_v2_plan_for(
owned_plan=plan.pool_topk_v2_plan, pool_lens=pool_seqlens
),
)
pool_seqlens = attn_metadata.pooled_cache_seqlens_int32
topk_v2_plan = attn_metadata.pooled_topk_v2_plan
pool_block_tables = attn_metadata.pooled_real_page_table
pool_schedule_metadata = attn_metadata.pooled_paged_mqa_schedule_metadata
@@ -846,6 +866,7 @@ class IndexerKPool(MultiPlatformOp):
block_tables, self.index_kpool
).contiguous()
pool_schedule_metadata = None
topk_v2_plan = None
else:
pool_seqlens = pool_seqlens[: seqlens_32.shape[0]]
pool_block_tables = pool_block_tables[
@@ -864,6 +885,7 @@ class IndexerKPool(MultiPlatformOp):
pool_context_lens,
pool_block_tables,
pool_schedule_metadata,
_topk_v2_plan_for(owned_plan=topk_v2_plan, pool_lens=pool_seqlens),
)
@staticmethod
@@ -1003,16 +1025,18 @@ class IndexerKPool(MultiPlatformOp):
and self._should_use_tilelang_paged_mqa_logits(q_fp8)
)
pool_seqlens, pool_context_lens, pool_block_tables, pool_schedule_metadata = (
self._get_kpool_decode_metadata(
metadata,
block_tables,
seqlens_32,
blocksize,
build_schedule_metadata=not (
use_aiter_paged_mqa or use_tilelang_paged_mqa
),
)
(
pool_seqlens,
pool_context_lens,
pool_block_tables,
pool_schedule_metadata,
topk_v2_plan,
) = self._get_kpool_decode_metadata(
metadata=metadata,
block_tables=block_tables,
seqlens_32=seqlens_32,
blocksize=blocksize,
build_schedule_metadata=not (use_aiter_paged_mqa or use_tilelang_paged_mqa),
)
pool_max_seq_len = pool_block_tables.shape[1] * blocksize
if use_aiter_paged_mqa:
@@ -1065,8 +1089,9 @@ class IndexerKPool(MultiPlatformOp):
page_table_1, topk_offsets, _ = self._kpool_fused_topk_mapping(metadata)
topk_result = self._topk_from_kpool_logits(
logits,
pool_seqlens,
logits=logits,
pool_lens=pool_seqlens,
topk_v2_plan=topk_v2_plan,
seq_lens=seqlens_32,
page_table=page_table_1,
topk_offsets=topk_offsets,
@@ -214,6 +214,7 @@ class DSAMetadataManagementMixin:
dst.pooled_paged_mqa_schedule_metadata,
src.pooled_paged_mqa_schedule_metadata,
)
and _match(dst.pooled_topk_v2_plan, src.pooled_topk_v2_plan)
and _match(dst.kpool_write_plan, src.kpool_write_plan)
):
return False
@@ -223,6 +224,7 @@ class DSAMetadataManagementMixin:
and _match(dst_plan.seqlens_per_q, src_plan.seqlens_per_q)
and _match(dst_plan.pool_schedule_metadata, src_plan.pool_schedule_metadata)
and _match(dst_plan.effective_n_per_batch, src_plan.effective_n_per_batch)
and _match(dst_plan.pool_topk_v2_plan, src_plan.pool_topk_v2_plan)
):
return False
return True
@@ -298,6 +300,8 @@ class DSAMetadataManagementMixin:
metadata.pooled_paged_mqa_schedule_metadata.copy_(
src_metadata.pooled_paged_mqa_schedule_metadata
)
if metadata.pooled_topk_v2_plan is not None:
metadata.pooled_topk_v2_plan.copy_(src_metadata.pooled_topk_v2_plan)
dst_plan = metadata.kpool_write_plan
src_plan = src_metadata.kpool_write_plan
@@ -315,3 +319,5 @@ class DSAMetadataManagementMixin:
dst_plan.pool_schedule_metadata.copy_(src_plan.pool_schedule_metadata)
if dst_plan.effective_n_per_batch is not None:
dst_plan.effective_n_per_batch.copy_(src_plan.effective_n_per_batch)
if dst_plan.pool_topk_v2_plan is not None:
dst_plan.pool_topk_v2_plan.copy_(src_plan.pool_topk_v2_plan)
@@ -4,10 +4,14 @@ import torch
import triton
import triton.language as tl
from sglang.srt.environ import envs
from sglang.srt.layers.attention.dsa.utils import (
INDEXER_K_CACHE_PRESHUFFLE_TILE,
aiter_can_use_preshuffle_paged_mqa,
)
from sglang.srt.utils import is_cuda
_is_cuda = is_cuda()
BLOCK_SIZE_K = 64
INDEX_HEAD_DIM = 128
@@ -586,6 +590,177 @@ def _append_kpool_tail_to_topk_kernel(
tl.store(out_ptr + row * out_stride_0 + cols * out_stride_1, value, mask=mask)
def _kpool_topk_v2_enabled() -> bool:
return _is_cuda and envs.SGLANG_OPT_USE_TOPK_V2.get()
def build_kpool_topk_v2_plan(pool_lens: torch.Tensor) -> Optional[torch.Tensor]:
if not _kpool_topk_v2_enabled():
return None
plan = pool_lens.new_empty((pool_lens.shape[0] + 1, 2), dtype=torch.int32)
refresh_kpool_topk_v2_plan(plan=plan, pool_lens=pool_lens)
return plan
def refresh_kpool_topk_v2_plan(*, plan: torch.Tensor, pool_lens: torch.Tensor) -> None:
from sglang.kernels.ops.attention.dsv4.topk import plan_topk_v2
plan_topk_v2(pool_lens.to(torch.int32), out=plan)
def _kpool_topk_v2_layout_ok(*, logits: torch.Tensor, pool_size: int) -> bool:
return (
_kpool_topk_v2_enabled()
and logits.dtype == torch.float32
and logits.stride(1) == 1
and logits.stride(0) % 4 == 0
and logits.data_ptr() % 16 == 0
and pool_size > 1
and pool_size & (pool_size - 1) == 0
)
def _can_use_kpool_topk_v2(
*,
logits: torch.Tensor,
pool_size: int,
row_starts: torch.Tensor | None,
page_table_row_index: torch.Tensor | None,
) -> bool:
return (
row_starts is None
and page_table_row_index is None
and logits.shape[1] % 4 == 0
and _kpool_topk_v2_layout_ok(logits=logits, pool_size=pool_size)
)
def _alloc_kpool_topk_out(
*, logits: torch.Tensor, rows: int, out_cols: int, out_rows: int | None
) -> torch.Tensor:
if out_rows is None or out_rows == rows:
return logits.new_empty((rows, out_cols), dtype=torch.int32)
return logits.new_full((out_rows, out_cols), -1, dtype=torch.int32)
def _kpool_topk_v2_ragged(
*,
logits: torch.Tensor,
group_lengths: torch.Tensor,
pool_size: int,
topk: int,
page_table: torch.Tensor | None,
page_table_row_index: torch.Tensor | None,
topk_offsets: torch.Tensor | None,
seq_lens: torch.Tensor | None,
row_starts: torch.Tensor | None,
out_rows: int | None,
) -> torch.Tensor:
from sglang.kernels.ops.attention.dsv4.topk import topk_transform_kpool_ragged_v2
rows = logits.shape[0]
out_cols = topk + (pool_size - 1 if seq_lens is not None else 0)
full = _alloc_kpool_topk_out(
logits=logits, rows=rows, out_cols=out_cols, out_rows=out_rows
)
topk_transform_kpool_ragged_v2(
scores=logits,
pool_lens=group_lengths.to(torch.int32),
out=full[:rows],
pool_size=pool_size,
row_starts=None if row_starts is None else row_starts.to(torch.int32),
token_seq_lens=None if seq_lens is None else seq_lens.to(torch.int32),
page_table=page_table,
page_table_row_index=(
None
if page_table_row_index is None
else page_table_row_index.to(torch.int32)
),
out_offsets=topk_offsets,
)
return full
def _kpool_topk_v2(
*,
logits: torch.Tensor,
group_lengths: torch.Tensor,
pool_size: int,
topk: int,
page_table: torch.Tensor | None,
topk_offsets: torch.Tensor | None,
seq_lens: torch.Tensor | None,
out_rows: int | None,
topk_v2_plan: torch.Tensor,
) -> torch.Tensor:
from sglang.kernels.ops.attention.dsv4.topk import topk_transform_kpool_v2
rows = logits.shape[0]
group_lengths = group_lengths.to(torch.int32)
out_cols = topk + (pool_size - 1 if seq_lens is not None else 0)
full = _alloc_kpool_topk_out(
logits=logits, rows=rows, out_cols=out_cols, out_rows=out_rows
)
topk_transform_kpool_v2(
scores=logits,
pool_lens=group_lengths,
out=full[:rows],
pool_size=pool_size,
metadata=topk_v2_plan,
token_seq_lens=None if seq_lens is None else seq_lens.to(torch.int32),
page_table=page_table,
out_offsets=topk_offsets,
)
return full
def _try_kpool_topk_v2(
*,
logits: torch.Tensor,
group_lengths: torch.Tensor,
kpool: int,
topk: int,
page_table: torch.Tensor | None,
page_table_row_index: torch.Tensor | None,
topk_offsets: torch.Tensor | None,
seq_lens: torch.Tensor | None,
row_starts: torch.Tensor | None,
out_rows: int | None,
topk_v2_plan: torch.Tensor | None,
) -> torch.Tensor | None:
if topk_v2_plan is not None and _can_use_kpool_topk_v2(
logits=logits,
pool_size=kpool,
row_starts=row_starts,
page_table_row_index=page_table_row_index,
):
return _kpool_topk_v2(
logits=logits,
group_lengths=group_lengths,
pool_size=kpool,
topk=topk,
page_table=page_table,
topk_offsets=topk_offsets,
seq_lens=seq_lens,
out_rows=out_rows,
topk_v2_plan=topk_v2_plan,
)
if _kpool_topk_v2_layout_ok(logits=logits, pool_size=kpool):
return _kpool_topk_v2_ragged(
logits=logits,
group_lengths=group_lengths,
pool_size=kpool,
topk=topk,
page_table=page_table,
page_table_row_index=page_table_row_index,
topk_offsets=topk_offsets,
seq_lens=seq_lens,
row_starts=row_starts,
out_rows=out_rows,
)
return None
def topk_from_pooled_history_logits(
logits: torch.Tensor,
group_lengths: torch.Tensor,
@@ -597,6 +772,7 @@ def topk_from_pooled_history_logits(
row_starts: torch.Tensor | None = None,
out_rows: int | None = None,
page_table_row_index: torch.Tensor | None = None,
topk_v2_plan: torch.Tensor | None = None,
) -> torch.Tensor:
assert logits.ndim == 2
assert group_lengths.ndim == 1
@@ -606,7 +782,6 @@ def topk_from_pooled_history_logits(
assert out_rows is None or out_rows >= logits.shape[0]
assert page_table_row_index is None or page_table is not None
_, cols = logits.shape
group_topk = history_group_budget_for_topk(topk, kpool)
if topk_offsets is not None and topk_offsets.ndim == 2:
assert topk_offsets.shape[1] == 1
@@ -624,6 +799,22 @@ def topk_from_pooled_history_logits(
f"is disabled. Got device={logits.device}, dtype={logits.dtype}."
)
v2_result = _try_kpool_topk_v2(
logits=logits,
group_lengths=group_lengths,
kpool=kpool,
topk=topk,
page_table=page_table,
page_table_row_index=page_table_row_index,
topk_offsets=topk_offsets,
seq_lens=seq_lens,
row_starts=row_starts,
out_rows=out_rows,
topk_v2_plan=topk_v2_plan,
)
if v2_result is not None:
return v2_result
if group_topk in (128, 160, 192, 224, 256, 512):
from sglang.kernels.ops.attention.dsa.kpool_topk_transform import (
fast_kpool_topk_transform_fused,
@@ -648,6 +839,35 @@ def topk_from_pooled_history_logits(
padded[: result.shape[0]] = result
return padded
return _kpool_topk_unfused(
logits=logits,
group_lengths=group_lengths,
kpool=kpool,
topk=topk,
group_topk=group_topk,
page_table=page_table,
page_table_row_index=page_table_row_index,
topk_offsets=topk_offsets,
seq_lens=seq_lens,
row_starts=row_starts,
out_rows=out_rows,
)
def _kpool_topk_unfused(
*,
logits: torch.Tensor,
group_lengths: torch.Tensor,
kpool: int,
topk: int,
group_topk: int,
page_table: torch.Tensor | None,
page_table_row_index: torch.Tensor | None,
topk_offsets: torch.Tensor | None,
seq_lens: torch.Tensor | None,
row_starts: torch.Tensor | None,
out_rows: int | None,
) -> torch.Tensor:
assert page_table_row_index is None, (
"page_table_row_index requires the fused fast_kpool group_topk path"
)
@@ -662,7 +882,7 @@ def topk_from_pooled_history_logits(
)
rank = torch.arange(group_topk, device=logits.device, dtype=torch.int32)
max_valid_groups = min(cols, group_topk)
max_valid_groups = min(logits.shape[1], group_topk)
valid_counts = torch.minimum(
group_lengths.to(torch.int32),
torch.full_like(group_lengths.to(torch.int32), max_valid_groups),
@@ -9,9 +9,11 @@ import torch
from sglang.srt.environ import envs
from sglang.srt.layers.attention.dsa.kpool_fp8_index import (
INDEX_HEAD_DIM,
build_kpool_topk_v2_plan,
build_pooled_page_table_64,
kpool_build_ragged_layout,
kpool_max_closed_pools,
refresh_kpool_topk_v2_plan,
update_kpool_write_plan_cuda_graph,
)
from sglang.srt.layers.attention.dsa.utils import dsa_use_prefill_cp
@@ -115,6 +117,7 @@ class KPoolWritePlan:
seqlens_per_q: Optional[torch.Tensor] = None
pool_schedule_metadata: Optional[torch.Tensor] = None
effective_n_per_batch: Optional[torch.Tensor] = None
pool_topk_v2_plan: Optional[torch.Tensor] = None
def _is_kpool_layout_enabled(kpool: int, physical_page_size: int) -> bool:
@@ -614,6 +617,7 @@ def init_pooled_paged_mqa_metadata(
pooled_cache_seqlens_int32=pool_seqlens,
pooled_real_page_table=pooled_page_table,
pooled_paged_mqa_schedule_metadata=schedule,
pooled_topk_v2_plan=build_kpool_topk_v2_plan(pool_seqlens),
)
@@ -669,6 +673,12 @@ def update_pooled_paged_mqa_metadata(
if new_schedule is not None:
metadata.pooled_paged_mqa_schedule_metadata.copy_(new_schedule)
if metadata.pooled_topk_v2_plan is not None:
refresh_kpool_topk_v2_plan(
plan=metadata.pooled_topk_v2_plan,
pool_lens=metadata.pooled_cache_seqlens_int32,
)
def _alloc_kpool_write_plan_buffers(
*,
@@ -733,6 +743,10 @@ def init_kpool_write_plan_capture(
index_page_size=index_page_size,
)
plan = dataclasses.replace(plan, pool_schedule_metadata=schedule)
if is_verify:
plan = dataclasses.replace(
plan, pool_topk_v2_plan=build_kpool_topk_v2_plan(plan.pool_seqlens_per_q)
)
return dataclasses.replace(metadata, kpool_write_plan=plan)
@@ -793,6 +807,11 @@ def update_kpool_write_plan(
if new_schedule is not None:
plan.pool_schedule_metadata.copy_(new_schedule)
if plan.pool_topk_v2_plan is not None:
refresh_kpool_topk_v2_plan(
plan=plan.pool_topk_v2_plan, pool_lens=plan.pool_seqlens_per_q
)
def init_kpool_write_plan(
metadata: DSAMetadata,
@@ -306,6 +306,7 @@ class DSAMetadata:
pooled_cache_seqlens_int32: Optional[torch.Tensor] = None
pooled_real_page_table: Optional[torch.Tensor] = None
pooled_paged_mqa_schedule_metadata: Optional[torch.Tensor] = None
pooled_topk_v2_plan: Optional[torch.Tensor] = None
kpool_extend_plan: Optional[KPoolExtendPlan] = None
kpool_write_plan: Optional[KPoolWritePlan] = None
+13 -2
View File
@@ -96,13 +96,24 @@ def addresses(metadata):
}
def _topk_v2_plan_lengths(metadata, name):
if name == "topk_v2_plan":
return metadata.dsa_seqlens_expanded
if name == "pooled_topk_v2_plan":
return metadata.pooled_cache_seqlens_int32
if name == "kpool.pool_topk_v2_plan":
return metadata.kpool_write_plan.pool_seqlens_per_q
return None
def assert_metadata_equal(test, actual, expected):
actual_buffers, expected_buffers = tensor_buffers(actual), tensor_buffers(expected)
test.assertEqual(actual_buffers.keys(), expected_buffers.keys())
for name, value in actual_buffers.items():
reference = expected_buffers[name]
if name == "topk_v2_plan":
if not topk_v2_plan_is_written(expected.dsa_seqlens_expanded):
plan_lengths = _topk_v2_plan_lengths(expected, name)
if plan_lengths is not None:
if not topk_v2_plan_is_written(plan_lengths):
continue
# Unused plan rows are intentionally uninitialized. Active rows are
# compacted by atomicAdd, so compare them in request order.