[Perf] dsa: Route the k-pool indexer top-k through the top-k v2 kernel (#43775)
Co-authored-by: Mohammad Angkad <mohammad.angkad@radixark.ai>
This commit is contained in:
co-authored by
Mohammad Angkad
parent
df078cc729
commit
377cd867fd
@@ -37,6 +37,7 @@ enum class TopKMode {
|
||||
INDICES, ///< raw selected indices into `out`; `page_table` unused
|
||||
PAGE_TABLE, ///< page-table-transformed indices into `out`
|
||||
DUAL_OUTPUT, ///< page-table-transformed indices into `out` and raw indices into `raw_indices`
|
||||
KPOOL,
|
||||
};
|
||||
|
||||
using Register2 = impl::TopKRegister<2>; // <= 8192, register-resident, 1 read
|
||||
@@ -85,11 +86,18 @@ struct PageTransform {
|
||||
const int32_t* __restrict__ page_table;
|
||||
uint32_t page_bits;
|
||||
int32_t* __restrict__ raw_out; // the row's raw output, written in DUAL_OUTPUT only
|
||||
int32_t offset = 0;
|
||||
uint32_t pool_bits = 0;
|
||||
uint32_t tail_count = 0;
|
||||
uint32_t out_cols = 0;
|
||||
|
||||
SGL_DEVICE int32_t page_to_indices(uint32_t i) const {
|
||||
const uint32_t mask = (1u << page_bits) - 1u;
|
||||
return (page_table[i >> page_bits] << page_bits) | (i & mask);
|
||||
}
|
||||
SGL_DEVICE int32_t kpool_token(uint32_t i) const {
|
||||
return page_table != nullptr ? page_to_indices(i) : static_cast<int32_t>(i) + offset;
|
||||
}
|
||||
};
|
||||
|
||||
struct TopKPagedParams {
|
||||
@@ -105,6 +113,11 @@ struct TopKPagedParams {
|
||||
uint32_t page_bits;
|
||||
uint32_t static_cluster_floor; // only used in small batch variant
|
||||
uint32_t batch_size;
|
||||
int64_t out_stride;
|
||||
const int32_t* __restrict__ token_seq_lens;
|
||||
const int32_t* __restrict__ out_offsets;
|
||||
uint32_t pool_bits;
|
||||
uint32_t out_cols;
|
||||
|
||||
SGL_DEVICE const GlobalMetadata& global() const {
|
||||
return *reinterpret_cast<const GlobalMetadata*>(metadata);
|
||||
@@ -116,19 +129,28 @@ struct TopKPagedParams {
|
||||
return metadata[1 + i];
|
||||
}
|
||||
SGL_DEVICE int32_t* get_output_ptr(uint32_t batch_id) const {
|
||||
return page_indices + batch_id * static_cast<int64_t>(topk);
|
||||
return page_indices + batch_id * out_stride;
|
||||
}
|
||||
template <TopKMode kMode>
|
||||
SGL_DEVICE PageTransform get_transform(uint32_t batch_id) const {
|
||||
return {
|
||||
auto transform = PageTransform{
|
||||
page_table == nullptr ? nullptr : page_table + batch_id * page_table_stride,
|
||||
page_bits,
|
||||
raw_indices == nullptr ? nullptr : raw_indices + batch_id * static_cast<int64_t>(topk)};
|
||||
if constexpr (kMode == TopKMode::KPOOL) {
|
||||
const uint32_t pool_mask = (1u << pool_bits) - 1u;
|
||||
transform.offset = out_offsets == nullptr ? 0 : out_offsets[batch_id];
|
||||
transform.pool_bits = pool_bits;
|
||||
transform.tail_count =
|
||||
token_seq_lens == nullptr ? 0u : static_cast<uint32_t>(token_seq_lens[batch_id]) & pool_mask;
|
||||
transform.out_cols = out_cols;
|
||||
}
|
||||
return transform;
|
||||
}
|
||||
SGL_DEVICE TopKProblem problem(uint32_t batch_id, uint32_t seq_len) const {
|
||||
const auto k = static_cast<int64_t>(topk);
|
||||
return TopKProblem{
|
||||
.in = scores + batch_id * score_stride,
|
||||
.out = page_indices + batch_id * k,
|
||||
.out = page_indices + batch_id * out_stride,
|
||||
.topk = topk,
|
||||
.seq_len = seq_len,
|
||||
};
|
||||
@@ -161,10 +183,34 @@ SGL_DEVICE void for_each_item(uint32_t topk, const F& f) {
|
||||
}
|
||||
}
|
||||
|
||||
template <typename F>
|
||||
SGL_DEVICE void
|
||||
kpool_expand(uint32_t num_pools, uint32_t topk, int32_t* out, const PageTransform& t, const F& selected) {
|
||||
const auto pool_bits = t.pool_bits;
|
||||
const uint32_t pool_mask = (1u << pool_bits) - 1u;
|
||||
const uint32_t history = min(num_pools, topk) << pool_bits;
|
||||
const uint32_t tail_end = history + t.tail_count;
|
||||
const uint32_t tail_base = num_pools << pool_bits;
|
||||
for (uint32_t col = threadIdx.x; col < t.out_cols; col += kBlockSize) {
|
||||
int32_t value = -1;
|
||||
if (col < history) {
|
||||
const int32_t pool_id = selected(col >> pool_bits);
|
||||
if (pool_id >= 0) value = t.kpool_token((static_cast<uint32_t>(pool_id) << pool_bits) | (col & pool_mask));
|
||||
} else if (col < tail_end) {
|
||||
value = t.kpool_token(tail_base + (col - history));
|
||||
}
|
||||
out[col] = value;
|
||||
}
|
||||
}
|
||||
|
||||
template <bool kPDL, TopKMode kMode>
|
||||
SGL_DEVICE void trivial_transform(const TopKProblem& problem, const PageTransform& transform) {
|
||||
device::PDLWaitPrimary<kPDL>();
|
||||
device::PDLTriggerSecondary<kPDL>();
|
||||
if constexpr (kMode == TopKMode::KPOOL) {
|
||||
return kpool_expand(
|
||||
problem.seq_len, problem.topk, problem.out, transform, [](uint32_t r) { return static_cast<int32_t>(r); });
|
||||
}
|
||||
for_each_item(problem.topk, [&](uint32_t tx, uint32_t) {
|
||||
if constexpr (kMode == TopKMode::INDICES) {
|
||||
problem.out[tx] = tx < problem.seq_len ? static_cast<int32_t>(tx) : -1;
|
||||
@@ -181,6 +227,10 @@ template <TopKMode kMode>
|
||||
SGL_DEVICE void paged_transform(const TopKProblem& problem, int32_t* out, const PageTransform& transform) {
|
||||
static_assert(kMode != TopKMode::INDICES, "paged_transform requires page-table output");
|
||||
static_assert(kMaxTopK % kBlockSize == 0);
|
||||
if constexpr (kMode == TopKMode::KPOOL) {
|
||||
const auto staged = problem.out;
|
||||
return kpool_expand(problem.seq_len, problem.topk, out, transform, [staged](uint32_t r) { return staged[r]; });
|
||||
}
|
||||
constexpr uint32_t kNumElems = kMaxTopK / kBlockSize;
|
||||
int32_t indices[kNumElems];
|
||||
for_each_item(problem.topk, [&](uint32_t tx, uint32_t i) {
|
||||
@@ -272,6 +322,109 @@ TOPK_KERNEL void topk_ragged_kernel(const __grid_constant__ TopKRaggedParams par
|
||||
// PDL trigger secondary at the end the block typically has no use, so ignore it
|
||||
}
|
||||
|
||||
struct TopKRaggedKpoolParams {
|
||||
float* __restrict__ scores;
|
||||
const int32_t* __restrict__ pool_lens;
|
||||
const int32_t* __restrict__ row_starts;
|
||||
const int32_t* __restrict__ token_seq_lens;
|
||||
const int32_t* __restrict__ page_table;
|
||||
const int32_t* __restrict__ page_table_row_index;
|
||||
const int32_t* __restrict__ out_offsets;
|
||||
int32_t* __restrict__ out;
|
||||
int64_t score_stride;
|
||||
int64_t page_table_stride;
|
||||
int64_t out_stride;
|
||||
uint32_t topk;
|
||||
uint32_t page_bits;
|
||||
uint32_t pool_bits;
|
||||
uint32_t out_cols;
|
||||
|
||||
SGL_DEVICE PageTransform get_transform(uint32_t bx) const {
|
||||
const auto table_row = page_table_row_index == nullptr ? bx : static_cast<uint32_t>(page_table_row_index[bx]);
|
||||
const uint32_t pool_mask = (1u << pool_bits) - 1u;
|
||||
auto transform = PageTransform{
|
||||
page_table == nullptr ? nullptr : page_table + static_cast<int64_t>(table_row) * page_table_stride,
|
||||
page_bits,
|
||||
nullptr};
|
||||
transform.offset = out_offsets == nullptr ? 0 : out_offsets[bx];
|
||||
transform.pool_bits = pool_bits;
|
||||
transform.tail_count = token_seq_lens == nullptr ? 0u : static_cast<uint32_t>(token_seq_lens[bx]) & pool_mask;
|
||||
transform.out_cols = out_cols;
|
||||
return transform;
|
||||
}
|
||||
};
|
||||
|
||||
using RaggedSmem = impl::MaxSmem<Register2::Smem, Register4::Smem, Streaming::Smem>;
|
||||
|
||||
// Writes `score` in place: the <= 3 columns ahead of an unaligned window are masked, as in topk_ragged_kernel.
|
||||
template <bool kPDL, TopKMode kMode>
|
||||
SGL_DEVICE void ragged_select_transform(
|
||||
float* score,
|
||||
const uint32_t row_start,
|
||||
const uint32_t seq_len,
|
||||
const uint32_t topk,
|
||||
int32_t* out,
|
||||
const PageTransform& transform,
|
||||
int32_t* s_topk_indices,
|
||||
RaggedSmem* smem) {
|
||||
constexpr uint32_t kVecSize = impl::TopKStreaming::kVecSize;
|
||||
auto problem = TopKProblem{
|
||||
.in = score + row_start,
|
||||
.out = out,
|
||||
.topk = topk,
|
||||
.seq_len = seq_len,
|
||||
};
|
||||
if (seq_len <= topk) {
|
||||
return trivial_transform<kPDL, kMode>(problem, transform);
|
||||
}
|
||||
|
||||
const auto rem = row_start % kVecSize;
|
||||
if (rem != 0) {
|
||||
// The mask has to land after the indexer has retired
|
||||
// Otherwise it may be accidentally overwritten by DG upstream
|
||||
device::PDLWaitPrimary<kPDL>();
|
||||
static_assert(kVecSize <= kBlockSize, "not enough threads ");
|
||||
if (const auto tx = threadIdx.x; tx < rem) {
|
||||
score[row_start - rem + tx] = impl::padding_value();
|
||||
}
|
||||
}
|
||||
using device::topk::broadcast;
|
||||
problem.in -= rem;
|
||||
problem.out = s_topk_indices; // write into stage buffer in smem first
|
||||
problem.seq_len = seq_len + rem;
|
||||
problem.bias = broadcast(-static_cast<int32_t>(rem));
|
||||
problem.input_start = broadcast(rem);
|
||||
|
||||
if (problem.seq_len <= Register2::kMaxSeqLen) {
|
||||
Register2::forward<kPDL>(problem, smem);
|
||||
} else if (problem.seq_len <= Register4::kMaxSeqLen) {
|
||||
Register4::forward<kPDL>(problem, smem);
|
||||
} else {
|
||||
Streaming::forward<kPDL>(problem, smem);
|
||||
}
|
||||
device::PDLTriggerSecondary<kPDL>();
|
||||
__syncthreads();
|
||||
problem.seq_len = seq_len;
|
||||
paged_transform<kMode>(problem, out, transform);
|
||||
}
|
||||
|
||||
template <bool kPDL>
|
||||
TOPK_KERNEL void topk_ragged_kpool_kernel(const __grid_constant__ TopKRaggedKpoolParams params) {
|
||||
device::enable_smem_spilling();
|
||||
__shared__ RaggedSmem smem;
|
||||
__shared__ int32_t s_topk_indices[kMaxTopK];
|
||||
const auto bx = blockIdx.x;
|
||||
ragged_select_transform<kPDL, TopKMode::KPOOL>(
|
||||
params.scores + bx * params.score_stride,
|
||||
params.row_starts == nullptr ? 0u : static_cast<uint32_t>(params.row_starts[bx]),
|
||||
static_cast<uint32_t>(params.pool_lens[bx]),
|
||||
params.topk,
|
||||
params.out + bx * params.out_stride,
|
||||
params.get_transform(bx),
|
||||
s_topk_indices,
|
||||
&smem);
|
||||
}
|
||||
|
||||
#ifdef USE_ROCM
|
||||
// Only the ROCm DSA prefill emits this layout today, so CUDA/XPU builds stay
|
||||
// unchanged. Nothing below is AMD-specific; the guard can be dropped later.
|
||||
@@ -315,57 +468,18 @@ struct TopKPackedParams {
|
||||
template <bool kPDL>
|
||||
TOPK_KERNEL void topk_packed_kernel(const __grid_constant__ TopKPackedParams params) {
|
||||
device::enable_smem_spilling();
|
||||
constexpr uint32_t kVecSize = impl::TopKStreaming::kVecSize;
|
||||
__shared__ impl::MaxSmem<Register2::Smem, Register4::Smem, Streaming::Smem> smem;
|
||||
__shared__ RaggedSmem smem;
|
||||
__shared__ int32_t s_topk_indices[kMaxTopK];
|
||||
|
||||
const auto bx = blockIdx.x;
|
||||
const auto seq_len = static_cast<uint32_t>(params.seq_lens[bx]);
|
||||
const auto row_start = static_cast<uint32_t>(params.row_starts[bx]);
|
||||
const auto topk = params.topk;
|
||||
const auto transform = params.get_transform(bx);
|
||||
const auto out = params.page_indices + bx * static_cast<int64_t>(topk);
|
||||
const auto score = params.scores + bx * params.score_stride;
|
||||
|
||||
auto problem = TopKProblem{
|
||||
.in = score + row_start,
|
||||
.out = out,
|
||||
.topk = topk,
|
||||
.seq_len = seq_len,
|
||||
};
|
||||
if (seq_len <= topk) {
|
||||
return trivial_transform<kPDL, TopKMode::PAGE_TABLE>(problem, transform);
|
||||
}
|
||||
|
||||
// Round the window down to a `kVecSize` boundary and mask the <= 3 columns
|
||||
// that pulls in, with the same bias / input_start as `topk_ragged_kernel`.
|
||||
const auto rem = row_start % kVecSize;
|
||||
if (rem != 0) {
|
||||
// The mask has to land after the indexer has retired
|
||||
// Otherwise it may be accidentally overwritten by DG upstream
|
||||
device::PDLWaitPrimary<kPDL>();
|
||||
static_assert(kVecSize <= kBlockSize, "not enough threads ");
|
||||
if (const auto tx = threadIdx.x; tx < rem) {
|
||||
score[row_start - rem + tx] = impl::padding_value();
|
||||
}
|
||||
}
|
||||
using device::topk::broadcast;
|
||||
problem.in -= rem;
|
||||
problem.out = s_topk_indices; // write into stage buffer in smem first
|
||||
problem.seq_len = seq_len + rem;
|
||||
problem.bias = broadcast(-static_cast<int32_t>(rem));
|
||||
problem.input_start = broadcast(rem);
|
||||
|
||||
if (problem.seq_len <= Register2::kMaxSeqLen) {
|
||||
Register2::forward<kPDL>(problem, &smem);
|
||||
} else if (problem.seq_len <= Register4::kMaxSeqLen) {
|
||||
Register4::forward<kPDL>(problem, &smem);
|
||||
} else {
|
||||
Streaming::forward<kPDL>(problem, &smem);
|
||||
}
|
||||
device::PDLTriggerSecondary<kPDL>();
|
||||
__syncthreads();
|
||||
paged_transform<TopKMode::PAGE_TABLE>(problem, out, transform);
|
||||
ragged_select_transform<kPDL, TopKMode::PAGE_TABLE>(
|
||||
params.scores + bx * params.score_stride,
|
||||
static_cast<uint32_t>(params.row_starts[bx]),
|
||||
static_cast<uint32_t>(params.seq_lens[bx]),
|
||||
params.topk,
|
||||
params.page_indices + bx * static_cast<int64_t>(params.topk),
|
||||
params.get_transform(bx),
|
||||
s_topk_indices,
|
||||
&smem);
|
||||
}
|
||||
#endif // USE_ROCM
|
||||
|
||||
@@ -395,7 +509,7 @@ TOPK_KERNEL void topk_main_kernel(const __grid_constant__ TopKPagedParams params
|
||||
const auto bx = blockIdx.x;
|
||||
auto problem = params.problem(bx);
|
||||
if (problem.seq_len <= problem.topk) {
|
||||
return trivial_transform<kPDLEarly, kMode>(problem, params.get_transform(bx));
|
||||
return trivial_transform<kPDLEarly, kMode>(problem, params.template get_transform<kMode>(bx));
|
||||
}
|
||||
if constexpr (kNeedStaging) {
|
||||
problem.out = s_topk_indices; // write into stage buffer in smem first
|
||||
@@ -422,7 +536,14 @@ TOPK_KERNEL void topk_main_kernel(const __grid_constant__ TopKPagedParams params
|
||||
device::PDLWaitPrimary<kPDLFinal>();
|
||||
problem.out = params.get_output_ptr(bx); // in-place transform
|
||||
device::PDLTriggerSecondary<kPDL>();
|
||||
return paged_transform<kMode>(problem, problem.out, params.get_transform(bx));
|
||||
if constexpr (kMode == TopKMode::KPOOL) {
|
||||
for_each_item(problem.topk, [&](uint32_t tx, uint32_t) { s_topk_indices[tx] = problem.out[tx]; });
|
||||
__syncthreads();
|
||||
const auto out = problem.out;
|
||||
problem.out = s_topk_indices;
|
||||
return paged_transform<kMode>(problem, out, params.template get_transform<kMode>(bx));
|
||||
}
|
||||
return paged_transform<kMode>(problem, problem.out, params.template get_transform<kMode>(bx));
|
||||
} else {
|
||||
return device::PDLTriggerSecondary<kPDL>();
|
||||
}
|
||||
@@ -432,7 +553,7 @@ TOPK_KERNEL void topk_main_kernel(const __grid_constant__ TopKPagedParams params
|
||||
device::PDLTriggerSecondary<kPDL>();
|
||||
if constexpr (kNeedStaging) {
|
||||
__syncthreads();
|
||||
paged_transform<kMode>(problem, params.get_output_ptr(bx), params.get_transform(bx));
|
||||
paged_transform<kMode>(problem, params.get_output_ptr(bx), params.template get_transform<kMode>(bx));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -493,7 +614,7 @@ CLUSTER_TOPK_KERNEL void topk_small_batch_cluster_kernel(const __grid_constant__
|
||||
const auto worker_rank = bx % kClusterSize;
|
||||
if (problem.seq_len <= problem.topk) {
|
||||
if (by != worker_rank) return;
|
||||
return trivial_transform<kPDL, kMode>(problem, params.get_transform(bx));
|
||||
return trivial_transform<kPDL, kMode>(problem, params.template get_transform<kMode>(bx));
|
||||
}
|
||||
|
||||
if constexpr (kNeedStaging) {
|
||||
@@ -517,7 +638,7 @@ CLUSTER_TOPK_KERNEL void topk_small_batch_cluster_kernel(const __grid_constant__
|
||||
cluster.sync();
|
||||
if (by != 0) return;
|
||||
problem.out = s_topk_indices;
|
||||
return paged_transform<kMode>(problem, params.get_output_ptr(bx), params.get_transform(bx));
|
||||
return paged_transform<kMode>(problem, params.get_output_ptr(bx), params.template get_transform<kMode>(bx));
|
||||
} else {
|
||||
return device::PDLTriggerSecondary<kPDL>();
|
||||
}
|
||||
@@ -526,7 +647,7 @@ CLUSTER_TOPK_KERNEL void topk_small_batch_cluster_kernel(const __grid_constant__
|
||||
device::PDLTriggerSecondary<kPDL>();
|
||||
if constexpr (kNeedStaging) {
|
||||
__syncthreads();
|
||||
paged_transform<kMode>(problem, params.get_output_ptr(bx), params.get_transform(bx));
|
||||
paged_transform<kMode>(problem, params.get_output_ptr(bx), params.template get_transform<kMode>(bx));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -869,7 +990,7 @@ TOPK_KERNEL void topk_split_select(const __grid_constant__ TopKPagedParams param
|
||||
if (rank != 0) return;
|
||||
__shared__ int32_t s_topk_indices[kNeedStaging ? kMaxTopK : 1];
|
||||
if (problem.seq_len <= problem.topk) {
|
||||
return trivial_transform<false, kMode>(problem, params.get_transform(row));
|
||||
return trivial_transform<false, kMode>(problem, params.template get_transform<kMode>(row));
|
||||
}
|
||||
if constexpr (kNeedStaging) problem.out = s_topk_indices;
|
||||
if (problem.seq_len <= kReg4MaxSeqLen) {
|
||||
@@ -879,7 +1000,7 @@ TOPK_KERNEL void topk_split_select(const __grid_constant__ TopKPagedParams param
|
||||
}
|
||||
if constexpr (kNeedStaging) {
|
||||
__syncthreads();
|
||||
paged_transform<kMode>(problem, params.get_output_ptr(row), params.get_transform(row));
|
||||
paged_transform<kMode>(problem, params.get_output_ptr(row), params.template get_transform<kMode>(row));
|
||||
}
|
||||
return;
|
||||
}
|
||||
@@ -899,7 +1020,7 @@ TOPK_KERNEL void topk_split_select(const __grid_constant__ TopKPagedParams param
|
||||
// problem.out is already the destination, and paged_transform reads every
|
||||
// slot into registers before writing any, so transforming in place is safe.
|
||||
__syncthreads();
|
||||
paged_transform<kMode>(problem, problem.out, params.get_transform(row));
|
||||
paged_transform<kMode>(problem, problem.out, params.template get_transform<kMode>(row));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1016,6 +1137,89 @@ struct TopKKernel {
|
||||
#endif
|
||||
}
|
||||
|
||||
static uint32_t get_static_cluster_floor(uint32_t batch_size) {
|
||||
// NOTE: 15 is exactly 0.5 wave which saturate all cluster-8 SMs on Hopper/Blackwell
|
||||
if constexpr (SGL_ARCH_BLACKWELL_OR_GREATER) {
|
||||
return batch_size <= 15 ? 24576 : 30720;
|
||||
} else if constexpr (SGL_ARCH_HOPPER_OR_GREATER) {
|
||||
return batch_size <= 15 ? 32768 : 65536;
|
||||
} else {
|
||||
return UINT_MAX;
|
||||
}
|
||||
}
|
||||
|
||||
template <TopKMode kMode>
|
||||
static void launch(const TopKPagedParams& params, const uint32_t max_seq_len, const DLDevice device) {
|
||||
using namespace host;
|
||||
const auto batch_size = params.batch_size;
|
||||
#if SUPPORT_CLUSTER
|
||||
const bool use_cluster = (max_seq_len > params.static_cluster_floor) && (batch_size <= kClusterMaxBatch);
|
||||
if (use_cluster) {
|
||||
if constexpr (kMaxCluster16BatchSize > 0) {
|
||||
if (batch_size <= kMaxCluster16BatchSize) {
|
||||
constexpr uint32_t kClusterSize = 16;
|
||||
// Widths above 8 are non-portable; the launch is rejected without this.
|
||||
const auto kernel = topk_small_batch_cluster_kernel<kUsePDL, kMode, kClusterSize, 1>;
|
||||
const auto kernel_ptr = reinterpret_cast<const void*>(kernel);
|
||||
runtime::init_per_device(device, [kernel_ptr] {
|
||||
CHECK_CUDA(::cudaFuncSetAttribute(kernel_ptr, ::cudaFuncAttributeNonPortableClusterSizeAllowed, 1));
|
||||
});
|
||||
return LaunchKernel({batch_size, kClusterSize}, kBlockSize, device)
|
||||
.config({.use_pdl = kUsePDL, .cluster_dim = dim3{1, kClusterSize}})
|
||||
.launch(kernel, params);
|
||||
}
|
||||
}
|
||||
|
||||
if constexpr (kNumPersistentClusters > 0) {
|
||||
if (batch_size <= kNumPersistentClusters) {
|
||||
constexpr uint32_t kClusterSize = 8;
|
||||
return LaunchKernel({batch_size, kClusterSize}, kBlockSize, device)
|
||||
.config({.use_pdl = kUsePDL, .cluster_dim = dim3{1, kClusterSize}})
|
||||
.launch(topk_small_batch_cluster_kernel<kUsePDL, kMode, kClusterSize, 2>, params);
|
||||
} else {
|
||||
constexpr uint32_t kClusterSize = 8;
|
||||
const uint32_t num_clusters = std::min(batch_size, kNumPersistentClusters);
|
||||
LaunchKernel({num_clusters, kClusterSize}, kBlockSize, device)
|
||||
.config({.use_pdl = kUsePDL, .cluster_dim = dim3{1, kClusterSize}})
|
||||
.launch(topk_persistent_cluster_kernel<kUsePDL, kClusterSize>, params);
|
||||
LaunchKernel(batch_size, kBlockSize, device)
|
||||
.config({.use_pdl = kUsePDL})
|
||||
.launch(topk_main_kernel<kUsePDL, /*kLevel=*/3, kMode>, params);
|
||||
return void();
|
||||
}
|
||||
}
|
||||
}
|
||||
#elif defined(USE_ROCM)
|
||||
// Split dispatch. One block per row leaves a long row latency bound on one
|
||||
// CU however idle the rest is; split_plan decides where a second launch pays.
|
||||
// PDL stays off: the hist -> select kernel boundary is the barrier.
|
||||
if constexpr (kMode != TopKMode::KPOOL) {
|
||||
if (const auto [split, split_ws] = split_plan(batch_size, max_seq_len, device); split >= kSplitMin) {
|
||||
LaunchKernel({batch_size, split}, kBlockSize, device)
|
||||
.config({.use_pdl = false})
|
||||
.launch(topk_split_hist, params, split_ws);
|
||||
LaunchKernel({batch_size, split}, kBlockSize, device)
|
||||
.config({.use_pdl = false})
|
||||
.launch(topk_split_select<kMode>, params, split_ws);
|
||||
return;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
if (max_seq_len <= kReg2MaxSeqLen) {
|
||||
LaunchKernel(batch_size, kBlockSize, device)
|
||||
.config({.use_pdl = kUsePDL})
|
||||
.launch(topk_main_kernel<kUsePDL, /*kLevel=*/0, kMode>, params);
|
||||
} else if (max_seq_len <= kReg4MaxSeqLen) {
|
||||
LaunchKernel(batch_size, kBlockSize, device)
|
||||
.config({.use_pdl = kUsePDL})
|
||||
.launch(topk_main_kernel<kUsePDL, /*kLevel=*/1, kMode>, params);
|
||||
} else {
|
||||
LaunchKernel(batch_size, kBlockSize, device)
|
||||
.config({.use_pdl = kUsePDL})
|
||||
.launch(topk_main_kernel<kUsePDL, /*kLevel=*/2, kMode>, params);
|
||||
}
|
||||
}
|
||||
|
||||
static void transform_paged(
|
||||
const tvm::ffi::TensorView scores,
|
||||
const tvm::ffi::TensorView seq_lens,
|
||||
@@ -1085,17 +1289,6 @@ struct TopKKernel {
|
||||
const auto max_seq_len = static_cast<uint32_t>(L.unwrap());
|
||||
const auto device = device_.unwrap();
|
||||
|
||||
constexpr auto get_static_cluster_floor = [](uint32_t batch_size) -> uint32_t {
|
||||
// NOTE: 15 is exactly 0.5 wave which saturate all cluster-8 SMs on Hopper/Blackwell
|
||||
if constexpr (SGL_ARCH_BLACKWELL_OR_GREATER) {
|
||||
return batch_size <= 15 ? 24576 : 30720;
|
||||
} else if constexpr (SGL_ARCH_HOPPER_OR_GREATER) {
|
||||
return batch_size <= 15 ? 32768 : 65536;
|
||||
} else {
|
||||
return UINT_MAX;
|
||||
}
|
||||
};
|
||||
|
||||
const auto params = TopKPagedParams{
|
||||
.scores = static_cast<const float*>(scores.data_ptr()),
|
||||
.seq_lens = static_cast<const int32_t*>(seq_lens.data_ptr()),
|
||||
@@ -1111,6 +1304,11 @@ struct TopKKernel {
|
||||
.static_cluster_floor = get_static_cluster_floor(batch_size),
|
||||
// used for persistent cluster kernel and main kernel
|
||||
.batch_size = batch_size,
|
||||
.out_stride = static_cast<int64_t>(topk),
|
||||
.token_seq_lens = nullptr,
|
||||
.out_offsets = nullptr,
|
||||
.pool_bits = 0,
|
||||
.out_cols = topk,
|
||||
};
|
||||
|
||||
const auto dispatch = [&]<typename F>(F&& f) {
|
||||
@@ -1128,72 +1326,116 @@ struct TopKKernel {
|
||||
Panic("Invalid mode, this path should be unreachable");
|
||||
}
|
||||
};
|
||||
dispatch([&]<TopKMode kMode>() {
|
||||
#if SUPPORT_CLUSTER
|
||||
const bool use_cluster = (max_seq_len > params.static_cluster_floor) && (batch_size <= kClusterMaxBatch);
|
||||
if (use_cluster) {
|
||||
if constexpr (kMaxCluster16BatchSize > 0) {
|
||||
if (batch_size <= kMaxCluster16BatchSize) {
|
||||
constexpr uint32_t kClusterSize = 16;
|
||||
// Widths above 8 are non-portable; the launch is rejected without this.
|
||||
const auto kernel = topk_small_batch_cluster_kernel<kUsePDL, kMode, kClusterSize, 1>;
|
||||
const auto kernel_ptr = reinterpret_cast<const void*>(kernel);
|
||||
runtime::init_per_device(device, [kernel_ptr] {
|
||||
CHECK_CUDA(::cudaFuncSetAttribute(kernel_ptr, ::cudaFuncAttributeNonPortableClusterSizeAllowed, 1));
|
||||
});
|
||||
return LaunchKernel({batch_size, kClusterSize}, kBlockSize, device)
|
||||
.config({.use_pdl = kUsePDL, .cluster_dim = dim3{1, kClusterSize}})
|
||||
.launch(kernel, params);
|
||||
}
|
||||
}
|
||||
dispatch([&]<TopKMode kMode>() { launch<kMode>(params, max_seq_len, device); });
|
||||
}
|
||||
|
||||
if constexpr (kNumPersistentClusters > 0) {
|
||||
if (batch_size <= kNumPersistentClusters) {
|
||||
constexpr uint32_t kClusterSize = 8;
|
||||
return LaunchKernel({batch_size, kClusterSize}, kBlockSize, device)
|
||||
.config({.use_pdl = kUsePDL, .cluster_dim = dim3{1, kClusterSize}})
|
||||
.launch(topk_small_batch_cluster_kernel<kUsePDL, kMode, kClusterSize, 2>, params);
|
||||
} else {
|
||||
constexpr uint32_t kClusterSize = 8;
|
||||
const uint32_t num_clusters = std::min(batch_size, kNumPersistentClusters);
|
||||
LaunchKernel({num_clusters, kClusterSize}, kBlockSize, device)
|
||||
.config({.use_pdl = kUsePDL, .cluster_dim = dim3{1, kClusterSize}})
|
||||
.launch(topk_persistent_cluster_kernel<kUsePDL, kClusterSize>, params);
|
||||
LaunchKernel(batch_size, kBlockSize, device)
|
||||
.config({.use_pdl = kUsePDL})
|
||||
.launch(topk_main_kernel<kUsePDL, /*kLevel=*/3, kMode>, params);
|
||||
return void();
|
||||
}
|
||||
}
|
||||
}
|
||||
#elif defined(USE_ROCM)
|
||||
// Split dispatch. One block per row leaves a long row latency bound on one
|
||||
// CU however idle the rest is; split_plan decides where a second launch pays.
|
||||
// PDL stays off: the hist -> select kernel boundary is the barrier.
|
||||
if (const auto [split, split_ws] = split_plan(batch_size, max_seq_len, device); split >= kSplitMin) {
|
||||
LaunchKernel({batch_size, split}, kBlockSize, device)
|
||||
.config({.use_pdl = false})
|
||||
.launch(topk_split_hist, params, split_ws);
|
||||
LaunchKernel({batch_size, split}, kBlockSize, device)
|
||||
.config({.use_pdl = false})
|
||||
.launch(topk_split_select<kMode>, params, split_ws);
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
if (max_seq_len <= kReg2MaxSeqLen) {
|
||||
LaunchKernel(batch_size, kBlockSize, device)
|
||||
.config({.use_pdl = kUsePDL})
|
||||
.launch(topk_main_kernel<kUsePDL, /*kLevel=*/0, kMode>, params);
|
||||
} else if (max_seq_len <= kReg4MaxSeqLen) {
|
||||
LaunchKernel(batch_size, kBlockSize, device)
|
||||
.config({.use_pdl = kUsePDL})
|
||||
.launch(topk_main_kernel<kUsePDL, /*kLevel=*/1, kMode>, params);
|
||||
} else {
|
||||
LaunchKernel(batch_size, kBlockSize, device)
|
||||
.config({.use_pdl = kUsePDL})
|
||||
.launch(topk_main_kernel<kUsePDL, /*kLevel=*/2, kMode>, params);
|
||||
}
|
||||
});
|
||||
/**
|
||||
* \brief GLM k-pool variant of `transform_paged`: top-k over per-pool scores,
|
||||
* fused with pool expansion, tail append and the output transform.
|
||||
*
|
||||
* Row `b` selects the `topk` best of `scores[b, :pool_lens[b]]`, where
|
||||
* `topk = (out.size(1) - tail_cols) / pool_size` and `tail_cols` is
|
||||
* `pool_size - 1` when `token_seq_lens` is given, else 0. Each selected pool `g`
|
||||
* expands to tokens `g * pool_size + [0, pool_size)`; then the
|
||||
* `token_seq_lens[b] % pool_size` tokens of the open tail pool follow. Tokens
|
||||
* map through `page_table` (page size `page_size`) when given, else get
|
||||
* `out_offsets[b]` added (0 when absent); the rest of the row is `-1`.
|
||||
*
|
||||
* \param scores 16-byte-aligned float32 rows; width and row stride must be multiples of 4.
|
||||
* \param pool_lens NON-NEGATIVE; `metadata` must be `plan` over these values.
|
||||
* \param metadata Read before the PDL wait, so it must be written before the kernel
|
||||
* that produces `scores` is launched, not between that kernel and this one.
|
||||
*/
|
||||
static void transform_kpool(
|
||||
const tvm::ffi::TensorView scores,
|
||||
const tvm::ffi::TensorView pool_lens,
|
||||
const tvm::ffi::Optional<tvm::ffi::TensorView> token_seq_lens,
|
||||
const tvm::ffi::Optional<tvm::ffi::TensorView> page_table,
|
||||
const tvm::ffi::Optional<tvm::ffi::TensorView> out_offsets,
|
||||
const tvm::ffi::TensorView out,
|
||||
const uint32_t page_size,
|
||||
const uint32_t pool_size,
|
||||
const tvm::ffi::TensorView metadata) {
|
||||
using namespace host;
|
||||
auto B = SymbolicSize{"batch_size"};
|
||||
auto L = SymbolicSize{"max_seq_len"};
|
||||
auto S = SymbolicSize{"score_stride"};
|
||||
auto C = SymbolicSize{"out_cols"};
|
||||
auto O = SymbolicSize{"out_stride"};
|
||||
auto device_ = SymbolicDevice{};
|
||||
device_.set_options<kDLGPU>();
|
||||
|
||||
TensorMatcher({B, L}) // score
|
||||
.with_strides({S, 1})
|
||||
.with_dtype<float>()
|
||||
.with_device(device_)
|
||||
.ensure_alignment(16)
|
||||
.verify(scores);
|
||||
TensorMatcher({B}) // pool_lens
|
||||
.with_dtype<int32_t>()
|
||||
.with_device(device_)
|
||||
.verify(pool_lens);
|
||||
TensorMatcher({B, C}) // out
|
||||
.with_strides({O, 1})
|
||||
.with_dtype<int32_t>()
|
||||
.with_device(device_)
|
||||
.verify(out);
|
||||
TensorMatcher({-1, 2}) // metadata
|
||||
.with_dtype<int32_t>()
|
||||
.with_device(device_)
|
||||
.verify(metadata);
|
||||
const int32_t* token_seq_lens_ptr = nullptr;
|
||||
if (token_seq_lens.has_value()) {
|
||||
TensorMatcher({B}).with_dtype<int32_t>().with_device(device_).verify(token_seq_lens.value());
|
||||
token_seq_lens_ptr = static_cast<const int32_t*>(token_seq_lens.value().data_ptr());
|
||||
}
|
||||
const int32_t* page_table_ptr = nullptr;
|
||||
int64_t page_table_stride = 0;
|
||||
if (page_table.has_value()) {
|
||||
TensorMatcher({B, -1}).with_strides({-1, 1}).with_dtype<int32_t>().with_device(device_).verify(
|
||||
page_table.value());
|
||||
page_table_ptr = static_cast<const int32_t*>(page_table.value().data_ptr());
|
||||
page_table_stride = page_table.value().stride(0);
|
||||
}
|
||||
const int32_t* out_offsets_ptr = nullptr;
|
||||
if (out_offsets.has_value()) {
|
||||
RuntimeCheck(!page_table.has_value(), "page_table and out_offsets are mutually exclusive");
|
||||
TensorMatcher({B}).with_dtype<int32_t>().with_device(device_).verify(out_offsets.value());
|
||||
out_offsets_ptr = static_cast<const int32_t*>(out_offsets.value().data_ptr());
|
||||
}
|
||||
|
||||
RuntimeCheck(std::has_single_bit(page_size), "page_size must be power of 2");
|
||||
RuntimeCheck(pool_size > 1 && std::has_single_bit(pool_size), "pool_size must be a power of 2 > 1");
|
||||
RuntimeCheck(S.unwrap() % 4 == 0, "score_stride must be a multiple of 4 (16-byte vectorized load)");
|
||||
CHECK_HOST(L.unwrap() % 4 == 0) << "score width must be a multiple of 4 (16-byte vectorized load)";
|
||||
RuntimeCheck(metadata.size(0) == B.unwrap() + 1, "invalid metadata shape");
|
||||
const auto out_cols = static_cast<uint32_t>(C.unwrap());
|
||||
const auto tail_cols = token_seq_lens.has_value() ? pool_size - 1 : 0u;
|
||||
RuntimeCheck(out_cols > tail_cols && (out_cols - tail_cols) % pool_size == 0, "invalid out width ", out_cols);
|
||||
const auto topk = (out_cols - tail_cols) / pool_size;
|
||||
RuntimeCheck(topk > 0 && topk <= kMaxTopK, "pool-level topk must be in (0, 2048]");
|
||||
|
||||
const auto batch_size = static_cast<uint32_t>(B.unwrap());
|
||||
const auto params = TopKPagedParams{
|
||||
.scores = static_cast<const float*>(scores.data_ptr()),
|
||||
.seq_lens = static_cast<const int32_t*>(pool_lens.data_ptr()),
|
||||
.page_table = page_table_ptr,
|
||||
.page_indices = static_cast<int32_t*>(out.data_ptr()),
|
||||
.raw_indices = nullptr,
|
||||
.metadata = static_cast<const PlanItem*>(metadata.data_ptr()),
|
||||
.score_stride = S.unwrap(),
|
||||
.page_table_stride = page_table_stride,
|
||||
.topk = topk,
|
||||
.page_bits = static_cast<uint32_t>(std::countr_zero(page_size)),
|
||||
.static_cluster_floor = get_static_cluster_floor(batch_size),
|
||||
.batch_size = batch_size,
|
||||
.out_stride = O.unwrap(),
|
||||
.token_seq_lens = token_seq_lens_ptr,
|
||||
.out_offsets = out_offsets_ptr,
|
||||
.pool_bits = static_cast<uint32_t>(std::countr_zero(pool_size)),
|
||||
.out_cols = out_cols,
|
||||
};
|
||||
launch<TopKMode::KPOOL>(params, static_cast<uint32_t>(L.unwrap()), device_.unwrap());
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1267,6 +1509,90 @@ struct TopKKernel {
|
||||
.launch(topk_ragged_kernel<kUsePDL>, params);
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief Ragged (prefill) variant of `transform_kpool`: per-row window, no plan.
|
||||
*
|
||||
* Row `b` selects among `scores[b, ks : ks + pool_lens[b]]` (`ks = row_starts[b]`,
|
||||
* 0 when absent), with the same expansion, tail and output transform as
|
||||
* `transform_kpool`. `page_table_row_index[b]` picks the page-table row when
|
||||
* several score rows share one; without it `page_table` needs one row per score row.
|
||||
*
|
||||
* `scores` is written in place: the <= 3 columns ahead of an unaligned window
|
||||
* are masked, as in `transform_ragged`. The buffer must have no other consumer.
|
||||
*/
|
||||
static void transform_kpool_ragged(
|
||||
const tvm::ffi::TensorView scores,
|
||||
const tvm::ffi::TensorView pool_lens,
|
||||
const tvm::ffi::Optional<tvm::ffi::TensorView> row_starts,
|
||||
const tvm::ffi::Optional<tvm::ffi::TensorView> token_seq_lens,
|
||||
const tvm::ffi::Optional<tvm::ffi::TensorView> page_table,
|
||||
const tvm::ffi::Optional<tvm::ffi::TensorView> page_table_row_index,
|
||||
const tvm::ffi::Optional<tvm::ffi::TensorView> out_offsets,
|
||||
const tvm::ffi::TensorView out,
|
||||
const uint32_t page_size,
|
||||
const uint32_t pool_size) {
|
||||
using namespace host;
|
||||
auto B = SymbolicSize{"batch_size"};
|
||||
auto S = SymbolicSize{"score_stride"};
|
||||
auto C = SymbolicSize{"out_cols"};
|
||||
auto O = SymbolicSize{"out_stride"};
|
||||
auto device_ = SymbolicDevice{};
|
||||
device_.set_options<kDLGPU>();
|
||||
|
||||
TensorMatcher({B, -1}).with_strides({S, 1}).with_dtype<float>().with_device(device_).ensure_alignment(16).verify(
|
||||
scores);
|
||||
TensorMatcher({B}).with_dtype<int32_t>().with_device(device_).verify(pool_lens);
|
||||
TensorMatcher({B, C}).with_strides({O, 1}).with_dtype<int32_t>().with_device(device_).verify(out);
|
||||
const auto optional_rows = [&](const tvm::ffi::Optional<tvm::ffi::TensorView>& t) -> const int32_t* {
|
||||
if (!t.has_value()) return nullptr;
|
||||
TensorMatcher({B}).with_dtype<int32_t>().with_device(device_).verify(t.value());
|
||||
return static_cast<const int32_t*>(t.value().data_ptr());
|
||||
};
|
||||
const int32_t* page_table_ptr = nullptr;
|
||||
int64_t page_table_stride = 0;
|
||||
if (page_table.has_value()) {
|
||||
TensorMatcher({-1, -1}).with_strides({-1, 1}).with_dtype<int32_t>().with_device(device_).verify(
|
||||
page_table.value());
|
||||
if (!page_table_row_index.has_value()) {
|
||||
RuntimeCheck(page_table.value().size(0) == B.unwrap(), "page_table needs one row per score row");
|
||||
}
|
||||
page_table_ptr = static_cast<const int32_t*>(page_table.value().data_ptr());
|
||||
page_table_stride = page_table.value().stride(0);
|
||||
}
|
||||
RuntimeCheck(!page_table_row_index.has_value() || page_table.has_value(), "page_table_row_index needs page_table");
|
||||
RuntimeCheck(!(page_table.has_value() && out_offsets.has_value()), "page_table and out_offsets are exclusive");
|
||||
|
||||
RuntimeCheck(std::has_single_bit(page_size), "page_size must be power of 2");
|
||||
RuntimeCheck(pool_size > 1 && std::has_single_bit(pool_size), "pool_size must be a power of 2 > 1");
|
||||
RuntimeCheck(S.unwrap() % 4 == 0, "score_stride must be a multiple of 4 (16-byte vectorized load)");
|
||||
const auto out_cols = static_cast<uint32_t>(C.unwrap());
|
||||
const auto tail_cols = token_seq_lens.has_value() ? pool_size - 1 : 0u;
|
||||
RuntimeCheck(out_cols > tail_cols && (out_cols - tail_cols) % pool_size == 0, "invalid out width ", out_cols);
|
||||
const auto topk = (out_cols - tail_cols) / pool_size;
|
||||
RuntimeCheck(topk > 0 && topk <= kMaxTopK, "pool-level topk must be in (0, 2048]");
|
||||
|
||||
const auto params = TopKRaggedKpoolParams{
|
||||
.scores = static_cast<float*>(scores.data_ptr()),
|
||||
.pool_lens = static_cast<const int32_t*>(pool_lens.data_ptr()),
|
||||
.row_starts = optional_rows(row_starts),
|
||||
.token_seq_lens = optional_rows(token_seq_lens),
|
||||
.page_table = page_table_ptr,
|
||||
.page_table_row_index = optional_rows(page_table_row_index),
|
||||
.out_offsets = optional_rows(out_offsets),
|
||||
.out = static_cast<int32_t*>(out.data_ptr()),
|
||||
.score_stride = S.unwrap(),
|
||||
.page_table_stride = page_table_stride,
|
||||
.out_stride = O.unwrap(),
|
||||
.topk = topk,
|
||||
.page_bits = static_cast<uint32_t>(std::countr_zero(page_size)),
|
||||
.pool_bits = static_cast<uint32_t>(std::countr_zero(pool_size)),
|
||||
.out_cols = out_cols,
|
||||
};
|
||||
LaunchKernel(static_cast<uint32_t>(B.unwrap()), kBlockSize, device_.unwrap())
|
||||
.config({.use_pdl = kUsePDL})
|
||||
.launch(topk_ragged_kpool_kernel<kUsePDL>, params);
|
||||
}
|
||||
|
||||
#ifdef USE_ROCM // see the packed kernel above
|
||||
/**
|
||||
* \brief Packed (DSA extend prefill) variant of `transform_paged`: per-row
|
||||
|
||||
@@ -304,6 +304,22 @@ register_kernel(
|
||||
capabilities=CUDA_ONLY,
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="attention.topk_transform_kpool_v2",
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.attention.dsv4.topk:topk_transform_kpool_v2",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="attention.topk_transform_kpool_ragged_v2",
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.attention.dsv4.topk:topk_transform_kpool_ragged_v2",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="attention.fast_topk",
|
||||
|
||||
@@ -25,6 +25,8 @@ from .elementwise import (
|
||||
from .fp8_wo_a import sglang_per_token_group_quant_fp8_dsv4_wo_a
|
||||
from .topk import (
|
||||
plan_topk_v2,
|
||||
topk_transform_kpool_ragged_v2,
|
||||
topk_transform_kpool_v2,
|
||||
topk_transform_paged,
|
||||
topk_transform_paged_v2,
|
||||
topk_transform_ragged_v2,
|
||||
@@ -50,6 +52,8 @@ __all__ = [
|
||||
"get_paged_mqa_logits_metadata",
|
||||
"triton_create_paged_compress_data",
|
||||
"topk_transform_paged",
|
||||
"topk_transform_kpool_v2",
|
||||
"topk_transform_kpool_ragged_v2",
|
||||
"topk_transform_paged_v2",
|
||||
"topk_transform_ragged_v2",
|
||||
"plan_topk_v2",
|
||||
|
||||
@@ -54,6 +54,8 @@ def _jit_topk_v2_module():
|
||||
("topk_transform_paged", f"{kernel}::transform_paged"),
|
||||
("topk_transform_ragged", f"{kernel}::transform_ragged"),
|
||||
("topk_plan", f"{kernel}::plan"),
|
||||
("topk_transform_kpool", f"{kernel}::transform_kpool"),
|
||||
("topk_transform_kpool_ragged", f"{kernel}::transform_kpool_ragged"),
|
||||
]
|
||||
if is_hip_runtime():
|
||||
# transform_packed only exists under USE_ROCM, see topk_v2.cuh
|
||||
@@ -153,7 +155,12 @@ def topk_transform_paged_torch(
|
||||
_PLAN_METADATA_INTS_PER_BATCH = 2
|
||||
|
||||
|
||||
def plan_topk_v2(seq_lens: torch.Tensor, static_threshold: int = -1) -> torch.Tensor:
|
||||
def plan_topk_v2(
|
||||
seq_lens: torch.Tensor,
|
||||
static_threshold: int = -1,
|
||||
*,
|
||||
out: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Preprocess the per-batch routing plan for :func:`topk_transform_paged_v2`.
|
||||
NOTE: every entry of ``seq_lens`` must be NON-NEGATIVE.
|
||||
@@ -164,7 +171,11 @@ def plan_topk_v2(seq_lens: torch.Tensor, static_threshold: int = -1) -> torch.Te
|
||||
"""
|
||||
module = _jit_topk_v2_module()
|
||||
bs = seq_lens.shape[0]
|
||||
metadata = seq_lens.new_empty(bs + 1, _PLAN_METADATA_INTS_PER_BATCH)
|
||||
metadata = (
|
||||
seq_lens.new_empty(bs + 1, _PLAN_METADATA_INTS_PER_BATCH)
|
||||
if out is None
|
||||
else out
|
||||
)
|
||||
module.topk_plan(seq_lens, metadata, static_threshold)
|
||||
return metadata
|
||||
|
||||
@@ -281,6 +292,70 @@ def topk_transform_paged_v2(
|
||||
)
|
||||
|
||||
|
||||
def topk_transform_kpool_v2(
|
||||
scores: torch.Tensor,
|
||||
pool_lens: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
pool_size: int,
|
||||
metadata: torch.Tensor,
|
||||
*,
|
||||
token_seq_lens: Optional[torch.Tensor] = None,
|
||||
page_table: Optional[torch.Tensor] = None,
|
||||
page_size: int = 1,
|
||||
out_offsets: Optional[torch.Tensor] = None,
|
||||
) -> None:
|
||||
"""GLM k-pool top-k over per-pool scores, fused with pool expansion, the
|
||||
``token_seq_lens % pool_size`` tail and the page-table or offset transform.
|
||||
|
||||
``metadata`` is :func:`plan_topk_v2` over ``pool_lens``. The kernel reads it
|
||||
before its PDL wait, so plan before launching the kernel that writes ``scores``.
|
||||
"""
|
||||
_jit_topk_v2_module().topk_transform_kpool(
|
||||
scores,
|
||||
pool_lens,
|
||||
token_seq_lens,
|
||||
page_table,
|
||||
out_offsets,
|
||||
out,
|
||||
page_size,
|
||||
pool_size,
|
||||
metadata,
|
||||
)
|
||||
|
||||
|
||||
def topk_transform_kpool_ragged_v2(
|
||||
scores: torch.Tensor,
|
||||
pool_lens: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
pool_size: int,
|
||||
*,
|
||||
row_starts: Optional[torch.Tensor] = None,
|
||||
token_seq_lens: Optional[torch.Tensor] = None,
|
||||
page_table: Optional[torch.Tensor] = None,
|
||||
page_table_row_index: Optional[torch.Tensor] = None,
|
||||
page_size: int = 1,
|
||||
out_offsets: Optional[torch.Tensor] = None,
|
||||
) -> None:
|
||||
"""Ragged (prefill) :func:`topk_transform_kpool_v2`: row ``i`` selects among
|
||||
``scores[i, ks : ks + pool_lens[i]]`` (``ks = row_starts[i]``, 0 when omitted);
|
||||
no plan is needed.
|
||||
|
||||
NOTE: ``scores`` is written in place, as in :func:`topk_transform_ragged_v2`.
|
||||
"""
|
||||
_jit_topk_v2_module().topk_transform_kpool_ragged(
|
||||
scores,
|
||||
pool_lens,
|
||||
row_starts,
|
||||
token_seq_lens,
|
||||
page_table,
|
||||
page_table_row_index,
|
||||
out_offsets,
|
||||
out,
|
||||
page_size,
|
||||
pool_size,
|
||||
)
|
||||
|
||||
|
||||
def topk_transform_packed_v2(
|
||||
scores: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
|
||||
@@ -75,6 +75,14 @@ def _should_fuse_kpool_topk(metadata: BaseIndexerMetadata) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _topk_v2_plan_for(
|
||||
*, owned_plan: Optional[torch.Tensor], pool_lens: torch.Tensor
|
||||
) -> Optional[torch.Tensor]:
|
||||
if owned_plan is not None and owned_plan.shape[0] == pool_lens.shape[0] + 1:
|
||||
return owned_plan
|
||||
return None
|
||||
|
||||
|
||||
def _slice_rows(
|
||||
*, tensor: Optional[torch.Tensor], rows: slice
|
||||
) -> Optional[torch.Tensor]:
|
||||
@@ -763,6 +771,7 @@ class IndexerKPool(MultiPlatformOp):
|
||||
row_starts: Optional[torch.Tensor] = None,
|
||||
out_rows: Optional[int] = None,
|
||||
page_table_row_index: Optional[torch.Tensor] = None,
|
||||
topk_v2_plan: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
from sglang.srt.layers.attention.dsa.kpool_fp8_index import (
|
||||
topk_from_pooled_history_logits,
|
||||
@@ -791,6 +800,7 @@ class IndexerKPool(MultiPlatformOp):
|
||||
row_starts=row_starts,
|
||||
out_rows=out_rows,
|
||||
page_table_row_index=page_table_row_index,
|
||||
topk_v2_plan=topk_v2_plan,
|
||||
)
|
||||
|
||||
def _get_kpool_decode_metadata(
|
||||
@@ -800,7 +810,13 @@ class IndexerKPool(MultiPlatformOp):
|
||||
seqlens_32: torch.Tensor,
|
||||
blocksize: int,
|
||||
build_schedule_metadata: bool = True,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
|
||||
) -> Tuple[
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
Optional[torch.Tensor],
|
||||
Optional[torch.Tensor],
|
||||
]:
|
||||
from sglang.srt.layers.attention.dsa.kpool_fp8_index import (
|
||||
build_pooled_page_table_64,
|
||||
)
|
||||
@@ -828,9 +844,13 @@ class IndexerKPool(MultiPlatformOp):
|
||||
pool_context_lens,
|
||||
pool_block_tables,
|
||||
pool_schedule_metadata,
|
||||
_topk_v2_plan_for(
|
||||
owned_plan=plan.pool_topk_v2_plan, pool_lens=pool_seqlens
|
||||
),
|
||||
)
|
||||
|
||||
pool_seqlens = attn_metadata.pooled_cache_seqlens_int32
|
||||
topk_v2_plan = attn_metadata.pooled_topk_v2_plan
|
||||
pool_block_tables = attn_metadata.pooled_real_page_table
|
||||
pool_schedule_metadata = attn_metadata.pooled_paged_mqa_schedule_metadata
|
||||
|
||||
@@ -846,6 +866,7 @@ class IndexerKPool(MultiPlatformOp):
|
||||
block_tables, self.index_kpool
|
||||
).contiguous()
|
||||
pool_schedule_metadata = None
|
||||
topk_v2_plan = None
|
||||
else:
|
||||
pool_seqlens = pool_seqlens[: seqlens_32.shape[0]]
|
||||
pool_block_tables = pool_block_tables[
|
||||
@@ -864,6 +885,7 @@ class IndexerKPool(MultiPlatformOp):
|
||||
pool_context_lens,
|
||||
pool_block_tables,
|
||||
pool_schedule_metadata,
|
||||
_topk_v2_plan_for(owned_plan=topk_v2_plan, pool_lens=pool_seqlens),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -1003,16 +1025,18 @@ class IndexerKPool(MultiPlatformOp):
|
||||
and self._should_use_tilelang_paged_mqa_logits(q_fp8)
|
||||
)
|
||||
|
||||
pool_seqlens, pool_context_lens, pool_block_tables, pool_schedule_metadata = (
|
||||
self._get_kpool_decode_metadata(
|
||||
metadata,
|
||||
block_tables,
|
||||
seqlens_32,
|
||||
blocksize,
|
||||
build_schedule_metadata=not (
|
||||
use_aiter_paged_mqa or use_tilelang_paged_mqa
|
||||
),
|
||||
)
|
||||
(
|
||||
pool_seqlens,
|
||||
pool_context_lens,
|
||||
pool_block_tables,
|
||||
pool_schedule_metadata,
|
||||
topk_v2_plan,
|
||||
) = self._get_kpool_decode_metadata(
|
||||
metadata=metadata,
|
||||
block_tables=block_tables,
|
||||
seqlens_32=seqlens_32,
|
||||
blocksize=blocksize,
|
||||
build_schedule_metadata=not (use_aiter_paged_mqa or use_tilelang_paged_mqa),
|
||||
)
|
||||
pool_max_seq_len = pool_block_tables.shape[1] * blocksize
|
||||
if use_aiter_paged_mqa:
|
||||
@@ -1065,8 +1089,9 @@ class IndexerKPool(MultiPlatformOp):
|
||||
|
||||
page_table_1, topk_offsets, _ = self._kpool_fused_topk_mapping(metadata)
|
||||
topk_result = self._topk_from_kpool_logits(
|
||||
logits,
|
||||
pool_seqlens,
|
||||
logits=logits,
|
||||
pool_lens=pool_seqlens,
|
||||
topk_v2_plan=topk_v2_plan,
|
||||
seq_lens=seqlens_32,
|
||||
page_table=page_table_1,
|
||||
topk_offsets=topk_offsets,
|
||||
|
||||
@@ -214,6 +214,7 @@ class DSAMetadataManagementMixin:
|
||||
dst.pooled_paged_mqa_schedule_metadata,
|
||||
src.pooled_paged_mqa_schedule_metadata,
|
||||
)
|
||||
and _match(dst.pooled_topk_v2_plan, src.pooled_topk_v2_plan)
|
||||
and _match(dst.kpool_write_plan, src.kpool_write_plan)
|
||||
):
|
||||
return False
|
||||
@@ -223,6 +224,7 @@ class DSAMetadataManagementMixin:
|
||||
and _match(dst_plan.seqlens_per_q, src_plan.seqlens_per_q)
|
||||
and _match(dst_plan.pool_schedule_metadata, src_plan.pool_schedule_metadata)
|
||||
and _match(dst_plan.effective_n_per_batch, src_plan.effective_n_per_batch)
|
||||
and _match(dst_plan.pool_topk_v2_plan, src_plan.pool_topk_v2_plan)
|
||||
):
|
||||
return False
|
||||
return True
|
||||
@@ -298,6 +300,8 @@ class DSAMetadataManagementMixin:
|
||||
metadata.pooled_paged_mqa_schedule_metadata.copy_(
|
||||
src_metadata.pooled_paged_mqa_schedule_metadata
|
||||
)
|
||||
if metadata.pooled_topk_v2_plan is not None:
|
||||
metadata.pooled_topk_v2_plan.copy_(src_metadata.pooled_topk_v2_plan)
|
||||
|
||||
dst_plan = metadata.kpool_write_plan
|
||||
src_plan = src_metadata.kpool_write_plan
|
||||
@@ -315,3 +319,5 @@ class DSAMetadataManagementMixin:
|
||||
dst_plan.pool_schedule_metadata.copy_(src_plan.pool_schedule_metadata)
|
||||
if dst_plan.effective_n_per_batch is not None:
|
||||
dst_plan.effective_n_per_batch.copy_(src_plan.effective_n_per_batch)
|
||||
if dst_plan.pool_topk_v2_plan is not None:
|
||||
dst_plan.pool_topk_v2_plan.copy_(src_plan.pool_topk_v2_plan)
|
||||
|
||||
@@ -4,10 +4,14 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.attention.dsa.utils import (
|
||||
INDEXER_K_CACHE_PRESHUFFLE_TILE,
|
||||
aiter_can_use_preshuffle_paged_mqa,
|
||||
)
|
||||
from sglang.srt.utils import is_cuda
|
||||
|
||||
_is_cuda = is_cuda()
|
||||
|
||||
BLOCK_SIZE_K = 64
|
||||
INDEX_HEAD_DIM = 128
|
||||
@@ -586,6 +590,177 @@ def _append_kpool_tail_to_topk_kernel(
|
||||
tl.store(out_ptr + row * out_stride_0 + cols * out_stride_1, value, mask=mask)
|
||||
|
||||
|
||||
def _kpool_topk_v2_enabled() -> bool:
|
||||
return _is_cuda and envs.SGLANG_OPT_USE_TOPK_V2.get()
|
||||
|
||||
|
||||
def build_kpool_topk_v2_plan(pool_lens: torch.Tensor) -> Optional[torch.Tensor]:
|
||||
if not _kpool_topk_v2_enabled():
|
||||
return None
|
||||
plan = pool_lens.new_empty((pool_lens.shape[0] + 1, 2), dtype=torch.int32)
|
||||
refresh_kpool_topk_v2_plan(plan=plan, pool_lens=pool_lens)
|
||||
return plan
|
||||
|
||||
|
||||
def refresh_kpool_topk_v2_plan(*, plan: torch.Tensor, pool_lens: torch.Tensor) -> None:
|
||||
from sglang.kernels.ops.attention.dsv4.topk import plan_topk_v2
|
||||
|
||||
plan_topk_v2(pool_lens.to(torch.int32), out=plan)
|
||||
|
||||
|
||||
def _kpool_topk_v2_layout_ok(*, logits: torch.Tensor, pool_size: int) -> bool:
|
||||
return (
|
||||
_kpool_topk_v2_enabled()
|
||||
and logits.dtype == torch.float32
|
||||
and logits.stride(1) == 1
|
||||
and logits.stride(0) % 4 == 0
|
||||
and logits.data_ptr() % 16 == 0
|
||||
and pool_size > 1
|
||||
and pool_size & (pool_size - 1) == 0
|
||||
)
|
||||
|
||||
|
||||
def _can_use_kpool_topk_v2(
|
||||
*,
|
||||
logits: torch.Tensor,
|
||||
pool_size: int,
|
||||
row_starts: torch.Tensor | None,
|
||||
page_table_row_index: torch.Tensor | None,
|
||||
) -> bool:
|
||||
return (
|
||||
row_starts is None
|
||||
and page_table_row_index is None
|
||||
and logits.shape[1] % 4 == 0
|
||||
and _kpool_topk_v2_layout_ok(logits=logits, pool_size=pool_size)
|
||||
)
|
||||
|
||||
|
||||
def _alloc_kpool_topk_out(
|
||||
*, logits: torch.Tensor, rows: int, out_cols: int, out_rows: int | None
|
||||
) -> torch.Tensor:
|
||||
if out_rows is None or out_rows == rows:
|
||||
return logits.new_empty((rows, out_cols), dtype=torch.int32)
|
||||
return logits.new_full((out_rows, out_cols), -1, dtype=torch.int32)
|
||||
|
||||
|
||||
def _kpool_topk_v2_ragged(
|
||||
*,
|
||||
logits: torch.Tensor,
|
||||
group_lengths: torch.Tensor,
|
||||
pool_size: int,
|
||||
topk: int,
|
||||
page_table: torch.Tensor | None,
|
||||
page_table_row_index: torch.Tensor | None,
|
||||
topk_offsets: torch.Tensor | None,
|
||||
seq_lens: torch.Tensor | None,
|
||||
row_starts: torch.Tensor | None,
|
||||
out_rows: int | None,
|
||||
) -> torch.Tensor:
|
||||
from sglang.kernels.ops.attention.dsv4.topk import topk_transform_kpool_ragged_v2
|
||||
|
||||
rows = logits.shape[0]
|
||||
out_cols = topk + (pool_size - 1 if seq_lens is not None else 0)
|
||||
full = _alloc_kpool_topk_out(
|
||||
logits=logits, rows=rows, out_cols=out_cols, out_rows=out_rows
|
||||
)
|
||||
topk_transform_kpool_ragged_v2(
|
||||
scores=logits,
|
||||
pool_lens=group_lengths.to(torch.int32),
|
||||
out=full[:rows],
|
||||
pool_size=pool_size,
|
||||
row_starts=None if row_starts is None else row_starts.to(torch.int32),
|
||||
token_seq_lens=None if seq_lens is None else seq_lens.to(torch.int32),
|
||||
page_table=page_table,
|
||||
page_table_row_index=(
|
||||
None
|
||||
if page_table_row_index is None
|
||||
else page_table_row_index.to(torch.int32)
|
||||
),
|
||||
out_offsets=topk_offsets,
|
||||
)
|
||||
return full
|
||||
|
||||
|
||||
def _kpool_topk_v2(
|
||||
*,
|
||||
logits: torch.Tensor,
|
||||
group_lengths: torch.Tensor,
|
||||
pool_size: int,
|
||||
topk: int,
|
||||
page_table: torch.Tensor | None,
|
||||
topk_offsets: torch.Tensor | None,
|
||||
seq_lens: torch.Tensor | None,
|
||||
out_rows: int | None,
|
||||
topk_v2_plan: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
from sglang.kernels.ops.attention.dsv4.topk import topk_transform_kpool_v2
|
||||
|
||||
rows = logits.shape[0]
|
||||
group_lengths = group_lengths.to(torch.int32)
|
||||
out_cols = topk + (pool_size - 1 if seq_lens is not None else 0)
|
||||
full = _alloc_kpool_topk_out(
|
||||
logits=logits, rows=rows, out_cols=out_cols, out_rows=out_rows
|
||||
)
|
||||
topk_transform_kpool_v2(
|
||||
scores=logits,
|
||||
pool_lens=group_lengths,
|
||||
out=full[:rows],
|
||||
pool_size=pool_size,
|
||||
metadata=topk_v2_plan,
|
||||
token_seq_lens=None if seq_lens is None else seq_lens.to(torch.int32),
|
||||
page_table=page_table,
|
||||
out_offsets=topk_offsets,
|
||||
)
|
||||
return full
|
||||
|
||||
|
||||
def _try_kpool_topk_v2(
|
||||
*,
|
||||
logits: torch.Tensor,
|
||||
group_lengths: torch.Tensor,
|
||||
kpool: int,
|
||||
topk: int,
|
||||
page_table: torch.Tensor | None,
|
||||
page_table_row_index: torch.Tensor | None,
|
||||
topk_offsets: torch.Tensor | None,
|
||||
seq_lens: torch.Tensor | None,
|
||||
row_starts: torch.Tensor | None,
|
||||
out_rows: int | None,
|
||||
topk_v2_plan: torch.Tensor | None,
|
||||
) -> torch.Tensor | None:
|
||||
if topk_v2_plan is not None and _can_use_kpool_topk_v2(
|
||||
logits=logits,
|
||||
pool_size=kpool,
|
||||
row_starts=row_starts,
|
||||
page_table_row_index=page_table_row_index,
|
||||
):
|
||||
return _kpool_topk_v2(
|
||||
logits=logits,
|
||||
group_lengths=group_lengths,
|
||||
pool_size=kpool,
|
||||
topk=topk,
|
||||
page_table=page_table,
|
||||
topk_offsets=topk_offsets,
|
||||
seq_lens=seq_lens,
|
||||
out_rows=out_rows,
|
||||
topk_v2_plan=topk_v2_plan,
|
||||
)
|
||||
if _kpool_topk_v2_layout_ok(logits=logits, pool_size=kpool):
|
||||
return _kpool_topk_v2_ragged(
|
||||
logits=logits,
|
||||
group_lengths=group_lengths,
|
||||
pool_size=kpool,
|
||||
topk=topk,
|
||||
page_table=page_table,
|
||||
page_table_row_index=page_table_row_index,
|
||||
topk_offsets=topk_offsets,
|
||||
seq_lens=seq_lens,
|
||||
row_starts=row_starts,
|
||||
out_rows=out_rows,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def topk_from_pooled_history_logits(
|
||||
logits: torch.Tensor,
|
||||
group_lengths: torch.Tensor,
|
||||
@@ -597,6 +772,7 @@ def topk_from_pooled_history_logits(
|
||||
row_starts: torch.Tensor | None = None,
|
||||
out_rows: int | None = None,
|
||||
page_table_row_index: torch.Tensor | None = None,
|
||||
topk_v2_plan: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert logits.ndim == 2
|
||||
assert group_lengths.ndim == 1
|
||||
@@ -606,7 +782,6 @@ def topk_from_pooled_history_logits(
|
||||
assert out_rows is None or out_rows >= logits.shape[0]
|
||||
assert page_table_row_index is None or page_table is not None
|
||||
|
||||
_, cols = logits.shape
|
||||
group_topk = history_group_budget_for_topk(topk, kpool)
|
||||
if topk_offsets is not None and topk_offsets.ndim == 2:
|
||||
assert topk_offsets.shape[1] == 1
|
||||
@@ -624,6 +799,22 @@ def topk_from_pooled_history_logits(
|
||||
f"is disabled. Got device={logits.device}, dtype={logits.dtype}."
|
||||
)
|
||||
|
||||
v2_result = _try_kpool_topk_v2(
|
||||
logits=logits,
|
||||
group_lengths=group_lengths,
|
||||
kpool=kpool,
|
||||
topk=topk,
|
||||
page_table=page_table,
|
||||
page_table_row_index=page_table_row_index,
|
||||
topk_offsets=topk_offsets,
|
||||
seq_lens=seq_lens,
|
||||
row_starts=row_starts,
|
||||
out_rows=out_rows,
|
||||
topk_v2_plan=topk_v2_plan,
|
||||
)
|
||||
if v2_result is not None:
|
||||
return v2_result
|
||||
|
||||
if group_topk in (128, 160, 192, 224, 256, 512):
|
||||
from sglang.kernels.ops.attention.dsa.kpool_topk_transform import (
|
||||
fast_kpool_topk_transform_fused,
|
||||
@@ -648,6 +839,35 @@ def topk_from_pooled_history_logits(
|
||||
padded[: result.shape[0]] = result
|
||||
return padded
|
||||
|
||||
return _kpool_topk_unfused(
|
||||
logits=logits,
|
||||
group_lengths=group_lengths,
|
||||
kpool=kpool,
|
||||
topk=topk,
|
||||
group_topk=group_topk,
|
||||
page_table=page_table,
|
||||
page_table_row_index=page_table_row_index,
|
||||
topk_offsets=topk_offsets,
|
||||
seq_lens=seq_lens,
|
||||
row_starts=row_starts,
|
||||
out_rows=out_rows,
|
||||
)
|
||||
|
||||
|
||||
def _kpool_topk_unfused(
|
||||
*,
|
||||
logits: torch.Tensor,
|
||||
group_lengths: torch.Tensor,
|
||||
kpool: int,
|
||||
topk: int,
|
||||
group_topk: int,
|
||||
page_table: torch.Tensor | None,
|
||||
page_table_row_index: torch.Tensor | None,
|
||||
topk_offsets: torch.Tensor | None,
|
||||
seq_lens: torch.Tensor | None,
|
||||
row_starts: torch.Tensor | None,
|
||||
out_rows: int | None,
|
||||
) -> torch.Tensor:
|
||||
assert page_table_row_index is None, (
|
||||
"page_table_row_index requires the fused fast_kpool group_topk path"
|
||||
)
|
||||
@@ -662,7 +882,7 @@ def topk_from_pooled_history_logits(
|
||||
)
|
||||
|
||||
rank = torch.arange(group_topk, device=logits.device, dtype=torch.int32)
|
||||
max_valid_groups = min(cols, group_topk)
|
||||
max_valid_groups = min(logits.shape[1], group_topk)
|
||||
valid_counts = torch.minimum(
|
||||
group_lengths.to(torch.int32),
|
||||
torch.full_like(group_lengths.to(torch.int32), max_valid_groups),
|
||||
|
||||
@@ -9,9 +9,11 @@ import torch
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.attention.dsa.kpool_fp8_index import (
|
||||
INDEX_HEAD_DIM,
|
||||
build_kpool_topk_v2_plan,
|
||||
build_pooled_page_table_64,
|
||||
kpool_build_ragged_layout,
|
||||
kpool_max_closed_pools,
|
||||
refresh_kpool_topk_v2_plan,
|
||||
update_kpool_write_plan_cuda_graph,
|
||||
)
|
||||
from sglang.srt.layers.attention.dsa.utils import dsa_use_prefill_cp
|
||||
@@ -115,6 +117,7 @@ class KPoolWritePlan:
|
||||
seqlens_per_q: Optional[torch.Tensor] = None
|
||||
pool_schedule_metadata: Optional[torch.Tensor] = None
|
||||
effective_n_per_batch: Optional[torch.Tensor] = None
|
||||
pool_topk_v2_plan: Optional[torch.Tensor] = None
|
||||
|
||||
|
||||
def _is_kpool_layout_enabled(kpool: int, physical_page_size: int) -> bool:
|
||||
@@ -614,6 +617,7 @@ def init_pooled_paged_mqa_metadata(
|
||||
pooled_cache_seqlens_int32=pool_seqlens,
|
||||
pooled_real_page_table=pooled_page_table,
|
||||
pooled_paged_mqa_schedule_metadata=schedule,
|
||||
pooled_topk_v2_plan=build_kpool_topk_v2_plan(pool_seqlens),
|
||||
)
|
||||
|
||||
|
||||
@@ -669,6 +673,12 @@ def update_pooled_paged_mqa_metadata(
|
||||
if new_schedule is not None:
|
||||
metadata.pooled_paged_mqa_schedule_metadata.copy_(new_schedule)
|
||||
|
||||
if metadata.pooled_topk_v2_plan is not None:
|
||||
refresh_kpool_topk_v2_plan(
|
||||
plan=metadata.pooled_topk_v2_plan,
|
||||
pool_lens=metadata.pooled_cache_seqlens_int32,
|
||||
)
|
||||
|
||||
|
||||
def _alloc_kpool_write_plan_buffers(
|
||||
*,
|
||||
@@ -733,6 +743,10 @@ def init_kpool_write_plan_capture(
|
||||
index_page_size=index_page_size,
|
||||
)
|
||||
plan = dataclasses.replace(plan, pool_schedule_metadata=schedule)
|
||||
if is_verify:
|
||||
plan = dataclasses.replace(
|
||||
plan, pool_topk_v2_plan=build_kpool_topk_v2_plan(plan.pool_seqlens_per_q)
|
||||
)
|
||||
return dataclasses.replace(metadata, kpool_write_plan=plan)
|
||||
|
||||
|
||||
@@ -793,6 +807,11 @@ def update_kpool_write_plan(
|
||||
if new_schedule is not None:
|
||||
plan.pool_schedule_metadata.copy_(new_schedule)
|
||||
|
||||
if plan.pool_topk_v2_plan is not None:
|
||||
refresh_kpool_topk_v2_plan(
|
||||
plan=plan.pool_topk_v2_plan, pool_lens=plan.pool_seqlens_per_q
|
||||
)
|
||||
|
||||
|
||||
def init_kpool_write_plan(
|
||||
metadata: DSAMetadata,
|
||||
|
||||
@@ -306,6 +306,7 @@ class DSAMetadata:
|
||||
pooled_cache_seqlens_int32: Optional[torch.Tensor] = None
|
||||
pooled_real_page_table: Optional[torch.Tensor] = None
|
||||
pooled_paged_mqa_schedule_metadata: Optional[torch.Tensor] = None
|
||||
pooled_topk_v2_plan: Optional[torch.Tensor] = None
|
||||
kpool_extend_plan: Optional[KPoolExtendPlan] = None
|
||||
kpool_write_plan: Optional[KPoolWritePlan] = None
|
||||
|
||||
|
||||
@@ -96,13 +96,24 @@ def addresses(metadata):
|
||||
}
|
||||
|
||||
|
||||
def _topk_v2_plan_lengths(metadata, name):
|
||||
if name == "topk_v2_plan":
|
||||
return metadata.dsa_seqlens_expanded
|
||||
if name == "pooled_topk_v2_plan":
|
||||
return metadata.pooled_cache_seqlens_int32
|
||||
if name == "kpool.pool_topk_v2_plan":
|
||||
return metadata.kpool_write_plan.pool_seqlens_per_q
|
||||
return None
|
||||
|
||||
|
||||
def assert_metadata_equal(test, actual, expected):
|
||||
actual_buffers, expected_buffers = tensor_buffers(actual), tensor_buffers(expected)
|
||||
test.assertEqual(actual_buffers.keys(), expected_buffers.keys())
|
||||
for name, value in actual_buffers.items():
|
||||
reference = expected_buffers[name]
|
||||
if name == "topk_v2_plan":
|
||||
if not topk_v2_plan_is_written(expected.dsa_seqlens_expanded):
|
||||
plan_lengths = _topk_v2_plan_lengths(expected, name)
|
||||
if plan_lengths is not None:
|
||||
if not topk_v2_plan_is_written(plan_lengths):
|
||||
continue
|
||||
# Unused plan rows are intentionally uninitialized. Active rows are
|
||||
# compacted by atomicAdd, so compare them in request order.
|
||||
|
||||
Reference in New Issue
Block a user