diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/topk_v2.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/topk_v2.cuh index 751eec4a7a..27b5f61744 100644 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/topk_v2.cuh +++ b/python/sglang/kernels/jit/csrc/deepseek_v4/topk_v2.cuh @@ -37,6 +37,7 @@ enum class TopKMode { INDICES, ///< raw selected indices into `out`; `page_table` unused PAGE_TABLE, ///< page-table-transformed indices into `out` DUAL_OUTPUT, ///< page-table-transformed indices into `out` and raw indices into `raw_indices` + KPOOL, }; using Register2 = impl::TopKRegister<2>; // <= 8192, register-resident, 1 read @@ -85,11 +86,18 @@ struct PageTransform { const int32_t* __restrict__ page_table; uint32_t page_bits; int32_t* __restrict__ raw_out; // the row's raw output, written in DUAL_OUTPUT only + int32_t offset = 0; + uint32_t pool_bits = 0; + uint32_t tail_count = 0; + uint32_t out_cols = 0; SGL_DEVICE int32_t page_to_indices(uint32_t i) const { const uint32_t mask = (1u << page_bits) - 1u; return (page_table[i >> page_bits] << page_bits) | (i & mask); } + SGL_DEVICE int32_t kpool_token(uint32_t i) const { + return page_table != nullptr ? page_to_indices(i) : static_cast(i) + offset; + } }; struct TopKPagedParams { @@ -105,6 +113,11 @@ struct TopKPagedParams { uint32_t page_bits; uint32_t static_cluster_floor; // only used in small batch variant uint32_t batch_size; + int64_t out_stride; + const int32_t* __restrict__ token_seq_lens; + const int32_t* __restrict__ out_offsets; + uint32_t pool_bits; + uint32_t out_cols; SGL_DEVICE const GlobalMetadata& global() const { return *reinterpret_cast(metadata); @@ -116,19 +129,28 @@ struct TopKPagedParams { return metadata[1 + i]; } SGL_DEVICE int32_t* get_output_ptr(uint32_t batch_id) const { - return page_indices + batch_id * static_cast(topk); + return page_indices + batch_id * out_stride; } + template SGL_DEVICE PageTransform get_transform(uint32_t batch_id) const { - return { + auto transform = PageTransform{ page_table == nullptr ? nullptr : page_table + batch_id * page_table_stride, page_bits, raw_indices == nullptr ? nullptr : raw_indices + batch_id * static_cast(topk)}; + if constexpr (kMode == TopKMode::KPOOL) { + const uint32_t pool_mask = (1u << pool_bits) - 1u; + transform.offset = out_offsets == nullptr ? 0 : out_offsets[batch_id]; + transform.pool_bits = pool_bits; + transform.tail_count = + token_seq_lens == nullptr ? 0u : static_cast(token_seq_lens[batch_id]) & pool_mask; + transform.out_cols = out_cols; + } + return transform; } SGL_DEVICE TopKProblem problem(uint32_t batch_id, uint32_t seq_len) const { - const auto k = static_cast(topk); return TopKProblem{ .in = scores + batch_id * score_stride, - .out = page_indices + batch_id * k, + .out = page_indices + batch_id * out_stride, .topk = topk, .seq_len = seq_len, }; @@ -161,10 +183,34 @@ SGL_DEVICE void for_each_item(uint32_t topk, const F& f) { } } +template +SGL_DEVICE void +kpool_expand(uint32_t num_pools, uint32_t topk, int32_t* out, const PageTransform& t, const F& selected) { + const auto pool_bits = t.pool_bits; + const uint32_t pool_mask = (1u << pool_bits) - 1u; + const uint32_t history = min(num_pools, topk) << pool_bits; + const uint32_t tail_end = history + t.tail_count; + const uint32_t tail_base = num_pools << pool_bits; + for (uint32_t col = threadIdx.x; col < t.out_cols; col += kBlockSize) { + int32_t value = -1; + if (col < history) { + const int32_t pool_id = selected(col >> pool_bits); + if (pool_id >= 0) value = t.kpool_token((static_cast(pool_id) << pool_bits) | (col & pool_mask)); + } else if (col < tail_end) { + value = t.kpool_token(tail_base + (col - history)); + } + out[col] = value; + } +} + template SGL_DEVICE void trivial_transform(const TopKProblem& problem, const PageTransform& transform) { device::PDLWaitPrimary(); device::PDLTriggerSecondary(); + if constexpr (kMode == TopKMode::KPOOL) { + return kpool_expand( + problem.seq_len, problem.topk, problem.out, transform, [](uint32_t r) { return static_cast(r); }); + } for_each_item(problem.topk, [&](uint32_t tx, uint32_t) { if constexpr (kMode == TopKMode::INDICES) { problem.out[tx] = tx < problem.seq_len ? static_cast(tx) : -1; @@ -181,6 +227,10 @@ template SGL_DEVICE void paged_transform(const TopKProblem& problem, int32_t* out, const PageTransform& transform) { static_assert(kMode != TopKMode::INDICES, "paged_transform requires page-table output"); static_assert(kMaxTopK % kBlockSize == 0); + if constexpr (kMode == TopKMode::KPOOL) { + const auto staged = problem.out; + return kpool_expand(problem.seq_len, problem.topk, out, transform, [staged](uint32_t r) { return staged[r]; }); + } constexpr uint32_t kNumElems = kMaxTopK / kBlockSize; int32_t indices[kNumElems]; for_each_item(problem.topk, [&](uint32_t tx, uint32_t i) { @@ -272,6 +322,109 @@ TOPK_KERNEL void topk_ragged_kernel(const __grid_constant__ TopKRaggedParams par // PDL trigger secondary at the end the block typically has no use, so ignore it } +struct TopKRaggedKpoolParams { + float* __restrict__ scores; + const int32_t* __restrict__ pool_lens; + const int32_t* __restrict__ row_starts; + const int32_t* __restrict__ token_seq_lens; + const int32_t* __restrict__ page_table; + const int32_t* __restrict__ page_table_row_index; + const int32_t* __restrict__ out_offsets; + int32_t* __restrict__ out; + int64_t score_stride; + int64_t page_table_stride; + int64_t out_stride; + uint32_t topk; + uint32_t page_bits; + uint32_t pool_bits; + uint32_t out_cols; + + SGL_DEVICE PageTransform get_transform(uint32_t bx) const { + const auto table_row = page_table_row_index == nullptr ? bx : static_cast(page_table_row_index[bx]); + const uint32_t pool_mask = (1u << pool_bits) - 1u; + auto transform = PageTransform{ + page_table == nullptr ? nullptr : page_table + static_cast(table_row) * page_table_stride, + page_bits, + nullptr}; + transform.offset = out_offsets == nullptr ? 0 : out_offsets[bx]; + transform.pool_bits = pool_bits; + transform.tail_count = token_seq_lens == nullptr ? 0u : static_cast(token_seq_lens[bx]) & pool_mask; + transform.out_cols = out_cols; + return transform; + } +}; + +using RaggedSmem = impl::MaxSmem; + +// Writes `score` in place: the <= 3 columns ahead of an unaligned window are masked, as in topk_ragged_kernel. +template +SGL_DEVICE void ragged_select_transform( + float* score, + const uint32_t row_start, + const uint32_t seq_len, + const uint32_t topk, + int32_t* out, + const PageTransform& transform, + int32_t* s_topk_indices, + RaggedSmem* smem) { + constexpr uint32_t kVecSize = impl::TopKStreaming::kVecSize; + auto problem = TopKProblem{ + .in = score + row_start, + .out = out, + .topk = topk, + .seq_len = seq_len, + }; + if (seq_len <= topk) { + return trivial_transform(problem, transform); + } + + const auto rem = row_start % kVecSize; + if (rem != 0) { + // The mask has to land after the indexer has retired + // Otherwise it may be accidentally overwritten by DG upstream + device::PDLWaitPrimary(); + static_assert(kVecSize <= kBlockSize, "not enough threads "); + if (const auto tx = threadIdx.x; tx < rem) { + score[row_start - rem + tx] = impl::padding_value(); + } + } + using device::topk::broadcast; + problem.in -= rem; + problem.out = s_topk_indices; // write into stage buffer in smem first + problem.seq_len = seq_len + rem; + problem.bias = broadcast(-static_cast(rem)); + problem.input_start = broadcast(rem); + + if (problem.seq_len <= Register2::kMaxSeqLen) { + Register2::forward(problem, smem); + } else if (problem.seq_len <= Register4::kMaxSeqLen) { + Register4::forward(problem, smem); + } else { + Streaming::forward(problem, smem); + } + device::PDLTriggerSecondary(); + __syncthreads(); + problem.seq_len = seq_len; + paged_transform(problem, out, transform); +} + +template +TOPK_KERNEL void topk_ragged_kpool_kernel(const __grid_constant__ TopKRaggedKpoolParams params) { + device::enable_smem_spilling(); + __shared__ RaggedSmem smem; + __shared__ int32_t s_topk_indices[kMaxTopK]; + const auto bx = blockIdx.x; + ragged_select_transform( + params.scores + bx * params.score_stride, + params.row_starts == nullptr ? 0u : static_cast(params.row_starts[bx]), + static_cast(params.pool_lens[bx]), + params.topk, + params.out + bx * params.out_stride, + params.get_transform(bx), + s_topk_indices, + &smem); +} + #ifdef USE_ROCM // Only the ROCm DSA prefill emits this layout today, so CUDA/XPU builds stay // unchanged. Nothing below is AMD-specific; the guard can be dropped later. @@ -315,57 +468,18 @@ struct TopKPackedParams { template TOPK_KERNEL void topk_packed_kernel(const __grid_constant__ TopKPackedParams params) { device::enable_smem_spilling(); - constexpr uint32_t kVecSize = impl::TopKStreaming::kVecSize; - __shared__ impl::MaxSmem smem; + __shared__ RaggedSmem smem; __shared__ int32_t s_topk_indices[kMaxTopK]; - const auto bx = blockIdx.x; - const auto seq_len = static_cast(params.seq_lens[bx]); - const auto row_start = static_cast(params.row_starts[bx]); - const auto topk = params.topk; - const auto transform = params.get_transform(bx); - const auto out = params.page_indices + bx * static_cast(topk); - const auto score = params.scores + bx * params.score_stride; - - auto problem = TopKProblem{ - .in = score + row_start, - .out = out, - .topk = topk, - .seq_len = seq_len, - }; - if (seq_len <= topk) { - return trivial_transform(problem, transform); - } - - // Round the window down to a `kVecSize` boundary and mask the <= 3 columns - // that pulls in, with the same bias / input_start as `topk_ragged_kernel`. - const auto rem = row_start % kVecSize; - if (rem != 0) { - // The mask has to land after the indexer has retired - // Otherwise it may be accidentally overwritten by DG upstream - device::PDLWaitPrimary(); - static_assert(kVecSize <= kBlockSize, "not enough threads "); - if (const auto tx = threadIdx.x; tx < rem) { - score[row_start - rem + tx] = impl::padding_value(); - } - } - using device::topk::broadcast; - problem.in -= rem; - problem.out = s_topk_indices; // write into stage buffer in smem first - problem.seq_len = seq_len + rem; - problem.bias = broadcast(-static_cast(rem)); - problem.input_start = broadcast(rem); - - if (problem.seq_len <= Register2::kMaxSeqLen) { - Register2::forward(problem, &smem); - } else if (problem.seq_len <= Register4::kMaxSeqLen) { - Register4::forward(problem, &smem); - } else { - Streaming::forward(problem, &smem); - } - device::PDLTriggerSecondary(); - __syncthreads(); - paged_transform(problem, out, transform); + ragged_select_transform( + params.scores + bx * params.score_stride, + static_cast(params.row_starts[bx]), + static_cast(params.seq_lens[bx]), + params.topk, + params.page_indices + bx * static_cast(params.topk), + params.get_transform(bx), + s_topk_indices, + &smem); } #endif // USE_ROCM @@ -395,7 +509,7 @@ TOPK_KERNEL void topk_main_kernel(const __grid_constant__ TopKPagedParams params const auto bx = blockIdx.x; auto problem = params.problem(bx); if (problem.seq_len <= problem.topk) { - return trivial_transform(problem, params.get_transform(bx)); + return trivial_transform(problem, params.template get_transform(bx)); } if constexpr (kNeedStaging) { problem.out = s_topk_indices; // write into stage buffer in smem first @@ -422,7 +536,14 @@ TOPK_KERNEL void topk_main_kernel(const __grid_constant__ TopKPagedParams params device::PDLWaitPrimary(); problem.out = params.get_output_ptr(bx); // in-place transform device::PDLTriggerSecondary(); - return paged_transform(problem, problem.out, params.get_transform(bx)); + if constexpr (kMode == TopKMode::KPOOL) { + for_each_item(problem.topk, [&](uint32_t tx, uint32_t) { s_topk_indices[tx] = problem.out[tx]; }); + __syncthreads(); + const auto out = problem.out; + problem.out = s_topk_indices; + return paged_transform(problem, out, params.template get_transform(bx)); + } + return paged_transform(problem, problem.out, params.template get_transform(bx)); } else { return device::PDLTriggerSecondary(); } @@ -432,7 +553,7 @@ TOPK_KERNEL void topk_main_kernel(const __grid_constant__ TopKPagedParams params device::PDLTriggerSecondary(); if constexpr (kNeedStaging) { __syncthreads(); - paged_transform(problem, params.get_output_ptr(bx), params.get_transform(bx)); + paged_transform(problem, params.get_output_ptr(bx), params.template get_transform(bx)); } } @@ -493,7 +614,7 @@ CLUSTER_TOPK_KERNEL void topk_small_batch_cluster_kernel(const __grid_constant__ const auto worker_rank = bx % kClusterSize; if (problem.seq_len <= problem.topk) { if (by != worker_rank) return; - return trivial_transform(problem, params.get_transform(bx)); + return trivial_transform(problem, params.template get_transform(bx)); } if constexpr (kNeedStaging) { @@ -517,7 +638,7 @@ CLUSTER_TOPK_KERNEL void topk_small_batch_cluster_kernel(const __grid_constant__ cluster.sync(); if (by != 0) return; problem.out = s_topk_indices; - return paged_transform(problem, params.get_output_ptr(bx), params.get_transform(bx)); + return paged_transform(problem, params.get_output_ptr(bx), params.template get_transform(bx)); } else { return device::PDLTriggerSecondary(); } @@ -526,7 +647,7 @@ CLUSTER_TOPK_KERNEL void topk_small_batch_cluster_kernel(const __grid_constant__ device::PDLTriggerSecondary(); if constexpr (kNeedStaging) { __syncthreads(); - paged_transform(problem, params.get_output_ptr(bx), params.get_transform(bx)); + paged_transform(problem, params.get_output_ptr(bx), params.template get_transform(bx)); } } @@ -869,7 +990,7 @@ TOPK_KERNEL void topk_split_select(const __grid_constant__ TopKPagedParams param if (rank != 0) return; __shared__ int32_t s_topk_indices[kNeedStaging ? kMaxTopK : 1]; if (problem.seq_len <= problem.topk) { - return trivial_transform(problem, params.get_transform(row)); + return trivial_transform(problem, params.template get_transform(row)); } if constexpr (kNeedStaging) problem.out = s_topk_indices; if (problem.seq_len <= kReg4MaxSeqLen) { @@ -879,7 +1000,7 @@ TOPK_KERNEL void topk_split_select(const __grid_constant__ TopKPagedParams param } if constexpr (kNeedStaging) { __syncthreads(); - paged_transform(problem, params.get_output_ptr(row), params.get_transform(row)); + paged_transform(problem, params.get_output_ptr(row), params.template get_transform(row)); } return; } @@ -899,7 +1020,7 @@ TOPK_KERNEL void topk_split_select(const __grid_constant__ TopKPagedParams param // problem.out is already the destination, and paged_transform reads every // slot into registers before writing any, so transforming in place is safe. __syncthreads(); - paged_transform(problem, problem.out, params.get_transform(row)); + paged_transform(problem, problem.out, params.template get_transform(row)); } } @@ -1016,6 +1137,89 @@ struct TopKKernel { #endif } + static uint32_t get_static_cluster_floor(uint32_t batch_size) { + // NOTE: 15 is exactly 0.5 wave which saturate all cluster-8 SMs on Hopper/Blackwell + if constexpr (SGL_ARCH_BLACKWELL_OR_GREATER) { + return batch_size <= 15 ? 24576 : 30720; + } else if constexpr (SGL_ARCH_HOPPER_OR_GREATER) { + return batch_size <= 15 ? 32768 : 65536; + } else { + return UINT_MAX; + } + } + + template + static void launch(const TopKPagedParams& params, const uint32_t max_seq_len, const DLDevice device) { + using namespace host; + const auto batch_size = params.batch_size; +#if SUPPORT_CLUSTER + const bool use_cluster = (max_seq_len > params.static_cluster_floor) && (batch_size <= kClusterMaxBatch); + if (use_cluster) { + if constexpr (kMaxCluster16BatchSize > 0) { + if (batch_size <= kMaxCluster16BatchSize) { + constexpr uint32_t kClusterSize = 16; + // Widths above 8 are non-portable; the launch is rejected without this. + const auto kernel = topk_small_batch_cluster_kernel; + const auto kernel_ptr = reinterpret_cast(kernel); + runtime::init_per_device(device, [kernel_ptr] { + CHECK_CUDA(::cudaFuncSetAttribute(kernel_ptr, ::cudaFuncAttributeNonPortableClusterSizeAllowed, 1)); + }); + return LaunchKernel({batch_size, kClusterSize}, kBlockSize, device) + .config({.use_pdl = kUsePDL, .cluster_dim = dim3{1, kClusterSize}}) + .launch(kernel, params); + } + } + + if constexpr (kNumPersistentClusters > 0) { + if (batch_size <= kNumPersistentClusters) { + constexpr uint32_t kClusterSize = 8; + return LaunchKernel({batch_size, kClusterSize}, kBlockSize, device) + .config({.use_pdl = kUsePDL, .cluster_dim = dim3{1, kClusterSize}}) + .launch(topk_small_batch_cluster_kernel, params); + } else { + constexpr uint32_t kClusterSize = 8; + const uint32_t num_clusters = std::min(batch_size, kNumPersistentClusters); + LaunchKernel({num_clusters, kClusterSize}, kBlockSize, device) + .config({.use_pdl = kUsePDL, .cluster_dim = dim3{1, kClusterSize}}) + .launch(topk_persistent_cluster_kernel, params); + LaunchKernel(batch_size, kBlockSize, device) + .config({.use_pdl = kUsePDL}) + .launch(topk_main_kernel, params); + return void(); + } + } + } +#elif defined(USE_ROCM) + // Split dispatch. One block per row leaves a long row latency bound on one + // CU however idle the rest is; split_plan decides where a second launch pays. + // PDL stays off: the hist -> select kernel boundary is the barrier. + if constexpr (kMode != TopKMode::KPOOL) { + if (const auto [split, split_ws] = split_plan(batch_size, max_seq_len, device); split >= kSplitMin) { + LaunchKernel({batch_size, split}, kBlockSize, device) + .config({.use_pdl = false}) + .launch(topk_split_hist, params, split_ws); + LaunchKernel({batch_size, split}, kBlockSize, device) + .config({.use_pdl = false}) + .launch(topk_split_select, params, split_ws); + return; + } + } +#endif + if (max_seq_len <= kReg2MaxSeqLen) { + LaunchKernel(batch_size, kBlockSize, device) + .config({.use_pdl = kUsePDL}) + .launch(topk_main_kernel, params); + } else if (max_seq_len <= kReg4MaxSeqLen) { + LaunchKernel(batch_size, kBlockSize, device) + .config({.use_pdl = kUsePDL}) + .launch(topk_main_kernel, params); + } else { + LaunchKernel(batch_size, kBlockSize, device) + .config({.use_pdl = kUsePDL}) + .launch(topk_main_kernel, params); + } + } + static void transform_paged( const tvm::ffi::TensorView scores, const tvm::ffi::TensorView seq_lens, @@ -1085,17 +1289,6 @@ struct TopKKernel { const auto max_seq_len = static_cast(L.unwrap()); const auto device = device_.unwrap(); - constexpr auto get_static_cluster_floor = [](uint32_t batch_size) -> uint32_t { - // NOTE: 15 is exactly 0.5 wave which saturate all cluster-8 SMs on Hopper/Blackwell - if constexpr (SGL_ARCH_BLACKWELL_OR_GREATER) { - return batch_size <= 15 ? 24576 : 30720; - } else if constexpr (SGL_ARCH_HOPPER_OR_GREATER) { - return batch_size <= 15 ? 32768 : 65536; - } else { - return UINT_MAX; - } - }; - const auto params = TopKPagedParams{ .scores = static_cast(scores.data_ptr()), .seq_lens = static_cast(seq_lens.data_ptr()), @@ -1111,6 +1304,11 @@ struct TopKKernel { .static_cluster_floor = get_static_cluster_floor(batch_size), // used for persistent cluster kernel and main kernel .batch_size = batch_size, + .out_stride = static_cast(topk), + .token_seq_lens = nullptr, + .out_offsets = nullptr, + .pool_bits = 0, + .out_cols = topk, }; const auto dispatch = [&](F&& f) { @@ -1128,72 +1326,116 @@ struct TopKKernel { Panic("Invalid mode, this path should be unreachable"); } }; - dispatch([&]() { -#if SUPPORT_CLUSTER - const bool use_cluster = (max_seq_len > params.static_cluster_floor) && (batch_size <= kClusterMaxBatch); - if (use_cluster) { - if constexpr (kMaxCluster16BatchSize > 0) { - if (batch_size <= kMaxCluster16BatchSize) { - constexpr uint32_t kClusterSize = 16; - // Widths above 8 are non-portable; the launch is rejected without this. - const auto kernel = topk_small_batch_cluster_kernel; - const auto kernel_ptr = reinterpret_cast(kernel); - runtime::init_per_device(device, [kernel_ptr] { - CHECK_CUDA(::cudaFuncSetAttribute(kernel_ptr, ::cudaFuncAttributeNonPortableClusterSizeAllowed, 1)); - }); - return LaunchKernel({batch_size, kClusterSize}, kBlockSize, device) - .config({.use_pdl = kUsePDL, .cluster_dim = dim3{1, kClusterSize}}) - .launch(kernel, params); - } - } + dispatch([&]() { launch(params, max_seq_len, device); }); + } - if constexpr (kNumPersistentClusters > 0) { - if (batch_size <= kNumPersistentClusters) { - constexpr uint32_t kClusterSize = 8; - return LaunchKernel({batch_size, kClusterSize}, kBlockSize, device) - .config({.use_pdl = kUsePDL, .cluster_dim = dim3{1, kClusterSize}}) - .launch(topk_small_batch_cluster_kernel, params); - } else { - constexpr uint32_t kClusterSize = 8; - const uint32_t num_clusters = std::min(batch_size, kNumPersistentClusters); - LaunchKernel({num_clusters, kClusterSize}, kBlockSize, device) - .config({.use_pdl = kUsePDL, .cluster_dim = dim3{1, kClusterSize}}) - .launch(topk_persistent_cluster_kernel, params); - LaunchKernel(batch_size, kBlockSize, device) - .config({.use_pdl = kUsePDL}) - .launch(topk_main_kernel, params); - return void(); - } - } - } -#elif defined(USE_ROCM) - // Split dispatch. One block per row leaves a long row latency bound on one - // CU however idle the rest is; split_plan decides where a second launch pays. - // PDL stays off: the hist -> select kernel boundary is the barrier. - if (const auto [split, split_ws] = split_plan(batch_size, max_seq_len, device); split >= kSplitMin) { - LaunchKernel({batch_size, split}, kBlockSize, device) - .config({.use_pdl = false}) - .launch(topk_split_hist, params, split_ws); - LaunchKernel({batch_size, split}, kBlockSize, device) - .config({.use_pdl = false}) - .launch(topk_split_select, params, split_ws); - return; - } -#endif - if (max_seq_len <= kReg2MaxSeqLen) { - LaunchKernel(batch_size, kBlockSize, device) - .config({.use_pdl = kUsePDL}) - .launch(topk_main_kernel, params); - } else if (max_seq_len <= kReg4MaxSeqLen) { - LaunchKernel(batch_size, kBlockSize, device) - .config({.use_pdl = kUsePDL}) - .launch(topk_main_kernel, params); - } else { - LaunchKernel(batch_size, kBlockSize, device) - .config({.use_pdl = kUsePDL}) - .launch(topk_main_kernel, params); - } - }); + /** + * \brief GLM k-pool variant of `transform_paged`: top-k over per-pool scores, + * fused with pool expansion, tail append and the output transform. + * + * Row `b` selects the `topk` best of `scores[b, :pool_lens[b]]`, where + * `topk = (out.size(1) - tail_cols) / pool_size` and `tail_cols` is + * `pool_size - 1` when `token_seq_lens` is given, else 0. Each selected pool `g` + * expands to tokens `g * pool_size + [0, pool_size)`; then the + * `token_seq_lens[b] % pool_size` tokens of the open tail pool follow. Tokens + * map through `page_table` (page size `page_size`) when given, else get + * `out_offsets[b]` added (0 when absent); the rest of the row is `-1`. + * + * \param scores 16-byte-aligned float32 rows; width and row stride must be multiples of 4. + * \param pool_lens NON-NEGATIVE; `metadata` must be `plan` over these values. + * \param metadata Read before the PDL wait, so it must be written before the kernel + * that produces `scores` is launched, not between that kernel and this one. + */ + static void transform_kpool( + const tvm::ffi::TensorView scores, + const tvm::ffi::TensorView pool_lens, + const tvm::ffi::Optional token_seq_lens, + const tvm::ffi::Optional page_table, + const tvm::ffi::Optional out_offsets, + const tvm::ffi::TensorView out, + const uint32_t page_size, + const uint32_t pool_size, + const tvm::ffi::TensorView metadata) { + using namespace host; + auto B = SymbolicSize{"batch_size"}; + auto L = SymbolicSize{"max_seq_len"}; + auto S = SymbolicSize{"score_stride"}; + auto C = SymbolicSize{"out_cols"}; + auto O = SymbolicSize{"out_stride"}; + auto device_ = SymbolicDevice{}; + device_.set_options(); + + TensorMatcher({B, L}) // score + .with_strides({S, 1}) + .with_dtype() + .with_device(device_) + .ensure_alignment(16) + .verify(scores); + TensorMatcher({B}) // pool_lens + .with_dtype() + .with_device(device_) + .verify(pool_lens); + TensorMatcher({B, C}) // out + .with_strides({O, 1}) + .with_dtype() + .with_device(device_) + .verify(out); + TensorMatcher({-1, 2}) // metadata + .with_dtype() + .with_device(device_) + .verify(metadata); + const int32_t* token_seq_lens_ptr = nullptr; + if (token_seq_lens.has_value()) { + TensorMatcher({B}).with_dtype().with_device(device_).verify(token_seq_lens.value()); + token_seq_lens_ptr = static_cast(token_seq_lens.value().data_ptr()); + } + const int32_t* page_table_ptr = nullptr; + int64_t page_table_stride = 0; + if (page_table.has_value()) { + TensorMatcher({B, -1}).with_strides({-1, 1}).with_dtype().with_device(device_).verify( + page_table.value()); + page_table_ptr = static_cast(page_table.value().data_ptr()); + page_table_stride = page_table.value().stride(0); + } + const int32_t* out_offsets_ptr = nullptr; + if (out_offsets.has_value()) { + RuntimeCheck(!page_table.has_value(), "page_table and out_offsets are mutually exclusive"); + TensorMatcher({B}).with_dtype().with_device(device_).verify(out_offsets.value()); + out_offsets_ptr = static_cast(out_offsets.value().data_ptr()); + } + + RuntimeCheck(std::has_single_bit(page_size), "page_size must be power of 2"); + RuntimeCheck(pool_size > 1 && std::has_single_bit(pool_size), "pool_size must be a power of 2 > 1"); + RuntimeCheck(S.unwrap() % 4 == 0, "score_stride must be a multiple of 4 (16-byte vectorized load)"); + CHECK_HOST(L.unwrap() % 4 == 0) << "score width must be a multiple of 4 (16-byte vectorized load)"; + RuntimeCheck(metadata.size(0) == B.unwrap() + 1, "invalid metadata shape"); + const auto out_cols = static_cast(C.unwrap()); + const auto tail_cols = token_seq_lens.has_value() ? pool_size - 1 : 0u; + RuntimeCheck(out_cols > tail_cols && (out_cols - tail_cols) % pool_size == 0, "invalid out width ", out_cols); + const auto topk = (out_cols - tail_cols) / pool_size; + RuntimeCheck(topk > 0 && topk <= kMaxTopK, "pool-level topk must be in (0, 2048]"); + + const auto batch_size = static_cast(B.unwrap()); + const auto params = TopKPagedParams{ + .scores = static_cast(scores.data_ptr()), + .seq_lens = static_cast(pool_lens.data_ptr()), + .page_table = page_table_ptr, + .page_indices = static_cast(out.data_ptr()), + .raw_indices = nullptr, + .metadata = static_cast(metadata.data_ptr()), + .score_stride = S.unwrap(), + .page_table_stride = page_table_stride, + .topk = topk, + .page_bits = static_cast(std::countr_zero(page_size)), + .static_cluster_floor = get_static_cluster_floor(batch_size), + .batch_size = batch_size, + .out_stride = O.unwrap(), + .token_seq_lens = token_seq_lens_ptr, + .out_offsets = out_offsets_ptr, + .pool_bits = static_cast(std::countr_zero(pool_size)), + .out_cols = out_cols, + }; + launch(params, static_cast(L.unwrap()), device_.unwrap()); } /** @@ -1267,6 +1509,90 @@ struct TopKKernel { .launch(topk_ragged_kernel, params); } + /** + * \brief Ragged (prefill) variant of `transform_kpool`: per-row window, no plan. + * + * Row `b` selects among `scores[b, ks : ks + pool_lens[b]]` (`ks = row_starts[b]`, + * 0 when absent), with the same expansion, tail and output transform as + * `transform_kpool`. `page_table_row_index[b]` picks the page-table row when + * several score rows share one; without it `page_table` needs one row per score row. + * + * `scores` is written in place: the <= 3 columns ahead of an unaligned window + * are masked, as in `transform_ragged`. The buffer must have no other consumer. + */ + static void transform_kpool_ragged( + const tvm::ffi::TensorView scores, + const tvm::ffi::TensorView pool_lens, + const tvm::ffi::Optional row_starts, + const tvm::ffi::Optional token_seq_lens, + const tvm::ffi::Optional page_table, + const tvm::ffi::Optional page_table_row_index, + const tvm::ffi::Optional out_offsets, + const tvm::ffi::TensorView out, + const uint32_t page_size, + const uint32_t pool_size) { + using namespace host; + auto B = SymbolicSize{"batch_size"}; + auto S = SymbolicSize{"score_stride"}; + auto C = SymbolicSize{"out_cols"}; + auto O = SymbolicSize{"out_stride"}; + auto device_ = SymbolicDevice{}; + device_.set_options(); + + TensorMatcher({B, -1}).with_strides({S, 1}).with_dtype().with_device(device_).ensure_alignment(16).verify( + scores); + TensorMatcher({B}).with_dtype().with_device(device_).verify(pool_lens); + TensorMatcher({B, C}).with_strides({O, 1}).with_dtype().with_device(device_).verify(out); + const auto optional_rows = [&](const tvm::ffi::Optional& t) -> const int32_t* { + if (!t.has_value()) return nullptr; + TensorMatcher({B}).with_dtype().with_device(device_).verify(t.value()); + return static_cast(t.value().data_ptr()); + }; + const int32_t* page_table_ptr = nullptr; + int64_t page_table_stride = 0; + if (page_table.has_value()) { + TensorMatcher({-1, -1}).with_strides({-1, 1}).with_dtype().with_device(device_).verify( + page_table.value()); + if (!page_table_row_index.has_value()) { + RuntimeCheck(page_table.value().size(0) == B.unwrap(), "page_table needs one row per score row"); + } + page_table_ptr = static_cast(page_table.value().data_ptr()); + page_table_stride = page_table.value().stride(0); + } + RuntimeCheck(!page_table_row_index.has_value() || page_table.has_value(), "page_table_row_index needs page_table"); + RuntimeCheck(!(page_table.has_value() && out_offsets.has_value()), "page_table and out_offsets are exclusive"); + + RuntimeCheck(std::has_single_bit(page_size), "page_size must be power of 2"); + RuntimeCheck(pool_size > 1 && std::has_single_bit(pool_size), "pool_size must be a power of 2 > 1"); + RuntimeCheck(S.unwrap() % 4 == 0, "score_stride must be a multiple of 4 (16-byte vectorized load)"); + const auto out_cols = static_cast(C.unwrap()); + const auto tail_cols = token_seq_lens.has_value() ? pool_size - 1 : 0u; + RuntimeCheck(out_cols > tail_cols && (out_cols - tail_cols) % pool_size == 0, "invalid out width ", out_cols); + const auto topk = (out_cols - tail_cols) / pool_size; + RuntimeCheck(topk > 0 && topk <= kMaxTopK, "pool-level topk must be in (0, 2048]"); + + const auto params = TopKRaggedKpoolParams{ + .scores = static_cast(scores.data_ptr()), + .pool_lens = static_cast(pool_lens.data_ptr()), + .row_starts = optional_rows(row_starts), + .token_seq_lens = optional_rows(token_seq_lens), + .page_table = page_table_ptr, + .page_table_row_index = optional_rows(page_table_row_index), + .out_offsets = optional_rows(out_offsets), + .out = static_cast(out.data_ptr()), + .score_stride = S.unwrap(), + .page_table_stride = page_table_stride, + .out_stride = O.unwrap(), + .topk = topk, + .page_bits = static_cast(std::countr_zero(page_size)), + .pool_bits = static_cast(std::countr_zero(pool_size)), + .out_cols = out_cols, + }; + LaunchKernel(static_cast(B.unwrap()), kBlockSize, device_.unwrap()) + .config({.use_pdl = kUsePDL}) + .launch(topk_ragged_kpool_kernel, params); + } + #ifdef USE_ROCM // see the packed kernel above /** * \brief Packed (DSA extend prefill) variant of `transform_paged`: per-row diff --git a/python/sglang/kernels/ops/attention/__init__.py b/python/sglang/kernels/ops/attention/__init__.py index f8fe9fc3de..e982c4ffac 100644 --- a/python/sglang/kernels/ops/attention/__init__.py +++ b/python/sglang/kernels/ops/attention/__init__.py @@ -304,6 +304,22 @@ register_kernel( capabilities=CUDA_ONLY, ) ) +register_kernel( + KernelSpec( + op="attention.topk_transform_kpool_v2", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.attention.dsv4.topk:topk_transform_kpool_v2", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) +register_kernel( + KernelSpec( + op="attention.topk_transform_kpool_ragged_v2", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.attention.dsv4.topk:topk_transform_kpool_ragged_v2", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) register_kernel( KernelSpec( op="attention.fast_topk", diff --git a/python/sglang/kernels/ops/attention/dsv4/__init__.py b/python/sglang/kernels/ops/attention/dsv4/__init__.py index 3b4e6647ab..faac4666d1 100644 --- a/python/sglang/kernels/ops/attention/dsv4/__init__.py +++ b/python/sglang/kernels/ops/attention/dsv4/__init__.py @@ -25,6 +25,8 @@ from .elementwise import ( from .fp8_wo_a import sglang_per_token_group_quant_fp8_dsv4_wo_a from .topk import ( plan_topk_v2, + topk_transform_kpool_ragged_v2, + topk_transform_kpool_v2, topk_transform_paged, topk_transform_paged_v2, topk_transform_ragged_v2, @@ -50,6 +52,8 @@ __all__ = [ "get_paged_mqa_logits_metadata", "triton_create_paged_compress_data", "topk_transform_paged", + "topk_transform_kpool_v2", + "topk_transform_kpool_ragged_v2", "topk_transform_paged_v2", "topk_transform_ragged_v2", "plan_topk_v2", diff --git a/python/sglang/kernels/ops/attention/dsv4/topk.py b/python/sglang/kernels/ops/attention/dsv4/topk.py index a68cb99bb9..24c3bbd963 100644 --- a/python/sglang/kernels/ops/attention/dsv4/topk.py +++ b/python/sglang/kernels/ops/attention/dsv4/topk.py @@ -54,6 +54,8 @@ def _jit_topk_v2_module(): ("topk_transform_paged", f"{kernel}::transform_paged"), ("topk_transform_ragged", f"{kernel}::transform_ragged"), ("topk_plan", f"{kernel}::plan"), + ("topk_transform_kpool", f"{kernel}::transform_kpool"), + ("topk_transform_kpool_ragged", f"{kernel}::transform_kpool_ragged"), ] if is_hip_runtime(): # transform_packed only exists under USE_ROCM, see topk_v2.cuh @@ -153,7 +155,12 @@ def topk_transform_paged_torch( _PLAN_METADATA_INTS_PER_BATCH = 2 -def plan_topk_v2(seq_lens: torch.Tensor, static_threshold: int = -1) -> torch.Tensor: +def plan_topk_v2( + seq_lens: torch.Tensor, + static_threshold: int = -1, + *, + out: Optional[torch.Tensor] = None, +) -> torch.Tensor: """ Preprocess the per-batch routing plan for :func:`topk_transform_paged_v2`. NOTE: every entry of ``seq_lens`` must be NON-NEGATIVE. @@ -164,7 +171,11 @@ def plan_topk_v2(seq_lens: torch.Tensor, static_threshold: int = -1) -> torch.Te """ module = _jit_topk_v2_module() bs = seq_lens.shape[0] - metadata = seq_lens.new_empty(bs + 1, _PLAN_METADATA_INTS_PER_BATCH) + metadata = ( + seq_lens.new_empty(bs + 1, _PLAN_METADATA_INTS_PER_BATCH) + if out is None + else out + ) module.topk_plan(seq_lens, metadata, static_threshold) return metadata @@ -281,6 +292,70 @@ def topk_transform_paged_v2( ) +def topk_transform_kpool_v2( + scores: torch.Tensor, + pool_lens: torch.Tensor, + out: torch.Tensor, + pool_size: int, + metadata: torch.Tensor, + *, + token_seq_lens: Optional[torch.Tensor] = None, + page_table: Optional[torch.Tensor] = None, + page_size: int = 1, + out_offsets: Optional[torch.Tensor] = None, +) -> None: + """GLM k-pool top-k over per-pool scores, fused with pool expansion, the + ``token_seq_lens % pool_size`` tail and the page-table or offset transform. + + ``metadata`` is :func:`plan_topk_v2` over ``pool_lens``. The kernel reads it + before its PDL wait, so plan before launching the kernel that writes ``scores``. + """ + _jit_topk_v2_module().topk_transform_kpool( + scores, + pool_lens, + token_seq_lens, + page_table, + out_offsets, + out, + page_size, + pool_size, + metadata, + ) + + +def topk_transform_kpool_ragged_v2( + scores: torch.Tensor, + pool_lens: torch.Tensor, + out: torch.Tensor, + pool_size: int, + *, + row_starts: Optional[torch.Tensor] = None, + token_seq_lens: Optional[torch.Tensor] = None, + page_table: Optional[torch.Tensor] = None, + page_table_row_index: Optional[torch.Tensor] = None, + page_size: int = 1, + out_offsets: Optional[torch.Tensor] = None, +) -> None: + """Ragged (prefill) :func:`topk_transform_kpool_v2`: row ``i`` selects among + ``scores[i, ks : ks + pool_lens[i]]`` (``ks = row_starts[i]``, 0 when omitted); + no plan is needed. + + NOTE: ``scores`` is written in place, as in :func:`topk_transform_ragged_v2`. + """ + _jit_topk_v2_module().topk_transform_kpool_ragged( + scores, + pool_lens, + row_starts, + token_seq_lens, + page_table, + page_table_row_index, + out_offsets, + out, + page_size, + pool_size, + ) + + def topk_transform_packed_v2( scores: torch.Tensor, seq_lens: torch.Tensor, diff --git a/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py b/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py index 8aae41a421..0a25452985 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py @@ -75,6 +75,14 @@ def _should_fuse_kpool_topk(metadata: BaseIndexerMetadata) -> bool: ) +def _topk_v2_plan_for( + *, owned_plan: Optional[torch.Tensor], pool_lens: torch.Tensor +) -> Optional[torch.Tensor]: + if owned_plan is not None and owned_plan.shape[0] == pool_lens.shape[0] + 1: + return owned_plan + return None + + def _slice_rows( *, tensor: Optional[torch.Tensor], rows: slice ) -> Optional[torch.Tensor]: @@ -763,6 +771,7 @@ class IndexerKPool(MultiPlatformOp): row_starts: Optional[torch.Tensor] = None, out_rows: Optional[int] = None, page_table_row_index: Optional[torch.Tensor] = None, + topk_v2_plan: Optional[torch.Tensor] = None, ) -> torch.Tensor: from sglang.srt.layers.attention.dsa.kpool_fp8_index import ( topk_from_pooled_history_logits, @@ -791,6 +800,7 @@ class IndexerKPool(MultiPlatformOp): row_starts=row_starts, out_rows=out_rows, page_table_row_index=page_table_row_index, + topk_v2_plan=topk_v2_plan, ) def _get_kpool_decode_metadata( @@ -800,7 +810,13 @@ class IndexerKPool(MultiPlatformOp): seqlens_32: torch.Tensor, blocksize: int, build_schedule_metadata: bool = True, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + ) -> Tuple[ + torch.Tensor, + torch.Tensor, + torch.Tensor, + Optional[torch.Tensor], + Optional[torch.Tensor], + ]: from sglang.srt.layers.attention.dsa.kpool_fp8_index import ( build_pooled_page_table_64, ) @@ -828,9 +844,13 @@ class IndexerKPool(MultiPlatformOp): pool_context_lens, pool_block_tables, pool_schedule_metadata, + _topk_v2_plan_for( + owned_plan=plan.pool_topk_v2_plan, pool_lens=pool_seqlens + ), ) pool_seqlens = attn_metadata.pooled_cache_seqlens_int32 + topk_v2_plan = attn_metadata.pooled_topk_v2_plan pool_block_tables = attn_metadata.pooled_real_page_table pool_schedule_metadata = attn_metadata.pooled_paged_mqa_schedule_metadata @@ -846,6 +866,7 @@ class IndexerKPool(MultiPlatformOp): block_tables, self.index_kpool ).contiguous() pool_schedule_metadata = None + topk_v2_plan = None else: pool_seqlens = pool_seqlens[: seqlens_32.shape[0]] pool_block_tables = pool_block_tables[ @@ -864,6 +885,7 @@ class IndexerKPool(MultiPlatformOp): pool_context_lens, pool_block_tables, pool_schedule_metadata, + _topk_v2_plan_for(owned_plan=topk_v2_plan, pool_lens=pool_seqlens), ) @staticmethod @@ -1003,16 +1025,18 @@ class IndexerKPool(MultiPlatformOp): and self._should_use_tilelang_paged_mqa_logits(q_fp8) ) - pool_seqlens, pool_context_lens, pool_block_tables, pool_schedule_metadata = ( - self._get_kpool_decode_metadata( - metadata, - block_tables, - seqlens_32, - blocksize, - build_schedule_metadata=not ( - use_aiter_paged_mqa or use_tilelang_paged_mqa - ), - ) + ( + pool_seqlens, + pool_context_lens, + pool_block_tables, + pool_schedule_metadata, + topk_v2_plan, + ) = self._get_kpool_decode_metadata( + metadata=metadata, + block_tables=block_tables, + seqlens_32=seqlens_32, + blocksize=blocksize, + build_schedule_metadata=not (use_aiter_paged_mqa or use_tilelang_paged_mqa), ) pool_max_seq_len = pool_block_tables.shape[1] * blocksize if use_aiter_paged_mqa: @@ -1065,8 +1089,9 @@ class IndexerKPool(MultiPlatformOp): page_table_1, topk_offsets, _ = self._kpool_fused_topk_mapping(metadata) topk_result = self._topk_from_kpool_logits( - logits, - pool_seqlens, + logits=logits, + pool_lens=pool_seqlens, + topk_v2_plan=topk_v2_plan, seq_lens=seqlens_32, page_table=page_table_1, topk_offsets=topk_offsets, diff --git a/python/sglang/srt/layers/attention/dsa/dsa_metadata_manager.py b/python/sglang/srt/layers/attention/dsa/dsa_metadata_manager.py index 733df7a87f..1a84ccb509 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_metadata_manager.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_metadata_manager.py @@ -214,6 +214,7 @@ class DSAMetadataManagementMixin: dst.pooled_paged_mqa_schedule_metadata, src.pooled_paged_mqa_schedule_metadata, ) + and _match(dst.pooled_topk_v2_plan, src.pooled_topk_v2_plan) and _match(dst.kpool_write_plan, src.kpool_write_plan) ): return False @@ -223,6 +224,7 @@ class DSAMetadataManagementMixin: and _match(dst_plan.seqlens_per_q, src_plan.seqlens_per_q) and _match(dst_plan.pool_schedule_metadata, src_plan.pool_schedule_metadata) and _match(dst_plan.effective_n_per_batch, src_plan.effective_n_per_batch) + and _match(dst_plan.pool_topk_v2_plan, src_plan.pool_topk_v2_plan) ): return False return True @@ -298,6 +300,8 @@ class DSAMetadataManagementMixin: metadata.pooled_paged_mqa_schedule_metadata.copy_( src_metadata.pooled_paged_mqa_schedule_metadata ) + if metadata.pooled_topk_v2_plan is not None: + metadata.pooled_topk_v2_plan.copy_(src_metadata.pooled_topk_v2_plan) dst_plan = metadata.kpool_write_plan src_plan = src_metadata.kpool_write_plan @@ -315,3 +319,5 @@ class DSAMetadataManagementMixin: dst_plan.pool_schedule_metadata.copy_(src_plan.pool_schedule_metadata) if dst_plan.effective_n_per_batch is not None: dst_plan.effective_n_per_batch.copy_(src_plan.effective_n_per_batch) + if dst_plan.pool_topk_v2_plan is not None: + dst_plan.pool_topk_v2_plan.copy_(src_plan.pool_topk_v2_plan) diff --git a/python/sglang/srt/layers/attention/dsa/kpool_fp8_index.py b/python/sglang/srt/layers/attention/dsa/kpool_fp8_index.py index 614b93ef22..af3b3845f2 100644 --- a/python/sglang/srt/layers/attention/dsa/kpool_fp8_index.py +++ b/python/sglang/srt/layers/attention/dsa/kpool_fp8_index.py @@ -4,10 +4,14 @@ import torch import triton import triton.language as tl +from sglang.srt.environ import envs from sglang.srt.layers.attention.dsa.utils import ( INDEXER_K_CACHE_PRESHUFFLE_TILE, aiter_can_use_preshuffle_paged_mqa, ) +from sglang.srt.utils import is_cuda + +_is_cuda = is_cuda() BLOCK_SIZE_K = 64 INDEX_HEAD_DIM = 128 @@ -586,6 +590,177 @@ def _append_kpool_tail_to_topk_kernel( tl.store(out_ptr + row * out_stride_0 + cols * out_stride_1, value, mask=mask) +def _kpool_topk_v2_enabled() -> bool: + return _is_cuda and envs.SGLANG_OPT_USE_TOPK_V2.get() + + +def build_kpool_topk_v2_plan(pool_lens: torch.Tensor) -> Optional[torch.Tensor]: + if not _kpool_topk_v2_enabled(): + return None + plan = pool_lens.new_empty((pool_lens.shape[0] + 1, 2), dtype=torch.int32) + refresh_kpool_topk_v2_plan(plan=plan, pool_lens=pool_lens) + return plan + + +def refresh_kpool_topk_v2_plan(*, plan: torch.Tensor, pool_lens: torch.Tensor) -> None: + from sglang.kernels.ops.attention.dsv4.topk import plan_topk_v2 + + plan_topk_v2(pool_lens.to(torch.int32), out=plan) + + +def _kpool_topk_v2_layout_ok(*, logits: torch.Tensor, pool_size: int) -> bool: + return ( + _kpool_topk_v2_enabled() + and logits.dtype == torch.float32 + and logits.stride(1) == 1 + and logits.stride(0) % 4 == 0 + and logits.data_ptr() % 16 == 0 + and pool_size > 1 + and pool_size & (pool_size - 1) == 0 + ) + + +def _can_use_kpool_topk_v2( + *, + logits: torch.Tensor, + pool_size: int, + row_starts: torch.Tensor | None, + page_table_row_index: torch.Tensor | None, +) -> bool: + return ( + row_starts is None + and page_table_row_index is None + and logits.shape[1] % 4 == 0 + and _kpool_topk_v2_layout_ok(logits=logits, pool_size=pool_size) + ) + + +def _alloc_kpool_topk_out( + *, logits: torch.Tensor, rows: int, out_cols: int, out_rows: int | None +) -> torch.Tensor: + if out_rows is None or out_rows == rows: + return logits.new_empty((rows, out_cols), dtype=torch.int32) + return logits.new_full((out_rows, out_cols), -1, dtype=torch.int32) + + +def _kpool_topk_v2_ragged( + *, + logits: torch.Tensor, + group_lengths: torch.Tensor, + pool_size: int, + topk: int, + page_table: torch.Tensor | None, + page_table_row_index: torch.Tensor | None, + topk_offsets: torch.Tensor | None, + seq_lens: torch.Tensor | None, + row_starts: torch.Tensor | None, + out_rows: int | None, +) -> torch.Tensor: + from sglang.kernels.ops.attention.dsv4.topk import topk_transform_kpool_ragged_v2 + + rows = logits.shape[0] + out_cols = topk + (pool_size - 1 if seq_lens is not None else 0) + full = _alloc_kpool_topk_out( + logits=logits, rows=rows, out_cols=out_cols, out_rows=out_rows + ) + topk_transform_kpool_ragged_v2( + scores=logits, + pool_lens=group_lengths.to(torch.int32), + out=full[:rows], + pool_size=pool_size, + row_starts=None if row_starts is None else row_starts.to(torch.int32), + token_seq_lens=None if seq_lens is None else seq_lens.to(torch.int32), + page_table=page_table, + page_table_row_index=( + None + if page_table_row_index is None + else page_table_row_index.to(torch.int32) + ), + out_offsets=topk_offsets, + ) + return full + + +def _kpool_topk_v2( + *, + logits: torch.Tensor, + group_lengths: torch.Tensor, + pool_size: int, + topk: int, + page_table: torch.Tensor | None, + topk_offsets: torch.Tensor | None, + seq_lens: torch.Tensor | None, + out_rows: int | None, + topk_v2_plan: torch.Tensor, +) -> torch.Tensor: + from sglang.kernels.ops.attention.dsv4.topk import topk_transform_kpool_v2 + + rows = logits.shape[0] + group_lengths = group_lengths.to(torch.int32) + out_cols = topk + (pool_size - 1 if seq_lens is not None else 0) + full = _alloc_kpool_topk_out( + logits=logits, rows=rows, out_cols=out_cols, out_rows=out_rows + ) + topk_transform_kpool_v2( + scores=logits, + pool_lens=group_lengths, + out=full[:rows], + pool_size=pool_size, + metadata=topk_v2_plan, + token_seq_lens=None if seq_lens is None else seq_lens.to(torch.int32), + page_table=page_table, + out_offsets=topk_offsets, + ) + return full + + +def _try_kpool_topk_v2( + *, + logits: torch.Tensor, + group_lengths: torch.Tensor, + kpool: int, + topk: int, + page_table: torch.Tensor | None, + page_table_row_index: torch.Tensor | None, + topk_offsets: torch.Tensor | None, + seq_lens: torch.Tensor | None, + row_starts: torch.Tensor | None, + out_rows: int | None, + topk_v2_plan: torch.Tensor | None, +) -> torch.Tensor | None: + if topk_v2_plan is not None and _can_use_kpool_topk_v2( + logits=logits, + pool_size=kpool, + row_starts=row_starts, + page_table_row_index=page_table_row_index, + ): + return _kpool_topk_v2( + logits=logits, + group_lengths=group_lengths, + pool_size=kpool, + topk=topk, + page_table=page_table, + topk_offsets=topk_offsets, + seq_lens=seq_lens, + out_rows=out_rows, + topk_v2_plan=topk_v2_plan, + ) + if _kpool_topk_v2_layout_ok(logits=logits, pool_size=kpool): + return _kpool_topk_v2_ragged( + logits=logits, + group_lengths=group_lengths, + pool_size=kpool, + topk=topk, + page_table=page_table, + page_table_row_index=page_table_row_index, + topk_offsets=topk_offsets, + seq_lens=seq_lens, + row_starts=row_starts, + out_rows=out_rows, + ) + return None + + def topk_from_pooled_history_logits( logits: torch.Tensor, group_lengths: torch.Tensor, @@ -597,6 +772,7 @@ def topk_from_pooled_history_logits( row_starts: torch.Tensor | None = None, out_rows: int | None = None, page_table_row_index: torch.Tensor | None = None, + topk_v2_plan: torch.Tensor | None = None, ) -> torch.Tensor: assert logits.ndim == 2 assert group_lengths.ndim == 1 @@ -606,7 +782,6 @@ def topk_from_pooled_history_logits( assert out_rows is None or out_rows >= logits.shape[0] assert page_table_row_index is None or page_table is not None - _, cols = logits.shape group_topk = history_group_budget_for_topk(topk, kpool) if topk_offsets is not None and topk_offsets.ndim == 2: assert topk_offsets.shape[1] == 1 @@ -624,6 +799,22 @@ def topk_from_pooled_history_logits( f"is disabled. Got device={logits.device}, dtype={logits.dtype}." ) + v2_result = _try_kpool_topk_v2( + logits=logits, + group_lengths=group_lengths, + kpool=kpool, + topk=topk, + page_table=page_table, + page_table_row_index=page_table_row_index, + topk_offsets=topk_offsets, + seq_lens=seq_lens, + row_starts=row_starts, + out_rows=out_rows, + topk_v2_plan=topk_v2_plan, + ) + if v2_result is not None: + return v2_result + if group_topk in (128, 160, 192, 224, 256, 512): from sglang.kernels.ops.attention.dsa.kpool_topk_transform import ( fast_kpool_topk_transform_fused, @@ -648,6 +839,35 @@ def topk_from_pooled_history_logits( padded[: result.shape[0]] = result return padded + return _kpool_topk_unfused( + logits=logits, + group_lengths=group_lengths, + kpool=kpool, + topk=topk, + group_topk=group_topk, + page_table=page_table, + page_table_row_index=page_table_row_index, + topk_offsets=topk_offsets, + seq_lens=seq_lens, + row_starts=row_starts, + out_rows=out_rows, + ) + + +def _kpool_topk_unfused( + *, + logits: torch.Tensor, + group_lengths: torch.Tensor, + kpool: int, + topk: int, + group_topk: int, + page_table: torch.Tensor | None, + page_table_row_index: torch.Tensor | None, + topk_offsets: torch.Tensor | None, + seq_lens: torch.Tensor | None, + row_starts: torch.Tensor | None, + out_rows: int | None, +) -> torch.Tensor: assert page_table_row_index is None, ( "page_table_row_index requires the fused fast_kpool group_topk path" ) @@ -662,7 +882,7 @@ def topk_from_pooled_history_logits( ) rank = torch.arange(group_topk, device=logits.device, dtype=torch.int32) - max_valid_groups = min(cols, group_topk) + max_valid_groups = min(logits.shape[1], group_topk) valid_counts = torch.minimum( group_lengths.to(torch.int32), torch.full_like(group_lengths.to(torch.int32), max_valid_groups), diff --git a/python/sglang/srt/layers/attention/dsa/kpool_plan.py b/python/sglang/srt/layers/attention/dsa/kpool_plan.py index 3902a0a840..c6bbc8cc2f 100644 --- a/python/sglang/srt/layers/attention/dsa/kpool_plan.py +++ b/python/sglang/srt/layers/attention/dsa/kpool_plan.py @@ -9,9 +9,11 @@ import torch from sglang.srt.environ import envs from sglang.srt.layers.attention.dsa.kpool_fp8_index import ( INDEX_HEAD_DIM, + build_kpool_topk_v2_plan, build_pooled_page_table_64, kpool_build_ragged_layout, kpool_max_closed_pools, + refresh_kpool_topk_v2_plan, update_kpool_write_plan_cuda_graph, ) from sglang.srt.layers.attention.dsa.utils import dsa_use_prefill_cp @@ -115,6 +117,7 @@ class KPoolWritePlan: seqlens_per_q: Optional[torch.Tensor] = None pool_schedule_metadata: Optional[torch.Tensor] = None effective_n_per_batch: Optional[torch.Tensor] = None + pool_topk_v2_plan: Optional[torch.Tensor] = None def _is_kpool_layout_enabled(kpool: int, physical_page_size: int) -> bool: @@ -614,6 +617,7 @@ def init_pooled_paged_mqa_metadata( pooled_cache_seqlens_int32=pool_seqlens, pooled_real_page_table=pooled_page_table, pooled_paged_mqa_schedule_metadata=schedule, + pooled_topk_v2_plan=build_kpool_topk_v2_plan(pool_seqlens), ) @@ -669,6 +673,12 @@ def update_pooled_paged_mqa_metadata( if new_schedule is not None: metadata.pooled_paged_mqa_schedule_metadata.copy_(new_schedule) + if metadata.pooled_topk_v2_plan is not None: + refresh_kpool_topk_v2_plan( + plan=metadata.pooled_topk_v2_plan, + pool_lens=metadata.pooled_cache_seqlens_int32, + ) + def _alloc_kpool_write_plan_buffers( *, @@ -733,6 +743,10 @@ def init_kpool_write_plan_capture( index_page_size=index_page_size, ) plan = dataclasses.replace(plan, pool_schedule_metadata=schedule) + if is_verify: + plan = dataclasses.replace( + plan, pool_topk_v2_plan=build_kpool_topk_v2_plan(plan.pool_seqlens_per_q) + ) return dataclasses.replace(metadata, kpool_write_plan=plan) @@ -793,6 +807,11 @@ def update_kpool_write_plan( if new_schedule is not None: plan.pool_schedule_metadata.copy_(new_schedule) + if plan.pool_topk_v2_plan is not None: + refresh_kpool_topk_v2_plan( + plan=plan.pool_topk_v2_plan, pool_lens=plan.pool_seqlens_per_q + ) + def init_kpool_write_plan( metadata: DSAMetadata, diff --git a/python/sglang/srt/layers/attention/dsa_backend.py b/python/sglang/srt/layers/attention/dsa_backend.py index ae1de8455e..f98fbc8440 100644 --- a/python/sglang/srt/layers/attention/dsa_backend.py +++ b/python/sglang/srt/layers/attention/dsa_backend.py @@ -306,6 +306,7 @@ class DSAMetadata: pooled_cache_seqlens_int32: Optional[torch.Tensor] = None pooled_real_page_table: Optional[torch.Tensor] = None pooled_paged_mqa_schedule_metadata: Optional[torch.Tensor] = None + pooled_topk_v2_plan: Optional[torch.Tensor] = None kpool_extend_plan: Optional[KPoolExtendPlan] = None kpool_write_plan: Optional[KPoolWritePlan] = None diff --git a/python/sglang/test/kits/dsa_metadata_kit.py b/python/sglang/test/kits/dsa_metadata_kit.py index 6843e7a39e..56c84a283b 100644 --- a/python/sglang/test/kits/dsa_metadata_kit.py +++ b/python/sglang/test/kits/dsa_metadata_kit.py @@ -96,13 +96,24 @@ def addresses(metadata): } +def _topk_v2_plan_lengths(metadata, name): + if name == "topk_v2_plan": + return metadata.dsa_seqlens_expanded + if name == "pooled_topk_v2_plan": + return metadata.pooled_cache_seqlens_int32 + if name == "kpool.pool_topk_v2_plan": + return metadata.kpool_write_plan.pool_seqlens_per_q + return None + + def assert_metadata_equal(test, actual, expected): actual_buffers, expected_buffers = tensor_buffers(actual), tensor_buffers(expected) test.assertEqual(actual_buffers.keys(), expected_buffers.keys()) for name, value in actual_buffers.items(): reference = expected_buffers[name] - if name == "topk_v2_plan": - if not topk_v2_plan_is_written(expected.dsa_seqlens_expanded): + plan_lengths = _topk_v2_plan_lengths(expected, name) + if plan_lengths is not None: + if not topk_v2_plan_is_written(plan_lengths): continue # Unused plan rows are intentionally uninitialized. Active rows are # compacted by atomicAdd, so compare them in request order. diff --git a/test/registered/kernels/ops/attention/test_dsa_metadata_replay.py b/test/registered/kernels/ops/attention/test_dsa_metadata_replay.py index 8e203cb1f0..63e99f0fea 100644 --- a/test/registered/kernels/ops/attention/test_dsa_metadata_replay.py +++ b/test/registered/kernels/ops/attention/test_dsa_metadata_replay.py @@ -56,7 +56,7 @@ class TestDSAMetadataReplay(CustomTestCase): self.assertEqual(pointers, addresses(fused.forward_metadata)) if mode.is_target_verify(): metadata = fused.forward_metadata - _, _, table, _ = IndexerKPool._get_kpool_decode_metadata( + _, _, table, _, _ = IndexerKPool._get_kpool_decode_metadata( SimpleNamespace(index_kpool=POOL), SimpleNamespace(attn_metadata=metadata), metadata.real_page_table, diff --git a/test/registered/unit/layers/test_kpool_stream_scheduling.py b/test/registered/unit/layers/test_kpool_stream_scheduling.py index c059fda4cf..c0f5b0779a 100644 --- a/test/registered/unit/layers/test_kpool_stream_scheduling.py +++ b/test/registered/unit/layers/test_kpool_stream_scheduling.py @@ -417,6 +417,7 @@ def _encoding_topk(*, width, calls): row_starts, out_rows, page_table_row_index, + topk_v2_plan=None, ): calls.append( {"page_table": page_table, "page_table_row_index": page_table_row_index}