diff --git a/python/sglang/kernels/jit/csrc/deep_select/vendor/cuda_kernels/common_parts.cuh b/python/sglang/kernels/jit/csrc/deep_select/vendor/cuda_kernels/common_parts.cuh index ae7748428d..ec33a9b052 100644 --- a/python/sglang/kernels/jit/csrc/deep_select/vendor/cuda_kernels/common_parts.cuh +++ b/python/sglang/kernels/jit/csrc/deep_select/vendor/cuda_kernels/common_parts.cuh @@ -649,9 +649,10 @@ public: // Save result to `smem.reconstruct_pivot_bucket` and `smem.reconstruct_num_should_select` // If `CHECK_IF_SHOULD_SELECT_WHOLE_BUCKET`, return whether we should select all elements in that bucket. // Should be called by a whole warp - template + // [SGLang] Any plan with `reconstruct_pivot_bucket` and `reconstruct_num_should_select` will do + template static __device__ __forceinline__ - bool find_pivot_in_histogram(SharedMemoryPlanBase &smem, const uint32_t *bucket_counter, uint32_t topk, uint32_t lane_idx) { + bool find_pivot_in_histogram(PlanT &smem, const uint32_t *bucket_counter, uint32_t topk, uint32_t lane_idx) { bool should_select_whole_bucket = false; // Each lane loads 8 counters static_assert(NUM_RECONSTRUCT_BUCKETS == 32 * 8); @@ -954,8 +955,10 @@ public: ); } - // Take the appropriate action when we see a NaN (i.e. `trap()` or write 0x3f3f3f3f) - static __device__ __forceinline__ + // Take the appropriate action when we see a NaN: `trap()`, or [SGLang] write the row as if every selected + // index were 0, so downstream gathers stay in range. Called by the whole CTA; out of line, since inlined copies + // of this cold path only cost instruction fetch on every launch. + static __device__ __noinline__ void take_action_when_have_nan(const TopkSelectArgs &args, uint32_t batch_idx) { if (args.abort_when_nan_found) { if (threadIdx.x == 0) { @@ -964,8 +967,19 @@ public: __syncthreads(); ku::trap(); } else { - if (threadIdx.x == 0) { - *((OutIdxT*)args.output_index + batch_idx * args.stride_output_index_batch) = 0x3F3F3F3F; + OutIdxT* result_indices = (OutIdxT*)args.output_index + (uint64_t)batch_idx * args.stride_output_index_batch; + OutIdxT index0; + if constexpr (Config::page_transform) { + const int* page_table_row = args.page_table + (uint64_t)batch_idx * args.stride_page_table_batch; + index0 = (OutIdxT)EpilogueT::page_transform_index(0u, true, 0, page_table_row, args.page_bits); + } else { + index0 = (OutIdxT)(args.output_idx_offset != nullptr ? __ldg(args.output_idx_offset + batch_idx) : 0); + } + for (uint32_t i = threadIdx.x; i < args.topk; i += NUM_THREADS) result_indices[i] = index0; + if constexpr (Config::return_value) { + const ValueT value0 = ((const ValueT*)args.input)[(uint64_t)batch_idx * args.stride_input_batch]; + ValueT* result_values = (ValueT*)args.output_value + (uint64_t)batch_idx * args.stride_output_value_batch; + for (uint32_t i = threadIdx.x; i < args.topk; i += NUM_THREADS) result_values[i] = value0; } } } diff --git a/python/sglang/kernels/jit/csrc/deep_select/vendor/cuda_kernels/v3_cluster/topk_select.cuh b/python/sglang/kernels/jit/csrc/deep_select/vendor/cuda_kernels/v3_cluster/topk_select.cuh index 47e8fba698..f3b9d7e2b6 100644 --- a/python/sglang/kernels/jit/csrc/deep_select/vendor/cuda_kernels/v3_cluster/topk_select.cuh +++ b/python/sglang/kernels/jit/csrc/deep_select/vendor/cuda_kernels/v3_cluster/topk_select.cuh @@ -1,10 +1,14 @@ /* TopK select kernel bf16 cluster variant. Use multiple CTAs to calculate one sequence to avoid wave quantization -Algorithm: -- Each CTA computes its local top-k pairs -- Each CTA sends its local top-k pairs to CTA0 -- CTA0 selects top-k from those pairs +[SGLang] Algorithm: +- The row is split contiguously across the CTAs and streamed through a ring of bulk copies +- A slice of at most RAW_WINDOW elements is its own candidate set. A longer slice first takes a pivot from every + CTA's leading RAW_WINDOW elements, then keeps only the elements above it while the rest streams in +- The cluster runs a two-pass (MSB, LSB) radix select over all candidates: every CTA pushes its 256-bin histogram + to every CTA, so each one finds the same pivot on its own +- Each warp claims output slots with one atomic on CTA0's cursor and stores its selected indices directly. With + `sorted_index`, each CTA instead sorts its selection by index and stores it after the lower CTAs' selections */ #pragma once @@ -13,6 +17,7 @@ Algorithm: #include #include #include +#include #include "structs.h" #include "cuda_kernels/utils.cuh" @@ -20,16 +25,6 @@ Algorithm: namespace topk_select_bf16_cluster { -CUTE_DEVICE -static void st_async_32b(uint32_t dst_addr, const uint32_t& data, transac_bar_t &mbar) { - uint32_t mbar_addr = cute::cast_smem_ptr_to_uint(&mbar); - asm volatile ( - "st.async.weak.shared::cluster.mbarrier::complete_tx::bytes.s32 [%0], {%1}, [%2]; \n" - : - : "r"(dst_addr), "r"(data), "r"(mbar_addr) - ); -} - template class TopkSelectKernelBF16Cluster : public topk_select_common::TopkSelectKernelBF16Base { using BF16Base = topk_select_common::TopkSelectKernelBF16Base; @@ -41,33 +36,883 @@ public: using EpilogueT = typename BF16Base::EpilogueT; using BF16Base::NUM_THREADS; using BF16Base::MAX_TOPK; - using BF16Base::NUM_EXTRA_SLOTS; - using BF16Base::NUM_ELEMS_PER_SEG; - using BF16Base::NUM_ELEMS_PER_ROUND; - using BF16Base::NUM_SEGS_IN_INIT_WINDOW; - using BF16Base::NUM_INIT_ROUNDS_MAX; - using BF16Base::NUM_TAIL_ELEMS; - using BF16Base::NUM_TAIL_SEGS; - using BF16Base::NUM_SEGS_PER_ROUND; static_assert(Config::target_occupancy == 1); static_assert(MAX_TOPK == 512 || MAX_TOPK == 1024); - static_assert(Config::cluster_size == 8 || Config::cluster_size == 16); - static_assert(NUM_ELEMS_PER_ROUND * sizeof(ValueT) % 1024 == 0); - static_assert((uint64_t)(MAX_VOCAB_SIZE / NUM_ELEMS_PER_SEG) * (MAX_VOCAB_SIZE / NUM_ELEMS_PER_SEG) - + BF16Base::PERM_ADD_BASE <= 0xFFFFFFFFull); + static_assert(Config::cluster_size == 2 || Config::cluster_size == 3 || Config::cluster_size == 4 + || Config::cluster_size == 8 || Config::cluster_size == 16); static constexpr uint32_t CLUSTER_SIZE = Config::cluster_size; - static constexpr uint32_t NUM_GATHER_PAIRS = Config::cluster_size * MAX_TOPK; - static constexpr uint32_t NUM_GATHER_UNITS = NUM_GATHER_PAIRS / 2; - static constexpr uint32_t NUM_UINT32_GATHER_PER_THREAD = ((NUM_GATHER_UNITS + NUM_THREADS - 1) / NUM_THREADS) | 1u; - struct SharedMemoryPlanBF16Cluster : SharedMemoryPlanBase { - kerutils::transac_bar_t gather_val_bar; - kerutils::transac_bar_t gather_bar; - CUTE_ALIGNAS(16) uint32_t gathered_num_survivors[Config::cluster_size]; + + + // A histogram travels as 256 u16 bins; the MSB one is followed by a 16-byte chunk whose first word is the + // sender's NaN flag + static constexpr uint32_t XCHG_MSB_WORDS = 128 + 4; + static constexpr uint32_t XCHG_LSB_WORDS = 128; + using RecvMsbT = uint32_t[Config::cluster_size][XCHG_MSB_WORDS]; + using RecvLsbT = uint32_t[Config::cluster_size][XCHG_LSB_WORDS]; + struct RecvPair { + CUTE_ALIGNAS(16) RecvMsbT msb; + CUTE_ALIGNAS(16) RecvLsbT lsb; }; - static_assert(NUM_GATHER_UNITS % NUM_THREADS == 0); - static_assert(NUM_GATHER_PAIRS <= 0xFFFF); - static_assert(2 * NUM_UINT32_GATHER_PER_THREAD <= 256); + // A slice streams through the ring in stages of two unit rows, one 16-byte unit per thread per row + static constexpr uint32_t UNIT_ROW = NUM_THREADS * 8; + static constexpr uint32_t STAGE = 2 * UNIT_ROW; + static constexpr uint32_t RAW_WINDOW = 32768; + static constexpr uint32_t WINDOW_ROWS = RAW_WINDOW / UNIT_ROW; + static constexpr uint32_t WINDOW_STAGES = RAW_WINDOW / STAGE; + static_assert(NUM_THREADS == 512 && WINDOW_ROWS * 4 <= 32); + // Room for a long slice's candidates; arbitrary, it only has to beat topk plus a typical stage's hits + static constexpr uint32_t CAND_CAP = 4096; + static_assert(CAND_CAP >= MAX_TOPK && CAND_CAP % NUM_THREADS == 0); + struct StreamTail { + // The window exchange is over before the first candidate is written + union { + RecvPair window_recv; + uint64_t cand[CAND_CAP]; // (value bits << 32) | vocab index + }; + RecvPair final_recv; + CUTE_ALIGNAS(16) uint32_t xh_msb[256]; + CUTE_ALIGNAS(16) uint32_t xh_lsb[BF16Base::NUM_RECONSTRUCT_BUCKETS + 4]; // [256] is the LSB pass's sink + CUTE_ALIGNAS(16) uint32_t xh_sum[256]; + uint32_t warp_cnt[BF16Base::NUM_WARPS]; + kerutils::transac_bar_t xbar[2]; // window exchange + kerutils::transac_bar_t ybar[2]; // final exchange + uint64_t xctr; // CTA0's copy is the row's output cursor: (count above the pivot) << 32 | (ties) + uint32_t cand_cnt; + uint32_t eq_cnt; + uint32_t sel_cnt; // `sorted_index`: this CTA's selected elements, and the ties among them + uint32_t sel_eq; + uint32_t rank_gt[Config::cluster_size]; // `sorted_index`: every CTA's count above the pivot and at it + uint32_t rank_eq[Config::cluster_size]; + uint32_t warp_pre[BF16Base::NUM_WARPS]; // `sorted_index`: selected elements before each warp's bitmap run + uint32_t reconstruct_pivot_bucket; + uint32_t reconstruct_num_should_select; + }; + // The ring takes whatever the dynamic shared memory leaves; 225 KB keeps clear of the 227 KB opt-in limit + static constexpr uint32_t SMEM_BUDGET = 225 * 1024; + static constexpr uint32_t NUM_STAGES = cute::min(16u, (uint32_t)((SMEM_BUDGET - (sizeof(StreamTail) + 1023) / 1024 * 1024 - 16 * (sizeof(transac_bar_t) + 4)) / (STAGE * sizeof(ValueT)))); + static_assert(NUM_STAGES > WINDOW_STAGES); + // Stages per streaming step; arbitrary, measured better than one stage per step + static constexpr uint32_t STREAM_BATCH = 4; + static_assert(STREAM_BATCH <= NUM_STAGES - WINDOW_STAGES); + struct StreamPlan : StreamTail { + CUTE_ALIGNAS(1024) ValueT ring[NUM_STAGES][STAGE]; + kerutils::transac_bar_t full[NUM_STAGES]; + uint32_t stage_off[NUM_STAGES]; // slice offset of the stage in each ring slot, published by its barrier + }; + using PlanT = StreamPlan; + // A row of at most topk elements takes the base kernel's epilogue, laid out as its plan + static_assert(sizeof(SharedMemoryPlanBase) <= sizeof(StreamPlan)); + + static __device__ __forceinline__ void st_async_v4(uint32_t dst, uint32_t a, uint32_t b, uint32_t c, uint32_t d, uint32_t mbar) { + asm volatile("st.async.shared::cluster.mbarrier::complete_tx::bytes.v4.b32 [%0], {%1, %2, %3, %4}, [%5];" + :: "r"(dst), "r"(a), "r"(b), "r"(c), "r"(d), "r"(mbar) : "memory"); + } + + static __device__ __forceinline__ void bulk_g2s(void *dst, const void *src, uint32_t bytes, transac_bar_t &bar) { + asm volatile("cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" + :: "r"(cute::cast_smem_ptr_to_uint(dst)), "l"(src), "r"(bytes), "r"(cute::cast_smem_ptr_to_uint(&bar)) : "memory"); + } + + // Cluster-scope acquire: pairs with the release of the `st.async` complete-tx, which orders everything the + // sender's CTA did before it (CTA0 zeroing `xctr` included) before this CTA's later accesses + static __device__ __forceinline__ void wait_acquire_cluster(transac_bar_t &bar, uint32_t phase) { + asm volatile( + "{\n.reg .pred p;\n" + "LAB_WAIT_%=:\n" + "mbarrier.try_wait.parity.acquire.cluster.shared::cta.b64 p, [%0], %1;\n" + "@!p bra LAB_WAIT_%=;\n}\n" + :: "r"(cute::cast_smem_ptr_to_uint(&bar)), "r"(phase) : "memory"); + } + + // +0 and -0 get distinct histogram keys but compare equal, so -0 becomes +0 before either sees it + static __device__ __forceinline__ uint32_t canonical_zero_x2(uint32_t v) { + uint32_t r; + asm("add.rn.bf16x2 %0, %1, %2;" : "=r"(r) : "r"(v), "r"(0u)); + return r; + } + + static __device__ __forceinline__ bool has_nan_x2(uint32_t v) { + return (v & 0x7FFFu) > 0x7F80u || ((v >> 16) & 0x7FFFu) > 0x7F80u; + } + + // Pack this CTA's u32 bins into u16 and push them into slot `rank` of every CTA's `recv` + template + static __device__ __forceinline__ void push_hist(const uint32_t *bins, uint32_t (&recv)[CLUSTER_SIZE][WORDS], + transac_bar_t &bar, uint32_t rank, uint32_t nan_flag) { + constexpr uint32_t NCHUNK = WORDS / 4; + const uint32_t recv_base = cute::cast_smem_ptr_to_uint(&recv[rank][0]); + const uint32_t bar_addr = cute::cast_smem_ptr_to_uint(&bar); + CUTE_UNROLL + for (uint32_t i = threadIdx.x; i < NCHUNK * CLUSTER_SIZE; i += NUM_THREADS) { + uint32_t c = i % NCHUNK, j = i / NCHUNK; + uint32_t w0 = nan_flag, w1 = 0, w2 = 0, w3 = 0; + if (c < 32) { + uint4 lo = *reinterpret_cast(bins + c * 8); + uint4 hi = *reinterpret_cast(bins + c * 8 + 4); + w0 = lo.x | (lo.y << 16); w1 = lo.z | (lo.w << 16); w2 = hi.x | (hi.y << 16); w3 = hi.z | (hi.w << 16); + } + st_async_v4(cute::set_block_rank(recv_base + c * 16, j), w0, w1, w2, w3, cute::set_block_rank(bar_addr, j)); + } + } + + template + static __device__ __forceinline__ uint32_t sum_bin(const uint32_t (&recv)[CLUSTER_SIZE][WORDS], uint32_t bin) { + uint32_t all = 0; + CUTE_UNROLL + for (uint32_t j = 0; j < CLUSTER_SIZE; ++j) all += (recv[j][bin >> 1] >> (16 * (bin & 1))) & 0xFFFFu; + return all; + } + + static __device__ __forceinline__ void clear_local_hists(StreamPlan &smem) { + for (uint32_t i = threadIdx.x; i < 256; i += NUM_THREADS) smem.xh_msb[i] = 0u; + for (uint32_t i = threadIdx.x; i < BF16Base::NUM_RECONSTRUCT_BUCKETS + 4; i += NUM_THREADS) smem.xh_lsb[i] = 0u; + } + + // Masks as in the main loop: two `set.*.bf16x2` + prmt + dp4a give a nibble per 4 elements, bit e for element e + // (half e % 2 of word e / 2). GT_OP is `gt` or `gtu`; the invalid 0xFFFF is a NaN, so `gt` never takes it. + template + static __device__ __forceinline__ void pivot_masks(const uint32_t (&raw)[NU], uint32_t num_words, uint32_t p2, uint64_t >, uint64_t &eq) { + static_assert(NU % 2 == 0 && NU <= 32); + uint32_t gtw[2] = {0, 0}, eqw[2] = {0, 0}; + auto group = [&](uint32_t g) { + if (2 * g >= num_words) return; + if constexpr (UNORDERED) { + asm("{\n.reg .b32 r0, r1, ind, nib;\n" + "set.gtu.s32.bf16x2 r0, %2, %4;\nset.gtu.s32.bf16x2 r1, %3, %4;\n" + "prmt.b32 ind, r0, r1, 0x7531;\ndp4a.s32.s32 nib, ind, 0xF8FCFEFF, 0;\nmad.lo.u32 %0, %0, 16, nib;\n" + "set.eq.s32.bf16x2 r0, %2, %4;\nset.eq.s32.bf16x2 r1, %3, %4;\n" + "prmt.b32 ind, r0, r1, 0x7531;\ndp4a.s32.s32 nib, ind, 0xF8FCFEFF, 0;\nmad.lo.u32 %1, %1, 16, nib;\n}\n" + : "+r"(gtw[g / 8]), "+r"(eqw[g / 8]) : "r"(raw[2 * g]), "r"(raw[2 * g + 1]), "r"(p2)); + } else { + asm("{\n.reg .b32 r0, r1, ind, nib;\n" + "set.gt.s32.bf16x2 r0, %2, %4;\nset.gt.s32.bf16x2 r1, %3, %4;\n" + "prmt.b32 ind, r0, r1, 0x7531;\ndp4a.s32.s32 nib, ind, 0xF8FCFEFF, 0;\nmad.lo.u32 %0, %0, 16, nib;\n" + "set.eq.s32.bf16x2 r0, %2, %4;\nset.eq.s32.bf16x2 r1, %3, %4;\n" + "prmt.b32 ind, r0, r1, 0x7531;\ndp4a.s32.s32 nib, ind, 0xF8FCFEFF, 0;\nmad.lo.u32 %1, %1, 16, nib;\n}\n" + : "+r"(gtw[g / 8]), "+r"(eqw[g / 8]) : "r"(raw[2 * g]), "r"(raw[2 * g + 1]), "r"(p2)); + } + }; + // Groups run from the top down; the upper 16 words are skipped as a whole when no thread has them + if constexpr (NU > 16) { + if (num_words > 16) { + CUTE_UNROLL + for (int32_t g = NU / 2 - 1; g >= 8; --g) group((uint32_t)g); + } + } + CUTE_UNROLL + for (int32_t g = (NU > 16 ? 8 : NU / 2) - 1; g >= 0; --g) group((uint32_t)g); + gt = gtw[0] | ((uint64_t)gtw[1] << 32); + eq = eqw[0] | ((uint64_t)eqw[1] << 32); + } + + struct ClusterPivot { uint32_t key; uint32_t need_eq; bool nan; }; + + // Four stage offsets as scalars: an array here gets its address taken and lands in local memory + struct Quad { + uint32_t v0, v1, v2, v3; + __device__ __forceinline__ uint32_t at(uint32_t i) const { + return i == 0 ? v0 : i == 1 ? v1 : i == 2 ? v2 : v3; + } + }; + static_assert(WINDOW_STAGES == 4 && STREAM_BATCH == 4); + + // The pivot of the top-k over every CTA's candidates. `raw` holds NU packed bf16x2 words, the first `num_words` + // live; an invalid element holds 0xFFFF (distorted key 0) and is counted in `n_invalid`. `xh_msb` holds this + // CTA's MSB histogram, invalid elements excluded. FIRST waits on the cluster barrier armed at kernel start + // (every CTA's barriers are initialized); `on_synced` runs on thread 0 once the whole CTA is past its loads. + template + static __device__ __forceinline__ ClusterPivot cluster_pivot( + StreamPlan &smem, RecvPair &recv, transac_bar_t (&bar)[2], uint32_t rank, uint32_t warp_idx, uint32_t lane_idx, + uint32_t topk, bool nan_local, const uint32_t (&raw)[NU], uint32_t num_words, uint32_t n_invalid, OnSyncedF &&on_synced + ) { + const uint32_t tid = threadIdx.x; + const nv_bfloat162 (&rawv)[NU] = reinterpret_cast(raw); + const uint32_t nan_cta = __syncthreads_or(nan_local) != 0; // also publishes `xh_msb` + if (tid == 0) on_synced(); + if constexpr (FIRST) ku::barrier_cluster_wait_acquire(); + push_hist(smem.xh_msb, recv.msb, bar[0], rank, nan_cta); + wait_acquire_cluster(bar[0], 0); + if (tid < 256) smem.xh_sum[tid] = sum_bin(recv.msb, tid); + const uint32_t nan_any = tid < CLUSTER_SIZE ? recv.msb[tid][128] : 0u; + if (__syncthreads_or(nan_any)) return {0u, 0u, true}; + if (warp_idx == 0) BF16Base::template find_pivot_in_histogram(smem, smem.xh_sum, topk, lane_idx); + __syncthreads(); + const uint32_t pivot_hi = smem.reconstruct_pivot_bucket; + const uint32_t need_in_bucket = smem.reconstruct_num_should_select; + + BF16Base::template histogram_radix_lsb_for_pivot_msb(smem.xh_lsb, pivot_hi, rawv, num_words); + if (pivot_hi == 0 && n_invalid != 0) atomicAdd_block(&smem.xh_lsb[0], 0u - n_invalid); + __syncthreads(); + push_hist(smem.xh_lsb, recv.lsb, bar[1], rank, 0u); + wait_acquire_cluster(bar[1], 0); + if (tid < 256) smem.xh_sum[tid] = sum_bin(recv.lsb, tid); // warp 0 finished the MSB search before the barrier above + __syncthreads(); + if (warp_idx == 0) BF16Base::template find_pivot_in_histogram(smem, smem.xh_sum, need_in_bucket, lane_idx); + __syncthreads(); + const ClusterPivot result{(pivot_hi << 8) | smem.reconstruct_pivot_bucket, smem.reconstruct_num_should_select, false}; + // Both pushes read their bins into registers, so the local histograms are free for the next pass + clear_local_hists(smem); + return result; + } + + // Writes vocab index `idx` (and its value) to output slot `pos` of the row + static __device__ __forceinline__ auto output_writer(const TopkSelectArgs &args, uint32_t batch_idx) { + OutIdxT* result_indices = (OutIdxT*)args.output_index + (uint64_t)batch_idx * args.stride_output_index_batch; + const int32_t idx_offset = args.output_idx_offset != nullptr ? __ldg(args.output_idx_offset + batch_idx) : 0; + const int* page_table_row = Config::page_transform ? args.page_table + (uint64_t)batch_idx * args.stride_page_table_batch : nullptr; + const uint32_t page_bits = args.page_bits; + const ValueT* input_row = (const ValueT*)args.input + (uint64_t)batch_idx * args.stride_input_batch; + ValueT* result_values = Config::return_value ? (ValueT*)args.output_value + (uint64_t)batch_idx * args.stride_output_value_batch : nullptr; + return [=](uint32_t pos, uint32_t idx) { + if constexpr (Config::page_transform) { + result_indices[pos] = (OutIdxT)EpilogueT::page_transform_index(idx, true, 0, page_table_row, page_bits); + } else { + result_indices[pos] = (OutIdxT)idx + idx_offset; + } + if constexpr (Config::return_value) result_values[pos] = input_row[idx]; + }; + } + + // Store this CTA's share of the top-k given the cluster pivot: slots [0, topk - need_eq) take every candidate + // above it, the rest the first `need_eq` ties to claim one. `index(j, b)` is the vocab index of element G * j + b. + template + static __device__ __forceinline__ void cluster_emit( + StreamPlan &smem, const TopkSelectArgs &args, uint32_t batch_idx, uint32_t lane_idx, ClusterPivot pivot, + const uint32_t (&raw)[NU], uint32_t num_words, IndexF &&index + ) { + constexpr uint32_t NE = 2 * NU; + static_assert(NE % G == 0 && G <= 32); + const uint32_t pivot_raw = topk_select_common::un_distort((uint16_t)pivot.key); + uint64_t gt, eq; + pivot_masks(raw, num_words, pivot_raw | (pivot_raw << 16), gt, eq); + + const uint32_t my_gt = (uint32_t)__popcll(gt), my_eq = (uint32_t)__popcll(eq); + const uint32_t packed = (my_gt << 16) | my_eq; // a warp holds at most 32 * NE <= 2048 of each + const uint32_t lane_off = warp_level_exclusive_prefix_sum(packed, lane_idx); + const uint32_t warp_tot = __reduce_add_sync(0xFFFFFFFFu, packed); + uint64_t base = 0; + if (lane_idx == 0 && warp_tot != 0) { + const uint32_t cursor = cute::set_block_rank(cute::cast_smem_ptr_to_uint(&smem.xctr), 0); + const uint64_t inc = ((uint64_t)(warp_tot >> 16) << 32) | (warp_tot & 0xFFFFu); + asm volatile("atom.shared::cluster.add.u64 %0, [%1], %2;" : "=l"(base) : "r"(cursor), "l"(inc) : "memory"); + } + base = __shfl_sync(0xFFFFFFFFu, base, 0); + // CTA0 must outlive every CTA's atomic on its `xctr`; having the result means this warp's is done + ku::barrier_cluster_arrive_relaxed(); + uint32_t gt_pos = (uint32_t)(base >> 32) + (lane_off >> 16); + const uint32_t eq_rank = (uint32_t)base + (lane_off & 0xFFFFu); + const uint32_t eq_take = pivot.need_eq > eq_rank ? min(my_eq, pivot.need_eq - eq_rank) : 0u; + while ((uint32_t)__popcll(eq) > eq_take) eq &= ~(1ull << (63 - __clzll(eq))); + uint32_t eq_pos = args.topk - pivot.need_eq + eq_rank; + + auto store = output_writer(args, batch_idx); + auto emit_unit = [&](uint32_t j) { + uint32_t mg = (uint32_t)(gt >> (G * j)) & (uint32_t)((1ull << G) - 1); + uint32_t me = (uint32_t)(eq >> (G * j)) & (uint32_t)((1ull << G) - 1); + while (mg != 0) { + uint32_t b = __ffs(mg) - 1; + mg &= mg - 1; + store(gt_pos++, index(j, b)); + } + while (me != 0) { + uint32_t b = __ffs(me) - 1; + me &= me - 1; + store(eq_pos++, index(j, b)); + } + }; + constexpr uint32_t LOW_UNITS = (NE > 32 ? 32 : NE) / G; + CUTE_UNROLL + for (uint32_t j = 0; j < LOW_UNITS; ++j) emit_unit(j); + if constexpr (NE > 32) { + if (((gt | eq) >> 32) != 0) { + CUTE_UNROLL + for (uint32_t j = LOW_UNITS; j < NE / G; ++j) emit_unit(j); + } + } + ku::barrier_cluster_wait_acquire(); + } + + // `sorted_index`: CTA r owns a contiguous slice past those of CTAs < r, so its selected elements start at the + // count the lower CTAs select; ties go to the lowest CTAs first. Every CTA reads each CTA's counts off the + // histograms of the last exchange (`recv`). Within the slice, an element's rank among the selected is a prefix + // popcount over a bitmap of them, which the idle ring holds: quads [0, Q) of bits, then quads [Q, 2Q) of each + // word's prefix within its warp's run. + static constexpr uint32_t SORTED_MAX_SLICE = NUM_STAGES * STAGE * (uint32_t)sizeof(ValueT) / 8 * 32; + + template + static __device__ __forceinline__ void cluster_emit_sorted( + StreamPlan &smem, const RecvPair &recv, const TopkSelectArgs &args, uint32_t batch_idx, uint32_t rank, + uint32_t warp_idx, uint32_t lane_idx, ClusterPivot pivot, const uint32_t (&raw)[NU], uint32_t num_words, + uint32_t start, uint32_t len, IndexF &&index + ) { + constexpr uint32_t NE = 2 * NU; + static_assert(NE % G == 0 && NE <= 64); + const uint32_t num_q = ku::ceil_div(len, 128u); + uint4 *bits4 = reinterpret_cast(&smem.ring[0][0]); + uint4 *prefix4 = bits4 + num_q; + uint32_t *bits = reinterpret_cast(bits4), *prefix = reinterpret_cast(prefix4); + // The ring is idle (every stage issued has been consumed) + for (uint32_t i = threadIdx.x; i < num_q; i += NUM_THREADS) bits4[i] = make_uint4(0u, 0u, 0u, 0u); + // Warp j counts CTA j's elements above the pivot key and at it; word w of a histogram holds bins 2w, 2w + 1 + if (warp_idx < CLUSTER_SIZE) { + const uint32_t hi = pivot.key >> 8, lo = pivot.key & 0xFFu; + uint32_t gt = 0, eq = 0; + CUTE_UNROLL + for (uint32_t w = lane_idx; w < 128; w += 32) { + const uint32_t m = recv.msb[warp_idx][w], l = recv.lsb[warp_idx][w]; + CUTE_UNROLL + for (uint32_t h = 0; h < 2; ++h) { + const uint32_t bin = 2 * w + h, mc = (m >> (16 * h)) & 0xFFFFu, lc = (l >> (16 * h)) & 0xFFFFu; + gt += (bin > hi ? mc : 0u) + (bin > lo ? lc : 0u); + eq += bin == lo ? lc : 0u; + } + } + gt = __reduce_add_sync(0xFFFFFFFFu, gt); + eq = __reduce_add_sync(0xFFFFFFFFu, eq); + if (lane_idx == 0) { + smem.rank_gt[warp_idx] = gt; + smem.rank_eq[warp_idx] = eq; + } + } + const uint32_t pivot_raw = topk_select_common::un_distort((uint16_t)pivot.key); + uint64_t gt, eq; + pivot_masks(raw, num_words, pivot_raw | (pivot_raw << 16), gt, eq); + __syncthreads(); + uint32_t base = 0, eq_left = pivot.need_eq; + CUTE_UNROLL + for (uint32_t j = 0; j < CLUSTER_SIZE; ++j) { + if (j < rank) { + const uint32_t t = min(smem.rank_eq[j], eq_left); + eq_left -= t; + base += smem.rank_gt[j] + t; + } + } + const uint32_t my_take = min(smem.rank_eq[rank], eq_left); + + // Any `my_take` of this CTA's ties will do: the bitmap orders whatever is kept + const uint32_t my_eq = (uint32_t)__popcll(eq); + const uint32_t warp_eq = __reduce_add_sync(0xFFFFFFFFu, my_eq); + uint32_t eq_base = 0; + if (lane_idx == 0 && warp_eq != 0) eq_base = atomicAdd_block(&smem.sel_eq, warp_eq); + eq_base = __shfl_sync(0xFFFFFFFFu, eq_base, 0) + warp_level_exclusive_prefix_sum(my_eq, lane_idx); + const uint32_t eq_take = my_take > eq_base ? min(my_eq, my_take - eq_base) : 0u; + while ((uint32_t)__popcll(eq) > eq_take) eq &= ~(1ull << (63 - __clzll(eq))); + const uint64_t keep = gt | eq; + for (uint64_t m = keep; m != 0; m &= m - 1) { + const uint32_t e = __ffsll(m) - 1; + const uint32_t off = index(e / G, e % G) - start; + atomicOr_block(&bits[off / 32], 1u << (off % 32)); + } + __syncthreads(); + + // One pass of exclusive prefixes: warp i takes a power-of-two run of quads, its lanes interleaved over it + // (a thread-contiguous run would put a warp's accesses in one bank); the runs' bases are summed after + constexpr uint32_t LOG_WARPS = 31 - __builtin_clz(BF16Base::NUM_WARPS); + static_assert((1u << LOG_WARPS) == BF16Base::NUM_WARPS); + const uint32_t log_run = max(5u, 32u - __clz(max((max(num_q, 1u) - 1) >> LOG_WARPS, 1u))); // a CTA past the row end has len 0 + const uint32_t q_lo = min(warp_idx << log_run, num_q), q_hi = min(q_lo + (1u << log_run), num_q); + uint32_t run = 0; + for (uint32_t q0 = q_lo; q0 < q_hi; q0 += 32) { + const uint32_t q = q0 + lane_idx; + const uint4 v = q < q_hi ? bits4[q] : make_uint4(0u, 0u, 0u, 0u); + const uint32_t c0 = __popc(v.x), c1 = __popc(v.y), c2 = __popc(v.z), c3 = __popc(v.w); + const uint32_t local = c0 + c1 + c2 + c3; + const uint32_t incl = warp_level_inclusive_prefix_sum(local, lane_idx); + const uint32_t e0 = run + incl - local; + if (q < q_hi) prefix4[q] = make_uint4(e0, e0 + c0, e0 + c0 + c1, e0 + c0 + c1 + c2); + run += __shfl_sync(0xFFFFFFFFu, incl, 31); + } + if (lane_idx == 0) smem.warp_cnt[warp_idx] = run; + __syncthreads(); + if (warp_idx == 0) { + const uint32_t c = lane_idx < BF16Base::NUM_WARPS ? smem.warp_cnt[lane_idx] : 0u; + const uint32_t e = warp_level_exclusive_prefix_sum(c, lane_idx); + if (lane_idx < BF16Base::NUM_WARPS) smem.warp_pre[lane_idx] = e; + } + __syncthreads(); + + auto store = output_writer(args, batch_idx); + for (uint64_t m = keep; m != 0; m &= m - 1) { + const uint32_t e = __ffsll(m) - 1; + const uint32_t off = index(e / G, e % G) - start; + const uint32_t w = off / 32; + store(base + smem.warp_pre[w >> (log_run + 2)] + prefix[w] + __popc(bits[w] & ((1u << (off % 32)) - 1u)), start + off); + } + } + + // Load the window, ring stages 0..WINDOW_STAGES-1 (phase 0), into `vals` as WINDOW_ROWS units per thread, + // building the MSB histogram as each row lands. Unit row r is the (r % 2)-th half of the slice stage at + // element offset `win_off[r / 2]`; thread t holds unit t of it as words [4r, 4r + 4). Returns the number of live + // unit rows of this thread (a row past the slice's end is never followed by a live one); words past them are + // never read. + static __device__ __forceinline__ uint32_t load_window( + StreamPlan &smem, uint32_t len, uint32_t num_stages, Quad win_off, + uint32_t (&vals)[WINDOW_ROWS * 4], uint32_t &n_invalid, bool &nan_local + ) { + const ValueT *buf = &smem.ring[0][0]; + uint32_t nan_acc = 0, num_units = 0; + auto load_row = [&](auto row) { + constexpr uint32_t r = decltype(row)::value; + if (r / 2 >= num_stages) return; + if (r % 2 == 0) smem.full[r / 2].wait(0); + // Offset in the slice of this thread's unit, and in the window + const uint32_t e0 = win_off.at(r / 2) + (r % 2) * UNIT_ROW + threadIdx.x * 8; + if (e0 < len) { + topk_select_common::ld_shared<4>(vals + r * 4, reinterpret_cast(buf + r * UNIT_ROW + threadIdx.x * 8)); + const bool full = e0 + 8 <= len; + CUTE_UNROLL + for (uint32_t w = 0; w < 4; ++w) { + if (full) asm("min.NaN.bf16x2 %0, %0, %1;" : "+r"(nan_acc) : "r"(vals[r * 4 + w])); + vals[r * 4 + w] = canonical_zero_x2(vals[r * 4 + w]); + } + if (!full) { + // Only the last unit of the slice runs past its end + CUTE_UNROLL + for (uint32_t w = 0; w < 4; ++w) { + const bool ok0 = e0 + 2 * w < len, ok1 = e0 + 2 * w + 1 < len; + uint32_t v = vals[r * 4 + w]; + nan_local |= (ok0 && (v & 0x7FFFu) > 0x7F80u) || (ok1 && ((v >> 16) & 0x7FFFu) > 0x7F80u); + n_invalid += (ok0 ? 0u : 1u) + (ok1 ? 0u : 1u); + v = ok0 ? v : (v | 0xFFFFu); + v = ok1 ? v : (v | 0xFFFF0000u); + vals[r * 4 + w] = v; + } + } + BF16Base::histogram_radix_msb(smem.xh_msb, reinterpret_cast(vals[r * 4]), 4u); + num_units = r + 1; + } + }; + // The rows past the first half are live only for a long slice: one block-uniform branch skips them all + constexpr uint32_t HALF = WINDOW_ROWS > 4 ? 4 : WINDOW_ROWS; + [&](std::integer_sequence) { + (load_row(std::integral_constant{}), ...); + }(std::make_integer_sequence{}); + if constexpr (WINDOW_ROWS > HALF) { + if (HALF / 2 < num_stages) { + [&](std::integer_sequence) { + (load_row(std::integral_constant{}), ...); + }(std::make_integer_sequence{}); + } + } + uint32_t nan_flag; + asm("set.nan.u32.bf16x2 %0, %1, %1;" : "=r"(nan_flag) : "r"(nan_acc)); + nan_local |= nan_flag != 0; + if (n_invalid != 0) atomicAdd_block(&smem.xh_msb[0], 0u - n_invalid); + return num_units; + } + + // Slow path for a stage whose hits do not fit: replace the candidates [0, num_cand) and every element of the + // stage with their top `topk` (a superset of this CTA's share), and return the new threshold (raw bf16 x2). + // Kept out of line: it runs at most a few times per launch, and inlined copies only cost cold instruction fetch + struct Reduced { uint32_t threshold; bool nan; }; + static __device__ __noinline__ Reduced reduce_candidates( + StreamPlan &smem, uint32_t num_cand, const ValueT *stage_buf, uint32_t stage_slice_off, uint32_t len, + uint32_t slice_start, uint32_t topk, uint32_t warp_idx, uint32_t lane_idx + ) { + bool nan_seen = false; + constexpr uint32_t NC = CAND_CAP / NUM_THREADS; // candidates per thread + constexpr uint32_t NW = NC / 2 + 8; // their words, then the stage's two units + static_assert(NC % 2 == 0); + uint32_t vals[NW]; + uint32_t cidx[NC]; + uint32_t n_invalid = 0; + CUTE_UNROLL + for (uint32_t e = 0; e < NC; ++e) { + const uint32_t c = threadIdx.x + e * NUM_THREADS; + const uint64_t pair = c < num_cand ? smem.cand[c] : 0ull; + cidx[e] = (uint32_t)pair; + uint32_t v = (uint32_t)(pair >> 32) & 0xFFFFu; + nan_seen |= c < num_cand && (v & 0x7FFFu) > 0x7F80u; + v = v == 0x8000u ? 0u : v; + v = c < num_cand ? v : 0xFFFFu; + n_invalid += c < num_cand ? 0u : 1u; + if (e % 2 == 0) vals[e / 2] = v; else vals[e / 2] |= v << 16; + } + CUTE_UNROLL + for (uint32_t h = 0; h < 2; ++h) { + const uint32_t off = h * UNIT_ROW + threadIdx.x * 8; + topk_select_common::ld_shared<4>(vals + NC / 2 + h * 4, reinterpret_cast(stage_buf + off)); + CUTE_UNROLL + for (uint32_t w = 0; w < 4; ++w) { + uint32_t v = vals[NC / 2 + h * 4 + w]; + const bool ok0 = stage_slice_off + off + 2 * w < len, ok1 = stage_slice_off + off + 2 * w + 1 < len; + nan_seen |= (ok0 && (v & 0x7FFFu) > 0x7F80u) || (ok1 && ((v >> 16) & 0x7FFFu) > 0x7F80u); + v = canonical_zero_x2(v); + v = ok0 ? v : (v | 0xFFFFu); + v = ok1 ? v : (v | 0xFFFF0000u); + n_invalid += (ok0 ? 0u : 1u) + (ok1 ? 0u : 1u); + vals[NC / 2 + h * 4 + w] = v; + } + } + const nv_bfloat162 (&rawv)[NW] = reinterpret_cast(vals); + BF16Base::template histogram_radix_msb(smem.xh_msb, rawv, NW); + if (n_invalid != 0) atomicAdd_block(&smem.xh_msb[0], 0u - n_invalid); + __syncthreads(); + if (warp_idx == 0) BF16Base::template find_pivot_in_histogram(smem, smem.xh_msb, topk, lane_idx); + __syncthreads(); + const uint32_t pivot_hi = smem.reconstruct_pivot_bucket; + const uint32_t need_in_bucket = smem.reconstruct_num_should_select; + BF16Base::template histogram_radix_lsb_for_pivot_msb(smem.xh_lsb, pivot_hi, rawv, NW); + if (pivot_hi == 0 && n_invalid != 0) atomicAdd_block(&smem.xh_lsb[0], 0u - n_invalid); + __syncthreads(); + if (warp_idx == 0) BF16Base::template find_pivot_in_histogram(smem, smem.xh_lsb, need_in_bucket, lane_idx); + __syncthreads(); + const uint32_t pivot_raw = topk_select_common::un_distort((uint16_t)((pivot_hi << 8) | smem.reconstruct_pivot_bucket)); + const uint32_t p2 = pivot_raw | (pivot_raw << 16); + auto census = BF16Base::template get_census_counts(rawv, NW, p2); + // Ends with a barrier, so every load of `cand` above is done before the stores below + auto eqgt = BF16Base::compute_equal_quota_and_prefix(census.cnt_gt, census.cnt_eq, topk, warp_idx, lane_idx, smem.warp_cnt); + uint32_t pos = eqgt.start_pos_in_collector, quota = eqgt.eq_quota; + auto keep = [&](uint32_t w, uint32_t h, uint32_t idx) { + const uint32_t v = (vals[w] >> (16 * h)) & 0xFFFFu; + const uint32_t v2 = v | (v << 16); + uint32_t g, q; + asm("set.gt.u32.bf16x2 %0, %1, %2;" : "=r"(g) : "r"(v2), "r"(p2)); + asm("set.eq.u32.bf16x2 %0, %1, %2;" : "=r"(q) : "r"(v2), "r"(p2)); + const bool take = g != 0 || (q != 0 && quota > 0); + if (take) { + if (g == 0) --quota; + smem.cand[pos++] = ((uint64_t)v << 32) | idx; + } + }; + CUTE_UNROLL + for (uint32_t e = 0; e < NC; ++e) keep(e / 2, e % 2, cidx[e]); + CUTE_UNROLL + for (uint32_t h = 0; h < 2; ++h) { + CUTE_UNROLL + for (uint32_t b = 0; b < 8; ++b) + keep(NC / 2 + h * 4 + b / 2, b % 2, slice_start + stage_slice_off + h * UNIT_ROW + threadIdx.x * 8 + b); + } + clear_local_hists(smem); + if (threadIdx.x == 0) smem.cand_cnt = topk; + __syncthreads(); + return {p2, nan_seen}; + } + + // A NaN in the cluster's windows: the stages [WINDOW_STAGES, num_issued) issued during the exchange must land + // before the CTA exits. Out of line: inlined, this cold path slows the stream loop. + static __device__ __noinline__ void window_nan_exit( + StreamPlan &smem, const TopkSelectArgs &args, uint32_t batch_idx, uint32_t rank, uint32_t num_issued + ) { + for (uint32_t g = WINDOW_STAGES; g < num_issued; ++g) smem.full[g].wait(0); + if (rank == 0) BF16Base::take_action_when_have_nan(args, batch_idx); + } + + static __device__ __forceinline__ void select_devfunc( + StreamPlan &smem, const TopkSelectArgs &args, uint32_t batch_idx, uint32_t end_vocab_idx, + uint32_t rank, uint32_t warp_idx, uint32_t lane_idx + ) { + const uint32_t slice = ku::ceil_div(ku::ceil_div(end_vocab_idx, (uint32_t)CLUSTER_SIZE), 64u) * 64u; + const uint32_t start = min(rank * slice, end_vocab_idx); + const uint32_t len = min(start + slice, end_vocab_idx) - start; + const uint32_t num_stages = ku::ceil_div(len, (uint32_t)STAGE); + const ValueT *src = (const ValueT*)args.input + (uint64_t)batch_idx * args.stride_input_batch + start; + // Visit order of the slice's stages. A slice longer than the window puts its + // last stage (the row's tail on the last CTA, which tends to hold heavy scores, and the slice's only partial + // stage) in the window, and visits the full stages in a pseudo-random order like the one-CTA-per-row kernel: + // i -> (i * mul + base) % P, a bijection since mul is a residue of the prime 2^31 - 1, which exceeds P. + // Offsets are computed once and then only advanced: inlined divisions at every use cost more in code size + // (cold instruction fetch on every launch) than in instructions. + const bool permuted = num_stages > WINDOW_STAGES; + const uint32_t num_full = num_stages - 1; + const uint32_t perm_mul = permuted ? 0x7FFFFFFFu % num_full : 1u; + auto perm_next = [&](uint32_t cur) { + cur += perm_mul; + return cur >= num_full ? cur - num_full : cur; + }; + // Window stages: the first WINDOW_STAGES - 1 visits of the permutation, then the last stage + uint32_t perm_head = permuted ? (rank * 0x9E3779B1u + batch_idx * 0x85EBCA77u) % num_full : 0u; + Quad win_off; + win_off.v0 = (permuted ? perm_head : 0u) * STAGE; + perm_head = perm_next(perm_head); + win_off.v1 = (permuted ? perm_head : 1u) * STAGE; + perm_head = perm_next(perm_head); + win_off.v2 = (permuted ? perm_head : 2u) * STAGE; + perm_head = perm_next(perm_head); + win_off.v3 = (permuted ? num_full : 3u) * STAGE; + // `perm_head` is now the permuted index of the first streamed stage + auto issue_stage_at = [&](uint32_t g, uint32_t off) { + // Rounded up to 16 bytes; the overrun stays inside the row, whose end is padded to 128 bytes + const uint32_t bytes = (min((uint32_t)STAGE, len - off) * (uint32_t)sizeof(ValueT) + 15) & ~15u; + transac_bar_t &bar = smem.full[g % NUM_STAGES]; + smem.stage_off[g % NUM_STAGES] = off; + bar.arrive_and_expect_tx(bytes); + bulk_g2s(smem.ring[g % NUM_STAGES], src + off, bytes, bar); + }; + // Thread 0 issues the stages after the window in visit order from this pointer + uint32_t perm_issue = perm_head; + auto issue_next = [&](uint32_t g) { + if (g < num_stages) issue_stage_at(g, perm_issue * STAGE); + perm_issue = perm_next(perm_issue); + }; + + clear_local_hists(smem); + if (warp_idx == 0 && cute::elect_one_sync()) { + CUTE_UNROLL + for (uint32_t s = 0; s < NUM_STAGES; ++s) smem.full[s].init(1); + CUTE_UNROLL + for (uint32_t i = 0; i < 2; ++i) { + smem.xbar[i].init(1); + smem.ybar[i].init(1); + } + smem.xbar[0].arrive_and_expect_tx(CLUSTER_SIZE * XCHG_MSB_WORDS * 4); + smem.xbar[1].arrive_and_expect_tx(CLUSTER_SIZE * XCHG_LSB_WORDS * 4); + smem.ybar[0].arrive_and_expect_tx(CLUSTER_SIZE * XCHG_MSB_WORDS * 4); + smem.ybar[1].arrive_and_expect_tx(CLUSTER_SIZE * XCHG_LSB_WORDS * 4); + smem.xctr = 0; + smem.cand_cnt = 0; + smem.eq_cnt = 0; + smem.sel_cnt = 0; + smem.sel_eq = 0; + cutlass::arch::fence_barrier_init(); + // The window first: the stream behind it is useless until the window's pivot comes back, so it is + // issued once the window has landed and arrives during the exchange instead of competing with it + CUTE_UNROLL + for (uint32_t g = 0; g < WINDOW_STAGES; ++g) { + if (g < num_stages) issue_stage_at(g, win_off.at(g)); + } + } + __syncthreads(); + ku::barrier_cluster_arrive_relaxed(); // paired with the wait of the first exchange + + uint32_t vals[WINDOW_ROWS * 4]; + uint32_t n_invalid = 0; + bool nan_local = false; + const uint32_t num_units = load_window(smem, len, num_stages, win_off, vals, n_invalid, nan_local); + // `j` is a compile-time unit row in `cluster_emit`; a select chain keeps `win_off` in registers otherwise + auto window_index = [&](uint32_t j, uint32_t b) { + return start + win_off.at(j / 2) + (j % 2) * UNIT_ROW + threadIdx.x * 8 + b; + }; + + // Decided on `slice`, which every CTA of the cluster shares, so they all take the same exchanges + if (slice <= RAW_WINDOW) { + // The whole slice is in `vals`: it is the candidate set + auto pivot = cluster_pivot(smem, smem.window_recv, smem.xbar, rank, warp_idx, lane_idx, args.topk, + nan_local, vals, num_units * 4, n_invalid, [] {}); + if (pivot.nan) { + if (rank == 0) BF16Base::take_action_when_have_nan(args, batch_idx); + return; + } + if constexpr (Config::sorted_index) { + cluster_emit_sorted(smem, smem.window_recv, args, batch_idx, rank, warp_idx, lane_idx, + pivot, vals, num_units * 4, start, len, window_index); + } else { + cluster_emit(smem, args, batch_idx, lane_idx, pivot, vals, num_units * 4, window_index); + } + return; + } + + // A long slice: the window's pivot over the whole cluster bounds every later candidate from below. The rest + // of the ring fills while it is exchanged; the window stages stay until their candidates are copied out. + const auto window_pivot = cluster_pivot(smem, smem.window_recv, smem.xbar, rank, warp_idx, lane_idx, args.topk, + nan_local, vals, num_units * 4, n_invalid, [&] { + CUTE_UNROLL + for (uint32_t g = WINDOW_STAGES; g < NUM_STAGES; ++g) issue_next(g); + }); + if (window_pivot.nan) { + window_nan_exit(smem, args, batch_idx, rank, min((uint32_t)NUM_STAGES, num_stages)); + return; + } + const uint32_t pivot_raw = topk_select_common::un_distort((uint16_t)window_pivot.key); + uint32_t threshold = pivot_raw | (pivot_raw << 16); + { + // Keep the window's elements above the pivot and up to `need_eq` ties: at least topk of the cluster's + // window elements are at least the pivot, so every element of the top-k is kept or still to stream in + uint64_t gt, eq; + pivot_masks(vals, num_units * 4, threshold, gt, eq); + const uint32_t my_eq = (uint32_t)__popcll(eq); + const uint32_t warp_eq = __reduce_add_sync(0xFFFFFFFFu, my_eq); + uint32_t eq_base = 0; + if (lane_idx == 0 && warp_eq != 0) eq_base = atomicAdd_block(&smem.eq_cnt, warp_eq); + eq_base = __shfl_sync(0xFFFFFFFFu, eq_base, 0) + warp_level_exclusive_prefix_sum(my_eq, lane_idx); + const uint32_t eq_take = window_pivot.need_eq > eq_base ? min(my_eq, window_pivot.need_eq - eq_base) : 0u; + while ((uint32_t)__popcll(eq) > eq_take) eq &= ~(1ull << (63 - __clzll(eq))); + const uint64_t keep = gt | eq; + const uint32_t my_keep = (uint32_t)__popcll(keep); + const uint32_t warp_keep = __reduce_add_sync(0xFFFFFFFFu, my_keep); + uint32_t pos = 0; + if (lane_idx == 0 && warp_keep != 0) pos = atomicAdd_block(&smem.cand_cnt, warp_keep); + pos = __shfl_sync(0xFFFFFFFFu, pos, 0) + warp_level_exclusive_prefix_sum(my_keep, lane_idx); + // The window is still in ring stages 0..WINDOW_STAGES-1, contiguous + const ValueT *window_buf = &smem.ring[0][0]; + uint64_t m = keep; + while (m != 0) { + const uint32_t e = __ffsll(m) - 1; + m &= m - 1; + const uint32_t off = (e / 8) * UNIT_ROW + threadIdx.x * 8 + e % 8; + uint32_t v; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(v) : "r"(cute::cast_smem_ptr_to_uint(window_buf + off))); + smem.cand[pos++] = ((uint64_t)v << 32) | (start + smem.stage_off[e / 16] + off % STAGE); + } + } + bool nan_seen = false; + __syncthreads(); + if (threadIdx.x == 0) { + CUTE_UNROLL + for (uint32_t g = NUM_STAGES; g < NUM_STAGES + WINDOW_STAGES; ++g) issue_next(g); + } + + // Stream the rest: thread t takes unit t of both unit rows of a stage, 16 elements as 8 words. Stages go in + // batches of STREAM_BATCH under one atomic per warp and one barrier: per stage, the dependent chain of + // atomic, prefix and barrier costs more than the 16 elements' work. + // Every streamed stage is full: the slice's partial stage is in the window + auto load_stage = [&](uint32_t slot, uint32_t phase, uint32_t (&w)[8]) { + const ValueT *stage_buf = smem.ring[slot]; + smem.full[slot].wait(phase); + topk_select_common::ld_shared<4>(w, reinterpret_cast(stage_buf + threadIdx.x * 8)); + topk_select_common::ld_shared<4>(w + 4, reinterpret_cast(stage_buf + UNIT_ROW + threadIdx.x * 8)); + }; + // Bit e: element e of the thread's 16 (unit row e / 8, offset e % 8) beats the threshold. NaN counts as a + // hit (`gtu`), so every NaN reaches the candidates and the final exchange reports it. + auto hit_mask = [&](const uint32_t (&w)[8]) { + uint32_t m = 0; + CUTE_UNROLL + for (int32_t q = 3; q >= 0; --q) { + asm("{\n.reg .b32 r0, r1, ind, nib;\n" + "set.gtu.s32.bf16x2 r0, %1, %3;\nset.gtu.s32.bf16x2 r1, %2, %3;\n" + "prmt.b32 ind, r0, r1, 0x7531;\ndp4a.s32.s32 nib, ind, 0xF8FCFEFF, 0;\nmad.lo.u32 %0, %0, 16, nib;\n}\n" + : "+r"(m) : "r"(w[2 * q]), "r"(w[2 * q + 1]), "r"(threshold)); + } + return m; + }; + auto append_stage = [&](uint32_t stage_off, uint32_t slot, uint32_t hit, uint32_t &pos) { + const ValueT *stage_buf = smem.ring[slot]; + while (hit != 0) { + const uint32_t e = __ffs(hit) - 1; + hit &= hit - 1; + const uint32_t off = (e / 8) * UNIT_ROW + threadIdx.x * 8 + e % 8; + uint32_t v; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(v) : "r"(cute::cast_smem_ptr_to_uint(stage_buf + off))); + smem.cand[pos++] = ((uint64_t)v << 32) | (start + stage_off + off); + } + }; + // Claims room for `my_hits` per thread with one atomic per warp; false when it does not fit + auto claim = [&](uint32_t my_hits, uint32_t &pos) { + if (!__any_sync(0xFFFFFFFFu, my_hits != 0)) return true; + const uint32_t warp_hits = __reduce_add_sync(0xFFFFFFFFu, my_hits); + pos = 0; + if (lane_idx == 0) pos = atomicAdd_block(&smem.cand_cnt, warp_hits); + pos = __shfl_sync(0xFFFFFFFFu, pos, 0) + warp_level_exclusive_prefix_sum(my_hits, lane_idx); + return pos + my_hits <= CAND_CAP; + }; + // After a failed claim the whole CTA is past the barrier and no atomic is in flight: rewind the count by + // everything claimed since + auto rewind = [&](uint32_t my_hits) { + const uint32_t warp_hits = __reduce_add_sync(0xFFFFFFFFu, my_hits); + if (lane_idx == 0) smem.warp_cnt[warp_idx] = warp_hits; + __syncthreads(); + uint32_t claimed = 0; + CUTE_UNROLL + for (uint32_t i = 0; i < BF16Base::NUM_WARPS; ++i) claimed += smem.warp_cnt[i]; + const uint32_t before = smem.cand_cnt - claimed; + __syncthreads(); + if (threadIdx.x == 0) smem.cand_cnt = before; + __syncthreads(); + return before; + }; + static_assert(STREAM_BATCH * 16 <= 64); + for (uint32_t g0 = WINDOW_STAGES; g0 < num_stages; g0 += STREAM_BATCH) { + // Slot and phase of each stage of the batch, without a division per stage + uint32_t slot[STREAM_BATCH], phase[STREAM_BATCH]; + slot[0] = g0 % NUM_STAGES; + phase[0] = (g0 / NUM_STAGES) & 1; + CUTE_UNROLL + for (uint32_t k = 1; k < STREAM_BATCH; ++k) { + const bool wrap = slot[k - 1] + 1 == NUM_STAGES; + slot[k] = wrap ? 0u : slot[k - 1] + 1; + phase[k] = wrap ? phase[k - 1] ^ 1u : phase[k - 1]; + } + uint32_t w[STREAM_BATCH][8]; + uint32_t hit[STREAM_BATCH]; + uint64_t hits = 0; + CUTE_UNROLL + for (uint32_t k = 0; k < STREAM_BATCH; ++k) { + hit[k] = 0; + if (g0 + k < num_stages) { + load_stage(slot[k], phase[k], w[k]); + hit[k] = hit_mask(w[k]); + hits |= (uint64_t)hit[k] << (16 * k); + } + } + const uint32_t my_hits = (uint32_t)__popcll(hits); + uint32_t pos; + const bool fits = claim(my_hits, pos); + if (fits) { + // One loop over the whole batch's hits + while (hits != 0) { + const uint32_t e = __ffsll(hits) - 1; + hits &= hits - 1; + const uint32_t k = e / 16, b = e % 16; + uint32_t sl = slot[0] + k; + sl = sl >= NUM_STAGES ? sl - NUM_STAGES : sl; + const uint32_t off = (b / 8) * UNIT_ROW + threadIdx.x * 8 + b % 8; + uint32_t v; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(v) : "r"(cute::cast_smem_ptr_to_uint(smem.ring[sl] + off))); + smem.cand[pos++] = ((uint64_t)v << 32) | (start + smem.stage_off[sl] + off); + } + } + if (__syncthreads_or(!fits)) { + // Rare: redo the batch stage by stage, reducing the candidates whenever a stage does not fit. One copy + // of this code (no unrolling), reloading each stage from the ring, which is not refilled yet. + rewind(my_hits); + #pragma unroll 1 + for (uint32_t g = g0; g < min(g0 + STREAM_BATCH, num_stages); ++g) { + const uint32_t sl = g % NUM_STAGES; + const uint32_t so = smem.stage_off[sl]; + uint32_t ws[8]; + load_stage(sl, (g / NUM_STAGES) & 1, ws); + const uint32_t h = hit_mask(ws); + const bool ok = claim(__popc(h), pos); + if (ok) append_stage(so, sl, h, pos); + if (__syncthreads_or(!ok)) { + const uint32_t before = rewind(__popc(h)); + const Reduced red = reduce_candidates(smem, before, smem.ring[sl], so, len, start, + args.topk, warp_idx, lane_idx); + threshold = red.threshold; + nan_seen |= red.nan; + } + } + } + if (threadIdx.x == 0) { + CUTE_UNROLL + for (uint32_t k = 0; k < STREAM_BATCH; ++k) issue_next(g0 + k + NUM_STAGES); + } + } + // Every atomic on `cand_cnt` happened before the last barrier above (or the one after the window's appends) + __syncthreads(); + const uint32_t num_cand = smem.cand_cnt; + + // The final exchange over the candidates + constexpr uint32_t NC = CAND_CAP / NUM_THREADS; + uint32_t cvals[NC / 2]; + CUTE_UNROLL + for (uint32_t e = 0; e < NC; ++e) { + const uint32_t c = threadIdx.x + e * NUM_THREADS; + uint32_t v = c < num_cand ? (uint32_t)(smem.cand[c] >> 32) & 0xFFFFu : 0xFFFFu; + nan_seen |= c < num_cand && (v & 0x7FFFu) > 0x7F80u; + v = v == 0x8000u ? 0u : v; // -0 to +0, see `canonical_zero_x2` + if (e % 2 == 0) cvals[e / 2] = v; else cvals[e / 2] |= v << 16; + } + const uint32_t my_cand = num_cand > threadIdx.x ? ku::ceil_div(num_cand - threadIdx.x, (uint32_t)NUM_THREADS) : 0u; + const uint32_t num_words = (my_cand + 1) / 2; + const uint32_t c_invalid = my_cand % 2; // the odd tail of the last live word + BF16Base::template histogram_radix_msb(smem.xh_msb, reinterpret_cast(cvals), num_words); + if (c_invalid != 0) atomicAdd_block(&smem.xh_msb[0], 0u - c_invalid); + auto pivot = cluster_pivot(smem, smem.final_recv, smem.ybar, rank, warp_idx, lane_idx, args.topk, + nan_seen, cvals, num_words, c_invalid, [] {}); + if (pivot.nan) { + if (rank == 0) BF16Base::take_action_when_have_nan(args, batch_idx); + return; + } + auto cand_index = [&](uint32_t j, uint32_t) { return (uint32_t)smem.cand[threadIdx.x + j * NUM_THREADS]; }; + if constexpr (Config::sorted_index) { + cluster_emit_sorted(smem, smem.final_recv, args, batch_idx, rank, warp_idx, lane_idx, + pivot, cvals, num_words, start, len, cand_index); + } else { + cluster_emit(smem, args, batch_idx, lane_idx, pivot, cvals, num_words, cand_index); + } + } static __device__ __forceinline__ void topk_select_kernel_devfunc(const TopkSelectArgs &args, const TmaParams &tma_params) { uint32_t rank_in_cluster = blockIdx.x; @@ -75,7 +920,7 @@ public: uint32_t end_vocab_idx = args.end_ptr == nullptr ? args.vocab_size : __ldg(args.end_ptr + batch_idx); extern __shared__ CUTE_ALIGNAS(1024) char wksp_buf[]; - SharedMemoryPlanBF16Cluster &smem = *reinterpret_cast(wksp_buf); + SharedMemoryPlanBase &smem = *reinterpret_cast(wksp_buf); uint32_t warp_idx = cutlass::canonical_warp_idx_sync(); uint32_t lane_idx = threadIdx.x % 32; @@ -94,173 +939,7 @@ public: return; } - uint32_t survivor_buf_idx = 0; // current candidate buffer (A/B, swapped by reconstruct) - - BF16Base::init_shared_memory(smem, warp_idx, [&]{ - smem.gather_val_bar.init(1); - smem.gather_bar.init(1); - }); - - // Still, divide the whole input into the "perm" part and the "tail" part - // The "tail" part contains the last 1 ~ NUM_TAIL_ELEMS elements and the "perm" part has the other elements. - uint32_t num_input_segs = ku::ceil_div(end_vocab_idx, (uint32_t)NUM_ELEMS_PER_SEG); - uint32_t num_perm_segs = num_input_segs <= NUM_SEGS_IN_INIT_WINDOW ? 0u : (num_input_segs - 1) / NUM_TAIL_SEGS * NUM_TAIL_SEGS; - uint32_t num_perm_elems = num_perm_segs * NUM_ELEMS_PER_SEG; - uint32_t num_tail_elems_padded = num_perm_segs != 0 ? (uint32_t)NUM_TAIL_ELEMS : num_input_segs * NUM_ELEMS_PER_SEG; - - // The "tail" part is handled by CTA0 in the cluster - // "local" means the current CTA (in the cluster) - uint32_t num_local_tail_elems_padded = rank_in_cluster == 0 ? num_tail_elems_padded : 0u; - uint32_t num_local_tail_segs = num_local_tail_elems_padded / NUM_ELEMS_PER_SEG; - uint32_t num_local_tail_elems = rank_in_cluster == 0 ? end_vocab_idx - num_perm_elems : 0u; - - // rank r takes the r-th contiguous interval of the row's visit order, with equal visit elements per rank - uint32_t num_total_elems_padded = num_tail_elems_padded + num_perm_elems; - // Rank r's slice of the padded row starts at r * num_total_elems_padded / cluster_size; the tail is - // rank 0's alone, so clamp that boundary into the perm part (element index within the perm part). - auto perm_boundary = [&](uint32_t rank) { - static_assert((uint64_t)Config::cluster_size * (MAX_VOCAB_SIZE + NUM_TAIL_ELEMS) <= 0xFFFFFFFFull); - uint32_t boundary = rank * num_total_elems_padded / Config::cluster_size; - return boundary > num_tail_elems_padded ? boundary - num_tail_elems_padded : 0u; - }; - uint32_t local_start_seg_idx = perm_boundary(rank_in_cluster) / NUM_ELEMS_PER_SEG; - uint32_t local_end_seg_idx = perm_boundary(rank_in_cluster + 1) / NUM_ELEMS_PER_SEG; - uint32_t num_local_perm_segs = local_end_seg_idx - local_start_seg_idx; - uint32_t num_local_elems = num_local_tail_elems_padded + num_local_perm_segs * NUM_ELEMS_PER_SEG; - - uint32_t num_local_rounds = ku::ceil_div(num_local_elems, (uint32_t)NUM_ELEMS_PER_ROUND); - uint32_t num_local_init_rounds = min(num_local_rounds, (uint32_t)NUM_INIT_ROUNDS_MAX); - - uint32_t num_local_segs = num_local_tail_segs + num_local_perm_segs; - uint32_t used_segs = num_local_init_rounds * NUM_SEGS_PER_ROUND; - uint32_t rem_segs = num_local_segs > used_segs ? num_local_segs - used_segs : 0u; - uint32_t warp_rem_segs = warp_idx < rem_segs ? rem_segs - warp_idx : 0u; - uint32_t warp_round_limit = ku::ceil_div(warp_rem_segs, (uint32_t)NUM_SEGS_PER_ROUND); - - // Each CTA gets its local top-k elements - bool nan_seen = false; - BF16Base::template scan_segs( - tma_params, smem, - batch_idx, end_vocab_idx, args.topk, warp_idx, lane_idx, - num_perm_segs, local_start_seg_idx, num_local_perm_segs, num_local_tail_elems_padded, num_local_tail_elems, - survivor_buf_idx, - nan_seen, - [&](uint32_t m) { return m < warp_round_limit; } - ); - - // CTA-wide OR; this is also the barrier that makes the scan's writes to the survivor buffer visible to the gather below - nan_seen = __syncthreads_or(nan_seen) != 0; - - // The cluster barrier must not complete before every CTA has finished its own scan: the gather - // below writes into CTA0's shared memory, so a CTA that finished early would otherwise clobber - // CTA0's `tma_load_buf` / `incoming_topk_pairs` while CTA0 is still scanning them. - ku::barrier_cluster_arrive_relaxed(); // Paired with the `barrier_cluster_wait_acquire` below - ku::barrier_cluster_wait_acquire(); - - static_assert(NUM_GATHER_UNITS * sizeof(uint32_t) <= sizeof(smem.incoming_topk_pairs)); - void* val_dst = (void*)(int64_t)cute::set_block_rank( - cute::cast_smem_ptr_to_uint((uint32_t*)smem.incoming_topk_pairs + rank_in_cluster * (MAX_TOPK / 2)), 0); - - static_assert(NUM_GATHER_PAIRS * sizeof(uint64_t) <= sizeof(SharedMemoryPlanBase::tma_load_buf)); - void* dst = (void*)(int64_t)cute::set_block_rank( - cute::cast_smem_ptr_to_uint(reinterpret_cast(smem.tma_load_buf) + rank_in_cluster * MAX_TOPK), 0); - - static_assert(MAX_TOPK*sizeof(ValueT) % 16 == 0); - constexpr uint32_t NUM_VAL_CHUNKS = MAX_TOPK * sizeof(ValueT) / 16; // 16B store - constexpr uint32_t NUM_VAL_CHUNKS_PER_THREAD = ku::ceil_div(NUM_VAL_CHUNKS, NUM_THREADS); - - if (warp_idx == 0 && cute::elect_one_sync()) { - if (rank_in_cluster == 0) { - smem.gather_val_bar.arrive_and_expect_tx((NUM_VAL_CHUNKS * 16 + sizeof(uint32_t)) * CLUSTER_SIZE); - smem.gather_bar.arrive_and_expect_tx((MAX_TOPK * (uint32_t)sizeof(uint64_t)) * CLUSTER_SIZE); - } - st_async_32b( - cute::set_block_rank(cute::cast_smem_ptr_to_uint(smem.gathered_num_survivors + rank_in_cluster), 0), - (uint32_t)nan_seen, - smem.gather_val_bar - ); - } - - // Store values first... so that CTA0 can begin pivot selection earlier - CUTE_UNROLL - for (uint32_t i = 0; i < NUM_VAL_CHUNKS_PER_THREAD; ++i) { - uint32_t c = i * NUM_THREADS + threadIdx.x; - if constexpr (NUM_VAL_CHUNKS % NUM_THREADS != 0) { - if (c >= NUM_VAL_CHUNKS) break; - } - uint32_t vw[4]; - CUTE_UNROLL - for (uint32_t j = 0; j < 4; ++j) { - uint32_t pair2[4]; - topk_select_common::ld_shared<4>(pair2, (const uint32_t*)(smem.surviving_topk_pairs[survivor_buf_idx] + 8 * c + 2 * j)); - vw[j] = __byte_perm(pair2[1], pair2[3], 0x5410); - } - ku::st_async((char*)val_dst + c * sizeof(uint4), make_uint4(vw[0], vw[1], vw[2], vw[3]), - smem.gather_val_bar); - } - - // And then store those (value, index) pairs - static_assert(MAX_TOPK * sizeof(uint64_t) % 16 == 0); - constexpr uint32_t NUM_INDEX_VALUE_CHUNKS = MAX_TOPK * sizeof(uint64_t) / 16; - static_assert(NUM_INDEX_VALUE_CHUNKS % NUM_THREADS == 0); - CUTE_UNROLL - for (uint32_t i = 0; i < NUM_INDEX_VALUE_CHUNKS / NUM_THREADS; ++i) { - uint32_t s = 2 * (i * NUM_THREADS + threadIdx.x); - ulonglong2 pp = *reinterpret_cast(smem.surviving_topk_pairs[survivor_buf_idx] + s); - ku::st_async((char*)dst + s * sizeof(uint64_t), pp, smem.gather_bar); - } - - if (rank_in_cluster != 0) { - return; - } - - // The following code is CTA0 only - smem.gather_val_bar.wait(0); - BF16Base::clear_reconstruct_histograms(smem, threadIdx.x); - __syncthreads(); - - static_assert(Config::cluster_size <= 32); - uint32_t stored_num_survivors = lane_idx < Config::cluster_size ? smem.gathered_num_survivors[lane_idx] : 0u; - nan_seen |= stored_num_survivors; - - uint32_t unit_base = threadIdx.x * NUM_UINT32_GATHER_PER_THREAD; - uint32_t num_my_units = unit_base < NUM_GATHER_UNITS ? min((uint32_t)NUM_UINT32_GATHER_PER_THREAD, NUM_GATHER_UNITS - unit_base) : 0u; - - const uint32_t *gather_vals = (const uint32_t*)smem.incoming_topk_pairs; - nv_bfloat162 values[NUM_UINT32_GATHER_PER_THREAD]; - CUTE_UNROLL - for (uint32_t i = 0; i < NUM_UINT32_GATHER_PER_THREAD; i++) { - if (i == num_my_units) break; - values[i] = topk_select_common::u32_to_bf16x2(gather_vals[unit_base + i]); - } - BF16Base::histogram_radix_msb(smem.reconstruct_bucket_counter[0], values, num_my_units); - __syncthreads(); - - auto [pivot_value_x2_bits, out_prefix, eq_quota, cnt_nan] = - BF16Base::template compute_pivot_and_quota(args.topk, values, num_my_units, warp_idx, lane_idx, smem); - nan_seen |= cnt_nan != 0; - - smem.gather_bar.wait(0); - __syncthreads(); - - uint32_t out_ptr = cute::cast_smem_ptr_to_uint(smem.surviving_topk_pairs[survivor_buf_idx ^ 1]) + out_prefix * (uint32_t)sizeof(uint64_t); - uint32_t gather_pairs_base = cute::cast_smem_ptr_to_uint(smem.tma_load_buf); - CUTE_UNROLL - for (uint32_t m = 0; m < NUM_UINT32_GATHER_PER_THREAD; m++) { - if (m == num_my_units) break; - BF16Base::template copy_selected_pairs_to_survivor(out_ptr, eq_quota, values[m], pivot_value_x2_bits, gather_pairs_base + 2 * (unit_base + m) * (uint32_t)sizeof(uint64_t)); - } - survivor_buf_idx ^= 1; - - nan_seen = __syncthreads_or(nan_seen) != 0; - if (nan_seen) { - BF16Base::take_action_when_have_nan(args, batch_idx); - return; - } - - BF16Base::template stage_output_and_epilogue( - smem, *reinterpret_cast(smem.incoming_topk_pairs), args, - batch_idx, end_vocab_idx, survivor_buf_idx, warp_idx, lane_idx); + select_devfunc(*reinterpret_cast(wksp_buf), args, batch_idx, end_vocab_idx, rank_in_cluster, warp_idx, lane_idx); } }; @@ -280,10 +959,13 @@ void run_topk_select_kernel(const TopkSelectArgs &args) { using Kernel = TopkSelectKernelBF16Cluster; KU_ASSERT(args.topk <= Kernel::MAX_TOPK, "topk is too large. Maximum allowed: %d\n", Kernel::MAX_TOPK); + // `vocab_size` bounds every row's `end` + KU_ASSERT(!Config::sorted_index || ku::ceil_div(args.vocab_size, (uint32_t)Kernel::CLUSTER_SIZE) <= Kernel::SORTED_MAX_SLICE - 63, + "`sorted_index` slice too long for the cluster kernel"); static_assert(INPUT_STRIDE_ALIGNMENT_REQUIREMENT % 16 == 0); auto kernel = topk_kernel; - constexpr size_t smem_size = sizeof(typename Kernel::SharedMemoryPlanBF16Cluster); + constexpr size_t smem_size = sizeof(typename Kernel::PlanT); KU_ASSERT(smem_size * Kernel::TARGET_OCCUPANCY <= args.shared_memory_size_per_sm); KU_CUDA_CHECK(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); if constexpr (Kernel::CLUSTER_SIZE > 8) { diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/block_amax.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/block_amax.cuh index 330073800b..63ae02bb0d 100644 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/block_amax.cuh +++ b/python/sglang/kernels/jit/csrc/deepseek_v4/block_amax.cuh @@ -9,30 +9,39 @@ #include #include +#include #include #include +#include namespace sglang { /// Level-one keys of the two-level indexer: the max of each kBlockTokens-score -/// block, the row's newest block forced to +inf. Contract: BlockAmaxKernel. +/// block, the row's newest block forced to +inf. Scores and keys are each fp32 +/// or bf16; a bf16 score's max is exact in either key type. Contract: BlockAmaxKernel. +template struct BlockAmaxConfig { - using DType = float; + using DType = InT; + using KeyT = OutT; static constexpr uint32_t kBlockTokens = 8; // scores per key static constexpr uint32_t kBlockSize = 512; - static constexpr uint32_t kNumItems = 2; // keys per thread static constexpr uint32_t kOccupancy = 4; - static constexpr uint32_t kKeysPerCTA = kBlockSize * kNumItems; - // One block is 32 B: a single load on Blackwell, two 16 B loads before it. - static constexpr uint32_t kVecSize = device::kMaxVecBytes / sizeof(DType); + // One fp32 block is 32 B: a single load on Blackwell, two 16 B loads before it. + // A bf16 block is one 16 B load anywhere. + static constexpr uint32_t kBlockBytes = kBlockTokens * sizeof(DType); + static constexpr uint32_t kVecBytes = std::min(kBlockBytes, device::kMaxVecBytes); + static constexpr uint32_t kVecSize = kVecBytes / sizeof(DType); static constexpr uint32_t kVecsPerBlock = kBlockTokens / kVecSize; + static constexpr uint32_t kNumItems = 64 / kBlockBytes; // 64B scores (16 registers) per thread + static constexpr uint32_t kKeysPerCTA = kBlockSize * kNumItems; static_assert(kVecsPerBlock * kVecSize == kBlockTokens); using vec_t = device::AlignedVector; }; +template struct BlockAmaxParams { - const BlockAmaxConfig::DType* __restrict__ scores; - BlockAmaxConfig::DType* __restrict__ amax_scores; + const InT* __restrict__ scores; + OutT* __restrict__ amax_scores; const int32_t* __restrict__ seq_len; int64_t stride_scores; // in elements int64_t stride_amax_scores; // in elements @@ -41,12 +50,11 @@ struct BlockAmaxParams { /// grid = (rows, ceil(max_keys / kKeysPerCTA)); a CTA owns kKeysPerCTA consecutive /// keys of one row, a thread kNumItems keys kBlockSize apart (coalesced loads). -template -__global__ __launch_bounds__(BlockAmaxConfig::kBlockSize, BlockAmaxConfig::kOccupancy) // - void amax8_varlen_kernel(const __grid_constant__ BlockAmaxParams params) { +template +__global__ __launch_bounds__(BlockAmaxConfig::kBlockSize, BlockAmaxConfig::kOccupancy) // + void amax8_varlen_kernel(const __grid_constant__ BlockAmaxParams params) { using namespace device; - using C = BlockAmaxConfig; - using T = typename C::DType; + using C = BlockAmaxConfig; using vec_t = typename C::vec_t; const auto bx = blockIdx.x; const auto by = blockIdx.y; @@ -81,25 +89,41 @@ __global__ __launch_bounds__(BlockAmaxConfig::kBlockSize, BlockAmaxConfig::kOccu for (uint32_t i = 0; i < C::kNumItems; ++i) { const auto idx = first_key + tx + i * C::kBlockSize; if (idx < num_keys) { - T key = vec[i][0][0]; + // fmaxf and __hmax2 ignore a NaN score, torch.amax would propagate it + if constexpr (std::is_same_v) { + float key = vec[i][0][0]; #pragma unroll - for (uint32_t v = 0; v < C::kVecsPerBlock; ++v) { + for (uint32_t v = 0; v < C::kVecsPerBlock; ++v) { #pragma unroll - for (uint32_t j = 0; j < C::kVecSize; ++j) { - key = fmaxf(key, vec[i][v][j]); // a NaN score is ignored, torch.amax would propagate it + for (uint32_t j = 0; j < C::kVecSize; ++j) { + key = fmaxf(key, vec[i][v][j]); + } } + const auto fp32_inf = std::numeric_limits::infinity(); + out[idx] = cast(idx + 1 == num_keys ? fp32_inf : key); + } else { + const auto vec_ptr = reinterpret_cast(vec[i][0].data()); + static_assert(C::kVecSize % 2 == 0); + bf16x2_t key2 = vec_ptr[0]; +#pragma unroll + for (uint32_t v = 1; v < C::kVecsPerBlock * C::kVecSize / 2; ++v) { + key2 = __hmax2(key2, vec_ptr[v]); + } + const auto key = __hmax(key2.x, key2.y); + const auto bf16_inf = cast(std::numeric_limits::infinity()); + out[idx] = cast(idx + 1 == num_keys ? bf16_inf : key); } - out[idx] = idx + 1 == num_keys ? std::numeric_limits::infinity() : key; } } } /// Host entry: `amax_scores[b, i] = max(scores[b, 8 i : 8 i + 8])` for /// `i < ceil(seq_len[b] / 8)`, the last of them +inf; rows with at most `topk` -/// blocks untouched. `scores` rows must stay 32 B aligned (stride % 8 == 0). +/// blocks untouched. `scores` (`InT`) rows must keep every block of 8 aligned +/// (stride % 8 == 0); `amax_scores` is `OutT`. Each type is fp32 or bf16. /// The grid covers `amax_scores`' width, so the caller sizes it for the longest /// row: `seq_len[b] <= 8 * amax_scores.shape[1]` for every row (not checked). -template +template struct BlockAmaxKernel { static void amax8_varlen( const tvm::ffi::TensorView scores, @@ -107,40 +131,34 @@ struct BlockAmaxKernel { const tvm::ffi::TensorView amax_scores, const uint32_t topk) { using namespace host; - using C = BlockAmaxConfig; + using C = BlockAmaxConfig; auto B = SymbolicSize{"batch_size"}; auto L = SymbolicSize{"max_seq_len"}; - auto S = SymbolicSize{"stride_scores"}; auto K = SymbolicSize{"max_keys"}; - auto O = SymbolicSize{"stride_amax_scores"}; auto device_ = SymbolicDevice{}; device_.set_options(); TensorMatcher({B, L}) // scores - .with_strides({S, 1}) + .with_strides({-1, 1}) .with_dtype() .with_device(device_) + .ensure_alignment(C::kBlockBytes) .verify(scores); TensorMatcher({B}) // seq_lens .with_dtype() .with_device(device_) .verify(seq_lens); TensorMatcher({B, K}) // amax_scores - .with_strides({O, 1}) - .with_dtype() + .with_strides({-1, 1}) + .with_dtype() .with_device(device_) .verify(amax_scores); - RuntimeCheck(S.unwrap() % C::kBlockTokens == 0, "stride_scores must keep every block 32 B aligned"); - RuntimeCheck( - reinterpret_cast(scores.data_ptr()) % (C::kBlockTokens * sizeof(typename C::DType)) == 0, - "scores must be 32 B aligned"); - RuntimeCheck(K.unwrap() > 0, "amax_scores must hold at least one key per row"); const auto max_keys = K.unwrap(); // ceil(longest row / 8), sized by the caller - const auto params = BlockAmaxParams{ - .scores = static_cast(scores.data_ptr()), - .amax_scores = static_cast(amax_scores.data_ptr()), + const auto params = BlockAmaxParams{ + .scores = static_cast(scores.data_ptr()), + .amax_scores = static_cast(amax_scores.data_ptr()), .seq_len = static_cast(seq_lens.data_ptr()), - .stride_scores = S.unwrap(), - .stride_amax_scores = O.unwrap(), + .stride_scores = scores.stride(0), + .stride_amax_scores = amax_scores.stride(0), .topk = topk, }; const auto grid = dim3( @@ -148,7 +166,7 @@ struct BlockAmaxKernel { static_cast(div_ceil(max_keys, static_cast(C::kKeysPerCTA)))); LaunchKernel(grid, C::kBlockSize, device_.unwrap()) .config({.use_pdl = kPDL}) - .launch(amax8_varlen_kernel, params); + .launch(amax8_varlen_kernel, params); } }; diff --git a/python/sglang/kernels/jit/csrc/occupancy/cluster_probe.cuh b/python/sglang/kernels/jit/csrc/occupancy/cluster_probe.cuh index 1a990c715e..0082212083 100644 --- a/python/sglang/kernels/jit/csrc/occupancy/cluster_probe.cuh +++ b/python/sglang/kernels/jit/csrc/occupancy/cluster_probe.cuh @@ -30,24 +30,26 @@ uint32_t get_max_active_clusters(uint32_t cluster_size, uint32_t num_waves) { const auto smem = (std::min(budget - std::min(budget, reserved), static_cast(smem_per_block))) & ~1023u; const auto num_warps = std::max(1u, std::min(1024u, static_cast(max_threads_per_sm) / num_waves) / 32); - // Widths above 8 are non-portable and the query rejects them without this. - CHECK_CUDA(cudaFuncSetAttribute( - reinterpret_cast(dummy_probe_kernel), cudaFuncAttributeNonPortableClusterSizeAllowed, 1)); - CHECK_CUDA(cudaFuncSetAttribute( - reinterpret_cast(dummy_probe_kernel), - cudaFuncAttributeMaxDynamicSharedMemorySize, - static_cast(smem))); - - cudaLaunchConfig_t config = {}; // stream/dynamicSmemBytes must not be garbage - config.gridDim = dim3{cluster_size, 1024u}; - config.blockDim = dim3{32, num_warps}; + const auto kernel = reinterpret_cast(dummy_probe_kernel); + if (cluster_size > 8) { + // Widths above 8 are non-portable and the query rejects them without this. + CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributeNonPortableClusterSizeAllowed, 1)); + } + CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem)); + int probed_occupancy = 0; + const auto block_size = 32 * num_warps; + CHECK_CUDA(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&probed_occupancy, dummy_probe_kernel, block_size, smem)); + CHECK_HOST(probed_occupancy == static_cast(num_waves)); + cudaLaunchConfig_t config = {}; + config.gridDim = dim3{cluster_size}; + config.blockDim = dim3{block_size}; config.dynamicSmemBytes = smem; config.numAttrs = 1; cudaLaunchAttribute attr = {}; attr.id = cudaLaunchAttributeClusterDimension; attr.val.clusterDim = {cluster_size, 1, 1}; config.attrs = &attr; - CHECK_CUDA(cudaOccupancyMaxActiveClusters(&num_clusters, dummy_probe_kernel, &config)); + CHECK_CUDA(cudaOccupancyMaxActiveClusters(&num_clusters, kernel, &config)); return num_clusters; #endif } diff --git a/python/sglang/kernels/jit/utils/__init__.py b/python/sglang/kernels/jit/utils/__init__.py index 0e52be9226..2469116a82 100644 --- a/python/sglang/kernels/jit/utils/__init__.py +++ b/python/sglang/kernels/jit/utils/__init__.py @@ -21,6 +21,7 @@ from sglang.kernels.jit.utils.compile import ( cuda_stubs_dir, load_jit, make_cpp_args, + parallel_compile, ) __all__ = [ @@ -34,6 +35,7 @@ __all__ = [ "is_musa_runtime", "make_cpp_args", "load_jit", + "parallel_compile", "cuda_stubs_dir", "override_jit_cuda_arch", "get_activation_cuda_cflags", diff --git a/python/sglang/kernels/jit/utils/compile/__init__.py b/python/sglang/kernels/jit/utils/compile/__init__.py index fe757b5904..14e7dfb5d8 100644 --- a/python/sglang/kernels/jit/utils/compile/__init__.py +++ b/python/sglang/kernels/jit/utils/compile/__init__.py @@ -13,6 +13,7 @@ Modules, in dependency order (there are no cycles):: ninja generating and running build.ninja, reading its depfiles cache build_key / deps_key, cache layout, publication loader load_jit + parallel parallel_compile: several builds at once """ from sglang.kernels.jit.utils.compile.cpp_args import ( @@ -22,6 +23,7 @@ from sglang.kernels.jit.utils.compile.cpp_args import ( make_cpp_args, ) from sglang.kernels.jit.utils.compile.loader import load_jit +from sglang.kernels.jit.utils.compile.parallel import parallel_compile from sglang.kernels.jit.utils.compile.paths import ( DEFAULT_CFLAGS, DEFAULT_INCLUDE, @@ -43,4 +45,5 @@ __all__ = [ "cuda_stubs_dir", "load_jit", "make_cpp_args", + "parallel_compile", ] diff --git a/python/sglang/kernels/jit/utils/compile/parallel.py b/python/sglang/kernels/jit/utils/compile/parallel.py new file mode 100644 index 0000000000..f602768d93 --- /dev/null +++ b/python/sglang/kernels/jit/utils/compile/parallel.py @@ -0,0 +1,32 @@ +"""``parallel_compile``: build several JIT modules at once.""" + +from __future__ import annotations + +import os +from concurrent.futures import ThreadPoolExecutor +from typing import Callable, Iterable, List, Optional, TypeVar + +T = TypeVar("T") + + +def parallel_compile( + builders: Iterable[Callable[[], T]], + max_workers: Optional[int] = None, +) -> List[T]: + """Run module builders concurrently and return their results in order. + + A builder is a call that ends in :func:`load_jit`, typically a ``cache_once`` + module factory with its arguments bound. Each build is its own ninja/nvcc + subprocess under its own build lock, so the threads only wait on them, and a + cached module returns at once. Use it to build every variant a dispatch may + reach up front (e.g. before CUDA graph capture) instead of one at a time on + first use. Every builder runs to completion; the first exception is raised + after that. + """ + builders = list(builders) + if len(builders) <= 1: + return [build() for build in builders] + workers = max_workers or min(len(builders), (os.cpu_count() or 1) + 4) + with ThreadPoolExecutor(max_workers=workers, thread_name_prefix="jit-build") as ex: + futures = [ex.submit(build) for build in builders] + return [future.result() for future in futures] diff --git a/python/sglang/kernels/ops/attention/deep_select.py b/python/sglang/kernels/ops/attention/deep_select.py index d6338f839b..5bbc7609b6 100644 --- a/python/sglang/kernels/ops/attention/deep_select.py +++ b/python/sglang/kernels/ops/attention/deep_select.py @@ -1,5 +1,6 @@ from __future__ import annotations +import functools from typing import TYPE_CHECKING, Dict, NamedTuple, Optional, Tuple import torch @@ -10,6 +11,7 @@ from sglang.kernels.jit.utils import ( cache_once, load_jit, make_cpp_args, + parallel_compile, ) if TYPE_CHECKING: @@ -19,7 +21,7 @@ if TYPE_CHECKING: _INPUT_STRIDE_ALIGNMENT_BYTES = 128 _OUTPUT_STRIDE_ALIGNMENT_BYTES = 32 _CLUSTER_MAX_TOPK = 1024 -_CLUSTER_MAX_BATCH_SIZE = 6 +_CLUSTER_MIN_VOCAB_SIZE = 16384 class _Config(NamedTuple): @@ -79,11 +81,26 @@ _NORMAL_CONFIGS: Dict[Tuple[torch.dtype, int], Tuple[_Config, ...]] = { SUPPORTED_CUDA_ARCHS = ((9, 0), (10, 0), (10, 3)) +# (max rows, cluster size), widest first: the most rows each cluster size runs in one wave @cache_once -def _cluster_tuning(device_capability: Tuple[int, int]) -> Tuple[int, int]: - # Return (cluster_size, min_vocab_size) - major = device_capability[0] - return (8, 128 * 1024) if major == 9 else (16, 512 * 1024) +def _cluster_tiers() -> Tuple[Tuple[int, int], ...]: + from sglang.kernels.jit.utils.occupancy import ( + NoSchedulableClustersError, + get_max_active_clusters, + ) + + tiers = [] + # NOTE: only try these cluster sizes; add more variants if we need + for cluster_size in (16, 8, 4, 3, 2): + try: + # The cluster kernel runs one CTA per SM + max_rows = get_max_active_clusters(cluster_size, occupancy=1) + except NoSchedulableClustersError: + continue + # A narrower cluster earns a tier only by fitting more rows in one wave + if not tiers or max_rows > tiers[-1][0]: + tiers.append((max_rows, cluster_size)) + return tuple(tiers) @cache_once @@ -120,7 +137,8 @@ def _jit_deep_select_module( raise ValueError( "the cluster kernel is tuned for bfloat16 at max_topk 1024 only" ) - configs = [_Config(max_topk, 256, 1, 4096, 4096, 16, cluster_size)] + # The cluster kernel streams its own stages; this tuning only fixes 512 threads + configs = [_Config(max_topk, 512, 1, 8192, 4096, 5, cluster_size)] host_dispatch = "TopkCluster" classes = [ config.to_cpp_class_name( @@ -187,6 +205,51 @@ def is_deep_select_supported() -> bool: return (major, minor) in SUPPORTED_CUDA_ARCHS +class _CachedModuleDispatch: + def __init__(self, mods: list[Module], rows: list[int], fallback: Module): + self._clusters = tuple(zip(rows, mods, strict=True)) + self._fallback = fallback + + def __call__(self, rows: int, vocab_size: int): + # NOTE: deficiency: cluster + sorted_index supports up to 1M context + if vocab_size > _CLUSTER_MIN_VOCAB_SIZE: + for max_rows, module in self._clusters: + if rows <= max_rows: + return module + return self._fallback + + +@cache_once +def _precompile( + *, + dtype: torch.dtype, + topk: int, + sorted: bool, + indices_type: torch.dtype, + sorted_index: bool, + return_value: bool, + page_transform: bool, +) -> _CachedModuleDispatch: + signature = (dtype, indices_type, sorted, sorted_index, return_value) + jobs = [(_get_max_topk_bucket(topk, False), 1)] + rows = [] + if dtype == torch.bfloat16 and topk <= _CLUSTER_MAX_TOPK: + for max_rows, cluster_size in _cluster_tiers(): + jobs.append((_get_max_topk_bucket(topk, True), cluster_size)) + rows.append(max_rows) + modules = parallel_compile( + functools.partial( + _jit_deep_select_module, + *signature, + bucket, + cluster_size, + page_transform, + ) + for bucket, cluster_size in jobs + ) + return _CachedModuleDispatch(modules[1:], rows, modules[0]) + + def topk( input: torch.Tensor, topk: int, @@ -201,8 +264,8 @@ def topk( output_idx_offset: Optional[torch.Tensor] = None, idx_oob_fill_value: int = 2147483647, value_oob_fill_value: float = float("-inf"), - return_value: bool = True, - abort_when_nan_found: bool = True, + return_value: bool = False, + abort_when_nan_found: bool = False, ) -> Tuple[Optional[torch.Tensor], torch.Tensor]: """Select the largest ``topk`` values of every row of ``input``. @@ -239,7 +302,7 @@ def topk_page_transform( sorted_index: bool = False, output_idx: Optional[torch.Tensor] = None, idx_oob_fill_value: int = -1, - abort_when_nan_found: bool = True, + abort_when_nan_found: bool = False, ) -> torch.Tensor: """Top-k indices of every row of ``input``, written through a page table. @@ -291,19 +354,21 @@ def _launch_topk_impl( abort_when_nan_found: bool, ) -> Tuple[Optional[torch.Tensor], torch.Tensor]: rows, vocab_size = input.shape - device_capability = torch.cuda.get_device_capability(input.device) - cluster_size, cluster_min_vocab_size = _cluster_tuning(device_capability) - use_cluster = ( - input.dtype == torch.bfloat16 - and rows <= _CLUSTER_MAX_BATCH_SIZE - and vocab_size >= cluster_min_vocab_size - and topk <= _CLUSTER_MAX_TOPK + dtype = input.dtype + dispatch = _precompile( + dtype=dtype, + topk=topk, + sorted=sorted, + indices_type=indices_type, + sorted_index=sorted_index, + return_value=return_value, + page_transform=page_table is not None, ) values = None if return_value: values = aligned_new_empty( (rows, topk), - input.dtype, + dtype, input.device, alignment_bytes=_OUTPUT_STRIDE_ALIGNMENT_BYTES, ) @@ -314,20 +379,10 @@ def _launch_topk_impl( input.device, alignment_bytes=_OUTPUT_STRIDE_ALIGNMENT_BYTES, ) - module = _jit_deep_select_module( - input.dtype, - indices_type, - sorted, - sorted_index, - return_value, - _get_max_topk_bucket(topk, use_cluster), - cluster_size if use_cluster else 1, - page_table is not None, - ) input_storage_bytes = ( input.untyped_storage().nbytes() - input.storage_offset() * input.element_size() ) - module.topk( + dispatch(rows, vocab_size).topk( input, values, output_idx, diff --git a/python/sglang/kernels/ops/attention/dsv4/candidate_blocks.py b/python/sglang/kernels/ops/attention/dsv4/candidate_blocks.py index 4f29230705..ba4abad43a 100644 --- a/python/sglang/kernels/ops/attention/dsv4/candidate_blocks.py +++ b/python/sglang/kernels/ops/attention/dsv4/candidate_blocks.py @@ -80,8 +80,8 @@ def candidate_row_lens( @cache_once -def _jit_block_amax_module(): - args = make_cpp_args(is_arch_support_pdl()) +def _jit_block_amax_module(in_dtype: torch.dtype, out_dtype: torch.dtype): + args = make_cpp_args(is_arch_support_pdl(), in_dtype, out_dtype) return load_jit( make_name("block_amax"), *args, @@ -105,15 +105,17 @@ def amax8_varlen( selected anyway); ``topk=0`` never skips. ``out`` is allocated as ``[rows, ceil(max_seqlen / 8)]`` when not given, ``max_seqlen`` defaulting to the width of ``scores``; every ``seq_lens[b]`` must fit in ``8 * out.shape[1]``. - fp32 only for now; ``scores`` rows must be 32-byte aligned (stride a multiple - of 8). Returns ``out``. + ``scores`` and ``out`` are each fp32 or bf16 (``out`` defaults to fp32), the + rows of ``scores`` aligned to a block of 8 (stride a multiple of 8). Returns + ``out``. """ if out is None: num_tokens, max_len = scores.shape if max_seqlen == 0: max_seqlen = max_len - out = scores.new_empty(num_tokens, (max_seqlen + 7) // 8) - _jit_block_amax_module().amax8_varlen(scores, seq_lens, out, topk) + out = scores.new_empty(num_tokens, (max_seqlen + 7) // 8, dtype=torch.float32) + module = _jit_block_amax_module(scores.dtype, out.dtype) + module.amax8_varlen(scores, seq_lens, out, topk) return out @@ -136,7 +138,9 @@ def amax_topk_blocks( plan = plan_topk_v2(nblocks) # block maxima, the newest block +inf; the top-k reads each row up to nblocks # only, so nothing past a row's keys is initialised (v2 needs stride % 4 == 0) - keys = logits.new_empty(rows, -(-max_seq_len // (4 * block)) * 4) + keys = logits.new_empty( + rows, -(-max_seq_len // (4 * block)) * 4, dtype=torch.float32 + ) amax8_varlen(logits, seq_lens, out=keys) blocks = torch.empty(rows, topk_blocks, dtype=torch.int32, device=logits.device) topk_transform_paged_v2(keys, nblocks, None, blocks, 1, plan) diff --git a/python/sglang/kernels/ops/attention/dsv4/index_logits.py b/python/sglang/kernels/ops/attention/dsv4/index_logits.py index 7832d770c5..b24711a14b 100644 --- a/python/sglang/kernels/ops/attention/dsv4/index_logits.py +++ b/python/sglang/kernels/ops/attention/dsv4/index_logits.py @@ -8,6 +8,7 @@ from typing import Iterator, Tuple import torch +from sglang.srt.environ import envs from sglang.srt.layers.attention.mqa_logits_utils import ( mqa_logits_row_bytes, mqa_logits_rows_per_chunk, @@ -16,16 +17,31 @@ from sglang.srt.utils.common import ceil_align from .candidate_table import CANDIDATE_BLOCK_SIZE +_LOGITS_DTYPE = ( + torch.bfloat16 if envs.SGLANG_OPT_DSV41_BF16_INDEX_LOGITS.get() else torch.float32 +) +_LOGITS_DTYPE_BYTES = _LOGITS_DTYPE.itemsize +# Width alignment of the flat logits: covers block-wise reads of up to 64 columns, and 128 B of bf16, which +# DeepSelect reads past a row's end (DeepGEMM pads the stride but its tensor's storage ends at the last row) +FLAT_LOGITS_WIDTH_ALIGN = 64 + def flat_index_logits_rows_per_tile( - rows: int, width: int, *, heads: int, budget_bytes: int + rows: int, + width: int, + *, + heads: int, + budget_bytes: int, + item_size: int = _LOGITS_DTYPE_BYTES, ) -> int: - """Rows per fp32 logits tile within ``budget_bytes``, at the kernel's row - alignment.""" + """Rows per logits tile within ``budget_bytes`` (sized as fp32, which also + bounds bf16), at the kernel's row alignment.""" row_alignment = 128 // heads + # NOTE: for bf16, the estimated row bytes is 2 times the actual bytes + effective_width = width * item_size // 4 rows_per_chunk = mqa_logits_rows_per_chunk( num_rows=ceil_align(rows, row_alignment), - row_bytes=mqa_logits_row_bytes(width), + row_bytes=mqa_logits_row_bytes(effective_width), budget_bytes=budget_bytes, ) if rows_per_chunk is None: @@ -42,16 +58,15 @@ def flat_index_logits_tiles( lengths: torch.Tensor, context_lengths: list[int], budget_bytes: int, - width_align: int = 4, ) -> Iterator[tuple[slice, torch.Tensor]]: - """``(rows, logits)`` per row tile, each fp32 tile within ``budget_bytes``: + """``(rows, logits)`` per row tile, each tile within ``budget_bytes``: ``logits[i, j]`` scores query row ``rows.start + i`` against ``kv[starts + j]``, garbage past the row's ``lengths``; the width is ``max(context_lengths)`` - aligned to ``width_align``.""" + aligned to ``FLAT_LOGITS_WIDTH_ALIGN``.""" from deep_gemm import fp8_fp4_mqa_logits rows = q[0].shape[0] - width = ceil_align(max(context_lengths, default=0), width_align) + width = ceil_align(max(context_lengths, default=0), FLAT_LOGITS_WIDTH_ALIGN) if rows == 0 or width == 0: return rows_per_chunk = flat_index_logits_rows_per_tile( @@ -70,6 +85,7 @@ def flat_index_logits_tiles( tile_starts + lengths[tile], False, width, + logits_dtype=_LOGITS_DTYPE, ), ) @@ -98,6 +114,7 @@ def deep_gemm_fp4_paged_mqa_logits( deep_gemm_metadata, max_seq_len, False, + logits_dtype=_LOGITS_DTYPE, ) diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index c1bdf79f56..8d094af930 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1631,6 +1631,9 @@ class Envs: # Run the DeepSeek-V4.1 ratio-1/2 prefill indexer on the torch path instead # of the DeepGEMM dense fp4 logits kernel (test oracle / fallback). SGLANG_DSV41_TORCH_PREFILL_INDEXER = EnvBool(False) + # DeepSeek-V4.1 ratio-1/2 index layers: the DeepGEMM fp4 logits kernels store bf16 instead + # of fp32 (the reference ranks bf16 scores); every consumer of those logits must take bf16. + SGLANG_OPT_DSV41_BF16_INDEX_LOGITS = EnvBool(False) SGLANG_FP8_PAGED_MQA_LOGITS_TORCH = EnvBool(False) SGLANG_OPT_FLASHMLA_SPARSE_PREFILL = EnvBool(True) # gfx950 DeepSeek-V4.1 prefill: attend with aiter's OPUS sparse kernel over a bf16 diff --git a/python/sglang/srt/layers/attention/dsv4/indexer.py b/python/sglang/srt/layers/attention/dsv4/indexer.py index f0bac3d410..f11ec3f368 100644 --- a/python/sglang/srt/layers/attention/dsv4/indexer.py +++ b/python/sglang/srt/layers/attention/dsv4/indexer.py @@ -443,6 +443,26 @@ def topk_transform_flashinfer_fused( ) +def topk_transform_paged_bf16( + logits: torch.Tensor, + seq_lens: torch.Tensor, + page_table: torch.Tensor, + out_page_indices: torch.Tensor, + page_size: int, +) -> None: + """DeepSelect paged top-k of bf16 logits: pool slots only, unordered.""" + from sglang.kernels.ops.attention.deep_select import topk_page_transform + + topk_page_transform( + logits, + out_page_indices.shape[1], + page_table=page_table, + page_size=page_size, + end=seq_lens.reshape(-1).to(torch.int32), + output_idx=out_page_indices, + ) + + def topk_transform_paged_from_metadata( logits: torch.Tensor, metadata, @@ -464,6 +484,16 @@ def topk_transform_paged_from_metadata( page_table = metadata.page_table[rows] out_page_indices = page_indices[rows] out_raw_indices = raw_indices[rows] if raw_indices is not None else None + if logits.dtype == torch.bfloat16: + assert out_raw_indices is None + topk_transform_paged_bf16( + logits, + seq_lens, + page_table, + out_page_indices, + metadata.compressed_page_size, + ) + return if metadata.use_topk_v2: topk_transform_paged_v2( logits, diff --git a/python/sglang/srt/layers/attention/dsv4/v41_indexer/dense_blocks.py b/python/sglang/srt/layers/attention/dsv4/v41_indexer/dense_blocks.py index b9ed473ce0..eda27cf9f9 100644 --- a/python/sglang/srt/layers/attention/dsv4/v41_indexer/dense_blocks.py +++ b/python/sglang/srt/layers/attention/dsv4/v41_indexer/dense_blocks.py @@ -14,6 +14,7 @@ from sglang.kernels.ops.attention.dsv4.candidate_blocks import ( select_candidate_block_ids, topk_among_blocks, ) +from sglang.kernels.ops.attention.dsv4.index_logits import FLAT_LOGITS_WIDTH_ALIGN from .scoring import ( DeepGEMMPrefillData, @@ -23,6 +24,7 @@ from .scoring import ( prefill_requests, score_tiles, select_decode, + topk_ragged, write_decode, write_prefill, ) @@ -246,14 +248,12 @@ def _publish_prefill_blocks( topk_blocks: int, block_size: int, ) -> Tuple[torch.Tensor, torch.Tensor]: - from sglang.kernels.ops.attention.dsv4 import topk_transform_ragged_v2 - selected = data.empty_selection(topk) width = min(topk_blocks, -(-max(data.lens_per_request, default=0) // block_size)) blocks = torch.full( (data.num_rows, width), -1, dtype=torch.int32, device=selected.device ) - for tile, logits in score_tiles(data, kv, width_align=4): + for tile, logits in score_tiles(data, kv): _publish_tile_blocks( data=data, tile=tile, @@ -262,11 +262,11 @@ def _publish_prefill_blocks( topk_blocks=topk_blocks, block_size=block_size, ) - topk_transform_ragged_v2( + topk_ragged( logits, data.compress_lens[tile], - out_offsets=data.request_starts[tile], - out_indices=selected[tile], + out=selected[tile], + offsets=data.request_starts[tile], ) # Free this tile's logits before the generator scores the next one. del logits @@ -307,7 +307,9 @@ def _consume_prefill_blocks( block_size: int, ) -> torch.Tensor: selected = data.empty_selection(topk) - for tile, logits in score_tiles(data, kv, width_align=block_size): + # the block-aligned slices of `_consume_tile_blocks` stay within the logits width + assert FLAT_LOGITS_WIDTH_ALIGN % block_size == 0 + for tile, logits in score_tiles(data, kv): _consume_tile_blocks( data=data, tile=tile, diff --git a/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py b/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py index 487d1b96a7..4423b2033c 100644 --- a/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py +++ b/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py @@ -5,7 +5,7 @@ a selection back into pool slots.""" from __future__ import annotations import itertools -from typing import TYPE_CHECKING, Generator, Iterator, List, Optional, Tuple +from typing import TYPE_CHECKING, Iterator, List, Optional, Tuple import msgspec import torch @@ -207,12 +207,7 @@ def get_deep_gemm_prefill_data( ) -def score_tiles( - data: DeepGEMMPrefillData, - kv: Tuple[torch.Tensor, torch.Tensor], - *, - width_align: int, -) -> Generator[Tuple[slice, torch.Tensor], None, None]: +def score_tiles(data: DeepGEMMPrefillData, kv: Tuple[torch.Tensor, torch.Tensor]): yield from flat_index_logits_tiles( q=(data.q_fp4, data.q_sf), kv=kv, @@ -221,10 +216,35 @@ def score_tiles( lengths=data.compress_lens, context_lengths=data.lens_per_request, budget_bytes=_DEEP_GEMM_SCORE_BUDGET_BYTES, - width_align=width_align, ) +def topk_ragged( + logits: torch.Tensor, + lens: torch.Tensor, + *, + out: torch.Tensor, + offsets: torch.Tensor, +) -> None: + """Top-k of each row's first ``lens`` columns plus ``offsets`` into ``out``, + ``-1`` padded, unordered: DeepSelect for bf16 logits, top-k v2 for fp32.""" + if logits.dtype == torch.bfloat16: + from sglang.kernels.ops.attention.deep_select import topk + + topk( + logits, + out.shape[1], + end=lens, + output_idx=out, + output_idx_offset=offsets, + idx_oob_fill_value=-1, + ) + return + from sglang.kernels.ops.attention.dsv4 import topk_transform_ragged_v2 + + topk_transform_ragged_v2(logits, lens, out_offsets=offsets, out_indices=out) + + def dense_prefill_topk( data: DeepGEMMPrefillData, kv: Tuple[torch.Tensor, torch.Tensor], @@ -232,16 +252,11 @@ def dense_prefill_topk( out: torch.Tensor, ) -> None: """Request-local compressed positions into ``out``, ``-1`` padded, unordered.""" - from sglang.kernels.ops.attention.dsv4 import topk_transform_ragged_v2 - # Score columns are request-relative, so a zero offset emits request-local positions. zero_offsets = torch.zeros_like(data.request_starts) - for tile, logits in score_tiles(data, kv, width_align=4): - topk_transform_ragged_v2( - logits, - data.compress_lens[tile], - out_offsets=zero_offsets[tile], - out_indices=out[tile], + for tile, logits in score_tiles(data, kv): + topk_ragged( + logits, data.compress_lens[tile], out=out[tile], offsets=zero_offsets[tile] ) # Free this tile's logits before the generator scores the next one. del logits diff --git a/python/sglang/srt/layers/attention/dsv4/v41_indexer/sparse_table.py b/python/sglang/srt/layers/attention/dsv4/v41_indexer/sparse_table.py index 85f4947b1d..efc7ede22c 100644 --- a/python/sglang/srt/layers/attention/dsv4/v41_indexer/sparse_table.py +++ b/python/sglang/srt/layers/attention/dsv4/v41_indexer/sparse_table.py @@ -28,7 +28,10 @@ from sglang.kernels.ops.attention.dsv4.topk import ( topk_transform_ragged_v2, topk_transform_sparse, ) -from sglang.srt.layers.attention.dsv4.indexer import topk_transform_paged_from_metadata +from sglang.srt.layers.attention.dsv4.indexer import ( + topk_transform_paged_bf16, + topk_transform_paged_from_metadata, +) from sglang.srt.layers.attention.dsv4.metadata import expand_index_page_table from sglang.srt.utils.common import async_h2d @@ -39,6 +42,7 @@ from .scoring import ( get_deep_gemm_prefill_data, get_index_k_cache, score_tiles, + topk_ragged, ) from .types import ( CandidateMetadata, @@ -188,14 +192,23 @@ class SparseTableBackend: # The block-selection chain reads logits after the main stream moves on. logits.record_stream(self.alt_stream) # TODO(candidate): one kernel for both selections below (dense logits read once) - topk_transform_paged_v2( - logits, - seq_lens, - metadata.page_table, - inputs.out_page_indices, - metadata.compressed_page_size, - metadata.topk_metadata, - ) + if logits.dtype == torch.bfloat16: + topk_transform_paged_bf16( + logits, + seq_lens, + metadata.page_table, + inputs.out_page_indices, + metadata.compressed_page_size, + ) + else: + topk_transform_paged_v2( + logits, + seq_lens, + metadata.page_table, + inputs.out_page_indices, + metadata.compressed_page_size, + metadata.topk_metadata, + ) # per row: block count for the block top-k, sparse-row length for the consumers with torch.cuda.stream(self.alt_stream): nblocks, row_valid_lens = candidate_row_lens(seq_lens, self.topk_blocks) @@ -403,20 +416,17 @@ def publish_prefill_table( blocks = torch.empty(rows, topk_blocks, dtype=torch.int32, device=device) zero_offsets = torch.zeros(rows, dtype=torch.int32, device=device) # the block keys read the score rows through 32-byte vectors - for tile, logits in score_tiles(data, kv, width_align=8): + for tile, logits in score_tiles(data, kv): lens = data.compress_lens[tile] - topk_transform_ragged_v2( - logits, - lens, - out_offsets=zero_offsets[tile], - out_indices=out_positions[tile], - ) + topk_ragged(logits, lens, out=out_positions[tile], offsets=zero_offsets[tile]) # keys past a row's block count stay unset; the top-k reads a row # up to its block count only (v2 wants the stride a multiple of 4) width_blocks = ( logits.shape[1] + CANDIDATE_BLOCK_SIZE - 1 ) // CANDIDATE_BLOCK_SIZE - keys = logits.new_empty(logits.shape[0], (width_blocks + 3) // 4 * 4) + keys = logits.new_empty( + logits.shape[0], (width_blocks + 3) // 4 * 4, dtype=torch.float32 + ) amax8_varlen(logits, lens, out=keys) topk_transform_ragged_v2( keys, diff --git a/test/registered/kernels/ops/attention/test_deep_select.py b/test/registered/kernels/ops/attention/test_deep_select.py index f51a6369e4..692d3a5dbb 100644 --- a/test/registered/kernels/ops/attention/test_deep_select.py +++ b/test/registered/kernels/ops/attention/test_deep_select.py @@ -79,10 +79,13 @@ def _check_topk( assert torch.all(row_values[~valid] == value_fill), f"row {r}" -@pytest.mark.parametrize("rows,width", [(4, 8192), (200, 30000)]) +# (8, 300_000) and (40, 1_000_000) take the cluster kernel: a slice that fits its window, and one that streams +@pytest.mark.parametrize( + "rows,width", [(4, 8192), (200, 30000), (8, 300_000), (40, 1_000_000)] +) def test_topk_matches_reference(rows, width): input = _make_input(rows, width, torch.bfloat16) - values, indices = topk(input, 512) + values, indices = topk(input, 512, return_value=True) assert indices.dtype == torch.int32 _check_topk(input, 512, indices, values=values) @@ -99,7 +102,7 @@ def test_sorted_index_int64(): def test_sorted_values_fp32(): input = _make_input(8, 20000, torch.float32) - values, indices = topk(input, 1024, sorted=True) + values, indices = topk(input, 1024, sorted=True, return_value=True) expected = torch.topk(input, 1024) torch.testing.assert_close(values, expected.values, rtol=0, atol=0) torch.testing.assert_close( @@ -113,7 +116,12 @@ def test_variable_row_end(): input = _make_input(len(ends), width, torch.bfloat16) end = torch.tensor(ends, dtype=torch.int32, device="cuda") values, indices = topk( - input, k, end=end, idx_oob_fill_value=-1, value_oob_fill_value=-5.0 + input, + k, + end=end, + idx_oob_fill_value=-1, + value_oob_fill_value=-5.0, + return_value=True, ) _check_topk(input, k, indices, end=end, values=values, idx_fill=-1, value_fill=-5.0) @@ -126,13 +134,33 @@ def test_caller_output_and_index_offset(): output = backing[:, :k] offset = torch.randint(0, 1 << 20, (rows,), dtype=torch.int32, device="cuda") - values, indices = topk(input, k, output_idx=output, output_idx_offset=offset) + values, indices = topk( + input, k, output_idx=output, output_idx_offset=offset, return_value=True + ) assert indices.data_ptr() == output.data_ptr() assert torch.all(backing[:, k:] == -7), "wrote past the output row" _check_topk(input, k, indices, values=values, offset=offset) +@pytest.mark.parametrize( + "rows,width,nan_col", + [(3, 8192, 100), (3, 300_000, 5), (3, 1_000_000, 999_990), (3, 1_000_000, 500_000)], +) +def test_nan_row_selects_index_zero(rows, width, nan_col): + """Without abort, a row holding a NaN selects index 0 (plus its offset) in every + slot instead of an out-of-range sentinel; the other rows are unaffected.""" + k = 512 + input = _make_input(rows, width, torch.bfloat16) + input[1, nan_col] = float("nan") + offset = torch.randint(0, 1 << 20, (rows,), dtype=torch.int32, device="cuda") + values, indices = topk(input, k, output_idx_offset=offset, return_value=True) + assert torch.all(indices[1] == offset[1]) + assert torch.equal(values[1], input[1, :1].expand(k)) + keep = torch.tensor([0, 2], device="cuda") + _check_topk(input[keep], k, indices[keep], values=values[keep], offset=offset[keep]) + + def _random_page_table(rows: int, width: int, page_size: int): """Distinct page ids per slot, plus the inverse map from page id to logical page.""" num_pages = (width + page_size - 1) // page_size