diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/topk_v2.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/topk_v2.cuh index 4272353273..751eec4a7a 100644 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/topk_v2.cuh +++ b/python/sglang/kernels/jit/csrc/deepseek_v4/topk_v2.cuh @@ -49,7 +49,25 @@ constexpr uint32_t kMaxTopK = impl::TopKConfig::kMaxTopK; constexpr uint32_t kReg2MaxSeqLen = Register2::kMaxSeqLen; // 8192 constexpr uint32_t kReg4MaxSeqLen = Register4::kMaxSeqLen; // 16384 +#ifdef USE_ROCM +// HIP reads the second __launch_bounds__ argument as waves per SIMD, CUDA as +// blocks per SM, so kOccupancy asks ROCm for 2 waves/SIMD -- below the 4 a +// 1024-thread block already forces. Do NOT translate kOccupancy into this: +// "kOccupancy blocks per CU" is 8 waves/SIMD, which caps the allocator at +// 512/8 = 64 VGPRs and spills, and a second resident block is unreachable with +// a grid of batch_size. wave64 is a literal because __AMDGCN_WAVEFRONT_SIZE__ +// is gone in ROCm 7 and warpSize is not constexpr. +inline constexpr uint32_t kSimdsPerCu = 4; +inline constexpr uint32_t kWavefrontSize = 64; // CDNA +inline constexpr uint32_t kWavesPerBlock = kBlockSize / kWavefrontSize; +inline constexpr uint32_t kMinWavesPerSimd = kWavesPerBlock / kSimdsPerCu; +static_assert(kMinWavesPerSimd > 0, "kBlockSize must cover at least one wave per SIMD"); +#define TOPK_KERNEL __global__ __launch_bounds__(kBlockSize, kMinWavesPerSimd) +#else +// topk_small_batch_cluster_kernel shadows kOccupancy with its own template +// parameter and CLUSTER_TOPK_KERNEL relies on that, so leave the CUDA spelling. #define TOPK_KERNEL __global__ __launch_bounds__(kBlockSize, kOccupancy) +#endif /// Metadata tensor rows (each 8 B / 2 int32). Row 0 is the global plan result; /// rows 1..N are the (batch_id, seq_len) of items routed to the cluster pool. @@ -644,8 +662,9 @@ struct SplitWorkspace { uint32_t floor; ///< same value the host dispatched on }; -/// 12 histogram bits, the width TopKStreaming uses: a 10-bit threshold bin holds -/// more unequal scores than kMaxNumTie can stage, and drops the rest in silence. +/// 12 histogram bits, the width TopKStreaming uses: a 10-bit threshold bin overflows +/// kMaxNumTie far more often, and each overflowing row costs the epilogue a +/// refine_ties rescan of the whole row in one block. struct TopKSplit : impl::TopKRadixBase<12> { using Base = impl::TopKRadixBase<12>; static_assert(kHistSize % kBlockSize == 0, "the histogram is transferred kHistItems bins per thread"); @@ -723,8 +742,9 @@ struct TopKSplit : impl::TopKRadixBase<12> { // The full row's histogram and the full row's seq_len, so every rank picks // the same bin and the appends below agree on what "above" means. find_threshold(problem.topk, problem.seq_len, smem, [&](uint32_t threshold_bin) { - smem->v_hi = impl::coarse_bin_lower_bound(threshold_bin + 1); - smem->v_lo = impl::coarse_bin_lower_bound(threshold_bin + 0); + const auto [v_lo, v_hi] = impl::coarse_bin_bounds(threshold_bin); + smem->v_hi = v_hi; + smem->v_lo = v_lo; }); const auto topk = problem.topk; @@ -751,7 +771,8 @@ struct TopKSplit : impl::TopKRadixBase<12> { const auto n_eq = min(smem->count_eq, kMaxNumTie); if (tx == 0) { smem->base_gt = atomicAdd(&ctr->count_gt, n_gt); - smem->base_eq = atomicAdd(&ctr->count_eq, n_eq); + // Uncapped: finish_ties needs the row's exact bin size to detect overflow and refine. + smem->base_eq = atomicAdd(&ctr->count_eq, smem->count_eq); } __syncthreads(); const auto base_gt = smem->base_gt; @@ -788,15 +809,27 @@ struct TopKSplit : impl::TopKRadixBase<12> { } /// Fill the slots the threshold bin has to break ties for, staging the ties - /// into LDS first since handle_tie's ranking pass is all-to-all. + /// into LDS first since handle_tie's ranking pass is all-to-all. `problem` + /// must be the whole row: an overflowing bin is re-derived by rescanning it. SGL_DEVICE static void finish_ties(const TopKProblem& problem, const impl::TieValue* ties, Smem* smem) { const auto tx = threadIdx.x; - const auto above_count = smem->total_gt; - const auto tie_count = min(smem->total_eq, kMaxNumTie); + uint32_t above_count, tie_count; + if (smem->total_eq > kMaxNumTie) [[unlikely]] { + // The global list holds an arrival-order subset. refine_ties emits after + // the slots every rank already took, so seed its counter with their total. + if (tx == 0) smem->count_gt = smem->total_gt; + __syncthreads(); + refine_ties(problem, smem, smem->v_lo, smem->v_hi, smem->total_eq); + above_count = smem->count_gt; + tie_count = min(smem->count_eq, kMaxNumTie); + } else { + above_count = smem->total_gt; + tie_count = smem->total_eq; + for (uint32_t t = tx; t < tie_count; t += kBlockSize) + smem->tie_values[t] = ties[t]; + __syncthreads(); + } const auto remain_topk = above_count < problem.topk ? problem.topk - above_count : 0; - for (uint32_t t = tx; t < tie_count; t += kBlockSize) - smem->tie_values[t] = ties[t]; - __syncthreads(); handle_tie(smem->tie_values, problem, above_count, tie_count, remain_topk, &smem->tie_handle); } }; diff --git a/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/topk_impl.cuh b/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/topk_impl.cuh index b0b2088540..369868a0f0 100644 --- a/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/topk_impl.cuh +++ b/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/topk_impl.cuh @@ -14,7 +14,8 @@ /// - the dispatcher selects cluster size 8 or 16 from the probed occupancy. /// /// Algorithm: fp16 coarse histogram -> threshold bin -> fp32-boundary collect -> -/// exact radix tie-break. +/// exact radix tie-break, plus an exact-key refinement when the threshold bin +/// overflows the staging buffer (refine_ties). #pragma once @@ -90,59 +91,35 @@ SGL_DEVICE uint32_t extract_coarse_bin(float x) { return (b ^ (s | 0x80000000u)) >> (32 - kBits); } -SGL_DEVICE uint16_t coarse_bin_to_bits_finite(uint32_t bin) { - const uint16_t ob = static_cast(bin); - return (ob & 0x8000) ? static_cast(ob ^ 0x8000) : static_cast(~ob); -} - -// Smallest fp32 `v` for which `extract_coarse_bin(v) >= bin`, i.e. the -// lower fp32 boundary of coarse bin `bin`. The collect pass classifies with two -// comparisons against these instead of recomputing the fp16 bin per element, so -// this must agree with `extract_coarse_bin` on every value -- a score sitting -// exactly on a boundary included. Two pairs no fp32 threshold can separate are -// left: -0.0 at the zero bin, and +inf at a NaN-key bin. +/// fp32 bounds {lo, hi} of coarse bin `bin`: "above" iff v >= hi, "tie" iff lo <= v < hi, +/// agreeing with extract_coarse_bin on every non-NaN score. A zero hi moves to +/// the smallest subnormal, since fp32 compares cannot put -0.0 below +0.0. template -SGL_DEVICE float coarse_bin_lower_bound(uint32_t bin) { +SGL_DEVICE float2 coarse_bin_bounds(uint32_t bin) { constexpr uint32_t kShift = 16 - kBits; - constexpr uint32_t kInfBin = 0xFC00u >> kShift; // bin holding the +inf key - const uint32_t key = bin << kShift; // ordered16 key at the low edge - // ordered16 -> fp16 value (inverse of the transform in extract_coarse_bin); - constexpr auto to_finite_val = [](uint32_t okey) -> float { - const uint16_t hb = coarse_bin_to_bits_finite(okey); - return cast(*reinterpret_cast(&hb)); + static_assert(kShift >= 1, "edge keys must be even for the rounding rule below"); + constexpr uint32_t kNegInfBin = 0x03FFu >> kShift; // bin holding the -inf key + constexpr uint32_t kInfBin = 0xFC00u >> kShift; // bin holding the +inf key + // ordered16 edge keys, {lo, hi} packed in the {low, high} halves. + const uint32_t keys = (bin << kShift) | ((bin + 1) << (kShift + 16)); + // ordered16 -> fp16 bits per half: ~key below 0x8000 (negative), key ^ 0x8000 above. + const auto to_fp32 = [](uint32_t k) { + const uint32_t bits = k ^ 0x80008000u ^ ((~k >> 15) & 0x00010001u) * 0x7FFFu; + return cast(*reinterpret_cast(&bits)); }; - constexpr auto step_up = [](float v) -> float { - const int32_t b = __float_as_int(v); - return __int_as_float(b >= 0 ? b + 1 : b - 1); - }; - // Fast path, hoisted above the per-key special cases so both keys are - // range-checked at once: `key` and `key - 1` both land in the finite band - // [0x0401, 0xFBFF] -- every boundary a finite-score threshold produces. fp16 - // rounds to nearest, so the boundary is the midpoint between the fp16 values - // at `key` and `key - 1`. - if (key - 0x0401u <= 0xFBFFu - 0x0401u) { - const float mid = 0.5f * (to_finite_val(key) + to_finite_val(key - 1)); - // fp32 -> fp16 rounds to nearest EVEN, so on the ~half of bins whose fp16 - // value has an odd significand the midpoint still bins as `bin - 1`. - return (coarse_bin_to_bits_finite(key) & 1u) ? step_up(mid) : mid; - } - // Slow path: an edge of `bin` touches the +/-inf keys or NaN key space. The - // ordered-key line is: [0, 0x03FF) negative-NaN space, 0x03FF = -inf, - // [0x0400, 0xFC00) finite, 0xFC00 = +inf, (0xFC00, 0xFFFF] positive-NaN - // space. The +/-inf keys stand in as +/-65536, one ideal step past fp16 max, - // so the midpoint lands on the +/-65520 fp32 -> fp16 overflow threshold. - if (bin == 0) return -infinity_value(); // every value bins at >= 0 - if (bin > kInfBin) return infinity_value(); // NaN key space: nothing bins that high - const auto to_val = [&](uint32_t okey) -> float { - if (okey < 0x03FFu) return -infinity_value(); - if (okey == 0x03FFu) return -65536.0f; - if (okey == 0xFC00u) return 65536.0f; - return to_finite_val(okey); - }; - // The +/-65536 stand-ins are not real fp16 neighbours, so the parity rule - // does not apply here; test the property directly instead. - const float mid = 0.5f * (to_val(key) + to_val(key - 1)); - return extract_coarse_bin(mid) < bin ? step_up(mid) : mid; + const auto [v_lo, v_hi] = to_fp32(keys); + const auto [u_lo, u_hi] = to_fp32(keys - 0x00010001u); + // Exact midpoints of adjacent fp16 values; the clamp puts one against inf on the + // +/-65520 overflow point. + const auto midpoint = [](float a, float b) { return fminf(fmaxf(0.5f * (a + b), -65520.0f), 65520.0f); }; + const int32_t lo = __float_as_int(midpoint(v_lo, u_lo)); + const int32_t hi = __float_as_int(midpoint(v_hi, u_hi)); + // Ties round to even and edge keys are even, so negative edges sit one ulp above + // the midpoint. A zero hi steps past +0. + const float lo_edge = __int_as_float(lo + (lo >> 31)); + const float hi_edge = __int_as_float(hi + (hi >> 31) + (hi == 0)); + // Edges inside the NaN key space: every score is >= lo, none is >= hi. + return {bin <= kNegInfBin ? -infinity_value() : lo_edge, bin >= kInfBin ? infinity_value() : hi_edge}; } SGL_DEVICE uint32_t warp_sum_bool(bool pred, uint32_t mask = 0xFFFFFFFF) { @@ -200,6 +177,17 @@ struct TopKConfig { // by downstream sparse attention. static constexpr uint32_t kMaxNumTie = 2048; static constexpr uint32_t kRadixSize = 1 << 8; + // Radix width of the threshold-bin refinement (refine_ties). Wider + // than kRadixSize so that ceil(32 / kRefineBits) == 3 rounds cover the whole + // key: the histogram overlays the tie staging buffer, which that path + // re-derives anyway, so the extra bins are free. 12 is the widest that fits. + static constexpr uint32_t kRefineBits = 12; + static constexpr uint32_t kRefineSize = 1 << kRefineBits; + static constexpr uint32_t kRefineItems = kRefineSize / kBlockSize; + static_assert( + kRefineSize * sizeof(uint32_t) <= kMaxNumTie * sizeof(TieValue), + "the refinement histogram must fit the tie staging buffer it overlays"); + static_assert(kRefineSize % kBlockSize == 0); static constexpr uint32_t kTopKItems = (kMaxTopK + kBlockSize - 1) / kBlockSize; // tie candidates owned per thread in the strided handle_tie loops static constexpr uint32_t kTieItems = kMaxNumTie / kBlockSize; @@ -308,6 +296,44 @@ struct TopKConfig { } } + /// Threshold-digit search over the kRefineSize-bin refinement histogram in + /// `hist`, by the same rule radix_tie_select uses. One thread owns kRefineItems + /// contiguous bins. Publishes the result in `smem->match`. Block-wide; ends + /// with a barrier. + SGL_DEVICE static void refine_find_threshold( // + const uint32_t* hist, + const uint32_t total_active, + const uint32_t topk_remain, + TieHandleSmem* smem) { + const auto tx = threadIdx.x; + const auto lane_id = tx % kWarpSize; + const auto warp_id = broadcast(tx / kWarpSize); + + uint32_t counts[kRefineItems]; + uint32_t local_sum = 0; +#pragma unroll + for (uint32_t i = 0; i < kRefineItems; ++i) { + counts[i] = hist[tx * kRefineItems + i]; + local_sum += counts[i]; + } + const auto warp_inc = warp::inclusive_sum(local_sum, lane_id); + if (lane_id == kWarpSize - 1) smem->warp_sum[warp_id] = warp_inc; + __syncthreads(); + + // Inclusive prefix over all bins below this thread's first one. + uint32_t prefix = warp::reduce_sum(lane_id < warp_id ? smem->warp_sum[lane_id] : 0); + prefix += warp_inc - local_sum; +#pragma unroll + for (uint32_t i = 0; i < kRefineItems; ++i) { + prefix += counts[i]; + const auto above = total_active - prefix; // active elements in bins ABOVE this one + if (above < topk_remain && above + counts[i] >= topk_remain) { + smem->match = {tx * kRefineItems + i, above, counts[i]}; + } + } + __syncthreads(); + } + /// Exact radix select over the tie candidates: each thread owns kItems /// strided elements (inactive beyond num_ties). Requires /// num_ties <= kItems * kBlockSize. @@ -431,7 +457,13 @@ struct TopKRadixBase : TopKConfig { union { alignas(16) uint32_t histogram[kHistSize]; struct { - TieValue tie_values[kMaxNumTie]; + // refine_ties re-derives the candidates instead of trusting + // what the collect pass staged, so the buffer is free to carry that + // path's radix histogram until its staging pass writes values back. + union { + TieValue tie_values[kMaxNumTie]; + uint32_t refine_hist[kRefineSize]; + }; TieHandleSmem tie_handle; }; }; @@ -524,6 +556,96 @@ struct TopKRadixBase : TopKConfig { __syncthreads(); } + + /// Re-stages the tie buffer exactly when the threshold bin overflows it: radix + /// rounds narrow the bin to one bucket, then a final scan emits what lies above + /// it and stages the bucket for the caller's handle_tie. + SGL_DEVICE static void refine_ties( // + const TopKProblem& problem, + Smem* smem, + const float v_lo, + const float v_hi, + const uint32_t count_eq) { + constexpr uint32_t kRefineRounds = (32 + kRefineBits - 1) / kRefineBits; + const auto topk = problem.topk; + if (smem->count_gt >= topk) [[unlikely]] + return; + const auto tx = threadIdx.x; + const auto handle = &smem->tie_handle; + const auto hist = smem->refine_hist; + // for_each_input's elements with one copy of `fn` and reversed lanes, so this + // cold scan shares no load address (hence no registers) with the fast path. + const auto scan = [&](auto&& fn) { + const auto seq_len = problem.seq_len; + const auto num_vec = (seq_len + kVecSize - 1) / kVecSize; + uint32_t vi = kBlockSize - 1 - tx; + vec_t cur; + if (vi < num_vec) cur.load(problem.in, vi); +#pragma unroll 1 + while (vi < num_vec) { + const auto next_vi = vi + kBlockSize; + vec_t next; + if (next_vi < num_vec) next.load(problem.in, next_vi); +#pragma unroll + for (uint32_t j = 0; j < kVecSize; ++j) { + if (vi * kVecSize + j < seq_len) fn(cur[j], vi * kVecSize + j); + } + cur = next; + vi = next_vi; + } + }; + + // refine_find_threshold needs 0 < remain <= active; its match rule keeps it. + uint32_t remain = topk - smem->count_gt; + uint32_t active = count_eq; + // Candidates are [bucket_lo, bucket_lo + bucket_mask]; round 0 takes the whole bin. + uint32_t bucket_lo = 0; + uint32_t bucket_mask = ~0u; + +#pragma unroll + for (uint32_t round = 0; round < kRefineRounds; ++round) { + // round 0: bits [20, 32), round 1: bits [8, 20), round 2: bits [0, 8) + const uint32_t lo = (round + 1) * kRefineBits < 32 ? 32 - (round + 1) * kRefineBits : 0; +#pragma unroll + for (uint32_t i = 0; i < kRefineItems; ++i) { + hist[tx * kRefineItems + i] = 0; + } + __syncthreads(); + // bucket_lo is aligned to the previous digit, so (key - bucket_lo) >> lo is this digit. + scan([&](float val, uint32_t) { + if (!(val >= v_lo && val < v_hi)) return; + const auto d = extract_exact_bin(val) - bucket_lo; + if (d <= bucket_mask) atomicAdd(&hist[d >> lo], 1); + }); + __syncthreads(); + refine_find_threshold(hist, active, remain, handle); + const auto match = handle->match; + bucket_lo |= match.bin << lo; + bucket_mask = (1u << lo) - 1u; + remain -= match.above_count; + active = match.equal_count; + if (lo == 0 || active <= kMaxNumTie) break; + } + + // Candidates are never NaN, so keys stay <= key(+inf) and bucket_hi cannot wrap. + // Staging overwrites the histogram, which refine_find_threshold has consumed. + const auto bucket_hi = bucket_lo + bucket_mask + 1u; + if (tx == 0) smem->count_eq = 0; + __syncthreads(); + scan([&](float val, uint32_t idx) { + if (!(val >= v_lo && val < v_hi)) return; + const auto key = extract_exact_bin(val); + if (key >= bucket_hi) { + const auto pos = atomicAdd(&smem->count_gt, 1); + if (pos < topk) [[likely]] + problem.emit(pos, idx); + } else if (key >= bucket_lo) { + const auto slot = atomicAdd(&smem->count_eq, 1); + if (slot < kMaxNumTie) smem->tie_values[slot] = {val, idx}; + } + }); + __syncthreads(); + } }; // --------------------------------------------------------------------------- @@ -588,8 +710,7 @@ struct TopKRegister : TopKRadixBase<12> { // Phase 2: Find the threshold bin find_threshold(problem.topk, num_full * kVecSize, smem, [&](uint32_t threshold_bin) { - const auto v_hi = coarse_bin_lower_bound(threshold_bin + 1); - const auto v_lo = coarse_bin_lower_bound(threshold_bin + 0); + const auto [v_lo, v_hi] = coarse_bin_bounds(threshold_bin); smem->v_hi = v_hi; smem->v_lo = v_lo; }); @@ -624,6 +745,10 @@ struct TopKRegister : TopKRadixBase<12> { // Phase 4: Handle ties. __syncthreads(); + if (smem->count_eq > kMaxNumTie) [[unlikely]] { + // Buffer holds an arrival-order subset; re-stage it on the exact key. + refine_ties(problem, smem, v_lo, v_hi, smem->count_eq); + } const auto count_gt = smem->count_gt; const auto count_eq = smem->count_eq; const auto remain_topk = count_gt < topk ? topk - count_gt : 0; @@ -667,8 +792,7 @@ struct TopKStreaming : TopKRadixBase<12> { // Phase 2: Find the threshold bin find_threshold(problem.topk, problem.seq_len, smem, [&](uint32_t threshold_bin) { - const auto v_hi = coarse_bin_lower_bound(threshold_bin + 1); - const auto v_lo = coarse_bin_lower_bound(threshold_bin + 0); + const auto [v_lo, v_hi] = coarse_bin_bounds(threshold_bin); smem->v_hi = v_hi; smem->v_lo = v_lo; }); @@ -701,6 +825,10 @@ struct TopKStreaming : TopKRadixBase<12> { // "above" and "tie" sets. above_count is < topk by the threshold-bin invariant, // so the count_gt guard above effectively never triggers. __syncthreads(); + if (smem->count_eq > kMaxNumTie) [[unlikely]] { + // See the register path. + refine_ties(problem, smem, v_lo, v_hi, smem->count_eq); + } const auto count_gt = smem->count_gt; const auto count_eq = smem->count_eq; const auto remain_topk = count_gt < topk ? topk - count_gt : 0; @@ -715,6 +843,11 @@ struct TopKStreaming : TopKRadixBase<12> { // // CUDA only: thread-block clusters and distributed shared memory have no CDNA // equivalent. +// +// Still truncates an overflowing threshold bin, unlike the register, streaming +// and ROCm split paths: the candidate set is split across kClusterSize ranks +// and no single block resolves the row, so refining needs cluster-wide +// histogram and emit counters rather than the block-local ones refine_ties uses. // --------------------------------------------------------------------------- #if SUPPORT_CLUSTER @@ -853,8 +986,9 @@ struct TopKCluster : TopKRadixBase<10> { // Phase 3. rank-0 find threshold and write to local smem for other ranks to read find_threshold(problem.topk, problem.seq_len, smem, [&](uint32_t threshold_bin) { - smem->v_hi = coarse_bin_lower_bound(threshold_bin + 1); - smem->v_lo = coarse_bin_lower_bound(threshold_bin + 0); + const auto [v_lo, v_hi] = coarse_bin_bounds(threshold_bin); + smem->v_hi = v_hi; + smem->v_lo = v_lo; }); barrier_cluster_arrive_release(); // bar-2 arrive: publishes v_hi / v_lo diff --git a/test/registered/kernels/ops/attention/test_topk_v2.py b/test/registered/kernels/ops/attention/test_topk_v2.py index 99866d2e6a..8ddb596383 100644 --- a/test/registered/kernels/ops/attention/test_topk_v2.py +++ b/test/registered/kernels/ops/attention/test_topk_v2.py @@ -87,8 +87,14 @@ FIXED_CONFIGS = [ ] -def _assert_topk_close(scores_cpu, ref_raw, our_raw, bs, seq_lens, k): - """Set-compare our top-k raw indices vs torch's, tolerating equal-score ties.""" +def _assert_topk_close( + scores_cpu, ref_raw, our_raw, bs, seq_lens, k, max_permit_error=MAX_PERMIT_ERROR +): + """Set-compare our top-k raw indices vs torch's, tolerating equal-score ties. + + ``max_permit_error`` covers the fp16 coarse histogram swapping near-equal + scores; tests that pin exactness pass 0. + """ bad = 0 for i in range(bs): L = int(seq_lens[i]) @@ -105,7 +111,7 @@ def _assert_topk_close(scores_cpu, ref_raw, our_raw, bs, seq_lens, k): assert len(our) == min(k, L), ( f"b={i} L={L} k={k}: {len(our)} valid != {min(k, L)}" ) - assert bad <= MAX_PERMIT_ERROR, f"{bad=} > {MAX_PERMIT_ERROR}" + assert bad <= max_permit_error, f"{bad=} > {max_permit_error}" def _make_page_table(batch, num_pages, mode, device, per_row=False): @@ -311,6 +317,98 @@ def test_topk_v2_dual_output(batch: int, seq: int) -> None: _assert_topk_close(scores.cpu(), ref_raw, direct_raw, batch, seq_lens.cpu(), k) +# --- exactness when the coarse histogram cannot discriminate ----------------- +# The tests above draw from ``torch.randn``, which spreads scores over many +# coarse bins and leaves the threshold bin far below the tie-staging capacity. +# These distributions force the threshold bin to overflow the staging buffer. +NARROW_DISTRIBUTIONS = [ + "narrow", + "narrow_bin", + "tiny", + "two_values", + "signed_zero", + "all_equal", +] + + +def _single_bin_scores(kind, batch, width, device): + """Scores whose coarse bin carries no ordering information. + + The bin comes from the top bits of the fp16 cast, so a band narrower than + one fp16 ulp -- or a magnitude that underflows fp16 -- gives one bin. + """ + if kind == "narrow": + # Narrower than one fp32 ulp at 1.0 (1.19e-7): pins the tie count. + u = torch.rand(batch, width, dtype=torch.float32, device=device) * 2.0 - 1.0 + return 1.0 + u * 1e-6 + if kind == "narrow_bin": + # One coarse bin, ~1.7e5 distinct fp32 values: pins the ordering too. + return 1.0 + torch.rand(batch, width, dtype=torch.float32, device=device) * 0.02 + if kind == "tiny": + # Below the fp16 subnormal floor: every score casts to zero. + return ( + 0.5 + torch.rand(batch, width, dtype=torch.float32, device=device) + ) * 1e-30 + if kind == "two_values": + # Two adjacent fp32 values, ~half the row each: the refinement has to + # consume all 32 bits before the survivors are bit-identical, and more + # than kMaxNumTie of them still remain at that point. + hi = torch.tensor(1.0, dtype=torch.float32, device=device).nextafter( + torch.tensor(2.0, dtype=torch.float32, device=device) + ) + pick = torch.rand(batch, width, device=device) < 0.5 + return torch.where(pick, hi, torch.ones_like(hi)) + if kind == "signed_zero": + # The threshold bin ends at +0.0. Its upper boundary must move to the + # smallest positive subnormal so -0.0 stays in the tie set instead of + # racing true positive values for output slots. + out = torch.full((batch, width), -1.0, dtype=torch.float32, device=device) + out[:, : width // 2] = -0.0 + out[:, -1024:] = 1e-6 + return out + # Control: the bin overflows too, but the candidates are bit-identical, so + # any subset is correct and this must keep passing. + return torch.full((batch, width), 0.5, dtype=torch.float32, device=device) + + +def _narrow_seed(dist, *rest): + """Reproducible across processes -- ``hash(str)`` is salted per interpreter.""" + seed = NARROW_DISTRIBUTIONS.index(dist) + for v in rest: + seed = seed * 100003 + int(v) + return seed + + +# Longest row that never reaches the CUDA cluster path, which still truncates an +# overflowing threshold bin (the smallest get_static_cluster_floor in topk_v2.cuh). +CUDA_MAX_NON_CLUSTER_SEQ = 24576 + + +@pytest.mark.parametrize("dist", NARROW_DISTRIBUTIONS) +@pytest.mark.parametrize("batch,seq", [(6, 8192), (4, 20480), (4, 32768), (4, 65536)]) +@torch.inference_mode() +def test_topk_v2_single_coarse_bin_is_exact(batch: int, seq: int, dist: str) -> None: + """No tolerance here: the threshold bin holds the whole row at k=2048, so a + dropped candidate is a wrong answer rather than a tie swap. Covers + register-path and streaming-path shapes, and a ROCm split-path shape (above + split_floor); tie staging is per-template. + """ + if not is_hip() and seq > CUDA_MAX_NON_CLUSTER_SEQ: + pytest.skip("CUDA may route this shape to the unrefined cluster path") + torch.manual_seed(_narrow_seed(dist, batch, seq)) + device = "cuda" + k = 2048 + width = (seq + 3) & ~3 + scores = _single_bin_scores(dist, batch, width, device)[:, :seq] + seq_lens = torch.full((batch,), seq, dtype=torch.int32, device=device) + + our_raw = _run_raw(scores, seq_lens, k) + ref_raw = _reference(scores, seq_lens, k) + _assert_topk_close( + scores.cpu(), ref_raw, our_raw, batch, seq_lens.cpu(), k, max_permit_error=0 + ) + + # --- ragged entry point ------------------------------------------------------ # Rows select inside `[row_start, row_start + seq_len)` of their score row and # emit `position + offset`. The window start is an arbitrary token offset, so @@ -616,12 +714,12 @@ def test_topk_v2_split_duplicate_scores(batch: int, seq: int, nvals: int) -> Non """Duplicate-heavy rows: the cross-block tie merge must stay exact. The split path resolves the threshold bin in whichever block arrives last, - out of a tie list that every block appended to, so it is exact only while - that bin holds no more candidates than the list. Rows of ``nvals`` distinct - scores put ``seq / nvals`` exact duplicates at the cut, which is what fills - it; the shapes are long and narrow enough to reach the split gate. CUDA - runs its cluster path here, whose coarser histogram makes the same bin four - times as wide, so this is the ROCm path's test. + out of a tie list that every block appended to; a bin that overflows the + list is rescanned instead (test_topk_v2_single_coarse_bin_is_exact). Rows + of ``nvals`` distinct scores put ``seq / nvals`` exact duplicates at the + cut, which is what fills it; the shapes are long and narrow enough to reach + the split gate. CUDA runs its cluster path here, whose coarser histogram + makes the same bin four times as wide, so this is the ROCm path's test. """ k = 2048 g = torch.Generator(device="cuda").manual_seed(nvals * 7 + seq)