[Fix] dsa: Make the fused top-k exact on an overflowing threshold bin (#37941)
Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: DarkSharpness <2040703891@qq.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Cursor
DarkSharpness
Claude Opus 5.5
parent
9ea5472833
commit
86f4ede4fb
@@ -49,7 +49,25 @@ constexpr uint32_t kMaxTopK = impl::TopKConfig::kMaxTopK;
|
||||
constexpr uint32_t kReg2MaxSeqLen = Register2::kMaxSeqLen; // 8192
|
||||
constexpr uint32_t kReg4MaxSeqLen = Register4::kMaxSeqLen; // 16384
|
||||
|
||||
#ifdef USE_ROCM
|
||||
// HIP reads the second __launch_bounds__ argument as waves per SIMD, CUDA as
|
||||
// blocks per SM, so kOccupancy asks ROCm for 2 waves/SIMD -- below the 4 a
|
||||
// 1024-thread block already forces. Do NOT translate kOccupancy into this:
|
||||
// "kOccupancy blocks per CU" is 8 waves/SIMD, which caps the allocator at
|
||||
// 512/8 = 64 VGPRs and spills, and a second resident block is unreachable with
|
||||
// a grid of batch_size. wave64 is a literal because __AMDGCN_WAVEFRONT_SIZE__
|
||||
// is gone in ROCm 7 and warpSize is not constexpr.
|
||||
inline constexpr uint32_t kSimdsPerCu = 4;
|
||||
inline constexpr uint32_t kWavefrontSize = 64; // CDNA
|
||||
inline constexpr uint32_t kWavesPerBlock = kBlockSize / kWavefrontSize;
|
||||
inline constexpr uint32_t kMinWavesPerSimd = kWavesPerBlock / kSimdsPerCu;
|
||||
static_assert(kMinWavesPerSimd > 0, "kBlockSize must cover at least one wave per SIMD");
|
||||
#define TOPK_KERNEL __global__ __launch_bounds__(kBlockSize, kMinWavesPerSimd)
|
||||
#else
|
||||
// topk_small_batch_cluster_kernel shadows kOccupancy with its own template
|
||||
// parameter and CLUSTER_TOPK_KERNEL relies on that, so leave the CUDA spelling.
|
||||
#define TOPK_KERNEL __global__ __launch_bounds__(kBlockSize, kOccupancy)
|
||||
#endif
|
||||
|
||||
/// Metadata tensor rows (each 8 B / 2 int32). Row 0 is the global plan result;
|
||||
/// rows 1..N are the (batch_id, seq_len) of items routed to the cluster pool.
|
||||
@@ -644,8 +662,9 @@ struct SplitWorkspace {
|
||||
uint32_t floor; ///< same value the host dispatched on
|
||||
};
|
||||
|
||||
/// 12 histogram bits, the width TopKStreaming uses: a 10-bit threshold bin holds
|
||||
/// more unequal scores than kMaxNumTie can stage, and drops the rest in silence.
|
||||
/// 12 histogram bits, the width TopKStreaming uses: a 10-bit threshold bin overflows
|
||||
/// kMaxNumTie far more often, and each overflowing row costs the epilogue a
|
||||
/// refine_ties rescan of the whole row in one block.
|
||||
struct TopKSplit : impl::TopKRadixBase<12> {
|
||||
using Base = impl::TopKRadixBase<12>;
|
||||
static_assert(kHistSize % kBlockSize == 0, "the histogram is transferred kHistItems bins per thread");
|
||||
@@ -723,8 +742,9 @@ struct TopKSplit : impl::TopKRadixBase<12> {
|
||||
// The full row's histogram and the full row's seq_len, so every rank picks
|
||||
// the same bin and the appends below agree on what "above" means.
|
||||
find_threshold(problem.topk, problem.seq_len, smem, [&](uint32_t threshold_bin) {
|
||||
smem->v_hi = impl::coarse_bin_lower_bound<kHistBits>(threshold_bin + 1);
|
||||
smem->v_lo = impl::coarse_bin_lower_bound<kHistBits>(threshold_bin + 0);
|
||||
const auto [v_lo, v_hi] = impl::coarse_bin_bounds<kHistBits>(threshold_bin);
|
||||
smem->v_hi = v_hi;
|
||||
smem->v_lo = v_lo;
|
||||
});
|
||||
|
||||
const auto topk = problem.topk;
|
||||
@@ -751,7 +771,8 @@ struct TopKSplit : impl::TopKRadixBase<12> {
|
||||
const auto n_eq = min(smem->count_eq, kMaxNumTie);
|
||||
if (tx == 0) {
|
||||
smem->base_gt = atomicAdd(&ctr->count_gt, n_gt);
|
||||
smem->base_eq = atomicAdd(&ctr->count_eq, n_eq);
|
||||
// Uncapped: finish_ties needs the row's exact bin size to detect overflow and refine.
|
||||
smem->base_eq = atomicAdd(&ctr->count_eq, smem->count_eq);
|
||||
}
|
||||
__syncthreads();
|
||||
const auto base_gt = smem->base_gt;
|
||||
@@ -788,15 +809,27 @@ struct TopKSplit : impl::TopKRadixBase<12> {
|
||||
}
|
||||
|
||||
/// Fill the slots the threshold bin has to break ties for, staging the ties
|
||||
/// into LDS first since handle_tie's ranking pass is all-to-all.
|
||||
/// into LDS first since handle_tie's ranking pass is all-to-all. `problem`
|
||||
/// must be the whole row: an overflowing bin is re-derived by rescanning it.
|
||||
SGL_DEVICE static void finish_ties(const TopKProblem& problem, const impl::TieValue* ties, Smem* smem) {
|
||||
const auto tx = threadIdx.x;
|
||||
const auto above_count = smem->total_gt;
|
||||
const auto tie_count = min(smem->total_eq, kMaxNumTie);
|
||||
uint32_t above_count, tie_count;
|
||||
if (smem->total_eq > kMaxNumTie) [[unlikely]] {
|
||||
// The global list holds an arrival-order subset. refine_ties emits after
|
||||
// the slots every rank already took, so seed its counter with their total.
|
||||
if (tx == 0) smem->count_gt = smem->total_gt;
|
||||
__syncthreads();
|
||||
refine_ties(problem, smem, smem->v_lo, smem->v_hi, smem->total_eq);
|
||||
above_count = smem->count_gt;
|
||||
tie_count = min(smem->count_eq, kMaxNumTie);
|
||||
} else {
|
||||
above_count = smem->total_gt;
|
||||
tie_count = smem->total_eq;
|
||||
for (uint32_t t = tx; t < tie_count; t += kBlockSize)
|
||||
smem->tie_values[t] = ties[t];
|
||||
__syncthreads();
|
||||
}
|
||||
const auto remain_topk = above_count < problem.topk ? problem.topk - above_count : 0;
|
||||
for (uint32_t t = tx; t < tie_count; t += kBlockSize)
|
||||
smem->tie_values[t] = ties[t];
|
||||
__syncthreads();
|
||||
handle_tie(smem->tie_values, problem, above_count, tie_count, remain_topk, &smem->tie_handle);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -14,7 +14,8 @@
|
||||
/// - the dispatcher selects cluster size 8 or 16 from the probed occupancy.
|
||||
///
|
||||
/// Algorithm: fp16 coarse histogram -> threshold bin -> fp32-boundary collect ->
|
||||
/// exact radix tie-break.
|
||||
/// exact radix tie-break, plus an exact-key refinement when the threshold bin
|
||||
/// overflows the staging buffer (refine_ties).
|
||||
|
||||
#pragma once
|
||||
|
||||
@@ -90,59 +91,35 @@ SGL_DEVICE uint32_t extract_coarse_bin(float x) {
|
||||
return (b ^ (s | 0x80000000u)) >> (32 - kBits);
|
||||
}
|
||||
|
||||
SGL_DEVICE uint16_t coarse_bin_to_bits_finite(uint32_t bin) {
|
||||
const uint16_t ob = static_cast<uint16_t>(bin);
|
||||
return (ob & 0x8000) ? static_cast<uint16_t>(ob ^ 0x8000) : static_cast<uint16_t>(~ob);
|
||||
}
|
||||
|
||||
// Smallest fp32 `v` for which `extract_coarse_bin<kBits>(v) >= bin`, i.e. the
|
||||
// lower fp32 boundary of coarse bin `bin`. The collect pass classifies with two
|
||||
// comparisons against these instead of recomputing the fp16 bin per element, so
|
||||
// this must agree with `extract_coarse_bin` on every value -- a score sitting
|
||||
// exactly on a boundary included. Two pairs no fp32 threshold can separate are
|
||||
// left: -0.0 at the zero bin, and +inf at a NaN-key bin.
|
||||
/// fp32 bounds {lo, hi} of coarse bin `bin`: "above" iff v >= hi, "tie" iff lo <= v < hi,
|
||||
/// agreeing with extract_coarse_bin<kBits> on every non-NaN score. A zero hi moves to
|
||||
/// the smallest subnormal, since fp32 compares cannot put -0.0 below +0.0.
|
||||
template <uint32_t kBits>
|
||||
SGL_DEVICE float coarse_bin_lower_bound(uint32_t bin) {
|
||||
SGL_DEVICE float2 coarse_bin_bounds(uint32_t bin) {
|
||||
constexpr uint32_t kShift = 16 - kBits;
|
||||
constexpr uint32_t kInfBin = 0xFC00u >> kShift; // bin holding the +inf key
|
||||
const uint32_t key = bin << kShift; // ordered16 key at the low edge
|
||||
// ordered16 -> fp16 value (inverse of the transform in extract_coarse_bin);
|
||||
constexpr auto to_finite_val = [](uint32_t okey) -> float {
|
||||
const uint16_t hb = coarse_bin_to_bits_finite(okey);
|
||||
return cast<float>(*reinterpret_cast<const fp16_t*>(&hb));
|
||||
static_assert(kShift >= 1, "edge keys must be even for the rounding rule below");
|
||||
constexpr uint32_t kNegInfBin = 0x03FFu >> kShift; // bin holding the -inf key
|
||||
constexpr uint32_t kInfBin = 0xFC00u >> kShift; // bin holding the +inf key
|
||||
// ordered16 edge keys, {lo, hi} packed in the {low, high} halves.
|
||||
const uint32_t keys = (bin << kShift) | ((bin + 1) << (kShift + 16));
|
||||
// ordered16 -> fp16 bits per half: ~key below 0x8000 (negative), key ^ 0x8000 above.
|
||||
const auto to_fp32 = [](uint32_t k) {
|
||||
const uint32_t bits = k ^ 0x80008000u ^ ((~k >> 15) & 0x00010001u) * 0x7FFFu;
|
||||
return cast<fp32x2_t>(*reinterpret_cast<const fp16x2_t*>(&bits));
|
||||
};
|
||||
constexpr auto step_up = [](float v) -> float {
|
||||
const int32_t b = __float_as_int(v);
|
||||
return __int_as_float(b >= 0 ? b + 1 : b - 1);
|
||||
};
|
||||
// Fast path, hoisted above the per-key special cases so both keys are
|
||||
// range-checked at once: `key` and `key - 1` both land in the finite band
|
||||
// [0x0401, 0xFBFF] -- every boundary a finite-score threshold produces. fp16
|
||||
// rounds to nearest, so the boundary is the midpoint between the fp16 values
|
||||
// at `key` and `key - 1`.
|
||||
if (key - 0x0401u <= 0xFBFFu - 0x0401u) {
|
||||
const float mid = 0.5f * (to_finite_val(key) + to_finite_val(key - 1));
|
||||
// fp32 -> fp16 rounds to nearest EVEN, so on the ~half of bins whose fp16
|
||||
// value has an odd significand the midpoint still bins as `bin - 1`.
|
||||
return (coarse_bin_to_bits_finite(key) & 1u) ? step_up(mid) : mid;
|
||||
}
|
||||
// Slow path: an edge of `bin` touches the +/-inf keys or NaN key space. The
|
||||
// ordered-key line is: [0, 0x03FF) negative-NaN space, 0x03FF = -inf,
|
||||
// [0x0400, 0xFC00) finite, 0xFC00 = +inf, (0xFC00, 0xFFFF] positive-NaN
|
||||
// space. The +/-inf keys stand in as +/-65536, one ideal step past fp16 max,
|
||||
// so the midpoint lands on the +/-65520 fp32 -> fp16 overflow threshold.
|
||||
if (bin == 0) return -infinity_value(); // every value bins at >= 0
|
||||
if (bin > kInfBin) return infinity_value(); // NaN key space: nothing bins that high
|
||||
const auto to_val = [&](uint32_t okey) -> float {
|
||||
if (okey < 0x03FFu) return -infinity_value();
|
||||
if (okey == 0x03FFu) return -65536.0f;
|
||||
if (okey == 0xFC00u) return 65536.0f;
|
||||
return to_finite_val(okey);
|
||||
};
|
||||
// The +/-65536 stand-ins are not real fp16 neighbours, so the parity rule
|
||||
// does not apply here; test the property directly instead.
|
||||
const float mid = 0.5f * (to_val(key) + to_val(key - 1));
|
||||
return extract_coarse_bin<kBits>(mid) < bin ? step_up(mid) : mid;
|
||||
const auto [v_lo, v_hi] = to_fp32(keys);
|
||||
const auto [u_lo, u_hi] = to_fp32(keys - 0x00010001u);
|
||||
// Exact midpoints of adjacent fp16 values; the clamp puts one against inf on the
|
||||
// +/-65520 overflow point.
|
||||
const auto midpoint = [](float a, float b) { return fminf(fmaxf(0.5f * (a + b), -65520.0f), 65520.0f); };
|
||||
const int32_t lo = __float_as_int(midpoint(v_lo, u_lo));
|
||||
const int32_t hi = __float_as_int(midpoint(v_hi, u_hi));
|
||||
// Ties round to even and edge keys are even, so negative edges sit one ulp above
|
||||
// the midpoint. A zero hi steps past +0.
|
||||
const float lo_edge = __int_as_float(lo + (lo >> 31));
|
||||
const float hi_edge = __int_as_float(hi + (hi >> 31) + (hi == 0));
|
||||
// Edges inside the NaN key space: every score is >= lo, none is >= hi.
|
||||
return {bin <= kNegInfBin ? -infinity_value() : lo_edge, bin >= kInfBin ? infinity_value() : hi_edge};
|
||||
}
|
||||
|
||||
SGL_DEVICE uint32_t warp_sum_bool(bool pred, uint32_t mask = 0xFFFFFFFF) {
|
||||
@@ -200,6 +177,17 @@ struct TopKConfig {
|
||||
// by downstream sparse attention.
|
||||
static constexpr uint32_t kMaxNumTie = 2048;
|
||||
static constexpr uint32_t kRadixSize = 1 << 8;
|
||||
// Radix width of the threshold-bin refinement (refine_ties). Wider
|
||||
// than kRadixSize so that ceil(32 / kRefineBits) == 3 rounds cover the whole
|
||||
// key: the histogram overlays the tie staging buffer, which that path
|
||||
// re-derives anyway, so the extra bins are free. 12 is the widest that fits.
|
||||
static constexpr uint32_t kRefineBits = 12;
|
||||
static constexpr uint32_t kRefineSize = 1 << kRefineBits;
|
||||
static constexpr uint32_t kRefineItems = kRefineSize / kBlockSize;
|
||||
static_assert(
|
||||
kRefineSize * sizeof(uint32_t) <= kMaxNumTie * sizeof(TieValue),
|
||||
"the refinement histogram must fit the tie staging buffer it overlays");
|
||||
static_assert(kRefineSize % kBlockSize == 0);
|
||||
static constexpr uint32_t kTopKItems = (kMaxTopK + kBlockSize - 1) / kBlockSize;
|
||||
// tie candidates owned per thread in the strided handle_tie loops
|
||||
static constexpr uint32_t kTieItems = kMaxNumTie / kBlockSize;
|
||||
@@ -308,6 +296,44 @@ struct TopKConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// Threshold-digit search over the kRefineSize-bin refinement histogram in
|
||||
/// `hist`, by the same rule radix_tie_select uses. One thread owns kRefineItems
|
||||
/// contiguous bins. Publishes the result in `smem->match`. Block-wide; ends
|
||||
/// with a barrier.
|
||||
SGL_DEVICE static void refine_find_threshold( //
|
||||
const uint32_t* hist,
|
||||
const uint32_t total_active,
|
||||
const uint32_t topk_remain,
|
||||
TieHandleSmem* smem) {
|
||||
const auto tx = threadIdx.x;
|
||||
const auto lane_id = tx % kWarpSize;
|
||||
const auto warp_id = broadcast(tx / kWarpSize);
|
||||
|
||||
uint32_t counts[kRefineItems];
|
||||
uint32_t local_sum = 0;
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < kRefineItems; ++i) {
|
||||
counts[i] = hist[tx * kRefineItems + i];
|
||||
local_sum += counts[i];
|
||||
}
|
||||
const auto warp_inc = warp::inclusive_sum(local_sum, lane_id);
|
||||
if (lane_id == kWarpSize - 1) smem->warp_sum[warp_id] = warp_inc;
|
||||
__syncthreads();
|
||||
|
||||
// Inclusive prefix over all bins below this thread's first one.
|
||||
uint32_t prefix = warp::reduce_sum(lane_id < warp_id ? smem->warp_sum[lane_id] : 0);
|
||||
prefix += warp_inc - local_sum;
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < kRefineItems; ++i) {
|
||||
prefix += counts[i];
|
||||
const auto above = total_active - prefix; // active elements in bins ABOVE this one
|
||||
if (above < topk_remain && above + counts[i] >= topk_remain) {
|
||||
smem->match = {tx * kRefineItems + i, above, counts[i]};
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
/// Exact radix select over the tie candidates: each thread owns kItems
|
||||
/// strided elements (inactive beyond num_ties). Requires
|
||||
/// num_ties <= kItems * kBlockSize.
|
||||
@@ -431,7 +457,13 @@ struct TopKRadixBase : TopKConfig {
|
||||
union {
|
||||
alignas(16) uint32_t histogram[kHistSize];
|
||||
struct {
|
||||
TieValue tie_values[kMaxNumTie];
|
||||
// refine_ties re-derives the candidates instead of trusting
|
||||
// what the collect pass staged, so the buffer is free to carry that
|
||||
// path's radix histogram until its staging pass writes values back.
|
||||
union {
|
||||
TieValue tie_values[kMaxNumTie];
|
||||
uint32_t refine_hist[kRefineSize];
|
||||
};
|
||||
TieHandleSmem tie_handle;
|
||||
};
|
||||
};
|
||||
@@ -524,6 +556,96 @@ struct TopKRadixBase : TopKConfig {
|
||||
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
/// Re-stages the tie buffer exactly when the threshold bin overflows it: radix
|
||||
/// rounds narrow the bin to one bucket, then a final scan emits what lies above
|
||||
/// it and stages the bucket for the caller's handle_tie.
|
||||
SGL_DEVICE static void refine_ties( //
|
||||
const TopKProblem& problem,
|
||||
Smem* smem,
|
||||
const float v_lo,
|
||||
const float v_hi,
|
||||
const uint32_t count_eq) {
|
||||
constexpr uint32_t kRefineRounds = (32 + kRefineBits - 1) / kRefineBits;
|
||||
const auto topk = problem.topk;
|
||||
if (smem->count_gt >= topk) [[unlikely]]
|
||||
return;
|
||||
const auto tx = threadIdx.x;
|
||||
const auto handle = &smem->tie_handle;
|
||||
const auto hist = smem->refine_hist;
|
||||
// for_each_input's elements with one copy of `fn` and reversed lanes, so this
|
||||
// cold scan shares no load address (hence no registers) with the fast path.
|
||||
const auto scan = [&](auto&& fn) {
|
||||
const auto seq_len = problem.seq_len;
|
||||
const auto num_vec = (seq_len + kVecSize - 1) / kVecSize;
|
||||
uint32_t vi = kBlockSize - 1 - tx;
|
||||
vec_t cur;
|
||||
if (vi < num_vec) cur.load(problem.in, vi);
|
||||
#pragma unroll 1
|
||||
while (vi < num_vec) {
|
||||
const auto next_vi = vi + kBlockSize;
|
||||
vec_t next;
|
||||
if (next_vi < num_vec) next.load(problem.in, next_vi);
|
||||
#pragma unroll
|
||||
for (uint32_t j = 0; j < kVecSize; ++j) {
|
||||
if (vi * kVecSize + j < seq_len) fn(cur[j], vi * kVecSize + j);
|
||||
}
|
||||
cur = next;
|
||||
vi = next_vi;
|
||||
}
|
||||
};
|
||||
|
||||
// refine_find_threshold needs 0 < remain <= active; its match rule keeps it.
|
||||
uint32_t remain = topk - smem->count_gt;
|
||||
uint32_t active = count_eq;
|
||||
// Candidates are [bucket_lo, bucket_lo + bucket_mask]; round 0 takes the whole bin.
|
||||
uint32_t bucket_lo = 0;
|
||||
uint32_t bucket_mask = ~0u;
|
||||
|
||||
#pragma unroll
|
||||
for (uint32_t round = 0; round < kRefineRounds; ++round) {
|
||||
// round 0: bits [20, 32), round 1: bits [8, 20), round 2: bits [0, 8)
|
||||
const uint32_t lo = (round + 1) * kRefineBits < 32 ? 32 - (round + 1) * kRefineBits : 0;
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < kRefineItems; ++i) {
|
||||
hist[tx * kRefineItems + i] = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
// bucket_lo is aligned to the previous digit, so (key - bucket_lo) >> lo is this digit.
|
||||
scan([&](float val, uint32_t) {
|
||||
if (!(val >= v_lo && val < v_hi)) return;
|
||||
const auto d = extract_exact_bin(val) - bucket_lo;
|
||||
if (d <= bucket_mask) atomicAdd(&hist[d >> lo], 1);
|
||||
});
|
||||
__syncthreads();
|
||||
refine_find_threshold(hist, active, remain, handle);
|
||||
const auto match = handle->match;
|
||||
bucket_lo |= match.bin << lo;
|
||||
bucket_mask = (1u << lo) - 1u;
|
||||
remain -= match.above_count;
|
||||
active = match.equal_count;
|
||||
if (lo == 0 || active <= kMaxNumTie) break;
|
||||
}
|
||||
|
||||
// Candidates are never NaN, so keys stay <= key(+inf) and bucket_hi cannot wrap.
|
||||
// Staging overwrites the histogram, which refine_find_threshold has consumed.
|
||||
const auto bucket_hi = bucket_lo + bucket_mask + 1u;
|
||||
if (tx == 0) smem->count_eq = 0;
|
||||
__syncthreads();
|
||||
scan([&](float val, uint32_t idx) {
|
||||
if (!(val >= v_lo && val < v_hi)) return;
|
||||
const auto key = extract_exact_bin(val);
|
||||
if (key >= bucket_hi) {
|
||||
const auto pos = atomicAdd(&smem->count_gt, 1);
|
||||
if (pos < topk) [[likely]]
|
||||
problem.emit(pos, idx);
|
||||
} else if (key >= bucket_lo) {
|
||||
const auto slot = atomicAdd(&smem->count_eq, 1);
|
||||
if (slot < kMaxNumTie) smem->tie_values[slot] = {val, idx};
|
||||
}
|
||||
});
|
||||
__syncthreads();
|
||||
}
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -588,8 +710,7 @@ struct TopKRegister : TopKRadixBase<12> {
|
||||
|
||||
// Phase 2: Find the threshold bin
|
||||
find_threshold(problem.topk, num_full * kVecSize, smem, [&](uint32_t threshold_bin) {
|
||||
const auto v_hi = coarse_bin_lower_bound<kHistBits>(threshold_bin + 1);
|
||||
const auto v_lo = coarse_bin_lower_bound<kHistBits>(threshold_bin + 0);
|
||||
const auto [v_lo, v_hi] = coarse_bin_bounds<kHistBits>(threshold_bin);
|
||||
smem->v_hi = v_hi;
|
||||
smem->v_lo = v_lo;
|
||||
});
|
||||
@@ -624,6 +745,10 @@ struct TopKRegister : TopKRadixBase<12> {
|
||||
|
||||
// Phase 4: Handle ties.
|
||||
__syncthreads();
|
||||
if (smem->count_eq > kMaxNumTie) [[unlikely]] {
|
||||
// Buffer holds an arrival-order subset; re-stage it on the exact key.
|
||||
refine_ties(problem, smem, v_lo, v_hi, smem->count_eq);
|
||||
}
|
||||
const auto count_gt = smem->count_gt;
|
||||
const auto count_eq = smem->count_eq;
|
||||
const auto remain_topk = count_gt < topk ? topk - count_gt : 0;
|
||||
@@ -667,8 +792,7 @@ struct TopKStreaming : TopKRadixBase<12> {
|
||||
|
||||
// Phase 2: Find the threshold bin
|
||||
find_threshold(problem.topk, problem.seq_len, smem, [&](uint32_t threshold_bin) {
|
||||
const auto v_hi = coarse_bin_lower_bound<kHistBits>(threshold_bin + 1);
|
||||
const auto v_lo = coarse_bin_lower_bound<kHistBits>(threshold_bin + 0);
|
||||
const auto [v_lo, v_hi] = coarse_bin_bounds<kHistBits>(threshold_bin);
|
||||
smem->v_hi = v_hi;
|
||||
smem->v_lo = v_lo;
|
||||
});
|
||||
@@ -701,6 +825,10 @@ struct TopKStreaming : TopKRadixBase<12> {
|
||||
// "above" and "tie" sets. above_count is < topk by the threshold-bin invariant,
|
||||
// so the count_gt guard above effectively never triggers.
|
||||
__syncthreads();
|
||||
if (smem->count_eq > kMaxNumTie) [[unlikely]] {
|
||||
// See the register path.
|
||||
refine_ties(problem, smem, v_lo, v_hi, smem->count_eq);
|
||||
}
|
||||
const auto count_gt = smem->count_gt;
|
||||
const auto count_eq = smem->count_eq;
|
||||
const auto remain_topk = count_gt < topk ? topk - count_gt : 0;
|
||||
@@ -715,6 +843,11 @@ struct TopKStreaming : TopKRadixBase<12> {
|
||||
//
|
||||
// CUDA only: thread-block clusters and distributed shared memory have no CDNA
|
||||
// equivalent.
|
||||
//
|
||||
// Still truncates an overflowing threshold bin, unlike the register, streaming
|
||||
// and ROCm split paths: the candidate set is split across kClusterSize ranks
|
||||
// and no single block resolves the row, so refining needs cluster-wide
|
||||
// histogram and emit counters rather than the block-local ones refine_ties uses.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#if SUPPORT_CLUSTER
|
||||
@@ -853,8 +986,9 @@ struct TopKCluster : TopKRadixBase<10> {
|
||||
|
||||
// Phase 3. rank-0 find threshold and write to local smem for other ranks to read
|
||||
find_threshold(problem.topk, problem.seq_len, smem, [&](uint32_t threshold_bin) {
|
||||
smem->v_hi = coarse_bin_lower_bound<kHistBits>(threshold_bin + 1);
|
||||
smem->v_lo = coarse_bin_lower_bound<kHistBits>(threshold_bin + 0);
|
||||
const auto [v_lo, v_hi] = coarse_bin_bounds<kHistBits>(threshold_bin);
|
||||
smem->v_hi = v_hi;
|
||||
smem->v_lo = v_lo;
|
||||
});
|
||||
|
||||
barrier_cluster_arrive_release(); // bar-2 arrive: publishes v_hi / v_lo
|
||||
|
||||
Reference in New Issue
Block a user