[Perf] [AMD] deepseek-v4: Skip top-k refinement for proven positive-zero ties (#44927)
This commit is contained in:
@@ -772,7 +772,7 @@ struct alignas(128) SplitCounters {
|
||||
uint32_t count_gt;
|
||||
uint32_t count_eq;
|
||||
uint32_t arrive;
|
||||
uint32_t _pad;
|
||||
uint32_t tie_nonzero;
|
||||
};
|
||||
|
||||
struct SplitWorkspace {
|
||||
@@ -871,6 +871,8 @@ struct TopKSplit : impl::TopKRadixBase<12> {
|
||||
const auto topk = problem.topk;
|
||||
const auto v_hi = smem->v_hi;
|
||||
const auto v_lo = smem->v_lo;
|
||||
const bool zero_bin = is_zero_bin(v_lo);
|
||||
if (zero_bin && tx == 0) smem->tie_nonzero = 0;
|
||||
__syncthreads();
|
||||
|
||||
for_each_input(problem.in + chunk.start, chunk.len, [&](float val, uint32_t local) {
|
||||
@@ -882,6 +884,7 @@ struct TopKSplit : impl::TopKRadixBase<12> {
|
||||
smem->staged[pos] = static_cast<int32_t>(idx);
|
||||
} else if (val >= v_lo) {
|
||||
const auto slot = atomicAdd(&smem->count_eq, 1u);
|
||||
if (zero_bin && __float_as_uint(val) != 0u) atomicOr(&smem->tie_nonzero, 1u);
|
||||
if (slot < kMaxNumTie) [[likely]]
|
||||
smem->tie_values[slot] = {val, idx};
|
||||
}
|
||||
@@ -894,6 +897,7 @@ struct TopKSplit : impl::TopKRadixBase<12> {
|
||||
smem->base_gt = atomicAdd(&ctr->count_gt, n_gt);
|
||||
// Uncapped: finish_ties needs the row's exact bin size to detect overflow and refine.
|
||||
smem->base_eq = atomicAdd(&ctr->count_eq, smem->count_eq);
|
||||
if (zero_bin && smem->tie_nonzero != 0) atomicOr(&ctr->tie_nonzero, 1u);
|
||||
}
|
||||
__syncthreads();
|
||||
const auto base_gt = smem->base_gt;
|
||||
@@ -923,6 +927,7 @@ struct TopKSplit : impl::TopKRadixBase<12> {
|
||||
// served out of this CU's own stale cache.
|
||||
smem->total_gt = atomicAdd(&ctr->count_gt, 0u);
|
||||
smem->total_eq = atomicAdd(&ctr->count_eq, 0u);
|
||||
if (is_zero_bin(smem->v_lo)) smem->tie_nonzero = atomicAdd(&ctr->tie_nonzero, 0u);
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
@@ -935,7 +940,8 @@ struct TopKSplit : impl::TopKRadixBase<12> {
|
||||
SGL_DEVICE static void finish_ties(const TopKProblem& problem, const impl::TieValue* ties, Smem* smem) {
|
||||
const auto tx = threadIdx.x;
|
||||
uint32_t above_count, tie_count;
|
||||
if (smem->total_eq > kMaxNumTie) [[unlikely]] {
|
||||
const bool zero_ties = is_zero_bin(smem->v_lo) && smem->tie_nonzero == 0;
|
||||
if (smem->total_eq > kMaxNumTie && !zero_ties) [[unlikely]] {
|
||||
// The global list holds an arrival-order subset. refine_ties emits after
|
||||
// the slots every rank already took, so seed its counter with their total.
|
||||
if (tx == 0) smem->count_gt = smem->total_gt;
|
||||
@@ -945,7 +951,7 @@ struct TopKSplit : impl::TopKRadixBase<12> {
|
||||
tie_count = min(smem->count_eq, kMaxNumTie);
|
||||
} else {
|
||||
above_count = smem->total_gt;
|
||||
tie_count = smem->total_eq;
|
||||
tie_count = zero_ties ? min(smem->total_eq, kMaxNumTie) : smem->total_eq;
|
||||
for (uint32_t t = tx; t < tie_count; t += kBlockSize)
|
||||
smem->tie_values[t] = ties[t];
|
||||
__syncthreads();
|
||||
|
||||
@@ -449,7 +449,10 @@ struct TopKRadixBase : TopKConfig {
|
||||
uint32_t count_gt;
|
||||
float v_hi;
|
||||
float v_lo;
|
||||
uint32_t warp_sum[kNumWarps];
|
||||
union {
|
||||
uint32_t warp_sum[kNumWarps];
|
||||
uint32_t tie_nonzero;
|
||||
};
|
||||
// The coarse histogram is dead once find_threshold() has published its
|
||||
// boundaries, and the tie machinery only comes alive after that, so the two
|
||||
// phases overlay. tie_handle and tie_values are live TOGETHER, so they sit
|
||||
@@ -470,6 +473,14 @@ struct TopKRadixBase : TopKConfig {
|
||||
};
|
||||
|
||||
protected:
|
||||
SGL_DEVICE static bool is_zero_bin(float lower) {
|
||||
#ifdef USE_ROCM
|
||||
return __float_as_uint(lower) == 0u;
|
||||
#else
|
||||
return false;
|
||||
#endif
|
||||
}
|
||||
|
||||
template <uint32_t N = 1, typename F>
|
||||
SGL_DEVICE static void for_each_input(const float* __restrict__ in, uint32_t seq_len, F&& fn) {
|
||||
constexpr auto kStride = N * kBlockSize;
|
||||
@@ -719,6 +730,12 @@ struct TopKRegister : TopKRadixBase<12> {
|
||||
const auto topk = problem.topk;
|
||||
const auto v_hi = smem->v_hi;
|
||||
const auto v_lo = smem->v_lo;
|
||||
const bool zero_bin = is_zero_bin(v_lo);
|
||||
if (zero_bin) {
|
||||
// find_threshold has finished using the overlaid warp sums.
|
||||
if (tx == 0) smem->tie_nonzero = 0;
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < kLocalVecs; ++i) {
|
||||
@@ -736,6 +753,7 @@ struct TopKRegister : TopKRadixBase<12> {
|
||||
}
|
||||
} else if (val >= v_lo) {
|
||||
const auto pos = atomicAdd(&smem->count_eq, 1);
|
||||
if (zero_bin && __float_as_uint(val) != 0u) atomicOr(&smem->tie_nonzero, 1u);
|
||||
if (pos < kMaxNumTie) [[likely]] {
|
||||
smem->tie_values[pos] = {val, idx};
|
||||
}
|
||||
@@ -745,7 +763,7 @@ struct TopKRegister : TopKRadixBase<12> {
|
||||
|
||||
// Phase 4: Handle ties.
|
||||
__syncthreads();
|
||||
if (smem->count_eq > kMaxNumTie) [[unlikely]] {
|
||||
if (smem->count_eq > kMaxNumTie && (!zero_bin || smem->tie_nonzero != 0)) [[unlikely]] {
|
||||
// Buffer holds an arrival-order subset; re-stage it on the exact key.
|
||||
refine_ties(problem, smem, v_lo, v_hi, smem->count_eq);
|
||||
}
|
||||
@@ -805,6 +823,11 @@ struct TopKStreaming : TopKRadixBase<12> {
|
||||
const auto v_hi = smem->v_hi;
|
||||
const auto v_lo = smem->v_lo;
|
||||
const auto topk = problem.topk;
|
||||
const bool zero_bin = is_zero_bin(v_lo);
|
||||
if (zero_bin) {
|
||||
if (tx == 0) smem->tie_nonzero = 0;
|
||||
__syncthreads();
|
||||
}
|
||||
for_each_input(problem.in, problem.seq_len, [&](float val, uint32_t idx) {
|
||||
if (val >= v_hi) {
|
||||
const auto pos = atomicAdd(&smem->count_gt, 1);
|
||||
@@ -813,6 +836,7 @@ struct TopKStreaming : TopKRadixBase<12> {
|
||||
}
|
||||
} else if (val >= v_lo) {
|
||||
const auto pos = atomicAdd(&smem->count_eq, 1);
|
||||
if (zero_bin && __float_as_uint(val) != 0u) atomicOr(&smem->tie_nonzero, 1u);
|
||||
if (pos < kMaxNumTie) [[likely]] {
|
||||
smem->tie_values[pos] = {val, idx};
|
||||
}
|
||||
@@ -825,7 +849,7 @@ struct TopKStreaming : TopKRadixBase<12> {
|
||||
// "above" and "tie" sets. above_count is < topk by the threshold-bin invariant,
|
||||
// so the count_gt guard above effectively never triggers.
|
||||
__syncthreads();
|
||||
if (smem->count_eq > kMaxNumTie) [[unlikely]] {
|
||||
if (smem->count_eq > kMaxNumTie && (!zero_bin || smem->tie_nonzero != 0)) [[unlikely]] {
|
||||
// See the register path.
|
||||
refine_ties(problem, smem, v_lo, v_hi, smem->count_eq);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user