From 8d3eee46a7e128a6e98acb164c926d44ed211783 Mon Sep 17 00:00:00 2001 From: Duyi-Wang Date: Mon, 19 Oct 2026 11:27:47 +0800 Subject: [PATCH] [Perf] [AMD] deepseek-v4: Skip top-k refinement for proven positive-zero ties (#44927) --- .../kernels/jit/csrc/deepseek_v4/topk_v2.cuh | 12 ++- .../sgl_kernel/deepseek_v4/topk_impl.cuh | 30 ++++++- .../kernels/ops/attention/test_topk_v2.py | 83 +++++++++++++++++++ 3 files changed, 119 insertions(+), 6 deletions(-) diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/topk_v2.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/topk_v2.cuh index 27b5f61744..73f34a0cbe 100644 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/topk_v2.cuh +++ b/python/sglang/kernels/jit/csrc/deepseek_v4/topk_v2.cuh @@ -772,7 +772,7 @@ struct alignas(128) SplitCounters { uint32_t count_gt; uint32_t count_eq; uint32_t arrive; - uint32_t _pad; + uint32_t tie_nonzero; }; struct SplitWorkspace { @@ -871,6 +871,8 @@ struct TopKSplit : impl::TopKRadixBase<12> { const auto topk = problem.topk; const auto v_hi = smem->v_hi; const auto v_lo = smem->v_lo; + const bool zero_bin = is_zero_bin(v_lo); + if (zero_bin && tx == 0) smem->tie_nonzero = 0; __syncthreads(); for_each_input(problem.in + chunk.start, chunk.len, [&](float val, uint32_t local) { @@ -882,6 +884,7 @@ struct TopKSplit : impl::TopKRadixBase<12> { smem->staged[pos] = static_cast(idx); } else if (val >= v_lo) { const auto slot = atomicAdd(&smem->count_eq, 1u); + if (zero_bin && __float_as_uint(val) != 0u) atomicOr(&smem->tie_nonzero, 1u); if (slot < kMaxNumTie) [[likely]] smem->tie_values[slot] = {val, idx}; } @@ -894,6 +897,7 @@ struct TopKSplit : impl::TopKRadixBase<12> { smem->base_gt = atomicAdd(&ctr->count_gt, n_gt); // Uncapped: finish_ties needs the row's exact bin size to detect overflow and refine. smem->base_eq = atomicAdd(&ctr->count_eq, smem->count_eq); + if (zero_bin && smem->tie_nonzero != 0) atomicOr(&ctr->tie_nonzero, 1u); } __syncthreads(); const auto base_gt = smem->base_gt; @@ -923,6 +927,7 @@ struct TopKSplit : impl::TopKRadixBase<12> { // served out of this CU's own stale cache. smem->total_gt = atomicAdd(&ctr->count_gt, 0u); smem->total_eq = atomicAdd(&ctr->count_eq, 0u); + if (is_zero_bin(smem->v_lo)) smem->tie_nonzero = atomicAdd(&ctr->tie_nonzero, 0u); } } __syncthreads(); @@ -935,7 +940,8 @@ struct TopKSplit : impl::TopKRadixBase<12> { SGL_DEVICE static void finish_ties(const TopKProblem& problem, const impl::TieValue* ties, Smem* smem) { const auto tx = threadIdx.x; uint32_t above_count, tie_count; - if (smem->total_eq > kMaxNumTie) [[unlikely]] { + const bool zero_ties = is_zero_bin(smem->v_lo) && smem->tie_nonzero == 0; + if (smem->total_eq > kMaxNumTie && !zero_ties) [[unlikely]] { // The global list holds an arrival-order subset. refine_ties emits after // the slots every rank already took, so seed its counter with their total. if (tx == 0) smem->count_gt = smem->total_gt; @@ -945,7 +951,7 @@ struct TopKSplit : impl::TopKRadixBase<12> { tie_count = min(smem->count_eq, kMaxNumTie); } else { above_count = smem->total_gt; - tie_count = smem->total_eq; + tie_count = zero_ties ? min(smem->total_eq, kMaxNumTie) : smem->total_eq; for (uint32_t t = tx; t < tie_count; t += kBlockSize) smem->tie_values[t] = ties[t]; __syncthreads(); diff --git a/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/topk_impl.cuh b/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/topk_impl.cuh index 369868a0f0..f44184bc24 100644 --- a/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/topk_impl.cuh +++ b/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/topk_impl.cuh @@ -449,7 +449,10 @@ struct TopKRadixBase : TopKConfig { uint32_t count_gt; float v_hi; float v_lo; - uint32_t warp_sum[kNumWarps]; + union { + uint32_t warp_sum[kNumWarps]; + uint32_t tie_nonzero; + }; // The coarse histogram is dead once find_threshold() has published its // boundaries, and the tie machinery only comes alive after that, so the two // phases overlay. tie_handle and tie_values are live TOGETHER, so they sit @@ -470,6 +473,14 @@ struct TopKRadixBase : TopKConfig { }; protected: + SGL_DEVICE static bool is_zero_bin(float lower) { +#ifdef USE_ROCM + return __float_as_uint(lower) == 0u; +#else + return false; +#endif + } + template SGL_DEVICE static void for_each_input(const float* __restrict__ in, uint32_t seq_len, F&& fn) { constexpr auto kStride = N * kBlockSize; @@ -719,6 +730,12 @@ struct TopKRegister : TopKRadixBase<12> { const auto topk = problem.topk; const auto v_hi = smem->v_hi; const auto v_lo = smem->v_lo; + const bool zero_bin = is_zero_bin(v_lo); + if (zero_bin) { + // find_threshold has finished using the overlaid warp sums. + if (tx == 0) smem->tie_nonzero = 0; + __syncthreads(); + } #pragma unroll for (uint32_t i = 0; i < kLocalVecs; ++i) { @@ -736,6 +753,7 @@ struct TopKRegister : TopKRadixBase<12> { } } else if (val >= v_lo) { const auto pos = atomicAdd(&smem->count_eq, 1); + if (zero_bin && __float_as_uint(val) != 0u) atomicOr(&smem->tie_nonzero, 1u); if (pos < kMaxNumTie) [[likely]] { smem->tie_values[pos] = {val, idx}; } @@ -745,7 +763,7 @@ struct TopKRegister : TopKRadixBase<12> { // Phase 4: Handle ties. __syncthreads(); - if (smem->count_eq > kMaxNumTie) [[unlikely]] { + if (smem->count_eq > kMaxNumTie && (!zero_bin || smem->tie_nonzero != 0)) [[unlikely]] { // Buffer holds an arrival-order subset; re-stage it on the exact key. refine_ties(problem, smem, v_lo, v_hi, smem->count_eq); } @@ -805,6 +823,11 @@ struct TopKStreaming : TopKRadixBase<12> { const auto v_hi = smem->v_hi; const auto v_lo = smem->v_lo; const auto topk = problem.topk; + const bool zero_bin = is_zero_bin(v_lo); + if (zero_bin) { + if (tx == 0) smem->tie_nonzero = 0; + __syncthreads(); + } for_each_input(problem.in, problem.seq_len, [&](float val, uint32_t idx) { if (val >= v_hi) { const auto pos = atomicAdd(&smem->count_gt, 1); @@ -813,6 +836,7 @@ struct TopKStreaming : TopKRadixBase<12> { } } else if (val >= v_lo) { const auto pos = atomicAdd(&smem->count_eq, 1); + if (zero_bin && __float_as_uint(val) != 0u) atomicOr(&smem->tie_nonzero, 1u); if (pos < kMaxNumTie) [[likely]] { smem->tie_values[pos] = {val, idx}; } @@ -825,7 +849,7 @@ struct TopKStreaming : TopKRadixBase<12> { // "above" and "tie" sets. above_count is < topk by the threshold-bin invariant, // so the count_gt guard above effectively never triggers. __syncthreads(); - if (smem->count_eq > kMaxNumTie) [[unlikely]] { + if (smem->count_eq > kMaxNumTie && (!zero_bin || smem->tie_nonzero != 0)) [[unlikely]] { // See the register path. refine_ties(problem, smem, v_lo, v_hi, smem->count_eq); } diff --git a/test/registered/kernels/ops/attention/test_topk_v2.py b/test/registered/kernels/ops/attention/test_topk_v2.py index 8ddb596383..11271bbe0c 100644 --- a/test/registered/kernels/ops/attention/test_topk_v2.py +++ b/test/registered/kernels/ops/attention/test_topk_v2.py @@ -733,5 +733,88 @@ def test_topk_v2_split_duplicate_scores(batch: int, seq: int, nvals: int) -> Non _assert_topk_values(scores[row], out[row], k) +@pytest.mark.skipif(not is_hip(), reason="ROCm zero-candidate overflow shortcut") +@pytest.mark.parametrize("k", [1024, 2048]) +@pytest.mark.parametrize( + "batch,seq", + [ + (8, 8192), # Register2 + (8, 16384), # Register4 + (68, 65536), # Streaming: below this batch's split floor + (56, 262400), # Split: late values land in the last long chunk + ], +) +@torch.inference_mode() +def test_topk_v2_zero_candidates_graph_replay(batch: int, seq: int, k: int) -> None: + """Only a proven all-+0 candidate bucket may bypass overflow refinement. + + The row itself is not constant: k-1 positive scores are above the bucket, + and negative scores are below it. More than 2048 +0 candidates precede + the final positions, where a small positive value must win the last slot. + These positions are also late within the last split chunk; atomic staging + order is deliberately not assumed. Reuse one graph across +0, nonzero, + and +0 inputs to exercise the persistent split counters as well. + """ + device = "cuda" + scores_cpu = torch.zeros((batch, seq), dtype=torch.float32) + scores_cpu[:, : k - 1] = 2.0 + scores_cpu[:, k + 128 : k + 640] = -1.0 + assert (scores_cpu[0] == 0).sum() > 2048 + scores = scores_cpu.to(device) + lengths = torch.full((batch,), seq, dtype=torch.int32, device=device) + page_table = ( + torch.arange((seq + PAGE_SIZE - 1) // PAGE_SIZE, device=device) + .to(torch.int32) + .expand(batch, -1) + .contiguous() + ) + metadata = _plan(lengths) + out = torch.full((batch, k), -2, dtype=torch.int32, device=device) + + def transform(): + topk_transform_paged_v2(scores, lengths, page_table, out, PAGE_SIZE, metadata) + + # Compile and initialize persistent scratch outside graph capture. + for _ in range(3): + transform() + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + transform() + + # Both positives round to binary16 +0 but must survive exact fp32 ranking. + # Write bits directly so CPU DAZ/FTZ cannot erase the subnormal test input. + scores_cpu_bits = scores_cpu.view(torch.int32) + for late_bits, negative_zero in [ + (0, False), + (0x31800000, False), # 2**-28 + (0, False), + (1, False), # smallest positive fp32 subnormal + (0, False), + (0, True), + (0, False), + (0x31800000, True), + (0, False), + ]: + scores_cpu_bits[:, -2] = -2147483648 if negative_zero else 0 + scores_cpu_bits[:, -1] = late_bits + scores.copy_(scores_cpu) + assert torch.equal( + scores[:, -2:].view(torch.int32).cpu(), + scores_cpu[:, -2:].view(torch.int32), + ) + out.fill_(-2) + for _ in range(3): + graph.replay() + torch.cuda.synchronize() + # Identity page mapping makes outputs raw indices. The CPU oracle + # preserves subnormals and permits any legal, unique equal-score set. + out_cpu = out.cpu() + if late_bits != 0: + assert (out_cpu == seq - 1).any(dim=1).all() + for row in range(batch): + _assert_topk_values(scores_cpu[row], out_cpu[row], k) + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"]))