diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/topk_v2.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/topk_v2.cuh index 24059de224..fae0ece4c6 100644 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/topk_v2.cuh +++ b/python/sglang/kernels/jit/csrc/deepseek_v4/topk_v2.cuh @@ -19,10 +19,13 @@ #include #include +#include #include #include #include #include +#include +#include namespace sglang { @@ -604,6 +607,341 @@ __global__ __launch_bounds__(kBlockSize, 1) void topk_plan_cluster( #endif // SUPPORT_CLUSTER +#ifdef USE_ROCM +// Split path (ROCm): no clusters/DSMEM on CDNA, so a row spans blocks over two launches with the +// kernel boundary as the barrier: hist accumulates chunk histograms in global memory, select appends +// candidates, and the last block resolves ties and re-zeroes the row (so the scratch is single-stream). + +constexpr uint32_t kSplitMax = 32; ///< most blocks one row may take +constexpr uint32_t kSplitMin = 4; ///< fewest that pays for the second launch + +/// Blocks a launch may spread its rows over: the cross-block cost grows with +/// rows * split, while the scan it buys back only shrinks as L / split. +constexpr uint32_t kSplitBlocks = 64; + +/// Shortest row worth splitting: the second launch and the once-per-row +/// epilogue have to be covered, and their cost tracks the batch, not the spread. +inline constexpr uint32_t split_floor(uint32_t batch_size) { + return batch_size <= 8 ? 40960 : batch_size <= 16 ? 49152 : batch_size <= 32 ? 65536 : 114688; +} + +/// A cache line each: rows reserve their output slots with atomics on these, +/// and packing four rows into one line makes those atomics serialize across +/// rows that have nothing to do with each other. +struct alignas(128) SplitCounters { + uint32_t count_gt; + uint32_t count_eq; + uint32_t arrive; + uint32_t _pad; +}; + +struct SplitWorkspace { + uint32_t* __restrict__ hist; ///< [rows][kHistSize], accumulated, left zeroed + SplitCounters* __restrict__ ctr; ///< [rows] + impl::TieValue* __restrict__ ties; ///< [rows][kMaxNumTie] + uint32_t split; + uint32_t floor; ///< same value the host dispatched on +}; + +/// 12 histogram bits, the width TopKStreaming uses: a 10-bit threshold bin holds +/// more unequal scores than kMaxNumTie can stage, and drops the rest in silence. +struct TopKSplit : impl::TopKRadixBase<12> { + using Base = impl::TopKRadixBase<12>; + static_assert(kHistSize % kBlockSize == 0, "the histogram is transferred kHistItems bins per thread"); + /// Bins per thread, in the contiguous tx * kHistItems layout the base uses. + static constexpr uint32_t kHistItems = kHistSize / kBlockSize; + static constexpr uint32_t kWarp = kBlockSize / impl::TopKConfig::kNumWarps; + + struct Smem : Base::Smem { + uint32_t base_gt, base_eq, total_gt, total_eq, is_last; + int32_t staged[kMaxTopK]; + }; + + struct Chunk { + uint32_t start, len; + }; + + /// This rank's slice. Chunk starts are rounded up to a whole wavefront of + /// vector loads so that for_each_input's 16-byte path stays aligned on every + /// rank, not just the first. + SGL_DEVICE static Chunk chunk_of(uint32_t seq_len, uint32_t rank, uint32_t split) { + constexpr uint32_t kAlign = kWarp * kVecSize; + const uint32_t per = (seq_len + split - 1) / split; + const uint32_t size = ((per + kAlign - 1) / kAlign) * kAlign; + const uint32_t start = min(rank * size, seq_len); + return {start, min(start + size, seq_len) - start}; + } + + SGL_DEVICE static void + histogram_chunk(const TopKProblem& problem, Chunk chunk, uint32_t* __restrict__ row_hist, Smem* smem) { + const auto tx = threadIdx.x; + init_histogram(smem->histogram, tx); + __syncthreads(); + for_each_input(problem.in + chunk.start, chunk.len, [&](float val, uint32_t) { + atomicAdd(&smem->histogram[impl::extract_coarse_bin(val)], 1); + }); + __syncthreads(); + // One atomic per bin per rank, so `split` of them per address at worst. +#pragma unroll + for (uint32_t i = 0; i < kHistItems; ++i) { + const auto bin = tx * kHistItems + i; + if (const auto n = smem->histogram[bin]; n != 0) atomicAdd(&row_hist[bin], n); + } + } + + /// Hand a row's histogram back zeroed, so the next launch needs no reset. + SGL_DEVICE static void clear_row(uint32_t* __restrict__ row_hist) { +#pragma unroll + for (uint32_t i = 0; i < kHistItems; ++i) + row_hist[threadIdx.x * kHistItems + i] = 0; + } + + /// Scan this rank's chunk and append what clears the threshold. Staged in LDS + /// and committed with one global atomic per block: the threshold bin is + /// unbounded, and per-candidate global atomics serialize on one address. + SGL_DEVICE static void select_chunk( + const TopKProblem& problem, + Chunk chunk, + const uint32_t* __restrict__ row_hist, + SplitCounters* __restrict__ ctr, + impl::TieValue* __restrict__ ties, + Smem* smem) { + const auto tx = threadIdx.x; +#pragma unroll + for (uint32_t i = 0; i < kHistItems; ++i) { + const auto bin = tx * kHistItems + i; + smem->histogram[bin] = row_hist[bin]; + } + if (tx == 0) { + smem->count_eq = 0; + smem->count_gt = 0; + smem->v_hi = impl::padding_value(); + smem->v_lo = impl::padding_value(); + } + __syncthreads(); + // The full row's histogram and the full row's seq_len, so every rank picks + // the same bin and the appends below agree on what "above" means. + find_threshold(problem.topk, problem.seq_len, smem, [&](uint32_t threshold_bin) { + smem->v_hi = impl::coarse_bin_lower_bound(threshold_bin + 1); + smem->v_lo = impl::coarse_bin_lower_bound(threshold_bin + 0); + }); + + const auto topk = problem.topk; + const auto v_hi = smem->v_hi; + const auto v_lo = smem->v_lo; + __syncthreads(); + + for_each_input(problem.in + chunk.start, chunk.len, [&](float val, uint32_t local) { + const auto idx = chunk.start + local; + if (val >= v_hi) { + const auto pos = atomicAdd(&smem->count_gt, 1u); + // The whole row has fewer than topk of these, so this rank has too. + if (pos < topk) [[likely]] + smem->staged[pos] = static_cast(idx); + } else if (val >= v_lo) { + const auto slot = atomicAdd(&smem->count_eq, 1u); + if (slot < kMaxNumTie) [[likely]] + smem->tie_values[slot] = {val, idx}; + } + }); + __syncthreads(); + + const auto n_gt = min(smem->count_gt, topk); + const auto n_eq = min(smem->count_eq, kMaxNumTie); + if (tx == 0) { + smem->base_gt = atomicAdd(&ctr->count_gt, n_gt); + smem->base_eq = atomicAdd(&ctr->count_eq, n_eq); + } + __syncthreads(); + const auto base_gt = smem->base_gt; + const auto base_eq = smem->base_eq; + + for (uint32_t t = tx; t < n_gt; t += kBlockSize) { + if (base_gt + t < topk) problem.emit(base_gt + t, static_cast(smem->staged[t])); + } + for (uint32_t t = tx; t < n_eq; t += kBlockSize) { + if (base_eq + t < kMaxNumTie) ties[base_eq + t] = smem->tie_values[t]; + } + } + + /// True in exactly one block per row, once every rank's appends are visible. + /// Nobody waits: the epilogue simply runs wherever the last arrival lands. + SGL_DEVICE static bool arrive_last(SplitCounters* __restrict__ ctr, uint32_t split, Smem* smem) { + __syncthreads(); // this block's appends are done and visible to thread 0 + if (threadIdx.x == 0) { + // Fences from thread 0 only: a device-scope fence is cross-L2 on a + // multi-die part, and the __syncthreads above already covers the block. + __threadfence(); // release: the appends land before the arrival does + const bool last = atomicAdd(&ctr->arrive, 1u) == split - 1; + smem->is_last = last ? 1u : 0u; + if (last) { + __threadfence(); // acquire the other ranks' appends + // Through the atomic path that wrote them: a plain load could be + // served out of this CU's own stale cache. + smem->total_gt = atomicAdd(&ctr->count_gt, 0u); + smem->total_eq = atomicAdd(&ctr->count_eq, 0u); + } + } + __syncthreads(); + return smem->is_last != 0; + } + + /// Fill the slots the threshold bin has to break ties for, staging the ties + /// into LDS first since handle_tie's ranking pass is all-to-all. + SGL_DEVICE static void finish_ties(const TopKProblem& problem, const impl::TieValue* ties, Smem* smem) { + const auto tx = threadIdx.x; + const auto above_count = smem->total_gt; + const auto tie_count = min(smem->total_eq, kMaxNumTie); + const auto remain_topk = above_count < problem.topk ? problem.topk - above_count : 0; + for (uint32_t t = tx; t < tie_count; t += kBlockSize) + smem->tie_values[t] = ties[t]; + __syncthreads(); + handle_tie(smem->tie_values, problem, above_count, tie_count, remain_topk, &smem->tie_handle); + } +}; + +TOPK_KERNEL void topk_split_hist(const __grid_constant__ TopKPagedParams params, const SplitWorkspace ws) { + device::enable_smem_spilling(); + const auto row = blockIdx.x; + const auto rank = blockIdx.y; + const auto tx = threadIdx.x; + // One launch boundary ahead of the only reader, so no fence is needed. + if (rank == 0 && tx < sizeof(SplitCounters) / sizeof(uint32_t)) { + reinterpret_cast(&ws.ctr[row])[tx] = 0; + } + + const auto problem = params.problem(row); + if (problem.seq_len <= ws.floor) return; // the select pass takes it whole + + __shared__ impl::MaxSmem smem; + const auto chunk = TopKSplit::chunk_of(problem.seq_len, rank, ws.split); + auto* row_hist = ws.hist + static_cast(row) * TopKSplit::kHistSize; + TopKSplit::histogram_chunk(problem, chunk, row_hist, reinterpret_cast(&smem)); +} + +template +TOPK_KERNEL void topk_split_select(const __grid_constant__ TopKPagedParams params, const SplitWorkspace ws) { + device::enable_smem_spilling(); + const auto row = blockIdx.x; + const auto rank = blockIdx.y; + auto problem = params.problem(row); + constexpr bool kNeedStaging = kMode != TopKMode::INDICES; + __shared__ impl::MaxSmem smem; + + // Rows too short to be worth splitting were skipped by the histogram pass; + // rank 0 runs them on the ordinary one-block paths and the rest retire, the + // same election the cluster kernel makes for its short items. + if (problem.seq_len <= ws.floor) { + if (rank != 0) return; + __shared__ int32_t s_topk_indices[kNeedStaging ? kMaxTopK : 1]; + if (problem.seq_len <= problem.topk) { + return trivial_transform(problem, params.get_transform(row)); + } + if constexpr (kNeedStaging) problem.out = s_topk_indices; + if (problem.seq_len <= kReg4MaxSeqLen) { + Register4::forward(problem, &smem); + } else { + Streaming::forward(problem, &smem); + } + if constexpr (kNeedStaging) { + __syncthreads(); + paged_transform(problem, params.get_output_ptr(row), params.get_transform(row)); + } + return; + } + + auto* const split_smem = reinterpret_cast(&smem); + auto* const ctr = &ws.ctr[row]; + auto* const ties = ws.ties + static_cast(row) * TopKSplit::kMaxNumTie; + const auto chunk = TopKSplit::chunk_of(problem.seq_len, rank, ws.split); + auto* const row_hist = ws.hist + static_cast(row) * TopKSplit::kHistSize; + + TopKSplit::select_chunk(problem, chunk, row_hist, ctr, ties, split_smem); + if (!TopKSplit::arrive_last(ctr, ws.split, split_smem)) return; + + TopKSplit::clear_row(row_hist); + TopKSplit::finish_ties(problem, ties, split_smem); + if constexpr (kNeedStaging) { + // problem.out is already the destination, and paged_transform reads every + // slot into registers before writing any, so transforming in place is safe. + __syncthreads(); + paged_transform(problem, problem.out, params.get_transform(row)); + } +} + +/// Per-device scratch for the split path, sized for the worst case and never +/// freed: HIP graphs bake the address into their kernel arguments. The gate +/// bounds it by the CU count, not the batch -- about 2 MB on a 256-CU part. +struct SplitResources { + int cu = 0; + uint32_t max_rows = 0; + SplitWorkspace ws{}; + + SplitResources() = default; // the "no split path here" state + + explicit SplitResources(int device_id) { + hipDeviceProp_t prop{}; + if (hipGetDeviceProperties(&prop, device_id) != hipSuccess) { + (void)hipGetLastError(); // do not leave it for the next launch to trip on + return; + } + cu = prop.multiProcessorCount; + max_rows = std::max(cu / kSplitMin, 1); + + const size_t hist_bytes = static_cast(max_rows) * TopKSplit::kHistSize * sizeof(uint32_t); + const size_t ctr_bytes = static_cast(max_rows) * sizeof(SplitCounters); + const size_t tie_bytes = static_cast(max_rows) * TopKSplit::kMaxNumTie * sizeof(impl::TieValue); + + int prev = 0; + (void)hipGetDevice(&prev); + (void)hipSetDevice(device_id); + void* base = nullptr; + const auto total = hist_bytes + ctr_bytes + tie_bytes; + // Zeroed once here; from then on each launch leaves the histogram clean. + const bool ok = hipMalloc(&base, total) == hipSuccess && hipMemset(base, 0, total) == hipSuccess; + (void)hipSetDevice(prev); + if (!ok) { + (void)hipGetLastError(); + cu = 0; // the caller falls back to one block per row + return; + } + auto* p = static_cast(base); + ws.hist = reinterpret_cast(p); + p += hist_bytes; + ws.ctr = reinterpret_cast(p); + p += ctr_bytes; + ws.ties = reinterpret_cast(p); + } +}; + +inline const SplitResources& split_resources(int device_id) { + // One slot per device: the buffer is a raw device pointer, so a single shared + // one would be valid on exactly one of them. + constexpr int kMaxDevices = 16; + static std::once_flag once[kMaxDevices]; + static const SplitResources* slots[kMaxDevices] = {}; + static const SplitResources kNone{}; + if (device_id < 0 || device_id >= kMaxDevices) return kNone; + std::call_once(once[device_id], [device_id] { slots[device_id] = new SplitResources(device_id); }); + return *slots[device_id]; +} + +/// How many blocks to give each row (zero = one block per row) and their scratch. +/// Both bounds are measured: split_floor for length, kSplitBlocks for spread. +inline auto split_plan(uint32_t batch_size, uint32_t max_seq_len, DLDevice device) + -> std::pair { + const auto& res = split_resources(device.device_id); + if (res.cu <= 0 || batch_size == 0 || batch_size > res.max_rows) return {0, {}}; + const auto split = std::clamp(kSplitBlocks / batch_size, kSplitMin, kSplitMax); + const auto floor = split_floor(batch_size); + if (split < kSplitMin || max_seq_len <= floor) return {0, {}}; + auto ws = res.ws; + ws.split = split; + ws.floor = floor; + return {split, ws}; +} +#endif // USE_ROCM + template struct TopKKernel { static void plan( // @@ -796,6 +1134,19 @@ struct TopKKernel { } } } +#elif defined(USE_ROCM) + // Split dispatch. One block per row leaves a long row latency bound on one + // CU however idle the rest is; split_plan decides where a second launch pays. + // PDL stays off: the hist -> select kernel boundary is the barrier. + if (const auto [split, split_ws] = split_plan(batch_size, max_seq_len, device); split >= kSplitMin) { + LaunchKernel({batch_size, split}, kBlockSize, device) + .config({.use_pdl = false}) + .launch(topk_split_hist, params, split_ws); + LaunchKernel({batch_size, split}, kBlockSize, device) + .config({.use_pdl = false}) + .launch(topk_split_select, params, split_ws); + return; + } #endif if (max_seq_len <= kReg2MaxSeqLen) { LaunchKernel(batch_size, kBlockSize, device) diff --git a/test/registered/kernels/ops/attention/test_topk_v2.py b/test/registered/kernels/ops/attention/test_topk_v2.py index 3869c4d28e..99866d2e6a 100644 --- a/test/registered/kernels/ops/attention/test_topk_v2.py +++ b/test/registered/kernels/ops/attention/test_topk_v2.py @@ -608,5 +608,32 @@ def test_topk_v2_packed_level_boundary(boundary: int, residue: int) -> None: _assert_topk_close(window.unsqueeze(0), [ref], [our], 1, [L], k) +@pytest.mark.skipif(not is_hip(), reason="ROCm split path") +@pytest.mark.parametrize("nvals", [64, 1024, 16384]) +@pytest.mark.parametrize("batch,seq", [(1, 131072), (8, 131072)]) +@torch.inference_mode() +def test_topk_v2_split_duplicate_scores(batch: int, seq: int, nvals: int) -> None: + """Duplicate-heavy rows: the cross-block tie merge must stay exact. + + The split path resolves the threshold bin in whichever block arrives last, + out of a tie list that every block appended to, so it is exact only while + that bin holds no more candidates than the list. Rows of ``nvals`` distinct + scores put ``seq / nvals`` exact duplicates at the cut, which is what fills + it; the shapes are long and narrow enough to reach the split gate. CUDA + runs its cluster path here, whose coarser histogram makes the same bin four + times as wide, so this is the ROCm path's test. + """ + k = 2048 + g = torch.Generator(device="cuda").manual_seed(nvals * 7 + seq) + scores = torch.randint(0, nvals, (batch, seq), generator=g, device="cuda").float() + seq_lens = torch.full((batch,), seq, dtype=torch.int32, device="cuda") + out = torch.full((batch, k), -2, dtype=torch.int32, device="cuda") + + topk_transform_paged_v2(scores, seq_lens, None, out, PAGE_SIZE, _plan(seq_lens)) + + for row in range(batch): + _assert_topk_values(scores[row], out[row], k) + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"]))