[ROCm] topk v2: split one long row across blocks, the CDNA cluster-path equivalent (#39931)
Co-authored-by: Bingxu Chen <bingxche@amd.com>
This commit is contained in:
co-authored by
Bingxu Chen
parent
1f299e9f2f
commit
b792228b35
@@ -19,10 +19,13 @@
|
||||
#include <dlpack/dlpack.h>
|
||||
#include <tvm/ffi/container/tensor.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <bit>
|
||||
#include <climits>
|
||||
#include <cstdint>
|
||||
#include <iterator>
|
||||
#include <mutex>
|
||||
#include <utility>
|
||||
|
||||
namespace sglang {
|
||||
|
||||
@@ -604,6 +607,341 @@ __global__ __launch_bounds__(kBlockSize, 1) void topk_plan_cluster(
|
||||
|
||||
#endif // SUPPORT_CLUSTER
|
||||
|
||||
#ifdef USE_ROCM
|
||||
// Split path (ROCm): no clusters/DSMEM on CDNA, so a row spans blocks over two launches with the
|
||||
// kernel boundary as the barrier: hist accumulates chunk histograms in global memory, select appends
|
||||
// candidates, and the last block resolves ties and re-zeroes the row (so the scratch is single-stream).
|
||||
|
||||
constexpr uint32_t kSplitMax = 32; ///< most blocks one row may take
|
||||
constexpr uint32_t kSplitMin = 4; ///< fewest that pays for the second launch
|
||||
|
||||
/// Blocks a launch may spread its rows over: the cross-block cost grows with
|
||||
/// rows * split, while the scan it buys back only shrinks as L / split.
|
||||
constexpr uint32_t kSplitBlocks = 64;
|
||||
|
||||
/// Shortest row worth splitting: the second launch and the once-per-row
|
||||
/// epilogue have to be covered, and their cost tracks the batch, not the spread.
|
||||
inline constexpr uint32_t split_floor(uint32_t batch_size) {
|
||||
return batch_size <= 8 ? 40960 : batch_size <= 16 ? 49152 : batch_size <= 32 ? 65536 : 114688;
|
||||
}
|
||||
|
||||
/// A cache line each: rows reserve their output slots with atomics on these,
|
||||
/// and packing four rows into one line makes those atomics serialize across
|
||||
/// rows that have nothing to do with each other.
|
||||
struct alignas(128) SplitCounters {
|
||||
uint32_t count_gt;
|
||||
uint32_t count_eq;
|
||||
uint32_t arrive;
|
||||
uint32_t _pad;
|
||||
};
|
||||
|
||||
struct SplitWorkspace {
|
||||
uint32_t* __restrict__ hist; ///< [rows][kHistSize], accumulated, left zeroed
|
||||
SplitCounters* __restrict__ ctr; ///< [rows]
|
||||
impl::TieValue* __restrict__ ties; ///< [rows][kMaxNumTie]
|
||||
uint32_t split;
|
||||
uint32_t floor; ///< same value the host dispatched on
|
||||
};
|
||||
|
||||
/// 12 histogram bits, the width TopKStreaming uses: a 10-bit threshold bin holds
|
||||
/// more unequal scores than kMaxNumTie can stage, and drops the rest in silence.
|
||||
struct TopKSplit : impl::TopKRadixBase<12> {
|
||||
using Base = impl::TopKRadixBase<12>;
|
||||
static_assert(kHistSize % kBlockSize == 0, "the histogram is transferred kHistItems bins per thread");
|
||||
/// Bins per thread, in the contiguous tx * kHistItems layout the base uses.
|
||||
static constexpr uint32_t kHistItems = kHistSize / kBlockSize;
|
||||
static constexpr uint32_t kWarp = kBlockSize / impl::TopKConfig::kNumWarps;
|
||||
|
||||
struct Smem : Base::Smem {
|
||||
uint32_t base_gt, base_eq, total_gt, total_eq, is_last;
|
||||
int32_t staged[kMaxTopK];
|
||||
};
|
||||
|
||||
struct Chunk {
|
||||
uint32_t start, len;
|
||||
};
|
||||
|
||||
/// This rank's slice. Chunk starts are rounded up to a whole wavefront of
|
||||
/// vector loads so that for_each_input's 16-byte path stays aligned on every
|
||||
/// rank, not just the first.
|
||||
SGL_DEVICE static Chunk chunk_of(uint32_t seq_len, uint32_t rank, uint32_t split) {
|
||||
constexpr uint32_t kAlign = kWarp * kVecSize;
|
||||
const uint32_t per = (seq_len + split - 1) / split;
|
||||
const uint32_t size = ((per + kAlign - 1) / kAlign) * kAlign;
|
||||
const uint32_t start = min(rank * size, seq_len);
|
||||
return {start, min(start + size, seq_len) - start};
|
||||
}
|
||||
|
||||
SGL_DEVICE static void
|
||||
histogram_chunk(const TopKProblem& problem, Chunk chunk, uint32_t* __restrict__ row_hist, Smem* smem) {
|
||||
const auto tx = threadIdx.x;
|
||||
init_histogram(smem->histogram, tx);
|
||||
__syncthreads();
|
||||
for_each_input(problem.in + chunk.start, chunk.len, [&](float val, uint32_t) {
|
||||
atomicAdd(&smem->histogram[impl::extract_coarse_bin<kHistBits>(val)], 1);
|
||||
});
|
||||
__syncthreads();
|
||||
// One atomic per bin per rank, so `split` of them per address at worst.
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < kHistItems; ++i) {
|
||||
const auto bin = tx * kHistItems + i;
|
||||
if (const auto n = smem->histogram[bin]; n != 0) atomicAdd(&row_hist[bin], n);
|
||||
}
|
||||
}
|
||||
|
||||
/// Hand a row's histogram back zeroed, so the next launch needs no reset.
|
||||
SGL_DEVICE static void clear_row(uint32_t* __restrict__ row_hist) {
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < kHistItems; ++i)
|
||||
row_hist[threadIdx.x * kHistItems + i] = 0;
|
||||
}
|
||||
|
||||
/// Scan this rank's chunk and append what clears the threshold. Staged in LDS
|
||||
/// and committed with one global atomic per block: the threshold bin is
|
||||
/// unbounded, and per-candidate global atomics serialize on one address.
|
||||
SGL_DEVICE static void select_chunk(
|
||||
const TopKProblem& problem,
|
||||
Chunk chunk,
|
||||
const uint32_t* __restrict__ row_hist,
|
||||
SplitCounters* __restrict__ ctr,
|
||||
impl::TieValue* __restrict__ ties,
|
||||
Smem* smem) {
|
||||
const auto tx = threadIdx.x;
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < kHistItems; ++i) {
|
||||
const auto bin = tx * kHistItems + i;
|
||||
smem->histogram[bin] = row_hist[bin];
|
||||
}
|
||||
if (tx == 0) {
|
||||
smem->count_eq = 0;
|
||||
smem->count_gt = 0;
|
||||
smem->v_hi = impl::padding_value();
|
||||
smem->v_lo = impl::padding_value();
|
||||
}
|
||||
__syncthreads();
|
||||
// The full row's histogram and the full row's seq_len, so every rank picks
|
||||
// the same bin and the appends below agree on what "above" means.
|
||||
find_threshold(problem.topk, problem.seq_len, smem, [&](uint32_t threshold_bin) {
|
||||
smem->v_hi = impl::coarse_bin_lower_bound<kHistBits>(threshold_bin + 1);
|
||||
smem->v_lo = impl::coarse_bin_lower_bound<kHistBits>(threshold_bin + 0);
|
||||
});
|
||||
|
||||
const auto topk = problem.topk;
|
||||
const auto v_hi = smem->v_hi;
|
||||
const auto v_lo = smem->v_lo;
|
||||
__syncthreads();
|
||||
|
||||
for_each_input(problem.in + chunk.start, chunk.len, [&](float val, uint32_t local) {
|
||||
const auto idx = chunk.start + local;
|
||||
if (val >= v_hi) {
|
||||
const auto pos = atomicAdd(&smem->count_gt, 1u);
|
||||
// The whole row has fewer than topk of these, so this rank has too.
|
||||
if (pos < topk) [[likely]]
|
||||
smem->staged[pos] = static_cast<int32_t>(idx);
|
||||
} else if (val >= v_lo) {
|
||||
const auto slot = atomicAdd(&smem->count_eq, 1u);
|
||||
if (slot < kMaxNumTie) [[likely]]
|
||||
smem->tie_values[slot] = {val, idx};
|
||||
}
|
||||
});
|
||||
__syncthreads();
|
||||
|
||||
const auto n_gt = min(smem->count_gt, topk);
|
||||
const auto n_eq = min(smem->count_eq, kMaxNumTie);
|
||||
if (tx == 0) {
|
||||
smem->base_gt = atomicAdd(&ctr->count_gt, n_gt);
|
||||
smem->base_eq = atomicAdd(&ctr->count_eq, n_eq);
|
||||
}
|
||||
__syncthreads();
|
||||
const auto base_gt = smem->base_gt;
|
||||
const auto base_eq = smem->base_eq;
|
||||
|
||||
for (uint32_t t = tx; t < n_gt; t += kBlockSize) {
|
||||
if (base_gt + t < topk) problem.emit(base_gt + t, static_cast<uint32_t>(smem->staged[t]));
|
||||
}
|
||||
for (uint32_t t = tx; t < n_eq; t += kBlockSize) {
|
||||
if (base_eq + t < kMaxNumTie) ties[base_eq + t] = smem->tie_values[t];
|
||||
}
|
||||
}
|
||||
|
||||
/// True in exactly one block per row, once every rank's appends are visible.
|
||||
/// Nobody waits: the epilogue simply runs wherever the last arrival lands.
|
||||
SGL_DEVICE static bool arrive_last(SplitCounters* __restrict__ ctr, uint32_t split, Smem* smem) {
|
||||
__syncthreads(); // this block's appends are done and visible to thread 0
|
||||
if (threadIdx.x == 0) {
|
||||
// Fences from thread 0 only: a device-scope fence is cross-L2 on a
|
||||
// multi-die part, and the __syncthreads above already covers the block.
|
||||
__threadfence(); // release: the appends land before the arrival does
|
||||
const bool last = atomicAdd(&ctr->arrive, 1u) == split - 1;
|
||||
smem->is_last = last ? 1u : 0u;
|
||||
if (last) {
|
||||
__threadfence(); // acquire the other ranks' appends
|
||||
// Through the atomic path that wrote them: a plain load could be
|
||||
// served out of this CU's own stale cache.
|
||||
smem->total_gt = atomicAdd(&ctr->count_gt, 0u);
|
||||
smem->total_eq = atomicAdd(&ctr->count_eq, 0u);
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
return smem->is_last != 0;
|
||||
}
|
||||
|
||||
/// Fill the slots the threshold bin has to break ties for, staging the ties
|
||||
/// into LDS first since handle_tie's ranking pass is all-to-all.
|
||||
SGL_DEVICE static void finish_ties(const TopKProblem& problem, const impl::TieValue* ties, Smem* smem) {
|
||||
const auto tx = threadIdx.x;
|
||||
const auto above_count = smem->total_gt;
|
||||
const auto tie_count = min(smem->total_eq, kMaxNumTie);
|
||||
const auto remain_topk = above_count < problem.topk ? problem.topk - above_count : 0;
|
||||
for (uint32_t t = tx; t < tie_count; t += kBlockSize)
|
||||
smem->tie_values[t] = ties[t];
|
||||
__syncthreads();
|
||||
handle_tie(smem->tie_values, problem, above_count, tie_count, remain_topk, &smem->tie_handle);
|
||||
}
|
||||
};
|
||||
|
||||
TOPK_KERNEL void topk_split_hist(const __grid_constant__ TopKPagedParams params, const SplitWorkspace ws) {
|
||||
device::enable_smem_spilling();
|
||||
const auto row = blockIdx.x;
|
||||
const auto rank = blockIdx.y;
|
||||
const auto tx = threadIdx.x;
|
||||
// One launch boundary ahead of the only reader, so no fence is needed.
|
||||
if (rank == 0 && tx < sizeof(SplitCounters) / sizeof(uint32_t)) {
|
||||
reinterpret_cast<uint32_t*>(&ws.ctr[row])[tx] = 0;
|
||||
}
|
||||
|
||||
const auto problem = params.problem(row);
|
||||
if (problem.seq_len <= ws.floor) return; // the select pass takes it whole
|
||||
|
||||
__shared__ impl::MaxSmem<TopKSplit::Smem> smem;
|
||||
const auto chunk = TopKSplit::chunk_of(problem.seq_len, rank, ws.split);
|
||||
auto* row_hist = ws.hist + static_cast<size_t>(row) * TopKSplit::kHistSize;
|
||||
TopKSplit::histogram_chunk(problem, chunk, row_hist, reinterpret_cast<TopKSplit::Smem*>(&smem));
|
||||
}
|
||||
|
||||
template <TopKMode kMode>
|
||||
TOPK_KERNEL void topk_split_select(const __grid_constant__ TopKPagedParams params, const SplitWorkspace ws) {
|
||||
device::enable_smem_spilling();
|
||||
const auto row = blockIdx.x;
|
||||
const auto rank = blockIdx.y;
|
||||
auto problem = params.problem(row);
|
||||
constexpr bool kNeedStaging = kMode != TopKMode::INDICES;
|
||||
__shared__ impl::MaxSmem<Register4::Smem, Streaming::Smem, TopKSplit::Smem> smem;
|
||||
|
||||
// Rows too short to be worth splitting were skipped by the histogram pass;
|
||||
// rank 0 runs them on the ordinary one-block paths and the rest retire, the
|
||||
// same election the cluster kernel makes for its short items.
|
||||
if (problem.seq_len <= ws.floor) {
|
||||
if (rank != 0) return;
|
||||
__shared__ int32_t s_topk_indices[kNeedStaging ? kMaxTopK : 1];
|
||||
if (problem.seq_len <= problem.topk) {
|
||||
return trivial_transform<false, kMode>(problem, params.get_transform(row));
|
||||
}
|
||||
if constexpr (kNeedStaging) problem.out = s_topk_indices;
|
||||
if (problem.seq_len <= kReg4MaxSeqLen) {
|
||||
Register4::forward<false>(problem, &smem);
|
||||
} else {
|
||||
Streaming::forward<false>(problem, &smem);
|
||||
}
|
||||
if constexpr (kNeedStaging) {
|
||||
__syncthreads();
|
||||
paged_transform<kMode>(problem, params.get_output_ptr(row), params.get_transform(row));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
auto* const split_smem = reinterpret_cast<TopKSplit::Smem*>(&smem);
|
||||
auto* const ctr = &ws.ctr[row];
|
||||
auto* const ties = ws.ties + static_cast<size_t>(row) * TopKSplit::kMaxNumTie;
|
||||
const auto chunk = TopKSplit::chunk_of(problem.seq_len, rank, ws.split);
|
||||
auto* const row_hist = ws.hist + static_cast<size_t>(row) * TopKSplit::kHistSize;
|
||||
|
||||
TopKSplit::select_chunk(problem, chunk, row_hist, ctr, ties, split_smem);
|
||||
if (!TopKSplit::arrive_last(ctr, ws.split, split_smem)) return;
|
||||
|
||||
TopKSplit::clear_row(row_hist);
|
||||
TopKSplit::finish_ties(problem, ties, split_smem);
|
||||
if constexpr (kNeedStaging) {
|
||||
// problem.out is already the destination, and paged_transform reads every
|
||||
// slot into registers before writing any, so transforming in place is safe.
|
||||
__syncthreads();
|
||||
paged_transform<kMode>(problem, problem.out, params.get_transform(row));
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-device scratch for the split path, sized for the worst case and never
|
||||
/// freed: HIP graphs bake the address into their kernel arguments. The gate
|
||||
/// bounds it by the CU count, not the batch -- about 2 MB on a 256-CU part.
|
||||
struct SplitResources {
|
||||
int cu = 0;
|
||||
uint32_t max_rows = 0;
|
||||
SplitWorkspace ws{};
|
||||
|
||||
SplitResources() = default; // the "no split path here" state
|
||||
|
||||
explicit SplitResources(int device_id) {
|
||||
hipDeviceProp_t prop{};
|
||||
if (hipGetDeviceProperties(&prop, device_id) != hipSuccess) {
|
||||
(void)hipGetLastError(); // do not leave it for the next launch to trip on
|
||||
return;
|
||||
}
|
||||
cu = prop.multiProcessorCount;
|
||||
max_rows = std::max<uint32_t>(cu / kSplitMin, 1);
|
||||
|
||||
const size_t hist_bytes = static_cast<size_t>(max_rows) * TopKSplit::kHistSize * sizeof(uint32_t);
|
||||
const size_t ctr_bytes = static_cast<size_t>(max_rows) * sizeof(SplitCounters);
|
||||
const size_t tie_bytes = static_cast<size_t>(max_rows) * TopKSplit::kMaxNumTie * sizeof(impl::TieValue);
|
||||
|
||||
int prev = 0;
|
||||
(void)hipGetDevice(&prev);
|
||||
(void)hipSetDevice(device_id);
|
||||
void* base = nullptr;
|
||||
const auto total = hist_bytes + ctr_bytes + tie_bytes;
|
||||
// Zeroed once here; from then on each launch leaves the histogram clean.
|
||||
const bool ok = hipMalloc(&base, total) == hipSuccess && hipMemset(base, 0, total) == hipSuccess;
|
||||
(void)hipSetDevice(prev);
|
||||
if (!ok) {
|
||||
(void)hipGetLastError();
|
||||
cu = 0; // the caller falls back to one block per row
|
||||
return;
|
||||
}
|
||||
auto* p = static_cast<char*>(base);
|
||||
ws.hist = reinterpret_cast<uint32_t*>(p);
|
||||
p += hist_bytes;
|
||||
ws.ctr = reinterpret_cast<SplitCounters*>(p);
|
||||
p += ctr_bytes;
|
||||
ws.ties = reinterpret_cast<impl::TieValue*>(p);
|
||||
}
|
||||
};
|
||||
|
||||
inline const SplitResources& split_resources(int device_id) {
|
||||
// One slot per device: the buffer is a raw device pointer, so a single shared
|
||||
// one would be valid on exactly one of them.
|
||||
constexpr int kMaxDevices = 16;
|
||||
static std::once_flag once[kMaxDevices];
|
||||
static const SplitResources* slots[kMaxDevices] = {};
|
||||
static const SplitResources kNone{};
|
||||
if (device_id < 0 || device_id >= kMaxDevices) return kNone;
|
||||
std::call_once(once[device_id], [device_id] { slots[device_id] = new SplitResources(device_id); });
|
||||
return *slots[device_id];
|
||||
}
|
||||
|
||||
/// How many blocks to give each row (zero = one block per row) and their scratch.
|
||||
/// Both bounds are measured: split_floor for length, kSplitBlocks for spread.
|
||||
inline auto split_plan(uint32_t batch_size, uint32_t max_seq_len, DLDevice device)
|
||||
-> std::pair<uint32_t, SplitWorkspace> {
|
||||
const auto& res = split_resources(device.device_id);
|
||||
if (res.cu <= 0 || batch_size == 0 || batch_size > res.max_rows) return {0, {}};
|
||||
const auto split = std::clamp<uint32_t>(kSplitBlocks / batch_size, kSplitMin, kSplitMax);
|
||||
const auto floor = split_floor(batch_size);
|
||||
if (split < kSplitMin || max_seq_len <= floor) return {0, {}};
|
||||
auto ws = res.ws;
|
||||
ws.split = split;
|
||||
ws.floor = floor;
|
||||
return {split, ws};
|
||||
}
|
||||
#endif // USE_ROCM
|
||||
|
||||
template <bool kUsePDL>
|
||||
struct TopKKernel {
|
||||
static void plan( //
|
||||
@@ -796,6 +1134,19 @@ struct TopKKernel {
|
||||
}
|
||||
}
|
||||
}
|
||||
#elif defined(USE_ROCM)
|
||||
// Split dispatch. One block per row leaves a long row latency bound on one
|
||||
// CU however idle the rest is; split_plan decides where a second launch pays.
|
||||
// PDL stays off: the hist -> select kernel boundary is the barrier.
|
||||
if (const auto [split, split_ws] = split_plan(batch_size, max_seq_len, device); split >= kSplitMin) {
|
||||
LaunchKernel({batch_size, split}, kBlockSize, device)
|
||||
.config({.use_pdl = false})
|
||||
.launch(topk_split_hist, params, split_ws);
|
||||
LaunchKernel({batch_size, split}, kBlockSize, device)
|
||||
.config({.use_pdl = false})
|
||||
.launch(topk_split_select<kMode>, params, split_ws);
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
if (max_seq_len <= kReg2MaxSeqLen) {
|
||||
LaunchKernel(batch_size, kBlockSize, device)
|
||||
|
||||
Reference in New Issue
Block a user