[ROCm] Select DSA indexer top-k wave size by arch (wave32 on gfx1250) (#41687)

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Thomas Wang <thomawan@amd.com>
This commit is contained in:
jiaryang
2026-09-30 00:47:04 -07:00
committed by GitHub
co-authored by Cursor Thomas Wang
parent 51cae5f303
commit 682f4ef32f
@@ -91,7 +91,22 @@ limitations under the License.
namespace sgl {
namespace dsa_topk {
// Only device code may depend on this: the host pass always sees 64. RDNA targets
// built wave64 define the same macros as wave32, so they are rejected by
// check_wave_size rather than told apart here.
#if defined(__GFX10__) || defined(__GFX11__) || defined(__GFX12__)
constexpr uint32_t kWaveSize = 32; // RDNA and gfx1250 run 32-lane waves
#else
constexpr uint32_t kWaveSize = 64;
#endif
// Folds to nothing when kWaveSize matches the target; otherwise every scan and
// ballot below is wrong and the kernel would emit out-of-range indices.
__device__ __forceinline__ void check_wave_size() {
if (__builtin_amdgcn_wavefrontsize() != kWaveSize) {
__builtin_trap();
}
}
// Ordered 32-bit key: flips the sign bit for positives and inverts negatives, so an
// unsigned compare on the result matches a float compare on the input.
@@ -771,6 +786,7 @@ __device__ void coop_topk_select(const CoopParams<TopK>& params, CoopSmem<TopK,
template <uint32_t TopK, uint32_t kHistBits, uint32_t TieCap, uint32_t BlockSize>
__global__ __launch_bounds__(BlockSize) void coop_topk_kernel(CoopParams<TopK> params) {
check_wave_size();
__shared__ CoopSmem<TopK, kHistBits, TieCap, BlockSize> s;
coop_topk_select<TopK, kHistBits, TieCap, BlockSize>(params, s);
@@ -900,6 +916,7 @@ row_bounds(const CoopMbParams<TopK>& p, uint32_t row, int32_t& row_start, uint32
// atomics per block follow it.
template <uint32_t TopK, uint32_t kHistBits, uint32_t TieCap, uint32_t BlockSize>
__global__ __launch_bounds__(BlockSize) void coop_mb_hist0(CoopMbParams<TopK> p, uint32_t G) {
check_wave_size();
constexpr uint32_t kHistBins = 1u << kHistBits;
constexpr uint32_t kLowBits = 16 - kHistBits;
__shared__ uint32_t hist[kHistBins];
@@ -952,6 +969,7 @@ __global__ __launch_bounds__(BlockSize) void coop_mb_hist0(CoopMbParams<TopK> p,
// the diffuse path.
template <uint32_t TopK, uint32_t kHistBits, uint32_t TieCap, uint32_t BlockSize>
__global__ __launch_bounds__(BlockSize) void coop_mb_hist1(CoopMbParams<TopK> p, uint32_t G) {
check_wave_size();
constexpr uint32_t kLowBits = 16 - kHistBits;
constexpr uint32_t kLowBins = 1u << kLowBits;
__shared__ uint32_t scratch[BlockSize / kWaveSize];
@@ -1025,6 +1043,7 @@ __global__ __launch_bounds__(BlockSize) void coop_mb_hist1(CoopMbParams<TopK> p,
// bandwidth-bound and measured about 4 us at G=4.
template <uint32_t TopK, uint32_t kHistBits, uint32_t TieCap, uint32_t BlockSize>
__global__ __launch_bounds__(BlockSize) void coop_mb_scatter(CoopMbParams<TopK> p, uint32_t G) {
check_wave_size();
constexpr uint32_t kLowBits = 16 - kHistBits;
constexpr uint32_t kLowBins = 1u << kLowBits;
constexpr uint32_t kNumWaves = BlockSize / kWaveSize;
@@ -1163,6 +1182,7 @@ __device__ void coop_mb_refine_row(const CoopMbParams<TopK>& p, CoopSmem<TopK, k
// already handled in coop_mb_hist0, which is the last kernel that touches them.
template <uint32_t TopK, uint32_t kHistBits, uint32_t TieCap, uint32_t BlockSize>
__global__ __launch_bounds__(BlockSize) void coop_mb_refine(CoopMbParams<TopK> p) {
check_wave_size();
__shared__ CoopSmem<TopK, kHistBits, TieCap, BlockSize> s;
coop_mb_refine_row<TopK, kHistBits, TieCap, BlockSize>(p, s);