[ROCm] GLM-5.2: gfx950 four-kernel fused DSA indexer decode path (#38583)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
JohnQinAMD
2026-09-29 21:52:00 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent be66c1b460
commit 964fd6af92
14 changed files with 3176 additions and 4 deletions
@@ -0,0 +1,344 @@
/// Dual bf16 GEMV for the gfx950 DSA indexer: Q [M,4096] and KW [M,160] in one launch.
#pragma once
#ifndef USE_ROCM
#error "dual_gemv_bf16.cuh targets CDNA3/4; it uses MFMA and wave64 v_dot2c"
#endif
#include <sgl_kernel/tensor.h>
#include <sgl_kernel/utils.h>
#include <sgl_kernel/runtime.cuh>
#include <sgl_kernel/utils.cuh>
#include <tvm/ffi/container/tensor.h>
namespace sglang {
// Local bf16_t is the builtin __bf16 that ext_vector_type needs, not sglang::bf16_t.
namespace dsa_gfx950::dual_gemv {
using namespace ::sglang::host;
typedef unsigned short ushort_t;
typedef __bf16 bf16_t;
typedef bf16_t bf16x2_t __attribute__((ext_vector_type(2)));
typedef bf16_t bf16x8_t __attribute__((ext_vector_type(8)));
typedef float float4_t __attribute__((ext_vector_type(4)));
constexpr int kThreads = 256;
constexpr int kKq = 2048;
constexpr int kKk = 6144;
constexpr int kMaxRows = 48; // fused_decode.MAX_ROWS
constexpr int kQCols = 16;
constexpr int kQSteps = kKq / 128;
constexpr int kQStageRows = 16;
constexpr int kQStageLd = kThreads + 1; // 16-byte pad per row against LDS bank conflicts
constexpr int kKwVecs = kKk / (8 * kThreads);
__device__ __forceinline__ float dot2_bf16(unsigned int x, unsigned int w, float acc) {
return __builtin_amdgcn_fdot2_f32_bf16(__builtin_bit_cast(bf16x2_t, x), __builtin_bit_cast(bf16x2_t, w), acc, false);
}
// round-to-nearest-even, matching torch's fp32 -> bf16 cast
__device__ __forceinline__ ushort_t f32_to_bf16_rne(float f) {
unsigned int u = __float_as_uint(f);
if ((u & 0x7fffffffu) > 0x7f800000u) { // NaN -> quiet NaN
return static_cast<ushort_t>((u >> 16) | 0x0040u);
}
unsigned int lsb = (u >> 16) & 1u;
u += 0x7fffu + lsb;
return static_cast<ushort_t>(u >> 16);
}
// K offset of MFMA step s, in hipBLASLt MT16x16x512's reduction order so Q matches it bit for bit.
__device__ __forceinline__ int q_step_k(int s) {
return (s >> 2) * 512 + (s & 3) * 32;
}
// 16 Q columns for all M rows; the weights load once and the first 16 x rows go through LDS.
__device__ __forceinline__ void gemv_q(
const bf16_t* __restrict__ x,
const bf16_t* __restrict__ w,
ushort_t* __restrict__ O,
long ldo,
int M,
int col0,
float* __restrict__ smem) {
const int wave = threadIdx.x >> 6;
const int lane = threadIdx.x & 63;
const int frag = lane & 15;
const int kofs = wave * 128 + (lane >> 4) * 8;
const bf16_t* wp = w + (long)(col0 + frag) * kKq + kofs;
const bf16_t* xp = x + (long)frag * kKq + kofs;
bf16x8_t a[kQSteps];
bf16x8_t b[kQSteps];
{
bf16x8_t xv[kQStageRows];
#pragma unroll
for (int r = 0; r < kQStageRows; ++r)
if (r < M) xv[r] = *reinterpret_cast<const bf16x8_t*>(x + (long)r * kKq + threadIdx.x * 8);
#pragma unroll
for (int s = 0; s < kQSteps; ++s)
b[s] = *reinterpret_cast<const bf16x8_t*>(wp + q_step_k(s));
bf16x8_t* xs = reinterpret_cast<bf16x8_t*>(smem);
#pragma unroll
for (int r = 0; r < kQStageRows; ++r)
if (r < M) xs[r * kQStageLd + threadIdx.x] = xv[r];
__syncthreads();
#pragma unroll
for (int s = 0; s < kQSteps; ++s)
a[s] = frag < M ? xs[frag * kQStageLd + (kofs + q_step_k(s)) / 8] : bf16x8_t{};
__syncthreads();
}
const int groups = (M + 15) >> 4;
for (int g = 0; g < groups; ++g) {
const bool more = g + 1 < groups;
const bool next_valid = 16 * (g + 1) + frag < M;
const bf16_t* xn = xp + (long)16 * (g + 1) * kKq;
float4_t acc = {};
#pragma unroll
for (int s = 0; s < kQSteps; ++s) {
acc = __builtin_amdgcn_mfma_f32_16x16x32_bf16(a[s], b[s], acc, 0, 0, 0);
if (more) {
a[s] = next_valid ? *reinterpret_cast<const bf16x8_t*>(xn + q_step_k(s)) : bf16x8_t{};
}
}
// [group][wave][column][row], so the wave-order sum below reads stride 256.
*reinterpret_cast<float4_t*>(smem + ((g * 4 + wave) * 16 + frag) * 16 + (lane >> 4) * 4) = acc;
}
__syncthreads();
for (int i = threadIdx.x; i < M * kQCols; i += kThreads) {
const int r = i >> 4;
const int c = i & 15;
const float* p = smem + ((r >> 4) * 64 + c) * 16 + (r & 15);
float sum = p[0] + p[256];
sum += p[512];
sum += p[768];
O[(long)r * ldo + col0 + c] = f32_to_bf16_rne(sum);
}
}
// NCOL KW columns for R rows from row0.
template <int R, int NCOL>
__device__ __forceinline__ void gemv_kw(
const uint4* __restrict__ Xv,
long ldxv,
const uint4* __restrict__ Wv,
long ldwv,
ushort_t* __restrict__ O,
long ldo,
int M,
int row0,
int col0,
int N,
float* __restrict__ smem) {
const int tid = threadIdx.x;
unsigned int xr[R][kKwVecs][4];
#pragma unroll
for (int m = 0; m < R; ++m) {
#pragma unroll
for (int p = 0; p < kKwVecs; ++p) {
uint4 t = {0u, 0u, 0u, 0u};
if (row0 + m < M) t = Xv[(long)(row0 + m) * ldxv + tid + p * kThreads];
xr[m][p][0] = t.x;
xr[m][p][1] = t.y;
xr[m][p][2] = t.z;
xr[m][p][3] = t.w;
}
}
uint4 wr[NCOL][kKwVecs];
#pragma unroll
for (int c = 0; c < NCOL; ++c) {
const int col = col0 + c;
// clamp instead of branch: out-of-range columns are computed and dropped
const uint4* wp = Wv + (long)(col < N ? col : 0) * ldwv + tid;
#pragma unroll
for (int p = 0; p < kKwVecs; ++p) {
wr[c][p] = wp[p * kThreads];
}
}
float acc[NCOL][R] = {};
#pragma unroll
for (int c = 0; c < NCOL; ++c) {
#pragma unroll
for (int p = 0; p < kKwVecs; ++p) {
unsigned int w4[4] = {wr[c][p].x, wr[c][p].y, wr[c][p].z, wr[c][p].w};
#pragma unroll
for (int e = 0; e < 4; ++e) {
#pragma unroll
for (int m = 0; m < R; ++m)
acc[c][m] = dot2_bf16(xr[m][p][e], w4[e], acc[c][m]);
}
}
}
const int wave = tid >> 6;
const int wlane = tid & 63;
#pragma unroll
for (int off = 32; off > 0; off >>= 1) {
#pragma unroll
for (int c = 0; c < NCOL; ++c)
#pragma unroll
for (int m = 0; m < R; ++m)
acc[c][m] += __shfl_down(acc[c][m], off, 64);
}
if (wlane == 0) {
#pragma unroll
for (int c = 0; c < NCOL; ++c)
#pragma unroll
for (int m = 0; m < R; ++m)
smem[(wave * NCOL + c) * R + m] = acc[c][m];
}
__syncthreads();
if (tid < NCOL * R) {
const int c = tid / R;
const int m = tid - c * R;
float s = 0.0f;
#pragma unroll
for (int w = 0; w < 4; ++w)
s += smem[(w * NCOL + c) * R + m];
const int col = col0 + c;
if (col < N && row0 + m < M) O[(long)(row0 + m) * ldo + col] = f32_to_bf16_rne(s);
}
}
// The KW blocks lead the grid so their loads queue ahead of the Q weight stream.
template <int R, int NCOL>
__global__ __launch_bounds__(kThreads) void dual_gemv_kernel(
const bf16_t* __restrict__ Xq,
const bf16_t* __restrict__ Wq,
ushort_t* __restrict__ Oq,
long ldoq,
int nblk_q,
const uint4* __restrict__ Xk,
long ldxk,
const uint4* __restrict__ Wk,
long ldwk,
ushort_t* __restrict__ Ok,
long ldok,
int Nk,
int nblk_kc,
int M) {
constexpr int kSmemQ = kQStageRows * kQStageLd * 4;
static_assert(kSmemQ >= (kMaxRows / 16) * 4 * 16 * 16 && kSmemQ >= 4 * NCOL * R);
__shared__ __attribute__((aligned(16))) float smem[kSmemQ];
const int nblk_k = gridDim.x - nblk_q;
int bid = blockIdx.x;
bid = bid < nblk_k ? bid + nblk_q : bid - nblk_k;
if (bid < nblk_q) {
gemv_q(Xq, Wq, Oq, ldoq, M, bid * kQCols, smem);
} else {
const int b = bid - nblk_q;
const int rc = b / nblk_kc;
const int cb = b - rc * nblk_kc;
gemv_kw<R, NCOL>(Xk, ldxk, Wk, ldwk, Ok, ldok, M, rc * R, cb * NCOL, Nk, smem);
}
}
struct Args {
const bf16_t* Xq;
const bf16_t* Wq;
ushort_t* Oq;
long ldoq;
int Nq;
const uint4* Xk;
long ldxk;
const uint4* Wk;
long ldwk;
ushort_t* Ok;
long ldok;
int Nk;
int M;
hipStream_t stream;
};
template <int R, int NCOL>
void launch(const Args& a) {
const int nblk_q = a.Nq / kQCols;
const int nblk_kc = (a.Nk + NCOL - 1) / NCOL;
const int nblk_kr = (a.M + R - 1) / R;
dual_gemv_kernel<R, NCOL><<<dim3(nblk_q + nblk_kc * nblk_kr), dim3(kThreads), 0, a.stream>>>(
a.Xq, a.Wq, a.Oq, a.ldoq, nblk_q, a.Xk, a.ldxk, a.Wk, a.ldwk, a.Ok, a.ldok, a.Nk, nblk_kc, a.M);
}
// KW block shape (rows x columns) per M range.
inline void dispatch(const Args& a) {
if (a.M == 1) {
launch<1, 1>(a);
} else if (a.M == 2) {
launch<2, 1>(a);
} else if (a.M <= 4) {
launch<4, 1>(a);
} else if (a.M <= 6) {
launch<2, 2>(a);
} else if (a.M <= 8) {
launch<4, 2>(a);
} else if (a.M <= 24) {
launch<4, 4>(a);
} else {
launch<8, 4>(a);
}
}
} // namespace dsa_gfx950::dual_gemv
struct DualGemvBf16Kernel {
/// Oq = Xq @ Wq^T and Ok = Xk @ Wk^T, all bf16; outputs are overwritten.
static void
run(const tvm::ffi::TensorView Xq,
const tvm::ffi::TensorView Wq,
const tvm::ffi::TensorView Oq,
const tvm::ffi::TensorView Xk,
const tvm::ffi::TensorView Wk,
const tvm::ffi::TensorView Ok) {
using namespace host;
namespace impl = dsa_gfx950::dual_gemv;
auto M_ = SymbolicSize{"num_rows"};
auto Kq_ = SymbolicSize{"k_q"};
auto Nq_ = SymbolicSize{"n_q"};
auto Kk_ = SymbolicSize{"k_k"};
auto Nk_ = SymbolicSize{"n_k"};
auto device_ = SymbolicDevice{};
device_.set_options<kDLCUDA>();
// Q reads x at row * K, so a padded Xq or Wq would silently read wrong rows.
TensorMatcher({M_, Kq_}).with_dtype<bf16_t>().with_device(device_).with_strides({Kq_, 1}).verify(Xq);
TensorMatcher({Nq_, Kq_}).with_dtype<bf16_t>().with_device(device_).with_strides({Kq_, 1}).verify(Wq);
TensorMatcher({M_, Nq_}).with_dtype<bf16_t>().with_device(device_).with_strides({-1, 1}).verify(Oq);
TensorMatcher({M_, Kk_}).with_dtype<bf16_t>().with_device(device_).with_strides({-1, 1}).verify(Xk);
TensorMatcher({Nk_, Kk_}).with_dtype<bf16_t>().with_device(device_).with_strides({-1, 1}).verify(Wk);
TensorMatcher({M_, Nk_}).with_dtype<bf16_t>().with_device(device_).with_strides({-1, 1}).verify(Ok);
const auto M = static_cast<int>(M_.unwrap());
RuntimeCheck(M >= 1 && M <= impl::kMaxRows, "dual_gemv supports M in [1,", impl::kMaxRows, "], got ", M);
const auto Kq = static_cast<int>(Kq_.unwrap());
const auto Kk = static_cast<int>(Kk_.unwrap());
RuntimeCheck(
Kk == impl::kKk && Kq == impl::kKq, "dual_gemv is built for Kk=6144, Kq=2048; got Kk=", Kk, " Kq=", Kq);
impl::Args a;
a.Xq = reinterpret_cast<const impl::bf16_t*>(Xq.data_ptr());
a.Wq = reinterpret_cast<const impl::bf16_t*>(Wq.data_ptr());
a.Oq = reinterpret_cast<impl::ushort_t*>(Oq.data_ptr());
a.Nq = static_cast<int>(Nq_.unwrap());
a.ldoq = Oq.stride(0);
a.Xk = reinterpret_cast<const uint4*>(Xk.data_ptr());
a.Wk = reinterpret_cast<const uint4*>(Wk.data_ptr());
a.Ok = reinterpret_cast<impl::ushort_t*>(Ok.data_ptr());
a.Nk = static_cast<int>(Nk_.unwrap());
a.ldxk = Xk.stride(0) / 8;
a.ldwk = Wk.stride(0) / 8;
a.ldok = Ok.stride(0);
a.M = M;
a.stream = LaunchKernel::resolve_device(device_.unwrap());
RuntimeCheck(a.Nq % impl::kQCols == 0, "dual_gemv needs n_q % 16 == 0, got ", a.Nq);
RuntimeCheck(Xk.stride(0) % 8 == 0 && Wk.stride(0) % 8 == 0, "the K operands must start on an 8-element boundary");
impl::dispatch(a);
RuntimeDeviceCheck();
}
};
} // namespace sglang
@@ -0,0 +1,324 @@
/// GLM-5.2 DSA indexer, the logits kernel: paged MQA FP8 logits plus the fine
/// histogram and coarse summary topk_transform.cuh selects from. One kernel
/// ships, logits_hist_m<8, 12>; WARPS and HB are template arguments.
#pragma once
#ifndef USE_ROCM
#error "paged_mqa_logits.cuh targets gfx950; it is raw CDNA4 MFMA and wave64"
#endif
#include <sgl_kernel/tensor.h>
#include <sgl_kernel/utils.h>
#include <sgl_kernel/runtime.cuh>
#include <sgl_kernel/type.cuh>
#include <sgl_kernel/utils.cuh>
#include <hip/hip_fp16.h>
#include <hip/hip_runtime.h>
#include <tvm/ffi/container/tensor.h>
#include <cstdint>
namespace sglang {
namespace dsa_gfx950::logits {
typedef __attribute__((__vector_size__(8 * sizeof(int)))) int i32x8;
typedef __attribute__((__vector_size__(4 * sizeof(int)))) int i32x4;
typedef __attribute__((__vector_size__(4 * sizeof(float)))) float f32x4;
#define PAGE_TOK 64
#define HD 128
#define HEADS 32
// topk_transform.cuh's TOPK: both kernels must agree on which rows get binned.
#define LG_TOPK 2048
#define TOK_STRIDE 132
#define PAGE_BYTES (PAGE_TOK * TOK_STRIDE) // 8448
#define K_BYTES (PAGE_TOK * HD) // 8192
union V8 {
i32x8 v;
i32x4 h[2];
};
__device__ __forceinline__ f32x4 mfma128(i32x8 a, i32x8 b, f32x4 c) {
return __builtin_amdgcn_mfma_scale_f32_16x16x128_f8f6f4(a, b, c, 0, 0, 0, 127, 0, 127);
}
// wave64 lane exchanges on the VALU instead of through LDS:
// xor32 <- v_permlane64_b32 (lane l reads lane l^32)
// xor16 <- v_permlanex16_b32 with identity selectors (lane l reads lane l^16)
typedef __attribute__((__vector_size__(2 * sizeof(int)))) int i32x2;
__device__ __forceinline__ float xor32(float x, int lane) {
int v = __builtin_bit_cast(int, x);
i32x2 r = __builtin_amdgcn_permlane32_swap(v, v, false, false);
return __builtin_bit_cast(float, (lane & 32) ? r[0] : r[1]);
}
__device__ __forceinline__ float xor16(float x, int lane) {
int v = __builtin_bit_cast(int, x);
i32x2 r = __builtin_amdgcn_permlane16_swap(v, v, false, false);
return __builtin_bit_cast(float, (lane & 16) ? r[0] : r[1]);
}
// The top-k stage's histogram, built here while this kernel waits on KV loads.
// It shares three invariants with topk_transform.cuh: the key (order_bin_fast),
// the npages clamp, and do_hist's TOPK.
#define LG_CBITS 6
#define LG_CBINS (1 << LG_CBITS)
// Fused paged-MQA FP8 logits + in-loop fine histogram + coarse summary. The
// histogram is what lets the top-k stage skip a separate scoring pass.
// order_key16(x) >> LOWB in 4 VALU ops after the cvt, with no branch and no
// v_cndmask. Bit-identical to topk_transform.cuh's order_key16(x) >> LOWB for
// every finite x and for +/-0.
template <int LOWB>
__device__ __forceinline__ uint32_t order_bin_fast(float x) {
const uint32_t h = (uint32_t)__half_as_ushort(__float2half_rn(x));
const uint32_t m = (uint32_t)((int32_t)__float_as_int(x) >> 31);
return (h >> LOWB) ^ (0x8000u >> LOWB) ^ (m & (0x7fffu >> LOWB));
}
template <int WARPS, int HB>
__global__ __launch_bounds__(WARPS * 64) void logits_hist_m(
const uint8_t* __restrict__ q,
const uint8_t* __restrict__ kv,
const float* __restrict__ wgt,
const int* __restrict__ seqlens,
const int* __restrict__ ptable,
float* __restrict__ out,
unsigned int* __restrict__ ghist,
int max_pages,
int out_stride) {
constexpr int NBIN = 1 << HB;
constexpr int LOWB = 16 - HB;
constexpr int GHS = NBIN + LG_CBINS;
constexpr int NTH = WARPS * 64;
constexpr int PERT = NBIN / NTH; // GLOBAL bins per thread (coarse map)
constexpr int CSH = HB - LG_CBITS;
constexpr int GRP = NTH / LG_CBINS;
static_assert(NBIN >= 2 && (NBIN % 2) == 0, "paired flush needs even bins");
static_assert(NBIN % NTH == 0 && NTH >= LG_CBINS, "");
static_assert(PERT <= LG_CBINS && (LG_CBINS % PERT) == 0, "a thread's contiguous run must sit inside one coarse bin");
__shared__ unsigned int s_hist[NBIN];
const int row = blockIdx.y;
const int lane = threadIdx.x & 63;
const int warp = threadIdx.x >> 6;
const int seqlen = seqlens[row];
// seqlen is device data, so no host check can bound it. Without the clamp
// an oversized row walks into the NEXT row's page table -- valid memory,
// plausible page ids, confidently wrong logits.
const int npages_raw = (seqlen + PAGE_TOK - 1) / PAGE_TOK;
// Bound the walk by BOTH the page table and `out`. The host only requires
// that `out` cover whole pages -- a captured table is as wide as the graph
// while the row is only as long as seqlens says -- so a table wider than
// `out` is legal, and without the second term an oversized seqlen writes
// past the end of the row.
const int pages_out = out_stride / PAGE_TOK;
const int npages_cap = max_pages < pages_out ? max_pages : pages_out;
const int npages = npages_raw < npages_cap ? npages_raw : npages_cap;
const int g = lane >> 4, c = lane & 15;
// Nothing to rank when the row is shorter than topk: everything is selected,
// so the histogram, its two barriers and the epilogue all compile out.
//
// This predicate MUST use the same clamped length the paired top-k kernel
// applies (it clamps row_ends by the page-table width before its own
// row_len <= TOPK early return). A row binned here but skipped there is
// never zeroed, and the histogram stays dirty for the life of the process.
const int seqlen_eff = seqlen < npages_cap * PAGE_TOK ? seqlen : npages_cap * PAGE_TOK;
const bool do_hist = (seqlen_eff > LG_TOPK);
if (do_hist) {
for (int i = threadIdx.x; i < NBIN; i += NTH)
s_hist[i] = 0u;
}
const uint8_t* qp = q + (size_t)row * (size_t)(HEADS * HD) + (size_t)c * HD + g * 32;
V8 qa0, qa1;
qa0.h[0] = *(const i32x4*)(qp);
qa0.h[1] = *(const i32x4*)(qp + 16);
qa1.h[0] = *(const i32x4*)(qp + 16 * HD);
qa1.h[1] = *(const i32x4*)(qp + 16 * HD + 16);
const float* wp = wgt + (size_t)row * HEADS + g * 4;
f32x4 w0 = *(const f32x4*)(wp);
f32x4 w1 = *(const f32x4*)(wp + 16);
const int nwaves = gridDim.x * WARPS;
const int wid = blockIdx.x * WARPS + warp;
const int koff = g * 512 + c * 16;
if (do_hist) __syncthreads();
unsigned int* __restrict__ gh = ghist + (size_t)row * GHS;
const int* __restrict__ pt = ptable + (size_t)row * max_pages;
for (int p = wid; p < npages; p += nwaves) {
const int phys = pt[p];
const uint8_t* base = kv + (size_t)phys * PAGE_BYTES;
const uint8_t* kb = base + koff;
V8 b[4];
#pragma unroll
for (int tt = 0; tt < 4; ++tt) {
b[tt].h[0] = *(const i32x4*)(kb + tt * 2048);
b[tt].h[1] = *(const i32x4*)(kb + tt * 2048 + 256);
}
const float ks = ((const float*)(base + K_BYTES))[lane];
float s[4];
#pragma unroll
for (int tt = 0; tt < 4; ++tt) {
const f32x4 z = {0.f, 0.f, 0.f, 0.f};
f32x4 a0 = mfma128(qa0.v, b[tt].v, z);
f32x4 a1 = mfma128(qa1.v, b[tt].v, z);
float acc = 0.f;
#pragma unroll
for (int i = 0; i < 4; ++i)
acc = fmaf(fmaxf(a0[i], 0.f), w0[i], acc);
#pragma unroll
for (int i = 0; i < 4; ++i)
acc = fmaf(fmaxf(a1[i], 0.f), w1[i], acc);
s[tt] = acc;
}
const bool hi = (g & 2) != 0;
float A0 = (hi ? s[2] : s[0]) + xor32(hi ? s[0] : s[2], lane);
float A1 = (hi ? s[3] : s[1]) + xor32(hi ? s[1] : s[3], lane);
const bool odd = (g & 1) != 0;
float T = (odd ? A1 : A0) + xor16(odd ? A0 : A1, lane);
const int pos = p * PAGE_TOK + lane;
const bool live = pos < seqlen;
const float v = T * ks;
out[(size_t)row * out_stride + pos] = live ? v : -INFINITY;
if (do_hist && live) {
const uint32_t bb = order_bin_fast<LOWB>(v);
atomicAdd(&s_hist[bb], 1u);
}
}
if (do_hist) {
__syncthreads();
// ---- coarse summary, mapped over the GLOBAL bin index ----
// Thread tx owns global bins [tx*PERT, tx*PERT+PERT), which lies inside
// coarse bin (tx*PERT)>>CSH. Needs no barrier of its own.
const int g0 = (int)threadIdx.x * PERT;
unsigned int acc = 0u;
#pragma unroll
for (int j = 0; j < PERT; ++j)
acc += s_hist[g0 + j];
#pragma unroll
for (int o = GRP >> 1; o >= 1; o >>= 1)
acc += __shfl_down(acc, o, 64);
if ((threadIdx.x & (GRP - 1)) == 0 && acc) atomicAdd(&gh[NBIN + (g0 >> CSH)], acc);
for (int i = threadIdx.x; i < NBIN / 2; i += NTH) {
const unsigned long long vv = (unsigned long long)s_hist[2 * i] | ((unsigned long long)s_hist[2 * i + 1] << 32);
if (vv) atomicAdd((unsigned long long*)(gh + 2 * i), vv);
}
}
}
/// The fp8 dtype the MFMA reads as raw OCP e4m3 bytes. Checked by hand rather
/// than through ``with_dtype<fp8_e4m3_t>``: on ROCm ``sglang::fp8_e4m3_t`` is
/// ``uint8_t``, which ``DLDataTypeTrait`` maps to kDLUInt -- that would reject a
/// genuine fp8 tensor and accept a uint8 one, exactly backwards.
inline auto is_fp8_e4m3fn(DLDataType dtype) -> bool {
return dtype.lanes == 1 && dtype.bits == 8 && dtype.code == kDLFloat8_e4m3fn;
}
} // namespace dsa_gfx950::logits
struct PagedMqaLogitsKernel {
static void
run(const tvm::ffi::TensorView q,
const tvm::ffi::TensorView kv,
const tvm::ffi::TensorView weights,
const tvm::ffi::TensorView seqlens,
const tvm::ffi::TensorView page_table,
const tvm::ffi::TensorView out,
const tvm::ffi::TensorView ghist,
int64_t blocks_per_row) {
using namespace host;
namespace impl = dsa_gfx950::logits;
// The launch below is logits_hist_m<8, HOST_HB>, so the histogram row
// stride the kernel writes is fixed here too. topk_transform.cuh checks
// the same width from its side; without this one a mismatch overruns into
// the next row's counters.
constexpr int HOST_HB = 12;
constexpr int64_t HOST_GHS = (int64_t{1} << HOST_HB) + LG_CBINS;
auto rows_ = SymbolicSize{"rows"};
auto pages_ = SymbolicSize{"table_pages"};
auto cols_ = SymbolicSize{"logits_cols"};
auto device_ = SymbolicDevice{};
device_.set_options<kDLCUDA>();
// Rebinding one symbolic dtype across q and kv is the "kv must have the
// same fp8 dtype as q" check; the options stay open because e4m3fn has no
// DLDataTypeTrait on ROCm.
auto fp8_ = SymbolicDType{};
// No with_strides means TensorMatcher demands full contiguity, which is
// what each of these wants.
TensorMatcher({rows_, int64_t{HEADS}, int64_t{HD}}).with_dtype(fp8_).with_device(device_).verify(q);
// PAGE_TOK and TOK_STRIDE are compiled in; a cache laid out differently
// would be read as garbage rather than refused, hence the row width here.
TensorMatcher({-1, int64_t{PAGE_TOK} * TOK_STRIDE}).with_dtype(fp8_).with_device(device_).verify(kv);
RuntimeCheck(
impl::is_fp8_e4m3fn(fp8_.unwrap()), "q and the kv cache must be fp8 e4m3fn; they are read as raw bytes");
TensorMatcher({rows_, int64_t{HEADS}}).with_dtype<fp32_t>().with_device(device_).verify(weights);
TensorMatcher({rows_, pages_}).with_dtype<int32_t>().with_device(device_).verify(page_table);
TensorMatcher({rows_, HOST_GHS})
.with_dtype<int32_t>()
.with_device(device_)
.with_strides({HOST_GHS, 1})
.verify(ghist);
// `out` is a column slice of the workspace, so only the row is packed.
TensorMatcher({rows_, cols_}).with_dtype<fp32_t>().with_device(device_).with_strides({-1, 1}).verify(out);
// One entry per row, but the caller may hand over a longer buffer.
TensorMatcher({-1}).with_dtype<int32_t>().with_device(device_).verify(seqlens);
RuntimeCheck(
seqlens.size(0) >= rows_.unwrap(),
"seqlens needs one entry per row: got ",
seqlens.size(0),
" for ",
rows_.unwrap(),
" rows");
// A captured page table is as wide as the graph while the row is as long
// as seqlens says, so it may exceed `out`; what must hold is that `out`
// covers whole pages.
RuntimeCheck(cols_.unwrap() % PAGE_TOK == 0, "out row must be a whole number of pages, got ", cols_.unwrap());
LaunchKernel(
dim3(static_cast<int>(blocks_per_row), static_cast<int>(rows_.unwrap())), dim3(8 * 64), device_.unwrap())(
impl::logits_hist_m<8, HOST_HB>,
reinterpret_cast<const uint8_t*>(q.data_ptr()),
reinterpret_cast<const uint8_t*>(kv.data_ptr()),
static_cast<fp32_t*>(weights.data_ptr()),
static_cast<int*>(seqlens.data_ptr()),
static_cast<int*>(page_table.data_ptr()),
static_cast<fp32_t*>(out.data_ptr()),
reinterpret_cast<unsigned int*>(ghist.data_ptr()),
static_cast<int>(pages_.unwrap()),
static_cast<int>(out.stride(0)));
}
};
} // namespace sglang
// PAGE_TOK / HD / TOK_STRIDE are generic enough to collide with another header
// compiled into the same translation unit; they exist only for the kernel above.
#undef PAGE_TOK
#undef HD
#undef HEADS
#undef LG_TOPK
#undef TOK_STRIDE
#undef PAGE_BYTES
#undef K_BYTES
#undef LG_CBITS
#undef LG_CBINS
@@ -0,0 +1,336 @@
/// A copy of aiter's indexer_qk_rope_quant_and_cache_kernel with an inline
/// 128-point Hadamard, so the front end keeps rotate_activation and the fusion
/// at once. aiter is untouched; this includes its headers (JIT dependency
/// "aiter"), so the maths match.
#pragma once
#ifndef USE_ROCM
#error "qk_rope_hadamard_quant.cuh targets gfx950; it folds a head onto one wave64"
#endif
#include <sgl_kernel/tensor.h>
#include <sgl_kernel/utils.h>
#include <sgl_kernel/runtime.cuh>
#include <sgl_kernel/type.cuh>
#include <sgl_kernel/utils.cuh>
#include <hip/hip_bf16.h>
#include <hip/hip_runtime.h>
#include <tvm/ffi/container/tensor.h>
#include "hip_reduce.h"
#include "opus/opus.hpp"
#include <algorithm>
#include <cstdint>
namespace sglang {
namespace dsa_gfx950::qk_rope {
using cache_t = opus::fp8_t;
using scalar_t = opus::bf16_t;
constexpr int HEAD_DIM = 128;
constexpr int ROPE_DIM = 64;
constexpr int LOG2_HEAD_DIM = 7;
// Folded onto one 64-lane wavefront: lane t owns dim t and t+64, so both halves
// of every reduction are an independent DPP tree. Zero LDS, zero syncthreads,
// and bit-exact against the 128-thread form operand for operand.
constexpr int HALF_DIM = HEAD_DIM / 2; // 64 == one wavefront
template <typename F>
__device__ __forceinline__ float wred64(float v, F op) {
return wave_reduce<float, F, 64, true>(v, op);
}
// Interleaved rope on the low 64 dims, held one-per-lane in `v`. The pair
// comes from a cross-lane shuffle instead of LDS; the arithmetic is the same.
__device__ __forceinline__ float
rope_lo(float v, int t, const scalar_t* __restrict__ cos_ptr, const scalar_t* __restrict__ sin_ptr) {
const float pair_val = __shfl_xor(v, 1, 64);
const int cos_idx = t / 2;
const float cos_v = static_cast<float>(cos_ptr[cos_idx]);
const float sin_v = static_cast<float>(sin_ptr[cos_idx]);
v = (t % 2 == 0) ? (v * cos_v - pair_val * sin_v) : (v * cos_v + pair_val * sin_v);
return static_cast<float>(static_cast<scalar_t>(v));
}
// 128-point Hadamard on a pair (lo = dim t, hi = dim t+64) held in one lane.
__device__ __forceinline__ void hadamard128_wave(float& lo, float& hi, int t, float scale) {
#pragma unroll
for (int s = 0; s < LOG2_HEAD_DIM - 1; ++s) { // strides 1, 2, 4, 8, 16, 32
const int d = 1 << s;
const float plo = __shfl_xor(lo, d, 64);
const float phi = __shfl_xor(hi, d, 64);
const bool a = ((t & d) == 0); // (dim & d) is the same for dim and dim+64
lo = a ? (lo + plo) : (plo - lo);
hi = a ? (hi + phi) : (phi - hi);
}
// stride 64: partner of dim t is dim t+64 -- both already in this lane.
const float nlo = lo + hi;
const float nhi = lo - hi; // (dim & 64) != 0 -> partner - v
lo = nlo * scale;
hi = nhi * scale;
}
// The k-side epilogue runs in its own block rather than at the end of the
// head_idx==0 blocks, where it would serialise behind them.
__global__ __launch_bounds__(HALF_DIM) void qk_rope_hadamard_quant_kernel(
const scalar_t* __restrict__ q,
cache_t* __restrict__ q_out,
const scalar_t* __restrict__ weights,
float* __restrict__ weights_out,
const scalar_t* __restrict__ k,
cache_t* __restrict__ kv_cache,
const int64_t* __restrict__ slot_mapping,
const float* __restrict__ norm_weight,
const float* __restrict__ norm_bias,
const int64_t* __restrict__ positions,
const scalar_t* __restrict__ cos_cache,
const scalar_t* __restrict__ sin_cache,
const int n_heads,
const int cache_block_size,
const int cache_stride,
const int q_stride_t,
const int q_stride_h,
const int q_out_stride_t,
const int q_out_stride_h,
const int weights_stride_t,
const int weights_out_stride_t,
const int k_stride_t,
const int cos_stride0,
const int sin_stride0,
const float epsilon,
const float weights_scale,
const float hadamard_scale,
const int max_position) {
const int64_t token_idx = blockIdx.x;
const int head_idx = blockIdx.y; // n_heads + 1 of them: the last is the k side
const int t = threadIdx.x; // owns dim t and dim t + 64
const bool do_q = head_idx < n_heads;
const int64_t slot_idx = slot_mapping[token_idx];
// Clamped unconditionally: pos indexes cos_cache/sin_cache on both branches,
// and clamping only the slot_idx < 0 one left the common path free to read at
// an arbitrary offset.
int64_t pos = positions[token_idx];
pos = pos < 0 ? 0 : (pos >= max_position ? max_position - 1 : pos);
const scalar_t* cos_ptr = cos_cache + pos * cos_stride0;
const scalar_t* sin_ptr = sin_cache + pos * sin_stride0;
auto max_func = [](float a, float b) { return fmaxf(a, b); };
auto sum_func = [](float a, float b) { return a + b; };
const float q_fp8_max = static_cast<float>(opus::finfo<cache_t>::max());
// ------------------------------- q side ---------------------------------
if (do_q) {
const scalar_t* q_row = q + token_idx * q_stride_t + head_idx * q_stride_h;
float lo = static_cast<float>(q_row[t]);
float hi = static_cast<float>(q_row[t + HALF_DIM]);
lo = rope_lo(lo, t, cos_ptr, sin_ptr); // dims >=64 untouched
hadamard128_wave(lo, hi, t, hadamard_scale);
lo = static_cast<float>(static_cast<scalar_t>(lo));
hi = static_cast<float>(static_cast<scalar_t>(hi));
const float a0 = wred64(fabsf(lo), max_func);
const float a1 = wred64(fabsf(hi), max_func);
const float q_amax = max_func(a1, a0); // block_reduce's order
const float q_inv_fp8_max = 1.0f / q_fp8_max;
// ue8m0: the scale is rounded up to a power of two.
const float q_scale = exp2f(ceilf(log2f(fmaxf(q_amax, 1e-10f) * q_inv_fp8_max)));
const float q_inv_scale = 1.0f / q_scale;
cache_t* qo = q_out + token_idx * q_out_stride_t + head_idx * q_out_stride_h;
qo[t] = opus::cast<cache_t>(lo * q_inv_scale);
qo[t + HALF_DIM] = opus::cast<cache_t>(hi * q_inv_scale);
if (t == 0) {
const float w = static_cast<float>(weights[token_idx * weights_stride_t + head_idx]);
const float head_scale = rsqrtf(static_cast<float>(n_heads));
const scalar_t w_head = static_cast<scalar_t>(w * head_scale);
const float softmax_scale = weights_scale / head_scale;
weights_out[token_idx * weights_out_stride_t + head_idx] = static_cast<float>(w_head) * q_scale * softmax_scale;
}
}
if (do_q || slot_idx < 0) return;
// ------------------------------- k side ---------------------------------
const scalar_t* k_row = k + token_idx * k_stride_t;
const float xlo = static_cast<float>(k_row[t]);
const float xhi = static_cast<float>(k_row[t + HALF_DIM]);
const float s0 = wred64(xlo, sum_func);
const float s1 = wred64(xhi, sum_func);
const float mean = sum_func(s1, s0) / static_cast<float>(HEAD_DIM);
const float clo = xlo - mean;
const float chi = xhi - mean;
const float q0 = wred64(clo * clo, sum_func);
const float q1 = wred64(chi * chi, sum_func);
const float ss = sum_func(q1, q0);
const float inv_std = rsqrtf(ss / static_cast<float>(HEAD_DIM) + epsilon);
float klo = clo * inv_std * norm_weight[t] + norm_bias[t];
float khi = chi * inv_std * norm_weight[t + HALF_DIM] + norm_bias[t + HALF_DIM];
klo = static_cast<float>(static_cast<scalar_t>(klo));
khi = static_cast<float>(static_cast<scalar_t>(khi));
klo = rope_lo(klo, t, cos_ptr, sin_ptr);
hadamard128_wave(klo, khi, t, hadamard_scale);
klo = static_cast<float>(static_cast<scalar_t>(klo));
khi = static_cast<float>(static_cast<scalar_t>(khi));
const float m0 = wred64(fabsf(klo), max_func);
const float m1 = wred64(fabsf(khi), max_func);
const float k_amax = max_func(m1, m0);
const float k_scale = exp2f(ceilf(log2f(fmaxf(k_amax, 1e-4f) / q_fp8_max))); // ue8m0
const int64_t block_idx = slot_idx / cache_block_size;
const int64_t block_offset = slot_idx % cache_block_size;
const int64_t page_base = block_idx * cache_block_size * cache_stride;
if (t == 0) {
// One fp32 scale per token, after the page's K bytes: quant block == HEAD_DIM.
const int64_t dst_scale_idx = page_base + cache_block_size * HEAD_DIM + block_offset * 4;
reinterpret_cast<float*>(kv_cache)[dst_scale_idx / 4] = k_scale;
}
const float k_inv_scale = 1.0f / k_scale;
// Preshuffled page layout: 16x16 (token x dim) tiles.
constexpr int TILE = 16;
#pragma unroll
for (int h = 0; h < 2; ++h) {
const int dim = t + h * HALF_DIM;
const int token_tile_id = block_offset / TILE;
const int token_in_tile = block_offset % TILE;
const int col_tile_id = dim / TILE;
const int col_in_tile = dim % TILE;
const int64_t dst_offset = page_base + token_tile_id * (TILE * HEAD_DIM) + col_tile_id * (TILE * TILE) +
token_in_tile * TILE + col_in_tile;
kv_cache[dst_offset] = opus::cast<cache_t>((h == 0 ? klo : khi) * k_inv_scale);
}
}
} // namespace dsa_gfx950::qk_rope
struct QkRopeHadamardQuantKernel {
/// AITER indexer_qk_rope_quant_and_cache plus the inline 128-point Hadamard,
/// specialised to ue8m0 scales, the preshuffled cache, interleaved rope and a
/// 128-wide quant block; the Python gate admits nothing else.
static void
run(const tvm::ffi::TensorView q,
const tvm::ffi::TensorView q_out,
const tvm::ffi::TensorView weights,
const tvm::ffi::TensorView weights_out,
const tvm::ffi::TensorView k,
const tvm::ffi::TensorView kv_cache,
const tvm::ffi::TensorView slot_mapping,
const tvm::ffi::TensorView norm_weight,
const tvm::ffi::TensorView norm_bias,
const tvm::ffi::TensorView positions,
const tvm::ffi::TensorView cos_cache,
const tvm::ffi::TensorView sin_cache,
double epsilon,
double weights_scale) {
using namespace host;
namespace impl = dsa_gfx950::qk_rope;
/// ``q_out`` and ``kv_cache`` are reinterpret_cast to a 1-byte cache_t and
/// written as raw fp8, so what matters is the width, not the flavour: a
/// wider dtype fills half the row and leaves the rest untouched.
const auto byte_width = [](DLDataType dtype) -> int64_t { return (int64_t{dtype.bits} * dtype.lanes + 7) / 8; };
const int num_tokens = static_cast<int>(std::min(k.size(0), slot_mapping.size(0)));
// num_tokens becomes gridDim.x and indexes all six per-token tensors, but
// only two of them took part in the min above.
RuntimeCheck(
q.size(0) >= num_tokens && q_out.size(0) >= num_tokens && weights.size(0) >= num_tokens &&
weights_out.size(0) >= num_tokens && positions.size(0) >= num_tokens,
"every per-token tensor must have at least num_tokens rows");
const int head_dim = static_cast<int>(k.size(1));
const int n_heads = static_cast<int>(q.size(1));
RuntimeCheck(cos_cache.dim() == 2 && sin_cache.dim() == 2, "cos/sin must be 2-D");
const int rope_dim = static_cast<int>(cos_cache.size(1)) * 2;
const int cache_block_size = static_cast<int>(kv_cache.size(1));
const int cache_stride = static_cast<int>(kv_cache.size(2));
const int max_position = static_cast<int>(cos_cache.size(0));
RuntimeCheck(head_dim == impl::HEAD_DIM, "head_dim must be 128");
RuntimeCheck(rope_dim == impl::ROPE_DIM, "rope_dim must be 64");
RuntimeCheck(is_type<bf16_t>(q.dtype()) && is_type<bf16_t>(k.dtype()), "q/k must be bf16");
RuntimeCheck(is_type<bf16_t>(weights.dtype()), "weights must be bf16");
RuntimeCheck(
byte_width(q_out.dtype()) == 1 && byte_width(kv_cache.dtype()) == 1,
"q_out and kv_cache must be 1-byte fp8, got ",
q_out.dtype(),
"/",
kv_cache.dtype());
RuntimeCheck(is_type<fp32_t>(weights_out.dtype()), "weights_out must be fp32");
RuntimeCheck(
is_type<fp32_t>(norm_weight.dtype()) && is_type<fp32_t>(norm_bias.dtype()), "norm params must be fp32");
RuntimeCheck(cos_cache.stride(1) == 1 && sin_cache.stride(1) == 1, "cos/sin last dim must be contiguous");
RuntimeCheck(is_type<int64_t>(slot_mapping.dtype()), "slot_mapping must be int64");
RuntimeCheck(is_type<int64_t>(positions.dtype()), "positions must be int64");
const auto* qp = reinterpret_cast<const impl::scalar_t*>(q.data_ptr());
auto* qop = reinterpret_cast<impl::cache_t*>(q_out.data_ptr());
const auto* wp = reinterpret_cast<const impl::scalar_t*>(weights.data_ptr());
auto* wop = static_cast<fp32_t*>(weights_out.data_ptr());
const auto* kp = reinterpret_cast<const impl::scalar_t*>(k.data_ptr());
auto* cp = reinterpret_cast<impl::cache_t*>(kv_cache.data_ptr());
const auto* sp = static_cast<const int64_t*>(slot_mapping.data_ptr());
const auto* nwp = static_cast<const fp32_t*>(norm_weight.data_ptr());
const auto* nbp = static_cast<const fp32_t*>(norm_bias.data_ptr());
const auto* pp = static_cast<const int64_t*>(positions.data_ptr());
const auto* cosp = reinterpret_cast<const impl::scalar_t*>(cos_cache.data_ptr());
const auto* sinp = reinterpret_cast<const impl::scalar_t*>(sin_cache.data_ptr());
// rotate_activation(x) == hadamard_transform(x, scale=x.size(-1) ** -0.5)
const float had_scale = 1.0f / sqrtf(static_cast<float>(impl::HEAD_DIM));
// The wave path indexes the head dimension directly, so the layout
// invariants it relies on are asserted rather than carried as arguments.
RuntimeCheck(q.stride(2) == 1 && q_out.stride(2) == 1 && k.stride(1) == 1, "wave kernel needs contiguous head dim");
RuntimeCheck(
weights.stride(1) == 1 && weights_out.stride(1) == 1, "wave kernel needs contiguous head axis on weights");
LaunchKernel(dim3(num_tokens, n_heads + 1), dim3(impl::HALF_DIM), q.device())(
impl::qk_rope_hadamard_quant_kernel,
qp,
qop,
wp,
wop,
kp,
cp,
sp,
nwp,
nbp,
pp,
cosp,
sinp,
n_heads,
cache_block_size,
cache_stride,
static_cast<int>(q.stride(0)),
static_cast<int>(q.stride(1)),
static_cast<int>(q_out.stride(0)),
static_cast<int>(q_out.stride(1)),
static_cast<int>(weights.stride(0)),
static_cast<int>(weights_out.stride(0)),
static_cast<int>(k.stride(0)),
static_cast<int>(cos_cache.stride(0)),
static_cast<int>(sin_cache.stride(0)),
static_cast<float>(epsilon),
static_cast<float>(weights_scale),
had_scale,
max_position);
}
};
} // namespace sglang
File diff suppressed because it is too large Load Diff
+20
View File
@@ -28,6 +28,26 @@ def register_dependency(name: str):
return decorator
@register_dependency("aiter")
def get_aiter_include_paths() -> List[str]:
"""aiter's C++ headers, for ROCm kernels that must compute exactly what aiter
computes (hip_reduce.h, opus/opus.hpp)."""
aiter_root = _find_package_root("aiter")
if aiter_root is None:
raise RuntimeError(
"Cannot find the aiter package. Install aiter to get the required"
" headers for JIT compilation."
)
include = aiter_root.parent / "csrc" / "include"
if not (include / "hip_reduce.h").is_file():
raise RuntimeError(
f"aiter C++ headers not found under {include}; a kernel that includes"
" them cannot be built against this aiter installation."
)
return [str(include)]
@register_dependency("flashinfer")
def get_flashinfer_include_paths() -> List[str]:
include_paths: List[str] = []
@@ -0,0 +1,13 @@
"""gfx950 fused DSA indexer decode path: four raw-HIP kernels replacing the
12-launch ROCm chain. Enabled wherever supported; the gate is
``utils.gfx950_fused_indexer_runtime_ok``."""
from sglang.kernels.ops.attention.dsa.hip_gfx950.fused_decode import ( # noqa: F401
CACHE_TOK_STRIDE,
MAX_ROWS,
PAGE_SIZE,
Gfx950FusedIndexer,
model_shape_supported,
prealloc_workspace,
supported_hardware,
)
@@ -0,0 +1,331 @@
"""Marshals production tensors into the four fused kernels' ABIs and owns the
workspace. Output matches aiter.dsa_topk_transform exactly: int32
physical page_size=1 slots, -1 padded, cache byte-compatible."""
from __future__ import annotations
import logging
from typing import Dict, Optional, Tuple
import torch
from sglang.kernels.jit.utils.common import is_hip_runtime
from sglang.kernels.ops.attention.dsa.hip_gfx950 import loader
logger = logging.getLogger(__name__)
_WARNED_TOO_NARROW = False
_WARNED_NO_WORKSPACE = False
# Kernel-source constraints. Every one of these is enforced by a RuntimeCheck or a
# compile-time constant in csrc/, not by taste:
HEAD_DIM = 128 # qk_rope_hadamard_quant.cuh / paged_mqa_logits.cuh
ROPE_DIM = 64 # qk_rope_hadamard_quant.cuh
N_HEADS = 32 # validated shape (grid.y of the qk kernel, MFMA n of the logits kernel)
INDEX_TOPK = 2048 # topk_transform.cuh: constexpr TOPK = 2048u
PAGE_SIZE = 64 # paged_mqa_logits.cuh: PAGE_TOK
CACHE_TOK_STRIDE = 132 # paged_mqa_logits.cuh: TOK_STRIDE (128 K bytes + 4 scale)
Q_LORA_RANK = 2048 # dual_gemv_bf16.cuh: Kq
HIDDEN_SIZE = 6144 # dual_gemv_bf16.cuh: Kk
QUANT_BLOCK = 128 # qk_rope_hadamard_quant.cuh: quant block == HEAD_DIM
MAX_ROWS = 48 # bs 8 x num_draft_tokens 6; dual_gemv_bf16.cuh kMaxRows matches it
def supported_hardware() -> bool:
"""Whether this device is one the kernels target, before asking whether the
shape fits or whether they build. They are raw gfx950: MFMA, wave64 DPP and
an e4m3fn fp8 cache throughout."""
if not is_hip_runtime() or not torch.cuda.is_available():
return False
return "gfx950" in torch.cuda.get_device_properties(0).gcnArchName
def model_shape_supported(
*,
head_dim: int,
rope_head_dim: int,
n_heads: int,
index_topk: int,
q_lora_rank: int,
hidden_size: int,
quant_block: int,
scale_fmt: Optional[str],
is_neox_style: bool,
k_norm,
num_init_tokens: int,
num_local_tokens: int,
) -> bool:
"""Static (per-model) half of the gate. Evaluated once in ``Indexer.__init__``."""
from sglang.srt.layers.layernorm import LayerNorm
return (
supported_hardware()
and head_dim == HEAD_DIM
and rope_head_dim == ROPE_DIM
and n_heads == N_HEADS
and index_topk == INDEX_TOPK
and q_lora_rank == Q_LORA_RANK
and hidden_size == HIDDEN_SIZE
and quant_block == QUANT_BLOCK
and scale_fmt == "ue8m0"
# GLM-5.2 sets indexer_rope_interleave=True -> is_neox_style False, which
# is what the qk kernel's interleaved rope implements.
and not is_neox_style
# The kernel does a LayerNorm with weight AND bias; RMSNorm has no bias
# and different math (index_k_norm_type == "rms").
and isinstance(k_norm, LayerNorm)
# _mask_init_and_local_tokens must be a no-op: the fused pipeline has no
# place to inject the +inf streaming-attention mask between B and C.
and num_init_tokens == 0
and num_local_tokens == 0
)
class _Workspace:
"""Persistent buffers, allocated once outside any capture at the maximum size
the process can ask for, so a captured graph never records a graph-private
pointer and no later call can force a reallocation."""
def __init__(self, *, device, max_cols: int, fp8_dtype):
rows = MAX_ROWS
cap = max(INDEX_TOPK, max_cols)
gh_stride = loader.modules()[3].hist_stride()
self.max_cols = max_cols
self.logits = torch.empty((rows, max_cols), dtype=torch.float32, device=device)
self.q_fp8 = torch.empty(
(rows, N_HEADS, HEAD_DIM), dtype=fp8_dtype, device=device
)
self.head_gate = torch.empty(
(rows, N_HEADS), dtype=torch.float32, device=device
)
self.q_proj = torch.empty(
(rows, N_HEADS * HEAD_DIM), dtype=torch.bfloat16, device=device
)
self.kw = torch.empty(
(rows, HEAD_DIM + N_HEADS), dtype=torch.bfloat16, device=device
)
self.ghist = torch.zeros((rows, gh_stride), dtype=torch.int32, device=device)
self.cursor = torch.zeros((rows, 32), dtype=torch.int32, device=device)
self.cand_idx = torch.empty((rows, cap), dtype=torch.int32, device=device)
self.cand_val = torch.empty((rows, cap), dtype=torch.float32, device=device)
# Row-expanded page table: metadata is per request, the kernels index
# per row. Flat so a narrower width still views contiguously.
self.page_table = torch.empty(
(rows * (max_cols // PAGE_SIZE),), dtype=torch.int32, device=device
)
# One workspace per (device, fp8 dtype), shared by all layers: the buffers are
# scratch, so per-layer copies would multiply the footprint by 79.
_WORKSPACES: Dict[Tuple[int, torch.dtype], _Workspace] = {}
def _workspace_key(device, fp8_dtype) -> Tuple[int, torch.dtype]:
index = device.index if isinstance(device, torch.device) else device
if index is None:
index = torch.cuda.current_device()
return (int(index), fp8_dtype)
def prealloc_workspace(*, device, max_cols: int, fp8_dtype) -> bool:
"""Create the shared workspace before the server sizes its memory pools, which
divide whatever mem_get_info reports at that moment. Idempotent; returns
False if one exists."""
key = _workspace_key(device, fp8_dtype)
if key in _WORKSPACES:
return False
_WORKSPACES[key] = _Workspace(device=device, max_cols=max_cols, fp8_dtype=fp8_dtype)
logger.info(
"gfx950 fused DSA indexer: workspace preallocated (max_cols=%d, "
"%.0f MiB) before pool sizing",
max_cols,
sum(
t.numel() * t.element_size()
for t in vars(_WORKSPACES[key]).values()
if isinstance(t, torch.Tensor)
)
/ (1 << 20),
)
return True
class Gfx950FusedIndexer:
"""Per-``Indexer`` state: merged weights, fp32 norm params, 2-D rope cache."""
def __init__(self, indexer):
self.indexer = indexer
self.workspace: Optional[_Workspace] = None
self._const = None
# softmax_scale * n_heads**-0.5, folded exactly as _get_logits_head_gate
# does (weights * n_heads**-0.5, then * q_scale * softmax_scale).
self.weights_scale = float(indexer.n_heads) ** -0.5 * indexer.softmax_scale
# -- setup-time derived constants (weights are frozen after load) ---------
#
# Derived lazily on purpose: this object is built in Indexer.__init__, which
# runs before load_weights, so nothing here can be hoisted into __init__.
# The first decode is after both weight load and the pre-capture warmup, so
# the torch.cat below is never recorded into a CUDA graph.
#
# Known limitation: w_kw / norm_w / norm_b are copies taken once. An
# in-place weight update afterwards (RL weight sync writes through
# param.data.copy_, keeping the same storage) is not observed here, while
# w_q_b is a view and does track it -- so a reload would leave the two
# halves of the same GEMV on different weights. There is no per-module
# update hook to subscribe to, so a deployment that hot-swaps weights must
# run with --no-enable-dsa-fused-indexer.
def _constants(self):
if self._const is None:
ix = self.indexer
# L1: [wk ; weights_proj] -> one [160, 6144] bf16 weight. A pure
# weight-loader style merge: both are ReplicatedLinear over the same
# x, so one 6144->160 GEMM is arithmetically the two GEMMs.
w_kw = torch.cat(
[ix.wk.weight.detach(), ix.weights_proj.weight.detach()], dim=0
).contiguous()
assert w_kw.dtype == torch.bfloat16, w_kw.dtype
w_q_b = ix.wq_b.weight.detach()
assert w_q_b.dtype == torch.bfloat16 and w_q_b.is_contiguous()
norm_w = ix.k_norm.weight.detach().float().contiguous()
norm_b = ix.k_norm.bias.detach().float().contiguous()
cos2d, sin2d = _rope_2d(ix.rotary_emb, ROPE_DIM)
self._const = (w_q_b, w_kw, norm_w, norm_b, cos2d, sin2d)
return self._const
def ensure_workspace(self, *, device, max_cols: int, fp8_dtype) -> bool:
"""Bind the preallocated workspace; False if there is none or it is narrower
than this call's page table. Never allocates: Indexer.__init__ owns that."""
ws = _WORKSPACES.get(_workspace_key(device, fp8_dtype))
if ws is None:
# Say so once, for the same reason as the width refusal below.
global _WARNED_NO_WORKSPACE
if not _WARNED_NO_WORKSPACE:
_WARNED_NO_WORKSPACE = True
logger.warning(
"gfx950 fused DSA indexer: no workspace was preallocated, so "
"the standard indexer runs instead."
)
return False
self.workspace = ws
if max_cols > ws.max_cols:
# Say so once. This refusal turns the whole feature off for the
# rest of the run while every other log line still reports it as
# enabled, so an A/B against it silently measures two identical arms.
global _WARNED_TOO_NARROW
if not _WARNED_TOO_NARROW:
_WARNED_TOO_NARROW = True
logger.warning(
"gfx950 fused DSA indexer: workspace is %d columns wide "
"but this call needs %d, so the standard indexer runs "
"instead. The workspace must cover the decode graph's "
"page-table width.",
ws.max_cols,
max_cols,
)
return False
return True
# -- the four launches ---------------------------------------------------
def run(
self,
*,
x: torch.Tensor,
q_lora: torch.Tensor,
positions: torch.Tensor,
out_cache_loc: torch.Tensor,
kv_cache: torch.Tensor,
kv_cache_read: torch.Tensor,
seqlens_int32: torch.Tensor,
row_ends_int32: torch.Tensor,
page_table_64: torch.Tensor,
rows: int,
) -> torch.Tensor:
gemv, qk, logits_mod, topk_mod = loader.modules()
w_q_b, w_kw, norm_w, norm_b, cos2d, sin2d = self._constants()
ws = self.workspace
assert ws is not None
cols = page_table_64.shape[1] * PAGE_SIZE
q_proj = ws.q_proj[:rows]
kw = ws.kw[:rows]
q_fp8 = ws.q_fp8[:rows]
head_gate = ws.head_gate[:rows]
logits = ws.logits[:rows, :cols]
# node 1: oq = q_lora @ wq_b.T and ok = x @ [wk ; weights_proj].T, one launch.
gemv.dual_gemv_bf16(q_lora[:rows], w_q_b, q_proj, x[:rows], w_kw, kw)
# node 2: k_norm | rope | Hadamard(q,k) | act_quant(q) |
# indexer_k_quant_and_cache(k) | head gate. The Hadamard is inline, so
# the gate checks the standard path applies it too (hadamard_preserved).
qk.indexer_qk_rope_hadamard_quant_and_cache(
q_proj.view(rows, N_HEADS, HEAD_DIM),
q_fp8,
kw[:, HEAD_DIM:],
head_gate,
kw[:, :HEAD_DIM],
kv_cache,
out_cache_loc,
norm_w,
norm_b,
positions,
cos2d,
sin2d,
float(self.indexer.k_norm.variance_epsilon),
float(self.weights_scale),
)
# Allocated before node 3, not between the two: logits_hist dirties the
# histogram, and an OOM in between would leave it dirty for every later call.
out = torch.empty((rows, INDEX_TOPK), dtype=torch.int32, device=logits.device)
# --- node 3: paged MQA fp8 logits + fine histogram + coarse summary --
logits_mod.logits_hist(
q_fp8,
kv_cache_read,
head_gate,
seqlens_int32,
page_table_64,
logits,
ws.ghist[:rows],
loader.LOGITS_BLOCKS_PER_ROW,
)
# --- node 4: top-k(2048) + page transform, fused refinement ---------
topk_mod.topk_transform(
logits,
row_ends_int32,
page_table_64,
out,
ws.ghist[:rows],
ws.cursor[:rows],
ws.cand_idx[:rows],
ws.cand_val[:rows],
loader.TOPK_G,
PAGE_SIZE,
)
return out
def _rope_2d(rotary_emb, rope_dim: int):
"""cos/sin as 2-D [max_pos, rope_dim // 2] bf16 with unit last stride. Both
halves are strided views, so contiguous() copies; the pair is cached on the
shared rope module rather than copied per layer."""
half = rope_dim // 2
cached = getattr(rotary_emb, "_dsa_fused_rope_2d", None)
if cached is not None and cached[0].shape[1] == half:
return cached
cos = getattr(rotary_emb, "cos_cache", None)
sin = getattr(rotary_emb, "sin_cache", None)
if cos is None or sin is None:
cache = rotary_emb.cos_sin_cache
cos, sin = cache[..., :half], cache[..., half:]
cos = cos.reshape(cos.shape[0], half).contiguous()
sin = sin.reshape(sin.shape[0], half).contiguous()
assert cos.dtype == torch.bfloat16 and sin.dtype == torch.bfloat16, (
f"gfx950 fused indexer needs a bf16 rope cache, got {cos.dtype}/{sin.dtype}"
)
rotary_emb._dsa_fused_rope_2d = (cos, sin)
return cos, sin
@@ -0,0 +1,93 @@
"""JIT build + load of the four gfx950 DSA-indexer kernels, on the first gated
call -- never in a timed region or a graph capture.
Four modules rather than one because the flags differ per kernel: the logits
kernel wants its own -mllvm set, the qk kernel needs aiter's headers.
"""
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Optional, Tuple
from sglang.kernels.jit.utils import cache_once, load_jit
if TYPE_CHECKING:
from tvm_ffi.module import Module
logger = logging.getLogger(__name__)
# The accepted configuration of each kernel. Changing one of these means
# editing the kernel it belongs to; the sources reject any other value rather
# than mis-launching.
LOGITS_HIST_BITS = 12 # must match logits_hist_m<8, 12> and topk_transform.cuh
LOGITS_BLOCKS_PER_ROW = 48
TOPK_G = 64
@cache_once
def modules() -> Tuple[Module, Module, Module, Module]:
"""(gemv, qk, logits, topk) modules, or raise. qk goes early: it is the one
that can fail for a reason outside this tree (a missing or stale aiter)."""
gemv = load_jit(
"dsa_gfx950_dual_gemv",
cuda_files=["dsa_gfx950/dual_gemv_bf16.cuh"],
cuda_wrappers=[("dual_gemv_bf16", "DualGemvBf16Kernel::run")],
extra_cuda_cflags=["-O3"],
)
# Includes aiter's hip_reduce.h and opus.hpp, so the numeric helpers are the
# same code aiter's own fused indexer uses. opus.hpp needs the C++20 the JIT
# defaults already pass.
qk = load_jit(
"dsa_gfx950_qk",
cuda_files=["dsa_gfx950/qk_rope_hadamard_quant.cuh"],
cuda_wrappers=[
(
"indexer_qk_rope_hadamard_quant_and_cache",
"QkRopeHadamardQuantKernel::run",
)
],
extra_cuda_cflags=["-O3", "-Wno-unused-result"],
extra_dependencies=["aiter"],
)
logits = load_jit(
"dsa_gfx950_logits",
cuda_files=["dsa_gfx950/paged_mqa_logits.cuh"],
cuda_wrappers=[("logits_hist", "PagedMqaLogitsKernel::run")],
extra_cuda_cflags=[
"-O3",
"-fno-honor-nans",
"-mllvm",
"-amdgpu-mfma-vgpr-form",
"-mllvm",
"-amdgpu-early-inline-all=true",
"-mllvm",
"-amdgpu-function-calls=false",
"-Wno-unused-result",
],
)
# The width is a module argument as well as a -D. The -D already keys the
# cache; the argument keeps two widths in separate modules, so their exports
# cannot collide in one process.
topk = load_jit(
"dsa_gfx950_topk",
f"hist_bits{LOGITS_HIST_BITS}",
cuda_files=["dsa_gfx950/topk_transform.cuh"],
cuda_wrappers=[
("topk_transform", "TopKTransformKernel::run"),
("hist_stride", "TopKHistStride::run"),
],
extra_cuda_cflags=["-O3", f"-DDSA_TOPK_HIST_BITS={LOGITS_HIST_BITS}"],
)
return gemv, qk, logits, topk
@cache_once
def modules_or_none() -> Optional[Tuple[Module, ...]]:
"""Build once; on failure log and return None. Whether the caller falls back
or refuses is the gate's decision, so this does not claim either."""
try:
return modules()
except Exception as e: # noqa: BLE001 - a build failure must never be fatal
logger.warning("gfx950 fused DSA indexer unavailable, build failed: %s", e)
return None
@@ -326,6 +326,19 @@ class ExecKernel(msgspec.Struct):
bool,
"Enable the experimental FP4 C4 indexer path for DeepSeek V4. Default keeps the existing indexer implementation.",
] = False
enable_dsa_fused_indexer: A[
Optional[bool],
Arg(
help="Use the four-kernel fused DSA indexer decode path in place of "
"the 12-launch aiter/torch chain. By default this is enabled "
"wherever it is supported -- gfx950 (MI355X) with aiter preshuffle "
"and an fp8 e4m3fn index cache -- and the runtime declines with a "
"logged reason wherever any of that is missing. Pass "
"--no-enable-dsa-fused-indexer to force the standard path.",
action=argparse.BooleanOptionalAction,
resolvable=True,
),
] = None
class ExecMamba(msgspec.Struct):
@@ -34,6 +34,9 @@ from sglang.srt.layers.attention.dsa.paged_mqa_logits_backend import (
)
from sglang.srt.layers.attention.dsa.utils import (
aiter_can_use_preshuffle_paged_mqa,
gfx950_fused_indexer_runtime_ok,
gfx950_model_shape_supported,
hadamard_preserved,
is_dsa_enable_prefill_cp,
is_graph_dsa_split_op_surface,
)
@@ -80,6 +83,21 @@ _FUSED_FP8_WRITER_LOGGED = False
_is_cuda = is_cuda()
_is_hip = is_hip()
# Every per-call decline in _gfx950_fused_decode falls through to the standard
# path, which is silent by construction. Without a trace, a config that turns
# the feature off -- rows past the row cap is the easy one to hit -- looks
# identical to one where it is on, and an A/B measures two identical arms.
_FUSED_DECLINE_LOGGED: set = set()
def _decline_fused(reason: str) -> None:
"""Log the first occurrence of each distinct decline reason, then stay quiet."""
if reason not in _FUSED_DECLINE_LOGGED:
_FUSED_DECLINE_LOGGED.add(reason)
logger.info("gfx950 fused DSA indexer declined this call: %s", reason)
_is_npu = is_npu()
_is_xpu = is_xpu()
@@ -377,10 +395,6 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
"using the legacy ROCm indexer writer"
)
elif self.use_aiter_fused_fp8_writer:
global _FUSED_FP8_WRITER_LOGGED
if not _FUSED_FP8_WRITER_LOGGED:
logger.info("Enabled AITER fused FP8 DSA indexer writer")
_FUSED_FP8_WRITER_LOGGED = True
self._fused_fp8_rope_cache_key = None
self._fused_fp8_cos_cache = None
self._fused_fp8_sin_cache = None
@@ -394,6 +408,114 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
get_exec().kernel.dsa_paged_mqa_logits_backend
)
# gfx950 fused decode indexer. All three conditions below must hold; the
# runtime half is dsa/utils.gfx950_fused_indexer_runtime_ok.
gfx950_shape_ok = gfx950_model_shape_supported(
head_dim=self.head_dim,
rope_head_dim=self.rope_head_dim,
n_heads=self.n_heads,
index_topk=self.index_topk,
q_lora_rank=self.q_lora_rank,
hidden_size=self.hidden_size,
quant_block=self.block_size,
scale_fmt=self.scale_fmt,
is_neox_style=is_neox_style,
k_norm=self.k_norm,
num_init_tokens=self.num_init_tokens,
num_local_tokens=self.num_local_tokens,
)
# The fused GEMV pre-merges [wk ; weights_proj] into one bf16 weight,
# so both must be bf16; a checkpoint that quantises one of them cannot
# take this path. Resolve every module defensively: use_dsa_indexer_fusion
# builds wk_weights_proj and leaves wk/weights_proj undefined, and a
# quantised checkpoint names its parameter something other than `weight`,
# so neither the modules nor their `.weight` can be assumed to exist.
wk_w = getattr(getattr(self, "wk", None), "weight", None)
wp_w = getattr(getattr(self, "weights_proj", None), "weight", None)
wq_b_w = getattr(getattr(self, "wq_b", None), "weight", None)
gfx950_weights_ok = (
wk_w is not None
and wp_w is not None
and wq_b_w is not None
and wk_w.dtype == torch.bfloat16
and wp_w.dtype == torch.bfloat16
and wq_b_w.dtype == torch.bfloat16
)
if gfx950_shape_ok and not gfx950_weights_ok:
logger.info(
"gfx950 fused DSA indexer disabled: needs bf16 wk/weights_proj/"
"wq_b, got %s/%s/%s",
getattr(wk_w, "dtype", None),
getattr(wp_w, "dtype", None),
getattr(wq_b_w, "dtype", None),
)
# Order matters: gfx950_fused_indexer_runtime_ok() ends by JIT-building
# four HIP extensions, so it goes last. Evaluated first it would compile
# them on every gfx950 server -- minutes, serialised across TP ranks on
# torch's build lock -- for models the shape gate then rejects anyway.
self.use_gfx950_fused_indexer = (
not self.use_dsa_indexer_fusion
and gfx950_shape_ok
and gfx950_weights_ok
and gfx950_fused_indexer_runtime_ok()
)
self._gfx950_fused = None
if (
# Only when asked for by name: unset, this path is a default.
get_exec().kernel.enable_dsa_fused_indexer is True
and not self.use_gfx950_fused_indexer
):
logger.info(
"gfx950 fused DSA indexer not used on this layer: "
"dsa_indexer_fusion=%s shape_supported=%s weights_bf16=%s",
self.use_dsa_indexer_fusion,
gfx950_shape_ok,
gfx950_weights_ok,
)
if self.use_gfx950_fused_indexer:
# The fused kernels write Hadamard-rotated index-K, and prefill must
# write the same format; the AITER fused FP8 writer skips the rotation.
self.use_aiter_fused_fp8_writer = False
self.k_norm = LayerNorm(self.head_dim, dtype=torch.bfloat16)
# Off rather than fatal: without the fused path both phases share a format.
self.use_gfx950_fused_indexer = hadamard_preserved(self)
if self.use_gfx950_fused_indexer:
from sglang.kernels.ops.attention.dsa.hip_gfx950 import (
Gfx950FusedIndexer,
prealloc_workspace,
)
self._gfx950_fused = Gfx950FusedIndexer(self)
# Take the workspace here, while weights load, so calculate_pool_sizes still
# sees the real free memory. Deferring it left the pools sized against memory
# that was about to be taken.
_PAGE_SLACK = 4 # pages of headroom over the derived width
max_ctx = getattr(config, "max_position_embeddings", 0) or 0
extra = 4
try:
from sglang.srt.runtime_context import get_spec
if get_spec().speculative_num_draft_tokens is not None:
extra += int(get_spec().speculative_num_draft_tokens)
except Exception: # spec config not available: the +4 still applies
pass
if max_ctx:
pages = -(-(max_ctx + extra) // 64) + _PAGE_SLACK
max_ctx = pages * 64
if max_ctx:
prealloc_workspace(
device=torch.device("cuda", torch.cuda.current_device()),
max_cols=max_ctx,
fp8_dtype=fp8_dtype,
)
# After the gfx950 fused indexer's gate, which may turn the writer off.
global _FUSED_FP8_WRITER_LOGGED
if self.use_aiter_fused_fp8_writer and not _FUSED_FP8_WRITER_LOGGED:
logger.info("Enabled AITER fused FP8 DSA indexer writer")
_FUSED_FP8_WRITER_LOGGED = True
@contextlib.contextmanager
def _with_real_sm_count(self):
# When pipeline parallelism is enabled, each PP rank initiates a recv operation after the _pp_launch_batch
@@ -558,6 +680,146 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
return x
return rotate_activation(x)
def _gfx950_fused_decode(
self,
x,
q_lora: torch.Tensor,
positions: torch.Tensor,
forward_batch: ForwardBatch,
layer_id: int,
metadata: Optional[BaseIndexerMetadata],
in_graph: bool,
) -> Optional[torch.Tensor]:
"""Per-call half of the gate plus the four launches. Returns the (rows,
index_topk) physical-slot tensor, or None to mean this call is not served
here -- forward_cuda then continues into the standard path unchanged."""
from sglang.kernels.ops.attention.dsa.hip_gfx950 import (
CACHE_TOK_STRIDE,
MAX_ROWS,
PAGE_SIZE,
)
from sglang.srt.mem_cache.dsa_cache_layer_split import (
LayerSplitDSATokenToKVPool,
)
if metadata is None or in_graph:
return None
# draft-extend-v2 rows are an extend chunk, not one query per row.
if not (
forward_batch.forward_mode.is_decode_or_idle()
or forward_batch.forward_mode.is_target_verify()
):
return None
# These kernels emit physical page_size=1 slots, which is what the caller
# wants only while the fused top-k transform is in play. When the topk is
# forced unfused -- SGLANG_DSA_FUSE_TOPK=0, hisparse on decode, or a PD
# seed without local-slot remap -- the attention backend transforms the
# indexer's output itself and therefore expects logical positions, so
# handing it slots would index the page table with slot ids and corrupt
# the gather with no error anywhere.
if metadata.force_unfused_topk:
_decline_fused("the top-k transform is forced unfused")
return None
# CP gathers K outside the kernel, which this path writes itself.
if forward_batch.attn_cp_metadata is not None:
_decline_fused("context parallelism gathers K outside the kernel")
return None
# The fused GEMV takes bf16 hidden states, not the quantised tuple.
if isinstance(x, tuple) or x.dtype != torch.bfloat16:
_decline_fused("hidden states are not plain bf16")
return None
# LoRA owns base+delta; the weights this path consumes are base only --
# wk/weights_proj pre-merged into one bf16 tensor and wq_b taken straight
# from the module. All three are LoRA-targetable (lora/utils.py maps
# indexer.wq_b alongside indexer.wk and indexer.weights_proj), so an
# adapter on any of them has to fall through to the standard path.
if any(
getattr(getattr(self, name, None), "set_lora", False)
for name in ("weights_proj", "wk", "wq_b")
):
_decline_fused("a LoRA adapter owns wk / weights_proj / wq_b")
return None
pool = get_token_to_kv_pool()
if pool.page_size != PAGE_SIZE:
_decline_fused(f"pool page_size is {pool.page_size}, not {PAGE_SIZE}")
return None
# DSA cache layer split: the standard path stores index-K per layer, and the
# fused path addresses the same buffer directly.
if isinstance(pool, LayerSplitDSATokenToKVPool):
# Ordering, not ownership, is what rules this out. The read buffer is
# materialised before state.run() writes this step's K, and for a
# split pool materialising it *is* the owner broadcast (invalidate()
# clears remote_layer_id, so get_broadcastable_buffer copies straight
# away). The current token would therefore be missing from the very
# logits that rank it. The standard path stores first and reads
# after; matching that here means splitting run() in two, so decline
# instead -- this combination is untested for this path anyway.
_decline_fused("the DSA cache layer-split pool needs store-then-read")
return None
rows = x.shape[0]
if not (1 <= rows <= MAX_ROWS):
_decline_fused(f"row count is outside [1, {MAX_ROWS}]")
return None
# Per-token lengths; decode aliases this to the per-request tensor.
seqlens = metadata.get_seqlens_expanded()
page_table_64 = metadata.get_page_table_64()
if seqlens.shape[0] != rows:
return None
if page_table_64 is None or page_table_64.shape[0] == 0:
return None
if not page_table_64.is_contiguous() or page_table_64.dtype != torch.int32:
return None
out_cache_loc = forward_batch.out_cache_loc
if (
out_cache_loc.dtype != torch.int64
or positions.dtype != torch.int64
or out_cache_loc.shape[0] < rows
or positions.shape[0] < rows
or not out_cache_loc.is_contiguous()
or not positions.is_contiguous()
):
return None
state = self._gfx950_fused
if not state.ensure_workspace(
device=x.device,
max_cols=page_table_64.shape[1] * PAGE_SIZE,
fp8_dtype=fp8_dtype,
):
return None
# Draft rows share their request's table; broadcast into the
# preallocated buffer, since no allocation is legal under capture.
n_pt, blocks = page_table_64.shape
if n_pt < rows:
if rows % n_pt or rows * blocks > state.workspace.page_table.numel():
return None
dst = state.workspace.page_table[: rows * blocks].view(rows, blocks)
dst.view(n_pt, rows // n_pt, blocks).copy_(page_table_64.unsqueeze(1))
page_table_64 = dst
else:
page_table_64 = page_table_64[:rows]
if hasattr(pool, "invalidate_index_buffer_for_layer"):
pool.invalidate_index_buffer_for_layer(layer_id)
# The owned/read buffer split the standard path uses.
kv_write = pool.get_index_k_with_scale_buffer(layer_id=layer_id).view(fp8_dtype)
kv_read = self._get_index_k_read_buffer(pool, layer_id).view(fp8_dtype)
return state.run(
x=x,
q_lora=q_lora,
positions=positions[:rows],
out_cache_loc=out_cache_loc[:rows],
kv_cache=kv_write.view(-1, PAGE_SIZE, CACHE_TOK_STRIDE),
kv_cache_read=kv_read,
seqlens_int32=seqlens,
row_ends_int32=seqlens,
page_table_64=page_table_64,
rows=rows,
)
def _should_skip_logits_computation(self, forward_batch: ForwardBatch) -> bool:
# topk_transform selects every valid page slot when kv_len <= index_topk;
# logits are unnecessary in that case.
@@ -1691,6 +1953,25 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
topk_result = _broadcast_indexer_topk_from_rank0(topk_result)
return maybe_capture_indexer_topk(layer_id, topk_result)
# gfx950 fused decode indexer (4 kernels). Returns None -- and touches
# nothing -- whenever any part of its gate is false, so the standard path
# below is reached completely unchanged.
if self.use_gfx950_fused_indexer:
fused_topk = self._gfx950_fused_decode(
x,
q_lora,
positions,
forward_batch,
layer_id,
metadata,
in_piecewise_or_breakable_cuda_graph,
)
if fused_topk is not None:
fused_topk = _broadcast_indexer_topk_from_rank0(
fused_topk if return_indices else None
)
return maybe_capture_indexer_topk(layer_id, fused_topk)
# When weights_proj is LoRA-wrapped, use an eager module call so the
# wrapper owns base+delta and no LoRA kernel runs under torch.compile.
# Fusion folds weights_proj into wk_weights_proj, so weights_proj is
@@ -1,3 +1,4 @@
import logging
from functools import lru_cache
from typing import TYPE_CHECKING, Optional
@@ -22,6 +23,8 @@ from sglang.srt.runtime_context import (
from sglang.srt.utils import get_bool_env_var, is_cuda, is_hip, is_musa
from sglang.srt.utils.common import ceil_div
logger = logging.getLogger(__name__)
@lru_cache(maxsize=1)
def aiter_can_use_preshuffle_paged_mqa() -> bool:
@@ -61,6 +64,82 @@ def aiter_can_use_preshuffle_paged_mqa() -> bool:
return False
@lru_cache(maxsize=1)
def gfx950_fused_indexer_runtime_ok() -> bool:
"""Whether this runtime can serve the gfx950 fused indexer: aiter with
preshuffled paged MQA, and kernels that build.
Reached only on gfx950, since fused_decode.supported_hardware() is evaluated
first. Every decline here is therefore a configuration or toolchain error;
it is logged, as a warning when the path was asked for by name."""
from sglang.srt.runtime_context import get_exec
requested = get_exec().kernel.enable_dsa_fused_indexer
if requested is False:
return False # asked for the standard path; not worth a line per server
# Every decline logs its reason. Without this the path is invisible: a run
# with the switch on and one with it off produce identical logs, and telling
# the two apart cost a day of bisecting benchmark results.
def _refuse(reason: str) -> bool:
# Asked for by name: warn, but still start on the standard path.
log = logger.warning if requested is True else logger.info
log("gfx950 fused DSA indexer disabled: %s", reason)
return False
# No hardware term here: fused_decode.supported_hardware() is the hardware
# half of the gate and runs first, so anything reaching this point is
# already on gfx950. What is left is what a deployment can get wrong.
if not get_bool_env_var("SGLANG_USE_AITER"):
return _refuse("SGLANG_USE_AITER is not set")
if not aiter_can_use_preshuffle_paged_mqa():
return _refuse("aiter cannot use preshuffled paged MQA logits")
from sglang.kernels.ops.attention.dsa.hip_gfx950 import loader
# modules_or_none logged the build error; do not repeat the compiler output.
if loader.modules_or_none() is None:
return _refuse("the kernels failed to build, see the warning above")
logger.info("gfx950 fused DSA indexer enabled")
return True
def gfx950_model_shape_supported(**kwargs) -> bool:
"""Static per-model half of the gate: shapes and dtypes that cannot change
after load."""
from sglang.kernels.ops.attention.dsa.hip_gfx950 import model_shape_supported
return model_shape_supported(**kwargs)
def hadamard_preserved(indexer) -> bool:
"""Whether Indexer._maybe_rotate still applies the Hadamard the fused kernels
fold in. If not, the fused path must stay off, or prefill and decode would
write different index-K formats."""
device = indexer.k_norm.weight.device
probe = torch.zeros(1, indexer.head_dim, dtype=torch.bfloat16, device=device)
probe[0, 0] = 1.0
rotated = indexer._maybe_rotate(probe)
# A 128-point Hadamard sends e_0 to a vector whose every entry is 128**-0.5;
# the identity leaves 127 zeros. Check the magnitude too, so a transform that
# is merely dense does not pass for the rotation the kernels assume.
expected = float(indexer.head_dim) ** -0.5
if not bool(
(rotated != 0).all()
and torch.allclose(
rotated.float().abs(),
torch.full_like(rotated.float(), expected),
rtol=0.05,
atol=0.0,
)
):
logger.warning(
"gfx950 fused DSA indexer disabled: Indexer._maybe_rotate does not "
"apply the Hadamard rotation the fused kernels assume"
)
return False
return True
# Tile size for the indexer FP8 K-cache preshuffle layout. Store and gather
# kernels reorganize each page into (tile x tile) blocks so the aiter preshuffle
# paged-MQA gather can consume the cache directly.