[ROCm] GLM-5.2: gfx950 four-kernel fused DSA indexer decode path (#38583)
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
be66c1b460
commit
964fd6af92
@@ -0,0 +1,344 @@
|
||||
/// Dual bf16 GEMV for the gfx950 DSA indexer: Q [M,4096] and KW [M,160] in one launch.
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifndef USE_ROCM
|
||||
#error "dual_gemv_bf16.cuh targets CDNA3/4; it uses MFMA and wave64 v_dot2c"
|
||||
#endif
|
||||
|
||||
#include <sgl_kernel/tensor.h>
|
||||
#include <sgl_kernel/utils.h>
|
||||
|
||||
#include <sgl_kernel/runtime.cuh>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
|
||||
#include <tvm/ffi/container/tensor.h>
|
||||
|
||||
namespace sglang {
|
||||
|
||||
// Local bf16_t is the builtin __bf16 that ext_vector_type needs, not sglang::bf16_t.
|
||||
namespace dsa_gfx950::dual_gemv {
|
||||
|
||||
using namespace ::sglang::host;
|
||||
|
||||
typedef unsigned short ushort_t;
|
||||
typedef __bf16 bf16_t;
|
||||
typedef bf16_t bf16x2_t __attribute__((ext_vector_type(2)));
|
||||
typedef bf16_t bf16x8_t __attribute__((ext_vector_type(8)));
|
||||
typedef float float4_t __attribute__((ext_vector_type(4)));
|
||||
|
||||
constexpr int kThreads = 256;
|
||||
constexpr int kKq = 2048;
|
||||
constexpr int kKk = 6144;
|
||||
constexpr int kMaxRows = 48; // fused_decode.MAX_ROWS
|
||||
constexpr int kQCols = 16;
|
||||
constexpr int kQSteps = kKq / 128;
|
||||
constexpr int kQStageRows = 16;
|
||||
constexpr int kQStageLd = kThreads + 1; // 16-byte pad per row against LDS bank conflicts
|
||||
constexpr int kKwVecs = kKk / (8 * kThreads);
|
||||
|
||||
__device__ __forceinline__ float dot2_bf16(unsigned int x, unsigned int w, float acc) {
|
||||
return __builtin_amdgcn_fdot2_f32_bf16(__builtin_bit_cast(bf16x2_t, x), __builtin_bit_cast(bf16x2_t, w), acc, false);
|
||||
}
|
||||
|
||||
// round-to-nearest-even, matching torch's fp32 -> bf16 cast
|
||||
__device__ __forceinline__ ushort_t f32_to_bf16_rne(float f) {
|
||||
unsigned int u = __float_as_uint(f);
|
||||
if ((u & 0x7fffffffu) > 0x7f800000u) { // NaN -> quiet NaN
|
||||
return static_cast<ushort_t>((u >> 16) | 0x0040u);
|
||||
}
|
||||
unsigned int lsb = (u >> 16) & 1u;
|
||||
u += 0x7fffu + lsb;
|
||||
return static_cast<ushort_t>(u >> 16);
|
||||
}
|
||||
|
||||
// K offset of MFMA step s, in hipBLASLt MT16x16x512's reduction order so Q matches it bit for bit.
|
||||
__device__ __forceinline__ int q_step_k(int s) {
|
||||
return (s >> 2) * 512 + (s & 3) * 32;
|
||||
}
|
||||
|
||||
// 16 Q columns for all M rows; the weights load once and the first 16 x rows go through LDS.
|
||||
__device__ __forceinline__ void gemv_q(
|
||||
const bf16_t* __restrict__ x,
|
||||
const bf16_t* __restrict__ w,
|
||||
ushort_t* __restrict__ O,
|
||||
long ldo,
|
||||
int M,
|
||||
int col0,
|
||||
float* __restrict__ smem) {
|
||||
const int wave = threadIdx.x >> 6;
|
||||
const int lane = threadIdx.x & 63;
|
||||
const int frag = lane & 15;
|
||||
const int kofs = wave * 128 + (lane >> 4) * 8;
|
||||
const bf16_t* wp = w + (long)(col0 + frag) * kKq + kofs;
|
||||
const bf16_t* xp = x + (long)frag * kKq + kofs;
|
||||
bf16x8_t a[kQSteps];
|
||||
bf16x8_t b[kQSteps];
|
||||
{
|
||||
bf16x8_t xv[kQStageRows];
|
||||
#pragma unroll
|
||||
for (int r = 0; r < kQStageRows; ++r)
|
||||
if (r < M) xv[r] = *reinterpret_cast<const bf16x8_t*>(x + (long)r * kKq + threadIdx.x * 8);
|
||||
#pragma unroll
|
||||
for (int s = 0; s < kQSteps; ++s)
|
||||
b[s] = *reinterpret_cast<const bf16x8_t*>(wp + q_step_k(s));
|
||||
bf16x8_t* xs = reinterpret_cast<bf16x8_t*>(smem);
|
||||
#pragma unroll
|
||||
for (int r = 0; r < kQStageRows; ++r)
|
||||
if (r < M) xs[r * kQStageLd + threadIdx.x] = xv[r];
|
||||
__syncthreads();
|
||||
#pragma unroll
|
||||
for (int s = 0; s < kQSteps; ++s)
|
||||
a[s] = frag < M ? xs[frag * kQStageLd + (kofs + q_step_k(s)) / 8] : bf16x8_t{};
|
||||
__syncthreads();
|
||||
}
|
||||
const int groups = (M + 15) >> 4;
|
||||
for (int g = 0; g < groups; ++g) {
|
||||
const bool more = g + 1 < groups;
|
||||
const bool next_valid = 16 * (g + 1) + frag < M;
|
||||
const bf16_t* xn = xp + (long)16 * (g + 1) * kKq;
|
||||
float4_t acc = {};
|
||||
#pragma unroll
|
||||
for (int s = 0; s < kQSteps; ++s) {
|
||||
acc = __builtin_amdgcn_mfma_f32_16x16x32_bf16(a[s], b[s], acc, 0, 0, 0);
|
||||
if (more) {
|
||||
a[s] = next_valid ? *reinterpret_cast<const bf16x8_t*>(xn + q_step_k(s)) : bf16x8_t{};
|
||||
}
|
||||
}
|
||||
// [group][wave][column][row], so the wave-order sum below reads stride 256.
|
||||
*reinterpret_cast<float4_t*>(smem + ((g * 4 + wave) * 16 + frag) * 16 + (lane >> 4) * 4) = acc;
|
||||
}
|
||||
__syncthreads();
|
||||
for (int i = threadIdx.x; i < M * kQCols; i += kThreads) {
|
||||
const int r = i >> 4;
|
||||
const int c = i & 15;
|
||||
const float* p = smem + ((r >> 4) * 64 + c) * 16 + (r & 15);
|
||||
float sum = p[0] + p[256];
|
||||
sum += p[512];
|
||||
sum += p[768];
|
||||
O[(long)r * ldo + col0 + c] = f32_to_bf16_rne(sum);
|
||||
}
|
||||
}
|
||||
|
||||
// NCOL KW columns for R rows from row0.
|
||||
template <int R, int NCOL>
|
||||
__device__ __forceinline__ void gemv_kw(
|
||||
const uint4* __restrict__ Xv,
|
||||
long ldxv,
|
||||
const uint4* __restrict__ Wv,
|
||||
long ldwv,
|
||||
ushort_t* __restrict__ O,
|
||||
long ldo,
|
||||
int M,
|
||||
int row0,
|
||||
int col0,
|
||||
int N,
|
||||
float* __restrict__ smem) {
|
||||
const int tid = threadIdx.x;
|
||||
unsigned int xr[R][kKwVecs][4];
|
||||
#pragma unroll
|
||||
for (int m = 0; m < R; ++m) {
|
||||
#pragma unroll
|
||||
for (int p = 0; p < kKwVecs; ++p) {
|
||||
uint4 t = {0u, 0u, 0u, 0u};
|
||||
if (row0 + m < M) t = Xv[(long)(row0 + m) * ldxv + tid + p * kThreads];
|
||||
xr[m][p][0] = t.x;
|
||||
xr[m][p][1] = t.y;
|
||||
xr[m][p][2] = t.z;
|
||||
xr[m][p][3] = t.w;
|
||||
}
|
||||
}
|
||||
uint4 wr[NCOL][kKwVecs];
|
||||
#pragma unroll
|
||||
for (int c = 0; c < NCOL; ++c) {
|
||||
const int col = col0 + c;
|
||||
// clamp instead of branch: out-of-range columns are computed and dropped
|
||||
const uint4* wp = Wv + (long)(col < N ? col : 0) * ldwv + tid;
|
||||
#pragma unroll
|
||||
for (int p = 0; p < kKwVecs; ++p) {
|
||||
wr[c][p] = wp[p * kThreads];
|
||||
}
|
||||
}
|
||||
float acc[NCOL][R] = {};
|
||||
#pragma unroll
|
||||
for (int c = 0; c < NCOL; ++c) {
|
||||
#pragma unroll
|
||||
for (int p = 0; p < kKwVecs; ++p) {
|
||||
unsigned int w4[4] = {wr[c][p].x, wr[c][p].y, wr[c][p].z, wr[c][p].w};
|
||||
#pragma unroll
|
||||
for (int e = 0; e < 4; ++e) {
|
||||
#pragma unroll
|
||||
for (int m = 0; m < R; ++m)
|
||||
acc[c][m] = dot2_bf16(xr[m][p][e], w4[e], acc[c][m]);
|
||||
}
|
||||
}
|
||||
}
|
||||
const int wave = tid >> 6;
|
||||
const int wlane = tid & 63;
|
||||
#pragma unroll
|
||||
for (int off = 32; off > 0; off >>= 1) {
|
||||
#pragma unroll
|
||||
for (int c = 0; c < NCOL; ++c)
|
||||
#pragma unroll
|
||||
for (int m = 0; m < R; ++m)
|
||||
acc[c][m] += __shfl_down(acc[c][m], off, 64);
|
||||
}
|
||||
if (wlane == 0) {
|
||||
#pragma unroll
|
||||
for (int c = 0; c < NCOL; ++c)
|
||||
#pragma unroll
|
||||
for (int m = 0; m < R; ++m)
|
||||
smem[(wave * NCOL + c) * R + m] = acc[c][m];
|
||||
}
|
||||
__syncthreads();
|
||||
if (tid < NCOL * R) {
|
||||
const int c = tid / R;
|
||||
const int m = tid - c * R;
|
||||
float s = 0.0f;
|
||||
#pragma unroll
|
||||
for (int w = 0; w < 4; ++w)
|
||||
s += smem[(w * NCOL + c) * R + m];
|
||||
const int col = col0 + c;
|
||||
if (col < N && row0 + m < M) O[(long)(row0 + m) * ldo + col] = f32_to_bf16_rne(s);
|
||||
}
|
||||
}
|
||||
|
||||
// The KW blocks lead the grid so their loads queue ahead of the Q weight stream.
|
||||
template <int R, int NCOL>
|
||||
__global__ __launch_bounds__(kThreads) void dual_gemv_kernel(
|
||||
const bf16_t* __restrict__ Xq,
|
||||
const bf16_t* __restrict__ Wq,
|
||||
ushort_t* __restrict__ Oq,
|
||||
long ldoq,
|
||||
int nblk_q,
|
||||
const uint4* __restrict__ Xk,
|
||||
long ldxk,
|
||||
const uint4* __restrict__ Wk,
|
||||
long ldwk,
|
||||
ushort_t* __restrict__ Ok,
|
||||
long ldok,
|
||||
int Nk,
|
||||
int nblk_kc,
|
||||
int M) {
|
||||
constexpr int kSmemQ = kQStageRows * kQStageLd * 4;
|
||||
static_assert(kSmemQ >= (kMaxRows / 16) * 4 * 16 * 16 && kSmemQ >= 4 * NCOL * R);
|
||||
__shared__ __attribute__((aligned(16))) float smem[kSmemQ];
|
||||
const int nblk_k = gridDim.x - nblk_q;
|
||||
int bid = blockIdx.x;
|
||||
bid = bid < nblk_k ? bid + nblk_q : bid - nblk_k;
|
||||
if (bid < nblk_q) {
|
||||
gemv_q(Xq, Wq, Oq, ldoq, M, bid * kQCols, smem);
|
||||
} else {
|
||||
const int b = bid - nblk_q;
|
||||
const int rc = b / nblk_kc;
|
||||
const int cb = b - rc * nblk_kc;
|
||||
gemv_kw<R, NCOL>(Xk, ldxk, Wk, ldwk, Ok, ldok, M, rc * R, cb * NCOL, Nk, smem);
|
||||
}
|
||||
}
|
||||
|
||||
struct Args {
|
||||
const bf16_t* Xq;
|
||||
const bf16_t* Wq;
|
||||
ushort_t* Oq;
|
||||
long ldoq;
|
||||
int Nq;
|
||||
const uint4* Xk;
|
||||
long ldxk;
|
||||
const uint4* Wk;
|
||||
long ldwk;
|
||||
ushort_t* Ok;
|
||||
long ldok;
|
||||
int Nk;
|
||||
int M;
|
||||
hipStream_t stream;
|
||||
};
|
||||
|
||||
template <int R, int NCOL>
|
||||
void launch(const Args& a) {
|
||||
const int nblk_q = a.Nq / kQCols;
|
||||
const int nblk_kc = (a.Nk + NCOL - 1) / NCOL;
|
||||
const int nblk_kr = (a.M + R - 1) / R;
|
||||
dual_gemv_kernel<R, NCOL><<<dim3(nblk_q + nblk_kc * nblk_kr), dim3(kThreads), 0, a.stream>>>(
|
||||
a.Xq, a.Wq, a.Oq, a.ldoq, nblk_q, a.Xk, a.ldxk, a.Wk, a.ldwk, a.Ok, a.ldok, a.Nk, nblk_kc, a.M);
|
||||
}
|
||||
|
||||
// KW block shape (rows x columns) per M range.
|
||||
inline void dispatch(const Args& a) {
|
||||
if (a.M == 1) {
|
||||
launch<1, 1>(a);
|
||||
} else if (a.M == 2) {
|
||||
launch<2, 1>(a);
|
||||
} else if (a.M <= 4) {
|
||||
launch<4, 1>(a);
|
||||
} else if (a.M <= 6) {
|
||||
launch<2, 2>(a);
|
||||
} else if (a.M <= 8) {
|
||||
launch<4, 2>(a);
|
||||
} else if (a.M <= 24) {
|
||||
launch<4, 4>(a);
|
||||
} else {
|
||||
launch<8, 4>(a);
|
||||
}
|
||||
}
|
||||
} // namespace dsa_gfx950::dual_gemv
|
||||
|
||||
struct DualGemvBf16Kernel {
|
||||
/// Oq = Xq @ Wq^T and Ok = Xk @ Wk^T, all bf16; outputs are overwritten.
|
||||
static void
|
||||
run(const tvm::ffi::TensorView Xq,
|
||||
const tvm::ffi::TensorView Wq,
|
||||
const tvm::ffi::TensorView Oq,
|
||||
const tvm::ffi::TensorView Xk,
|
||||
const tvm::ffi::TensorView Wk,
|
||||
const tvm::ffi::TensorView Ok) {
|
||||
using namespace host;
|
||||
namespace impl = dsa_gfx950::dual_gemv;
|
||||
|
||||
auto M_ = SymbolicSize{"num_rows"};
|
||||
auto Kq_ = SymbolicSize{"k_q"};
|
||||
auto Nq_ = SymbolicSize{"n_q"};
|
||||
auto Kk_ = SymbolicSize{"k_k"};
|
||||
auto Nk_ = SymbolicSize{"n_k"};
|
||||
auto device_ = SymbolicDevice{};
|
||||
device_.set_options<kDLCUDA>();
|
||||
|
||||
// Q reads x at row * K, so a padded Xq or Wq would silently read wrong rows.
|
||||
TensorMatcher({M_, Kq_}).with_dtype<bf16_t>().with_device(device_).with_strides({Kq_, 1}).verify(Xq);
|
||||
TensorMatcher({Nq_, Kq_}).with_dtype<bf16_t>().with_device(device_).with_strides({Kq_, 1}).verify(Wq);
|
||||
TensorMatcher({M_, Nq_}).with_dtype<bf16_t>().with_device(device_).with_strides({-1, 1}).verify(Oq);
|
||||
TensorMatcher({M_, Kk_}).with_dtype<bf16_t>().with_device(device_).with_strides({-1, 1}).verify(Xk);
|
||||
TensorMatcher({Nk_, Kk_}).with_dtype<bf16_t>().with_device(device_).with_strides({-1, 1}).verify(Wk);
|
||||
TensorMatcher({M_, Nk_}).with_dtype<bf16_t>().with_device(device_).with_strides({-1, 1}).verify(Ok);
|
||||
|
||||
const auto M = static_cast<int>(M_.unwrap());
|
||||
RuntimeCheck(M >= 1 && M <= impl::kMaxRows, "dual_gemv supports M in [1,", impl::kMaxRows, "], got ", M);
|
||||
const auto Kq = static_cast<int>(Kq_.unwrap());
|
||||
const auto Kk = static_cast<int>(Kk_.unwrap());
|
||||
RuntimeCheck(
|
||||
Kk == impl::kKk && Kq == impl::kKq, "dual_gemv is built for Kk=6144, Kq=2048; got Kk=", Kk, " Kq=", Kq);
|
||||
|
||||
impl::Args a;
|
||||
a.Xq = reinterpret_cast<const impl::bf16_t*>(Xq.data_ptr());
|
||||
a.Wq = reinterpret_cast<const impl::bf16_t*>(Wq.data_ptr());
|
||||
a.Oq = reinterpret_cast<impl::ushort_t*>(Oq.data_ptr());
|
||||
a.Nq = static_cast<int>(Nq_.unwrap());
|
||||
a.ldoq = Oq.stride(0);
|
||||
a.Xk = reinterpret_cast<const uint4*>(Xk.data_ptr());
|
||||
a.Wk = reinterpret_cast<const uint4*>(Wk.data_ptr());
|
||||
a.Ok = reinterpret_cast<impl::ushort_t*>(Ok.data_ptr());
|
||||
a.Nk = static_cast<int>(Nk_.unwrap());
|
||||
a.ldxk = Xk.stride(0) / 8;
|
||||
a.ldwk = Wk.stride(0) / 8;
|
||||
a.ldok = Ok.stride(0);
|
||||
a.M = M;
|
||||
a.stream = LaunchKernel::resolve_device(device_.unwrap());
|
||||
|
||||
RuntimeCheck(a.Nq % impl::kQCols == 0, "dual_gemv needs n_q % 16 == 0, got ", a.Nq);
|
||||
RuntimeCheck(Xk.stride(0) % 8 == 0 && Wk.stride(0) % 8 == 0, "the K operands must start on an 8-element boundary");
|
||||
|
||||
impl::dispatch(a);
|
||||
RuntimeDeviceCheck();
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace sglang
|
||||
@@ -0,0 +1,324 @@
|
||||
/// GLM-5.2 DSA indexer, the logits kernel: paged MQA FP8 logits plus the fine
|
||||
/// histogram and coarse summary topk_transform.cuh selects from. One kernel
|
||||
/// ships, logits_hist_m<8, 12>; WARPS and HB are template arguments.
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifndef USE_ROCM
|
||||
#error "paged_mqa_logits.cuh targets gfx950; it is raw CDNA4 MFMA and wave64"
|
||||
#endif
|
||||
|
||||
#include <sgl_kernel/tensor.h>
|
||||
#include <sgl_kernel/utils.h>
|
||||
|
||||
#include <sgl_kernel/runtime.cuh>
|
||||
#include <sgl_kernel/type.cuh>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
|
||||
#include <hip/hip_fp16.h>
|
||||
#include <hip/hip_runtime.h>
|
||||
#include <tvm/ffi/container/tensor.h>
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
namespace sglang {
|
||||
|
||||
namespace dsa_gfx950::logits {
|
||||
|
||||
typedef __attribute__((__vector_size__(8 * sizeof(int)))) int i32x8;
|
||||
typedef __attribute__((__vector_size__(4 * sizeof(int)))) int i32x4;
|
||||
typedef __attribute__((__vector_size__(4 * sizeof(float)))) float f32x4;
|
||||
|
||||
#define PAGE_TOK 64
|
||||
#define HD 128
|
||||
#define HEADS 32
|
||||
// topk_transform.cuh's TOPK: both kernels must agree on which rows get binned.
|
||||
#define LG_TOPK 2048
|
||||
#define TOK_STRIDE 132
|
||||
#define PAGE_BYTES (PAGE_TOK * TOK_STRIDE) // 8448
|
||||
#define K_BYTES (PAGE_TOK * HD) // 8192
|
||||
|
||||
union V8 {
|
||||
i32x8 v;
|
||||
i32x4 h[2];
|
||||
};
|
||||
|
||||
__device__ __forceinline__ f32x4 mfma128(i32x8 a, i32x8 b, f32x4 c) {
|
||||
return __builtin_amdgcn_mfma_scale_f32_16x16x128_f8f6f4(a, b, c, 0, 0, 0, 127, 0, 127);
|
||||
}
|
||||
|
||||
// wave64 lane exchanges on the VALU instead of through LDS:
|
||||
// xor32 <- v_permlane64_b32 (lane l reads lane l^32)
|
||||
// xor16 <- v_permlanex16_b32 with identity selectors (lane l reads lane l^16)
|
||||
typedef __attribute__((__vector_size__(2 * sizeof(int)))) int i32x2;
|
||||
__device__ __forceinline__ float xor32(float x, int lane) {
|
||||
int v = __builtin_bit_cast(int, x);
|
||||
i32x2 r = __builtin_amdgcn_permlane32_swap(v, v, false, false);
|
||||
return __builtin_bit_cast(float, (lane & 32) ? r[0] : r[1]);
|
||||
}
|
||||
__device__ __forceinline__ float xor16(float x, int lane) {
|
||||
int v = __builtin_bit_cast(int, x);
|
||||
i32x2 r = __builtin_amdgcn_permlane16_swap(v, v, false, false);
|
||||
return __builtin_bit_cast(float, (lane & 16) ? r[0] : r[1]);
|
||||
}
|
||||
|
||||
// The top-k stage's histogram, built here while this kernel waits on KV loads.
|
||||
// It shares three invariants with topk_transform.cuh: the key (order_bin_fast),
|
||||
// the npages clamp, and do_hist's TOPK.
|
||||
#define LG_CBITS 6
|
||||
#define LG_CBINS (1 << LG_CBITS)
|
||||
|
||||
// Fused paged-MQA FP8 logits + in-loop fine histogram + coarse summary. The
|
||||
// histogram is what lets the top-k stage skip a separate scoring pass.
|
||||
|
||||
// order_key16(x) >> LOWB in 4 VALU ops after the cvt, with no branch and no
|
||||
// v_cndmask. Bit-identical to topk_transform.cuh's order_key16(x) >> LOWB for
|
||||
// every finite x and for +/-0.
|
||||
template <int LOWB>
|
||||
__device__ __forceinline__ uint32_t order_bin_fast(float x) {
|
||||
const uint32_t h = (uint32_t)__half_as_ushort(__float2half_rn(x));
|
||||
const uint32_t m = (uint32_t)((int32_t)__float_as_int(x) >> 31);
|
||||
return (h >> LOWB) ^ (0x8000u >> LOWB) ^ (m & (0x7fffu >> LOWB));
|
||||
}
|
||||
|
||||
template <int WARPS, int HB>
|
||||
__global__ __launch_bounds__(WARPS * 64) void logits_hist_m(
|
||||
const uint8_t* __restrict__ q,
|
||||
const uint8_t* __restrict__ kv,
|
||||
const float* __restrict__ wgt,
|
||||
const int* __restrict__ seqlens,
|
||||
const int* __restrict__ ptable,
|
||||
float* __restrict__ out,
|
||||
unsigned int* __restrict__ ghist,
|
||||
int max_pages,
|
||||
int out_stride) {
|
||||
constexpr int NBIN = 1 << HB;
|
||||
constexpr int LOWB = 16 - HB;
|
||||
constexpr int GHS = NBIN + LG_CBINS;
|
||||
constexpr int NTH = WARPS * 64;
|
||||
constexpr int PERT = NBIN / NTH; // GLOBAL bins per thread (coarse map)
|
||||
constexpr int CSH = HB - LG_CBITS;
|
||||
constexpr int GRP = NTH / LG_CBINS;
|
||||
static_assert(NBIN >= 2 && (NBIN % 2) == 0, "paired flush needs even bins");
|
||||
static_assert(NBIN % NTH == 0 && NTH >= LG_CBINS, "");
|
||||
static_assert(PERT <= LG_CBINS && (LG_CBINS % PERT) == 0, "a thread's contiguous run must sit inside one coarse bin");
|
||||
__shared__ unsigned int s_hist[NBIN];
|
||||
|
||||
const int row = blockIdx.y;
|
||||
const int lane = threadIdx.x & 63;
|
||||
const int warp = threadIdx.x >> 6;
|
||||
const int seqlen = seqlens[row];
|
||||
// seqlen is device data, so no host check can bound it. Without the clamp
|
||||
// an oversized row walks into the NEXT row's page table -- valid memory,
|
||||
// plausible page ids, confidently wrong logits.
|
||||
const int npages_raw = (seqlen + PAGE_TOK - 1) / PAGE_TOK;
|
||||
// Bound the walk by BOTH the page table and `out`. The host only requires
|
||||
// that `out` cover whole pages -- a captured table is as wide as the graph
|
||||
// while the row is only as long as seqlens says -- so a table wider than
|
||||
// `out` is legal, and without the second term an oversized seqlen writes
|
||||
// past the end of the row.
|
||||
const int pages_out = out_stride / PAGE_TOK;
|
||||
const int npages_cap = max_pages < pages_out ? max_pages : pages_out;
|
||||
const int npages = npages_raw < npages_cap ? npages_raw : npages_cap;
|
||||
const int g = lane >> 4, c = lane & 15;
|
||||
// Nothing to rank when the row is shorter than topk: everything is selected,
|
||||
// so the histogram, its two barriers and the epilogue all compile out.
|
||||
//
|
||||
// This predicate MUST use the same clamped length the paired top-k kernel
|
||||
// applies (it clamps row_ends by the page-table width before its own
|
||||
// row_len <= TOPK early return). A row binned here but skipped there is
|
||||
// never zeroed, and the histogram stays dirty for the life of the process.
|
||||
const int seqlen_eff = seqlen < npages_cap * PAGE_TOK ? seqlen : npages_cap * PAGE_TOK;
|
||||
const bool do_hist = (seqlen_eff > LG_TOPK);
|
||||
|
||||
if (do_hist) {
|
||||
for (int i = threadIdx.x; i < NBIN; i += NTH)
|
||||
s_hist[i] = 0u;
|
||||
}
|
||||
|
||||
const uint8_t* qp = q + (size_t)row * (size_t)(HEADS * HD) + (size_t)c * HD + g * 32;
|
||||
V8 qa0, qa1;
|
||||
qa0.h[0] = *(const i32x4*)(qp);
|
||||
qa0.h[1] = *(const i32x4*)(qp + 16);
|
||||
qa1.h[0] = *(const i32x4*)(qp + 16 * HD);
|
||||
qa1.h[1] = *(const i32x4*)(qp + 16 * HD + 16);
|
||||
const float* wp = wgt + (size_t)row * HEADS + g * 4;
|
||||
f32x4 w0 = *(const f32x4*)(wp);
|
||||
f32x4 w1 = *(const f32x4*)(wp + 16);
|
||||
|
||||
const int nwaves = gridDim.x * WARPS;
|
||||
const int wid = blockIdx.x * WARPS + warp;
|
||||
const int koff = g * 512 + c * 16;
|
||||
|
||||
if (do_hist) __syncthreads();
|
||||
|
||||
unsigned int* __restrict__ gh = ghist + (size_t)row * GHS;
|
||||
|
||||
const int* __restrict__ pt = ptable + (size_t)row * max_pages;
|
||||
|
||||
for (int p = wid; p < npages; p += nwaves) {
|
||||
const int phys = pt[p];
|
||||
const uint8_t* base = kv + (size_t)phys * PAGE_BYTES;
|
||||
const uint8_t* kb = base + koff;
|
||||
V8 b[4];
|
||||
#pragma unroll
|
||||
for (int tt = 0; tt < 4; ++tt) {
|
||||
b[tt].h[0] = *(const i32x4*)(kb + tt * 2048);
|
||||
b[tt].h[1] = *(const i32x4*)(kb + tt * 2048 + 256);
|
||||
}
|
||||
const float ks = ((const float*)(base + K_BYTES))[lane];
|
||||
float s[4];
|
||||
#pragma unroll
|
||||
for (int tt = 0; tt < 4; ++tt) {
|
||||
const f32x4 z = {0.f, 0.f, 0.f, 0.f};
|
||||
f32x4 a0 = mfma128(qa0.v, b[tt].v, z);
|
||||
f32x4 a1 = mfma128(qa1.v, b[tt].v, z);
|
||||
float acc = 0.f;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 4; ++i)
|
||||
acc = fmaf(fmaxf(a0[i], 0.f), w0[i], acc);
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 4; ++i)
|
||||
acc = fmaf(fmaxf(a1[i], 0.f), w1[i], acc);
|
||||
s[tt] = acc;
|
||||
}
|
||||
const bool hi = (g & 2) != 0;
|
||||
float A0 = (hi ? s[2] : s[0]) + xor32(hi ? s[0] : s[2], lane);
|
||||
float A1 = (hi ? s[3] : s[1]) + xor32(hi ? s[1] : s[3], lane);
|
||||
const bool odd = (g & 1) != 0;
|
||||
float T = (odd ? A1 : A0) + xor16(odd ? A0 : A1, lane);
|
||||
const int pos = p * PAGE_TOK + lane;
|
||||
const bool live = pos < seqlen;
|
||||
const float v = T * ks;
|
||||
out[(size_t)row * out_stride + pos] = live ? v : -INFINITY;
|
||||
if (do_hist && live) {
|
||||
const uint32_t bb = order_bin_fast<LOWB>(v);
|
||||
atomicAdd(&s_hist[bb], 1u);
|
||||
}
|
||||
}
|
||||
|
||||
if (do_hist) {
|
||||
__syncthreads();
|
||||
|
||||
// ---- coarse summary, mapped over the GLOBAL bin index ----
|
||||
// Thread tx owns global bins [tx*PERT, tx*PERT+PERT), which lies inside
|
||||
// coarse bin (tx*PERT)>>CSH. Needs no barrier of its own.
|
||||
const int g0 = (int)threadIdx.x * PERT;
|
||||
unsigned int acc = 0u;
|
||||
#pragma unroll
|
||||
for (int j = 0; j < PERT; ++j)
|
||||
acc += s_hist[g0 + j];
|
||||
#pragma unroll
|
||||
for (int o = GRP >> 1; o >= 1; o >>= 1)
|
||||
acc += __shfl_down(acc, o, 64);
|
||||
if ((threadIdx.x & (GRP - 1)) == 0 && acc) atomicAdd(&gh[NBIN + (g0 >> CSH)], acc);
|
||||
|
||||
for (int i = threadIdx.x; i < NBIN / 2; i += NTH) {
|
||||
const unsigned long long vv = (unsigned long long)s_hist[2 * i] | ((unsigned long long)s_hist[2 * i + 1] << 32);
|
||||
if (vv) atomicAdd((unsigned long long*)(gh + 2 * i), vv);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The fp8 dtype the MFMA reads as raw OCP e4m3 bytes. Checked by hand rather
|
||||
/// than through ``with_dtype<fp8_e4m3_t>``: on ROCm ``sglang::fp8_e4m3_t`` is
|
||||
/// ``uint8_t``, which ``DLDataTypeTrait`` maps to kDLUInt -- that would reject a
|
||||
/// genuine fp8 tensor and accept a uint8 one, exactly backwards.
|
||||
inline auto is_fp8_e4m3fn(DLDataType dtype) -> bool {
|
||||
return dtype.lanes == 1 && dtype.bits == 8 && dtype.code == kDLFloat8_e4m3fn;
|
||||
}
|
||||
|
||||
} // namespace dsa_gfx950::logits
|
||||
|
||||
struct PagedMqaLogitsKernel {
|
||||
static void
|
||||
run(const tvm::ffi::TensorView q,
|
||||
const tvm::ffi::TensorView kv,
|
||||
const tvm::ffi::TensorView weights,
|
||||
const tvm::ffi::TensorView seqlens,
|
||||
const tvm::ffi::TensorView page_table,
|
||||
const tvm::ffi::TensorView out,
|
||||
const tvm::ffi::TensorView ghist,
|
||||
int64_t blocks_per_row) {
|
||||
using namespace host;
|
||||
namespace impl = dsa_gfx950::logits;
|
||||
|
||||
// The launch below is logits_hist_m<8, HOST_HB>, so the histogram row
|
||||
// stride the kernel writes is fixed here too. topk_transform.cuh checks
|
||||
// the same width from its side; without this one a mismatch overruns into
|
||||
// the next row's counters.
|
||||
constexpr int HOST_HB = 12;
|
||||
constexpr int64_t HOST_GHS = (int64_t{1} << HOST_HB) + LG_CBINS;
|
||||
|
||||
auto rows_ = SymbolicSize{"rows"};
|
||||
auto pages_ = SymbolicSize{"table_pages"};
|
||||
auto cols_ = SymbolicSize{"logits_cols"};
|
||||
auto device_ = SymbolicDevice{};
|
||||
device_.set_options<kDLCUDA>();
|
||||
// Rebinding one symbolic dtype across q and kv is the "kv must have the
|
||||
// same fp8 dtype as q" check; the options stay open because e4m3fn has no
|
||||
// DLDataTypeTrait on ROCm.
|
||||
auto fp8_ = SymbolicDType{};
|
||||
|
||||
// No with_strides means TensorMatcher demands full contiguity, which is
|
||||
// what each of these wants.
|
||||
TensorMatcher({rows_, int64_t{HEADS}, int64_t{HD}}).with_dtype(fp8_).with_device(device_).verify(q);
|
||||
// PAGE_TOK and TOK_STRIDE are compiled in; a cache laid out differently
|
||||
// would be read as garbage rather than refused, hence the row width here.
|
||||
TensorMatcher({-1, int64_t{PAGE_TOK} * TOK_STRIDE}).with_dtype(fp8_).with_device(device_).verify(kv);
|
||||
RuntimeCheck(
|
||||
impl::is_fp8_e4m3fn(fp8_.unwrap()), "q and the kv cache must be fp8 e4m3fn; they are read as raw bytes");
|
||||
|
||||
TensorMatcher({rows_, int64_t{HEADS}}).with_dtype<fp32_t>().with_device(device_).verify(weights);
|
||||
TensorMatcher({rows_, pages_}).with_dtype<int32_t>().with_device(device_).verify(page_table);
|
||||
TensorMatcher({rows_, HOST_GHS})
|
||||
.with_dtype<int32_t>()
|
||||
.with_device(device_)
|
||||
.with_strides({HOST_GHS, 1})
|
||||
.verify(ghist);
|
||||
// `out` is a column slice of the workspace, so only the row is packed.
|
||||
TensorMatcher({rows_, cols_}).with_dtype<fp32_t>().with_device(device_).with_strides({-1, 1}).verify(out);
|
||||
|
||||
// One entry per row, but the caller may hand over a longer buffer.
|
||||
TensorMatcher({-1}).with_dtype<int32_t>().with_device(device_).verify(seqlens);
|
||||
RuntimeCheck(
|
||||
seqlens.size(0) >= rows_.unwrap(),
|
||||
"seqlens needs one entry per row: got ",
|
||||
seqlens.size(0),
|
||||
" for ",
|
||||
rows_.unwrap(),
|
||||
" rows");
|
||||
|
||||
// A captured page table is as wide as the graph while the row is as long
|
||||
// as seqlens says, so it may exceed `out`; what must hold is that `out`
|
||||
// covers whole pages.
|
||||
RuntimeCheck(cols_.unwrap() % PAGE_TOK == 0, "out row must be a whole number of pages, got ", cols_.unwrap());
|
||||
|
||||
LaunchKernel(
|
||||
dim3(static_cast<int>(blocks_per_row), static_cast<int>(rows_.unwrap())), dim3(8 * 64), device_.unwrap())(
|
||||
impl::logits_hist_m<8, HOST_HB>,
|
||||
reinterpret_cast<const uint8_t*>(q.data_ptr()),
|
||||
reinterpret_cast<const uint8_t*>(kv.data_ptr()),
|
||||
static_cast<fp32_t*>(weights.data_ptr()),
|
||||
static_cast<int*>(seqlens.data_ptr()),
|
||||
static_cast<int*>(page_table.data_ptr()),
|
||||
static_cast<fp32_t*>(out.data_ptr()),
|
||||
reinterpret_cast<unsigned int*>(ghist.data_ptr()),
|
||||
static_cast<int>(pages_.unwrap()),
|
||||
static_cast<int>(out.stride(0)));
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace sglang
|
||||
|
||||
// PAGE_TOK / HD / TOK_STRIDE are generic enough to collide with another header
|
||||
// compiled into the same translation unit; they exist only for the kernel above.
|
||||
#undef PAGE_TOK
|
||||
#undef HD
|
||||
#undef HEADS
|
||||
#undef LG_TOPK
|
||||
#undef TOK_STRIDE
|
||||
#undef PAGE_BYTES
|
||||
#undef K_BYTES
|
||||
#undef LG_CBITS
|
||||
#undef LG_CBINS
|
||||
@@ -0,0 +1,336 @@
|
||||
/// A copy of aiter's indexer_qk_rope_quant_and_cache_kernel with an inline
|
||||
/// 128-point Hadamard, so the front end keeps rotate_activation and the fusion
|
||||
/// at once. aiter is untouched; this includes its headers (JIT dependency
|
||||
/// "aiter"), so the maths match.
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifndef USE_ROCM
|
||||
#error "qk_rope_hadamard_quant.cuh targets gfx950; it folds a head onto one wave64"
|
||||
#endif
|
||||
|
||||
#include <sgl_kernel/tensor.h>
|
||||
#include <sgl_kernel/utils.h>
|
||||
|
||||
#include <sgl_kernel/runtime.cuh>
|
||||
#include <sgl_kernel/type.cuh>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
|
||||
#include <hip/hip_bf16.h>
|
||||
#include <hip/hip_runtime.h>
|
||||
#include <tvm/ffi/container/tensor.h>
|
||||
|
||||
#include "hip_reduce.h"
|
||||
#include "opus/opus.hpp"
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
|
||||
namespace sglang {
|
||||
|
||||
namespace dsa_gfx950::qk_rope {
|
||||
|
||||
using cache_t = opus::fp8_t;
|
||||
using scalar_t = opus::bf16_t;
|
||||
|
||||
constexpr int HEAD_DIM = 128;
|
||||
constexpr int ROPE_DIM = 64;
|
||||
constexpr int LOG2_HEAD_DIM = 7;
|
||||
|
||||
// Folded onto one 64-lane wavefront: lane t owns dim t and t+64, so both halves
|
||||
// of every reduction are an independent DPP tree. Zero LDS, zero syncthreads,
|
||||
// and bit-exact against the 128-thread form operand for operand.
|
||||
|
||||
constexpr int HALF_DIM = HEAD_DIM / 2; // 64 == one wavefront
|
||||
|
||||
template <typename F>
|
||||
__device__ __forceinline__ float wred64(float v, F op) {
|
||||
return wave_reduce<float, F, 64, true>(v, op);
|
||||
}
|
||||
|
||||
// Interleaved rope on the low 64 dims, held one-per-lane in `v`. The pair
|
||||
// comes from a cross-lane shuffle instead of LDS; the arithmetic is the same.
|
||||
__device__ __forceinline__ float
|
||||
rope_lo(float v, int t, const scalar_t* __restrict__ cos_ptr, const scalar_t* __restrict__ sin_ptr) {
|
||||
const float pair_val = __shfl_xor(v, 1, 64);
|
||||
const int cos_idx = t / 2;
|
||||
const float cos_v = static_cast<float>(cos_ptr[cos_idx]);
|
||||
const float sin_v = static_cast<float>(sin_ptr[cos_idx]);
|
||||
v = (t % 2 == 0) ? (v * cos_v - pair_val * sin_v) : (v * cos_v + pair_val * sin_v);
|
||||
return static_cast<float>(static_cast<scalar_t>(v));
|
||||
}
|
||||
|
||||
// 128-point Hadamard on a pair (lo = dim t, hi = dim t+64) held in one lane.
|
||||
__device__ __forceinline__ void hadamard128_wave(float& lo, float& hi, int t, float scale) {
|
||||
#pragma unroll
|
||||
for (int s = 0; s < LOG2_HEAD_DIM - 1; ++s) { // strides 1, 2, 4, 8, 16, 32
|
||||
const int d = 1 << s;
|
||||
const float plo = __shfl_xor(lo, d, 64);
|
||||
const float phi = __shfl_xor(hi, d, 64);
|
||||
const bool a = ((t & d) == 0); // (dim & d) is the same for dim and dim+64
|
||||
lo = a ? (lo + plo) : (plo - lo);
|
||||
hi = a ? (hi + phi) : (phi - hi);
|
||||
}
|
||||
// stride 64: partner of dim t is dim t+64 -- both already in this lane.
|
||||
const float nlo = lo + hi;
|
||||
const float nhi = lo - hi; // (dim & 64) != 0 -> partner - v
|
||||
lo = nlo * scale;
|
||||
hi = nhi * scale;
|
||||
}
|
||||
|
||||
// The k-side epilogue runs in its own block rather than at the end of the
|
||||
// head_idx==0 blocks, where it would serialise behind them.
|
||||
__global__ __launch_bounds__(HALF_DIM) void qk_rope_hadamard_quant_kernel(
|
||||
const scalar_t* __restrict__ q,
|
||||
cache_t* __restrict__ q_out,
|
||||
const scalar_t* __restrict__ weights,
|
||||
float* __restrict__ weights_out,
|
||||
const scalar_t* __restrict__ k,
|
||||
cache_t* __restrict__ kv_cache,
|
||||
const int64_t* __restrict__ slot_mapping,
|
||||
const float* __restrict__ norm_weight,
|
||||
const float* __restrict__ norm_bias,
|
||||
const int64_t* __restrict__ positions,
|
||||
const scalar_t* __restrict__ cos_cache,
|
||||
const scalar_t* __restrict__ sin_cache,
|
||||
const int n_heads,
|
||||
const int cache_block_size,
|
||||
const int cache_stride,
|
||||
const int q_stride_t,
|
||||
const int q_stride_h,
|
||||
const int q_out_stride_t,
|
||||
const int q_out_stride_h,
|
||||
const int weights_stride_t,
|
||||
const int weights_out_stride_t,
|
||||
const int k_stride_t,
|
||||
const int cos_stride0,
|
||||
const int sin_stride0,
|
||||
const float epsilon,
|
||||
const float weights_scale,
|
||||
const float hadamard_scale,
|
||||
const int max_position) {
|
||||
const int64_t token_idx = blockIdx.x;
|
||||
const int head_idx = blockIdx.y; // n_heads + 1 of them: the last is the k side
|
||||
const int t = threadIdx.x; // owns dim t and dim t + 64
|
||||
const bool do_q = head_idx < n_heads;
|
||||
|
||||
const int64_t slot_idx = slot_mapping[token_idx];
|
||||
// Clamped unconditionally: pos indexes cos_cache/sin_cache on both branches,
|
||||
// and clamping only the slot_idx < 0 one left the common path free to read at
|
||||
// an arbitrary offset.
|
||||
int64_t pos = positions[token_idx];
|
||||
pos = pos < 0 ? 0 : (pos >= max_position ? max_position - 1 : pos);
|
||||
const scalar_t* cos_ptr = cos_cache + pos * cos_stride0;
|
||||
const scalar_t* sin_ptr = sin_cache + pos * sin_stride0;
|
||||
|
||||
auto max_func = [](float a, float b) { return fmaxf(a, b); };
|
||||
auto sum_func = [](float a, float b) { return a + b; };
|
||||
const float q_fp8_max = static_cast<float>(opus::finfo<cache_t>::max());
|
||||
|
||||
// ------------------------------- q side ---------------------------------
|
||||
if (do_q) {
|
||||
const scalar_t* q_row = q + token_idx * q_stride_t + head_idx * q_stride_h;
|
||||
float lo = static_cast<float>(q_row[t]);
|
||||
float hi = static_cast<float>(q_row[t + HALF_DIM]);
|
||||
|
||||
lo = rope_lo(lo, t, cos_ptr, sin_ptr); // dims >=64 untouched
|
||||
|
||||
hadamard128_wave(lo, hi, t, hadamard_scale);
|
||||
lo = static_cast<float>(static_cast<scalar_t>(lo));
|
||||
hi = static_cast<float>(static_cast<scalar_t>(hi));
|
||||
|
||||
const float a0 = wred64(fabsf(lo), max_func);
|
||||
const float a1 = wred64(fabsf(hi), max_func);
|
||||
const float q_amax = max_func(a1, a0); // block_reduce's order
|
||||
|
||||
const float q_inv_fp8_max = 1.0f / q_fp8_max;
|
||||
// ue8m0: the scale is rounded up to a power of two.
|
||||
const float q_scale = exp2f(ceilf(log2f(fmaxf(q_amax, 1e-10f) * q_inv_fp8_max)));
|
||||
const float q_inv_scale = 1.0f / q_scale;
|
||||
cache_t* qo = q_out + token_idx * q_out_stride_t + head_idx * q_out_stride_h;
|
||||
qo[t] = opus::cast<cache_t>(lo * q_inv_scale);
|
||||
qo[t + HALF_DIM] = opus::cast<cache_t>(hi * q_inv_scale);
|
||||
if (t == 0) {
|
||||
const float w = static_cast<float>(weights[token_idx * weights_stride_t + head_idx]);
|
||||
const float head_scale = rsqrtf(static_cast<float>(n_heads));
|
||||
const scalar_t w_head = static_cast<scalar_t>(w * head_scale);
|
||||
const float softmax_scale = weights_scale / head_scale;
|
||||
weights_out[token_idx * weights_out_stride_t + head_idx] = static_cast<float>(w_head) * q_scale * softmax_scale;
|
||||
}
|
||||
}
|
||||
|
||||
if (do_q || slot_idx < 0) return;
|
||||
|
||||
// ------------------------------- k side ---------------------------------
|
||||
const scalar_t* k_row = k + token_idx * k_stride_t;
|
||||
const float xlo = static_cast<float>(k_row[t]);
|
||||
const float xhi = static_cast<float>(k_row[t + HALF_DIM]);
|
||||
|
||||
const float s0 = wred64(xlo, sum_func);
|
||||
const float s1 = wred64(xhi, sum_func);
|
||||
const float mean = sum_func(s1, s0) / static_cast<float>(HEAD_DIM);
|
||||
|
||||
const float clo = xlo - mean;
|
||||
const float chi = xhi - mean;
|
||||
const float q0 = wred64(clo * clo, sum_func);
|
||||
const float q1 = wred64(chi * chi, sum_func);
|
||||
const float ss = sum_func(q1, q0);
|
||||
const float inv_std = rsqrtf(ss / static_cast<float>(HEAD_DIM) + epsilon);
|
||||
|
||||
float klo = clo * inv_std * norm_weight[t] + norm_bias[t];
|
||||
float khi = chi * inv_std * norm_weight[t + HALF_DIM] + norm_bias[t + HALF_DIM];
|
||||
klo = static_cast<float>(static_cast<scalar_t>(klo));
|
||||
khi = static_cast<float>(static_cast<scalar_t>(khi));
|
||||
|
||||
klo = rope_lo(klo, t, cos_ptr, sin_ptr);
|
||||
|
||||
hadamard128_wave(klo, khi, t, hadamard_scale);
|
||||
klo = static_cast<float>(static_cast<scalar_t>(klo));
|
||||
khi = static_cast<float>(static_cast<scalar_t>(khi));
|
||||
|
||||
const float m0 = wred64(fabsf(klo), max_func);
|
||||
const float m1 = wred64(fabsf(khi), max_func);
|
||||
const float k_amax = max_func(m1, m0);
|
||||
const float k_scale = exp2f(ceilf(log2f(fmaxf(k_amax, 1e-4f) / q_fp8_max))); // ue8m0
|
||||
|
||||
const int64_t block_idx = slot_idx / cache_block_size;
|
||||
const int64_t block_offset = slot_idx % cache_block_size;
|
||||
const int64_t page_base = block_idx * cache_block_size * cache_stride;
|
||||
if (t == 0) {
|
||||
// One fp32 scale per token, after the page's K bytes: quant block == HEAD_DIM.
|
||||
const int64_t dst_scale_idx = page_base + cache_block_size * HEAD_DIM + block_offset * 4;
|
||||
reinterpret_cast<float*>(kv_cache)[dst_scale_idx / 4] = k_scale;
|
||||
}
|
||||
const float k_inv_scale = 1.0f / k_scale;
|
||||
|
||||
// Preshuffled page layout: 16x16 (token x dim) tiles.
|
||||
constexpr int TILE = 16;
|
||||
#pragma unroll
|
||||
for (int h = 0; h < 2; ++h) {
|
||||
const int dim = t + h * HALF_DIM;
|
||||
const int token_tile_id = block_offset / TILE;
|
||||
const int token_in_tile = block_offset % TILE;
|
||||
const int col_tile_id = dim / TILE;
|
||||
const int col_in_tile = dim % TILE;
|
||||
const int64_t dst_offset = page_base + token_tile_id * (TILE * HEAD_DIM) + col_tile_id * (TILE * TILE) +
|
||||
token_in_tile * TILE + col_in_tile;
|
||||
kv_cache[dst_offset] = opus::cast<cache_t>((h == 0 ? klo : khi) * k_inv_scale);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace dsa_gfx950::qk_rope
|
||||
|
||||
struct QkRopeHadamardQuantKernel {
|
||||
/// AITER indexer_qk_rope_quant_and_cache plus the inline 128-point Hadamard,
|
||||
/// specialised to ue8m0 scales, the preshuffled cache, interleaved rope and a
|
||||
/// 128-wide quant block; the Python gate admits nothing else.
|
||||
static void
|
||||
run(const tvm::ffi::TensorView q,
|
||||
const tvm::ffi::TensorView q_out,
|
||||
const tvm::ffi::TensorView weights,
|
||||
const tvm::ffi::TensorView weights_out,
|
||||
const tvm::ffi::TensorView k,
|
||||
const tvm::ffi::TensorView kv_cache,
|
||||
const tvm::ffi::TensorView slot_mapping,
|
||||
const tvm::ffi::TensorView norm_weight,
|
||||
const tvm::ffi::TensorView norm_bias,
|
||||
const tvm::ffi::TensorView positions,
|
||||
const tvm::ffi::TensorView cos_cache,
|
||||
const tvm::ffi::TensorView sin_cache,
|
||||
double epsilon,
|
||||
double weights_scale) {
|
||||
using namespace host;
|
||||
namespace impl = dsa_gfx950::qk_rope;
|
||||
|
||||
/// ``q_out`` and ``kv_cache`` are reinterpret_cast to a 1-byte cache_t and
|
||||
/// written as raw fp8, so what matters is the width, not the flavour: a
|
||||
/// wider dtype fills half the row and leaves the rest untouched.
|
||||
const auto byte_width = [](DLDataType dtype) -> int64_t { return (int64_t{dtype.bits} * dtype.lanes + 7) / 8; };
|
||||
|
||||
const int num_tokens = static_cast<int>(std::min(k.size(0), slot_mapping.size(0)));
|
||||
// num_tokens becomes gridDim.x and indexes all six per-token tensors, but
|
||||
// only two of them took part in the min above.
|
||||
RuntimeCheck(
|
||||
q.size(0) >= num_tokens && q_out.size(0) >= num_tokens && weights.size(0) >= num_tokens &&
|
||||
weights_out.size(0) >= num_tokens && positions.size(0) >= num_tokens,
|
||||
"every per-token tensor must have at least num_tokens rows");
|
||||
const int head_dim = static_cast<int>(k.size(1));
|
||||
const int n_heads = static_cast<int>(q.size(1));
|
||||
RuntimeCheck(cos_cache.dim() == 2 && sin_cache.dim() == 2, "cos/sin must be 2-D");
|
||||
const int rope_dim = static_cast<int>(cos_cache.size(1)) * 2;
|
||||
const int cache_block_size = static_cast<int>(kv_cache.size(1));
|
||||
const int cache_stride = static_cast<int>(kv_cache.size(2));
|
||||
const int max_position = static_cast<int>(cos_cache.size(0));
|
||||
|
||||
RuntimeCheck(head_dim == impl::HEAD_DIM, "head_dim must be 128");
|
||||
RuntimeCheck(rope_dim == impl::ROPE_DIM, "rope_dim must be 64");
|
||||
RuntimeCheck(is_type<bf16_t>(q.dtype()) && is_type<bf16_t>(k.dtype()), "q/k must be bf16");
|
||||
RuntimeCheck(is_type<bf16_t>(weights.dtype()), "weights must be bf16");
|
||||
RuntimeCheck(
|
||||
byte_width(q_out.dtype()) == 1 && byte_width(kv_cache.dtype()) == 1,
|
||||
"q_out and kv_cache must be 1-byte fp8, got ",
|
||||
q_out.dtype(),
|
||||
"/",
|
||||
kv_cache.dtype());
|
||||
RuntimeCheck(is_type<fp32_t>(weights_out.dtype()), "weights_out must be fp32");
|
||||
RuntimeCheck(
|
||||
is_type<fp32_t>(norm_weight.dtype()) && is_type<fp32_t>(norm_bias.dtype()), "norm params must be fp32");
|
||||
RuntimeCheck(cos_cache.stride(1) == 1 && sin_cache.stride(1) == 1, "cos/sin last dim must be contiguous");
|
||||
RuntimeCheck(is_type<int64_t>(slot_mapping.dtype()), "slot_mapping must be int64");
|
||||
RuntimeCheck(is_type<int64_t>(positions.dtype()), "positions must be int64");
|
||||
|
||||
const auto* qp = reinterpret_cast<const impl::scalar_t*>(q.data_ptr());
|
||||
auto* qop = reinterpret_cast<impl::cache_t*>(q_out.data_ptr());
|
||||
const auto* wp = reinterpret_cast<const impl::scalar_t*>(weights.data_ptr());
|
||||
auto* wop = static_cast<fp32_t*>(weights_out.data_ptr());
|
||||
const auto* kp = reinterpret_cast<const impl::scalar_t*>(k.data_ptr());
|
||||
auto* cp = reinterpret_cast<impl::cache_t*>(kv_cache.data_ptr());
|
||||
const auto* sp = static_cast<const int64_t*>(slot_mapping.data_ptr());
|
||||
const auto* nwp = static_cast<const fp32_t*>(norm_weight.data_ptr());
|
||||
const auto* nbp = static_cast<const fp32_t*>(norm_bias.data_ptr());
|
||||
const auto* pp = static_cast<const int64_t*>(positions.data_ptr());
|
||||
const auto* cosp = reinterpret_cast<const impl::scalar_t*>(cos_cache.data_ptr());
|
||||
const auto* sinp = reinterpret_cast<const impl::scalar_t*>(sin_cache.data_ptr());
|
||||
|
||||
// rotate_activation(x) == hadamard_transform(x, scale=x.size(-1) ** -0.5)
|
||||
const float had_scale = 1.0f / sqrtf(static_cast<float>(impl::HEAD_DIM));
|
||||
|
||||
// The wave path indexes the head dimension directly, so the layout
|
||||
// invariants it relies on are asserted rather than carried as arguments.
|
||||
RuntimeCheck(q.stride(2) == 1 && q_out.stride(2) == 1 && k.stride(1) == 1, "wave kernel needs contiguous head dim");
|
||||
RuntimeCheck(
|
||||
weights.stride(1) == 1 && weights_out.stride(1) == 1, "wave kernel needs contiguous head axis on weights");
|
||||
|
||||
LaunchKernel(dim3(num_tokens, n_heads + 1), dim3(impl::HALF_DIM), q.device())(
|
||||
impl::qk_rope_hadamard_quant_kernel,
|
||||
qp,
|
||||
qop,
|
||||
wp,
|
||||
wop,
|
||||
kp,
|
||||
cp,
|
||||
sp,
|
||||
nwp,
|
||||
nbp,
|
||||
pp,
|
||||
cosp,
|
||||
sinp,
|
||||
n_heads,
|
||||
cache_block_size,
|
||||
cache_stride,
|
||||
static_cast<int>(q.stride(0)),
|
||||
static_cast<int>(q.stride(1)),
|
||||
static_cast<int>(q_out.stride(0)),
|
||||
static_cast<int>(q_out.stride(1)),
|
||||
static_cast<int>(weights.stride(0)),
|
||||
static_cast<int>(weights_out.stride(0)),
|
||||
static_cast<int>(k.stride(0)),
|
||||
static_cast<int>(cos_cache.stride(0)),
|
||||
static_cast<int>(sin_cache.stride(0)),
|
||||
static_cast<float>(epsilon),
|
||||
static_cast<float>(weights_scale),
|
||||
had_scale,
|
||||
max_position);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace sglang
|
||||
File diff suppressed because it is too large
Load Diff
@@ -28,6 +28,26 @@ def register_dependency(name: str):
|
||||
return decorator
|
||||
|
||||
|
||||
@register_dependency("aiter")
|
||||
def get_aiter_include_paths() -> List[str]:
|
||||
"""aiter's C++ headers, for ROCm kernels that must compute exactly what aiter
|
||||
computes (hip_reduce.h, opus/opus.hpp)."""
|
||||
aiter_root = _find_package_root("aiter")
|
||||
if aiter_root is None:
|
||||
raise RuntimeError(
|
||||
"Cannot find the aiter package. Install aiter to get the required"
|
||||
" headers for JIT compilation."
|
||||
)
|
||||
|
||||
include = aiter_root.parent / "csrc" / "include"
|
||||
if not (include / "hip_reduce.h").is_file():
|
||||
raise RuntimeError(
|
||||
f"aiter C++ headers not found under {include}; a kernel that includes"
|
||||
" them cannot be built against this aiter installation."
|
||||
)
|
||||
return [str(include)]
|
||||
|
||||
|
||||
@register_dependency("flashinfer")
|
||||
def get_flashinfer_include_paths() -> List[str]:
|
||||
include_paths: List[str] = []
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
"""gfx950 fused DSA indexer decode path: four raw-HIP kernels replacing the
|
||||
12-launch ROCm chain. Enabled wherever supported; the gate is
|
||||
``utils.gfx950_fused_indexer_runtime_ok``."""
|
||||
|
||||
from sglang.kernels.ops.attention.dsa.hip_gfx950.fused_decode import ( # noqa: F401
|
||||
CACHE_TOK_STRIDE,
|
||||
MAX_ROWS,
|
||||
PAGE_SIZE,
|
||||
Gfx950FusedIndexer,
|
||||
model_shape_supported,
|
||||
prealloc_workspace,
|
||||
supported_hardware,
|
||||
)
|
||||
@@ -0,0 +1,331 @@
|
||||
"""Marshals production tensors into the four fused kernels' ABIs and owns the
|
||||
workspace. Output matches aiter.dsa_topk_transform exactly: int32
|
||||
physical page_size=1 slots, -1 padded, cache byte-compatible."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Dict, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.jit.utils.common import is_hip_runtime
|
||||
from sglang.kernels.ops.attention.dsa.hip_gfx950 import loader
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_WARNED_TOO_NARROW = False
|
||||
_WARNED_NO_WORKSPACE = False
|
||||
|
||||
# Kernel-source constraints. Every one of these is enforced by a RuntimeCheck or a
|
||||
# compile-time constant in csrc/, not by taste:
|
||||
HEAD_DIM = 128 # qk_rope_hadamard_quant.cuh / paged_mqa_logits.cuh
|
||||
ROPE_DIM = 64 # qk_rope_hadamard_quant.cuh
|
||||
N_HEADS = 32 # validated shape (grid.y of the qk kernel, MFMA n of the logits kernel)
|
||||
INDEX_TOPK = 2048 # topk_transform.cuh: constexpr TOPK = 2048u
|
||||
PAGE_SIZE = 64 # paged_mqa_logits.cuh: PAGE_TOK
|
||||
CACHE_TOK_STRIDE = 132 # paged_mqa_logits.cuh: TOK_STRIDE (128 K bytes + 4 scale)
|
||||
Q_LORA_RANK = 2048 # dual_gemv_bf16.cuh: Kq
|
||||
HIDDEN_SIZE = 6144 # dual_gemv_bf16.cuh: Kk
|
||||
QUANT_BLOCK = 128 # qk_rope_hadamard_quant.cuh: quant block == HEAD_DIM
|
||||
MAX_ROWS = 48 # bs 8 x num_draft_tokens 6; dual_gemv_bf16.cuh kMaxRows matches it
|
||||
|
||||
|
||||
def supported_hardware() -> bool:
|
||||
"""Whether this device is one the kernels target, before asking whether the
|
||||
shape fits or whether they build. They are raw gfx950: MFMA, wave64 DPP and
|
||||
an e4m3fn fp8 cache throughout."""
|
||||
if not is_hip_runtime() or not torch.cuda.is_available():
|
||||
return False
|
||||
return "gfx950" in torch.cuda.get_device_properties(0).gcnArchName
|
||||
|
||||
|
||||
def model_shape_supported(
|
||||
*,
|
||||
head_dim: int,
|
||||
rope_head_dim: int,
|
||||
n_heads: int,
|
||||
index_topk: int,
|
||||
q_lora_rank: int,
|
||||
hidden_size: int,
|
||||
quant_block: int,
|
||||
scale_fmt: Optional[str],
|
||||
is_neox_style: bool,
|
||||
k_norm,
|
||||
num_init_tokens: int,
|
||||
num_local_tokens: int,
|
||||
) -> bool:
|
||||
"""Static (per-model) half of the gate. Evaluated once in ``Indexer.__init__``."""
|
||||
from sglang.srt.layers.layernorm import LayerNorm
|
||||
|
||||
return (
|
||||
supported_hardware()
|
||||
and head_dim == HEAD_DIM
|
||||
and rope_head_dim == ROPE_DIM
|
||||
and n_heads == N_HEADS
|
||||
and index_topk == INDEX_TOPK
|
||||
and q_lora_rank == Q_LORA_RANK
|
||||
and hidden_size == HIDDEN_SIZE
|
||||
and quant_block == QUANT_BLOCK
|
||||
and scale_fmt == "ue8m0"
|
||||
# GLM-5.2 sets indexer_rope_interleave=True -> is_neox_style False, which
|
||||
# is what the qk kernel's interleaved rope implements.
|
||||
and not is_neox_style
|
||||
# The kernel does a LayerNorm with weight AND bias; RMSNorm has no bias
|
||||
# and different math (index_k_norm_type == "rms").
|
||||
and isinstance(k_norm, LayerNorm)
|
||||
# _mask_init_and_local_tokens must be a no-op: the fused pipeline has no
|
||||
# place to inject the +inf streaming-attention mask between B and C.
|
||||
and num_init_tokens == 0
|
||||
and num_local_tokens == 0
|
||||
)
|
||||
|
||||
|
||||
class _Workspace:
|
||||
"""Persistent buffers, allocated once outside any capture at the maximum size
|
||||
the process can ask for, so a captured graph never records a graph-private
|
||||
pointer and no later call can force a reallocation."""
|
||||
|
||||
def __init__(self, *, device, max_cols: int, fp8_dtype):
|
||||
rows = MAX_ROWS
|
||||
cap = max(INDEX_TOPK, max_cols)
|
||||
gh_stride = loader.modules()[3].hist_stride()
|
||||
self.max_cols = max_cols
|
||||
self.logits = torch.empty((rows, max_cols), dtype=torch.float32, device=device)
|
||||
self.q_fp8 = torch.empty(
|
||||
(rows, N_HEADS, HEAD_DIM), dtype=fp8_dtype, device=device
|
||||
)
|
||||
self.head_gate = torch.empty(
|
||||
(rows, N_HEADS), dtype=torch.float32, device=device
|
||||
)
|
||||
self.q_proj = torch.empty(
|
||||
(rows, N_HEADS * HEAD_DIM), dtype=torch.bfloat16, device=device
|
||||
)
|
||||
self.kw = torch.empty(
|
||||
(rows, HEAD_DIM + N_HEADS), dtype=torch.bfloat16, device=device
|
||||
)
|
||||
self.ghist = torch.zeros((rows, gh_stride), dtype=torch.int32, device=device)
|
||||
self.cursor = torch.zeros((rows, 32), dtype=torch.int32, device=device)
|
||||
self.cand_idx = torch.empty((rows, cap), dtype=torch.int32, device=device)
|
||||
self.cand_val = torch.empty((rows, cap), dtype=torch.float32, device=device)
|
||||
# Row-expanded page table: metadata is per request, the kernels index
|
||||
# per row. Flat so a narrower width still views contiguously.
|
||||
self.page_table = torch.empty(
|
||||
(rows * (max_cols // PAGE_SIZE),), dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
|
||||
# One workspace per (device, fp8 dtype), shared by all layers: the buffers are
|
||||
# scratch, so per-layer copies would multiply the footprint by 79.
|
||||
_WORKSPACES: Dict[Tuple[int, torch.dtype], _Workspace] = {}
|
||||
|
||||
|
||||
def _workspace_key(device, fp8_dtype) -> Tuple[int, torch.dtype]:
|
||||
index = device.index if isinstance(device, torch.device) else device
|
||||
if index is None:
|
||||
index = torch.cuda.current_device()
|
||||
return (int(index), fp8_dtype)
|
||||
|
||||
|
||||
def prealloc_workspace(*, device, max_cols: int, fp8_dtype) -> bool:
|
||||
"""Create the shared workspace before the server sizes its memory pools, which
|
||||
divide whatever mem_get_info reports at that moment. Idempotent; returns
|
||||
False if one exists."""
|
||||
key = _workspace_key(device, fp8_dtype)
|
||||
if key in _WORKSPACES:
|
||||
return False
|
||||
_WORKSPACES[key] = _Workspace(device=device, max_cols=max_cols, fp8_dtype=fp8_dtype)
|
||||
logger.info(
|
||||
"gfx950 fused DSA indexer: workspace preallocated (max_cols=%d, "
|
||||
"%.0f MiB) before pool sizing",
|
||||
max_cols,
|
||||
sum(
|
||||
t.numel() * t.element_size()
|
||||
for t in vars(_WORKSPACES[key]).values()
|
||||
if isinstance(t, torch.Tensor)
|
||||
)
|
||||
/ (1 << 20),
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
class Gfx950FusedIndexer:
|
||||
"""Per-``Indexer`` state: merged weights, fp32 norm params, 2-D rope cache."""
|
||||
|
||||
def __init__(self, indexer):
|
||||
self.indexer = indexer
|
||||
self.workspace: Optional[_Workspace] = None
|
||||
self._const = None
|
||||
# softmax_scale * n_heads**-0.5, folded exactly as _get_logits_head_gate
|
||||
# does (weights * n_heads**-0.5, then * q_scale * softmax_scale).
|
||||
self.weights_scale = float(indexer.n_heads) ** -0.5 * indexer.softmax_scale
|
||||
|
||||
# -- setup-time derived constants (weights are frozen after load) ---------
|
||||
#
|
||||
# Derived lazily on purpose: this object is built in Indexer.__init__, which
|
||||
# runs before load_weights, so nothing here can be hoisted into __init__.
|
||||
# The first decode is after both weight load and the pre-capture warmup, so
|
||||
# the torch.cat below is never recorded into a CUDA graph.
|
||||
#
|
||||
# Known limitation: w_kw / norm_w / norm_b are copies taken once. An
|
||||
# in-place weight update afterwards (RL weight sync writes through
|
||||
# param.data.copy_, keeping the same storage) is not observed here, while
|
||||
# w_q_b is a view and does track it -- so a reload would leave the two
|
||||
# halves of the same GEMV on different weights. There is no per-module
|
||||
# update hook to subscribe to, so a deployment that hot-swaps weights must
|
||||
# run with --no-enable-dsa-fused-indexer.
|
||||
def _constants(self):
|
||||
if self._const is None:
|
||||
ix = self.indexer
|
||||
# L1: [wk ; weights_proj] -> one [160, 6144] bf16 weight. A pure
|
||||
# weight-loader style merge: both are ReplicatedLinear over the same
|
||||
# x, so one 6144->160 GEMM is arithmetically the two GEMMs.
|
||||
w_kw = torch.cat(
|
||||
[ix.wk.weight.detach(), ix.weights_proj.weight.detach()], dim=0
|
||||
).contiguous()
|
||||
assert w_kw.dtype == torch.bfloat16, w_kw.dtype
|
||||
w_q_b = ix.wq_b.weight.detach()
|
||||
assert w_q_b.dtype == torch.bfloat16 and w_q_b.is_contiguous()
|
||||
|
||||
norm_w = ix.k_norm.weight.detach().float().contiguous()
|
||||
norm_b = ix.k_norm.bias.detach().float().contiguous()
|
||||
|
||||
cos2d, sin2d = _rope_2d(ix.rotary_emb, ROPE_DIM)
|
||||
self._const = (w_q_b, w_kw, norm_w, norm_b, cos2d, sin2d)
|
||||
return self._const
|
||||
|
||||
def ensure_workspace(self, *, device, max_cols: int, fp8_dtype) -> bool:
|
||||
"""Bind the preallocated workspace; False if there is none or it is narrower
|
||||
than this call's page table. Never allocates: Indexer.__init__ owns that."""
|
||||
ws = _WORKSPACES.get(_workspace_key(device, fp8_dtype))
|
||||
if ws is None:
|
||||
# Say so once, for the same reason as the width refusal below.
|
||||
global _WARNED_NO_WORKSPACE
|
||||
if not _WARNED_NO_WORKSPACE:
|
||||
_WARNED_NO_WORKSPACE = True
|
||||
logger.warning(
|
||||
"gfx950 fused DSA indexer: no workspace was preallocated, so "
|
||||
"the standard indexer runs instead."
|
||||
)
|
||||
return False
|
||||
self.workspace = ws
|
||||
if max_cols > ws.max_cols:
|
||||
# Say so once. This refusal turns the whole feature off for the
|
||||
# rest of the run while every other log line still reports it as
|
||||
# enabled, so an A/B against it silently measures two identical arms.
|
||||
global _WARNED_TOO_NARROW
|
||||
if not _WARNED_TOO_NARROW:
|
||||
_WARNED_TOO_NARROW = True
|
||||
logger.warning(
|
||||
"gfx950 fused DSA indexer: workspace is %d columns wide "
|
||||
"but this call needs %d, so the standard indexer runs "
|
||||
"instead. The workspace must cover the decode graph's "
|
||||
"page-table width.",
|
||||
ws.max_cols,
|
||||
max_cols,
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
# -- the four launches ---------------------------------------------------
|
||||
def run(
|
||||
self,
|
||||
*,
|
||||
x: torch.Tensor,
|
||||
q_lora: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
out_cache_loc: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
kv_cache_read: torch.Tensor,
|
||||
seqlens_int32: torch.Tensor,
|
||||
row_ends_int32: torch.Tensor,
|
||||
page_table_64: torch.Tensor,
|
||||
rows: int,
|
||||
) -> torch.Tensor:
|
||||
gemv, qk, logits_mod, topk_mod = loader.modules()
|
||||
w_q_b, w_kw, norm_w, norm_b, cos2d, sin2d = self._constants()
|
||||
ws = self.workspace
|
||||
assert ws is not None
|
||||
|
||||
cols = page_table_64.shape[1] * PAGE_SIZE
|
||||
q_proj = ws.q_proj[:rows]
|
||||
kw = ws.kw[:rows]
|
||||
q_fp8 = ws.q_fp8[:rows]
|
||||
head_gate = ws.head_gate[:rows]
|
||||
logits = ws.logits[:rows, :cols]
|
||||
|
||||
# node 1: oq = q_lora @ wq_b.T and ok = x @ [wk ; weights_proj].T, one launch.
|
||||
gemv.dual_gemv_bf16(q_lora[:rows], w_q_b, q_proj, x[:rows], w_kw, kw)
|
||||
|
||||
# node 2: k_norm | rope | Hadamard(q,k) | act_quant(q) |
|
||||
# indexer_k_quant_and_cache(k) | head gate. The Hadamard is inline, so
|
||||
# the gate checks the standard path applies it too (hadamard_preserved).
|
||||
qk.indexer_qk_rope_hadamard_quant_and_cache(
|
||||
q_proj.view(rows, N_HEADS, HEAD_DIM),
|
||||
q_fp8,
|
||||
kw[:, HEAD_DIM:],
|
||||
head_gate,
|
||||
kw[:, :HEAD_DIM],
|
||||
kv_cache,
|
||||
out_cache_loc,
|
||||
norm_w,
|
||||
norm_b,
|
||||
positions,
|
||||
cos2d,
|
||||
sin2d,
|
||||
float(self.indexer.k_norm.variance_epsilon),
|
||||
float(self.weights_scale),
|
||||
)
|
||||
|
||||
# Allocated before node 3, not between the two: logits_hist dirties the
|
||||
# histogram, and an OOM in between would leave it dirty for every later call.
|
||||
out = torch.empty((rows, INDEX_TOPK), dtype=torch.int32, device=logits.device)
|
||||
|
||||
# --- node 3: paged MQA fp8 logits + fine histogram + coarse summary --
|
||||
logits_mod.logits_hist(
|
||||
q_fp8,
|
||||
kv_cache_read,
|
||||
head_gate,
|
||||
seqlens_int32,
|
||||
page_table_64,
|
||||
logits,
|
||||
ws.ghist[:rows],
|
||||
loader.LOGITS_BLOCKS_PER_ROW,
|
||||
)
|
||||
|
||||
# --- node 4: top-k(2048) + page transform, fused refinement ---------
|
||||
topk_mod.topk_transform(
|
||||
logits,
|
||||
row_ends_int32,
|
||||
page_table_64,
|
||||
out,
|
||||
ws.ghist[:rows],
|
||||
ws.cursor[:rows],
|
||||
ws.cand_idx[:rows],
|
||||
ws.cand_val[:rows],
|
||||
loader.TOPK_G,
|
||||
PAGE_SIZE,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _rope_2d(rotary_emb, rope_dim: int):
|
||||
"""cos/sin as 2-D [max_pos, rope_dim // 2] bf16 with unit last stride. Both
|
||||
halves are strided views, so contiguous() copies; the pair is cached on the
|
||||
shared rope module rather than copied per layer."""
|
||||
half = rope_dim // 2
|
||||
cached = getattr(rotary_emb, "_dsa_fused_rope_2d", None)
|
||||
if cached is not None and cached[0].shape[1] == half:
|
||||
return cached
|
||||
cos = getattr(rotary_emb, "cos_cache", None)
|
||||
sin = getattr(rotary_emb, "sin_cache", None)
|
||||
if cos is None or sin is None:
|
||||
cache = rotary_emb.cos_sin_cache
|
||||
cos, sin = cache[..., :half], cache[..., half:]
|
||||
cos = cos.reshape(cos.shape[0], half).contiguous()
|
||||
sin = sin.reshape(sin.shape[0], half).contiguous()
|
||||
assert cos.dtype == torch.bfloat16 and sin.dtype == torch.bfloat16, (
|
||||
f"gfx950 fused indexer needs a bf16 rope cache, got {cos.dtype}/{sin.dtype}"
|
||||
)
|
||||
rotary_emb._dsa_fused_rope_2d = (cos, sin)
|
||||
return cos, sin
|
||||
@@ -0,0 +1,93 @@
|
||||
"""JIT build + load of the four gfx950 DSA-indexer kernels, on the first gated
|
||||
call -- never in a timed region or a graph capture.
|
||||
|
||||
Four modules rather than one because the flags differ per kernel: the logits
|
||||
kernel wants its own -mllvm set, the qk kernel needs aiter's headers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Optional, Tuple
|
||||
|
||||
from sglang.kernels.jit.utils import cache_once, load_jit
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from tvm_ffi.module import Module
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# The accepted configuration of each kernel. Changing one of these means
|
||||
# editing the kernel it belongs to; the sources reject any other value rather
|
||||
# than mis-launching.
|
||||
LOGITS_HIST_BITS = 12 # must match logits_hist_m<8, 12> and topk_transform.cuh
|
||||
LOGITS_BLOCKS_PER_ROW = 48
|
||||
TOPK_G = 64
|
||||
|
||||
|
||||
@cache_once
|
||||
def modules() -> Tuple[Module, Module, Module, Module]:
|
||||
"""(gemv, qk, logits, topk) modules, or raise. qk goes early: it is the one
|
||||
that can fail for a reason outside this tree (a missing or stale aiter)."""
|
||||
gemv = load_jit(
|
||||
"dsa_gfx950_dual_gemv",
|
||||
cuda_files=["dsa_gfx950/dual_gemv_bf16.cuh"],
|
||||
cuda_wrappers=[("dual_gemv_bf16", "DualGemvBf16Kernel::run")],
|
||||
extra_cuda_cflags=["-O3"],
|
||||
)
|
||||
# Includes aiter's hip_reduce.h and opus.hpp, so the numeric helpers are the
|
||||
# same code aiter's own fused indexer uses. opus.hpp needs the C++20 the JIT
|
||||
# defaults already pass.
|
||||
qk = load_jit(
|
||||
"dsa_gfx950_qk",
|
||||
cuda_files=["dsa_gfx950/qk_rope_hadamard_quant.cuh"],
|
||||
cuda_wrappers=[
|
||||
(
|
||||
"indexer_qk_rope_hadamard_quant_and_cache",
|
||||
"QkRopeHadamardQuantKernel::run",
|
||||
)
|
||||
],
|
||||
extra_cuda_cflags=["-O3", "-Wno-unused-result"],
|
||||
extra_dependencies=["aiter"],
|
||||
)
|
||||
logits = load_jit(
|
||||
"dsa_gfx950_logits",
|
||||
cuda_files=["dsa_gfx950/paged_mqa_logits.cuh"],
|
||||
cuda_wrappers=[("logits_hist", "PagedMqaLogitsKernel::run")],
|
||||
extra_cuda_cflags=[
|
||||
"-O3",
|
||||
"-fno-honor-nans",
|
||||
"-mllvm",
|
||||
"-amdgpu-mfma-vgpr-form",
|
||||
"-mllvm",
|
||||
"-amdgpu-early-inline-all=true",
|
||||
"-mllvm",
|
||||
"-amdgpu-function-calls=false",
|
||||
"-Wno-unused-result",
|
||||
],
|
||||
)
|
||||
# The width is a module argument as well as a -D. The -D already keys the
|
||||
# cache; the argument keeps two widths in separate modules, so their exports
|
||||
# cannot collide in one process.
|
||||
topk = load_jit(
|
||||
"dsa_gfx950_topk",
|
||||
f"hist_bits{LOGITS_HIST_BITS}",
|
||||
cuda_files=["dsa_gfx950/topk_transform.cuh"],
|
||||
cuda_wrappers=[
|
||||
("topk_transform", "TopKTransformKernel::run"),
|
||||
("hist_stride", "TopKHistStride::run"),
|
||||
],
|
||||
extra_cuda_cflags=["-O3", f"-DDSA_TOPK_HIST_BITS={LOGITS_HIST_BITS}"],
|
||||
)
|
||||
return gemv, qk, logits, topk
|
||||
|
||||
|
||||
@cache_once
|
||||
def modules_or_none() -> Optional[Tuple[Module, ...]]:
|
||||
"""Build once; on failure log and return None. Whether the caller falls back
|
||||
or refuses is the gate's decision, so this does not claim either."""
|
||||
try:
|
||||
return modules()
|
||||
except Exception as e: # noqa: BLE001 - a build failure must never be fatal
|
||||
logger.warning("gfx950 fused DSA indexer unavailable, build failed: %s", e)
|
||||
return None
|
||||
@@ -326,6 +326,19 @@ class ExecKernel(msgspec.Struct):
|
||||
bool,
|
||||
"Enable the experimental FP4 C4 indexer path for DeepSeek V4. Default keeps the existing indexer implementation.",
|
||||
] = False
|
||||
enable_dsa_fused_indexer: A[
|
||||
Optional[bool],
|
||||
Arg(
|
||||
help="Use the four-kernel fused DSA indexer decode path in place of "
|
||||
"the 12-launch aiter/torch chain. By default this is enabled "
|
||||
"wherever it is supported -- gfx950 (MI355X) with aiter preshuffle "
|
||||
"and an fp8 e4m3fn index cache -- and the runtime declines with a "
|
||||
"logged reason wherever any of that is missing. Pass "
|
||||
"--no-enable-dsa-fused-indexer to force the standard path.",
|
||||
action=argparse.BooleanOptionalAction,
|
||||
resolvable=True,
|
||||
),
|
||||
] = None
|
||||
|
||||
|
||||
class ExecMamba(msgspec.Struct):
|
||||
|
||||
@@ -34,6 +34,9 @@ from sglang.srt.layers.attention.dsa.paged_mqa_logits_backend import (
|
||||
)
|
||||
from sglang.srt.layers.attention.dsa.utils import (
|
||||
aiter_can_use_preshuffle_paged_mqa,
|
||||
gfx950_fused_indexer_runtime_ok,
|
||||
gfx950_model_shape_supported,
|
||||
hadamard_preserved,
|
||||
is_dsa_enable_prefill_cp,
|
||||
is_graph_dsa_split_op_surface,
|
||||
)
|
||||
@@ -80,6 +83,21 @@ _FUSED_FP8_WRITER_LOGGED = False
|
||||
|
||||
_is_cuda = is_cuda()
|
||||
_is_hip = is_hip()
|
||||
|
||||
# Every per-call decline in _gfx950_fused_decode falls through to the standard
|
||||
# path, which is silent by construction. Without a trace, a config that turns
|
||||
# the feature off -- rows past the row cap is the easy one to hit -- looks
|
||||
# identical to one where it is on, and an A/B measures two identical arms.
|
||||
_FUSED_DECLINE_LOGGED: set = set()
|
||||
|
||||
|
||||
def _decline_fused(reason: str) -> None:
|
||||
"""Log the first occurrence of each distinct decline reason, then stay quiet."""
|
||||
if reason not in _FUSED_DECLINE_LOGGED:
|
||||
_FUSED_DECLINE_LOGGED.add(reason)
|
||||
logger.info("gfx950 fused DSA indexer declined this call: %s", reason)
|
||||
|
||||
|
||||
_is_npu = is_npu()
|
||||
_is_xpu = is_xpu()
|
||||
|
||||
@@ -377,10 +395,6 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
|
||||
"using the legacy ROCm indexer writer"
|
||||
)
|
||||
elif self.use_aiter_fused_fp8_writer:
|
||||
global _FUSED_FP8_WRITER_LOGGED
|
||||
if not _FUSED_FP8_WRITER_LOGGED:
|
||||
logger.info("Enabled AITER fused FP8 DSA indexer writer")
|
||||
_FUSED_FP8_WRITER_LOGGED = True
|
||||
self._fused_fp8_rope_cache_key = None
|
||||
self._fused_fp8_cos_cache = None
|
||||
self._fused_fp8_sin_cache = None
|
||||
@@ -394,6 +408,114 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
|
||||
get_exec().kernel.dsa_paged_mqa_logits_backend
|
||||
)
|
||||
|
||||
# gfx950 fused decode indexer. All three conditions below must hold; the
|
||||
# runtime half is dsa/utils.gfx950_fused_indexer_runtime_ok.
|
||||
gfx950_shape_ok = gfx950_model_shape_supported(
|
||||
head_dim=self.head_dim,
|
||||
rope_head_dim=self.rope_head_dim,
|
||||
n_heads=self.n_heads,
|
||||
index_topk=self.index_topk,
|
||||
q_lora_rank=self.q_lora_rank,
|
||||
hidden_size=self.hidden_size,
|
||||
quant_block=self.block_size,
|
||||
scale_fmt=self.scale_fmt,
|
||||
is_neox_style=is_neox_style,
|
||||
k_norm=self.k_norm,
|
||||
num_init_tokens=self.num_init_tokens,
|
||||
num_local_tokens=self.num_local_tokens,
|
||||
)
|
||||
# The fused GEMV pre-merges [wk ; weights_proj] into one bf16 weight,
|
||||
# so both must be bf16; a checkpoint that quantises one of them cannot
|
||||
# take this path. Resolve every module defensively: use_dsa_indexer_fusion
|
||||
# builds wk_weights_proj and leaves wk/weights_proj undefined, and a
|
||||
# quantised checkpoint names its parameter something other than `weight`,
|
||||
# so neither the modules nor their `.weight` can be assumed to exist.
|
||||
wk_w = getattr(getattr(self, "wk", None), "weight", None)
|
||||
wp_w = getattr(getattr(self, "weights_proj", None), "weight", None)
|
||||
wq_b_w = getattr(getattr(self, "wq_b", None), "weight", None)
|
||||
gfx950_weights_ok = (
|
||||
wk_w is not None
|
||||
and wp_w is not None
|
||||
and wq_b_w is not None
|
||||
and wk_w.dtype == torch.bfloat16
|
||||
and wp_w.dtype == torch.bfloat16
|
||||
and wq_b_w.dtype == torch.bfloat16
|
||||
)
|
||||
if gfx950_shape_ok and not gfx950_weights_ok:
|
||||
logger.info(
|
||||
"gfx950 fused DSA indexer disabled: needs bf16 wk/weights_proj/"
|
||||
"wq_b, got %s/%s/%s",
|
||||
getattr(wk_w, "dtype", None),
|
||||
getattr(wp_w, "dtype", None),
|
||||
getattr(wq_b_w, "dtype", None),
|
||||
)
|
||||
# Order matters: gfx950_fused_indexer_runtime_ok() ends by JIT-building
|
||||
# four HIP extensions, so it goes last. Evaluated first it would compile
|
||||
# them on every gfx950 server -- minutes, serialised across TP ranks on
|
||||
# torch's build lock -- for models the shape gate then rejects anyway.
|
||||
self.use_gfx950_fused_indexer = (
|
||||
not self.use_dsa_indexer_fusion
|
||||
and gfx950_shape_ok
|
||||
and gfx950_weights_ok
|
||||
and gfx950_fused_indexer_runtime_ok()
|
||||
)
|
||||
self._gfx950_fused = None
|
||||
if (
|
||||
# Only when asked for by name: unset, this path is a default.
|
||||
get_exec().kernel.enable_dsa_fused_indexer is True
|
||||
and not self.use_gfx950_fused_indexer
|
||||
):
|
||||
logger.info(
|
||||
"gfx950 fused DSA indexer not used on this layer: "
|
||||
"dsa_indexer_fusion=%s shape_supported=%s weights_bf16=%s",
|
||||
self.use_dsa_indexer_fusion,
|
||||
gfx950_shape_ok,
|
||||
gfx950_weights_ok,
|
||||
)
|
||||
if self.use_gfx950_fused_indexer:
|
||||
# The fused kernels write Hadamard-rotated index-K, and prefill must
|
||||
# write the same format; the AITER fused FP8 writer skips the rotation.
|
||||
self.use_aiter_fused_fp8_writer = False
|
||||
self.k_norm = LayerNorm(self.head_dim, dtype=torch.bfloat16)
|
||||
# Off rather than fatal: without the fused path both phases share a format.
|
||||
self.use_gfx950_fused_indexer = hadamard_preserved(self)
|
||||
if self.use_gfx950_fused_indexer:
|
||||
from sglang.kernels.ops.attention.dsa.hip_gfx950 import (
|
||||
Gfx950FusedIndexer,
|
||||
prealloc_workspace,
|
||||
)
|
||||
|
||||
self._gfx950_fused = Gfx950FusedIndexer(self)
|
||||
|
||||
# Take the workspace here, while weights load, so calculate_pool_sizes still
|
||||
# sees the real free memory. Deferring it left the pools sized against memory
|
||||
# that was about to be taken.
|
||||
_PAGE_SLACK = 4 # pages of headroom over the derived width
|
||||
max_ctx = getattr(config, "max_position_embeddings", 0) or 0
|
||||
extra = 4
|
||||
try:
|
||||
from sglang.srt.runtime_context import get_spec
|
||||
|
||||
if get_spec().speculative_num_draft_tokens is not None:
|
||||
extra += int(get_spec().speculative_num_draft_tokens)
|
||||
except Exception: # spec config not available: the +4 still applies
|
||||
pass
|
||||
if max_ctx:
|
||||
pages = -(-(max_ctx + extra) // 64) + _PAGE_SLACK
|
||||
max_ctx = pages * 64
|
||||
if max_ctx:
|
||||
prealloc_workspace(
|
||||
device=torch.device("cuda", torch.cuda.current_device()),
|
||||
max_cols=max_ctx,
|
||||
fp8_dtype=fp8_dtype,
|
||||
)
|
||||
|
||||
# After the gfx950 fused indexer's gate, which may turn the writer off.
|
||||
global _FUSED_FP8_WRITER_LOGGED
|
||||
if self.use_aiter_fused_fp8_writer and not _FUSED_FP8_WRITER_LOGGED:
|
||||
logger.info("Enabled AITER fused FP8 DSA indexer writer")
|
||||
_FUSED_FP8_WRITER_LOGGED = True
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _with_real_sm_count(self):
|
||||
# When pipeline parallelism is enabled, each PP rank initiates a recv operation after the _pp_launch_batch
|
||||
@@ -558,6 +680,146 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
|
||||
return x
|
||||
return rotate_activation(x)
|
||||
|
||||
def _gfx950_fused_decode(
|
||||
self,
|
||||
x,
|
||||
q_lora: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layer_id: int,
|
||||
metadata: Optional[BaseIndexerMetadata],
|
||||
in_graph: bool,
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""Per-call half of the gate plus the four launches. Returns the (rows,
|
||||
index_topk) physical-slot tensor, or None to mean this call is not served
|
||||
here -- forward_cuda then continues into the standard path unchanged."""
|
||||
from sglang.kernels.ops.attention.dsa.hip_gfx950 import (
|
||||
CACHE_TOK_STRIDE,
|
||||
MAX_ROWS,
|
||||
PAGE_SIZE,
|
||||
)
|
||||
from sglang.srt.mem_cache.dsa_cache_layer_split import (
|
||||
LayerSplitDSATokenToKVPool,
|
||||
)
|
||||
|
||||
if metadata is None or in_graph:
|
||||
return None
|
||||
# draft-extend-v2 rows are an extend chunk, not one query per row.
|
||||
if not (
|
||||
forward_batch.forward_mode.is_decode_or_idle()
|
||||
or forward_batch.forward_mode.is_target_verify()
|
||||
):
|
||||
return None
|
||||
# These kernels emit physical page_size=1 slots, which is what the caller
|
||||
# wants only while the fused top-k transform is in play. When the topk is
|
||||
# forced unfused -- SGLANG_DSA_FUSE_TOPK=0, hisparse on decode, or a PD
|
||||
# seed without local-slot remap -- the attention backend transforms the
|
||||
# indexer's output itself and therefore expects logical positions, so
|
||||
# handing it slots would index the page table with slot ids and corrupt
|
||||
# the gather with no error anywhere.
|
||||
if metadata.force_unfused_topk:
|
||||
_decline_fused("the top-k transform is forced unfused")
|
||||
return None
|
||||
# CP gathers K outside the kernel, which this path writes itself.
|
||||
if forward_batch.attn_cp_metadata is not None:
|
||||
_decline_fused("context parallelism gathers K outside the kernel")
|
||||
return None
|
||||
# The fused GEMV takes bf16 hidden states, not the quantised tuple.
|
||||
if isinstance(x, tuple) or x.dtype != torch.bfloat16:
|
||||
_decline_fused("hidden states are not plain bf16")
|
||||
return None
|
||||
# LoRA owns base+delta; the weights this path consumes are base only --
|
||||
# wk/weights_proj pre-merged into one bf16 tensor and wq_b taken straight
|
||||
# from the module. All three are LoRA-targetable (lora/utils.py maps
|
||||
# indexer.wq_b alongside indexer.wk and indexer.weights_proj), so an
|
||||
# adapter on any of them has to fall through to the standard path.
|
||||
if any(
|
||||
getattr(getattr(self, name, None), "set_lora", False)
|
||||
for name in ("weights_proj", "wk", "wq_b")
|
||||
):
|
||||
_decline_fused("a LoRA adapter owns wk / weights_proj / wq_b")
|
||||
return None
|
||||
|
||||
pool = get_token_to_kv_pool()
|
||||
if pool.page_size != PAGE_SIZE:
|
||||
_decline_fused(f"pool page_size is {pool.page_size}, not {PAGE_SIZE}")
|
||||
return None
|
||||
# DSA cache layer split: the standard path stores index-K per layer, and the
|
||||
# fused path addresses the same buffer directly.
|
||||
if isinstance(pool, LayerSplitDSATokenToKVPool):
|
||||
# Ordering, not ownership, is what rules this out. The read buffer is
|
||||
# materialised before state.run() writes this step's K, and for a
|
||||
# split pool materialising it *is* the owner broadcast (invalidate()
|
||||
# clears remote_layer_id, so get_broadcastable_buffer copies straight
|
||||
# away). The current token would therefore be missing from the very
|
||||
# logits that rank it. The standard path stores first and reads
|
||||
# after; matching that here means splitting run() in two, so decline
|
||||
# instead -- this combination is untested for this path anyway.
|
||||
_decline_fused("the DSA cache layer-split pool needs store-then-read")
|
||||
return None
|
||||
|
||||
rows = x.shape[0]
|
||||
if not (1 <= rows <= MAX_ROWS):
|
||||
_decline_fused(f"row count is outside [1, {MAX_ROWS}]")
|
||||
return None
|
||||
# Per-token lengths; decode aliases this to the per-request tensor.
|
||||
seqlens = metadata.get_seqlens_expanded()
|
||||
page_table_64 = metadata.get_page_table_64()
|
||||
if seqlens.shape[0] != rows:
|
||||
return None
|
||||
if page_table_64 is None or page_table_64.shape[0] == 0:
|
||||
return None
|
||||
if not page_table_64.is_contiguous() or page_table_64.dtype != torch.int32:
|
||||
return None
|
||||
out_cache_loc = forward_batch.out_cache_loc
|
||||
if (
|
||||
out_cache_loc.dtype != torch.int64
|
||||
or positions.dtype != torch.int64
|
||||
or out_cache_loc.shape[0] < rows
|
||||
or positions.shape[0] < rows
|
||||
or not out_cache_loc.is_contiguous()
|
||||
or not positions.is_contiguous()
|
||||
):
|
||||
return None
|
||||
|
||||
state = self._gfx950_fused
|
||||
if not state.ensure_workspace(
|
||||
device=x.device,
|
||||
max_cols=page_table_64.shape[1] * PAGE_SIZE,
|
||||
fp8_dtype=fp8_dtype,
|
||||
):
|
||||
return None
|
||||
|
||||
# Draft rows share their request's table; broadcast into the
|
||||
# preallocated buffer, since no allocation is legal under capture.
|
||||
n_pt, blocks = page_table_64.shape
|
||||
if n_pt < rows:
|
||||
if rows % n_pt or rows * blocks > state.workspace.page_table.numel():
|
||||
return None
|
||||
dst = state.workspace.page_table[: rows * blocks].view(rows, blocks)
|
||||
dst.view(n_pt, rows // n_pt, blocks).copy_(page_table_64.unsqueeze(1))
|
||||
page_table_64 = dst
|
||||
else:
|
||||
page_table_64 = page_table_64[:rows]
|
||||
|
||||
if hasattr(pool, "invalidate_index_buffer_for_layer"):
|
||||
pool.invalidate_index_buffer_for_layer(layer_id)
|
||||
# The owned/read buffer split the standard path uses.
|
||||
kv_write = pool.get_index_k_with_scale_buffer(layer_id=layer_id).view(fp8_dtype)
|
||||
kv_read = self._get_index_k_read_buffer(pool, layer_id).view(fp8_dtype)
|
||||
return state.run(
|
||||
x=x,
|
||||
q_lora=q_lora,
|
||||
positions=positions[:rows],
|
||||
out_cache_loc=out_cache_loc[:rows],
|
||||
kv_cache=kv_write.view(-1, PAGE_SIZE, CACHE_TOK_STRIDE),
|
||||
kv_cache_read=kv_read,
|
||||
seqlens_int32=seqlens,
|
||||
row_ends_int32=seqlens,
|
||||
page_table_64=page_table_64,
|
||||
rows=rows,
|
||||
)
|
||||
|
||||
def _should_skip_logits_computation(self, forward_batch: ForwardBatch) -> bool:
|
||||
# topk_transform selects every valid page slot when kv_len <= index_topk;
|
||||
# logits are unnecessary in that case.
|
||||
@@ -1691,6 +1953,25 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
|
||||
topk_result = _broadcast_indexer_topk_from_rank0(topk_result)
|
||||
return maybe_capture_indexer_topk(layer_id, topk_result)
|
||||
|
||||
# gfx950 fused decode indexer (4 kernels). Returns None -- and touches
|
||||
# nothing -- whenever any part of its gate is false, so the standard path
|
||||
# below is reached completely unchanged.
|
||||
if self.use_gfx950_fused_indexer:
|
||||
fused_topk = self._gfx950_fused_decode(
|
||||
x,
|
||||
q_lora,
|
||||
positions,
|
||||
forward_batch,
|
||||
layer_id,
|
||||
metadata,
|
||||
in_piecewise_or_breakable_cuda_graph,
|
||||
)
|
||||
if fused_topk is not None:
|
||||
fused_topk = _broadcast_indexer_topk_from_rank0(
|
||||
fused_topk if return_indices else None
|
||||
)
|
||||
return maybe_capture_indexer_topk(layer_id, fused_topk)
|
||||
|
||||
# When weights_proj is LoRA-wrapped, use an eager module call so the
|
||||
# wrapper owns base+delta and no LoRA kernel runs under torch.compile.
|
||||
# Fusion folds weights_proj into wk_weights_proj, so weights_proj is
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import logging
|
||||
from functools import lru_cache
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
@@ -22,6 +23,8 @@ from sglang.srt.runtime_context import (
|
||||
from sglang.srt.utils import get_bool_env_var, is_cuda, is_hip, is_musa
|
||||
from sglang.srt.utils.common import ceil_div
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def aiter_can_use_preshuffle_paged_mqa() -> bool:
|
||||
@@ -61,6 +64,82 @@ def aiter_can_use_preshuffle_paged_mqa() -> bool:
|
||||
return False
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def gfx950_fused_indexer_runtime_ok() -> bool:
|
||||
"""Whether this runtime can serve the gfx950 fused indexer: aiter with
|
||||
preshuffled paged MQA, and kernels that build.
|
||||
|
||||
Reached only on gfx950, since fused_decode.supported_hardware() is evaluated
|
||||
first. Every decline here is therefore a configuration or toolchain error;
|
||||
it is logged, as a warning when the path was asked for by name."""
|
||||
from sglang.srt.runtime_context import get_exec
|
||||
|
||||
requested = get_exec().kernel.enable_dsa_fused_indexer
|
||||
if requested is False:
|
||||
return False # asked for the standard path; not worth a line per server
|
||||
|
||||
# Every decline logs its reason. Without this the path is invisible: a run
|
||||
# with the switch on and one with it off produce identical logs, and telling
|
||||
# the two apart cost a day of bisecting benchmark results.
|
||||
def _refuse(reason: str) -> bool:
|
||||
# Asked for by name: warn, but still start on the standard path.
|
||||
log = logger.warning if requested is True else logger.info
|
||||
log("gfx950 fused DSA indexer disabled: %s", reason)
|
||||
return False
|
||||
|
||||
# No hardware term here: fused_decode.supported_hardware() is the hardware
|
||||
# half of the gate and runs first, so anything reaching this point is
|
||||
# already on gfx950. What is left is what a deployment can get wrong.
|
||||
if not get_bool_env_var("SGLANG_USE_AITER"):
|
||||
return _refuse("SGLANG_USE_AITER is not set")
|
||||
if not aiter_can_use_preshuffle_paged_mqa():
|
||||
return _refuse("aiter cannot use preshuffled paged MQA logits")
|
||||
from sglang.kernels.ops.attention.dsa.hip_gfx950 import loader
|
||||
|
||||
# modules_or_none logged the build error; do not repeat the compiler output.
|
||||
if loader.modules_or_none() is None:
|
||||
return _refuse("the kernels failed to build, see the warning above")
|
||||
logger.info("gfx950 fused DSA indexer enabled")
|
||||
return True
|
||||
|
||||
|
||||
def gfx950_model_shape_supported(**kwargs) -> bool:
|
||||
"""Static per-model half of the gate: shapes and dtypes that cannot change
|
||||
after load."""
|
||||
from sglang.kernels.ops.attention.dsa.hip_gfx950 import model_shape_supported
|
||||
|
||||
return model_shape_supported(**kwargs)
|
||||
|
||||
|
||||
def hadamard_preserved(indexer) -> bool:
|
||||
"""Whether Indexer._maybe_rotate still applies the Hadamard the fused kernels
|
||||
fold in. If not, the fused path must stay off, or prefill and decode would
|
||||
write different index-K formats."""
|
||||
device = indexer.k_norm.weight.device
|
||||
probe = torch.zeros(1, indexer.head_dim, dtype=torch.bfloat16, device=device)
|
||||
probe[0, 0] = 1.0
|
||||
rotated = indexer._maybe_rotate(probe)
|
||||
# A 128-point Hadamard sends e_0 to a vector whose every entry is 128**-0.5;
|
||||
# the identity leaves 127 zeros. Check the magnitude too, so a transform that
|
||||
# is merely dense does not pass for the rotation the kernels assume.
|
||||
expected = float(indexer.head_dim) ** -0.5
|
||||
if not bool(
|
||||
(rotated != 0).all()
|
||||
and torch.allclose(
|
||||
rotated.float().abs(),
|
||||
torch.full_like(rotated.float(), expected),
|
||||
rtol=0.05,
|
||||
atol=0.0,
|
||||
)
|
||||
):
|
||||
logger.warning(
|
||||
"gfx950 fused DSA indexer disabled: Indexer._maybe_rotate does not "
|
||||
"apply the Hadamard rotation the fused kernels assume"
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
# Tile size for the indexer FP8 K-cache preshuffle layout. Store and gather
|
||||
# kernels reorganize each page into (tile x tile) blocks so the aiter preshuffle
|
||||
# paged-MQA gather can consume the cache directly.
|
||||
|
||||
Reference in New Issue
Block a user