dsv4.1-amd: KV cache layouts, FP4 indexer, compressor and router kernels (#41019)

Co-authored-by: x <x>
This commit is contained in:
Kevin Mi
2026-09-26 19:44:47 -07:00
committed by GitHub
co-authored by x
parent 425a1f8f24
commit effb752188
26 changed files with 2766 additions and 116 deletions
@@ -267,7 +267,7 @@ struct FlashCompress1Kernel {
auto N = SymbolicSize{"num_tokens"};
auto device_ = SymbolicDevice{};
device_.set_options<kDLCUDA>();
device_.set_options<kDLGPU>();
TensorMatcher({N, kHeadDim}) //
.with_dtype<bf16_t>()
@@ -133,8 +133,12 @@ SGL_DEVICE IndexPacked index_rope_quant_pack(fp32x2_t head, fp32x2_t tail, fp32x
// `inv_scale_ue8m0` instead of the reference's division: the scale is a
// power of two, so both are exact except at the unreachable exponent 254.
const auto inv_scale = deepseek_v4::fp8::inv_scale_ue8m0(static_cast<int32_t>(out.exponent[half]));
#ifndef USE_ROCM
const auto code = __nv_cvt_float2_to_fp4x2(
fp32x2_t{data[half * 2] * inv_scale, data[half * 2 + 1] * inv_scale}, __NV_E2M1, cudaRoundNearest);
#else
const auto code = fp4::e2m1x2_code(fp32x2_t{data[half * 2] * inv_scale, data[half * 2 + 1] * inv_scale});
#endif
out.payload[half] = clear_negative_zero(static_cast<uint32_t>(code));
}
return out;
@@ -147,8 +151,15 @@ SGL_DEVICE IndexPacked index_rope_quant_pack(fp32x2_t head, fp32x2_t tail, fp32x
/// the warp. Every lane must reach it: the shuffles are warp-wide.
SGL_DEVICE uint32_t index_scale_word(const uint32_t (&exponent)[2]) {
using namespace device;
#ifndef USE_ROCM
const auto exp_1 = __shfl_sync(warp::kFullMask, exponent[0], kWarpThreads / 2);
const auto exp_3 = __shfl_sync(warp::kFullMask, exponent[1], kWarpThreads / 2);
#else
// A wave holds two of these 32-lane rows, so the shuffle width has to be the
// row, not the wave, for lane 16 to be this row's lane 16.
const auto exp_1 = __shfl(exponent[0], kWarpThreads / 2, kWarpThreads);
const auto exp_3 = __shfl(exponent[1], kWarpThreads / 2, kWarpThreads);
#endif
return exponent[0] | (exp_1 << 8) | (exponent[1] << 16) | (exp_3 << 24);
}
@@ -310,7 +321,7 @@ struct IndexKKernel {
auto N = SymbolicSize{"num_tokens"};
auto device_ = SymbolicDevice{};
device_.set_options<kDLCUDA>();
device_.set_options<kDLGPU>();
TensorMatcher({N, kHeadDim}).with_dtype<bf16_t>().with_device(device_).verify(input);
TensorMatcher({kHeadDim}).with_dtype<bf16_t>().with_device(device_).verify(norm_weight);
@@ -390,7 +401,7 @@ struct IndexQKernel {
auto H = SymbolicSize{"heads"};
auto R = SymbolicSize{"num_rows"};
auto device_ = SymbolicDevice{};
device_.set_options<kDLCUDA>();
device_.set_options<kDLGPU>();
TensorMatcher({N, H, kHeadDim}).with_dtype<bf16_t>().with_device(device_).verify(input);
// Real/imag interleaved, so the trailing dim is kRopeDim, not kRopeDim / 2.
@@ -0,0 +1,156 @@
/// Index-K write of `fp4_indexer_rope.cuh` into the split FlyDSL layout: payload `[npages, 1, 4, kPageSize, 16]`
/// (chunk `c` holds elements `[32c, 32c + 32)`) and ue8m0 exponents `[npages, 1, 4, kPageSize]` with
/// the slot axis transposed as a 16 x 4 tile -- the bytes `store_fp4_index_k_cache_split` writes.
#pragma once
#ifndef USE_ROCM
#error "fp4_indexer_rope_hip.cuh writes the FlyDSL index-K layout, which exists on ROCm only"
#endif
#include "fp4_indexer_rope.cuh"
namespace sglang {
struct IndexKSplitParams {
const bf16_t* __restrict__ input; // [num_tokens, kHeadDim] bf16, pre-norm
const bf16_t* __restrict__ norm_weight; // [kHeadDim] bf16
const float* __restrict__ freqs_cis; // [max_pos, kRopeDim] fp32, real/imag interleaved
const void* __restrict__ positions; // [num_tokens] PosT
const int64_t* __restrict__ loc; // [num_tokens] index-K slot; 0 publishes nothing
uint8_t* __restrict__ payload; // [npages, 1, 4, kPageSize, 16] uint8
uint8_t* __restrict__ scale; // [npages, 1, 4, kPageSize] uint8
uint32_t num_tokens;
float eps;
};
template <bool kUsePDL, int64_t kHeadDim, int64_t kRopeDim, uint32_t kPageSize, uint32_t kRatio, typename PosT>
__global__ __launch_bounds__(kFp4RopeWarpsPerCTA* device::kWarpThreads) void flash_index_k_split_kernel(
const IndexKSplitParams params) {
using namespace device;
static_assert(kPageSize == 64, "the 16 x 4 slot transpose of the scale spans a 64-slot page");
static_assert(
kHeadDim == 128 && kRopeDim == 64,
"the one-warp tiling is specific to a 128-wide row whose second half is the RoPE tail");
static_assert(kHeadDim == 2 * kWarpThreads * 2, "a lane owns one bf16x2 of each half");
static_assert(std::has_single_bit(kRatio), "group_pos is derived by masking, so the ratio must be a power of two");
using bf16_vec_t = AlignedVector<bf16x2_t, 1>;
using fp32_vec_t = AlignedVector<float, 2>;
const auto lane = threadIdx.x % kWarpThreads;
const auto row = blockIdx.x * kFp4RopeWarpsPerCTA + threadIdx.x / kWarpThreads;
// Warp-uniform, so the reductions below still see a full warp.
if (row >= params.num_tokens) return;
// step metadata, not the predecessor's output: loaded ahead of the PDL gate; slot 0 rows drop at the store
const auto slot_id = params.loc[row];
const auto position = static_cast<int64_t>(static_cast<const PosT*>(params.positions)[row]);
PDLWaitPrimary<kUsePDL>();
bf16_vec_t head_in, tail_in, head_w, tail_w;
head_in.load(params.input + row * kHeadDim, lane);
tail_in.load(params.input + row * kHeadDim, lane + kWarpThreads);
head_w.load(params.norm_weight, lane);
tail_w.load(params.norm_weight, lane + kWarpThreads);
fp32_vec_t freq;
freq.load(params.freqs_cis + (position & ~static_cast<int64_t>(kRatio - 1)) * kRopeDim, lane);
fp32x2_t head, tail;
{
const auto [h0, h1] = cast<fp32x2_t>(head_in[0]);
const auto [t0, t1] = cast<fp32x2_t>(tail_in[0]);
const auto sqrsum = warp::reduce_sum(h0 * h0 + h1 * h1 + t0 * t0 + t1 * t1);
const auto inv_rms = math::rsqrt(sqrsum * (1.0f / static_cast<float>(kHeadDim)) + params.eps);
const auto [wh0, wh1] = cast<fp32x2_t>(head_w[0]);
const auto [wt0, wt1] = cast<fp32x2_t>(tail_w[0]);
// `k_norm` materializes a bf16 tensor, so the norm result is rounded before
// anything downstream sees it -- the RoPE below included.
head = cast<fp32x2_t>(cast<bf16x2_t>(fp32x2_t{wh0 * (h0 * inv_rms), wh1 * (h1 * inv_rms)}));
tail = cast<fp32x2_t>(cast<bf16x2_t>(fp32x2_t{wt0 * (t0 * inv_rms), wt1 * (t1 * inv_rms)}));
}
const auto packed = index_rope_quant_pack(head, tail, fp32x2_t{freq[0], freq[1]});
// slot 0 is reserved: padded graph rows and, at ratio > 1, rows completing no group publish nothing
if (slot_id <= 0) return;
const auto page = slot_id / kPageSize;
const auto slot = slot_id % kPageSize;
// byte j of the slot is chunk j / 16, byte j % 16: the head pair at byte lane, the tail pair at lane + 32
const auto payload_ptr = params.payload + page * (4 * kPageSize * 16) + slot * 16;
payload_ptr[(lane / 16) * (kPageSize * 16) + lane % 16] = static_cast<uint8_t>(packed.payload[0]);
payload_ptr[(2 + lane / 16) * (kPageSize * 16) + lane % 16] = static_cast<uint8_t>(packed.payload[1]);
// Lane 0 holds the exponents of blocks 0 and 2, lane 16 those of 1 and 3.
if (lane % (kWarpThreads / 2) == 0) {
const auto shuffled = (slot % 16) * 4 + slot / 16;
const auto scale_ptr = params.scale + page * (4 * kPageSize) + shuffled;
const auto first = lane / (kWarpThreads / 2);
scale_ptr[first * kPageSize] = static_cast<uint8_t>(packed.exponent[0]);
scale_ptr[(2 + first) * kPageSize] = static_cast<uint8_t>(packed.exponent[1]);
}
}
/// \brief Host side of `flash_index_k_split_kernel`.
template <int64_t kHeadDim, int64_t kRopeDim, uint32_t kPageSize, uint32_t kRatio, bool kUsePDL>
struct FlashIndexKSplitKernel {
static constexpr uint32_t kBlockSize = kFp4RopeWarpsPerCTA * device::kWarpThreads;
template <typename PosT>
static constexpr auto kernel = flash_index_k_split_kernel<kUsePDL, kHeadDim, kRopeDim, kPageSize, kRatio, PosT>;
/// `IndexKKernel::run_index_k`'s arguments, the cache replaced by
/// \param payload `[npages, 1, 4, kPageSize, 16]` uint8 (the pool's
/// `float4_e2m1fn_x2` buffer viewed as bytes).
/// \param scale `[npages, 1, 4, kPageSize]` uint8.
static void
run(const tvm::ffi::TensorView input,
const tvm::ffi::TensorView norm_weight,
const tvm::ffi::TensorView freqs_cis,
const tvm::ffi::TensorView positions,
const tvm::ffi::TensorView loc,
const tvm::ffi::TensorView payload,
const tvm::ffi::TensorView scale,
const float eps) {
using namespace host;
auto N = SymbolicSize{"num_tokens"};
auto P = SymbolicSize{"npages"};
auto device_ = SymbolicDevice{};
device_.set_options<kDLGPU>();
TensorMatcher({N, kHeadDim}).with_dtype<bf16_t>().with_device(device_).verify(input);
TensorMatcher({kHeadDim}).with_dtype<bf16_t>().with_device(device_).verify(norm_weight);
// Real/imag interleaved, so the trailing dim is kRopeDim, not kRopeDim / 2.
TensorMatcher({-1, kRopeDim}).with_dtype<fp32_t>().with_device(device_).verify(freqs_cis);
auto pos_dtype = SymbolicDType{};
TensorMatcher({N}).with_dtype<int32_t, int64_t>(pos_dtype).with_device(device_).verify(positions);
TensorMatcher({N}).with_dtype<int64_t>().with_device(device_).verify(loc);
TensorMatcher({P, 1, 4, kPageSize, 16}).with_dtype<uint8_t>().with_device(device_).verify(payload);
TensorMatcher({P, 1, 4, kPageSize}).with_dtype<uint8_t>().with_device(device_).verify(scale);
const auto num_tokens = static_cast<uint32_t>(N.unwrap());
if (num_tokens == 0) return;
const auto params = IndexKSplitParams{
.input = static_cast<const bf16_t*>(input.data_ptr()),
.norm_weight = static_cast<const bf16_t*>(norm_weight.data_ptr()),
.freqs_cis = static_cast<const float*>(freqs_cis.data_ptr()),
.positions = positions.data_ptr(),
.loc = static_cast<const int64_t*>(loc.data_ptr()),
.payload = static_cast<uint8_t*>(payload.data_ptr()),
.scale = static_cast<uint8_t*>(scale.data_ptr()),
.num_tokens = num_tokens,
.eps = eps,
};
const auto k_int32 = kernel<int32_t>;
const auto k_int64 = kernel<int64_t>;
const auto k = pos_dtype.is_type<int32_t>() ? k_int32 : k_int64;
LaunchKernel(div_ceil(num_tokens, kFp4RopeWarpsPerCTA), kBlockSize, device_.unwrap()) //
.enable_pdl(kUsePDL)(k, params);
}
};
} // namespace sglang
@@ -15,6 +15,7 @@
#include <bit>
#include <cstdint>
#include <type_traits>
namespace sglang {
@@ -258,6 +259,12 @@ struct FusedKNormRopeFlashMLAParams {
int64_t kv_stride_batch;
uint32_t batch_size;
float eps;
// kRopeQ (ROCm): (B, num_q_heads, kHeadDim) DType; the trailing kRopeDim of every head
// are rotated in place with this token's frequencies by the same launch.
void* __restrict__ q = nullptr;
int64_t q_stride_batch = 0;
int64_t q_stride_head = 0;
uint32_t num_q_heads = 0;
};
template <
@@ -267,7 +274,8 @@ template <
typename PosT,
int32_t kPageBits,
deepseek_v4::KVLayout kLayout,
bool kUsePDL>
bool kUsePDL,
bool kRopeQ = false>
K_KERNEL void fused_k_norm_rope_flashmla(const __grid_constant__ FusedKNormRopeFlashMLAParams params) {
using namespace device;
@@ -327,6 +335,30 @@ K_KERNEL void fused_k_norm_rope_flashmla(const __grid_constant__ FusedKNormRopeF
}
}
// Before the out_loc check: a row without a KV slot still ropes its query heads.
if constexpr (kRopeQ) {
// Query rope, pair j of head h at q[h * stride + kHeadDim - kRopeDim + 2j]: the cross product
// is rounded, then one fma with the cosine, so the bf16 result is bitwise the Triton flat
// rope kernel's on gfx950.
static_assert(std::is_same_v<DType, bf16_t>, "the in-place query rope reads q as bf16 pairs");
constexpr uint32_t kPairsPerHead = kRopeDim / 2;
const auto q_row = static_cast<DType*>(params.q) + work_id * params.q_stride_batch + (kHeadDim - kRopeDim);
const auto n_pairs = params.num_q_heads * kPairsPerHead;
for (uint32_t p = tx; p < n_pairs; p += kFusedKBlockSize) {
const auto head = p / kPairsPerHead;
const auto pair = p % kPairsPerHead;
auto* ptr = reinterpret_cast<bf16x2_t*>(q_row + head * params.q_stride_head) + pair;
const auto qv = cast<fp32x2_t>(*ptr);
const auto cos_v = freqs_cis[2 * pair];
const auto sin_v = freqs_cis[2 * pair + 1];
const float rot_real = -__fmul_rn(qv.y, sin_v);
const float rot_imag = __fmul_rn(qv.x, sin_v);
const float out_real = fmaf(qv.x, cos_v, rot_real);
const float out_imag = fmaf(qv.y, cos_v, rot_imag);
*ptr = cast<bf16x2_t>(fp32x2_t{out_real, out_imag});
}
}
// A negative out_loc marks a slot with no KV write target (e.g. the -1
// sentinel from the full->SWA translation for out-of-window tokens or
// padded rows); skip the row instead of writing out of bounds. Checked
@@ -398,9 +430,9 @@ struct FusedKNormRopeFlashMLAKernel {
static_assert(1 << kLogPageSize == kPageSize);
static_assert(kHeadDim == 512 && kRopeDim == 64, "FlashMLA layout requires (512, 64)");
template <typename PosT>
template <typename PosT, bool kRopeQ>
static constexpr auto kernel =
fused_k_norm_rope_flashmla<DType, kHeadDim, kRopeDim, PosT, kLogPageSize, kLayout, kUsePDL>;
fused_k_norm_rope_flashmla<DType, kHeadDim, kRopeDim, PosT, kLogPageSize, kLayout, kUsePDL, kRopeQ>;
static void forward(
const tvm::ffi::TensorView kv,
@@ -410,6 +442,33 @@ struct FusedKNormRopeFlashMLAKernel {
const tvm::ffi::TensorView out_loc,
const tvm::ffi::TensorView kvcache,
float eps) {
forward_impl<false>(kv, kv_weight, freqs_cis, positions, out_loc, kvcache, eps, nullptr);
}
/// `forward` with the same tokens' query heads `q` (B, H, kHeadDim) roped in place (ROCm).
static void forward_with_q(
const tvm::ffi::TensorView kv,
const tvm::ffi::TensorView kv_weight,
const tvm::ffi::TensorView freqs_cis,
const tvm::ffi::TensorView positions,
const tvm::ffi::TensorView out_loc,
const tvm::ffi::TensorView kvcache,
float eps,
const tvm::ffi::TensorView q) {
forward_impl<true>(kv, kv_weight, freqs_cis, positions, out_loc, kvcache, eps, &q);
}
private:
template <bool kRopeQ>
static void forward_impl(
const tvm::ffi::TensorView kv,
const tvm::ffi::TensorView kv_weight,
const tvm::ffi::TensorView freqs_cis,
const tvm::ffi::TensorView positions,
const tvm::ffi::TensorView out_loc,
const tvm::ffi::TensorView kvcache,
float eps,
const tvm::ffi::TensorView* q) {
using namespace host;
auto B = SymbolicSize{"batch_size"};
@@ -447,7 +506,7 @@ struct FusedKNormRopeFlashMLAKernel {
const auto batch_size = static_cast<uint32_t>(B.unwrap());
if (batch_size == 0) return;
const auto params = FusedKNormRopeFlashMLAParams{
auto params = FusedKNormRopeFlashMLAParams{
.kv = kv.data_ptr(),
.kv_weight = kv_weight.data_ptr(),
.freqs_cis = static_cast<const float*>(freqs_cis.data_ptr()),
@@ -458,8 +517,22 @@ struct FusedKNormRopeFlashMLAKernel {
.batch_size = batch_size,
.eps = eps,
};
const auto k_int32 = kernel<int32_t>;
const auto k_int64 = kernel<int64_t>;
if constexpr (kRopeQ) {
auto H = SymbolicSize{"num_q_heads"};
TensorMatcher({B, H, kHeadDim}) //
.with_strides({-1, -1, 1})
.with_dtype<DType>()
.with_device(device_)
.verify(*q);
// The rope pairs are read and written as 4-byte packs.
RuntimeCheck(q->stride(1) % 2 == 0 && q->stride(0) % 2 == 0, "q strides must be even");
params.q = q->data_ptr();
params.q_stride_batch = q->stride(0);
params.q_stride_head = q->stride(1);
params.num_q_heads = static_cast<uint32_t>(H.unwrap());
}
const auto k_int32 = kernel<int32_t, kRopeQ>;
const auto k_int64 = kernel<int64_t, kRopeQ>;
const auto k = pos_dtype.is_type<int32_t>() ? k_int32 : k_int64;
LaunchKernel(batch_size, kFusedKBlockSize, device_.unwrap()) //
.enable_pdl(kUsePDL)(k, params);
@@ -15,7 +15,9 @@
#include <bit>
#include <cstdint>
#ifndef USE_ROCM
#include <cuda_fp8.h>
#endif
#include <optional>
namespace sglang {
@@ -23,18 +23,66 @@ constexpr float kAmaxFloor = 6.0f * 1.1754943508222875e-38f;
constexpr uint32_t kBlockSize = 32;
/// Compressed-KV elements sharing one E4M3 scale.
constexpr uint32_t kCompressedKVBlockSize = 16;
#ifdef USE_ROCM
/// \brief `cvt.rn.satfinite.e2m1x2.f32` for one value, in software: RNE onto the
/// e2m1 grid {0, .5, 1, 1.5, 2, 3, 4, 6}, saturated at 6, sign kept (also for -0).
SGL_DEVICE uint32_t e2m1_code(float x) {
const float mag = fminf(fabsf(x), kMax);
const float step = mag < 2.0f ? 0.5f : (mag < 4.0f ? 1.0f : 2.0f);
const float q = rintf(mag / step) * step;
// q in {0, .5, 1, 1.5} -> 2q; {2, 3} -> q + 2; {4, 6} -> q / 2 + 4.
const uint32_t idx = q < 2.0f ? static_cast<uint32_t>(q * 2.0f)
: (q < 4.0f ? static_cast<uint32_t>(q) + 2u : static_cast<uint32_t>(q * 0.5f) + 4u);
return idx | (__float_as_uint(x) >> 31 << 3);
}
/// \brief The e2m1 grid value a code stands for, exact in fp32.
SGL_DEVICE float e2m1_value(uint32_t code) {
const uint32_t idx = code & 0x7u;
const float mag = idx < 4u ? static_cast<float>(idx) * 0.5f
: (idx < 6u ? static_cast<float>(idx) - 2.0f : static_cast<float>(idx - 4u) * 2.0f);
return (code & 0x8u) ? -mag : mag;
}
/// \brief `__nv_cvt_float2_to_fp4x2(.., __NV_E2M1, cudaRoundNearest)`: `.x` in
/// the low nibble, `.y` in the high one.
SGL_DEVICE uint32_t e2m1x2_code(fp32x2_t x) {
return e2m1_code(x.x) | (e2m1_code(x.y) << 4);
}
/// \brief `float(__nv_fp8_e4m3(raw))` for `raw` in `[2^-9, 448]`: RNE onto the E4M3FN
/// grid, gfx950's `v_cvt_pk_fp8_f32` in hardware, software elsewhere (gfx942's fp8 is FNUZ).
SGL_DEVICE float e4m3_round_rn(float raw) {
#if defined(__HIP_DEVICE_COMPILE__) && defined(__gfx950__)
const auto packed = __builtin_amdgcn_cvt_pk_fp8_f32(raw, raw, 0, false);
return __builtin_amdgcn_cvt_f32_fp8(packed, 0);
#else
const int exponent = ilogbf(raw);
const float step = exponent < -6 ? 0x1p-9f : ldexpf(1.0f, exponent - 3);
return rintf(raw / step) * step;
#endif
}
#endif
/// \brief Round amax / 6 to a positive finite E4M3 scale, ties to even.
SGL_DEVICE float compressed_kv_scale(float amax) {
const auto raw = fminf(fmaxf(amax * (1.0f / kMax), 0x1p-9f), 448.0f);
#ifdef USE_ROCM
return e4m3_round_rn(raw);
#else
return static_cast<float>(__nv_fp8_e4m3(raw));
#endif
}
/// \brief Quantize compressed KV with its E4M3 scale and return dequantized values.
SGL_DEVICE fp32x2_t fake_quant_compressed_kv_x2(fp32x2_t x, float scale) {
const fp32x2_t scaled{__fdiv_rn(x.x, scale) + 0.0f, __fdiv_rn(x.y, scale) + 0.0f};
#ifdef USE_ROCM
const fp32x2_t grid{e2m1_value(e2m1_code(scaled.x)), e2m1_value(e2m1_code(scaled.y))};
#else
const auto code = __nv_cvt_float2_to_fp4x2(scaled, __NV_E2M1, cudaRoundNearest);
const auto grid = device::cast<fp32x2_t>(fp16x2_t{__nv_cvt_fp4x2_to_halfraw2(code, __NV_E2M1)});
#endif
return {grid.x * scale, grid.y * scale};
}
@@ -50,8 +98,12 @@ SGL_DEVICE fp32x2_t block_scale(float amax) {
/// scaling clears negative zero to match `torch.sign(0) == 0` in `torch_quant.round_fp4`.
SGL_DEVICE fp32x2_t fake_quant_x2(fp32x2_t x, float scale, float inv_scale) {
const fp32x2_t scaled{__fmaf_rn(x.x, inv_scale, 0.0f), __fmaf_rn(x.y, inv_scale, 0.0f)};
#ifdef USE_ROCM
const fp32x2_t grid{e2m1_value(e2m1_code(scaled.x)), e2m1_value(e2m1_code(scaled.y))};
#else
const auto code = __nv_cvt_float2_to_fp4x2(scaled, __NV_E2M1, cudaRoundNearest);
const auto grid = device::cast<fp32x2_t>(fp16x2_t{__nv_cvt_fp4x2_to_halfraw2(code, __NV_E2M1)});
#endif
return {grid.x * scale, grid.y * scale};
}
@@ -5,8 +5,10 @@
#include <sgl_kernel/vec.cuh>
#include <sgl_kernel/warp.cuh>
#include <sgl_kernel/deepseek_v4/fp4_utils.cuh>
#include <sgl_kernel/deepseek_v4/fp8_utils.cuh>
#include <bit>
#include <cstdint>
#ifndef USE_ROCM
#include <cuda_fp4.h>
@@ -111,8 +113,6 @@ struct PagedKV {
}
};
#ifndef USE_ROCM
namespace v41 {
/// The row helpers below quantize one 512-wide token spread over `512 / kVecSize` threads,
@@ -152,7 +152,11 @@ SGL_DEVICE void store_row_fp8(uint8_t* data_row, uint8_t* scale_row, uint32_t tx
#pragma unroll
for (uint32_t i = 0; i < kVecSize / 2; ++i) {
// `cvt.rn.satfinite.e4m3x2` directly: |x / scale| <= 448 needs no clamp.
#ifdef USE_ROCM
out[i] = fp8::pack_fp8(v[2 * i] * inv_scale, v[2 * i + 1] * inv_scale);
#else
out[i] = fp8x2_e4m3_t{fp32x2_t{v[2 * i] * inv_scale, v[2 * i + 1] * inv_scale}};
#endif
}
out.store(data_row, tx);
// Every lane of the tile holds the exponent; they all store the same byte.
@@ -170,18 +174,29 @@ SGL_DEVICE void store_row_fp4(uint8_t* data_row, uint8_t* scale_row, uint32_t tx
static_assert(kVecSize % 2 == 0 && kTileLanes >= 1 && (kTileLanes & (kTileLanes - 1)) == 0);
const float amax = warp::reduce_max<kTileLanes>(vec_amax(v));
#ifdef USE_ROCM
const float scale = fp4::e4m3_round_rn(fminf(fmaxf(__fdiv_rn(amax, 6.0f), 0x1p-9f), 448.0f));
// the low byte of the pack's 16-bit storage, whether fp8x2_e4m3_t is an integer or HIP's fp8 pair
const auto scale_bits = static_cast<uint8_t>(std::bit_cast<uint16_t>(fp8::pack_fp8(scale, scale)));
#else
const __nv_fp8_e4m3 scale_e4m3{fminf(fmaxf(__fdiv_rn(amax, 6.0f), 0x1p-9f), 448.0f)};
const float scale = static_cast<float>(scale_e4m3);
const uint8_t scale_bits = scale_e4m3.__x;
#endif
AlignedVector<uint8_t, kVecSize / 2> out;
#pragma unroll
for (uint32_t i = 0; i < kVecSize / 2; ++i) {
// IEEE division by the rounded scale, as the reference does; a reciprocal multiply could cross an e2m1 tie.
#ifdef USE_ROCM
out[i] = static_cast<uint8_t>(fp4::e2m1x2_code({__fdiv_rn(v[2 * i], scale), __fdiv_rn(v[2 * i + 1], scale)}));
#else
out[i] = static_cast<uint8_t>(__nv_cvt_float2_to_fp4x2(
fp32x2_t{__fdiv_rn(v[2 * i], scale), __fdiv_rn(v[2 * i + 1], scale)}, __NV_E2M1, cudaRoundNearest));
#endif
}
out.store(data_row, tx);
// Every lane of the tile holds the scale; they all store the same byte.
scale_row[tx / kTileLanes] = scale_e4m3.__x;
scale_row[tx / kTileLanes] = scale_bits;
}
/// Dispatch on the layout for a 512-wide row held as `kVecSize` consecutive fp32 per thread.
@@ -205,24 +220,6 @@ store_row(uint8_t* data_row, uint8_t* scale_row, uint32_t tx, const device::Alig
} // namespace v41
#else // USE_ROCM
namespace v41 {
template <KVLayout>
inline constexpr bool dependent_false_v = false;
/// A discarded `if constexpr` branch is still parsed, so the V4 callers need the name to
/// resolve on ROCm; instantiating it is the error.
template <KVLayout kLayout, typename... Ts>
SGL_DEVICE void store_row(uint8_t*, uint8_t*, uint32_t, Ts&&...) {
static_assert(dependent_false_v<kLayout>, "the V4.1 KV layouts are CUDA only");
}
} // namespace v41
#endif // USE_ROCM
} // namespace deepseek_v4
} // namespace sglang
@@ -11,6 +11,7 @@ from sglang.kernels.jit.utils import (
load_jit,
make_cpp_args,
)
from sglang.srt.utils import is_gfx95_supported
from .kv_layout import KVLayout
from .utils import make_name
@@ -95,16 +96,17 @@ def fused_store_cache(
RoPE tail in-kernel, so ``input`` must then be the un-rotated latent.
"""
layout = KVLayout.parse(layout)
if is_hip_runtime():
assert layout is KVLayout.V4 and freqs_cis is None, (
"the V4.1 KV layouts are CUDA (sm100) only"
)
if is_hip_runtime() and layout is KVLayout.V4:
assert freqs_cis is None, "the V4 store receives already-rotated inputs"
from sglang.kernels.ops.kvcache.triton_store_cache import (
triton_fused_store_cache,
)
triton_fused_store_cache(input, cache, indices, page_size=page_size, type=type)
else:
assert not is_hip_runtime() or is_gfx95_supported(), (
"V4.1 KV stores on HIP require gfx950"
)
module = _jit_fused_store_module(
name=type,
input_dtype=input.dtype,
@@ -89,9 +89,9 @@ def _c2_decode_pool_kernel(
e1 = libdevice.exp(score - m)
denom = e0 + e1
# The + 0.0 below prevents FMA contraction: torch rounds both products first.
# libdevice.div_rn matches torch division; Triton's / is an approximate reciprocal.
t0 = p_kv * libdevice.div_rn(e0, denom)
t1 = kv * libdevice.div_rn(e1, denom)
# tl.div_rn matches torch division; Triton's / is an approximate reciprocal on CUDA.
t0 = p_kv * tl.div_rn(e0, denom)
t1 = kv * tl.div_rn(e1, denom)
t0 = t0 + 0.0
t1 = t1 + 0.0
pooled = t0 + t1
@@ -14,7 +14,7 @@ from sglang.srt.layers.attention.dsa.utils import (
INDEXER_K_CACHE_PRESHUFFLE_TILE,
aiter_can_use_preshuffle_paged_mqa,
)
from sglang.srt.utils import is_hip, is_xpu
from sglang.srt.utils import is_gfx95_supported, is_hip, is_xpu
from .kv_layout import KVLayout
from .utils import make_name
@@ -467,7 +467,10 @@ def compress_norm_rope_store(
assert kv.shape[-1] == 512 and not use_fp4 and not bf16_store, (
"the V4.1 layouts are paged FlashMLA main-KV caches"
)
assert not is_hip() and not _is_xpu, "the V4.1 KV layouts are CUDA (sm100) only"
assert not _is_xpu, "the V4.1 KV layouts are not supported on XPU"
assert not is_hip() or is_gfx95_supported(), (
"V4.1 KV stores on HIP require gfx950"
)
if use_fp4:
assert kv.shape[-1] == 128
if is_hip() and use_fp4:
@@ -7,6 +7,7 @@ import triton.language as tl
from sglang.kernels.jit.utils import get_jit_cuda_arch, is_hip_runtime
from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout
from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
from sglang.srt.utils import is_gfx95_supported
fp8_dtype = torch.float8_e4m3fnuz if is_fp8_fnuz() else torch.float8_e4m3fn
@@ -108,9 +109,13 @@ def dequantize_k_cache_paged_v41(
"""
layout = KVLayout.parse(layout)
assert layout in (KVLayout.V41, KVLayout.V41_FP4), layout
if is_hip_runtime() or get_jit_cuda_arch().major < 10:
if is_hip_runtime():
supported = is_gfx95_supported()
else:
supported = get_jit_cuda_arch().major >= 10
if not supported:
raise RuntimeError(
"DeepSeek V4.1 KV cache dequantization requires CUDA SM100 or newer"
"DeepSeek V4.1 KV cache dequantization requires CUDA SM100 or newer, or gfx950"
)
assert quant_k_cache.is_contiguous()
assert page_table_1_flattened.dtype in (torch.int32, torch.int64)
@@ -153,6 +158,7 @@ def dequantize_k_cache_paged_v41(
SCALE_BYTES=layout.scale_bytes,
TILE_SIZE=layout.tile_size,
S_OFFSET_BYTES=layout.scale_offset(page_size),
IS_HIP=is_hip_runtime(),
)
return out
@@ -367,6 +373,18 @@ def _e2m1_code_to_fp32(code):
return (mag.to(tl.int32, bitcast=True) | sign).to(tl.float32, bitcast=True)
@triton.jit
def _v41_dequant_to_bf16(value, IS_HIP: tl.constexpr):
result = value.to(tl.bfloat16)
if IS_HIP:
# HIP conversion can produce negative NaNs; match the torch format
# reference's canonical positive BF16 NaN without changing finite bits.
bits = result.to(tl.uint16, bitcast=True)
bits = tl.where((bits & 0x7FFF) > 0x7F80, 0x7FC0, bits).to(tl.uint16)
result = bits.to(tl.bfloat16, bitcast=True)
return result
@triton.jit
def _dequantize_k_cache_paged_v41_fp8_kernel(
output_ptr,
@@ -380,6 +398,7 @@ def _dequantize_k_cache_paged_v41_fp8_kernel(
SCALE_BYTES: tl.constexpr,
TILE_SIZE: tl.constexpr,
S_OFFSET_BYTES: tl.constexpr,
IS_HIP: tl.constexpr,
):
# V41: 512 e4m3 values per token, then 16 ue8m0 scales (one per 32 values).
tl.static_assert(DATA_BYTES == 512 and SCALE_BYTES == 16 and TILE_SIZE == 32)
@@ -397,7 +416,7 @@ def _dequantize_k_cache_paged_v41_fp8_kernel(
out = vals * _ue8m0_to_fp32(scale_u8)
tl.store(
output_ptr + token_id * output_stride_0 + offs,
out.to(output_ptr.dtype.element_ty),
_v41_dequant_to_bf16(out, IS_HIP),
)
@@ -414,6 +433,7 @@ def _dequantize_k_cache_paged_v41_fp4_kernel(
SCALE_BYTES: tl.constexpr,
TILE_SIZE: tl.constexpr,
S_OFFSET_BYTES: tl.constexpr,
IS_HIP: tl.constexpr,
):
# V41_FP4: 512 e2m1 codes packed two per byte (even index in the low nibble),
# then 32 e4m3 scales (one per 16 values).
@@ -435,8 +455,8 @@ def _dequantize_k_cache_paged_v41_fp4_kernel(
lo = _e2m1_code_to_fp32(packed & 0xF) * scale
hi = _e2m1_code_to_fp32(packed >> 4) * scale
out_base = output_ptr + token_id * output_stride_0
tl.store(out_base + 2 * boffs, lo.to(output_ptr.dtype.element_ty))
tl.store(out_base + 2 * boffs + 1, hi.to(output_ptr.dtype.element_ty))
tl.store(out_base + 2 * boffs, _v41_dequant_to_bf16(lo, IS_HIP))
tl.store(out_base + 2 * boffs + 1, _v41_dequant_to_bf16(hi, IS_HIP))
@triton.jit
@@ -72,6 +72,28 @@ def _jit_main_k_norm_rope_flashmla_module(
)
@cache_once
def _jit_main_k_norm_rope_q_flashmla_module(
dtype: torch.dtype,
head_dim: int,
rope_dim: int,
page_size: int,
layout: KVLayout,
):
"""ROCm only: the K kernel above with the in-place query rope in the same launch."""
args = make_cpp_args(
dtype, head_dim, rope_dim, page_size, layout.cpp_name, is_arch_support_pdl()
)
return load_jit(
make_name("main_k_norm_rope_q_flashmla"),
*args,
cuda_files=["deepseek_v4/main_norm_rope.cuh"],
cuda_wrappers=[
("forward_with_q", f"FusedKNormRopeFlashMLAKernel<{args}>::forward_with_q"),
],
)
@cache_once
def _jit_main_q_indexer_rope_hadamard_quant_module(dtype: torch.dtype):
"""C4 indexer Q kernel: RoPE + 128-pt Hadamard + fp8 act-quant"""
@@ -278,6 +300,7 @@ def fused_k_norm_rope_flashmla(
kvcache: torch.Tensor,
page_size: int,
layout: Union[KVLayout, str] = KVLayout.V4,
q: Optional[torch.Tensor] = None,
) -> None:
"""RMSNorm + RoPE ``kv`` and write it into the ``layout`` paged FlashMLA
cache at ``out_loc``."""
@@ -287,9 +310,18 @@ def fused_k_norm_rope_flashmla(
rope_dim = freqs_real.shape[-1]
if _is_xpu:
assert layout is KVLayout.V4, "the V4.1 KV layouts are CUDA (sm100) only"
assert q is None, "the XPU K kernel does not rope q"
fused_k_norm_rope_flashmla_xpu(
kv, kv_weight, freqs_real, positions, out_loc, kvcache, eps, page_size
)
elif q is not None:
assert _is_hip, "only the ROCm K launch ropes q"
module = _jit_main_k_norm_rope_q_flashmla_module(
kv.dtype, head_dim, rope_dim, page_size, layout
)
module.forward_with_q(
kv, kv_weight, freqs_real, positions, out_loc, kvcache, eps, q
)
else:
module = _jit_main_k_norm_rope_flashmla_module(
kv.dtype, head_dim, rope_dim, page_size, layout
@@ -74,17 +74,19 @@ def _fp4_e2m1_code_rne(x):
@triton.jit
def _quantize_fp4_indexer_kernel(
x,
x_fp4,
x_sf,
def quantize_fp4_indexer_row(
values,
v0,
v1,
BLOCK_N: tl.constexpr,
GROUP_N: tl.constexpr,
RNE: tl.constexpr,
):
token_id = tl.program_id(0)
"""One row of quantize_fp4_indexer_tensor: values is the fp32 [BLOCK_N] row,
v0 / v1 its even / odd elements as fp32 [BLOCK_N // 2]. Returns
(packed codes [BLOCK_N // 2], packed ue8m0 exponents), the two stores of the
standalone kernel below."""
offs = tl.arange(0, BLOCK_N)
values = tl.load(x + token_id * BLOCK_N + offs).to(tl.float32)
abs_values = tl.abs(values)
amax0 = tl.max(tl.where(offs < GROUP_N, abs_values, 0.0), axis=0)
@@ -109,7 +111,6 @@ def _quantize_fp4_indexer_kernel(
exp3 = _ceil_ue8m0_exp(sf3)
packed_sf = exp0 | (exp1 << 8) | (exp2 << 16) | (exp3 << 24)
tl.store(x_sf + token_id, packed_sf)
pair_offsets = tl.arange(0, BLOCK_N // 2)
offs0 = pair_offsets * 2
@@ -121,8 +122,8 @@ def _quantize_fp4_indexer_kernel(
scale0 = (scale_exp0 << 23).to(tl.float32, bitcast=True)
scale1 = (scale_exp1 << 23).to(tl.float32, bitcast=True)
v0 = tl.load(x + token_id * BLOCK_N + offs0).to(tl.float32) / scale0
v1 = tl.load(x + token_id * BLOCK_N + offs1).to(tl.float32) / scale1
v0 = v0 / scale0
v1 = v1 / scale1
if RNE:
code0 = _fp4_e2m1_code_rne(v0)
code1 = _fp4_e2m1_code_rne(v1)
@@ -130,6 +131,29 @@ def _quantize_fp4_indexer_kernel(
code0 = _fp4_e2m1_code(v0)
code1 = _fp4_e2m1_code(v1)
packed = (code0 & 0x0F) | ((code1 & 0x0F) << 4)
return packed, packed_sf
@triton.jit
def _quantize_fp4_indexer_kernel(
x,
x_fp4,
x_sf,
BLOCK_N: tl.constexpr,
GROUP_N: tl.constexpr,
RNE: tl.constexpr,
):
token_id = tl.program_id(0)
offs = tl.arange(0, BLOCK_N)
pair_offsets = tl.arange(0, BLOCK_N // 2)
row = x + token_id * BLOCK_N
values = tl.load(row + offs).to(tl.float32)
v0 = tl.load(row + pair_offsets * 2).to(tl.float32)
v1 = tl.load(row + pair_offsets * 2 + 1).to(tl.float32)
packed, packed_sf = quantize_fp4_indexer_row(
values, v0, v1, BLOCK_N=BLOCK_N, GROUP_N=GROUP_N, RNE=RNE
)
tl.store(x_sf + token_id, packed_sf)
tl.store(x_fp4 + token_id * (BLOCK_N // 2) + pair_offsets, packed)
@@ -6,6 +6,18 @@ import os
from typing import TYPE_CHECKING, NamedTuple, Optional, Tuple, Union
import torch
import triton
import triton.language as tl
from sglang.kernels.ops.attention.dsv4.fp4_indexer import (
quantize_fp4_indexer_row,
quantize_fp4_indexer_tensor,
)
from sglang.kernels.ops.attention.dsv4.fp4_rope_fake_quant import (
FP4_AMAX_FLOOR,
rope_tail_fake_quant_fp4_row,
)
from sglang.kernels.ops.gemm.router_gemv_hip import rocm_router_gemv_split_k
if TYPE_CHECKING:
from sglang.kernels.ops.attention.dsv4.compress import (
@@ -28,6 +40,8 @@ _DECODE_BASE_CTA_TARGET = 1024
# Preserve per-query parallelism when the batch itself exceeds one CTA per CU.
_DECODE_CTAS_PER_QUERY = 4
_PREFILL_BASE_CTA_TARGET = 1024
# FlyDSL compiles one kernel per page-table width: bucket it so a new context length pays no JIT
LOW_RATIO_PAGE_TABLE_BUCKET = 64
# AITER varctx cta_info row: [batch_packed, chunk_start, chunk_count, ctx_len].
_DECODE_CTA_INFO_WIDTH = 4
@@ -129,23 +143,32 @@ def _decode_cta_count(num_queries: int, max_seq_len: int) -> int:
return min(available_ctas, target_ctas)
def _guarded_pages(logical_width: int) -> int:
def _guarded_pages(logical_width: int, page_table_bucket: int = 4) -> int:
"""Page columns after padding for 256-token scheduling."""
return max(4, (logical_width + 3) // 4 * 4)
return max(
4,
(logical_width + page_table_bucket - 1)
// page_table_bucket
* page_table_bucket,
)
def _guard_page_table(page_table: torch.Tensor, out: Optional[torch.Tensor] = None):
def _guard_page_table(
page_table: torch.Tensor,
out: Optional[torch.Tensor] = None,
page_table_bucket: int = 4,
):
"""Pad page tables for 256-token scheduling and one-chunk lookahead."""
from sglang.kernels.ops.attention.dsv4.fp4_indexer_schedule_hip import (
pad_page_table,
)
return pad_page_table(page_table, out=out)
return pad_page_table(page_table, out=out, page_table_bucket=page_table_bucket)
def logits_rows_per_chunk(page_table: torch.Tensor) -> int:
def logits_rows_per_chunk(page_table: torch.Tensor, page_table_bucket: int = 4) -> int:
"""Rows whose logits fit the pooled block, for callers that loop by row."""
width = _guarded_pages(page_table.shape[1]) * _KV_BLOCK_SIZE
width = _guarded_pages(page_table.shape[1], page_table_bucket) * _KV_BLOCK_SIZE
return max(1, _LOGITS_BUDGET_ELEMS // width)
@@ -188,6 +211,7 @@ def _alloc_logits(
def prepare_fp4_decode_workspace(
page_table: torch.Tensor,
c4_seq_lens: torch.Tensor,
page_table_bucket: int = 4,
) -> FP4DecodeWorkspace:
"""Build the decode page-table, schedule, and logits buffers.
@@ -195,11 +219,13 @@ def prepare_fp4_decode_workspace(
kernel touches is reachable from the returned workspace, so none of it can
be handed out again by a later capture sharing the graph memory pool.
"""
from aiter.ops.flydsl.kernels.mqa_logits.pa_mqa_logits_fp4 import (
compute_varctx_schedule,
from sglang.kernels.ops.attention.dsv4.fp4_indexer_schedule_hip import (
build_decode_schedule,
)
guarded, max_seq_len = _guard_page_table(page_table)
guarded, max_seq_len = _guard_page_table(
page_table, page_table_bucket=page_table_bucket
)
c4_seq_lens = _as_int32_1d(c4_seq_lens)
num_queries = guarded.shape[0]
cta_count = _decode_cta_count(num_queries, max_seq_len)
@@ -208,13 +234,8 @@ def prepare_fp4_decode_workspace(
dtype=torch.int32,
device=guarded.device,
)
schedule_scratch, _, _ = compute_varctx_schedule(
c4_seq_lens,
block_k=256,
parallel_unit_num=cta_count,
max_seq_len=max_seq_len,
next_n=1,
cta_info_out=cta_info,
schedule_scratch = build_decode_schedule(
c4_seq_lens, cta_info_out=cta_info, max_seq_len=max_seq_len
)
return FP4DecodeWorkspace(
guarded, c4_seq_lens, cta_info, cta_count, max_seq_len, schedule_scratch
@@ -225,6 +246,7 @@ def prepare_fp4_prefill_workspace(
page_table: torch.Tensor,
c4_seq_lens: torch.Tensor,
workspace: Optional[FP4PrefillWorkspace] = None,
page_table_bucket: int = 4,
) -> FP4PrefillWorkspace:
"""Build or refresh the prefill page-table, schedule, and logits buffers.
@@ -247,7 +269,7 @@ def prepare_fp4_prefill_workspace(
c4_seq_lens = _as_int32_1d(c4_seq_lens)
if workspace is None:
rows, _, padded_width = padded_page_table_shape(page_table)
rows, _, padded_width = padded_page_table_shape(page_table, page_table_bucket)
cta_count = max(_PREFILL_BASE_CTA_TARGET, rows)
device = page_table.device
buffers = PrefillScheduleBuffers(rows, device)
@@ -280,6 +302,7 @@ def prepare_fp4_prefill_workspace(
block_k=256,
guarded_out=workspace.guarded_page_table,
buffers=workspace.schedule_buffers,
page_table_bucket=page_table_bucket,
)
return workspace
@@ -297,6 +320,7 @@ def aiter_fp4_paged_mqa_logits(
is_decode: bool,
decode_workspace: Optional[FP4DecodeWorkspace] = None,
prefill_workspace: Optional[FP4PrefillWorkspace] = None,
page_table_bucket: int = 4,
) -> torch.Tensor:
"""Compute FP4 Q/K indexer logits with the decode or prefill FlyDSL kernel."""
from aiter.ops.flydsl import (
@@ -318,7 +342,26 @@ def aiter_fp4_paged_mqa_logits(
page_table = workspace.guarded_page_table
max_seq_len = workspace.max_seq_len
elif is_decode:
page_table, max_seq_len = _guard_page_table(page_table)
# No usable workspace: build the schedule here too, rather than letting
# AITER build it with its varctx kernel, which cannot reserve its scratch
# at large batches.
from sglang.kernels.ops.attention.dsv4.fp4_indexer_schedule_hip import (
build_decode_schedule,
)
page_table, max_seq_len = _guard_page_table(
page_table, page_table_bucket=page_table_bucket
)
cta_count = _decode_cta_count(num_tokens, max_seq_len)
cta_info = torch.empty(
(cta_count, _DECODE_CTA_INFO_WIDTH),
dtype=torch.int32,
device=page_table.device,
)
scratch = build_decode_schedule(
c4_seq_lens, cta_info_out=cta_info, max_seq_len=max_seq_len
)
fallback_schedule = (cta_info, cta_count, scratch)
else:
# No usable workspace (DP padding or truncated activations): build the
# schedule here rather than letting AITER rebuild it from ~29 torch ops
@@ -332,7 +375,7 @@ def aiter_fp4_paged_mqa_logits(
padded_page_table_shape,
)
_, _, padded_width = padded_page_table_shape(page_table)
_, _, padded_width = padded_page_table_shape(page_table, page_table_bucket)
max_seq_len = padded_width * _KV_BLOCK_SIZE
cta_count = max(_PREFILL_BASE_CTA_TARGET, num_tokens)
cta_info = torch.empty(
@@ -347,6 +390,7 @@ def aiter_fp4_paged_mqa_logits(
parallel_unit_num=cta_count,
max_seq_len=max_seq_len,
block_k=256,
page_table_bucket=page_table_bucket,
)
fallback_schedule = (cta_info, cta_count, buffers)
q_payload = q_fp4.view(torch.uint8)
@@ -365,16 +409,16 @@ def aiter_fp4_paged_mqa_logits(
}
if is_decode:
pinned = (
{}
if workspace is None
else {
if workspace is None:
cta_info, cta_count, _ = fallback_schedule
pinned = {"cta_info": cta_info, "total_ctas": cta_count}
else:
pinned = {
"cta_info": workspace.cta_info,
"total_ctas": workspace.cta_count,
}
)
logits = flydsl_pa_mqa_logits_fp4(
q_payload.reshape(num_tokens, 1, _HEADS, _HEAD_DIM // 2),
q_payload.reshape(num_tokens, 1, q_fp4.shape[-2], _HEAD_DIM // 2),
q_scale.reshape(num_tokens, 1, *_Q_SCALE_SHAPE),
k_payload,
k_scale,
@@ -497,3 +541,362 @@ def aiter_k_indexer_fp4_cache_write(
shuffle_scale=True,
do_rotate_act=True,
)
# FlyDSL split payload / scale layout; bytes equal AITER's fp4 writer up to the sign of zero
@triton.jit
def _store_fp4_index_k_split_kernel(
k_fp4,
k_sf,
payload,
scale,
loc,
page_size: tl.constexpr,
BLOCK: tl.constexpr,
):
token_id = tl.program_id(0)
cache_loc = tl.load(loc + token_id)
page = cache_loc // page_size
page_offset = cache_loc - page * page_size
# payload [page, 1, 4 chunks, page_size, 16 bytes]: chunk c holds dims [32c, 32c + 32).
offsets = tl.arange(0, BLOCK)
chunk = offsets // 16
byte = offsets - chunk * 16
k = tl.load(k_fp4 + token_id * BLOCK + offsets)
tl.store(
payload
+ page * (4 * page_size * 16)
+ chunk * (page_size * 16)
+ page_offset * 16
+ byte,
k,
)
# scale [page, 1, 4, page_size]: the slot axis is a 16 x 4 tile transposed (the FlyDSL K ABI)
shuffled = (page_offset % 16) * 4 + page_offset // 16
sf = tl.load(k_sf + token_id)
sf_offsets = tl.arange(0, 4)
sf_bytes = ((sf >> (sf_offsets * 8)) & 0xFF).to(tl.uint8)
tl.store(
scale + page * (4 * page_size) + sf_offsets * page_size + shuffled, sf_bytes
)
def store_fp4_index_k_cache_split(
input: torch.Tensor,
payload: torch.Tensor,
scale: torch.Tensor,
loc: torch.Tensor,
*,
page_size: int,
rne: bool = False,
) -> None:
"""Quantize input [n, 128] to fp4 (per-32 ue8m0) and scatter row i to slot
loc[i] of the split FlyDSL K layout (payload [pages, 1, 4, page_size, 16],
scale [pages, 1, 4, page_size] with the slot axis 16 x 4 transposed)."""
assert input.shape[-1] == _HEAD_DIM
assert page_size == _KV_BLOCK_SIZE, "the 16 x 4 slot transpose spans a 64-slot page"
assert payload.shape[1:] == (1, 4, page_size, 16), payload.shape
assert scale.shape[1:] == (1, 4, page_size), scale.shape
k_fp4, k_sf = quantize_fp4_indexer_tensor(input.contiguous(), rne=rne)
n_tokens = k_fp4.shape[0]
if n_tokens == 0:
return
_store_fp4_index_k_split_kernel[(n_tokens,)](
k_fp4.view(torch.uint8),
k_sf,
payload.view(torch.uint8),
scale,
loc,
page_size,
BLOCK=64,
)
def read_fp4_index_k_split(
payload: torch.Tensor, scale: torch.Tensor, slots: torch.Tensor, *, page_size: int
) -> Tuple[torch.Tensor, torch.Tensor]:
"""Inverse of store_fp4_index_k_cache_split: (payload int8 [n, 64], scales
int32 [n] with chunk c's e8m0 byte at bits 8c..8c+7), the layout of
quantize_fp4_indexer_tensor."""
assert page_size == _KV_BLOCK_SIZE, "the 16 x 4 slot transpose spans a 64-slot page"
slots = slots.to(torch.int64)
page, off = slots // page_size, slots % page_size
rows = payload.view(torch.uint8)[page, 0, :, off, :] # [n, 4, 16]
rows = rows.reshape(-1, 64).view(torch.int8)
shuffled = (off % 16) * 4 + off // 16
sf = scale[page, 0, :, shuffled].to(torch.int32) # [n, 4]
packed = sf[:, 0] | (sf[:, 1] << 8) | (sf[:, 2] << 16) | (sf[:, 3] << 24)
return rows, packed
@triton.jit
def _quantize_fp4_query_flydsl_kernel(
x,
x_fp4,
q_scale,
HEADS: tl.constexpr,
BLOCK_N: tl.constexpr,
GROUP_N: tl.constexpr,
):
"""One program per (token, head slot of 64): quantize_fp4_indexer_row (RNE codes) with
the e8m0 byte of chunk c stored straight into the FlyDSL scale layout
[t, 0, c, h % 16, h // 16]; head slots past HEADS write zero bytes."""
token_id = tl.program_id(0)
h = tl.program_id(1)
scale_base = q_scale + token_id * (4 * 16 * 4) + (h % 16) * 4 + h // 16
chunks = tl.arange(0, 4)
if h >= HEADS:
tl.store(scale_base + chunks * (16 * 4), tl.zeros([4], dtype=tl.uint8))
return
row = token_id * HEADS + h
values = tl.load(x + row * BLOCK_N + tl.arange(0, BLOCK_N)).to(tl.float32)
v0, v1 = tl.split(tl.reshape(values, (BLOCK_N // 2, 2)))
packed, packed_sf = quantize_fp4_indexer_row(
values, v0, v1, BLOCK_N=BLOCK_N, GROUP_N=GROUP_N, RNE=True
)
sf_bytes = ((packed_sf >> (8 * chunks)) & 0xFF).to(tl.uint8)
tl.store(scale_base + chunks * (16 * 4), sf_bytes)
tl.store(x_fp4 + row * (BLOCK_N // 2) + tl.arange(0, BLOCK_N // 2), packed)
def pack_fp4_query_flydsl(q: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
"""fp4-grid query [T, H, 128] -> (payload int8 [T, H, 64], scale uint8 [T, 1, 4, 16, 4]) in the
FlyDSL MQA-logits layout: head h, chunk c's e8m0 byte at [t, 0, c, h % 16, h // 16] (H <= 64),
RNE as the CUDA low-ratio path."""
num_tokens, heads = q.shape[0], q.shape[1]
assert heads % 16 == 0 and heads <= 64, (
f"{heads} heads: the FlyDSL scale layout holds 16-head groups up to 64"
)
assert q.shape[-1] == _HEAD_DIM
x = q.contiguous().view(-1, _HEAD_DIM)
q_fp4 = torch.empty(
(num_tokens, heads, _HEAD_DIM // 2), dtype=torch.int8, device=q.device
)
q_scale = torch.empty((num_tokens, 1, 4, 16, 4), dtype=torch.uint8, device=q.device)
if num_tokens > 0:
_quantize_fp4_query_flydsl_kernel[(num_tokens, 64)](
x,
q_fp4,
q_scale,
HEADS=heads,
BLOCK_N=_HEAD_DIM,
GROUP_N=_GROUP_SIZE,
)
return q_fp4, q_scale
@triton.jit
def _reduce_scale_bf16_block(
block,
part_ptr,
out_ptr,
M,
stride_ps,
stride_pm,
stride_om,
scale,
N: tl.constexpr,
SPLIT_K: tl.constexpr,
BLOCK: tl.constexpr,
):
"""One BLOCK of _reduce_scale_bf16_kernel (shared with the fused index-Q launch)."""
offs = block * BLOCK + tl.arange(0, BLOCK)
m = offs // N
n = offs % N
mask = m < M
acc = tl.load(part_ptr + m * stride_pm + n, mask=mask, other=0.0)
for s in tl.static_range(1, SPLIT_K):
acc += tl.load(
part_ptr + s * stride_ps + m * stride_pm + n, mask=mask, other=0.0
)
# bf16(bf16(sum) * scale): the linear's bf16 output, then the aten multiply rounded to bf16
w = acc.to(tl.bfloat16).to(tl.float32) * scale
tl.store(out_ptr + m * stride_om + n, w.to(tl.bfloat16), mask=mask)
@triton.jit
def _reduce_scale_bf16_kernel(
part_ptr,
out_ptr,
M,
stride_ps,
stride_pm,
stride_om,
scale,
N: tl.constexpr,
SPLIT_K: tl.constexpr,
BLOCK: tl.constexpr,
):
_reduce_scale_bf16_block(
tl.program_id(0),
part_ptr,
out_ptr,
M,
stride_ps,
stride_pm,
stride_om,
scale,
N=N,
SPLIT_K=SPLIT_K,
BLOCK=BLOCK,
)
@triton.jit
def _index_q_pack_weights_kernel(
q_ptr, # [T, H * 128] bf16 (wq_b output)
f_ptr, # [max_pos, RD // 2, 2] fp32: the real view of the complex freqs table
pos_ptr, # [T] int64
q_fp4_ptr, # [T, H, 64] int8
q_scale_ptr, # [T, 1, 4, 16, 4] uint8
part_ptr, # [SPLIT_K, T, H] fp32 head-weight partials
weights_ptr, # [T, H] bf16
stride_qt,
stride_ps,
stride_pm,
weight_scale,
T,
num_pos,
H: tl.constexpr,
D: tl.constexpr,
RD: tl.constexpr,
AMAX_FLOOR: tl.constexpr,
SPLIT_K: tl.constexpr,
W_BLOCK: tl.constexpr,
):
"""Grid (T * H + 1,). Program t * H + h: the query row through rope_tail_fake_quant_fp4_row
(rounded to bf16 as the standalone kernel stores it) and quantize_fp4_indexer_row (RNE), in
the FlyDSL MQA-logits layout; program T * H: the head weights' _reduce_scale_bf16_block."""
pid = tl.program_id(0)
if pid < T * H:
t = pid // H
h = pid % H
pos = tl.load(pos_ptr + t)
# the caller owns positions < num_pos; an out-of-range row reads entry 0 instead of past the table
pos = tl.where((pos >= 0) & (pos < num_pos), pos, 0)
fq = rope_tail_fake_quant_fp4_row(
q_ptr + t.to(tl.int64) * stride_qt + h * D,
f_ptr + pos * RD,
D=D,
RD=RD,
BLK=32,
AMAX_FLOOR=AMAX_FLOOR,
INVERSE=False,
COMPRESSED_KV=False,
)
# the standalone path stores the fake-quant as bf16 and the packer reloads it
values = fq.to(tl.bfloat16).to(tl.float32)
v0, v1 = tl.split(tl.reshape(values, (D // 2, 2)))
packed, packed_sf = quantize_fp4_indexer_row(
values, v0, v1, BLOCK_N=D, GROUP_N=32, RNE=True
)
tl.store(q_fp4_ptr + pid.to(tl.int64) * (D // 2) + tl.arange(0, D // 2), packed)
# scale bytes: chunk c of head h at [t, 0, c, h % 16, h // 16]
c = tl.arange(0, 4)
sf_bytes = ((packed_sf >> (8 * c)) & 0xFF).to(tl.uint8)
base = q_scale_ptr + t.to(tl.int64) * (4 * 16 * 4) + c * (16 * 4) + (h % 16) * 4
tl.store(base + h // 16, sf_bytes)
if h < 16:
# groups this head count does not have stay zero
for g in tl.static_range(H // 16, 4):
tl.store(base + g, tl.zeros((4,), dtype=tl.uint8))
else:
_reduce_scale_bf16_block(
0,
part_ptr,
weights_ptr,
T,
stride_ps,
stride_pm,
H,
weight_scale,
N=H,
SPLIT_K=SPLIT_K,
BLOCK=W_BLOCK,
)
def index_q_rope_pack_weights_flydsl(
q: torch.Tensor,
freqs_cis: torch.Tensor,
positions: torch.Tensor,
rope_dim: int,
head_weight_partials: torch.Tensor,
weight_scale: float,
*,
num_heads: int,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""The HIP index-Q inputs in one launch: (q_fp4 [T, H, 64] int8, q_scale [T, 1, 4, 16, 4]
uint8, weights [T, H] bf16), bitwise pack_fp4_query_flydsl(_rope_fq4(q.view(T, H, 128),
freqs_cis[positions], rope_dim)) and indexer_head_weights's reduce of the
[split_k, T, H] fp32 head_weight_partials."""
T = q.shape[0]
H = num_heads
assert q.dtype == torch.bfloat16 and q.dim() == 2 and q.shape[1] == H * 128
assert q.stride(1) == 1
assert H % 16 == 0 and 0 < H <= 64, H
assert freqs_cis.dtype == torch.complex64 and freqs_cis.shape[1] == rope_dim // 2
assert positions.shape == (T,)
split_k, pt, ph = head_weight_partials.shape
assert (pt, ph) == (T, H) and head_weight_partials.dtype == torch.float32
f_real = torch.view_as_real(freqs_cis)
assert f_real.is_contiguous()
q_fp4 = torch.empty((T, H, 64), dtype=torch.int8, device=q.device)
q_scale = torch.empty((T, 1, 4, 16, 4), dtype=torch.uint8, device=q.device)
weights = torch.empty((T, H), dtype=torch.bfloat16, device=q.device)
if T == 0:
return q_fp4, q_scale, weights
w_block = triton.next_power_of_2(T * H)
assert w_block <= 4096, (
f"T * H = {T * H} exceeds the 4096-wide head-weight reduce block"
)
_index_q_pack_weights_kernel[(T * H + 1,)](
q,
f_real,
positions,
q_fp4,
q_scale,
head_weight_partials,
weights,
q.stride(0),
head_weight_partials.stride(0),
head_weight_partials.stride(1),
float(weight_scale),
T,
f_real.shape[0],
H=H,
D=128,
RD=rope_dim,
AMAX_FLOOR=FP4_AMAX_FLOOR,
SPLIT_K=split_k,
W_BLOCK=w_block,
num_warps=4,
)
return q_fp4, q_scale, weights
def indexer_head_weights(
x: torch.Tensor, weight: torch.Tensor, scale: float
) -> torch.Tensor:
"""bf16(bf16(x @ weight.T) * scale) as a contiguous bf16 [M, N], the layout the
FlyDSL logits kernels take. x bf16 [M, K] and weight bf16 [N, K], with
M <= rocm_gemv_split_k_max_tokens(n=N, k=K, weight_dtype=torch.bfloat16)."""
partials = rocm_router_gemv_split_k(x, weight)
split_k, M, N = partials.shape
out = torch.empty((M, N), dtype=torch.bfloat16, device=x.device)
block = triton.next_power_of_2(M * N)
_reduce_scale_bf16_kernel[(triton.cdiv(M * N, block),)](
partials,
out,
M,
partials.stride(0),
partials.stride(1),
out.stride(0),
float(scale),
N=N,
SPLIT_K=split_k,
BLOCK=block,
num_warps=4,
)
return out
@@ -0,0 +1,65 @@
"""ROCm index-K store of the FP4 indexer into the split FlyDSL layout (payload and scale buffers)."""
from __future__ import annotations
from typing import TYPE_CHECKING
import torch
from sglang.kernels.jit.utils import (
cache_once,
is_arch_support_pdl,
load_jit,
make_cpp_args,
)
from .utils import make_name
if TYPE_CHECKING:
from tvm_ffi.module import Module
@cache_once
def _jit_index_k_split_module(
head_dim: int, rope_dim: int, page_size: int, ratio: int
) -> Module:
args = make_cpp_args(head_dim, rope_dim, page_size, ratio, is_arch_support_pdl())
return load_jit(
make_name("index_k_rope_pack_split"),
*args,
cuda_files=["deepseek_v4/fp4_indexer_rope_hip.cuh"],
cuda_wrappers=[("index_k_split", f"FlashIndexKSplitKernel<{args}>::run")],
)
def index_k_norm_rope_pack_store_split(
input: torch.Tensor,
norm_weight: torch.Tensor,
eps: float,
freqs_cis: torch.Tensor,
positions: torch.Tensor,
loc: torch.Tensor,
payload: torch.Tensor,
scale: torch.Tensor,
*,
ratio: int,
) -> None:
"""`fp4_indexer_rope.index_k_norm_rope_pack_store` into the split
FlyDSL K layout (payload [npages, 1, 4, page_size, 16], scale [npages, 1, 4,
page_size] uint8); the bytes equal store_fp4_index_k_cache_split's."""
head_dim = input.shape[-1]
page_size = payload.shape[3]
assert payload.shape[1:] == (1, 4, page_size, 16), payload.shape
assert scale.shape[1:] == (1, 4, page_size), scale.shape
_jit_index_k_split_module(
head_dim, freqs_cis.shape[-1], page_size, ratio
).index_k_split(
input,
norm_weight,
freqs_cis,
positions,
loc,
payload.view(torch.uint8),
scale,
float(eps),
)
@@ -12,6 +12,10 @@ Everything the preamble computes is a reduction or a scan over the row count,
so it collapses into a single kernel; the only real dependency is that
``_prefill_cta_info_kernel`` needs the finished prefix sums. That leaves two
dispatches for the whole schedule.
The decode schedule (AITER's ``compute_varctx_schedule``) is built the same way:
AITER's single kernel holds [256, rows] tiles in registers and spills past a few
thousand rows, so its scratch cannot be reserved once the KV pool owns the memory.
"""
from __future__ import annotations
@@ -167,7 +171,9 @@ def _pad_page_table_kernel(
)
def padded_page_table_shape(page_table: torch.Tensor) -> Tuple[int, int, int]:
def padded_page_table_shape(
page_table: torch.Tensor, page_table_bucket: int = 4
) -> Tuple[int, int, int]:
"""Rows, logical width, and the 256-token-scheduling padded width.
The padding rule is shared with the logits sizing in ``fp4_indexer_hip``;
@@ -176,7 +182,7 @@ def padded_page_table_shape(page_table: torch.Tensor) -> Tuple[int, int, int]:
from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import _guarded_pages
rows, logical_width = page_table.shape
return rows, logical_width, _guarded_pages(logical_width)
return rows, logical_width, _guarded_pages(logical_width, page_table_bucket)
def _as_int32_2d(page_table: torch.Tensor) -> torch.Tensor:
@@ -189,7 +195,9 @@ def _as_int32_2d(page_table: torch.Tensor) -> torch.Tensor:
def pad_page_table(
page_table: torch.Tensor, out: Optional[torch.Tensor] = None
page_table: torch.Tensor,
out: Optional[torch.Tensor] = None,
page_table_bucket: int = 4,
) -> Tuple[torch.Tensor, int]:
"""Pad a page table for 256-token scheduling in a single dispatch.
@@ -197,7 +205,9 @@ def pad_page_table(
output element, so the destination never needs pre-zeroing.
"""
page_table = _as_int32_2d(page_table)
rows, logical_width, padded_width = padded_page_table_shape(page_table)
rows, logical_width, padded_width = padded_page_table_shape(
page_table, page_table_bucket
)
if out is None:
out = page_table.new_empty((rows, padded_width + 4))
else:
@@ -269,6 +279,7 @@ def build_prefill_schedule(
block_k: int = 256,
guarded_out: Optional[torch.Tensor] = None,
buffers: Optional[PrefillScheduleBuffers] = None,
page_table_bucket: int = 4,
) -> Tuple[torch.Tensor, PrefillScheduleBuffers]:
"""Pad the page table and build the FP4 prefill schedule in two dispatches.
@@ -282,7 +293,9 @@ def build_prefill_schedule(
)
page_table = _as_int32_2d(page_table)
rows, logical_width, padded_width = padded_page_table_shape(page_table)
rows, logical_width, padded_width = padded_page_table_shape(
page_table, page_table_bucket
)
total_rows = local_ends.shape[0]
assert total_rows <= rows, (
f"local_ends rows {total_rows} exceed the page table's {rows}; the "
@@ -363,3 +376,153 @@ def build_prefill_schedule(
BLOCK_P=BLOCK_P,
)
return guarded_out, buffers
# Rows per block of the decode prep kernel's loops: its registers stay fixed as the
# batch grows, where AITER's varctx kernel holds every row in one tile.
_DECODE_ROW_BLOCK = 1024
# Slots per program of the decode cta_info kernel, as AITER's varctx kernel.
_DECODE_SLOT_BLOCK = 256
@triton.jit
def _decode_schedule_prep_kernel(
ctx_ptr, # [B] int32 c4 context lengths
incl_ptr, # [B] int32 out, inclusive prefix sum of per-row CTA counts
scalars_ptr, # [2] int32 out, [safe, total_splits]
B,
P,
block_k,
s_max,
ROW_BLOCK: tl.constexpr,
):
"""AITER varctx schedule preamble (next_n = 1): the split factor and the prefix sums."""
off = tl.arange(0, ROW_BLOCK)
zero = tl.sum(tl.zeros([ROW_BLOCK], tl.int32), axis=0)
max_chunks = zero + 1
any_total = zero
for start in range(0, B, ROW_BLOCK):
live = start + off < B
ctx = tl.load(ctx_ptr + start + off, mask=live, other=0)
chunks = tl.where(live, (ctx + block_k - 1) // block_k, 0)
max_chunks = tl.maximum(max_chunks, tl.max(chunks, axis=0))
any_total += tl.sum((chunks + s_max - 1) // s_max, axis=0)
# Smallest split factor s with sum_i ceil(chunks_i / s) <= P, as AITER's search.
lo = zero + 1
hi = zero + s_max
for _ in tl.static_range(32):
searching = lo < hi
mid = (lo + hi) // 2
total = zero
for start in range(0, B, ROW_BLOCK):
live = start + off < B
ctx = tl.load(ctx_ptr + start + off, mask=live, other=0)
chunks = tl.where(live, (ctx + block_k - 1) // block_k, 0)
total += tl.sum((chunks + mid - 1) // mid, axis=0)
fits = total <= P
hi = tl.where(searching & fits, mid, hi)
lo = tl.where(searching & (fits == 0), mid + 1, lo)
safe = tl.where(any_total <= P, lo, max_chunks)
carry = zero
for start in range(0, B, ROW_BLOCK):
live = start + off < B
ctx = tl.load(ctx_ptr + start + off, mask=live, other=0)
chunks = tl.where(live, (ctx + block_k - 1) // block_k, 0)
ctas = (chunks + safe - 1) // safe
tl.store(incl_ptr + start + off, tl.cumsum(ctas, axis=0) + carry, mask=live)
carry += tl.sum(ctas, axis=0)
tl.store(scalars_ptr, safe)
tl.store(scalars_ptr + 1, carry)
@triton.jit
def _decode_cta_info_kernel(
ctx_ptr, # [B] int32 c4 context lengths
incl_ptr, # [B] int32 inclusive prefix sum of per-row CTA counts
scalars_ptr, # [2] int32 [safe, total_splits]
cta_info_ptr, # [P, 4] int32 out, [batch, chunk_start, chunk_count, ctx_len]
B,
P,
block_k,
BLOCK_S: tl.constexpr,
):
"""AITER varctx cta_info rows (next_n = 1), each slot's row found by a binary search
over incl in global memory, as AITER's _prefill_cta_info_kernel does."""
safe = tl.load(scalars_ptr)
total_splits = tl.load(scalars_ptr + 1)
slot = tl.program_id(0) * BLOCK_S + tl.arange(0, BLOCK_S)
smask = slot < P
# searchsorted(incl, slot, right=True) = count(incl <= slot)
lo = tl.zeros([BLOCK_S], tl.int32)
hi = tl.full([BLOCK_S], B, tl.int32)
for _ in tl.static_range(32):
mid = (lo + hi) // 2
incl_mid = tl.load(
incl_ptr + tl.minimum(mid, B - 1), mask=mid < B, other=2147483647
)
go_right = incl_mid <= slot
lo = tl.where(go_right, mid + 1, lo)
hi = tl.where(go_right, hi, mid)
row = tl.minimum(lo, B - 1)
ctx = tl.load(ctx_ptr + row, mask=smask, other=0)
chunks = (ctx + block_k - 1) // block_k
excl = tl.load(incl_ptr + row, mask=smask, other=0) - (chunks + safe - 1) // safe
valid = slot < total_splits
vi = valid.to(tl.int32)
start = (slot - excl) * safe # pre-mask (count uses this)
count = tl.maximum(tl.minimum(safe, chunks - start), 0)
base = slot * 4
tl.store(cta_info_ptr + base + 0, row * vi, mask=smask)
tl.store(cta_info_ptr + base + 1, start * vi, mask=smask)
tl.store(cta_info_ptr + base + 2, tl.where(valid, count, 1), mask=smask)
tl.store(cta_info_ptr + base + 3, ctx * vi, mask=smask)
def build_decode_schedule(
context_lens: torch.Tensor,
*,
cta_info_out: torch.Tensor,
max_seq_len: int,
block_k: int = 256,
) -> torch.Tensor:
"""AITER's ``compute_varctx_schedule`` (next_n = 1) into ``cta_info_out`` [P, 4] in two
dispatches whose registers do not grow with the row count. Returns the scratch the
kernels read ([safe, total_splits] then the per-row prefix sums): a captured graph
must keep it alive."""
context_lens = context_lens.to(torch.int32)
rows = context_lens.shape[0]
ctas = cta_info_out.shape[0]
assert rows <= ctas, (
f"{rows} rows on {ctas} CTAs: the persistent grid must give every row a CTA"
)
scratch = torch.empty(rows + 2, dtype=torch.int32, device=context_lens.device)
if rows == 0:
return scratch
scalars, incl = scratch[:2], scratch[2:]
_decode_schedule_prep_kernel[(1,)](
context_lens,
incl,
scalars,
rows,
ctas,
block_k,
max(1, triton.cdiv(max_seq_len, block_k)),
ROW_BLOCK=_DECODE_ROW_BLOCK,
)
_decode_cta_info_kernel[(triton.cdiv(ctas, _DECODE_SLOT_BLOCK),)](
context_lens,
incl,
scalars,
cta_info_out,
rows,
ctas,
block_k,
BLOCK_S=_DECODE_SLOT_BLOCK,
)
return scratch
@@ -2,6 +2,8 @@
from __future__ import annotations
from typing import Optional
import torch
import triton
import triton.language as tl
@@ -11,14 +13,9 @@ from sglang.kernels.ops.attention.dsv4.torch_quant import FP4_AMAX_FLOOR
@triton.jit
def _rope_tail_fake_quant_fp4_kernel(
x_ptr,
f_ptr,
out_ptr,
x_stride_r,
out_stride_r,
f_stride_t,
rows_per_token,
def rope_tail_fake_quant_fp4_row(
x_row_ptr,
f_row_ptr,
D: tl.constexpr,
RD: tl.constexpr,
BLK: tl.constexpr,
@@ -26,10 +23,10 @@ def _rope_tail_fake_quant_fp4_kernel(
INVERSE: tl.constexpr,
COMPRESSED_KV: tl.constexpr,
):
r = tl.program_id(0)
t = r // rows_per_token
"""One row of rope_tail_fake_quant_fp4 as an fp32 [D] vector: x_row_ptr points
at the row, f_row_ptr at the token's [RD // 2, 2] real view of the complex freqs."""
offs = tl.arange(0, D)
v = tl.load(x_ptr + r * x_stride_r + offs).to(tl.float32)
v = tl.load(x_row_ptr + offs).to(tl.float32)
# ---- rope_tail: adjacent pairs of the last RD features as one complex number
head_len = D - RD
@@ -37,15 +34,13 @@ def _rope_tail_fake_quant_fp4_kernel(
pos = offs - head_len
j = pos // 2
is_im = (pos % 2) == 1
re = tl.load(x_ptr + x_stride_r * r + head_len + 2 * j, mask=in_tail, other=0.0).to(
re = tl.load(x_row_ptr + head_len + 2 * j, mask=in_tail, other=0.0).to(tl.float32)
im = tl.load(x_row_ptr + head_len + 2 * j + 1, mask=in_tail, other=0.0).to(
tl.float32
)
im = tl.load(
x_ptr + x_stride_r * r + head_len + 2 * j + 1, mask=in_tail, other=0.0
).to(tl.float32)
# freqs is a real/imag-interleaved view: stride 2 between complex pairs.
fr = tl.load(f_ptr + t * f_stride_t + 2 * j, mask=in_tail, other=1.0)
fi = tl.load(f_ptr + t * f_stride_t + 2 * j + 1, mask=in_tail, other=0.0)
fr = tl.load(f_row_ptr + 2 * j, mask=in_tail, other=1.0)
fi = tl.load(f_row_ptr + 2 * j + 1, mask=in_tail, other=0.0)
if INVERSE:
fi = -fi
rot = tl.where(is_im, re * fi + im * fr, re * fr - im * fi)
@@ -75,8 +70,48 @@ def _rope_tail_fake_quant_fp4_kernel(
# torch.round is round-half-to-even; torch.sign(0) is 0
sgn = tl.where(s > 0, 1.0, tl.where(s < 0, -1.0, 0.0))
q = libdevice.rint(mag / step) * step * sgn
out = tl.reshape(q * scale[:, None], (D,))
tl.store(out_ptr + r * out_stride_r + offs, out.to(out_ptr.dtype.element_ty))
return tl.reshape(q * scale[:, None], (D,))
@triton.jit
def _rope_tail_fake_quant_fp4_kernel(
x_ptr,
f_ptr,
pos_ptr,
out_ptr,
x_stride_r,
out_stride_r,
f_stride_t,
rows_per_token,
num_pos,
D: tl.constexpr,
RD: tl.constexpr,
BLK: tl.constexpr,
AMAX_FLOOR: tl.constexpr,
INVERSE: tl.constexpr,
COMPRESSED_KV: tl.constexpr,
HAS_POS: tl.constexpr,
):
r = tl.program_id(0)
t = r // rows_per_token
if HAS_POS:
# freqs is the whole table: row t reads its position's entry (the freqs[positions] gather folded in)
t = tl.load(pos_ptr + t).to(tl.int64)
# the caller owns positions < num_pos; an out-of-range row reads entry 0 instead of past the table
t = tl.where((t >= 0) & (t < num_pos), t, 0)
out = rope_tail_fake_quant_fp4_row(
x_ptr + r * x_stride_r,
f_ptr + t * f_stride_t,
D=D,
RD=RD,
BLK=BLK,
AMAX_FLOOR=AMAX_FLOOR,
INVERSE=INVERSE,
COMPRESSED_KV=COMPRESSED_KV,
)
tl.store(
out_ptr + r * out_stride_r + tl.arange(0, D), out.to(out_ptr.dtype.element_ty)
)
def rope_tail_fake_quant_fp4(
@@ -87,10 +122,12 @@ def rope_tail_fake_quant_fp4(
block_size: int = 32,
*,
compressed_kv: bool = False,
positions: Optional[torch.Tensor] = None,
) -> torch.Tensor:
"""RoPE and FP4 round-trip: per-16 E4M3 for compressed KV, per-32 UE8M0 otherwise.
x: [T, ..., D] contiguous in the last dim; freqs: complex [T, rope_dim // 2].
x: [T, ..., D] contiguous in the last dim; freqs: complex [T, rope_dim // 2], or with
positions ([T] int) the whole table that row t reads at positions[t].
"""
x = x.contiguous()
if compressed_kv:
@@ -105,20 +142,29 @@ def rope_tail_fake_quant_fp4(
return out
rows_per_token = rows // x.shape[0]
f_real = torch.view_as_real(freqs.contiguous()).contiguous()
if positions is not None:
assert positions.dim() == 1 and positions.shape[0] == x.shape[0], (
positions.shape,
x.shape,
)
positions = positions.contiguous()
_rope_tail_fake_quant_fp4_kernel[(rows,)](
x2,
f_real,
positions if positions is not None else f_real,
out.reshape(-1, d),
x2.stride(0),
d,
f_real.stride(0),
rows_per_token,
f_real.shape[0],
D=d,
RD=rope_dim,
BLK=block_size,
AMAX_FLOOR=FP4_AMAX_FLOOR,
INVERSE=inverse,
COMPRESSED_KV=compressed_kv,
HAS_POS=positions is not None,
num_warps=4,
)
return out
@@ -17,6 +17,7 @@ import torch
from sglang.kernels.jit.utils import (
cache_once,
is_arch_support_pdl,
is_hip_runtime,
load_jit,
make_cpp_args,
)
@@ -98,7 +99,8 @@ def c1_decode_norm_rope_store(
freqs_cis,
positions,
out_loc,
k_cache,
# HIP's fp8_e4m3_t is uint8_t, so the kernel matches the fp8 pool as bytes
k_cache.view(torch.uint8) if is_hip_runtime() else k_cache,
float(eps),
)
return out
@@ -176,7 +178,8 @@ def c2_decode_norm_rope_store(
raw_out_loc,
eps,
freqs_cis,
k_cache,
# HIP's fp8_e4m3_t is uint8_t, so the kernel matches the fp8 pool as bytes
k_cache.view(torch.uint8) if is_hip_runtime() else k_cache,
ring_size,
draft_len,
)
+1 -10
View File
@@ -5,13 +5,6 @@ from typing import Optional
import torch
from sglang.srt.environ import envs
from sglang.srt.utils import get_bool_env_var, is_hip
_is_hip = is_hip()
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip
if _use_aiter:
from aiter.tuned_gemm import tgemm
_linear_bf16_fp32_algo = envs.SGLANG_OPT_BF16_FP32_GEMM_ALGO.get()
_HPC_GEMM_WEIGHT_CACHE_ATTR = "_sglang_bf16xfp32_weight_cache"
@@ -147,9 +140,7 @@ def linear_bf16_fp32(
*,
hpc_kernel_min_m: Optional[int] = None,
) -> torch.Tensor:
if _use_aiter and y.dtype == torch.bfloat16:
return tgemm.mm(x, y, otype=x.dtype).float()
elif hpc_kernel_min_m is not None:
if hpc_kernel_min_m is not None:
output = _linear_bf16_fp32_hpc(x, y, min_m=hpc_kernel_min_m)
if output is not None:
return output
@@ -0,0 +1,149 @@
"""Skinny bf16 GEMV on ROCm for decode row counts: [M, K] x [N, K] as fp32 split-K
partials whose fixed-order sum is batch invariant (the DeepSeek-V4 router and indexer head
weights)."""
from __future__ import annotations
import torch
import triton
import triton.language as tl
from sglang.srt.utils import is_gfx95_supported, is_hip
# the GEMV walks 16-row tiles inside one program, so every M runs the same tile (batch invariance)
ROCM_ROUTER_MAX_TOKENS = 64
_BLOCK_M = 16
_BLOCK_N = 16
_BLOCK_K = 512
_MAX_SPLIT_K = 32
def rocm_gemv_split_k_max_tokens(*, n: int, k: int, weight_dtype: torch.dtype) -> int:
"""Rows up to which rocm_router_gemv_split_k serves an [M, k] @ [n, k].T bf16
GEMV (one 16-wide N tile per 512 of K, 16-row tiles), -1 when the device or the shape
rules it out."""
if not (is_hip() and is_gfx95_supported()):
return -1
if weight_dtype != torch.bfloat16:
return -1
if n <= 0 or n % _BLOCK_N != 0:
return -1
if k % _BLOCK_K != 0 or not 0 < k // _BLOCK_K <= _MAX_SPLIT_K:
return -1
return ROCM_ROUTER_MAX_TOKENS
@triton.jit
def _router_gemv_split_k_kernel(
x_ptr,
w_ptr,
part_ptr,
M,
stride_xm,
stride_wn,
stride_ps,
stride_pm,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
BLOCK_K: tl.constexpr,
M_TILES: tl.constexpr,
):
pid_n = tl.program_id(0)
pid_k = tl.program_id(1)
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
offs_k = pid_k * BLOCK_K + tl.arange(0, BLOCK_K)
w = tl.load(w_ptr + offs_n[None, :] * stride_wn + offs_k[:, None])
# the weight tile is read once; every row tile is the same BLOCK_M x BLOCK_K dot
for t in tl.static_range(M_TILES):
offs_m = t * BLOCK_M + tl.arange(0, BLOCK_M)
row_mask = offs_m < M
x = tl.load(
x_ptr + offs_m[:, None] * stride_xm + offs_k[None, :],
mask=row_mask[:, None],
other=0.0,
)
acc = tl.dot(x, w)
tl.store(
part_ptr
+ pid_k * stride_ps
+ offs_m[:, None] * stride_pm
+ offs_n[None, :],
acc,
mask=row_mask[:, None],
)
def rocm_router_gemv_split_k(x: torch.Tensor, w: torch.Tensor) -> torch.Tensor:
"""x[M, K] @ w[N, K].T as fp32 split-K partials [K // 512, M, N], summed over dim 0 in
order by rocm_router_reduce_partials or the fused gate."""
M, K = x.shape
N, K_w = w.shape
assert K == K_w and K % _BLOCK_K == 0 and N % _BLOCK_N == 0
assert 0 < M <= ROCM_ROUTER_MAX_TOKENS, (
f"{M} rows: the split-K GEMV serves at most {ROCM_ROUTER_MAX_TOKENS}"
)
assert x.dtype == torch.bfloat16 and w.dtype == torch.bfloat16
assert x.stride(1) == 1 and w.stride(1) == 1
split_k = K // _BLOCK_K
partials = torch.empty((split_k, M, N), dtype=torch.float32, device=x.device)
_router_gemv_split_k_kernel[(N // _BLOCK_N, split_k)](
x,
w,
partials,
M,
x.stride(0),
w.stride(0),
partials.stride(0),
partials.stride(1),
BLOCK_M=_BLOCK_M,
BLOCK_N=_BLOCK_N,
BLOCK_K=_BLOCK_K,
M_TILES=triton.cdiv(M, _BLOCK_M),
num_warps=4,
)
return partials
@triton.jit
def _reduce_partials_kernel(
part_ptr,
out_ptr,
M,
stride_ps,
stride_pm,
stride_om,
N: tl.constexpr,
SPLIT_K: tl.constexpr,
BLOCK: tl.constexpr,
):
offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
m = offs // N
n = offs % N
mask = m < M
acc = tl.load(part_ptr + m * stride_pm + n, mask=mask, other=0.0)
for s in tl.static_range(1, SPLIT_K):
acc += tl.load(
part_ptr + s * stride_ps + m * stride_pm + n, mask=mask, other=0.0
)
tl.store(out_ptr + m * stride_om + n, acc, mask=mask)
def rocm_router_reduce_partials(partials: torch.Tensor, out: torch.Tensor) -> None:
"""out[M, N] = partials[0] + partials[1] + ... in that order, in fp32:
the same sum the fused gate computes."""
split_k, M, N = partials.shape
assert out.shape == (M, N) and out.dtype == torch.float32 and out.stride(1) == 1
block = 1024
_reduce_partials_kernel[(triton.cdiv(M * N, block),)](
partials,
out,
M,
partials.stride(0),
partials.stride(1),
out.stride(0),
N=N,
SPLIT_K=split_k,
BLOCK=block,
num_warps=4,
)
@@ -0,0 +1,307 @@
"""DeepSeek-V4 MoE router gate at decode row counts on ROCm: a sqrtsoftplus top-k over the
split-K GEMV partials of router_gemv_hip, bitwise aiter's topk_gating_kernel_opt, ties
included."""
from __future__ import annotations
from typing import Optional, Tuple
import torch
import triton
import triton.language as tl
from sglang.kernels.ops.gemm.router_gemv_hip import rocm_gemv_split_k_max_tokens
# aiter's wave64 kernel: EPT = 384 / 64 = 6 slots per lane; the sorting network below is N == 6
_WARP = 64
_GATE_NUM_EXPERTS = _WARP * 6
_MAX_TOPK = 16
_FLT_MAX = tl.constexpr(3.4028234663852886e38)
_LOG2_E = tl.constexpr(1.4426950408889634)
_LN_2 = tl.constexpr(0.6931471805599453)
def rocm_router_max_tokens(
*,
num_experts: int,
hidden_size: int,
topk: int,
weight_dtype: torch.dtype,
) -> int:
"""Rows up to which rocm_router_gemv_split_k + rocm_router_gate
serve the router, -1 when the device or the shape rules them out."""
if num_experts != _GATE_NUM_EXPERTS or not 0 < topk <= _MAX_TOPK:
return -1
return rocm_gemv_split_k_max_tokens(
n=num_experts, k=hidden_size, weight_dtype=weight_dtype
)
@triton.jit
def _score(x):
# aiter compute_score<SQRTSOFTPLUS>: +inf clamps to FLT_MAX, NaN passes, sqrt correctly rounded
x = tl.where(x > _FLT_MAX, _FLT_MAX, x)
t = tl.exp2(x * _LOG2_E)
sp = tl.where(x > 20.0, x, tl.log2(1.0 + t) * _LN_2)
return tl.sqrt_rn(sp)
@triton.jit
def _cas(vi, oi, ii, vj, oj, ij):
# aiter _CAS_DESC: swap when vals[i] < vals[j]; equal values stay put.
c = vi < vj
return (
tl.where(c, vj, vi),
tl.where(c, oj, oi),
tl.where(c, ij, ii),
tl.where(c, vi, vj),
tl.where(c, oi, oj),
tl.where(c, ii, ij),
)
@triton.jit
def _slot(cursor, a0, a1, a2, a3, a4, a5, other):
r = tl.where(cursor == 0, a0, other)
r = tl.where(cursor == 1, a1, r)
r = tl.where(cursor == 2, a2, r)
r = tl.where(cursor == 3, a3, r)
r = tl.where(cursor == 4, a4, r)
r = tl.where(cursor == 5, a5, r)
return r
@triton.jit
def _load_logits(
logits_ptr, part_ptr, row, e, stride_lm, stride_ps, stride_pm, SPLIT_K: tl.constexpr
):
if SPLIT_K == 0:
return tl.load(logits_ptr + row * stride_lm + e).to(tl.float32)
else:
acc = tl.load(part_ptr + row * stride_pm + e)
for s in tl.static_range(1, SPLIT_K):
acc += tl.load(part_ptr + s * stride_ps + row * stride_pm + e)
return acc
@triton.jit
def _gate_row(
logits_ptr,
part_ptr,
bias_ptr,
row,
stride_lm,
stride_ps,
stride_pm,
routed_scaling_factor,
SPLIT_K: tl.constexpr,
WRITE_LOGITS: tl.constexpr,
HAS_BIAS: tl.constexpr,
RENORM: tl.constexpr,
TOPK: tl.constexpr,
):
"""aiter's topk_gating_kernel_opt for one row on one 64-lane wave: returns the
scaled weights and the ids as [64] lane tensors (lane k < TOPK holds slot k)."""
lane = tl.arange(0, 64)
# Slot i of a lane holds expert lane + 64 * i, as in aiter's register kernel.
i0 = lane
i1 = lane + 64
i2 = lane + 128
i3 = lane + 192
i4 = lane + 256
i5 = lane + 320
x0 = _load_logits(
logits_ptr, part_ptr, row, i0, stride_lm, stride_ps, stride_pm, SPLIT_K
)
x1 = _load_logits(
logits_ptr, part_ptr, row, i1, stride_lm, stride_ps, stride_pm, SPLIT_K
)
x2 = _load_logits(
logits_ptr, part_ptr, row, i2, stride_lm, stride_ps, stride_pm, SPLIT_K
)
x3 = _load_logits(
logits_ptr, part_ptr, row, i3, stride_lm, stride_ps, stride_pm, SPLIT_K
)
x4 = _load_logits(
logits_ptr, part_ptr, row, i4, stride_lm, stride_ps, stride_pm, SPLIT_K
)
x5 = _load_logits(
logits_ptr, part_ptr, row, i5, stride_lm, stride_ps, stride_pm, SPLIT_K
)
if WRITE_LOGITS:
tl.store(logits_ptr + row * stride_lm + i0, x0)
tl.store(logits_ptr + row * stride_lm + i1, x1)
tl.store(logits_ptr + row * stride_lm + i2, x2)
tl.store(logits_ptr + row * stride_lm + i3, x3)
tl.store(logits_ptr + row * stride_lm + i4, x4)
tl.store(logits_ptr + row * stride_lm + i5, x5)
s0 = _score(x0)
s1 = _score(x1)
s2 = _score(x2)
s3 = _score(x3)
s4 = _score(x4)
s5 = _score(x5)
# as in aiter, a NaN score weighs 0 and its choice value is -inf
o0 = tl.where(s0 != s0, 0.0, s0)
o1 = tl.where(s1 != s1, 0.0, s1)
o2 = tl.where(s2 != s2, 0.0, s2)
o3 = tl.where(s3 != s3, 0.0, s3)
o4 = tl.where(s4 != s4, 0.0, s4)
o5 = tl.where(s5 != s5, 0.0, s5)
if HAS_BIAS:
v0 = s0 + tl.load(bias_ptr + i0).to(tl.float32)
v1 = s1 + tl.load(bias_ptr + i1).to(tl.float32)
v2 = s2 + tl.load(bias_ptr + i2).to(tl.float32)
v3 = s3 + tl.load(bias_ptr + i3).to(tl.float32)
v4 = s4 + tl.load(bias_ptr + i4).to(tl.float32)
v5 = s5 + tl.load(bias_ptr + i5).to(tl.float32)
else:
v0 = s0
v1 = s1
v2 = s2
v3 = s3
v4 = s4
v5 = s5
ninf = float("-inf")
v0 = tl.where(v0 != v0, ninf, v0)
v1 = tl.where(v1 != v1, ninf, v1)
v2 = tl.where(v2 != v2, ninf, v2)
v3 = tl.where(v3 != v3, ninf, v3)
v4 = tl.where(v4 != v4, ninf, v4)
v5 = tl.where(v5 != v5, ninf, v5)
# aiter sort_network_desc<6>: the 12-comparator network, same order.
v0, o0, i0, v1, o1, i1 = _cas(v0, o0, i0, v1, o1, i1)
v2, o2, i2, v3, o3, i3 = _cas(v2, o2, i2, v3, o3, i3)
v4, o4, i4, v5, o5, i5 = _cas(v4, o4, i4, v5, o5, i5)
v0, o0, i0, v2, o2, i2 = _cas(v0, o0, i0, v2, o2, i2)
v1, o1, i1, v4, o4, i4 = _cas(v1, o1, i1, v4, o4, i4)
v3, o3, i3, v5, o5, i5 = _cas(v3, o3, i3, v5, o5, i5)
v0, o0, i0, v1, o1, i1 = _cas(v0, o0, i0, v1, o1, i1)
v2, o2, i2, v3, o3, i3 = _cas(v2, o2, i2, v3, o3, i3)
v4, o4, i4, v5, o5, i5 = _cas(v4, o4, i4, v5, o5, i5)
v1, o1, i1, v2, o2, i2 = _cas(v1, o1, i1, v2, o2, i2)
v3, o3, i3, v4, o4, i4 = _cas(v3, o3, i3, v4, o4, i4)
v2, o2, i2, v3, o3, i3 = _cas(v2, o2, i2, v3, o3, i3)
# k-way merge: lanes offer their heads, the max wins, ties to the lowest lane (aiter ballot + ctz)
cursor = tl.zeros([64], dtype=tl.int32)
sel_val = tl.zeros([64], dtype=tl.float32)
sel_idx = tl.zeros([64], dtype=tl.int32)
total = 0.0
for k in tl.static_range(TOPK):
my_val = _slot(cursor, v0, v1, v2, v3, v4, v5, ninf)
my_idx = _slot(cursor, i0, i1, i2, i3, i4, i5, 0)
max_val = tl.max(my_val, axis=0)
win_lane = tl.min(tl.where(my_val == max_val, lane, 64), axis=0)
win_idx = tl.sum(tl.where(lane == win_lane, my_idx, 0), axis=0)
i_won = (cursor < 6) & (my_idx == win_idx)
my_orig = tl.where(i_won, _slot(cursor, o0, o1, o2, o3, o4, o5, 0.0), 0.0)
cursor = cursor + i_won.to(tl.int32)
src_lane = win_idx & 63
weight_bits = tl.sum(
tl.where(lane == src_lane, my_orig.to(tl.int32, bitcast=True), 0), axis=0
)
weight = weight_bits.to(tl.float32, bitcast=True)
sel_val = tl.where(lane == k, weight, sel_val)
sel_idx = tl.where(lane == k, win_idx, sel_idx)
total = total + weight
if RENORM:
scale = routed_scaling_factor / tl.maximum(total, 1e-20)
else:
scale = routed_scaling_factor * 1.0
return sel_val * scale, sel_idx
@triton.jit
def _router_gate_kernel(
logits_ptr, # [M, 384] read when SPLIT_K == 0, written when WRITE_LOGITS
part_ptr, # [SPLIT_K, M, 384] fp32 (SPLIT_K > 0)
bias_ptr, # [384] fp32 or bf16 (HAS_BIAS)
weights_ptr, # [M, TOPK] fp32
ids_ptr, # [M, TOPK] int32
stride_lm,
stride_ps,
stride_pm,
stride_om,
routed_scaling_factor,
SPLIT_K: tl.constexpr,
WRITE_LOGITS: tl.constexpr,
HAS_BIAS: tl.constexpr,
RENORM: tl.constexpr,
TOPK: tl.constexpr,
):
row = tl.program_id(0)
weights, ids = _gate_row(
logits_ptr,
part_ptr,
bias_ptr,
row,
stride_lm,
stride_ps,
stride_pm,
routed_scaling_factor,
SPLIT_K,
WRITE_LOGITS,
HAS_BIAS,
RENORM,
TOPK,
)
lane = tl.arange(0, 64)
out_mask = lane < TOPK
tl.store(weights_ptr + row * stride_om + lane, weights, mask=out_mask)
tl.store(ids_ptr + row * stride_om + lane, ids, mask=out_mask)
def rocm_router_gate(
gating_output: torch.Tensor,
correction_bias: Optional[torch.Tensor],
topk: int,
renormalize: bool,
routed_scaling_factor: Optional[float],
*,
partials: Optional[torch.Tensor] = None,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""aiter topk_gating(..., score_func="sqrtsoftplus") for 384 experts: fp32 weights and int32
ids [M, topk]; with partials their fixed-order sum is gated and written into
gating_output."""
M, num_experts = gating_output.shape
assert num_experts == _GATE_NUM_EXPERTS and 0 < topk <= _MAX_TOPK
assert gating_output.stride(1) == 1
if partials is not None:
assert (
partials.shape[1:] == (M, num_experts) and partials.dtype == torch.float32
)
assert gating_output.dtype == torch.float32
split_k = partials.shape[0]
stride_ps, stride_pm = partials.stride(0), partials.stride(1)
else:
partials = gating_output
split_k = 0
stride_ps = stride_pm = 0
if correction_bias is not None:
assert (
correction_bias.shape == (num_experts,) and correction_bias.is_contiguous()
)
weights = torch.empty((M, topk), dtype=torch.float32, device=gating_output.device)
ids = torch.empty((M, topk), dtype=torch.int32, device=gating_output.device)
if M == 0:
return weights, ids
_router_gate_kernel[(M,)](
gating_output,
partials,
correction_bias if correction_bias is not None else gating_output,
weights,
ids,
gating_output.stride(0),
stride_ps,
stride_pm,
weights.stride(0),
float(1.0 if routed_scaling_factor is None else routed_scaling_factor),
SPLIT_K=split_k,
WRITE_LOGITS=split_k > 0,
HAS_BIAS=correction_bias is not None,
RENORM=bool(renormalize),
TOPK=topk,
num_warps=1,
)
return weights, ids
@@ -0,0 +1,91 @@
"""Independent packed-FP4 (V41_FP4 layout) cache oracle for the V4.1 store and reader tests."""
from typing import Optional
import torch
KV_DIM = 512
FP4_GROUP = 16 # values per e4m3 scale
E2M1_BYTES_PER_TOKEN = KV_DIM // 2
E4M3_SCALE_BYTES_PER_TOKEN = KV_DIM // FP4_GROUP
FP4_TOKEN_BYTES = E2M1_BYTES_PER_TOKEN + E4M3_SCALE_BYTES_PER_TOKEN
E2M1_MAX = 6.0
E4M3_MAX = 448.0
MIN_SCALE = 2.0**-9
_E2M1_MAGNITUDES = (0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0)
def quantize_to_e2m1_codes(x: torch.Tensor) -> torch.Tensor:
"""Round to the nearest e2m1 value with the semantics of
cvt.rn.satfinite.e2m1x2.f32 (ties to even, saturating to +-6) and return
the 4-bit codes as uint8. The sign is kept for values that round to zero
(-0.0 and small negatives give the code 0x8); NaN maps to code 0."""
x = x.float()
mags = torch.tensor(_E2M1_MAGNITUDES, dtype=torch.float32, device=x.device)
sign = torch.signbit(x).to(torch.uint8) << 3
a = torch.nan_to_num(x.abs(), nan=0.0, posinf=6.0).clamp_max(6.0)
mids = (mags[:-1] + mags[1:]) / 2
code = torch.bucketize(a, mids, right=True)
on_tie = (a.unsqueeze(-1) == mids).any(dim=-1)
tie_code = torch.bucketize(a, mids, right=False)
code = torch.where(on_tie, tie_code + (tie_code & 1), code)
return sign | code.to(torch.uint8)
def dequantize_e2m1_codes(codes: torch.Tensor) -> torch.Tensor:
mags = torch.tensor(_E2M1_MAGNITUDES, dtype=torch.float32, device=codes.device)
val = mags[(codes & 7).long()]
return torch.where((codes & 8) != 0, -val, val)
def quantize_k_cache_v41_fp4(
k: torch.Tensor, page_bytes: Optional[int] = None
) -> torch.Tensor:
"""k [num_pages, page_size, 512] -> uint8 [num_pages, page_bytes]
pages of the V41_FP4 layout: 256 B of e2m1 codes per token (even index in
the low nibble), then 32 e4m3 scales per token (one per 16 values),
scale = e4m3(clamp(amax / 6, 2**-9, 448)). A NaN element poisons its
tile: NaN scale, zero codes."""
num_pages, page_size, d = k.shape
assert d == KV_DIM
x = k.float().view(num_pages, page_size, KV_DIM // FP4_GROUP, FP4_GROUP)
amax = torch.nan_to_num(x.abs(), nan=float("inf")).amax(dim=-1)
scale = torch.clamp(amax / E2M1_MAX, MIN_SCALE, E4M3_MAX).to(torch.float8_e4m3fn)
scale = torch.where(torch.isinf(amax), torch.full_like(scale, float("nan")), scale)
codes = quantize_to_e2m1_codes(x / scale.float().unsqueeze(-1))
codes = codes.view(num_pages, page_size, KV_DIM)
packed = codes[..., 0::2] | (codes[..., 1::2] << 4)
raw = page_size * FP4_TOKEN_BYTES
if page_bytes is None:
page_bytes = -(-raw // 256) * 256 # pages start on a 256-byte boundary
assert page_bytes >= raw
data_bytes = page_size * E2M1_BYTES_PER_TOKEN
out = torch.zeros((num_pages, page_bytes), dtype=torch.uint8, device=k.device)
out[:, :data_bytes] = packed.reshape(num_pages, data_bytes)
out[:, data_bytes:raw] = scale.view(torch.uint8).reshape(
num_pages, raw - data_bytes
)
return out
def dequantize_k_cache_v41_fp4(pages: torch.Tensor, page_size: int) -> torch.Tensor:
"""Inverse of quantize_k_cache_v41_fp4: [num_pages, page_size, 512]
bf16. e2m1 * e4m3 has at most 2 + 4 significant bits, so the product is
exact in bf16, as in the kernel."""
num_pages = pages.shape[0]
pages = pages.view(torch.uint8)
data_bytes = page_size * E2M1_BYTES_PER_TOKEN
data = pages[:, :data_bytes].reshape(num_pages, page_size, E2M1_BYTES_PER_TOKEN)
scale = pages[:, data_bytes : page_size * FP4_TOKEN_BYTES].reshape(
num_pages, page_size, E4M3_SCALE_BYTES_PER_TOKEN
)
codes = torch.empty(
(num_pages, page_size, KV_DIM), dtype=torch.uint8, device=pages.device
)
codes[..., 0::2] = data & 0xF
codes[..., 1::2] = data >> 4
values = dequantize_e2m1_codes(codes).view(
num_pages, page_size, KV_DIM // FP4_GROUP, FP4_GROUP
)
out = values * scale.view(torch.float8_e4m3fn).float().unsqueeze(-1)
return out.view(num_pages, page_size, KV_DIM).to(torch.bfloat16)