dsv4.1-amd: KV cache layouts, FP4 indexer, compressor and router kernels (#41019)
Co-authored-by: x <x>
This commit is contained in:
@@ -267,7 +267,7 @@ struct FlashCompress1Kernel {
|
||||
|
||||
auto N = SymbolicSize{"num_tokens"};
|
||||
auto device_ = SymbolicDevice{};
|
||||
device_.set_options<kDLCUDA>();
|
||||
device_.set_options<kDLGPU>();
|
||||
|
||||
TensorMatcher({N, kHeadDim}) //
|
||||
.with_dtype<bf16_t>()
|
||||
|
||||
@@ -133,8 +133,12 @@ SGL_DEVICE IndexPacked index_rope_quant_pack(fp32x2_t head, fp32x2_t tail, fp32x
|
||||
// `inv_scale_ue8m0` instead of the reference's division: the scale is a
|
||||
// power of two, so both are exact except at the unreachable exponent 254.
|
||||
const auto inv_scale = deepseek_v4::fp8::inv_scale_ue8m0(static_cast<int32_t>(out.exponent[half]));
|
||||
#ifndef USE_ROCM
|
||||
const auto code = __nv_cvt_float2_to_fp4x2(
|
||||
fp32x2_t{data[half * 2] * inv_scale, data[half * 2 + 1] * inv_scale}, __NV_E2M1, cudaRoundNearest);
|
||||
#else
|
||||
const auto code = fp4::e2m1x2_code(fp32x2_t{data[half * 2] * inv_scale, data[half * 2 + 1] * inv_scale});
|
||||
#endif
|
||||
out.payload[half] = clear_negative_zero(static_cast<uint32_t>(code));
|
||||
}
|
||||
return out;
|
||||
@@ -147,8 +151,15 @@ SGL_DEVICE IndexPacked index_rope_quant_pack(fp32x2_t head, fp32x2_t tail, fp32x
|
||||
/// the warp. Every lane must reach it: the shuffles are warp-wide.
|
||||
SGL_DEVICE uint32_t index_scale_word(const uint32_t (&exponent)[2]) {
|
||||
using namespace device;
|
||||
#ifndef USE_ROCM
|
||||
const auto exp_1 = __shfl_sync(warp::kFullMask, exponent[0], kWarpThreads / 2);
|
||||
const auto exp_3 = __shfl_sync(warp::kFullMask, exponent[1], kWarpThreads / 2);
|
||||
#else
|
||||
// A wave holds two of these 32-lane rows, so the shuffle width has to be the
|
||||
// row, not the wave, for lane 16 to be this row's lane 16.
|
||||
const auto exp_1 = __shfl(exponent[0], kWarpThreads / 2, kWarpThreads);
|
||||
const auto exp_3 = __shfl(exponent[1], kWarpThreads / 2, kWarpThreads);
|
||||
#endif
|
||||
return exponent[0] | (exp_1 << 8) | (exponent[1] << 16) | (exp_3 << 24);
|
||||
}
|
||||
|
||||
@@ -310,7 +321,7 @@ struct IndexKKernel {
|
||||
|
||||
auto N = SymbolicSize{"num_tokens"};
|
||||
auto device_ = SymbolicDevice{};
|
||||
device_.set_options<kDLCUDA>();
|
||||
device_.set_options<kDLGPU>();
|
||||
|
||||
TensorMatcher({N, kHeadDim}).with_dtype<bf16_t>().with_device(device_).verify(input);
|
||||
TensorMatcher({kHeadDim}).with_dtype<bf16_t>().with_device(device_).verify(norm_weight);
|
||||
@@ -390,7 +401,7 @@ struct IndexQKernel {
|
||||
auto H = SymbolicSize{"heads"};
|
||||
auto R = SymbolicSize{"num_rows"};
|
||||
auto device_ = SymbolicDevice{};
|
||||
device_.set_options<kDLCUDA>();
|
||||
device_.set_options<kDLGPU>();
|
||||
|
||||
TensorMatcher({N, H, kHeadDim}).with_dtype<bf16_t>().with_device(device_).verify(input);
|
||||
// Real/imag interleaved, so the trailing dim is kRopeDim, not kRopeDim / 2.
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
/// Index-K write of `fp4_indexer_rope.cuh` into the split FlyDSL layout: payload `[npages, 1, 4, kPageSize, 16]`
|
||||
/// (chunk `c` holds elements `[32c, 32c + 32)`) and ue8m0 exponents `[npages, 1, 4, kPageSize]` with
|
||||
/// the slot axis transposed as a 16 x 4 tile -- the bytes `store_fp4_index_k_cache_split` writes.
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifndef USE_ROCM
|
||||
#error "fp4_indexer_rope_hip.cuh writes the FlyDSL index-K layout, which exists on ROCm only"
|
||||
#endif
|
||||
|
||||
#include "fp4_indexer_rope.cuh"
|
||||
|
||||
namespace sglang {
|
||||
|
||||
struct IndexKSplitParams {
|
||||
const bf16_t* __restrict__ input; // [num_tokens, kHeadDim] bf16, pre-norm
|
||||
const bf16_t* __restrict__ norm_weight; // [kHeadDim] bf16
|
||||
const float* __restrict__ freqs_cis; // [max_pos, kRopeDim] fp32, real/imag interleaved
|
||||
const void* __restrict__ positions; // [num_tokens] PosT
|
||||
const int64_t* __restrict__ loc; // [num_tokens] index-K slot; 0 publishes nothing
|
||||
uint8_t* __restrict__ payload; // [npages, 1, 4, kPageSize, 16] uint8
|
||||
uint8_t* __restrict__ scale; // [npages, 1, 4, kPageSize] uint8
|
||||
uint32_t num_tokens;
|
||||
float eps;
|
||||
};
|
||||
|
||||
template <bool kUsePDL, int64_t kHeadDim, int64_t kRopeDim, uint32_t kPageSize, uint32_t kRatio, typename PosT>
|
||||
__global__ __launch_bounds__(kFp4RopeWarpsPerCTA* device::kWarpThreads) void flash_index_k_split_kernel(
|
||||
const IndexKSplitParams params) {
|
||||
using namespace device;
|
||||
|
||||
static_assert(kPageSize == 64, "the 16 x 4 slot transpose of the scale spans a 64-slot page");
|
||||
static_assert(
|
||||
kHeadDim == 128 && kRopeDim == 64,
|
||||
"the one-warp tiling is specific to a 128-wide row whose second half is the RoPE tail");
|
||||
static_assert(kHeadDim == 2 * kWarpThreads * 2, "a lane owns one bf16x2 of each half");
|
||||
static_assert(std::has_single_bit(kRatio), "group_pos is derived by masking, so the ratio must be a power of two");
|
||||
|
||||
using bf16_vec_t = AlignedVector<bf16x2_t, 1>;
|
||||
using fp32_vec_t = AlignedVector<float, 2>;
|
||||
|
||||
const auto lane = threadIdx.x % kWarpThreads;
|
||||
const auto row = blockIdx.x * kFp4RopeWarpsPerCTA + threadIdx.x / kWarpThreads;
|
||||
// Warp-uniform, so the reductions below still see a full warp.
|
||||
if (row >= params.num_tokens) return;
|
||||
|
||||
// step metadata, not the predecessor's output: loaded ahead of the PDL gate; slot 0 rows drop at the store
|
||||
const auto slot_id = params.loc[row];
|
||||
const auto position = static_cast<int64_t>(static_cast<const PosT*>(params.positions)[row]);
|
||||
PDLWaitPrimary<kUsePDL>();
|
||||
|
||||
bf16_vec_t head_in, tail_in, head_w, tail_w;
|
||||
head_in.load(params.input + row * kHeadDim, lane);
|
||||
tail_in.load(params.input + row * kHeadDim, lane + kWarpThreads);
|
||||
head_w.load(params.norm_weight, lane);
|
||||
tail_w.load(params.norm_weight, lane + kWarpThreads);
|
||||
fp32_vec_t freq;
|
||||
freq.load(params.freqs_cis + (position & ~static_cast<int64_t>(kRatio - 1)) * kRopeDim, lane);
|
||||
|
||||
fp32x2_t head, tail;
|
||||
{
|
||||
const auto [h0, h1] = cast<fp32x2_t>(head_in[0]);
|
||||
const auto [t0, t1] = cast<fp32x2_t>(tail_in[0]);
|
||||
const auto sqrsum = warp::reduce_sum(h0 * h0 + h1 * h1 + t0 * t0 + t1 * t1);
|
||||
const auto inv_rms = math::rsqrt(sqrsum * (1.0f / static_cast<float>(kHeadDim)) + params.eps);
|
||||
const auto [wh0, wh1] = cast<fp32x2_t>(head_w[0]);
|
||||
const auto [wt0, wt1] = cast<fp32x2_t>(tail_w[0]);
|
||||
// `k_norm` materializes a bf16 tensor, so the norm result is rounded before
|
||||
// anything downstream sees it -- the RoPE below included.
|
||||
head = cast<fp32x2_t>(cast<bf16x2_t>(fp32x2_t{wh0 * (h0 * inv_rms), wh1 * (h1 * inv_rms)}));
|
||||
tail = cast<fp32x2_t>(cast<bf16x2_t>(fp32x2_t{wt0 * (t0 * inv_rms), wt1 * (t1 * inv_rms)}));
|
||||
}
|
||||
|
||||
const auto packed = index_rope_quant_pack(head, tail, fp32x2_t{freq[0], freq[1]});
|
||||
|
||||
// slot 0 is reserved: padded graph rows and, at ratio > 1, rows completing no group publish nothing
|
||||
if (slot_id <= 0) return;
|
||||
const auto page = slot_id / kPageSize;
|
||||
const auto slot = slot_id % kPageSize;
|
||||
|
||||
// byte j of the slot is chunk j / 16, byte j % 16: the head pair at byte lane, the tail pair at lane + 32
|
||||
const auto payload_ptr = params.payload + page * (4 * kPageSize * 16) + slot * 16;
|
||||
payload_ptr[(lane / 16) * (kPageSize * 16) + lane % 16] = static_cast<uint8_t>(packed.payload[0]);
|
||||
payload_ptr[(2 + lane / 16) * (kPageSize * 16) + lane % 16] = static_cast<uint8_t>(packed.payload[1]);
|
||||
|
||||
// Lane 0 holds the exponents of blocks 0 and 2, lane 16 those of 1 and 3.
|
||||
if (lane % (kWarpThreads / 2) == 0) {
|
||||
const auto shuffled = (slot % 16) * 4 + slot / 16;
|
||||
const auto scale_ptr = params.scale + page * (4 * kPageSize) + shuffled;
|
||||
const auto first = lane / (kWarpThreads / 2);
|
||||
scale_ptr[first * kPageSize] = static_cast<uint8_t>(packed.exponent[0]);
|
||||
scale_ptr[(2 + first) * kPageSize] = static_cast<uint8_t>(packed.exponent[1]);
|
||||
}
|
||||
}
|
||||
|
||||
/// \brief Host side of `flash_index_k_split_kernel`.
|
||||
template <int64_t kHeadDim, int64_t kRopeDim, uint32_t kPageSize, uint32_t kRatio, bool kUsePDL>
|
||||
struct FlashIndexKSplitKernel {
|
||||
static constexpr uint32_t kBlockSize = kFp4RopeWarpsPerCTA * device::kWarpThreads;
|
||||
|
||||
template <typename PosT>
|
||||
static constexpr auto kernel = flash_index_k_split_kernel<kUsePDL, kHeadDim, kRopeDim, kPageSize, kRatio, PosT>;
|
||||
|
||||
/// `IndexKKernel::run_index_k`'s arguments, the cache replaced by
|
||||
/// \param payload `[npages, 1, 4, kPageSize, 16]` uint8 (the pool's
|
||||
/// `float4_e2m1fn_x2` buffer viewed as bytes).
|
||||
/// \param scale `[npages, 1, 4, kPageSize]` uint8.
|
||||
static void
|
||||
run(const tvm::ffi::TensorView input,
|
||||
const tvm::ffi::TensorView norm_weight,
|
||||
const tvm::ffi::TensorView freqs_cis,
|
||||
const tvm::ffi::TensorView positions,
|
||||
const tvm::ffi::TensorView loc,
|
||||
const tvm::ffi::TensorView payload,
|
||||
const tvm::ffi::TensorView scale,
|
||||
const float eps) {
|
||||
using namespace host;
|
||||
|
||||
auto N = SymbolicSize{"num_tokens"};
|
||||
auto P = SymbolicSize{"npages"};
|
||||
auto device_ = SymbolicDevice{};
|
||||
device_.set_options<kDLGPU>();
|
||||
|
||||
TensorMatcher({N, kHeadDim}).with_dtype<bf16_t>().with_device(device_).verify(input);
|
||||
TensorMatcher({kHeadDim}).with_dtype<bf16_t>().with_device(device_).verify(norm_weight);
|
||||
// Real/imag interleaved, so the trailing dim is kRopeDim, not kRopeDim / 2.
|
||||
TensorMatcher({-1, kRopeDim}).with_dtype<fp32_t>().with_device(device_).verify(freqs_cis);
|
||||
auto pos_dtype = SymbolicDType{};
|
||||
TensorMatcher({N}).with_dtype<int32_t, int64_t>(pos_dtype).with_device(device_).verify(positions);
|
||||
TensorMatcher({N}).with_dtype<int64_t>().with_device(device_).verify(loc);
|
||||
TensorMatcher({P, 1, 4, kPageSize, 16}).with_dtype<uint8_t>().with_device(device_).verify(payload);
|
||||
TensorMatcher({P, 1, 4, kPageSize}).with_dtype<uint8_t>().with_device(device_).verify(scale);
|
||||
|
||||
const auto num_tokens = static_cast<uint32_t>(N.unwrap());
|
||||
if (num_tokens == 0) return;
|
||||
|
||||
const auto params = IndexKSplitParams{
|
||||
.input = static_cast<const bf16_t*>(input.data_ptr()),
|
||||
.norm_weight = static_cast<const bf16_t*>(norm_weight.data_ptr()),
|
||||
.freqs_cis = static_cast<const float*>(freqs_cis.data_ptr()),
|
||||
.positions = positions.data_ptr(),
|
||||
.loc = static_cast<const int64_t*>(loc.data_ptr()),
|
||||
.payload = static_cast<uint8_t*>(payload.data_ptr()),
|
||||
.scale = static_cast<uint8_t*>(scale.data_ptr()),
|
||||
.num_tokens = num_tokens,
|
||||
.eps = eps,
|
||||
};
|
||||
const auto k_int32 = kernel<int32_t>;
|
||||
const auto k_int64 = kernel<int64_t>;
|
||||
const auto k = pos_dtype.is_type<int32_t>() ? k_int32 : k_int64;
|
||||
LaunchKernel(div_ceil(num_tokens, kFp4RopeWarpsPerCTA), kBlockSize, device_.unwrap()) //
|
||||
.enable_pdl(kUsePDL)(k, params);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace sglang
|
||||
@@ -15,6 +15,7 @@
|
||||
|
||||
#include <bit>
|
||||
#include <cstdint>
|
||||
#include <type_traits>
|
||||
|
||||
namespace sglang {
|
||||
|
||||
@@ -258,6 +259,12 @@ struct FusedKNormRopeFlashMLAParams {
|
||||
int64_t kv_stride_batch;
|
||||
uint32_t batch_size;
|
||||
float eps;
|
||||
// kRopeQ (ROCm): (B, num_q_heads, kHeadDim) DType; the trailing kRopeDim of every head
|
||||
// are rotated in place with this token's frequencies by the same launch.
|
||||
void* __restrict__ q = nullptr;
|
||||
int64_t q_stride_batch = 0;
|
||||
int64_t q_stride_head = 0;
|
||||
uint32_t num_q_heads = 0;
|
||||
};
|
||||
|
||||
template <
|
||||
@@ -267,7 +274,8 @@ template <
|
||||
typename PosT,
|
||||
int32_t kPageBits,
|
||||
deepseek_v4::KVLayout kLayout,
|
||||
bool kUsePDL>
|
||||
bool kUsePDL,
|
||||
bool kRopeQ = false>
|
||||
K_KERNEL void fused_k_norm_rope_flashmla(const __grid_constant__ FusedKNormRopeFlashMLAParams params) {
|
||||
using namespace device;
|
||||
|
||||
@@ -327,6 +335,30 @@ K_KERNEL void fused_k_norm_rope_flashmla(const __grid_constant__ FusedKNormRopeF
|
||||
}
|
||||
}
|
||||
|
||||
// Before the out_loc check: a row without a KV slot still ropes its query heads.
|
||||
if constexpr (kRopeQ) {
|
||||
// Query rope, pair j of head h at q[h * stride + kHeadDim - kRopeDim + 2j]: the cross product
|
||||
// is rounded, then one fma with the cosine, so the bf16 result is bitwise the Triton flat
|
||||
// rope kernel's on gfx950.
|
||||
static_assert(std::is_same_v<DType, bf16_t>, "the in-place query rope reads q as bf16 pairs");
|
||||
constexpr uint32_t kPairsPerHead = kRopeDim / 2;
|
||||
const auto q_row = static_cast<DType*>(params.q) + work_id * params.q_stride_batch + (kHeadDim - kRopeDim);
|
||||
const auto n_pairs = params.num_q_heads * kPairsPerHead;
|
||||
for (uint32_t p = tx; p < n_pairs; p += kFusedKBlockSize) {
|
||||
const auto head = p / kPairsPerHead;
|
||||
const auto pair = p % kPairsPerHead;
|
||||
auto* ptr = reinterpret_cast<bf16x2_t*>(q_row + head * params.q_stride_head) + pair;
|
||||
const auto qv = cast<fp32x2_t>(*ptr);
|
||||
const auto cos_v = freqs_cis[2 * pair];
|
||||
const auto sin_v = freqs_cis[2 * pair + 1];
|
||||
const float rot_real = -__fmul_rn(qv.y, sin_v);
|
||||
const float rot_imag = __fmul_rn(qv.x, sin_v);
|
||||
const float out_real = fmaf(qv.x, cos_v, rot_real);
|
||||
const float out_imag = fmaf(qv.y, cos_v, rot_imag);
|
||||
*ptr = cast<bf16x2_t>(fp32x2_t{out_real, out_imag});
|
||||
}
|
||||
}
|
||||
|
||||
// A negative out_loc marks a slot with no KV write target (e.g. the -1
|
||||
// sentinel from the full->SWA translation for out-of-window tokens or
|
||||
// padded rows); skip the row instead of writing out of bounds. Checked
|
||||
@@ -398,9 +430,9 @@ struct FusedKNormRopeFlashMLAKernel {
|
||||
static_assert(1 << kLogPageSize == kPageSize);
|
||||
static_assert(kHeadDim == 512 && kRopeDim == 64, "FlashMLA layout requires (512, 64)");
|
||||
|
||||
template <typename PosT>
|
||||
template <typename PosT, bool kRopeQ>
|
||||
static constexpr auto kernel =
|
||||
fused_k_norm_rope_flashmla<DType, kHeadDim, kRopeDim, PosT, kLogPageSize, kLayout, kUsePDL>;
|
||||
fused_k_norm_rope_flashmla<DType, kHeadDim, kRopeDim, PosT, kLogPageSize, kLayout, kUsePDL, kRopeQ>;
|
||||
|
||||
static void forward(
|
||||
const tvm::ffi::TensorView kv,
|
||||
@@ -410,6 +442,33 @@ struct FusedKNormRopeFlashMLAKernel {
|
||||
const tvm::ffi::TensorView out_loc,
|
||||
const tvm::ffi::TensorView kvcache,
|
||||
float eps) {
|
||||
forward_impl<false>(kv, kv_weight, freqs_cis, positions, out_loc, kvcache, eps, nullptr);
|
||||
}
|
||||
|
||||
/// `forward` with the same tokens' query heads `q` (B, H, kHeadDim) roped in place (ROCm).
|
||||
static void forward_with_q(
|
||||
const tvm::ffi::TensorView kv,
|
||||
const tvm::ffi::TensorView kv_weight,
|
||||
const tvm::ffi::TensorView freqs_cis,
|
||||
const tvm::ffi::TensorView positions,
|
||||
const tvm::ffi::TensorView out_loc,
|
||||
const tvm::ffi::TensorView kvcache,
|
||||
float eps,
|
||||
const tvm::ffi::TensorView q) {
|
||||
forward_impl<true>(kv, kv_weight, freqs_cis, positions, out_loc, kvcache, eps, &q);
|
||||
}
|
||||
|
||||
private:
|
||||
template <bool kRopeQ>
|
||||
static void forward_impl(
|
||||
const tvm::ffi::TensorView kv,
|
||||
const tvm::ffi::TensorView kv_weight,
|
||||
const tvm::ffi::TensorView freqs_cis,
|
||||
const tvm::ffi::TensorView positions,
|
||||
const tvm::ffi::TensorView out_loc,
|
||||
const tvm::ffi::TensorView kvcache,
|
||||
float eps,
|
||||
const tvm::ffi::TensorView* q) {
|
||||
using namespace host;
|
||||
|
||||
auto B = SymbolicSize{"batch_size"};
|
||||
@@ -447,7 +506,7 @@ struct FusedKNormRopeFlashMLAKernel {
|
||||
const auto batch_size = static_cast<uint32_t>(B.unwrap());
|
||||
if (batch_size == 0) return;
|
||||
|
||||
const auto params = FusedKNormRopeFlashMLAParams{
|
||||
auto params = FusedKNormRopeFlashMLAParams{
|
||||
.kv = kv.data_ptr(),
|
||||
.kv_weight = kv_weight.data_ptr(),
|
||||
.freqs_cis = static_cast<const float*>(freqs_cis.data_ptr()),
|
||||
@@ -458,8 +517,22 @@ struct FusedKNormRopeFlashMLAKernel {
|
||||
.batch_size = batch_size,
|
||||
.eps = eps,
|
||||
};
|
||||
const auto k_int32 = kernel<int32_t>;
|
||||
const auto k_int64 = kernel<int64_t>;
|
||||
if constexpr (kRopeQ) {
|
||||
auto H = SymbolicSize{"num_q_heads"};
|
||||
TensorMatcher({B, H, kHeadDim}) //
|
||||
.with_strides({-1, -1, 1})
|
||||
.with_dtype<DType>()
|
||||
.with_device(device_)
|
||||
.verify(*q);
|
||||
// The rope pairs are read and written as 4-byte packs.
|
||||
RuntimeCheck(q->stride(1) % 2 == 0 && q->stride(0) % 2 == 0, "q strides must be even");
|
||||
params.q = q->data_ptr();
|
||||
params.q_stride_batch = q->stride(0);
|
||||
params.q_stride_head = q->stride(1);
|
||||
params.num_q_heads = static_cast<uint32_t>(H.unwrap());
|
||||
}
|
||||
const auto k_int32 = kernel<int32_t, kRopeQ>;
|
||||
const auto k_int64 = kernel<int64_t, kRopeQ>;
|
||||
const auto k = pos_dtype.is_type<int32_t>() ? k_int32 : k_int64;
|
||||
LaunchKernel(batch_size, kFusedKBlockSize, device_.unwrap()) //
|
||||
.enable_pdl(kUsePDL)(k, params);
|
||||
|
||||
@@ -15,7 +15,9 @@
|
||||
|
||||
#include <bit>
|
||||
#include <cstdint>
|
||||
#ifndef USE_ROCM
|
||||
#include <cuda_fp8.h>
|
||||
#endif
|
||||
#include <optional>
|
||||
|
||||
namespace sglang {
|
||||
|
||||
@@ -23,18 +23,66 @@ constexpr float kAmaxFloor = 6.0f * 1.1754943508222875e-38f;
|
||||
constexpr uint32_t kBlockSize = 32;
|
||||
/// Compressed-KV elements sharing one E4M3 scale.
|
||||
constexpr uint32_t kCompressedKVBlockSize = 16;
|
||||
#ifdef USE_ROCM
|
||||
/// \brief `cvt.rn.satfinite.e2m1x2.f32` for one value, in software: RNE onto the
|
||||
/// e2m1 grid {0, .5, 1, 1.5, 2, 3, 4, 6}, saturated at 6, sign kept (also for -0).
|
||||
SGL_DEVICE uint32_t e2m1_code(float x) {
|
||||
const float mag = fminf(fabsf(x), kMax);
|
||||
const float step = mag < 2.0f ? 0.5f : (mag < 4.0f ? 1.0f : 2.0f);
|
||||
const float q = rintf(mag / step) * step;
|
||||
// q in {0, .5, 1, 1.5} -> 2q; {2, 3} -> q + 2; {4, 6} -> q / 2 + 4.
|
||||
const uint32_t idx = q < 2.0f ? static_cast<uint32_t>(q * 2.0f)
|
||||
: (q < 4.0f ? static_cast<uint32_t>(q) + 2u : static_cast<uint32_t>(q * 0.5f) + 4u);
|
||||
return idx | (__float_as_uint(x) >> 31 << 3);
|
||||
}
|
||||
|
||||
/// \brief The e2m1 grid value a code stands for, exact in fp32.
|
||||
SGL_DEVICE float e2m1_value(uint32_t code) {
|
||||
const uint32_t idx = code & 0x7u;
|
||||
const float mag = idx < 4u ? static_cast<float>(idx) * 0.5f
|
||||
: (idx < 6u ? static_cast<float>(idx) - 2.0f : static_cast<float>(idx - 4u) * 2.0f);
|
||||
return (code & 0x8u) ? -mag : mag;
|
||||
}
|
||||
|
||||
/// \brief `__nv_cvt_float2_to_fp4x2(.., __NV_E2M1, cudaRoundNearest)`: `.x` in
|
||||
/// the low nibble, `.y` in the high one.
|
||||
SGL_DEVICE uint32_t e2m1x2_code(fp32x2_t x) {
|
||||
return e2m1_code(x.x) | (e2m1_code(x.y) << 4);
|
||||
}
|
||||
|
||||
/// \brief `float(__nv_fp8_e4m3(raw))` for `raw` in `[2^-9, 448]`: RNE onto the E4M3FN
|
||||
/// grid, gfx950's `v_cvt_pk_fp8_f32` in hardware, software elsewhere (gfx942's fp8 is FNUZ).
|
||||
SGL_DEVICE float e4m3_round_rn(float raw) {
|
||||
#if defined(__HIP_DEVICE_COMPILE__) && defined(__gfx950__)
|
||||
const auto packed = __builtin_amdgcn_cvt_pk_fp8_f32(raw, raw, 0, false);
|
||||
return __builtin_amdgcn_cvt_f32_fp8(packed, 0);
|
||||
#else
|
||||
const int exponent = ilogbf(raw);
|
||||
const float step = exponent < -6 ? 0x1p-9f : ldexpf(1.0f, exponent - 3);
|
||||
return rintf(raw / step) * step;
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
|
||||
/// \brief Round amax / 6 to a positive finite E4M3 scale, ties to even.
|
||||
SGL_DEVICE float compressed_kv_scale(float amax) {
|
||||
const auto raw = fminf(fmaxf(amax * (1.0f / kMax), 0x1p-9f), 448.0f);
|
||||
#ifdef USE_ROCM
|
||||
return e4m3_round_rn(raw);
|
||||
#else
|
||||
return static_cast<float>(__nv_fp8_e4m3(raw));
|
||||
#endif
|
||||
}
|
||||
|
||||
/// \brief Quantize compressed KV with its E4M3 scale and return dequantized values.
|
||||
SGL_DEVICE fp32x2_t fake_quant_compressed_kv_x2(fp32x2_t x, float scale) {
|
||||
const fp32x2_t scaled{__fdiv_rn(x.x, scale) + 0.0f, __fdiv_rn(x.y, scale) + 0.0f};
|
||||
#ifdef USE_ROCM
|
||||
const fp32x2_t grid{e2m1_value(e2m1_code(scaled.x)), e2m1_value(e2m1_code(scaled.y))};
|
||||
#else
|
||||
const auto code = __nv_cvt_float2_to_fp4x2(scaled, __NV_E2M1, cudaRoundNearest);
|
||||
const auto grid = device::cast<fp32x2_t>(fp16x2_t{__nv_cvt_fp4x2_to_halfraw2(code, __NV_E2M1)});
|
||||
#endif
|
||||
return {grid.x * scale, grid.y * scale};
|
||||
}
|
||||
|
||||
@@ -50,8 +98,12 @@ SGL_DEVICE fp32x2_t block_scale(float amax) {
|
||||
/// scaling clears negative zero to match `torch.sign(0) == 0` in `torch_quant.round_fp4`.
|
||||
SGL_DEVICE fp32x2_t fake_quant_x2(fp32x2_t x, float scale, float inv_scale) {
|
||||
const fp32x2_t scaled{__fmaf_rn(x.x, inv_scale, 0.0f), __fmaf_rn(x.y, inv_scale, 0.0f)};
|
||||
#ifdef USE_ROCM
|
||||
const fp32x2_t grid{e2m1_value(e2m1_code(scaled.x)), e2m1_value(e2m1_code(scaled.y))};
|
||||
#else
|
||||
const auto code = __nv_cvt_float2_to_fp4x2(scaled, __NV_E2M1, cudaRoundNearest);
|
||||
const auto grid = device::cast<fp32x2_t>(fp16x2_t{__nv_cvt_fp4x2_to_halfraw2(code, __NV_E2M1)});
|
||||
#endif
|
||||
return {grid.x * scale, grid.y * scale};
|
||||
}
|
||||
|
||||
|
||||
@@ -5,8 +5,10 @@
|
||||
#include <sgl_kernel/vec.cuh>
|
||||
#include <sgl_kernel/warp.cuh>
|
||||
|
||||
#include <sgl_kernel/deepseek_v4/fp4_utils.cuh>
|
||||
#include <sgl_kernel/deepseek_v4/fp8_utils.cuh>
|
||||
|
||||
#include <bit>
|
||||
#include <cstdint>
|
||||
#ifndef USE_ROCM
|
||||
#include <cuda_fp4.h>
|
||||
@@ -111,8 +113,6 @@ struct PagedKV {
|
||||
}
|
||||
};
|
||||
|
||||
#ifndef USE_ROCM
|
||||
|
||||
namespace v41 {
|
||||
|
||||
/// The row helpers below quantize one 512-wide token spread over `512 / kVecSize` threads,
|
||||
@@ -152,7 +152,11 @@ SGL_DEVICE void store_row_fp8(uint8_t* data_row, uint8_t* scale_row, uint32_t tx
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < kVecSize / 2; ++i) {
|
||||
// `cvt.rn.satfinite.e4m3x2` directly: |x / scale| <= 448 needs no clamp.
|
||||
#ifdef USE_ROCM
|
||||
out[i] = fp8::pack_fp8(v[2 * i] * inv_scale, v[2 * i + 1] * inv_scale);
|
||||
#else
|
||||
out[i] = fp8x2_e4m3_t{fp32x2_t{v[2 * i] * inv_scale, v[2 * i + 1] * inv_scale}};
|
||||
#endif
|
||||
}
|
||||
out.store(data_row, tx);
|
||||
// Every lane of the tile holds the exponent; they all store the same byte.
|
||||
@@ -170,18 +174,29 @@ SGL_DEVICE void store_row_fp4(uint8_t* data_row, uint8_t* scale_row, uint32_t tx
|
||||
static_assert(kVecSize % 2 == 0 && kTileLanes >= 1 && (kTileLanes & (kTileLanes - 1)) == 0);
|
||||
|
||||
const float amax = warp::reduce_max<kTileLanes>(vec_amax(v));
|
||||
#ifdef USE_ROCM
|
||||
const float scale = fp4::e4m3_round_rn(fminf(fmaxf(__fdiv_rn(amax, 6.0f), 0x1p-9f), 448.0f));
|
||||
// the low byte of the pack's 16-bit storage, whether fp8x2_e4m3_t is an integer or HIP's fp8 pair
|
||||
const auto scale_bits = static_cast<uint8_t>(std::bit_cast<uint16_t>(fp8::pack_fp8(scale, scale)));
|
||||
#else
|
||||
const __nv_fp8_e4m3 scale_e4m3{fminf(fmaxf(__fdiv_rn(amax, 6.0f), 0x1p-9f), 448.0f)};
|
||||
const float scale = static_cast<float>(scale_e4m3);
|
||||
const uint8_t scale_bits = scale_e4m3.__x;
|
||||
#endif
|
||||
AlignedVector<uint8_t, kVecSize / 2> out;
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < kVecSize / 2; ++i) {
|
||||
// IEEE division by the rounded scale, as the reference does; a reciprocal multiply could cross an e2m1 tie.
|
||||
#ifdef USE_ROCM
|
||||
out[i] = static_cast<uint8_t>(fp4::e2m1x2_code({__fdiv_rn(v[2 * i], scale), __fdiv_rn(v[2 * i + 1], scale)}));
|
||||
#else
|
||||
out[i] = static_cast<uint8_t>(__nv_cvt_float2_to_fp4x2(
|
||||
fp32x2_t{__fdiv_rn(v[2 * i], scale), __fdiv_rn(v[2 * i + 1], scale)}, __NV_E2M1, cudaRoundNearest));
|
||||
#endif
|
||||
}
|
||||
out.store(data_row, tx);
|
||||
// Every lane of the tile holds the scale; they all store the same byte.
|
||||
scale_row[tx / kTileLanes] = scale_e4m3.__x;
|
||||
scale_row[tx / kTileLanes] = scale_bits;
|
||||
}
|
||||
|
||||
/// Dispatch on the layout for a 512-wide row held as `kVecSize` consecutive fp32 per thread.
|
||||
@@ -205,24 +220,6 @@ store_row(uint8_t* data_row, uint8_t* scale_row, uint32_t tx, const device::Alig
|
||||
|
||||
} // namespace v41
|
||||
|
||||
#else // USE_ROCM
|
||||
|
||||
namespace v41 {
|
||||
|
||||
template <KVLayout>
|
||||
inline constexpr bool dependent_false_v = false;
|
||||
|
||||
/// A discarded `if constexpr` branch is still parsed, so the V4 callers need the name to
|
||||
/// resolve on ROCm; instantiating it is the error.
|
||||
template <KVLayout kLayout, typename... Ts>
|
||||
SGL_DEVICE void store_row(uint8_t*, uint8_t*, uint32_t, Ts&&...) {
|
||||
static_assert(dependent_false_v<kLayout>, "the V4.1 KV layouts are CUDA only");
|
||||
}
|
||||
|
||||
} // namespace v41
|
||||
|
||||
#endif // USE_ROCM
|
||||
|
||||
} // namespace deepseek_v4
|
||||
|
||||
} // namespace sglang
|
||||
|
||||
@@ -11,6 +11,7 @@ from sglang.kernels.jit.utils import (
|
||||
load_jit,
|
||||
make_cpp_args,
|
||||
)
|
||||
from sglang.srt.utils import is_gfx95_supported
|
||||
|
||||
from .kv_layout import KVLayout
|
||||
from .utils import make_name
|
||||
@@ -95,16 +96,17 @@ def fused_store_cache(
|
||||
RoPE tail in-kernel, so ``input`` must then be the un-rotated latent.
|
||||
"""
|
||||
layout = KVLayout.parse(layout)
|
||||
if is_hip_runtime():
|
||||
assert layout is KVLayout.V4 and freqs_cis is None, (
|
||||
"the V4.1 KV layouts are CUDA (sm100) only"
|
||||
)
|
||||
if is_hip_runtime() and layout is KVLayout.V4:
|
||||
assert freqs_cis is None, "the V4 store receives already-rotated inputs"
|
||||
from sglang.kernels.ops.kvcache.triton_store_cache import (
|
||||
triton_fused_store_cache,
|
||||
)
|
||||
|
||||
triton_fused_store_cache(input, cache, indices, page_size=page_size, type=type)
|
||||
else:
|
||||
assert not is_hip_runtime() or is_gfx95_supported(), (
|
||||
"V4.1 KV stores on HIP require gfx950"
|
||||
)
|
||||
module = _jit_fused_store_module(
|
||||
name=type,
|
||||
input_dtype=input.dtype,
|
||||
|
||||
@@ -89,9 +89,9 @@ def _c2_decode_pool_kernel(
|
||||
e1 = libdevice.exp(score - m)
|
||||
denom = e0 + e1
|
||||
# The + 0.0 below prevents FMA contraction: torch rounds both products first.
|
||||
# libdevice.div_rn matches torch division; Triton's / is an approximate reciprocal.
|
||||
t0 = p_kv * libdevice.div_rn(e0, denom)
|
||||
t1 = kv * libdevice.div_rn(e1, denom)
|
||||
# tl.div_rn matches torch division; Triton's / is an approximate reciprocal on CUDA.
|
||||
t0 = p_kv * tl.div_rn(e0, denom)
|
||||
t1 = kv * tl.div_rn(e1, denom)
|
||||
t0 = t0 + 0.0
|
||||
t1 = t1 + 0.0
|
||||
pooled = t0 + t1
|
||||
|
||||
@@ -14,7 +14,7 @@ from sglang.srt.layers.attention.dsa.utils import (
|
||||
INDEXER_K_CACHE_PRESHUFFLE_TILE,
|
||||
aiter_can_use_preshuffle_paged_mqa,
|
||||
)
|
||||
from sglang.srt.utils import is_hip, is_xpu
|
||||
from sglang.srt.utils import is_gfx95_supported, is_hip, is_xpu
|
||||
|
||||
from .kv_layout import KVLayout
|
||||
from .utils import make_name
|
||||
@@ -467,7 +467,10 @@ def compress_norm_rope_store(
|
||||
assert kv.shape[-1] == 512 and not use_fp4 and not bf16_store, (
|
||||
"the V4.1 layouts are paged FlashMLA main-KV caches"
|
||||
)
|
||||
assert not is_hip() and not _is_xpu, "the V4.1 KV layouts are CUDA (sm100) only"
|
||||
assert not _is_xpu, "the V4.1 KV layouts are not supported on XPU"
|
||||
assert not is_hip() or is_gfx95_supported(), (
|
||||
"V4.1 KV stores on HIP require gfx950"
|
||||
)
|
||||
if use_fp4:
|
||||
assert kv.shape[-1] == 128
|
||||
if is_hip() and use_fp4:
|
||||
|
||||
@@ -7,6 +7,7 @@ import triton.language as tl
|
||||
from sglang.kernels.jit.utils import get_jit_cuda_arch, is_hip_runtime
|
||||
from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
|
||||
from sglang.srt.utils import is_gfx95_supported
|
||||
|
||||
fp8_dtype = torch.float8_e4m3fnuz if is_fp8_fnuz() else torch.float8_e4m3fn
|
||||
|
||||
@@ -108,9 +109,13 @@ def dequantize_k_cache_paged_v41(
|
||||
"""
|
||||
layout = KVLayout.parse(layout)
|
||||
assert layout in (KVLayout.V41, KVLayout.V41_FP4), layout
|
||||
if is_hip_runtime() or get_jit_cuda_arch().major < 10:
|
||||
if is_hip_runtime():
|
||||
supported = is_gfx95_supported()
|
||||
else:
|
||||
supported = get_jit_cuda_arch().major >= 10
|
||||
if not supported:
|
||||
raise RuntimeError(
|
||||
"DeepSeek V4.1 KV cache dequantization requires CUDA SM100 or newer"
|
||||
"DeepSeek V4.1 KV cache dequantization requires CUDA SM100 or newer, or gfx950"
|
||||
)
|
||||
assert quant_k_cache.is_contiguous()
|
||||
assert page_table_1_flattened.dtype in (torch.int32, torch.int64)
|
||||
@@ -153,6 +158,7 @@ def dequantize_k_cache_paged_v41(
|
||||
SCALE_BYTES=layout.scale_bytes,
|
||||
TILE_SIZE=layout.tile_size,
|
||||
S_OFFSET_BYTES=layout.scale_offset(page_size),
|
||||
IS_HIP=is_hip_runtime(),
|
||||
)
|
||||
return out
|
||||
|
||||
@@ -367,6 +373,18 @@ def _e2m1_code_to_fp32(code):
|
||||
return (mag.to(tl.int32, bitcast=True) | sign).to(tl.float32, bitcast=True)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _v41_dequant_to_bf16(value, IS_HIP: tl.constexpr):
|
||||
result = value.to(tl.bfloat16)
|
||||
if IS_HIP:
|
||||
# HIP conversion can produce negative NaNs; match the torch format
|
||||
# reference's canonical positive BF16 NaN without changing finite bits.
|
||||
bits = result.to(tl.uint16, bitcast=True)
|
||||
bits = tl.where((bits & 0x7FFF) > 0x7F80, 0x7FC0, bits).to(tl.uint16)
|
||||
result = bits.to(tl.bfloat16, bitcast=True)
|
||||
return result
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _dequantize_k_cache_paged_v41_fp8_kernel(
|
||||
output_ptr,
|
||||
@@ -380,6 +398,7 @@ def _dequantize_k_cache_paged_v41_fp8_kernel(
|
||||
SCALE_BYTES: tl.constexpr,
|
||||
TILE_SIZE: tl.constexpr,
|
||||
S_OFFSET_BYTES: tl.constexpr,
|
||||
IS_HIP: tl.constexpr,
|
||||
):
|
||||
# V41: 512 e4m3 values per token, then 16 ue8m0 scales (one per 32 values).
|
||||
tl.static_assert(DATA_BYTES == 512 and SCALE_BYTES == 16 and TILE_SIZE == 32)
|
||||
@@ -397,7 +416,7 @@ def _dequantize_k_cache_paged_v41_fp8_kernel(
|
||||
out = vals * _ue8m0_to_fp32(scale_u8)
|
||||
tl.store(
|
||||
output_ptr + token_id * output_stride_0 + offs,
|
||||
out.to(output_ptr.dtype.element_ty),
|
||||
_v41_dequant_to_bf16(out, IS_HIP),
|
||||
)
|
||||
|
||||
|
||||
@@ -414,6 +433,7 @@ def _dequantize_k_cache_paged_v41_fp4_kernel(
|
||||
SCALE_BYTES: tl.constexpr,
|
||||
TILE_SIZE: tl.constexpr,
|
||||
S_OFFSET_BYTES: tl.constexpr,
|
||||
IS_HIP: tl.constexpr,
|
||||
):
|
||||
# V41_FP4: 512 e2m1 codes packed two per byte (even index in the low nibble),
|
||||
# then 32 e4m3 scales (one per 16 values).
|
||||
@@ -435,8 +455,8 @@ def _dequantize_k_cache_paged_v41_fp4_kernel(
|
||||
lo = _e2m1_code_to_fp32(packed & 0xF) * scale
|
||||
hi = _e2m1_code_to_fp32(packed >> 4) * scale
|
||||
out_base = output_ptr + token_id * output_stride_0
|
||||
tl.store(out_base + 2 * boffs, lo.to(output_ptr.dtype.element_ty))
|
||||
tl.store(out_base + 2 * boffs + 1, hi.to(output_ptr.dtype.element_ty))
|
||||
tl.store(out_base + 2 * boffs, _v41_dequant_to_bf16(lo, IS_HIP))
|
||||
tl.store(out_base + 2 * boffs + 1, _v41_dequant_to_bf16(hi, IS_HIP))
|
||||
|
||||
|
||||
@triton.jit
|
||||
|
||||
@@ -72,6 +72,28 @@ def _jit_main_k_norm_rope_flashmla_module(
|
||||
)
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_main_k_norm_rope_q_flashmla_module(
|
||||
dtype: torch.dtype,
|
||||
head_dim: int,
|
||||
rope_dim: int,
|
||||
page_size: int,
|
||||
layout: KVLayout,
|
||||
):
|
||||
"""ROCm only: the K kernel above with the in-place query rope in the same launch."""
|
||||
args = make_cpp_args(
|
||||
dtype, head_dim, rope_dim, page_size, layout.cpp_name, is_arch_support_pdl()
|
||||
)
|
||||
return load_jit(
|
||||
make_name("main_k_norm_rope_q_flashmla"),
|
||||
*args,
|
||||
cuda_files=["deepseek_v4/main_norm_rope.cuh"],
|
||||
cuda_wrappers=[
|
||||
("forward_with_q", f"FusedKNormRopeFlashMLAKernel<{args}>::forward_with_q"),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_main_q_indexer_rope_hadamard_quant_module(dtype: torch.dtype):
|
||||
"""C4 indexer Q kernel: RoPE + 128-pt Hadamard + fp8 act-quant"""
|
||||
@@ -278,6 +300,7 @@ def fused_k_norm_rope_flashmla(
|
||||
kvcache: torch.Tensor,
|
||||
page_size: int,
|
||||
layout: Union[KVLayout, str] = KVLayout.V4,
|
||||
q: Optional[torch.Tensor] = None,
|
||||
) -> None:
|
||||
"""RMSNorm + RoPE ``kv`` and write it into the ``layout`` paged FlashMLA
|
||||
cache at ``out_loc``."""
|
||||
@@ -287,9 +310,18 @@ def fused_k_norm_rope_flashmla(
|
||||
rope_dim = freqs_real.shape[-1]
|
||||
if _is_xpu:
|
||||
assert layout is KVLayout.V4, "the V4.1 KV layouts are CUDA (sm100) only"
|
||||
assert q is None, "the XPU K kernel does not rope q"
|
||||
fused_k_norm_rope_flashmla_xpu(
|
||||
kv, kv_weight, freqs_real, positions, out_loc, kvcache, eps, page_size
|
||||
)
|
||||
elif q is not None:
|
||||
assert _is_hip, "only the ROCm K launch ropes q"
|
||||
module = _jit_main_k_norm_rope_q_flashmla_module(
|
||||
kv.dtype, head_dim, rope_dim, page_size, layout
|
||||
)
|
||||
module.forward_with_q(
|
||||
kv, kv_weight, freqs_real, positions, out_loc, kvcache, eps, q
|
||||
)
|
||||
else:
|
||||
module = _jit_main_k_norm_rope_flashmla_module(
|
||||
kv.dtype, head_dim, rope_dim, page_size, layout
|
||||
|
||||
@@ -74,17 +74,19 @@ def _fp4_e2m1_code_rne(x):
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _quantize_fp4_indexer_kernel(
|
||||
x,
|
||||
x_fp4,
|
||||
x_sf,
|
||||
def quantize_fp4_indexer_row(
|
||||
values,
|
||||
v0,
|
||||
v1,
|
||||
BLOCK_N: tl.constexpr,
|
||||
GROUP_N: tl.constexpr,
|
||||
RNE: tl.constexpr,
|
||||
):
|
||||
token_id = tl.program_id(0)
|
||||
"""One row of quantize_fp4_indexer_tensor: values is the fp32 [BLOCK_N] row,
|
||||
v0 / v1 its even / odd elements as fp32 [BLOCK_N // 2]. Returns
|
||||
(packed codes [BLOCK_N // 2], packed ue8m0 exponents), the two stores of the
|
||||
standalone kernel below."""
|
||||
offs = tl.arange(0, BLOCK_N)
|
||||
values = tl.load(x + token_id * BLOCK_N + offs).to(tl.float32)
|
||||
abs_values = tl.abs(values)
|
||||
|
||||
amax0 = tl.max(tl.where(offs < GROUP_N, abs_values, 0.0), axis=0)
|
||||
@@ -109,7 +111,6 @@ def _quantize_fp4_indexer_kernel(
|
||||
exp3 = _ceil_ue8m0_exp(sf3)
|
||||
|
||||
packed_sf = exp0 | (exp1 << 8) | (exp2 << 16) | (exp3 << 24)
|
||||
tl.store(x_sf + token_id, packed_sf)
|
||||
|
||||
pair_offsets = tl.arange(0, BLOCK_N // 2)
|
||||
offs0 = pair_offsets * 2
|
||||
@@ -121,8 +122,8 @@ def _quantize_fp4_indexer_kernel(
|
||||
scale0 = (scale_exp0 << 23).to(tl.float32, bitcast=True)
|
||||
scale1 = (scale_exp1 << 23).to(tl.float32, bitcast=True)
|
||||
|
||||
v0 = tl.load(x + token_id * BLOCK_N + offs0).to(tl.float32) / scale0
|
||||
v1 = tl.load(x + token_id * BLOCK_N + offs1).to(tl.float32) / scale1
|
||||
v0 = v0 / scale0
|
||||
v1 = v1 / scale1
|
||||
if RNE:
|
||||
code0 = _fp4_e2m1_code_rne(v0)
|
||||
code1 = _fp4_e2m1_code_rne(v1)
|
||||
@@ -130,6 +131,29 @@ def _quantize_fp4_indexer_kernel(
|
||||
code0 = _fp4_e2m1_code(v0)
|
||||
code1 = _fp4_e2m1_code(v1)
|
||||
packed = (code0 & 0x0F) | ((code1 & 0x0F) << 4)
|
||||
return packed, packed_sf
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _quantize_fp4_indexer_kernel(
|
||||
x,
|
||||
x_fp4,
|
||||
x_sf,
|
||||
BLOCK_N: tl.constexpr,
|
||||
GROUP_N: tl.constexpr,
|
||||
RNE: tl.constexpr,
|
||||
):
|
||||
token_id = tl.program_id(0)
|
||||
offs = tl.arange(0, BLOCK_N)
|
||||
pair_offsets = tl.arange(0, BLOCK_N // 2)
|
||||
row = x + token_id * BLOCK_N
|
||||
values = tl.load(row + offs).to(tl.float32)
|
||||
v0 = tl.load(row + pair_offsets * 2).to(tl.float32)
|
||||
v1 = tl.load(row + pair_offsets * 2 + 1).to(tl.float32)
|
||||
packed, packed_sf = quantize_fp4_indexer_row(
|
||||
values, v0, v1, BLOCK_N=BLOCK_N, GROUP_N=GROUP_N, RNE=RNE
|
||||
)
|
||||
tl.store(x_sf + token_id, packed_sf)
|
||||
tl.store(x_fp4 + token_id * (BLOCK_N // 2) + pair_offsets, packed)
|
||||
|
||||
|
||||
|
||||
@@ -6,6 +6,18 @@ import os
|
||||
from typing import TYPE_CHECKING, NamedTuple, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer import (
|
||||
quantize_fp4_indexer_row,
|
||||
quantize_fp4_indexer_tensor,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_rope_fake_quant import (
|
||||
FP4_AMAX_FLOOR,
|
||||
rope_tail_fake_quant_fp4_row,
|
||||
)
|
||||
from sglang.kernels.ops.gemm.router_gemv_hip import rocm_router_gemv_split_k
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.kernels.ops.attention.dsv4.compress import (
|
||||
@@ -28,6 +40,8 @@ _DECODE_BASE_CTA_TARGET = 1024
|
||||
# Preserve per-query parallelism when the batch itself exceeds one CTA per CU.
|
||||
_DECODE_CTAS_PER_QUERY = 4
|
||||
_PREFILL_BASE_CTA_TARGET = 1024
|
||||
# FlyDSL compiles one kernel per page-table width: bucket it so a new context length pays no JIT
|
||||
LOW_RATIO_PAGE_TABLE_BUCKET = 64
|
||||
# AITER varctx cta_info row: [batch_packed, chunk_start, chunk_count, ctx_len].
|
||||
_DECODE_CTA_INFO_WIDTH = 4
|
||||
|
||||
@@ -129,23 +143,32 @@ def _decode_cta_count(num_queries: int, max_seq_len: int) -> int:
|
||||
return min(available_ctas, target_ctas)
|
||||
|
||||
|
||||
def _guarded_pages(logical_width: int) -> int:
|
||||
def _guarded_pages(logical_width: int, page_table_bucket: int = 4) -> int:
|
||||
"""Page columns after padding for 256-token scheduling."""
|
||||
return max(4, (logical_width + 3) // 4 * 4)
|
||||
return max(
|
||||
4,
|
||||
(logical_width + page_table_bucket - 1)
|
||||
// page_table_bucket
|
||||
* page_table_bucket,
|
||||
)
|
||||
|
||||
|
||||
def _guard_page_table(page_table: torch.Tensor, out: Optional[torch.Tensor] = None):
|
||||
def _guard_page_table(
|
||||
page_table: torch.Tensor,
|
||||
out: Optional[torch.Tensor] = None,
|
||||
page_table_bucket: int = 4,
|
||||
):
|
||||
"""Pad page tables for 256-token scheduling and one-chunk lookahead."""
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer_schedule_hip import (
|
||||
pad_page_table,
|
||||
)
|
||||
|
||||
return pad_page_table(page_table, out=out)
|
||||
return pad_page_table(page_table, out=out, page_table_bucket=page_table_bucket)
|
||||
|
||||
|
||||
def logits_rows_per_chunk(page_table: torch.Tensor) -> int:
|
||||
def logits_rows_per_chunk(page_table: torch.Tensor, page_table_bucket: int = 4) -> int:
|
||||
"""Rows whose logits fit the pooled block, for callers that loop by row."""
|
||||
width = _guarded_pages(page_table.shape[1]) * _KV_BLOCK_SIZE
|
||||
width = _guarded_pages(page_table.shape[1], page_table_bucket) * _KV_BLOCK_SIZE
|
||||
return max(1, _LOGITS_BUDGET_ELEMS // width)
|
||||
|
||||
|
||||
@@ -188,6 +211,7 @@ def _alloc_logits(
|
||||
def prepare_fp4_decode_workspace(
|
||||
page_table: torch.Tensor,
|
||||
c4_seq_lens: torch.Tensor,
|
||||
page_table_bucket: int = 4,
|
||||
) -> FP4DecodeWorkspace:
|
||||
"""Build the decode page-table, schedule, and logits buffers.
|
||||
|
||||
@@ -195,11 +219,13 @@ def prepare_fp4_decode_workspace(
|
||||
kernel touches is reachable from the returned workspace, so none of it can
|
||||
be handed out again by a later capture sharing the graph memory pool.
|
||||
"""
|
||||
from aiter.ops.flydsl.kernels.mqa_logits.pa_mqa_logits_fp4 import (
|
||||
compute_varctx_schedule,
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer_schedule_hip import (
|
||||
build_decode_schedule,
|
||||
)
|
||||
|
||||
guarded, max_seq_len = _guard_page_table(page_table)
|
||||
guarded, max_seq_len = _guard_page_table(
|
||||
page_table, page_table_bucket=page_table_bucket
|
||||
)
|
||||
c4_seq_lens = _as_int32_1d(c4_seq_lens)
|
||||
num_queries = guarded.shape[0]
|
||||
cta_count = _decode_cta_count(num_queries, max_seq_len)
|
||||
@@ -208,13 +234,8 @@ def prepare_fp4_decode_workspace(
|
||||
dtype=torch.int32,
|
||||
device=guarded.device,
|
||||
)
|
||||
schedule_scratch, _, _ = compute_varctx_schedule(
|
||||
c4_seq_lens,
|
||||
block_k=256,
|
||||
parallel_unit_num=cta_count,
|
||||
max_seq_len=max_seq_len,
|
||||
next_n=1,
|
||||
cta_info_out=cta_info,
|
||||
schedule_scratch = build_decode_schedule(
|
||||
c4_seq_lens, cta_info_out=cta_info, max_seq_len=max_seq_len
|
||||
)
|
||||
return FP4DecodeWorkspace(
|
||||
guarded, c4_seq_lens, cta_info, cta_count, max_seq_len, schedule_scratch
|
||||
@@ -225,6 +246,7 @@ def prepare_fp4_prefill_workspace(
|
||||
page_table: torch.Tensor,
|
||||
c4_seq_lens: torch.Tensor,
|
||||
workspace: Optional[FP4PrefillWorkspace] = None,
|
||||
page_table_bucket: int = 4,
|
||||
) -> FP4PrefillWorkspace:
|
||||
"""Build or refresh the prefill page-table, schedule, and logits buffers.
|
||||
|
||||
@@ -247,7 +269,7 @@ def prepare_fp4_prefill_workspace(
|
||||
|
||||
c4_seq_lens = _as_int32_1d(c4_seq_lens)
|
||||
if workspace is None:
|
||||
rows, _, padded_width = padded_page_table_shape(page_table)
|
||||
rows, _, padded_width = padded_page_table_shape(page_table, page_table_bucket)
|
||||
cta_count = max(_PREFILL_BASE_CTA_TARGET, rows)
|
||||
device = page_table.device
|
||||
buffers = PrefillScheduleBuffers(rows, device)
|
||||
@@ -280,6 +302,7 @@ def prepare_fp4_prefill_workspace(
|
||||
block_k=256,
|
||||
guarded_out=workspace.guarded_page_table,
|
||||
buffers=workspace.schedule_buffers,
|
||||
page_table_bucket=page_table_bucket,
|
||||
)
|
||||
return workspace
|
||||
|
||||
@@ -297,6 +320,7 @@ def aiter_fp4_paged_mqa_logits(
|
||||
is_decode: bool,
|
||||
decode_workspace: Optional[FP4DecodeWorkspace] = None,
|
||||
prefill_workspace: Optional[FP4PrefillWorkspace] = None,
|
||||
page_table_bucket: int = 4,
|
||||
) -> torch.Tensor:
|
||||
"""Compute FP4 Q/K indexer logits with the decode or prefill FlyDSL kernel."""
|
||||
from aiter.ops.flydsl import (
|
||||
@@ -318,7 +342,26 @@ def aiter_fp4_paged_mqa_logits(
|
||||
page_table = workspace.guarded_page_table
|
||||
max_seq_len = workspace.max_seq_len
|
||||
elif is_decode:
|
||||
page_table, max_seq_len = _guard_page_table(page_table)
|
||||
# No usable workspace: build the schedule here too, rather than letting
|
||||
# AITER build it with its varctx kernel, which cannot reserve its scratch
|
||||
# at large batches.
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer_schedule_hip import (
|
||||
build_decode_schedule,
|
||||
)
|
||||
|
||||
page_table, max_seq_len = _guard_page_table(
|
||||
page_table, page_table_bucket=page_table_bucket
|
||||
)
|
||||
cta_count = _decode_cta_count(num_tokens, max_seq_len)
|
||||
cta_info = torch.empty(
|
||||
(cta_count, _DECODE_CTA_INFO_WIDTH),
|
||||
dtype=torch.int32,
|
||||
device=page_table.device,
|
||||
)
|
||||
scratch = build_decode_schedule(
|
||||
c4_seq_lens, cta_info_out=cta_info, max_seq_len=max_seq_len
|
||||
)
|
||||
fallback_schedule = (cta_info, cta_count, scratch)
|
||||
else:
|
||||
# No usable workspace (DP padding or truncated activations): build the
|
||||
# schedule here rather than letting AITER rebuild it from ~29 torch ops
|
||||
@@ -332,7 +375,7 @@ def aiter_fp4_paged_mqa_logits(
|
||||
padded_page_table_shape,
|
||||
)
|
||||
|
||||
_, _, padded_width = padded_page_table_shape(page_table)
|
||||
_, _, padded_width = padded_page_table_shape(page_table, page_table_bucket)
|
||||
max_seq_len = padded_width * _KV_BLOCK_SIZE
|
||||
cta_count = max(_PREFILL_BASE_CTA_TARGET, num_tokens)
|
||||
cta_info = torch.empty(
|
||||
@@ -347,6 +390,7 @@ def aiter_fp4_paged_mqa_logits(
|
||||
parallel_unit_num=cta_count,
|
||||
max_seq_len=max_seq_len,
|
||||
block_k=256,
|
||||
page_table_bucket=page_table_bucket,
|
||||
)
|
||||
fallback_schedule = (cta_info, cta_count, buffers)
|
||||
q_payload = q_fp4.view(torch.uint8)
|
||||
@@ -365,16 +409,16 @@ def aiter_fp4_paged_mqa_logits(
|
||||
}
|
||||
|
||||
if is_decode:
|
||||
pinned = (
|
||||
{}
|
||||
if workspace is None
|
||||
else {
|
||||
if workspace is None:
|
||||
cta_info, cta_count, _ = fallback_schedule
|
||||
pinned = {"cta_info": cta_info, "total_ctas": cta_count}
|
||||
else:
|
||||
pinned = {
|
||||
"cta_info": workspace.cta_info,
|
||||
"total_ctas": workspace.cta_count,
|
||||
}
|
||||
)
|
||||
logits = flydsl_pa_mqa_logits_fp4(
|
||||
q_payload.reshape(num_tokens, 1, _HEADS, _HEAD_DIM // 2),
|
||||
q_payload.reshape(num_tokens, 1, q_fp4.shape[-2], _HEAD_DIM // 2),
|
||||
q_scale.reshape(num_tokens, 1, *_Q_SCALE_SHAPE),
|
||||
k_payload,
|
||||
k_scale,
|
||||
@@ -497,3 +541,362 @@ def aiter_k_indexer_fp4_cache_write(
|
||||
shuffle_scale=True,
|
||||
do_rotate_act=True,
|
||||
)
|
||||
|
||||
|
||||
# FlyDSL split payload / scale layout; bytes equal AITER's fp4 writer up to the sign of zero
|
||||
@triton.jit
|
||||
def _store_fp4_index_k_split_kernel(
|
||||
k_fp4,
|
||||
k_sf,
|
||||
payload,
|
||||
scale,
|
||||
loc,
|
||||
page_size: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
token_id = tl.program_id(0)
|
||||
cache_loc = tl.load(loc + token_id)
|
||||
page = cache_loc // page_size
|
||||
page_offset = cache_loc - page * page_size
|
||||
|
||||
# payload [page, 1, 4 chunks, page_size, 16 bytes]: chunk c holds dims [32c, 32c + 32).
|
||||
offsets = tl.arange(0, BLOCK)
|
||||
chunk = offsets // 16
|
||||
byte = offsets - chunk * 16
|
||||
k = tl.load(k_fp4 + token_id * BLOCK + offsets)
|
||||
tl.store(
|
||||
payload
|
||||
+ page * (4 * page_size * 16)
|
||||
+ chunk * (page_size * 16)
|
||||
+ page_offset * 16
|
||||
+ byte,
|
||||
k,
|
||||
)
|
||||
# scale [page, 1, 4, page_size]: the slot axis is a 16 x 4 tile transposed (the FlyDSL K ABI)
|
||||
shuffled = (page_offset % 16) * 4 + page_offset // 16
|
||||
sf = tl.load(k_sf + token_id)
|
||||
sf_offsets = tl.arange(0, 4)
|
||||
sf_bytes = ((sf >> (sf_offsets * 8)) & 0xFF).to(tl.uint8)
|
||||
tl.store(
|
||||
scale + page * (4 * page_size) + sf_offsets * page_size + shuffled, sf_bytes
|
||||
)
|
||||
|
||||
|
||||
def store_fp4_index_k_cache_split(
|
||||
input: torch.Tensor,
|
||||
payload: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
loc: torch.Tensor,
|
||||
*,
|
||||
page_size: int,
|
||||
rne: bool = False,
|
||||
) -> None:
|
||||
"""Quantize input [n, 128] to fp4 (per-32 ue8m0) and scatter row i to slot
|
||||
loc[i] of the split FlyDSL K layout (payload [pages, 1, 4, page_size, 16],
|
||||
scale [pages, 1, 4, page_size] with the slot axis 16 x 4 transposed)."""
|
||||
assert input.shape[-1] == _HEAD_DIM
|
||||
assert page_size == _KV_BLOCK_SIZE, "the 16 x 4 slot transpose spans a 64-slot page"
|
||||
assert payload.shape[1:] == (1, 4, page_size, 16), payload.shape
|
||||
assert scale.shape[1:] == (1, 4, page_size), scale.shape
|
||||
k_fp4, k_sf = quantize_fp4_indexer_tensor(input.contiguous(), rne=rne)
|
||||
n_tokens = k_fp4.shape[0]
|
||||
if n_tokens == 0:
|
||||
return
|
||||
_store_fp4_index_k_split_kernel[(n_tokens,)](
|
||||
k_fp4.view(torch.uint8),
|
||||
k_sf,
|
||||
payload.view(torch.uint8),
|
||||
scale,
|
||||
loc,
|
||||
page_size,
|
||||
BLOCK=64,
|
||||
)
|
||||
|
||||
|
||||
def read_fp4_index_k_split(
|
||||
payload: torch.Tensor, scale: torch.Tensor, slots: torch.Tensor, *, page_size: int
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Inverse of store_fp4_index_k_cache_split: (payload int8 [n, 64], scales
|
||||
int32 [n] with chunk c's e8m0 byte at bits 8c..8c+7), the layout of
|
||||
quantize_fp4_indexer_tensor."""
|
||||
assert page_size == _KV_BLOCK_SIZE, "the 16 x 4 slot transpose spans a 64-slot page"
|
||||
slots = slots.to(torch.int64)
|
||||
page, off = slots // page_size, slots % page_size
|
||||
rows = payload.view(torch.uint8)[page, 0, :, off, :] # [n, 4, 16]
|
||||
rows = rows.reshape(-1, 64).view(torch.int8)
|
||||
shuffled = (off % 16) * 4 + off // 16
|
||||
sf = scale[page, 0, :, shuffled].to(torch.int32) # [n, 4]
|
||||
packed = sf[:, 0] | (sf[:, 1] << 8) | (sf[:, 2] << 16) | (sf[:, 3] << 24)
|
||||
return rows, packed
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _quantize_fp4_query_flydsl_kernel(
|
||||
x,
|
||||
x_fp4,
|
||||
q_scale,
|
||||
HEADS: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
GROUP_N: tl.constexpr,
|
||||
):
|
||||
"""One program per (token, head slot of 64): quantize_fp4_indexer_row (RNE codes) with
|
||||
the e8m0 byte of chunk c stored straight into the FlyDSL scale layout
|
||||
[t, 0, c, h % 16, h // 16]; head slots past HEADS write zero bytes."""
|
||||
token_id = tl.program_id(0)
|
||||
h = tl.program_id(1)
|
||||
scale_base = q_scale + token_id * (4 * 16 * 4) + (h % 16) * 4 + h // 16
|
||||
chunks = tl.arange(0, 4)
|
||||
if h >= HEADS:
|
||||
tl.store(scale_base + chunks * (16 * 4), tl.zeros([4], dtype=tl.uint8))
|
||||
return
|
||||
row = token_id * HEADS + h
|
||||
values = tl.load(x + row * BLOCK_N + tl.arange(0, BLOCK_N)).to(tl.float32)
|
||||
v0, v1 = tl.split(tl.reshape(values, (BLOCK_N // 2, 2)))
|
||||
packed, packed_sf = quantize_fp4_indexer_row(
|
||||
values, v0, v1, BLOCK_N=BLOCK_N, GROUP_N=GROUP_N, RNE=True
|
||||
)
|
||||
sf_bytes = ((packed_sf >> (8 * chunks)) & 0xFF).to(tl.uint8)
|
||||
tl.store(scale_base + chunks * (16 * 4), sf_bytes)
|
||||
tl.store(x_fp4 + row * (BLOCK_N // 2) + tl.arange(0, BLOCK_N // 2), packed)
|
||||
|
||||
|
||||
def pack_fp4_query_flydsl(q: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""fp4-grid query [T, H, 128] -> (payload int8 [T, H, 64], scale uint8 [T, 1, 4, 16, 4]) in the
|
||||
FlyDSL MQA-logits layout: head h, chunk c's e8m0 byte at [t, 0, c, h % 16, h // 16] (H <= 64),
|
||||
RNE as the CUDA low-ratio path."""
|
||||
num_tokens, heads = q.shape[0], q.shape[1]
|
||||
assert heads % 16 == 0 and heads <= 64, (
|
||||
f"{heads} heads: the FlyDSL scale layout holds 16-head groups up to 64"
|
||||
)
|
||||
assert q.shape[-1] == _HEAD_DIM
|
||||
x = q.contiguous().view(-1, _HEAD_DIM)
|
||||
q_fp4 = torch.empty(
|
||||
(num_tokens, heads, _HEAD_DIM // 2), dtype=torch.int8, device=q.device
|
||||
)
|
||||
q_scale = torch.empty((num_tokens, 1, 4, 16, 4), dtype=torch.uint8, device=q.device)
|
||||
if num_tokens > 0:
|
||||
_quantize_fp4_query_flydsl_kernel[(num_tokens, 64)](
|
||||
x,
|
||||
q_fp4,
|
||||
q_scale,
|
||||
HEADS=heads,
|
||||
BLOCK_N=_HEAD_DIM,
|
||||
GROUP_N=_GROUP_SIZE,
|
||||
)
|
||||
return q_fp4, q_scale
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _reduce_scale_bf16_block(
|
||||
block,
|
||||
part_ptr,
|
||||
out_ptr,
|
||||
M,
|
||||
stride_ps,
|
||||
stride_pm,
|
||||
stride_om,
|
||||
scale,
|
||||
N: tl.constexpr,
|
||||
SPLIT_K: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
"""One BLOCK of _reduce_scale_bf16_kernel (shared with the fused index-Q launch)."""
|
||||
offs = block * BLOCK + tl.arange(0, BLOCK)
|
||||
m = offs // N
|
||||
n = offs % N
|
||||
mask = m < M
|
||||
acc = tl.load(part_ptr + m * stride_pm + n, mask=mask, other=0.0)
|
||||
for s in tl.static_range(1, SPLIT_K):
|
||||
acc += tl.load(
|
||||
part_ptr + s * stride_ps + m * stride_pm + n, mask=mask, other=0.0
|
||||
)
|
||||
# bf16(bf16(sum) * scale): the linear's bf16 output, then the aten multiply rounded to bf16
|
||||
w = acc.to(tl.bfloat16).to(tl.float32) * scale
|
||||
tl.store(out_ptr + m * stride_om + n, w.to(tl.bfloat16), mask=mask)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _reduce_scale_bf16_kernel(
|
||||
part_ptr,
|
||||
out_ptr,
|
||||
M,
|
||||
stride_ps,
|
||||
stride_pm,
|
||||
stride_om,
|
||||
scale,
|
||||
N: tl.constexpr,
|
||||
SPLIT_K: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
_reduce_scale_bf16_block(
|
||||
tl.program_id(0),
|
||||
part_ptr,
|
||||
out_ptr,
|
||||
M,
|
||||
stride_ps,
|
||||
stride_pm,
|
||||
stride_om,
|
||||
scale,
|
||||
N=N,
|
||||
SPLIT_K=SPLIT_K,
|
||||
BLOCK=BLOCK,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _index_q_pack_weights_kernel(
|
||||
q_ptr, # [T, H * 128] bf16 (wq_b output)
|
||||
f_ptr, # [max_pos, RD // 2, 2] fp32: the real view of the complex freqs table
|
||||
pos_ptr, # [T] int64
|
||||
q_fp4_ptr, # [T, H, 64] int8
|
||||
q_scale_ptr, # [T, 1, 4, 16, 4] uint8
|
||||
part_ptr, # [SPLIT_K, T, H] fp32 head-weight partials
|
||||
weights_ptr, # [T, H] bf16
|
||||
stride_qt,
|
||||
stride_ps,
|
||||
stride_pm,
|
||||
weight_scale,
|
||||
T,
|
||||
num_pos,
|
||||
H: tl.constexpr,
|
||||
D: tl.constexpr,
|
||||
RD: tl.constexpr,
|
||||
AMAX_FLOOR: tl.constexpr,
|
||||
SPLIT_K: tl.constexpr,
|
||||
W_BLOCK: tl.constexpr,
|
||||
):
|
||||
"""Grid (T * H + 1,). Program t * H + h: the query row through rope_tail_fake_quant_fp4_row
|
||||
(rounded to bf16 as the standalone kernel stores it) and quantize_fp4_indexer_row (RNE), in
|
||||
the FlyDSL MQA-logits layout; program T * H: the head weights' _reduce_scale_bf16_block."""
|
||||
pid = tl.program_id(0)
|
||||
if pid < T * H:
|
||||
t = pid // H
|
||||
h = pid % H
|
||||
pos = tl.load(pos_ptr + t)
|
||||
# the caller owns positions < num_pos; an out-of-range row reads entry 0 instead of past the table
|
||||
pos = tl.where((pos >= 0) & (pos < num_pos), pos, 0)
|
||||
fq = rope_tail_fake_quant_fp4_row(
|
||||
q_ptr + t.to(tl.int64) * stride_qt + h * D,
|
||||
f_ptr + pos * RD,
|
||||
D=D,
|
||||
RD=RD,
|
||||
BLK=32,
|
||||
AMAX_FLOOR=AMAX_FLOOR,
|
||||
INVERSE=False,
|
||||
COMPRESSED_KV=False,
|
||||
)
|
||||
# the standalone path stores the fake-quant as bf16 and the packer reloads it
|
||||
values = fq.to(tl.bfloat16).to(tl.float32)
|
||||
v0, v1 = tl.split(tl.reshape(values, (D // 2, 2)))
|
||||
packed, packed_sf = quantize_fp4_indexer_row(
|
||||
values, v0, v1, BLOCK_N=D, GROUP_N=32, RNE=True
|
||||
)
|
||||
tl.store(q_fp4_ptr + pid.to(tl.int64) * (D // 2) + tl.arange(0, D // 2), packed)
|
||||
# scale bytes: chunk c of head h at [t, 0, c, h % 16, h // 16]
|
||||
c = tl.arange(0, 4)
|
||||
sf_bytes = ((packed_sf >> (8 * c)) & 0xFF).to(tl.uint8)
|
||||
base = q_scale_ptr + t.to(tl.int64) * (4 * 16 * 4) + c * (16 * 4) + (h % 16) * 4
|
||||
tl.store(base + h // 16, sf_bytes)
|
||||
if h < 16:
|
||||
# groups this head count does not have stay zero
|
||||
for g in tl.static_range(H // 16, 4):
|
||||
tl.store(base + g, tl.zeros((4,), dtype=tl.uint8))
|
||||
else:
|
||||
_reduce_scale_bf16_block(
|
||||
0,
|
||||
part_ptr,
|
||||
weights_ptr,
|
||||
T,
|
||||
stride_ps,
|
||||
stride_pm,
|
||||
H,
|
||||
weight_scale,
|
||||
N=H,
|
||||
SPLIT_K=SPLIT_K,
|
||||
BLOCK=W_BLOCK,
|
||||
)
|
||||
|
||||
|
||||
def index_q_rope_pack_weights_flydsl(
|
||||
q: torch.Tensor,
|
||||
freqs_cis: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
rope_dim: int,
|
||||
head_weight_partials: torch.Tensor,
|
||||
weight_scale: float,
|
||||
*,
|
||||
num_heads: int,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""The HIP index-Q inputs in one launch: (q_fp4 [T, H, 64] int8, q_scale [T, 1, 4, 16, 4]
|
||||
uint8, weights [T, H] bf16), bitwise pack_fp4_query_flydsl(_rope_fq4(q.view(T, H, 128),
|
||||
freqs_cis[positions], rope_dim)) and indexer_head_weights's reduce of the
|
||||
[split_k, T, H] fp32 head_weight_partials."""
|
||||
T = q.shape[0]
|
||||
H = num_heads
|
||||
assert q.dtype == torch.bfloat16 and q.dim() == 2 and q.shape[1] == H * 128
|
||||
assert q.stride(1) == 1
|
||||
assert H % 16 == 0 and 0 < H <= 64, H
|
||||
assert freqs_cis.dtype == torch.complex64 and freqs_cis.shape[1] == rope_dim // 2
|
||||
assert positions.shape == (T,)
|
||||
split_k, pt, ph = head_weight_partials.shape
|
||||
assert (pt, ph) == (T, H) and head_weight_partials.dtype == torch.float32
|
||||
f_real = torch.view_as_real(freqs_cis)
|
||||
assert f_real.is_contiguous()
|
||||
q_fp4 = torch.empty((T, H, 64), dtype=torch.int8, device=q.device)
|
||||
q_scale = torch.empty((T, 1, 4, 16, 4), dtype=torch.uint8, device=q.device)
|
||||
weights = torch.empty((T, H), dtype=torch.bfloat16, device=q.device)
|
||||
if T == 0:
|
||||
return q_fp4, q_scale, weights
|
||||
w_block = triton.next_power_of_2(T * H)
|
||||
assert w_block <= 4096, (
|
||||
f"T * H = {T * H} exceeds the 4096-wide head-weight reduce block"
|
||||
)
|
||||
_index_q_pack_weights_kernel[(T * H + 1,)](
|
||||
q,
|
||||
f_real,
|
||||
positions,
|
||||
q_fp4,
|
||||
q_scale,
|
||||
head_weight_partials,
|
||||
weights,
|
||||
q.stride(0),
|
||||
head_weight_partials.stride(0),
|
||||
head_weight_partials.stride(1),
|
||||
float(weight_scale),
|
||||
T,
|
||||
f_real.shape[0],
|
||||
H=H,
|
||||
D=128,
|
||||
RD=rope_dim,
|
||||
AMAX_FLOOR=FP4_AMAX_FLOOR,
|
||||
SPLIT_K=split_k,
|
||||
W_BLOCK=w_block,
|
||||
num_warps=4,
|
||||
)
|
||||
return q_fp4, q_scale, weights
|
||||
|
||||
|
||||
def indexer_head_weights(
|
||||
x: torch.Tensor, weight: torch.Tensor, scale: float
|
||||
) -> torch.Tensor:
|
||||
"""bf16(bf16(x @ weight.T) * scale) as a contiguous bf16 [M, N], the layout the
|
||||
FlyDSL logits kernels take. x bf16 [M, K] and weight bf16 [N, K], with
|
||||
M <= rocm_gemv_split_k_max_tokens(n=N, k=K, weight_dtype=torch.bfloat16)."""
|
||||
partials = rocm_router_gemv_split_k(x, weight)
|
||||
split_k, M, N = partials.shape
|
||||
out = torch.empty((M, N), dtype=torch.bfloat16, device=x.device)
|
||||
block = triton.next_power_of_2(M * N)
|
||||
_reduce_scale_bf16_kernel[(triton.cdiv(M * N, block),)](
|
||||
partials,
|
||||
out,
|
||||
M,
|
||||
partials.stride(0),
|
||||
partials.stride(1),
|
||||
out.stride(0),
|
||||
float(scale),
|
||||
N=N,
|
||||
SPLIT_K=split_k,
|
||||
BLOCK=block,
|
||||
num_warps=4,
|
||||
)
|
||||
return out
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
"""ROCm index-K store of the FP4 indexer into the split FlyDSL layout (payload and scale buffers)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.jit.utils import (
|
||||
cache_once,
|
||||
is_arch_support_pdl,
|
||||
load_jit,
|
||||
make_cpp_args,
|
||||
)
|
||||
|
||||
from .utils import make_name
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from tvm_ffi.module import Module
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_index_k_split_module(
|
||||
head_dim: int, rope_dim: int, page_size: int, ratio: int
|
||||
) -> Module:
|
||||
args = make_cpp_args(head_dim, rope_dim, page_size, ratio, is_arch_support_pdl())
|
||||
return load_jit(
|
||||
make_name("index_k_rope_pack_split"),
|
||||
*args,
|
||||
cuda_files=["deepseek_v4/fp4_indexer_rope_hip.cuh"],
|
||||
cuda_wrappers=[("index_k_split", f"FlashIndexKSplitKernel<{args}>::run")],
|
||||
)
|
||||
|
||||
|
||||
def index_k_norm_rope_pack_store_split(
|
||||
input: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
eps: float,
|
||||
freqs_cis: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
loc: torch.Tensor,
|
||||
payload: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
*,
|
||||
ratio: int,
|
||||
) -> None:
|
||||
"""`fp4_indexer_rope.index_k_norm_rope_pack_store` into the split
|
||||
FlyDSL K layout (payload [npages, 1, 4, page_size, 16], scale [npages, 1, 4,
|
||||
page_size] uint8); the bytes equal store_fp4_index_k_cache_split's."""
|
||||
head_dim = input.shape[-1]
|
||||
page_size = payload.shape[3]
|
||||
assert payload.shape[1:] == (1, 4, page_size, 16), payload.shape
|
||||
assert scale.shape[1:] == (1, 4, page_size), scale.shape
|
||||
_jit_index_k_split_module(
|
||||
head_dim, freqs_cis.shape[-1], page_size, ratio
|
||||
).index_k_split(
|
||||
input,
|
||||
norm_weight,
|
||||
freqs_cis,
|
||||
positions,
|
||||
loc,
|
||||
payload.view(torch.uint8),
|
||||
scale,
|
||||
float(eps),
|
||||
)
|
||||
@@ -12,6 +12,10 @@ Everything the preamble computes is a reduction or a scan over the row count,
|
||||
so it collapses into a single kernel; the only real dependency is that
|
||||
``_prefill_cta_info_kernel`` needs the finished prefix sums. That leaves two
|
||||
dispatches for the whole schedule.
|
||||
|
||||
The decode schedule (AITER's ``compute_varctx_schedule``) is built the same way:
|
||||
AITER's single kernel holds [256, rows] tiles in registers and spills past a few
|
||||
thousand rows, so its scratch cannot be reserved once the KV pool owns the memory.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -167,7 +171,9 @@ def _pad_page_table_kernel(
|
||||
)
|
||||
|
||||
|
||||
def padded_page_table_shape(page_table: torch.Tensor) -> Tuple[int, int, int]:
|
||||
def padded_page_table_shape(
|
||||
page_table: torch.Tensor, page_table_bucket: int = 4
|
||||
) -> Tuple[int, int, int]:
|
||||
"""Rows, logical width, and the 256-token-scheduling padded width.
|
||||
|
||||
The padding rule is shared with the logits sizing in ``fp4_indexer_hip``;
|
||||
@@ -176,7 +182,7 @@ def padded_page_table_shape(page_table: torch.Tensor) -> Tuple[int, int, int]:
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import _guarded_pages
|
||||
|
||||
rows, logical_width = page_table.shape
|
||||
return rows, logical_width, _guarded_pages(logical_width)
|
||||
return rows, logical_width, _guarded_pages(logical_width, page_table_bucket)
|
||||
|
||||
|
||||
def _as_int32_2d(page_table: torch.Tensor) -> torch.Tensor:
|
||||
@@ -189,7 +195,9 @@ def _as_int32_2d(page_table: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
|
||||
def pad_page_table(
|
||||
page_table: torch.Tensor, out: Optional[torch.Tensor] = None
|
||||
page_table: torch.Tensor,
|
||||
out: Optional[torch.Tensor] = None,
|
||||
page_table_bucket: int = 4,
|
||||
) -> Tuple[torch.Tensor, int]:
|
||||
"""Pad a page table for 256-token scheduling in a single dispatch.
|
||||
|
||||
@@ -197,7 +205,9 @@ def pad_page_table(
|
||||
output element, so the destination never needs pre-zeroing.
|
||||
"""
|
||||
page_table = _as_int32_2d(page_table)
|
||||
rows, logical_width, padded_width = padded_page_table_shape(page_table)
|
||||
rows, logical_width, padded_width = padded_page_table_shape(
|
||||
page_table, page_table_bucket
|
||||
)
|
||||
if out is None:
|
||||
out = page_table.new_empty((rows, padded_width + 4))
|
||||
else:
|
||||
@@ -269,6 +279,7 @@ def build_prefill_schedule(
|
||||
block_k: int = 256,
|
||||
guarded_out: Optional[torch.Tensor] = None,
|
||||
buffers: Optional[PrefillScheduleBuffers] = None,
|
||||
page_table_bucket: int = 4,
|
||||
) -> Tuple[torch.Tensor, PrefillScheduleBuffers]:
|
||||
"""Pad the page table and build the FP4 prefill schedule in two dispatches.
|
||||
|
||||
@@ -282,7 +293,9 @@ def build_prefill_schedule(
|
||||
)
|
||||
|
||||
page_table = _as_int32_2d(page_table)
|
||||
rows, logical_width, padded_width = padded_page_table_shape(page_table)
|
||||
rows, logical_width, padded_width = padded_page_table_shape(
|
||||
page_table, page_table_bucket
|
||||
)
|
||||
total_rows = local_ends.shape[0]
|
||||
assert total_rows <= rows, (
|
||||
f"local_ends rows {total_rows} exceed the page table's {rows}; the "
|
||||
@@ -363,3 +376,153 @@ def build_prefill_schedule(
|
||||
BLOCK_P=BLOCK_P,
|
||||
)
|
||||
return guarded_out, buffers
|
||||
|
||||
|
||||
# Rows per block of the decode prep kernel's loops: its registers stay fixed as the
|
||||
# batch grows, where AITER's varctx kernel holds every row in one tile.
|
||||
_DECODE_ROW_BLOCK = 1024
|
||||
# Slots per program of the decode cta_info kernel, as AITER's varctx kernel.
|
||||
_DECODE_SLOT_BLOCK = 256
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _decode_schedule_prep_kernel(
|
||||
ctx_ptr, # [B] int32 c4 context lengths
|
||||
incl_ptr, # [B] int32 out, inclusive prefix sum of per-row CTA counts
|
||||
scalars_ptr, # [2] int32 out, [safe, total_splits]
|
||||
B,
|
||||
P,
|
||||
block_k,
|
||||
s_max,
|
||||
ROW_BLOCK: tl.constexpr,
|
||||
):
|
||||
"""AITER varctx schedule preamble (next_n = 1): the split factor and the prefix sums."""
|
||||
off = tl.arange(0, ROW_BLOCK)
|
||||
zero = tl.sum(tl.zeros([ROW_BLOCK], tl.int32), axis=0)
|
||||
|
||||
max_chunks = zero + 1
|
||||
any_total = zero
|
||||
for start in range(0, B, ROW_BLOCK):
|
||||
live = start + off < B
|
||||
ctx = tl.load(ctx_ptr + start + off, mask=live, other=0)
|
||||
chunks = tl.where(live, (ctx + block_k - 1) // block_k, 0)
|
||||
max_chunks = tl.maximum(max_chunks, tl.max(chunks, axis=0))
|
||||
any_total += tl.sum((chunks + s_max - 1) // s_max, axis=0)
|
||||
|
||||
# Smallest split factor s with sum_i ceil(chunks_i / s) <= P, as AITER's search.
|
||||
lo = zero + 1
|
||||
hi = zero + s_max
|
||||
for _ in tl.static_range(32):
|
||||
searching = lo < hi
|
||||
mid = (lo + hi) // 2
|
||||
total = zero
|
||||
for start in range(0, B, ROW_BLOCK):
|
||||
live = start + off < B
|
||||
ctx = tl.load(ctx_ptr + start + off, mask=live, other=0)
|
||||
chunks = tl.where(live, (ctx + block_k - 1) // block_k, 0)
|
||||
total += tl.sum((chunks + mid - 1) // mid, axis=0)
|
||||
fits = total <= P
|
||||
hi = tl.where(searching & fits, mid, hi)
|
||||
lo = tl.where(searching & (fits == 0), mid + 1, lo)
|
||||
safe = tl.where(any_total <= P, lo, max_chunks)
|
||||
|
||||
carry = zero
|
||||
for start in range(0, B, ROW_BLOCK):
|
||||
live = start + off < B
|
||||
ctx = tl.load(ctx_ptr + start + off, mask=live, other=0)
|
||||
chunks = tl.where(live, (ctx + block_k - 1) // block_k, 0)
|
||||
ctas = (chunks + safe - 1) // safe
|
||||
tl.store(incl_ptr + start + off, tl.cumsum(ctas, axis=0) + carry, mask=live)
|
||||
carry += tl.sum(ctas, axis=0)
|
||||
tl.store(scalars_ptr, safe)
|
||||
tl.store(scalars_ptr + 1, carry)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _decode_cta_info_kernel(
|
||||
ctx_ptr, # [B] int32 c4 context lengths
|
||||
incl_ptr, # [B] int32 inclusive prefix sum of per-row CTA counts
|
||||
scalars_ptr, # [2] int32 [safe, total_splits]
|
||||
cta_info_ptr, # [P, 4] int32 out, [batch, chunk_start, chunk_count, ctx_len]
|
||||
B,
|
||||
P,
|
||||
block_k,
|
||||
BLOCK_S: tl.constexpr,
|
||||
):
|
||||
"""AITER varctx cta_info rows (next_n = 1), each slot's row found by a binary search
|
||||
over incl in global memory, as AITER's _prefill_cta_info_kernel does."""
|
||||
safe = tl.load(scalars_ptr)
|
||||
total_splits = tl.load(scalars_ptr + 1)
|
||||
slot = tl.program_id(0) * BLOCK_S + tl.arange(0, BLOCK_S)
|
||||
smask = slot < P
|
||||
|
||||
# searchsorted(incl, slot, right=True) = count(incl <= slot)
|
||||
lo = tl.zeros([BLOCK_S], tl.int32)
|
||||
hi = tl.full([BLOCK_S], B, tl.int32)
|
||||
for _ in tl.static_range(32):
|
||||
mid = (lo + hi) // 2
|
||||
incl_mid = tl.load(
|
||||
incl_ptr + tl.minimum(mid, B - 1), mask=mid < B, other=2147483647
|
||||
)
|
||||
go_right = incl_mid <= slot
|
||||
lo = tl.where(go_right, mid + 1, lo)
|
||||
hi = tl.where(go_right, hi, mid)
|
||||
row = tl.minimum(lo, B - 1)
|
||||
|
||||
ctx = tl.load(ctx_ptr + row, mask=smask, other=0)
|
||||
chunks = (ctx + block_k - 1) // block_k
|
||||
excl = tl.load(incl_ptr + row, mask=smask, other=0) - (chunks + safe - 1) // safe
|
||||
|
||||
valid = slot < total_splits
|
||||
vi = valid.to(tl.int32)
|
||||
start = (slot - excl) * safe # pre-mask (count uses this)
|
||||
count = tl.maximum(tl.minimum(safe, chunks - start), 0)
|
||||
base = slot * 4
|
||||
tl.store(cta_info_ptr + base + 0, row * vi, mask=smask)
|
||||
tl.store(cta_info_ptr + base + 1, start * vi, mask=smask)
|
||||
tl.store(cta_info_ptr + base + 2, tl.where(valid, count, 1), mask=smask)
|
||||
tl.store(cta_info_ptr + base + 3, ctx * vi, mask=smask)
|
||||
|
||||
|
||||
def build_decode_schedule(
|
||||
context_lens: torch.Tensor,
|
||||
*,
|
||||
cta_info_out: torch.Tensor,
|
||||
max_seq_len: int,
|
||||
block_k: int = 256,
|
||||
) -> torch.Tensor:
|
||||
"""AITER's ``compute_varctx_schedule`` (next_n = 1) into ``cta_info_out`` [P, 4] in two
|
||||
dispatches whose registers do not grow with the row count. Returns the scratch the
|
||||
kernels read ([safe, total_splits] then the per-row prefix sums): a captured graph
|
||||
must keep it alive."""
|
||||
context_lens = context_lens.to(torch.int32)
|
||||
rows = context_lens.shape[0]
|
||||
ctas = cta_info_out.shape[0]
|
||||
assert rows <= ctas, (
|
||||
f"{rows} rows on {ctas} CTAs: the persistent grid must give every row a CTA"
|
||||
)
|
||||
scratch = torch.empty(rows + 2, dtype=torch.int32, device=context_lens.device)
|
||||
if rows == 0:
|
||||
return scratch
|
||||
scalars, incl = scratch[:2], scratch[2:]
|
||||
_decode_schedule_prep_kernel[(1,)](
|
||||
context_lens,
|
||||
incl,
|
||||
scalars,
|
||||
rows,
|
||||
ctas,
|
||||
block_k,
|
||||
max(1, triton.cdiv(max_seq_len, block_k)),
|
||||
ROW_BLOCK=_DECODE_ROW_BLOCK,
|
||||
)
|
||||
_decode_cta_info_kernel[(triton.cdiv(ctas, _DECODE_SLOT_BLOCK),)](
|
||||
context_lens,
|
||||
incl,
|
||||
scalars,
|
||||
cta_info_out,
|
||||
rows,
|
||||
ctas,
|
||||
block_k,
|
||||
BLOCK_S=_DECODE_SLOT_BLOCK,
|
||||
)
|
||||
return scratch
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
@@ -11,14 +13,9 @@ from sglang.kernels.ops.attention.dsv4.torch_quant import FP4_AMAX_FLOOR
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _rope_tail_fake_quant_fp4_kernel(
|
||||
x_ptr,
|
||||
f_ptr,
|
||||
out_ptr,
|
||||
x_stride_r,
|
||||
out_stride_r,
|
||||
f_stride_t,
|
||||
rows_per_token,
|
||||
def rope_tail_fake_quant_fp4_row(
|
||||
x_row_ptr,
|
||||
f_row_ptr,
|
||||
D: tl.constexpr,
|
||||
RD: tl.constexpr,
|
||||
BLK: tl.constexpr,
|
||||
@@ -26,10 +23,10 @@ def _rope_tail_fake_quant_fp4_kernel(
|
||||
INVERSE: tl.constexpr,
|
||||
COMPRESSED_KV: tl.constexpr,
|
||||
):
|
||||
r = tl.program_id(0)
|
||||
t = r // rows_per_token
|
||||
"""One row of rope_tail_fake_quant_fp4 as an fp32 [D] vector: x_row_ptr points
|
||||
at the row, f_row_ptr at the token's [RD // 2, 2] real view of the complex freqs."""
|
||||
offs = tl.arange(0, D)
|
||||
v = tl.load(x_ptr + r * x_stride_r + offs).to(tl.float32)
|
||||
v = tl.load(x_row_ptr + offs).to(tl.float32)
|
||||
|
||||
# ---- rope_tail: adjacent pairs of the last RD features as one complex number
|
||||
head_len = D - RD
|
||||
@@ -37,15 +34,13 @@ def _rope_tail_fake_quant_fp4_kernel(
|
||||
pos = offs - head_len
|
||||
j = pos // 2
|
||||
is_im = (pos % 2) == 1
|
||||
re = tl.load(x_ptr + x_stride_r * r + head_len + 2 * j, mask=in_tail, other=0.0).to(
|
||||
re = tl.load(x_row_ptr + head_len + 2 * j, mask=in_tail, other=0.0).to(tl.float32)
|
||||
im = tl.load(x_row_ptr + head_len + 2 * j + 1, mask=in_tail, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
im = tl.load(
|
||||
x_ptr + x_stride_r * r + head_len + 2 * j + 1, mask=in_tail, other=0.0
|
||||
).to(tl.float32)
|
||||
# freqs is a real/imag-interleaved view: stride 2 between complex pairs.
|
||||
fr = tl.load(f_ptr + t * f_stride_t + 2 * j, mask=in_tail, other=1.0)
|
||||
fi = tl.load(f_ptr + t * f_stride_t + 2 * j + 1, mask=in_tail, other=0.0)
|
||||
fr = tl.load(f_row_ptr + 2 * j, mask=in_tail, other=1.0)
|
||||
fi = tl.load(f_row_ptr + 2 * j + 1, mask=in_tail, other=0.0)
|
||||
if INVERSE:
|
||||
fi = -fi
|
||||
rot = tl.where(is_im, re * fi + im * fr, re * fr - im * fi)
|
||||
@@ -75,8 +70,48 @@ def _rope_tail_fake_quant_fp4_kernel(
|
||||
# torch.round is round-half-to-even; torch.sign(0) is 0
|
||||
sgn = tl.where(s > 0, 1.0, tl.where(s < 0, -1.0, 0.0))
|
||||
q = libdevice.rint(mag / step) * step * sgn
|
||||
out = tl.reshape(q * scale[:, None], (D,))
|
||||
tl.store(out_ptr + r * out_stride_r + offs, out.to(out_ptr.dtype.element_ty))
|
||||
return tl.reshape(q * scale[:, None], (D,))
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _rope_tail_fake_quant_fp4_kernel(
|
||||
x_ptr,
|
||||
f_ptr,
|
||||
pos_ptr,
|
||||
out_ptr,
|
||||
x_stride_r,
|
||||
out_stride_r,
|
||||
f_stride_t,
|
||||
rows_per_token,
|
||||
num_pos,
|
||||
D: tl.constexpr,
|
||||
RD: tl.constexpr,
|
||||
BLK: tl.constexpr,
|
||||
AMAX_FLOOR: tl.constexpr,
|
||||
INVERSE: tl.constexpr,
|
||||
COMPRESSED_KV: tl.constexpr,
|
||||
HAS_POS: tl.constexpr,
|
||||
):
|
||||
r = tl.program_id(0)
|
||||
t = r // rows_per_token
|
||||
if HAS_POS:
|
||||
# freqs is the whole table: row t reads its position's entry (the freqs[positions] gather folded in)
|
||||
t = tl.load(pos_ptr + t).to(tl.int64)
|
||||
# the caller owns positions < num_pos; an out-of-range row reads entry 0 instead of past the table
|
||||
t = tl.where((t >= 0) & (t < num_pos), t, 0)
|
||||
out = rope_tail_fake_quant_fp4_row(
|
||||
x_ptr + r * x_stride_r,
|
||||
f_ptr + t * f_stride_t,
|
||||
D=D,
|
||||
RD=RD,
|
||||
BLK=BLK,
|
||||
AMAX_FLOOR=AMAX_FLOOR,
|
||||
INVERSE=INVERSE,
|
||||
COMPRESSED_KV=COMPRESSED_KV,
|
||||
)
|
||||
tl.store(
|
||||
out_ptr + r * out_stride_r + tl.arange(0, D), out.to(out_ptr.dtype.element_ty)
|
||||
)
|
||||
|
||||
|
||||
def rope_tail_fake_quant_fp4(
|
||||
@@ -87,10 +122,12 @@ def rope_tail_fake_quant_fp4(
|
||||
block_size: int = 32,
|
||||
*,
|
||||
compressed_kv: bool = False,
|
||||
positions: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
"""RoPE and FP4 round-trip: per-16 E4M3 for compressed KV, per-32 UE8M0 otherwise.
|
||||
|
||||
x: [T, ..., D] contiguous in the last dim; freqs: complex [T, rope_dim // 2].
|
||||
x: [T, ..., D] contiguous in the last dim; freqs: complex [T, rope_dim // 2], or with
|
||||
positions ([T] int) the whole table that row t reads at positions[t].
|
||||
"""
|
||||
x = x.contiguous()
|
||||
if compressed_kv:
|
||||
@@ -105,20 +142,29 @@ def rope_tail_fake_quant_fp4(
|
||||
return out
|
||||
rows_per_token = rows // x.shape[0]
|
||||
f_real = torch.view_as_real(freqs.contiguous()).contiguous()
|
||||
if positions is not None:
|
||||
assert positions.dim() == 1 and positions.shape[0] == x.shape[0], (
|
||||
positions.shape,
|
||||
x.shape,
|
||||
)
|
||||
positions = positions.contiguous()
|
||||
_rope_tail_fake_quant_fp4_kernel[(rows,)](
|
||||
x2,
|
||||
f_real,
|
||||
positions if positions is not None else f_real,
|
||||
out.reshape(-1, d),
|
||||
x2.stride(0),
|
||||
d,
|
||||
f_real.stride(0),
|
||||
rows_per_token,
|
||||
f_real.shape[0],
|
||||
D=d,
|
||||
RD=rope_dim,
|
||||
BLK=block_size,
|
||||
AMAX_FLOOR=FP4_AMAX_FLOOR,
|
||||
INVERSE=inverse,
|
||||
COMPRESSED_KV=compressed_kv,
|
||||
HAS_POS=positions is not None,
|
||||
num_warps=4,
|
||||
)
|
||||
return out
|
||||
|
||||
@@ -17,6 +17,7 @@ import torch
|
||||
from sglang.kernels.jit.utils import (
|
||||
cache_once,
|
||||
is_arch_support_pdl,
|
||||
is_hip_runtime,
|
||||
load_jit,
|
||||
make_cpp_args,
|
||||
)
|
||||
@@ -98,7 +99,8 @@ def c1_decode_norm_rope_store(
|
||||
freqs_cis,
|
||||
positions,
|
||||
out_loc,
|
||||
k_cache,
|
||||
# HIP's fp8_e4m3_t is uint8_t, so the kernel matches the fp8 pool as bytes
|
||||
k_cache.view(torch.uint8) if is_hip_runtime() else k_cache,
|
||||
float(eps),
|
||||
)
|
||||
return out
|
||||
@@ -176,7 +178,8 @@ def c2_decode_norm_rope_store(
|
||||
raw_out_loc,
|
||||
eps,
|
||||
freqs_cis,
|
||||
k_cache,
|
||||
# HIP's fp8_e4m3_t is uint8_t, so the kernel matches the fp8 pool as bytes
|
||||
k_cache.view(torch.uint8) if is_hip_runtime() else k_cache,
|
||||
ring_size,
|
||||
draft_len,
|
||||
)
|
||||
|
||||
@@ -5,13 +5,6 @@ from typing import Optional
|
||||
import torch
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.utils import get_bool_env_var, is_hip
|
||||
|
||||
_is_hip = is_hip()
|
||||
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip
|
||||
|
||||
if _use_aiter:
|
||||
from aiter.tuned_gemm import tgemm
|
||||
|
||||
_linear_bf16_fp32_algo = envs.SGLANG_OPT_BF16_FP32_GEMM_ALGO.get()
|
||||
_HPC_GEMM_WEIGHT_CACHE_ATTR = "_sglang_bf16xfp32_weight_cache"
|
||||
@@ -147,9 +140,7 @@ def linear_bf16_fp32(
|
||||
*,
|
||||
hpc_kernel_min_m: Optional[int] = None,
|
||||
) -> torch.Tensor:
|
||||
if _use_aiter and y.dtype == torch.bfloat16:
|
||||
return tgemm.mm(x, y, otype=x.dtype).float()
|
||||
elif hpc_kernel_min_m is not None:
|
||||
if hpc_kernel_min_m is not None:
|
||||
output = _linear_bf16_fp32_hpc(x, y, min_m=hpc_kernel_min_m)
|
||||
if output is not None:
|
||||
return output
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
"""Skinny bf16 GEMV on ROCm for decode row counts: [M, K] x [N, K] as fp32 split-K
|
||||
partials whose fixed-order sum is batch invariant (the DeepSeek-V4 router and indexer head
|
||||
weights)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.srt.utils import is_gfx95_supported, is_hip
|
||||
|
||||
# the GEMV walks 16-row tiles inside one program, so every M runs the same tile (batch invariance)
|
||||
ROCM_ROUTER_MAX_TOKENS = 64
|
||||
|
||||
_BLOCK_M = 16
|
||||
_BLOCK_N = 16
|
||||
_BLOCK_K = 512
|
||||
_MAX_SPLIT_K = 32
|
||||
|
||||
|
||||
def rocm_gemv_split_k_max_tokens(*, n: int, k: int, weight_dtype: torch.dtype) -> int:
|
||||
"""Rows up to which rocm_router_gemv_split_k serves an [M, k] @ [n, k].T bf16
|
||||
GEMV (one 16-wide N tile per 512 of K, 16-row tiles), -1 when the device or the shape
|
||||
rules it out."""
|
||||
if not (is_hip() and is_gfx95_supported()):
|
||||
return -1
|
||||
if weight_dtype != torch.bfloat16:
|
||||
return -1
|
||||
if n <= 0 or n % _BLOCK_N != 0:
|
||||
return -1
|
||||
if k % _BLOCK_K != 0 or not 0 < k // _BLOCK_K <= _MAX_SPLIT_K:
|
||||
return -1
|
||||
return ROCM_ROUTER_MAX_TOKENS
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _router_gemv_split_k_kernel(
|
||||
x_ptr,
|
||||
w_ptr,
|
||||
part_ptr,
|
||||
M,
|
||||
stride_xm,
|
||||
stride_wn,
|
||||
stride_ps,
|
||||
stride_pm,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
M_TILES: tl.constexpr,
|
||||
):
|
||||
pid_n = tl.program_id(0)
|
||||
pid_k = tl.program_id(1)
|
||||
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
offs_k = pid_k * BLOCK_K + tl.arange(0, BLOCK_K)
|
||||
w = tl.load(w_ptr + offs_n[None, :] * stride_wn + offs_k[:, None])
|
||||
# the weight tile is read once; every row tile is the same BLOCK_M x BLOCK_K dot
|
||||
for t in tl.static_range(M_TILES):
|
||||
offs_m = t * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
row_mask = offs_m < M
|
||||
x = tl.load(
|
||||
x_ptr + offs_m[:, None] * stride_xm + offs_k[None, :],
|
||||
mask=row_mask[:, None],
|
||||
other=0.0,
|
||||
)
|
||||
acc = tl.dot(x, w)
|
||||
tl.store(
|
||||
part_ptr
|
||||
+ pid_k * stride_ps
|
||||
+ offs_m[:, None] * stride_pm
|
||||
+ offs_n[None, :],
|
||||
acc,
|
||||
mask=row_mask[:, None],
|
||||
)
|
||||
|
||||
|
||||
def rocm_router_gemv_split_k(x: torch.Tensor, w: torch.Tensor) -> torch.Tensor:
|
||||
"""x[M, K] @ w[N, K].T as fp32 split-K partials [K // 512, M, N], summed over dim 0 in
|
||||
order by rocm_router_reduce_partials or the fused gate."""
|
||||
M, K = x.shape
|
||||
N, K_w = w.shape
|
||||
assert K == K_w and K % _BLOCK_K == 0 and N % _BLOCK_N == 0
|
||||
assert 0 < M <= ROCM_ROUTER_MAX_TOKENS, (
|
||||
f"{M} rows: the split-K GEMV serves at most {ROCM_ROUTER_MAX_TOKENS}"
|
||||
)
|
||||
assert x.dtype == torch.bfloat16 and w.dtype == torch.bfloat16
|
||||
assert x.stride(1) == 1 and w.stride(1) == 1
|
||||
split_k = K // _BLOCK_K
|
||||
partials = torch.empty((split_k, M, N), dtype=torch.float32, device=x.device)
|
||||
_router_gemv_split_k_kernel[(N // _BLOCK_N, split_k)](
|
||||
x,
|
||||
w,
|
||||
partials,
|
||||
M,
|
||||
x.stride(0),
|
||||
w.stride(0),
|
||||
partials.stride(0),
|
||||
partials.stride(1),
|
||||
BLOCK_M=_BLOCK_M,
|
||||
BLOCK_N=_BLOCK_N,
|
||||
BLOCK_K=_BLOCK_K,
|
||||
M_TILES=triton.cdiv(M, _BLOCK_M),
|
||||
num_warps=4,
|
||||
)
|
||||
return partials
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _reduce_partials_kernel(
|
||||
part_ptr,
|
||||
out_ptr,
|
||||
M,
|
||||
stride_ps,
|
||||
stride_pm,
|
||||
stride_om,
|
||||
N: tl.constexpr,
|
||||
SPLIT_K: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
|
||||
m = offs // N
|
||||
n = offs % N
|
||||
mask = m < M
|
||||
acc = tl.load(part_ptr + m * stride_pm + n, mask=mask, other=0.0)
|
||||
for s in tl.static_range(1, SPLIT_K):
|
||||
acc += tl.load(
|
||||
part_ptr + s * stride_ps + m * stride_pm + n, mask=mask, other=0.0
|
||||
)
|
||||
tl.store(out_ptr + m * stride_om + n, acc, mask=mask)
|
||||
|
||||
|
||||
def rocm_router_reduce_partials(partials: torch.Tensor, out: torch.Tensor) -> None:
|
||||
"""out[M, N] = partials[0] + partials[1] + ... in that order, in fp32:
|
||||
the same sum the fused gate computes."""
|
||||
split_k, M, N = partials.shape
|
||||
assert out.shape == (M, N) and out.dtype == torch.float32 and out.stride(1) == 1
|
||||
block = 1024
|
||||
_reduce_partials_kernel[(triton.cdiv(M * N, block),)](
|
||||
partials,
|
||||
out,
|
||||
M,
|
||||
partials.stride(0),
|
||||
partials.stride(1),
|
||||
out.stride(0),
|
||||
N=N,
|
||||
SPLIT_K=split_k,
|
||||
BLOCK=block,
|
||||
num_warps=4,
|
||||
)
|
||||
@@ -0,0 +1,307 @@
|
||||
"""DeepSeek-V4 MoE router gate at decode row counts on ROCm: a sqrtsoftplus top-k over the
|
||||
split-K GEMV partials of router_gemv_hip, bitwise aiter's topk_gating_kernel_opt, ties
|
||||
included."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional, Tuple
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.gemm.router_gemv_hip import rocm_gemv_split_k_max_tokens
|
||||
|
||||
# aiter's wave64 kernel: EPT = 384 / 64 = 6 slots per lane; the sorting network below is N == 6
|
||||
_WARP = 64
|
||||
_GATE_NUM_EXPERTS = _WARP * 6
|
||||
_MAX_TOPK = 16
|
||||
|
||||
_FLT_MAX = tl.constexpr(3.4028234663852886e38)
|
||||
_LOG2_E = tl.constexpr(1.4426950408889634)
|
||||
_LN_2 = tl.constexpr(0.6931471805599453)
|
||||
|
||||
|
||||
def rocm_router_max_tokens(
|
||||
*,
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
topk: int,
|
||||
weight_dtype: torch.dtype,
|
||||
) -> int:
|
||||
"""Rows up to which rocm_router_gemv_split_k + rocm_router_gate
|
||||
serve the router, -1 when the device or the shape rules them out."""
|
||||
if num_experts != _GATE_NUM_EXPERTS or not 0 < topk <= _MAX_TOPK:
|
||||
return -1
|
||||
return rocm_gemv_split_k_max_tokens(
|
||||
n=num_experts, k=hidden_size, weight_dtype=weight_dtype
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _score(x):
|
||||
# aiter compute_score<SQRTSOFTPLUS>: +inf clamps to FLT_MAX, NaN passes, sqrt correctly rounded
|
||||
x = tl.where(x > _FLT_MAX, _FLT_MAX, x)
|
||||
t = tl.exp2(x * _LOG2_E)
|
||||
sp = tl.where(x > 20.0, x, tl.log2(1.0 + t) * _LN_2)
|
||||
return tl.sqrt_rn(sp)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _cas(vi, oi, ii, vj, oj, ij):
|
||||
# aiter _CAS_DESC: swap when vals[i] < vals[j]; equal values stay put.
|
||||
c = vi < vj
|
||||
return (
|
||||
tl.where(c, vj, vi),
|
||||
tl.where(c, oj, oi),
|
||||
tl.where(c, ij, ii),
|
||||
tl.where(c, vi, vj),
|
||||
tl.where(c, oi, oj),
|
||||
tl.where(c, ii, ij),
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _slot(cursor, a0, a1, a2, a3, a4, a5, other):
|
||||
r = tl.where(cursor == 0, a0, other)
|
||||
r = tl.where(cursor == 1, a1, r)
|
||||
r = tl.where(cursor == 2, a2, r)
|
||||
r = tl.where(cursor == 3, a3, r)
|
||||
r = tl.where(cursor == 4, a4, r)
|
||||
r = tl.where(cursor == 5, a5, r)
|
||||
return r
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _load_logits(
|
||||
logits_ptr, part_ptr, row, e, stride_lm, stride_ps, stride_pm, SPLIT_K: tl.constexpr
|
||||
):
|
||||
if SPLIT_K == 0:
|
||||
return tl.load(logits_ptr + row * stride_lm + e).to(tl.float32)
|
||||
else:
|
||||
acc = tl.load(part_ptr + row * stride_pm + e)
|
||||
for s in tl.static_range(1, SPLIT_K):
|
||||
acc += tl.load(part_ptr + s * stride_ps + row * stride_pm + e)
|
||||
return acc
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _gate_row(
|
||||
logits_ptr,
|
||||
part_ptr,
|
||||
bias_ptr,
|
||||
row,
|
||||
stride_lm,
|
||||
stride_ps,
|
||||
stride_pm,
|
||||
routed_scaling_factor,
|
||||
SPLIT_K: tl.constexpr,
|
||||
WRITE_LOGITS: tl.constexpr,
|
||||
HAS_BIAS: tl.constexpr,
|
||||
RENORM: tl.constexpr,
|
||||
TOPK: tl.constexpr,
|
||||
):
|
||||
"""aiter's topk_gating_kernel_opt for one row on one 64-lane wave: returns the
|
||||
scaled weights and the ids as [64] lane tensors (lane k < TOPK holds slot k)."""
|
||||
lane = tl.arange(0, 64)
|
||||
# Slot i of a lane holds expert lane + 64 * i, as in aiter's register kernel.
|
||||
i0 = lane
|
||||
i1 = lane + 64
|
||||
i2 = lane + 128
|
||||
i3 = lane + 192
|
||||
i4 = lane + 256
|
||||
i5 = lane + 320
|
||||
x0 = _load_logits(
|
||||
logits_ptr, part_ptr, row, i0, stride_lm, stride_ps, stride_pm, SPLIT_K
|
||||
)
|
||||
x1 = _load_logits(
|
||||
logits_ptr, part_ptr, row, i1, stride_lm, stride_ps, stride_pm, SPLIT_K
|
||||
)
|
||||
x2 = _load_logits(
|
||||
logits_ptr, part_ptr, row, i2, stride_lm, stride_ps, stride_pm, SPLIT_K
|
||||
)
|
||||
x3 = _load_logits(
|
||||
logits_ptr, part_ptr, row, i3, stride_lm, stride_ps, stride_pm, SPLIT_K
|
||||
)
|
||||
x4 = _load_logits(
|
||||
logits_ptr, part_ptr, row, i4, stride_lm, stride_ps, stride_pm, SPLIT_K
|
||||
)
|
||||
x5 = _load_logits(
|
||||
logits_ptr, part_ptr, row, i5, stride_lm, stride_ps, stride_pm, SPLIT_K
|
||||
)
|
||||
if WRITE_LOGITS:
|
||||
tl.store(logits_ptr + row * stride_lm + i0, x0)
|
||||
tl.store(logits_ptr + row * stride_lm + i1, x1)
|
||||
tl.store(logits_ptr + row * stride_lm + i2, x2)
|
||||
tl.store(logits_ptr + row * stride_lm + i3, x3)
|
||||
tl.store(logits_ptr + row * stride_lm + i4, x4)
|
||||
tl.store(logits_ptr + row * stride_lm + i5, x5)
|
||||
s0 = _score(x0)
|
||||
s1 = _score(x1)
|
||||
s2 = _score(x2)
|
||||
s3 = _score(x3)
|
||||
s4 = _score(x4)
|
||||
s5 = _score(x5)
|
||||
# as in aiter, a NaN score weighs 0 and its choice value is -inf
|
||||
o0 = tl.where(s0 != s0, 0.0, s0)
|
||||
o1 = tl.where(s1 != s1, 0.0, s1)
|
||||
o2 = tl.where(s2 != s2, 0.0, s2)
|
||||
o3 = tl.where(s3 != s3, 0.0, s3)
|
||||
o4 = tl.where(s4 != s4, 0.0, s4)
|
||||
o5 = tl.where(s5 != s5, 0.0, s5)
|
||||
if HAS_BIAS:
|
||||
v0 = s0 + tl.load(bias_ptr + i0).to(tl.float32)
|
||||
v1 = s1 + tl.load(bias_ptr + i1).to(tl.float32)
|
||||
v2 = s2 + tl.load(bias_ptr + i2).to(tl.float32)
|
||||
v3 = s3 + tl.load(bias_ptr + i3).to(tl.float32)
|
||||
v4 = s4 + tl.load(bias_ptr + i4).to(tl.float32)
|
||||
v5 = s5 + tl.load(bias_ptr + i5).to(tl.float32)
|
||||
else:
|
||||
v0 = s0
|
||||
v1 = s1
|
||||
v2 = s2
|
||||
v3 = s3
|
||||
v4 = s4
|
||||
v5 = s5
|
||||
ninf = float("-inf")
|
||||
v0 = tl.where(v0 != v0, ninf, v0)
|
||||
v1 = tl.where(v1 != v1, ninf, v1)
|
||||
v2 = tl.where(v2 != v2, ninf, v2)
|
||||
v3 = tl.where(v3 != v3, ninf, v3)
|
||||
v4 = tl.where(v4 != v4, ninf, v4)
|
||||
v5 = tl.where(v5 != v5, ninf, v5)
|
||||
# aiter sort_network_desc<6>: the 12-comparator network, same order.
|
||||
v0, o0, i0, v1, o1, i1 = _cas(v0, o0, i0, v1, o1, i1)
|
||||
v2, o2, i2, v3, o3, i3 = _cas(v2, o2, i2, v3, o3, i3)
|
||||
v4, o4, i4, v5, o5, i5 = _cas(v4, o4, i4, v5, o5, i5)
|
||||
v0, o0, i0, v2, o2, i2 = _cas(v0, o0, i0, v2, o2, i2)
|
||||
v1, o1, i1, v4, o4, i4 = _cas(v1, o1, i1, v4, o4, i4)
|
||||
v3, o3, i3, v5, o5, i5 = _cas(v3, o3, i3, v5, o5, i5)
|
||||
v0, o0, i0, v1, o1, i1 = _cas(v0, o0, i0, v1, o1, i1)
|
||||
v2, o2, i2, v3, o3, i3 = _cas(v2, o2, i2, v3, o3, i3)
|
||||
v4, o4, i4, v5, o5, i5 = _cas(v4, o4, i4, v5, o5, i5)
|
||||
v1, o1, i1, v2, o2, i2 = _cas(v1, o1, i1, v2, o2, i2)
|
||||
v3, o3, i3, v4, o4, i4 = _cas(v3, o3, i3, v4, o4, i4)
|
||||
v2, o2, i2, v3, o3, i3 = _cas(v2, o2, i2, v3, o3, i3)
|
||||
# k-way merge: lanes offer their heads, the max wins, ties to the lowest lane (aiter ballot + ctz)
|
||||
cursor = tl.zeros([64], dtype=tl.int32)
|
||||
sel_val = tl.zeros([64], dtype=tl.float32)
|
||||
sel_idx = tl.zeros([64], dtype=tl.int32)
|
||||
total = 0.0
|
||||
for k in tl.static_range(TOPK):
|
||||
my_val = _slot(cursor, v0, v1, v2, v3, v4, v5, ninf)
|
||||
my_idx = _slot(cursor, i0, i1, i2, i3, i4, i5, 0)
|
||||
max_val = tl.max(my_val, axis=0)
|
||||
win_lane = tl.min(tl.where(my_val == max_val, lane, 64), axis=0)
|
||||
win_idx = tl.sum(tl.where(lane == win_lane, my_idx, 0), axis=0)
|
||||
i_won = (cursor < 6) & (my_idx == win_idx)
|
||||
my_orig = tl.where(i_won, _slot(cursor, o0, o1, o2, o3, o4, o5, 0.0), 0.0)
|
||||
cursor = cursor + i_won.to(tl.int32)
|
||||
src_lane = win_idx & 63
|
||||
weight_bits = tl.sum(
|
||||
tl.where(lane == src_lane, my_orig.to(tl.int32, bitcast=True), 0), axis=0
|
||||
)
|
||||
weight = weight_bits.to(tl.float32, bitcast=True)
|
||||
sel_val = tl.where(lane == k, weight, sel_val)
|
||||
sel_idx = tl.where(lane == k, win_idx, sel_idx)
|
||||
total = total + weight
|
||||
if RENORM:
|
||||
scale = routed_scaling_factor / tl.maximum(total, 1e-20)
|
||||
else:
|
||||
scale = routed_scaling_factor * 1.0
|
||||
return sel_val * scale, sel_idx
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _router_gate_kernel(
|
||||
logits_ptr, # [M, 384] read when SPLIT_K == 0, written when WRITE_LOGITS
|
||||
part_ptr, # [SPLIT_K, M, 384] fp32 (SPLIT_K > 0)
|
||||
bias_ptr, # [384] fp32 or bf16 (HAS_BIAS)
|
||||
weights_ptr, # [M, TOPK] fp32
|
||||
ids_ptr, # [M, TOPK] int32
|
||||
stride_lm,
|
||||
stride_ps,
|
||||
stride_pm,
|
||||
stride_om,
|
||||
routed_scaling_factor,
|
||||
SPLIT_K: tl.constexpr,
|
||||
WRITE_LOGITS: tl.constexpr,
|
||||
HAS_BIAS: tl.constexpr,
|
||||
RENORM: tl.constexpr,
|
||||
TOPK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
weights, ids = _gate_row(
|
||||
logits_ptr,
|
||||
part_ptr,
|
||||
bias_ptr,
|
||||
row,
|
||||
stride_lm,
|
||||
stride_ps,
|
||||
stride_pm,
|
||||
routed_scaling_factor,
|
||||
SPLIT_K,
|
||||
WRITE_LOGITS,
|
||||
HAS_BIAS,
|
||||
RENORM,
|
||||
TOPK,
|
||||
)
|
||||
lane = tl.arange(0, 64)
|
||||
out_mask = lane < TOPK
|
||||
tl.store(weights_ptr + row * stride_om + lane, weights, mask=out_mask)
|
||||
tl.store(ids_ptr + row * stride_om + lane, ids, mask=out_mask)
|
||||
|
||||
|
||||
def rocm_router_gate(
|
||||
gating_output: torch.Tensor,
|
||||
correction_bias: Optional[torch.Tensor],
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
routed_scaling_factor: Optional[float],
|
||||
*,
|
||||
partials: Optional[torch.Tensor] = None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""aiter topk_gating(..., score_func="sqrtsoftplus") for 384 experts: fp32 weights and int32
|
||||
ids [M, topk]; with partials their fixed-order sum is gated and written into
|
||||
gating_output."""
|
||||
M, num_experts = gating_output.shape
|
||||
assert num_experts == _GATE_NUM_EXPERTS and 0 < topk <= _MAX_TOPK
|
||||
assert gating_output.stride(1) == 1
|
||||
if partials is not None:
|
||||
assert (
|
||||
partials.shape[1:] == (M, num_experts) and partials.dtype == torch.float32
|
||||
)
|
||||
assert gating_output.dtype == torch.float32
|
||||
split_k = partials.shape[0]
|
||||
stride_ps, stride_pm = partials.stride(0), partials.stride(1)
|
||||
else:
|
||||
partials = gating_output
|
||||
split_k = 0
|
||||
stride_ps = stride_pm = 0
|
||||
if correction_bias is not None:
|
||||
assert (
|
||||
correction_bias.shape == (num_experts,) and correction_bias.is_contiguous()
|
||||
)
|
||||
weights = torch.empty((M, topk), dtype=torch.float32, device=gating_output.device)
|
||||
ids = torch.empty((M, topk), dtype=torch.int32, device=gating_output.device)
|
||||
if M == 0:
|
||||
return weights, ids
|
||||
_router_gate_kernel[(M,)](
|
||||
gating_output,
|
||||
partials,
|
||||
correction_bias if correction_bias is not None else gating_output,
|
||||
weights,
|
||||
ids,
|
||||
gating_output.stride(0),
|
||||
stride_ps,
|
||||
stride_pm,
|
||||
weights.stride(0),
|
||||
float(1.0 if routed_scaling_factor is None else routed_scaling_factor),
|
||||
SPLIT_K=split_k,
|
||||
WRITE_LOGITS=split_k > 0,
|
||||
HAS_BIAS=correction_bias is not None,
|
||||
RENORM=bool(renormalize),
|
||||
TOPK=topk,
|
||||
num_warps=1,
|
||||
)
|
||||
return weights, ids
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Independent packed-FP4 (V41_FP4 layout) cache oracle for the V4.1 store and reader tests."""
|
||||
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
KV_DIM = 512
|
||||
FP4_GROUP = 16 # values per e4m3 scale
|
||||
E2M1_BYTES_PER_TOKEN = KV_DIM // 2
|
||||
E4M3_SCALE_BYTES_PER_TOKEN = KV_DIM // FP4_GROUP
|
||||
FP4_TOKEN_BYTES = E2M1_BYTES_PER_TOKEN + E4M3_SCALE_BYTES_PER_TOKEN
|
||||
E2M1_MAX = 6.0
|
||||
E4M3_MAX = 448.0
|
||||
MIN_SCALE = 2.0**-9
|
||||
_E2M1_MAGNITUDES = (0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0)
|
||||
|
||||
|
||||
def quantize_to_e2m1_codes(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Round to the nearest e2m1 value with the semantics of
|
||||
cvt.rn.satfinite.e2m1x2.f32 (ties to even, saturating to +-6) and return
|
||||
the 4-bit codes as uint8. The sign is kept for values that round to zero
|
||||
(-0.0 and small negatives give the code 0x8); NaN maps to code 0."""
|
||||
x = x.float()
|
||||
mags = torch.tensor(_E2M1_MAGNITUDES, dtype=torch.float32, device=x.device)
|
||||
sign = torch.signbit(x).to(torch.uint8) << 3
|
||||
a = torch.nan_to_num(x.abs(), nan=0.0, posinf=6.0).clamp_max(6.0)
|
||||
mids = (mags[:-1] + mags[1:]) / 2
|
||||
code = torch.bucketize(a, mids, right=True)
|
||||
on_tie = (a.unsqueeze(-1) == mids).any(dim=-1)
|
||||
tie_code = torch.bucketize(a, mids, right=False)
|
||||
code = torch.where(on_tie, tie_code + (tie_code & 1), code)
|
||||
return sign | code.to(torch.uint8)
|
||||
|
||||
|
||||
def dequantize_e2m1_codes(codes: torch.Tensor) -> torch.Tensor:
|
||||
mags = torch.tensor(_E2M1_MAGNITUDES, dtype=torch.float32, device=codes.device)
|
||||
val = mags[(codes & 7).long()]
|
||||
return torch.where((codes & 8) != 0, -val, val)
|
||||
|
||||
|
||||
def quantize_k_cache_v41_fp4(
|
||||
k: torch.Tensor, page_bytes: Optional[int] = None
|
||||
) -> torch.Tensor:
|
||||
"""k [num_pages, page_size, 512] -> uint8 [num_pages, page_bytes]
|
||||
pages of the V41_FP4 layout: 256 B of e2m1 codes per token (even index in
|
||||
the low nibble), then 32 e4m3 scales per token (one per 16 values),
|
||||
scale = e4m3(clamp(amax / 6, 2**-9, 448)). A NaN element poisons its
|
||||
tile: NaN scale, zero codes."""
|
||||
num_pages, page_size, d = k.shape
|
||||
assert d == KV_DIM
|
||||
x = k.float().view(num_pages, page_size, KV_DIM // FP4_GROUP, FP4_GROUP)
|
||||
amax = torch.nan_to_num(x.abs(), nan=float("inf")).amax(dim=-1)
|
||||
scale = torch.clamp(amax / E2M1_MAX, MIN_SCALE, E4M3_MAX).to(torch.float8_e4m3fn)
|
||||
scale = torch.where(torch.isinf(amax), torch.full_like(scale, float("nan")), scale)
|
||||
codes = quantize_to_e2m1_codes(x / scale.float().unsqueeze(-1))
|
||||
codes = codes.view(num_pages, page_size, KV_DIM)
|
||||
packed = codes[..., 0::2] | (codes[..., 1::2] << 4)
|
||||
raw = page_size * FP4_TOKEN_BYTES
|
||||
if page_bytes is None:
|
||||
page_bytes = -(-raw // 256) * 256 # pages start on a 256-byte boundary
|
||||
assert page_bytes >= raw
|
||||
data_bytes = page_size * E2M1_BYTES_PER_TOKEN
|
||||
out = torch.zeros((num_pages, page_bytes), dtype=torch.uint8, device=k.device)
|
||||
out[:, :data_bytes] = packed.reshape(num_pages, data_bytes)
|
||||
out[:, data_bytes:raw] = scale.view(torch.uint8).reshape(
|
||||
num_pages, raw - data_bytes
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def dequantize_k_cache_v41_fp4(pages: torch.Tensor, page_size: int) -> torch.Tensor:
|
||||
"""Inverse of quantize_k_cache_v41_fp4: [num_pages, page_size, 512]
|
||||
bf16. e2m1 * e4m3 has at most 2 + 4 significant bits, so the product is
|
||||
exact in bf16, as in the kernel."""
|
||||
num_pages = pages.shape[0]
|
||||
pages = pages.view(torch.uint8)
|
||||
data_bytes = page_size * E2M1_BYTES_PER_TOKEN
|
||||
data = pages[:, :data_bytes].reshape(num_pages, page_size, E2M1_BYTES_PER_TOKEN)
|
||||
scale = pages[:, data_bytes : page_size * FP4_TOKEN_BYTES].reshape(
|
||||
num_pages, page_size, E4M3_SCALE_BYTES_PER_TOKEN
|
||||
)
|
||||
codes = torch.empty(
|
||||
(num_pages, page_size, KV_DIM), dtype=torch.uint8, device=pages.device
|
||||
)
|
||||
codes[..., 0::2] = data & 0xF
|
||||
codes[..., 1::2] = data >> 4
|
||||
values = dequantize_e2m1_codes(codes).view(
|
||||
num_pages, page_size, KV_DIM // FP4_GROUP, FP4_GROUP
|
||||
)
|
||||
out = values * scale.view(torch.float8_e4m3fn).float().unsqueeze(-1)
|
||||
return out.view(num_pages, page_size, KV_DIM).to(torch.bfloat16)
|
||||
Reference in New Issue
Block a user