diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/c1.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/c1.cuh index ca1200c3ad..1e22d39f7a 100644 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/c1.cuh +++ b/python/sglang/kernels/jit/csrc/deepseek_v4/c1.cuh @@ -267,7 +267,7 @@ struct FlashCompress1Kernel { auto N = SymbolicSize{"num_tokens"}; auto device_ = SymbolicDevice{}; - device_.set_options(); + device_.set_options(); TensorMatcher({N, kHeadDim}) // .with_dtype() diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/fp4_indexer_rope.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/fp4_indexer_rope.cuh index 6558a4f373..cfb268ae7a 100644 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/fp4_indexer_rope.cuh +++ b/python/sglang/kernels/jit/csrc/deepseek_v4/fp4_indexer_rope.cuh @@ -133,8 +133,12 @@ SGL_DEVICE IndexPacked index_rope_quant_pack(fp32x2_t head, fp32x2_t tail, fp32x // `inv_scale_ue8m0` instead of the reference's division: the scale is a // power of two, so both are exact except at the unreachable exponent 254. const auto inv_scale = deepseek_v4::fp8::inv_scale_ue8m0(static_cast(out.exponent[half])); +#ifndef USE_ROCM const auto code = __nv_cvt_float2_to_fp4x2( fp32x2_t{data[half * 2] * inv_scale, data[half * 2 + 1] * inv_scale}, __NV_E2M1, cudaRoundNearest); +#else + const auto code = fp4::e2m1x2_code(fp32x2_t{data[half * 2] * inv_scale, data[half * 2 + 1] * inv_scale}); +#endif out.payload[half] = clear_negative_zero(static_cast(code)); } return out; @@ -147,8 +151,15 @@ SGL_DEVICE IndexPacked index_rope_quant_pack(fp32x2_t head, fp32x2_t tail, fp32x /// the warp. Every lane must reach it: the shuffles are warp-wide. SGL_DEVICE uint32_t index_scale_word(const uint32_t (&exponent)[2]) { using namespace device; +#ifndef USE_ROCM const auto exp_1 = __shfl_sync(warp::kFullMask, exponent[0], kWarpThreads / 2); const auto exp_3 = __shfl_sync(warp::kFullMask, exponent[1], kWarpThreads / 2); +#else + // A wave holds two of these 32-lane rows, so the shuffle width has to be the + // row, not the wave, for lane 16 to be this row's lane 16. + const auto exp_1 = __shfl(exponent[0], kWarpThreads / 2, kWarpThreads); + const auto exp_3 = __shfl(exponent[1], kWarpThreads / 2, kWarpThreads); +#endif return exponent[0] | (exp_1 << 8) | (exponent[1] << 16) | (exp_3 << 24); } @@ -310,7 +321,7 @@ struct IndexKKernel { auto N = SymbolicSize{"num_tokens"}; auto device_ = SymbolicDevice{}; - device_.set_options(); + device_.set_options(); TensorMatcher({N, kHeadDim}).with_dtype().with_device(device_).verify(input); TensorMatcher({kHeadDim}).with_dtype().with_device(device_).verify(norm_weight); @@ -390,7 +401,7 @@ struct IndexQKernel { auto H = SymbolicSize{"heads"}; auto R = SymbolicSize{"num_rows"}; auto device_ = SymbolicDevice{}; - device_.set_options(); + device_.set_options(); TensorMatcher({N, H, kHeadDim}).with_dtype().with_device(device_).verify(input); // Real/imag interleaved, so the trailing dim is kRopeDim, not kRopeDim / 2. diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/fp4_indexer_rope_hip.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/fp4_indexer_rope_hip.cuh new file mode 100644 index 0000000000..a702e09130 --- /dev/null +++ b/python/sglang/kernels/jit/csrc/deepseek_v4/fp4_indexer_rope_hip.cuh @@ -0,0 +1,156 @@ +/// Index-K write of `fp4_indexer_rope.cuh` into the split FlyDSL layout: payload `[npages, 1, 4, kPageSize, 16]` +/// (chunk `c` holds elements `[32c, 32c + 32)`) and ue8m0 exponents `[npages, 1, 4, kPageSize]` with +/// the slot axis transposed as a 16 x 4 tile -- the bytes `store_fp4_index_k_cache_split` writes. + +#pragma once + +#ifndef USE_ROCM +#error "fp4_indexer_rope_hip.cuh writes the FlyDSL index-K layout, which exists on ROCm only" +#endif + +#include "fp4_indexer_rope.cuh" + +namespace sglang { + +struct IndexKSplitParams { + const bf16_t* __restrict__ input; // [num_tokens, kHeadDim] bf16, pre-norm + const bf16_t* __restrict__ norm_weight; // [kHeadDim] bf16 + const float* __restrict__ freqs_cis; // [max_pos, kRopeDim] fp32, real/imag interleaved + const void* __restrict__ positions; // [num_tokens] PosT + const int64_t* __restrict__ loc; // [num_tokens] index-K slot; 0 publishes nothing + uint8_t* __restrict__ payload; // [npages, 1, 4, kPageSize, 16] uint8 + uint8_t* __restrict__ scale; // [npages, 1, 4, kPageSize] uint8 + uint32_t num_tokens; + float eps; +}; + +template +__global__ __launch_bounds__(kFp4RopeWarpsPerCTA* device::kWarpThreads) void flash_index_k_split_kernel( + const IndexKSplitParams params) { + using namespace device; + + static_assert(kPageSize == 64, "the 16 x 4 slot transpose of the scale spans a 64-slot page"); + static_assert( + kHeadDim == 128 && kRopeDim == 64, + "the one-warp tiling is specific to a 128-wide row whose second half is the RoPE tail"); + static_assert(kHeadDim == 2 * kWarpThreads * 2, "a lane owns one bf16x2 of each half"); + static_assert(std::has_single_bit(kRatio), "group_pos is derived by masking, so the ratio must be a power of two"); + + using bf16_vec_t = AlignedVector; + using fp32_vec_t = AlignedVector; + + const auto lane = threadIdx.x % kWarpThreads; + const auto row = blockIdx.x * kFp4RopeWarpsPerCTA + threadIdx.x / kWarpThreads; + // Warp-uniform, so the reductions below still see a full warp. + if (row >= params.num_tokens) return; + + // step metadata, not the predecessor's output: loaded ahead of the PDL gate; slot 0 rows drop at the store + const auto slot_id = params.loc[row]; + const auto position = static_cast(static_cast(params.positions)[row]); + PDLWaitPrimary(); + + bf16_vec_t head_in, tail_in, head_w, tail_w; + head_in.load(params.input + row * kHeadDim, lane); + tail_in.load(params.input + row * kHeadDim, lane + kWarpThreads); + head_w.load(params.norm_weight, lane); + tail_w.load(params.norm_weight, lane + kWarpThreads); + fp32_vec_t freq; + freq.load(params.freqs_cis + (position & ~static_cast(kRatio - 1)) * kRopeDim, lane); + + fp32x2_t head, tail; + { + const auto [h0, h1] = cast(head_in[0]); + const auto [t0, t1] = cast(tail_in[0]); + const auto sqrsum = warp::reduce_sum(h0 * h0 + h1 * h1 + t0 * t0 + t1 * t1); + const auto inv_rms = math::rsqrt(sqrsum * (1.0f / static_cast(kHeadDim)) + params.eps); + const auto [wh0, wh1] = cast(head_w[0]); + const auto [wt0, wt1] = cast(tail_w[0]); + // `k_norm` materializes a bf16 tensor, so the norm result is rounded before + // anything downstream sees it -- the RoPE below included. + head = cast(cast(fp32x2_t{wh0 * (h0 * inv_rms), wh1 * (h1 * inv_rms)})); + tail = cast(cast(fp32x2_t{wt0 * (t0 * inv_rms), wt1 * (t1 * inv_rms)})); + } + + const auto packed = index_rope_quant_pack(head, tail, fp32x2_t{freq[0], freq[1]}); + + // slot 0 is reserved: padded graph rows and, at ratio > 1, rows completing no group publish nothing + if (slot_id <= 0) return; + const auto page = slot_id / kPageSize; + const auto slot = slot_id % kPageSize; + + // byte j of the slot is chunk j / 16, byte j % 16: the head pair at byte lane, the tail pair at lane + 32 + const auto payload_ptr = params.payload + page * (4 * kPageSize * 16) + slot * 16; + payload_ptr[(lane / 16) * (kPageSize * 16) + lane % 16] = static_cast(packed.payload[0]); + payload_ptr[(2 + lane / 16) * (kPageSize * 16) + lane % 16] = static_cast(packed.payload[1]); + + // Lane 0 holds the exponents of blocks 0 and 2, lane 16 those of 1 and 3. + if (lane % (kWarpThreads / 2) == 0) { + const auto shuffled = (slot % 16) * 4 + slot / 16; + const auto scale_ptr = params.scale + page * (4 * kPageSize) + shuffled; + const auto first = lane / (kWarpThreads / 2); + scale_ptr[first * kPageSize] = static_cast(packed.exponent[0]); + scale_ptr[(2 + first) * kPageSize] = static_cast(packed.exponent[1]); + } +} + +/// \brief Host side of `flash_index_k_split_kernel`. +template +struct FlashIndexKSplitKernel { + static constexpr uint32_t kBlockSize = kFp4RopeWarpsPerCTA * device::kWarpThreads; + + template + static constexpr auto kernel = flash_index_k_split_kernel; + + /// `IndexKKernel::run_index_k`'s arguments, the cache replaced by + /// \param payload `[npages, 1, 4, kPageSize, 16]` uint8 (the pool's + /// `float4_e2m1fn_x2` buffer viewed as bytes). + /// \param scale `[npages, 1, 4, kPageSize]` uint8. + static void + run(const tvm::ffi::TensorView input, + const tvm::ffi::TensorView norm_weight, + const tvm::ffi::TensorView freqs_cis, + const tvm::ffi::TensorView positions, + const tvm::ffi::TensorView loc, + const tvm::ffi::TensorView payload, + const tvm::ffi::TensorView scale, + const float eps) { + using namespace host; + + auto N = SymbolicSize{"num_tokens"}; + auto P = SymbolicSize{"npages"}; + auto device_ = SymbolicDevice{}; + device_.set_options(); + + TensorMatcher({N, kHeadDim}).with_dtype().with_device(device_).verify(input); + TensorMatcher({kHeadDim}).with_dtype().with_device(device_).verify(norm_weight); + // Real/imag interleaved, so the trailing dim is kRopeDim, not kRopeDim / 2. + TensorMatcher({-1, kRopeDim}).with_dtype().with_device(device_).verify(freqs_cis); + auto pos_dtype = SymbolicDType{}; + TensorMatcher({N}).with_dtype(pos_dtype).with_device(device_).verify(positions); + TensorMatcher({N}).with_dtype().with_device(device_).verify(loc); + TensorMatcher({P, 1, 4, kPageSize, 16}).with_dtype().with_device(device_).verify(payload); + TensorMatcher({P, 1, 4, kPageSize}).with_dtype().with_device(device_).verify(scale); + + const auto num_tokens = static_cast(N.unwrap()); + if (num_tokens == 0) return; + + const auto params = IndexKSplitParams{ + .input = static_cast(input.data_ptr()), + .norm_weight = static_cast(norm_weight.data_ptr()), + .freqs_cis = static_cast(freqs_cis.data_ptr()), + .positions = positions.data_ptr(), + .loc = static_cast(loc.data_ptr()), + .payload = static_cast(payload.data_ptr()), + .scale = static_cast(scale.data_ptr()), + .num_tokens = num_tokens, + .eps = eps, + }; + const auto k_int32 = kernel; + const auto k_int64 = kernel; + const auto k = pos_dtype.is_type() ? k_int32 : k_int64; + LaunchKernel(div_ceil(num_tokens, kFp4RopeWarpsPerCTA), kBlockSize, device_.unwrap()) // + .enable_pdl(kUsePDL)(k, params); + } +}; + +} // namespace sglang diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/main_norm_rope.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/main_norm_rope.cuh index 481f230e8c..94ef8bc30f 100644 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/main_norm_rope.cuh +++ b/python/sglang/kernels/jit/csrc/deepseek_v4/main_norm_rope.cuh @@ -15,6 +15,7 @@ #include #include +#include namespace sglang { @@ -258,6 +259,12 @@ struct FusedKNormRopeFlashMLAParams { int64_t kv_stride_batch; uint32_t batch_size; float eps; + // kRopeQ (ROCm): (B, num_q_heads, kHeadDim) DType; the trailing kRopeDim of every head + // are rotated in place with this token's frequencies by the same launch. + void* __restrict__ q = nullptr; + int64_t q_stride_batch = 0; + int64_t q_stride_head = 0; + uint32_t num_q_heads = 0; }; template < @@ -267,7 +274,8 @@ template < typename PosT, int32_t kPageBits, deepseek_v4::KVLayout kLayout, - bool kUsePDL> + bool kUsePDL, + bool kRopeQ = false> K_KERNEL void fused_k_norm_rope_flashmla(const __grid_constant__ FusedKNormRopeFlashMLAParams params) { using namespace device; @@ -327,6 +335,30 @@ K_KERNEL void fused_k_norm_rope_flashmla(const __grid_constant__ FusedKNormRopeF } } + // Before the out_loc check: a row without a KV slot still ropes its query heads. + if constexpr (kRopeQ) { + // Query rope, pair j of head h at q[h * stride + kHeadDim - kRopeDim + 2j]: the cross product + // is rounded, then one fma with the cosine, so the bf16 result is bitwise the Triton flat + // rope kernel's on gfx950. + static_assert(std::is_same_v, "the in-place query rope reads q as bf16 pairs"); + constexpr uint32_t kPairsPerHead = kRopeDim / 2; + const auto q_row = static_cast(params.q) + work_id * params.q_stride_batch + (kHeadDim - kRopeDim); + const auto n_pairs = params.num_q_heads * kPairsPerHead; + for (uint32_t p = tx; p < n_pairs; p += kFusedKBlockSize) { + const auto head = p / kPairsPerHead; + const auto pair = p % kPairsPerHead; + auto* ptr = reinterpret_cast(q_row + head * params.q_stride_head) + pair; + const auto qv = cast(*ptr); + const auto cos_v = freqs_cis[2 * pair]; + const auto sin_v = freqs_cis[2 * pair + 1]; + const float rot_real = -__fmul_rn(qv.y, sin_v); + const float rot_imag = __fmul_rn(qv.x, sin_v); + const float out_real = fmaf(qv.x, cos_v, rot_real); + const float out_imag = fmaf(qv.y, cos_v, rot_imag); + *ptr = cast(fp32x2_t{out_real, out_imag}); + } + } + // A negative out_loc marks a slot with no KV write target (e.g. the -1 // sentinel from the full->SWA translation for out-of-window tokens or // padded rows); skip the row instead of writing out of bounds. Checked @@ -398,9 +430,9 @@ struct FusedKNormRopeFlashMLAKernel { static_assert(1 << kLogPageSize == kPageSize); static_assert(kHeadDim == 512 && kRopeDim == 64, "FlashMLA layout requires (512, 64)"); - template + template static constexpr auto kernel = - fused_k_norm_rope_flashmla; + fused_k_norm_rope_flashmla; static void forward( const tvm::ffi::TensorView kv, @@ -410,6 +442,33 @@ struct FusedKNormRopeFlashMLAKernel { const tvm::ffi::TensorView out_loc, const tvm::ffi::TensorView kvcache, float eps) { + forward_impl(kv, kv_weight, freqs_cis, positions, out_loc, kvcache, eps, nullptr); + } + + /// `forward` with the same tokens' query heads `q` (B, H, kHeadDim) roped in place (ROCm). + static void forward_with_q( + const tvm::ffi::TensorView kv, + const tvm::ffi::TensorView kv_weight, + const tvm::ffi::TensorView freqs_cis, + const tvm::ffi::TensorView positions, + const tvm::ffi::TensorView out_loc, + const tvm::ffi::TensorView kvcache, + float eps, + const tvm::ffi::TensorView q) { + forward_impl(kv, kv_weight, freqs_cis, positions, out_loc, kvcache, eps, &q); + } + + private: + template + static void forward_impl( + const tvm::ffi::TensorView kv, + const tvm::ffi::TensorView kv_weight, + const tvm::ffi::TensorView freqs_cis, + const tvm::ffi::TensorView positions, + const tvm::ffi::TensorView out_loc, + const tvm::ffi::TensorView kvcache, + float eps, + const tvm::ffi::TensorView* q) { using namespace host; auto B = SymbolicSize{"batch_size"}; @@ -447,7 +506,7 @@ struct FusedKNormRopeFlashMLAKernel { const auto batch_size = static_cast(B.unwrap()); if (batch_size == 0) return; - const auto params = FusedKNormRopeFlashMLAParams{ + auto params = FusedKNormRopeFlashMLAParams{ .kv = kv.data_ptr(), .kv_weight = kv_weight.data_ptr(), .freqs_cis = static_cast(freqs_cis.data_ptr()), @@ -458,8 +517,22 @@ struct FusedKNormRopeFlashMLAKernel { .batch_size = batch_size, .eps = eps, }; - const auto k_int32 = kernel; - const auto k_int64 = kernel; + if constexpr (kRopeQ) { + auto H = SymbolicSize{"num_q_heads"}; + TensorMatcher({B, H, kHeadDim}) // + .with_strides({-1, -1, 1}) + .with_dtype() + .with_device(device_) + .verify(*q); + // The rope pairs are read and written as 4-byte packs. + RuntimeCheck(q->stride(1) % 2 == 0 && q->stride(0) % 2 == 0, "q strides must be even"); + params.q = q->data_ptr(); + params.q_stride_batch = q->stride(0); + params.q_stride_head = q->stride(1); + params.num_q_heads = static_cast(H.unwrap()); + } + const auto k_int32 = kernel; + const auto k_int64 = kernel; const auto k = pos_dtype.is_type() ? k_int32 : k_int64; LaunchKernel(batch_size, kFusedKBlockSize, device_.unwrap()) // .enable_pdl(kUsePDL)(k, params); diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/store.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/store.cuh index 5a7dc36da1..37f13be3d6 100644 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/store.cuh +++ b/python/sglang/kernels/jit/csrc/deepseek_v4/store.cuh @@ -15,7 +15,9 @@ #include #include +#ifndef USE_ROCM #include +#endif #include namespace sglang { diff --git a/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/fp4_utils.cuh b/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/fp4_utils.cuh index bbb593f8c4..c148521811 100644 --- a/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/fp4_utils.cuh +++ b/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/fp4_utils.cuh @@ -23,18 +23,66 @@ constexpr float kAmaxFloor = 6.0f * 1.1754943508222875e-38f; constexpr uint32_t kBlockSize = 32; /// Compressed-KV elements sharing one E4M3 scale. constexpr uint32_t kCompressedKVBlockSize = 16; +#ifdef USE_ROCM +/// \brief `cvt.rn.satfinite.e2m1x2.f32` for one value, in software: RNE onto the +/// e2m1 grid {0, .5, 1, 1.5, 2, 3, 4, 6}, saturated at 6, sign kept (also for -0). +SGL_DEVICE uint32_t e2m1_code(float x) { + const float mag = fminf(fabsf(x), kMax); + const float step = mag < 2.0f ? 0.5f : (mag < 4.0f ? 1.0f : 2.0f); + const float q = rintf(mag / step) * step; + // q in {0, .5, 1, 1.5} -> 2q; {2, 3} -> q + 2; {4, 6} -> q / 2 + 4. + const uint32_t idx = q < 2.0f ? static_cast(q * 2.0f) + : (q < 4.0f ? static_cast(q) + 2u : static_cast(q * 0.5f) + 4u); + return idx | (__float_as_uint(x) >> 31 << 3); +} + +/// \brief The e2m1 grid value a code stands for, exact in fp32. +SGL_DEVICE float e2m1_value(uint32_t code) { + const uint32_t idx = code & 0x7u; + const float mag = idx < 4u ? static_cast(idx) * 0.5f + : (idx < 6u ? static_cast(idx) - 2.0f : static_cast(idx - 4u) * 2.0f); + return (code & 0x8u) ? -mag : mag; +} + +/// \brief `__nv_cvt_float2_to_fp4x2(.., __NV_E2M1, cudaRoundNearest)`: `.x` in +/// the low nibble, `.y` in the high one. +SGL_DEVICE uint32_t e2m1x2_code(fp32x2_t x) { + return e2m1_code(x.x) | (e2m1_code(x.y) << 4); +} + +/// \brief `float(__nv_fp8_e4m3(raw))` for `raw` in `[2^-9, 448]`: RNE onto the E4M3FN +/// grid, gfx950's `v_cvt_pk_fp8_f32` in hardware, software elsewhere (gfx942's fp8 is FNUZ). +SGL_DEVICE float e4m3_round_rn(float raw) { +#if defined(__HIP_DEVICE_COMPILE__) && defined(__gfx950__) + const auto packed = __builtin_amdgcn_cvt_pk_fp8_f32(raw, raw, 0, false); + return __builtin_amdgcn_cvt_f32_fp8(packed, 0); +#else + const int exponent = ilogbf(raw); + const float step = exponent < -6 ? 0x1p-9f : ldexpf(1.0f, exponent - 3); + return rintf(raw / step) * step; +#endif +} +#endif /// \brief Round amax / 6 to a positive finite E4M3 scale, ties to even. SGL_DEVICE float compressed_kv_scale(float amax) { const auto raw = fminf(fmaxf(amax * (1.0f / kMax), 0x1p-9f), 448.0f); +#ifdef USE_ROCM + return e4m3_round_rn(raw); +#else return static_cast(__nv_fp8_e4m3(raw)); +#endif } /// \brief Quantize compressed KV with its E4M3 scale and return dequantized values. SGL_DEVICE fp32x2_t fake_quant_compressed_kv_x2(fp32x2_t x, float scale) { const fp32x2_t scaled{__fdiv_rn(x.x, scale) + 0.0f, __fdiv_rn(x.y, scale) + 0.0f}; +#ifdef USE_ROCM + const fp32x2_t grid{e2m1_value(e2m1_code(scaled.x)), e2m1_value(e2m1_code(scaled.y))}; +#else const auto code = __nv_cvt_float2_to_fp4x2(scaled, __NV_E2M1, cudaRoundNearest); const auto grid = device::cast(fp16x2_t{__nv_cvt_fp4x2_to_halfraw2(code, __NV_E2M1)}); +#endif return {grid.x * scale, grid.y * scale}; } @@ -50,8 +98,12 @@ SGL_DEVICE fp32x2_t block_scale(float amax) { /// scaling clears negative zero to match `torch.sign(0) == 0` in `torch_quant.round_fp4`. SGL_DEVICE fp32x2_t fake_quant_x2(fp32x2_t x, float scale, float inv_scale) { const fp32x2_t scaled{__fmaf_rn(x.x, inv_scale, 0.0f), __fmaf_rn(x.y, inv_scale, 0.0f)}; +#ifdef USE_ROCM + const fp32x2_t grid{e2m1_value(e2m1_code(scaled.x)), e2m1_value(e2m1_code(scaled.y))}; +#else const auto code = __nv_cvt_float2_to_fp4x2(scaled, __NV_E2M1, cudaRoundNearest); const auto grid = device::cast(fp16x2_t{__nv_cvt_fp4x2_to_halfraw2(code, __NV_E2M1)}); +#endif return {grid.x * scale, grid.y * scale}; } diff --git a/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/kv_layout.cuh b/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/kv_layout.cuh index 6c9f9579ba..456203bbe5 100644 --- a/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/kv_layout.cuh +++ b/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/kv_layout.cuh @@ -5,8 +5,10 @@ #include #include +#include #include +#include #include #ifndef USE_ROCM #include @@ -111,8 +113,6 @@ struct PagedKV { } }; -#ifndef USE_ROCM - namespace v41 { /// The row helpers below quantize one 512-wide token spread over `512 / kVecSize` threads, @@ -152,7 +152,11 @@ SGL_DEVICE void store_row_fp8(uint8_t* data_row, uint8_t* scale_row, uint32_t tx #pragma unroll for (uint32_t i = 0; i < kVecSize / 2; ++i) { // `cvt.rn.satfinite.e4m3x2` directly: |x / scale| <= 448 needs no clamp. +#ifdef USE_ROCM + out[i] = fp8::pack_fp8(v[2 * i] * inv_scale, v[2 * i + 1] * inv_scale); +#else out[i] = fp8x2_e4m3_t{fp32x2_t{v[2 * i] * inv_scale, v[2 * i + 1] * inv_scale}}; +#endif } out.store(data_row, tx); // Every lane of the tile holds the exponent; they all store the same byte. @@ -170,18 +174,29 @@ SGL_DEVICE void store_row_fp4(uint8_t* data_row, uint8_t* scale_row, uint32_t tx static_assert(kVecSize % 2 == 0 && kTileLanes >= 1 && (kTileLanes & (kTileLanes - 1)) == 0); const float amax = warp::reduce_max(vec_amax(v)); +#ifdef USE_ROCM + const float scale = fp4::e4m3_round_rn(fminf(fmaxf(__fdiv_rn(amax, 6.0f), 0x1p-9f), 448.0f)); + // the low byte of the pack's 16-bit storage, whether fp8x2_e4m3_t is an integer or HIP's fp8 pair + const auto scale_bits = static_cast(std::bit_cast(fp8::pack_fp8(scale, scale))); +#else const __nv_fp8_e4m3 scale_e4m3{fminf(fmaxf(__fdiv_rn(amax, 6.0f), 0x1p-9f), 448.0f)}; const float scale = static_cast(scale_e4m3); + const uint8_t scale_bits = scale_e4m3.__x; +#endif AlignedVector out; #pragma unroll for (uint32_t i = 0; i < kVecSize / 2; ++i) { // IEEE division by the rounded scale, as the reference does; a reciprocal multiply could cross an e2m1 tie. +#ifdef USE_ROCM + out[i] = static_cast(fp4::e2m1x2_code({__fdiv_rn(v[2 * i], scale), __fdiv_rn(v[2 * i + 1], scale)})); +#else out[i] = static_cast(__nv_cvt_float2_to_fp4x2( fp32x2_t{__fdiv_rn(v[2 * i], scale), __fdiv_rn(v[2 * i + 1], scale)}, __NV_E2M1, cudaRoundNearest)); +#endif } out.store(data_row, tx); // Every lane of the tile holds the scale; they all store the same byte. - scale_row[tx / kTileLanes] = scale_e4m3.__x; + scale_row[tx / kTileLanes] = scale_bits; } /// Dispatch on the layout for a 512-wide row held as `kVecSize` consecutive fp32 per thread. @@ -205,24 +220,6 @@ store_row(uint8_t* data_row, uint8_t* scale_row, uint32_t tx, const device::Alig } // namespace v41 -#else // USE_ROCM - -namespace v41 { - -template -inline constexpr bool dependent_false_v = false; - -/// A discarded `if constexpr` branch is still parsed, so the V4 callers need the name to -/// resolve on ROCm; instantiating it is the error. -template -SGL_DEVICE void store_row(uint8_t*, uint8_t*, uint32_t, Ts&&...) { - static_assert(dependent_false_v, "the V4.1 KV layouts are CUDA only"); -} - -} // namespace v41 - -#endif // USE_ROCM - } // namespace deepseek_v4 } // namespace sglang diff --git a/python/sglang/kernels/ops/attention/dsv4/attn.py b/python/sglang/kernels/ops/attention/dsv4/attn.py index dba73d79ee..448e328599 100644 --- a/python/sglang/kernels/ops/attention/dsv4/attn.py +++ b/python/sglang/kernels/ops/attention/dsv4/attn.py @@ -11,6 +11,7 @@ from sglang.kernels.jit.utils import ( load_jit, make_cpp_args, ) +from sglang.srt.utils import is_gfx95_supported from .kv_layout import KVLayout from .utils import make_name @@ -95,16 +96,17 @@ def fused_store_cache( RoPE tail in-kernel, so ``input`` must then be the un-rotated latent. """ layout = KVLayout.parse(layout) - if is_hip_runtime(): - assert layout is KVLayout.V4 and freqs_cis is None, ( - "the V4.1 KV layouts are CUDA (sm100) only" - ) + if is_hip_runtime() and layout is KVLayout.V4: + assert freqs_cis is None, "the V4 store receives already-rotated inputs" from sglang.kernels.ops.kvcache.triton_store_cache import ( triton_fused_store_cache, ) triton_fused_store_cache(input, cache, indices, page_size=page_size, type=type) else: + assert not is_hip_runtime() or is_gfx95_supported(), ( + "V4.1 KV stores on HIP require gfx950" + ) module = _jit_fused_store_module( name=type, input_dtype=input.dtype, diff --git a/python/sglang/kernels/ops/attention/dsv4/c2_decode_pool.py b/python/sglang/kernels/ops/attention/dsv4/c2_decode_pool.py index 5f9f9bcffe..5cc1a1d3fe 100644 --- a/python/sglang/kernels/ops/attention/dsv4/c2_decode_pool.py +++ b/python/sglang/kernels/ops/attention/dsv4/c2_decode_pool.py @@ -89,9 +89,9 @@ def _c2_decode_pool_kernel( e1 = libdevice.exp(score - m) denom = e0 + e1 # The + 0.0 below prevents FMA contraction: torch rounds both products first. - # libdevice.div_rn matches torch division; Triton's / is an approximate reciprocal. - t0 = p_kv * libdevice.div_rn(e0, denom) - t1 = kv * libdevice.div_rn(e1, denom) + # tl.div_rn matches torch division; Triton's / is an approximate reciprocal on CUDA. + t0 = p_kv * tl.div_rn(e0, denom) + t1 = kv * tl.div_rn(e1, denom) t0 = t0 + 0.0 t1 = t1 + 0.0 pooled = t0 + t1 diff --git a/python/sglang/kernels/ops/attention/dsv4/compress.py b/python/sglang/kernels/ops/attention/dsv4/compress.py index 2d27cddbe3..4048d2fac4 100644 --- a/python/sglang/kernels/ops/attention/dsv4/compress.py +++ b/python/sglang/kernels/ops/attention/dsv4/compress.py @@ -14,7 +14,7 @@ from sglang.srt.layers.attention.dsa.utils import ( INDEXER_K_CACHE_PRESHUFFLE_TILE, aiter_can_use_preshuffle_paged_mqa, ) -from sglang.srt.utils import is_hip, is_xpu +from sglang.srt.utils import is_gfx95_supported, is_hip, is_xpu from .kv_layout import KVLayout from .utils import make_name @@ -467,7 +467,10 @@ def compress_norm_rope_store( assert kv.shape[-1] == 512 and not use_fp4 and not bf16_store, ( "the V4.1 layouts are paged FlashMLA main-KV caches" ) - assert not is_hip() and not _is_xpu, "the V4.1 KV layouts are CUDA (sm100) only" + assert not _is_xpu, "the V4.1 KV layouts are not supported on XPU" + assert not is_hip() or is_gfx95_supported(), ( + "V4.1 KV stores on HIP require gfx950" + ) if use_fp4: assert kv.shape[-1] == 128 if is_hip() and use_fp4: diff --git a/python/sglang/kernels/ops/attention/dsv4/dequant_k_cache.py b/python/sglang/kernels/ops/attention/dsv4/dequant_k_cache.py index 2cbe2e0069..20ad157570 100644 --- a/python/sglang/kernels/ops/attention/dsv4/dequant_k_cache.py +++ b/python/sglang/kernels/ops/attention/dsv4/dequant_k_cache.py @@ -7,6 +7,7 @@ import triton.language as tl from sglang.kernels.jit.utils import get_jit_cuda_arch, is_hip_runtime from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz +from sglang.srt.utils import is_gfx95_supported fp8_dtype = torch.float8_e4m3fnuz if is_fp8_fnuz() else torch.float8_e4m3fn @@ -108,9 +109,13 @@ def dequantize_k_cache_paged_v41( """ layout = KVLayout.parse(layout) assert layout in (KVLayout.V41, KVLayout.V41_FP4), layout - if is_hip_runtime() or get_jit_cuda_arch().major < 10: + if is_hip_runtime(): + supported = is_gfx95_supported() + else: + supported = get_jit_cuda_arch().major >= 10 + if not supported: raise RuntimeError( - "DeepSeek V4.1 KV cache dequantization requires CUDA SM100 or newer" + "DeepSeek V4.1 KV cache dequantization requires CUDA SM100 or newer, or gfx950" ) assert quant_k_cache.is_contiguous() assert page_table_1_flattened.dtype in (torch.int32, torch.int64) @@ -153,6 +158,7 @@ def dequantize_k_cache_paged_v41( SCALE_BYTES=layout.scale_bytes, TILE_SIZE=layout.tile_size, S_OFFSET_BYTES=layout.scale_offset(page_size), + IS_HIP=is_hip_runtime(), ) return out @@ -367,6 +373,18 @@ def _e2m1_code_to_fp32(code): return (mag.to(tl.int32, bitcast=True) | sign).to(tl.float32, bitcast=True) +@triton.jit +def _v41_dequant_to_bf16(value, IS_HIP: tl.constexpr): + result = value.to(tl.bfloat16) + if IS_HIP: + # HIP conversion can produce negative NaNs; match the torch format + # reference's canonical positive BF16 NaN without changing finite bits. + bits = result.to(tl.uint16, bitcast=True) + bits = tl.where((bits & 0x7FFF) > 0x7F80, 0x7FC0, bits).to(tl.uint16) + result = bits.to(tl.bfloat16, bitcast=True) + return result + + @triton.jit def _dequantize_k_cache_paged_v41_fp8_kernel( output_ptr, @@ -380,6 +398,7 @@ def _dequantize_k_cache_paged_v41_fp8_kernel( SCALE_BYTES: tl.constexpr, TILE_SIZE: tl.constexpr, S_OFFSET_BYTES: tl.constexpr, + IS_HIP: tl.constexpr, ): # V41: 512 e4m3 values per token, then 16 ue8m0 scales (one per 32 values). tl.static_assert(DATA_BYTES == 512 and SCALE_BYTES == 16 and TILE_SIZE == 32) @@ -397,7 +416,7 @@ def _dequantize_k_cache_paged_v41_fp8_kernel( out = vals * _ue8m0_to_fp32(scale_u8) tl.store( output_ptr + token_id * output_stride_0 + offs, - out.to(output_ptr.dtype.element_ty), + _v41_dequant_to_bf16(out, IS_HIP), ) @@ -414,6 +433,7 @@ def _dequantize_k_cache_paged_v41_fp4_kernel( SCALE_BYTES: tl.constexpr, TILE_SIZE: tl.constexpr, S_OFFSET_BYTES: tl.constexpr, + IS_HIP: tl.constexpr, ): # V41_FP4: 512 e2m1 codes packed two per byte (even index in the low nibble), # then 32 e4m3 scales (one per 16 values). @@ -435,8 +455,8 @@ def _dequantize_k_cache_paged_v41_fp4_kernel( lo = _e2m1_code_to_fp32(packed & 0xF) * scale hi = _e2m1_code_to_fp32(packed >> 4) * scale out_base = output_ptr + token_id * output_stride_0 - tl.store(out_base + 2 * boffs, lo.to(output_ptr.dtype.element_ty)) - tl.store(out_base + 2 * boffs + 1, hi.to(output_ptr.dtype.element_ty)) + tl.store(out_base + 2 * boffs, _v41_dequant_to_bf16(lo, IS_HIP)) + tl.store(out_base + 2 * boffs + 1, _v41_dequant_to_bf16(hi, IS_HIP)) @triton.jit diff --git a/python/sglang/kernels/ops/attention/dsv4/elementwise.py b/python/sglang/kernels/ops/attention/dsv4/elementwise.py index 457829d429..c0ab88af3e 100644 --- a/python/sglang/kernels/ops/attention/dsv4/elementwise.py +++ b/python/sglang/kernels/ops/attention/dsv4/elementwise.py @@ -72,6 +72,28 @@ def _jit_main_k_norm_rope_flashmla_module( ) +@cache_once +def _jit_main_k_norm_rope_q_flashmla_module( + dtype: torch.dtype, + head_dim: int, + rope_dim: int, + page_size: int, + layout: KVLayout, +): + """ROCm only: the K kernel above with the in-place query rope in the same launch.""" + args = make_cpp_args( + dtype, head_dim, rope_dim, page_size, layout.cpp_name, is_arch_support_pdl() + ) + return load_jit( + make_name("main_k_norm_rope_q_flashmla"), + *args, + cuda_files=["deepseek_v4/main_norm_rope.cuh"], + cuda_wrappers=[ + ("forward_with_q", f"FusedKNormRopeFlashMLAKernel<{args}>::forward_with_q"), + ], + ) + + @cache_once def _jit_main_q_indexer_rope_hadamard_quant_module(dtype: torch.dtype): """C4 indexer Q kernel: RoPE + 128-pt Hadamard + fp8 act-quant""" @@ -278,6 +300,7 @@ def fused_k_norm_rope_flashmla( kvcache: torch.Tensor, page_size: int, layout: Union[KVLayout, str] = KVLayout.V4, + q: Optional[torch.Tensor] = None, ) -> None: """RMSNorm + RoPE ``kv`` and write it into the ``layout`` paged FlashMLA cache at ``out_loc``.""" @@ -287,9 +310,18 @@ def fused_k_norm_rope_flashmla( rope_dim = freqs_real.shape[-1] if _is_xpu: assert layout is KVLayout.V4, "the V4.1 KV layouts are CUDA (sm100) only" + assert q is None, "the XPU K kernel does not rope q" fused_k_norm_rope_flashmla_xpu( kv, kv_weight, freqs_real, positions, out_loc, kvcache, eps, page_size ) + elif q is not None: + assert _is_hip, "only the ROCm K launch ropes q" + module = _jit_main_k_norm_rope_q_flashmla_module( + kv.dtype, head_dim, rope_dim, page_size, layout + ) + module.forward_with_q( + kv, kv_weight, freqs_real, positions, out_loc, kvcache, eps, q + ) else: module = _jit_main_k_norm_rope_flashmla_module( kv.dtype, head_dim, rope_dim, page_size, layout diff --git a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer.py b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer.py index d22c4b5fb3..28bb762599 100644 --- a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer.py +++ b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer.py @@ -74,17 +74,19 @@ def _fp4_e2m1_code_rne(x): @triton.jit -def _quantize_fp4_indexer_kernel( - x, - x_fp4, - x_sf, +def quantize_fp4_indexer_row( + values, + v0, + v1, BLOCK_N: tl.constexpr, GROUP_N: tl.constexpr, RNE: tl.constexpr, ): - token_id = tl.program_id(0) + """One row of quantize_fp4_indexer_tensor: values is the fp32 [BLOCK_N] row, + v0 / v1 its even / odd elements as fp32 [BLOCK_N // 2]. Returns + (packed codes [BLOCK_N // 2], packed ue8m0 exponents), the two stores of the + standalone kernel below.""" offs = tl.arange(0, BLOCK_N) - values = tl.load(x + token_id * BLOCK_N + offs).to(tl.float32) abs_values = tl.abs(values) amax0 = tl.max(tl.where(offs < GROUP_N, abs_values, 0.0), axis=0) @@ -109,7 +111,6 @@ def _quantize_fp4_indexer_kernel( exp3 = _ceil_ue8m0_exp(sf3) packed_sf = exp0 | (exp1 << 8) | (exp2 << 16) | (exp3 << 24) - tl.store(x_sf + token_id, packed_sf) pair_offsets = tl.arange(0, BLOCK_N // 2) offs0 = pair_offsets * 2 @@ -121,8 +122,8 @@ def _quantize_fp4_indexer_kernel( scale0 = (scale_exp0 << 23).to(tl.float32, bitcast=True) scale1 = (scale_exp1 << 23).to(tl.float32, bitcast=True) - v0 = tl.load(x + token_id * BLOCK_N + offs0).to(tl.float32) / scale0 - v1 = tl.load(x + token_id * BLOCK_N + offs1).to(tl.float32) / scale1 + v0 = v0 / scale0 + v1 = v1 / scale1 if RNE: code0 = _fp4_e2m1_code_rne(v0) code1 = _fp4_e2m1_code_rne(v1) @@ -130,6 +131,29 @@ def _quantize_fp4_indexer_kernel( code0 = _fp4_e2m1_code(v0) code1 = _fp4_e2m1_code(v1) packed = (code0 & 0x0F) | ((code1 & 0x0F) << 4) + return packed, packed_sf + + +@triton.jit +def _quantize_fp4_indexer_kernel( + x, + x_fp4, + x_sf, + BLOCK_N: tl.constexpr, + GROUP_N: tl.constexpr, + RNE: tl.constexpr, +): + token_id = tl.program_id(0) + offs = tl.arange(0, BLOCK_N) + pair_offsets = tl.arange(0, BLOCK_N // 2) + row = x + token_id * BLOCK_N + values = tl.load(row + offs).to(tl.float32) + v0 = tl.load(row + pair_offsets * 2).to(tl.float32) + v1 = tl.load(row + pair_offsets * 2 + 1).to(tl.float32) + packed, packed_sf = quantize_fp4_indexer_row( + values, v0, v1, BLOCK_N=BLOCK_N, GROUP_N=GROUP_N, RNE=RNE + ) + tl.store(x_sf + token_id, packed_sf) tl.store(x_fp4 + token_id * (BLOCK_N // 2) + pair_offsets, packed) diff --git a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_hip.py b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_hip.py index 3982898a30..cef8fab212 100644 --- a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_hip.py +++ b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_hip.py @@ -6,6 +6,18 @@ import os from typing import TYPE_CHECKING, NamedTuple, Optional, Tuple, Union import torch +import triton +import triton.language as tl + +from sglang.kernels.ops.attention.dsv4.fp4_indexer import ( + quantize_fp4_indexer_row, + quantize_fp4_indexer_tensor, +) +from sglang.kernels.ops.attention.dsv4.fp4_rope_fake_quant import ( + FP4_AMAX_FLOOR, + rope_tail_fake_quant_fp4_row, +) +from sglang.kernels.ops.gemm.router_gemv_hip import rocm_router_gemv_split_k if TYPE_CHECKING: from sglang.kernels.ops.attention.dsv4.compress import ( @@ -28,6 +40,8 @@ _DECODE_BASE_CTA_TARGET = 1024 # Preserve per-query parallelism when the batch itself exceeds one CTA per CU. _DECODE_CTAS_PER_QUERY = 4 _PREFILL_BASE_CTA_TARGET = 1024 +# FlyDSL compiles one kernel per page-table width: bucket it so a new context length pays no JIT +LOW_RATIO_PAGE_TABLE_BUCKET = 64 # AITER varctx cta_info row: [batch_packed, chunk_start, chunk_count, ctx_len]. _DECODE_CTA_INFO_WIDTH = 4 @@ -129,23 +143,32 @@ def _decode_cta_count(num_queries: int, max_seq_len: int) -> int: return min(available_ctas, target_ctas) -def _guarded_pages(logical_width: int) -> int: +def _guarded_pages(logical_width: int, page_table_bucket: int = 4) -> int: """Page columns after padding for 256-token scheduling.""" - return max(4, (logical_width + 3) // 4 * 4) + return max( + 4, + (logical_width + page_table_bucket - 1) + // page_table_bucket + * page_table_bucket, + ) -def _guard_page_table(page_table: torch.Tensor, out: Optional[torch.Tensor] = None): +def _guard_page_table( + page_table: torch.Tensor, + out: Optional[torch.Tensor] = None, + page_table_bucket: int = 4, +): """Pad page tables for 256-token scheduling and one-chunk lookahead.""" from sglang.kernels.ops.attention.dsv4.fp4_indexer_schedule_hip import ( pad_page_table, ) - return pad_page_table(page_table, out=out) + return pad_page_table(page_table, out=out, page_table_bucket=page_table_bucket) -def logits_rows_per_chunk(page_table: torch.Tensor) -> int: +def logits_rows_per_chunk(page_table: torch.Tensor, page_table_bucket: int = 4) -> int: """Rows whose logits fit the pooled block, for callers that loop by row.""" - width = _guarded_pages(page_table.shape[1]) * _KV_BLOCK_SIZE + width = _guarded_pages(page_table.shape[1], page_table_bucket) * _KV_BLOCK_SIZE return max(1, _LOGITS_BUDGET_ELEMS // width) @@ -188,6 +211,7 @@ def _alloc_logits( def prepare_fp4_decode_workspace( page_table: torch.Tensor, c4_seq_lens: torch.Tensor, + page_table_bucket: int = 4, ) -> FP4DecodeWorkspace: """Build the decode page-table, schedule, and logits buffers. @@ -195,11 +219,13 @@ def prepare_fp4_decode_workspace( kernel touches is reachable from the returned workspace, so none of it can be handed out again by a later capture sharing the graph memory pool. """ - from aiter.ops.flydsl.kernels.mqa_logits.pa_mqa_logits_fp4 import ( - compute_varctx_schedule, + from sglang.kernels.ops.attention.dsv4.fp4_indexer_schedule_hip import ( + build_decode_schedule, ) - guarded, max_seq_len = _guard_page_table(page_table) + guarded, max_seq_len = _guard_page_table( + page_table, page_table_bucket=page_table_bucket + ) c4_seq_lens = _as_int32_1d(c4_seq_lens) num_queries = guarded.shape[0] cta_count = _decode_cta_count(num_queries, max_seq_len) @@ -208,13 +234,8 @@ def prepare_fp4_decode_workspace( dtype=torch.int32, device=guarded.device, ) - schedule_scratch, _, _ = compute_varctx_schedule( - c4_seq_lens, - block_k=256, - parallel_unit_num=cta_count, - max_seq_len=max_seq_len, - next_n=1, - cta_info_out=cta_info, + schedule_scratch = build_decode_schedule( + c4_seq_lens, cta_info_out=cta_info, max_seq_len=max_seq_len ) return FP4DecodeWorkspace( guarded, c4_seq_lens, cta_info, cta_count, max_seq_len, schedule_scratch @@ -225,6 +246,7 @@ def prepare_fp4_prefill_workspace( page_table: torch.Tensor, c4_seq_lens: torch.Tensor, workspace: Optional[FP4PrefillWorkspace] = None, + page_table_bucket: int = 4, ) -> FP4PrefillWorkspace: """Build or refresh the prefill page-table, schedule, and logits buffers. @@ -247,7 +269,7 @@ def prepare_fp4_prefill_workspace( c4_seq_lens = _as_int32_1d(c4_seq_lens) if workspace is None: - rows, _, padded_width = padded_page_table_shape(page_table) + rows, _, padded_width = padded_page_table_shape(page_table, page_table_bucket) cta_count = max(_PREFILL_BASE_CTA_TARGET, rows) device = page_table.device buffers = PrefillScheduleBuffers(rows, device) @@ -280,6 +302,7 @@ def prepare_fp4_prefill_workspace( block_k=256, guarded_out=workspace.guarded_page_table, buffers=workspace.schedule_buffers, + page_table_bucket=page_table_bucket, ) return workspace @@ -297,6 +320,7 @@ def aiter_fp4_paged_mqa_logits( is_decode: bool, decode_workspace: Optional[FP4DecodeWorkspace] = None, prefill_workspace: Optional[FP4PrefillWorkspace] = None, + page_table_bucket: int = 4, ) -> torch.Tensor: """Compute FP4 Q/K indexer logits with the decode or prefill FlyDSL kernel.""" from aiter.ops.flydsl import ( @@ -318,7 +342,26 @@ def aiter_fp4_paged_mqa_logits( page_table = workspace.guarded_page_table max_seq_len = workspace.max_seq_len elif is_decode: - page_table, max_seq_len = _guard_page_table(page_table) + # No usable workspace: build the schedule here too, rather than letting + # AITER build it with its varctx kernel, which cannot reserve its scratch + # at large batches. + from sglang.kernels.ops.attention.dsv4.fp4_indexer_schedule_hip import ( + build_decode_schedule, + ) + + page_table, max_seq_len = _guard_page_table( + page_table, page_table_bucket=page_table_bucket + ) + cta_count = _decode_cta_count(num_tokens, max_seq_len) + cta_info = torch.empty( + (cta_count, _DECODE_CTA_INFO_WIDTH), + dtype=torch.int32, + device=page_table.device, + ) + scratch = build_decode_schedule( + c4_seq_lens, cta_info_out=cta_info, max_seq_len=max_seq_len + ) + fallback_schedule = (cta_info, cta_count, scratch) else: # No usable workspace (DP padding or truncated activations): build the # schedule here rather than letting AITER rebuild it from ~29 torch ops @@ -332,7 +375,7 @@ def aiter_fp4_paged_mqa_logits( padded_page_table_shape, ) - _, _, padded_width = padded_page_table_shape(page_table) + _, _, padded_width = padded_page_table_shape(page_table, page_table_bucket) max_seq_len = padded_width * _KV_BLOCK_SIZE cta_count = max(_PREFILL_BASE_CTA_TARGET, num_tokens) cta_info = torch.empty( @@ -347,6 +390,7 @@ def aiter_fp4_paged_mqa_logits( parallel_unit_num=cta_count, max_seq_len=max_seq_len, block_k=256, + page_table_bucket=page_table_bucket, ) fallback_schedule = (cta_info, cta_count, buffers) q_payload = q_fp4.view(torch.uint8) @@ -365,16 +409,16 @@ def aiter_fp4_paged_mqa_logits( } if is_decode: - pinned = ( - {} - if workspace is None - else { + if workspace is None: + cta_info, cta_count, _ = fallback_schedule + pinned = {"cta_info": cta_info, "total_ctas": cta_count} + else: + pinned = { "cta_info": workspace.cta_info, "total_ctas": workspace.cta_count, } - ) logits = flydsl_pa_mqa_logits_fp4( - q_payload.reshape(num_tokens, 1, _HEADS, _HEAD_DIM // 2), + q_payload.reshape(num_tokens, 1, q_fp4.shape[-2], _HEAD_DIM // 2), q_scale.reshape(num_tokens, 1, *_Q_SCALE_SHAPE), k_payload, k_scale, @@ -497,3 +541,362 @@ def aiter_k_indexer_fp4_cache_write( shuffle_scale=True, do_rotate_act=True, ) + + +# FlyDSL split payload / scale layout; bytes equal AITER's fp4 writer up to the sign of zero +@triton.jit +def _store_fp4_index_k_split_kernel( + k_fp4, + k_sf, + payload, + scale, + loc, + page_size: tl.constexpr, + BLOCK: tl.constexpr, +): + token_id = tl.program_id(0) + cache_loc = tl.load(loc + token_id) + page = cache_loc // page_size + page_offset = cache_loc - page * page_size + + # payload [page, 1, 4 chunks, page_size, 16 bytes]: chunk c holds dims [32c, 32c + 32). + offsets = tl.arange(0, BLOCK) + chunk = offsets // 16 + byte = offsets - chunk * 16 + k = tl.load(k_fp4 + token_id * BLOCK + offsets) + tl.store( + payload + + page * (4 * page_size * 16) + + chunk * (page_size * 16) + + page_offset * 16 + + byte, + k, + ) + # scale [page, 1, 4, page_size]: the slot axis is a 16 x 4 tile transposed (the FlyDSL K ABI) + shuffled = (page_offset % 16) * 4 + page_offset // 16 + sf = tl.load(k_sf + token_id) + sf_offsets = tl.arange(0, 4) + sf_bytes = ((sf >> (sf_offsets * 8)) & 0xFF).to(tl.uint8) + tl.store( + scale + page * (4 * page_size) + sf_offsets * page_size + shuffled, sf_bytes + ) + + +def store_fp4_index_k_cache_split( + input: torch.Tensor, + payload: torch.Tensor, + scale: torch.Tensor, + loc: torch.Tensor, + *, + page_size: int, + rne: bool = False, +) -> None: + """Quantize input [n, 128] to fp4 (per-32 ue8m0) and scatter row i to slot + loc[i] of the split FlyDSL K layout (payload [pages, 1, 4, page_size, 16], + scale [pages, 1, 4, page_size] with the slot axis 16 x 4 transposed).""" + assert input.shape[-1] == _HEAD_DIM + assert page_size == _KV_BLOCK_SIZE, "the 16 x 4 slot transpose spans a 64-slot page" + assert payload.shape[1:] == (1, 4, page_size, 16), payload.shape + assert scale.shape[1:] == (1, 4, page_size), scale.shape + k_fp4, k_sf = quantize_fp4_indexer_tensor(input.contiguous(), rne=rne) + n_tokens = k_fp4.shape[0] + if n_tokens == 0: + return + _store_fp4_index_k_split_kernel[(n_tokens,)]( + k_fp4.view(torch.uint8), + k_sf, + payload.view(torch.uint8), + scale, + loc, + page_size, + BLOCK=64, + ) + + +def read_fp4_index_k_split( + payload: torch.Tensor, scale: torch.Tensor, slots: torch.Tensor, *, page_size: int +) -> Tuple[torch.Tensor, torch.Tensor]: + """Inverse of store_fp4_index_k_cache_split: (payload int8 [n, 64], scales + int32 [n] with chunk c's e8m0 byte at bits 8c..8c+7), the layout of + quantize_fp4_indexer_tensor.""" + assert page_size == _KV_BLOCK_SIZE, "the 16 x 4 slot transpose spans a 64-slot page" + slots = slots.to(torch.int64) + page, off = slots // page_size, slots % page_size + rows = payload.view(torch.uint8)[page, 0, :, off, :] # [n, 4, 16] + rows = rows.reshape(-1, 64).view(torch.int8) + shuffled = (off % 16) * 4 + off // 16 + sf = scale[page, 0, :, shuffled].to(torch.int32) # [n, 4] + packed = sf[:, 0] | (sf[:, 1] << 8) | (sf[:, 2] << 16) | (sf[:, 3] << 24) + return rows, packed + + +@triton.jit +def _quantize_fp4_query_flydsl_kernel( + x, + x_fp4, + q_scale, + HEADS: tl.constexpr, + BLOCK_N: tl.constexpr, + GROUP_N: tl.constexpr, +): + """One program per (token, head slot of 64): quantize_fp4_indexer_row (RNE codes) with + the e8m0 byte of chunk c stored straight into the FlyDSL scale layout + [t, 0, c, h % 16, h // 16]; head slots past HEADS write zero bytes.""" + token_id = tl.program_id(0) + h = tl.program_id(1) + scale_base = q_scale + token_id * (4 * 16 * 4) + (h % 16) * 4 + h // 16 + chunks = tl.arange(0, 4) + if h >= HEADS: + tl.store(scale_base + chunks * (16 * 4), tl.zeros([4], dtype=tl.uint8)) + return + row = token_id * HEADS + h + values = tl.load(x + row * BLOCK_N + tl.arange(0, BLOCK_N)).to(tl.float32) + v0, v1 = tl.split(tl.reshape(values, (BLOCK_N // 2, 2))) + packed, packed_sf = quantize_fp4_indexer_row( + values, v0, v1, BLOCK_N=BLOCK_N, GROUP_N=GROUP_N, RNE=True + ) + sf_bytes = ((packed_sf >> (8 * chunks)) & 0xFF).to(tl.uint8) + tl.store(scale_base + chunks * (16 * 4), sf_bytes) + tl.store(x_fp4 + row * (BLOCK_N // 2) + tl.arange(0, BLOCK_N // 2), packed) + + +def pack_fp4_query_flydsl(q: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """fp4-grid query [T, H, 128] -> (payload int8 [T, H, 64], scale uint8 [T, 1, 4, 16, 4]) in the + FlyDSL MQA-logits layout: head h, chunk c's e8m0 byte at [t, 0, c, h % 16, h // 16] (H <= 64), + RNE as the CUDA low-ratio path.""" + num_tokens, heads = q.shape[0], q.shape[1] + assert heads % 16 == 0 and heads <= 64, ( + f"{heads} heads: the FlyDSL scale layout holds 16-head groups up to 64" + ) + assert q.shape[-1] == _HEAD_DIM + x = q.contiguous().view(-1, _HEAD_DIM) + q_fp4 = torch.empty( + (num_tokens, heads, _HEAD_DIM // 2), dtype=torch.int8, device=q.device + ) + q_scale = torch.empty((num_tokens, 1, 4, 16, 4), dtype=torch.uint8, device=q.device) + if num_tokens > 0: + _quantize_fp4_query_flydsl_kernel[(num_tokens, 64)]( + x, + q_fp4, + q_scale, + HEADS=heads, + BLOCK_N=_HEAD_DIM, + GROUP_N=_GROUP_SIZE, + ) + return q_fp4, q_scale + + +@triton.jit +def _reduce_scale_bf16_block( + block, + part_ptr, + out_ptr, + M, + stride_ps, + stride_pm, + stride_om, + scale, + N: tl.constexpr, + SPLIT_K: tl.constexpr, + BLOCK: tl.constexpr, +): + """One BLOCK of _reduce_scale_bf16_kernel (shared with the fused index-Q launch).""" + offs = block * BLOCK + tl.arange(0, BLOCK) + m = offs // N + n = offs % N + mask = m < M + acc = tl.load(part_ptr + m * stride_pm + n, mask=mask, other=0.0) + for s in tl.static_range(1, SPLIT_K): + acc += tl.load( + part_ptr + s * stride_ps + m * stride_pm + n, mask=mask, other=0.0 + ) + # bf16(bf16(sum) * scale): the linear's bf16 output, then the aten multiply rounded to bf16 + w = acc.to(tl.bfloat16).to(tl.float32) * scale + tl.store(out_ptr + m * stride_om + n, w.to(tl.bfloat16), mask=mask) + + +@triton.jit +def _reduce_scale_bf16_kernel( + part_ptr, + out_ptr, + M, + stride_ps, + stride_pm, + stride_om, + scale, + N: tl.constexpr, + SPLIT_K: tl.constexpr, + BLOCK: tl.constexpr, +): + _reduce_scale_bf16_block( + tl.program_id(0), + part_ptr, + out_ptr, + M, + stride_ps, + stride_pm, + stride_om, + scale, + N=N, + SPLIT_K=SPLIT_K, + BLOCK=BLOCK, + ) + + +@triton.jit +def _index_q_pack_weights_kernel( + q_ptr, # [T, H * 128] bf16 (wq_b output) + f_ptr, # [max_pos, RD // 2, 2] fp32: the real view of the complex freqs table + pos_ptr, # [T] int64 + q_fp4_ptr, # [T, H, 64] int8 + q_scale_ptr, # [T, 1, 4, 16, 4] uint8 + part_ptr, # [SPLIT_K, T, H] fp32 head-weight partials + weights_ptr, # [T, H] bf16 + stride_qt, + stride_ps, + stride_pm, + weight_scale, + T, + num_pos, + H: tl.constexpr, + D: tl.constexpr, + RD: tl.constexpr, + AMAX_FLOOR: tl.constexpr, + SPLIT_K: tl.constexpr, + W_BLOCK: tl.constexpr, +): + """Grid (T * H + 1,). Program t * H + h: the query row through rope_tail_fake_quant_fp4_row + (rounded to bf16 as the standalone kernel stores it) and quantize_fp4_indexer_row (RNE), in + the FlyDSL MQA-logits layout; program T * H: the head weights' _reduce_scale_bf16_block.""" + pid = tl.program_id(0) + if pid < T * H: + t = pid // H + h = pid % H + pos = tl.load(pos_ptr + t) + # the caller owns positions < num_pos; an out-of-range row reads entry 0 instead of past the table + pos = tl.where((pos >= 0) & (pos < num_pos), pos, 0) + fq = rope_tail_fake_quant_fp4_row( + q_ptr + t.to(tl.int64) * stride_qt + h * D, + f_ptr + pos * RD, + D=D, + RD=RD, + BLK=32, + AMAX_FLOOR=AMAX_FLOOR, + INVERSE=False, + COMPRESSED_KV=False, + ) + # the standalone path stores the fake-quant as bf16 and the packer reloads it + values = fq.to(tl.bfloat16).to(tl.float32) + v0, v1 = tl.split(tl.reshape(values, (D // 2, 2))) + packed, packed_sf = quantize_fp4_indexer_row( + values, v0, v1, BLOCK_N=D, GROUP_N=32, RNE=True + ) + tl.store(q_fp4_ptr + pid.to(tl.int64) * (D // 2) + tl.arange(0, D // 2), packed) + # scale bytes: chunk c of head h at [t, 0, c, h % 16, h // 16] + c = tl.arange(0, 4) + sf_bytes = ((packed_sf >> (8 * c)) & 0xFF).to(tl.uint8) + base = q_scale_ptr + t.to(tl.int64) * (4 * 16 * 4) + c * (16 * 4) + (h % 16) * 4 + tl.store(base + h // 16, sf_bytes) + if h < 16: + # groups this head count does not have stay zero + for g in tl.static_range(H // 16, 4): + tl.store(base + g, tl.zeros((4,), dtype=tl.uint8)) + else: + _reduce_scale_bf16_block( + 0, + part_ptr, + weights_ptr, + T, + stride_ps, + stride_pm, + H, + weight_scale, + N=H, + SPLIT_K=SPLIT_K, + BLOCK=W_BLOCK, + ) + + +def index_q_rope_pack_weights_flydsl( + q: torch.Tensor, + freqs_cis: torch.Tensor, + positions: torch.Tensor, + rope_dim: int, + head_weight_partials: torch.Tensor, + weight_scale: float, + *, + num_heads: int, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """The HIP index-Q inputs in one launch: (q_fp4 [T, H, 64] int8, q_scale [T, 1, 4, 16, 4] + uint8, weights [T, H] bf16), bitwise pack_fp4_query_flydsl(_rope_fq4(q.view(T, H, 128), + freqs_cis[positions], rope_dim)) and indexer_head_weights's reduce of the + [split_k, T, H] fp32 head_weight_partials.""" + T = q.shape[0] + H = num_heads + assert q.dtype == torch.bfloat16 and q.dim() == 2 and q.shape[1] == H * 128 + assert q.stride(1) == 1 + assert H % 16 == 0 and 0 < H <= 64, H + assert freqs_cis.dtype == torch.complex64 and freqs_cis.shape[1] == rope_dim // 2 + assert positions.shape == (T,) + split_k, pt, ph = head_weight_partials.shape + assert (pt, ph) == (T, H) and head_weight_partials.dtype == torch.float32 + f_real = torch.view_as_real(freqs_cis) + assert f_real.is_contiguous() + q_fp4 = torch.empty((T, H, 64), dtype=torch.int8, device=q.device) + q_scale = torch.empty((T, 1, 4, 16, 4), dtype=torch.uint8, device=q.device) + weights = torch.empty((T, H), dtype=torch.bfloat16, device=q.device) + if T == 0: + return q_fp4, q_scale, weights + w_block = triton.next_power_of_2(T * H) + assert w_block <= 4096, ( + f"T * H = {T * H} exceeds the 4096-wide head-weight reduce block" + ) + _index_q_pack_weights_kernel[(T * H + 1,)]( + q, + f_real, + positions, + q_fp4, + q_scale, + head_weight_partials, + weights, + q.stride(0), + head_weight_partials.stride(0), + head_weight_partials.stride(1), + float(weight_scale), + T, + f_real.shape[0], + H=H, + D=128, + RD=rope_dim, + AMAX_FLOOR=FP4_AMAX_FLOOR, + SPLIT_K=split_k, + W_BLOCK=w_block, + num_warps=4, + ) + return q_fp4, q_scale, weights + + +def indexer_head_weights( + x: torch.Tensor, weight: torch.Tensor, scale: float +) -> torch.Tensor: + """bf16(bf16(x @ weight.T) * scale) as a contiguous bf16 [M, N], the layout the + FlyDSL logits kernels take. x bf16 [M, K] and weight bf16 [N, K], with + M <= rocm_gemv_split_k_max_tokens(n=N, k=K, weight_dtype=torch.bfloat16).""" + partials = rocm_router_gemv_split_k(x, weight) + split_k, M, N = partials.shape + out = torch.empty((M, N), dtype=torch.bfloat16, device=x.device) + block = triton.next_power_of_2(M * N) + _reduce_scale_bf16_kernel[(triton.cdiv(M * N, block),)]( + partials, + out, + M, + partials.stride(0), + partials.stride(1), + out.stride(0), + float(scale), + N=N, + SPLIT_K=split_k, + BLOCK=block, + num_warps=4, + ) + return out diff --git a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_rope_hip.py b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_rope_hip.py new file mode 100644 index 0000000000..0d02c5fbc1 --- /dev/null +++ b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_rope_hip.py @@ -0,0 +1,65 @@ +"""ROCm index-K store of the FP4 indexer into the split FlyDSL layout (payload and scale buffers).""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch + +from sglang.kernels.jit.utils import ( + cache_once, + is_arch_support_pdl, + load_jit, + make_cpp_args, +) + +from .utils import make_name + +if TYPE_CHECKING: + from tvm_ffi.module import Module + + +@cache_once +def _jit_index_k_split_module( + head_dim: int, rope_dim: int, page_size: int, ratio: int +) -> Module: + args = make_cpp_args(head_dim, rope_dim, page_size, ratio, is_arch_support_pdl()) + return load_jit( + make_name("index_k_rope_pack_split"), + *args, + cuda_files=["deepseek_v4/fp4_indexer_rope_hip.cuh"], + cuda_wrappers=[("index_k_split", f"FlashIndexKSplitKernel<{args}>::run")], + ) + + +def index_k_norm_rope_pack_store_split( + input: torch.Tensor, + norm_weight: torch.Tensor, + eps: float, + freqs_cis: torch.Tensor, + positions: torch.Tensor, + loc: torch.Tensor, + payload: torch.Tensor, + scale: torch.Tensor, + *, + ratio: int, +) -> None: + """`fp4_indexer_rope.index_k_norm_rope_pack_store` into the split + FlyDSL K layout (payload [npages, 1, 4, page_size, 16], scale [npages, 1, 4, + page_size] uint8); the bytes equal store_fp4_index_k_cache_split's.""" + head_dim = input.shape[-1] + page_size = payload.shape[3] + assert payload.shape[1:] == (1, 4, page_size, 16), payload.shape + assert scale.shape[1:] == (1, 4, page_size), scale.shape + _jit_index_k_split_module( + head_dim, freqs_cis.shape[-1], page_size, ratio + ).index_k_split( + input, + norm_weight, + freqs_cis, + positions, + loc, + payload.view(torch.uint8), + scale, + float(eps), + ) diff --git a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_schedule_hip.py b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_schedule_hip.py index cd930dd373..f6fe697a2c 100644 --- a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_schedule_hip.py +++ b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_schedule_hip.py @@ -12,6 +12,10 @@ Everything the preamble computes is a reduction or a scan over the row count, so it collapses into a single kernel; the only real dependency is that ``_prefill_cta_info_kernel`` needs the finished prefix sums. That leaves two dispatches for the whole schedule. + +The decode schedule (AITER's ``compute_varctx_schedule``) is built the same way: +AITER's single kernel holds [256, rows] tiles in registers and spills past a few +thousand rows, so its scratch cannot be reserved once the KV pool owns the memory. """ from __future__ import annotations @@ -167,7 +171,9 @@ def _pad_page_table_kernel( ) -def padded_page_table_shape(page_table: torch.Tensor) -> Tuple[int, int, int]: +def padded_page_table_shape( + page_table: torch.Tensor, page_table_bucket: int = 4 +) -> Tuple[int, int, int]: """Rows, logical width, and the 256-token-scheduling padded width. The padding rule is shared with the logits sizing in ``fp4_indexer_hip``; @@ -176,7 +182,7 @@ def padded_page_table_shape(page_table: torch.Tensor) -> Tuple[int, int, int]: from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import _guarded_pages rows, logical_width = page_table.shape - return rows, logical_width, _guarded_pages(logical_width) + return rows, logical_width, _guarded_pages(logical_width, page_table_bucket) def _as_int32_2d(page_table: torch.Tensor) -> torch.Tensor: @@ -189,7 +195,9 @@ def _as_int32_2d(page_table: torch.Tensor) -> torch.Tensor: def pad_page_table( - page_table: torch.Tensor, out: Optional[torch.Tensor] = None + page_table: torch.Tensor, + out: Optional[torch.Tensor] = None, + page_table_bucket: int = 4, ) -> Tuple[torch.Tensor, int]: """Pad a page table for 256-token scheduling in a single dispatch. @@ -197,7 +205,9 @@ def pad_page_table( output element, so the destination never needs pre-zeroing. """ page_table = _as_int32_2d(page_table) - rows, logical_width, padded_width = padded_page_table_shape(page_table) + rows, logical_width, padded_width = padded_page_table_shape( + page_table, page_table_bucket + ) if out is None: out = page_table.new_empty((rows, padded_width + 4)) else: @@ -269,6 +279,7 @@ def build_prefill_schedule( block_k: int = 256, guarded_out: Optional[torch.Tensor] = None, buffers: Optional[PrefillScheduleBuffers] = None, + page_table_bucket: int = 4, ) -> Tuple[torch.Tensor, PrefillScheduleBuffers]: """Pad the page table and build the FP4 prefill schedule in two dispatches. @@ -282,7 +293,9 @@ def build_prefill_schedule( ) page_table = _as_int32_2d(page_table) - rows, logical_width, padded_width = padded_page_table_shape(page_table) + rows, logical_width, padded_width = padded_page_table_shape( + page_table, page_table_bucket + ) total_rows = local_ends.shape[0] assert total_rows <= rows, ( f"local_ends rows {total_rows} exceed the page table's {rows}; the " @@ -363,3 +376,153 @@ def build_prefill_schedule( BLOCK_P=BLOCK_P, ) return guarded_out, buffers + + +# Rows per block of the decode prep kernel's loops: its registers stay fixed as the +# batch grows, where AITER's varctx kernel holds every row in one tile. +_DECODE_ROW_BLOCK = 1024 +# Slots per program of the decode cta_info kernel, as AITER's varctx kernel. +_DECODE_SLOT_BLOCK = 256 + + +@triton.jit +def _decode_schedule_prep_kernel( + ctx_ptr, # [B] int32 c4 context lengths + incl_ptr, # [B] int32 out, inclusive prefix sum of per-row CTA counts + scalars_ptr, # [2] int32 out, [safe, total_splits] + B, + P, + block_k, + s_max, + ROW_BLOCK: tl.constexpr, +): + """AITER varctx schedule preamble (next_n = 1): the split factor and the prefix sums.""" + off = tl.arange(0, ROW_BLOCK) + zero = tl.sum(tl.zeros([ROW_BLOCK], tl.int32), axis=0) + + max_chunks = zero + 1 + any_total = zero + for start in range(0, B, ROW_BLOCK): + live = start + off < B + ctx = tl.load(ctx_ptr + start + off, mask=live, other=0) + chunks = tl.where(live, (ctx + block_k - 1) // block_k, 0) + max_chunks = tl.maximum(max_chunks, tl.max(chunks, axis=0)) + any_total += tl.sum((chunks + s_max - 1) // s_max, axis=0) + + # Smallest split factor s with sum_i ceil(chunks_i / s) <= P, as AITER's search. + lo = zero + 1 + hi = zero + s_max + for _ in tl.static_range(32): + searching = lo < hi + mid = (lo + hi) // 2 + total = zero + for start in range(0, B, ROW_BLOCK): + live = start + off < B + ctx = tl.load(ctx_ptr + start + off, mask=live, other=0) + chunks = tl.where(live, (ctx + block_k - 1) // block_k, 0) + total += tl.sum((chunks + mid - 1) // mid, axis=0) + fits = total <= P + hi = tl.where(searching & fits, mid, hi) + lo = tl.where(searching & (fits == 0), mid + 1, lo) + safe = tl.where(any_total <= P, lo, max_chunks) + + carry = zero + for start in range(0, B, ROW_BLOCK): + live = start + off < B + ctx = tl.load(ctx_ptr + start + off, mask=live, other=0) + chunks = tl.where(live, (ctx + block_k - 1) // block_k, 0) + ctas = (chunks + safe - 1) // safe + tl.store(incl_ptr + start + off, tl.cumsum(ctas, axis=0) + carry, mask=live) + carry += tl.sum(ctas, axis=0) + tl.store(scalars_ptr, safe) + tl.store(scalars_ptr + 1, carry) + + +@triton.jit +def _decode_cta_info_kernel( + ctx_ptr, # [B] int32 c4 context lengths + incl_ptr, # [B] int32 inclusive prefix sum of per-row CTA counts + scalars_ptr, # [2] int32 [safe, total_splits] + cta_info_ptr, # [P, 4] int32 out, [batch, chunk_start, chunk_count, ctx_len] + B, + P, + block_k, + BLOCK_S: tl.constexpr, +): + """AITER varctx cta_info rows (next_n = 1), each slot's row found by a binary search + over incl in global memory, as AITER's _prefill_cta_info_kernel does.""" + safe = tl.load(scalars_ptr) + total_splits = tl.load(scalars_ptr + 1) + slot = tl.program_id(0) * BLOCK_S + tl.arange(0, BLOCK_S) + smask = slot < P + + # searchsorted(incl, slot, right=True) = count(incl <= slot) + lo = tl.zeros([BLOCK_S], tl.int32) + hi = tl.full([BLOCK_S], B, tl.int32) + for _ in tl.static_range(32): + mid = (lo + hi) // 2 + incl_mid = tl.load( + incl_ptr + tl.minimum(mid, B - 1), mask=mid < B, other=2147483647 + ) + go_right = incl_mid <= slot + lo = tl.where(go_right, mid + 1, lo) + hi = tl.where(go_right, hi, mid) + row = tl.minimum(lo, B - 1) + + ctx = tl.load(ctx_ptr + row, mask=smask, other=0) + chunks = (ctx + block_k - 1) // block_k + excl = tl.load(incl_ptr + row, mask=smask, other=0) - (chunks + safe - 1) // safe + + valid = slot < total_splits + vi = valid.to(tl.int32) + start = (slot - excl) * safe # pre-mask (count uses this) + count = tl.maximum(tl.minimum(safe, chunks - start), 0) + base = slot * 4 + tl.store(cta_info_ptr + base + 0, row * vi, mask=smask) + tl.store(cta_info_ptr + base + 1, start * vi, mask=smask) + tl.store(cta_info_ptr + base + 2, tl.where(valid, count, 1), mask=smask) + tl.store(cta_info_ptr + base + 3, ctx * vi, mask=smask) + + +def build_decode_schedule( + context_lens: torch.Tensor, + *, + cta_info_out: torch.Tensor, + max_seq_len: int, + block_k: int = 256, +) -> torch.Tensor: + """AITER's ``compute_varctx_schedule`` (next_n = 1) into ``cta_info_out`` [P, 4] in two + dispatches whose registers do not grow with the row count. Returns the scratch the + kernels read ([safe, total_splits] then the per-row prefix sums): a captured graph + must keep it alive.""" + context_lens = context_lens.to(torch.int32) + rows = context_lens.shape[0] + ctas = cta_info_out.shape[0] + assert rows <= ctas, ( + f"{rows} rows on {ctas} CTAs: the persistent grid must give every row a CTA" + ) + scratch = torch.empty(rows + 2, dtype=torch.int32, device=context_lens.device) + if rows == 0: + return scratch + scalars, incl = scratch[:2], scratch[2:] + _decode_schedule_prep_kernel[(1,)]( + context_lens, + incl, + scalars, + rows, + ctas, + block_k, + max(1, triton.cdiv(max_seq_len, block_k)), + ROW_BLOCK=_DECODE_ROW_BLOCK, + ) + _decode_cta_info_kernel[(triton.cdiv(ctas, _DECODE_SLOT_BLOCK),)]( + context_lens, + incl, + scalars, + cta_info_out, + rows, + ctas, + block_k, + BLOCK_S=_DECODE_SLOT_BLOCK, + ) + return scratch diff --git a/python/sglang/kernels/ops/attention/dsv4/fp4_rope_fake_quant.py b/python/sglang/kernels/ops/attention/dsv4/fp4_rope_fake_quant.py index 6a20233bf5..adc6176e3d 100644 --- a/python/sglang/kernels/ops/attention/dsv4/fp4_rope_fake_quant.py +++ b/python/sglang/kernels/ops/attention/dsv4/fp4_rope_fake_quant.py @@ -2,6 +2,8 @@ from __future__ import annotations +from typing import Optional + import torch import triton import triton.language as tl @@ -11,14 +13,9 @@ from sglang.kernels.ops.attention.dsv4.torch_quant import FP4_AMAX_FLOOR @triton.jit -def _rope_tail_fake_quant_fp4_kernel( - x_ptr, - f_ptr, - out_ptr, - x_stride_r, - out_stride_r, - f_stride_t, - rows_per_token, +def rope_tail_fake_quant_fp4_row( + x_row_ptr, + f_row_ptr, D: tl.constexpr, RD: tl.constexpr, BLK: tl.constexpr, @@ -26,10 +23,10 @@ def _rope_tail_fake_quant_fp4_kernel( INVERSE: tl.constexpr, COMPRESSED_KV: tl.constexpr, ): - r = tl.program_id(0) - t = r // rows_per_token + """One row of rope_tail_fake_quant_fp4 as an fp32 [D] vector: x_row_ptr points + at the row, f_row_ptr at the token's [RD // 2, 2] real view of the complex freqs.""" offs = tl.arange(0, D) - v = tl.load(x_ptr + r * x_stride_r + offs).to(tl.float32) + v = tl.load(x_row_ptr + offs).to(tl.float32) # ---- rope_tail: adjacent pairs of the last RD features as one complex number head_len = D - RD @@ -37,15 +34,13 @@ def _rope_tail_fake_quant_fp4_kernel( pos = offs - head_len j = pos // 2 is_im = (pos % 2) == 1 - re = tl.load(x_ptr + x_stride_r * r + head_len + 2 * j, mask=in_tail, other=0.0).to( + re = tl.load(x_row_ptr + head_len + 2 * j, mask=in_tail, other=0.0).to(tl.float32) + im = tl.load(x_row_ptr + head_len + 2 * j + 1, mask=in_tail, other=0.0).to( tl.float32 ) - im = tl.load( - x_ptr + x_stride_r * r + head_len + 2 * j + 1, mask=in_tail, other=0.0 - ).to(tl.float32) # freqs is a real/imag-interleaved view: stride 2 between complex pairs. - fr = tl.load(f_ptr + t * f_stride_t + 2 * j, mask=in_tail, other=1.0) - fi = tl.load(f_ptr + t * f_stride_t + 2 * j + 1, mask=in_tail, other=0.0) + fr = tl.load(f_row_ptr + 2 * j, mask=in_tail, other=1.0) + fi = tl.load(f_row_ptr + 2 * j + 1, mask=in_tail, other=0.0) if INVERSE: fi = -fi rot = tl.where(is_im, re * fi + im * fr, re * fr - im * fi) @@ -75,8 +70,48 @@ def _rope_tail_fake_quant_fp4_kernel( # torch.round is round-half-to-even; torch.sign(0) is 0 sgn = tl.where(s > 0, 1.0, tl.where(s < 0, -1.0, 0.0)) q = libdevice.rint(mag / step) * step * sgn - out = tl.reshape(q * scale[:, None], (D,)) - tl.store(out_ptr + r * out_stride_r + offs, out.to(out_ptr.dtype.element_ty)) + return tl.reshape(q * scale[:, None], (D,)) + + +@triton.jit +def _rope_tail_fake_quant_fp4_kernel( + x_ptr, + f_ptr, + pos_ptr, + out_ptr, + x_stride_r, + out_stride_r, + f_stride_t, + rows_per_token, + num_pos, + D: tl.constexpr, + RD: tl.constexpr, + BLK: tl.constexpr, + AMAX_FLOOR: tl.constexpr, + INVERSE: tl.constexpr, + COMPRESSED_KV: tl.constexpr, + HAS_POS: tl.constexpr, +): + r = tl.program_id(0) + t = r // rows_per_token + if HAS_POS: + # freqs is the whole table: row t reads its position's entry (the freqs[positions] gather folded in) + t = tl.load(pos_ptr + t).to(tl.int64) + # the caller owns positions < num_pos; an out-of-range row reads entry 0 instead of past the table + t = tl.where((t >= 0) & (t < num_pos), t, 0) + out = rope_tail_fake_quant_fp4_row( + x_ptr + r * x_stride_r, + f_ptr + t * f_stride_t, + D=D, + RD=RD, + BLK=BLK, + AMAX_FLOOR=AMAX_FLOOR, + INVERSE=INVERSE, + COMPRESSED_KV=COMPRESSED_KV, + ) + tl.store( + out_ptr + r * out_stride_r + tl.arange(0, D), out.to(out_ptr.dtype.element_ty) + ) def rope_tail_fake_quant_fp4( @@ -87,10 +122,12 @@ def rope_tail_fake_quant_fp4( block_size: int = 32, *, compressed_kv: bool = False, + positions: Optional[torch.Tensor] = None, ) -> torch.Tensor: """RoPE and FP4 round-trip: per-16 E4M3 for compressed KV, per-32 UE8M0 otherwise. - x: [T, ..., D] contiguous in the last dim; freqs: complex [T, rope_dim // 2]. + x: [T, ..., D] contiguous in the last dim; freqs: complex [T, rope_dim // 2], or with + positions ([T] int) the whole table that row t reads at positions[t]. """ x = x.contiguous() if compressed_kv: @@ -105,20 +142,29 @@ def rope_tail_fake_quant_fp4( return out rows_per_token = rows // x.shape[0] f_real = torch.view_as_real(freqs.contiguous()).contiguous() + if positions is not None: + assert positions.dim() == 1 and positions.shape[0] == x.shape[0], ( + positions.shape, + x.shape, + ) + positions = positions.contiguous() _rope_tail_fake_quant_fp4_kernel[(rows,)]( x2, f_real, + positions if positions is not None else f_real, out.reshape(-1, d), x2.stride(0), d, f_real.stride(0), rows_per_token, + f_real.shape[0], D=d, RD=rope_dim, BLK=block_size, AMAX_FLOOR=FP4_AMAX_FLOOR, INVERSE=inverse, COMPRESSED_KV=compressed_kv, + HAS_POS=positions is not None, num_warps=4, ) return out diff --git a/python/sglang/kernels/ops/attention/dsv4/low_ratio_compress.py b/python/sglang/kernels/ops/attention/dsv4/low_ratio_compress.py index bbb559da97..fec2f2846e 100644 --- a/python/sglang/kernels/ops/attention/dsv4/low_ratio_compress.py +++ b/python/sglang/kernels/ops/attention/dsv4/low_ratio_compress.py @@ -17,6 +17,7 @@ import torch from sglang.kernels.jit.utils import ( cache_once, is_arch_support_pdl, + is_hip_runtime, load_jit, make_cpp_args, ) @@ -98,7 +99,8 @@ def c1_decode_norm_rope_store( freqs_cis, positions, out_loc, - k_cache, + # HIP's fp8_e4m3_t is uint8_t, so the kernel matches the fp8 pool as bytes + k_cache.view(torch.uint8) if is_hip_runtime() else k_cache, float(eps), ) return out @@ -176,7 +178,8 @@ def c2_decode_norm_rope_store( raw_out_loc, eps, freqs_cis, - k_cache, + # HIP's fp8_e4m3_t is uint8_t, so the kernel matches the fp8 pool as bytes + k_cache.view(torch.uint8) if is_hip_runtime() else k_cache, ring_size, draft_len, ) diff --git a/python/sglang/kernels/ops/gemm/bf16_fp32.py b/python/sglang/kernels/ops/gemm/bf16_fp32.py index 5e225649d3..50f40afa59 100644 --- a/python/sglang/kernels/ops/gemm/bf16_fp32.py +++ b/python/sglang/kernels/ops/gemm/bf16_fp32.py @@ -5,13 +5,6 @@ from typing import Optional import torch from sglang.srt.environ import envs -from sglang.srt.utils import get_bool_env_var, is_hip - -_is_hip = is_hip() -_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip - -if _use_aiter: - from aiter.tuned_gemm import tgemm _linear_bf16_fp32_algo = envs.SGLANG_OPT_BF16_FP32_GEMM_ALGO.get() _HPC_GEMM_WEIGHT_CACHE_ATTR = "_sglang_bf16xfp32_weight_cache" @@ -147,9 +140,7 @@ def linear_bf16_fp32( *, hpc_kernel_min_m: Optional[int] = None, ) -> torch.Tensor: - if _use_aiter and y.dtype == torch.bfloat16: - return tgemm.mm(x, y, otype=x.dtype).float() - elif hpc_kernel_min_m is not None: + if hpc_kernel_min_m is not None: output = _linear_bf16_fp32_hpc(x, y, min_m=hpc_kernel_min_m) if output is not None: return output diff --git a/python/sglang/kernels/ops/gemm/router_gemv_hip.py b/python/sglang/kernels/ops/gemm/router_gemv_hip.py new file mode 100644 index 0000000000..17e55139c4 --- /dev/null +++ b/python/sglang/kernels/ops/gemm/router_gemv_hip.py @@ -0,0 +1,149 @@ +"""Skinny bf16 GEMV on ROCm for decode row counts: [M, K] x [N, K] as fp32 split-K +partials whose fixed-order sum is batch invariant (the DeepSeek-V4 router and indexer head +weights).""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from sglang.srt.utils import is_gfx95_supported, is_hip + +# the GEMV walks 16-row tiles inside one program, so every M runs the same tile (batch invariance) +ROCM_ROUTER_MAX_TOKENS = 64 + +_BLOCK_M = 16 +_BLOCK_N = 16 +_BLOCK_K = 512 +_MAX_SPLIT_K = 32 + + +def rocm_gemv_split_k_max_tokens(*, n: int, k: int, weight_dtype: torch.dtype) -> int: + """Rows up to which rocm_router_gemv_split_k serves an [M, k] @ [n, k].T bf16 + GEMV (one 16-wide N tile per 512 of K, 16-row tiles), -1 when the device or the shape + rules it out.""" + if not (is_hip() and is_gfx95_supported()): + return -1 + if weight_dtype != torch.bfloat16: + return -1 + if n <= 0 or n % _BLOCK_N != 0: + return -1 + if k % _BLOCK_K != 0 or not 0 < k // _BLOCK_K <= _MAX_SPLIT_K: + return -1 + return ROCM_ROUTER_MAX_TOKENS + + +@triton.jit +def _router_gemv_split_k_kernel( + x_ptr, + w_ptr, + part_ptr, + M, + stride_xm, + stride_wn, + stride_ps, + stride_pm, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + M_TILES: tl.constexpr, +): + pid_n = tl.program_id(0) + pid_k = tl.program_id(1) + offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N) + offs_k = pid_k * BLOCK_K + tl.arange(0, BLOCK_K) + w = tl.load(w_ptr + offs_n[None, :] * stride_wn + offs_k[:, None]) + # the weight tile is read once; every row tile is the same BLOCK_M x BLOCK_K dot + for t in tl.static_range(M_TILES): + offs_m = t * BLOCK_M + tl.arange(0, BLOCK_M) + row_mask = offs_m < M + x = tl.load( + x_ptr + offs_m[:, None] * stride_xm + offs_k[None, :], + mask=row_mask[:, None], + other=0.0, + ) + acc = tl.dot(x, w) + tl.store( + part_ptr + + pid_k * stride_ps + + offs_m[:, None] * stride_pm + + offs_n[None, :], + acc, + mask=row_mask[:, None], + ) + + +def rocm_router_gemv_split_k(x: torch.Tensor, w: torch.Tensor) -> torch.Tensor: + """x[M, K] @ w[N, K].T as fp32 split-K partials [K // 512, M, N], summed over dim 0 in + order by rocm_router_reduce_partials or the fused gate.""" + M, K = x.shape + N, K_w = w.shape + assert K == K_w and K % _BLOCK_K == 0 and N % _BLOCK_N == 0 + assert 0 < M <= ROCM_ROUTER_MAX_TOKENS, ( + f"{M} rows: the split-K GEMV serves at most {ROCM_ROUTER_MAX_TOKENS}" + ) + assert x.dtype == torch.bfloat16 and w.dtype == torch.bfloat16 + assert x.stride(1) == 1 and w.stride(1) == 1 + split_k = K // _BLOCK_K + partials = torch.empty((split_k, M, N), dtype=torch.float32, device=x.device) + _router_gemv_split_k_kernel[(N // _BLOCK_N, split_k)]( + x, + w, + partials, + M, + x.stride(0), + w.stride(0), + partials.stride(0), + partials.stride(1), + BLOCK_M=_BLOCK_M, + BLOCK_N=_BLOCK_N, + BLOCK_K=_BLOCK_K, + M_TILES=triton.cdiv(M, _BLOCK_M), + num_warps=4, + ) + return partials + + +@triton.jit +def _reduce_partials_kernel( + part_ptr, + out_ptr, + M, + stride_ps, + stride_pm, + stride_om, + N: tl.constexpr, + SPLIT_K: tl.constexpr, + BLOCK: tl.constexpr, +): + offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) + m = offs // N + n = offs % N + mask = m < M + acc = tl.load(part_ptr + m * stride_pm + n, mask=mask, other=0.0) + for s in tl.static_range(1, SPLIT_K): + acc += tl.load( + part_ptr + s * stride_ps + m * stride_pm + n, mask=mask, other=0.0 + ) + tl.store(out_ptr + m * stride_om + n, acc, mask=mask) + + +def rocm_router_reduce_partials(partials: torch.Tensor, out: torch.Tensor) -> None: + """out[M, N] = partials[0] + partials[1] + ... in that order, in fp32: + the same sum the fused gate computes.""" + split_k, M, N = partials.shape + assert out.shape == (M, N) and out.dtype == torch.float32 and out.stride(1) == 1 + block = 1024 + _reduce_partials_kernel[(triton.cdiv(M * N, block),)]( + partials, + out, + M, + partials.stride(0), + partials.stride(1), + out.stride(0), + N=N, + SPLIT_K=split_k, + BLOCK=block, + num_warps=4, + ) diff --git a/python/sglang/kernels/ops/moe/rocm_router_gate.py b/python/sglang/kernels/ops/moe/rocm_router_gate.py new file mode 100644 index 0000000000..ac747c85c8 --- /dev/null +++ b/python/sglang/kernels/ops/moe/rocm_router_gate.py @@ -0,0 +1,307 @@ +"""DeepSeek-V4 MoE router gate at decode row counts on ROCm: a sqrtsoftplus top-k over the +split-K GEMV partials of router_gemv_hip, bitwise aiter's topk_gating_kernel_opt, ties +included.""" + +from __future__ import annotations + +from typing import Optional, Tuple + +import torch +import triton +import triton.language as tl + +from sglang.kernels.ops.gemm.router_gemv_hip import rocm_gemv_split_k_max_tokens + +# aiter's wave64 kernel: EPT = 384 / 64 = 6 slots per lane; the sorting network below is N == 6 +_WARP = 64 +_GATE_NUM_EXPERTS = _WARP * 6 +_MAX_TOPK = 16 + +_FLT_MAX = tl.constexpr(3.4028234663852886e38) +_LOG2_E = tl.constexpr(1.4426950408889634) +_LN_2 = tl.constexpr(0.6931471805599453) + + +def rocm_router_max_tokens( + *, + num_experts: int, + hidden_size: int, + topk: int, + weight_dtype: torch.dtype, +) -> int: + """Rows up to which rocm_router_gemv_split_k + rocm_router_gate + serve the router, -1 when the device or the shape rules them out.""" + if num_experts != _GATE_NUM_EXPERTS or not 0 < topk <= _MAX_TOPK: + return -1 + return rocm_gemv_split_k_max_tokens( + n=num_experts, k=hidden_size, weight_dtype=weight_dtype + ) + + +@triton.jit +def _score(x): + # aiter compute_score: +inf clamps to FLT_MAX, NaN passes, sqrt correctly rounded + x = tl.where(x > _FLT_MAX, _FLT_MAX, x) + t = tl.exp2(x * _LOG2_E) + sp = tl.where(x > 20.0, x, tl.log2(1.0 + t) * _LN_2) + return tl.sqrt_rn(sp) + + +@triton.jit +def _cas(vi, oi, ii, vj, oj, ij): + # aiter _CAS_DESC: swap when vals[i] < vals[j]; equal values stay put. + c = vi < vj + return ( + tl.where(c, vj, vi), + tl.where(c, oj, oi), + tl.where(c, ij, ii), + tl.where(c, vi, vj), + tl.where(c, oi, oj), + tl.where(c, ii, ij), + ) + + +@triton.jit +def _slot(cursor, a0, a1, a2, a3, a4, a5, other): + r = tl.where(cursor == 0, a0, other) + r = tl.where(cursor == 1, a1, r) + r = tl.where(cursor == 2, a2, r) + r = tl.where(cursor == 3, a3, r) + r = tl.where(cursor == 4, a4, r) + r = tl.where(cursor == 5, a5, r) + return r + + +@triton.jit +def _load_logits( + logits_ptr, part_ptr, row, e, stride_lm, stride_ps, stride_pm, SPLIT_K: tl.constexpr +): + if SPLIT_K == 0: + return tl.load(logits_ptr + row * stride_lm + e).to(tl.float32) + else: + acc = tl.load(part_ptr + row * stride_pm + e) + for s in tl.static_range(1, SPLIT_K): + acc += tl.load(part_ptr + s * stride_ps + row * stride_pm + e) + return acc + + +@triton.jit +def _gate_row( + logits_ptr, + part_ptr, + bias_ptr, + row, + stride_lm, + stride_ps, + stride_pm, + routed_scaling_factor, + SPLIT_K: tl.constexpr, + WRITE_LOGITS: tl.constexpr, + HAS_BIAS: tl.constexpr, + RENORM: tl.constexpr, + TOPK: tl.constexpr, +): + """aiter's topk_gating_kernel_opt for one row on one 64-lane wave: returns the + scaled weights and the ids as [64] lane tensors (lane k < TOPK holds slot k).""" + lane = tl.arange(0, 64) + # Slot i of a lane holds expert lane + 64 * i, as in aiter's register kernel. + i0 = lane + i1 = lane + 64 + i2 = lane + 128 + i3 = lane + 192 + i4 = lane + 256 + i5 = lane + 320 + x0 = _load_logits( + logits_ptr, part_ptr, row, i0, stride_lm, stride_ps, stride_pm, SPLIT_K + ) + x1 = _load_logits( + logits_ptr, part_ptr, row, i1, stride_lm, stride_ps, stride_pm, SPLIT_K + ) + x2 = _load_logits( + logits_ptr, part_ptr, row, i2, stride_lm, stride_ps, stride_pm, SPLIT_K + ) + x3 = _load_logits( + logits_ptr, part_ptr, row, i3, stride_lm, stride_ps, stride_pm, SPLIT_K + ) + x4 = _load_logits( + logits_ptr, part_ptr, row, i4, stride_lm, stride_ps, stride_pm, SPLIT_K + ) + x5 = _load_logits( + logits_ptr, part_ptr, row, i5, stride_lm, stride_ps, stride_pm, SPLIT_K + ) + if WRITE_LOGITS: + tl.store(logits_ptr + row * stride_lm + i0, x0) + tl.store(logits_ptr + row * stride_lm + i1, x1) + tl.store(logits_ptr + row * stride_lm + i2, x2) + tl.store(logits_ptr + row * stride_lm + i3, x3) + tl.store(logits_ptr + row * stride_lm + i4, x4) + tl.store(logits_ptr + row * stride_lm + i5, x5) + s0 = _score(x0) + s1 = _score(x1) + s2 = _score(x2) + s3 = _score(x3) + s4 = _score(x4) + s5 = _score(x5) + # as in aiter, a NaN score weighs 0 and its choice value is -inf + o0 = tl.where(s0 != s0, 0.0, s0) + o1 = tl.where(s1 != s1, 0.0, s1) + o2 = tl.where(s2 != s2, 0.0, s2) + o3 = tl.where(s3 != s3, 0.0, s3) + o4 = tl.where(s4 != s4, 0.0, s4) + o5 = tl.where(s5 != s5, 0.0, s5) + if HAS_BIAS: + v0 = s0 + tl.load(bias_ptr + i0).to(tl.float32) + v1 = s1 + tl.load(bias_ptr + i1).to(tl.float32) + v2 = s2 + tl.load(bias_ptr + i2).to(tl.float32) + v3 = s3 + tl.load(bias_ptr + i3).to(tl.float32) + v4 = s4 + tl.load(bias_ptr + i4).to(tl.float32) + v5 = s5 + tl.load(bias_ptr + i5).to(tl.float32) + else: + v0 = s0 + v1 = s1 + v2 = s2 + v3 = s3 + v4 = s4 + v5 = s5 + ninf = float("-inf") + v0 = tl.where(v0 != v0, ninf, v0) + v1 = tl.where(v1 != v1, ninf, v1) + v2 = tl.where(v2 != v2, ninf, v2) + v3 = tl.where(v3 != v3, ninf, v3) + v4 = tl.where(v4 != v4, ninf, v4) + v5 = tl.where(v5 != v5, ninf, v5) + # aiter sort_network_desc<6>: the 12-comparator network, same order. + v0, o0, i0, v1, o1, i1 = _cas(v0, o0, i0, v1, o1, i1) + v2, o2, i2, v3, o3, i3 = _cas(v2, o2, i2, v3, o3, i3) + v4, o4, i4, v5, o5, i5 = _cas(v4, o4, i4, v5, o5, i5) + v0, o0, i0, v2, o2, i2 = _cas(v0, o0, i0, v2, o2, i2) + v1, o1, i1, v4, o4, i4 = _cas(v1, o1, i1, v4, o4, i4) + v3, o3, i3, v5, o5, i5 = _cas(v3, o3, i3, v5, o5, i5) + v0, o0, i0, v1, o1, i1 = _cas(v0, o0, i0, v1, o1, i1) + v2, o2, i2, v3, o3, i3 = _cas(v2, o2, i2, v3, o3, i3) + v4, o4, i4, v5, o5, i5 = _cas(v4, o4, i4, v5, o5, i5) + v1, o1, i1, v2, o2, i2 = _cas(v1, o1, i1, v2, o2, i2) + v3, o3, i3, v4, o4, i4 = _cas(v3, o3, i3, v4, o4, i4) + v2, o2, i2, v3, o3, i3 = _cas(v2, o2, i2, v3, o3, i3) + # k-way merge: lanes offer their heads, the max wins, ties to the lowest lane (aiter ballot + ctz) + cursor = tl.zeros([64], dtype=tl.int32) + sel_val = tl.zeros([64], dtype=tl.float32) + sel_idx = tl.zeros([64], dtype=tl.int32) + total = 0.0 + for k in tl.static_range(TOPK): + my_val = _slot(cursor, v0, v1, v2, v3, v4, v5, ninf) + my_idx = _slot(cursor, i0, i1, i2, i3, i4, i5, 0) + max_val = tl.max(my_val, axis=0) + win_lane = tl.min(tl.where(my_val == max_val, lane, 64), axis=0) + win_idx = tl.sum(tl.where(lane == win_lane, my_idx, 0), axis=0) + i_won = (cursor < 6) & (my_idx == win_idx) + my_orig = tl.where(i_won, _slot(cursor, o0, o1, o2, o3, o4, o5, 0.0), 0.0) + cursor = cursor + i_won.to(tl.int32) + src_lane = win_idx & 63 + weight_bits = tl.sum( + tl.where(lane == src_lane, my_orig.to(tl.int32, bitcast=True), 0), axis=0 + ) + weight = weight_bits.to(tl.float32, bitcast=True) + sel_val = tl.where(lane == k, weight, sel_val) + sel_idx = tl.where(lane == k, win_idx, sel_idx) + total = total + weight + if RENORM: + scale = routed_scaling_factor / tl.maximum(total, 1e-20) + else: + scale = routed_scaling_factor * 1.0 + return sel_val * scale, sel_idx + + +@triton.jit +def _router_gate_kernel( + logits_ptr, # [M, 384] read when SPLIT_K == 0, written when WRITE_LOGITS + part_ptr, # [SPLIT_K, M, 384] fp32 (SPLIT_K > 0) + bias_ptr, # [384] fp32 or bf16 (HAS_BIAS) + weights_ptr, # [M, TOPK] fp32 + ids_ptr, # [M, TOPK] int32 + stride_lm, + stride_ps, + stride_pm, + stride_om, + routed_scaling_factor, + SPLIT_K: tl.constexpr, + WRITE_LOGITS: tl.constexpr, + HAS_BIAS: tl.constexpr, + RENORM: tl.constexpr, + TOPK: tl.constexpr, +): + row = tl.program_id(0) + weights, ids = _gate_row( + logits_ptr, + part_ptr, + bias_ptr, + row, + stride_lm, + stride_ps, + stride_pm, + routed_scaling_factor, + SPLIT_K, + WRITE_LOGITS, + HAS_BIAS, + RENORM, + TOPK, + ) + lane = tl.arange(0, 64) + out_mask = lane < TOPK + tl.store(weights_ptr + row * stride_om + lane, weights, mask=out_mask) + tl.store(ids_ptr + row * stride_om + lane, ids, mask=out_mask) + + +def rocm_router_gate( + gating_output: torch.Tensor, + correction_bias: Optional[torch.Tensor], + topk: int, + renormalize: bool, + routed_scaling_factor: Optional[float], + *, + partials: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + """aiter topk_gating(..., score_func="sqrtsoftplus") for 384 experts: fp32 weights and int32 + ids [M, topk]; with partials their fixed-order sum is gated and written into + gating_output.""" + M, num_experts = gating_output.shape + assert num_experts == _GATE_NUM_EXPERTS and 0 < topk <= _MAX_TOPK + assert gating_output.stride(1) == 1 + if partials is not None: + assert ( + partials.shape[1:] == (M, num_experts) and partials.dtype == torch.float32 + ) + assert gating_output.dtype == torch.float32 + split_k = partials.shape[0] + stride_ps, stride_pm = partials.stride(0), partials.stride(1) + else: + partials = gating_output + split_k = 0 + stride_ps = stride_pm = 0 + if correction_bias is not None: + assert ( + correction_bias.shape == (num_experts,) and correction_bias.is_contiguous() + ) + weights = torch.empty((M, topk), dtype=torch.float32, device=gating_output.device) + ids = torch.empty((M, topk), dtype=torch.int32, device=gating_output.device) + if M == 0: + return weights, ids + _router_gate_kernel[(M,)]( + gating_output, + partials, + correction_bias if correction_bias is not None else gating_output, + weights, + ids, + gating_output.stride(0), + stride_ps, + stride_pm, + weights.stride(0), + float(1.0 if routed_scaling_factor is None else routed_scaling_factor), + SPLIT_K=split_k, + WRITE_LOGITS=split_k > 0, + HAS_BIAS=correction_bias is not None, + RENORM=bool(renormalize), + TOPK=topk, + num_warps=1, + ) + return weights, ids diff --git a/python/sglang/test/kernels/deepseek_v4/dsv41_kv_quant_reference.py b/python/sglang/test/kernels/deepseek_v4/dsv41_kv_quant_reference.py new file mode 100644 index 0000000000..be0d173d9b --- /dev/null +++ b/python/sglang/test/kernels/deepseek_v4/dsv41_kv_quant_reference.py @@ -0,0 +1,91 @@ +"""Independent packed-FP4 (V41_FP4 layout) cache oracle for the V4.1 store and reader tests.""" + +from typing import Optional + +import torch + +KV_DIM = 512 +FP4_GROUP = 16 # values per e4m3 scale +E2M1_BYTES_PER_TOKEN = KV_DIM // 2 +E4M3_SCALE_BYTES_PER_TOKEN = KV_DIM // FP4_GROUP +FP4_TOKEN_BYTES = E2M1_BYTES_PER_TOKEN + E4M3_SCALE_BYTES_PER_TOKEN +E2M1_MAX = 6.0 +E4M3_MAX = 448.0 +MIN_SCALE = 2.0**-9 +_E2M1_MAGNITUDES = (0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0) + + +def quantize_to_e2m1_codes(x: torch.Tensor) -> torch.Tensor: + """Round to the nearest e2m1 value with the semantics of + cvt.rn.satfinite.e2m1x2.f32 (ties to even, saturating to +-6) and return + the 4-bit codes as uint8. The sign is kept for values that round to zero + (-0.0 and small negatives give the code 0x8); NaN maps to code 0.""" + x = x.float() + mags = torch.tensor(_E2M1_MAGNITUDES, dtype=torch.float32, device=x.device) + sign = torch.signbit(x).to(torch.uint8) << 3 + a = torch.nan_to_num(x.abs(), nan=0.0, posinf=6.0).clamp_max(6.0) + mids = (mags[:-1] + mags[1:]) / 2 + code = torch.bucketize(a, mids, right=True) + on_tie = (a.unsqueeze(-1) == mids).any(dim=-1) + tie_code = torch.bucketize(a, mids, right=False) + code = torch.where(on_tie, tie_code + (tie_code & 1), code) + return sign | code.to(torch.uint8) + + +def dequantize_e2m1_codes(codes: torch.Tensor) -> torch.Tensor: + mags = torch.tensor(_E2M1_MAGNITUDES, dtype=torch.float32, device=codes.device) + val = mags[(codes & 7).long()] + return torch.where((codes & 8) != 0, -val, val) + + +def quantize_k_cache_v41_fp4( + k: torch.Tensor, page_bytes: Optional[int] = None +) -> torch.Tensor: + """k [num_pages, page_size, 512] -> uint8 [num_pages, page_bytes] + pages of the V41_FP4 layout: 256 B of e2m1 codes per token (even index in + the low nibble), then 32 e4m3 scales per token (one per 16 values), + scale = e4m3(clamp(amax / 6, 2**-9, 448)). A NaN element poisons its + tile: NaN scale, zero codes.""" + num_pages, page_size, d = k.shape + assert d == KV_DIM + x = k.float().view(num_pages, page_size, KV_DIM // FP4_GROUP, FP4_GROUP) + amax = torch.nan_to_num(x.abs(), nan=float("inf")).amax(dim=-1) + scale = torch.clamp(amax / E2M1_MAX, MIN_SCALE, E4M3_MAX).to(torch.float8_e4m3fn) + scale = torch.where(torch.isinf(amax), torch.full_like(scale, float("nan")), scale) + codes = quantize_to_e2m1_codes(x / scale.float().unsqueeze(-1)) + codes = codes.view(num_pages, page_size, KV_DIM) + packed = codes[..., 0::2] | (codes[..., 1::2] << 4) + raw = page_size * FP4_TOKEN_BYTES + if page_bytes is None: + page_bytes = -(-raw // 256) * 256 # pages start on a 256-byte boundary + assert page_bytes >= raw + data_bytes = page_size * E2M1_BYTES_PER_TOKEN + out = torch.zeros((num_pages, page_bytes), dtype=torch.uint8, device=k.device) + out[:, :data_bytes] = packed.reshape(num_pages, data_bytes) + out[:, data_bytes:raw] = scale.view(torch.uint8).reshape( + num_pages, raw - data_bytes + ) + return out + + +def dequantize_k_cache_v41_fp4(pages: torch.Tensor, page_size: int) -> torch.Tensor: + """Inverse of quantize_k_cache_v41_fp4: [num_pages, page_size, 512] + bf16. e2m1 * e4m3 has at most 2 + 4 significant bits, so the product is + exact in bf16, as in the kernel.""" + num_pages = pages.shape[0] + pages = pages.view(torch.uint8) + data_bytes = page_size * E2M1_BYTES_PER_TOKEN + data = pages[:, :data_bytes].reshape(num_pages, page_size, E2M1_BYTES_PER_TOKEN) + scale = pages[:, data_bytes : page_size * FP4_TOKEN_BYTES].reshape( + num_pages, page_size, E4M3_SCALE_BYTES_PER_TOKEN + ) + codes = torch.empty( + (num_pages, page_size, KV_DIM), dtype=torch.uint8, device=pages.device + ) + codes[..., 0::2] = data & 0xF + codes[..., 1::2] = data >> 4 + values = dequantize_e2m1_codes(codes).view( + num_pages, page_size, KV_DIM // FP4_GROUP, FP4_GROUP + ) + out = values * scale.view(torch.float8_e4m3fn).float().unsqueeze(-1) + return out.view(num_pages, page_size, KV_DIM).to(torch.bfloat16) diff --git a/test/registered/kernels/ops/attention/dsv4/test_v41_kv_store.py b/test/registered/kernels/ops/attention/dsv4/test_v41_kv_store.py new file mode 100644 index 0000000000..41d0ae2060 --- /dev/null +++ b/test/registered/kernels/ops/attention/dsv4/test_v41_kv_store.py @@ -0,0 +1,668 @@ +"""Byte-exactness of the V4.1 (fp8 / fp4) FlashMLA KV cache store kernels. + +Every store kernel is compared byte for byte with the pure-torch quantizers of +the two formats (torch_quant.quantize_k_cache_v41 and +dsv41_kv_quant_reference.quantize_k_cache_v41_fp4), which follow the decode +kernel's own reference quantizer. +""" + +import math +import unittest +from typing import Tuple + +import torch + +from sglang.kernels.ops.attention.deepseek_v4_rope import set_batched_rope +from sglang.kernels.ops.attention.dsv4.attn import fused_store_cache +from sglang.kernels.ops.attention.dsv4.compress import ( + CompressorDecodePlan, + compress_norm_rope_store, +) +from sglang.kernels.ops.attention.dsv4.dequant_k_cache import dequantize_k_cache_paged +from sglang.kernels.ops.attention.dsv4.elementwise import ( + fused_k_norm_rope_flashmla, + fused_rope_inplace, +) +from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout +from sglang.kernels.ops.attention.dsv4.torch_quant import ( + dequantize_k_cache_v41, + fake_quant_compressed_kv, + quantize_k_cache_v41, +) +from sglang.srt.utils import is_gfx95_supported, is_hip +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci +from sglang.test.kernels.deepseek_v4 import dsv41_kv_quant_reference as tq +from sglang.test.test_utils import CustomTestCase + +# the V4.1 store kernels are the SM100 / gfx950 JIT kernels; the only Blackwell runner +# config is the four-GPU one +register_cuda_ci(est_time=60, stage="base-b-kernel-unit", runner_config="4-gpu-b200") +# backend-specific: the gfx950 builds of the V4.1 row quantizers, the dequant NaN +# canonicalization and the query rope in the K launch exist only on HIP +register_amd_ci(est_time=60, suite="stage-b-test-1-gpu-small-amd-mi35x") + +REFERENCE = { + KVLayout.V41: quantize_k_cache_v41, + KVLayout.V41_FP4: tq.quantize_k_cache_v41_fp4, +} +DEQUANT = { + KVLayout.V41: dequantize_k_cache_v41, + KVLayout.V41_FP4: tq.dequantize_k_cache_v41_fp4, +} +# One quantization step, relative: e4m3 has 3 mantissa bits, e2m1 one. +ONE_CODE_RTOL = {KVLayout.V41: 0.13, KVLayout.V41_FP4: 0.51} + + +def rope_tail( + x: torch.Tensor, freqs: torch.Tensor, rope_dim: int, inverse: bool = False +) -> torch.Tensor: + """Rotate the last rope_dim features of x [T, ..., D] with complex freqs [T, rope_dim // 2].""" + head, tail = x[..., :-rope_dim], x[..., -rope_dim:] + tc = torch.view_as_complex(tail.float().unflatten(-1, (-1, 2)).contiguous()) + f = freqs.conj() if inverse else freqs + f = f.view(x.shape[0], *([1] * (x.ndim - 2)), rope_dim // 2) + rotated = torch.view_as_real(tc * f).flatten(-2).to(x.dtype) + return torch.cat([head, rotated], dim=-1) + + +def token_rows( + pages: torch.Tensor, layout: KVLayout, page_size: int, locs: torch.Tensor +) -> Tuple[torch.Tensor, torch.Tensor]: + """The (data row, scale row) bytes of the tokens at locs.""" + locs = locs.long() + page, offset = locs // page_size, locs % page_size + data_cols = torch.arange(layout.data_bytes, device=pages.device) + scale_cols = torch.arange(layout.scale_bytes, device=pages.device) + data = pages[page[:, None], offset[:, None] * layout.data_bytes + data_cols] + scale = pages[ + page[:, None], + layout.scale_offset(page_size) + + offset[:, None] * layout.scale_bytes + + scale_cols, + ] + return data, scale + + +def random_rows( + n: int, generator: torch.Generator, device: str = "cuda" +) -> torch.Tensor: + """bf16 rows over a wide dynamic range, with zero, negative-zero and tiny tiles.""" + x = torch.randn(n, 512, generator=generator, device=device, dtype=torch.bfloat16) + scale = torch.exp2( + torch.randint(-10, 6, (n, 1), generator=generator, device=device).float() + ) + x = (x * scale).to(torch.bfloat16) + if n >= 3: + x[0, :32] = 0 + x[1, 32:48] = -0.0 + x[2, 100] = -1e-10 + return x + + +def reference_pages( + layout: KVLayout, + page_size: int, + num_pages: int, + locs: torch.Tensor, + values: torch.Tensor, + page_bytes: int, +) -> torch.Tensor: + full = torch.zeros( + num_pages, page_size, 512, device=values.device, dtype=values.dtype + ) + full.view(-1, 512)[locs.long()] = values + return REFERENCE[layout](full, page_bytes=page_bytes) + + +def _v41_store_kernels_available() -> bool: + if not torch.cuda.is_available(): + return False + if is_hip(): + return is_gfx95_supported() + return torch.cuda.get_device_capability()[0] >= 10 + + +@unittest.skipUnless( + _v41_store_kernels_available(), "the V4.1 store kernels need gfx950 or SM100" +) +class TestV41KVStore(CustomTestCase): + def assert_tokens_equal(self, cache, ref, layout, page_size, locs): + got_data, got_scale = token_rows(cache, layout, page_size, locs) + exp_data, exp_scale = token_rows(ref, layout, page_size, locs) + self.assertTrue(torch.equal(got_scale, exp_scale), "scale rows differ") + self.assertTrue(torch.equal(got_data, exp_data), "data rows differ") + + def assert_rows_close(self, cache, ref, layout, page_size, locs): + """For rows that went through the kernel's fp32 RMSNorm: torch sums the + squares in another order, and the fp32 ulp this can cost is occasionally + kept by a bf16 rounding boundary and then by the quantizer. Allow a + one-code difference in a handful of elements.""" + got_data, got_scale = token_rows(cache, layout, page_size, locs) + exp_data, exp_scale = token_rows(ref, layout, page_size, locs) + self.assertGreater((got_scale == exp_scale).float().mean().item(), 0.999) + self.assertGreater((got_data == exp_data).float().mean().item(), 0.999) + deq_got = DEQUANT[layout](cache, page_size).view(-1, 512)[locs.long()].float() + deq_exp = DEQUANT[layout](ref, page_size).view(-1, 512)[locs.long()].float() + torch.testing.assert_close( + deq_got, deq_exp, rtol=ONE_CODE_RTOL[layout], atol=0.06 + ) + + def assert_untouched_zero(self, cache, layout, page_size, locs): + num_slots = cache.shape[0] * page_size + written = torch.zeros(num_slots, dtype=torch.bool, device=cache.device) + written[locs.long()] = True + others = torch.arange(num_slots, device=cache.device)[~written] + data, scale = token_rows(cache, layout, page_size, others) + self.assertEqual(int(data.sum()) + int(scale.sum()), 0) + + def test_fused_store_cache(self): + """Ragged page fills: a random subset of slots is written, the rest stays zero.""" + + g = torch.Generator(device="cuda").manual_seed(0) + for layout in (KVLayout.V41, KVLayout.V41_FP4): + # A full page and a padded 2-token page, each with one index dtype + # (the layout, the pad and the dtype are the three template axes). + for page_size, num_pages, n, idx_dtype in ( + (64, 9, 333, torch.int32), + (2, 40, 37, torch.int64), + ): + with self.subTest( + layout=layout.name, page_size=page_size, idx=idx_dtype + ): + x = random_rows(n, g) + locs = torch.randperm( + num_pages * page_size, generator=g, device="cuda" + )[:n].to(idx_dtype) + cache = torch.zeros( + num_pages, + layout.page_bytes(page_size), + dtype=torch.uint8, + device="cuda", + ) + fused_store_cache( + x, + cache, + locs, + page_size=page_size, + type="flashmla", + layout=layout, + ) + ref = reference_pages( + layout, page_size, num_pages, locs, x, cache.shape[1] + ) + self.assert_tokens_equal(cache, ref, layout, page_size, locs) + self.assert_untouched_zero(cache, layout, page_size, locs) + + def test_fused_store_cache_with_rope(self): + """The in-kernel RoPE tail equals rope_tail (bf16-rounded) before quantizing, + so the fp4 cache holds exactly fake_quant_compressed_kv(rope_tail(x)).""" + + g = torch.Generator(device="cuda").manual_seed(1) + page_size, num_pages, n = 64, 5, 200 + for layout in (KVLayout.V41, KVLayout.V41_FP4): + with self.subTest(layout=layout.name): + x = random_rows(n, g) + angles = torch.randn(n, 32, generator=g, device="cuda") + freqs = torch.polar(torch.ones_like(angles), angles) + locs = torch.randperm( + num_pages * page_size, generator=g, device="cuda" + )[:n] + cache = torch.zeros( + num_pages, + layout.page_bytes(page_size), + dtype=torch.uint8, + device="cuda", + ) + fused_store_cache( + x, + cache, + locs, + page_size=page_size, + type="flashmla", + layout=layout, + freqs_cis=freqs, + ) + rotated = rope_tail(x, freqs, 64) + ref = reference_pages( + layout, page_size, num_pages, locs, rotated, cache.shape[1] + ) + self.assert_tokens_equal(cache, ref, layout, page_size, locs) + if layout is KVLayout.V41_FP4: + deq = tq.dequantize_k_cache_v41_fp4(cache, page_size).view(-1, 512)[ + locs + ] + self.assertTrue(torch.equal(deq, fake_quant_compressed_kv(rotated))) + + def test_boundary_tiles(self): + """Tie, saturation and subnormal-scale tiles follow the reference. (NaN / inf + rows are not fed: the kernels do not reproduce the reference's handling of them.)""" + + maxima = [ + 0, + 2**-12, + 6 * 2**-9, + 6 * 1.0625, + 6 * 1.1875, + 6 * 448, + 1e6, + 8.25, + 448.0, + 449.0, + ] + rows = [] + for m in maxima: + row = torch.full((512,), m, dtype=torch.bfloat16) + row[1::2] *= -1 + row[64:80] = torch.tensor( + [8.25, 4.125, 3.4375, -4.8125, 0.0, -0.0, 2.5, -3.5] * 2, + dtype=torch.bfloat16, + ) + rows.append(row) + x = torch.stack(rows).cuda() + n = x.shape[0] + page_size = 16 + locs = torch.arange(n, device="cuda", dtype=torch.int32) + for layout in (KVLayout.V41, KVLayout.V41_FP4): + with self.subTest(layout=layout.name): + cache = torch.zeros( + 1, layout.page_bytes(page_size), dtype=torch.uint8, device="cuda" + ) + fused_store_cache( + x, cache, locs, page_size=page_size, type="flashmla", layout=layout + ) + ref = reference_pages(layout, page_size, 1, locs, x, cache.shape[1]) + self.assert_tokens_equal(cache, ref, layout, page_size, locs) + self.assert_untouched_zero(cache, layout, page_size, locs) + + def test_fused_k_norm_rope_store(self): + """The fused RMSNorm + RoPE + store (the SWA write) equals norm -> rope_tail -> + fused_store_cache: exact-norm rows bitwise against the torch quantizer, and + general rows bitwise against the unfused kernel chain.""" + + g = torch.Generator(device="cuda").manual_seed(2) + page_size, num_pages, n = 256, 3, 300 + angles = torch.randn(1024, 32, generator=g, device="cuda") + freqs_table = torch.polar(torch.ones_like(angles), angles) + pos = torch.randint( + 0, 1024, (n,), generator=g, device="cuda", dtype=torch.int64 + ) + locs = torch.randperm(num_pages * page_size, generator=g, device="cuda")[:n].to( + torch.int32 + ) + locs[7] = -1 # a row without a write target is skipped + valid = locs >= 0 + for layout in (KVLayout.V41, KVLayout.V41_FP4): + with self.subTest(layout=layout.name, rows="exact-norm"): + # x = +-2^k per row with eps = 0 normalizes to +-1 exactly, so the + # normed row is sign * weight and the reference is exact. + signs = torch.where( + torch.rand(n, 512, generator=g, device="cuda") < 0.5, -1.0, 1.0 + ) + k = torch.randint(-6, 6, (n, 1), generator=g, device="cuda").float() + x = (signs * torch.exp2(k)).to(torch.bfloat16) + w = ( + torch.randn(512, generator=g, device="cuda") + * torch.exp2( + torch.randint(-6, 4, (512,), generator=g, device="cuda").float() + ) + ).to(torch.bfloat16) + cache = torch.zeros( + num_pages, + layout.page_bytes(page_size), + dtype=torch.uint8, + device="cuda", + ) + fused_k_norm_rope_flashmla( + x, w, 0.0, freqs_table, pos, locs, cache, page_size, layout=layout + ) + rotated = rope_tail( + (signs * w.float()).to(torch.bfloat16), freqs_table[pos], 64 + ) + ref = reference_pages( + layout, + page_size, + num_pages, + locs[valid], + rotated[valid], + cache.shape[1], + ) + self.assert_tokens_equal(cache, ref, layout, page_size, locs[valid]) + self.assert_untouched_zero(cache, layout, page_size, locs[valid]) + with self.subTest(layout=layout.name, rows="general"): + x = (torch.randn(n, 512, generator=g, device="cuda") * 3).to( + torch.bfloat16 + ) + w = (torch.randn(512, generator=g, device="cuda") * 0.3 + 1).to( + torch.bfloat16 + ) + eps = 1e-6 + cache = torch.zeros( + num_pages, + layout.page_bytes(page_size), + dtype=torch.uint8, + device="cuda", + ) + fused_k_norm_rope_flashmla( + x, w, eps, freqs_table, pos, locs, cache, page_size, layout=layout + ) + xf = x.float() + normed = ( + xf + * torch.rsqrt(xf.square().mean(-1, keepdim=True) + eps) + * w.float() + ).to(torch.bfloat16) + rotated = rope_tail(normed, freqs_table[pos], 64) + unfused = torch.zeros_like(cache) + fused_store_cache( + rotated[valid], + unfused, + locs[valid], + page_size=page_size, + type="flashmla", + layout=layout, + ) + self.assert_rows_close(cache, unfused, layout, page_size, locs[valid]) + + def test_compress_norm_rope_store(self): + """The ratio-4 / ratio-128 writer (norm + RoPE + store from a decode plan) in + the V4.1 layouts: bitwise on exact-norm rows, one-code close on general rows.""" + + g = torch.Generator(device="cuda").manual_seed(5) + ratio = 4 + angles = torch.randn(4096, 32, generator=g, device="cuda") + freqs = torch.polar(torch.ones_like(angles), angles) + n, page_size, num_pages = 150, 64, 8 + for layout in (KVLayout.V41, KVLayout.V41_FP4): + seq_lens = ( + torch.randint( + 1, 1000, (n,), generator=g, device="cuda", dtype=torch.int64 + ) + * ratio + ) + seq_lens[3] += 1 # not a group boundary: writes nothing + plan = CompressorDecodePlan.generate_legacy( + ratio, torch.arange(n, device="cuda", dtype=torch.int64), seq_lens + ) + valid = seq_lens % ratio == 0 + pos = seq_lens - ratio + out_loc = torch.randperm(num_pages * page_size, generator=g, device="cuda")[ + :n + ].to(torch.int64) + with self.subTest( + layout=layout.name, page_size=page_size, rows="exact-norm" + ): + signs = torch.where( + torch.rand(n, 512, generator=g, device="cuda") < 0.5, -1.0, 1.0 + ) + k = torch.randint(-6, 6, (n, 1), generator=g, device="cuda").float() + kv = (signs * torch.exp2(k)).to(torch.bfloat16) + w = ( + torch.randn(512, generator=g, device="cuda") + * torch.exp2( + torch.randint(-6, 4, (512,), generator=g, device="cuda").float() + ) + ).to(torch.bfloat16) + cache = torch.zeros( + num_pages, + layout.page_bytes(page_size), + dtype=torch.uint8, + device="cuda", + ) + compress_norm_rope_store( + kv, + plan, + norm_weight=w, + norm_eps=0.0, + freq_cis=freqs, + out_loc=out_loc, + kvcache=cache, + page_size=page_size, + layout=layout, + ) + rotated = rope_tail( + (signs * w.float()).to(torch.bfloat16), freqs[pos], 64 + ) + ref = reference_pages( + layout, + page_size, + num_pages, + out_loc[valid], + rotated[valid], + cache.shape[1], + ) + self.assert_tokens_equal(cache, ref, layout, page_size, out_loc[valid]) + self.assert_untouched_zero(cache, layout, page_size, out_loc[valid]) + with self.subTest(layout=layout.name, page_size=page_size, rows="general"): + eps = 1e-6 + kv = (torch.randn(n, 512, generator=g, device="cuda") * 2).to( + torch.bfloat16 + ) + w = (torch.randn(512, generator=g, device="cuda") * 0.3 + 1).to( + torch.bfloat16 + ) + cache = torch.zeros( + num_pages, + layout.page_bytes(page_size), + dtype=torch.uint8, + device="cuda", + ) + compress_norm_rope_store( + kv, + plan, + norm_weight=w, + norm_eps=eps, + freq_cis=freqs, + out_loc=out_loc, + kvcache=cache, + page_size=page_size, + layout=layout, + ) + xf = kv.float() + normed = ( + xf + * torch.rsqrt(xf.square().mean(-1, keepdim=True) + eps) + * w.float() + ).to(torch.bfloat16) + rotated = rope_tail(normed, freqs[pos], 64) + ref = reference_pages( + layout, + page_size, + num_pages, + out_loc[valid], + rotated[valid], + cache.shape[1], + ) + self.assert_rows_close(cache, ref, layout, page_size, out_loc[valid]) + + +CASES = { + KVLayout.V41: (quantize_k_cache_v41, dequantize_k_cache_v41), + KVLayout.V41_FP4: (tq.quantize_k_cache_v41_fp4, tq.dequantize_k_cache_v41_fp4), +} + + +def bits(t: torch.Tensor) -> torch.Tensor: + """bf16 as int16, so that -0.0 and NaN payloads compare exactly.""" + return t.contiguous().view(torch.int16) + + +@unittest.skipUnless(is_hip() and is_gfx95_supported(), "requires gfx950") +class TestV41KVDequant(CustomTestCase): + def _gather_ref(self, dequant, pages, page_size, ids): + return dequant(pages, page_size).view(-1, 512)[ids.long()].unsqueeze(1) + + def test_quantized_pages(self): + g = torch.Generator(device="cuda").manual_seed(0) + for layout, (quant, dequant) in CASES.items(): + for page_size, num_pages in ((64, 9), (2, 50)): + with self.subTest(layout=layout.name, page_size=page_size): + k = torch.randn( + num_pages, + page_size, + 512, + generator=g, + device="cuda", + dtype=torch.bfloat16, + ) + k = ( + k + * torch.exp2( + torch.randint( + -12, + 6, + (num_pages, page_size, 1), + generator=g, + device="cuda", + ).float() + ) + ).to(torch.bfloat16) + k[0, 0, :32] = 0 + k[0, 0, 32:48] = -0.0 + pages = quant(k, page_bytes=layout.page_bytes(page_size)) + ids = torch.randint( + 0, + num_pages * page_size, + (777,), + generator=g, + device="cuda", + dtype=torch.int32, + ) + got = dequantize_k_cache_paged(pages, ids, page_size, layout=layout) + self.assertTrue( + torch.equal( + bits(got), + bits(self._gather_ref(dequant, pages, page_size, ids)), + ) + ) + # The fp4 cache dequantizes to the model's fake-quantized value + # (compared by value: the fake quant maps an exact -0.0 to +0.0). + if layout is KVLayout.V41_FP4: + expect = fake_quant_compressed_kv( + k.view(-1, 512)[ids.long()] + ).unsqueeze(1) + self.assertTrue(torch.equal(got, expect)) + + def test_random_bytes_and_workspace_slice(self): + """Arbitrary payload bytes (scales in the quantizer's range) and an + out that is a strided slice of a larger workspace.""" + g = torch.Generator(device="cuda").manual_seed(1) + for layout, (_, dequant) in CASES.items(): + page_size, num_pages = 64, 7 + with self.subTest(layout=layout.name): + pages = torch.randint( + 0, + 256, + (num_pages, layout.page_bytes(page_size)), + generator=g, + dtype=torch.uint8, + device="cuda", + ) + if layout is KVLayout.V41: + lo = layout.scale_offset(page_size) + hi = lo + page_size * layout.scale_bytes + pages[:, lo:hi] = torch.randint( + 100, + 140, + (num_pages, hi - lo), + generator=g, + dtype=torch.uint8, + device="cuda", + ) + ids = torch.randint( + 0, + num_pages * page_size, + (300,), + generator=g, + device="cuda", + dtype=torch.int64, + ) + ref = self._gather_ref(dequant, pages, page_size, ids) + workspace = torch.zeros( + 305, 1, 512, dtype=torch.bfloat16, device="cuda" + ) + out = dequantize_k_cache_paged( + pages, ids, page_size, out=workspace[5:], layout=layout + ) + # NaN payloads (fp8 0x7F / e4m3 NaN scales) compare through their bits. + self.assertTrue(torch.equal(bits(workspace[5:]), bits(ref))) + self.assertEqual(int(workspace[:5].abs().sum()), 0) + + +HEAD_DIM, ROPE_DIM = 512, 64 +# the pool under test: one page size for the SWA and the compressed caches +PAGE_SIZE = 256 +FULL_SIZE = 4 * PAGE_SIZE + + +@unittest.skipUnless( + is_hip() and is_gfx95_supported(), + "the query rope rides the HIP K launch; its bitwise parity with the flat rope" + " kernel is claimed on gfx950 only", +) +class TestFusedKNormRopeFlashMLA(CustomTestCase): + def test_query_rope_in_the_k_launch(self): + """With q the K launch must rope every query head's trailing ROPE_DIM bitwise + like the flat rope kernel, leave the cache bytes and the nope part untouched, + and rope rows without a slot.""" + dev = "cuda" + num_tokens, heads, pos_dtype, seed = 300, 16, torch.int32, 2 + for layout in (KVLayout.V4, KVLayout.V41): + with self.subTest(layout=layout): + torch.manual_seed(seed) + kv = torch.randn(num_tokens, HEAD_DIM, device=dev, dtype=torch.bfloat16) + weight = (1 + 0.1 * torch.randn(HEAD_DIM, device=dev)).to( + torch.bfloat16 + ) + angles = torch.rand(8192, ROPE_DIM // 2, device=dev) * 2 * math.pi + freqs_cis = torch.polar(torch.ones_like(angles), angles) + positions = torch.randint(0, 8192, (num_tokens,), device=dev).to( + pos_dtype + ) + out_loc = torch.randperm(FULL_SIZE, device=dev)[:num_tokens] + out_loc = out_loc.to(torch.int32) + out_loc[1] = -1 + page_bytes = layout.page_bytes(PAGE_SIZE) + cache = torch.zeros( + FULL_SIZE // PAGE_SIZE, page_bytes, device=dev, dtype=torch.uint8 + ) + cache_q = cache.clone() + q = (torch.randn(num_tokens, heads, HEAD_DIM, device=dev) * 3).to( + torch.bfloat16 + ) + expected = q.clone() + # The model's standalone query rope (batched flat kernel). + set_batched_rope(True) + self.addCleanup(set_batched_rope, False) + fused_rope_inplace( + expected[..., -ROPE_DIM:], None, freqs_cis, positions + ) + got = q.clone() + fused_k_norm_rope_flashmla( + kv, + weight, + 1e-6, + freqs_cis, + positions, + out_loc, + cache, + PAGE_SIZE, + layout=layout, + ) + fused_k_norm_rope_flashmla( + kv, + weight, + 1e-6, + freqs_cis, + positions, + out_loc, + cache_q, + PAGE_SIZE, + q=got, + layout=layout, + ) + self.assertTrue(torch.equal(got, expected)) + self.assertTrue(torch.equal(cache_q, cache)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/attention/test_fp4_indexer_hip.py b/test/registered/kernels/ops/attention/test_fp4_indexer_hip.py index cf6aacb6ab..4d4559eb78 100644 --- a/test/registered/kernels/ops/attention/test_fp4_indexer_hip.py +++ b/test/registered/kernels/ops/attention/test_fp4_indexer_hip.py @@ -26,6 +26,7 @@ from sglang.kernels.ops.attention.dsv4 import ( compress_norm_rope_store, ) from sglang.kernels.ops.attention.dsv4.compress import CompressorPrefillPlan +from sglang.kernels.ops.attention.dsv4.fp4_indexer import quantize_fp4_indexer_tensor from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import ( FP4KWriteMetadata, _decode_cta_count, @@ -33,14 +34,29 @@ from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import ( aiter_fp4_paged_mqa_logits, aiter_k_indexer_fp4_cache_write, aiter_q_indexer_fp4, + index_q_rope_pack_weights_flydsl, + indexer_head_weights, + pack_fp4_query_flydsl, prepare_fp4_decode_workspace, prepare_fp4_k_write_metadata, prepare_fp4_prefill_workspace, + read_fp4_index_k_split, + store_fp4_index_k_cache_split, ) +from sglang.kernels.ops.attention.dsv4.fp4_indexer_rope_hip import ( + index_k_norm_rope_pack_store_split, +) +from sglang.kernels.ops.attention.dsv4.fp4_indexer_schedule_hip import ( + build_decode_schedule, +) +from sglang.kernels.ops.attention.dsv4.fp4_rope_fake_quant import ( + rope_tail_fake_quant_fp4, +) +from sglang.kernels.ops.gemm.router_gemv_hip import rocm_router_gemv_split_k from sglang.srt.utils import get_device, is_gfx95_supported, is_hip from sglang.test.ci.ci_register import register_amd_ci -register_amd_ci(est_time=120, suite="stage-b-test-1-gpu-small-amd-mi35x") +register_amd_ci(est_time=60, suite="stage-b-test-1-gpu-small-amd-mi35x") pytestmark = pytest.mark.skipif( not (is_hip() and is_gfx95_supported()), @@ -280,6 +296,42 @@ def test_quantize_fp4_indexer_tensor(num_tokens: int) -> None: torch.testing.assert_close(_canonical_zero(stored_fp4), _canonical_zero(ref_fp4)) +def test_index_q_pack_weights_matches_standalone() -> None: + """The one-launch index-Q path (RoPE, two-stage fp4 pack in the FlyDSL layout, head + weights) is bitwise the three standalone launches it replaces.""" + + torch.manual_seed(0) + num_tokens, num_heads = 16, 32 + rope_dim, hidden, max_pos = 64, 5120, 4096 + q = (torch.randn(num_tokens, num_heads * 128, device="cuda") * 3).bfloat16() + freqs = precompute_freqs_cis(rope_dim, max_pos, 0, 10000, 1, 32, 1).to("cuda") + pos = torch.randint(0, max_pos, (num_tokens,), device="cuda", dtype=torch.int64) + x = torch.randn(num_tokens, hidden, device="cuda").bfloat16() + w = (torch.randn(num_heads, hidden, device="cuda") * 0.02).bfloat16() + scale = 128**-0.5 * num_heads**-0.5 + + ref_q = rope_tail_fake_quant_fp4( + q.view(num_tokens, num_heads, 128), freqs[pos], rope_dim + ) + ref_fp4, ref_scale = pack_fp4_query_flydsl(ref_q) + ref_w = indexer_head_weights(x, w, scale) + + partials = rocm_router_gemv_split_k(x, w) + q_fp4, q_scale, weights = index_q_rope_pack_weights_flydsl( + q, freqs, pos, rope_dim, partials, scale, num_heads=num_heads + ) + # torch.equal ignores dtype + assert q_fp4.dtype == ref_fp4.dtype and q_scale.dtype == ref_scale.dtype + assert torch.equal(q_fp4, ref_fp4) + assert torch.equal(q_scale, ref_scale) + assert torch.equal(weights, ref_w) + exact_weights = (x.double() @ w.double().T * scale).bfloat16() + ulps = ( + weights.view(torch.int16).int() - exact_weights.view(torch.int16).int() + ).abs() + assert int(ulps.max()) <= 1 + + @pytest.mark.parametrize("num_tokens", [1, 16, 96]) def test_fp4_index_cache_store_layout(num_tokens: int) -> None: """Scattered slots land in the paged layout and touch nothing else.""" @@ -445,6 +497,36 @@ def test_decode_cta_count_stays_within_available_chunks( assert cta_count <= max(1024, num_queries * 4) +@pytest.mark.parametrize("num_queries", [1, 1536]) +def test_decode_schedule_matches_aiter_varctx_schedule(num_queries: int) -> None: + """The in-tree decode schedule writes AITER compute_varctx_schedule's cta_info rows and + split factor, including zero-length rows and prefix sums carried across row blocks.""" + from aiter.ops.flydsl.kernels.mqa_logits.pa_mqa_logits_fp4 import ( + compute_varctx_schedule, + ) + + torch.manual_seed(num_queries) + max_seq_len = 65536 + c4_seq_lens = torch.randint( + 0, max_seq_len, (num_queries,), device=get_device(), dtype=torch.int32 + ) + c4_seq_lens[::7] = 0 + cta_count = _decode_cta_count(num_queries, max_seq_len) + ref_safe, ref, _ = compute_varctx_schedule( + c4_seq_lens, + block_k=256, + parallel_unit_num=cta_count, + max_seq_len=max_seq_len, + next_n=1, + ) + cta_info = torch.empty_like(ref) + scratch = build_decode_schedule( + c4_seq_lens, cta_info_out=cta_info, max_seq_len=max_seq_len + ) + assert torch.equal(scratch[:1], ref_safe) + assert torch.equal(cta_info, ref) + + def _decode_plan(seq_lens: torch.Tensor, compress_ratio: int) -> CompressorDecodePlan: """Hand-build the 16-byte decode plan rows the metadata builder reads.""" words = torch.zeros((seq_lens.shape[0], 4), dtype=torch.int32, device=get_device()) @@ -825,5 +907,153 @@ def test_row_chunks_reproduce_the_unsplit_batch() -> None: ) +def pack_fp4_query_flydsl_torch(q: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """The three-launch form of pack_fp4_query_flydsl: the shared quantizer, then + zeros and a permuted copy into the scale layout.""" + num_tokens, heads = q.shape[0], q.shape[1] + assert heads % 16 == 0 and heads <= 64, heads + q_fp4, q_sf = quantize_fp4_indexer_tensor(q.flatten(0, 1), rne=True) + q_fp4 = q_fp4.view(num_tokens, heads, 64) + sf_bytes = q_sf.view(torch.uint8).view(num_tokens, heads // 16, 16, 4) + q_scale = torch.zeros((num_tokens, 1, 4, 16, 4), dtype=torch.uint8, device=q.device) + q_scale[:, 0, :, :, : heads // 16] = sf_bytes.permute(0, 3, 2, 1) + return q_fp4, q_scale + + +def test_pack_fp4_query_flydsl_single_launch(): + torch.manual_seed(17) + heads, dtype = 32, torch.bfloat16 + q = torch.randn(40, heads, 128, device=get_device(), dtype=dtype) * 4 + # exact fp4 grid points and tie values, zeros and a huge group + q[0, 0, :32] = torch.tensor( + [0.0, 0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5.0] * 4, + device=get_device(), + dtype=dtype, + ) + q[0, 0, 32:64] = 0.0 + q[0, 0, 64:96] = 3.0e4 + ref_fp4, ref_scale = pack_fp4_query_flydsl_torch(q) + fp4, scale = pack_fp4_query_flydsl(q) + assert fp4.dtype is ref_fp4.dtype and scale.dtype is ref_scale.dtype + assert torch.equal(fp4, ref_fp4) + assert torch.equal(scale, ref_scale) + empty = torch.empty(0, heads, 128, device=get_device(), dtype=dtype) + fp4, scale = pack_fp4_query_flydsl(empty) + assert fp4.shape == (0, heads, 64) and scale.shape == (0, 1, 4, 16, 4) + + +@pytest.mark.parametrize("compressed_kv", [False, True], ids=["False", "True"]) +def test_rope_fake_quant_gathers_freqs_by_position(compressed_kv: bool): + + torch.manual_seed(19) + table = torch.polar( + torch.ones(4096, 32, device=get_device()), + torch.rand(4096, 32, device=get_device()) * 6.283, + ) + x = torch.randn(17, 32, 128, device=get_device(), dtype=torch.bfloat16) * 3 + for pos_dtype in (torch.int64, torch.int32): + pos = torch.randint(0, 4096, (17,), device=get_device(), dtype=pos_dtype) + ref = rope_tail_fake_quant_fp4(x, table[pos], 64, compressed_kv=compressed_kv) + out = rope_tail_fake_quant_fp4( + x, table, 64, compressed_kv=compressed_kv, positions=pos + ) + assert torch.equal(out, ref) + + +@pytest.mark.parametrize("ratio", [1, 4]) +def test_index_k_split_writer_matches_the_triton_chain(ratio: int) -> None: + """index_k_norm_rope_pack_store_split writes the bytes of RMSNorm, then the Triton + rope_tail_fake_quant_fp4 and store_fp4_index_k_cache_split (rne); slot 0 stays empty.""" + + torch.manual_seed(ratio) + num_tokens, page_size, num_pages, max_pos = 100, 64, 4, 4096 + x = (torch.randn(num_tokens, 128, device="cuda") * 3).bfloat16() + x[3] = 0 + norm_weight = (1 + 0.1 * torch.randn(128, device="cuda")).bfloat16() + freqs_cis = precompute_freqs_cis(64, max_pos, 0, 10000, 1, 32, 1).to("cuda") + freqs = torch.view_as_real(freqs_cis).flatten(-2) + positions = torch.randint(0, max_pos, (num_tokens,), device="cuda") + loc = torch.randperm(num_pages * page_size - 1, device="cuda")[:num_tokens] + 1 + loc[:3] = 0 + + payload = torch.zeros( + num_pages, 1, 4, page_size, 16, dtype=torch.uint8, device="cuda" + ) + scale = torch.zeros(num_pages, 1, 4, page_size, dtype=torch.uint8, device="cuda") + index_k_norm_rope_pack_store_split( + x, norm_weight, 1e-6, freqs, positions, loc, payload, scale, ratio=ratio + ) + + xf = x.float() + normed = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + 1e-6) + normed = (normed * norm_weight.float()).bfloat16() + # a compressed row takes the rope of its group's first position + roped = rope_tail_fake_quant_fp4(normed, freqs_cis[positions & ~(ratio - 1)], 64) + ref_payload, ref_scale = torch.zeros_like(payload), torch.zeros_like(scale) + store_fp4_index_k_cache_split( + roped[3:], ref_payload, ref_scale, loc[3:], page_size=page_size, rne=True + ) + assert torch.equal(payload, ref_payload) + assert torch.equal(scale, ref_scale) + + +E2M1 = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]) +TIES = [0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5.0] +RNE = [0.0, 1.0, 1.0, 2.0, 2.0, 4.0, 4.0] + + +def _tie_row(scale: float, values) -> torch.Tensor: + """[128]: four 32-blocks of [6, +values, -values, 0...] times a power-of-two scale.""" + block = [6.0] + values + [-v for v in values] + block += [0.0] * (32 - len(block)) + return (torch.tensor(block) * scale).repeat(4) + + +def _unpack(packed: torch.Tensor) -> torch.Tensor: + """Packed e2m1 nibbles [..., 64] (low nibble first) -> values [..., 128].""" + p = packed.view(torch.uint8).to(torch.int64).cpu() + codes = torch.stack([p & 0xF, p >> 4], dim=-1).flatten(-2) + mag = E2M1[codes & 7] + return torch.where((codes & 8) != 0, -mag, mag) + + +def test_low_ratio_triton_paths_round_half_to_even_like_cuda() -> None: + """Same Triton quantizer as CUDA (rne=True): ties to even on both.""" + + scale = 2.0**-3 + e8m0 = 127 + int(torch.log2(torch.tensor(scale))) + expected = _tie_row(scale, RNE) + + def check(name, got): + assert torch.equal(got.float().cpu(), expected), f"{name}: {got[1:8].tolist()}" + + row = _tie_row(scale, TIES).cuda().to(torch.bfloat16) + assert torch.equal(row.float().cpu(), _tie_row(scale, TIES)) + + fp4, sf = quantize_fp4_indexer_tensor(row.view(1, 128), rne=True) + assert (sf.cpu() & 0xFF).item() == e8m0 + check("quantize_fp4_indexer_tensor(rne=True)", _unpack(fp4)[0] * scale) + + q_fp4, q_scale = pack_fp4_query_flydsl( + row.view(1, 1, 128).expand(1, 16, 128).contiguous() + ) + assert q_scale.unique().tolist() == [0, e8m0] + check("pack_fp4_query_flydsl", _unpack(q_fp4)[0, 0] * scale) + + payload = torch.zeros((1, 1, 4, 64, 16), dtype=torch.uint8, device="cuda") + k_scale = torch.zeros((1, 1, 4, 64), dtype=torch.uint8, device="cuda") + loc = torch.tensor([5], dtype=torch.int64, device="cuda") + store_fp4_index_k_cache_split( + row.view(1, 128), payload, k_scale, loc, page_size=64, rne=True + ) + k_fp4, k_sf = read_fp4_index_k_split(payload, k_scale, loc, page_size=64) + assert (k_sf.cpu() & 0xFF).item() == e8m0 + check("store_fp4_index_k_cache_split", _unpack(k_fp4)[0] * scale) + + freqs = torch.ones(1, 32, dtype=torch.complex64, device="cuda") + fq = rope_tail_fake_quant_fp4(row.view(1, 128), freqs, 64) + check("rope_tail_fake_quant_fp4", fq[0]) + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/kernels/ops/gemm/test_router_gemv_hip.py b/test/registered/kernels/ops/gemm/test_router_gemv_hip.py new file mode 100644 index 0000000000..0ab5b47d8f --- /dev/null +++ b/test/registered/kernels/ops/gemm/test_router_gemv_hip.py @@ -0,0 +1,59 @@ +"""The ROCm bf16 x bf16 -> fp32 GEMMs of DeepSeek-V4 keep fp32 accuracy.""" + +import unittest + +import torch + +from sglang.kernels.ops.gemm.bf16_fp32 import linear_bf16_fp32 +from sglang.kernels.ops.gemm.router_gemv_hip import ( + ROCM_ROUTER_MAX_TOKENS, + rocm_router_gemv_split_k, + rocm_router_reduce_partials, +) +from sglang.srt.utils import is_gfx95_supported, is_hip +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import CustomTestCase + +register_amd_ci(est_time=5, stage="stage-b", runner_config="1-gpu-small-amd-mi35x") + +NUM_EXPERTS, HIDDEN = 384, 5120 + + +@unittest.skipUnless(is_hip() and is_gfx95_supported(), "ROCm gfx95 only") +class TestRouterGemvHip(CustomTestCase): + def test_gemv_accuracy_and_batch_invariance(self): + """The split-K GEMV is within fp32 rounding of fp64, and every M runs the same + 16-row tile, so a row's result does not depend on the batch.""" + gen = torch.Generator(device="cuda").manual_seed(0) + weight = ( + torch.randn(NUM_EXPERTS, HIDDEN, device="cuda", generator=gen) * 0.02 + ).to(torch.bfloat16) + x = torch.randn( + ROCM_ROUTER_MAX_TOKENS, HIDDEN, device="cuda", generator=gen + ).to(torch.bfloat16) + ref = (x.double() @ weight.double().T).float() + full = torch.empty(ROCM_ROUTER_MAX_TOKENS, NUM_EXPERTS, device="cuda") + rocm_router_reduce_partials(rocm_router_gemv_split_k(x, weight), full) + self.assertTrue(torch.allclose(full, ref, atol=2e-3, rtol=1e-4)) + part = torch.empty(17, NUM_EXPERTS, device="cuda") + rocm_router_reduce_partials(rocm_router_gemv_split_k(x[:17], weight), part) + self.assertTrue(torch.equal(part, full[:17])) + + def test_linear_bf16_fp32_is_not_rounded_to_bf16(self): + """linear_bf16_fp32 feeds the V4 compressor and router logits in fp32; a route + that rounds its output to bf16 on ROCm lands far outside fp32 accumulation error.""" + torch.manual_seed(0) + x = torch.randn(37, HIDDEN, device="cuda", dtype=torch.bfloat16) + w = (torch.randn(512, HIDDEN, device="cuda") * 0.02).to(torch.bfloat16) + out = linear_bf16_fp32(x, w) + self.assertEqual(out.dtype, torch.float32) + # bf16 x bf16 products are exact in fp64, so this is the reference up to fp32 order + ref = x.double() @ w.double().t() + err = (out.double() - ref).abs().max() + bf16_err = (out.bfloat16().double() - ref).abs().max() + self.assertLess(err, bf16_err) + torch.testing.assert_close(out.double(), ref, rtol=1e-4, atol=1e-4) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/moe/test_rocm_router_gate.py b/test/registered/kernels/ops/moe/test_rocm_router_gate.py new file mode 100644 index 0000000000..a252b02dfa --- /dev/null +++ b/test/registered/kernels/ops/moe/test_rocm_router_gate.py @@ -0,0 +1,103 @@ +"""The ROCm decode router gate must reproduce aiter's topk_gating bit for bit.""" + +import unittest + +import torch + +from sglang.kernels.ops.gemm.router_gemv_hip import ( + ROCM_ROUTER_MAX_TOKENS, + rocm_router_gemv_split_k, + rocm_router_reduce_partials, +) +from sglang.kernels.ops.moe.rocm_router_gate import rocm_router_gate +from sglang.srt.utils import is_gfx95_supported, is_hip +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import CustomTestCase + +register_amd_ci(est_time=8, stage="stage-b", runner_config="1-gpu-small-amd-mi35x") + + +try: + from aiter import topk_gating as aiter_topk_gating +except ImportError: # aiter is absent off ROCm + aiter_topk_gating = None + + +NUM_EXPERTS = 384 +HIDDEN = 5120 +TOPK = 6 +ROUTED_SCALING = 1.5 + + +def _aiter_gate(logits, bias, topk, renorm, rsf): + weights = torch.empty( + logits.shape[0], topk, dtype=torch.float32, device=logits.device + ) + ids = torch.empty(logits.shape[0], topk, dtype=torch.int32, device=logits.device) + aiter_topk_gating( + weights, ids, logits, bias, renorm, rsf, score_func="sqrtsoftplus" + ) + return weights, ids + + +@unittest.skipUnless( + is_hip() and is_gfx95_supported() and aiter_topk_gating is not None, + "ROCm gfx95 + aiter only", +) +class TestRocmRouterGate(CustomTestCase): + def setUp(self): + self.device = torch.device("cuda") + self.gen = torch.Generator(device=self.device).manual_seed(0) + self.bias_bf16 = ( + torch.randn(NUM_EXPERTS, device=self.device, generator=self.gen) * 0.5 + ).to(torch.bfloat16) + + def _randn(self, *shape, scale=1.0): + return torch.randn(*shape, device=self.device, generator=self.gen) * scale + + def _assert_same_gate(self, logits, bias, renorm=True, rsf=ROUTED_SCALING, msg=""): + ref_w, ref_i = _aiter_gate(logits, bias, TOPK, renorm, rsf) + out_w, out_i = rocm_router_gate(logits, bias, TOPK, renorm, rsf) + self.assertTrue(torch.equal(ref_i, out_i), f"ids {msg}") + self.assertTrue(torch.equal(ref_w, out_w), f"weights {msg}") + + def test_gate_matches_aiter_on_ties_and_non_finite_logits(self): + """Tie order and the NaN / +-inf handling are aiter topk_gating's, bit for bit.""" + zero_bias = torch.zeros(NUM_EXPERTS, device=self.device, dtype=torch.bfloat16) + levels = torch.randint( + 0, 8, (512, NUM_EXPERTS), device=self.device, generator=self.gen + ).float() + self._assert_same_gate(levels - 3, self.bias_bf16, msg="8 levels") + self._assert_same_gate( + torch.zeros(16, NUM_EXPERTS, device=self.device), zero_bias, msg="all equal" + ) + for bad in (float("-inf"), float("nan"), float("inf")): + logits = self._randn(16, NUM_EXPERTS, scale=3.0) + logits[:, ::7] = bad + self._assert_same_gate(logits, self.bias_bf16, msg=str(bad)) + + def test_fused_split_k_gate_matches_reduce_then_gate(self): + """The decode router passes the GEMV partials straight to the gate: its in-launch + reduce must write the same logits as rocm_router_reduce_partials, and gate them + to the same weights and ids.""" + weight = (self._randn(NUM_EXPERTS, HIDDEN) * 0.02).to(torch.bfloat16) + x = self._randn(ROCM_ROUTER_MAX_TOKENS, HIDDEN).to(torch.bfloat16) + partials = rocm_router_gemv_split_k(x, weight) + ref_logits = torch.empty( + ROCM_ROUTER_MAX_TOKENS, NUM_EXPERTS, device=self.device + ) + rocm_router_reduce_partials(partials, ref_logits) + ref_w, ref_i = rocm_router_gate( + ref_logits, self.bias_bf16, TOPK, True, ROUTED_SCALING + ) + logits = torch.full_like(ref_logits, float("nan")) + out_w, out_i = rocm_router_gate( + logits, self.bias_bf16, TOPK, True, ROUTED_SCALING, partials=partials + ) + self.assertTrue(torch.equal(logits, ref_logits)) + self.assertTrue(torch.equal(out_i, ref_i)) + self.assertTrue(torch.equal(out_w, ref_w)) + + +if __name__ == "__main__": + unittest.main()