[DSv4.1] Fused c1/c2 compress for eager extend, faster c2 decode (#41660)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: BBuf <1182563586@qq.com>
(cherry picked from commit 5916999afb)
This commit is contained in:
DarkSharpness
2026-10-08 15:32:48 +08:00
committed by minke.yu
parent 62f1ed099e
commit 4792b61332
7 changed files with 642 additions and 74 deletions
+172 -38
View File
@@ -18,6 +18,8 @@
namespace sglang {
using device::math::fast_mod_div_u32_t;
/// \brief Ratio-2 decode compressor: pair-pool, RMSNorm and the main-KV write.
///
/// `kv_input` and `kv_state` rows are `2 * kHeadDim` floats, kv then score.
@@ -34,8 +36,7 @@ struct Compress2DecodeParams {
const int64_t* __restrict__ req; // [num_tokens], req_pool_idx per token
const void* __restrict__ raw_out_loc; // [num_tokens] LocT, the FULL slot; 0 marks a padded row
uint8_t* __restrict__ kvcache; // [npages, kPageBytes] uint8
/// Positions per request slot in the pair-state ring.
uint32_t ring_size;
fast_mod_div_u32_t ring_size; // Positions per request slot in the pair-state ring.
float eps;
};
@@ -43,21 +44,32 @@ struct Compress2DecodeParams {
/// At (512, 64) it also keeps the nope/rope split warp-aligned, as the fp8 amax reduction requires.
constexpr uint32_t kC2VecSize = 2;
/// \brief grid = num_tokens, block = kHeadDim / kC2VecSize.
/// \brief Where a completing row finds its partner, and who parks a pending row.
enum class C2Mode {
/// One row per request: the partner comes from the ring, a pending row parks there.
Decode,
/// `draft_len` consecutive positions per request: `blockIdx.x` is the position inside the
/// block, `blockIdx.y` the request, and every row but the first takes its partner from the
/// previous `kv_input` row instead of the ring.
Verify,
/// Arbitrary rows, paired as `_low_ratio_pair_partners` pairs them: the partner is the previous
/// row when it is the same request's previous position, else the ring. A pending row does
/// nothing here; `c2_prefill_store_kernel` parks the ring rows after this kernel read them.
Prefill,
};
/// \brief grid = num_tokens (Decode, Prefill) or `draft_len x batch` (Verify),
/// block = kHeadDim / kC2VecSize.
///
/// An odd position completes a group with its even predecessor; an even one parks in the state.
///
/// Under target-verify, `draft_len` consecutive positions per request: `blockIdx.x` is the
/// position inside the block, `blockIdx.y` the request, and every row but the first takes its
/// partner from the previous `kv_input` row instead of the ring.
///
/// The three reductions have different widths and are not interchangeable: the RMSNorm
/// statistic spans the row, an fp8 store scale 64 elements, an fp4 block 16.
///
/// kLayout is the cache's page format: V4 and V41 store the fake-quantized value; V41_FP4
/// stores the e2m1 codes and their e4m3 scales directly, so the fp4 rounding happens once.
template <
bool kVerify,
C2Mode kMode,
int64_t kHeadDim,
int64_t kRopeDim,
int32_t kPageBits,
@@ -91,16 +103,21 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
const auto tx = threadIdx.x;
// Verify gives each request a CTA column; decode a flat grid of one row each.
const auto row = kVerify ? blockIdx.y * gridDim.x + blockIdx.x : blockIdx.x;
const auto row = kMode == C2Mode::Verify ? blockIdx.y * gridDim.x + blockIdx.x : blockIdx.x;
PDLWaitPrimary<kUsePDL>();
// Slots fit in int32 whatever width the scheduler hands them in.
const auto raw_out_loc = static_cast<int32_t>(static_cast<const LocT*>(params.raw_out_loc)[row]);
// CUDA graph padding is a completely inert row: do not read schedule, input,
// state, or RoPE data, and do not publish output or update the cache.
if (raw_out_loc == 0) return PDLTriggerSecondary<kUsePDL>();
const auto pos = static_cast<const PosT*>(params.positions)[row];
// A completing row reads the slot left by `pos - 1`;
// a pending row writes its own slot, so reads and writes stay disjoint.
const auto* loc_ptr = static_cast<const LocT*>(params.raw_out_loc);
const auto* pos_ptr = static_cast<const PosT*>(params.positions);
const auto raw_out_loc = loc_ptr[row];
// Prefill rows are many, so a CUDA graph padding row exits before any load.
// Decode and verify rows are few: every row issues its loads to keep more in flight.
if constexpr (kMode == C2Mode::Prefill) {
if (raw_out_loc == 0) return PDLTriggerSecondary<kUsePDL>();
}
const auto pos = pos_ptr[row];
// Prefill leaves a pending row's latent unwritten; its index-K slot is -1, so nothing reads it.
if constexpr (kMode == C2Mode::Prefill) {
if ((pos & 1) == 0) return PDLTriggerSecondary<kUsePDL>();
}
const auto rid = params.req[row];
fp32_vec_t kv_new, score_new;
@@ -108,21 +125,32 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
score_new.load(params.kv_input + row * kStride, tx + kCTASize);
const auto ring = static_cast<int64_t>(rid) * params.ring_size;
const auto read_row = ring + (pos - 1 + params.ring_size) % params.ring_size;
const auto write_row = ring + pos % params.ring_size;
const auto write_offset = static_cast<uint32_t>(pos) % params.ring_size;
const auto read_offset = write_offset == 0 ? params.ring_size - 1 : write_offset - 1;
const auto write_row = ring + write_offset;
const auto read_row = ring + read_offset;
fp32_vec_t kv_old, score_old;
const float* partner = params.kv_state + read_row * kStride;
if constexpr (kVerify) {
if constexpr (kMode == C2Mode::Verify) {
if (blockIdx.x != 0) partner = params.kv_input + static_cast<int64_t>(row - 1) * kStride;
}
if constexpr (kMode == C2Mode::Prefill) {
if (row > 0 && params.req[row - 1] == rid && pos_ptr[row - 1] == pos - 1 && loc_ptr[row - 1] != 0) {
partner = params.kv_input + static_cast<int64_t>(row - 1) * kStride;
}
}
kv_old.load(partner, tx);
score_old.load(partner, tx + kCTASize);
if ((pos & 1) == 0) {
kv_new.store(params.kv_state + write_row * kStride, tx);
score_new.store(params.kv_state + write_row * kStride, tx + kCTASize);
return PDLTriggerSecondary<kUsePDL>();
if constexpr (kMode != C2Mode::Prefill) {
// CUDA graph padding case
if (raw_out_loc == 0) return PDLTriggerSecondary<kUsePDL>();
if ((pos & 1) == 0) {
kv_new.store(params.kv_state + write_row * kStride, tx);
score_new.store(params.kv_state + write_row * kStride, tx + kCTASize);
return PDLTriggerSecondary<kUsePDL>();
}
}
constexpr uint32_t kNumWarps = kCTASize / kWarpThreads;
@@ -203,9 +231,9 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
}
}
const int32_t out_loc = static_cast<uint32_t>(raw_out_loc) >> 1;
if constexpr (kLayout == KVLayout::V41_FP4) {
// The fp4 cache takes the rotated bf16 value as is: its row quantizer is the fake quant, minus the dequant.
const int32_t out_loc = raw_out_loc >> 1;
const auto kv_row = Paged::row(params.kvcache, out_loc);
return deepseek_v4::v41::store_row<kLayout>(kv_row.data, kv_row.scale, tx, staged);
}
@@ -228,7 +256,6 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
}
// `raw_out_loc / ratio`; ratio 2 makes it a shift.
const int32_t out_loc = raw_out_loc >> 1;
const auto kv_row = Paged::row(params.kvcache, out_loc);
if constexpr (kLayout == KVLayout::V41) {
@@ -264,21 +291,63 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
}
}
/// \brief Prefill ring write-back, launched after `flash_c2_decode_kernel<C2Mode::Prefill>`.
///
/// Each request's last `ring_size` rows park their `| kv | score |` at `req * ring_size +
/// pos % ring_size` for a later step's first partner. It cannot run inside the pooling kernel:
/// a chunk whose first row is odd and whose last row is even reads and writes the same ring
/// slot. grid = `ring_size x num_reqs`, CTA (k, i) parks the k-th row from the end of request i,
/// one float4 per thread; those rows are consecutive positions, so their slots are distinct.
template <int64_t kHeadDim, typename PosT, typename LocT, bool kUsePDL>
__global__ __launch_bounds__(kHeadDim / 2) void c2_prefill_store_kernel(
const Compress2DecodeParams params,
const int32_t* __restrict__ extend_start_loc,
const int32_t* __restrict__ extend_seq_lens) {
using namespace device;
constexpr int64_t kStride = kHeadDim * 2;
using vec_t = AlignedVector<float, 4>;
static_assert(kStride == 4 * (kHeadDim / 2));
// The pooling kernel reads the ring rows this one overwrites.
PDLWaitPrimary<kUsePDL>();
const auto bx = static_cast<int32_t>(blockIdx.x);
const auto len = extend_seq_lens[blockIdx.y];
const auto start_loc = extend_start_loc[blockIdx.y];
if (bx >= len) return;
const auto row = start_loc + (len - 1 - bx);
const auto rid = params.req[row];
const auto out = static_cast<const LocT*>(params.raw_out_loc)[row];
const auto pos = static_cast<uint32_t>(static_cast<const PosT*>(params.positions)[row]);
const auto dst_row = rid * params.ring_size + pos % params.ring_size;
if (out == 0) return;
vec_t buf;
buf.load(params.kv_input + row * kStride, threadIdx.x);
PDLTriggerSecondary<kUsePDL>();
buf.store(params.kv_state + dst_row * kStride, threadIdx.x);
}
template <int64_t kHeadDim, int64_t kRopeDim, uint32_t kPageSize, deepseek_v4::KVLayout kLayout, bool kUsePDL>
struct FlashCompress2Kernel {
static constexpr uint32_t kBlockSize = kHeadDim / kC2VecSize;
static constexpr int32_t kPageBits = std::bit_width(kPageSize) - 1;
static constexpr int64_t kPageBytes = deepseek_v4::kv_page_bytes<kLayout>(kPageSize);
static_assert(kLayout != deepseek_v4::KVLayout::V4 || kPageBytes == host::div_ceil(584ll * kPageSize, 576) * 576);
template <bool kVerify, typename PosT, typename LocT>
template <C2Mode kMode, typename PosT, typename LocT>
static constexpr auto kernel =
flash_c2_decode_kernel<kVerify, kHeadDim, kRopeDim, kPageBits, PosT, LocT, kLayout, kUsePDL>;
flash_c2_decode_kernel<kMode, kHeadDim, kRopeDim, kPageBits, PosT, LocT, kLayout, kUsePDL>;
template <typename PosT, typename LocT>
static constexpr auto store_kernel = c2_prefill_store_kernel<kHeadDim, PosT, LocT, kUsePDL>;
/// \brief The (`positions`, `raw_out_loc`) dtype pair, resolved at run time.
template <bool kVerify>
template <C2Mode kMode>
static auto select(const bool pos_i32, const bool loc_i32) {
if (pos_i32) return loc_i32 ? kernel<kVerify, int32_t, int32_t> : kernel<kVerify, int32_t, int64_t>;
return loc_i32 ? kernel<kVerify, int64_t, int32_t> : kernel<kVerify, int64_t, int64_t>;
if (pos_i32) return loc_i32 ? kernel<kMode, int32_t, int32_t> : kernel<kMode, int32_t, int64_t>;
return loc_i32 ? kernel<kMode, int64_t, int32_t> : kernel<kMode, int64_t, int64_t>;
}
static auto select_store(const bool pos_i32, const bool loc_i32) {
if (pos_i32) return loc_i32 ? store_kernel<int32_t, int32_t> : store_kernel<int32_t, int64_t>;
return loc_i32 ? store_kernel<int64_t, int32_t> : store_kernel<int64_t, int64_t>;
}
// The sum of squares reduces through a fixed-size shared array, so the CTA must be whole warps.
@@ -302,7 +371,7 @@ struct FlashCompress2Kernel {
const tvm::ffi::TensorView kvcache,
const int64_t ring_size,
const int64_t draft_len) {
launch(
launch<false>(
kv_input,
kv_state,
kv_output,
@@ -317,7 +386,62 @@ struct FlashCompress2Kernel {
draft_len);
}
/// \brief The extend-batch pooling kernel, then the ring write-back.
///
/// Rows may mix requests, prefix offsets and padding; a pending (even) row's `kv_output`
/// row is left unwritten. `extend_start_loc` / `extend_seq_lens` are int32 `[num_reqs]`:
/// request i owns rows `[start[i], start[i] + len[i])`; zero-length requests are skipped.
static void run_prefill_fusion(
const tvm::ffi::TensorView kv_input,
const tvm::ffi::TensorView kv_state,
const tvm::ffi::TensorView kv_output,
const tvm::ffi::TensorView norm_weight,
const tvm::ffi::TensorView positions,
const tvm::ffi::TensorView req,
const tvm::ffi::TensorView raw_out_loc,
const float eps,
const tvm::ffi::TensorView freqs_cis,
const tvm::ffi::TensorView kvcache,
const int64_t ring_size,
const tvm::ffi::TensorView extend_start_loc,
const tvm::ffi::TensorView extend_seq_lens) {
using namespace host;
auto B = SymbolicSize{"num_reqs"};
auto device_ = SymbolicDevice{};
device_.set_options<kDLGPU>();
TensorMatcher({B}).with_dtype<int32_t>().with_device(device_).verify(extend_start_loc);
TensorMatcher({B}).with_dtype<int32_t>().with_device(device_).verify(extend_seq_lens);
const auto requests = StoreRequests{
.extend_start_loc = static_cast<const int32_t*>(extend_start_loc.data_ptr()),
.extend_seq_lens = static_cast<const int32_t*>(extend_seq_lens.data_ptr()),
.num_reqs = static_cast<uint32_t>(B.unwrap()),
};
launch<true>(
kv_input,
kv_state,
kv_output,
norm_weight,
positions,
req,
raw_out_loc,
eps,
ring_size,
freqs_cis,
kvcache,
1,
&requests);
}
private:
struct StoreRequests {
const int32_t* extend_start_loc;
const int32_t* extend_seq_lens;
uint32_t num_reqs;
};
/// \tparam kPrefill `C2Mode::Prefill` plus the ring write-back of `requests`; otherwise
/// Decode, or Verify when `draft_len > 1`.
template <bool kPrefill>
static void launch(
const tvm::ffi::TensorView kv_input,
const tvm::ffi::TensorView kv_state,
@@ -330,7 +454,8 @@ struct FlashCompress2Kernel {
const int64_t ring_size,
const tvm::ffi::TensorView freqs_cis,
const tvm::ffi::TensorView kvcache,
const int64_t draft_len) {
const int64_t draft_len,
const StoreRequests* requests = nullptr) {
using namespace host;
auto N = SymbolicSize{"num_tokens"};
@@ -358,7 +483,7 @@ struct FlashCompress2Kernel {
const auto num_tokens = static_cast<uint32_t>(N.unwrap());
if (num_tokens == 0) return;
const auto is_verify = draft_len > 1;
const auto is_verify = !kPrefill && draft_len > 1;
CHECK_HOST(ring_size > 0 && draft_len >= 1);
CHECK_HOST(!is_verify || num_tokens % draft_len == 0);
CHECK_HOST(!is_verify || ring_size > draft_len)
@@ -373,19 +498,28 @@ struct FlashCompress2Kernel {
.req = static_cast<const int64_t*>(req.data_ptr()),
.raw_out_loc = raw_out_loc.data_ptr(),
.kvcache = static_cast<uint8_t*>(kvcache.data_ptr()),
.ring_size = static_cast<uint32_t>(ring_size),
.ring_size = static_cast<fast_mod_div_u32_t>(ring_size),
.eps = eps,
};
// `LaunchKernel` is move-only, so each arm builds its own.
const auto pos_i32 = pos_dtype.is_type<int32_t>();
const auto loc_i32 = loc_dtype.is_type<int32_t>();
if (is_verify) {
if constexpr (kPrefill) {
const auto k = select<C2Mode::Prefill>(pos_i32, loc_i32);
LaunchKernel(num_tokens, kBlockSize, device_.unwrap()) //
.enable_pdl(kUsePDL)(k, params);
if (requests->num_reqs == 0) return;
const auto store = select_store(pos_i32, loc_i32);
const auto grid = dim3{static_cast<uint32_t>(ring_size), requests->num_reqs};
LaunchKernel(grid, kHeadDim / 2, device_.unwrap()) //
.enable_pdl(kUsePDL)(store, params, requests->extend_start_loc, requests->extend_seq_lens);
} else if (is_verify) {
const auto block = static_cast<uint32_t>(draft_len);
const auto k = select<true>(pos_i32, loc_i32);
const auto k = select<C2Mode::Verify>(pos_i32, loc_i32);
LaunchKernel(dim3{block, num_tokens / block}, kBlockSize, device_.unwrap()) //
.enable_pdl(kUsePDL)(k, params);
} else {
const auto k = select<false>(pos_i32, loc_i32);
const auto k = select<C2Mode::Decode>(pos_i32, loc_i32);
LaunchKernel(num_tokens, kBlockSize, device_.unwrap()) //
.enable_pdl(kUsePDL)(k, params);
}
@@ -2,6 +2,7 @@
#include <sgl_kernel/tensor.h>
#include <sgl_kernel/utils.h>
#include <sgl_kernel/math.cuh>
#include <sgl_kernel/runtime.cuh>
#include <sgl_kernel/type.cuh>
#include <sgl_kernel/utils.cuh>
@@ -21,31 +22,7 @@ namespace sglang {
using device::distributed::PushWorkSpace;
using device::distributed::Semaphore;
// Runtime uint32 division as a multiply-high and a shift (round-up magic,
// exact below 2^31); cuda::fast_mod_div needs a newer CCCL than CUDA 13 bundles.
struct fast_mod_div_u32_t {
uint32_t divisor;
uint32_t magic;
uint32_t shift;
__host__ explicit fast_mod_div_u32_t(uint32_t d) : divisor(d), magic(0), shift(0) {
if (d > 1) {
const uint32_t log2_ceil = 32 - std::countl_zero(d - 1);
const uint32_t p = 31 + log2_ceil;
magic = static_cast<uint32_t>(((uint64_t{1} << p) + d - 1) / d);
shift = p - 32;
}
}
__device__ friend uint32_t operator/(uint32_t n, const fast_mod_div_u32_t& fd) {
return fd.divisor == 1 ? n : __umulhi(n, fd.magic) >> fd.shift;
}
__device__ friend uint32_t operator%(uint32_t n, const fast_mod_div_u32_t& fd) {
return n - (n / fd) * fd.divisor;
}
};
using device::math::fast_mod_div_u32_t;
template <uint32_t kWorldSize>
struct NVLinkCommPushParams {
@@ -168,8 +145,7 @@ PUSH_KERNEL void nvlink_push_kernel(const __grid_constant__ NVLinkCommPushParams
ptx::st_multimem_16B(vec, dst_ptr_mc, vid);
}
} else /* reduce-scatter only */ {
const auto token_id = vid / params.vecs_per_token_div;
const auto offset = vid % params.vecs_per_token_div;
const auto [token_id, offset] = math::div(vid, params.vecs_per_token_div);
// Both by a compile-time constant, so this is a mask and a shift.
const auto dst_rank = token_id % kWorldSize;
const auto dst_token_id = token_id / kWorldSize;
@@ -112,6 +112,53 @@ SGL_DEVICE float fma_f32_bf16(bf16_t a, float b, float acc) {
return fmaf(cast<fp32_t>(a), b, acc);
}
/**
* \brief Fast uint32 division and modulo by a compile-time constant divisor.
*
* This struct is only exact when the divisor is less than `2^31`. Larger dividends
* will lead to incorrect results. Be careful when using this struct.
*/
struct fast_mod_div_u32_t {
public:
explicit constexpr fast_mod_div_u32_t(uint32_t d) : m_divisor(d) {
if (d > 1) {
const uint32_t log2_ceil = 32 - std::countl_zero(d - 1);
const uint32_t p = 31 + log2_ceil;
m_magic = static_cast<uint32_t>(((uint64_t{1} << p) + d - 1) / d);
m_shift = p - 32;
} else {
m_magic = 0;
m_shift = 0;
}
}
template <typename T>
SGL_DEVICE friend uint32_t operator/(T n, const fast_mod_div_u32_t& d) {
static_assert(std::is_unsigned_v<T> && sizeof(T) <= sizeof(uint32_t));
return d.m_divisor == 1 ? n : __umulhi(n, d.m_magic) >> d.m_shift;
}
template <typename T>
SGL_DEVICE friend uint32_t operator%(T n, const fast_mod_div_u32_t& d) {
static_assert(std::is_unsigned_v<T> && sizeof(T) <= sizeof(uint32_t));
return n - (n / d) * d.m_divisor;
}
constexpr operator uint32_t() const {
return m_divisor;
}
private:
uint32_t m_divisor;
uint32_t m_magic;
uint32_t m_shift;
};
template <typename T>
SGL_DEVICE uint2 div(T n, const fast_mod_div_u32_t& d) {
return {n / d, n % d};
}
} // namespace device::math
} // namespace sglang
@@ -1,5 +1,6 @@
"""Fused ratio-1 and ratio-2 decode compressors: RMSNorm, RoPE and the FlashMLA
cache write in one launch.
"""Fused ratio-1 and ratio-2 compressors: RMSNorm, RoPE and the FlashMLA cache
write in one launch. The ratio-1 kernel is per token, so decode and extend share
it; ratio 2 has a decode/verify entry point and an extend one.
Ratio 1 takes the bf16 ``wkv`` projection as is: RoPE uses the token's own
position and the compressed slot equals the FULL slot. Ratio 2 pair-pools the
@@ -119,11 +120,12 @@ def _jit_c2_module(
is_arch_support_pdl(),
)
return load_jit(
make_name("c2_decode"),
make_name("c2"),
*args,
cuda_files=["deepseek_v4/c2.cuh"],
cuda_wrappers=[
("decode_fusion", f"FlashCompress2Kernel<{args}>::run_decode_fusion"),
("prefill_fusion", f"FlashCompress2Kernel<{args}>::run_prefill_fusion"),
],
)
@@ -149,6 +151,12 @@ def c2_decode_norm_rope_store(
The cache store uses ``raw_out_loc // 2`` as its slot.
A padded graph row is marked by ``raw_out_loc == 0`` and writes nothing, but
decode and verify rows issue their ring load before that check, so a padded
row's ``req`` and ``positions`` must still address an in-bounds ring slot.
The decode graph buffers zero both tails (``PaddingPolicy.ZERO``); a runner
that left stale request indices there would read out of the ring pool.
:param freqs_cis: ``[max_pos, rope_dim]`` fp32, real/imag interleaved --
``torch.view_as_real(freqs).flatten(-2)``. Indexed
in-kernel at ``positions - 1``, the position the latent
@@ -181,3 +189,60 @@ def c2_decode_norm_rope_store(
draft_len,
)
return out
def c2_prefill_norm_rope_store(
kv_input: torch.Tensor,
kv_state: torch.Tensor,
norm_weight: torch.Tensor,
positions: torch.Tensor,
req: torch.Tensor,
raw_out_loc: torch.Tensor,
eps: float,
freqs_cis: torch.Tensor,
k_cache: torch.Tensor,
extend_start_loc: torch.Tensor,
extend_seq_lens: torch.Tensor,
*,
page_size: int,
ring_size: int,
layout: Union[KVLayout, str] = KVLayout.V4,
out: Optional[torch.Tensor] = None,
) -> torch.Tensor:
""":func:`c2_decode_norm_rope_store` for an extend batch, then the ring write-back.
Rows may mix requests, prefix offsets and CUDA graph padding (``raw_out_loc
== 0``). An odd position pairs with the previous row when that row is the
same request's previous position, else with its ring slot. Afterwards each
request's last ``ring_size`` rows are saved to the ring, which a later step
reads for its first partner.
:param extend_start_loc: int32 ``[num_reqs]``, each request's first row.
:param extend_seq_lens: int32 ``[num_reqs]``, each request's row count.
:return: ``out``, ``[num_tokens, head_dim]`` bf16. Only the rows of odd
positions are written; the rest are left as they were.
"""
num_tokens, fused_dim = kv_input.shape
head_dim = fused_dim // 2
if out is None:
out = kv_input.new_empty((num_tokens, head_dim), dtype=torch.bfloat16)
layout = KVLayout.parse(layout)
module = _jit_c2_module(head_dim, freqs_cis.shape[-1], page_size, layout)
module.prefill_fusion(
kv_input,
kv_state,
out,
norm_weight,
positions,
req,
raw_out_loc,
eps,
freqs_cis,
# HIP's fp8_e4m3_t is uint8_t, so the kernel matches the fp8 pool as bytes
k_cache.view(torch.uint8) if is_hip_runtime() else k_cache,
ring_size,
extend_start_loc,
extend_seq_lens,
)
return out
@@ -2930,6 +2930,22 @@ class DeepseekV4AttnBackend(
self._low_ratio_compress_fused(
layer, x, req, pos, draft_len=self.speculative_num_draft_tokens
)
elif (
layer.compressor.use_fused_compress
and forward_batch.forward_mode.is_extend_without_speculative()
and forward_batch.extend_start_loc is not None
):
# DP padding extends extend_seq_lens but not extend_start_loc, whose
# length is the request count; padded rows belong to no request.
start_loc = forward_batch.extend_start_loc
seq_lens = forward_batch.extend_seq_lens[: start_loc.shape[0]]
self._low_ratio_compress_fused(
layer,
x,
req,
pos,
extend_offsets=(start_loc.to(torch.int32), seq_lens.to(torch.int32)),
)
else:
self._low_ratio_compress_torch(
layer,
@@ -3001,13 +3017,18 @@ class DeepseekV4AttnBackend(
fuse_index_store=_is_sm100_or_newer(),
)
def _low_ratio_compress_fused(self, layer, x, req, pos, *, draft_len=1) -> None:
def _low_ratio_compress_fused(
self, layer, x, req, pos, *, draft_len=1, extend_offsets=None
) -> None:
"""Decode or target-verify; an extend batch passes ``extend_offsets``, the int32
(first row, row count) of each request."""
from sglang.kernels.ops.attention.dsv4.fp4_indexer_rope import (
index_k_norm_rope_pack_store,
)
from sglang.kernels.ops.attention.dsv4.low_ratio_compress import (
c1_decode_norm_rope_store,
c2_decode_norm_rope_store,
c2_prefill_norm_rope_store,
)
pool = self.token_to_kv_pool
@@ -3040,7 +3061,7 @@ class DeepseekV4AttnBackend(
# CompressStatePool stores each request's pending pairs in a position ring.
# KVAndScore rows use | kv | score |, addressed as req * ring_size + pos % ring_size.
state = pool.get_attention_compress_states(layer_id)
latent = c2_decode_norm_rope_store(
args = (
compressor.project_fused(x),
state.kv_score_buffer.kv_score,
compressor.norm.weight.data,
@@ -3050,11 +3071,23 @@ class DeepseekV4AttnBackend(
compressor.norm.eps,
freqs_cis,
kv_cache,
page_size=page_size,
ring_size=state.ring_size,
draft_len=draft_len,
layout=kv_layout,
)
if extend_offsets is None:
latent = c2_decode_norm_rope_store(
*args,
page_size=page_size,
ring_size=state.ring_size,
draft_len=draft_len,
layout=kv_layout,
)
else:
latent = c2_prefill_norm_rope_store(
*args,
*extend_offsets,
page_size=page_size,
ring_size=state.ring_size,
layout=kv_layout,
)
out_loc = core.c2_out_loc
indexer = layer.indexer
@@ -0,0 +1,122 @@
"""Benchmark the fused ratio-2 decode compressor (``c2_decode_norm_rope_store``).
One CTA per token: pair-pool against the state ring, RMSNorm, RoPE, quantize and
store one compressed KV row. The line axis is the cache page format.
Shape axes:
- ``bs``: requests per batch.
- ``draft_len``: 1 is plain decode; > 1 is a target-verify block of that many
consecutive positions per request.
- ``parity``: the first position's parity. On decode, ``odd`` rows complete a
pair and run the full norm/RoPE/store path, ``even`` rows only park in the ring.
Constants follow the DeepSeek-V4 deployment: ``head_dim = 512``, ``rope_dim = 64``,
a 256-token FULL page, hence 128 compressed slots per page.
"""
import torch
from sglang.kernels.jit.benchmark import marker
from sglang.kernels.jit.benchmark.utils import (
DEFAULT_DEVICE,
create_empty,
create_random,
)
from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout
from sglang.kernels.ops.attention.dsv4.low_ratio_compress import (
c2_decode_norm_rope_store,
)
from sglang.srt.mem_cache.deepseek_v4_memory_pool import get_compress_state_ring_size
from sglang.test.ci.ci_register import register_cuda_ci
register_cuda_ci(
est_time=30, stage="base-b-kernel-benchmark", runner_config="1-gpu-large"
)
HEAD_DIM = 512
ROPE_DIM = 64
PAGE_SIZE = 256 // 2
MAX_POS = 65536
EPS = 1e-6
@marker.parametrize("parity", ["odd", "even", "half"], ["half"])
@marker.parametrize("draft_len", [1, 2, 4, 8], [1, 8])
@marker.parametrize("bs", marker.range(14, pattern="pow2"), [1, 1024])
@marker.benchmark("layout", ["V4", "V41", "V41_FP4"])
def benchmark(bs: int, draft_len: int, parity: str, layout: str):
if parity == "half" and bs % 2 != 0:
marker.skip("half parity requires even batch size")
layout = KVLayout[layout]
num_tokens = bs * draft_len
ring_size = get_compress_state_ring_size(2, draft_len if draft_len > 1 else 0)
torch.manual_seed(0)
kv_input = create_random(num_tokens, 2 * HEAD_DIM, dtype=torch.float32)
kv_state = create_random(bs * ring_size, 2 * HEAD_DIM, dtype=torch.float32)
norm_weight = create_random(HEAD_DIM)
freqs_cis = create_random(MAX_POS, ROPE_DIM, dtype=torch.float32)
req_ids = torch.randperm(bs, device=DEFAULT_DEVICE)
start = torch.randint(1, MAX_POS // 2 - draft_len, (bs,), device=DEFAULT_DEVICE)
if parity == "odd":
start = start * 2 + 1
elif parity == "even":
start = start * 2
else:
start = start * 2
start[::2] += 1 # half odd, half even
offsets = torch.arange(draft_len, device=DEFAULT_DEVICE)
positions = (start[:, None] + offsets).flatten()
req = req_ids.repeat_interleave(draft_len)
# Distinct scattered slots; slot 0 is left alone, it marks a padded row.
num_slots = num_tokens + 1
num_pages = -(-num_slots // PAGE_SIZE)
slots = torch.randperm(num_pages * PAGE_SIZE - 1, device=DEFAULT_DEVICE)
slots = slots[:num_tokens]
raw_out_loc = (slots + 1) * 2 + 1
k_cache = create_empty(
num_pages,
layout.page_bytes(PAGE_SIZE),
dtype=torch.uint8,
)
out = create_empty(num_tokens, HEAD_DIM)
# Traffic of the rows that complete a pair; parked rows write the state instead.
num_complete = int((positions % 2 == 1).sum())
num_parked = num_tokens - num_complete
row_bytes = 2 * HEAD_DIM * 4
complete_bytes = (
2 * row_bytes # kv_input + partner
+ ROPE_DIM * 4 # freqs_cis row
+ HEAD_DIM * 2 # out
+ layout.bytes_per_token
)
footprint = (
num_complete * complete_bytes
+ num_parked * 3 * row_bytes # kv_input + partner read, state write
+ num_tokens * 3 * 8 # positions, req, raw_out_loc
+ norm_weight.nbytes
)
return marker.do_bench(
c2_decode_norm_rope_store,
input_args=(kv_input, kv_state, norm_weight, positions, req, raw_out_loc),
input_kwargs=dict(
eps=EPS,
freqs_cis=freqs_cis,
k_cache=k_cache,
page_size=PAGE_SIZE,
ring_size=ring_size,
draft_len=draft_len,
layout=layout,
out=out,
),
memory_args=None,
memory_output=None,
extra_memory_footprint=footprint,
)
if __name__ == "__main__":
benchmark.run()
@@ -0,0 +1,191 @@
"""The ratio-2 compressor's eager-extend path against its decode path.
``c2_prefill_norm_rope_store`` pairs rows inside a chunk and parks each request's
last ``ring_size`` rows with a separate write-back kernel, while
``c2_decode_norm_rope_store`` takes every partner from the ring and parks every
pending row there. Driving one token stream both ways must leave the same main-KV
bytes and the same latents, so a chunk boundary that parks the wrong rows shows up
in the next chunk's output instead of staying invisible in the first chunk's.
"""
import unittest
from typing import List, Tuple
import torch
from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout
from sglang.kernels.ops.attention.dsv4.low_ratio_compress import (
c2_decode_norm_rope_store,
c2_prefill_norm_rope_store,
)
from sglang.srt.utils import is_cuda
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
# The c2 JIT kernels are the SM100 build; the only Blackwell runner config is the
# four-GPU one.
register_cuda_ci(est_time=60, stage="base-b-kernel-unit", runner_config="4-gpu-b200")
HEAD_DIM = 512
ROPE_DIM = 64
PAGE_SIZE = 64
NUM_PAGES = 16
EPS = 1e-6
# Request r's full slots start here, even so that a (2k, 2k+1) pair shares a
# compressed slot; never 0, which marks a padded row.
SLOT_BASE = 64
def _sm100() -> bool:
return is_cuda() and torch.cuda.get_device_capability()[0] == 10
class Stream:
"""One token stream: per-request prefix and chunk sizes, laid out as the rows
the backend would hand each call."""
def __init__(self, prefixes: List[int], chunks: List[List[int]]):
self.prefixes = prefixes
self.chunks = chunks # chunks[c][r] = rows of request r in chunk c
def chunk_rows(self, c: int) -> List[Tuple[int, int]]:
"""(request, position) of every row of chunk ``c``, requests in order."""
rows = []
for r, n in enumerate(self.chunks[c]):
start = self.prefixes[r] + sum(self.chunks[j][r] for j in range(c))
rows.extend((r, start + i) for i in range(n))
return rows
def _make_inputs(gen, rows: int) -> torch.Tensor:
"""``[rows, 2 * HEAD_DIM]`` fp32: the kv half then the score half."""
return torch.randn(
rows, 2 * HEAD_DIM, generator=gen, device="cuda", dtype=torch.float32
)
def _row_tensors(rows: List[Tuple[int, int]], pad: int):
"""``req``, ``positions`` and ``raw_out_loc`` for ``rows``, plus ``pad`` padded
rows. A padded row still has to address a live ring slot, so it takes request 0
and position 0; only ``raw_out_loc == 0`` marks it."""
req = [r for r, _ in rows] + [0] * pad
pos = [p for _, p in rows] + [0] * pad
loc = [SLOT_BASE * (1 + r) + p for r, p in rows] + [0] * pad
to = lambda v, d: torch.tensor(v, device="cuda", dtype=d)
return to(req, torch.int64), to(pos, torch.int32), to(loc, torch.int32)
class TestC2PrefillRing(CustomTestCase):
def _run(self, stream: Stream, layout: KVLayout, ring_size: int, seed: int):
"""The fused chunked path and the row-at-a-time decode path over the same
stream; returns (cache, latents) for each."""
gen = torch.Generator(device="cuda").manual_seed(seed)
num_reqs = len(stream.prefixes)
weight = torch.randn(
HEAD_DIM, generator=gen, device="cuda", dtype=torch.float32
).to(torch.bfloat16)
angles = torch.randn(4096, ROPE_DIM // 2, generator=gen, device="cuda")
freqs = torch.view_as_real(
torch.polar(torch.ones_like(angles), angles)
).flatten(-2)
# One fp32 input row per (request, position) in the stream, shared by both
# paths so any difference is the pairing, not the data.
all_rows = [
row for c in range(len(stream.chunks)) for row in stream.chunk_rows(c)
]
inputs = {row: _make_inputs(gen, 1)[0] for row in all_rows}
state_rows = num_reqs * ring_size + ring_size
caches, latents = [], []
for fused in (True, False):
cache = torch.zeros(
NUM_PAGES,
layout.page_bytes(PAGE_SIZE),
dtype=torch.uint8,
device="cuda",
)
state = torch.zeros(
state_rows, 2 * HEAD_DIM, dtype=torch.float32, device="cuda"
)
out = {}
if fused:
for c in range(len(stream.chunks)):
rows = stream.chunk_rows(c)
pad = 2
req, pos, loc = _row_tensors(rows, pad)
kv = torch.stack(
[inputs[row] for row in rows]
+ [torch.zeros(2 * HEAD_DIM, device="cuda")] * pad
)
starts, lens = [], []
at = 0
for r in range(num_reqs):
starts.append(at)
lens.append(stream.chunks[c][r])
at += stream.chunks[c][r]
latent = c2_prefill_norm_rope_store(
kv,
state,
weight,
pos,
req,
loc,
EPS,
freqs,
cache,
torch.tensor(starts, device="cuda", dtype=torch.int32),
torch.tensor(lens, device="cuda", dtype=torch.int32),
page_size=PAGE_SIZE,
ring_size=ring_size,
layout=layout,
)
for i, row in enumerate(rows):
out[row] = latent[i].clone()
else:
for row in all_rows:
req, pos, loc = _row_tensors([row], 0)
latent = c2_decode_norm_rope_store(
inputs[row][None, :],
state,
weight,
pos,
req,
loc,
EPS,
freqs,
cache,
page_size=PAGE_SIZE,
ring_size=ring_size,
layout=layout,
)
out[row] = latent[0].clone()
caches.append(cache)
# Only a completing (odd position) row writes a latent.
latents.append(
torch.stack([out[row] for row in all_rows if row[1] % 2 == 1])
)
return caches, latents
@unittest.skipUnless(_sm100(), "the c2 JIT kernels are the SM100 build")
def test_chunked_extend_matches_decode(self):
# Request 0 starts even, request 1 odd and spans more rows than the ring,
# request 2 odd with a single row in its first chunk.
stream = Stream(
prefixes=[0, 7, 1],
chunks=[[5, 4, 1], [3, 6, 2], [1, 1, 1]],
)
for layout in (KVLayout.V41, KVLayout.V41_FP4):
for ring_size in (2, 8):
with self.subTest(layout=layout.name, ring_size=ring_size):
caches, latents = self._run(stream, layout, ring_size, seed=7)
self.assertTrue(
torch.equal(caches[0], caches[1]),
f"main-KV bytes differ: {(caches[0] != caches[1]).sum()} of "
f"{caches[0].numel()}",
)
self.assertTrue(torch.equal(latents[0], latents[1]))
if __name__ == "__main__":
unittest.main()