[DSv4.1] Fused c1/c2 compress for eager extend, faster c2 decode (#41660)
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: BBuf <1182563586@qq.com>
(cherry picked from commit 5916999afb)
This commit is contained in:
@@ -18,6 +18,8 @@
|
||||
|
||||
namespace sglang {
|
||||
|
||||
using device::math::fast_mod_div_u32_t;
|
||||
|
||||
/// \brief Ratio-2 decode compressor: pair-pool, RMSNorm and the main-KV write.
|
||||
///
|
||||
/// `kv_input` and `kv_state` rows are `2 * kHeadDim` floats, kv then score.
|
||||
@@ -34,8 +36,7 @@ struct Compress2DecodeParams {
|
||||
const int64_t* __restrict__ req; // [num_tokens], req_pool_idx per token
|
||||
const void* __restrict__ raw_out_loc; // [num_tokens] LocT, the FULL slot; 0 marks a padded row
|
||||
uint8_t* __restrict__ kvcache; // [npages, kPageBytes] uint8
|
||||
/// Positions per request slot in the pair-state ring.
|
||||
uint32_t ring_size;
|
||||
fast_mod_div_u32_t ring_size; // Positions per request slot in the pair-state ring.
|
||||
float eps;
|
||||
};
|
||||
|
||||
@@ -43,21 +44,32 @@ struct Compress2DecodeParams {
|
||||
/// At (512, 64) it also keeps the nope/rope split warp-aligned, as the fp8 amax reduction requires.
|
||||
constexpr uint32_t kC2VecSize = 2;
|
||||
|
||||
/// \brief grid = num_tokens, block = kHeadDim / kC2VecSize.
|
||||
/// \brief Where a completing row finds its partner, and who parks a pending row.
|
||||
enum class C2Mode {
|
||||
/// One row per request: the partner comes from the ring, a pending row parks there.
|
||||
Decode,
|
||||
/// `draft_len` consecutive positions per request: `blockIdx.x` is the position inside the
|
||||
/// block, `blockIdx.y` the request, and every row but the first takes its partner from the
|
||||
/// previous `kv_input` row instead of the ring.
|
||||
Verify,
|
||||
/// Arbitrary rows, paired as `_low_ratio_pair_partners` pairs them: the partner is the previous
|
||||
/// row when it is the same request's previous position, else the ring. A pending row does
|
||||
/// nothing here; `c2_prefill_store_kernel` parks the ring rows after this kernel read them.
|
||||
Prefill,
|
||||
};
|
||||
|
||||
/// \brief grid = num_tokens (Decode, Prefill) or `draft_len x batch` (Verify),
|
||||
/// block = kHeadDim / kC2VecSize.
|
||||
///
|
||||
/// An odd position completes a group with its even predecessor; an even one parks in the state.
|
||||
///
|
||||
/// Under target-verify, `draft_len` consecutive positions per request: `blockIdx.x` is the
|
||||
/// position inside the block, `blockIdx.y` the request, and every row but the first takes its
|
||||
/// partner from the previous `kv_input` row instead of the ring.
|
||||
///
|
||||
/// The three reductions have different widths and are not interchangeable: the RMSNorm
|
||||
/// statistic spans the row, an fp8 store scale 64 elements, an fp4 block 16.
|
||||
///
|
||||
/// kLayout is the cache's page format: V4 and V41 store the fake-quantized value; V41_FP4
|
||||
/// stores the e2m1 codes and their e4m3 scales directly, so the fp4 rounding happens once.
|
||||
template <
|
||||
bool kVerify,
|
||||
C2Mode kMode,
|
||||
int64_t kHeadDim,
|
||||
int64_t kRopeDim,
|
||||
int32_t kPageBits,
|
||||
@@ -91,16 +103,21 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
|
||||
|
||||
const auto tx = threadIdx.x;
|
||||
// Verify gives each request a CTA column; decode a flat grid of one row each.
|
||||
const auto row = kVerify ? blockIdx.y * gridDim.x + blockIdx.x : blockIdx.x;
|
||||
const auto row = kMode == C2Mode::Verify ? blockIdx.y * gridDim.x + blockIdx.x : blockIdx.x;
|
||||
PDLWaitPrimary<kUsePDL>();
|
||||
// Slots fit in int32 whatever width the scheduler hands them in.
|
||||
const auto raw_out_loc = static_cast<int32_t>(static_cast<const LocT*>(params.raw_out_loc)[row]);
|
||||
// CUDA graph padding is a completely inert row: do not read schedule, input,
|
||||
// state, or RoPE data, and do not publish output or update the cache.
|
||||
if (raw_out_loc == 0) return PDLTriggerSecondary<kUsePDL>();
|
||||
const auto pos = static_cast<const PosT*>(params.positions)[row];
|
||||
// A completing row reads the slot left by `pos - 1`;
|
||||
// a pending row writes its own slot, so reads and writes stay disjoint.
|
||||
const auto* loc_ptr = static_cast<const LocT*>(params.raw_out_loc);
|
||||
const auto* pos_ptr = static_cast<const PosT*>(params.positions);
|
||||
const auto raw_out_loc = loc_ptr[row];
|
||||
// Prefill rows are many, so a CUDA graph padding row exits before any load.
|
||||
// Decode and verify rows are few: every row issues its loads to keep more in flight.
|
||||
if constexpr (kMode == C2Mode::Prefill) {
|
||||
if (raw_out_loc == 0) return PDLTriggerSecondary<kUsePDL>();
|
||||
}
|
||||
const auto pos = pos_ptr[row];
|
||||
// Prefill leaves a pending row's latent unwritten; its index-K slot is -1, so nothing reads it.
|
||||
if constexpr (kMode == C2Mode::Prefill) {
|
||||
if ((pos & 1) == 0) return PDLTriggerSecondary<kUsePDL>();
|
||||
}
|
||||
const auto rid = params.req[row];
|
||||
|
||||
fp32_vec_t kv_new, score_new;
|
||||
@@ -108,21 +125,32 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
|
||||
score_new.load(params.kv_input + row * kStride, tx + kCTASize);
|
||||
|
||||
const auto ring = static_cast<int64_t>(rid) * params.ring_size;
|
||||
const auto read_row = ring + (pos - 1 + params.ring_size) % params.ring_size;
|
||||
const auto write_row = ring + pos % params.ring_size;
|
||||
const auto write_offset = static_cast<uint32_t>(pos) % params.ring_size;
|
||||
const auto read_offset = write_offset == 0 ? params.ring_size - 1 : write_offset - 1;
|
||||
const auto write_row = ring + write_offset;
|
||||
const auto read_row = ring + read_offset;
|
||||
|
||||
fp32_vec_t kv_old, score_old;
|
||||
const float* partner = params.kv_state + read_row * kStride;
|
||||
if constexpr (kVerify) {
|
||||
if constexpr (kMode == C2Mode::Verify) {
|
||||
if (blockIdx.x != 0) partner = params.kv_input + static_cast<int64_t>(row - 1) * kStride;
|
||||
}
|
||||
if constexpr (kMode == C2Mode::Prefill) {
|
||||
if (row > 0 && params.req[row - 1] == rid && pos_ptr[row - 1] == pos - 1 && loc_ptr[row - 1] != 0) {
|
||||
partner = params.kv_input + static_cast<int64_t>(row - 1) * kStride;
|
||||
}
|
||||
}
|
||||
kv_old.load(partner, tx);
|
||||
score_old.load(partner, tx + kCTASize);
|
||||
|
||||
if ((pos & 1) == 0) {
|
||||
kv_new.store(params.kv_state + write_row * kStride, tx);
|
||||
score_new.store(params.kv_state + write_row * kStride, tx + kCTASize);
|
||||
return PDLTriggerSecondary<kUsePDL>();
|
||||
if constexpr (kMode != C2Mode::Prefill) {
|
||||
// CUDA graph padding case
|
||||
if (raw_out_loc == 0) return PDLTriggerSecondary<kUsePDL>();
|
||||
if ((pos & 1) == 0) {
|
||||
kv_new.store(params.kv_state + write_row * kStride, tx);
|
||||
score_new.store(params.kv_state + write_row * kStride, tx + kCTASize);
|
||||
return PDLTriggerSecondary<kUsePDL>();
|
||||
}
|
||||
}
|
||||
|
||||
constexpr uint32_t kNumWarps = kCTASize / kWarpThreads;
|
||||
@@ -203,9 +231,9 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
|
||||
}
|
||||
}
|
||||
|
||||
const int32_t out_loc = static_cast<uint32_t>(raw_out_loc) >> 1;
|
||||
if constexpr (kLayout == KVLayout::V41_FP4) {
|
||||
// The fp4 cache takes the rotated bf16 value as is: its row quantizer is the fake quant, minus the dequant.
|
||||
const int32_t out_loc = raw_out_loc >> 1;
|
||||
const auto kv_row = Paged::row(params.kvcache, out_loc);
|
||||
return deepseek_v4::v41::store_row<kLayout>(kv_row.data, kv_row.scale, tx, staged);
|
||||
}
|
||||
@@ -228,7 +256,6 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
|
||||
}
|
||||
|
||||
// `raw_out_loc / ratio`; ratio 2 makes it a shift.
|
||||
const int32_t out_loc = raw_out_loc >> 1;
|
||||
const auto kv_row = Paged::row(params.kvcache, out_loc);
|
||||
|
||||
if constexpr (kLayout == KVLayout::V41) {
|
||||
@@ -264,21 +291,63 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
|
||||
}
|
||||
}
|
||||
|
||||
/// \brief Prefill ring write-back, launched after `flash_c2_decode_kernel<C2Mode::Prefill>`.
|
||||
///
|
||||
/// Each request's last `ring_size` rows park their `| kv | score |` at `req * ring_size +
|
||||
/// pos % ring_size` for a later step's first partner. It cannot run inside the pooling kernel:
|
||||
/// a chunk whose first row is odd and whose last row is even reads and writes the same ring
|
||||
/// slot. grid = `ring_size x num_reqs`, CTA (k, i) parks the k-th row from the end of request i,
|
||||
/// one float4 per thread; those rows are consecutive positions, so their slots are distinct.
|
||||
template <int64_t kHeadDim, typename PosT, typename LocT, bool kUsePDL>
|
||||
__global__ __launch_bounds__(kHeadDim / 2) void c2_prefill_store_kernel(
|
||||
const Compress2DecodeParams params,
|
||||
const int32_t* __restrict__ extend_start_loc,
|
||||
const int32_t* __restrict__ extend_seq_lens) {
|
||||
using namespace device;
|
||||
constexpr int64_t kStride = kHeadDim * 2;
|
||||
using vec_t = AlignedVector<float, 4>;
|
||||
static_assert(kStride == 4 * (kHeadDim / 2));
|
||||
|
||||
// The pooling kernel reads the ring rows this one overwrites.
|
||||
PDLWaitPrimary<kUsePDL>();
|
||||
const auto bx = static_cast<int32_t>(blockIdx.x);
|
||||
const auto len = extend_seq_lens[blockIdx.y];
|
||||
const auto start_loc = extend_start_loc[blockIdx.y];
|
||||
if (bx >= len) return;
|
||||
const auto row = start_loc + (len - 1 - bx);
|
||||
const auto rid = params.req[row];
|
||||
const auto out = static_cast<const LocT*>(params.raw_out_loc)[row];
|
||||
const auto pos = static_cast<uint32_t>(static_cast<const PosT*>(params.positions)[row]);
|
||||
const auto dst_row = rid * params.ring_size + pos % params.ring_size;
|
||||
if (out == 0) return;
|
||||
vec_t buf;
|
||||
buf.load(params.kv_input + row * kStride, threadIdx.x);
|
||||
PDLTriggerSecondary<kUsePDL>();
|
||||
buf.store(params.kv_state + dst_row * kStride, threadIdx.x);
|
||||
}
|
||||
|
||||
template <int64_t kHeadDim, int64_t kRopeDim, uint32_t kPageSize, deepseek_v4::KVLayout kLayout, bool kUsePDL>
|
||||
struct FlashCompress2Kernel {
|
||||
static constexpr uint32_t kBlockSize = kHeadDim / kC2VecSize;
|
||||
static constexpr int32_t kPageBits = std::bit_width(kPageSize) - 1;
|
||||
static constexpr int64_t kPageBytes = deepseek_v4::kv_page_bytes<kLayout>(kPageSize);
|
||||
static_assert(kLayout != deepseek_v4::KVLayout::V4 || kPageBytes == host::div_ceil(584ll * kPageSize, 576) * 576);
|
||||
template <bool kVerify, typename PosT, typename LocT>
|
||||
template <C2Mode kMode, typename PosT, typename LocT>
|
||||
static constexpr auto kernel =
|
||||
flash_c2_decode_kernel<kVerify, kHeadDim, kRopeDim, kPageBits, PosT, LocT, kLayout, kUsePDL>;
|
||||
flash_c2_decode_kernel<kMode, kHeadDim, kRopeDim, kPageBits, PosT, LocT, kLayout, kUsePDL>;
|
||||
template <typename PosT, typename LocT>
|
||||
static constexpr auto store_kernel = c2_prefill_store_kernel<kHeadDim, PosT, LocT, kUsePDL>;
|
||||
|
||||
/// \brief The (`positions`, `raw_out_loc`) dtype pair, resolved at run time.
|
||||
template <bool kVerify>
|
||||
template <C2Mode kMode>
|
||||
static auto select(const bool pos_i32, const bool loc_i32) {
|
||||
if (pos_i32) return loc_i32 ? kernel<kVerify, int32_t, int32_t> : kernel<kVerify, int32_t, int64_t>;
|
||||
return loc_i32 ? kernel<kVerify, int64_t, int32_t> : kernel<kVerify, int64_t, int64_t>;
|
||||
if (pos_i32) return loc_i32 ? kernel<kMode, int32_t, int32_t> : kernel<kMode, int32_t, int64_t>;
|
||||
return loc_i32 ? kernel<kMode, int64_t, int32_t> : kernel<kMode, int64_t, int64_t>;
|
||||
}
|
||||
|
||||
static auto select_store(const bool pos_i32, const bool loc_i32) {
|
||||
if (pos_i32) return loc_i32 ? store_kernel<int32_t, int32_t> : store_kernel<int32_t, int64_t>;
|
||||
return loc_i32 ? store_kernel<int64_t, int32_t> : store_kernel<int64_t, int64_t>;
|
||||
}
|
||||
|
||||
// The sum of squares reduces through a fixed-size shared array, so the CTA must be whole warps.
|
||||
@@ -302,7 +371,7 @@ struct FlashCompress2Kernel {
|
||||
const tvm::ffi::TensorView kvcache,
|
||||
const int64_t ring_size,
|
||||
const int64_t draft_len) {
|
||||
launch(
|
||||
launch<false>(
|
||||
kv_input,
|
||||
kv_state,
|
||||
kv_output,
|
||||
@@ -317,7 +386,62 @@ struct FlashCompress2Kernel {
|
||||
draft_len);
|
||||
}
|
||||
|
||||
/// \brief The extend-batch pooling kernel, then the ring write-back.
|
||||
///
|
||||
/// Rows may mix requests, prefix offsets and padding; a pending (even) row's `kv_output`
|
||||
/// row is left unwritten. `extend_start_loc` / `extend_seq_lens` are int32 `[num_reqs]`:
|
||||
/// request i owns rows `[start[i], start[i] + len[i])`; zero-length requests are skipped.
|
||||
static void run_prefill_fusion(
|
||||
const tvm::ffi::TensorView kv_input,
|
||||
const tvm::ffi::TensorView kv_state,
|
||||
const tvm::ffi::TensorView kv_output,
|
||||
const tvm::ffi::TensorView norm_weight,
|
||||
const tvm::ffi::TensorView positions,
|
||||
const tvm::ffi::TensorView req,
|
||||
const tvm::ffi::TensorView raw_out_loc,
|
||||
const float eps,
|
||||
const tvm::ffi::TensorView freqs_cis,
|
||||
const tvm::ffi::TensorView kvcache,
|
||||
const int64_t ring_size,
|
||||
const tvm::ffi::TensorView extend_start_loc,
|
||||
const tvm::ffi::TensorView extend_seq_lens) {
|
||||
using namespace host;
|
||||
auto B = SymbolicSize{"num_reqs"};
|
||||
auto device_ = SymbolicDevice{};
|
||||
device_.set_options<kDLGPU>();
|
||||
TensorMatcher({B}).with_dtype<int32_t>().with_device(device_).verify(extend_start_loc);
|
||||
TensorMatcher({B}).with_dtype<int32_t>().with_device(device_).verify(extend_seq_lens);
|
||||
const auto requests = StoreRequests{
|
||||
.extend_start_loc = static_cast<const int32_t*>(extend_start_loc.data_ptr()),
|
||||
.extend_seq_lens = static_cast<const int32_t*>(extend_seq_lens.data_ptr()),
|
||||
.num_reqs = static_cast<uint32_t>(B.unwrap()),
|
||||
};
|
||||
launch<true>(
|
||||
kv_input,
|
||||
kv_state,
|
||||
kv_output,
|
||||
norm_weight,
|
||||
positions,
|
||||
req,
|
||||
raw_out_loc,
|
||||
eps,
|
||||
ring_size,
|
||||
freqs_cis,
|
||||
kvcache,
|
||||
1,
|
||||
&requests);
|
||||
}
|
||||
|
||||
private:
|
||||
struct StoreRequests {
|
||||
const int32_t* extend_start_loc;
|
||||
const int32_t* extend_seq_lens;
|
||||
uint32_t num_reqs;
|
||||
};
|
||||
|
||||
/// \tparam kPrefill `C2Mode::Prefill` plus the ring write-back of `requests`; otherwise
|
||||
/// Decode, or Verify when `draft_len > 1`.
|
||||
template <bool kPrefill>
|
||||
static void launch(
|
||||
const tvm::ffi::TensorView kv_input,
|
||||
const tvm::ffi::TensorView kv_state,
|
||||
@@ -330,7 +454,8 @@ struct FlashCompress2Kernel {
|
||||
const int64_t ring_size,
|
||||
const tvm::ffi::TensorView freqs_cis,
|
||||
const tvm::ffi::TensorView kvcache,
|
||||
const int64_t draft_len) {
|
||||
const int64_t draft_len,
|
||||
const StoreRequests* requests = nullptr) {
|
||||
using namespace host;
|
||||
|
||||
auto N = SymbolicSize{"num_tokens"};
|
||||
@@ -358,7 +483,7 @@ struct FlashCompress2Kernel {
|
||||
|
||||
const auto num_tokens = static_cast<uint32_t>(N.unwrap());
|
||||
if (num_tokens == 0) return;
|
||||
const auto is_verify = draft_len > 1;
|
||||
const auto is_verify = !kPrefill && draft_len > 1;
|
||||
CHECK_HOST(ring_size > 0 && draft_len >= 1);
|
||||
CHECK_HOST(!is_verify || num_tokens % draft_len == 0);
|
||||
CHECK_HOST(!is_verify || ring_size > draft_len)
|
||||
@@ -373,19 +498,28 @@ struct FlashCompress2Kernel {
|
||||
.req = static_cast<const int64_t*>(req.data_ptr()),
|
||||
.raw_out_loc = raw_out_loc.data_ptr(),
|
||||
.kvcache = static_cast<uint8_t*>(kvcache.data_ptr()),
|
||||
.ring_size = static_cast<uint32_t>(ring_size),
|
||||
.ring_size = static_cast<fast_mod_div_u32_t>(ring_size),
|
||||
.eps = eps,
|
||||
};
|
||||
// `LaunchKernel` is move-only, so each arm builds its own.
|
||||
const auto pos_i32 = pos_dtype.is_type<int32_t>();
|
||||
const auto loc_i32 = loc_dtype.is_type<int32_t>();
|
||||
if (is_verify) {
|
||||
if constexpr (kPrefill) {
|
||||
const auto k = select<C2Mode::Prefill>(pos_i32, loc_i32);
|
||||
LaunchKernel(num_tokens, kBlockSize, device_.unwrap()) //
|
||||
.enable_pdl(kUsePDL)(k, params);
|
||||
if (requests->num_reqs == 0) return;
|
||||
const auto store = select_store(pos_i32, loc_i32);
|
||||
const auto grid = dim3{static_cast<uint32_t>(ring_size), requests->num_reqs};
|
||||
LaunchKernel(grid, kHeadDim / 2, device_.unwrap()) //
|
||||
.enable_pdl(kUsePDL)(store, params, requests->extend_start_loc, requests->extend_seq_lens);
|
||||
} else if (is_verify) {
|
||||
const auto block = static_cast<uint32_t>(draft_len);
|
||||
const auto k = select<true>(pos_i32, loc_i32);
|
||||
const auto k = select<C2Mode::Verify>(pos_i32, loc_i32);
|
||||
LaunchKernel(dim3{block, num_tokens / block}, kBlockSize, device_.unwrap()) //
|
||||
.enable_pdl(kUsePDL)(k, params);
|
||||
} else {
|
||||
const auto k = select<false>(pos_i32, loc_i32);
|
||||
const auto k = select<C2Mode::Decode>(pos_i32, loc_i32);
|
||||
LaunchKernel(num_tokens, kBlockSize, device_.unwrap()) //
|
||||
.enable_pdl(kUsePDL)(k, params);
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
#include <sgl_kernel/tensor.h>
|
||||
#include <sgl_kernel/utils.h>
|
||||
|
||||
#include <sgl_kernel/math.cuh>
|
||||
#include <sgl_kernel/runtime.cuh>
|
||||
#include <sgl_kernel/type.cuh>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
@@ -21,31 +22,7 @@ namespace sglang {
|
||||
|
||||
using device::distributed::PushWorkSpace;
|
||||
using device::distributed::Semaphore;
|
||||
|
||||
// Runtime uint32 division as a multiply-high and a shift (round-up magic,
|
||||
// exact below 2^31); cuda::fast_mod_div needs a newer CCCL than CUDA 13 bundles.
|
||||
struct fast_mod_div_u32_t {
|
||||
uint32_t divisor;
|
||||
uint32_t magic;
|
||||
uint32_t shift;
|
||||
|
||||
__host__ explicit fast_mod_div_u32_t(uint32_t d) : divisor(d), magic(0), shift(0) {
|
||||
if (d > 1) {
|
||||
const uint32_t log2_ceil = 32 - std::countl_zero(d - 1);
|
||||
const uint32_t p = 31 + log2_ceil;
|
||||
magic = static_cast<uint32_t>(((uint64_t{1} << p) + d - 1) / d);
|
||||
shift = p - 32;
|
||||
}
|
||||
}
|
||||
|
||||
__device__ friend uint32_t operator/(uint32_t n, const fast_mod_div_u32_t& fd) {
|
||||
return fd.divisor == 1 ? n : __umulhi(n, fd.magic) >> fd.shift;
|
||||
}
|
||||
|
||||
__device__ friend uint32_t operator%(uint32_t n, const fast_mod_div_u32_t& fd) {
|
||||
return n - (n / fd) * fd.divisor;
|
||||
}
|
||||
};
|
||||
using device::math::fast_mod_div_u32_t;
|
||||
|
||||
template <uint32_t kWorldSize>
|
||||
struct NVLinkCommPushParams {
|
||||
@@ -168,8 +145,7 @@ PUSH_KERNEL void nvlink_push_kernel(const __grid_constant__ NVLinkCommPushParams
|
||||
ptx::st_multimem_16B(vec, dst_ptr_mc, vid);
|
||||
}
|
||||
} else /* reduce-scatter only */ {
|
||||
const auto token_id = vid / params.vecs_per_token_div;
|
||||
const auto offset = vid % params.vecs_per_token_div;
|
||||
const auto [token_id, offset] = math::div(vid, params.vecs_per_token_div);
|
||||
// Both by a compile-time constant, so this is a mask and a shift.
|
||||
const auto dst_rank = token_id % kWorldSize;
|
||||
const auto dst_token_id = token_id / kWorldSize;
|
||||
|
||||
@@ -112,6 +112,53 @@ SGL_DEVICE float fma_f32_bf16(bf16_t a, float b, float acc) {
|
||||
return fmaf(cast<fp32_t>(a), b, acc);
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief Fast uint32 division and modulo by a compile-time constant divisor.
|
||||
*
|
||||
* This struct is only exact when the divisor is less than `2^31`. Larger dividends
|
||||
* will lead to incorrect results. Be careful when using this struct.
|
||||
*/
|
||||
struct fast_mod_div_u32_t {
|
||||
public:
|
||||
explicit constexpr fast_mod_div_u32_t(uint32_t d) : m_divisor(d) {
|
||||
if (d > 1) {
|
||||
const uint32_t log2_ceil = 32 - std::countl_zero(d - 1);
|
||||
const uint32_t p = 31 + log2_ceil;
|
||||
m_magic = static_cast<uint32_t>(((uint64_t{1} << p) + d - 1) / d);
|
||||
m_shift = p - 32;
|
||||
} else {
|
||||
m_magic = 0;
|
||||
m_shift = 0;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
SGL_DEVICE friend uint32_t operator/(T n, const fast_mod_div_u32_t& d) {
|
||||
static_assert(std::is_unsigned_v<T> && sizeof(T) <= sizeof(uint32_t));
|
||||
return d.m_divisor == 1 ? n : __umulhi(n, d.m_magic) >> d.m_shift;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
SGL_DEVICE friend uint32_t operator%(T n, const fast_mod_div_u32_t& d) {
|
||||
static_assert(std::is_unsigned_v<T> && sizeof(T) <= sizeof(uint32_t));
|
||||
return n - (n / d) * d.m_divisor;
|
||||
}
|
||||
|
||||
constexpr operator uint32_t() const {
|
||||
return m_divisor;
|
||||
}
|
||||
|
||||
private:
|
||||
uint32_t m_divisor;
|
||||
uint32_t m_magic;
|
||||
uint32_t m_shift;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
SGL_DEVICE uint2 div(T n, const fast_mod_div_u32_t& d) {
|
||||
return {n / d, n % d};
|
||||
}
|
||||
|
||||
} // namespace device::math
|
||||
|
||||
} // namespace sglang
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Fused ratio-1 and ratio-2 decode compressors: RMSNorm, RoPE and the FlashMLA
|
||||
cache write in one launch.
|
||||
"""Fused ratio-1 and ratio-2 compressors: RMSNorm, RoPE and the FlashMLA cache
|
||||
write in one launch. The ratio-1 kernel is per token, so decode and extend share
|
||||
it; ratio 2 has a decode/verify entry point and an extend one.
|
||||
|
||||
Ratio 1 takes the bf16 ``wkv`` projection as is: RoPE uses the token's own
|
||||
position and the compressed slot equals the FULL slot. Ratio 2 pair-pools the
|
||||
@@ -119,11 +120,12 @@ def _jit_c2_module(
|
||||
is_arch_support_pdl(),
|
||||
)
|
||||
return load_jit(
|
||||
make_name("c2_decode"),
|
||||
make_name("c2"),
|
||||
*args,
|
||||
cuda_files=["deepseek_v4/c2.cuh"],
|
||||
cuda_wrappers=[
|
||||
("decode_fusion", f"FlashCompress2Kernel<{args}>::run_decode_fusion"),
|
||||
("prefill_fusion", f"FlashCompress2Kernel<{args}>::run_prefill_fusion"),
|
||||
],
|
||||
)
|
||||
|
||||
@@ -149,6 +151,12 @@ def c2_decode_norm_rope_store(
|
||||
|
||||
The cache store uses ``raw_out_loc // 2`` as its slot.
|
||||
|
||||
A padded graph row is marked by ``raw_out_loc == 0`` and writes nothing, but
|
||||
decode and verify rows issue their ring load before that check, so a padded
|
||||
row's ``req`` and ``positions`` must still address an in-bounds ring slot.
|
||||
The decode graph buffers zero both tails (``PaddingPolicy.ZERO``); a runner
|
||||
that left stale request indices there would read out of the ring pool.
|
||||
|
||||
:param freqs_cis: ``[max_pos, rope_dim]`` fp32, real/imag interleaved --
|
||||
``torch.view_as_real(freqs).flatten(-2)``. Indexed
|
||||
in-kernel at ``positions - 1``, the position the latent
|
||||
@@ -181,3 +189,60 @@ def c2_decode_norm_rope_store(
|
||||
draft_len,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def c2_prefill_norm_rope_store(
|
||||
kv_input: torch.Tensor,
|
||||
kv_state: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
req: torch.Tensor,
|
||||
raw_out_loc: torch.Tensor,
|
||||
eps: float,
|
||||
freqs_cis: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
extend_start_loc: torch.Tensor,
|
||||
extend_seq_lens: torch.Tensor,
|
||||
*,
|
||||
page_size: int,
|
||||
ring_size: int,
|
||||
layout: Union[KVLayout, str] = KVLayout.V4,
|
||||
out: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
""":func:`c2_decode_norm_rope_store` for an extend batch, then the ring write-back.
|
||||
|
||||
Rows may mix requests, prefix offsets and CUDA graph padding (``raw_out_loc
|
||||
== 0``). An odd position pairs with the previous row when that row is the
|
||||
same request's previous position, else with its ring slot. Afterwards each
|
||||
request's last ``ring_size`` rows are saved to the ring, which a later step
|
||||
reads for its first partner.
|
||||
|
||||
:param extend_start_loc: int32 ``[num_reqs]``, each request's first row.
|
||||
:param extend_seq_lens: int32 ``[num_reqs]``, each request's row count.
|
||||
:return: ``out``, ``[num_tokens, head_dim]`` bf16. Only the rows of odd
|
||||
positions are written; the rest are left as they were.
|
||||
"""
|
||||
num_tokens, fused_dim = kv_input.shape
|
||||
head_dim = fused_dim // 2
|
||||
if out is None:
|
||||
out = kv_input.new_empty((num_tokens, head_dim), dtype=torch.bfloat16)
|
||||
|
||||
layout = KVLayout.parse(layout)
|
||||
module = _jit_c2_module(head_dim, freqs_cis.shape[-1], page_size, layout)
|
||||
module.prefill_fusion(
|
||||
kv_input,
|
||||
kv_state,
|
||||
out,
|
||||
norm_weight,
|
||||
positions,
|
||||
req,
|
||||
raw_out_loc,
|
||||
eps,
|
||||
freqs_cis,
|
||||
# HIP's fp8_e4m3_t is uint8_t, so the kernel matches the fp8 pool as bytes
|
||||
k_cache.view(torch.uint8) if is_hip_runtime() else k_cache,
|
||||
ring_size,
|
||||
extend_start_loc,
|
||||
extend_seq_lens,
|
||||
)
|
||||
return out
|
||||
|
||||
@@ -2930,6 +2930,22 @@ class DeepseekV4AttnBackend(
|
||||
self._low_ratio_compress_fused(
|
||||
layer, x, req, pos, draft_len=self.speculative_num_draft_tokens
|
||||
)
|
||||
elif (
|
||||
layer.compressor.use_fused_compress
|
||||
and forward_batch.forward_mode.is_extend_without_speculative()
|
||||
and forward_batch.extend_start_loc is not None
|
||||
):
|
||||
# DP padding extends extend_seq_lens but not extend_start_loc, whose
|
||||
# length is the request count; padded rows belong to no request.
|
||||
start_loc = forward_batch.extend_start_loc
|
||||
seq_lens = forward_batch.extend_seq_lens[: start_loc.shape[0]]
|
||||
self._low_ratio_compress_fused(
|
||||
layer,
|
||||
x,
|
||||
req,
|
||||
pos,
|
||||
extend_offsets=(start_loc.to(torch.int32), seq_lens.to(torch.int32)),
|
||||
)
|
||||
else:
|
||||
self._low_ratio_compress_torch(
|
||||
layer,
|
||||
@@ -3001,13 +3017,18 @@ class DeepseekV4AttnBackend(
|
||||
fuse_index_store=_is_sm100_or_newer(),
|
||||
)
|
||||
|
||||
def _low_ratio_compress_fused(self, layer, x, req, pos, *, draft_len=1) -> None:
|
||||
def _low_ratio_compress_fused(
|
||||
self, layer, x, req, pos, *, draft_len=1, extend_offsets=None
|
||||
) -> None:
|
||||
"""Decode or target-verify; an extend batch passes ``extend_offsets``, the int32
|
||||
(first row, row count) of each request."""
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer_rope import (
|
||||
index_k_norm_rope_pack_store,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.low_ratio_compress import (
|
||||
c1_decode_norm_rope_store,
|
||||
c2_decode_norm_rope_store,
|
||||
c2_prefill_norm_rope_store,
|
||||
)
|
||||
|
||||
pool = self.token_to_kv_pool
|
||||
@@ -3040,7 +3061,7 @@ class DeepseekV4AttnBackend(
|
||||
# CompressStatePool stores each request's pending pairs in a position ring.
|
||||
# KVAndScore rows use | kv | score |, addressed as req * ring_size + pos % ring_size.
|
||||
state = pool.get_attention_compress_states(layer_id)
|
||||
latent = c2_decode_norm_rope_store(
|
||||
args = (
|
||||
compressor.project_fused(x),
|
||||
state.kv_score_buffer.kv_score,
|
||||
compressor.norm.weight.data,
|
||||
@@ -3050,11 +3071,23 @@ class DeepseekV4AttnBackend(
|
||||
compressor.norm.eps,
|
||||
freqs_cis,
|
||||
kv_cache,
|
||||
page_size=page_size,
|
||||
ring_size=state.ring_size,
|
||||
draft_len=draft_len,
|
||||
layout=kv_layout,
|
||||
)
|
||||
if extend_offsets is None:
|
||||
latent = c2_decode_norm_rope_store(
|
||||
*args,
|
||||
page_size=page_size,
|
||||
ring_size=state.ring_size,
|
||||
draft_len=draft_len,
|
||||
layout=kv_layout,
|
||||
)
|
||||
else:
|
||||
latent = c2_prefill_norm_rope_store(
|
||||
*args,
|
||||
*extend_offsets,
|
||||
page_size=page_size,
|
||||
ring_size=state.ring_size,
|
||||
layout=kv_layout,
|
||||
)
|
||||
out_loc = core.c2_out_loc
|
||||
|
||||
indexer = layer.indexer
|
||||
|
||||
Reference in New Issue
Block a user