From 5916999afbc595de62dffca077fd8249634b4e74 Mon Sep 17 00:00:00 2001 From: DarkSharpness <76582120+DarkSharpness@users.noreply.github.com> Date: Sat, 3 Oct 2026 19:31:05 +0800 Subject: [PATCH] [DSv4.1] Fused c1/c2 compress for eager extend, faster c2 decode (#41660) Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: BBuf <1182563586@qq.com> --- .../kernels/jit/csrc/deepseek_v4/c2.cuh | 210 ++++++++++++++---- .../jit/csrc/distributed/nvlink_comm.cuh | 30 +-- .../kernels/jit/include/sgl_kernel/math.cuh | 47 ++++ .../ops/attention/dsv4/low_ratio_compress.py | 71 +++++- .../layers/attention/deepseek_v4_backend.py | 45 +++- .../attention/bench_dsv4_c2_decode.py | 122 ++++++++++ .../attention/dsv4/test_c2_prefill_ring.py | 191 ++++++++++++++++ 7 files changed, 642 insertions(+), 74 deletions(-) create mode 100644 test/registered/kernels/benchmark/attention/bench_dsv4_c2_decode.py create mode 100644 test/registered/kernels/ops/attention/dsv4/test_c2_prefill_ring.py diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/c2.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/c2.cuh index 60c96ba924..f9784dbd59 100644 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/c2.cuh +++ b/python/sglang/kernels/jit/csrc/deepseek_v4/c2.cuh @@ -18,6 +18,8 @@ namespace sglang { +using device::math::fast_mod_div_u32_t; + /// \brief Ratio-2 decode compressor: pair-pool, RMSNorm and the main-KV write. /// /// `kv_input` and `kv_state` rows are `2 * kHeadDim` floats, kv then score. @@ -34,8 +36,7 @@ struct Compress2DecodeParams { const int64_t* __restrict__ req; // [num_tokens], req_pool_idx per token const void* __restrict__ raw_out_loc; // [num_tokens] LocT, the FULL slot; 0 marks a padded row uint8_t* __restrict__ kvcache; // [npages, kPageBytes] uint8 - /// Positions per request slot in the pair-state ring. - uint32_t ring_size; + fast_mod_div_u32_t ring_size; // Positions per request slot in the pair-state ring. float eps; }; @@ -43,21 +44,32 @@ struct Compress2DecodeParams { /// At (512, 64) it also keeps the nope/rope split warp-aligned, as the fp8 amax reduction requires. constexpr uint32_t kC2VecSize = 2; -/// \brief grid = num_tokens, block = kHeadDim / kC2VecSize. +/// \brief Where a completing row finds its partner, and who parks a pending row. +enum class C2Mode { + /// One row per request: the partner comes from the ring, a pending row parks there. + Decode, + /// `draft_len` consecutive positions per request: `blockIdx.x` is the position inside the + /// block, `blockIdx.y` the request, and every row but the first takes its partner from the + /// previous `kv_input` row instead of the ring. + Verify, + /// Arbitrary rows, paired as `_low_ratio_pair_partners` pairs them: the partner is the previous + /// row when it is the same request's previous position, else the ring. A pending row does + /// nothing here; `c2_prefill_store_kernel` parks the ring rows after this kernel read them. + Prefill, +}; + +/// \brief grid = num_tokens (Decode, Prefill) or `draft_len x batch` (Verify), +/// block = kHeadDim / kC2VecSize. /// /// An odd position completes a group with its even predecessor; an even one parks in the state. /// -/// Under target-verify, `draft_len` consecutive positions per request: `blockIdx.x` is the -/// position inside the block, `blockIdx.y` the request, and every row but the first takes its -/// partner from the previous `kv_input` row instead of the ring. -/// /// The three reductions have different widths and are not interchangeable: the RMSNorm /// statistic spans the row, an fp8 store scale 64 elements, an fp4 block 16. /// /// kLayout is the cache's page format: V4 and V41 store the fake-quantized value; V41_FP4 /// stores the e2m1 codes and their e4m3 scales directly, so the fp4 rounding happens once. template < - bool kVerify, + C2Mode kMode, int64_t kHeadDim, int64_t kRopeDim, int32_t kPageBits, @@ -91,16 +103,21 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel( const auto tx = threadIdx.x; // Verify gives each request a CTA column; decode a flat grid of one row each. - const auto row = kVerify ? blockIdx.y * gridDim.x + blockIdx.x : blockIdx.x; + const auto row = kMode == C2Mode::Verify ? blockIdx.y * gridDim.x + blockIdx.x : blockIdx.x; PDLWaitPrimary(); - // Slots fit in int32 whatever width the scheduler hands them in. - const auto raw_out_loc = static_cast(static_cast(params.raw_out_loc)[row]); - // CUDA graph padding is a completely inert row: do not read schedule, input, - // state, or RoPE data, and do not publish output or update the cache. - if (raw_out_loc == 0) return PDLTriggerSecondary(); - const auto pos = static_cast(params.positions)[row]; - // A completing row reads the slot left by `pos - 1`; - // a pending row writes its own slot, so reads and writes stay disjoint. + const auto* loc_ptr = static_cast(params.raw_out_loc); + const auto* pos_ptr = static_cast(params.positions); + const auto raw_out_loc = loc_ptr[row]; + // Prefill rows are many, so a CUDA graph padding row exits before any load. + // Decode and verify rows are few: every row issues its loads to keep more in flight. + if constexpr (kMode == C2Mode::Prefill) { + if (raw_out_loc == 0) return PDLTriggerSecondary(); + } + const auto pos = pos_ptr[row]; + // Prefill leaves a pending row's latent unwritten; its index-K slot is -1, so nothing reads it. + if constexpr (kMode == C2Mode::Prefill) { + if ((pos & 1) == 0) return PDLTriggerSecondary(); + } const auto rid = params.req[row]; fp32_vec_t kv_new, score_new; @@ -108,21 +125,32 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel( score_new.load(params.kv_input + row * kStride, tx + kCTASize); const auto ring = static_cast(rid) * params.ring_size; - const auto read_row = ring + (pos - 1 + params.ring_size) % params.ring_size; - const auto write_row = ring + pos % params.ring_size; + const auto write_offset = static_cast(pos) % params.ring_size; + const auto read_offset = write_offset == 0 ? params.ring_size - 1 : write_offset - 1; + const auto write_row = ring + write_offset; + const auto read_row = ring + read_offset; fp32_vec_t kv_old, score_old; const float* partner = params.kv_state + read_row * kStride; - if constexpr (kVerify) { + if constexpr (kMode == C2Mode::Verify) { if (blockIdx.x != 0) partner = params.kv_input + static_cast(row - 1) * kStride; } + if constexpr (kMode == C2Mode::Prefill) { + if (row > 0 && params.req[row - 1] == rid && pos_ptr[row - 1] == pos - 1 && loc_ptr[row - 1] != 0) { + partner = params.kv_input + static_cast(row - 1) * kStride; + } + } kv_old.load(partner, tx); score_old.load(partner, tx + kCTASize); - if ((pos & 1) == 0) { - kv_new.store(params.kv_state + write_row * kStride, tx); - score_new.store(params.kv_state + write_row * kStride, tx + kCTASize); - return PDLTriggerSecondary(); + if constexpr (kMode != C2Mode::Prefill) { + // CUDA graph padding case + if (raw_out_loc == 0) return PDLTriggerSecondary(); + if ((pos & 1) == 0) { + kv_new.store(params.kv_state + write_row * kStride, tx); + score_new.store(params.kv_state + write_row * kStride, tx + kCTASize); + return PDLTriggerSecondary(); + } } constexpr uint32_t kNumWarps = kCTASize / kWarpThreads; @@ -203,9 +231,9 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel( } } + const int32_t out_loc = static_cast(raw_out_loc) >> 1; if constexpr (kLayout == KVLayout::V41_FP4) { // The fp4 cache takes the rotated bf16 value as is: its row quantizer is the fake quant, minus the dequant. - const int32_t out_loc = raw_out_loc >> 1; const auto kv_row = Paged::row(params.kvcache, out_loc); return deepseek_v4::v41::store_row(kv_row.data, kv_row.scale, tx, staged); } @@ -228,7 +256,6 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel( } // `raw_out_loc / ratio`; ratio 2 makes it a shift. - const int32_t out_loc = raw_out_loc >> 1; const auto kv_row = Paged::row(params.kvcache, out_loc); if constexpr (kLayout == KVLayout::V41) { @@ -264,21 +291,63 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel( } } +/// \brief Prefill ring write-back, launched after `flash_c2_decode_kernel`. +/// +/// Each request's last `ring_size` rows park their `| kv | score |` at `req * ring_size + +/// pos % ring_size` for a later step's first partner. It cannot run inside the pooling kernel: +/// a chunk whose first row is odd and whose last row is even reads and writes the same ring +/// slot. grid = `ring_size x num_reqs`, CTA (k, i) parks the k-th row from the end of request i, +/// one float4 per thread; those rows are consecutive positions, so their slots are distinct. +template +__global__ __launch_bounds__(kHeadDim / 2) void c2_prefill_store_kernel( + const Compress2DecodeParams params, + const int32_t* __restrict__ extend_start_loc, + const int32_t* __restrict__ extend_seq_lens) { + using namespace device; + constexpr int64_t kStride = kHeadDim * 2; + using vec_t = AlignedVector; + static_assert(kStride == 4 * (kHeadDim / 2)); + + // The pooling kernel reads the ring rows this one overwrites. + PDLWaitPrimary(); + const auto bx = static_cast(blockIdx.x); + const auto len = extend_seq_lens[blockIdx.y]; + const auto start_loc = extend_start_loc[blockIdx.y]; + if (bx >= len) return; + const auto row = start_loc + (len - 1 - bx); + const auto rid = params.req[row]; + const auto out = static_cast(params.raw_out_loc)[row]; + const auto pos = static_cast(static_cast(params.positions)[row]); + const auto dst_row = rid * params.ring_size + pos % params.ring_size; + if (out == 0) return; + vec_t buf; + buf.load(params.kv_input + row * kStride, threadIdx.x); + PDLTriggerSecondary(); + buf.store(params.kv_state + dst_row * kStride, threadIdx.x); +} + template struct FlashCompress2Kernel { static constexpr uint32_t kBlockSize = kHeadDim / kC2VecSize; static constexpr int32_t kPageBits = std::bit_width(kPageSize) - 1; static constexpr int64_t kPageBytes = deepseek_v4::kv_page_bytes(kPageSize); static_assert(kLayout != deepseek_v4::KVLayout::V4 || kPageBytes == host::div_ceil(584ll * kPageSize, 576) * 576); - template + template static constexpr auto kernel = - flash_c2_decode_kernel; + flash_c2_decode_kernel; + template + static constexpr auto store_kernel = c2_prefill_store_kernel; /// \brief The (`positions`, `raw_out_loc`) dtype pair, resolved at run time. - template + template static auto select(const bool pos_i32, const bool loc_i32) { - if (pos_i32) return loc_i32 ? kernel : kernel; - return loc_i32 ? kernel : kernel; + if (pos_i32) return loc_i32 ? kernel : kernel; + return loc_i32 ? kernel : kernel; + } + + static auto select_store(const bool pos_i32, const bool loc_i32) { + if (pos_i32) return loc_i32 ? store_kernel : store_kernel; + return loc_i32 ? store_kernel : store_kernel; } // The sum of squares reduces through a fixed-size shared array, so the CTA must be whole warps. @@ -302,7 +371,7 @@ struct FlashCompress2Kernel { const tvm::ffi::TensorView kvcache, const int64_t ring_size, const int64_t draft_len) { - launch( + launch( kv_input, kv_state, kv_output, @@ -317,7 +386,62 @@ struct FlashCompress2Kernel { draft_len); } + /// \brief The extend-batch pooling kernel, then the ring write-back. + /// + /// Rows may mix requests, prefix offsets and padding; a pending (even) row's `kv_output` + /// row is left unwritten. `extend_start_loc` / `extend_seq_lens` are int32 `[num_reqs]`: + /// request i owns rows `[start[i], start[i] + len[i])`; zero-length requests are skipped. + static void run_prefill_fusion( + const tvm::ffi::TensorView kv_input, + const tvm::ffi::TensorView kv_state, + const tvm::ffi::TensorView kv_output, + const tvm::ffi::TensorView norm_weight, + const tvm::ffi::TensorView positions, + const tvm::ffi::TensorView req, + const tvm::ffi::TensorView raw_out_loc, + const float eps, + const tvm::ffi::TensorView freqs_cis, + const tvm::ffi::TensorView kvcache, + const int64_t ring_size, + const tvm::ffi::TensorView extend_start_loc, + const tvm::ffi::TensorView extend_seq_lens) { + using namespace host; + auto B = SymbolicSize{"num_reqs"}; + auto device_ = SymbolicDevice{}; + device_.set_options(); + TensorMatcher({B}).with_dtype().with_device(device_).verify(extend_start_loc); + TensorMatcher({B}).with_dtype().with_device(device_).verify(extend_seq_lens); + const auto requests = StoreRequests{ + .extend_start_loc = static_cast(extend_start_loc.data_ptr()), + .extend_seq_lens = static_cast(extend_seq_lens.data_ptr()), + .num_reqs = static_cast(B.unwrap()), + }; + launch( + kv_input, + kv_state, + kv_output, + norm_weight, + positions, + req, + raw_out_loc, + eps, + ring_size, + freqs_cis, + kvcache, + 1, + &requests); + } + private: + struct StoreRequests { + const int32_t* extend_start_loc; + const int32_t* extend_seq_lens; + uint32_t num_reqs; + }; + + /// \tparam kPrefill `C2Mode::Prefill` plus the ring write-back of `requests`; otherwise + /// Decode, or Verify when `draft_len > 1`. + template static void launch( const tvm::ffi::TensorView kv_input, const tvm::ffi::TensorView kv_state, @@ -330,7 +454,8 @@ struct FlashCompress2Kernel { const int64_t ring_size, const tvm::ffi::TensorView freqs_cis, const tvm::ffi::TensorView kvcache, - const int64_t draft_len) { + const int64_t draft_len, + const StoreRequests* requests = nullptr) { using namespace host; auto N = SymbolicSize{"num_tokens"}; @@ -358,7 +483,7 @@ struct FlashCompress2Kernel { const auto num_tokens = static_cast(N.unwrap()); if (num_tokens == 0) return; - const auto is_verify = draft_len > 1; + const auto is_verify = !kPrefill && draft_len > 1; CHECK_HOST(ring_size > 0 && draft_len >= 1); CHECK_HOST(!is_verify || num_tokens % draft_len == 0); CHECK_HOST(!is_verify || ring_size > draft_len) @@ -373,19 +498,28 @@ struct FlashCompress2Kernel { .req = static_cast(req.data_ptr()), .raw_out_loc = raw_out_loc.data_ptr(), .kvcache = static_cast(kvcache.data_ptr()), - .ring_size = static_cast(ring_size), + .ring_size = static_cast(ring_size), .eps = eps, }; // `LaunchKernel` is move-only, so each arm builds its own. const auto pos_i32 = pos_dtype.is_type(); const auto loc_i32 = loc_dtype.is_type(); - if (is_verify) { + if constexpr (kPrefill) { + const auto k = select(pos_i32, loc_i32); + LaunchKernel(num_tokens, kBlockSize, device_.unwrap()) // + .enable_pdl(kUsePDL)(k, params); + if (requests->num_reqs == 0) return; + const auto store = select_store(pos_i32, loc_i32); + const auto grid = dim3{static_cast(ring_size), requests->num_reqs}; + LaunchKernel(grid, kHeadDim / 2, device_.unwrap()) // + .enable_pdl(kUsePDL)(store, params, requests->extend_start_loc, requests->extend_seq_lens); + } else if (is_verify) { const auto block = static_cast(draft_len); - const auto k = select(pos_i32, loc_i32); + const auto k = select(pos_i32, loc_i32); LaunchKernel(dim3{block, num_tokens / block}, kBlockSize, device_.unwrap()) // .enable_pdl(kUsePDL)(k, params); } else { - const auto k = select(pos_i32, loc_i32); + const auto k = select(pos_i32, loc_i32); LaunchKernel(num_tokens, kBlockSize, device_.unwrap()) // .enable_pdl(kUsePDL)(k, params); } diff --git a/python/sglang/kernels/jit/csrc/distributed/nvlink_comm.cuh b/python/sglang/kernels/jit/csrc/distributed/nvlink_comm.cuh index 9a2ae037d3..c846ecbedc 100644 --- a/python/sglang/kernels/jit/csrc/distributed/nvlink_comm.cuh +++ b/python/sglang/kernels/jit/csrc/distributed/nvlink_comm.cuh @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -21,31 +22,7 @@ namespace sglang { using device::distributed::PushWorkSpace; using device::distributed::Semaphore; - -// Runtime uint32 division as a multiply-high and a shift (round-up magic, -// exact below 2^31); cuda::fast_mod_div needs a newer CCCL than CUDA 13 bundles. -struct fast_mod_div_u32_t { - uint32_t divisor; - uint32_t magic; - uint32_t shift; - - __host__ explicit fast_mod_div_u32_t(uint32_t d) : divisor(d), magic(0), shift(0) { - if (d > 1) { - const uint32_t log2_ceil = 32 - std::countl_zero(d - 1); - const uint32_t p = 31 + log2_ceil; - magic = static_cast(((uint64_t{1} << p) + d - 1) / d); - shift = p - 32; - } - } - - __device__ friend uint32_t operator/(uint32_t n, const fast_mod_div_u32_t& fd) { - return fd.divisor == 1 ? n : __umulhi(n, fd.magic) >> fd.shift; - } - - __device__ friend uint32_t operator%(uint32_t n, const fast_mod_div_u32_t& fd) { - return n - (n / fd) * fd.divisor; - } -}; +using device::math::fast_mod_div_u32_t; template struct NVLinkCommPushParams { @@ -168,8 +145,7 @@ PUSH_KERNEL void nvlink_push_kernel(const __grid_constant__ NVLinkCommPushParams ptx::st_multimem_16B(vec, dst_ptr_mc, vid); } } else /* reduce-scatter only */ { - const auto token_id = vid / params.vecs_per_token_div; - const auto offset = vid % params.vecs_per_token_div; + const auto [token_id, offset] = math::div(vid, params.vecs_per_token_div); // Both by a compile-time constant, so this is a mask and a shift. const auto dst_rank = token_id % kWorldSize; const auto dst_token_id = token_id / kWorldSize; diff --git a/python/sglang/kernels/jit/include/sgl_kernel/math.cuh b/python/sglang/kernels/jit/include/sgl_kernel/math.cuh index 92f4a40c76..cb8d381b94 100644 --- a/python/sglang/kernels/jit/include/sgl_kernel/math.cuh +++ b/python/sglang/kernels/jit/include/sgl_kernel/math.cuh @@ -112,6 +112,53 @@ SGL_DEVICE float fma_f32_bf16(bf16_t a, float b, float acc) { return fmaf(cast(a), b, acc); } +/** + * \brief Fast uint32 division and modulo by a compile-time constant divisor. + * + * This struct is only exact when the divisor is less than `2^31`. Larger dividends + * will lead to incorrect results. Be careful when using this struct. + */ +struct fast_mod_div_u32_t { + public: + explicit constexpr fast_mod_div_u32_t(uint32_t d) : m_divisor(d) { + if (d > 1) { + const uint32_t log2_ceil = 32 - std::countl_zero(d - 1); + const uint32_t p = 31 + log2_ceil; + m_magic = static_cast(((uint64_t{1} << p) + d - 1) / d); + m_shift = p - 32; + } else { + m_magic = 0; + m_shift = 0; + } + } + + template + SGL_DEVICE friend uint32_t operator/(T n, const fast_mod_div_u32_t& d) { + static_assert(std::is_unsigned_v && sizeof(T) <= sizeof(uint32_t)); + return d.m_divisor == 1 ? n : __umulhi(n, d.m_magic) >> d.m_shift; + } + + template + SGL_DEVICE friend uint32_t operator%(T n, const fast_mod_div_u32_t& d) { + static_assert(std::is_unsigned_v && sizeof(T) <= sizeof(uint32_t)); + return n - (n / d) * d.m_divisor; + } + + constexpr operator uint32_t() const { + return m_divisor; + } + + private: + uint32_t m_divisor; + uint32_t m_magic; + uint32_t m_shift; +}; + +template +SGL_DEVICE uint2 div(T n, const fast_mod_div_u32_t& d) { + return {n / d, n % d}; +} + } // namespace device::math } // namespace sglang diff --git a/python/sglang/kernels/ops/attention/dsv4/low_ratio_compress.py b/python/sglang/kernels/ops/attention/dsv4/low_ratio_compress.py index fec2f2846e..2f0ed270b8 100644 --- a/python/sglang/kernels/ops/attention/dsv4/low_ratio_compress.py +++ b/python/sglang/kernels/ops/attention/dsv4/low_ratio_compress.py @@ -1,5 +1,6 @@ -"""Fused ratio-1 and ratio-2 decode compressors: RMSNorm, RoPE and the FlashMLA -cache write in one launch. +"""Fused ratio-1 and ratio-2 compressors: RMSNorm, RoPE and the FlashMLA cache +write in one launch. The ratio-1 kernel is per token, so decode and extend share +it; ratio 2 has a decode/verify entry point and an extend one. Ratio 1 takes the bf16 ``wkv`` projection as is: RoPE uses the token's own position and the compressed slot equals the FULL slot. Ratio 2 pair-pools the @@ -121,11 +122,12 @@ def _jit_c2_module( is_arch_support_pdl(), ) return load_jit( - make_name("c2_decode"), + make_name("c2"), *args, cuda_files=["deepseek_v4/c2.cuh"], cuda_wrappers=[ ("decode_fusion", f"FlashCompress2Kernel<{args}>::run_decode_fusion"), + ("prefill_fusion", f"FlashCompress2Kernel<{args}>::run_prefill_fusion"), ], ) @@ -151,6 +153,12 @@ def c2_decode_norm_rope_store( The cache store uses ``raw_out_loc // 2`` as its slot. + A padded graph row is marked by ``raw_out_loc == 0`` and writes nothing, but + decode and verify rows issue their ring load before that check, so a padded + row's ``req`` and ``positions`` must still address an in-bounds ring slot. + The decode graph buffers zero both tails (``PaddingPolicy.ZERO``); a runner + that left stale request indices there would read out of the ring pool. + :param freqs_cis: ``[max_pos, rope_dim]`` fp32, real/imag interleaved -- ``torch.view_as_real(freqs).flatten(-2)``. Indexed in-kernel at ``positions - 1``, the position the latent @@ -184,3 +192,60 @@ def c2_decode_norm_rope_store( draft_len, ) return out + + +def c2_prefill_norm_rope_store( + kv_input: torch.Tensor, + kv_state: torch.Tensor, + norm_weight: torch.Tensor, + positions: torch.Tensor, + req: torch.Tensor, + raw_out_loc: torch.Tensor, + eps: float, + freqs_cis: torch.Tensor, + k_cache: torch.Tensor, + extend_start_loc: torch.Tensor, + extend_seq_lens: torch.Tensor, + *, + page_size: int, + ring_size: int, + layout: Union[KVLayout, str] = KVLayout.V4, + out: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """:func:`c2_decode_norm_rope_store` for an extend batch, then the ring write-back. + + Rows may mix requests, prefix offsets and CUDA graph padding (``raw_out_loc + == 0``). An odd position pairs with the previous row when that row is the + same request's previous position, else with its ring slot. Afterwards each + request's last ``ring_size`` rows are saved to the ring, which a later step + reads for its first partner. + + :param extend_start_loc: int32 ``[num_reqs]``, each request's first row. + :param extend_seq_lens: int32 ``[num_reqs]``, each request's row count. + :return: ``out``, ``[num_tokens, head_dim]`` bf16. Only the rows of odd + positions are written; the rest are left as they were. + """ + num_tokens, fused_dim = kv_input.shape + head_dim = fused_dim // 2 + if out is None: + out = kv_input.new_empty((num_tokens, head_dim), dtype=torch.bfloat16) + + layout = KVLayout.parse(layout) + module = _jit_c2_module(head_dim, freqs_cis.shape[-1], page_size, layout) + module.prefill_fusion( + kv_input, + kv_state, + out, + norm_weight, + positions, + req, + raw_out_loc, + eps, + freqs_cis, + # HIP's fp8_e4m3_t is uint8_t, so the kernel matches the fp8 pool as bytes + k_cache.view(torch.uint8) if is_hip_runtime() else k_cache, + ring_size, + extend_start_loc, + extend_seq_lens, + ) + return out diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index 6f3d1baa91..38d33033f2 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -2803,6 +2803,22 @@ class DeepseekV4AttnBackend( self._low_ratio_compress_fused( layer, x, req, pos, draft_len=self.speculative_num_draft_tokens ) + elif ( + layer.compressor.use_fused_compress + and forward_batch.forward_mode.is_extend_without_speculative() + and forward_batch.extend_start_loc is not None + ): + # DP padding extends extend_seq_lens but not extend_start_loc, whose + # length is the request count; padded rows belong to no request. + start_loc = forward_batch.extend_start_loc + seq_lens = forward_batch.extend_seq_lens[: start_loc.shape[0]] + self._low_ratio_compress_fused( + layer, + x, + req, + pos, + extend_offsets=(start_loc.to(torch.int32), seq_lens.to(torch.int32)), + ) else: self._low_ratio_compress_torch( layer, @@ -2872,13 +2888,18 @@ class DeepseekV4AttnBackend( fuse_index_store=is_sm100_or_newer(), ) - def _low_ratio_compress_fused(self, layer, x, req, pos, *, draft_len=1) -> None: + def _low_ratio_compress_fused( + self, layer, x, req, pos, *, draft_len=1, extend_offsets=None + ) -> None: + """Decode or target-verify; an extend batch passes ``extend_offsets``, the int32 + (first row, row count) of each request.""" from sglang.kernels.ops.attention.dsv4.fp4_indexer_rope import ( index_k_norm_rope_pack_store, ) from sglang.kernels.ops.attention.dsv4.low_ratio_compress import ( c1_decode_norm_rope_store, c2_decode_norm_rope_store, + c2_prefill_norm_rope_store, ) pool = self.token_to_kv_pool @@ -2911,7 +2932,7 @@ class DeepseekV4AttnBackend( # CompressStatePool stores each request's pending pairs in a position ring. # KVAndScore rows use | kv | score |, addressed as req * ring_size + pos % ring_size. state = pool.get_attention_compress_states(layer_id) - latent = c2_decode_norm_rope_store( + args = ( compressor.project_fused(x), state.kv_score_buffer.kv_score, compressor.norm.weight.data, @@ -2921,11 +2942,23 @@ class DeepseekV4AttnBackend( compressor.norm.eps, freqs_cis, kv_cache, - page_size=page_size, - ring_size=state.ring_size, - draft_len=draft_len, - layout=kv_layout, ) + if extend_offsets is None: + latent = c2_decode_norm_rope_store( + *args, + page_size=page_size, + ring_size=state.ring_size, + draft_len=draft_len, + layout=kv_layout, + ) + else: + latent = c2_prefill_norm_rope_store( + *args, + *extend_offsets, + page_size=page_size, + ring_size=state.ring_size, + layout=kv_layout, + ) out_loc = core.c2_out_loc indexer = layer.indexer diff --git a/test/registered/kernels/benchmark/attention/bench_dsv4_c2_decode.py b/test/registered/kernels/benchmark/attention/bench_dsv4_c2_decode.py new file mode 100644 index 0000000000..5bb050b59e --- /dev/null +++ b/test/registered/kernels/benchmark/attention/bench_dsv4_c2_decode.py @@ -0,0 +1,122 @@ +"""Benchmark the fused ratio-2 decode compressor (``c2_decode_norm_rope_store``). + +One CTA per token: pair-pool against the state ring, RMSNorm, RoPE, quantize and +store one compressed KV row. The line axis is the cache page format. + +Shape axes: + - ``bs``: requests per batch. + - ``draft_len``: 1 is plain decode; > 1 is a target-verify block of that many + consecutive positions per request. + - ``parity``: the first position's parity. On decode, ``odd`` rows complete a + pair and run the full norm/RoPE/store path, ``even`` rows only park in the ring. + +Constants follow the DeepSeek-V4 deployment: ``head_dim = 512``, ``rope_dim = 64``, +a 256-token FULL page, hence 128 compressed slots per page. +""" + +import torch + +from sglang.kernels.jit.benchmark import marker +from sglang.kernels.jit.benchmark.utils import ( + DEFAULT_DEVICE, + create_empty, + create_random, +) +from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout +from sglang.kernels.ops.attention.dsv4.low_ratio_compress import ( + c2_decode_norm_rope_store, +) +from sglang.srt.mem_cache.deepseek_v4_memory_pool import get_compress_state_ring_size +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci( + est_time=30, stage="base-b-kernel-benchmark", runner_config="1-gpu-large" +) + +HEAD_DIM = 512 +ROPE_DIM = 64 +PAGE_SIZE = 256 // 2 +MAX_POS = 65536 +EPS = 1e-6 + + +@marker.parametrize("parity", ["odd", "even", "half"], ["half"]) +@marker.parametrize("draft_len", [1, 2, 4, 8], [1, 8]) +@marker.parametrize("bs", marker.range(14, pattern="pow2"), [1, 1024]) +@marker.benchmark("layout", ["V4", "V41", "V41_FP4"]) +def benchmark(bs: int, draft_len: int, parity: str, layout: str): + if parity == "half" and bs % 2 != 0: + marker.skip("half parity requires even batch size") + layout = KVLayout[layout] + num_tokens = bs * draft_len + ring_size = get_compress_state_ring_size(2, draft_len if draft_len > 1 else 0) + torch.manual_seed(0) + + kv_input = create_random(num_tokens, 2 * HEAD_DIM, dtype=torch.float32) + kv_state = create_random(bs * ring_size, 2 * HEAD_DIM, dtype=torch.float32) + norm_weight = create_random(HEAD_DIM) + freqs_cis = create_random(MAX_POS, ROPE_DIM, dtype=torch.float32) + req_ids = torch.randperm(bs, device=DEFAULT_DEVICE) + start = torch.randint(1, MAX_POS // 2 - draft_len, (bs,), device=DEFAULT_DEVICE) + if parity == "odd": + start = start * 2 + 1 + elif parity == "even": + start = start * 2 + else: + start = start * 2 + start[::2] += 1 # half odd, half even + + offsets = torch.arange(draft_len, device=DEFAULT_DEVICE) + positions = (start[:, None] + offsets).flatten() + req = req_ids.repeat_interleave(draft_len) + + # Distinct scattered slots; slot 0 is left alone, it marks a padded row. + num_slots = num_tokens + 1 + num_pages = -(-num_slots // PAGE_SIZE) + slots = torch.randperm(num_pages * PAGE_SIZE - 1, device=DEFAULT_DEVICE) + slots = slots[:num_tokens] + raw_out_loc = (slots + 1) * 2 + 1 + k_cache = create_empty( + num_pages, + layout.page_bytes(PAGE_SIZE), + dtype=torch.uint8, + ) + out = create_empty(num_tokens, HEAD_DIM) + + # Traffic of the rows that complete a pair; parked rows write the state instead. + num_complete = int((positions % 2 == 1).sum()) + num_parked = num_tokens - num_complete + row_bytes = 2 * HEAD_DIM * 4 + complete_bytes = ( + 2 * row_bytes # kv_input + partner + + ROPE_DIM * 4 # freqs_cis row + + HEAD_DIM * 2 # out + + layout.bytes_per_token + ) + footprint = ( + num_complete * complete_bytes + + num_parked * 3 * row_bytes # kv_input + partner read, state write + + num_tokens * 3 * 8 # positions, req, raw_out_loc + + norm_weight.nbytes + ) + return marker.do_bench( + c2_decode_norm_rope_store, + input_args=(kv_input, kv_state, norm_weight, positions, req, raw_out_loc), + input_kwargs=dict( + eps=EPS, + freqs_cis=freqs_cis, + k_cache=k_cache, + page_size=PAGE_SIZE, + ring_size=ring_size, + draft_len=draft_len, + layout=layout, + out=out, + ), + memory_args=None, + memory_output=None, + extra_memory_footprint=footprint, + ) + + +if __name__ == "__main__": + benchmark.run() diff --git a/test/registered/kernels/ops/attention/dsv4/test_c2_prefill_ring.py b/test/registered/kernels/ops/attention/dsv4/test_c2_prefill_ring.py new file mode 100644 index 0000000000..598011ca31 --- /dev/null +++ b/test/registered/kernels/ops/attention/dsv4/test_c2_prefill_ring.py @@ -0,0 +1,191 @@ +"""The ratio-2 compressor's eager-extend path against its decode path. + +``c2_prefill_norm_rope_store`` pairs rows inside a chunk and parks each request's +last ``ring_size`` rows with a separate write-back kernel, while +``c2_decode_norm_rope_store`` takes every partner from the ring and parks every +pending row there. Driving one token stream both ways must leave the same main-KV +bytes and the same latents, so a chunk boundary that parks the wrong rows shows up +in the next chunk's output instead of staying invisible in the first chunk's. +""" + +import unittest +from typing import List, Tuple + +import torch + +from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout +from sglang.kernels.ops.attention.dsv4.low_ratio_compress import ( + c2_decode_norm_rope_store, + c2_prefill_norm_rope_store, +) +from sglang.srt.utils import is_cuda +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +# The c2 JIT kernels are the SM100 build; the only Blackwell runner config is the +# four-GPU one. +register_cuda_ci(est_time=60, stage="base-b-kernel-unit", runner_config="4-gpu-b200") + +HEAD_DIM = 512 +ROPE_DIM = 64 +PAGE_SIZE = 64 +NUM_PAGES = 16 +EPS = 1e-6 +# Request r's full slots start here, even so that a (2k, 2k+1) pair shares a +# compressed slot; never 0, which marks a padded row. +SLOT_BASE = 64 + + +def _sm100() -> bool: + return is_cuda() and torch.cuda.get_device_capability()[0] == 10 + + +class Stream: + """One token stream: per-request prefix and chunk sizes, laid out as the rows + the backend would hand each call.""" + + def __init__(self, prefixes: List[int], chunks: List[List[int]]): + self.prefixes = prefixes + self.chunks = chunks # chunks[c][r] = rows of request r in chunk c + + def chunk_rows(self, c: int) -> List[Tuple[int, int]]: + """(request, position) of every row of chunk ``c``, requests in order.""" + rows = [] + for r, n in enumerate(self.chunks[c]): + start = self.prefixes[r] + sum(self.chunks[j][r] for j in range(c)) + rows.extend((r, start + i) for i in range(n)) + return rows + + +def _make_inputs(gen, rows: int) -> torch.Tensor: + """``[rows, 2 * HEAD_DIM]`` fp32: the kv half then the score half.""" + return torch.randn( + rows, 2 * HEAD_DIM, generator=gen, device="cuda", dtype=torch.float32 + ) + + +def _row_tensors(rows: List[Tuple[int, int]], pad: int): + """``req``, ``positions`` and ``raw_out_loc`` for ``rows``, plus ``pad`` padded + rows. A padded row still has to address a live ring slot, so it takes request 0 + and position 0; only ``raw_out_loc == 0`` marks it.""" + req = [r for r, _ in rows] + [0] * pad + pos = [p for _, p in rows] + [0] * pad + loc = [SLOT_BASE * (1 + r) + p for r, p in rows] + [0] * pad + to = lambda v, d: torch.tensor(v, device="cuda", dtype=d) + return to(req, torch.int64), to(pos, torch.int32), to(loc, torch.int32) + + +class TestC2PrefillRing(CustomTestCase): + def _run(self, stream: Stream, layout: KVLayout, ring_size: int, seed: int): + """The fused chunked path and the row-at-a-time decode path over the same + stream; returns (cache, latents) for each.""" + gen = torch.Generator(device="cuda").manual_seed(seed) + num_reqs = len(stream.prefixes) + weight = torch.randn( + HEAD_DIM, generator=gen, device="cuda", dtype=torch.float32 + ).to(torch.bfloat16) + angles = torch.randn(4096, ROPE_DIM // 2, generator=gen, device="cuda") + freqs = torch.view_as_real( + torch.polar(torch.ones_like(angles), angles) + ).flatten(-2) + + # One fp32 input row per (request, position) in the stream, shared by both + # paths so any difference is the pairing, not the data. + all_rows = [ + row for c in range(len(stream.chunks)) for row in stream.chunk_rows(c) + ] + inputs = {row: _make_inputs(gen, 1)[0] for row in all_rows} + + state_rows = num_reqs * ring_size + ring_size + caches, latents = [], [] + for fused in (True, False): + cache = torch.zeros( + NUM_PAGES, + layout.page_bytes(PAGE_SIZE), + dtype=torch.uint8, + device="cuda", + ) + state = torch.zeros( + state_rows, 2 * HEAD_DIM, dtype=torch.float32, device="cuda" + ) + out = {} + if fused: + for c in range(len(stream.chunks)): + rows = stream.chunk_rows(c) + pad = 2 + req, pos, loc = _row_tensors(rows, pad) + kv = torch.stack( + [inputs[row] for row in rows] + + [torch.zeros(2 * HEAD_DIM, device="cuda")] * pad + ) + starts, lens = [], [] + at = 0 + for r in range(num_reqs): + starts.append(at) + lens.append(stream.chunks[c][r]) + at += stream.chunks[c][r] + latent = c2_prefill_norm_rope_store( + kv, + state, + weight, + pos, + req, + loc, + EPS, + freqs, + cache, + torch.tensor(starts, device="cuda", dtype=torch.int32), + torch.tensor(lens, device="cuda", dtype=torch.int32), + page_size=PAGE_SIZE, + ring_size=ring_size, + layout=layout, + ) + for i, row in enumerate(rows): + out[row] = latent[i].clone() + else: + for row in all_rows: + req, pos, loc = _row_tensors([row], 0) + latent = c2_decode_norm_rope_store( + inputs[row][None, :], + state, + weight, + pos, + req, + loc, + EPS, + freqs, + cache, + page_size=PAGE_SIZE, + ring_size=ring_size, + layout=layout, + ) + out[row] = latent[0].clone() + caches.append(cache) + # Only a completing (odd position) row writes a latent. + latents.append( + torch.stack([out[row] for row in all_rows if row[1] % 2 == 1]) + ) + return caches, latents + + @unittest.skipUnless(_sm100(), "the c2 JIT kernels are the SM100 build") + def test_chunked_extend_matches_decode(self): + # Request 0 starts even, request 1 odd and spans more rows than the ring, + # request 2 odd with a single row in its first chunk. + stream = Stream( + prefixes=[0, 7, 1], + chunks=[[5, 4, 1], [3, 6, 2], [1, 1, 1]], + ) + for layout in (KVLayout.V41, KVLayout.V41_FP4): + for ring_size in (2, 8): + with self.subTest(layout=layout.name, ring_size=ring_size): + caches, latents = self._run(stream, layout, ring_size, seed=7) + self.assertTrue( + torch.equal(caches[0], caches[1]), + f"main-KV bytes differ: {(caches[0] != caches[1]).sum()} of " + f"{caches[0].numel()}", + ) + self.assertTrue(torch.equal(latents[0], latents[1])) + + +if __name__ == "__main__": + unittest.main()