[Qwen4-Exp] Optional fp8 (e4m3) storage for the compressed QSA indexer cache (#39614)
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: YAMY1234 <74099316+YAMY1234@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
YAMY1234
parent
88f95f4b87
commit
eb9c9ee99d
@@ -13,6 +13,7 @@
|
||||
#include <tvm/ffi/container/tensor.h>
|
||||
|
||||
#include <cstdint>
|
||||
#include <type_traits>
|
||||
|
||||
namespace sglang {
|
||||
|
||||
@@ -46,6 +47,18 @@ SGL_DEVICE float eager_round<fp16_t>(float x) {
|
||||
#endif
|
||||
}
|
||||
|
||||
/// \brief Convert a value computed in the compute dtype to the cache storage
|
||||
/// dtype. Identity when both agree. The fp8 compressed cache takes a plain
|
||||
/// e4m3 cast with no scale, as the row is RMSNormed before it gets here.
|
||||
template <typename TOut, typename T>
|
||||
SGL_DEVICE TOut qsa_storage_cast(T value) {
|
||||
if constexpr (std::is_same_v<TOut, T>) {
|
||||
return value;
|
||||
} else {
|
||||
return DTypeTrait<TOut>::from(static_cast<float>(value));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief Apply (M)RoPE to a normed row staged in shared memory.
|
||||
*
|
||||
@@ -54,7 +67,8 @@ SGL_DEVICE float eager_round<fp16_t>(float x) {
|
||||
* uses all-zero maps; Qwen interleaved/sectioned MRoPE maps are built on the
|
||||
* host). The cos/sin cache row is [cos(half), sin(half)] of width rotary_dim.
|
||||
*
|
||||
* \tparam T Storage element type: bf16_t | fp16_t.
|
||||
* \tparam T Compute/ring element type: bf16_t | fp16_t.
|
||||
* \tparam TOut Destination storage type: T, or fp8_e4m3_t for the fp8 compressed cache.
|
||||
* \tparam kHeadDim Compile-time head dimension (multiple of 32).
|
||||
* \tparam kIsNeox true -> NeoX pairing (d, d+half); false -> GPT-J (2i, 2i+1).
|
||||
* \param smem_row Normed row [kHeadDim], one warp cooperates.
|
||||
@@ -64,21 +78,21 @@ SGL_DEVICE float eager_round<fp16_t>(float x) {
|
||||
* \param pos Resolved per-axis positions for this token (>= 3 entries).
|
||||
* \param rotary_dim Rotated prefix length; tail dims pass through.
|
||||
*/
|
||||
template <typename T, int kHeadDim, bool kIsNeox>
|
||||
template <typename T, typename TOut, int kHeadDim, bool kIsNeox>
|
||||
SGL_DEVICE void qsa_mrope_apply(
|
||||
const T* __restrict__ smem_row,
|
||||
T* __restrict__ out_row,
|
||||
TOut* __restrict__ out_row,
|
||||
const float* __restrict__ cos_sin_cache,
|
||||
const int32_t* __restrict__ axis_map,
|
||||
const int64_t* pos,
|
||||
const int32_t rotary_dim) {
|
||||
using namespace device;
|
||||
constexpr int kPerLane = kHeadDim / kWarpThreads;
|
||||
using vec_t = AlignedVector<T, kPerLane>;
|
||||
using out_vec_t = AlignedVector<TOut, kPerLane>;
|
||||
const uint32_t lane = threadIdx.x % kWarpThreads;
|
||||
const int32_t half = rotary_dim / 2;
|
||||
|
||||
vec_t ov;
|
||||
out_vec_t ov;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kPerLane; ++i) {
|
||||
const int32_t d = static_cast<int32_t>(lane * kPerLane) + i;
|
||||
@@ -119,7 +133,7 @@ SGL_DEVICE void qsa_mrope_apply(
|
||||
o = smem_row[d];
|
||||
}
|
||||
}
|
||||
ov[i] = o;
|
||||
ov[i] = qsa_storage_cast<TOut>(o);
|
||||
}
|
||||
ov.store(out_row, lane); // offset is in vector units
|
||||
}
|
||||
@@ -203,11 +217,12 @@ struct QsaIndexQPrepParams {
|
||||
* head, zero-fill of padded heads, raw token-K store and RoPE-position store.
|
||||
* One CTA (4 warps) per token; one warp per query head.
|
||||
*/
|
||||
template <typename T, int kHeadDim, bool kIsNeox, bool kUsePDL>
|
||||
template <typename T, typename TOut, int kHeadDim, bool kIsNeox, bool kUsePDL>
|
||||
__global__ __launch_bounds__(128) void qsa_index_q_prep_kernel(const QsaIndexQPrepParams __grid_constant__ params) {
|
||||
using namespace device;
|
||||
constexpr int kPerLane = kHeadDim / kWarpThreads;
|
||||
using vec_t = AlignedVector<T, kPerLane>;
|
||||
using out_vec_t = AlignedVector<TOut, kPerLane>;
|
||||
const uint32_t token = blockIdx.x;
|
||||
const uint32_t warp = threadIdx.x / kWarpThreads;
|
||||
const uint32_t lane = threadIdx.x % kWarpThreads;
|
||||
@@ -225,15 +240,16 @@ __global__ __launch_bounds__(128) void qsa_index_q_prep_kernel(const QsaIndexQPr
|
||||
}
|
||||
|
||||
for (int32_t h = static_cast<int32_t>(warp); h < params.q_heads_padded; h += 4) {
|
||||
T* out_row = static_cast<T*>(params.q_out) + (static_cast<int64_t>(token) * params.q_heads_padded + h) * kHeadDim;
|
||||
TOut* out_row =
|
||||
static_cast<TOut*>(params.q_out) + (static_cast<int64_t>(token) * params.q_heads_padded + h) * kHeadDim;
|
||||
if (h < params.num_q_heads) {
|
||||
const T* x_row = static_cast<const T*>(params.qk) + qk_row + h * kHeadDim;
|
||||
qsa_gemma_norm_row<T, kHeadDim>(x_row, static_cast<const T*>(params.weight), params.eps, smem_rows[warp]);
|
||||
qsa_mrope_apply<T, kHeadDim, kIsNeox>(
|
||||
qsa_mrope_apply<T, TOut, kHeadDim, kIsNeox>(
|
||||
smem_rows[warp], out_row, params.cos_sin_cache, params.axis_map, pos, params.rotary_dim);
|
||||
} else {
|
||||
vec_t zv;
|
||||
zv.fill(DTypeTrait<T>::from(0.0f));
|
||||
out_vec_t zv;
|
||||
zv.fill(DTypeTrait<TOut>::from(0.0f));
|
||||
zv.store(out_row, lane); // offset is in vector units
|
||||
}
|
||||
}
|
||||
@@ -274,7 +290,7 @@ struct QsaIndexKCompressParams {
|
||||
* MRoPE at the group-start position, store into the compressed cache.
|
||||
* One warp per group.
|
||||
*/
|
||||
template <typename T, int kHeadDim, bool kIsNeox, bool kUsePDL>
|
||||
template <typename T, typename TOut, int kHeadDim, bool kIsNeox, bool kUsePDL>
|
||||
__global__
|
||||
__launch_bounds__(128) void qsa_index_k_compress_kernel(const QsaIndexKCompressParams __grid_constant__ params) {
|
||||
using namespace device;
|
||||
@@ -345,8 +361,9 @@ __launch_bounds__(128) void qsa_index_k_compress_kernel(const QsaIndexKCompressP
|
||||
pos[a] = params.rope_position_buffer[static_cast<int64_t>(loc0) * 3 + a];
|
||||
}
|
||||
|
||||
T* out_row = static_cast<T*>(params.compressed_k_buffer) + static_cast<int64_t>(params.write_locs[group]) * kHeadDim;
|
||||
qsa_mrope_apply<T, kHeadDim, kIsNeox>(
|
||||
TOut* out_row =
|
||||
static_cast<TOut*>(params.compressed_k_buffer) + static_cast<int64_t>(params.write_locs[group]) * kHeadDim;
|
||||
qsa_mrope_apply<T, TOut, kHeadDim, kIsNeox>(
|
||||
smem_rows[warp], out_row, params.cos_sin_cache, params.axis_map, pos, params.rotary_dim);
|
||||
|
||||
device::PDLTriggerSecondary<kUsePDL>();
|
||||
@@ -355,12 +372,13 @@ __launch_bounds__(128) void qsa_index_k_compress_kernel(const QsaIndexKCompressP
|
||||
/**
|
||||
* \brief Validate inputs and launch `qsa_index_q_prep_kernel` (one CTA per token).
|
||||
*
|
||||
* \tparam T Element type: bf16_t | fp16_t.
|
||||
* \tparam T Input/ring element type: bf16_t | fp16_t.
|
||||
* \tparam TOut Output storage type: T, or fp8_e4m3_t (fp8 indexer cache).
|
||||
* \tparam kHeadDim Index head dimension: 64 | 128 | 256.
|
||||
* \tparam kIsNeox RoPE pairing style.
|
||||
* \tparam kUsePDL Whether to launch with PDL enabled.
|
||||
*/
|
||||
template <typename T, int kHeadDim, bool kIsNeox, bool kUsePDL>
|
||||
template <typename T, typename TOut, int kHeadDim, bool kIsNeox, bool kUsePDL>
|
||||
void qsa_index_q_prep(
|
||||
tvm::ffi::TensorView qk,
|
||||
tvm::ffi::TensorView q_out,
|
||||
@@ -383,7 +401,7 @@ void qsa_index_q_prep(
|
||||
|
||||
TensorMatcher({tokens, (num_q_heads + 1) * D}).with_dtype<T>().with_device(device).verify(qk);
|
||||
auto heads_padded = SymbolicSize{"heads_padded"};
|
||||
TensorMatcher({tokens, heads_padded, D}).with_dtype<T>().with_device(device).verify(q_out);
|
||||
TensorMatcher({tokens, heads_padded, D}).with_dtype<TOut>().with_device(device).verify(q_out);
|
||||
TensorMatcher({D}).with_dtype<T>().with_device(device).verify(weight);
|
||||
auto cache_rows = SymbolicSize{"cos_sin_cache_rows"};
|
||||
TensorMatcher({cache_rows, rotary_dim}).with_dtype<fp32_t>().with_device(device).verify(cos_sin_cache);
|
||||
@@ -421,13 +439,13 @@ void qsa_index_q_prep(
|
||||
.eps = eps,
|
||||
};
|
||||
LaunchKernel(static_cast<uint32_t>(num_tokens), 128, device.unwrap())
|
||||
.enable_pdl(kUsePDL)(qsa_index_q_prep_kernel<T, kHeadDim, kIsNeox, kUsePDL>, params);
|
||||
.enable_pdl(kUsePDL)(qsa_index_q_prep_kernel<T, TOut, kHeadDim, kIsNeox, kUsePDL>, params);
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief Validate inputs and launch `qsa_index_k_compress_kernel` (one warp per group).
|
||||
*/
|
||||
template <typename T, int kHeadDim, bool kIsNeox, bool kUsePDL>
|
||||
template <typename T, typename TOut, int kHeadDim, bool kIsNeox, bool kUsePDL>
|
||||
void qsa_index_k_compress(
|
||||
tvm::ffi::TensorView key_state_buffer,
|
||||
tvm::ffi::TensorView group_locs,
|
||||
@@ -456,7 +474,7 @@ void qsa_index_k_compress(
|
||||
TensorMatcher({D}).with_dtype<T>().with_device(device).verify(weight);
|
||||
TensorMatcher({groups}).with_dtype<int32_t>().with_device(device).verify(write_locs);
|
||||
auto compressed_slots = SymbolicSize{"compressed_slots"};
|
||||
TensorMatcher({compressed_slots, D}).with_dtype<T>().with_device(device).verify(compressed_k_buffer);
|
||||
TensorMatcher({compressed_slots, D}).with_dtype<TOut>().with_device(device).verify(compressed_k_buffer);
|
||||
|
||||
const int64_t num_groups = groups.unwrap();
|
||||
CHECK_HOST(num_groups > 0) << "qsa_index_k_compress: no groups";
|
||||
@@ -480,7 +498,7 @@ void qsa_index_k_compress(
|
||||
.eps = eps,
|
||||
};
|
||||
LaunchKernel(static_cast<uint32_t>(div_ceil(num_groups, 4)), 128, device.unwrap())
|
||||
.enable_pdl(kUsePDL)(qsa_index_k_compress_kernel<T, kHeadDim, kIsNeox, kUsePDL>, params);
|
||||
.enable_pdl(kUsePDL)(qsa_index_k_compress_kernel<T, TOut, kHeadDim, kIsNeox, kUsePDL>, params);
|
||||
}
|
||||
|
||||
} // namespace sglang
|
||||
|
||||
@@ -9,11 +9,14 @@ chain gather -> fp32 mean -> GemmaRMSNorm -> MRoPE(group-start position) ->
|
||||
compressed-cache store into one kernel launch.
|
||||
|
||||
Both kernels reproduce the eager numerics step by step (fp32 norm reduction,
|
||||
per-op rounding to the storage dtype during RoPE, fp32 group mean rounded to
|
||||
the storage dtype before the norm). Outputs are bit-comparable to the eager
|
||||
per-op rounding to the compute dtype during RoPE, fp32 group mean rounded to
|
||||
the compute dtype before the norm). Outputs are bit-comparable to the eager
|
||||
indexer path: the eager RMSNorm (flashinfer's CuTe DSL kernel) reduces sums of
|
||||
squares in an order that cannot be reproduced exactly, so a small fraction of
|
||||
rows (~1 in 30k) may flip by one bf16 ulp on a rounding boundary.
|
||||
|
||||
With an fp8 indexer cache the rows are cast to e4m3 at the store (no scale),
|
||||
so fp8 outputs match the eager path to one e4m3 ulp.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -35,16 +38,23 @@ if TYPE_CHECKING:
|
||||
|
||||
@cache_once
|
||||
def _jit_qsa_indexer_module(
|
||||
dtype: torch.dtype, head_dim: int, is_neox_style: bool
|
||||
dtype: torch.dtype, out_dtype: torch.dtype, head_dim: int, is_neox_style: bool
|
||||
) -> Module:
|
||||
"""Compile and cache the JIT QSA indexer module for one specialisation."""
|
||||
if dtype not in (torch.bfloat16, torch.float16):
|
||||
raise RuntimeError(f"Unsupported dtype {dtype}. Supported: bfloat16, float16")
|
||||
if out_dtype != dtype and out_dtype != torch.float8_e4m3fn:
|
||||
raise RuntimeError(
|
||||
f"Unsupported output dtype {out_dtype} for input dtype {dtype}. "
|
||||
"Supported: the input dtype, or float8_e4m3fn"
|
||||
)
|
||||
if head_dim not in (64, 128, 256):
|
||||
raise RuntimeError(
|
||||
f"Unsupported index head_dim {head_dim}. Supported: 64, 128, 256"
|
||||
)
|
||||
args = make_cpp_args(dtype, head_dim, is_neox_style, is_arch_support_pdl())
|
||||
args = make_cpp_args(
|
||||
dtype, out_dtype, head_dim, is_neox_style, is_arch_support_pdl()
|
||||
)
|
||||
return load_jit(
|
||||
"qsa_indexer",
|
||||
*args,
|
||||
@@ -70,6 +80,7 @@ def qsa_index_q_norm_rope_store(
|
||||
eps: float,
|
||||
is_neox_style: bool,
|
||||
q_heads_padded: Optional[int] = None,
|
||||
out_dtype: Optional[torch.dtype] = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Per-token fused index-Q prep.
|
||||
@@ -92,6 +103,8 @@ def qsa_index_q_norm_rope_store(
|
||||
is_neox_style : NeoX (True) or GPT-J (False) RoPE pairing
|
||||
q_heads_padded : output head count; heads >= num_q_heads are zero-filled
|
||||
(defaults to num_q_heads)
|
||||
out_dtype : dtype of the returned Q (default qk.dtype), fp8 e4m3 when
|
||||
the indexer cache is fp8
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -103,10 +116,12 @@ def qsa_index_q_norm_rope_store(
|
||||
q_heads_padded = num_q_heads
|
||||
if positions.ndim == 1:
|
||||
positions = positions.unsqueeze(0)
|
||||
if out_dtype is None:
|
||||
out_dtype = qk.dtype
|
||||
q_out = torch.empty(
|
||||
(num_tokens, q_heads_padded, head_dim), dtype=qk.dtype, device=qk.device
|
||||
(num_tokens, q_heads_padded, head_dim), dtype=out_dtype, device=qk.device
|
||||
)
|
||||
module = _jit_qsa_indexer_module(qk.dtype, head_dim, is_neox_style)
|
||||
module = _jit_qsa_indexer_module(qk.dtype, out_dtype, head_dim, is_neox_style)
|
||||
module.q_prep(
|
||||
qk,
|
||||
q_out,
|
||||
@@ -153,14 +168,17 @@ def qsa_index_k_compress_store(
|
||||
axis_map : CUDA int32 [rotary_dim // 2] position-axis per pair index
|
||||
weight : [head_dim] gemma norm weight (kernel applies 1 + w)
|
||||
write_locs : CUDA int32 [groups] compressed-cache slots to write
|
||||
compressed_k_buffer : CUDA [compressed_slots, head_dim] (written)
|
||||
compressed_k_buffer : CUDA [compressed_slots, head_dim] (written), ring dtype
|
||||
or fp8 e4m3
|
||||
compress_ratio : raw keys per compressed key
|
||||
rotary_dim : rotated prefix of each head row
|
||||
eps : RMSNorm epsilon
|
||||
is_neox_style : NeoX (True) or GPT-J (False) RoPE pairing
|
||||
"""
|
||||
head_dim = weight.shape[0]
|
||||
module = _jit_qsa_indexer_module(key_state_buffer.dtype, head_dim, is_neox_style)
|
||||
module = _jit_qsa_indexer_module(
|
||||
key_state_buffer.dtype, compressed_k_buffer.dtype, head_dim, is_neox_style
|
||||
)
|
||||
module.k_compress(
|
||||
key_state_buffer,
|
||||
group_locs,
|
||||
|
||||
@@ -211,6 +211,17 @@ class Model(msgspec.Struct):
|
||||
resolvable=True,
|
||||
),
|
||||
] = "auto"
|
||||
qsa_indexer_dtype: A[
|
||||
str,
|
||||
Arg(
|
||||
help=(
|
||||
"Storage dtype of the compressed QSA indexer cache and index query "
|
||||
'(Qwen4-Exp). "fp8_e4m3" stores them as plain e4m3 and scores blocks '
|
||||
"with fp8 GEMMs (CUDA SM90+). The main KV cache is unaffected."
|
||||
),
|
||||
choices=["auto", "bfloat16", "fp8_e4m3"],
|
||||
),
|
||||
] = "auto"
|
||||
modelopt_quant: A[
|
||||
Optional[Union[str, Dict]],
|
||||
(
|
||||
|
||||
@@ -75,6 +75,17 @@ def _qwen4_exp_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
)
|
||||
|
||||
profile = parse_qsa_profile(hf_config)
|
||||
if cfg.qsa_indexer_dtype == "fp8_e4m3":
|
||||
# fp8 scoring runs on TileLang fp8 GEMMs, validated on Hopper and Blackwell.
|
||||
platform = get_platform()
|
||||
if not (platform.is_cuda and (platform.is_sm90 or platform.is_sm100)):
|
||||
raise ValueError(
|
||||
"--qsa-indexer-dtype fp8_e4m3 requires a CUDA SM90/SM100 GPU"
|
||||
)
|
||||
if profile is None:
|
||||
raise ValueError(
|
||||
"--qsa-indexer-dtype fp8_e4m3 requires the compressed QSA indexer"
|
||||
)
|
||||
if profile is not None:
|
||||
# Compressed slot = full_slot // ratio; all backends need page-aligned pages.
|
||||
# mamba_radix_cache_strategy resolves later, so do not gate on it.
|
||||
|
||||
@@ -1766,7 +1766,8 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
raise RuntimeError(
|
||||
f"PD Disaggregation does NOT support PD different TP sizes for non-MLA {st.upper()} hybrid models yet."
|
||||
)
|
||||
if has_heterogeneous_attn_tp and is_qwen4_qsa_state:
|
||||
if is_qwen4_qsa_state and target_rank_registration_info is not None:
|
||||
# Reject QSA state layout mismatches up front.
|
||||
if len(dst_item_lens) != len(dst_data_ptrs):
|
||||
raise RuntimeError(
|
||||
f"Replicated {st.upper()} destination pointer/item-length "
|
||||
@@ -1791,6 +1792,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
"prefill and decode entries: "
|
||||
f"{layout_mismatches}"
|
||||
)
|
||||
if has_heterogeneous_attn_tp and is_qwen4_qsa_state:
|
||||
local_tp_rank_in_group = (
|
||||
self.kv_args.engine_rank % self.attn_tp_size
|
||||
)
|
||||
|
||||
@@ -108,6 +108,25 @@ def torch_qsa_mqa_decode(
|
||||
return logits
|
||||
|
||||
|
||||
# TileLang dtype names of the scoring-kernel operands. The logits stay fp32.
|
||||
_TILELANG_DTYPES = {
|
||||
torch.bfloat16: "bfloat16",
|
||||
torch.float8_e4m3fn: "float8_e4m3fn",
|
||||
}
|
||||
|
||||
|
||||
def _scoring_dtype(q: torch.Tensor, k: torch.Tensor) -> torch.dtype:
|
||||
"""The dtype both GEMM operands run in: fp8 only when Q and K already are."""
|
||||
if q.dtype == torch.float8_e4m3fn or k.dtype == torch.float8_e4m3fn:
|
||||
if q.dtype != k.dtype:
|
||||
raise ValueError(
|
||||
"QSA fp8 scoring needs Q and the compressed K in the same dtype, "
|
||||
f"got q={q.dtype} k={k.dtype}"
|
||||
)
|
||||
return torch.float8_e4m3fn
|
||||
return torch.bfloat16
|
||||
|
||||
|
||||
if HAS_TILELANG:
|
||||
|
||||
@tilelang.jit(
|
||||
@@ -124,21 +143,22 @@ if HAS_TILELANG:
|
||||
block_q: int = 32,
|
||||
num_stages: int = 3,
|
||||
threads: int = 512,
|
||||
dtype: str = "bfloat16",
|
||||
):
|
||||
rows = T.dynamic("rows")
|
||||
keys = T.dynamic("keys")
|
||||
|
||||
@T.prim_func
|
||||
def kernel(
|
||||
Q: T.Tensor([rows * heads, head_dim], T.bfloat16), # type: ignore
|
||||
K: T.Tensor([keys, head_dim], T.bfloat16), # type: ignore
|
||||
Q: T.Tensor([rows * heads, head_dim], dtype), # type: ignore
|
||||
K: T.Tensor([keys, head_dim], dtype), # type: ignore
|
||||
Logits: T.Tensor([rows, keys], T.float32), # type: ignore
|
||||
Starts: T.Tensor([rows], T.int32), # type: ignore
|
||||
Ends: T.Tensor([rows], T.int32), # type: ignore
|
||||
):
|
||||
with T.Kernel(T.ceildiv(rows, block_q), threads=threads) as bx:
|
||||
q_shared = T.alloc_shared([block_q * heads, head_dim], T.bfloat16)
|
||||
k_shared = T.alloc_shared([block_n, head_dim], T.bfloat16)
|
||||
q_shared = T.alloc_shared([block_q * heads, head_dim], dtype)
|
||||
k_shared = T.alloc_shared([block_n, head_dim], dtype)
|
||||
scores = T.alloc_fragment([block_n, block_q * heads], T.float32)
|
||||
scores_3d = T.reshape(scores, (block_n, block_q, heads))
|
||||
reduced = T.alloc_fragment([block_n, block_q], T.float32)
|
||||
@@ -208,6 +228,7 @@ if HAS_TILELANG:
|
||||
groups_per_cta: int = 1,
|
||||
num_stages: int = 3,
|
||||
threads: int = 128,
|
||||
dtype: str = "bfloat16",
|
||||
):
|
||||
# The MMA layout needs 64 GEMM rows; a compressed page has full_page // ratio,
|
||||
# so pages are packed as sub-pages of one 64-row tile.
|
||||
@@ -221,8 +242,8 @@ if HAS_TILELANG:
|
||||
|
||||
@T.prim_func
|
||||
def kernel(
|
||||
Q: T.Tensor([batch, 1, heads, head_dim], T.bfloat16), # type: ignore
|
||||
KCache: T.Tensor([pages, page_size, 1, head_dim], T.bfloat16), # type: ignore
|
||||
Q: T.Tensor([batch, 1, heads, head_dim], dtype), # type: ignore
|
||||
KCache: T.Tensor([pages, page_size, 1, head_dim], dtype), # type: ignore
|
||||
PageTable: T.Tensor([batch, max_pages], T.int32), # type: ignore
|
||||
ContextLens: T.Tensor([batch], T.int32), # type: ignore
|
||||
Logits: T.Tensor([batch, max_model_len], T.float32), # type: ignore
|
||||
@@ -233,8 +254,8 @@ if HAS_TILELANG:
|
||||
T.ceildiv(T.ceildiv(max_pages, sub_pages), groups_per_cta),
|
||||
threads=threads,
|
||||
) as (bx, group_block):
|
||||
q_shared = T.alloc_shared([heads, head_dim], T.bfloat16)
|
||||
k_shared = T.alloc_shared([GROUP, head_dim], T.bfloat16)
|
||||
q_shared = T.alloc_shared([heads, head_dim], dtype)
|
||||
k_shared = T.alloc_shared([GROUP, head_dim], dtype)
|
||||
scores = T.alloc_fragment([GROUP, heads], T.float32)
|
||||
reduced = T.alloc_fragment([GROUP], T.float32)
|
||||
T.copy(Q[bx, 0, :, :], q_shared)
|
||||
@@ -301,7 +322,8 @@ def tilelang_qsa_mqa_prefill(
|
||||
# A torch.cat of the padding rows would copy the whole [rows, keys] fp32 matrix,
|
||||
# doubling the dominant prefill buffer; allocate pre-padded instead.
|
||||
logits = torch.zeros((padded_rows, keys), dtype=torch.float32, device=q.device)
|
||||
q_padded = q.to(torch.bfloat16).contiguous()
|
||||
scoring_dtype = _scoring_dtype(q, k)
|
||||
q_padded = q.to(scoring_dtype).contiguous()
|
||||
starts = row_starts.to(device=q.device, dtype=torch.int32).contiguous()
|
||||
ends = row_ends.to(device=q.device, dtype=torch.int32).contiguous()
|
||||
if padding:
|
||||
@@ -309,9 +331,14 @@ def tilelang_qsa_mqa_prefill(
|
||||
starts = torch.cat([starts, starts[-1:].expand(padding)])
|
||||
ends = torch.cat([ends, ends[-1:].expand(padding)])
|
||||
|
||||
_tilelang_qsa_mqa_prefill_kernel(heads=heads, head_dim=head_dim, block_q=block_q)(
|
||||
_tilelang_qsa_mqa_prefill_kernel(
|
||||
heads=heads,
|
||||
head_dim=head_dim,
|
||||
block_q=block_q,
|
||||
dtype=_TILELANG_DTYPES[scoring_dtype],
|
||||
)(
|
||||
q_padded.reshape(-1, head_dim),
|
||||
k[:, 0].to(torch.bfloat16).contiguous(),
|
||||
k[:, 0].to(scoring_dtype).contiguous(),
|
||||
logits,
|
||||
starts,
|
||||
ends,
|
||||
@@ -359,7 +386,8 @@ def tilelang_qsa_mqa_decode(
|
||||
head_alignment,
|
||||
((query_heads + head_alignment - 1) // head_alignment) * head_alignment,
|
||||
)
|
||||
q_kernel = q.to(torch.bfloat16)
|
||||
scoring_dtype = _scoring_dtype(q, k_cache)
|
||||
q_kernel = q.to(scoring_dtype)
|
||||
if kernel_heads != query_heads:
|
||||
q_kernel = torch.cat(
|
||||
[
|
||||
@@ -369,10 +397,13 @@ def tilelang_qsa_mqa_decode(
|
||||
dim=1,
|
||||
)
|
||||
_tilelang_qsa_mqa_decode_kernel(
|
||||
heads=kernel_heads, head_dim=head_dim, page_size=page_size
|
||||
heads=kernel_heads,
|
||||
head_dim=head_dim,
|
||||
page_size=page_size,
|
||||
dtype=_TILELANG_DTYPES[scoring_dtype],
|
||||
)(
|
||||
q_kernel.unsqueeze(1).contiguous(),
|
||||
k_cache.to(torch.bfloat16).contiguous(),
|
||||
k_cache.to(scoring_dtype).contiguous(),
|
||||
page_table.to(device=q.device, dtype=torch.int32).contiguous(),
|
||||
context_lens.to(device=q.device, dtype=torch.int32).contiguous(),
|
||||
logits,
|
||||
|
||||
@@ -193,6 +193,7 @@ class QSAIndexer(MultiPlatformOp):
|
||||
self.q_layernorm.variance_epsilon,
|
||||
self.rotary_emb.is_neox_style,
|
||||
q_heads_padded=q_heads_padded,
|
||||
out_dtype=pool.qsa_compressed_dtype,
|
||||
)
|
||||
return q, token_k, True
|
||||
q_raw = qk[:, : self.index_n_heads * self.index_head_dim]
|
||||
@@ -200,6 +201,8 @@ class QSAIndexer(MultiPlatformOp):
|
||||
-1, self.index_n_heads, self.index_head_dim
|
||||
)
|
||||
q = self.apply_rope(positions, q)
|
||||
if pool is not None:
|
||||
q = q.to(pool.qsa_compressed_dtype)
|
||||
return q, token_k, False
|
||||
|
||||
def normalize_compressed_keys(
|
||||
|
||||
@@ -84,6 +84,7 @@ from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_memory,
|
||||
get_mm,
|
||||
get_model,
|
||||
get_parallel,
|
||||
get_schedule,
|
||||
get_spec,
|
||||
@@ -1922,9 +1923,16 @@ class KVCacheConfigurator:
|
||||
)
|
||||
from sglang.srt.mem_cache.qsa_kv_pool import (
|
||||
QSATokenToKVPool,
|
||||
resolve_qsa_indexer_dtype,
|
||||
)
|
||||
|
||||
qsa_profile = parse_qsa_profile(self.model_config.hf_config)
|
||||
qsa_indexer_dtype = get_model().qsa_indexer_dtype
|
||||
if qsa_indexer_dtype != "auto" and qsa_profile is None:
|
||||
raise ValueError(
|
||||
f"--qsa-indexer-dtype {qsa_indexer_dtype} needs a model with a "
|
||||
"compressed QSA indexer (Qwen4-Exp); this model has none"
|
||||
)
|
||||
if qsa_profile is None:
|
||||
pool_class = HybridLinearKVPool
|
||||
extra_args["use_mla"] = self.use_mla_backend
|
||||
@@ -1936,6 +1944,7 @@ class KVCacheConfigurator:
|
||||
qsa_compress_ratio=qsa_profile.compress_ratio,
|
||||
qsa_token_topk=qsa_profile.budget,
|
||||
num_request_slots=req_to_token_pool.req_to_token.shape[0],
|
||||
qsa_indexer_dtype=resolve_qsa_indexer_dtype(qsa_indexer_dtype),
|
||||
)
|
||||
token_to_kv_pool = pool_class(
|
||||
page_size=self.pool_page_size,
|
||||
|
||||
@@ -8,6 +8,7 @@ Qwen3Next-DSA) adds only the flat per-token index-K cache.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from contextlib import nullcontext
|
||||
from typing import List, Optional
|
||||
|
||||
@@ -16,6 +17,8 @@ import torch
|
||||
from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE
|
||||
from sglang.srt.mem_cache.memory_pool import GB, HybridLinearKVPool, MambaPool
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# State layer IDs are serialized as uint32 by the disaggregation protocols.
|
||||
# Reserve the value below PLE's request-wide sentinel for QSA's request-wide
|
||||
# RoPE ring, which is shared by all full-attention layers.
|
||||
@@ -26,6 +29,22 @@ def _index_k_bytes(*, kv_heads: int, head_dim: int, dtype: torch.dtype) -> int:
|
||||
return kv_heads * head_dim * dtype.itemsize
|
||||
|
||||
|
||||
# ``--qsa-indexer-dtype`` choices. The pending key ring stays bf16 regardless.
|
||||
QSA_INDEXER_DTYPE_CHOICES = ("auto", "bfloat16", "fp8_e4m3")
|
||||
|
||||
|
||||
def resolve_qsa_indexer_dtype(name: str) -> torch.dtype:
|
||||
"""Storage dtype of the compressed QSA indexer cache for a CLI value."""
|
||||
if name in ("auto", "bfloat16"):
|
||||
return torch.bfloat16
|
||||
if name == "fp8_e4m3":
|
||||
return torch.float8_e4m3fn
|
||||
raise ValueError(
|
||||
f"Unsupported --qsa-indexer-dtype {name!r}; expected one of "
|
||||
f"{QSA_INDEXER_DTYPE_CHOICES}"
|
||||
)
|
||||
|
||||
|
||||
class QSATokenToKVPool(HybridLinearKVPool):
|
||||
"""Hybrid KV pool with the minimal BF16 state required by simple QSA."""
|
||||
|
||||
@@ -33,16 +52,23 @@ class QSATokenToKVPool(HybridLinearKVPool):
|
||||
# ``compressed_slot = full_slot // ratio`` needs no ownership bookkeeping;
|
||||
# lifecycle rides the full-KV allocator and radix tree.
|
||||
# Full slot 0 is the reserved padding slot; compressed slot 0 is the inert dump.
|
||||
# Pending-ring dtype: the raw keys are averaged from here, so it stays bf16.
|
||||
index_state_dtype = torch.bfloat16
|
||||
|
||||
@classmethod
|
||||
def qsa_bytes_per_token(
|
||||
cls, *, kv_heads: int, head_dim: int, compress_ratio: int, num_layers: int
|
||||
cls,
|
||||
*,
|
||||
kv_heads: int,
|
||||
head_dim: int,
|
||||
compress_ratio: int,
|
||||
num_layers: int,
|
||||
compressed_dtype: torch.dtype = torch.bfloat16,
|
||||
) -> int:
|
||||
"""Per-token QSA index-cache cost: compressed keys only;
|
||||
the per-request pending ring is budgeted with the other per-request buffers."""
|
||||
index_k_bytes = _index_k_bytes(
|
||||
kv_heads=kv_heads, head_dim=head_dim, dtype=cls.index_state_dtype
|
||||
kv_heads=kv_heads, head_dim=head_dim, dtype=compressed_dtype
|
||||
)
|
||||
return index_k_bytes // compress_ratio * num_layers
|
||||
|
||||
@@ -68,6 +94,7 @@ class QSATokenToKVPool(HybridLinearKVPool):
|
||||
full_kv_pool_class: Optional[type] = None,
|
||||
quant_method=None,
|
||||
post_capture_active: bool = False,
|
||||
qsa_indexer_dtype: torch.dtype = torch.bfloat16,
|
||||
):
|
||||
if page_size <= 1 or page_size % qsa_compress_ratio != 0:
|
||||
raise ValueError(
|
||||
@@ -117,6 +144,18 @@ class QSATokenToKVPool(HybridLinearKVPool):
|
||||
self.qsa_index_kv_heads = int(qsa_index_kv_heads)
|
||||
self.qsa_token_topk = int(qsa_token_topk)
|
||||
self.qsa_block_topk = self.qsa_token_topk // self.qsa_compress_ratio
|
||||
if qsa_indexer_dtype not in (torch.bfloat16, torch.float8_e4m3fn):
|
||||
raise ValueError(
|
||||
"QSA compressed indexer cache dtype must be bfloat16 or "
|
||||
f"float8_e4m3fn, got {qsa_indexer_dtype}"
|
||||
)
|
||||
# Storage dtype of the compressed keys and the index Q (the GEMM operands).
|
||||
self.qsa_compressed_dtype = qsa_indexer_dtype
|
||||
logger.info(
|
||||
"QSA compressed indexer cache dtype: %s (pending ring %s)",
|
||||
self.qsa_compressed_dtype,
|
||||
self.index_state_dtype,
|
||||
)
|
||||
state_size = size + page_size
|
||||
# Compressed slots mirror the full-KV slot space 1:ratio; the "page"
|
||||
# seen by the scoring kernels is one full-KV page's worth of groups.
|
||||
@@ -172,7 +211,7 @@ class QSATokenToKVPool(HybridLinearKVPool):
|
||||
* self.qsa_index_kv_heads
|
||||
* self.qsa_index_head_dim,
|
||||
),
|
||||
dtype=self.index_state_dtype,
|
||||
dtype=self.qsa_compressed_dtype,
|
||||
device=device,
|
||||
)
|
||||
self.qsa_compressed_k_buffer_pool = [
|
||||
|
||||
@@ -45,6 +45,7 @@ from sglang.srt.runtime_context import (
|
||||
get_disagg,
|
||||
get_exec,
|
||||
get_memory,
|
||||
get_model,
|
||||
get_parallel,
|
||||
get_schedule,
|
||||
get_spec,
|
||||
@@ -455,6 +456,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
|
||||
)
|
||||
from sglang.srt.mem_cache.qsa_kv_pool import (
|
||||
QSATokenToKVPool,
|
||||
resolve_qsa_indexer_dtype,
|
||||
)
|
||||
|
||||
if num_layers == 0:
|
||||
@@ -467,6 +469,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
|
||||
head_dim=qsa_profile.head_dim,
|
||||
compress_ratio=qsa_profile.compress_ratio,
|
||||
num_layers=num_layers,
|
||||
compressed_dtype=resolve_qsa_indexer_dtype(get_model().qsa_indexer_dtype),
|
||||
)
|
||||
|
||||
def _compute_dsa_indexer_cell_size(
|
||||
|
||||
Reference in New Issue
Block a user