[Qwen4-Exp] Optional fp8 (e4m3) storage for the compressed QSA indexer cache (#39614)

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: YAMY1234 <74099316+YAMY1234@users.noreply.github.com>
This commit is contained in:
Jackey Hua
2026-09-29 22:11:46 -07:00
committed by GitHub
co-authored by Claude Fable 5.1 YAMY1234
parent 88f95f4b87
commit eb9c9ee99d
13 changed files with 416 additions and 54 deletions
@@ -13,6 +13,7 @@
#include <tvm/ffi/container/tensor.h>
#include <cstdint>
#include <type_traits>
namespace sglang {
@@ -46,6 +47,18 @@ SGL_DEVICE float eager_round<fp16_t>(float x) {
#endif
}
/// \brief Convert a value computed in the compute dtype to the cache storage
/// dtype. Identity when both agree. The fp8 compressed cache takes a plain
/// e4m3 cast with no scale, as the row is RMSNormed before it gets here.
template <typename TOut, typename T>
SGL_DEVICE TOut qsa_storage_cast(T value) {
if constexpr (std::is_same_v<TOut, T>) {
return value;
} else {
return DTypeTrait<TOut>::from(static_cast<float>(value));
}
}
/**
* \brief Apply (M)RoPE to a normed row staged in shared memory.
*
@@ -54,7 +67,8 @@ SGL_DEVICE float eager_round<fp16_t>(float x) {
* uses all-zero maps; Qwen interleaved/sectioned MRoPE maps are built on the
* host). The cos/sin cache row is [cos(half), sin(half)] of width rotary_dim.
*
* \tparam T Storage element type: bf16_t | fp16_t.
* \tparam T Compute/ring element type: bf16_t | fp16_t.
* \tparam TOut Destination storage type: T, or fp8_e4m3_t for the fp8 compressed cache.
* \tparam kHeadDim Compile-time head dimension (multiple of 32).
* \tparam kIsNeox true -> NeoX pairing (d, d+half); false -> GPT-J (2i, 2i+1).
* \param smem_row Normed row [kHeadDim], one warp cooperates.
@@ -64,21 +78,21 @@ SGL_DEVICE float eager_round<fp16_t>(float x) {
* \param pos Resolved per-axis positions for this token (>= 3 entries).
* \param rotary_dim Rotated prefix length; tail dims pass through.
*/
template <typename T, int kHeadDim, bool kIsNeox>
template <typename T, typename TOut, int kHeadDim, bool kIsNeox>
SGL_DEVICE void qsa_mrope_apply(
const T* __restrict__ smem_row,
T* __restrict__ out_row,
TOut* __restrict__ out_row,
const float* __restrict__ cos_sin_cache,
const int32_t* __restrict__ axis_map,
const int64_t* pos,
const int32_t rotary_dim) {
using namespace device;
constexpr int kPerLane = kHeadDim / kWarpThreads;
using vec_t = AlignedVector<T, kPerLane>;
using out_vec_t = AlignedVector<TOut, kPerLane>;
const uint32_t lane = threadIdx.x % kWarpThreads;
const int32_t half = rotary_dim / 2;
vec_t ov;
out_vec_t ov;
#pragma unroll
for (int i = 0; i < kPerLane; ++i) {
const int32_t d = static_cast<int32_t>(lane * kPerLane) + i;
@@ -119,7 +133,7 @@ SGL_DEVICE void qsa_mrope_apply(
o = smem_row[d];
}
}
ov[i] = o;
ov[i] = qsa_storage_cast<TOut>(o);
}
ov.store(out_row, lane); // offset is in vector units
}
@@ -203,11 +217,12 @@ struct QsaIndexQPrepParams {
* head, zero-fill of padded heads, raw token-K store and RoPE-position store.
* One CTA (4 warps) per token; one warp per query head.
*/
template <typename T, int kHeadDim, bool kIsNeox, bool kUsePDL>
template <typename T, typename TOut, int kHeadDim, bool kIsNeox, bool kUsePDL>
__global__ __launch_bounds__(128) void qsa_index_q_prep_kernel(const QsaIndexQPrepParams __grid_constant__ params) {
using namespace device;
constexpr int kPerLane = kHeadDim / kWarpThreads;
using vec_t = AlignedVector<T, kPerLane>;
using out_vec_t = AlignedVector<TOut, kPerLane>;
const uint32_t token = blockIdx.x;
const uint32_t warp = threadIdx.x / kWarpThreads;
const uint32_t lane = threadIdx.x % kWarpThreads;
@@ -225,15 +240,16 @@ __global__ __launch_bounds__(128) void qsa_index_q_prep_kernel(const QsaIndexQPr
}
for (int32_t h = static_cast<int32_t>(warp); h < params.q_heads_padded; h += 4) {
T* out_row = static_cast<T*>(params.q_out) + (static_cast<int64_t>(token) * params.q_heads_padded + h) * kHeadDim;
TOut* out_row =
static_cast<TOut*>(params.q_out) + (static_cast<int64_t>(token) * params.q_heads_padded + h) * kHeadDim;
if (h < params.num_q_heads) {
const T* x_row = static_cast<const T*>(params.qk) + qk_row + h * kHeadDim;
qsa_gemma_norm_row<T, kHeadDim>(x_row, static_cast<const T*>(params.weight), params.eps, smem_rows[warp]);
qsa_mrope_apply<T, kHeadDim, kIsNeox>(
qsa_mrope_apply<T, TOut, kHeadDim, kIsNeox>(
smem_rows[warp], out_row, params.cos_sin_cache, params.axis_map, pos, params.rotary_dim);
} else {
vec_t zv;
zv.fill(DTypeTrait<T>::from(0.0f));
out_vec_t zv;
zv.fill(DTypeTrait<TOut>::from(0.0f));
zv.store(out_row, lane); // offset is in vector units
}
}
@@ -274,7 +290,7 @@ struct QsaIndexKCompressParams {
* MRoPE at the group-start position, store into the compressed cache.
* One warp per group.
*/
template <typename T, int kHeadDim, bool kIsNeox, bool kUsePDL>
template <typename T, typename TOut, int kHeadDim, bool kIsNeox, bool kUsePDL>
__global__
__launch_bounds__(128) void qsa_index_k_compress_kernel(const QsaIndexKCompressParams __grid_constant__ params) {
using namespace device;
@@ -345,8 +361,9 @@ __launch_bounds__(128) void qsa_index_k_compress_kernel(const QsaIndexKCompressP
pos[a] = params.rope_position_buffer[static_cast<int64_t>(loc0) * 3 + a];
}
T* out_row = static_cast<T*>(params.compressed_k_buffer) + static_cast<int64_t>(params.write_locs[group]) * kHeadDim;
qsa_mrope_apply<T, kHeadDim, kIsNeox>(
TOut* out_row =
static_cast<TOut*>(params.compressed_k_buffer) + static_cast<int64_t>(params.write_locs[group]) * kHeadDim;
qsa_mrope_apply<T, TOut, kHeadDim, kIsNeox>(
smem_rows[warp], out_row, params.cos_sin_cache, params.axis_map, pos, params.rotary_dim);
device::PDLTriggerSecondary<kUsePDL>();
@@ -355,12 +372,13 @@ __launch_bounds__(128) void qsa_index_k_compress_kernel(const QsaIndexKCompressP
/**
* \brief Validate inputs and launch `qsa_index_q_prep_kernel` (one CTA per token).
*
* \tparam T Element type: bf16_t | fp16_t.
* \tparam T Input/ring element type: bf16_t | fp16_t.
* \tparam TOut Output storage type: T, or fp8_e4m3_t (fp8 indexer cache).
* \tparam kHeadDim Index head dimension: 64 | 128 | 256.
* \tparam kIsNeox RoPE pairing style.
* \tparam kUsePDL Whether to launch with PDL enabled.
*/
template <typename T, int kHeadDim, bool kIsNeox, bool kUsePDL>
template <typename T, typename TOut, int kHeadDim, bool kIsNeox, bool kUsePDL>
void qsa_index_q_prep(
tvm::ffi::TensorView qk,
tvm::ffi::TensorView q_out,
@@ -383,7 +401,7 @@ void qsa_index_q_prep(
TensorMatcher({tokens, (num_q_heads + 1) * D}).with_dtype<T>().with_device(device).verify(qk);
auto heads_padded = SymbolicSize{"heads_padded"};
TensorMatcher({tokens, heads_padded, D}).with_dtype<T>().with_device(device).verify(q_out);
TensorMatcher({tokens, heads_padded, D}).with_dtype<TOut>().with_device(device).verify(q_out);
TensorMatcher({D}).with_dtype<T>().with_device(device).verify(weight);
auto cache_rows = SymbolicSize{"cos_sin_cache_rows"};
TensorMatcher({cache_rows, rotary_dim}).with_dtype<fp32_t>().with_device(device).verify(cos_sin_cache);
@@ -421,13 +439,13 @@ void qsa_index_q_prep(
.eps = eps,
};
LaunchKernel(static_cast<uint32_t>(num_tokens), 128, device.unwrap())
.enable_pdl(kUsePDL)(qsa_index_q_prep_kernel<T, kHeadDim, kIsNeox, kUsePDL>, params);
.enable_pdl(kUsePDL)(qsa_index_q_prep_kernel<T, TOut, kHeadDim, kIsNeox, kUsePDL>, params);
}
/**
* \brief Validate inputs and launch `qsa_index_k_compress_kernel` (one warp per group).
*/
template <typename T, int kHeadDim, bool kIsNeox, bool kUsePDL>
template <typename T, typename TOut, int kHeadDim, bool kIsNeox, bool kUsePDL>
void qsa_index_k_compress(
tvm::ffi::TensorView key_state_buffer,
tvm::ffi::TensorView group_locs,
@@ -456,7 +474,7 @@ void qsa_index_k_compress(
TensorMatcher({D}).with_dtype<T>().with_device(device).verify(weight);
TensorMatcher({groups}).with_dtype<int32_t>().with_device(device).verify(write_locs);
auto compressed_slots = SymbolicSize{"compressed_slots"};
TensorMatcher({compressed_slots, D}).with_dtype<T>().with_device(device).verify(compressed_k_buffer);
TensorMatcher({compressed_slots, D}).with_dtype<TOut>().with_device(device).verify(compressed_k_buffer);
const int64_t num_groups = groups.unwrap();
CHECK_HOST(num_groups > 0) << "qsa_index_k_compress: no groups";
@@ -480,7 +498,7 @@ void qsa_index_k_compress(
.eps = eps,
};
LaunchKernel(static_cast<uint32_t>(div_ceil(num_groups, 4)), 128, device.unwrap())
.enable_pdl(kUsePDL)(qsa_index_k_compress_kernel<T, kHeadDim, kIsNeox, kUsePDL>, params);
.enable_pdl(kUsePDL)(qsa_index_k_compress_kernel<T, TOut, kHeadDim, kIsNeox, kUsePDL>, params);
}
} // namespace sglang
@@ -9,11 +9,14 @@ chain gather -> fp32 mean -> GemmaRMSNorm -> MRoPE(group-start position) ->
compressed-cache store into one kernel launch.
Both kernels reproduce the eager numerics step by step (fp32 norm reduction,
per-op rounding to the storage dtype during RoPE, fp32 group mean rounded to
the storage dtype before the norm). Outputs are bit-comparable to the eager
per-op rounding to the compute dtype during RoPE, fp32 group mean rounded to
the compute dtype before the norm). Outputs are bit-comparable to the eager
indexer path: the eager RMSNorm (flashinfer's CuTe DSL kernel) reduces sums of
squares in an order that cannot be reproduced exactly, so a small fraction of
rows (~1 in 30k) may flip by one bf16 ulp on a rounding boundary.
With an fp8 indexer cache the rows are cast to e4m3 at the store (no scale),
so fp8 outputs match the eager path to one e4m3 ulp.
"""
from __future__ import annotations
@@ -35,16 +38,23 @@ if TYPE_CHECKING:
@cache_once
def _jit_qsa_indexer_module(
dtype: torch.dtype, head_dim: int, is_neox_style: bool
dtype: torch.dtype, out_dtype: torch.dtype, head_dim: int, is_neox_style: bool
) -> Module:
"""Compile and cache the JIT QSA indexer module for one specialisation."""
if dtype not in (torch.bfloat16, torch.float16):
raise RuntimeError(f"Unsupported dtype {dtype}. Supported: bfloat16, float16")
if out_dtype != dtype and out_dtype != torch.float8_e4m3fn:
raise RuntimeError(
f"Unsupported output dtype {out_dtype} for input dtype {dtype}. "
"Supported: the input dtype, or float8_e4m3fn"
)
if head_dim not in (64, 128, 256):
raise RuntimeError(
f"Unsupported index head_dim {head_dim}. Supported: 64, 128, 256"
)
args = make_cpp_args(dtype, head_dim, is_neox_style, is_arch_support_pdl())
args = make_cpp_args(
dtype, out_dtype, head_dim, is_neox_style, is_arch_support_pdl()
)
return load_jit(
"qsa_indexer",
*args,
@@ -70,6 +80,7 @@ def qsa_index_q_norm_rope_store(
eps: float,
is_neox_style: bool,
q_heads_padded: Optional[int] = None,
out_dtype: Optional[torch.dtype] = None,
) -> torch.Tensor:
"""
Per-token fused index-Q prep.
@@ -92,6 +103,8 @@ def qsa_index_q_norm_rope_store(
is_neox_style : NeoX (True) or GPT-J (False) RoPE pairing
q_heads_padded : output head count; heads >= num_q_heads are zero-filled
(defaults to num_q_heads)
out_dtype : dtype of the returned Q (default qk.dtype), fp8 e4m3 when
the indexer cache is fp8
Returns
-------
@@ -103,10 +116,12 @@ def qsa_index_q_norm_rope_store(
q_heads_padded = num_q_heads
if positions.ndim == 1:
positions = positions.unsqueeze(0)
if out_dtype is None:
out_dtype = qk.dtype
q_out = torch.empty(
(num_tokens, q_heads_padded, head_dim), dtype=qk.dtype, device=qk.device
(num_tokens, q_heads_padded, head_dim), dtype=out_dtype, device=qk.device
)
module = _jit_qsa_indexer_module(qk.dtype, head_dim, is_neox_style)
module = _jit_qsa_indexer_module(qk.dtype, out_dtype, head_dim, is_neox_style)
module.q_prep(
qk,
q_out,
@@ -153,14 +168,17 @@ def qsa_index_k_compress_store(
axis_map : CUDA int32 [rotary_dim // 2] position-axis per pair index
weight : [head_dim] gemma norm weight (kernel applies 1 + w)
write_locs : CUDA int32 [groups] compressed-cache slots to write
compressed_k_buffer : CUDA [compressed_slots, head_dim] (written)
compressed_k_buffer : CUDA [compressed_slots, head_dim] (written), ring dtype
or fp8 e4m3
compress_ratio : raw keys per compressed key
rotary_dim : rotated prefix of each head row
eps : RMSNorm epsilon
is_neox_style : NeoX (True) or GPT-J (False) RoPE pairing
"""
head_dim = weight.shape[0]
module = _jit_qsa_indexer_module(key_state_buffer.dtype, head_dim, is_neox_style)
module = _jit_qsa_indexer_module(
key_state_buffer.dtype, compressed_k_buffer.dtype, head_dim, is_neox_style
)
module.k_compress(
key_state_buffer,
group_locs,
@@ -211,6 +211,17 @@ class Model(msgspec.Struct):
resolvable=True,
),
] = "auto"
qsa_indexer_dtype: A[
str,
Arg(
help=(
"Storage dtype of the compressed QSA indexer cache and index query "
'(Qwen4-Exp). "fp8_e4m3" stores them as plain e4m3 and scores blocks '
"with fp8 GEMMs (CUDA SM90+). The main KV cache is unaffected."
),
choices=["auto", "bfloat16", "fp8_e4m3"],
),
] = "auto"
modelopt_quant: A[
Optional[Union[str, Dict]],
(
@@ -75,6 +75,17 @@ def _qwen4_exp_overrides(server_args: Any, hf_config: Any) -> dict:
)
profile = parse_qsa_profile(hf_config)
if cfg.qsa_indexer_dtype == "fp8_e4m3":
# fp8 scoring runs on TileLang fp8 GEMMs, validated on Hopper and Blackwell.
platform = get_platform()
if not (platform.is_cuda and (platform.is_sm90 or platform.is_sm100)):
raise ValueError(
"--qsa-indexer-dtype fp8_e4m3 requires a CUDA SM90/SM100 GPU"
)
if profile is None:
raise ValueError(
"--qsa-indexer-dtype fp8_e4m3 requires the compressed QSA indexer"
)
if profile is not None:
# Compressed slot = full_slot // ratio; all backends need page-aligned pages.
# mamba_radix_cache_strategy resolves later, so do not gate on it.
@@ -1766,7 +1766,8 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
raise RuntimeError(
f"PD Disaggregation does NOT support PD different TP sizes for non-MLA {st.upper()} hybrid models yet."
)
if has_heterogeneous_attn_tp and is_qwen4_qsa_state:
if is_qwen4_qsa_state and target_rank_registration_info is not None:
# Reject QSA state layout mismatches up front.
if len(dst_item_lens) != len(dst_data_ptrs):
raise RuntimeError(
f"Replicated {st.upper()} destination pointer/item-length "
@@ -1791,6 +1792,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
"prefill and decode entries: "
f"{layout_mismatches}"
)
if has_heterogeneous_attn_tp and is_qwen4_qsa_state:
local_tp_rank_in_group = (
self.kv_args.engine_rank % self.attn_tp_size
)
+45 -14
View File
@@ -108,6 +108,25 @@ def torch_qsa_mqa_decode(
return logits
# TileLang dtype names of the scoring-kernel operands. The logits stay fp32.
_TILELANG_DTYPES = {
torch.bfloat16: "bfloat16",
torch.float8_e4m3fn: "float8_e4m3fn",
}
def _scoring_dtype(q: torch.Tensor, k: torch.Tensor) -> torch.dtype:
"""The dtype both GEMM operands run in: fp8 only when Q and K already are."""
if q.dtype == torch.float8_e4m3fn or k.dtype == torch.float8_e4m3fn:
if q.dtype != k.dtype:
raise ValueError(
"QSA fp8 scoring needs Q and the compressed K in the same dtype, "
f"got q={q.dtype} k={k.dtype}"
)
return torch.float8_e4m3fn
return torch.bfloat16
if HAS_TILELANG:
@tilelang.jit(
@@ -124,21 +143,22 @@ if HAS_TILELANG:
block_q: int = 32,
num_stages: int = 3,
threads: int = 512,
dtype: str = "bfloat16",
):
rows = T.dynamic("rows")
keys = T.dynamic("keys")
@T.prim_func
def kernel(
Q: T.Tensor([rows * heads, head_dim], T.bfloat16), # type: ignore
K: T.Tensor([keys, head_dim], T.bfloat16), # type: ignore
Q: T.Tensor([rows * heads, head_dim], dtype), # type: ignore
K: T.Tensor([keys, head_dim], dtype), # type: ignore
Logits: T.Tensor([rows, keys], T.float32), # type: ignore
Starts: T.Tensor([rows], T.int32), # type: ignore
Ends: T.Tensor([rows], T.int32), # type: ignore
):
with T.Kernel(T.ceildiv(rows, block_q), threads=threads) as bx:
q_shared = T.alloc_shared([block_q * heads, head_dim], T.bfloat16)
k_shared = T.alloc_shared([block_n, head_dim], T.bfloat16)
q_shared = T.alloc_shared([block_q * heads, head_dim], dtype)
k_shared = T.alloc_shared([block_n, head_dim], dtype)
scores = T.alloc_fragment([block_n, block_q * heads], T.float32)
scores_3d = T.reshape(scores, (block_n, block_q, heads))
reduced = T.alloc_fragment([block_n, block_q], T.float32)
@@ -208,6 +228,7 @@ if HAS_TILELANG:
groups_per_cta: int = 1,
num_stages: int = 3,
threads: int = 128,
dtype: str = "bfloat16",
):
# The MMA layout needs 64 GEMM rows; a compressed page has full_page // ratio,
# so pages are packed as sub-pages of one 64-row tile.
@@ -221,8 +242,8 @@ if HAS_TILELANG:
@T.prim_func
def kernel(
Q: T.Tensor([batch, 1, heads, head_dim], T.bfloat16), # type: ignore
KCache: T.Tensor([pages, page_size, 1, head_dim], T.bfloat16), # type: ignore
Q: T.Tensor([batch, 1, heads, head_dim], dtype), # type: ignore
KCache: T.Tensor([pages, page_size, 1, head_dim], dtype), # type: ignore
PageTable: T.Tensor([batch, max_pages], T.int32), # type: ignore
ContextLens: T.Tensor([batch], T.int32), # type: ignore
Logits: T.Tensor([batch, max_model_len], T.float32), # type: ignore
@@ -233,8 +254,8 @@ if HAS_TILELANG:
T.ceildiv(T.ceildiv(max_pages, sub_pages), groups_per_cta),
threads=threads,
) as (bx, group_block):
q_shared = T.alloc_shared([heads, head_dim], T.bfloat16)
k_shared = T.alloc_shared([GROUP, head_dim], T.bfloat16)
q_shared = T.alloc_shared([heads, head_dim], dtype)
k_shared = T.alloc_shared([GROUP, head_dim], dtype)
scores = T.alloc_fragment([GROUP, heads], T.float32)
reduced = T.alloc_fragment([GROUP], T.float32)
T.copy(Q[bx, 0, :, :], q_shared)
@@ -301,7 +322,8 @@ def tilelang_qsa_mqa_prefill(
# A torch.cat of the padding rows would copy the whole [rows, keys] fp32 matrix,
# doubling the dominant prefill buffer; allocate pre-padded instead.
logits = torch.zeros((padded_rows, keys), dtype=torch.float32, device=q.device)
q_padded = q.to(torch.bfloat16).contiguous()
scoring_dtype = _scoring_dtype(q, k)
q_padded = q.to(scoring_dtype).contiguous()
starts = row_starts.to(device=q.device, dtype=torch.int32).contiguous()
ends = row_ends.to(device=q.device, dtype=torch.int32).contiguous()
if padding:
@@ -309,9 +331,14 @@ def tilelang_qsa_mqa_prefill(
starts = torch.cat([starts, starts[-1:].expand(padding)])
ends = torch.cat([ends, ends[-1:].expand(padding)])
_tilelang_qsa_mqa_prefill_kernel(heads=heads, head_dim=head_dim, block_q=block_q)(
_tilelang_qsa_mqa_prefill_kernel(
heads=heads,
head_dim=head_dim,
block_q=block_q,
dtype=_TILELANG_DTYPES[scoring_dtype],
)(
q_padded.reshape(-1, head_dim),
k[:, 0].to(torch.bfloat16).contiguous(),
k[:, 0].to(scoring_dtype).contiguous(),
logits,
starts,
ends,
@@ -359,7 +386,8 @@ def tilelang_qsa_mqa_decode(
head_alignment,
((query_heads + head_alignment - 1) // head_alignment) * head_alignment,
)
q_kernel = q.to(torch.bfloat16)
scoring_dtype = _scoring_dtype(q, k_cache)
q_kernel = q.to(scoring_dtype)
if kernel_heads != query_heads:
q_kernel = torch.cat(
[
@@ -369,10 +397,13 @@ def tilelang_qsa_mqa_decode(
dim=1,
)
_tilelang_qsa_mqa_decode_kernel(
heads=kernel_heads, head_dim=head_dim, page_size=page_size
heads=kernel_heads,
head_dim=head_dim,
page_size=page_size,
dtype=_TILELANG_DTYPES[scoring_dtype],
)(
q_kernel.unsqueeze(1).contiguous(),
k_cache.to(torch.bfloat16).contiguous(),
k_cache.to(scoring_dtype).contiguous(),
page_table.to(device=q.device, dtype=torch.int32).contiguous(),
context_lens.to(device=q.device, dtype=torch.int32).contiguous(),
logits,
@@ -193,6 +193,7 @@ class QSAIndexer(MultiPlatformOp):
self.q_layernorm.variance_epsilon,
self.rotary_emb.is_neox_style,
q_heads_padded=q_heads_padded,
out_dtype=pool.qsa_compressed_dtype,
)
return q, token_k, True
q_raw = qk[:, : self.index_n_heads * self.index_head_dim]
@@ -200,6 +201,8 @@ class QSAIndexer(MultiPlatformOp):
-1, self.index_n_heads, self.index_head_dim
)
q = self.apply_rope(positions, q)
if pool is not None:
q = q.to(pool.qsa_compressed_dtype)
return q, token_k, False
def normalize_compressed_keys(
@@ -84,6 +84,7 @@ from sglang.srt.runtime_context import (
get_exec,
get_memory,
get_mm,
get_model,
get_parallel,
get_schedule,
get_spec,
@@ -1922,9 +1923,16 @@ class KVCacheConfigurator:
)
from sglang.srt.mem_cache.qsa_kv_pool import (
QSATokenToKVPool,
resolve_qsa_indexer_dtype,
)
qsa_profile = parse_qsa_profile(self.model_config.hf_config)
qsa_indexer_dtype = get_model().qsa_indexer_dtype
if qsa_indexer_dtype != "auto" and qsa_profile is None:
raise ValueError(
f"--qsa-indexer-dtype {qsa_indexer_dtype} needs a model with a "
"compressed QSA indexer (Qwen4-Exp); this model has none"
)
if qsa_profile is None:
pool_class = HybridLinearKVPool
extra_args["use_mla"] = self.use_mla_backend
@@ -1936,6 +1944,7 @@ class KVCacheConfigurator:
qsa_compress_ratio=qsa_profile.compress_ratio,
qsa_token_topk=qsa_profile.budget,
num_request_slots=req_to_token_pool.req_to_token.shape[0],
qsa_indexer_dtype=resolve_qsa_indexer_dtype(qsa_indexer_dtype),
)
token_to_kv_pool = pool_class(
page_size=self.pool_page_size,
+42 -3
View File
@@ -8,6 +8,7 @@ Qwen3Next-DSA) adds only the flat per-token index-K cache.
from __future__ import annotations
import logging
from contextlib import nullcontext
from typing import List, Optional
@@ -16,6 +17,8 @@ import torch
from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE
from sglang.srt.mem_cache.memory_pool import GB, HybridLinearKVPool, MambaPool
logger = logging.getLogger(__name__)
# State layer IDs are serialized as uint32 by the disaggregation protocols.
# Reserve the value below PLE's request-wide sentinel for QSA's request-wide
# RoPE ring, which is shared by all full-attention layers.
@@ -26,6 +29,22 @@ def _index_k_bytes(*, kv_heads: int, head_dim: int, dtype: torch.dtype) -> int:
return kv_heads * head_dim * dtype.itemsize
# ``--qsa-indexer-dtype`` choices. The pending key ring stays bf16 regardless.
QSA_INDEXER_DTYPE_CHOICES = ("auto", "bfloat16", "fp8_e4m3")
def resolve_qsa_indexer_dtype(name: str) -> torch.dtype:
"""Storage dtype of the compressed QSA indexer cache for a CLI value."""
if name in ("auto", "bfloat16"):
return torch.bfloat16
if name == "fp8_e4m3":
return torch.float8_e4m3fn
raise ValueError(
f"Unsupported --qsa-indexer-dtype {name!r}; expected one of "
f"{QSA_INDEXER_DTYPE_CHOICES}"
)
class QSATokenToKVPool(HybridLinearKVPool):
"""Hybrid KV pool with the minimal BF16 state required by simple QSA."""
@@ -33,16 +52,23 @@ class QSATokenToKVPool(HybridLinearKVPool):
# ``compressed_slot = full_slot // ratio`` needs no ownership bookkeeping;
# lifecycle rides the full-KV allocator and radix tree.
# Full slot 0 is the reserved padding slot; compressed slot 0 is the inert dump.
# Pending-ring dtype: the raw keys are averaged from here, so it stays bf16.
index_state_dtype = torch.bfloat16
@classmethod
def qsa_bytes_per_token(
cls, *, kv_heads: int, head_dim: int, compress_ratio: int, num_layers: int
cls,
*,
kv_heads: int,
head_dim: int,
compress_ratio: int,
num_layers: int,
compressed_dtype: torch.dtype = torch.bfloat16,
) -> int:
"""Per-token QSA index-cache cost: compressed keys only;
the per-request pending ring is budgeted with the other per-request buffers."""
index_k_bytes = _index_k_bytes(
kv_heads=kv_heads, head_dim=head_dim, dtype=cls.index_state_dtype
kv_heads=kv_heads, head_dim=head_dim, dtype=compressed_dtype
)
return index_k_bytes // compress_ratio * num_layers
@@ -68,6 +94,7 @@ class QSATokenToKVPool(HybridLinearKVPool):
full_kv_pool_class: Optional[type] = None,
quant_method=None,
post_capture_active: bool = False,
qsa_indexer_dtype: torch.dtype = torch.bfloat16,
):
if page_size <= 1 or page_size % qsa_compress_ratio != 0:
raise ValueError(
@@ -117,6 +144,18 @@ class QSATokenToKVPool(HybridLinearKVPool):
self.qsa_index_kv_heads = int(qsa_index_kv_heads)
self.qsa_token_topk = int(qsa_token_topk)
self.qsa_block_topk = self.qsa_token_topk // self.qsa_compress_ratio
if qsa_indexer_dtype not in (torch.bfloat16, torch.float8_e4m3fn):
raise ValueError(
"QSA compressed indexer cache dtype must be bfloat16 or "
f"float8_e4m3fn, got {qsa_indexer_dtype}"
)
# Storage dtype of the compressed keys and the index Q (the GEMM operands).
self.qsa_compressed_dtype = qsa_indexer_dtype
logger.info(
"QSA compressed indexer cache dtype: %s (pending ring %s)",
self.qsa_compressed_dtype,
self.index_state_dtype,
)
state_size = size + page_size
# Compressed slots mirror the full-KV slot space 1:ratio; the "page"
# seen by the scoring kernels is one full-KV page's worth of groups.
@@ -172,7 +211,7 @@ class QSATokenToKVPool(HybridLinearKVPool):
* self.qsa_index_kv_heads
* self.qsa_index_head_dim,
),
dtype=self.index_state_dtype,
dtype=self.qsa_compressed_dtype,
device=device,
)
self.qsa_compressed_k_buffer_pool = [
@@ -45,6 +45,7 @@ from sglang.srt.runtime_context import (
get_disagg,
get_exec,
get_memory,
get_model,
get_parallel,
get_schedule,
get_spec,
@@ -455,6 +456,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
)
from sglang.srt.mem_cache.qsa_kv_pool import (
QSATokenToKVPool,
resolve_qsa_indexer_dtype,
)
if num_layers == 0:
@@ -467,6 +469,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
head_dim=qsa_profile.head_dim,
compress_ratio=qsa_profile.compress_ratio,
num_layers=num_layers,
compressed_dtype=resolve_qsa_indexer_dtype(get_model().qsa_indexer_dtype),
)
def _compute_dsa_indexer_cell_size(