[Qwen4-Exp] Optional fp8 (e4m3) storage for the compressed QSA indexer cache (#39614)
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: YAMY1234 <74099316+YAMY1234@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
YAMY1234
parent
88f95f4b87
commit
eb9c9ee99d
@@ -13,6 +13,7 @@
|
||||
#include <tvm/ffi/container/tensor.h>
|
||||
|
||||
#include <cstdint>
|
||||
#include <type_traits>
|
||||
|
||||
namespace sglang {
|
||||
|
||||
@@ -46,6 +47,18 @@ SGL_DEVICE float eager_round<fp16_t>(float x) {
|
||||
#endif
|
||||
}
|
||||
|
||||
/// \brief Convert a value computed in the compute dtype to the cache storage
|
||||
/// dtype. Identity when both agree. The fp8 compressed cache takes a plain
|
||||
/// e4m3 cast with no scale, as the row is RMSNormed before it gets here.
|
||||
template <typename TOut, typename T>
|
||||
SGL_DEVICE TOut qsa_storage_cast(T value) {
|
||||
if constexpr (std::is_same_v<TOut, T>) {
|
||||
return value;
|
||||
} else {
|
||||
return DTypeTrait<TOut>::from(static_cast<float>(value));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief Apply (M)RoPE to a normed row staged in shared memory.
|
||||
*
|
||||
@@ -54,7 +67,8 @@ SGL_DEVICE float eager_round<fp16_t>(float x) {
|
||||
* uses all-zero maps; Qwen interleaved/sectioned MRoPE maps are built on the
|
||||
* host). The cos/sin cache row is [cos(half), sin(half)] of width rotary_dim.
|
||||
*
|
||||
* \tparam T Storage element type: bf16_t | fp16_t.
|
||||
* \tparam T Compute/ring element type: bf16_t | fp16_t.
|
||||
* \tparam TOut Destination storage type: T, or fp8_e4m3_t for the fp8 compressed cache.
|
||||
* \tparam kHeadDim Compile-time head dimension (multiple of 32).
|
||||
* \tparam kIsNeox true -> NeoX pairing (d, d+half); false -> GPT-J (2i, 2i+1).
|
||||
* \param smem_row Normed row [kHeadDim], one warp cooperates.
|
||||
@@ -64,21 +78,21 @@ SGL_DEVICE float eager_round<fp16_t>(float x) {
|
||||
* \param pos Resolved per-axis positions for this token (>= 3 entries).
|
||||
* \param rotary_dim Rotated prefix length; tail dims pass through.
|
||||
*/
|
||||
template <typename T, int kHeadDim, bool kIsNeox>
|
||||
template <typename T, typename TOut, int kHeadDim, bool kIsNeox>
|
||||
SGL_DEVICE void qsa_mrope_apply(
|
||||
const T* __restrict__ smem_row,
|
||||
T* __restrict__ out_row,
|
||||
TOut* __restrict__ out_row,
|
||||
const float* __restrict__ cos_sin_cache,
|
||||
const int32_t* __restrict__ axis_map,
|
||||
const int64_t* pos,
|
||||
const int32_t rotary_dim) {
|
||||
using namespace device;
|
||||
constexpr int kPerLane = kHeadDim / kWarpThreads;
|
||||
using vec_t = AlignedVector<T, kPerLane>;
|
||||
using out_vec_t = AlignedVector<TOut, kPerLane>;
|
||||
const uint32_t lane = threadIdx.x % kWarpThreads;
|
||||
const int32_t half = rotary_dim / 2;
|
||||
|
||||
vec_t ov;
|
||||
out_vec_t ov;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kPerLane; ++i) {
|
||||
const int32_t d = static_cast<int32_t>(lane * kPerLane) + i;
|
||||
@@ -119,7 +133,7 @@ SGL_DEVICE void qsa_mrope_apply(
|
||||
o = smem_row[d];
|
||||
}
|
||||
}
|
||||
ov[i] = o;
|
||||
ov[i] = qsa_storage_cast<TOut>(o);
|
||||
}
|
||||
ov.store(out_row, lane); // offset is in vector units
|
||||
}
|
||||
@@ -203,11 +217,12 @@ struct QsaIndexQPrepParams {
|
||||
* head, zero-fill of padded heads, raw token-K store and RoPE-position store.
|
||||
* One CTA (4 warps) per token; one warp per query head.
|
||||
*/
|
||||
template <typename T, int kHeadDim, bool kIsNeox, bool kUsePDL>
|
||||
template <typename T, typename TOut, int kHeadDim, bool kIsNeox, bool kUsePDL>
|
||||
__global__ __launch_bounds__(128) void qsa_index_q_prep_kernel(const QsaIndexQPrepParams __grid_constant__ params) {
|
||||
using namespace device;
|
||||
constexpr int kPerLane = kHeadDim / kWarpThreads;
|
||||
using vec_t = AlignedVector<T, kPerLane>;
|
||||
using out_vec_t = AlignedVector<TOut, kPerLane>;
|
||||
const uint32_t token = blockIdx.x;
|
||||
const uint32_t warp = threadIdx.x / kWarpThreads;
|
||||
const uint32_t lane = threadIdx.x % kWarpThreads;
|
||||
@@ -225,15 +240,16 @@ __global__ __launch_bounds__(128) void qsa_index_q_prep_kernel(const QsaIndexQPr
|
||||
}
|
||||
|
||||
for (int32_t h = static_cast<int32_t>(warp); h < params.q_heads_padded; h += 4) {
|
||||
T* out_row = static_cast<T*>(params.q_out) + (static_cast<int64_t>(token) * params.q_heads_padded + h) * kHeadDim;
|
||||
TOut* out_row =
|
||||
static_cast<TOut*>(params.q_out) + (static_cast<int64_t>(token) * params.q_heads_padded + h) * kHeadDim;
|
||||
if (h < params.num_q_heads) {
|
||||
const T* x_row = static_cast<const T*>(params.qk) + qk_row + h * kHeadDim;
|
||||
qsa_gemma_norm_row<T, kHeadDim>(x_row, static_cast<const T*>(params.weight), params.eps, smem_rows[warp]);
|
||||
qsa_mrope_apply<T, kHeadDim, kIsNeox>(
|
||||
qsa_mrope_apply<T, TOut, kHeadDim, kIsNeox>(
|
||||
smem_rows[warp], out_row, params.cos_sin_cache, params.axis_map, pos, params.rotary_dim);
|
||||
} else {
|
||||
vec_t zv;
|
||||
zv.fill(DTypeTrait<T>::from(0.0f));
|
||||
out_vec_t zv;
|
||||
zv.fill(DTypeTrait<TOut>::from(0.0f));
|
||||
zv.store(out_row, lane); // offset is in vector units
|
||||
}
|
||||
}
|
||||
@@ -274,7 +290,7 @@ struct QsaIndexKCompressParams {
|
||||
* MRoPE at the group-start position, store into the compressed cache.
|
||||
* One warp per group.
|
||||
*/
|
||||
template <typename T, int kHeadDim, bool kIsNeox, bool kUsePDL>
|
||||
template <typename T, typename TOut, int kHeadDim, bool kIsNeox, bool kUsePDL>
|
||||
__global__
|
||||
__launch_bounds__(128) void qsa_index_k_compress_kernel(const QsaIndexKCompressParams __grid_constant__ params) {
|
||||
using namespace device;
|
||||
@@ -345,8 +361,9 @@ __launch_bounds__(128) void qsa_index_k_compress_kernel(const QsaIndexKCompressP
|
||||
pos[a] = params.rope_position_buffer[static_cast<int64_t>(loc0) * 3 + a];
|
||||
}
|
||||
|
||||
T* out_row = static_cast<T*>(params.compressed_k_buffer) + static_cast<int64_t>(params.write_locs[group]) * kHeadDim;
|
||||
qsa_mrope_apply<T, kHeadDim, kIsNeox>(
|
||||
TOut* out_row =
|
||||
static_cast<TOut*>(params.compressed_k_buffer) + static_cast<int64_t>(params.write_locs[group]) * kHeadDim;
|
||||
qsa_mrope_apply<T, TOut, kHeadDim, kIsNeox>(
|
||||
smem_rows[warp], out_row, params.cos_sin_cache, params.axis_map, pos, params.rotary_dim);
|
||||
|
||||
device::PDLTriggerSecondary<kUsePDL>();
|
||||
@@ -355,12 +372,13 @@ __launch_bounds__(128) void qsa_index_k_compress_kernel(const QsaIndexKCompressP
|
||||
/**
|
||||
* \brief Validate inputs and launch `qsa_index_q_prep_kernel` (one CTA per token).
|
||||
*
|
||||
* \tparam T Element type: bf16_t | fp16_t.
|
||||
* \tparam T Input/ring element type: bf16_t | fp16_t.
|
||||
* \tparam TOut Output storage type: T, or fp8_e4m3_t (fp8 indexer cache).
|
||||
* \tparam kHeadDim Index head dimension: 64 | 128 | 256.
|
||||
* \tparam kIsNeox RoPE pairing style.
|
||||
* \tparam kUsePDL Whether to launch with PDL enabled.
|
||||
*/
|
||||
template <typename T, int kHeadDim, bool kIsNeox, bool kUsePDL>
|
||||
template <typename T, typename TOut, int kHeadDim, bool kIsNeox, bool kUsePDL>
|
||||
void qsa_index_q_prep(
|
||||
tvm::ffi::TensorView qk,
|
||||
tvm::ffi::TensorView q_out,
|
||||
@@ -383,7 +401,7 @@ void qsa_index_q_prep(
|
||||
|
||||
TensorMatcher({tokens, (num_q_heads + 1) * D}).with_dtype<T>().with_device(device).verify(qk);
|
||||
auto heads_padded = SymbolicSize{"heads_padded"};
|
||||
TensorMatcher({tokens, heads_padded, D}).with_dtype<T>().with_device(device).verify(q_out);
|
||||
TensorMatcher({tokens, heads_padded, D}).with_dtype<TOut>().with_device(device).verify(q_out);
|
||||
TensorMatcher({D}).with_dtype<T>().with_device(device).verify(weight);
|
||||
auto cache_rows = SymbolicSize{"cos_sin_cache_rows"};
|
||||
TensorMatcher({cache_rows, rotary_dim}).with_dtype<fp32_t>().with_device(device).verify(cos_sin_cache);
|
||||
@@ -421,13 +439,13 @@ void qsa_index_q_prep(
|
||||
.eps = eps,
|
||||
};
|
||||
LaunchKernel(static_cast<uint32_t>(num_tokens), 128, device.unwrap())
|
||||
.enable_pdl(kUsePDL)(qsa_index_q_prep_kernel<T, kHeadDim, kIsNeox, kUsePDL>, params);
|
||||
.enable_pdl(kUsePDL)(qsa_index_q_prep_kernel<T, TOut, kHeadDim, kIsNeox, kUsePDL>, params);
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief Validate inputs and launch `qsa_index_k_compress_kernel` (one warp per group).
|
||||
*/
|
||||
template <typename T, int kHeadDim, bool kIsNeox, bool kUsePDL>
|
||||
template <typename T, typename TOut, int kHeadDim, bool kIsNeox, bool kUsePDL>
|
||||
void qsa_index_k_compress(
|
||||
tvm::ffi::TensorView key_state_buffer,
|
||||
tvm::ffi::TensorView group_locs,
|
||||
@@ -456,7 +474,7 @@ void qsa_index_k_compress(
|
||||
TensorMatcher({D}).with_dtype<T>().with_device(device).verify(weight);
|
||||
TensorMatcher({groups}).with_dtype<int32_t>().with_device(device).verify(write_locs);
|
||||
auto compressed_slots = SymbolicSize{"compressed_slots"};
|
||||
TensorMatcher({compressed_slots, D}).with_dtype<T>().with_device(device).verify(compressed_k_buffer);
|
||||
TensorMatcher({compressed_slots, D}).with_dtype<TOut>().with_device(device).verify(compressed_k_buffer);
|
||||
|
||||
const int64_t num_groups = groups.unwrap();
|
||||
CHECK_HOST(num_groups > 0) << "qsa_index_k_compress: no groups";
|
||||
@@ -480,7 +498,7 @@ void qsa_index_k_compress(
|
||||
.eps = eps,
|
||||
};
|
||||
LaunchKernel(static_cast<uint32_t>(div_ceil(num_groups, 4)), 128, device.unwrap())
|
||||
.enable_pdl(kUsePDL)(qsa_index_k_compress_kernel<T, kHeadDim, kIsNeox, kUsePDL>, params);
|
||||
.enable_pdl(kUsePDL)(qsa_index_k_compress_kernel<T, TOut, kHeadDim, kIsNeox, kUsePDL>, params);
|
||||
}
|
||||
|
||||
} // namespace sglang
|
||||
|
||||
@@ -9,11 +9,14 @@ chain gather -> fp32 mean -> GemmaRMSNorm -> MRoPE(group-start position) ->
|
||||
compressed-cache store into one kernel launch.
|
||||
|
||||
Both kernels reproduce the eager numerics step by step (fp32 norm reduction,
|
||||
per-op rounding to the storage dtype during RoPE, fp32 group mean rounded to
|
||||
the storage dtype before the norm). Outputs are bit-comparable to the eager
|
||||
per-op rounding to the compute dtype during RoPE, fp32 group mean rounded to
|
||||
the compute dtype before the norm). Outputs are bit-comparable to the eager
|
||||
indexer path: the eager RMSNorm (flashinfer's CuTe DSL kernel) reduces sums of
|
||||
squares in an order that cannot be reproduced exactly, so a small fraction of
|
||||
rows (~1 in 30k) may flip by one bf16 ulp on a rounding boundary.
|
||||
|
||||
With an fp8 indexer cache the rows are cast to e4m3 at the store (no scale),
|
||||
so fp8 outputs match the eager path to one e4m3 ulp.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -35,16 +38,23 @@ if TYPE_CHECKING:
|
||||
|
||||
@cache_once
|
||||
def _jit_qsa_indexer_module(
|
||||
dtype: torch.dtype, head_dim: int, is_neox_style: bool
|
||||
dtype: torch.dtype, out_dtype: torch.dtype, head_dim: int, is_neox_style: bool
|
||||
) -> Module:
|
||||
"""Compile and cache the JIT QSA indexer module for one specialisation."""
|
||||
if dtype not in (torch.bfloat16, torch.float16):
|
||||
raise RuntimeError(f"Unsupported dtype {dtype}. Supported: bfloat16, float16")
|
||||
if out_dtype != dtype and out_dtype != torch.float8_e4m3fn:
|
||||
raise RuntimeError(
|
||||
f"Unsupported output dtype {out_dtype} for input dtype {dtype}. "
|
||||
"Supported: the input dtype, or float8_e4m3fn"
|
||||
)
|
||||
if head_dim not in (64, 128, 256):
|
||||
raise RuntimeError(
|
||||
f"Unsupported index head_dim {head_dim}. Supported: 64, 128, 256"
|
||||
)
|
||||
args = make_cpp_args(dtype, head_dim, is_neox_style, is_arch_support_pdl())
|
||||
args = make_cpp_args(
|
||||
dtype, out_dtype, head_dim, is_neox_style, is_arch_support_pdl()
|
||||
)
|
||||
return load_jit(
|
||||
"qsa_indexer",
|
||||
*args,
|
||||
@@ -70,6 +80,7 @@ def qsa_index_q_norm_rope_store(
|
||||
eps: float,
|
||||
is_neox_style: bool,
|
||||
q_heads_padded: Optional[int] = None,
|
||||
out_dtype: Optional[torch.dtype] = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Per-token fused index-Q prep.
|
||||
@@ -92,6 +103,8 @@ def qsa_index_q_norm_rope_store(
|
||||
is_neox_style : NeoX (True) or GPT-J (False) RoPE pairing
|
||||
q_heads_padded : output head count; heads >= num_q_heads are zero-filled
|
||||
(defaults to num_q_heads)
|
||||
out_dtype : dtype of the returned Q (default qk.dtype), fp8 e4m3 when
|
||||
the indexer cache is fp8
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -103,10 +116,12 @@ def qsa_index_q_norm_rope_store(
|
||||
q_heads_padded = num_q_heads
|
||||
if positions.ndim == 1:
|
||||
positions = positions.unsqueeze(0)
|
||||
if out_dtype is None:
|
||||
out_dtype = qk.dtype
|
||||
q_out = torch.empty(
|
||||
(num_tokens, q_heads_padded, head_dim), dtype=qk.dtype, device=qk.device
|
||||
(num_tokens, q_heads_padded, head_dim), dtype=out_dtype, device=qk.device
|
||||
)
|
||||
module = _jit_qsa_indexer_module(qk.dtype, head_dim, is_neox_style)
|
||||
module = _jit_qsa_indexer_module(qk.dtype, out_dtype, head_dim, is_neox_style)
|
||||
module.q_prep(
|
||||
qk,
|
||||
q_out,
|
||||
@@ -153,14 +168,17 @@ def qsa_index_k_compress_store(
|
||||
axis_map : CUDA int32 [rotary_dim // 2] position-axis per pair index
|
||||
weight : [head_dim] gemma norm weight (kernel applies 1 + w)
|
||||
write_locs : CUDA int32 [groups] compressed-cache slots to write
|
||||
compressed_k_buffer : CUDA [compressed_slots, head_dim] (written)
|
||||
compressed_k_buffer : CUDA [compressed_slots, head_dim] (written), ring dtype
|
||||
or fp8 e4m3
|
||||
compress_ratio : raw keys per compressed key
|
||||
rotary_dim : rotated prefix of each head row
|
||||
eps : RMSNorm epsilon
|
||||
is_neox_style : NeoX (True) or GPT-J (False) RoPE pairing
|
||||
"""
|
||||
head_dim = weight.shape[0]
|
||||
module = _jit_qsa_indexer_module(key_state_buffer.dtype, head_dim, is_neox_style)
|
||||
module = _jit_qsa_indexer_module(
|
||||
key_state_buffer.dtype, compressed_k_buffer.dtype, head_dim, is_neox_style
|
||||
)
|
||||
module.k_compress(
|
||||
key_state_buffer,
|
||||
group_locs,
|
||||
|
||||
@@ -211,6 +211,17 @@ class Model(msgspec.Struct):
|
||||
resolvable=True,
|
||||
),
|
||||
] = "auto"
|
||||
qsa_indexer_dtype: A[
|
||||
str,
|
||||
Arg(
|
||||
help=(
|
||||
"Storage dtype of the compressed QSA indexer cache and index query "
|
||||
'(Qwen4-Exp). "fp8_e4m3" stores them as plain e4m3 and scores blocks '
|
||||
"with fp8 GEMMs (CUDA SM90+). The main KV cache is unaffected."
|
||||
),
|
||||
choices=["auto", "bfloat16", "fp8_e4m3"],
|
||||
),
|
||||
] = "auto"
|
||||
modelopt_quant: A[
|
||||
Optional[Union[str, Dict]],
|
||||
(
|
||||
|
||||
@@ -75,6 +75,17 @@ def _qwen4_exp_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
)
|
||||
|
||||
profile = parse_qsa_profile(hf_config)
|
||||
if cfg.qsa_indexer_dtype == "fp8_e4m3":
|
||||
# fp8 scoring runs on TileLang fp8 GEMMs, validated on Hopper and Blackwell.
|
||||
platform = get_platform()
|
||||
if not (platform.is_cuda and (platform.is_sm90 or platform.is_sm100)):
|
||||
raise ValueError(
|
||||
"--qsa-indexer-dtype fp8_e4m3 requires a CUDA SM90/SM100 GPU"
|
||||
)
|
||||
if profile is None:
|
||||
raise ValueError(
|
||||
"--qsa-indexer-dtype fp8_e4m3 requires the compressed QSA indexer"
|
||||
)
|
||||
if profile is not None:
|
||||
# Compressed slot = full_slot // ratio; all backends need page-aligned pages.
|
||||
# mamba_radix_cache_strategy resolves later, so do not gate on it.
|
||||
|
||||
@@ -1766,7 +1766,8 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
raise RuntimeError(
|
||||
f"PD Disaggregation does NOT support PD different TP sizes for non-MLA {st.upper()} hybrid models yet."
|
||||
)
|
||||
if has_heterogeneous_attn_tp and is_qwen4_qsa_state:
|
||||
if is_qwen4_qsa_state and target_rank_registration_info is not None:
|
||||
# Reject QSA state layout mismatches up front.
|
||||
if len(dst_item_lens) != len(dst_data_ptrs):
|
||||
raise RuntimeError(
|
||||
f"Replicated {st.upper()} destination pointer/item-length "
|
||||
@@ -1791,6 +1792,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
"prefill and decode entries: "
|
||||
f"{layout_mismatches}"
|
||||
)
|
||||
if has_heterogeneous_attn_tp and is_qwen4_qsa_state:
|
||||
local_tp_rank_in_group = (
|
||||
self.kv_args.engine_rank % self.attn_tp_size
|
||||
)
|
||||
|
||||
@@ -108,6 +108,25 @@ def torch_qsa_mqa_decode(
|
||||
return logits
|
||||
|
||||
|
||||
# TileLang dtype names of the scoring-kernel operands. The logits stay fp32.
|
||||
_TILELANG_DTYPES = {
|
||||
torch.bfloat16: "bfloat16",
|
||||
torch.float8_e4m3fn: "float8_e4m3fn",
|
||||
}
|
||||
|
||||
|
||||
def _scoring_dtype(q: torch.Tensor, k: torch.Tensor) -> torch.dtype:
|
||||
"""The dtype both GEMM operands run in: fp8 only when Q and K already are."""
|
||||
if q.dtype == torch.float8_e4m3fn or k.dtype == torch.float8_e4m3fn:
|
||||
if q.dtype != k.dtype:
|
||||
raise ValueError(
|
||||
"QSA fp8 scoring needs Q and the compressed K in the same dtype, "
|
||||
f"got q={q.dtype} k={k.dtype}"
|
||||
)
|
||||
return torch.float8_e4m3fn
|
||||
return torch.bfloat16
|
||||
|
||||
|
||||
if HAS_TILELANG:
|
||||
|
||||
@tilelang.jit(
|
||||
@@ -124,21 +143,22 @@ if HAS_TILELANG:
|
||||
block_q: int = 32,
|
||||
num_stages: int = 3,
|
||||
threads: int = 512,
|
||||
dtype: str = "bfloat16",
|
||||
):
|
||||
rows = T.dynamic("rows")
|
||||
keys = T.dynamic("keys")
|
||||
|
||||
@T.prim_func
|
||||
def kernel(
|
||||
Q: T.Tensor([rows * heads, head_dim], T.bfloat16), # type: ignore
|
||||
K: T.Tensor([keys, head_dim], T.bfloat16), # type: ignore
|
||||
Q: T.Tensor([rows * heads, head_dim], dtype), # type: ignore
|
||||
K: T.Tensor([keys, head_dim], dtype), # type: ignore
|
||||
Logits: T.Tensor([rows, keys], T.float32), # type: ignore
|
||||
Starts: T.Tensor([rows], T.int32), # type: ignore
|
||||
Ends: T.Tensor([rows], T.int32), # type: ignore
|
||||
):
|
||||
with T.Kernel(T.ceildiv(rows, block_q), threads=threads) as bx:
|
||||
q_shared = T.alloc_shared([block_q * heads, head_dim], T.bfloat16)
|
||||
k_shared = T.alloc_shared([block_n, head_dim], T.bfloat16)
|
||||
q_shared = T.alloc_shared([block_q * heads, head_dim], dtype)
|
||||
k_shared = T.alloc_shared([block_n, head_dim], dtype)
|
||||
scores = T.alloc_fragment([block_n, block_q * heads], T.float32)
|
||||
scores_3d = T.reshape(scores, (block_n, block_q, heads))
|
||||
reduced = T.alloc_fragment([block_n, block_q], T.float32)
|
||||
@@ -208,6 +228,7 @@ if HAS_TILELANG:
|
||||
groups_per_cta: int = 1,
|
||||
num_stages: int = 3,
|
||||
threads: int = 128,
|
||||
dtype: str = "bfloat16",
|
||||
):
|
||||
# The MMA layout needs 64 GEMM rows; a compressed page has full_page // ratio,
|
||||
# so pages are packed as sub-pages of one 64-row tile.
|
||||
@@ -221,8 +242,8 @@ if HAS_TILELANG:
|
||||
|
||||
@T.prim_func
|
||||
def kernel(
|
||||
Q: T.Tensor([batch, 1, heads, head_dim], T.bfloat16), # type: ignore
|
||||
KCache: T.Tensor([pages, page_size, 1, head_dim], T.bfloat16), # type: ignore
|
||||
Q: T.Tensor([batch, 1, heads, head_dim], dtype), # type: ignore
|
||||
KCache: T.Tensor([pages, page_size, 1, head_dim], dtype), # type: ignore
|
||||
PageTable: T.Tensor([batch, max_pages], T.int32), # type: ignore
|
||||
ContextLens: T.Tensor([batch], T.int32), # type: ignore
|
||||
Logits: T.Tensor([batch, max_model_len], T.float32), # type: ignore
|
||||
@@ -233,8 +254,8 @@ if HAS_TILELANG:
|
||||
T.ceildiv(T.ceildiv(max_pages, sub_pages), groups_per_cta),
|
||||
threads=threads,
|
||||
) as (bx, group_block):
|
||||
q_shared = T.alloc_shared([heads, head_dim], T.bfloat16)
|
||||
k_shared = T.alloc_shared([GROUP, head_dim], T.bfloat16)
|
||||
q_shared = T.alloc_shared([heads, head_dim], dtype)
|
||||
k_shared = T.alloc_shared([GROUP, head_dim], dtype)
|
||||
scores = T.alloc_fragment([GROUP, heads], T.float32)
|
||||
reduced = T.alloc_fragment([GROUP], T.float32)
|
||||
T.copy(Q[bx, 0, :, :], q_shared)
|
||||
@@ -301,7 +322,8 @@ def tilelang_qsa_mqa_prefill(
|
||||
# A torch.cat of the padding rows would copy the whole [rows, keys] fp32 matrix,
|
||||
# doubling the dominant prefill buffer; allocate pre-padded instead.
|
||||
logits = torch.zeros((padded_rows, keys), dtype=torch.float32, device=q.device)
|
||||
q_padded = q.to(torch.bfloat16).contiguous()
|
||||
scoring_dtype = _scoring_dtype(q, k)
|
||||
q_padded = q.to(scoring_dtype).contiguous()
|
||||
starts = row_starts.to(device=q.device, dtype=torch.int32).contiguous()
|
||||
ends = row_ends.to(device=q.device, dtype=torch.int32).contiguous()
|
||||
if padding:
|
||||
@@ -309,9 +331,14 @@ def tilelang_qsa_mqa_prefill(
|
||||
starts = torch.cat([starts, starts[-1:].expand(padding)])
|
||||
ends = torch.cat([ends, ends[-1:].expand(padding)])
|
||||
|
||||
_tilelang_qsa_mqa_prefill_kernel(heads=heads, head_dim=head_dim, block_q=block_q)(
|
||||
_tilelang_qsa_mqa_prefill_kernel(
|
||||
heads=heads,
|
||||
head_dim=head_dim,
|
||||
block_q=block_q,
|
||||
dtype=_TILELANG_DTYPES[scoring_dtype],
|
||||
)(
|
||||
q_padded.reshape(-1, head_dim),
|
||||
k[:, 0].to(torch.bfloat16).contiguous(),
|
||||
k[:, 0].to(scoring_dtype).contiguous(),
|
||||
logits,
|
||||
starts,
|
||||
ends,
|
||||
@@ -359,7 +386,8 @@ def tilelang_qsa_mqa_decode(
|
||||
head_alignment,
|
||||
((query_heads + head_alignment - 1) // head_alignment) * head_alignment,
|
||||
)
|
||||
q_kernel = q.to(torch.bfloat16)
|
||||
scoring_dtype = _scoring_dtype(q, k_cache)
|
||||
q_kernel = q.to(scoring_dtype)
|
||||
if kernel_heads != query_heads:
|
||||
q_kernel = torch.cat(
|
||||
[
|
||||
@@ -369,10 +397,13 @@ def tilelang_qsa_mqa_decode(
|
||||
dim=1,
|
||||
)
|
||||
_tilelang_qsa_mqa_decode_kernel(
|
||||
heads=kernel_heads, head_dim=head_dim, page_size=page_size
|
||||
heads=kernel_heads,
|
||||
head_dim=head_dim,
|
||||
page_size=page_size,
|
||||
dtype=_TILELANG_DTYPES[scoring_dtype],
|
||||
)(
|
||||
q_kernel.unsqueeze(1).contiguous(),
|
||||
k_cache.to(torch.bfloat16).contiguous(),
|
||||
k_cache.to(scoring_dtype).contiguous(),
|
||||
page_table.to(device=q.device, dtype=torch.int32).contiguous(),
|
||||
context_lens.to(device=q.device, dtype=torch.int32).contiguous(),
|
||||
logits,
|
||||
|
||||
@@ -193,6 +193,7 @@ class QSAIndexer(MultiPlatformOp):
|
||||
self.q_layernorm.variance_epsilon,
|
||||
self.rotary_emb.is_neox_style,
|
||||
q_heads_padded=q_heads_padded,
|
||||
out_dtype=pool.qsa_compressed_dtype,
|
||||
)
|
||||
return q, token_k, True
|
||||
q_raw = qk[:, : self.index_n_heads * self.index_head_dim]
|
||||
@@ -200,6 +201,8 @@ class QSAIndexer(MultiPlatformOp):
|
||||
-1, self.index_n_heads, self.index_head_dim
|
||||
)
|
||||
q = self.apply_rope(positions, q)
|
||||
if pool is not None:
|
||||
q = q.to(pool.qsa_compressed_dtype)
|
||||
return q, token_k, False
|
||||
|
||||
def normalize_compressed_keys(
|
||||
|
||||
@@ -84,6 +84,7 @@ from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_memory,
|
||||
get_mm,
|
||||
get_model,
|
||||
get_parallel,
|
||||
get_schedule,
|
||||
get_spec,
|
||||
@@ -1922,9 +1923,16 @@ class KVCacheConfigurator:
|
||||
)
|
||||
from sglang.srt.mem_cache.qsa_kv_pool import (
|
||||
QSATokenToKVPool,
|
||||
resolve_qsa_indexer_dtype,
|
||||
)
|
||||
|
||||
qsa_profile = parse_qsa_profile(self.model_config.hf_config)
|
||||
qsa_indexer_dtype = get_model().qsa_indexer_dtype
|
||||
if qsa_indexer_dtype != "auto" and qsa_profile is None:
|
||||
raise ValueError(
|
||||
f"--qsa-indexer-dtype {qsa_indexer_dtype} needs a model with a "
|
||||
"compressed QSA indexer (Qwen4-Exp); this model has none"
|
||||
)
|
||||
if qsa_profile is None:
|
||||
pool_class = HybridLinearKVPool
|
||||
extra_args["use_mla"] = self.use_mla_backend
|
||||
@@ -1936,6 +1944,7 @@ class KVCacheConfigurator:
|
||||
qsa_compress_ratio=qsa_profile.compress_ratio,
|
||||
qsa_token_topk=qsa_profile.budget,
|
||||
num_request_slots=req_to_token_pool.req_to_token.shape[0],
|
||||
qsa_indexer_dtype=resolve_qsa_indexer_dtype(qsa_indexer_dtype),
|
||||
)
|
||||
token_to_kv_pool = pool_class(
|
||||
page_size=self.pool_page_size,
|
||||
|
||||
@@ -8,6 +8,7 @@ Qwen3Next-DSA) adds only the flat per-token index-K cache.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from contextlib import nullcontext
|
||||
from typing import List, Optional
|
||||
|
||||
@@ -16,6 +17,8 @@ import torch
|
||||
from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE
|
||||
from sglang.srt.mem_cache.memory_pool import GB, HybridLinearKVPool, MambaPool
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# State layer IDs are serialized as uint32 by the disaggregation protocols.
|
||||
# Reserve the value below PLE's request-wide sentinel for QSA's request-wide
|
||||
# RoPE ring, which is shared by all full-attention layers.
|
||||
@@ -26,6 +29,22 @@ def _index_k_bytes(*, kv_heads: int, head_dim: int, dtype: torch.dtype) -> int:
|
||||
return kv_heads * head_dim * dtype.itemsize
|
||||
|
||||
|
||||
# ``--qsa-indexer-dtype`` choices. The pending key ring stays bf16 regardless.
|
||||
QSA_INDEXER_DTYPE_CHOICES = ("auto", "bfloat16", "fp8_e4m3")
|
||||
|
||||
|
||||
def resolve_qsa_indexer_dtype(name: str) -> torch.dtype:
|
||||
"""Storage dtype of the compressed QSA indexer cache for a CLI value."""
|
||||
if name in ("auto", "bfloat16"):
|
||||
return torch.bfloat16
|
||||
if name == "fp8_e4m3":
|
||||
return torch.float8_e4m3fn
|
||||
raise ValueError(
|
||||
f"Unsupported --qsa-indexer-dtype {name!r}; expected one of "
|
||||
f"{QSA_INDEXER_DTYPE_CHOICES}"
|
||||
)
|
||||
|
||||
|
||||
class QSATokenToKVPool(HybridLinearKVPool):
|
||||
"""Hybrid KV pool with the minimal BF16 state required by simple QSA."""
|
||||
|
||||
@@ -33,16 +52,23 @@ class QSATokenToKVPool(HybridLinearKVPool):
|
||||
# ``compressed_slot = full_slot // ratio`` needs no ownership bookkeeping;
|
||||
# lifecycle rides the full-KV allocator and radix tree.
|
||||
# Full slot 0 is the reserved padding slot; compressed slot 0 is the inert dump.
|
||||
# Pending-ring dtype: the raw keys are averaged from here, so it stays bf16.
|
||||
index_state_dtype = torch.bfloat16
|
||||
|
||||
@classmethod
|
||||
def qsa_bytes_per_token(
|
||||
cls, *, kv_heads: int, head_dim: int, compress_ratio: int, num_layers: int
|
||||
cls,
|
||||
*,
|
||||
kv_heads: int,
|
||||
head_dim: int,
|
||||
compress_ratio: int,
|
||||
num_layers: int,
|
||||
compressed_dtype: torch.dtype = torch.bfloat16,
|
||||
) -> int:
|
||||
"""Per-token QSA index-cache cost: compressed keys only;
|
||||
the per-request pending ring is budgeted with the other per-request buffers."""
|
||||
index_k_bytes = _index_k_bytes(
|
||||
kv_heads=kv_heads, head_dim=head_dim, dtype=cls.index_state_dtype
|
||||
kv_heads=kv_heads, head_dim=head_dim, dtype=compressed_dtype
|
||||
)
|
||||
return index_k_bytes // compress_ratio * num_layers
|
||||
|
||||
@@ -68,6 +94,7 @@ class QSATokenToKVPool(HybridLinearKVPool):
|
||||
full_kv_pool_class: Optional[type] = None,
|
||||
quant_method=None,
|
||||
post_capture_active: bool = False,
|
||||
qsa_indexer_dtype: torch.dtype = torch.bfloat16,
|
||||
):
|
||||
if page_size <= 1 or page_size % qsa_compress_ratio != 0:
|
||||
raise ValueError(
|
||||
@@ -117,6 +144,18 @@ class QSATokenToKVPool(HybridLinearKVPool):
|
||||
self.qsa_index_kv_heads = int(qsa_index_kv_heads)
|
||||
self.qsa_token_topk = int(qsa_token_topk)
|
||||
self.qsa_block_topk = self.qsa_token_topk // self.qsa_compress_ratio
|
||||
if qsa_indexer_dtype not in (torch.bfloat16, torch.float8_e4m3fn):
|
||||
raise ValueError(
|
||||
"QSA compressed indexer cache dtype must be bfloat16 or "
|
||||
f"float8_e4m3fn, got {qsa_indexer_dtype}"
|
||||
)
|
||||
# Storage dtype of the compressed keys and the index Q (the GEMM operands).
|
||||
self.qsa_compressed_dtype = qsa_indexer_dtype
|
||||
logger.info(
|
||||
"QSA compressed indexer cache dtype: %s (pending ring %s)",
|
||||
self.qsa_compressed_dtype,
|
||||
self.index_state_dtype,
|
||||
)
|
||||
state_size = size + page_size
|
||||
# Compressed slots mirror the full-KV slot space 1:ratio; the "page"
|
||||
# seen by the scoring kernels is one full-KV page's worth of groups.
|
||||
@@ -172,7 +211,7 @@ class QSATokenToKVPool(HybridLinearKVPool):
|
||||
* self.qsa_index_kv_heads
|
||||
* self.qsa_index_head_dim,
|
||||
),
|
||||
dtype=self.index_state_dtype,
|
||||
dtype=self.qsa_compressed_dtype,
|
||||
device=device,
|
||||
)
|
||||
self.qsa_compressed_k_buffer_pool = [
|
||||
|
||||
@@ -45,6 +45,7 @@ from sglang.srt.runtime_context import (
|
||||
get_disagg,
|
||||
get_exec,
|
||||
get_memory,
|
||||
get_model,
|
||||
get_parallel,
|
||||
get_schedule,
|
||||
get_spec,
|
||||
@@ -455,6 +456,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
|
||||
)
|
||||
from sglang.srt.mem_cache.qsa_kv_pool import (
|
||||
QSATokenToKVPool,
|
||||
resolve_qsa_indexer_dtype,
|
||||
)
|
||||
|
||||
if num_layers == 0:
|
||||
@@ -467,6 +469,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
|
||||
head_dim=qsa_profile.head_dim,
|
||||
compress_ratio=qsa_profile.compress_ratio,
|
||||
num_layers=num_layers,
|
||||
compressed_dtype=resolve_qsa_indexer_dtype(get_model().qsa_indexer_dtype),
|
||||
)
|
||||
|
||||
def _compute_dsa_indexer_cell_size(
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
"""Fused QSA indexer-prep kernels must match the eager indexer path bit-for-bit,
|
||||
up to rare last-ulp RMSNorm flips (see assert_bit_comparable)."""
|
||||
up to rare last-ulp RMSNorm flips (see assert_bit_comparable). With an fp8
|
||||
indexer cache the same rows are cast to e4m3 at the store, so a bf16 last-ulp
|
||||
flip may move the e4m3 code by one step (see assert_fp8_within_one_ulp)."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
@@ -85,13 +87,22 @@ class FakePool:
|
||||
|
||||
index_state_dtype = torch.bfloat16
|
||||
|
||||
def __init__(self, num_slots, num_compressed, device, dtype=torch.bfloat16):
|
||||
def __init__(
|
||||
self,
|
||||
num_slots,
|
||||
num_compressed,
|
||||
device,
|
||||
dtype=torch.bfloat16,
|
||||
compressed_dtype=None,
|
||||
):
|
||||
# Ring in the compute dtype, compressed cache and index Q in the storage dtype.
|
||||
self.qsa_compressed_dtype = compressed_dtype or dtype
|
||||
self.key_state = torch.zeros(num_slots, 1, HEAD_DIM, dtype=dtype, device=device)
|
||||
self.qsa_rope_position_buffer = torch.zeros(
|
||||
num_slots, 3, dtype=torch.int64, device=device
|
||||
)
|
||||
self.compressed = torch.zeros(
|
||||
num_compressed, 1, HEAD_DIM, dtype=dtype, device=device
|
||||
num_compressed, 1, HEAD_DIM, dtype=self.qsa_compressed_dtype, device=device
|
||||
)
|
||||
|
||||
def get_qsa_key_state_buffer(self, layer_id):
|
||||
@@ -128,6 +139,17 @@ def assert_bit_comparable(actual, expected, max_frac=1e-5, max_abs=0.02):
|
||||
assert peak <= max_abs, f"largest deviation {peak} exceeds {max_abs}"
|
||||
|
||||
|
||||
def assert_fp8_within_one_ulp(actual, expected):
|
||||
"""One e4m3 code step, or 2^-8 absolute for subnormals and sign crossings."""
|
||||
assert actual.dtype == expected.dtype == torch.float8_e4m3fn
|
||||
code_diff = (
|
||||
actual.view(torch.uint8).int() - expected.view(torch.uint8).int()
|
||||
).abs()
|
||||
abs_diff = (actual.float() - expected.float()).abs()
|
||||
bad = int((~((code_diff <= 1) | (abs_diff <= 2**-8))).sum())
|
||||
assert bad == 0, f"{bad} fp8 elements differ by more than one ulp"
|
||||
|
||||
|
||||
def _eager_compress_reference(indexer, pool, group_locs, write_locs):
|
||||
"""The pre-fusion compression chain, via the indexer's own helpers."""
|
||||
key_groups = pool.get_qsa_key_state_buffer(0)[group_locs.long()]
|
||||
@@ -144,15 +166,18 @@ def _eager_compress_reference(indexer, pool, group_locs, write_locs):
|
||||
"mrope_section, mrope_interleaved",
|
||||
[([24, 20, 20], True), ([24, 20, 20], False), (None, False)],
|
||||
)
|
||||
def test_fused_compress_matches_eager(num_groups, mrope_section, mrope_interleaved):
|
||||
@pytest.mark.parametrize("compressed_dtype", [torch.bfloat16, torch.float8_e4m3fn])
|
||||
def test_fused_compress_matches_eager(
|
||||
num_groups, mrope_section, mrope_interleaved, compressed_dtype
|
||||
):
|
||||
device = torch.device("cuda")
|
||||
dtype = torch.bfloat16
|
||||
torch.manual_seed(num_groups)
|
||||
rotary = _make_rotary(mrope_section, mrope_interleaved, device, dtype)
|
||||
indexer = _make_indexer(rotary, device, dtype)
|
||||
|
||||
pool_ref = FakePool(8192, 4096, device, dtype)
|
||||
pool_new = FakePool(8192, 4096, device, dtype)
|
||||
pool_ref = FakePool(8192, 4096, device, dtype, compressed_dtype)
|
||||
pool_new = FakePool(8192, 4096, device, dtype, compressed_dtype)
|
||||
pool_new.key_state.copy_(
|
||||
pool_ref.key_state.copy_(
|
||||
torch.randn(8192, 1, HEAD_DIM, device=device, dtype=dtype)
|
||||
@@ -172,7 +197,122 @@ def test_fused_compress_matches_eager(num_groups, mrope_section, mrope_interleav
|
||||
_eager_compress_reference(indexer, pool_ref, group_locs, write_locs)
|
||||
indexer._fused_compress_store(pool_new, group_locs, write_locs)
|
||||
|
||||
assert_bit_comparable(pool_new.compressed, pool_ref.compressed)
|
||||
if compressed_dtype == torch.float8_e4m3fn:
|
||||
assert_fp8_within_one_ulp(pool_new.compressed, pool_ref.compressed)
|
||||
else:
|
||||
assert_bit_comparable(pool_new.compressed, pool_ref.compressed)
|
||||
|
||||
|
||||
def test_fused_q_prep_fp8_matches_eager_cast():
|
||||
"""fp8 Q from the fused prep must match the eager Q cast at the end (one ulp)."""
|
||||
device = torch.device("cuda")
|
||||
dtype = torch.bfloat16
|
||||
torch.manual_seed(11)
|
||||
rotary = _make_rotary([24, 20, 20], True, device, dtype)
|
||||
indexer = _make_indexer(rotary, device, dtype)
|
||||
tokens = 300
|
||||
hidden = torch.randn(tokens, HIDDEN, device=device, dtype=dtype)
|
||||
positions = (
|
||||
torch.arange(5000, 5000 + tokens, device=device)
|
||||
.unsqueeze(0)
|
||||
.expand(3, -1)
|
||||
.contiguous()
|
||||
)
|
||||
qk, _ = indexer.index_qk_proj(hidden)
|
||||
q_ref = indexer.q_layernorm(qk[:, : NUM_Q_HEADS * HEAD_DIM].reshape(-1, HEAD_DIM))
|
||||
q_ref = indexer.apply_rope(positions, q_ref.reshape(tokens, NUM_Q_HEADS, HEAD_DIM))
|
||||
pool = FakePool(4096, 4096, device, dtype, torch.float8_e4m3fn)
|
||||
cache_loc = torch.arange(1, tokens + 1, device=device)
|
||||
q_new, _, stored = indexer.project_qk(
|
||||
hidden, positions, pool=pool, cache_loc=cache_loc
|
||||
)
|
||||
assert stored
|
||||
assert q_new.dtype == torch.float8_e4m3fn
|
||||
assert_fp8_within_one_ulp(q_new, q_ref.to(torch.float8_e4m3fn))
|
||||
|
||||
|
||||
def _selected_score_multisets(logits, block_indices):
|
||||
"""Sorted logits of the selected blocks per row, so tied blocks compare equal."""
|
||||
out = []
|
||||
for row in range(block_indices.shape[0]):
|
||||
picked = block_indices[row][block_indices[row] >= 0].long()
|
||||
out.append(logits[row, picked].sort().values)
|
||||
return out
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn])
|
||||
def test_tilelang_mqa_matches_torch_reference(dtype):
|
||||
"""TileLang scoring vs the fp32 torch reference on the same operands. The
|
||||
selected blocks are compared as score multisets because fp8 ties at the
|
||||
top-k boundary."""
|
||||
from sglang.srt.layers.attention.qsa.kernel import qsa_fast_topk
|
||||
from sglang.srt.layers.attention.qsa.mqa import (
|
||||
HAS_TILELANG,
|
||||
tilelang_qsa_mqa_decode,
|
||||
tilelang_qsa_mqa_prefill,
|
||||
torch_qsa_mqa_decode,
|
||||
torch_qsa_mqa_prefill,
|
||||
)
|
||||
|
||||
if not HAS_TILELANG:
|
||||
pytest.skip("tilelang unavailable")
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(3)
|
||||
# Decode: paged compressed cache. qsa_fast_topk only supports the production
|
||||
# block budget (512).
|
||||
batch, max_pages, page_size, block_topk = 6, 256, 16, 512
|
||||
q = torch.randn(batch, NUM_Q_HEADS, HEAD_DIM, device=device).to(dtype)
|
||||
cache = torch.randn(512, page_size, 1, HEAD_DIM, device=device).to(dtype)
|
||||
page_table = torch.stack(
|
||||
[torch.randperm(512, device=device)[:max_pages] for _ in range(batch)]
|
||||
).to(torch.int32)
|
||||
context_lens = torch.randint(
|
||||
600, max_pages * page_size + 1, (batch,), device=device
|
||||
).to(torch.int32)
|
||||
max_model_len = max_pages * page_size
|
||||
ref = torch_qsa_mqa_decode(q, cache, page_table, context_lens, max_model_len)
|
||||
out = tilelang_qsa_mqa_decode(q, cache, page_table, context_lens, max_model_len)
|
||||
finite = torch.isfinite(ref)
|
||||
assert torch.equal(finite, torch.isfinite(out))
|
||||
torch.testing.assert_close(out[finite], ref[finite], rtol=2e-3, atol=2e-3)
|
||||
row_starts = torch.zeros_like(context_lens)
|
||||
sel_ref = _selected_score_multisets(
|
||||
ref, qsa_fast_topk(ref, row_starts, context_lens, topk=block_topk)
|
||||
)
|
||||
sel_out = _selected_score_multisets(
|
||||
ref, qsa_fast_topk(out, row_starts, context_lens, topk=block_topk)
|
||||
)
|
||||
for a, b in zip(sel_ref, sel_out):
|
||||
torch.testing.assert_close(a, b, rtol=2e-3, atol=2e-3)
|
||||
# Prefill: packed keys with ragged causal ranges.
|
||||
rows, keys = 70, 512
|
||||
qp = torch.randn(rows, NUM_Q_HEADS, HEAD_DIM, device=device).to(dtype)
|
||||
kp = torch.randn(keys, 1, HEAD_DIM, device=device).to(dtype)
|
||||
row_starts = torch.zeros(rows, dtype=torch.int32, device=device)
|
||||
row_ends = torch.randint(1, keys + 1, (rows,), device=device).to(torch.int32)
|
||||
ref_p = torch_qsa_mqa_prefill(qp, kp, row_starts, row_ends)
|
||||
out_p = tilelang_qsa_mqa_prefill(qp, kp, row_starts, row_ends)
|
||||
finite = torch.isfinite(ref_p)
|
||||
assert torch.equal(finite, torch.isfinite(out_p))
|
||||
torch.testing.assert_close(out_p[finite], ref_p[finite], rtol=2e-3, atol=2e-3)
|
||||
|
||||
|
||||
def test_tilelang_mqa_rejects_mixed_operand_dtypes():
|
||||
"""Mixed fp8/bf16 operands must fail instead of dequantizing the cache."""
|
||||
from sglang.srt.layers.attention.qsa.mqa import (
|
||||
HAS_TILELANG,
|
||||
tilelang_qsa_mqa_decode,
|
||||
)
|
||||
|
||||
if not HAS_TILELANG:
|
||||
pytest.skip("tilelang unavailable")
|
||||
device = torch.device("cuda")
|
||||
q = torch.randn(2, NUM_Q_HEADS, HEAD_DIM, device=device, dtype=torch.bfloat16)
|
||||
cache = torch.randn(8, 16, 1, HEAD_DIM, device=device).to(torch.float8_e4m3fn)
|
||||
page_table = torch.zeros(2, 4, dtype=torch.int32, device=device)
|
||||
context_lens = torch.full((2,), 32, dtype=torch.int32, device=device)
|
||||
with pytest.raises(ValueError, match="same dtype"):
|
||||
tilelang_qsa_mqa_decode(q, cache, page_table, context_lens, 64)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.int32, torch.int64])
|
||||
|
||||
@@ -458,6 +458,51 @@ class TestQwen4StateWire(unittest.TestCase):
|
||||
[(0, 2), (1, 3)],
|
||||
)
|
||||
|
||||
def _qsa_compressed_manager(self, *, src_item_len):
|
||||
manager = object.__new__(MooncakeKVManager)
|
||||
manager.attn_tp_size = 4
|
||||
manager.pp_size = 1
|
||||
manager.is_mla_backend = False
|
||||
manager.is_hybrid_mla_backend = False
|
||||
manager.kv_args = SimpleNamespace(
|
||||
engine_rank=0,
|
||||
state_types=[StateType.QSA_COMPRESSED],
|
||||
state_data_ptrs=[[1000]],
|
||||
state_item_lens=[[src_item_len]],
|
||||
state_dim_per_tensor=[[]],
|
||||
state_layer_ids=[[24]],
|
||||
)
|
||||
manager.sent = []
|
||||
manager._send_kvcache_generic = lambda **kw: manager.sent.append(kw) or 0
|
||||
return manager
|
||||
|
||||
def test_qsa_compressed_layout_mismatch_rejected_before_transfer(self):
|
||||
"""Equal attention TP with fp8 on one peer only halves that peer's
|
||||
compressed item length. The transfer uses the source length as the
|
||||
destination stride, so the mismatch must be rejected before any write."""
|
||||
req = SimpleNamespace(mooncake_session_id="session", dst_state_indices=[[3]])
|
||||
decode_info = SimpleNamespace(
|
||||
dst_attn_tp_size=4,
|
||||
dst_state_data_ptrs=[[2000]],
|
||||
dst_state_item_lens=[[16]],
|
||||
dst_state_dim_per_tensor=[[]],
|
||||
dst_state_layer_ids=[[24]],
|
||||
)
|
||||
manager = self._qsa_compressed_manager(src_item_len=32)
|
||||
with self.assertRaisesRegex(RuntimeError, "QSA_COMPRESSED layout differs"):
|
||||
MooncakeKVManager.maybe_send_extra(
|
||||
manager, req, [[3]], None, target_rank_registration_info=decode_info
|
||||
)
|
||||
self.assertEqual(manager.sent, [])
|
||||
|
||||
manager = self._qsa_compressed_manager(src_item_len=16)
|
||||
rc = MooncakeKVManager.maybe_send_extra(
|
||||
manager, req, [[3]], None, target_rank_registration_info=decode_info
|
||||
)
|
||||
self.assertEqual(rc, 0)
|
||||
self.assertEqual(len(manager.sent), 1)
|
||||
self.assertEqual(manager.sent[0]["item_lens"], [16])
|
||||
|
||||
def test_replicated_state_tp_policy(self):
|
||||
for src_tp, dst_tp, rank, expected in (
|
||||
(4, 1, 0, True),
|
||||
|
||||
@@ -929,6 +929,38 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
self._resolved(self._construct(*qwen4), "ple_offload_embedding")
|
||||
)
|
||||
|
||||
def test_qwen4_fp8_indexer_dtype_platform_gate(self):
|
||||
"""fp8_e4m3 needs CUDA SM90/SM100 and a compressed QSA indexer. The bf16
|
||||
spellings never consult the platform."""
|
||||
qwen4 = ("Qwen4ExpForConditionalGeneration", "qwen4_exp")
|
||||
compressed = {
|
||||
"indexer_n_heads": 4,
|
||||
"indexer_kv_heads": 1,
|
||||
"indexer_head_dim": 128,
|
||||
"indexer_budget": 2048,
|
||||
"indexer_compress_ratio": 4,
|
||||
}
|
||||
with override_platform(is_cuda=True, is_sm100=True):
|
||||
sa = self._construct(
|
||||
*qwen4, config_extra=compressed, qsa_indexer_dtype="fp8_e4m3"
|
||||
)
|
||||
self.assertEqual(self._resolved(sa, "qsa_indexer_dtype"), "fp8_e4m3")
|
||||
# No compressed indexer fields: no QSA profile, nothing to store in fp8.
|
||||
with self.assertRaisesRegex(ValueError, "compressed QSA indexer"):
|
||||
self._construct(*qwen4, qsa_indexer_dtype="fp8_e4m3")
|
||||
with override_platform(
|
||||
is_cuda=False, is_hip=True, is_sm90=False, is_sm100=False
|
||||
):
|
||||
with self.assertRaisesRegex(ValueError, "SM90/SM100"):
|
||||
self._construct(
|
||||
*qwen4, config_extra=compressed, qsa_indexer_dtype="fp8_e4m3"
|
||||
)
|
||||
for name in ("auto", "bfloat16"):
|
||||
sa = self._construct(
|
||||
*qwen4, config_extra=compressed, qsa_indexer_dtype=name
|
||||
)
|
||||
self.assertEqual(self._resolved(sa, "qsa_indexer_dtype"), name)
|
||||
|
||||
def test_minimax_m2_enables_tf32_matmul(self):
|
||||
sa = self._construct("MiniMaxM2ForCausalLM", "llama")
|
||||
self.assertTrue(self._resolved(sa, "enable_tf32_matmul"))
|
||||
|
||||
Reference in New Issue
Block a user