diff --git a/python/sglang/kernels/jit/csrc/attention/qsa_indexer.cuh b/python/sglang/kernels/jit/csrc/attention/qsa_indexer.cuh index 01efe40af3..689ff8c85e 100644 --- a/python/sglang/kernels/jit/csrc/attention/qsa_indexer.cuh +++ b/python/sglang/kernels/jit/csrc/attention/qsa_indexer.cuh @@ -13,6 +13,7 @@ #include #include +#include namespace sglang { @@ -46,6 +47,18 @@ SGL_DEVICE float eager_round(float x) { #endif } +/// \brief Convert a value computed in the compute dtype to the cache storage +/// dtype. Identity when both agree. The fp8 compressed cache takes a plain +/// e4m3 cast with no scale, as the row is RMSNormed before it gets here. +template +SGL_DEVICE TOut qsa_storage_cast(T value) { + if constexpr (std::is_same_v) { + return value; + } else { + return DTypeTrait::from(static_cast(value)); + } +} + /** * \brief Apply (M)RoPE to a normed row staged in shared memory. * @@ -54,7 +67,8 @@ SGL_DEVICE float eager_round(float x) { * uses all-zero maps; Qwen interleaved/sectioned MRoPE maps are built on the * host). The cos/sin cache row is [cos(half), sin(half)] of width rotary_dim. * - * \tparam T Storage element type: bf16_t | fp16_t. + * \tparam T Compute/ring element type: bf16_t | fp16_t. + * \tparam TOut Destination storage type: T, or fp8_e4m3_t for the fp8 compressed cache. * \tparam kHeadDim Compile-time head dimension (multiple of 32). * \tparam kIsNeox true -> NeoX pairing (d, d+half); false -> GPT-J (2i, 2i+1). * \param smem_row Normed row [kHeadDim], one warp cooperates. @@ -64,21 +78,21 @@ SGL_DEVICE float eager_round(float x) { * \param pos Resolved per-axis positions for this token (>= 3 entries). * \param rotary_dim Rotated prefix length; tail dims pass through. */ -template +template SGL_DEVICE void qsa_mrope_apply( const T* __restrict__ smem_row, - T* __restrict__ out_row, + TOut* __restrict__ out_row, const float* __restrict__ cos_sin_cache, const int32_t* __restrict__ axis_map, const int64_t* pos, const int32_t rotary_dim) { using namespace device; constexpr int kPerLane = kHeadDim / kWarpThreads; - using vec_t = AlignedVector; + using out_vec_t = AlignedVector; const uint32_t lane = threadIdx.x % kWarpThreads; const int32_t half = rotary_dim / 2; - vec_t ov; + out_vec_t ov; #pragma unroll for (int i = 0; i < kPerLane; ++i) { const int32_t d = static_cast(lane * kPerLane) + i; @@ -119,7 +133,7 @@ SGL_DEVICE void qsa_mrope_apply( o = smem_row[d]; } } - ov[i] = o; + ov[i] = qsa_storage_cast(o); } ov.store(out_row, lane); // offset is in vector units } @@ -203,11 +217,12 @@ struct QsaIndexQPrepParams { * head, zero-fill of padded heads, raw token-K store and RoPE-position store. * One CTA (4 warps) per token; one warp per query head. */ -template +template __global__ __launch_bounds__(128) void qsa_index_q_prep_kernel(const QsaIndexQPrepParams __grid_constant__ params) { using namespace device; constexpr int kPerLane = kHeadDim / kWarpThreads; using vec_t = AlignedVector; + using out_vec_t = AlignedVector; const uint32_t token = blockIdx.x; const uint32_t warp = threadIdx.x / kWarpThreads; const uint32_t lane = threadIdx.x % kWarpThreads; @@ -225,15 +240,16 @@ __global__ __launch_bounds__(128) void qsa_index_q_prep_kernel(const QsaIndexQPr } for (int32_t h = static_cast(warp); h < params.q_heads_padded; h += 4) { - T* out_row = static_cast(params.q_out) + (static_cast(token) * params.q_heads_padded + h) * kHeadDim; + TOut* out_row = + static_cast(params.q_out) + (static_cast(token) * params.q_heads_padded + h) * kHeadDim; if (h < params.num_q_heads) { const T* x_row = static_cast(params.qk) + qk_row + h * kHeadDim; qsa_gemma_norm_row(x_row, static_cast(params.weight), params.eps, smem_rows[warp]); - qsa_mrope_apply( + qsa_mrope_apply( smem_rows[warp], out_row, params.cos_sin_cache, params.axis_map, pos, params.rotary_dim); } else { - vec_t zv; - zv.fill(DTypeTrait::from(0.0f)); + out_vec_t zv; + zv.fill(DTypeTrait::from(0.0f)); zv.store(out_row, lane); // offset is in vector units } } @@ -274,7 +290,7 @@ struct QsaIndexKCompressParams { * MRoPE at the group-start position, store into the compressed cache. * One warp per group. */ -template +template __global__ __launch_bounds__(128) void qsa_index_k_compress_kernel(const QsaIndexKCompressParams __grid_constant__ params) { using namespace device; @@ -345,8 +361,9 @@ __launch_bounds__(128) void qsa_index_k_compress_kernel(const QsaIndexKCompressP pos[a] = params.rope_position_buffer[static_cast(loc0) * 3 + a]; } - T* out_row = static_cast(params.compressed_k_buffer) + static_cast(params.write_locs[group]) * kHeadDim; - qsa_mrope_apply( + TOut* out_row = + static_cast(params.compressed_k_buffer) + static_cast(params.write_locs[group]) * kHeadDim; + qsa_mrope_apply( smem_rows[warp], out_row, params.cos_sin_cache, params.axis_map, pos, params.rotary_dim); device::PDLTriggerSecondary(); @@ -355,12 +372,13 @@ __launch_bounds__(128) void qsa_index_k_compress_kernel(const QsaIndexKCompressP /** * \brief Validate inputs and launch `qsa_index_q_prep_kernel` (one CTA per token). * - * \tparam T Element type: bf16_t | fp16_t. + * \tparam T Input/ring element type: bf16_t | fp16_t. + * \tparam TOut Output storage type: T, or fp8_e4m3_t (fp8 indexer cache). * \tparam kHeadDim Index head dimension: 64 | 128 | 256. * \tparam kIsNeox RoPE pairing style. * \tparam kUsePDL Whether to launch with PDL enabled. */ -template +template void qsa_index_q_prep( tvm::ffi::TensorView qk, tvm::ffi::TensorView q_out, @@ -383,7 +401,7 @@ void qsa_index_q_prep( TensorMatcher({tokens, (num_q_heads + 1) * D}).with_dtype().with_device(device).verify(qk); auto heads_padded = SymbolicSize{"heads_padded"}; - TensorMatcher({tokens, heads_padded, D}).with_dtype().with_device(device).verify(q_out); + TensorMatcher({tokens, heads_padded, D}).with_dtype().with_device(device).verify(q_out); TensorMatcher({D}).with_dtype().with_device(device).verify(weight); auto cache_rows = SymbolicSize{"cos_sin_cache_rows"}; TensorMatcher({cache_rows, rotary_dim}).with_dtype().with_device(device).verify(cos_sin_cache); @@ -421,13 +439,13 @@ void qsa_index_q_prep( .eps = eps, }; LaunchKernel(static_cast(num_tokens), 128, device.unwrap()) - .enable_pdl(kUsePDL)(qsa_index_q_prep_kernel, params); + .enable_pdl(kUsePDL)(qsa_index_q_prep_kernel, params); } /** * \brief Validate inputs and launch `qsa_index_k_compress_kernel` (one warp per group). */ -template +template void qsa_index_k_compress( tvm::ffi::TensorView key_state_buffer, tvm::ffi::TensorView group_locs, @@ -456,7 +474,7 @@ void qsa_index_k_compress( TensorMatcher({D}).with_dtype().with_device(device).verify(weight); TensorMatcher({groups}).with_dtype().with_device(device).verify(write_locs); auto compressed_slots = SymbolicSize{"compressed_slots"}; - TensorMatcher({compressed_slots, D}).with_dtype().with_device(device).verify(compressed_k_buffer); + TensorMatcher({compressed_slots, D}).with_dtype().with_device(device).verify(compressed_k_buffer); const int64_t num_groups = groups.unwrap(); CHECK_HOST(num_groups > 0) << "qsa_index_k_compress: no groups"; @@ -480,7 +498,7 @@ void qsa_index_k_compress( .eps = eps, }; LaunchKernel(static_cast(div_ceil(num_groups, 4)), 128, device.unwrap()) - .enable_pdl(kUsePDL)(qsa_index_k_compress_kernel, params); + .enable_pdl(kUsePDL)(qsa_index_k_compress_kernel, params); } } // namespace sglang diff --git a/python/sglang/kernels/ops/attention/qsa_indexer.py b/python/sglang/kernels/ops/attention/qsa_indexer.py index 7e7f71f5f2..dcfa1c5ecb 100644 --- a/python/sglang/kernels/ops/attention/qsa_indexer.py +++ b/python/sglang/kernels/ops/attention/qsa_indexer.py @@ -9,11 +9,14 @@ chain gather -> fp32 mean -> GemmaRMSNorm -> MRoPE(group-start position) -> compressed-cache store into one kernel launch. Both kernels reproduce the eager numerics step by step (fp32 norm reduction, -per-op rounding to the storage dtype during RoPE, fp32 group mean rounded to -the storage dtype before the norm). Outputs are bit-comparable to the eager +per-op rounding to the compute dtype during RoPE, fp32 group mean rounded to +the compute dtype before the norm). Outputs are bit-comparable to the eager indexer path: the eager RMSNorm (flashinfer's CuTe DSL kernel) reduces sums of squares in an order that cannot be reproduced exactly, so a small fraction of rows (~1 in 30k) may flip by one bf16 ulp on a rounding boundary. + +With an fp8 indexer cache the rows are cast to e4m3 at the store (no scale), +so fp8 outputs match the eager path to one e4m3 ulp. """ from __future__ import annotations @@ -35,16 +38,23 @@ if TYPE_CHECKING: @cache_once def _jit_qsa_indexer_module( - dtype: torch.dtype, head_dim: int, is_neox_style: bool + dtype: torch.dtype, out_dtype: torch.dtype, head_dim: int, is_neox_style: bool ) -> Module: """Compile and cache the JIT QSA indexer module for one specialisation.""" if dtype not in (torch.bfloat16, torch.float16): raise RuntimeError(f"Unsupported dtype {dtype}. Supported: bfloat16, float16") + if out_dtype != dtype and out_dtype != torch.float8_e4m3fn: + raise RuntimeError( + f"Unsupported output dtype {out_dtype} for input dtype {dtype}. " + "Supported: the input dtype, or float8_e4m3fn" + ) if head_dim not in (64, 128, 256): raise RuntimeError( f"Unsupported index head_dim {head_dim}. Supported: 64, 128, 256" ) - args = make_cpp_args(dtype, head_dim, is_neox_style, is_arch_support_pdl()) + args = make_cpp_args( + dtype, out_dtype, head_dim, is_neox_style, is_arch_support_pdl() + ) return load_jit( "qsa_indexer", *args, @@ -70,6 +80,7 @@ def qsa_index_q_norm_rope_store( eps: float, is_neox_style: bool, q_heads_padded: Optional[int] = None, + out_dtype: Optional[torch.dtype] = None, ) -> torch.Tensor: """ Per-token fused index-Q prep. @@ -92,6 +103,8 @@ def qsa_index_q_norm_rope_store( is_neox_style : NeoX (True) or GPT-J (False) RoPE pairing q_heads_padded : output head count; heads >= num_q_heads are zero-filled (defaults to num_q_heads) + out_dtype : dtype of the returned Q (default qk.dtype), fp8 e4m3 when + the indexer cache is fp8 Returns ------- @@ -103,10 +116,12 @@ def qsa_index_q_norm_rope_store( q_heads_padded = num_q_heads if positions.ndim == 1: positions = positions.unsqueeze(0) + if out_dtype is None: + out_dtype = qk.dtype q_out = torch.empty( - (num_tokens, q_heads_padded, head_dim), dtype=qk.dtype, device=qk.device + (num_tokens, q_heads_padded, head_dim), dtype=out_dtype, device=qk.device ) - module = _jit_qsa_indexer_module(qk.dtype, head_dim, is_neox_style) + module = _jit_qsa_indexer_module(qk.dtype, out_dtype, head_dim, is_neox_style) module.q_prep( qk, q_out, @@ -153,14 +168,17 @@ def qsa_index_k_compress_store( axis_map : CUDA int32 [rotary_dim // 2] position-axis per pair index weight : [head_dim] gemma norm weight (kernel applies 1 + w) write_locs : CUDA int32 [groups] compressed-cache slots to write - compressed_k_buffer : CUDA [compressed_slots, head_dim] (written) + compressed_k_buffer : CUDA [compressed_slots, head_dim] (written), ring dtype + or fp8 e4m3 compress_ratio : raw keys per compressed key rotary_dim : rotated prefix of each head row eps : RMSNorm epsilon is_neox_style : NeoX (True) or GPT-J (False) RoPE pairing """ head_dim = weight.shape[0] - module = _jit_qsa_indexer_module(key_state_buffer.dtype, head_dim, is_neox_style) + module = _jit_qsa_indexer_module( + key_state_buffer.dtype, compressed_k_buffer.dtype, head_dim, is_neox_style + ) module.k_compress( key_state_buffer, group_locs, diff --git a/python/sglang/srt/arg_groups/fields/model.py b/python/sglang/srt/arg_groups/fields/model.py index b6ca074691..f2223c1ab4 100644 --- a/python/sglang/srt/arg_groups/fields/model.py +++ b/python/sglang/srt/arg_groups/fields/model.py @@ -211,6 +211,17 @@ class Model(msgspec.Struct): resolvable=True, ), ] = "auto" + qsa_indexer_dtype: A[ + str, + Arg( + help=( + "Storage dtype of the compressed QSA indexer cache and index query " + '(Qwen4-Exp). "fp8_e4m3" stores them as plain e4m3 and scores blocks ' + "with fp8 GEMMs (CUDA SM90+). The main KV cache is unaffected." + ), + choices=["auto", "bfloat16", "fp8_e4m3"], + ), + ] = "auto" modelopt_quant: A[ Optional[Union[str, Dict]], ( diff --git a/python/sglang/srt/arg_groups/model_overrides/qwen4_exp.py b/python/sglang/srt/arg_groups/model_overrides/qwen4_exp.py index 53d281f9be..c9cd9a15f9 100644 --- a/python/sglang/srt/arg_groups/model_overrides/qwen4_exp.py +++ b/python/sglang/srt/arg_groups/model_overrides/qwen4_exp.py @@ -75,6 +75,17 @@ def _qwen4_exp_overrides(server_args: Any, hf_config: Any) -> dict: ) profile = parse_qsa_profile(hf_config) + if cfg.qsa_indexer_dtype == "fp8_e4m3": + # fp8 scoring runs on TileLang fp8 GEMMs, validated on Hopper and Blackwell. + platform = get_platform() + if not (platform.is_cuda and (platform.is_sm90 or platform.is_sm100)): + raise ValueError( + "--qsa-indexer-dtype fp8_e4m3 requires a CUDA SM90/SM100 GPU" + ) + if profile is None: + raise ValueError( + "--qsa-indexer-dtype fp8_e4m3 requires the compressed QSA indexer" + ) if profile is not None: # Compressed slot = full_slot // ratio; all backends need page-aligned pages. # mamba_radix_cache_strategy resolves later, so do not gate on it. diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index 0f0edd888b..836d1d8e10 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -1766,7 +1766,8 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): raise RuntimeError( f"PD Disaggregation does NOT support PD different TP sizes for non-MLA {st.upper()} hybrid models yet." ) - if has_heterogeneous_attn_tp and is_qwen4_qsa_state: + if is_qwen4_qsa_state and target_rank_registration_info is not None: + # Reject QSA state layout mismatches up front. if len(dst_item_lens) != len(dst_data_ptrs): raise RuntimeError( f"Replicated {st.upper()} destination pointer/item-length " @@ -1791,6 +1792,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): "prefill and decode entries: " f"{layout_mismatches}" ) + if has_heterogeneous_attn_tp and is_qwen4_qsa_state: local_tp_rank_in_group = ( self.kv_args.engine_rank % self.attn_tp_size ) diff --git a/python/sglang/srt/layers/attention/qsa/mqa.py b/python/sglang/srt/layers/attention/qsa/mqa.py index 88e201ee06..529e302450 100644 --- a/python/sglang/srt/layers/attention/qsa/mqa.py +++ b/python/sglang/srt/layers/attention/qsa/mqa.py @@ -108,6 +108,25 @@ def torch_qsa_mqa_decode( return logits +# TileLang dtype names of the scoring-kernel operands. The logits stay fp32. +_TILELANG_DTYPES = { + torch.bfloat16: "bfloat16", + torch.float8_e4m3fn: "float8_e4m3fn", +} + + +def _scoring_dtype(q: torch.Tensor, k: torch.Tensor) -> torch.dtype: + """The dtype both GEMM operands run in: fp8 only when Q and K already are.""" + if q.dtype == torch.float8_e4m3fn or k.dtype == torch.float8_e4m3fn: + if q.dtype != k.dtype: + raise ValueError( + "QSA fp8 scoring needs Q and the compressed K in the same dtype, " + f"got q={q.dtype} k={k.dtype}" + ) + return torch.float8_e4m3fn + return torch.bfloat16 + + if HAS_TILELANG: @tilelang.jit( @@ -124,21 +143,22 @@ if HAS_TILELANG: block_q: int = 32, num_stages: int = 3, threads: int = 512, + dtype: str = "bfloat16", ): rows = T.dynamic("rows") keys = T.dynamic("keys") @T.prim_func def kernel( - Q: T.Tensor([rows * heads, head_dim], T.bfloat16), # type: ignore - K: T.Tensor([keys, head_dim], T.bfloat16), # type: ignore + Q: T.Tensor([rows * heads, head_dim], dtype), # type: ignore + K: T.Tensor([keys, head_dim], dtype), # type: ignore Logits: T.Tensor([rows, keys], T.float32), # type: ignore Starts: T.Tensor([rows], T.int32), # type: ignore Ends: T.Tensor([rows], T.int32), # type: ignore ): with T.Kernel(T.ceildiv(rows, block_q), threads=threads) as bx: - q_shared = T.alloc_shared([block_q * heads, head_dim], T.bfloat16) - k_shared = T.alloc_shared([block_n, head_dim], T.bfloat16) + q_shared = T.alloc_shared([block_q * heads, head_dim], dtype) + k_shared = T.alloc_shared([block_n, head_dim], dtype) scores = T.alloc_fragment([block_n, block_q * heads], T.float32) scores_3d = T.reshape(scores, (block_n, block_q, heads)) reduced = T.alloc_fragment([block_n, block_q], T.float32) @@ -208,6 +228,7 @@ if HAS_TILELANG: groups_per_cta: int = 1, num_stages: int = 3, threads: int = 128, + dtype: str = "bfloat16", ): # The MMA layout needs 64 GEMM rows; a compressed page has full_page // ratio, # so pages are packed as sub-pages of one 64-row tile. @@ -221,8 +242,8 @@ if HAS_TILELANG: @T.prim_func def kernel( - Q: T.Tensor([batch, 1, heads, head_dim], T.bfloat16), # type: ignore - KCache: T.Tensor([pages, page_size, 1, head_dim], T.bfloat16), # type: ignore + Q: T.Tensor([batch, 1, heads, head_dim], dtype), # type: ignore + KCache: T.Tensor([pages, page_size, 1, head_dim], dtype), # type: ignore PageTable: T.Tensor([batch, max_pages], T.int32), # type: ignore ContextLens: T.Tensor([batch], T.int32), # type: ignore Logits: T.Tensor([batch, max_model_len], T.float32), # type: ignore @@ -233,8 +254,8 @@ if HAS_TILELANG: T.ceildiv(T.ceildiv(max_pages, sub_pages), groups_per_cta), threads=threads, ) as (bx, group_block): - q_shared = T.alloc_shared([heads, head_dim], T.bfloat16) - k_shared = T.alloc_shared([GROUP, head_dim], T.bfloat16) + q_shared = T.alloc_shared([heads, head_dim], dtype) + k_shared = T.alloc_shared([GROUP, head_dim], dtype) scores = T.alloc_fragment([GROUP, heads], T.float32) reduced = T.alloc_fragment([GROUP], T.float32) T.copy(Q[bx, 0, :, :], q_shared) @@ -301,7 +322,8 @@ def tilelang_qsa_mqa_prefill( # A torch.cat of the padding rows would copy the whole [rows, keys] fp32 matrix, # doubling the dominant prefill buffer; allocate pre-padded instead. logits = torch.zeros((padded_rows, keys), dtype=torch.float32, device=q.device) - q_padded = q.to(torch.bfloat16).contiguous() + scoring_dtype = _scoring_dtype(q, k) + q_padded = q.to(scoring_dtype).contiguous() starts = row_starts.to(device=q.device, dtype=torch.int32).contiguous() ends = row_ends.to(device=q.device, dtype=torch.int32).contiguous() if padding: @@ -309,9 +331,14 @@ def tilelang_qsa_mqa_prefill( starts = torch.cat([starts, starts[-1:].expand(padding)]) ends = torch.cat([ends, ends[-1:].expand(padding)]) - _tilelang_qsa_mqa_prefill_kernel(heads=heads, head_dim=head_dim, block_q=block_q)( + _tilelang_qsa_mqa_prefill_kernel( + heads=heads, + head_dim=head_dim, + block_q=block_q, + dtype=_TILELANG_DTYPES[scoring_dtype], + )( q_padded.reshape(-1, head_dim), - k[:, 0].to(torch.bfloat16).contiguous(), + k[:, 0].to(scoring_dtype).contiguous(), logits, starts, ends, @@ -359,7 +386,8 @@ def tilelang_qsa_mqa_decode( head_alignment, ((query_heads + head_alignment - 1) // head_alignment) * head_alignment, ) - q_kernel = q.to(torch.bfloat16) + scoring_dtype = _scoring_dtype(q, k_cache) + q_kernel = q.to(scoring_dtype) if kernel_heads != query_heads: q_kernel = torch.cat( [ @@ -369,10 +397,13 @@ def tilelang_qsa_mqa_decode( dim=1, ) _tilelang_qsa_mqa_decode_kernel( - heads=kernel_heads, head_dim=head_dim, page_size=page_size + heads=kernel_heads, + head_dim=head_dim, + page_size=page_size, + dtype=_TILELANG_DTYPES[scoring_dtype], )( q_kernel.unsqueeze(1).contiguous(), - k_cache.to(torch.bfloat16).contiguous(), + k_cache.to(scoring_dtype).contiguous(), page_table.to(device=q.device, dtype=torch.int32).contiguous(), context_lens.to(device=q.device, dtype=torch.int32).contiguous(), logits, diff --git a/python/sglang/srt/layers/attention/qsa/qsa_indexer.py b/python/sglang/srt/layers/attention/qsa/qsa_indexer.py index ebfadbd4ab..7939525eca 100644 --- a/python/sglang/srt/layers/attention/qsa/qsa_indexer.py +++ b/python/sglang/srt/layers/attention/qsa/qsa_indexer.py @@ -193,6 +193,7 @@ class QSAIndexer(MultiPlatformOp): self.q_layernorm.variance_epsilon, self.rotary_emb.is_neox_style, q_heads_padded=q_heads_padded, + out_dtype=pool.qsa_compressed_dtype, ) return q, token_k, True q_raw = qk[:, : self.index_n_heads * self.index_head_dim] @@ -200,6 +201,8 @@ class QSAIndexer(MultiPlatformOp): -1, self.index_n_heads, self.index_head_dim ) q = self.apply_rope(positions, q) + if pool is not None: + q = q.to(pool.qsa_compressed_dtype) return q, token_k, False def normalize_compressed_keys( diff --git a/python/sglang/srt/mem_cache/kv_cache_configurator.py b/python/sglang/srt/mem_cache/kv_cache_configurator.py index 2094584b17..65761b087e 100644 --- a/python/sglang/srt/mem_cache/kv_cache_configurator.py +++ b/python/sglang/srt/mem_cache/kv_cache_configurator.py @@ -84,6 +84,7 @@ from sglang.srt.runtime_context import ( get_exec, get_memory, get_mm, + get_model, get_parallel, get_schedule, get_spec, @@ -1922,9 +1923,16 @@ class KVCacheConfigurator: ) from sglang.srt.mem_cache.qsa_kv_pool import ( QSATokenToKVPool, + resolve_qsa_indexer_dtype, ) qsa_profile = parse_qsa_profile(self.model_config.hf_config) + qsa_indexer_dtype = get_model().qsa_indexer_dtype + if qsa_indexer_dtype != "auto" and qsa_profile is None: + raise ValueError( + f"--qsa-indexer-dtype {qsa_indexer_dtype} needs a model with a " + "compressed QSA indexer (Qwen4-Exp); this model has none" + ) if qsa_profile is None: pool_class = HybridLinearKVPool extra_args["use_mla"] = self.use_mla_backend @@ -1936,6 +1944,7 @@ class KVCacheConfigurator: qsa_compress_ratio=qsa_profile.compress_ratio, qsa_token_topk=qsa_profile.budget, num_request_slots=req_to_token_pool.req_to_token.shape[0], + qsa_indexer_dtype=resolve_qsa_indexer_dtype(qsa_indexer_dtype), ) token_to_kv_pool = pool_class( page_size=self.pool_page_size, diff --git a/python/sglang/srt/mem_cache/qsa_kv_pool.py b/python/sglang/srt/mem_cache/qsa_kv_pool.py index b3e4f6010d..992411f4ef 100644 --- a/python/sglang/srt/mem_cache/qsa_kv_pool.py +++ b/python/sglang/srt/mem_cache/qsa_kv_pool.py @@ -8,6 +8,7 @@ Qwen3Next-DSA) adds only the flat per-token index-K cache. from __future__ import annotations +import logging from contextlib import nullcontext from typing import List, Optional @@ -16,6 +17,8 @@ import torch from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE from sglang.srt.mem_cache.memory_pool import GB, HybridLinearKVPool, MambaPool +logger = logging.getLogger(__name__) + # State layer IDs are serialized as uint32 by the disaggregation protocols. # Reserve the value below PLE's request-wide sentinel for QSA's request-wide # RoPE ring, which is shared by all full-attention layers. @@ -26,6 +29,22 @@ def _index_k_bytes(*, kv_heads: int, head_dim: int, dtype: torch.dtype) -> int: return kv_heads * head_dim * dtype.itemsize +# ``--qsa-indexer-dtype`` choices. The pending key ring stays bf16 regardless. +QSA_INDEXER_DTYPE_CHOICES = ("auto", "bfloat16", "fp8_e4m3") + + +def resolve_qsa_indexer_dtype(name: str) -> torch.dtype: + """Storage dtype of the compressed QSA indexer cache for a CLI value.""" + if name in ("auto", "bfloat16"): + return torch.bfloat16 + if name == "fp8_e4m3": + return torch.float8_e4m3fn + raise ValueError( + f"Unsupported --qsa-indexer-dtype {name!r}; expected one of " + f"{QSA_INDEXER_DTYPE_CHOICES}" + ) + + class QSATokenToKVPool(HybridLinearKVPool): """Hybrid KV pool with the minimal BF16 state required by simple QSA.""" @@ -33,16 +52,23 @@ class QSATokenToKVPool(HybridLinearKVPool): # ``compressed_slot = full_slot // ratio`` needs no ownership bookkeeping; # lifecycle rides the full-KV allocator and radix tree. # Full slot 0 is the reserved padding slot; compressed slot 0 is the inert dump. + # Pending-ring dtype: the raw keys are averaged from here, so it stays bf16. index_state_dtype = torch.bfloat16 @classmethod def qsa_bytes_per_token( - cls, *, kv_heads: int, head_dim: int, compress_ratio: int, num_layers: int + cls, + *, + kv_heads: int, + head_dim: int, + compress_ratio: int, + num_layers: int, + compressed_dtype: torch.dtype = torch.bfloat16, ) -> int: """Per-token QSA index-cache cost: compressed keys only; the per-request pending ring is budgeted with the other per-request buffers.""" index_k_bytes = _index_k_bytes( - kv_heads=kv_heads, head_dim=head_dim, dtype=cls.index_state_dtype + kv_heads=kv_heads, head_dim=head_dim, dtype=compressed_dtype ) return index_k_bytes // compress_ratio * num_layers @@ -68,6 +94,7 @@ class QSATokenToKVPool(HybridLinearKVPool): full_kv_pool_class: Optional[type] = None, quant_method=None, post_capture_active: bool = False, + qsa_indexer_dtype: torch.dtype = torch.bfloat16, ): if page_size <= 1 or page_size % qsa_compress_ratio != 0: raise ValueError( @@ -117,6 +144,18 @@ class QSATokenToKVPool(HybridLinearKVPool): self.qsa_index_kv_heads = int(qsa_index_kv_heads) self.qsa_token_topk = int(qsa_token_topk) self.qsa_block_topk = self.qsa_token_topk // self.qsa_compress_ratio + if qsa_indexer_dtype not in (torch.bfloat16, torch.float8_e4m3fn): + raise ValueError( + "QSA compressed indexer cache dtype must be bfloat16 or " + f"float8_e4m3fn, got {qsa_indexer_dtype}" + ) + # Storage dtype of the compressed keys and the index Q (the GEMM operands). + self.qsa_compressed_dtype = qsa_indexer_dtype + logger.info( + "QSA compressed indexer cache dtype: %s (pending ring %s)", + self.qsa_compressed_dtype, + self.index_state_dtype, + ) state_size = size + page_size # Compressed slots mirror the full-KV slot space 1:ratio; the "page" # seen by the scoring kernels is one full-KV page's worth of groups. @@ -172,7 +211,7 @@ class QSATokenToKVPool(HybridLinearKVPool): * self.qsa_index_kv_heads * self.qsa_index_head_dim, ), - dtype=self.index_state_dtype, + dtype=self.qsa_compressed_dtype, device=device, ) self.qsa_compressed_k_buffer_pool = [ diff --git a/python/sglang/srt/model_executor/pool_configurator.py b/python/sglang/srt/model_executor/pool_configurator.py index b0417d545f..4a1f60f8c2 100644 --- a/python/sglang/srt/model_executor/pool_configurator.py +++ b/python/sglang/srt/model_executor/pool_configurator.py @@ -45,6 +45,7 @@ from sglang.srt.runtime_context import ( get_disagg, get_exec, get_memory, + get_model, get_parallel, get_schedule, get_spec, @@ -455,6 +456,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator): ) from sglang.srt.mem_cache.qsa_kv_pool import ( QSATokenToKVPool, + resolve_qsa_indexer_dtype, ) if num_layers == 0: @@ -467,6 +469,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator): head_dim=qsa_profile.head_dim, compress_ratio=qsa_profile.compress_ratio, num_layers=num_layers, + compressed_dtype=resolve_qsa_indexer_dtype(get_model().qsa_indexer_dtype), ) def _compute_dsa_indexer_cell_size( diff --git a/test/registered/kernels/ops/attention/qsa/test_qsa_indexer.py b/test/registered/kernels/ops/attention/qsa/test_qsa_indexer.py index 9664b83f9c..acba97f16d 100644 --- a/test/registered/kernels/ops/attention/qsa/test_qsa_indexer.py +++ b/test/registered/kernels/ops/attention/qsa/test_qsa_indexer.py @@ -1,5 +1,7 @@ """Fused QSA indexer-prep kernels must match the eager indexer path bit-for-bit, -up to rare last-ulp RMSNorm flips (see assert_bit_comparable).""" +up to rare last-ulp RMSNorm flips (see assert_bit_comparable). With an fp8 +indexer cache the same rows are cast to e4m3 at the store, so a bf16 last-ulp +flip may move the e4m3 code by one step (see assert_fp8_within_one_ulp).""" from types import SimpleNamespace @@ -85,13 +87,22 @@ class FakePool: index_state_dtype = torch.bfloat16 - def __init__(self, num_slots, num_compressed, device, dtype=torch.bfloat16): + def __init__( + self, + num_slots, + num_compressed, + device, + dtype=torch.bfloat16, + compressed_dtype=None, + ): + # Ring in the compute dtype, compressed cache and index Q in the storage dtype. + self.qsa_compressed_dtype = compressed_dtype or dtype self.key_state = torch.zeros(num_slots, 1, HEAD_DIM, dtype=dtype, device=device) self.qsa_rope_position_buffer = torch.zeros( num_slots, 3, dtype=torch.int64, device=device ) self.compressed = torch.zeros( - num_compressed, 1, HEAD_DIM, dtype=dtype, device=device + num_compressed, 1, HEAD_DIM, dtype=self.qsa_compressed_dtype, device=device ) def get_qsa_key_state_buffer(self, layer_id): @@ -128,6 +139,17 @@ def assert_bit_comparable(actual, expected, max_frac=1e-5, max_abs=0.02): assert peak <= max_abs, f"largest deviation {peak} exceeds {max_abs}" +def assert_fp8_within_one_ulp(actual, expected): + """One e4m3 code step, or 2^-8 absolute for subnormals and sign crossings.""" + assert actual.dtype == expected.dtype == torch.float8_e4m3fn + code_diff = ( + actual.view(torch.uint8).int() - expected.view(torch.uint8).int() + ).abs() + abs_diff = (actual.float() - expected.float()).abs() + bad = int((~((code_diff <= 1) | (abs_diff <= 2**-8))).sum()) + assert bad == 0, f"{bad} fp8 elements differ by more than one ulp" + + def _eager_compress_reference(indexer, pool, group_locs, write_locs): """The pre-fusion compression chain, via the indexer's own helpers.""" key_groups = pool.get_qsa_key_state_buffer(0)[group_locs.long()] @@ -144,15 +166,18 @@ def _eager_compress_reference(indexer, pool, group_locs, write_locs): "mrope_section, mrope_interleaved", [([24, 20, 20], True), ([24, 20, 20], False), (None, False)], ) -def test_fused_compress_matches_eager(num_groups, mrope_section, mrope_interleaved): +@pytest.mark.parametrize("compressed_dtype", [torch.bfloat16, torch.float8_e4m3fn]) +def test_fused_compress_matches_eager( + num_groups, mrope_section, mrope_interleaved, compressed_dtype +): device = torch.device("cuda") dtype = torch.bfloat16 torch.manual_seed(num_groups) rotary = _make_rotary(mrope_section, mrope_interleaved, device, dtype) indexer = _make_indexer(rotary, device, dtype) - pool_ref = FakePool(8192, 4096, device, dtype) - pool_new = FakePool(8192, 4096, device, dtype) + pool_ref = FakePool(8192, 4096, device, dtype, compressed_dtype) + pool_new = FakePool(8192, 4096, device, dtype, compressed_dtype) pool_new.key_state.copy_( pool_ref.key_state.copy_( torch.randn(8192, 1, HEAD_DIM, device=device, dtype=dtype) @@ -172,7 +197,122 @@ def test_fused_compress_matches_eager(num_groups, mrope_section, mrope_interleav _eager_compress_reference(indexer, pool_ref, group_locs, write_locs) indexer._fused_compress_store(pool_new, group_locs, write_locs) - assert_bit_comparable(pool_new.compressed, pool_ref.compressed) + if compressed_dtype == torch.float8_e4m3fn: + assert_fp8_within_one_ulp(pool_new.compressed, pool_ref.compressed) + else: + assert_bit_comparable(pool_new.compressed, pool_ref.compressed) + + +def test_fused_q_prep_fp8_matches_eager_cast(): + """fp8 Q from the fused prep must match the eager Q cast at the end (one ulp).""" + device = torch.device("cuda") + dtype = torch.bfloat16 + torch.manual_seed(11) + rotary = _make_rotary([24, 20, 20], True, device, dtype) + indexer = _make_indexer(rotary, device, dtype) + tokens = 300 + hidden = torch.randn(tokens, HIDDEN, device=device, dtype=dtype) + positions = ( + torch.arange(5000, 5000 + tokens, device=device) + .unsqueeze(0) + .expand(3, -1) + .contiguous() + ) + qk, _ = indexer.index_qk_proj(hidden) + q_ref = indexer.q_layernorm(qk[:, : NUM_Q_HEADS * HEAD_DIM].reshape(-1, HEAD_DIM)) + q_ref = indexer.apply_rope(positions, q_ref.reshape(tokens, NUM_Q_HEADS, HEAD_DIM)) + pool = FakePool(4096, 4096, device, dtype, torch.float8_e4m3fn) + cache_loc = torch.arange(1, tokens + 1, device=device) + q_new, _, stored = indexer.project_qk( + hidden, positions, pool=pool, cache_loc=cache_loc + ) + assert stored + assert q_new.dtype == torch.float8_e4m3fn + assert_fp8_within_one_ulp(q_new, q_ref.to(torch.float8_e4m3fn)) + + +def _selected_score_multisets(logits, block_indices): + """Sorted logits of the selected blocks per row, so tied blocks compare equal.""" + out = [] + for row in range(block_indices.shape[0]): + picked = block_indices[row][block_indices[row] >= 0].long() + out.append(logits[row, picked].sort().values) + return out + + +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn]) +def test_tilelang_mqa_matches_torch_reference(dtype): + """TileLang scoring vs the fp32 torch reference on the same operands. The + selected blocks are compared as score multisets because fp8 ties at the + top-k boundary.""" + from sglang.srt.layers.attention.qsa.kernel import qsa_fast_topk + from sglang.srt.layers.attention.qsa.mqa import ( + HAS_TILELANG, + tilelang_qsa_mqa_decode, + tilelang_qsa_mqa_prefill, + torch_qsa_mqa_decode, + torch_qsa_mqa_prefill, + ) + + if not HAS_TILELANG: + pytest.skip("tilelang unavailable") + device = torch.device("cuda") + torch.manual_seed(3) + # Decode: paged compressed cache. qsa_fast_topk only supports the production + # block budget (512). + batch, max_pages, page_size, block_topk = 6, 256, 16, 512 + q = torch.randn(batch, NUM_Q_HEADS, HEAD_DIM, device=device).to(dtype) + cache = torch.randn(512, page_size, 1, HEAD_DIM, device=device).to(dtype) + page_table = torch.stack( + [torch.randperm(512, device=device)[:max_pages] for _ in range(batch)] + ).to(torch.int32) + context_lens = torch.randint( + 600, max_pages * page_size + 1, (batch,), device=device + ).to(torch.int32) + max_model_len = max_pages * page_size + ref = torch_qsa_mqa_decode(q, cache, page_table, context_lens, max_model_len) + out = tilelang_qsa_mqa_decode(q, cache, page_table, context_lens, max_model_len) + finite = torch.isfinite(ref) + assert torch.equal(finite, torch.isfinite(out)) + torch.testing.assert_close(out[finite], ref[finite], rtol=2e-3, atol=2e-3) + row_starts = torch.zeros_like(context_lens) + sel_ref = _selected_score_multisets( + ref, qsa_fast_topk(ref, row_starts, context_lens, topk=block_topk) + ) + sel_out = _selected_score_multisets( + ref, qsa_fast_topk(out, row_starts, context_lens, topk=block_topk) + ) + for a, b in zip(sel_ref, sel_out): + torch.testing.assert_close(a, b, rtol=2e-3, atol=2e-3) + # Prefill: packed keys with ragged causal ranges. + rows, keys = 70, 512 + qp = torch.randn(rows, NUM_Q_HEADS, HEAD_DIM, device=device).to(dtype) + kp = torch.randn(keys, 1, HEAD_DIM, device=device).to(dtype) + row_starts = torch.zeros(rows, dtype=torch.int32, device=device) + row_ends = torch.randint(1, keys + 1, (rows,), device=device).to(torch.int32) + ref_p = torch_qsa_mqa_prefill(qp, kp, row_starts, row_ends) + out_p = tilelang_qsa_mqa_prefill(qp, kp, row_starts, row_ends) + finite = torch.isfinite(ref_p) + assert torch.equal(finite, torch.isfinite(out_p)) + torch.testing.assert_close(out_p[finite], ref_p[finite], rtol=2e-3, atol=2e-3) + + +def test_tilelang_mqa_rejects_mixed_operand_dtypes(): + """Mixed fp8/bf16 operands must fail instead of dequantizing the cache.""" + from sglang.srt.layers.attention.qsa.mqa import ( + HAS_TILELANG, + tilelang_qsa_mqa_decode, + ) + + if not HAS_TILELANG: + pytest.skip("tilelang unavailable") + device = torch.device("cuda") + q = torch.randn(2, NUM_Q_HEADS, HEAD_DIM, device=device, dtype=torch.bfloat16) + cache = torch.randn(8, 16, 1, HEAD_DIM, device=device).to(torch.float8_e4m3fn) + page_table = torch.zeros(2, 4, dtype=torch.int32, device=device) + context_lens = torch.full((2,), 32, dtype=torch.int32, device=device) + with pytest.raises(ValueError, match="same dtype"): + tilelang_qsa_mqa_decode(q, cache, page_table, context_lens, 64) @pytest.mark.parametrize("dtype", [torch.int32, torch.int64]) diff --git a/test/registered/unit/disaggregation/test_disaggregation_wire.py b/test/registered/unit/disaggregation/test_disaggregation_wire.py index 1f5d15706b..80561bdef7 100644 --- a/test/registered/unit/disaggregation/test_disaggregation_wire.py +++ b/test/registered/unit/disaggregation/test_disaggregation_wire.py @@ -458,6 +458,51 @@ class TestQwen4StateWire(unittest.TestCase): [(0, 2), (1, 3)], ) + def _qsa_compressed_manager(self, *, src_item_len): + manager = object.__new__(MooncakeKVManager) + manager.attn_tp_size = 4 + manager.pp_size = 1 + manager.is_mla_backend = False + manager.is_hybrid_mla_backend = False + manager.kv_args = SimpleNamespace( + engine_rank=0, + state_types=[StateType.QSA_COMPRESSED], + state_data_ptrs=[[1000]], + state_item_lens=[[src_item_len]], + state_dim_per_tensor=[[]], + state_layer_ids=[[24]], + ) + manager.sent = [] + manager._send_kvcache_generic = lambda **kw: manager.sent.append(kw) or 0 + return manager + + def test_qsa_compressed_layout_mismatch_rejected_before_transfer(self): + """Equal attention TP with fp8 on one peer only halves that peer's + compressed item length. The transfer uses the source length as the + destination stride, so the mismatch must be rejected before any write.""" + req = SimpleNamespace(mooncake_session_id="session", dst_state_indices=[[3]]) + decode_info = SimpleNamespace( + dst_attn_tp_size=4, + dst_state_data_ptrs=[[2000]], + dst_state_item_lens=[[16]], + dst_state_dim_per_tensor=[[]], + dst_state_layer_ids=[[24]], + ) + manager = self._qsa_compressed_manager(src_item_len=32) + with self.assertRaisesRegex(RuntimeError, "QSA_COMPRESSED layout differs"): + MooncakeKVManager.maybe_send_extra( + manager, req, [[3]], None, target_rank_registration_info=decode_info + ) + self.assertEqual(manager.sent, []) + + manager = self._qsa_compressed_manager(src_item_len=16) + rc = MooncakeKVManager.maybe_send_extra( + manager, req, [[3]], None, target_rank_registration_info=decode_info + ) + self.assertEqual(rc, 0) + self.assertEqual(len(manager.sent), 1) + self.assertEqual(manager.sent[0]["item_lens"], [16]) + def test_replicated_state_tp_policy(self): for src_tp, dst_tp, rank, expected in ( (4, 1, 0, True), diff --git a/test/registered/unit/test_model_overrides.py b/test/registered/unit/test_model_overrides.py index a1b186ea6d..9036047bc8 100644 --- a/test/registered/unit/test_model_overrides.py +++ b/test/registered/unit/test_model_overrides.py @@ -929,6 +929,38 @@ class TestGoldenModelOverrides(_IsolatedPublish): self._resolved(self._construct(*qwen4), "ple_offload_embedding") ) + def test_qwen4_fp8_indexer_dtype_platform_gate(self): + """fp8_e4m3 needs CUDA SM90/SM100 and a compressed QSA indexer. The bf16 + spellings never consult the platform.""" + qwen4 = ("Qwen4ExpForConditionalGeneration", "qwen4_exp") + compressed = { + "indexer_n_heads": 4, + "indexer_kv_heads": 1, + "indexer_head_dim": 128, + "indexer_budget": 2048, + "indexer_compress_ratio": 4, + } + with override_platform(is_cuda=True, is_sm100=True): + sa = self._construct( + *qwen4, config_extra=compressed, qsa_indexer_dtype="fp8_e4m3" + ) + self.assertEqual(self._resolved(sa, "qsa_indexer_dtype"), "fp8_e4m3") + # No compressed indexer fields: no QSA profile, nothing to store in fp8. + with self.assertRaisesRegex(ValueError, "compressed QSA indexer"): + self._construct(*qwen4, qsa_indexer_dtype="fp8_e4m3") + with override_platform( + is_cuda=False, is_hip=True, is_sm90=False, is_sm100=False + ): + with self.assertRaisesRegex(ValueError, "SM90/SM100"): + self._construct( + *qwen4, config_extra=compressed, qsa_indexer_dtype="fp8_e4m3" + ) + for name in ("auto", "bfloat16"): + sa = self._construct( + *qwen4, config_extra=compressed, qsa_indexer_dtype=name + ) + self.assertEqual(self._resolved(sa, "qsa_indexer_dtype"), name) + def test_minimax_m2_enables_tf32_matmul(self): sa = self._construct("MiniMaxM2ForCausalLM", "llama") self.assertTrue(self._resolved(sa, "enable_tf32_matmul"))