diff --git a/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx b/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx index 2a458a345b..84a28e1183 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx @@ -122,6 +122,8 @@ Keep the checkpoint's KDA lower-bound setting unchanged. In particular, do not o On Blackwell, the recipes default to an FP8 KV cache with TRT-LLM DSA: on GB300 this pairing measured 2.9–5.7% higher throughput and about 1.8x the KV token capacity at identical pool bytes, with GSM8K accuracy within noise of BF16. BF16 KV with TileLang DSA remains selectable in the deployment panel and is the default on H100 and H200, where FP8 KV with TRT-LLM DSA is disabled. MI355X instead uses FP8 KV with TileLang DSA in the recorded ROCm recipe. Switch the dtype and both DSA backends together; TileLang DSA with FP8 KV is not a valid CUDA combination, and TRT-LLM DSA is not the MI355X backend. +On Hopper, BF16 KV can also run the sparse prefill in FP8 with `--dsa-prefill-backend flashmla_sparse_q8` (decode stays on `tilelang` and the cache stays BF16): on H200 this pairing measured 5.8–12.1% higher throughput on prefill-only requests, with GSM8K accuracy within noise of BF16 + TileLang. It is SM90-only and requires a build that includes [PR #41790](https://github.com/sgl-project/sglang/pull/41790); FP8 KV is not available with it for this model. + ### Extend the cache hierarchy Keep **HiCache** off when GPU memory is sufficient. Select **L1 + L2** to spill reusable cache entries into host memory. Select **+ L3** only after configuring Mooncake on every serving node; the generated command exposes the required configuration path. These options remain selectable but are marked **Not Verified** until the resulting command is validated on the chosen hardware. diff --git a/docs/docs/advanced_features/attention_backend.mdx b/docs/docs/advanced_features/attention_backend.mdx index 270a956b7b..a842451236 100644 --- a/docs/docs/advanced_features/attention_backend.mdx +++ b/docs/docs/advanced_features/attention_backend.mdx @@ -436,7 +436,7 @@ Internally, the DSA backend dispatches to different sub-backends for prefill and flashmla_sparse_q8 ✅ ❌ - Native FP8 (q8×kv8) sparse prefill on Hopper (SM90); requires --kv-cache-dtype fp8_e4m3 + Native FP8 (q8×kv8) sparse prefill on Hopper (SM90); BF16 or FP8 KV cache triton_sparse_mla diff --git a/docs/src/snippets/configs/zai-org/glm-5.3-flash-benchmarks.jsx b/docs/src/snippets/configs/zai-org/glm-5.3-flash-benchmarks.jsx index 7719cb2899..f8ac2b0fec 100644 --- a/docs/src/snippets/configs/zai-org/glm-5.3-flash-benchmarks.jsx +++ b/docs/src/snippets/configs/zai-org/glm-5.3-flash-benchmarks.jsx @@ -265,6 +265,12 @@ export const benchmarks = [ notes: "Full GSM8K (all 1,319 problems) on 8x H200 (TP8/EP8) with zai-org/GLM-5.3-Flash at f040cc72e6: 97.35% for the recommended selection; 97.19-97.57% across all 4 measured selections. Run with `sgl-eval run gsm8k --base-url http://localhost:30000/v1 --num-threads 32 --max-tokens 32768`; gsm8k's registered default leaves thinking off, so these are non-thinking numbers and are not directly comparable to the GB300 rows above. Accuracy only, no speed measurement.", }, + // BF16 + Q8 sparse prefill has only a relative A/B so far; these empty entries outrank the hw + strategy rows above. + // A defaultAccuracy added to glm-5.3-flash.jsx would show on them as if it were measured. + { match: { hw: "h200", strategy: "low-latency", kvDsaPair: "bf16-q8-prefill" } }, + { match: { hw: "h200", strategy: "high-throughput", kvDsaPair: "bf16-q8-prefill" } }, + { match: { hw: "h100", strategy: "low-latency", kvDsaPair: "bf16-q8-prefill" } }, + { match: { hw: "h100", strategy: "high-throughput", kvDsaPair: "bf16-q8-prefill" } }, // TODO: speed measured on the earlier TP8/EP8 deep_gemm command with adaptive MTP; re-measure on the current recipe. { match: { hw: "b200", strategy: "low-latency" }, diff --git a/docs/src/snippets/configs/zai-org/glm-5.3-flash.jsx b/docs/src/snippets/configs/zai-org/glm-5.3-flash.jsx index 40cece4e91..b0770f1c4b 100644 --- a/docs/src/snippets/configs/zai-org/glm-5.3-flash.jsx +++ b/docs/src/snippets/configs/zai-org/glm-5.3-flash.jsx @@ -91,6 +91,22 @@ export const config = { "--dsa-decode-backend tilelang", ], }, + // Order matters: a disabled pick is re-seated onto the first enabled + // option, so this entry must stay after BF16 + TileLang to keep the + // Hopper default unchanged. + { + id: "bf16-q8-prefill", + label: "BF16 + Q8 sparse prefill", + disabled: (s) => !["h100", "h200"].includes(s.hw), + disableReason: "The Q8KV8 sparse prefill kernel is SM90-only (Hopper).", + stripPrefixes: ["--kv-cache-dtype", "--dsa-prefill-backend", "--dsa-decode-backend"], + flags: [ + "--kv-cache-dtype bfloat16", + "--dsa-prefill-backend flashmla_sparse_q8", + "--dsa-decode-backend tilelang", + ], + hints: ["Measured on H200: higher prefill throughput than BF16 + TileLang; requires a build with PR #41790."], + }, ], }, { @@ -979,7 +995,9 @@ sgl-eval run gsm8k \\ verified: true, verificationStatus: (s) => s.bcg !== "off" ? "unverified" : - ["off", "l2"].includes(s.hicache) ? "verified" : "unverified", + ["off", "l2"].includes(s.hicache) + ? (s.kvDsaPair === "bf16-q8-prefill" ? "in-progress" : "verified") + : "unverified", env: [], flags: [ "--model-path {{MODEL_NAME}}", @@ -1027,7 +1045,9 @@ sgl-eval run gsm8k \\ verified: true, verificationStatus: (s) => s.bcg !== "off" ? "unverified" : - ["off", "l2"].includes(s.hicache) ? "verified" : "unverified", + ["off", "l2"].includes(s.hicache) + ? (s.kvDsaPair === "bf16-q8-prefill" ? "in-progress" : "verified") + : "unverified", env: [], flags: [ "--model-path {{MODEL_NAME}}", diff --git a/python/sglang/kernels/ops/attention/dsa/dequant_k_cache.py b/python/sglang/kernels/ops/attention/dsa/dequant_k_cache.py index befdff24c2..3108701f93 100644 --- a/python/sglang/kernels/ops/attention/dsa/dequant_k_cache.py +++ b/python/sglang/kernels/ops/attention/dsa/dequant_k_cache.py @@ -572,18 +572,22 @@ def _concat_cast_kv_fp8_pad_kernel( write zeros (the -1-sentinel landing pad the kernel's clamp maps to).""" row = tl.program_id(0).to(tl.int64) offs_n = tl.arange(0, NOPE) - offs_r = tl.arange(0, ROPE) head = NOPE + ROPE if row < num_tokens: v_n = tl.load(k_ptr + row * k_stride + offs_n) tl.store(out_ptr + row * head + offs_n, v_n.to(tl.float8e4nv)) - v_r = tl.load(kr_ptr + row * kr_stride + offs_r) - tl.store(out_ptr + row * head + NOPE + offs_r, v_r.to(tl.float8e4nv)) + # NoPE models pass ROPE=0; tl.arange rejects empty ranges, so skip statically. + if ROPE > 0: + offs_r = tl.arange(0, ROPE) + v_r = tl.load(kr_ptr + row * kr_stride + offs_r) + tl.store(out_ptr + row * head + NOPE + offs_r, v_r.to(tl.float8e4nv)) else: zero_n = tl.zeros([NOPE], dtype=tl.float32).to(tl.float8e4nv) - zero_r = tl.zeros([ROPE], dtype=tl.float32).to(tl.float8e4nv) tl.store(out_ptr + row * head + offs_n, zero_n) - tl.store(out_ptr + row * head + NOPE + offs_r, zero_r) + if ROPE > 0: + offs_r = tl.arange(0, ROPE) + zero_r = tl.zeros([ROPE], dtype=tl.float32).to(tl.float8e4nv) + tl.store(out_ptr + row * head + NOPE + offs_r, zero_r) def concat_cast_kv_fp8_pad( @@ -598,7 +602,7 @@ def concat_cast_kv_fp8_pad( tail (3 kernels + one [tokens, 576] bf16 alloc). Same bf16->fp8 store-cast the gather kernel uses (bit-identical bytes). - ``out``: [total_rows, 576] fp8 slice (total_rows = num_tokens + pad band); + ``out``: [total_rows, NOPE+ROPE] fp8 slice (total_rows = num_tokens + pad band); ``k``: [num_tokens, NOPE] bf16 view; ``k_rope``: [num_tokens, ROPE] bf16. """ total_rows, head = out.shape @@ -607,7 +611,7 @@ def concat_cast_kv_fp8_pad( assert head == nope + rope and out.dtype == torch.float8_e4m3fn k2 = k.view(num_tokens, nope) kr2 = k_rope.view(num_tokens, rope) - assert k2.stride(-1) == 1 and kr2.stride(-1) == 1 + assert k2.stride(-1) == 1 and (rope == 0 or kr2.stride(-1) == 1) _concat_cast_kv_fp8_pad_kernel[(total_rows,)]( out, k2, @@ -619,3 +623,95 @@ def concat_cast_kv_fp8_pad( ROPE=rope, ) return out + + +@triton.jit +def _gather_cast_kv_bf16_paged_vec_kernel( + kv_buffer_ptr, + out_ptr, + loc_ptr, + num_tokens, + total_rows, + buffer_stride: tl.constexpr, + out_stride: tl.constexpr, + HEAD_DIM: tl.constexpr, + TOKENS_PER_PROG: tl.constexpr, +): + pid = tl.program_id(0) + rows = pid * TOKENS_PER_PROG + tl.arange(0, TOKENS_PER_PROG) + gathered = rows < num_tokens + loc = tl.load(loc_ptr + rows, mask=gathered, other=0).to(tl.int64) + cols = tl.arange(0, HEAD_DIM) + src = tl.load( + kv_buffer_ptr + loc[:, None] * buffer_stride + cols[None, :], + mask=gathered[:, None], + other=0.0, + ) + # Rows past num_tokens load nothing and store that zero: they are the + # -1-sentinel landing pad, so the trailing zero_ fuses into this kernel. + tl.store( + out_ptr + rows[:, None].to(tl.int64) * out_stride + cols[None, :], + src, + mask=(rows < total_rows)[:, None], + ) + + +def gather_cast_kv_fp8_pad_paged( + out: torch.Tensor, + kv_cache: torch.Tensor, + page_table_1_flattened: torch.Tensor, + nope_dim: int, +) -> torch.Tensor: + """Q8KV8 prefix KV prep for a bf16 MLA pool: gather rows [latent | rope] by + token slot, store-cast them into the fp8 buffer, and zero the trailing + -1-sentinel landing-pad rows. + + ``out``: [num_tokens + pad_rows, head_dim] fp8, contents arbitrary; + ``kv_cache``: [slots, 1, head_dim] bf16; + ``page_table_1_flattened``: [num_tokens] int32 slot ids. + """ + head_dim = out.shape[-1] + num_tokens = page_table_1_flattened.shape[0] + assert out.dtype == torch.float8_e4m3fn and out.is_contiguous() + assert kv_cache.dtype == torch.bfloat16 and kv_cache.shape[-1] == head_dim + # The gather addresses the source as loc * stride(0) + arange(0, nope_dim), + # so only stride(0) is passed; a non-unit last-dim stride would silently read + # the wrong elements. concat_cast_kv_fp8_pad asserts the same invariant. + assert kv_cache.stride(-1) == 1 + if head_dim == nope_dim and head_dim & (head_dim - 1) == 0: + # NoPE: source and destination rows are both one contiguous HEAD_DIM run, + # so gather, cast and pad-zeroing fit in one kernel. The shared get + # kernel below runs one program per token -- 131072 CTAs at a 128k + # context -- and CTA count, not access width, is what binds there. + total_rows = out.shape[0] + _gather_cast_kv_bf16_paged_vec_kernel[ + (triton.cdiv(total_rows, _GATHER_TOKENS_PER_PROG),) + ]( + kv_cache, + out, + page_table_1_flattened, + num_tokens, + total_rows, + kv_cache.stride(0), + out.stride(0), + HEAD_DIM=head_dim, + TOKENS_PER_PROG=_GATHER_TOKENS_PER_PROG, + num_warps=4, + ) + return out + + # Local import: mla_buffer reaches into sglang.srt.runtime_context, and this + # module lives under sglang.kernels; importing it at module scope would make + # every kernels-side consumer pull in srt at import time. + from sglang.kernels.ops.kvcache.mla_buffer import get_mla_kv_buffer_triton + + if num_tokens > 0: + rows = out[:num_tokens] + get_mla_kv_buffer_triton( + kv_cache, + page_table_1_flattened, + rows[:, :nope_dim], + rows[:, nope_dim:] if head_dim > nope_dim else None, + ) + out[num_tokens:].zero_() + return out diff --git a/python/sglang/kernels/ops/kvcache/cache_ops.py b/python/sglang/kernels/ops/kvcache/cache_ops.py index 9278641fd0..803b15ccc8 100644 --- a/python/sglang/kernels/ops/kvcache/cache_ops.py +++ b/python/sglang/kernels/ops/kvcache/cache_ops.py @@ -390,9 +390,13 @@ def concat_and_cast_q_fp8_pad_kernel( no = tl.arange(0, NOPE) src_n = tl.load(q_nope_ptr + pid * nope_s0 + hr[:, None] * nope_s1 + no[None, :]) tl.store(qpad_head + no[None, :], src_n) - ro = tl.arange(0, ROPE) - src_r = tl.load(q_rope_ptr + pid * rope_s0 + hr[:, None] * rope_s1 + ro[None, :]) - tl.store(qpad_head + NOPE + ro[None, :], src_r) + # NoPE models pass ROPE=0; tl.arange rejects empty ranges, so skip statically. + if ROPE > 0: + ro = tl.arange(0, ROPE) + src_r = tl.load( + q_rope_ptr + pid * rope_s0 + hr[:, None] * rope_s1 + ro[None, :] + ) + tl.store(qpad_head + NOPE + ro[None, :], src_r) def concat_and_cast_q_fp8_pad(q_fp8_pad, q_nope, q_rope, num_heads): @@ -400,7 +404,7 @@ def concat_and_cast_q_fp8_pad(q_fp8_pad, q_nope, q_rope, num_heads): [:, :num_heads, :] slice of the padded fp8 q buffer. Bit-exact replacement for the two strided converting copy_() in the Q8KV8 prefill q-prep, ~3.7x faster. Requires num_heads / nope_dim / rope_dim to be powers of two (always true for DeepSeek: 128 - heads / any TP, 512 nope, 64 rope).""" + heads / any TP, 512 nope, 64 rope); rope_dim may also be 0 for NoPE models.""" num_tokens = q_nope.shape[0] nope_dim = q_nope.shape[-1] rope_dim = q_rope.shape[-1] diff --git a/python/sglang/srt/layers/attention/dsa/dsa_backend_kpool.py b/python/sglang/srt/layers/attention/dsa/dsa_backend_kpool.py index 23896054ad..6d32c5dd8d 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_backend_kpool.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_backend_kpool.py @@ -39,6 +39,15 @@ class DeepseekSparseAttnBackendKPoolMixin: dsa_impl: _DSA_IMPL_T, phase: str, ) -> None: + # q8 prefill consumes the tail columns as ragged ids, like the history + # picks. Only over a bfloat16 pool: gathering packed fp8 rows through + # kpool tail columns is untested, so that combination keeps the guard. + if ( + dsa_impl == "flashmla_sparse_q8" + and phase == "prefill" + and not self.dsa_kv_cache_store_fp8 + ): + return if ( topk_indices is None or self.dsa_index_kpool <= 1 diff --git a/python/sglang/srt/layers/attention/dsa_backend.py b/python/sglang/srt/layers/attention/dsa_backend.py index b6cab552e6..de11bd31f0 100644 --- a/python/sglang/srt/layers/attention/dsa_backend.py +++ b/python/sglang/srt/layers/attention/dsa_backend.py @@ -21,6 +21,7 @@ from sglang.srt.runtime_context import ( get_exec, get_parallel, get_platform, + get_schedule, get_spec, ) @@ -28,6 +29,7 @@ logger = logging.getLogger(__name__) from sglang.kernels.ops.attention.dsa.dequant_k_cache import ( concat_cast_kv_fp8_pad, dequantize_k_cache_paged, + gather_cast_kv_fp8_pad_paged, gather_dequant_requant_fp8_paged, ) from sglang.kernels.ops.attention.dsa.quant_k_cache import quantize_k_cache @@ -404,6 +406,77 @@ def _resolve_dsa_triton_union(*, union: int, deterministic: bool) -> int: return union +def _validate_flashmla_sparse_q8_backend( + *, + prefill_impl: str, + decode_impl: str, + kv_cache_dtype: torch.dtype, + kv_cache_store_fp8: bool, + qk_rope_head_dim: int, + device_sm_major: int, + hisparse_enabled: bool = False, + mixed_chunk_enabled: bool = False, +) -> None: + # flashmla_sparse_q8 casts q and KV to fp8 per call, so the KV cache may be + # bfloat16 or fp8_e4m3. + if prefill_impl == "flashmla_sparse_q8": + if kv_cache_dtype not in (torch.bfloat16, torch.float8_e4m3fn): + raise ValueError( + "--dsa-prefill-backend flashmla_sparse_q8 computes attention in FP8 " + "and reads a bfloat16 or fp8_e4m3 KV cache; " + f"got kv_cache_dtype={kv_cache_dtype}." + ) + # Not supported yet: an fp8_e4m3 pool on a NoPE model has not been validated + # through the q8 prefix gather, and the KPool tail guard stays closed for + # fp8 pools. + if kv_cache_store_fp8 and qk_rope_head_dim == 0: + raise ValueError( + "--dsa-prefill-backend flashmla_sparse_q8 does not support an " + "fp8_e4m3 KV cache for NoPE MLA models (qk_rope_head_dim=0) yet; use " + "--kv-cache-dtype bfloat16." + ) + # An fp8 dtype that is NOT stored packed leaves the pool in the raw + # 576-dim layout, which the bf16 prefix gather cannot read; without this + # the failure is a message-less assert on the first prefix extend. + if kv_cache_dtype == torch.float8_e4m3fn and not kv_cache_store_fp8: + raise ValueError( + "--dsa-prefill-backend flashmla_sparse_q8 requires either a " + "bfloat16 KV cache or the packed fp8_e4m3 pool; got an unpacked " + "fp8_e4m3 pool (a trtllm backend keeps the raw layout). Use " + "--kv-cache-dtype bfloat16." + ) + if device_sm_major != 9: + raise ValueError( + "--dsa-prefill-backend flashmla_sparse_q8 is SM90-only; got compute " + f"capability sm_{device_sm_major}x." + ) + # A mixed batch takes the PAGED top-k transform, which belongs to the + # bf16 kernel. forward_extend refuses it, but the flag is readable here, + # so the refusal belongs at launch rather than on the first mixed batch + # formed under load. + if mixed_chunk_enabled: + raise ValueError( + "--dsa-prefill-backend flashmla_sparse_q8 does not support " + "--enable-mixed-chunk: q8 prefill runs on RAGGED extend batches " + "only." + ) + # The q8 route hands top-k to the kernel as ragged ids and never binds the + # page table HiSparse would translate, so the two cannot be combined. + if hisparse_enabled: + raise ValueError( + "--dsa-prefill-backend flashmla_sparse_q8 is not supported together " + "with HiSparse: the q8 path consumes top-k as ragged ids and does " + "not carry the HiSparse page translation." + ) + if decode_impl == "flashmla_sparse_q8": + raise ValueError( + "--dsa-decode-backend flashmla_sparse_q8 is not supported: " + "flashmla_sparse_q8 is a prefill-only backend. Pair it with a decode " + "backend that reads the KV cache dtype (flashmla_kv for fp8_e4m3; " + "tilelang or fa3 for bfloat16)." + ) + + class DeepseekSparseAttnBackend( DSAMetadataManagementMixin, DeepseekSparseAttnBackendKPoolMixin, @@ -576,23 +649,17 @@ class DeepseekSparseAttnBackend( self.device_sm_major = self.device_capability[0] self.kv_cache_dtype = model_runner.kv_cache_dtype - # `flashmla_sparse_q8` = the native FP8 SM90 sparse-prefill kernel. It always - # runs FP8 (requires fp8_e4m3 KV) and is SM90-only, so validate both at - # construction: an unsupported config must fail at launch rather than - # mid-forward. `flashmla_sparse` remains the bf16 path with no such - # requirement. - if self.dsa_prefill_impl == "flashmla_sparse_q8": - if self.kv_cache_dtype != torch.float8_e4m3fn: - raise ValueError( - "--dsa-prefill-backend flashmla_sparse_q8 is native FP8 and requires " - f"--kv-cache-dtype fp8_e4m3 (got kv_cache_dtype={self.kv_cache_dtype}); " - "use --dsa-prefill-backend flashmla_sparse for the bf16 path." - ) - if self.device_sm_major != 9: - raise ValueError( - "--dsa-prefill-backend flashmla_sparse_q8 is SM90-only; got compute " - f"capability sm_{self.device_sm_major}x." - ) + # Unsupported flashmla_sparse_q8 configs must fail at launch, not mid-forward. + _validate_flashmla_sparse_q8_backend( + prefill_impl=self.dsa_prefill_impl, + decode_impl=self.dsa_decode_impl, + kv_cache_dtype=self.kv_cache_dtype, + kv_cache_store_fp8=self.dsa_kv_cache_store_fp8, + qk_rope_head_dim=self.qk_rope_head_dim, + device_sm_major=self.device_sm_major, + hisparse_enabled=self.hisparse_coordinator is not None, + mixed_chunk_enabled=get_schedule().enable_mixed_chunk, + ) if self.dsa_prefill_impl == "triton_sparse_mla": _validate_triton_sparse_mla_backend( @@ -602,21 +669,13 @@ class DeepseekSparseAttnBackend( index_kpool=self.dsa_index_kpool, ) - # `flashmla_sparse_q8` is prefill-only (FP8 decode goes through - # `flashmla_kv`); reject it as a decode backend, since argparse accepts it - # via the shared CLI choices. - if self.dsa_decode_impl == "flashmla_sparse_q8": - raise ValueError( - "--dsa-decode-backend flashmla_sparse_q8 is not supported: " - "flashmla_sparse_q8 is a prefill-only backend. For FP8, use " - "--dsa-prefill-backend flashmla_sparse_q8 together with " - "--dsa-decode-backend flashmla_kv." - ) - # Q8KV8 per-call device-tensor caches, populated lazily on the first # Q8KV8 dispatch (no-ops for other backends). self._q8kv8_identity_scale: Optional[torch.Tensor] = None self._q8kv8_qpad_buf: Optional[torch.Tensor] = None + # Grow-only -1-filled top-k pad buffer; see _pad_topk_width_buffered. + self._q8kv8_topk_pad_buf: Optional[torch.Tensor] = None + self._q8kv8_topk_pad_width: Optional[int] = None # Persistent (grow-only) fp8 KV destination for the Q8KV8 prefill # gather: [capacity_rows, 576]. Avoids a fresh torch.zeros # (alloc + full-buffer FillFunctor) per layer per call; only the @@ -2141,6 +2200,18 @@ class DeepseekSparseAttnBackend( topk_transform_method = self.get_topk_transform_method( forward_batch.forward_mode ) + # q8 is only wired on the RAGGED route; PAGED would run the bf16 kernel. + if ( + dsa_impl == "flashmla_sparse_q8" + and topk_indices is not None + and topk_transform_method != TopkTransformMethod.RAGGED + ): + raise NotImplementedError( + "--dsa-prefill-backend flashmla_sparse_q8 supports EXTEND prefill " + f"batches only; got forward_mode={forward_batch.forward_mode.name} " + "(SPLIT_PREFILL under --enable-pdmux, or DLLM_EXTEND under " + "--dllm-algorithm; --enable-mixed-chunk is rejected at launch)." + ) if self.use_fused_topk: if topk_indices is not None: @@ -2226,11 +2297,9 @@ class DeepseekSparseAttnBackend( _has_prefix = any(forward_batch.extend_prefix_lens_cpu) page_table_1 = topk_indices - # `flashmla_sparse_q8` = native FP8 sparse prefill (constructor - # guarantees fp8_e4m3 KV + SM90). The helper consumes q_nope/q_rope - # directly (fusing the concat with the bf16->fp8 cast), so no bf16 - # q_all is materialized on this path. The prefix path hands over the - # paged fp8 KV as-is; the non-prefix path passes the gathered bf16 KV. + # flashmla_sparse_q8: FP8 compute on SM90 over a bf16 or fp8_e4m3 KV + # cache. The helper fuses the q concat with the bf16->fp8 cast; prefix + # batches hand over the paged cache as-is, others pass fresh bf16 KV. if dsa_impl == "flashmla_sparse_q8": if _has_prefix: page_table_1_flattened = ( @@ -2809,6 +2878,45 @@ class DeepseekSparseAttnBackend( self._q8kv8_born_q_sentinel = buf return buf[:numel].view(num_tokens, num_heads, v_head_dim) + def _pad_topk_width_buffered( + self, topk_indices: torch.Tensor, *, multiple: int + ) -> torch.Tensor: + """Widen top-k to a multiple of ``multiple``, padding with -1. + + The destination is a persistent grow-only buffer rather than a fresh + ``torch.cat``: only the real columns change between calls, while a cat + reallocates and rewrites the whole ``[num_tokens, padded_width]`` int32 + tensor on every DSA layer call (~71 MB at an 8192-token chunk). The pad + columns are filled with -1 once at allocation and never written again, + which is exactly what the SM90 kernel's -1 clamp reads. Eager-mode DSA + runs layers sequentially on one stream, so single-buffer reuse is safe + here for the same reason it is for _q8kv8_qpad_buf. + """ + assert topk_indices.dim() == 2 + width = topk_indices.shape[-1] + pad = (-width) % multiple + if pad == 0: + return topk_indices + rows = topk_indices.shape[0] + buf = self._q8kv8_topk_pad_buf + # A different REAL top-k width needs a fresh buffer even when it pads to + # the same total: the tail is -1-filled only at allocation, so a buffer + # built for a wider call still carries real ids in the columns this call + # must read as padding. Comparing width + pad misses that, because 2051 + # and 2100 both pad to 2176. + if ( + buf is None + or buf.shape[0] < rows + or buf.shape[-1] != width + pad + or self._q8kv8_topk_pad_width != width + ): + buf = topk_indices.new_full((rows, width + pad), -1) + self._q8kv8_topk_pad_buf = buf + self._q8kv8_topk_pad_width = width + out = buf[:rows] + out[:, :width].copy_(topk_indices) + return out + def _forward_flashmla_sparse_q8kv8( self, q_nope: torch.Tensor, @@ -2827,21 +2935,26 @@ class DeepseekSparseAttnBackend( Same contract as ``_forward_flashmla_sparse`` but executed through the FP8 ``sparse_mla_q8kv8_prefill_fwd`` kernel. Identity per-tensor - scales (scalar 1.0) are used: a raw bf16->fp8 cast of q/kv is accurate - on real DeepSeek-V3 magnitudes, so no dynamic rescaling is applied. + scales (scalar 1.0) are used, matching the pre-existing fp8-pool path, + which dequantizes by the group scale and re-casts at identity scale + over the same range; no dynamic rescaling is applied. The Triton cast + saturates at the fp8_e4m3 maximum (+-448) rather than producing NaN. The kernel runs via its fixed full-topk entry (``attn_sink`` / ``topk_length`` left None), keeping control flow identical across DP ranks; -1 topk sentinels are clamped to distinct zero pad rows inside the kernel. - Two KV paths: + KV paths: * non-prefix extend: ``kv_bf16`` (the gathered bf16 KV) is cast into a zero-padded fp8 buffer. - * prefix extend: ``paged_kv_cache`` (fp8, 656 B/token: nope_fp8 + - per-group scales + rope_bf16) is gathered, dequantized per group, - and requantized to per-tensor fp8 in one fused Triton kernel - (``gather_dequant_requant_fp8_paged``) — no intermediate bf16 - materialization. + * prefix extend on an fp8 pool (656 B/token: nope_fp8 + per-group + scales + rope_bf16): ``gather_dequant_requant_fp8_paged`` gathers, + dequantizes per group and requantizes to per-tensor fp8 in one + fused Triton kernel. + * prefix extend on a bf16 pool: ``gather_cast_kv_fp8_pad_paged`` + gathers the rows and store-casts them to fp8. + The top-k width is -1 padded to a multiple of 128 before the pad band + is sized (kpool rows are index_topk + index_kpool - 1 wide). """ from sglang.kernels.ops.attention.sparse_mla_q8kv8_prefill_sm90 import ( sparse_mla_q8kv8_prefill_fwd, @@ -2939,6 +3052,8 @@ class DeepseekSparseAttnBackend( # they need it EVERY call because a previous, larger call may have # left real KV data there. The gather kernel fuses the pad-row # zeroing; the bf16 path zeroes the tail explicitly. + # The kernel walks top-k in pairs of 64-wide blocks (topk % 128 == 0). + page_table_1 = self._pad_topk_width_buffered(page_table_1, multiple=128) topk = page_table_1.shape[-1] if paged_kv_cache is not None: num_kv_tokens = page_table_1_flattened.shape[0] @@ -2955,13 +3070,20 @@ class DeepseekSparseAttnBackend( device=dev, ) self._q8kv8_kv_buf = kv_buf - if paged_kv_cache is not None: + if paged_kv_cache is not None and self.dsa_kv_cache_store_fp8: kv_padded = gather_dequant_requant_fp8_paged( paged_kv_cache, page_table_1_flattened, extra_rows=topk, out=kv_buf[:total_kv_rows], ).view(-1, 1, head_dim) + elif paged_kv_cache is not None: + kv_padded = gather_cast_kv_fp8_pad_paged( + out=kv_buf[:total_kv_rows], + kv_cache=paged_kv_cache, + page_table_1_flattened=page_table_1_flattened, + nope_dim=self.kv_lora_rank, + ).view(-1, 1, head_dim) elif kv_k is not None: # Fused non-prefix KV prep (SGLANG_ENABLE_DSA_Q8KV8_KV_CAT_FUSION): # cast-concat k/k_rope straight into the fp8 buffer + zero the pad @@ -3831,13 +3953,14 @@ class DeepseekSparseAttnBackend( """ if ( # disable for MTP - self.dsa_kv_cache_store_fp8 - # flashmla_sparse_q8 shares flashmla_sparse's RAGGED prefill routing — the q8 - # dispatch lives inside the RAGGED branch of forward_extend; without this the - # transform is PAGED, the q8 path is skipped, and the bf16 kernel crashes on - # fp8 KV ("kv must have dtype kBFloat16"). triton_sparse_mla is listed for - # the same reason: its dispatch also dequantizes inside the RAGGED branch - # and its kernel is bf16-only. + ( + self.dsa_kv_cache_store_fp8 + or self.dsa_prefill_impl == "flashmla_sparse_q8" + ) + # flashmla_sparse_q8 always gathers KV into a contiguous fp8 buffer; + # it is RAGGED whatever the pool dtype, since its dispatch lives there. + # triton_sparse_mla is listed for an fp8 pool: its dispatch also + # dequantizes inside the RAGGED branch and its kernel is bf16-only. and self.dsa_prefill_impl in ("flashmla_sparse", "flashmla_sparse_q8", "triton_sparse_mla") and forward_mode == ForwardMode.EXTEND diff --git a/test/registered/kernels/ops/attention/test_q8kv8_dsa_nope_bf16_prefill.py b/test/registered/kernels/ops/attention/test_q8kv8_dsa_nope_bf16_prefill.py new file mode 100644 index 0000000000..c0b6e1c90b --- /dev/null +++ b/test/registered/kernels/ops/attention/test_q8kv8_dsa_nope_bf16_prefill.py @@ -0,0 +1,202 @@ +"""DSA Q8KV8 sparse prefill on NoPE latents, bf16 KV pools and kpool top-k widths. + +Covers the GLM-5.3-Flash geometry: qk_rope_head_dim == 0 (zero-width rope views), +512-dim bf16 KV rows, and kpool top-k rows of width index_topk + kpool - 1 = 2051. +""" + +from __future__ import annotations + +import math +import sys + +import pytest +import torch + +from sglang.kernels.ops.attention.dsa.dequant_k_cache import ( + concat_cast_kv_fp8_pad, + gather_cast_kv_fp8_pad_paged, +) +from sglang.kernels.ops.kvcache.cache_ops import concat_and_cast_q_fp8_pad +from sglang.srt.utils import is_sm90_supported +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=90, stage="base-b-kernel-unit", runner_config="1-gpu-large") + +FP8 = torch.float8_e4m3fn +NOPE = 512 +KPOOL_TOPK_WIDTH = 2048 + 4 - 1 +SM_SCALE = 1.0 / math.sqrt(256) + + +def _fp8_triton_available() -> bool: + return torch.cuda.is_available() and torch.cuda.get_device_capability() >= (8, 9) + + +def _sm90_available() -> bool: + return torch.cuda.is_available() and is_sm90_supported() + + +def _bytes(t: torch.Tensor) -> torch.Tensor: + return t.contiguous().view(torch.uint8) + + +@pytest.mark.skipif(not _fp8_triton_available(), reason="Triton fp8 needs SM>=89") +@pytest.mark.parametrize("rope", [0, 64]) +@pytest.mark.parametrize("heads", [8, 64]) +@pytest.mark.parametrize("tokens", [1, 437]) +def test_concat_and_cast_q_fp8_pad_matches_torch_cast(rope, heads, tokens): + """NoPE models pass a zero-width q_rope; the rope>0 bytes must stay unchanged.""" + q_nope = ( + (torch.randn(heads, tokens, NOPE, device="cuda") * 0.5) + .to(torch.bfloat16) + .transpose(0, 1) + ) + q_rope = (torch.randn(tokens, heads, 256 + rope, device="cuda") * 0.5).to( + torch.bfloat16 + ) + q_rope = q_rope.split([256, rope], dim=-1)[1] + dst = torch.zeros(tokens + 3, 64, NOPE + rope, dtype=FP8, device="cuda") + # Poison in place: _bytes() would copy the strided heads < 64 slice. + dst[:, :heads].view(torch.uint8).fill_(0x7F) + concat_and_cast_q_fp8_pad(dst, q_nope, q_rope, heads) + expected = torch.cat([q_nope, q_rope], dim=-1).to(FP8) + assert torch.equal(_bytes(dst[:tokens, :heads]), _bytes(expected)) + assert (_bytes(dst[:, heads:]) == 0).all() + + +@pytest.mark.skipif(not _fp8_triton_available(), reason="Triton fp8 needs SM>=89") +@pytest.mark.parametrize("rope,pad_rows", [(0, 2176), (64, 2048)]) +def test_concat_cast_kv_fp8_pad_writes_rows_and_zero_band(rope, pad_rows): + tokens = 437 + head = NOPE + rope + big = torch.empty(tokens + pad_rows + 50, head, dtype=FP8, device="cuda") + _bytes(big).fill_(0x7F) + out = big[: tokens + pad_rows] + k = (torch.randn(tokens, NOPE, device="cuda") * 0.5).to(torch.bfloat16).unsqueeze(1) + k_rope = ( + (torch.randn(tokens, head, device="cuda") * 0.5) + .to(torch.bfloat16)[..., NOPE:] + .unsqueeze(1) + ) + concat_cast_kv_fp8_pad(out, k, k_rope, tokens) + expected = torch.cat([k, k_rope], dim=-1).view(tokens, head).to(FP8) + assert torch.equal(_bytes(out[:tokens]), _bytes(expected)) + assert (_bytes(out[tokens:]) == 0).all() + + +@pytest.mark.skipif(not _fp8_triton_available(), reason="Triton fp8 needs SM>=89") +@pytest.mark.parametrize("head_dim", [512, 576]) +def test_gather_cast_kv_fp8_pad_paged_matches_torch_cast(head_dim): + """A radix-hit prefix (paged gather) must produce the same fp8 bytes as the + non-prefix cast of the same rows.""" + slots, rows, pad_rows = 4096, 333, 2176 + pool = (torch.randn(slots, 1, head_dim, device="cuda") * 50).to(torch.bfloat16) + idx = torch.randint(0, slots, (rows,), dtype=torch.int32, device="cuda") + idx[:8] = idx[8] + out = torch.empty(rows + pad_rows, head_dim, dtype=FP8, device="cuda") + _bytes(out).fill_(0x7F) + gather_cast_kv_fp8_pad_paged( + out=out, kv_cache=pool, page_table_1_flattened=idx, nope_dim=NOPE + ) + expected = pool.view(-1, head_dim)[idx.long()].to(FP8) + assert torch.equal(_bytes(out[:rows]), _bytes(expected)) + assert (_bytes(out[rows:]) == 0).all() + + +def _make_backend(): + from sglang.srt.layers.attention.dsa_backend import DeepseekSparseAttnBackend + + backend = DeepseekSparseAttnBackend.__new__(DeepseekSparseAttnBackend) + backend._q8kv8_born_q_stash = None + backend._q8kv8_born_q_buf = None + backend._q8kv8_qpad_buf = None + backend._q8kv8_topk_pad_buf = None + backend._q8kv8_topk_pad_width = None + backend._q8kv8_identity_scale = None + backend._q8kv8_kv_buf = None + backend._q8kv8_topk_length_enabled = False + backend._q8kv8_out_bufs = None + backend.dsa_kv_cache_store_fp8 = False + backend.kv_lora_rank = NOPE + return backend + + +def _make_ragged_case(heads: int, seed: int): + """Two prefix-extend requests laid out RAGGED: indices address one buffer with + every request's full context end to end; rows are [picks][tail][-1 ...].""" + gen = torch.Generator(device="cuda").manual_seed(seed) + seq_lens, extend_lens = [300, 520], [4, 3] + tokens, kv_rows = sum(extend_lens), sum(seq_lens) + slots = 2 * kv_rows + pool = (torch.randn(slots, 1, NOPE, generator=gen, device="cuda") * 0.25).to( + torch.bfloat16 + ) + flat = torch.randperm(slots, generator=gen, device="cuda")[:kv_rows].to(torch.int32) + ragged_kv = pool[flat.long(), 0, :] + indices = torch.full( + (tokens, KPOOL_TOPK_WIDTH), -1, dtype=torch.int32, device="cuda" + ) + row, base = 0, 0 + for seq_len, ext in zip(seq_lens, extend_lens): + for i in range(ext): + picks = torch.randperm(seq_len, generator=gen, device="cuda")[ + : 1 + (i * 97) % 257 + ] + indices[row, : picks.numel()] = (picks + base).to(torch.int32) + row += 1 + base += seq_len + q_nope = (torch.randn(tokens, heads, NOPE, generator=gen, device="cuda") * 0.25).to( + torch.bfloat16 + ) + q_rope = q_nope.new_empty((tokens, heads, 0)) + return q_nope, q_rope, pool, flat, ragged_kv, indices + + +def _torch_reference(q_nope, kv, indices): + q, kv = q_nope.float(), kv.float() + out = torch.empty_like(q) + for i in range(q.shape[0]): + keys = kv[indices[i][indices[i] >= 0].long()] + out[i] = torch.softmax((q[i] @ keys.T) * SM_SCALE, dim=-1) @ keys + return out + + +@pytest.mark.skipif(not _sm90_available(), reason="Q8KV8 sparse prefill requires SM90") +@pytest.mark.parametrize("heads", [16, 64]) +@pytest.mark.parametrize("topk_length", [False, True]) +def test_q8_helper_nope_bf16_pool_kpool_width(heads, topk_length): + q_nope, q_rope, pool, flat, ragged_kv, indices = _make_ragged_case(heads, seed=11) + backend = _make_backend() + backend._q8kv8_topk_length_enabled = topk_length + common = dict( + q_nope=q_nope, q_rope=q_rope, v_head_dim=NOPE, sm_scale=SM_SCALE, layer_id=0 + ) + prefix = backend._forward_flashmla_sparse_q8kv8( + kv_bf16=None, + page_table_1=indices, + paged_kv_cache=pool, + page_table_1_flattened=flat, + **common, + ).clone() + non_prefix = backend._forward_flashmla_sparse_q8kv8( + kv_bf16=ragged_kv, page_table_1=indices, **common + ).clone() + padded = torch.cat( + (indices, indices.new_full((indices.shape[0], 2176 - KPOOL_TOPK_WIDTH), -1)), + dim=-1, + ) + pre_padded = backend._forward_flashmla_sparse_q8kv8( + kv_bf16=ragged_kv, page_table_1=padded, **common + ).clone() + torch.cuda.synchronize() + assert prefix.shape == (q_nope.shape[0], heads, NOPE) + assert torch.equal(prefix, non_prefix) + assert torch.equal(non_prefix, pre_padded) + ref = _torch_reference(q_nope, ragged_kv, indices) + assert torch.isfinite(prefix.float()).all() + assert (prefix.float() - ref).abs().mean().item() < 0.03 + torch.testing.assert_close(prefix.float(), ref, atol=2.5e-1, rtol=3.0e-1) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/unit/layers/attention/test_dsa_q8kv8_bf16_prefill_routing.py b/test/registered/unit/layers/attention/test_dsa_q8kv8_bf16_prefill_routing.py new file mode 100644 index 0000000000..71049c763b --- /dev/null +++ b/test/registered/unit/layers/attention/test_dsa_q8kv8_bf16_prefill_routing.py @@ -0,0 +1,227 @@ +from types import SimpleNamespace + +import pytest +import torch + +from sglang.srt.layers.attention.dsa.dsa_backend_kpool import ( + DeepseekSparseAttnBackendKPoolMixin, +) +from sglang.srt.layers.attention.dsa.dsa_topk_backend import TopkTransformMethod +from sglang.srt.layers.attention.dsa_backend import ( + DeepseekSparseAttnBackend, + _validate_flashmla_sparse_q8_backend, +) +from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +RAGGED = TopkTransformMethod.RAGGED +PAGED = TopkTransformMethod.PAGED + + +def _make_backend(prefill_impl: str, *, fp8_kv_cache: bool): + backend = DeepseekSparseAttnBackend.__new__(DeepseekSparseAttnBackend) + backend.dsa_kv_cache_store_fp8 = fp8_kv_cache + backend.dsa_prefill_impl = prefill_impl + return backend + + +@pytest.mark.parametrize( + "prefill_impl,fp8_kv_cache,forward_mode,expected", + [ + # The q8 helper only exists on the RAGGED route, whatever the pool dtype. + ("flashmla_sparse_q8", False, ForwardMode.EXTEND, RAGGED), + ("flashmla_sparse_q8", True, ForwardMode.EXTEND, RAGGED), + # bf16 flashmla_sparse reads the pool directly through PAGED slot ids. + ("flashmla_sparse", False, ForwardMode.EXTEND, PAGED), + ("flashmla_sparse", True, ForwardMode.EXTEND, RAGGED), + ("flashmla_sparse_q8", False, ForwardMode.DECODE, PAGED), + ("flashmla_sparse_q8", False, ForwardMode.TARGET_VERIFY, PAGED), + ("flashmla_sparse_q8", False, ForwardMode.DRAFT_EXTEND_V2, PAGED), + ], +) +def test_topk_transform_method(prefill_impl, fp8_kv_cache, forward_mode, expected): + backend = _make_backend(prefill_impl, fp8_kv_cache=fp8_kv_cache) + assert backend.get_topk_transform_method(forward_mode) == expected + + +@pytest.mark.parametrize("kv_cache_dtype", [torch.bfloat16, torch.float8_e4m3fn]) +def test_q8_validator_accepts_bf16_and_fp8_kv_on_sm90(kv_cache_dtype): + _validate_flashmla_sparse_q8_backend( + prefill_impl="flashmla_sparse_q8", + decode_impl="tilelang", + kv_cache_dtype=kv_cache_dtype, + kv_cache_store_fp8=kv_cache_dtype == torch.float8_e4m3fn, + qk_rope_head_dim=64, + device_sm_major=9, + ) + + +@pytest.mark.parametrize( + "prefill_impl,decode_impl,kv_cache_dtype,store_fp8,rope,device_sm_major", + [ + ("flashmla_sparse_q8", "tilelang", torch.float16, False, 64, 9), + ("flashmla_sparse_q8", "trtllm", torch.bfloat16, False, 64, 10), + ("tilelang", "flashmla_sparse_q8", torch.bfloat16, False, 64, 9), + # An fp8 pool on a NoPE model is not supported yet. + ("flashmla_sparse_q8", "flashmla_kv", torch.float8_e4m3fn, True, 0, 9), + ], +) +def test_q8_validator_rejects_unsupported_configs( + prefill_impl, decode_impl, kv_cache_dtype, store_fp8, rope, device_sm_major +): + with pytest.raises(ValueError): + _validate_flashmla_sparse_q8_backend( + prefill_impl=prefill_impl, + decode_impl=decode_impl, + kv_cache_dtype=kv_cache_dtype, + kv_cache_store_fp8=store_fp8, + qk_rope_head_dim=rope, + device_sm_major=device_sm_major, + ) + + +@pytest.mark.parametrize( + "extra", + [ + pytest.param({"hisparse_enabled": True}, id="hisparse"), + pytest.param({"mixed_chunk_enabled": True}, id="mixed_chunk"), + ], +) +def test_q8_validator_rejects_combinations_that_used_to_fail_mid_forward(extra): + """Both used to surface only once a batch of the wrong shape was formed -- + HiSparse as an UnboundLocalError, mixed chunks as a NotImplementedError + hours into serving. The launch flags are visible at construction.""" + kwargs = dict( + prefill_impl="flashmla_sparse_q8", + decode_impl="tilelang", + kv_cache_dtype=torch.bfloat16, + kv_cache_store_fp8=False, + qk_rope_head_dim=0, + device_sm_major=9, + ) + kwargs.update(extra) + with pytest.raises(ValueError): + _validate_flashmla_sparse_q8_backend(**kwargs) + + +def test_q8_validator_rejects_an_unpacked_fp8_pool(): + """An fp8 dtype whose pool is not stored packed stays in the raw 576-dim + layout that the bf16 prefix gather cannot read; it previously surfaced as a + message-less assert on the first prefix extend.""" + with pytest.raises(ValueError): + _validate_flashmla_sparse_q8_backend( + prefill_impl="flashmla_sparse_q8", + decode_impl="tilelang", + kv_cache_dtype=torch.float8_e4m3fn, + kv_cache_store_fp8=False, + qk_rope_head_dim=64, + device_sm_major=9, + ) + + +def _make_padder(): + backend = DeepseekSparseAttnBackend.__new__(DeepseekSparseAttnBackend) + backend._q8kv8_topk_pad_buf = None + backend._q8kv8_topk_pad_width = None + return backend + + +def test_pad_topk_width_pads_kpool_width_with_minus_one(): + indices = torch.arange(2 * 2051, dtype=torch.int32).view(2, 2051) + padded = _make_padder()._pad_topk_width_buffered(indices, multiple=128) + assert padded.shape == (2, 2176) and padded.is_contiguous() + assert torch.equal(padded[:, :2051], indices) + assert (padded[:, 2051:] == -1).all() + + +@pytest.mark.parametrize("width", [2048, 2176]) +def test_pad_topk_width_is_zero_copy_on_aligned_widths(width): + indices = torch.zeros((3, width), dtype=torch.int32) + assert _make_padder()._pad_topk_width_buffered(indices, multiple=128) is indices + + +def test_pad_topk_width_rounds_64_aligned_width_to_128(): + """The SM90 kernel walks top-k in pairs of 64-wide blocks; a width that is + only a multiple of 64 (tilelang's padding) is still rejected.""" + indices = torch.zeros((1, 2112), dtype=torch.int32) + padded = _make_padder()._pad_topk_width_buffered(indices, multiple=128) + assert padded.shape == (1, 2176) + + +def test_pad_topk_width_reuses_the_buffer_without_leaking_stale_ids(): + """The pad columns are -1-filled once at allocation and never rewritten, so a + later call with fewer rows must still read -1 there rather than ids left by + an earlier, larger call.""" + padder = _make_padder() + first = padder._pad_topk_width_buffered( + torch.full((4, 2051), 7, dtype=torch.int32), multiple=128 + ) + assert (first[:, 2051:] == -1).all() + buf = padder._q8kv8_topk_pad_buf + second = padder._pad_topk_width_buffered( + torch.full((2, 2051), 9, dtype=torch.int32), multiple=128 + ) + assert padder._q8kv8_topk_pad_buf is buf, "fewer rows must not reallocate" + assert (second[:, :2051] == 9).all() + assert (second[:, 2051:] == -1).all() + + +def test_pad_topk_width_does_not_share_a_buffer_across_different_real_widths(): + """Two different real widths can pad to the same total: 2051 and 2100 both + round to 2176 at multiple=128. Reusing the buffer across them would leave the + wider call's real ids in columns the narrower call must read as padding, and + the SM90 kernel's -1 clamp would consume them as genuine top-k indices.""" + padder = _make_padder() + wide = padder._pad_topk_width_buffered( + torch.arange(1, 2100 + 1, dtype=torch.int32).unsqueeze(0), multiple=128 + ) + assert wide.shape == (1, 2176) + narrow = padder._pad_topk_width_buffered( + torch.full((1, 2051), 7, dtype=torch.int32), multiple=128 + ) + assert narrow.shape == (1, 2176) + assert (narrow[:, :2051] == 7).all() + assert (narrow[:, 2051:] == -1).all(), ( + "stale ids from the wider call leaked into the pad band" + ) + + +def test_kpool_tail_guard_admits_q8_for_prefill_only(): + backend = SimpleNamespace( + dsa_index_kpool=4, device_sm_major=9, dsa_kv_cache_store_fp8=False + ) + indices = torch.full((2, 2051), -1, dtype=torch.int32) + mixin = DeepseekSparseAttnBackendKPoolMixin + assert ( + mixin._resolve_kpool_tail_backend(backend, indices, "flashmla_sparse_q8") + == "flashmla_sparse_q8" + ) + assert ( + mixin._resolve_kpool_tail_backend(backend, indices, "flashmla_sparse") == "fa3" + ) + mixin._check_kpool_tail_backend(backend, indices, "flashmla_sparse_q8", "prefill") + with pytest.raises(NotImplementedError): + mixin._check_kpool_tail_backend( + backend, indices, "flashmla_sparse_q8", "decode" + ) + with pytest.raises(NotImplementedError): + mixin._check_kpool_tail_backend(backend, indices, "flashmla_kv", "prefill") + + +def test_kpool_tail_guard_still_rejects_q8_over_a_packed_fp8_pool(): + """Gathering packed fp8 rows through kpool tail columns is untested, so the + guard must stay in force there even though q8 prefill is admitted on bf16.""" + backend = SimpleNamespace( + dsa_index_kpool=4, device_sm_major=9, dsa_kv_cache_store_fp8=True + ) + indices = torch.full((2, 2051), -1, dtype=torch.int32) + with pytest.raises(NotImplementedError): + DeepseekSparseAttnBackendKPoolMixin._check_kpool_tail_backend( + backend, indices, "flashmla_sparse_q8", "prefill" + ) + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-v"]))