From 57d4adeb8331cf2e4b0a61dddd3103db037b67c1 Mon Sep 17 00:00:00 2001 From: DarkSharpness <76582120+DarkSharpness@users.noreply.github.com> Date: Wed, 30 Sep 2026 08:16:52 +0800 Subject: [PATCH] [Kernel] Expose FlashMLA kv_format in sgl-kernel sparse decode (#41336) --- .../sglang/kernels/aot/cmake/flashmla.cmake | 6 +- .../kernels/aot/csrc/flashmla_extension.cc | 7 +- .../aot/python/sgl_kernel/flash_mla.py | 40 +++++- .../sglang/kernels/aot/tests/test_flashmla.py | 134 ++++++++++++++++++ 4 files changed, 179 insertions(+), 8 deletions(-) diff --git a/python/sglang/kernels/aot/cmake/flashmla.cmake b/python/sglang/kernels/aot/cmake/flashmla.cmake index ae506d7a2b..24050ed35e 100644 --- a/python/sglang/kernels/aot/cmake/flashmla.cmake +++ b/python/sglang/kernels/aot/cmake/flashmla.cmake @@ -1,9 +1,9 @@ # flash_mla -# DeepSeek v4.1 kernels merged into the SGLang fork (sgl-project/FlashMLA@3e18517). +# DeepSeek v4.1 kernels, 528 B/token caches read as V4.1 by default (sgl-project/FlashMLA@3620b63). FetchContent_Declare( repo-flashmla - URL https://${GITHUB_ARTIFACTORY}/sgl-project/FlashMLA/archive/3e18517fb055a6c9608eef5a1f1347fb1a047bbd.tar.gz - URL_HASH SHA256=ab2af4657683a1bbaa707a2781a5e2c2792eba574ff36905f1832598e67fea79 + URL https://${GITHUB_ARTIFACTORY}/sgl-project/FlashMLA/archive/3620b63fc4ebe33950c50e1b7701b6607f44150e.tar.gz + URL_HASH SHA256=7d7f9819bc1b121ef8226281de7265cd97be0cbe15d6450575fc8e3f241041a2 ) FetchContent_Populate(repo-flashmla) diff --git a/python/sglang/kernels/aot/csrc/flashmla_extension.cc b/python/sglang/kernels/aot/csrc/flashmla_extension.cc index c4f1550365..595ecc0fa5 100644 --- a/python/sglang/kernels/aot/csrc/flashmla_extension.cc +++ b/python/sglang/kernels/aot/csrc/flashmla_extension.cc @@ -58,7 +58,8 @@ static std::tuple, std::option const std::optional& extra_indices, const std::optional& extra_topk_length, int64_t d_v, - double sm_scale) { + double sm_scale, + const std::optional& kv_format) { return sparse_attn_decode_interface( q, kv, @@ -72,7 +73,7 @@ static std::tuple, std::option extra_topk_length, static_cast(d_v), static_cast(sm_scale), - std::nullopt); + kv_format); } static std::tuple, std::optional> sgl_dense_decode_fwd( @@ -128,7 +129,7 @@ TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) { m.def( "sparse_decode_fwd(Tensor q, Tensor kv, Tensor indices, Tensor? topk_length, Tensor? attn_sink, " "Tensor? tile_scheduler_metadata, Tensor? num_splits, Tensor? extra_kv, Tensor? extra_indices, " - "Tensor? extra_topk_length, int d_v, float sm_scale) -> (Tensor, Tensor, Tensor?, Tensor?)"); + "Tensor? extra_topk_length, int d_v, float sm_scale, str? kv_format=None) -> (Tensor, Tensor, Tensor?, Tensor?)"); m.impl("sparse_decode_fwd", torch::kCUDA, &sgl_sparse_decode_fwd); m.def( diff --git a/python/sglang/kernels/aot/python/sgl_kernel/flash_mla.py b/python/sglang/kernels/aot/python/sgl_kernel/flash_mla.py index 2a15aad9ca..10574e54b5 100644 --- a/python/sglang/kernels/aot/python/sgl_kernel/flash_mla.py +++ b/python/sglang/kernels/aot/python/sgl_kernel/flash_mla.py @@ -14,6 +14,9 @@ _IMPORT_ERROR = ImportError( "Failed to load sgl_kernel.flashmla_ops extension. Ensure CUDA Driver >= 12.4" ) +# 512 fp8 values + 4 fp32 scales; FlashMLA's V4.1 format has the same size. +_V32_NO_ROPE_BYTES_PER_TOKEN = 528 + @dataclasses.dataclass class FlashMLASchedMeta: @@ -103,6 +106,7 @@ def flash_mla_with_kvcache( extra_indices_in_kvcache: Optional[torch.Tensor] = None, topk_length: Optional[torch.Tensor] = None, extra_topk_length: Optional[torch.Tensor] = None, + kv_format: Optional[str] = None, ) -> Tuple[torch.Tensor, torch.Tensor]: """ Arguments: @@ -119,6 +123,7 @@ def flash_mla_with_kvcache( descale_k: (batch_size), torch.float32. Descaling factors for K, used for fp8 quantization. is_fp8_kvcache: bool. Whether the k_cache and v_cache are in fp8 format. For the format of FP8 KV cache, please refer to README.md indices: (batch_size, seq_len_q, topk), torch.int32. If not None, sparse attention will be enabled, and only tokens in the `indices` array will be attended to. Invalid indices should be set to -1 or numbers >= total_seq_len_kv. For details about how to set up `indices`, please refer to README.md. + kv_format: Optional[str]. Sparse attention only: the layout of `k_cache` ("V32", "V32_NO_ROPE", "V4", "V41"). When omitted it is detected from the shape. V3.2-no-RoPE and V4.1 are both 528 bytes per token with head_dim 512: with FlashMLASchedMeta such a cache is read as V41, with tensor tile_scheduler_metadata as V32_NO_ROPE. Returns: out: (batch_size, seq_len_q, num_heads_q, head_dim_v). @@ -147,6 +152,7 @@ def flash_mla_with_kvcache( extra_indices_in_kvcache=extra_indices_in_kvcache, topk_length=topk_length, extra_topk_length=extra_topk_length, + kv_format=kv_format, ) assert num_splits is not None @@ -163,7 +169,34 @@ def flash_mla_with_kvcache( "descale_q and descale_k should be both None or both not None" ) - if indices is None and q.element_size() == 1: + if indices is not None: + assert is_fp8_kvcache, "is_fp8_kvcache must be True for sparse attention" + # The tensor-metadata API predates V4.1, so its 528 B/token cache is V3.2 without RoPE. + if ( + kv_format is None + and q.shape[-1] == 512 + and k_cache.shape[-1] == _V32_NO_ROPE_BYTES_PER_TOKEN + ): + kv_format = "V32_NO_ROPE" + out, softmax_lse, _, _ = torch.ops.sgl_kernel.sparse_decode_fwd.default( + q, + k_cache, + indices, + topk_length, + attn_sink, + tile_scheduler_metadata, + num_splits, + extra_k_cache, + extra_indices_in_kvcache, + extra_topk_length, + head_dim_v, + softmax_scale, + kv_format, + ) + return out, softmax_lse + + assert kv_format is None, "kv_format only applies to sparse attention" + if q.element_size() == 1: out, softmax_lse = torch.ops.sgl_kernel.fwd_kvcache_mla_fp8.default( q, k_cache, @@ -189,7 +222,7 @@ def flash_mla_with_kvcache( tile_scheduler_metadata, num_splits, is_fp8_kvcache, - indices, + None, attn_sink, extra_k_cache, extra_indices_in_kvcache, @@ -216,6 +249,7 @@ def _flash_mla_with_kvcache_sched_meta( extra_indices_in_kvcache: Optional[torch.Tensor], topk_length: Optional[torch.Tensor], extra_topk_length: Optional[torch.Tensor], + kv_format: Optional[str], ) -> Tuple[torch.Tensor, torch.Tensor]: assert num_splits is None, "num_splits must be None with FlashMLASchedMeta" @@ -279,9 +313,11 @@ def _flash_mla_with_kvcache_sched_meta( extra_topk_length, head_dim_v, softmax_scale, + kv_format, ) ) else: + assert kv_format is None, "kv_format only applies to sparse attention" assert block_table is not None and cache_seqlens is not None assert attn_sink is None assert extra_k_cache is None diff --git a/python/sglang/kernels/aot/tests/test_flashmla.py b/python/sglang/kernels/aot/tests/test_flashmla.py index f9b7574b61..e9c725f1a5 100644 --- a/python/sglang/kernels/aot/tests/test_flashmla.py +++ b/python/sglang/kernels/aot/tests/test_flashmla.py @@ -657,5 +657,139 @@ def test_flash_mla_fp8( cal_diff(lse_flash, lse_torch, "lse") +def _sparse_decode_inputs(b: int, s_q: int, h_q: int, s_k: int, topk: int, d: int): + block_size = 64 + num_blocks_per_seq = cdiv(s_k, block_size) + block_table = torch.randperm( + b * num_blocks_per_seq, dtype=torch.int32, device="cuda" + ).view(b, num_blocks_per_seq) + cache_seqlens = torch.full((b,), s_k, dtype=torch.int32, device="cuda") + q = torch.randn(b, s_q, h_q, d, dtype=torch.bfloat16, device="cuda").clamp_(-1, 1) + blocked_k = ( + (torch.randn(b * num_blocks_per_seq, block_size, 1, d, device="cuda") / 10) + .clamp_(-1, 1) + .to(torch.bfloat16) + ) + abs_indices = torch.stack( + [torch.randperm(s_k, device="cuda")[:topk] for _ in range(b * s_q)] + ).view(b, s_q, topk) + abs_indices[torch.rand(abs_indices.shape, device="cuda") < 0.05] = -1 + block_ids = torch.gather( + block_table, 1, (abs_indices.clamp(min=0) // block_size).view(b, -1) + ).view(b, s_q, topk) + indices_in_kvcache = block_ids * block_size + abs_indices.clamp(min=0) % block_size + indices_in_kvcache[abs_indices < 0] = -1 + return ( + q, + blocked_k, + block_table, + cache_seqlens, + abs_indices.int(), + indices_in_kvcache.int(), + ) + + +def _quantize_v41_k_cache(k: torch.Tensor) -> torch.Tensor: + """(num_blocks, block_size, 1, 512) bf16 -> V4.1 pages: fp8 data rows, then ue8m0 per-32 scale rows.""" + num_blocks, block_size = k.shape[:2] + x = k.view(num_blocks, block_size, 16, 32).float() + exponent = torch.ceil(torch.log2(x.abs().amax(dim=-1).clamp(min=1e-10) / 448.0)) + data = (x / torch.exp2(exponent).unsqueeze(-1)).to(torch.float8_e4m3fn) + pages = torch.empty( + num_blocks, block_size * 528, dtype=torch.uint8, device=k.device + ) + pages[:, : block_size * 512] = data.view(torch.uint8).view(num_blocks, -1) + pages[:, block_size * 512 :] = (exponent + 127).to(torch.uint8).view(num_blocks, -1) + return pages.view(num_blocks, block_size, 1, 528) + + +def _dequantize_v41_k_cache(pages: torch.Tensor) -> torch.Tensor: + num_blocks, block_size = pages.shape[:2] + flat = pages.view(num_blocks, block_size * 528) + data = flat[:, : block_size * 512].view(torch.float8_e4m3fn).float() + exponent = flat[:, block_size * 512 :].float() - 127 + x = data.view(num_blocks, block_size, 16, 32) * torch.exp2( + exponent.view(num_blocks, block_size, 16, 1) + ) + return x.to(torch.bfloat16).view(num_blocks, block_size, 1, 512) + + +@pytest.mark.skipif( + torch.cuda.get_device_capability()[0] not in (9, 10), + reason="FlashMLA sparse decode needs SM90 or SM100", +) +@pytest.mark.parametrize("h_q", [64, 128]) +def test_sparse_decode_tensor_metadata_zero_rope_cache(h_q: int): + """A 528 B/token zero-RoPE cache through tensor metadata without kv_format must decode as V3.2 without RoPE, not V4.1.""" + torch.manual_seed(h_q) + b, s_q, s_k, topk, d = 2, 1, 4096, 2048, 512 + q, blocked_k, block_table, cache_seqlens, abs_indices, indices = ( + _sparse_decode_inputs(b, s_q, h_q, s_k, topk, d) + ) + k_cache = quantize_k_cache(blocked_k, d) + tile_scheduler_metadata, num_splits = get_mla_metadata( + cache_seqlens, s_q * h_q, 1, h_q, True, topk + ) + out, lse = flash_mla_with_kvcache( + q, + k_cache, + block_table, + cache_seqlens, + d, + tile_scheduler_metadata, + num_splits, + is_fp8_kvcache=True, + indices=indices, + ) + out_ref, lse_ref = reference_torch_decode( + cache_seqlens, + block_table, + q, + dequantize_k_cache(k_cache, dv=d, d=d), + d, + False, + abs_indices, + ) + torch.testing.assert_close(out, out_ref, atol=8e-4, rtol=2.01 / 128) + torch.testing.assert_close(lse, lse_ref, atol=1e-6, rtol=8.01 / 65536) + + +@pytest.mark.skipif( + torch.cuda.get_device_capability()[0] != 10, + reason="FlashMLA V4.1 KV cache formats need SM100", +) +@pytest.mark.parametrize("h_q", [64, 128]) +def test_sparse_decode_sched_meta_v41_cache(h_q: int): + """A 528 B/token V4.1 cache through FlashMLASchedMeta without kv_format must decode as V4.1, not V3.2 without RoPE.""" + torch.manual_seed(h_q) + b, s_q, s_k, topk, d = 2, 1, 4096, 2048, 512 + q, blocked_k, block_table, cache_seqlens, abs_indices, indices = ( + _sparse_decode_inputs(b, s_q, h_q, s_k, topk, d) + ) + k_cache = _quantize_v41_k_cache(blocked_k) + sched_meta, _ = get_mla_metadata() + out, lse = flash_mla_with_kvcache( + q, + k_cache, + None, + None, + d, + sched_meta, + is_fp8_kvcache=True, + indices=indices, + ) + out_ref, lse_ref = reference_torch_decode( + cache_seqlens, + block_table, + q, + _dequantize_v41_k_cache(k_cache), + d, + False, + abs_indices, + ) + torch.testing.assert_close(out, out_ref, atol=8e-4, rtol=2.01 / 128) + torch.testing.assert_close(lse, lse_ref, atol=1e-6, rtol=8.01 / 65536) + + if __name__ == "__main__": sys.exit(pytest.main([__file__]))