[Kernel] Expose FlashMLA kv_format in sgl-kernel sparse decode (#41336)

This commit is contained in:
DarkSharpness
2026-09-29 17:16:52 -07:00
committed by GitHub
parent ba3e04c063
commit 57d4adeb83
4 changed files with 179 additions and 8 deletions
@@ -1,9 +1,9 @@
# flash_mla
# DeepSeek v4.1 kernels merged into the SGLang fork (sgl-project/FlashMLA@3e18517).
# DeepSeek v4.1 kernels, 528 B/token caches read as V4.1 by default (sgl-project/FlashMLA@3620b63).
FetchContent_Declare(
repo-flashmla
URL https://${GITHUB_ARTIFACTORY}/sgl-project/FlashMLA/archive/3e18517fb055a6c9608eef5a1f1347fb1a047bbd.tar.gz
URL_HASH SHA256=ab2af4657683a1bbaa707a2781a5e2c2792eba574ff36905f1832598e67fea79
URL https://${GITHUB_ARTIFACTORY}/sgl-project/FlashMLA/archive/3620b63fc4ebe33950c50e1b7701b6607f44150e.tar.gz
URL_HASH SHA256=7d7f9819bc1b121ef8226281de7265cd97be0cbe15d6450575fc8e3f241041a2
)
FetchContent_Populate(repo-flashmla)
@@ -58,7 +58,8 @@ static std::tuple<at::Tensor, at::Tensor, std::optional<at::Tensor>, std::option
const std::optional<at::Tensor>& extra_indices,
const std::optional<at::Tensor>& extra_topk_length,
int64_t d_v,
double sm_scale) {
double sm_scale,
const std::optional<std::string>& kv_format) {
return sparse_attn_decode_interface(
q,
kv,
@@ -72,7 +73,7 @@ static std::tuple<at::Tensor, at::Tensor, std::optional<at::Tensor>, std::option
extra_topk_length,
static_cast<int>(d_v),
static_cast<float>(sm_scale),
std::nullopt);
kv_format);
}
static std::tuple<at::Tensor, at::Tensor, std::optional<at::Tensor>, std::optional<at::Tensor>> sgl_dense_decode_fwd(
@@ -128,7 +129,7 @@ TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) {
m.def(
"sparse_decode_fwd(Tensor q, Tensor kv, Tensor indices, Tensor? topk_length, Tensor? attn_sink, "
"Tensor? tile_scheduler_metadata, Tensor? num_splits, Tensor? extra_kv, Tensor? extra_indices, "
"Tensor? extra_topk_length, int d_v, float sm_scale) -> (Tensor, Tensor, Tensor?, Tensor?)");
"Tensor? extra_topk_length, int d_v, float sm_scale, str? kv_format=None) -> (Tensor, Tensor, Tensor?, Tensor?)");
m.impl("sparse_decode_fwd", torch::kCUDA, &sgl_sparse_decode_fwd);
m.def(
@@ -14,6 +14,9 @@ _IMPORT_ERROR = ImportError(
"Failed to load sgl_kernel.flashmla_ops extension. Ensure CUDA Driver >= 12.4"
)
# 512 fp8 values + 4 fp32 scales; FlashMLA's V4.1 format has the same size.
_V32_NO_ROPE_BYTES_PER_TOKEN = 528
@dataclasses.dataclass
class FlashMLASchedMeta:
@@ -103,6 +106,7 @@ def flash_mla_with_kvcache(
extra_indices_in_kvcache: Optional[torch.Tensor] = None,
topk_length: Optional[torch.Tensor] = None,
extra_topk_length: Optional[torch.Tensor] = None,
kv_format: Optional[str] = None,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""
Arguments:
@@ -119,6 +123,7 @@ def flash_mla_with_kvcache(
descale_k: (batch_size), torch.float32. Descaling factors for K, used for fp8 quantization.
is_fp8_kvcache: bool. Whether the k_cache and v_cache are in fp8 format. For the format of FP8 KV cache, please refer to README.md
indices: (batch_size, seq_len_q, topk), torch.int32. If not None, sparse attention will be enabled, and only tokens in the `indices` array will be attended to. Invalid indices should be set to -1 or numbers >= total_seq_len_kv. For details about how to set up `indices`, please refer to README.md.
kv_format: Optional[str]. Sparse attention only: the layout of `k_cache` ("V32", "V32_NO_ROPE", "V4", "V41"). When omitted it is detected from the shape. V3.2-no-RoPE and V4.1 are both 528 bytes per token with head_dim 512: with FlashMLASchedMeta such a cache is read as V41, with tensor tile_scheduler_metadata as V32_NO_ROPE.
Returns:
out: (batch_size, seq_len_q, num_heads_q, head_dim_v).
@@ -147,6 +152,7 @@ def flash_mla_with_kvcache(
extra_indices_in_kvcache=extra_indices_in_kvcache,
topk_length=topk_length,
extra_topk_length=extra_topk_length,
kv_format=kv_format,
)
assert num_splits is not None
@@ -163,7 +169,34 @@ def flash_mla_with_kvcache(
"descale_q and descale_k should be both None or both not None"
)
if indices is None and q.element_size() == 1:
if indices is not None:
assert is_fp8_kvcache, "is_fp8_kvcache must be True for sparse attention"
# The tensor-metadata API predates V4.1, so its 528 B/token cache is V3.2 without RoPE.
if (
kv_format is None
and q.shape[-1] == 512
and k_cache.shape[-1] == _V32_NO_ROPE_BYTES_PER_TOKEN
):
kv_format = "V32_NO_ROPE"
out, softmax_lse, _, _ = torch.ops.sgl_kernel.sparse_decode_fwd.default(
q,
k_cache,
indices,
topk_length,
attn_sink,
tile_scheduler_metadata,
num_splits,
extra_k_cache,
extra_indices_in_kvcache,
extra_topk_length,
head_dim_v,
softmax_scale,
kv_format,
)
return out, softmax_lse
assert kv_format is None, "kv_format only applies to sparse attention"
if q.element_size() == 1:
out, softmax_lse = torch.ops.sgl_kernel.fwd_kvcache_mla_fp8.default(
q,
k_cache,
@@ -189,7 +222,7 @@ def flash_mla_with_kvcache(
tile_scheduler_metadata,
num_splits,
is_fp8_kvcache,
indices,
None,
attn_sink,
extra_k_cache,
extra_indices_in_kvcache,
@@ -216,6 +249,7 @@ def _flash_mla_with_kvcache_sched_meta(
extra_indices_in_kvcache: Optional[torch.Tensor],
topk_length: Optional[torch.Tensor],
extra_topk_length: Optional[torch.Tensor],
kv_format: Optional[str],
) -> Tuple[torch.Tensor, torch.Tensor]:
assert num_splits is None, "num_splits must be None with FlashMLASchedMeta"
@@ -279,9 +313,11 @@ def _flash_mla_with_kvcache_sched_meta(
extra_topk_length,
head_dim_v,
softmax_scale,
kv_format,
)
)
else:
assert kv_format is None, "kv_format only applies to sparse attention"
assert block_table is not None and cache_seqlens is not None
assert attn_sink is None
assert extra_k_cache is None
@@ -657,5 +657,139 @@ def test_flash_mla_fp8(
cal_diff(lse_flash, lse_torch, "lse")
def _sparse_decode_inputs(b: int, s_q: int, h_q: int, s_k: int, topk: int, d: int):
block_size = 64
num_blocks_per_seq = cdiv(s_k, block_size)
block_table = torch.randperm(
b * num_blocks_per_seq, dtype=torch.int32, device="cuda"
).view(b, num_blocks_per_seq)
cache_seqlens = torch.full((b,), s_k, dtype=torch.int32, device="cuda")
q = torch.randn(b, s_q, h_q, d, dtype=torch.bfloat16, device="cuda").clamp_(-1, 1)
blocked_k = (
(torch.randn(b * num_blocks_per_seq, block_size, 1, d, device="cuda") / 10)
.clamp_(-1, 1)
.to(torch.bfloat16)
)
abs_indices = torch.stack(
[torch.randperm(s_k, device="cuda")[:topk] for _ in range(b * s_q)]
).view(b, s_q, topk)
abs_indices[torch.rand(abs_indices.shape, device="cuda") < 0.05] = -1
block_ids = torch.gather(
block_table, 1, (abs_indices.clamp(min=0) // block_size).view(b, -1)
).view(b, s_q, topk)
indices_in_kvcache = block_ids * block_size + abs_indices.clamp(min=0) % block_size
indices_in_kvcache[abs_indices < 0] = -1
return (
q,
blocked_k,
block_table,
cache_seqlens,
abs_indices.int(),
indices_in_kvcache.int(),
)
def _quantize_v41_k_cache(k: torch.Tensor) -> torch.Tensor:
"""(num_blocks, block_size, 1, 512) bf16 -> V4.1 pages: fp8 data rows, then ue8m0 per-32 scale rows."""
num_blocks, block_size = k.shape[:2]
x = k.view(num_blocks, block_size, 16, 32).float()
exponent = torch.ceil(torch.log2(x.abs().amax(dim=-1).clamp(min=1e-10) / 448.0))
data = (x / torch.exp2(exponent).unsqueeze(-1)).to(torch.float8_e4m3fn)
pages = torch.empty(
num_blocks, block_size * 528, dtype=torch.uint8, device=k.device
)
pages[:, : block_size * 512] = data.view(torch.uint8).view(num_blocks, -1)
pages[:, block_size * 512 :] = (exponent + 127).to(torch.uint8).view(num_blocks, -1)
return pages.view(num_blocks, block_size, 1, 528)
def _dequantize_v41_k_cache(pages: torch.Tensor) -> torch.Tensor:
num_blocks, block_size = pages.shape[:2]
flat = pages.view(num_blocks, block_size * 528)
data = flat[:, : block_size * 512].view(torch.float8_e4m3fn).float()
exponent = flat[:, block_size * 512 :].float() - 127
x = data.view(num_blocks, block_size, 16, 32) * torch.exp2(
exponent.view(num_blocks, block_size, 16, 1)
)
return x.to(torch.bfloat16).view(num_blocks, block_size, 1, 512)
@pytest.mark.skipif(
torch.cuda.get_device_capability()[0] not in (9, 10),
reason="FlashMLA sparse decode needs SM90 or SM100",
)
@pytest.mark.parametrize("h_q", [64, 128])
def test_sparse_decode_tensor_metadata_zero_rope_cache(h_q: int):
"""A 528 B/token zero-RoPE cache through tensor metadata without kv_format must decode as V3.2 without RoPE, not V4.1."""
torch.manual_seed(h_q)
b, s_q, s_k, topk, d = 2, 1, 4096, 2048, 512
q, blocked_k, block_table, cache_seqlens, abs_indices, indices = (
_sparse_decode_inputs(b, s_q, h_q, s_k, topk, d)
)
k_cache = quantize_k_cache(blocked_k, d)
tile_scheduler_metadata, num_splits = get_mla_metadata(
cache_seqlens, s_q * h_q, 1, h_q, True, topk
)
out, lse = flash_mla_with_kvcache(
q,
k_cache,
block_table,
cache_seqlens,
d,
tile_scheduler_metadata,
num_splits,
is_fp8_kvcache=True,
indices=indices,
)
out_ref, lse_ref = reference_torch_decode(
cache_seqlens,
block_table,
q,
dequantize_k_cache(k_cache, dv=d, d=d),
d,
False,
abs_indices,
)
torch.testing.assert_close(out, out_ref, atol=8e-4, rtol=2.01 / 128)
torch.testing.assert_close(lse, lse_ref, atol=1e-6, rtol=8.01 / 65536)
@pytest.mark.skipif(
torch.cuda.get_device_capability()[0] != 10,
reason="FlashMLA V4.1 KV cache formats need SM100",
)
@pytest.mark.parametrize("h_q", [64, 128])
def test_sparse_decode_sched_meta_v41_cache(h_q: int):
"""A 528 B/token V4.1 cache through FlashMLASchedMeta without kv_format must decode as V4.1, not V3.2 without RoPE."""
torch.manual_seed(h_q)
b, s_q, s_k, topk, d = 2, 1, 4096, 2048, 512
q, blocked_k, block_table, cache_seqlens, abs_indices, indices = (
_sparse_decode_inputs(b, s_q, h_q, s_k, topk, d)
)
k_cache = _quantize_v41_k_cache(blocked_k)
sched_meta, _ = get_mla_metadata()
out, lse = flash_mla_with_kvcache(
q,
k_cache,
None,
None,
d,
sched_meta,
is_fp8_kvcache=True,
indices=indices,
)
out_ref, lse_ref = reference_torch_decode(
cache_seqlens,
block_table,
q,
_dequantize_v41_k_cache(k_cache),
d,
False,
abs_indices,
)
torch.testing.assert_close(out, out_ref, atol=8e-4, rtol=2.01 / 128)
torch.testing.assert_close(lse, lse_ref, atol=1e-6, rtol=8.01 / 65536)
if __name__ == "__main__":
sys.exit(pytest.main([__file__]))