[Kernel] Expose FlashMLA kv_format in sgl-kernel sparse decode (#41336)
This commit is contained in:
@@ -1,9 +1,9 @@
|
||||
# flash_mla
|
||||
# DeepSeek v4.1 kernels merged into the SGLang fork (sgl-project/FlashMLA@3e18517).
|
||||
# DeepSeek v4.1 kernels, 528 B/token caches read as V4.1 by default (sgl-project/FlashMLA@3620b63).
|
||||
FetchContent_Declare(
|
||||
repo-flashmla
|
||||
URL https://${GITHUB_ARTIFACTORY}/sgl-project/FlashMLA/archive/3e18517fb055a6c9608eef5a1f1347fb1a047bbd.tar.gz
|
||||
URL_HASH SHA256=ab2af4657683a1bbaa707a2781a5e2c2792eba574ff36905f1832598e67fea79
|
||||
URL https://${GITHUB_ARTIFACTORY}/sgl-project/FlashMLA/archive/3620b63fc4ebe33950c50e1b7701b6607f44150e.tar.gz
|
||||
URL_HASH SHA256=7d7f9819bc1b121ef8226281de7265cd97be0cbe15d6450575fc8e3f241041a2
|
||||
)
|
||||
FetchContent_Populate(repo-flashmla)
|
||||
|
||||
|
||||
@@ -58,7 +58,8 @@ static std::tuple<at::Tensor, at::Tensor, std::optional<at::Tensor>, std::option
|
||||
const std::optional<at::Tensor>& extra_indices,
|
||||
const std::optional<at::Tensor>& extra_topk_length,
|
||||
int64_t d_v,
|
||||
double sm_scale) {
|
||||
double sm_scale,
|
||||
const std::optional<std::string>& kv_format) {
|
||||
return sparse_attn_decode_interface(
|
||||
q,
|
||||
kv,
|
||||
@@ -72,7 +73,7 @@ static std::tuple<at::Tensor, at::Tensor, std::optional<at::Tensor>, std::option
|
||||
extra_topk_length,
|
||||
static_cast<int>(d_v),
|
||||
static_cast<float>(sm_scale),
|
||||
std::nullopt);
|
||||
kv_format);
|
||||
}
|
||||
|
||||
static std::tuple<at::Tensor, at::Tensor, std::optional<at::Tensor>, std::optional<at::Tensor>> sgl_dense_decode_fwd(
|
||||
@@ -128,7 +129,7 @@ TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) {
|
||||
m.def(
|
||||
"sparse_decode_fwd(Tensor q, Tensor kv, Tensor indices, Tensor? topk_length, Tensor? attn_sink, "
|
||||
"Tensor? tile_scheduler_metadata, Tensor? num_splits, Tensor? extra_kv, Tensor? extra_indices, "
|
||||
"Tensor? extra_topk_length, int d_v, float sm_scale) -> (Tensor, Tensor, Tensor?, Tensor?)");
|
||||
"Tensor? extra_topk_length, int d_v, float sm_scale, str? kv_format=None) -> (Tensor, Tensor, Tensor?, Tensor?)");
|
||||
m.impl("sparse_decode_fwd", torch::kCUDA, &sgl_sparse_decode_fwd);
|
||||
|
||||
m.def(
|
||||
|
||||
@@ -14,6 +14,9 @@ _IMPORT_ERROR = ImportError(
|
||||
"Failed to load sgl_kernel.flashmla_ops extension. Ensure CUDA Driver >= 12.4"
|
||||
)
|
||||
|
||||
# 512 fp8 values + 4 fp32 scales; FlashMLA's V4.1 format has the same size.
|
||||
_V32_NO_ROPE_BYTES_PER_TOKEN = 528
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class FlashMLASchedMeta:
|
||||
@@ -103,6 +106,7 @@ def flash_mla_with_kvcache(
|
||||
extra_indices_in_kvcache: Optional[torch.Tensor] = None,
|
||||
topk_length: Optional[torch.Tensor] = None,
|
||||
extra_topk_length: Optional[torch.Tensor] = None,
|
||||
kv_format: Optional[str] = None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Arguments:
|
||||
@@ -119,6 +123,7 @@ def flash_mla_with_kvcache(
|
||||
descale_k: (batch_size), torch.float32. Descaling factors for K, used for fp8 quantization.
|
||||
is_fp8_kvcache: bool. Whether the k_cache and v_cache are in fp8 format. For the format of FP8 KV cache, please refer to README.md
|
||||
indices: (batch_size, seq_len_q, topk), torch.int32. If not None, sparse attention will be enabled, and only tokens in the `indices` array will be attended to. Invalid indices should be set to -1 or numbers >= total_seq_len_kv. For details about how to set up `indices`, please refer to README.md.
|
||||
kv_format: Optional[str]. Sparse attention only: the layout of `k_cache` ("V32", "V32_NO_ROPE", "V4", "V41"). When omitted it is detected from the shape. V3.2-no-RoPE and V4.1 are both 528 bytes per token with head_dim 512: with FlashMLASchedMeta such a cache is read as V41, with tensor tile_scheduler_metadata as V32_NO_ROPE.
|
||||
|
||||
Returns:
|
||||
out: (batch_size, seq_len_q, num_heads_q, head_dim_v).
|
||||
@@ -147,6 +152,7 @@ def flash_mla_with_kvcache(
|
||||
extra_indices_in_kvcache=extra_indices_in_kvcache,
|
||||
topk_length=topk_length,
|
||||
extra_topk_length=extra_topk_length,
|
||||
kv_format=kv_format,
|
||||
)
|
||||
|
||||
assert num_splits is not None
|
||||
@@ -163,7 +169,34 @@ def flash_mla_with_kvcache(
|
||||
"descale_q and descale_k should be both None or both not None"
|
||||
)
|
||||
|
||||
if indices is None and q.element_size() == 1:
|
||||
if indices is not None:
|
||||
assert is_fp8_kvcache, "is_fp8_kvcache must be True for sparse attention"
|
||||
# The tensor-metadata API predates V4.1, so its 528 B/token cache is V3.2 without RoPE.
|
||||
if (
|
||||
kv_format is None
|
||||
and q.shape[-1] == 512
|
||||
and k_cache.shape[-1] == _V32_NO_ROPE_BYTES_PER_TOKEN
|
||||
):
|
||||
kv_format = "V32_NO_ROPE"
|
||||
out, softmax_lse, _, _ = torch.ops.sgl_kernel.sparse_decode_fwd.default(
|
||||
q,
|
||||
k_cache,
|
||||
indices,
|
||||
topk_length,
|
||||
attn_sink,
|
||||
tile_scheduler_metadata,
|
||||
num_splits,
|
||||
extra_k_cache,
|
||||
extra_indices_in_kvcache,
|
||||
extra_topk_length,
|
||||
head_dim_v,
|
||||
softmax_scale,
|
||||
kv_format,
|
||||
)
|
||||
return out, softmax_lse
|
||||
|
||||
assert kv_format is None, "kv_format only applies to sparse attention"
|
||||
if q.element_size() == 1:
|
||||
out, softmax_lse = torch.ops.sgl_kernel.fwd_kvcache_mla_fp8.default(
|
||||
q,
|
||||
k_cache,
|
||||
@@ -189,7 +222,7 @@ def flash_mla_with_kvcache(
|
||||
tile_scheduler_metadata,
|
||||
num_splits,
|
||||
is_fp8_kvcache,
|
||||
indices,
|
||||
None,
|
||||
attn_sink,
|
||||
extra_k_cache,
|
||||
extra_indices_in_kvcache,
|
||||
@@ -216,6 +249,7 @@ def _flash_mla_with_kvcache_sched_meta(
|
||||
extra_indices_in_kvcache: Optional[torch.Tensor],
|
||||
topk_length: Optional[torch.Tensor],
|
||||
extra_topk_length: Optional[torch.Tensor],
|
||||
kv_format: Optional[str],
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
assert num_splits is None, "num_splits must be None with FlashMLASchedMeta"
|
||||
|
||||
@@ -279,9 +313,11 @@ def _flash_mla_with_kvcache_sched_meta(
|
||||
extra_topk_length,
|
||||
head_dim_v,
|
||||
softmax_scale,
|
||||
kv_format,
|
||||
)
|
||||
)
|
||||
else:
|
||||
assert kv_format is None, "kv_format only applies to sparse attention"
|
||||
assert block_table is not None and cache_seqlens is not None
|
||||
assert attn_sink is None
|
||||
assert extra_k_cache is None
|
||||
|
||||
@@ -657,5 +657,139 @@ def test_flash_mla_fp8(
|
||||
cal_diff(lse_flash, lse_torch, "lse")
|
||||
|
||||
|
||||
def _sparse_decode_inputs(b: int, s_q: int, h_q: int, s_k: int, topk: int, d: int):
|
||||
block_size = 64
|
||||
num_blocks_per_seq = cdiv(s_k, block_size)
|
||||
block_table = torch.randperm(
|
||||
b * num_blocks_per_seq, dtype=torch.int32, device="cuda"
|
||||
).view(b, num_blocks_per_seq)
|
||||
cache_seqlens = torch.full((b,), s_k, dtype=torch.int32, device="cuda")
|
||||
q = torch.randn(b, s_q, h_q, d, dtype=torch.bfloat16, device="cuda").clamp_(-1, 1)
|
||||
blocked_k = (
|
||||
(torch.randn(b * num_blocks_per_seq, block_size, 1, d, device="cuda") / 10)
|
||||
.clamp_(-1, 1)
|
||||
.to(torch.bfloat16)
|
||||
)
|
||||
abs_indices = torch.stack(
|
||||
[torch.randperm(s_k, device="cuda")[:topk] for _ in range(b * s_q)]
|
||||
).view(b, s_q, topk)
|
||||
abs_indices[torch.rand(abs_indices.shape, device="cuda") < 0.05] = -1
|
||||
block_ids = torch.gather(
|
||||
block_table, 1, (abs_indices.clamp(min=0) // block_size).view(b, -1)
|
||||
).view(b, s_q, topk)
|
||||
indices_in_kvcache = block_ids * block_size + abs_indices.clamp(min=0) % block_size
|
||||
indices_in_kvcache[abs_indices < 0] = -1
|
||||
return (
|
||||
q,
|
||||
blocked_k,
|
||||
block_table,
|
||||
cache_seqlens,
|
||||
abs_indices.int(),
|
||||
indices_in_kvcache.int(),
|
||||
)
|
||||
|
||||
|
||||
def _quantize_v41_k_cache(k: torch.Tensor) -> torch.Tensor:
|
||||
"""(num_blocks, block_size, 1, 512) bf16 -> V4.1 pages: fp8 data rows, then ue8m0 per-32 scale rows."""
|
||||
num_blocks, block_size = k.shape[:2]
|
||||
x = k.view(num_blocks, block_size, 16, 32).float()
|
||||
exponent = torch.ceil(torch.log2(x.abs().amax(dim=-1).clamp(min=1e-10) / 448.0))
|
||||
data = (x / torch.exp2(exponent).unsqueeze(-1)).to(torch.float8_e4m3fn)
|
||||
pages = torch.empty(
|
||||
num_blocks, block_size * 528, dtype=torch.uint8, device=k.device
|
||||
)
|
||||
pages[:, : block_size * 512] = data.view(torch.uint8).view(num_blocks, -1)
|
||||
pages[:, block_size * 512 :] = (exponent + 127).to(torch.uint8).view(num_blocks, -1)
|
||||
return pages.view(num_blocks, block_size, 1, 528)
|
||||
|
||||
|
||||
def _dequantize_v41_k_cache(pages: torch.Tensor) -> torch.Tensor:
|
||||
num_blocks, block_size = pages.shape[:2]
|
||||
flat = pages.view(num_blocks, block_size * 528)
|
||||
data = flat[:, : block_size * 512].view(torch.float8_e4m3fn).float()
|
||||
exponent = flat[:, block_size * 512 :].float() - 127
|
||||
x = data.view(num_blocks, block_size, 16, 32) * torch.exp2(
|
||||
exponent.view(num_blocks, block_size, 16, 1)
|
||||
)
|
||||
return x.to(torch.bfloat16).view(num_blocks, block_size, 1, 512)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.cuda.get_device_capability()[0] not in (9, 10),
|
||||
reason="FlashMLA sparse decode needs SM90 or SM100",
|
||||
)
|
||||
@pytest.mark.parametrize("h_q", [64, 128])
|
||||
def test_sparse_decode_tensor_metadata_zero_rope_cache(h_q: int):
|
||||
"""A 528 B/token zero-RoPE cache through tensor metadata without kv_format must decode as V3.2 without RoPE, not V4.1."""
|
||||
torch.manual_seed(h_q)
|
||||
b, s_q, s_k, topk, d = 2, 1, 4096, 2048, 512
|
||||
q, blocked_k, block_table, cache_seqlens, abs_indices, indices = (
|
||||
_sparse_decode_inputs(b, s_q, h_q, s_k, topk, d)
|
||||
)
|
||||
k_cache = quantize_k_cache(blocked_k, d)
|
||||
tile_scheduler_metadata, num_splits = get_mla_metadata(
|
||||
cache_seqlens, s_q * h_q, 1, h_q, True, topk
|
||||
)
|
||||
out, lse = flash_mla_with_kvcache(
|
||||
q,
|
||||
k_cache,
|
||||
block_table,
|
||||
cache_seqlens,
|
||||
d,
|
||||
tile_scheduler_metadata,
|
||||
num_splits,
|
||||
is_fp8_kvcache=True,
|
||||
indices=indices,
|
||||
)
|
||||
out_ref, lse_ref = reference_torch_decode(
|
||||
cache_seqlens,
|
||||
block_table,
|
||||
q,
|
||||
dequantize_k_cache(k_cache, dv=d, d=d),
|
||||
d,
|
||||
False,
|
||||
abs_indices,
|
||||
)
|
||||
torch.testing.assert_close(out, out_ref, atol=8e-4, rtol=2.01 / 128)
|
||||
torch.testing.assert_close(lse, lse_ref, atol=1e-6, rtol=8.01 / 65536)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.cuda.get_device_capability()[0] != 10,
|
||||
reason="FlashMLA V4.1 KV cache formats need SM100",
|
||||
)
|
||||
@pytest.mark.parametrize("h_q", [64, 128])
|
||||
def test_sparse_decode_sched_meta_v41_cache(h_q: int):
|
||||
"""A 528 B/token V4.1 cache through FlashMLASchedMeta without kv_format must decode as V4.1, not V3.2 without RoPE."""
|
||||
torch.manual_seed(h_q)
|
||||
b, s_q, s_k, topk, d = 2, 1, 4096, 2048, 512
|
||||
q, blocked_k, block_table, cache_seqlens, abs_indices, indices = (
|
||||
_sparse_decode_inputs(b, s_q, h_q, s_k, topk, d)
|
||||
)
|
||||
k_cache = _quantize_v41_k_cache(blocked_k)
|
||||
sched_meta, _ = get_mla_metadata()
|
||||
out, lse = flash_mla_with_kvcache(
|
||||
q,
|
||||
k_cache,
|
||||
None,
|
||||
None,
|
||||
d,
|
||||
sched_meta,
|
||||
is_fp8_kvcache=True,
|
||||
indices=indices,
|
||||
)
|
||||
out_ref, lse_ref = reference_torch_decode(
|
||||
cache_seqlens,
|
||||
block_table,
|
||||
q,
|
||||
_dequantize_v41_k_cache(k_cache),
|
||||
d,
|
||||
False,
|
||||
abs_indices,
|
||||
)
|
||||
torch.testing.assert_close(out, out_ref, atol=8e-4, rtol=2.01 / 128)
|
||||
torch.testing.assert_close(lse, lse_ref, atol=1e-6, rtol=8.01 / 65536)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__]))
|
||||
|
||||
Reference in New Issue
Block a user