[Kernel] Expose bundled FlashMLA MegaAttention kernels and APIs (#43918)

This commit is contained in:
Khoa Pham
2026-10-12 14:50:04 -07:00
committed by GitHub
parent 6781da8e5b
commit 93809202a3
5 changed files with 419 additions and 0 deletions
@@ -0,0 +1,23 @@
import torch
from sgl_kernel.flash_mla_mega import flash_mla_mega_prefill
from sglang.kernels.jit.benchmark import marker
@marker.parametrize("tokens", [1, 32, 128, 512], [128])
@marker.parametrize("heads", [64, 128], [64])
@marker.benchmark("provider", ["flashmla_mega"])
def benchmark(tokens: int, heads: int, provider: str):
q = torch.randn(tokens, heads, 512, device="cuda", dtype=torch.bfloat16)
kv = torch.randn(4096, 1, 512, device="cuda", dtype=torch.bfloat16)
indices = torch.randint(4096, (tokens, 1, 640), device="cuda", dtype=torch.int32)
positions = torch.zeros(tokens, device="cuda", dtype=torch.int32)
table = torch.cat((torch.ones(1, 32), torch.zeros(1, 32)), dim=-1).cuda()
return marker.do_bench(
flash_mla_mega_prefill,
input_args=(q, kv, indices, positions, table, 512**-0.5),
)
if __name__ == "__main__":
benchmark.run()
@@ -127,6 +127,7 @@ set(FlashMLA_SOURCES
if(FLASHMLA_ENABLE_SM100)
list(APPEND FlashMLA_SOURCES
"csrc/flashmla_mega_extension.cc"
# sm100 dense prefill/bwd.
${repo-flashmla_SOURCE_DIR}/csrc/kernels/sm100/prefill/dense/fmha_cutlass_fwd_sm100.cu
${repo-flashmla_SOURCE_DIR}/csrc/kernels/sm100/prefill/dense/fmha_cutlass_bwd_sm100.cu
@@ -156,6 +157,21 @@ if(FLASHMLA_ENABLE_SM100)
${repo-flashmla_SOURCE_DIR}/csrc/kernels/sm100/prefill/sparse/fwd_for_small_topk/head128/instantiations/phase1_decode_k512_v41fp4.cu
${repo-flashmla_SOURCE_DIR}/csrc/kernels/sm100/prefill/sparse/fwd_for_small_topk/head128/instantiations/phase1_decode_k512_v41fp4_splitkv.cu
)
set(FLASHMLA_MEGA_ROOT ${repo-flashmla_SOURCE_DIR}/csrc/kernels/sm100/prefill/sparse/fused_norm_rope_attn_rope_cast_fwd)
foreach(HEADS 64 128)
foreach(NORM norm nonorm)
list(APPEND FlashMLA_SOURCES
${FLASHMLA_MEGA_ROOT}/core_attn/instantiations/v4_h${HEADS}_prefill_${NORM}.cu
${FLASHMLA_MEGA_ROOT}/core_attn/instantiations/v4_h${HEADS}_decode_${NORM}.cu
${FLASHMLA_MEGA_ROOT}/core_attn/instantiations/v41_h${HEADS}_decode_${NORM}.cu
${FLASHMLA_MEGA_ROOT}/core_attn/instantiations/v41fp4_h${HEADS}_decode_${NORM}.cu
)
endforeach()
endforeach()
list(APPEND FlashMLA_SOURCES
${FLASHMLA_MEGA_ROOT}/permute_q_b_proj/kernel.cu
${FLASHMLA_MEGA_ROOT}/permute_wv_proj/kernel.cu
)
endif()
Python_add_library(flashmla_ops MODULE USE_SABI ${SKBUILD_SABI_VERSION} WITH_SOABI ${FlashMLA_SOURCES})
@@ -0,0 +1,100 @@
#include <torch/all.h>
#include <torch/library.h>
#include "api/fused_norm_rope_attn_rope_cast_fwd.cpp"
static std::vector<at::Tensor> sgl_flash_mla_mega_prefill(
const at::Tensor& q,
const at::Tensor& kv,
const at::Tensor& indices,
const at::Tensor& positions,
const at::Tensor& cos_sin_cache,
double sm_scale,
int64_t n_wv_group,
const std::optional<at::Tensor>& attn_sink,
const std::optional<at::Tensor>& topk_length) {
TORCH_CHECK(n_wv_group > 0 && q.size(1) == n_wv_group * 8, "MegaAttention requires eight heads per WO-A group");
return fused_norm_rope_attn_rope_cast_fwd(
q,
kv,
indices,
static_cast<float>(sm_scale),
512,
attn_sink,
topk_length,
false,
0.0f,
positions,
false,
64,
cos_sin_cache,
static_cast<uint32_t>(n_wv_group),
32,
true,
true,
true);
}
static std::vector<at::Tensor> sgl_flash_mla_mega_decode(
const at::Tensor& q,
const at::Tensor& kv,
const at::Tensor& indices,
const at::Tensor& positions,
const at::Tensor& cos_sin_cache,
double sm_scale,
int64_t n_wv_group,
const std::optional<at::Tensor>& attn_sink,
const std::optional<at::Tensor>& topk_length,
const std::optional<at::Tensor>& extra_kv,
const std::optional<at::Tensor>& extra_indices,
const std::optional<at::Tensor>& extra_topk_length) {
TORCH_CHECK(n_wv_group > 0 && q.size(1) == n_wv_group * 8, "MegaAttention requires eight heads per WO-A group");
return fused_norm_rope_attn_rope_cast_decode(
q,
kv,
indices,
static_cast<float>(sm_scale),
512,
attn_sink,
topk_length,
extra_kv,
extra_indices,
extra_topk_length,
false,
0.0f,
positions,
false,
64,
cos_sin_cache,
static_cast<uint32_t>(n_wv_group),
32,
true,
true,
true);
}
static std::vector<at::Tensor>
sgl_flash_mla_mega_permute_q(const at::Tensor& weight, const at::Tensor& scale, int64_t heads) {
TORCH_CHECK(heads == 64 || heads == 128, "MegaAttention requires 64 or 128 query heads");
return permute_q_b_proj(weight, scale, static_cast<int>(heads), 512);
}
static std::vector<at::Tensor> sgl_flash_mla_mega_permute_wo(const at::Tensor& weight, const at::Tensor& scale) {
return permute_wv_proj(weight, scale, 8, 512);
}
TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) {
m.def(
"flash_mla_mega_prefill(Tensor q, Tensor kv, Tensor indices, Tensor positions, Tensor cos_sin_cache, "
"float sm_scale, int n_wv_group, Tensor? attn_sink=None, Tensor? topk_length=None) -> Tensor[]");
m.impl("flash_mla_mega_prefill", torch::kCUDA, &sgl_flash_mla_mega_prefill);
m.def(
"flash_mla_mega_decode(Tensor q, Tensor kv, Tensor indices, Tensor positions, Tensor cos_sin_cache, "
"float sm_scale, int n_wv_group, Tensor? attn_sink=None, Tensor? topk_length=None, "
"Tensor? extra_kv=None, Tensor? extra_indices=None, Tensor? extra_topk_length=None) -> Tensor[]");
m.impl("flash_mla_mega_decode", torch::kCUDA, &sgl_flash_mla_mega_decode);
m.def("flash_mla_mega_permute_q(Tensor weight, Tensor scale, int heads) -> Tensor[]");
m.impl("flash_mla_mega_permute_q", torch::kCUDA, &sgl_flash_mla_mega_permute_q);
m.def("flash_mla_mega_permute_wo(Tensor weight, Tensor scale) -> Tensor[]");
m.impl("flash_mla_mega_permute_wo", torch::kCUDA, &sgl_flash_mla_mega_permute_wo);
}
@@ -0,0 +1,96 @@
from typing import Optional
import torch
from sgl_kernel.flash_mla import _flashmla_import_error
def is_flash_mla_mega_available() -> bool:
return _flashmla_import_error is None and hasattr(
torch.ops.sgl_kernel, "flash_mla_mega_decode"
)
def _require_mega() -> None:
if not is_flash_mla_mega_available():
raise ImportError(
"FlashMLA MegaAttention requires sgl-kernel built with CUDA >= 12.9"
)
def flash_mla_mega_prefill(
q: torch.Tensor,
kv: torch.Tensor,
indices: torch.Tensor,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
softmax_scale: float,
*,
attn_sink: Optional[torch.Tensor] = None,
topk_length: Optional[torch.Tensor] = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
_require_mega()
return tuple(
torch.ops.sgl_kernel.flash_mla_mega_prefill.default(
q,
kv,
indices,
positions,
cos_sin_cache,
softmax_scale,
q.shape[1] // 8,
attn_sink,
topk_length,
)
)
def flash_mla_mega_decode(
q: torch.Tensor,
kv: torch.Tensor,
indices: torch.Tensor,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
softmax_scale: float,
*,
attn_sink: Optional[torch.Tensor] = None,
topk_length: Optional[torch.Tensor] = None,
extra_kv: Optional[torch.Tensor] = None,
extra_indices: Optional[torch.Tensor] = None,
extra_topk_length: Optional[torch.Tensor] = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
_require_mega()
return tuple(
torch.ops.sgl_kernel.flash_mla_mega_decode.default(
q,
kv,
indices,
positions,
cos_sin_cache,
softmax_scale,
q.shape[1] // 8,
attn_sink,
topk_length,
extra_kv,
extra_indices,
extra_topk_length,
)
)
def permute_mega_q_weight(
weight: torch.Tensor,
scale: torch.Tensor,
num_heads: int,
) -> tuple[torch.Tensor, torch.Tensor]:
_require_mega()
return tuple(
torch.ops.sgl_kernel.flash_mla_mega_permute_q.default(weight, scale, num_heads)
)
def permute_mega_wo_weight(
weight: torch.Tensor,
scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
_require_mega()
return tuple(torch.ops.sgl_kernel.flash_mla_mega_permute_wo.default(weight, scale))
@@ -0,0 +1,184 @@
import sys
import pytest
import torch
from sgl_kernel.flash_mla_mega import (
flash_mla_mega_decode,
flash_mla_mega_prefill,
is_flash_mla_mega_available,
permute_mega_q_weight,
permute_mega_wo_weight,
)
pytestmark = pytest.mark.skipif(
not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10,
reason="FlashMLA MegaAttention requires SM100/SM103",
)
def _pack_q(q):
tokens, heads, dim = q.shape
return q.view(tokens, heads, dim // 16, 16).transpose(1, 2).contiguous().view_as(q)
def _unpack_output(data, scale):
tokens, groups, hidden = data.shape
sf = scale.contiguous().view(torch.uint8).float()
values = data.float().view(tokens, groups, hidden // 32, 32)
values = values * torch.exp2(sf - 127).unsqueeze(-1)
return (
values.view(tokens, groups, 16, 8, 32)
.transpose(2, 3)
.reshape(tokens, groups * 8, 512)
)
def _rope(x, positions, table, inverse=False):
out = x.float().clone()
cos, sin = table[positions.long()].chunk(2, dim=-1)
cos, sin = cos[:, None], sin[:, None]
if inverse:
sin = -sin
even, odd = out[..., -64::2].clone(), out[..., -63::2].clone()
out[..., -64::2] = even * cos - odd * sin
out[..., -63::2] = even * sin + odd * cos
return out
def _reference(q, kv, indices, lens, positions, table, sink):
q = _rope(q, positions, table).bfloat16().float()
selected = kv[indices.clamp_min(0), 0].float()
logits = torch.einsum("thd,tkd->thk", q, selected) / 512**0.5
valid = (indices >= 0) & (
torch.arange(indices.shape[-1], device=q.device)[None] < lens[:, None]
)
logits.masked_fill_(~valid[:, None], -torch.inf)
logits = torch.cat((logits, sink[None, :, None].expand(q.shape[0], -1, -1)), dim=-1)
probabilities = logits.softmax(-1)[..., :-1]
out = torch.einsum("thk,tkd->thd", probabilities, selected)
return _rope(out, positions, table, inverse=True)
def _inputs(heads):
torch.manual_seed(123)
device = "cuda"
q = torch.randn(3, heads, 512, device=device, dtype=torch.bfloat16)
kv = torch.randn(128, 1, 512, device=device, dtype=torch.bfloat16)
indices = torch.arange(128, device=device, dtype=torch.int32).repeat(3, 1)
indices[1, 48:64] = -1
lens = torch.tensor([128, 96, 64], device=device, dtype=torch.int32)
positions = torch.tensor([0, 17, 255], device=device, dtype=torch.int32)
angles = (
torch.arange(256, device=device)[:, None]
/ torch.arange(1, 33, device=device)[None]
)
table = torch.cat((angles.cos(), angles.sin()), dim=-1).float()
sink = torch.randn(heads, device=device, dtype=torch.float32)
return q, kv, indices, lens, positions, table, sink
@pytest.mark.parametrize("heads", [64, 128])
def test_mega_prefill_matches_unfused_reference(heads):
assert is_flash_mla_mega_available(), "The SM100 wheel must expose MegaAttention"
q, kv, indices, lens, positions, table, sink = _inputs(heads)
data, scale, _, _ = flash_mla_mega_prefill(
_pack_q(q),
kv,
indices[:, None],
positions,
table,
512**-0.5,
attn_sink=sink,
topk_length=lens,
)
expected = _reference(q, kv, indices, lens, positions, table, sink)
actual = _unpack_output(data, scale)
relative_rmse = (
actual - expected
).square().mean().sqrt() / expected.square().mean().sqrt()
assert relative_rmse < 0.04
assert scale.dtype == torch.int32 and scale.stride(0) == 1
assert scale.stride(-1) % 4 == 0
def test_mega_decode_matches_gathered_prefill():
q, kv, indices, lens, positions, table, sink = _inputs(64)
groups = kv.float().view(128, 16, 32)
sf = torch.exp2(torch.ceil(torch.log2(groups.abs().amax(-1).clamp_min(1e-4) / 448)))
codes = (groups / sf[..., None]).to(torch.float8_e4m3fn)
dequant = (codes.float() * sf[..., None]).view_as(kv).bfloat16()
raw = torch.empty(128 * 528, device="cuda", dtype=torch.uint8)
raw[: 128 * 512] = codes.view(torch.uint8).reshape(-1)
raw[128 * 512 :] = (sf.log2() + 127).to(torch.uint8).reshape(-1)
cache = raw.view(1, 128, 1, 528)
packed_q = _pack_q(q)
decoded = flash_mla_mega_decode(
packed_q,
cache,
indices,
positions,
table,
512**-0.5,
attn_sink=sink,
topk_length=lens,
)
prefilled = flash_mla_mega_prefill(
packed_q,
dequant,
indices[:, None],
positions,
table,
512**-0.5,
attn_sink=sink,
topk_length=lens,
)
torch.testing.assert_close(
_unpack_output(*decoded[:2]),
_unpack_output(*prefilled[:2]),
rtol=0.05,
atol=0.01,
)
def test_mega_weight_permutations_preserve_scale_association():
heads, rows, width = 64, 64, 128
q_weight = (
torch.randint(0, 112, (heads * 512 * width,), device="cuda")
.to(torch.uint8)
.view(torch.float8_e4m3fn)
.view(heads * 512, width)
)
q_scale = torch.arange(heads * 512, device="cuda", dtype=torch.int32)[:, None]
pq, ps = permute_mega_q_weight(q_weight, q_scale, heads)
q_perm = (
torch.arange(heads * 512, device="cuda")
.view(heads, 32, 16)
.transpose(0, 1)
.reshape(-1)
)
torch.testing.assert_close(pq.view(torch.uint8), q_weight.view(torch.uint8)[q_perm])
torch.testing.assert_close(ps, q_scale[q_perm])
wo = (
torch.randint(0, 112, (rows * 4096,), device="cuda")
.to(torch.uint8)
.view(torch.float8_e4m3fn)
.view(1, rows, 4096)
)
sf_bytes = (
torch.arange(128, device="cuda", dtype=torch.uint8)[None, None]
.expand(1, rows, -1)
.contiguous()
)
sf = sf_bytes.view(torch.int32).transpose(1, 2).contiguous().transpose(1, 2)
pw, pws = permute_mega_wo_weight(wo, sf)
wo_perm = (
torch.arange(4096, device="cuda").view(8, 16, 32).transpose(0, 1).reshape(-1)
)
torch.testing.assert_close(pw.view(torch.uint8), wo.view(torch.uint8)[..., wo_perm])
expected_sf = sf_bytes[..., wo_perm[::32] // 32]
torch.testing.assert_close(pws.contiguous().view(torch.uint8), expected_sf)
if __name__ == "__main__":
sys.exit(pytest.main([__file__]))