From 93809202a3ffcf764c9bc0a9673ae83be69efc20 Mon Sep 17 00:00:00 2001 From: Khoa Pham Date: Mon, 12 Oct 2026 14:50:04 -0700 Subject: [PATCH] [Kernel] Expose bundled FlashMLA MegaAttention kernels and APIs (#43918) --- .../aot/benchmark/bench_flashmla_mega.py | 23 +++ .../sglang/kernels/aot/cmake/flashmla.cmake | 16 ++ .../aot/csrc/flashmla_mega_extension.cc | 100 ++++++++++ .../aot/python/sgl_kernel/flash_mla_mega.py | 96 +++++++++ .../kernels/aot/tests/test_flashmla_mega.py | 184 ++++++++++++++++++ 5 files changed, 419 insertions(+) create mode 100644 python/sglang/kernels/aot/benchmark/bench_flashmla_mega.py create mode 100644 python/sglang/kernels/aot/csrc/flashmla_mega_extension.cc create mode 100644 python/sglang/kernels/aot/python/sgl_kernel/flash_mla_mega.py create mode 100644 python/sglang/kernels/aot/tests/test_flashmla_mega.py diff --git a/python/sglang/kernels/aot/benchmark/bench_flashmla_mega.py b/python/sglang/kernels/aot/benchmark/bench_flashmla_mega.py new file mode 100644 index 0000000000..01b7ad91db --- /dev/null +++ b/python/sglang/kernels/aot/benchmark/bench_flashmla_mega.py @@ -0,0 +1,23 @@ +import torch +from sgl_kernel.flash_mla_mega import flash_mla_mega_prefill + +from sglang.kernels.jit.benchmark import marker + + +@marker.parametrize("tokens", [1, 32, 128, 512], [128]) +@marker.parametrize("heads", [64, 128], [64]) +@marker.benchmark("provider", ["flashmla_mega"]) +def benchmark(tokens: int, heads: int, provider: str): + q = torch.randn(tokens, heads, 512, device="cuda", dtype=torch.bfloat16) + kv = torch.randn(4096, 1, 512, device="cuda", dtype=torch.bfloat16) + indices = torch.randint(4096, (tokens, 1, 640), device="cuda", dtype=torch.int32) + positions = torch.zeros(tokens, device="cuda", dtype=torch.int32) + table = torch.cat((torch.ones(1, 32), torch.zeros(1, 32)), dim=-1).cuda() + return marker.do_bench( + flash_mla_mega_prefill, + input_args=(q, kv, indices, positions, table, 512**-0.5), + ) + + +if __name__ == "__main__": + benchmark.run() diff --git a/python/sglang/kernels/aot/cmake/flashmla.cmake b/python/sglang/kernels/aot/cmake/flashmla.cmake index 24050ed35e..285cc8ab60 100644 --- a/python/sglang/kernels/aot/cmake/flashmla.cmake +++ b/python/sglang/kernels/aot/cmake/flashmla.cmake @@ -127,6 +127,7 @@ set(FlashMLA_SOURCES if(FLASHMLA_ENABLE_SM100) list(APPEND FlashMLA_SOURCES + "csrc/flashmla_mega_extension.cc" # sm100 dense prefill/bwd. ${repo-flashmla_SOURCE_DIR}/csrc/kernels/sm100/prefill/dense/fmha_cutlass_fwd_sm100.cu ${repo-flashmla_SOURCE_DIR}/csrc/kernels/sm100/prefill/dense/fmha_cutlass_bwd_sm100.cu @@ -156,6 +157,21 @@ if(FLASHMLA_ENABLE_SM100) ${repo-flashmla_SOURCE_DIR}/csrc/kernels/sm100/prefill/sparse/fwd_for_small_topk/head128/instantiations/phase1_decode_k512_v41fp4.cu ${repo-flashmla_SOURCE_DIR}/csrc/kernels/sm100/prefill/sparse/fwd_for_small_topk/head128/instantiations/phase1_decode_k512_v41fp4_splitkv.cu ) + set(FLASHMLA_MEGA_ROOT ${repo-flashmla_SOURCE_DIR}/csrc/kernels/sm100/prefill/sparse/fused_norm_rope_attn_rope_cast_fwd) + foreach(HEADS 64 128) + foreach(NORM norm nonorm) + list(APPEND FlashMLA_SOURCES + ${FLASHMLA_MEGA_ROOT}/core_attn/instantiations/v4_h${HEADS}_prefill_${NORM}.cu + ${FLASHMLA_MEGA_ROOT}/core_attn/instantiations/v4_h${HEADS}_decode_${NORM}.cu + ${FLASHMLA_MEGA_ROOT}/core_attn/instantiations/v41_h${HEADS}_decode_${NORM}.cu + ${FLASHMLA_MEGA_ROOT}/core_attn/instantiations/v41fp4_h${HEADS}_decode_${NORM}.cu + ) + endforeach() + endforeach() + list(APPEND FlashMLA_SOURCES + ${FLASHMLA_MEGA_ROOT}/permute_q_b_proj/kernel.cu + ${FLASHMLA_MEGA_ROOT}/permute_wv_proj/kernel.cu + ) endif() Python_add_library(flashmla_ops MODULE USE_SABI ${SKBUILD_SABI_VERSION} WITH_SOABI ${FlashMLA_SOURCES}) diff --git a/python/sglang/kernels/aot/csrc/flashmla_mega_extension.cc b/python/sglang/kernels/aot/csrc/flashmla_mega_extension.cc new file mode 100644 index 0000000000..3de16fc625 --- /dev/null +++ b/python/sglang/kernels/aot/csrc/flashmla_mega_extension.cc @@ -0,0 +1,100 @@ +#include +#include + +#include "api/fused_norm_rope_attn_rope_cast_fwd.cpp" + +static std::vector sgl_flash_mla_mega_prefill( + const at::Tensor& q, + const at::Tensor& kv, + const at::Tensor& indices, + const at::Tensor& positions, + const at::Tensor& cos_sin_cache, + double sm_scale, + int64_t n_wv_group, + const std::optional& attn_sink, + const std::optional& topk_length) { + TORCH_CHECK(n_wv_group > 0 && q.size(1) == n_wv_group * 8, "MegaAttention requires eight heads per WO-A group"); + return fused_norm_rope_attn_rope_cast_fwd( + q, + kv, + indices, + static_cast(sm_scale), + 512, + attn_sink, + topk_length, + false, + 0.0f, + positions, + false, + 64, + cos_sin_cache, + static_cast(n_wv_group), + 32, + true, + true, + true); +} + +static std::vector sgl_flash_mla_mega_decode( + const at::Tensor& q, + const at::Tensor& kv, + const at::Tensor& indices, + const at::Tensor& positions, + const at::Tensor& cos_sin_cache, + double sm_scale, + int64_t n_wv_group, + const std::optional& attn_sink, + const std::optional& topk_length, + const std::optional& extra_kv, + const std::optional& extra_indices, + const std::optional& extra_topk_length) { + TORCH_CHECK(n_wv_group > 0 && q.size(1) == n_wv_group * 8, "MegaAttention requires eight heads per WO-A group"); + return fused_norm_rope_attn_rope_cast_decode( + q, + kv, + indices, + static_cast(sm_scale), + 512, + attn_sink, + topk_length, + extra_kv, + extra_indices, + extra_topk_length, + false, + 0.0f, + positions, + false, + 64, + cos_sin_cache, + static_cast(n_wv_group), + 32, + true, + true, + true); +} + +static std::vector +sgl_flash_mla_mega_permute_q(const at::Tensor& weight, const at::Tensor& scale, int64_t heads) { + TORCH_CHECK(heads == 64 || heads == 128, "MegaAttention requires 64 or 128 query heads"); + return permute_q_b_proj(weight, scale, static_cast(heads), 512); +} + +static std::vector sgl_flash_mla_mega_permute_wo(const at::Tensor& weight, const at::Tensor& scale) { + return permute_wv_proj(weight, scale, 8, 512); +} + +TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) { + m.def( + "flash_mla_mega_prefill(Tensor q, Tensor kv, Tensor indices, Tensor positions, Tensor cos_sin_cache, " + "float sm_scale, int n_wv_group, Tensor? attn_sink=None, Tensor? topk_length=None) -> Tensor[]"); + m.impl("flash_mla_mega_prefill", torch::kCUDA, &sgl_flash_mla_mega_prefill); + m.def( + "flash_mla_mega_decode(Tensor q, Tensor kv, Tensor indices, Tensor positions, Tensor cos_sin_cache, " + "float sm_scale, int n_wv_group, Tensor? attn_sink=None, Tensor? topk_length=None, " + "Tensor? extra_kv=None, Tensor? extra_indices=None, Tensor? extra_topk_length=None) -> Tensor[]"); + m.impl("flash_mla_mega_decode", torch::kCUDA, &sgl_flash_mla_mega_decode); + m.def("flash_mla_mega_permute_q(Tensor weight, Tensor scale, int heads) -> Tensor[]"); + m.impl("flash_mla_mega_permute_q", torch::kCUDA, &sgl_flash_mla_mega_permute_q); + m.def("flash_mla_mega_permute_wo(Tensor weight, Tensor scale) -> Tensor[]"); + m.impl("flash_mla_mega_permute_wo", torch::kCUDA, &sgl_flash_mla_mega_permute_wo); +} diff --git a/python/sglang/kernels/aot/python/sgl_kernel/flash_mla_mega.py b/python/sglang/kernels/aot/python/sgl_kernel/flash_mla_mega.py new file mode 100644 index 0000000000..cec8c3b724 --- /dev/null +++ b/python/sglang/kernels/aot/python/sgl_kernel/flash_mla_mega.py @@ -0,0 +1,96 @@ +from typing import Optional + +import torch +from sgl_kernel.flash_mla import _flashmla_import_error + + +def is_flash_mla_mega_available() -> bool: + return _flashmla_import_error is None and hasattr( + torch.ops.sgl_kernel, "flash_mla_mega_decode" + ) + + +def _require_mega() -> None: + if not is_flash_mla_mega_available(): + raise ImportError( + "FlashMLA MegaAttention requires sgl-kernel built with CUDA >= 12.9" + ) + + +def flash_mla_mega_prefill( + q: torch.Tensor, + kv: torch.Tensor, + indices: torch.Tensor, + positions: torch.Tensor, + cos_sin_cache: torch.Tensor, + softmax_scale: float, + *, + attn_sink: Optional[torch.Tensor] = None, + topk_length: Optional[torch.Tensor] = None, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + _require_mega() + return tuple( + torch.ops.sgl_kernel.flash_mla_mega_prefill.default( + q, + kv, + indices, + positions, + cos_sin_cache, + softmax_scale, + q.shape[1] // 8, + attn_sink, + topk_length, + ) + ) + + +def flash_mla_mega_decode( + q: torch.Tensor, + kv: torch.Tensor, + indices: torch.Tensor, + positions: torch.Tensor, + cos_sin_cache: torch.Tensor, + softmax_scale: float, + *, + attn_sink: Optional[torch.Tensor] = None, + topk_length: Optional[torch.Tensor] = None, + extra_kv: Optional[torch.Tensor] = None, + extra_indices: Optional[torch.Tensor] = None, + extra_topk_length: Optional[torch.Tensor] = None, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + _require_mega() + return tuple( + torch.ops.sgl_kernel.flash_mla_mega_decode.default( + q, + kv, + indices, + positions, + cos_sin_cache, + softmax_scale, + q.shape[1] // 8, + attn_sink, + topk_length, + extra_kv, + extra_indices, + extra_topk_length, + ) + ) + + +def permute_mega_q_weight( + weight: torch.Tensor, + scale: torch.Tensor, + num_heads: int, +) -> tuple[torch.Tensor, torch.Tensor]: + _require_mega() + return tuple( + torch.ops.sgl_kernel.flash_mla_mega_permute_q.default(weight, scale, num_heads) + ) + + +def permute_mega_wo_weight( + weight: torch.Tensor, + scale: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + _require_mega() + return tuple(torch.ops.sgl_kernel.flash_mla_mega_permute_wo.default(weight, scale)) diff --git a/python/sglang/kernels/aot/tests/test_flashmla_mega.py b/python/sglang/kernels/aot/tests/test_flashmla_mega.py new file mode 100644 index 0000000000..002f295af1 --- /dev/null +++ b/python/sglang/kernels/aot/tests/test_flashmla_mega.py @@ -0,0 +1,184 @@ +import sys + +import pytest +import torch +from sgl_kernel.flash_mla_mega import ( + flash_mla_mega_decode, + flash_mla_mega_prefill, + is_flash_mla_mega_available, + permute_mega_q_weight, + permute_mega_wo_weight, +) + +pytestmark = pytest.mark.skipif( + not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10, + reason="FlashMLA MegaAttention requires SM100/SM103", +) + + +def _pack_q(q): + tokens, heads, dim = q.shape + return q.view(tokens, heads, dim // 16, 16).transpose(1, 2).contiguous().view_as(q) + + +def _unpack_output(data, scale): + tokens, groups, hidden = data.shape + sf = scale.contiguous().view(torch.uint8).float() + values = data.float().view(tokens, groups, hidden // 32, 32) + values = values * torch.exp2(sf - 127).unsqueeze(-1) + return ( + values.view(tokens, groups, 16, 8, 32) + .transpose(2, 3) + .reshape(tokens, groups * 8, 512) + ) + + +def _rope(x, positions, table, inverse=False): + out = x.float().clone() + cos, sin = table[positions.long()].chunk(2, dim=-1) + cos, sin = cos[:, None], sin[:, None] + if inverse: + sin = -sin + even, odd = out[..., -64::2].clone(), out[..., -63::2].clone() + out[..., -64::2] = even * cos - odd * sin + out[..., -63::2] = even * sin + odd * cos + return out + + +def _reference(q, kv, indices, lens, positions, table, sink): + q = _rope(q, positions, table).bfloat16().float() + selected = kv[indices.clamp_min(0), 0].float() + logits = torch.einsum("thd,tkd->thk", q, selected) / 512**0.5 + valid = (indices >= 0) & ( + torch.arange(indices.shape[-1], device=q.device)[None] < lens[:, None] + ) + logits.masked_fill_(~valid[:, None], -torch.inf) + logits = torch.cat((logits, sink[None, :, None].expand(q.shape[0], -1, -1)), dim=-1) + probabilities = logits.softmax(-1)[..., :-1] + out = torch.einsum("thk,tkd->thd", probabilities, selected) + return _rope(out, positions, table, inverse=True) + + +def _inputs(heads): + torch.manual_seed(123) + device = "cuda" + q = torch.randn(3, heads, 512, device=device, dtype=torch.bfloat16) + kv = torch.randn(128, 1, 512, device=device, dtype=torch.bfloat16) + indices = torch.arange(128, device=device, dtype=torch.int32).repeat(3, 1) + indices[1, 48:64] = -1 + lens = torch.tensor([128, 96, 64], device=device, dtype=torch.int32) + positions = torch.tensor([0, 17, 255], device=device, dtype=torch.int32) + angles = ( + torch.arange(256, device=device)[:, None] + / torch.arange(1, 33, device=device)[None] + ) + table = torch.cat((angles.cos(), angles.sin()), dim=-1).float() + sink = torch.randn(heads, device=device, dtype=torch.float32) + return q, kv, indices, lens, positions, table, sink + + +@pytest.mark.parametrize("heads", [64, 128]) +def test_mega_prefill_matches_unfused_reference(heads): + assert is_flash_mla_mega_available(), "The SM100 wheel must expose MegaAttention" + q, kv, indices, lens, positions, table, sink = _inputs(heads) + data, scale, _, _ = flash_mla_mega_prefill( + _pack_q(q), + kv, + indices[:, None], + positions, + table, + 512**-0.5, + attn_sink=sink, + topk_length=lens, + ) + expected = _reference(q, kv, indices, lens, positions, table, sink) + actual = _unpack_output(data, scale) + relative_rmse = ( + actual - expected + ).square().mean().sqrt() / expected.square().mean().sqrt() + assert relative_rmse < 0.04 + assert scale.dtype == torch.int32 and scale.stride(0) == 1 + assert scale.stride(-1) % 4 == 0 + + +def test_mega_decode_matches_gathered_prefill(): + q, kv, indices, lens, positions, table, sink = _inputs(64) + groups = kv.float().view(128, 16, 32) + sf = torch.exp2(torch.ceil(torch.log2(groups.abs().amax(-1).clamp_min(1e-4) / 448))) + codes = (groups / sf[..., None]).to(torch.float8_e4m3fn) + dequant = (codes.float() * sf[..., None]).view_as(kv).bfloat16() + raw = torch.empty(128 * 528, device="cuda", dtype=torch.uint8) + raw[: 128 * 512] = codes.view(torch.uint8).reshape(-1) + raw[128 * 512 :] = (sf.log2() + 127).to(torch.uint8).reshape(-1) + cache = raw.view(1, 128, 1, 528) + packed_q = _pack_q(q) + decoded = flash_mla_mega_decode( + packed_q, + cache, + indices, + positions, + table, + 512**-0.5, + attn_sink=sink, + topk_length=lens, + ) + prefilled = flash_mla_mega_prefill( + packed_q, + dequant, + indices[:, None], + positions, + table, + 512**-0.5, + attn_sink=sink, + topk_length=lens, + ) + torch.testing.assert_close( + _unpack_output(*decoded[:2]), + _unpack_output(*prefilled[:2]), + rtol=0.05, + atol=0.01, + ) + + +def test_mega_weight_permutations_preserve_scale_association(): + heads, rows, width = 64, 64, 128 + q_weight = ( + torch.randint(0, 112, (heads * 512 * width,), device="cuda") + .to(torch.uint8) + .view(torch.float8_e4m3fn) + .view(heads * 512, width) + ) + q_scale = torch.arange(heads * 512, device="cuda", dtype=torch.int32)[:, None] + pq, ps = permute_mega_q_weight(q_weight, q_scale, heads) + q_perm = ( + torch.arange(heads * 512, device="cuda") + .view(heads, 32, 16) + .transpose(0, 1) + .reshape(-1) + ) + torch.testing.assert_close(pq.view(torch.uint8), q_weight.view(torch.uint8)[q_perm]) + torch.testing.assert_close(ps, q_scale[q_perm]) + + wo = ( + torch.randint(0, 112, (rows * 4096,), device="cuda") + .to(torch.uint8) + .view(torch.float8_e4m3fn) + .view(1, rows, 4096) + ) + sf_bytes = ( + torch.arange(128, device="cuda", dtype=torch.uint8)[None, None] + .expand(1, rows, -1) + .contiguous() + ) + sf = sf_bytes.view(torch.int32).transpose(1, 2).contiguous().transpose(1, 2) + pw, pws = permute_mega_wo_weight(wo, sf) + wo_perm = ( + torch.arange(4096, device="cuda").view(8, 16, 32).transpose(0, 1).reshape(-1) + ) + torch.testing.assert_close(pw.view(torch.uint8), wo.view(torch.uint8)[..., wo_perm]) + expected_sf = sf_bytes[..., wo_perm[::32] // 32] + torch.testing.assert_close(pws.contiguous().view(torch.uint8), expected_sf) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__]))