[Kernel] Expose bundled FlashMLA MegaAttention kernels and APIs (#43918)
This commit is contained in:
@@ -0,0 +1,23 @@
|
||||
import torch
|
||||
from sgl_kernel.flash_mla_mega import flash_mla_mega_prefill
|
||||
|
||||
from sglang.kernels.jit.benchmark import marker
|
||||
|
||||
|
||||
@marker.parametrize("tokens", [1, 32, 128, 512], [128])
|
||||
@marker.parametrize("heads", [64, 128], [64])
|
||||
@marker.benchmark("provider", ["flashmla_mega"])
|
||||
def benchmark(tokens: int, heads: int, provider: str):
|
||||
q = torch.randn(tokens, heads, 512, device="cuda", dtype=torch.bfloat16)
|
||||
kv = torch.randn(4096, 1, 512, device="cuda", dtype=torch.bfloat16)
|
||||
indices = torch.randint(4096, (tokens, 1, 640), device="cuda", dtype=torch.int32)
|
||||
positions = torch.zeros(tokens, device="cuda", dtype=torch.int32)
|
||||
table = torch.cat((torch.ones(1, 32), torch.zeros(1, 32)), dim=-1).cuda()
|
||||
return marker.do_bench(
|
||||
flash_mla_mega_prefill,
|
||||
input_args=(q, kv, indices, positions, table, 512**-0.5),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
benchmark.run()
|
||||
@@ -127,6 +127,7 @@ set(FlashMLA_SOURCES
|
||||
|
||||
if(FLASHMLA_ENABLE_SM100)
|
||||
list(APPEND FlashMLA_SOURCES
|
||||
"csrc/flashmla_mega_extension.cc"
|
||||
# sm100 dense prefill/bwd.
|
||||
${repo-flashmla_SOURCE_DIR}/csrc/kernels/sm100/prefill/dense/fmha_cutlass_fwd_sm100.cu
|
||||
${repo-flashmla_SOURCE_DIR}/csrc/kernels/sm100/prefill/dense/fmha_cutlass_bwd_sm100.cu
|
||||
@@ -156,6 +157,21 @@ if(FLASHMLA_ENABLE_SM100)
|
||||
${repo-flashmla_SOURCE_DIR}/csrc/kernels/sm100/prefill/sparse/fwd_for_small_topk/head128/instantiations/phase1_decode_k512_v41fp4.cu
|
||||
${repo-flashmla_SOURCE_DIR}/csrc/kernels/sm100/prefill/sparse/fwd_for_small_topk/head128/instantiations/phase1_decode_k512_v41fp4_splitkv.cu
|
||||
)
|
||||
set(FLASHMLA_MEGA_ROOT ${repo-flashmla_SOURCE_DIR}/csrc/kernels/sm100/prefill/sparse/fused_norm_rope_attn_rope_cast_fwd)
|
||||
foreach(HEADS 64 128)
|
||||
foreach(NORM norm nonorm)
|
||||
list(APPEND FlashMLA_SOURCES
|
||||
${FLASHMLA_MEGA_ROOT}/core_attn/instantiations/v4_h${HEADS}_prefill_${NORM}.cu
|
||||
${FLASHMLA_MEGA_ROOT}/core_attn/instantiations/v4_h${HEADS}_decode_${NORM}.cu
|
||||
${FLASHMLA_MEGA_ROOT}/core_attn/instantiations/v41_h${HEADS}_decode_${NORM}.cu
|
||||
${FLASHMLA_MEGA_ROOT}/core_attn/instantiations/v41fp4_h${HEADS}_decode_${NORM}.cu
|
||||
)
|
||||
endforeach()
|
||||
endforeach()
|
||||
list(APPEND FlashMLA_SOURCES
|
||||
${FLASHMLA_MEGA_ROOT}/permute_q_b_proj/kernel.cu
|
||||
${FLASHMLA_MEGA_ROOT}/permute_wv_proj/kernel.cu
|
||||
)
|
||||
endif()
|
||||
|
||||
Python_add_library(flashmla_ops MODULE USE_SABI ${SKBUILD_SABI_VERSION} WITH_SOABI ${FlashMLA_SOURCES})
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
#include <torch/all.h>
|
||||
#include <torch/library.h>
|
||||
|
||||
#include "api/fused_norm_rope_attn_rope_cast_fwd.cpp"
|
||||
|
||||
static std::vector<at::Tensor> sgl_flash_mla_mega_prefill(
|
||||
const at::Tensor& q,
|
||||
const at::Tensor& kv,
|
||||
const at::Tensor& indices,
|
||||
const at::Tensor& positions,
|
||||
const at::Tensor& cos_sin_cache,
|
||||
double sm_scale,
|
||||
int64_t n_wv_group,
|
||||
const std::optional<at::Tensor>& attn_sink,
|
||||
const std::optional<at::Tensor>& topk_length) {
|
||||
TORCH_CHECK(n_wv_group > 0 && q.size(1) == n_wv_group * 8, "MegaAttention requires eight heads per WO-A group");
|
||||
return fused_norm_rope_attn_rope_cast_fwd(
|
||||
q,
|
||||
kv,
|
||||
indices,
|
||||
static_cast<float>(sm_scale),
|
||||
512,
|
||||
attn_sink,
|
||||
topk_length,
|
||||
false,
|
||||
0.0f,
|
||||
positions,
|
||||
false,
|
||||
64,
|
||||
cos_sin_cache,
|
||||
static_cast<uint32_t>(n_wv_group),
|
||||
32,
|
||||
true,
|
||||
true,
|
||||
true);
|
||||
}
|
||||
|
||||
static std::vector<at::Tensor> sgl_flash_mla_mega_decode(
|
||||
const at::Tensor& q,
|
||||
const at::Tensor& kv,
|
||||
const at::Tensor& indices,
|
||||
const at::Tensor& positions,
|
||||
const at::Tensor& cos_sin_cache,
|
||||
double sm_scale,
|
||||
int64_t n_wv_group,
|
||||
const std::optional<at::Tensor>& attn_sink,
|
||||
const std::optional<at::Tensor>& topk_length,
|
||||
const std::optional<at::Tensor>& extra_kv,
|
||||
const std::optional<at::Tensor>& extra_indices,
|
||||
const std::optional<at::Tensor>& extra_topk_length) {
|
||||
TORCH_CHECK(n_wv_group > 0 && q.size(1) == n_wv_group * 8, "MegaAttention requires eight heads per WO-A group");
|
||||
return fused_norm_rope_attn_rope_cast_decode(
|
||||
q,
|
||||
kv,
|
||||
indices,
|
||||
static_cast<float>(sm_scale),
|
||||
512,
|
||||
attn_sink,
|
||||
topk_length,
|
||||
extra_kv,
|
||||
extra_indices,
|
||||
extra_topk_length,
|
||||
false,
|
||||
0.0f,
|
||||
positions,
|
||||
false,
|
||||
64,
|
||||
cos_sin_cache,
|
||||
static_cast<uint32_t>(n_wv_group),
|
||||
32,
|
||||
true,
|
||||
true,
|
||||
true);
|
||||
}
|
||||
|
||||
static std::vector<at::Tensor>
|
||||
sgl_flash_mla_mega_permute_q(const at::Tensor& weight, const at::Tensor& scale, int64_t heads) {
|
||||
TORCH_CHECK(heads == 64 || heads == 128, "MegaAttention requires 64 or 128 query heads");
|
||||
return permute_q_b_proj(weight, scale, static_cast<int>(heads), 512);
|
||||
}
|
||||
|
||||
static std::vector<at::Tensor> sgl_flash_mla_mega_permute_wo(const at::Tensor& weight, const at::Tensor& scale) {
|
||||
return permute_wv_proj(weight, scale, 8, 512);
|
||||
}
|
||||
|
||||
TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) {
|
||||
m.def(
|
||||
"flash_mla_mega_prefill(Tensor q, Tensor kv, Tensor indices, Tensor positions, Tensor cos_sin_cache, "
|
||||
"float sm_scale, int n_wv_group, Tensor? attn_sink=None, Tensor? topk_length=None) -> Tensor[]");
|
||||
m.impl("flash_mla_mega_prefill", torch::kCUDA, &sgl_flash_mla_mega_prefill);
|
||||
m.def(
|
||||
"flash_mla_mega_decode(Tensor q, Tensor kv, Tensor indices, Tensor positions, Tensor cos_sin_cache, "
|
||||
"float sm_scale, int n_wv_group, Tensor? attn_sink=None, Tensor? topk_length=None, "
|
||||
"Tensor? extra_kv=None, Tensor? extra_indices=None, Tensor? extra_topk_length=None) -> Tensor[]");
|
||||
m.impl("flash_mla_mega_decode", torch::kCUDA, &sgl_flash_mla_mega_decode);
|
||||
m.def("flash_mla_mega_permute_q(Tensor weight, Tensor scale, int heads) -> Tensor[]");
|
||||
m.impl("flash_mla_mega_permute_q", torch::kCUDA, &sgl_flash_mla_mega_permute_q);
|
||||
m.def("flash_mla_mega_permute_wo(Tensor weight, Tensor scale) -> Tensor[]");
|
||||
m.impl("flash_mla_mega_permute_wo", torch::kCUDA, &sgl_flash_mla_mega_permute_wo);
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
from sgl_kernel.flash_mla import _flashmla_import_error
|
||||
|
||||
|
||||
def is_flash_mla_mega_available() -> bool:
|
||||
return _flashmla_import_error is None and hasattr(
|
||||
torch.ops.sgl_kernel, "flash_mla_mega_decode"
|
||||
)
|
||||
|
||||
|
||||
def _require_mega() -> None:
|
||||
if not is_flash_mla_mega_available():
|
||||
raise ImportError(
|
||||
"FlashMLA MegaAttention requires sgl-kernel built with CUDA >= 12.9"
|
||||
)
|
||||
|
||||
|
||||
def flash_mla_mega_prefill(
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
softmax_scale: float,
|
||||
*,
|
||||
attn_sink: Optional[torch.Tensor] = None,
|
||||
topk_length: Optional[torch.Tensor] = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
_require_mega()
|
||||
return tuple(
|
||||
torch.ops.sgl_kernel.flash_mla_mega_prefill.default(
|
||||
q,
|
||||
kv,
|
||||
indices,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
softmax_scale,
|
||||
q.shape[1] // 8,
|
||||
attn_sink,
|
||||
topk_length,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def flash_mla_mega_decode(
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
softmax_scale: float,
|
||||
*,
|
||||
attn_sink: Optional[torch.Tensor] = None,
|
||||
topk_length: Optional[torch.Tensor] = None,
|
||||
extra_kv: Optional[torch.Tensor] = None,
|
||||
extra_indices: Optional[torch.Tensor] = None,
|
||||
extra_topk_length: Optional[torch.Tensor] = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
_require_mega()
|
||||
return tuple(
|
||||
torch.ops.sgl_kernel.flash_mla_mega_decode.default(
|
||||
q,
|
||||
kv,
|
||||
indices,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
softmax_scale,
|
||||
q.shape[1] // 8,
|
||||
attn_sink,
|
||||
topk_length,
|
||||
extra_kv,
|
||||
extra_indices,
|
||||
extra_topk_length,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def permute_mega_q_weight(
|
||||
weight: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
num_heads: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
_require_mega()
|
||||
return tuple(
|
||||
torch.ops.sgl_kernel.flash_mla_mega_permute_q.default(weight, scale, num_heads)
|
||||
)
|
||||
|
||||
|
||||
def permute_mega_wo_weight(
|
||||
weight: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
_require_mega()
|
||||
return tuple(torch.ops.sgl_kernel.flash_mla_mega_permute_wo.default(weight, scale))
|
||||
@@ -0,0 +1,184 @@
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from sgl_kernel.flash_mla_mega import (
|
||||
flash_mla_mega_decode,
|
||||
flash_mla_mega_prefill,
|
||||
is_flash_mla_mega_available,
|
||||
permute_mega_q_weight,
|
||||
permute_mega_wo_weight,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10,
|
||||
reason="FlashMLA MegaAttention requires SM100/SM103",
|
||||
)
|
||||
|
||||
|
||||
def _pack_q(q):
|
||||
tokens, heads, dim = q.shape
|
||||
return q.view(tokens, heads, dim // 16, 16).transpose(1, 2).contiguous().view_as(q)
|
||||
|
||||
|
||||
def _unpack_output(data, scale):
|
||||
tokens, groups, hidden = data.shape
|
||||
sf = scale.contiguous().view(torch.uint8).float()
|
||||
values = data.float().view(tokens, groups, hidden // 32, 32)
|
||||
values = values * torch.exp2(sf - 127).unsqueeze(-1)
|
||||
return (
|
||||
values.view(tokens, groups, 16, 8, 32)
|
||||
.transpose(2, 3)
|
||||
.reshape(tokens, groups * 8, 512)
|
||||
)
|
||||
|
||||
|
||||
def _rope(x, positions, table, inverse=False):
|
||||
out = x.float().clone()
|
||||
cos, sin = table[positions.long()].chunk(2, dim=-1)
|
||||
cos, sin = cos[:, None], sin[:, None]
|
||||
if inverse:
|
||||
sin = -sin
|
||||
even, odd = out[..., -64::2].clone(), out[..., -63::2].clone()
|
||||
out[..., -64::2] = even * cos - odd * sin
|
||||
out[..., -63::2] = even * sin + odd * cos
|
||||
return out
|
||||
|
||||
|
||||
def _reference(q, kv, indices, lens, positions, table, sink):
|
||||
q = _rope(q, positions, table).bfloat16().float()
|
||||
selected = kv[indices.clamp_min(0), 0].float()
|
||||
logits = torch.einsum("thd,tkd->thk", q, selected) / 512**0.5
|
||||
valid = (indices >= 0) & (
|
||||
torch.arange(indices.shape[-1], device=q.device)[None] < lens[:, None]
|
||||
)
|
||||
logits.masked_fill_(~valid[:, None], -torch.inf)
|
||||
logits = torch.cat((logits, sink[None, :, None].expand(q.shape[0], -1, -1)), dim=-1)
|
||||
probabilities = logits.softmax(-1)[..., :-1]
|
||||
out = torch.einsum("thk,tkd->thd", probabilities, selected)
|
||||
return _rope(out, positions, table, inverse=True)
|
||||
|
||||
|
||||
def _inputs(heads):
|
||||
torch.manual_seed(123)
|
||||
device = "cuda"
|
||||
q = torch.randn(3, heads, 512, device=device, dtype=torch.bfloat16)
|
||||
kv = torch.randn(128, 1, 512, device=device, dtype=torch.bfloat16)
|
||||
indices = torch.arange(128, device=device, dtype=torch.int32).repeat(3, 1)
|
||||
indices[1, 48:64] = -1
|
||||
lens = torch.tensor([128, 96, 64], device=device, dtype=torch.int32)
|
||||
positions = torch.tensor([0, 17, 255], device=device, dtype=torch.int32)
|
||||
angles = (
|
||||
torch.arange(256, device=device)[:, None]
|
||||
/ torch.arange(1, 33, device=device)[None]
|
||||
)
|
||||
table = torch.cat((angles.cos(), angles.sin()), dim=-1).float()
|
||||
sink = torch.randn(heads, device=device, dtype=torch.float32)
|
||||
return q, kv, indices, lens, positions, table, sink
|
||||
|
||||
|
||||
@pytest.mark.parametrize("heads", [64, 128])
|
||||
def test_mega_prefill_matches_unfused_reference(heads):
|
||||
assert is_flash_mla_mega_available(), "The SM100 wheel must expose MegaAttention"
|
||||
q, kv, indices, lens, positions, table, sink = _inputs(heads)
|
||||
data, scale, _, _ = flash_mla_mega_prefill(
|
||||
_pack_q(q),
|
||||
kv,
|
||||
indices[:, None],
|
||||
positions,
|
||||
table,
|
||||
512**-0.5,
|
||||
attn_sink=sink,
|
||||
topk_length=lens,
|
||||
)
|
||||
expected = _reference(q, kv, indices, lens, positions, table, sink)
|
||||
actual = _unpack_output(data, scale)
|
||||
relative_rmse = (
|
||||
actual - expected
|
||||
).square().mean().sqrt() / expected.square().mean().sqrt()
|
||||
assert relative_rmse < 0.04
|
||||
assert scale.dtype == torch.int32 and scale.stride(0) == 1
|
||||
assert scale.stride(-1) % 4 == 0
|
||||
|
||||
|
||||
def test_mega_decode_matches_gathered_prefill():
|
||||
q, kv, indices, lens, positions, table, sink = _inputs(64)
|
||||
groups = kv.float().view(128, 16, 32)
|
||||
sf = torch.exp2(torch.ceil(torch.log2(groups.abs().amax(-1).clamp_min(1e-4) / 448)))
|
||||
codes = (groups / sf[..., None]).to(torch.float8_e4m3fn)
|
||||
dequant = (codes.float() * sf[..., None]).view_as(kv).bfloat16()
|
||||
raw = torch.empty(128 * 528, device="cuda", dtype=torch.uint8)
|
||||
raw[: 128 * 512] = codes.view(torch.uint8).reshape(-1)
|
||||
raw[128 * 512 :] = (sf.log2() + 127).to(torch.uint8).reshape(-1)
|
||||
cache = raw.view(1, 128, 1, 528)
|
||||
packed_q = _pack_q(q)
|
||||
decoded = flash_mla_mega_decode(
|
||||
packed_q,
|
||||
cache,
|
||||
indices,
|
||||
positions,
|
||||
table,
|
||||
512**-0.5,
|
||||
attn_sink=sink,
|
||||
topk_length=lens,
|
||||
)
|
||||
prefilled = flash_mla_mega_prefill(
|
||||
packed_q,
|
||||
dequant,
|
||||
indices[:, None],
|
||||
positions,
|
||||
table,
|
||||
512**-0.5,
|
||||
attn_sink=sink,
|
||||
topk_length=lens,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
_unpack_output(*decoded[:2]),
|
||||
_unpack_output(*prefilled[:2]),
|
||||
rtol=0.05,
|
||||
atol=0.01,
|
||||
)
|
||||
|
||||
|
||||
def test_mega_weight_permutations_preserve_scale_association():
|
||||
heads, rows, width = 64, 64, 128
|
||||
q_weight = (
|
||||
torch.randint(0, 112, (heads * 512 * width,), device="cuda")
|
||||
.to(torch.uint8)
|
||||
.view(torch.float8_e4m3fn)
|
||||
.view(heads * 512, width)
|
||||
)
|
||||
q_scale = torch.arange(heads * 512, device="cuda", dtype=torch.int32)[:, None]
|
||||
pq, ps = permute_mega_q_weight(q_weight, q_scale, heads)
|
||||
q_perm = (
|
||||
torch.arange(heads * 512, device="cuda")
|
||||
.view(heads, 32, 16)
|
||||
.transpose(0, 1)
|
||||
.reshape(-1)
|
||||
)
|
||||
torch.testing.assert_close(pq.view(torch.uint8), q_weight.view(torch.uint8)[q_perm])
|
||||
torch.testing.assert_close(ps, q_scale[q_perm])
|
||||
|
||||
wo = (
|
||||
torch.randint(0, 112, (rows * 4096,), device="cuda")
|
||||
.to(torch.uint8)
|
||||
.view(torch.float8_e4m3fn)
|
||||
.view(1, rows, 4096)
|
||||
)
|
||||
sf_bytes = (
|
||||
torch.arange(128, device="cuda", dtype=torch.uint8)[None, None]
|
||||
.expand(1, rows, -1)
|
||||
.contiguous()
|
||||
)
|
||||
sf = sf_bytes.view(torch.int32).transpose(1, 2).contiguous().transpose(1, 2)
|
||||
pw, pws = permute_mega_wo_weight(wo, sf)
|
||||
wo_perm = (
|
||||
torch.arange(4096, device="cuda").view(8, 16, 32).transpose(0, 1).reshape(-1)
|
||||
)
|
||||
torch.testing.assert_close(pw.view(torch.uint8), wo.view(torch.uint8)[..., wo_perm])
|
||||
expected_sf = sf_bytes[..., wo_perm[::32] // 32]
|
||||
torch.testing.assert_close(pws.contiguous().view(torch.uint8), expected_sf)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__]))
|
||||
Reference in New Issue
Block a user