[Perf] Optimize MiMo local audio attention on Hopper (#42101)
This commit is contained in:
@@ -0,0 +1,115 @@
|
||||
"""Attention for MiMo's independent groups of at most four audio tokens."""
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _mimo_local_attention_kernel(
|
||||
Q,
|
||||
K,
|
||||
V,
|
||||
O,
|
||||
num_batch_heads,
|
||||
NUM_HEADS: tl.constexpr,
|
||||
NUM_TOKENS: tl.constexpr,
|
||||
stride_qb: tl.constexpr,
|
||||
stride_qh: tl.constexpr,
|
||||
stride_qt: tl.constexpr,
|
||||
stride_kb: tl.constexpr,
|
||||
stride_kh: tl.constexpr,
|
||||
stride_kt: tl.constexpr,
|
||||
stride_vb: tl.constexpr,
|
||||
stride_vh: tl.constexpr,
|
||||
stride_vt: tl.constexpr,
|
||||
SCALE: tl.constexpr,
|
||||
CAUSAL: tl.constexpr,
|
||||
):
|
||||
bh = tl.program_id(0).to(tl.int64) * 4 + tl.arange(0, 4)
|
||||
batch, head = bh // NUM_HEADS, bh % NUM_HEADS
|
||||
token, dim = tl.arange(0, 4), tl.arange(0, 16)
|
||||
valid = (bh[:, None, None] < num_batch_heads) & (token[None, :, None] < NUM_TOKENS)
|
||||
q = tl.load(
|
||||
Q
|
||||
+ batch[:, None, None] * stride_qb
|
||||
+ head[:, None, None] * stride_qh
|
||||
+ token[None, :, None] * stride_qt
|
||||
+ dim[None, None, :],
|
||||
valid,
|
||||
other=0,
|
||||
).to(tl.float32)
|
||||
k = tl.load(
|
||||
K
|
||||
+ batch[:, None, None] * stride_kb
|
||||
+ head[:, None, None] * stride_kh
|
||||
+ token[None, :, None] * stride_kt
|
||||
+ dim[None, None, :],
|
||||
valid,
|
||||
other=0,
|
||||
).to(tl.float32)
|
||||
v = tl.load(
|
||||
V
|
||||
+ batch[:, None, None] * stride_vb
|
||||
+ head[:, None, None] * stride_vh
|
||||
+ token[None, :, None] * stride_vt
|
||||
+ dim[None, None, :],
|
||||
valid,
|
||||
other=0,
|
||||
).to(tl.float32)
|
||||
scores = tl.sum(q[:, :, None, :] * k[:, None, :, :], axis=3) * SCALE
|
||||
allowed = token[None, None, :] < NUM_TOKENS
|
||||
if CAUSAL:
|
||||
allowed = allowed & (token[None, None, :] <= token[None, :, None])
|
||||
scores = tl.where(allowed, scores, -float("inf"))
|
||||
weights = tl.exp(scores - tl.max(scores, axis=2)[:, :, None])
|
||||
denominator = tl.sum(weights, axis=2)
|
||||
# Match the tested cuDNN path: round the unnormalized exponential weights,
|
||||
# then normalize the weighted sum. Rounding normalized weights differs.
|
||||
weights = weights.to(Q.dtype.element_ty).to(tl.float32)
|
||||
result = tl.sum(weights[:, :, :, None] * v[:, None, :, :], axis=2)
|
||||
result /= denominator[:, :, None]
|
||||
offsets = (
|
||||
(batch[:, None, None] * NUM_TOKENS + token[None, :, None]) * NUM_HEADS
|
||||
+ head[:, None, None]
|
||||
) * 16 + dim[None, None, :]
|
||||
tl.store(O + offsets, result, valid)
|
||||
|
||||
|
||||
def mimo_local_attention(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
scale: float = 0.25,
|
||||
is_causal: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""Attend within audio groups of 1–4 tokens with head dimension 16.
|
||||
|
||||
Q/K/V have shape [batch, head, token, dim]; output is contiguous
|
||||
[batch, token, head, dim]. Only the optional causal mask is supported.
|
||||
"""
|
||||
assert q.is_cuda and q.device == k.device == v.device
|
||||
assert q.dtype == k.dtype == v.dtype and q.dtype in (torch.bfloat16, torch.float16)
|
||||
assert q.shape == k.shape == v.shape and q.ndim == 4
|
||||
batch, heads, tokens, dim = q.shape
|
||||
assert 0 < tokens <= 4 and dim == 16
|
||||
assert q.stride(-1) == k.stride(-1) == v.stride(-1) == 1
|
||||
out = torch.empty((batch, tokens, heads, dim), device=q.device, dtype=q.dtype)
|
||||
if batch * heads:
|
||||
_mimo_local_attention_kernel[(triton.cdiv(batch * heads, 4),)](
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
out,
|
||||
batch * heads,
|
||||
heads,
|
||||
tokens,
|
||||
*q.stride()[:3],
|
||||
*k.stride()[:3],
|
||||
*v.stride()[:3],
|
||||
scale,
|
||||
is_causal,
|
||||
num_warps=1,
|
||||
enable_fp_fusion=False,
|
||||
)
|
||||
return out
|
||||
@@ -14,8 +14,13 @@ import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from einops import rearrange
|
||||
from transformers import AttentionInterface
|
||||
from transformers.activations import ACT2FN
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
from transformers.masking_utils import (
|
||||
ALL_MASK_ATTENTION_FUNCTIONS,
|
||||
AttentionMaskInterface,
|
||||
)
|
||||
from transformers.modeling_utils import PreTrainedModel
|
||||
from transformers.models.qwen2.configuration_qwen2 import Qwen2Config
|
||||
from transformers.models.qwen2.modeling_qwen2 import Qwen2Model
|
||||
@@ -26,10 +31,58 @@ from sglang.srt.runtime_context import (
|
||||
get_model,
|
||||
get_parallel,
|
||||
)
|
||||
from sglang.srt.utils import is_sm90_supported
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _mimo_local_attention_forward(
|
||||
module,
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
attention_mask,
|
||||
dropout=0.0,
|
||||
scaling=None,
|
||||
is_causal=None,
|
||||
position_bias=None,
|
||||
**kwargs,
|
||||
):
|
||||
from transformers.integrations.sdpa_attention import sdpa_attention_forward
|
||||
|
||||
if (
|
||||
query.dtype in (torch.float16, torch.bfloat16)
|
||||
and attention_mask is None
|
||||
and position_bias is None
|
||||
and dropout == 0.0
|
||||
and not kwargs.get("output_attentions", False)
|
||||
):
|
||||
from sglang.kernels.ops.attention.mimo_local_attention import (
|
||||
mimo_local_attention,
|
||||
)
|
||||
|
||||
causal = is_causal if is_causal is not None else module.is_causal
|
||||
return mimo_local_attention(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
scale=scaling if scaling is not None else 0.25,
|
||||
is_causal=bool(causal),
|
||||
), None
|
||||
return sdpa_attention_forward(
|
||||
module,
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
attention_mask,
|
||||
dropout=dropout,
|
||||
scaling=scaling,
|
||||
is_causal=is_causal,
|
||||
position_bias=position_bias,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def _compute_default_rope_parameters(
|
||||
config=None, device=None, seq_len=None, **rope_kwargs
|
||||
):
|
||||
@@ -1233,6 +1286,19 @@ class AudioEncoderMixin:
|
||||
partial_rotary_factor=config.partial_rotary_factor,
|
||||
)
|
||||
input_local_config.head_dim = config.input_local_head_dim
|
||||
input_local_config._attn_implementation = "sdpa"
|
||||
if (
|
||||
is_sm90_supported()
|
||||
and input_local_config.head_dim == 16
|
||||
and 0 < self.audio_group_size <= 4
|
||||
):
|
||||
attention_name = "sglang_mimo_local"
|
||||
AttentionInterface.register(attention_name, _mimo_local_attention_forward)
|
||||
# Reuse SDPA's mask preparation for the custom attention implementation.
|
||||
AttentionMaskInterface.register(
|
||||
attention_name, ALL_MASK_ATTENTION_FUNCTIONS["sdpa"]
|
||||
)
|
||||
input_local_config._attn_implementation = attention_name
|
||||
self.input_local_transformer = Qwen2Model(input_local_config)
|
||||
if not config.add_post_norm:
|
||||
self.input_local_transformer.norm = nn.Identity()
|
||||
|
||||
Reference in New Issue
Block a user