diff --git a/python/sglang/kernels/ops/attention/mimo_local_attention.py b/python/sglang/kernels/ops/attention/mimo_local_attention.py new file mode 100644 index 0000000000..04b89b9466 --- /dev/null +++ b/python/sglang/kernels/ops/attention/mimo_local_attention.py @@ -0,0 +1,115 @@ +"""Attention for MiMo's independent groups of at most four audio tokens.""" + +import torch +import triton +import triton.language as tl + + +@triton.jit +def _mimo_local_attention_kernel( + Q, + K, + V, + O, + num_batch_heads, + NUM_HEADS: tl.constexpr, + NUM_TOKENS: tl.constexpr, + stride_qb: tl.constexpr, + stride_qh: tl.constexpr, + stride_qt: tl.constexpr, + stride_kb: tl.constexpr, + stride_kh: tl.constexpr, + stride_kt: tl.constexpr, + stride_vb: tl.constexpr, + stride_vh: tl.constexpr, + stride_vt: tl.constexpr, + SCALE: tl.constexpr, + CAUSAL: tl.constexpr, +): + bh = tl.program_id(0).to(tl.int64) * 4 + tl.arange(0, 4) + batch, head = bh // NUM_HEADS, bh % NUM_HEADS + token, dim = tl.arange(0, 4), tl.arange(0, 16) + valid = (bh[:, None, None] < num_batch_heads) & (token[None, :, None] < NUM_TOKENS) + q = tl.load( + Q + + batch[:, None, None] * stride_qb + + head[:, None, None] * stride_qh + + token[None, :, None] * stride_qt + + dim[None, None, :], + valid, + other=0, + ).to(tl.float32) + k = tl.load( + K + + batch[:, None, None] * stride_kb + + head[:, None, None] * stride_kh + + token[None, :, None] * stride_kt + + dim[None, None, :], + valid, + other=0, + ).to(tl.float32) + v = tl.load( + V + + batch[:, None, None] * stride_vb + + head[:, None, None] * stride_vh + + token[None, :, None] * stride_vt + + dim[None, None, :], + valid, + other=0, + ).to(tl.float32) + scores = tl.sum(q[:, :, None, :] * k[:, None, :, :], axis=3) * SCALE + allowed = token[None, None, :] < NUM_TOKENS + if CAUSAL: + allowed = allowed & (token[None, None, :] <= token[None, :, None]) + scores = tl.where(allowed, scores, -float("inf")) + weights = tl.exp(scores - tl.max(scores, axis=2)[:, :, None]) + denominator = tl.sum(weights, axis=2) + # Match the tested cuDNN path: round the unnormalized exponential weights, + # then normalize the weighted sum. Rounding normalized weights differs. + weights = weights.to(Q.dtype.element_ty).to(tl.float32) + result = tl.sum(weights[:, :, :, None] * v[:, None, :, :], axis=2) + result /= denominator[:, :, None] + offsets = ( + (batch[:, None, None] * NUM_TOKENS + token[None, :, None]) * NUM_HEADS + + head[:, None, None] + ) * 16 + dim[None, None, :] + tl.store(O + offsets, result, valid) + + +def mimo_local_attention( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + scale: float = 0.25, + is_causal: bool = False, +) -> torch.Tensor: + """Attend within audio groups of 1–4 tokens with head dimension 16. + + Q/K/V have shape [batch, head, token, dim]; output is contiguous + [batch, token, head, dim]. Only the optional causal mask is supported. + """ + assert q.is_cuda and q.device == k.device == v.device + assert q.dtype == k.dtype == v.dtype and q.dtype in (torch.bfloat16, torch.float16) + assert q.shape == k.shape == v.shape and q.ndim == 4 + batch, heads, tokens, dim = q.shape + assert 0 < tokens <= 4 and dim == 16 + assert q.stride(-1) == k.stride(-1) == v.stride(-1) == 1 + out = torch.empty((batch, tokens, heads, dim), device=q.device, dtype=q.dtype) + if batch * heads: + _mimo_local_attention_kernel[(triton.cdiv(batch * heads, 4),)]( + q, + k, + v, + out, + batch * heads, + heads, + tokens, + *q.stride()[:3], + *k.stride()[:3], + *v.stride()[:3], + scale, + is_causal, + num_warps=1, + enable_fp_fusion=False, + ) + return out diff --git a/python/sglang/srt/models/mimo_audio.py b/python/sglang/srt/models/mimo_audio.py index 8c0e9c8734..551330edfd 100644 --- a/python/sglang/srt/models/mimo_audio.py +++ b/python/sglang/srt/models/mimo_audio.py @@ -14,8 +14,13 @@ import torch import torch.nn as nn import torch.nn.functional as F from einops import rearrange +from transformers import AttentionInterface from transformers.activations import ACT2FN from transformers.configuration_utils import PretrainedConfig +from transformers.masking_utils import ( + ALL_MASK_ATTENTION_FUNCTIONS, + AttentionMaskInterface, +) from transformers.modeling_utils import PreTrainedModel from transformers.models.qwen2.configuration_qwen2 import Qwen2Config from transformers.models.qwen2.modeling_qwen2 import Qwen2Model @@ -26,10 +31,58 @@ from sglang.srt.runtime_context import ( get_model, get_parallel, ) +from sglang.srt.utils import is_sm90_supported logger = logging.getLogger(__name__) +def _mimo_local_attention_forward( + module, + query, + key, + value, + attention_mask, + dropout=0.0, + scaling=None, + is_causal=None, + position_bias=None, + **kwargs, +): + from transformers.integrations.sdpa_attention import sdpa_attention_forward + + if ( + query.dtype in (torch.float16, torch.bfloat16) + and attention_mask is None + and position_bias is None + and dropout == 0.0 + and not kwargs.get("output_attentions", False) + ): + from sglang.kernels.ops.attention.mimo_local_attention import ( + mimo_local_attention, + ) + + causal = is_causal if is_causal is not None else module.is_causal + return mimo_local_attention( + query, + key, + value, + scale=scaling if scaling is not None else 0.25, + is_causal=bool(causal), + ), None + return sdpa_attention_forward( + module, + query, + key, + value, + attention_mask, + dropout=dropout, + scaling=scaling, + is_causal=is_causal, + position_bias=position_bias, + **kwargs, + ) + + def _compute_default_rope_parameters( config=None, device=None, seq_len=None, **rope_kwargs ): @@ -1233,6 +1286,19 @@ class AudioEncoderMixin: partial_rotary_factor=config.partial_rotary_factor, ) input_local_config.head_dim = config.input_local_head_dim + input_local_config._attn_implementation = "sdpa" + if ( + is_sm90_supported() + and input_local_config.head_dim == 16 + and 0 < self.audio_group_size <= 4 + ): + attention_name = "sglang_mimo_local" + AttentionInterface.register(attention_name, _mimo_local_attention_forward) + # Reuse SDPA's mask preparation for the custom attention implementation. + AttentionMaskInterface.register( + attention_name, ALL_MASK_ATTENTION_FUNCTIONS["sdpa"] + ) + input_local_config._attn_implementation = attention_name self.input_local_transformer = Qwen2Model(input_local_config) if not config.add_post_norm: self.input_local_transformer.norm = nn.Identity()