[diffusion] optimization: fuse Anima FP32 split-half Q/K RoPE (#42609)

This commit is contained in:
CuzMi
2026-10-06 10:18:09 +08:00
committed by GitHub
parent 1abd356595
commit 67959f80ab
5 changed files with 248 additions and 6 deletions
@@ -301,6 +301,13 @@ _SPECS: tuple[tuple[str, KernelBackend, str, frozenset, str], ...] = (
_CUDA,
"Paired LTX-2.5 decoder 3D RoPE.",
),
(
"diffusion.rope_rotate_half_fp32",
KernelBackend.TRITON,
"rope.rope_rotate_half_fp32:fused_rope_rotate_half_fp32",
_CUDA,
"Paired split-half RoPE with FP32 arithmetic and one output cast.",
),
(
"diffusion.rope_rotate_half",
KernelBackend.TRITON,
@@ -667,6 +674,8 @@ _EXPORTS: dict[str, str] = {
"fused_inplace_qknorm_rope": "rope.qknorm_rope_jit",
"fused_qknorm_rope_pack_kv": "rope.qknorm_rope_jit",
"try_fused_qwen_qkv_epilogue": "rope.qwen_qkv_epilogue_jit",
"can_use_fused_rope_rotate_half_fp32": "rope.rope_rotate_half_fp32",
"fused_rope_rotate_half_fp32": "rope.rope_rotate_half_fp32",
"fused_rope_rotate_half_bitexact": "rope.rope_rotate_half_bitexact",
"fused_interleaved_rope_fp64": "rope.interleaved_rope_fp64_jit",
"fused_inplace_helios_qk_rope": "rope.helios_qk_rope_jit",
@@ -0,0 +1,92 @@
# SPDX-License-Identifier: Apache-2.0
"""Paired split-half RoPE with FP32 products and sum, then one output cast.
Unlike the BF16-rounded RoPE kernel, this matches Cosmos/Anima eager math.
Cos/sin have shape (S, D) and are shared by every batch and attention head.
"""
import torch
import triton
import triton.language as tl
@triton.jit
def _rope_rotate_half_fp32_kernel(
Q,
K,
Cos,
Sin,
OutQ,
OutK,
N: tl.constexpr,
S: tl.constexpr,
H: tl.constexpr,
D: tl.constexpr,
BLOCK: tl.constexpr,
):
offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
mask = offsets < N
col = offsets % D
rotated = offsets + tl.where(col < D // 2, D // 2, -D // 2)
freq = ((offsets // (H * D)) % S) * D + col
c = tl.load(Cos + freq, mask, other=0)
s = tl.load(Sin + freq, mask, other=0)
if tl.program_id(1) == 0:
x = tl.load(Q + offsets, mask, other=0).to(tl.float32)
r = tl.load(Q + rotated, mask, other=0).to(tl.float32)
else:
x = tl.load(K + offsets, mask, other=0).to(tl.float32)
r = tl.load(K + rotated, mask, other=0).to(tl.float32)
r = tl.where(col < D // 2, -r, r)
# Preserve the two FP32 multiplication boundaries; do not contract to FMA.
out = x * c + r * s
if tl.program_id(1) == 0:
tl.store(OutQ + offsets, out, mask)
else:
tl.store(OutK + offsets, out, mask)
def can_use_fused_rope_rotate_half_fp32(q, k, cos, sin):
"""Conservative inference-only contract; unsupported inputs stay eager."""
return (
q.is_cuda
and torch.version.hip is None
and q.dtype in (torch.float16, torch.bfloat16)
and q.ndim == 4
and all(n > 0 for n in q.shape)
and q.shape[-1] == 128
and k.shape == q.shape
and k.dtype == q.dtype
and cos.shape == sin.shape == (q.shape[1], q.shape[-1])
and cos.dtype == sin.dtype == torch.float32
and all(t.device == q.device and t.is_contiguous() for t in (q, k, cos, sin))
and not (
torch.is_grad_enabled() and any(t.requires_grad for t in (q, k, cos, sin))
)
)
def fused_rope_rotate_half_fp32(q, k, cos, sin):
"""Return fresh Q/K tensors; never mutate inputs or allocate FP32 activations."""
if not can_use_fused_rope_rotate_half_fp32(q, k, cos, sin):
raise ValueError(
"Expected contiguous CUDA FP16/BF16 Q/K [B,S,H,128] and FP32 cos/sin [S,128]"
)
out_q, out_k = torch.empty_like(q), torch.empty_like(k)
_, seq, heads, dim = q.shape
with torch.cuda.device(q.device):
_rope_rotate_half_fp32_kernel[(triton.cdiv(q.numel(), 1024), 2)](
q,
k,
cos,
sin,
out_q,
out_k,
q.numel(),
seq,
heads,
dim,
BLOCK=1024,
enable_fp_fusion=False,
)
return out_q, out_k
@@ -8,6 +8,7 @@ import torch
from diffusers.models.embeddings import Timesteps
from torch import nn
from sglang.kernels.ops.diffusion import BitExactFusionGate, tensors_equal
from sglang.multimodal_gen.configs.models.dits.anima import AnimaDiTConfig
from sglang.multimodal_gen.configs.models.fsdp import is_module_list_entry_in
from sglang.multimodal_gen.runtime.distributed import get_tp_world_size
@@ -28,6 +29,55 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im
LayerwiseOffloadableModuleMixin,
)
from sglang.multimodal_gen.runtime.models.dits.base import CachableDiT
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
logger = init_logger(__name__)
_ANIMA_ROPE = BitExactFusionGate("Anima FP32 RoPE", per_signature=True)
def _anima_rope_eager(q, k, cos, sin):
cos, sin = cos[None, :, None, :], sin[None, :, None, :]
q1, q2 = q.chunk(2, dim=-1)
k1, k2 = k.chunk(2, dim=-1)
return (
(q.float() * cos + torch.cat((-q2, q1), -1).float() * sin).to(q.dtype),
(k.float() * cos + torch.cat((-k2, k1), -1).float() * sin).to(k.dtype),
)
def _anima_rope(q, k, cos, sin):
if (
not torch.compiler.is_compiling()
and q.is_cuda
and torch.version.hip is None
and not _ANIMA_ROPE.disabled
):
from sglang.kernels.ops.diffusion import (
can_use_fused_rope_rotate_half_fp32,
fused_rope_rotate_half_fp32,
)
if can_use_fused_rope_rotate_half_fp32(q, k, cos, sin):
sig = (q.device, q.dtype, q.shape)
verified = _ANIMA_ROPE.is_verified(sig)
if verified or not torch.cuda.is_current_stream_capturing():
try:
out = fused_rope_rotate_half_fp32(q, k, cos, sin)
except Exception as exc:
_ANIMA_ROPE.on_exception(exc, logger=logger)
else:
if verified:
return out
return _ANIMA_ROPE.accept_or_fallback(
out,
_anima_rope_eager(q, k, cos, sin),
sig=sig,
equal=tensors_equal,
logger=logger,
)
return _anima_rope_eager(q, k, cos, sin)
def _is_anima_block(name, module):
@@ -170,12 +220,7 @@ class AnimaAttention(nn.Module):
v = self.to_v(context)[0].unflatten(-1, (self.heads, self.head_dim))
q, k = self.norm_q(q), self.norm_k(k)
if rope is not None:
cos, sin = (r[None, :, None, :] for r in rope)
# Cosmos uses split-half RoPE in fp32, then rounds once
q1, q2 = q.chunk(2, dim=-1)
k1, k2 = k.chunk(2, dim=-1)
q = (q.float() * cos + torch.cat((-q2, q1), -1).float() * sin).to(q.dtype)
k = (k.float() * cos + torch.cat((-k2, k1), -1).float() * sin).to(k.dtype)
q, k = _anima_rope(q, k, *rope)
out = self.attn(q, k, v, attn_mask_meta=attn_mask_meta).flatten(2)
return self.to_out[0](out)[0]