[diffusion] optimization: fuse the video VAE decoder's RMSNorm and QK RoPE for MiniMax-H3 (#41906)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -238,6 +238,27 @@ _SPECS: tuple[tuple[str, KernelBackend, str, frozenset, str], ...] = (
|
||||
_CUDA,
|
||||
"Fused in-place QK RMS-norm + RoPE.",
|
||||
),
|
||||
(
|
||||
"diffusion.h3_vae_rmsnorm",
|
||||
KernelBackend.TRITON,
|
||||
"norm.h3_vae_fused_norm:h3_vae_rmsnorm",
|
||||
_CUDA,
|
||||
"FP32-accumulation RMSNorm for the MiniMax-H3 video VAE decoder.",
|
||||
),
|
||||
(
|
||||
"diffusion.h3_vae_scale_add_rmsnorm",
|
||||
KernelBackend.TRITON,
|
||||
"norm.h3_vae_fused_norm:h3_vae_scale_add_rmsnorm",
|
||||
_CUDA,
|
||||
"Scaled residual add plus the following RMSNorm for the H3 VAE decoder.",
|
||||
),
|
||||
(
|
||||
"diffusion.h3_vae_qk_rmsnorm_rope",
|
||||
KernelBackend.TRITON,
|
||||
"norm.h3_vae_fused_norm:h3_vae_qk_rmsnorm_rope",
|
||||
_CUDA,
|
||||
"Affine-free Q/K RMSNorm plus partial NeoX RoPE for the H3 VAE decoder.",
|
||||
),
|
||||
(
|
||||
"diffusion.flux2_layernorm_modulate_fp8_quant",
|
||||
KernelBackend.KDA,
|
||||
@@ -769,6 +790,9 @@ _EXPORTS: dict[str, str] = {
|
||||
"unmount_lingbot_video_gated_residual": "sites.lingbot_video_gated_residual_site",
|
||||
"can_use_rmsnorm_scale_shift_per_token": "norm.rmsnorm_scale_shift_triton",
|
||||
"rmsnorm_scale_shift_per_token": "norm.rmsnorm_scale_shift_triton",
|
||||
"h3_vae_rmsnorm": "norm.h3_vae_fused_norm",
|
||||
"h3_vae_scale_add_rmsnorm": "norm.h3_vae_fused_norm",
|
||||
"h3_vae_qk_rmsnorm_rope": "norm.h3_vae_fused_norm",
|
||||
"mark_sana_video_linear_attention_site": "sites.sana_video_linear_attention_site",
|
||||
"mount_sana_video_linear_attention": "sites.sana_video_linear_attention_site",
|
||||
"sana_video_linear_attention_active": "sites.sana_video_linear_attention_site",
|
||||
|
||||
@@ -0,0 +1,299 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Fused norm kernels for the MiniMax-H3 video VAE decoder.
|
||||
|
||||
The eager block casts the residual to FP32, runs RMSNorm, then adds
|
||||
``update * scale`` as a separate kernel. Under FP16 autocast that add
|
||||
promotes the residual to FP32, so later layers carry an FP32 residual and an
|
||||
FP16 attention/FFN update. ``h3_vae_scale_add_rmsnorm`` does the add and the
|
||||
next RMSNorm in one launch: the sum stays in registers for the variance, and
|
||||
the stored residual matches the eager FP32 add.
|
||||
|
||||
``h3_vae_qk_rmsnorm_rope`` is the affine-free head-64 path: RMSNorm, round to
|
||||
the activation dtype, then partial NeoX RoPE on the first 48 channels.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
import triton # type: ignore
|
||||
import triton.language as tl # type: ignore
|
||||
|
||||
from sglang.kernels.ops.diffusion.common.numerics import mul_rn_f32
|
||||
|
||||
_MAX_BLOCK = 4096
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _rmsnorm_kernel(
|
||||
out_ptr,
|
||||
x_ptr,
|
||||
weight_ptr,
|
||||
rows,
|
||||
eps,
|
||||
D: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
OUT_KIND: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
if row >= rows:
|
||||
return
|
||||
cols = tl.arange(0, BLOCK)
|
||||
mask = cols < D
|
||||
x = tl.load(x_ptr + row * D + cols, mask=mask, other=0.0).to(tl.float32)
|
||||
rstd = tl.rsqrt(tl.sum(x * x, axis=0) / D + eps)
|
||||
weight = tl.load(weight_ptr + cols, mask=mask, other=0.0).to(tl.float32)
|
||||
y = x * rstd * weight
|
||||
if OUT_KIND == 1:
|
||||
y = y.to(tl.float16)
|
||||
elif OUT_KIND == 2:
|
||||
y = y.to(tl.bfloat16)
|
||||
tl.store(out_ptr + row * D + cols, y, mask=mask)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _scale_add_rmsnorm_kernel(
|
||||
res_in_ptr,
|
||||
update_ptr,
|
||||
res_out_ptr,
|
||||
out_ptr,
|
||||
scale_ptr,
|
||||
weight_ptr,
|
||||
rows,
|
||||
eps,
|
||||
D: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
if row >= rows:
|
||||
return
|
||||
cols = tl.arange(0, BLOCK)
|
||||
mask = cols < D
|
||||
residual = tl.load(res_in_ptr + row * D + cols, mask=mask, other=0.0).to(tl.float32)
|
||||
update = tl.load(update_ptr + row * D + cols, mask=mask, other=0.0).to(tl.float32)
|
||||
scale = tl.load(scale_ptr + cols, mask=mask, other=0.0).to(tl.float32)
|
||||
# Same rounding as the eager FP32 residual add: a rounded multiply, not an FMA.
|
||||
acc = residual + mul_rn_f32(update, scale)
|
||||
tl.store(res_out_ptr + row * D + cols, acc, mask=mask)
|
||||
rstd = tl.rsqrt(tl.sum(acc * acc, axis=0) / D + eps)
|
||||
weight = tl.load(weight_ptr + cols, mask=mask, other=0.0).to(tl.float32)
|
||||
tl.store(out_ptr + row * D + cols, acc * rstd * weight, mask=mask)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _qk_rmsnorm_rope_kernel(
|
||||
q_ptr,
|
||||
k_ptr,
|
||||
qo_ptr,
|
||||
ko_ptr,
|
||||
cos_ptr,
|
||||
sin_ptr,
|
||||
stride_qb,
|
||||
stride_qs,
|
||||
stride_qh,
|
||||
stride_kb,
|
||||
stride_ks,
|
||||
stride_kh,
|
||||
seq_len,
|
||||
n_heads,
|
||||
eps,
|
||||
):
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
heads = n_heads
|
||||
token = row // heads
|
||||
head = row % heads
|
||||
seq = token % seq_len
|
||||
batch = token // seq_len
|
||||
cols = tl.arange(0, 64)
|
||||
rope_mask = cols < 48
|
||||
q_base = q_ptr + batch * stride_qb + seq * stride_qs + head * stride_qh
|
||||
k_base = k_ptr + batch * stride_kb + seq * stride_ks + head * stride_kh
|
||||
q = tl.load(q_base + cols).to(tl.float32)
|
||||
k = tl.load(k_base + cols).to(tl.float32)
|
||||
q = (q * tl.rsqrt(tl.sum(q * q, axis=0) / 64 + eps)).to(tl.float16)
|
||||
k = (k * tl.rsqrt(tl.sum(k * k, axis=0) / 64 + eps)).to(tl.float16)
|
||||
cos = tl.load(cos_ptr + seq * 48 + cols, mask=rope_mask, other=0.0).to(tl.float16)
|
||||
sin = tl.load(sin_ptr + seq * 48 + cols, mask=rope_mask, other=0.0).to(tl.float16)
|
||||
# NeoX rotate-half: [-x2, x1] on the first 48 channels.
|
||||
src = tl.where(cols < 24, cols + 24, cols - 24)
|
||||
sign = tl.where(cols < 24, -1.0, 1.0)
|
||||
q_rot = (tl.gather(q.to(tl.float32), src, 0) * sign).to(tl.float16)
|
||||
k_rot = (tl.gather(k.to(tl.float32), src, 0) * sign).to(tl.float16)
|
||||
yq = (
|
||||
(q.to(tl.float32) * cos.to(tl.float32)).to(tl.float16).to(tl.float32)
|
||||
+ (q_rot.to(tl.float32) * sin.to(tl.float32)).to(tl.float16).to(tl.float32)
|
||||
).to(tl.float16)
|
||||
yk = (
|
||||
(k.to(tl.float32) * cos.to(tl.float32)).to(tl.float16).to(tl.float32)
|
||||
+ (k_rot.to(tl.float32) * sin.to(tl.float32)).to(tl.float16).to(tl.float32)
|
||||
).to(tl.float16)
|
||||
yq = tl.where(rope_mask, yq, q)
|
||||
yk = tl.where(rope_mask, yk, k)
|
||||
tl.store(qo_ptr + row * 64 + cols, yq)
|
||||
tl.store(ko_ptr + row * 64 + cols, yk)
|
||||
|
||||
|
||||
def _out_kind(dtype: torch.dtype) -> int:
|
||||
if dtype == torch.float32:
|
||||
return 0
|
||||
if dtype == torch.float16:
|
||||
return 1
|
||||
if dtype == torch.bfloat16:
|
||||
return 2
|
||||
raise ValueError(f"unsupported H3 VAE norm dtype {dtype}")
|
||||
|
||||
|
||||
def _launch_config(dim: int) -> tuple[int, int]:
|
||||
block = triton.next_power_of_2(dim)
|
||||
if block > _MAX_BLOCK:
|
||||
raise ValueError(f"H3 VAE fused RMSNorm dim {dim} exceeds {_MAX_BLOCK}")
|
||||
num_warps = min(8, max(1, block // 128))
|
||||
return block, num_warps
|
||||
|
||||
|
||||
def _rows_of(tensor: torch.Tensor) -> tuple[torch.Tensor, int, int]:
|
||||
if tensor.shape[-1] > _MAX_BLOCK or tensor.numel() == 0:
|
||||
raise ValueError("H3 VAE fused RMSNorm input is empty or wider than one tile")
|
||||
flat = tensor.reshape(-1, tensor.shape[-1])
|
||||
if not flat.is_contiguous():
|
||||
flat = flat.contiguous()
|
||||
return flat, flat.shape[0], flat.shape[1]
|
||||
|
||||
|
||||
def h3_vae_rmsnorm(
|
||||
hidden: torch.Tensor, weight: torch.Tensor, eps: float
|
||||
) -> torch.Tensor:
|
||||
"""FP32 RMSNorm. The stored dtype matches ``hidden`` so it lines up with ``.to(dtype)``."""
|
||||
flat, rows, dim = _rows_of(hidden)
|
||||
if weight.shape != (dim,) or weight.device != hidden.device:
|
||||
raise ValueError("H3 VAE RMSNorm weight must match the last dimension")
|
||||
out = torch.empty(flat.shape, device=hidden.device, dtype=hidden.dtype)
|
||||
block, num_warps = _launch_config(dim)
|
||||
_rmsnorm_kernel[(rows,)](
|
||||
out,
|
||||
flat,
|
||||
weight.contiguous(),
|
||||
rows,
|
||||
float(eps),
|
||||
D=dim,
|
||||
BLOCK=block,
|
||||
OUT_KIND=_out_kind(hidden.dtype),
|
||||
num_warps=num_warps,
|
||||
)
|
||||
return out.view(hidden.shape)
|
||||
|
||||
|
||||
def h3_vae_scale_add_rmsnorm(
|
||||
residual: torch.Tensor,
|
||||
update: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
eps: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""``residual + update * scale`` in FP32, then RMSNorm of that sum.
|
||||
|
||||
The residual is updated in place when it is already a contiguous FP32
|
||||
tensor. Otherwise a new FP32 residual is returned, matching the dtype the
|
||||
eager autocast add promotes to.
|
||||
"""
|
||||
if residual.shape != update.shape:
|
||||
raise ValueError(
|
||||
"H3 VAE scale-add expects residual and update to share a shape"
|
||||
)
|
||||
flat, rows, dim = _rows_of(residual)
|
||||
update_flat, _, update_dim = _rows_of(update)
|
||||
if update_dim != dim or scale.shape != (dim,) or weight.shape != (dim,):
|
||||
raise ValueError("H3 VAE scale-add vectors must match the residual width")
|
||||
if (
|
||||
scale.device != residual.device
|
||||
or weight.device != residual.device
|
||||
or update.device != residual.device
|
||||
):
|
||||
raise ValueError("H3 VAE scale-add tensors must share a device")
|
||||
inplace = (
|
||||
residual.dtype == torch.float32
|
||||
and flat.data_ptr() == residual.data_ptr()
|
||||
and flat.is_contiguous()
|
||||
)
|
||||
res_out = (
|
||||
flat
|
||||
if inplace
|
||||
else torch.empty(flat.shape, device=residual.device, dtype=torch.float32)
|
||||
)
|
||||
normed = torch.empty(flat.shape, device=residual.device, dtype=torch.float32)
|
||||
block, num_warps = _launch_config(dim)
|
||||
_scale_add_rmsnorm_kernel[(rows,)](
|
||||
flat,
|
||||
update_flat,
|
||||
res_out,
|
||||
normed,
|
||||
scale.contiguous(),
|
||||
weight.contiguous(),
|
||||
rows,
|
||||
float(eps),
|
||||
D=dim,
|
||||
BLOCK=block,
|
||||
num_warps=num_warps,
|
||||
)
|
||||
out_shape = residual.shape
|
||||
residual_out = res_out.view(out_shape)
|
||||
if inplace:
|
||||
return residual, normed.view(out_shape)
|
||||
return residual_out, normed.view(out_shape)
|
||||
|
||||
|
||||
def h3_vae_qk_rmsnorm_rope(
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
eps: float,
|
||||
cos: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor] | None:
|
||||
"""Affine-free RMSNorm plus partial NeoX RoPE for head 64 / rotary 48.
|
||||
|
||||
Returns None when the tensors are outside that contract so the caller can
|
||||
keep the unfused norm and RoPE.
|
||||
"""
|
||||
if (
|
||||
not query.is_cuda
|
||||
or query.shape != key.shape
|
||||
or query.dtype != key.dtype
|
||||
or query.dtype != torch.float16
|
||||
or query.dim() != 4
|
||||
or query.shape[-1] != 64
|
||||
or query.stride(-1) != 1
|
||||
or key.stride(-1) != 1
|
||||
or cos.shape != sin.shape
|
||||
or cos.dim() != 2
|
||||
or cos.shape[-1] != 48
|
||||
or cos.shape[0] != query.shape[1]
|
||||
or cos.device != query.device
|
||||
or sin.device != query.device
|
||||
or key.device != query.device
|
||||
):
|
||||
return None
|
||||
batch, seq_len, n_heads, _ = query.shape
|
||||
rows = batch * seq_len * n_heads
|
||||
if rows == 0:
|
||||
return None
|
||||
query_out = torch.empty(rows, 64, device=query.device, dtype=query.dtype)
|
||||
key_out = torch.empty(rows, 64, device=key.device, dtype=key.dtype)
|
||||
_qk_rmsnorm_rope_kernel[(rows,)](
|
||||
query,
|
||||
key,
|
||||
query_out,
|
||||
key_out,
|
||||
cos.contiguous(),
|
||||
sin.contiguous(),
|
||||
query.stride(0),
|
||||
query.stride(1),
|
||||
query.stride(2),
|
||||
key.stride(0),
|
||||
key.stride(1),
|
||||
key.stride(2),
|
||||
seq_len,
|
||||
n_heads,
|
||||
float(eps),
|
||||
num_warps=1,
|
||||
)
|
||||
view = (batch, seq_len, n_heads, 64)
|
||||
return query_out.view(view), key_out.view(view)
|
||||
@@ -42,6 +42,40 @@ def _vit_norm_input(module, hidden_states):
|
||||
return hidden_states.to(weight.dtype if weight is not None else hidden_states.dtype)
|
||||
|
||||
|
||||
def _try_fused_qk_rmsnorm_rope(norm_q, norm_k, query, key, rotary_pos_emb):
|
||||
if not _env_flag("MINIMAX_H3_VAE_DECODER_FUSED_NORM", "1"):
|
||||
return None
|
||||
if (
|
||||
rotary_pos_emb is None
|
||||
or torch.is_grad_enabled()
|
||||
or torch.compiler.is_compiling()
|
||||
or not isinstance(norm_q, nn.RMSNorm)
|
||||
or not isinstance(norm_k, nn.RMSNorm)
|
||||
or norm_q.weight is not None
|
||||
or norm_k.weight is not None
|
||||
or norm_q.eps != norm_k.eps
|
||||
):
|
||||
return None
|
||||
cos, sin = rotary_pos_emb[:2]
|
||||
if cos.dim() != 4 or sin.shape != cos.shape or cos.shape[2] != 1:
|
||||
return None
|
||||
# Stacked tiles broadcast one rotary table. A real batch stride would mean
|
||||
# each tile has its own positions, which this kernel does not index.
|
||||
if cos.shape[0] != 1 and cos.stride(0) != 0:
|
||||
return None
|
||||
if sin.shape[0] != 1 and sin.stride(0) != 0:
|
||||
return None
|
||||
from sglang.kernels.ops.diffusion import h3_vae_qk_rmsnorm_rope
|
||||
|
||||
return h3_vae_qk_rmsnorm_rope(
|
||||
query,
|
||||
key,
|
||||
norm_q.eps,
|
||||
cos[0, :, 0, :],
|
||||
sin[0, :, 0, :],
|
||||
)
|
||||
|
||||
|
||||
def _apply_qk_norm(module, hidden_states):
|
||||
if (
|
||||
_env_flag("MINIMAX_H3_VAE_DECODER_VIT_FP32_NORM", "1")
|
||||
@@ -138,13 +172,20 @@ class Attention(nn.Module):
|
||||
qkv = qkv.view(batch_size, seq_len, -1, 3 * self.dim_head)
|
||||
query, key, value = torch.chunk(qkv, 3, dim=-1)
|
||||
|
||||
if self.norm_q is not None:
|
||||
query = _apply_qk_norm(self.norm_q, query)
|
||||
if self.norm_k is not None:
|
||||
key = _apply_qk_norm(self.norm_k, key)
|
||||
|
||||
if rotary_pos_emb is not None:
|
||||
query, key = apply_rotary_pos_emb_qk(query, key, rotary_pos_emb)
|
||||
fused_qk = None
|
||||
if self.norm_q is not None and self.norm_k is not None:
|
||||
fused_qk = _try_fused_qk_rmsnorm_rope(
|
||||
self.norm_q, self.norm_k, query, key, rotary_pos_emb
|
||||
)
|
||||
if fused_qk is not None:
|
||||
query, key = fused_qk
|
||||
else:
|
||||
if self.norm_q is not None:
|
||||
query = _apply_qk_norm(self.norm_q, query)
|
||||
if self.norm_k is not None:
|
||||
key = _apply_qk_norm(self.norm_k, key)
|
||||
if rotary_pos_emb is not None:
|
||||
query, key = apply_rotary_pos_emb_qk(query, key, rotary_pos_emb)
|
||||
|
||||
if self.attn is not None and query.dtype in (torch.float16, torch.bfloat16):
|
||||
hidden_states = self.attn(query, key, value)
|
||||
|
||||
@@ -9,8 +9,8 @@ from diffusers.configuration_utils import ConfigMixin, register_to_config
|
||||
from diffusers.models.modeling_utils import ModelMixin
|
||||
from diffusers.utils import logging
|
||||
|
||||
from .base_module import RotaryEmbeddingND, TransformerBlock
|
||||
from .vit_utils import create_token_ids, prepare_rotary_pos_emb
|
||||
from .base_module import RotaryEmbeddingND, TransformerBlock, _scaled_residual_add
|
||||
from .vit_utils import _env_flag, create_token_ids, prepare_rotary_pos_emb
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
@@ -28,6 +28,96 @@ def _cuda_autocast_disabled(tensor: torch.Tensor):
|
||||
return torch.autocast("cuda", enabled=False) if tensor.is_cuda else nullcontext()
|
||||
|
||||
|
||||
def _fused_blocks_available(blocks, hidden_states) -> bool:
|
||||
if not _env_flag("MINIMAX_H3_VAE_DECODER_FUSED_NORM", "1"):
|
||||
return False
|
||||
if (
|
||||
not hidden_states.is_cuda
|
||||
or hidden_states.dim() != 3
|
||||
or hidden_states.dtype not in (torch.float16, torch.bfloat16, torch.float32)
|
||||
or torch.is_grad_enabled()
|
||||
or torch.compiler.is_compiling()
|
||||
or len(blocks) == 0
|
||||
or hidden_states.shape[-1] > 4096
|
||||
):
|
||||
return False
|
||||
dim = hidden_states.shape[-1]
|
||||
for block in blocks:
|
||||
# Layerwise offload hooks live on the block. This loop calls attn and
|
||||
# ff directly, so those hooks would not see the weights move.
|
||||
if block._forward_pre_hooks or block._forward_hooks:
|
||||
return False
|
||||
if not getattr(block, "use_scale", False):
|
||||
return False
|
||||
norm1, norm2 = block.norm1, block.norm2
|
||||
if not isinstance(norm1, nn.RMSNorm) or not isinstance(norm2, nn.RMSNorm):
|
||||
return False
|
||||
if (
|
||||
norm1.weight is None
|
||||
or norm2.weight is None
|
||||
or norm1.weight.shape != (dim,)
|
||||
or norm2.weight.shape != (dim,)
|
||||
or block.scale1.shape != (dim,)
|
||||
or block.scale2.shape != (dim,)
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
_FUSED_BLOCKS_LOGGED = False
|
||||
|
||||
|
||||
def _run_fused_transformer_blocks(blocks, hidden_states, rotary_pos_emb):
|
||||
global _FUSED_BLOCKS_LOGGED
|
||||
from sglang.kernels.ops.diffusion import h3_vae_rmsnorm, h3_vae_scale_add_rmsnorm
|
||||
|
||||
if not _FUSED_BLOCKS_LOGGED:
|
||||
_FUSED_BLOCKS_LOGGED = True
|
||||
logger.info(
|
||||
"MiniMax H3 video VAE decoder fuses each scaled residual add into the next RMSNorm"
|
||||
)
|
||||
pending = None
|
||||
pending_scale = None
|
||||
for block in blocks:
|
||||
if pending is None:
|
||||
normed = h3_vae_rmsnorm(hidden_states, block.norm1.weight, block.norm1.eps)
|
||||
else:
|
||||
hidden_states, normed = h3_vae_scale_add_rmsnorm(
|
||||
hidden_states,
|
||||
pending,
|
||||
pending_scale,
|
||||
block.norm1.weight,
|
||||
block.norm1.eps,
|
||||
)
|
||||
attn_output = block.attn(normed, rotary_pos_emb)
|
||||
hidden_states, normed = h3_vae_scale_add_rmsnorm(
|
||||
hidden_states,
|
||||
attn_output,
|
||||
block.scale1,
|
||||
block.norm2.weight,
|
||||
block.norm2.eps,
|
||||
)
|
||||
pending = block.ff(normed)
|
||||
pending_scale = block.scale2
|
||||
return _scaled_residual_add(hidden_states, pending, pending_scale)
|
||||
|
||||
|
||||
def _expand_rotary_batch(rotary_pos_emb, batch: int):
|
||||
"""Broadcast a batch-1 rotary cache across stacked spatial tiles."""
|
||||
cos, sin, *extra = rotary_pos_emb
|
||||
if cos.shape[0] == batch:
|
||||
return rotary_pos_emb
|
||||
if cos.shape[0] != 1:
|
||||
raise ValueError(
|
||||
f"MiniMax H3 VAE rotary cache batch must be 1 or {batch}, got {cos.shape[0]}"
|
||||
)
|
||||
return (
|
||||
cos.expand(batch, -1, -1, -1),
|
||||
sin.expand(batch, -1, -1, -1),
|
||||
*extra,
|
||||
)
|
||||
|
||||
|
||||
def _pack_tensors_3d(tensors, patch_size, patch_size_t):
|
||||
batch_size, num_channels_tensors, temporal, height, width = tensors.shape
|
||||
|
||||
@@ -134,6 +224,10 @@ class ViTBase(ModelMixin, ConfigMixin):
|
||||
return hidden_states, img_ids
|
||||
|
||||
def forward_transformer_blocks(self, hidden_states, rotary_pos_emb):
|
||||
if _fused_blocks_available(self.transformer_blocks, hidden_states):
|
||||
return _run_fused_transformer_blocks(
|
||||
self.transformer_blocks, hidden_states, rotary_pos_emb
|
||||
)
|
||||
for block in self.transformer_blocks:
|
||||
hidden_states = block(hidden_states, rotary_pos_emb)
|
||||
return hidden_states
|
||||
@@ -326,10 +420,15 @@ class ViT3DDecoder(ViTBase):
|
||||
if cache_hit:
|
||||
rotary_pos_emb = cache_record[2]
|
||||
else:
|
||||
# Every stacked tile has the same coordinates, so one rotary
|
||||
# table is broadcast across the batch. That keeps the fused
|
||||
# NeoX kernel, which indexes a single sequence, on the batched path.
|
||||
rotary_pos_emb = prepare_rotary_pos_emb(
|
||||
self.pos_embed(img_ids),
|
||||
self.pos_embed(img_ids[:1]),
|
||||
dtype=rotary_dtype,
|
||||
)
|
||||
if B > 1:
|
||||
rotary_pos_emb = _expand_rotary_batch(rotary_pos_emb, B)
|
||||
if cache_enabled:
|
||||
self._rotary_pos_emb_cache = (
|
||||
cache_key,
|
||||
@@ -337,8 +436,7 @@ class ViT3DDecoder(ViTBase):
|
||||
rotary_pos_emb,
|
||||
)
|
||||
|
||||
for block in self.transformer_blocks:
|
||||
hidden_states = block(hidden_states, rotary_pos_emb)
|
||||
hidden_states = self.forward_transformer_blocks(hidden_states, rotary_pos_emb)
|
||||
|
||||
hidden_states = self.norm_out(hidden_states)
|
||||
|
||||
|
||||
@@ -222,27 +222,30 @@ def apply_rotary_pos_emb_qk(
|
||||
and query.dtype == key.dtype
|
||||
and query.dtype in (torch.float16, torch.bfloat16)
|
||||
and query.dim() == 4
|
||||
and query.shape[0] == 1
|
||||
and not torch.compiler.is_compiling()
|
||||
):
|
||||
_, _, cache, positions = rotary_pos_emb
|
||||
batch, seq_len = query.shape[:2]
|
||||
if (
|
||||
cache.is_cuda
|
||||
and cache.dtype == query.dtype
|
||||
and cache.dim() == 2
|
||||
and cache.shape[0] == query.shape[1]
|
||||
and cache.shape[0] == seq_len
|
||||
and cache.shape[1] <= query.shape[-1]
|
||||
and positions.is_cuda
|
||||
and positions.shape == (query.shape[1],)
|
||||
and positions.shape == (seq_len,)
|
||||
):
|
||||
from sgl_kernel import rotary_embedding
|
||||
|
||||
# Stacked spatial tiles share one rotary table. Flatten the
|
||||
# batch and replay that table once per tile.
|
||||
query = query.contiguous()
|
||||
key = key.contiguous()
|
||||
token_positions = positions if batch == 1 else positions.repeat(batch)
|
||||
rotary_embedding(
|
||||
positions,
|
||||
query.view(query.shape[1], -1),
|
||||
key.view(key.shape[1], -1),
|
||||
token_positions,
|
||||
query.view(batch * seq_len, -1),
|
||||
key.view(batch * seq_len, -1),
|
||||
query.shape[-1],
|
||||
cache,
|
||||
True,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""MiniMax-H3 released VAE decode contract."""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import textwrap
|
||||
@@ -24,6 +25,12 @@ from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.attention im
|
||||
Attention,
|
||||
_apply_qk_norm,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.vae_vit import (
|
||||
ViT3DDecoder,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.vit_utils import (
|
||||
apply_rotary_pos_emb_qk,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
|
||||
|
||||
|
||||
@@ -129,8 +136,7 @@ def test_audio_snake_first_call_matches_repeated_calls():
|
||||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
textwrap.dedent(
|
||||
"""
|
||||
textwrap.dedent("""
|
||||
import torch
|
||||
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_audio_vae.audio_vae import Snake1d
|
||||
|
||||
@@ -146,9 +152,109 @@ def test_audio_snake_first_call_matches_repeated_calls():
|
||||
torch.testing.assert_close(activation(x), first, rtol=0, atol=0)
|
||||
torch.testing.assert_close(x, original_x, rtol=0, atol=0)
|
||||
torch.testing.assert_close(activation.alpha, original_alpha, rtol=0, atol=0)
|
||||
"""
|
||||
),
|
||||
"""),
|
||||
],
|
||||
check=True,
|
||||
timeout=120,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_batched_neox_rope_matches_per_tile_rows():
|
||||
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.vit_utils import (
|
||||
prepare_rotary_pos_emb,
|
||||
)
|
||||
|
||||
cos = torch.randn(1, 5, 1, 8, device="cuda", dtype=torch.float16)
|
||||
sin = torch.randn(1, 5, 1, 8, device="cuda", dtype=torch.float16)
|
||||
packed = prepare_rotary_pos_emb((cos, sin), dtype=torch.float16)
|
||||
query = torch.randn(3, 5, 2, 16, device="cuda", dtype=torch.float16)
|
||||
key = torch.randn(3, 5, 2, 16, device="cuda", dtype=torch.float16)
|
||||
expanded = (
|
||||
packed[0].expand(3, -1, -1, -1),
|
||||
packed[1].expand(3, -1, -1, -1),
|
||||
*packed[2:],
|
||||
)
|
||||
batched_q, batched_k = apply_rotary_pos_emb_qk(query.clone(), key.clone(), expanded)
|
||||
rows_q = []
|
||||
rows_k = []
|
||||
for index in range(3):
|
||||
row_q, row_k = apply_rotary_pos_emb_qk(
|
||||
query[index : index + 1],
|
||||
key[index : index + 1],
|
||||
packed,
|
||||
)
|
||||
rows_q.append(row_q)
|
||||
rows_k.append(row_k)
|
||||
|
||||
torch.testing.assert_close(batched_q, torch.cat(rows_q, dim=0))
|
||||
torch.testing.assert_close(batched_k, torch.cat(rows_k, dim=0))
|
||||
|
||||
|
||||
def _h3_shaped_decoder():
|
||||
decoder = ViT3DDecoder(
|
||||
patch_size=1,
|
||||
patch_size_t=1,
|
||||
in_channels=4,
|
||||
out_channels=3,
|
||||
num_layers=2,
|
||||
heads=4,
|
||||
dim_head=64,
|
||||
norm_type="rms_norm",
|
||||
norm_affine=True,
|
||||
qk_norm_type="rms_norm",
|
||||
qk_norm_affine=False,
|
||||
ffn_activation_fn="silu",
|
||||
ffn_use_gated=True,
|
||||
rope_dim_ratio=0.75,
|
||||
rope_theta=100.0,
|
||||
num_register_tokens=0,
|
||||
).cuda()
|
||||
decoder.eval()
|
||||
for block in decoder.transformer_blocks:
|
||||
block.scale1.data.normal_(0, 0.1)
|
||||
block.scale2.data.normal_(0, 0.1)
|
||||
decoder.prepare_autocast_linear_weights(torch.float16)
|
||||
return decoder
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_fused_decoder_matches_unfused_forward():
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import (
|
||||
set_forward_context,
|
||||
)
|
||||
|
||||
decoder = _h3_shaped_decoder()
|
||||
latent = torch.randn(2, 4, 1, 2, 2, device="cuda", dtype=torch.float16)
|
||||
previous = os.environ.get("MINIMAX_H3_VAE_DECODER_FUSED_NORM")
|
||||
from sglang.kernels.ops import diffusion as diffusion_ops
|
||||
|
||||
try:
|
||||
with (
|
||||
torch.no_grad(),
|
||||
torch.autocast("cuda", dtype=torch.float16),
|
||||
set_forward_context(current_timestep=0, attn_metadata=None),
|
||||
mock.patch.object(
|
||||
diffusion_ops,
|
||||
"h3_vae_scale_add_rmsnorm",
|
||||
wraps=diffusion_ops.h3_vae_scale_add_rmsnorm,
|
||||
) as scale_add,
|
||||
mock.patch.object(
|
||||
diffusion_ops,
|
||||
"h3_vae_qk_rmsnorm_rope",
|
||||
wraps=diffusion_ops.h3_vae_qk_rmsnorm_rope,
|
||||
) as qk_rope,
|
||||
):
|
||||
os.environ["MINIMAX_H3_VAE_DECODER_FUSED_NORM"] = "0"
|
||||
eager = decoder(latent)
|
||||
os.environ["MINIMAX_H3_VAE_DECODER_FUSED_NORM"] = "1"
|
||||
fused = decoder(latent)
|
||||
assert scale_add.call_count == 3
|
||||
assert qk_rope.call_count == 2
|
||||
finally:
|
||||
if previous is None:
|
||||
os.environ.pop("MINIMAX_H3_VAE_DECODER_FUSED_NORM", None)
|
||||
else:
|
||||
os.environ["MINIMAX_H3_VAE_DECODER_FUSED_NORM"] = previous
|
||||
|
||||
torch.testing.assert_close(fused, eager, rtol=1e-3, atol=1e-3)
|
||||
|
||||
Reference in New Issue
Block a user