[diffusion] optimization: fuse the video VAE decoder's RMSNorm and QK RoPE for MiniMax-H3 (#41906)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
WenhaoZhang
2026-10-05 17:30:39 +08:00
committed by GitHub
co-authored by Mick Qian
parent 0263fdacb2
commit 284cda01fb
7 changed files with 672 additions and 22 deletions
@@ -238,6 +238,27 @@ _SPECS: tuple[tuple[str, KernelBackend, str, frozenset, str], ...] = (
_CUDA,
"Fused in-place QK RMS-norm + RoPE.",
),
(
"diffusion.h3_vae_rmsnorm",
KernelBackend.TRITON,
"norm.h3_vae_fused_norm:h3_vae_rmsnorm",
_CUDA,
"FP32-accumulation RMSNorm for the MiniMax-H3 video VAE decoder.",
),
(
"diffusion.h3_vae_scale_add_rmsnorm",
KernelBackend.TRITON,
"norm.h3_vae_fused_norm:h3_vae_scale_add_rmsnorm",
_CUDA,
"Scaled residual add plus the following RMSNorm for the H3 VAE decoder.",
),
(
"diffusion.h3_vae_qk_rmsnorm_rope",
KernelBackend.TRITON,
"norm.h3_vae_fused_norm:h3_vae_qk_rmsnorm_rope",
_CUDA,
"Affine-free Q/K RMSNorm plus partial NeoX RoPE for the H3 VAE decoder.",
),
(
"diffusion.flux2_layernorm_modulate_fp8_quant",
KernelBackend.KDA,
@@ -769,6 +790,9 @@ _EXPORTS: dict[str, str] = {
"unmount_lingbot_video_gated_residual": "sites.lingbot_video_gated_residual_site",
"can_use_rmsnorm_scale_shift_per_token": "norm.rmsnorm_scale_shift_triton",
"rmsnorm_scale_shift_per_token": "norm.rmsnorm_scale_shift_triton",
"h3_vae_rmsnorm": "norm.h3_vae_fused_norm",
"h3_vae_scale_add_rmsnorm": "norm.h3_vae_fused_norm",
"h3_vae_qk_rmsnorm_rope": "norm.h3_vae_fused_norm",
"mark_sana_video_linear_attention_site": "sites.sana_video_linear_attention_site",
"mount_sana_video_linear_attention": "sites.sana_video_linear_attention_site",
"sana_video_linear_attention_active": "sites.sana_video_linear_attention_site",
@@ -0,0 +1,299 @@
# SPDX-License-Identifier: Apache-2.0
"""Fused norm kernels for the MiniMax-H3 video VAE decoder.
The eager block casts the residual to FP32, runs RMSNorm, then adds
``update * scale`` as a separate kernel. Under FP16 autocast that add
promotes the residual to FP32, so later layers carry an FP32 residual and an
FP16 attention/FFN update. ``h3_vae_scale_add_rmsnorm`` does the add and the
next RMSNorm in one launch: the sum stays in registers for the variance, and
the stored residual matches the eager FP32 add.
``h3_vae_qk_rmsnorm_rope`` is the affine-free head-64 path: RMSNorm, round to
the activation dtype, then partial NeoX RoPE on the first 48 channels.
"""
from __future__ import annotations
import torch
import triton # type: ignore
import triton.language as tl # type: ignore
from sglang.kernels.ops.diffusion.common.numerics import mul_rn_f32
_MAX_BLOCK = 4096
@triton.jit
def _rmsnorm_kernel(
out_ptr,
x_ptr,
weight_ptr,
rows,
eps,
D: tl.constexpr,
BLOCK: tl.constexpr,
OUT_KIND: tl.constexpr,
):
row = tl.program_id(0)
if row >= rows:
return
cols = tl.arange(0, BLOCK)
mask = cols < D
x = tl.load(x_ptr + row * D + cols, mask=mask, other=0.0).to(tl.float32)
rstd = tl.rsqrt(tl.sum(x * x, axis=0) / D + eps)
weight = tl.load(weight_ptr + cols, mask=mask, other=0.0).to(tl.float32)
y = x * rstd * weight
if OUT_KIND == 1:
y = y.to(tl.float16)
elif OUT_KIND == 2:
y = y.to(tl.bfloat16)
tl.store(out_ptr + row * D + cols, y, mask=mask)
@triton.jit
def _scale_add_rmsnorm_kernel(
res_in_ptr,
update_ptr,
res_out_ptr,
out_ptr,
scale_ptr,
weight_ptr,
rows,
eps,
D: tl.constexpr,
BLOCK: tl.constexpr,
):
row = tl.program_id(0)
if row >= rows:
return
cols = tl.arange(0, BLOCK)
mask = cols < D
residual = tl.load(res_in_ptr + row * D + cols, mask=mask, other=0.0).to(tl.float32)
update = tl.load(update_ptr + row * D + cols, mask=mask, other=0.0).to(tl.float32)
scale = tl.load(scale_ptr + cols, mask=mask, other=0.0).to(tl.float32)
# Same rounding as the eager FP32 residual add: a rounded multiply, not an FMA.
acc = residual + mul_rn_f32(update, scale)
tl.store(res_out_ptr + row * D + cols, acc, mask=mask)
rstd = tl.rsqrt(tl.sum(acc * acc, axis=0) / D + eps)
weight = tl.load(weight_ptr + cols, mask=mask, other=0.0).to(tl.float32)
tl.store(out_ptr + row * D + cols, acc * rstd * weight, mask=mask)
@triton.jit
def _qk_rmsnorm_rope_kernel(
q_ptr,
k_ptr,
qo_ptr,
ko_ptr,
cos_ptr,
sin_ptr,
stride_qb,
stride_qs,
stride_qh,
stride_kb,
stride_ks,
stride_kh,
seq_len,
n_heads,
eps,
):
row = tl.program_id(0).to(tl.int64)
heads = n_heads
token = row // heads
head = row % heads
seq = token % seq_len
batch = token // seq_len
cols = tl.arange(0, 64)
rope_mask = cols < 48
q_base = q_ptr + batch * stride_qb + seq * stride_qs + head * stride_qh
k_base = k_ptr + batch * stride_kb + seq * stride_ks + head * stride_kh
q = tl.load(q_base + cols).to(tl.float32)
k = tl.load(k_base + cols).to(tl.float32)
q = (q * tl.rsqrt(tl.sum(q * q, axis=0) / 64 + eps)).to(tl.float16)
k = (k * tl.rsqrt(tl.sum(k * k, axis=0) / 64 + eps)).to(tl.float16)
cos = tl.load(cos_ptr + seq * 48 + cols, mask=rope_mask, other=0.0).to(tl.float16)
sin = tl.load(sin_ptr + seq * 48 + cols, mask=rope_mask, other=0.0).to(tl.float16)
# NeoX rotate-half: [-x2, x1] on the first 48 channels.
src = tl.where(cols < 24, cols + 24, cols - 24)
sign = tl.where(cols < 24, -1.0, 1.0)
q_rot = (tl.gather(q.to(tl.float32), src, 0) * sign).to(tl.float16)
k_rot = (tl.gather(k.to(tl.float32), src, 0) * sign).to(tl.float16)
yq = (
(q.to(tl.float32) * cos.to(tl.float32)).to(tl.float16).to(tl.float32)
+ (q_rot.to(tl.float32) * sin.to(tl.float32)).to(tl.float16).to(tl.float32)
).to(tl.float16)
yk = (
(k.to(tl.float32) * cos.to(tl.float32)).to(tl.float16).to(tl.float32)
+ (k_rot.to(tl.float32) * sin.to(tl.float32)).to(tl.float16).to(tl.float32)
).to(tl.float16)
yq = tl.where(rope_mask, yq, q)
yk = tl.where(rope_mask, yk, k)
tl.store(qo_ptr + row * 64 + cols, yq)
tl.store(ko_ptr + row * 64 + cols, yk)
def _out_kind(dtype: torch.dtype) -> int:
if dtype == torch.float32:
return 0
if dtype == torch.float16:
return 1
if dtype == torch.bfloat16:
return 2
raise ValueError(f"unsupported H3 VAE norm dtype {dtype}")
def _launch_config(dim: int) -> tuple[int, int]:
block = triton.next_power_of_2(dim)
if block > _MAX_BLOCK:
raise ValueError(f"H3 VAE fused RMSNorm dim {dim} exceeds {_MAX_BLOCK}")
num_warps = min(8, max(1, block // 128))
return block, num_warps
def _rows_of(tensor: torch.Tensor) -> tuple[torch.Tensor, int, int]:
if tensor.shape[-1] > _MAX_BLOCK or tensor.numel() == 0:
raise ValueError("H3 VAE fused RMSNorm input is empty or wider than one tile")
flat = tensor.reshape(-1, tensor.shape[-1])
if not flat.is_contiguous():
flat = flat.contiguous()
return flat, flat.shape[0], flat.shape[1]
def h3_vae_rmsnorm(
hidden: torch.Tensor, weight: torch.Tensor, eps: float
) -> torch.Tensor:
"""FP32 RMSNorm. The stored dtype matches ``hidden`` so it lines up with ``.to(dtype)``."""
flat, rows, dim = _rows_of(hidden)
if weight.shape != (dim,) or weight.device != hidden.device:
raise ValueError("H3 VAE RMSNorm weight must match the last dimension")
out = torch.empty(flat.shape, device=hidden.device, dtype=hidden.dtype)
block, num_warps = _launch_config(dim)
_rmsnorm_kernel[(rows,)](
out,
flat,
weight.contiguous(),
rows,
float(eps),
D=dim,
BLOCK=block,
OUT_KIND=_out_kind(hidden.dtype),
num_warps=num_warps,
)
return out.view(hidden.shape)
def h3_vae_scale_add_rmsnorm(
residual: torch.Tensor,
update: torch.Tensor,
scale: torch.Tensor,
weight: torch.Tensor,
eps: float,
) -> tuple[torch.Tensor, torch.Tensor]:
"""``residual + update * scale`` in FP32, then RMSNorm of that sum.
The residual is updated in place when it is already a contiguous FP32
tensor. Otherwise a new FP32 residual is returned, matching the dtype the
eager autocast add promotes to.
"""
if residual.shape != update.shape:
raise ValueError(
"H3 VAE scale-add expects residual and update to share a shape"
)
flat, rows, dim = _rows_of(residual)
update_flat, _, update_dim = _rows_of(update)
if update_dim != dim or scale.shape != (dim,) or weight.shape != (dim,):
raise ValueError("H3 VAE scale-add vectors must match the residual width")
if (
scale.device != residual.device
or weight.device != residual.device
or update.device != residual.device
):
raise ValueError("H3 VAE scale-add tensors must share a device")
inplace = (
residual.dtype == torch.float32
and flat.data_ptr() == residual.data_ptr()
and flat.is_contiguous()
)
res_out = (
flat
if inplace
else torch.empty(flat.shape, device=residual.device, dtype=torch.float32)
)
normed = torch.empty(flat.shape, device=residual.device, dtype=torch.float32)
block, num_warps = _launch_config(dim)
_scale_add_rmsnorm_kernel[(rows,)](
flat,
update_flat,
res_out,
normed,
scale.contiguous(),
weight.contiguous(),
rows,
float(eps),
D=dim,
BLOCK=block,
num_warps=num_warps,
)
out_shape = residual.shape
residual_out = res_out.view(out_shape)
if inplace:
return residual, normed.view(out_shape)
return residual_out, normed.view(out_shape)
def h3_vae_qk_rmsnorm_rope(
query: torch.Tensor,
key: torch.Tensor,
eps: float,
cos: torch.Tensor,
sin: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor] | None:
"""Affine-free RMSNorm plus partial NeoX RoPE for head 64 / rotary 48.
Returns None when the tensors are outside that contract so the caller can
keep the unfused norm and RoPE.
"""
if (
not query.is_cuda
or query.shape != key.shape
or query.dtype != key.dtype
or query.dtype != torch.float16
or query.dim() != 4
or query.shape[-1] != 64
or query.stride(-1) != 1
or key.stride(-1) != 1
or cos.shape != sin.shape
or cos.dim() != 2
or cos.shape[-1] != 48
or cos.shape[0] != query.shape[1]
or cos.device != query.device
or sin.device != query.device
or key.device != query.device
):
return None
batch, seq_len, n_heads, _ = query.shape
rows = batch * seq_len * n_heads
if rows == 0:
return None
query_out = torch.empty(rows, 64, device=query.device, dtype=query.dtype)
key_out = torch.empty(rows, 64, device=key.device, dtype=key.dtype)
_qk_rmsnorm_rope_kernel[(rows,)](
query,
key,
query_out,
key_out,
cos.contiguous(),
sin.contiguous(),
query.stride(0),
query.stride(1),
query.stride(2),
key.stride(0),
key.stride(1),
key.stride(2),
seq_len,
n_heads,
float(eps),
num_warps=1,
)
view = (batch, seq_len, n_heads, 64)
return query_out.view(view), key_out.view(view)
@@ -42,6 +42,40 @@ def _vit_norm_input(module, hidden_states):
return hidden_states.to(weight.dtype if weight is not None else hidden_states.dtype)
def _try_fused_qk_rmsnorm_rope(norm_q, norm_k, query, key, rotary_pos_emb):
if not _env_flag("MINIMAX_H3_VAE_DECODER_FUSED_NORM", "1"):
return None
if (
rotary_pos_emb is None
or torch.is_grad_enabled()
or torch.compiler.is_compiling()
or not isinstance(norm_q, nn.RMSNorm)
or not isinstance(norm_k, nn.RMSNorm)
or norm_q.weight is not None
or norm_k.weight is not None
or norm_q.eps != norm_k.eps
):
return None
cos, sin = rotary_pos_emb[:2]
if cos.dim() != 4 or sin.shape != cos.shape or cos.shape[2] != 1:
return None
# Stacked tiles broadcast one rotary table. A real batch stride would mean
# each tile has its own positions, which this kernel does not index.
if cos.shape[0] != 1 and cos.stride(0) != 0:
return None
if sin.shape[0] != 1 and sin.stride(0) != 0:
return None
from sglang.kernels.ops.diffusion import h3_vae_qk_rmsnorm_rope
return h3_vae_qk_rmsnorm_rope(
query,
key,
norm_q.eps,
cos[0, :, 0, :],
sin[0, :, 0, :],
)
def _apply_qk_norm(module, hidden_states):
if (
_env_flag("MINIMAX_H3_VAE_DECODER_VIT_FP32_NORM", "1")
@@ -138,13 +172,20 @@ class Attention(nn.Module):
qkv = qkv.view(batch_size, seq_len, -1, 3 * self.dim_head)
query, key, value = torch.chunk(qkv, 3, dim=-1)
if self.norm_q is not None:
query = _apply_qk_norm(self.norm_q, query)
if self.norm_k is not None:
key = _apply_qk_norm(self.norm_k, key)
if rotary_pos_emb is not None:
query, key = apply_rotary_pos_emb_qk(query, key, rotary_pos_emb)
fused_qk = None
if self.norm_q is not None and self.norm_k is not None:
fused_qk = _try_fused_qk_rmsnorm_rope(
self.norm_q, self.norm_k, query, key, rotary_pos_emb
)
if fused_qk is not None:
query, key = fused_qk
else:
if self.norm_q is not None:
query = _apply_qk_norm(self.norm_q, query)
if self.norm_k is not None:
key = _apply_qk_norm(self.norm_k, key)
if rotary_pos_emb is not None:
query, key = apply_rotary_pos_emb_qk(query, key, rotary_pos_emb)
if self.attn is not None and query.dtype in (torch.float16, torch.bfloat16):
hidden_states = self.attn(query, key, value)
@@ -9,8 +9,8 @@ from diffusers.configuration_utils import ConfigMixin, register_to_config
from diffusers.models.modeling_utils import ModelMixin
from diffusers.utils import logging
from .base_module import RotaryEmbeddingND, TransformerBlock
from .vit_utils import create_token_ids, prepare_rotary_pos_emb
from .base_module import RotaryEmbeddingND, TransformerBlock, _scaled_residual_add
from .vit_utils import _env_flag, create_token_ids, prepare_rotary_pos_emb
logger = logging.get_logger(__name__)
@@ -28,6 +28,96 @@ def _cuda_autocast_disabled(tensor: torch.Tensor):
return torch.autocast("cuda", enabled=False) if tensor.is_cuda else nullcontext()
def _fused_blocks_available(blocks, hidden_states) -> bool:
if not _env_flag("MINIMAX_H3_VAE_DECODER_FUSED_NORM", "1"):
return False
if (
not hidden_states.is_cuda
or hidden_states.dim() != 3
or hidden_states.dtype not in (torch.float16, torch.bfloat16, torch.float32)
or torch.is_grad_enabled()
or torch.compiler.is_compiling()
or len(blocks) == 0
or hidden_states.shape[-1] > 4096
):
return False
dim = hidden_states.shape[-1]
for block in blocks:
# Layerwise offload hooks live on the block. This loop calls attn and
# ff directly, so those hooks would not see the weights move.
if block._forward_pre_hooks or block._forward_hooks:
return False
if not getattr(block, "use_scale", False):
return False
norm1, norm2 = block.norm1, block.norm2
if not isinstance(norm1, nn.RMSNorm) or not isinstance(norm2, nn.RMSNorm):
return False
if (
norm1.weight is None
or norm2.weight is None
or norm1.weight.shape != (dim,)
or norm2.weight.shape != (dim,)
or block.scale1.shape != (dim,)
or block.scale2.shape != (dim,)
):
return False
return True
_FUSED_BLOCKS_LOGGED = False
def _run_fused_transformer_blocks(blocks, hidden_states, rotary_pos_emb):
global _FUSED_BLOCKS_LOGGED
from sglang.kernels.ops.diffusion import h3_vae_rmsnorm, h3_vae_scale_add_rmsnorm
if not _FUSED_BLOCKS_LOGGED:
_FUSED_BLOCKS_LOGGED = True
logger.info(
"MiniMax H3 video VAE decoder fuses each scaled residual add into the next RMSNorm"
)
pending = None
pending_scale = None
for block in blocks:
if pending is None:
normed = h3_vae_rmsnorm(hidden_states, block.norm1.weight, block.norm1.eps)
else:
hidden_states, normed = h3_vae_scale_add_rmsnorm(
hidden_states,
pending,
pending_scale,
block.norm1.weight,
block.norm1.eps,
)
attn_output = block.attn(normed, rotary_pos_emb)
hidden_states, normed = h3_vae_scale_add_rmsnorm(
hidden_states,
attn_output,
block.scale1,
block.norm2.weight,
block.norm2.eps,
)
pending = block.ff(normed)
pending_scale = block.scale2
return _scaled_residual_add(hidden_states, pending, pending_scale)
def _expand_rotary_batch(rotary_pos_emb, batch: int):
"""Broadcast a batch-1 rotary cache across stacked spatial tiles."""
cos, sin, *extra = rotary_pos_emb
if cos.shape[0] == batch:
return rotary_pos_emb
if cos.shape[0] != 1:
raise ValueError(
f"MiniMax H3 VAE rotary cache batch must be 1 or {batch}, got {cos.shape[0]}"
)
return (
cos.expand(batch, -1, -1, -1),
sin.expand(batch, -1, -1, -1),
*extra,
)
def _pack_tensors_3d(tensors, patch_size, patch_size_t):
batch_size, num_channels_tensors, temporal, height, width = tensors.shape
@@ -134,6 +224,10 @@ class ViTBase(ModelMixin, ConfigMixin):
return hidden_states, img_ids
def forward_transformer_blocks(self, hidden_states, rotary_pos_emb):
if _fused_blocks_available(self.transformer_blocks, hidden_states):
return _run_fused_transformer_blocks(
self.transformer_blocks, hidden_states, rotary_pos_emb
)
for block in self.transformer_blocks:
hidden_states = block(hidden_states, rotary_pos_emb)
return hidden_states
@@ -326,10 +420,15 @@ class ViT3DDecoder(ViTBase):
if cache_hit:
rotary_pos_emb = cache_record[2]
else:
# Every stacked tile has the same coordinates, so one rotary
# table is broadcast across the batch. That keeps the fused
# NeoX kernel, which indexes a single sequence, on the batched path.
rotary_pos_emb = prepare_rotary_pos_emb(
self.pos_embed(img_ids),
self.pos_embed(img_ids[:1]),
dtype=rotary_dtype,
)
if B > 1:
rotary_pos_emb = _expand_rotary_batch(rotary_pos_emb, B)
if cache_enabled:
self._rotary_pos_emb_cache = (
cache_key,
@@ -337,8 +436,7 @@ class ViT3DDecoder(ViTBase):
rotary_pos_emb,
)
for block in self.transformer_blocks:
hidden_states = block(hidden_states, rotary_pos_emb)
hidden_states = self.forward_transformer_blocks(hidden_states, rotary_pos_emb)
hidden_states = self.norm_out(hidden_states)
@@ -222,27 +222,30 @@ def apply_rotary_pos_emb_qk(
and query.dtype == key.dtype
and query.dtype in (torch.float16, torch.bfloat16)
and query.dim() == 4
and query.shape[0] == 1
and not torch.compiler.is_compiling()
):
_, _, cache, positions = rotary_pos_emb
batch, seq_len = query.shape[:2]
if (
cache.is_cuda
and cache.dtype == query.dtype
and cache.dim() == 2
and cache.shape[0] == query.shape[1]
and cache.shape[0] == seq_len
and cache.shape[1] <= query.shape[-1]
and positions.is_cuda
and positions.shape == (query.shape[1],)
and positions.shape == (seq_len,)
):
from sgl_kernel import rotary_embedding
# Stacked spatial tiles share one rotary table. Flatten the
# batch and replay that table once per tile.
query = query.contiguous()
key = key.contiguous()
token_positions = positions if batch == 1 else positions.repeat(batch)
rotary_embedding(
positions,
query.view(query.shape[1], -1),
key.view(key.shape[1], -1),
token_positions,
query.view(batch * seq_len, -1),
key.view(batch * seq_len, -1),
query.shape[-1],
cache,
True,
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
"""MiniMax-H3 released VAE decode contract."""
import os
import subprocess
import sys
import textwrap
@@ -24,6 +25,12 @@ from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.attention im
Attention,
_apply_qk_norm,
)
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.vae_vit import (
ViT3DDecoder,
)
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.vit_utils import (
apply_rotary_pos_emb_qk,
)
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
@@ -129,8 +136,7 @@ def test_audio_snake_first_call_matches_repeated_calls():
[
sys.executable,
"-c",
textwrap.dedent(
"""
textwrap.dedent("""
import torch
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_audio_vae.audio_vae import Snake1d
@@ -146,9 +152,109 @@ def test_audio_snake_first_call_matches_repeated_calls():
torch.testing.assert_close(activation(x), first, rtol=0, atol=0)
torch.testing.assert_close(x, original_x, rtol=0, atol=0)
torch.testing.assert_close(activation.alpha, original_alpha, rtol=0, atol=0)
"""
),
"""),
],
check=True,
timeout=120,
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_batched_neox_rope_matches_per_tile_rows():
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.vit_utils import (
prepare_rotary_pos_emb,
)
cos = torch.randn(1, 5, 1, 8, device="cuda", dtype=torch.float16)
sin = torch.randn(1, 5, 1, 8, device="cuda", dtype=torch.float16)
packed = prepare_rotary_pos_emb((cos, sin), dtype=torch.float16)
query = torch.randn(3, 5, 2, 16, device="cuda", dtype=torch.float16)
key = torch.randn(3, 5, 2, 16, device="cuda", dtype=torch.float16)
expanded = (
packed[0].expand(3, -1, -1, -1),
packed[1].expand(3, -1, -1, -1),
*packed[2:],
)
batched_q, batched_k = apply_rotary_pos_emb_qk(query.clone(), key.clone(), expanded)
rows_q = []
rows_k = []
for index in range(3):
row_q, row_k = apply_rotary_pos_emb_qk(
query[index : index + 1],
key[index : index + 1],
packed,
)
rows_q.append(row_q)
rows_k.append(row_k)
torch.testing.assert_close(batched_q, torch.cat(rows_q, dim=0))
torch.testing.assert_close(batched_k, torch.cat(rows_k, dim=0))
def _h3_shaped_decoder():
decoder = ViT3DDecoder(
patch_size=1,
patch_size_t=1,
in_channels=4,
out_channels=3,
num_layers=2,
heads=4,
dim_head=64,
norm_type="rms_norm",
norm_affine=True,
qk_norm_type="rms_norm",
qk_norm_affine=False,
ffn_activation_fn="silu",
ffn_use_gated=True,
rope_dim_ratio=0.75,
rope_theta=100.0,
num_register_tokens=0,
).cuda()
decoder.eval()
for block in decoder.transformer_blocks:
block.scale1.data.normal_(0, 0.1)
block.scale2.data.normal_(0, 0.1)
decoder.prepare_autocast_linear_weights(torch.float16)
return decoder
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_fused_decoder_matches_unfused_forward():
from sglang.multimodal_gen.runtime.managers.forward_context import (
set_forward_context,
)
decoder = _h3_shaped_decoder()
latent = torch.randn(2, 4, 1, 2, 2, device="cuda", dtype=torch.float16)
previous = os.environ.get("MINIMAX_H3_VAE_DECODER_FUSED_NORM")
from sglang.kernels.ops import diffusion as diffusion_ops
try:
with (
torch.no_grad(),
torch.autocast("cuda", dtype=torch.float16),
set_forward_context(current_timestep=0, attn_metadata=None),
mock.patch.object(
diffusion_ops,
"h3_vae_scale_add_rmsnorm",
wraps=diffusion_ops.h3_vae_scale_add_rmsnorm,
) as scale_add,
mock.patch.object(
diffusion_ops,
"h3_vae_qk_rmsnorm_rope",
wraps=diffusion_ops.h3_vae_qk_rmsnorm_rope,
) as qk_rope,
):
os.environ["MINIMAX_H3_VAE_DECODER_FUSED_NORM"] = "0"
eager = decoder(latent)
os.environ["MINIMAX_H3_VAE_DECODER_FUSED_NORM"] = "1"
fused = decoder(latent)
assert scale_add.call_count == 3
assert qk_rope.call_count == 2
finally:
if previous is None:
os.environ.pop("MINIMAX_H3_VAE_DECODER_FUSED_NORM", None)
else:
os.environ["MINIMAX_H3_VAE_DECODER_FUSED_NORM"] = previous
torch.testing.assert_close(fused, eager, rtol=1e-3, atol=1e-3)