From 284cda01fb2c029892c421844fb1e1fe90f930e1 Mon Sep 17 00:00:00 2001 From: WenhaoZhang <42087078+niehen6174@users.noreply.github.com> Date: Mon, 5 Oct 2026 17:30:39 +0800 Subject: [PATCH] [diffusion] optimization: fuse the video VAE decoder's RMSNorm and QK RoPE for MiniMax-H3 (#41906) Co-authored-by: Mick Qian --- .../sglang/kernels/ops/diffusion/__init__.py | 24 ++ .../ops/diffusion/norm/h3_vae_fused_norm.py | 299 ++++++++++++++++++ .../vaes/minimax_h3_video_vae/attention.py | 55 +++- .../vaes/minimax_h3_video_vae/vae_vit.py | 108 ++++++- .../vaes/minimax_h3_video_vae/vit_utils.py | 15 +- .../test_minimax_h3_vae_parallel_modes.py | 114 ++++++- .../ops/diffusion/test_h3_vae_fused_norm.py | 79 +++++ 7 files changed, 672 insertions(+), 22 deletions(-) create mode 100644 python/sglang/kernels/ops/diffusion/norm/h3_vae_fused_norm.py create mode 100644 test/registered/kernels/ops/diffusion/test_h3_vae_fused_norm.py diff --git a/python/sglang/kernels/ops/diffusion/__init__.py b/python/sglang/kernels/ops/diffusion/__init__.py index 6fd1ce982e..9b11749570 100644 --- a/python/sglang/kernels/ops/diffusion/__init__.py +++ b/python/sglang/kernels/ops/diffusion/__init__.py @@ -238,6 +238,27 @@ _SPECS: tuple[tuple[str, KernelBackend, str, frozenset, str], ...] = ( _CUDA, "Fused in-place QK RMS-norm + RoPE.", ), + ( + "diffusion.h3_vae_rmsnorm", + KernelBackend.TRITON, + "norm.h3_vae_fused_norm:h3_vae_rmsnorm", + _CUDA, + "FP32-accumulation RMSNorm for the MiniMax-H3 video VAE decoder.", + ), + ( + "diffusion.h3_vae_scale_add_rmsnorm", + KernelBackend.TRITON, + "norm.h3_vae_fused_norm:h3_vae_scale_add_rmsnorm", + _CUDA, + "Scaled residual add plus the following RMSNorm for the H3 VAE decoder.", + ), + ( + "diffusion.h3_vae_qk_rmsnorm_rope", + KernelBackend.TRITON, + "norm.h3_vae_fused_norm:h3_vae_qk_rmsnorm_rope", + _CUDA, + "Affine-free Q/K RMSNorm plus partial NeoX RoPE for the H3 VAE decoder.", + ), ( "diffusion.flux2_layernorm_modulate_fp8_quant", KernelBackend.KDA, @@ -769,6 +790,9 @@ _EXPORTS: dict[str, str] = { "unmount_lingbot_video_gated_residual": "sites.lingbot_video_gated_residual_site", "can_use_rmsnorm_scale_shift_per_token": "norm.rmsnorm_scale_shift_triton", "rmsnorm_scale_shift_per_token": "norm.rmsnorm_scale_shift_triton", + "h3_vae_rmsnorm": "norm.h3_vae_fused_norm", + "h3_vae_scale_add_rmsnorm": "norm.h3_vae_fused_norm", + "h3_vae_qk_rmsnorm_rope": "norm.h3_vae_fused_norm", "mark_sana_video_linear_attention_site": "sites.sana_video_linear_attention_site", "mount_sana_video_linear_attention": "sites.sana_video_linear_attention_site", "sana_video_linear_attention_active": "sites.sana_video_linear_attention_site", diff --git a/python/sglang/kernels/ops/diffusion/norm/h3_vae_fused_norm.py b/python/sglang/kernels/ops/diffusion/norm/h3_vae_fused_norm.py new file mode 100644 index 0000000000..ff58b910da --- /dev/null +++ b/python/sglang/kernels/ops/diffusion/norm/h3_vae_fused_norm.py @@ -0,0 +1,299 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Fused norm kernels for the MiniMax-H3 video VAE decoder. + +The eager block casts the residual to FP32, runs RMSNorm, then adds +``update * scale`` as a separate kernel. Under FP16 autocast that add +promotes the residual to FP32, so later layers carry an FP32 residual and an +FP16 attention/FFN update. ``h3_vae_scale_add_rmsnorm`` does the add and the +next RMSNorm in one launch: the sum stays in registers for the variance, and +the stored residual matches the eager FP32 add. + +``h3_vae_qk_rmsnorm_rope`` is the affine-free head-64 path: RMSNorm, round to +the activation dtype, then partial NeoX RoPE on the first 48 channels. +""" + +from __future__ import annotations + +import torch +import triton # type: ignore +import triton.language as tl # type: ignore + +from sglang.kernels.ops.diffusion.common.numerics import mul_rn_f32 + +_MAX_BLOCK = 4096 + + +@triton.jit +def _rmsnorm_kernel( + out_ptr, + x_ptr, + weight_ptr, + rows, + eps, + D: tl.constexpr, + BLOCK: tl.constexpr, + OUT_KIND: tl.constexpr, +): + row = tl.program_id(0) + if row >= rows: + return + cols = tl.arange(0, BLOCK) + mask = cols < D + x = tl.load(x_ptr + row * D + cols, mask=mask, other=0.0).to(tl.float32) + rstd = tl.rsqrt(tl.sum(x * x, axis=0) / D + eps) + weight = tl.load(weight_ptr + cols, mask=mask, other=0.0).to(tl.float32) + y = x * rstd * weight + if OUT_KIND == 1: + y = y.to(tl.float16) + elif OUT_KIND == 2: + y = y.to(tl.bfloat16) + tl.store(out_ptr + row * D + cols, y, mask=mask) + + +@triton.jit +def _scale_add_rmsnorm_kernel( + res_in_ptr, + update_ptr, + res_out_ptr, + out_ptr, + scale_ptr, + weight_ptr, + rows, + eps, + D: tl.constexpr, + BLOCK: tl.constexpr, +): + row = tl.program_id(0) + if row >= rows: + return + cols = tl.arange(0, BLOCK) + mask = cols < D + residual = tl.load(res_in_ptr + row * D + cols, mask=mask, other=0.0).to(tl.float32) + update = tl.load(update_ptr + row * D + cols, mask=mask, other=0.0).to(tl.float32) + scale = tl.load(scale_ptr + cols, mask=mask, other=0.0).to(tl.float32) + # Same rounding as the eager FP32 residual add: a rounded multiply, not an FMA. + acc = residual + mul_rn_f32(update, scale) + tl.store(res_out_ptr + row * D + cols, acc, mask=mask) + rstd = tl.rsqrt(tl.sum(acc * acc, axis=0) / D + eps) + weight = tl.load(weight_ptr + cols, mask=mask, other=0.0).to(tl.float32) + tl.store(out_ptr + row * D + cols, acc * rstd * weight, mask=mask) + + +@triton.jit +def _qk_rmsnorm_rope_kernel( + q_ptr, + k_ptr, + qo_ptr, + ko_ptr, + cos_ptr, + sin_ptr, + stride_qb, + stride_qs, + stride_qh, + stride_kb, + stride_ks, + stride_kh, + seq_len, + n_heads, + eps, +): + row = tl.program_id(0).to(tl.int64) + heads = n_heads + token = row // heads + head = row % heads + seq = token % seq_len + batch = token // seq_len + cols = tl.arange(0, 64) + rope_mask = cols < 48 + q_base = q_ptr + batch * stride_qb + seq * stride_qs + head * stride_qh + k_base = k_ptr + batch * stride_kb + seq * stride_ks + head * stride_kh + q = tl.load(q_base + cols).to(tl.float32) + k = tl.load(k_base + cols).to(tl.float32) + q = (q * tl.rsqrt(tl.sum(q * q, axis=0) / 64 + eps)).to(tl.float16) + k = (k * tl.rsqrt(tl.sum(k * k, axis=0) / 64 + eps)).to(tl.float16) + cos = tl.load(cos_ptr + seq * 48 + cols, mask=rope_mask, other=0.0).to(tl.float16) + sin = tl.load(sin_ptr + seq * 48 + cols, mask=rope_mask, other=0.0).to(tl.float16) + # NeoX rotate-half: [-x2, x1] on the first 48 channels. + src = tl.where(cols < 24, cols + 24, cols - 24) + sign = tl.where(cols < 24, -1.0, 1.0) + q_rot = (tl.gather(q.to(tl.float32), src, 0) * sign).to(tl.float16) + k_rot = (tl.gather(k.to(tl.float32), src, 0) * sign).to(tl.float16) + yq = ( + (q.to(tl.float32) * cos.to(tl.float32)).to(tl.float16).to(tl.float32) + + (q_rot.to(tl.float32) * sin.to(tl.float32)).to(tl.float16).to(tl.float32) + ).to(tl.float16) + yk = ( + (k.to(tl.float32) * cos.to(tl.float32)).to(tl.float16).to(tl.float32) + + (k_rot.to(tl.float32) * sin.to(tl.float32)).to(tl.float16).to(tl.float32) + ).to(tl.float16) + yq = tl.where(rope_mask, yq, q) + yk = tl.where(rope_mask, yk, k) + tl.store(qo_ptr + row * 64 + cols, yq) + tl.store(ko_ptr + row * 64 + cols, yk) + + +def _out_kind(dtype: torch.dtype) -> int: + if dtype == torch.float32: + return 0 + if dtype == torch.float16: + return 1 + if dtype == torch.bfloat16: + return 2 + raise ValueError(f"unsupported H3 VAE norm dtype {dtype}") + + +def _launch_config(dim: int) -> tuple[int, int]: + block = triton.next_power_of_2(dim) + if block > _MAX_BLOCK: + raise ValueError(f"H3 VAE fused RMSNorm dim {dim} exceeds {_MAX_BLOCK}") + num_warps = min(8, max(1, block // 128)) + return block, num_warps + + +def _rows_of(tensor: torch.Tensor) -> tuple[torch.Tensor, int, int]: + if tensor.shape[-1] > _MAX_BLOCK or tensor.numel() == 0: + raise ValueError("H3 VAE fused RMSNorm input is empty or wider than one tile") + flat = tensor.reshape(-1, tensor.shape[-1]) + if not flat.is_contiguous(): + flat = flat.contiguous() + return flat, flat.shape[0], flat.shape[1] + + +def h3_vae_rmsnorm( + hidden: torch.Tensor, weight: torch.Tensor, eps: float +) -> torch.Tensor: + """FP32 RMSNorm. The stored dtype matches ``hidden`` so it lines up with ``.to(dtype)``.""" + flat, rows, dim = _rows_of(hidden) + if weight.shape != (dim,) or weight.device != hidden.device: + raise ValueError("H3 VAE RMSNorm weight must match the last dimension") + out = torch.empty(flat.shape, device=hidden.device, dtype=hidden.dtype) + block, num_warps = _launch_config(dim) + _rmsnorm_kernel[(rows,)]( + out, + flat, + weight.contiguous(), + rows, + float(eps), + D=dim, + BLOCK=block, + OUT_KIND=_out_kind(hidden.dtype), + num_warps=num_warps, + ) + return out.view(hidden.shape) + + +def h3_vae_scale_add_rmsnorm( + residual: torch.Tensor, + update: torch.Tensor, + scale: torch.Tensor, + weight: torch.Tensor, + eps: float, +) -> tuple[torch.Tensor, torch.Tensor]: + """``residual + update * scale`` in FP32, then RMSNorm of that sum. + + The residual is updated in place when it is already a contiguous FP32 + tensor. Otherwise a new FP32 residual is returned, matching the dtype the + eager autocast add promotes to. + """ + if residual.shape != update.shape: + raise ValueError( + "H3 VAE scale-add expects residual and update to share a shape" + ) + flat, rows, dim = _rows_of(residual) + update_flat, _, update_dim = _rows_of(update) + if update_dim != dim or scale.shape != (dim,) or weight.shape != (dim,): + raise ValueError("H3 VAE scale-add vectors must match the residual width") + if ( + scale.device != residual.device + or weight.device != residual.device + or update.device != residual.device + ): + raise ValueError("H3 VAE scale-add tensors must share a device") + inplace = ( + residual.dtype == torch.float32 + and flat.data_ptr() == residual.data_ptr() + and flat.is_contiguous() + ) + res_out = ( + flat + if inplace + else torch.empty(flat.shape, device=residual.device, dtype=torch.float32) + ) + normed = torch.empty(flat.shape, device=residual.device, dtype=torch.float32) + block, num_warps = _launch_config(dim) + _scale_add_rmsnorm_kernel[(rows,)]( + flat, + update_flat, + res_out, + normed, + scale.contiguous(), + weight.contiguous(), + rows, + float(eps), + D=dim, + BLOCK=block, + num_warps=num_warps, + ) + out_shape = residual.shape + residual_out = res_out.view(out_shape) + if inplace: + return residual, normed.view(out_shape) + return residual_out, normed.view(out_shape) + + +def h3_vae_qk_rmsnorm_rope( + query: torch.Tensor, + key: torch.Tensor, + eps: float, + cos: torch.Tensor, + sin: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor] | None: + """Affine-free RMSNorm plus partial NeoX RoPE for head 64 / rotary 48. + + Returns None when the tensors are outside that contract so the caller can + keep the unfused norm and RoPE. + """ + if ( + not query.is_cuda + or query.shape != key.shape + or query.dtype != key.dtype + or query.dtype != torch.float16 + or query.dim() != 4 + or query.shape[-1] != 64 + or query.stride(-1) != 1 + or key.stride(-1) != 1 + or cos.shape != sin.shape + or cos.dim() != 2 + or cos.shape[-1] != 48 + or cos.shape[0] != query.shape[1] + or cos.device != query.device + or sin.device != query.device + or key.device != query.device + ): + return None + batch, seq_len, n_heads, _ = query.shape + rows = batch * seq_len * n_heads + if rows == 0: + return None + query_out = torch.empty(rows, 64, device=query.device, dtype=query.dtype) + key_out = torch.empty(rows, 64, device=key.device, dtype=key.dtype) + _qk_rmsnorm_rope_kernel[(rows,)]( + query, + key, + query_out, + key_out, + cos.contiguous(), + sin.contiguous(), + query.stride(0), + query.stride(1), + query.stride(2), + key.stride(0), + key.stride(1), + key.stride(2), + seq_len, + n_heads, + float(eps), + num_warps=1, + ) + view = (batch, seq_len, n_heads, 64) + return query_out.view(view), key_out.view(view) diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/attention.py b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/attention.py index ec683a0da9..5307a6a5a8 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/attention.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/attention.py @@ -42,6 +42,40 @@ def _vit_norm_input(module, hidden_states): return hidden_states.to(weight.dtype if weight is not None else hidden_states.dtype) +def _try_fused_qk_rmsnorm_rope(norm_q, norm_k, query, key, rotary_pos_emb): + if not _env_flag("MINIMAX_H3_VAE_DECODER_FUSED_NORM", "1"): + return None + if ( + rotary_pos_emb is None + or torch.is_grad_enabled() + or torch.compiler.is_compiling() + or not isinstance(norm_q, nn.RMSNorm) + or not isinstance(norm_k, nn.RMSNorm) + or norm_q.weight is not None + or norm_k.weight is not None + or norm_q.eps != norm_k.eps + ): + return None + cos, sin = rotary_pos_emb[:2] + if cos.dim() != 4 or sin.shape != cos.shape or cos.shape[2] != 1: + return None + # Stacked tiles broadcast one rotary table. A real batch stride would mean + # each tile has its own positions, which this kernel does not index. + if cos.shape[0] != 1 and cos.stride(0) != 0: + return None + if sin.shape[0] != 1 and sin.stride(0) != 0: + return None + from sglang.kernels.ops.diffusion import h3_vae_qk_rmsnorm_rope + + return h3_vae_qk_rmsnorm_rope( + query, + key, + norm_q.eps, + cos[0, :, 0, :], + sin[0, :, 0, :], + ) + + def _apply_qk_norm(module, hidden_states): if ( _env_flag("MINIMAX_H3_VAE_DECODER_VIT_FP32_NORM", "1") @@ -138,13 +172,20 @@ class Attention(nn.Module): qkv = qkv.view(batch_size, seq_len, -1, 3 * self.dim_head) query, key, value = torch.chunk(qkv, 3, dim=-1) - if self.norm_q is not None: - query = _apply_qk_norm(self.norm_q, query) - if self.norm_k is not None: - key = _apply_qk_norm(self.norm_k, key) - - if rotary_pos_emb is not None: - query, key = apply_rotary_pos_emb_qk(query, key, rotary_pos_emb) + fused_qk = None + if self.norm_q is not None and self.norm_k is not None: + fused_qk = _try_fused_qk_rmsnorm_rope( + self.norm_q, self.norm_k, query, key, rotary_pos_emb + ) + if fused_qk is not None: + query, key = fused_qk + else: + if self.norm_q is not None: + query = _apply_qk_norm(self.norm_q, query) + if self.norm_k is not None: + key = _apply_qk_norm(self.norm_k, key) + if rotary_pos_emb is not None: + query, key = apply_rotary_pos_emb_qk(query, key, rotary_pos_emb) if self.attn is not None and query.dtype in (torch.float16, torch.bfloat16): hidden_states = self.attn(query, key, value) diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vae_vit.py b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vae_vit.py index 9bcbade80d..fafb9433c6 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vae_vit.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vae_vit.py @@ -9,8 +9,8 @@ from diffusers.configuration_utils import ConfigMixin, register_to_config from diffusers.models.modeling_utils import ModelMixin from diffusers.utils import logging -from .base_module import RotaryEmbeddingND, TransformerBlock -from .vit_utils import create_token_ids, prepare_rotary_pos_emb +from .base_module import RotaryEmbeddingND, TransformerBlock, _scaled_residual_add +from .vit_utils import _env_flag, create_token_ids, prepare_rotary_pos_emb logger = logging.get_logger(__name__) @@ -28,6 +28,96 @@ def _cuda_autocast_disabled(tensor: torch.Tensor): return torch.autocast("cuda", enabled=False) if tensor.is_cuda else nullcontext() +def _fused_blocks_available(blocks, hidden_states) -> bool: + if not _env_flag("MINIMAX_H3_VAE_DECODER_FUSED_NORM", "1"): + return False + if ( + not hidden_states.is_cuda + or hidden_states.dim() != 3 + or hidden_states.dtype not in (torch.float16, torch.bfloat16, torch.float32) + or torch.is_grad_enabled() + or torch.compiler.is_compiling() + or len(blocks) == 0 + or hidden_states.shape[-1] > 4096 + ): + return False + dim = hidden_states.shape[-1] + for block in blocks: + # Layerwise offload hooks live on the block. This loop calls attn and + # ff directly, so those hooks would not see the weights move. + if block._forward_pre_hooks or block._forward_hooks: + return False + if not getattr(block, "use_scale", False): + return False + norm1, norm2 = block.norm1, block.norm2 + if not isinstance(norm1, nn.RMSNorm) or not isinstance(norm2, nn.RMSNorm): + return False + if ( + norm1.weight is None + or norm2.weight is None + or norm1.weight.shape != (dim,) + or norm2.weight.shape != (dim,) + or block.scale1.shape != (dim,) + or block.scale2.shape != (dim,) + ): + return False + return True + + +_FUSED_BLOCKS_LOGGED = False + + +def _run_fused_transformer_blocks(blocks, hidden_states, rotary_pos_emb): + global _FUSED_BLOCKS_LOGGED + from sglang.kernels.ops.diffusion import h3_vae_rmsnorm, h3_vae_scale_add_rmsnorm + + if not _FUSED_BLOCKS_LOGGED: + _FUSED_BLOCKS_LOGGED = True + logger.info( + "MiniMax H3 video VAE decoder fuses each scaled residual add into the next RMSNorm" + ) + pending = None + pending_scale = None + for block in blocks: + if pending is None: + normed = h3_vae_rmsnorm(hidden_states, block.norm1.weight, block.norm1.eps) + else: + hidden_states, normed = h3_vae_scale_add_rmsnorm( + hidden_states, + pending, + pending_scale, + block.norm1.weight, + block.norm1.eps, + ) + attn_output = block.attn(normed, rotary_pos_emb) + hidden_states, normed = h3_vae_scale_add_rmsnorm( + hidden_states, + attn_output, + block.scale1, + block.norm2.weight, + block.norm2.eps, + ) + pending = block.ff(normed) + pending_scale = block.scale2 + return _scaled_residual_add(hidden_states, pending, pending_scale) + + +def _expand_rotary_batch(rotary_pos_emb, batch: int): + """Broadcast a batch-1 rotary cache across stacked spatial tiles.""" + cos, sin, *extra = rotary_pos_emb + if cos.shape[0] == batch: + return rotary_pos_emb + if cos.shape[0] != 1: + raise ValueError( + f"MiniMax H3 VAE rotary cache batch must be 1 or {batch}, got {cos.shape[0]}" + ) + return ( + cos.expand(batch, -1, -1, -1), + sin.expand(batch, -1, -1, -1), + *extra, + ) + + def _pack_tensors_3d(tensors, patch_size, patch_size_t): batch_size, num_channels_tensors, temporal, height, width = tensors.shape @@ -134,6 +224,10 @@ class ViTBase(ModelMixin, ConfigMixin): return hidden_states, img_ids def forward_transformer_blocks(self, hidden_states, rotary_pos_emb): + if _fused_blocks_available(self.transformer_blocks, hidden_states): + return _run_fused_transformer_blocks( + self.transformer_blocks, hidden_states, rotary_pos_emb + ) for block in self.transformer_blocks: hidden_states = block(hidden_states, rotary_pos_emb) return hidden_states @@ -326,10 +420,15 @@ class ViT3DDecoder(ViTBase): if cache_hit: rotary_pos_emb = cache_record[2] else: + # Every stacked tile has the same coordinates, so one rotary + # table is broadcast across the batch. That keeps the fused + # NeoX kernel, which indexes a single sequence, on the batched path. rotary_pos_emb = prepare_rotary_pos_emb( - self.pos_embed(img_ids), + self.pos_embed(img_ids[:1]), dtype=rotary_dtype, ) + if B > 1: + rotary_pos_emb = _expand_rotary_batch(rotary_pos_emb, B) if cache_enabled: self._rotary_pos_emb_cache = ( cache_key, @@ -337,8 +436,7 @@ class ViT3DDecoder(ViTBase): rotary_pos_emb, ) - for block in self.transformer_blocks: - hidden_states = block(hidden_states, rotary_pos_emb) + hidden_states = self.forward_transformer_blocks(hidden_states, rotary_pos_emb) hidden_states = self.norm_out(hidden_states) diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vit_utils.py b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vit_utils.py index 000c9b6c8d..c40e1baaa0 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vit_utils.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vit_utils.py @@ -222,27 +222,30 @@ def apply_rotary_pos_emb_qk( and query.dtype == key.dtype and query.dtype in (torch.float16, torch.bfloat16) and query.dim() == 4 - and query.shape[0] == 1 and not torch.compiler.is_compiling() ): _, _, cache, positions = rotary_pos_emb + batch, seq_len = query.shape[:2] if ( cache.is_cuda and cache.dtype == query.dtype and cache.dim() == 2 - and cache.shape[0] == query.shape[1] + and cache.shape[0] == seq_len and cache.shape[1] <= query.shape[-1] and positions.is_cuda - and positions.shape == (query.shape[1],) + and positions.shape == (seq_len,) ): from sgl_kernel import rotary_embedding + # Stacked spatial tiles share one rotary table. Flatten the + # batch and replay that table once per tile. query = query.contiguous() key = key.contiguous() + token_positions = positions if batch == 1 else positions.repeat(batch) rotary_embedding( - positions, - query.view(query.shape[1], -1), - key.view(key.shape[1], -1), + token_positions, + query.view(batch * seq_len, -1), + key.view(batch * seq_len, -1), query.shape[-1], cache, True, diff --git a/python/sglang/multimodal_gen/test/unit/test_minimax_h3_vae_parallel_modes.py b/python/sglang/multimodal_gen/test/unit/test_minimax_h3_vae_parallel_modes.py index a6567a0e92..3647ff63a4 100644 --- a/python/sglang/multimodal_gen/test/unit/test_minimax_h3_vae_parallel_modes.py +++ b/python/sglang/multimodal_gen/test/unit/test_minimax_h3_vae_parallel_modes.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 """MiniMax-H3 released VAE decode contract.""" +import os import subprocess import sys import textwrap @@ -24,6 +25,12 @@ from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.attention im Attention, _apply_qk_norm, ) +from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.vae_vit import ( + ViT3DDecoder, +) +from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.vit_utils import ( + apply_rotary_pos_emb_qk, +) from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum @@ -129,8 +136,7 @@ def test_audio_snake_first_call_matches_repeated_calls(): [ sys.executable, "-c", - textwrap.dedent( - """ + textwrap.dedent(""" import torch from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_audio_vae.audio_vae import Snake1d @@ -146,9 +152,109 @@ def test_audio_snake_first_call_matches_repeated_calls(): torch.testing.assert_close(activation(x), first, rtol=0, atol=0) torch.testing.assert_close(x, original_x, rtol=0, atol=0) torch.testing.assert_close(activation.alpha, original_alpha, rtol=0, atol=0) - """ - ), + """), ], check=True, timeout=120, ) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_batched_neox_rope_matches_per_tile_rows(): + from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.vit_utils import ( + prepare_rotary_pos_emb, + ) + + cos = torch.randn(1, 5, 1, 8, device="cuda", dtype=torch.float16) + sin = torch.randn(1, 5, 1, 8, device="cuda", dtype=torch.float16) + packed = prepare_rotary_pos_emb((cos, sin), dtype=torch.float16) + query = torch.randn(3, 5, 2, 16, device="cuda", dtype=torch.float16) + key = torch.randn(3, 5, 2, 16, device="cuda", dtype=torch.float16) + expanded = ( + packed[0].expand(3, -1, -1, -1), + packed[1].expand(3, -1, -1, -1), + *packed[2:], + ) + batched_q, batched_k = apply_rotary_pos_emb_qk(query.clone(), key.clone(), expanded) + rows_q = [] + rows_k = [] + for index in range(3): + row_q, row_k = apply_rotary_pos_emb_qk( + query[index : index + 1], + key[index : index + 1], + packed, + ) + rows_q.append(row_q) + rows_k.append(row_k) + + torch.testing.assert_close(batched_q, torch.cat(rows_q, dim=0)) + torch.testing.assert_close(batched_k, torch.cat(rows_k, dim=0)) + + +def _h3_shaped_decoder(): + decoder = ViT3DDecoder( + patch_size=1, + patch_size_t=1, + in_channels=4, + out_channels=3, + num_layers=2, + heads=4, + dim_head=64, + norm_type="rms_norm", + norm_affine=True, + qk_norm_type="rms_norm", + qk_norm_affine=False, + ffn_activation_fn="silu", + ffn_use_gated=True, + rope_dim_ratio=0.75, + rope_theta=100.0, + num_register_tokens=0, + ).cuda() + decoder.eval() + for block in decoder.transformer_blocks: + block.scale1.data.normal_(0, 0.1) + block.scale2.data.normal_(0, 0.1) + decoder.prepare_autocast_linear_weights(torch.float16) + return decoder + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_fused_decoder_matches_unfused_forward(): + from sglang.multimodal_gen.runtime.managers.forward_context import ( + set_forward_context, + ) + + decoder = _h3_shaped_decoder() + latent = torch.randn(2, 4, 1, 2, 2, device="cuda", dtype=torch.float16) + previous = os.environ.get("MINIMAX_H3_VAE_DECODER_FUSED_NORM") + from sglang.kernels.ops import diffusion as diffusion_ops + + try: + with ( + torch.no_grad(), + torch.autocast("cuda", dtype=torch.float16), + set_forward_context(current_timestep=0, attn_metadata=None), + mock.patch.object( + diffusion_ops, + "h3_vae_scale_add_rmsnorm", + wraps=diffusion_ops.h3_vae_scale_add_rmsnorm, + ) as scale_add, + mock.patch.object( + diffusion_ops, + "h3_vae_qk_rmsnorm_rope", + wraps=diffusion_ops.h3_vae_qk_rmsnorm_rope, + ) as qk_rope, + ): + os.environ["MINIMAX_H3_VAE_DECODER_FUSED_NORM"] = "0" + eager = decoder(latent) + os.environ["MINIMAX_H3_VAE_DECODER_FUSED_NORM"] = "1" + fused = decoder(latent) + assert scale_add.call_count == 3 + assert qk_rope.call_count == 2 + finally: + if previous is None: + os.environ.pop("MINIMAX_H3_VAE_DECODER_FUSED_NORM", None) + else: + os.environ["MINIMAX_H3_VAE_DECODER_FUSED_NORM"] = previous + + torch.testing.assert_close(fused, eager, rtol=1e-3, atol=1e-3) diff --git a/test/registered/kernels/ops/diffusion/test_h3_vae_fused_norm.py b/test/registered/kernels/ops/diffusion/test_h3_vae_fused_norm.py new file mode 100644 index 0000000000..b7315f8a13 --- /dev/null +++ b/test/registered/kernels/ops/diffusion/test_h3_vae_fused_norm.py @@ -0,0 +1,79 @@ +"""MiniMax-H3 video VAE fused RMSNorm and Q/K RoPE.""" + +import sys + +import pytest +import torch +import torch.nn.functional as F + +from sglang.kernels.ops.diffusion import ( + h3_vae_qk_rmsnorm_rope, + h3_vae_rmsnorm, + h3_vae_scale_add_rmsnorm, +) +from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.vit_utils import ( + _apply_rotary_pos_emb_impl, +) +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="1-gpu-large") + + +@torch.inference_mode() +def test_rmsnorm_matches_pytorch_fp32_accumulation(): + rows, dim = 128, 2048 + hidden = torch.randn(4, rows // 4, dim, device="cuda") + weight = torch.randn(dim, device="cuda") + actual = h3_vae_rmsnorm(hidden, weight, 1e-5) + expected = F.rms_norm(hidden, (dim,), weight, 1e-5) + torch.testing.assert_close(actual, expected, rtol=1e-4, atol=1e-5) + + half = torch.randn(rows, dim, device="cuda", dtype=torch.float16) + actual_half = h3_vae_rmsnorm(half, weight, 1e-5) + expected_half = F.rms_norm(half.float(), (dim,), weight, 1e-5).half() + torch.testing.assert_close(actual_half, expected_half, rtol=1e-3, atol=1e-2) + + +@torch.inference_mode() +def test_scale_add_residual_matches_eager_product(): + rows, dim = 64, 256 + residual = torch.randn(2, rows // 2, dim, device="cuda") + update = torch.randn_like(residual, dtype=torch.float16) + scale = torch.randn(dim, device="cuda") + weight = torch.randn(dim, device="cuda") + original = residual.clone() + fused_residual, fused_norm = h3_vae_scale_add_rmsnorm( + residual, update, scale, weight, 1e-5 + ) + expected_residual = original + update.float() * scale + expected_norm = F.rms_norm(expected_residual, (dim,), weight, 1e-5) + torch.testing.assert_close(fused_residual, expected_residual, rtol=0, atol=0) + torch.testing.assert_close(fused_norm, expected_norm, rtol=1e-4, atol=1e-5) + assert fused_residual.data_ptr() == residual.data_ptr() + + +@torch.inference_mode() +def test_qk_rmsnorm_rope_matches_eager_neox(): + batch, seq_len, heads = 2, 17, 4 + packed = torch.randn(batch, seq_len, heads, 192, device="cuda", dtype=torch.float16) + query, key, _ = packed.chunk(3, dim=-1) + half_cos = torch.randn(seq_len, 24, device="cuda", dtype=torch.float16) + half_sin = torch.randn(seq_len, 24, device="cuda", dtype=torch.float16) + cos = torch.cat((half_cos, half_cos), dim=-1) + sin = torch.cat((half_sin, half_sin), dim=-1) + actual_q, actual_k = h3_vae_qk_rmsnorm_rope(query, key, 1e-5, cos, sin) + norm = torch.nn.RMSNorm(64, eps=1e-5, elementwise_affine=False).cuda() + rotary = ( + cos.view(1, seq_len, 1, 48).expand(batch, -1, -1, -1), + sin.view(1, seq_len, 1, 48).expand(batch, -1, -1, -1), + ) + with torch.autocast("cuda", enabled=False): + expected_q = _apply_rotary_pos_emb_impl(norm(query), rotary) + expected_k = _apply_rotary_pos_emb_impl(norm(key), rotary) + torch.testing.assert_close(actual_q, expected_q, rtol=1e-3, atol=1e-2) + torch.testing.assert_close(actual_k, expected_k, rtol=1e-3, atol=1e-2) + assert h3_vae_qk_rmsnorm_rope(query.float(), key.float(), 1e-5, cos, sin) is None + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"]))