[diffusion] perf: fuse MiniMax-H3's RMSNorm and indexed AdaLN under quality lossless (#43327)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-09 22:39:11 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5.5
parent 408f5d057a
commit 121aa8dcc7
7 changed files with 248 additions and 7 deletions
@@ -823,7 +823,13 @@ _EXPORTS: dict[str, str] = {
"mount_lingbot_video_gated_residual": "sites.lingbot_video_gated_residual_site",
"try_lingbot_video_gated_residual": "sites.lingbot_video_gated_residual_site",
"unmount_lingbot_video_gated_residual": "sites.lingbot_video_gated_residual_site",
"mark_minimax_h3_norm_modulate_site": "sites.minimax_h3_norm_modulate_site",
"minimax_h3_norm_modulate_active": "sites.minimax_h3_norm_modulate_site",
"mount_minimax_h3_norm_modulate": "sites.minimax_h3_norm_modulate_site",
"unmount_minimax_h3_norm_modulate": "sites.minimax_h3_norm_modulate_site",
"can_use_rmsnorm_scale_shift_per_token": "norm.rmsnorm_scale_shift_triton",
"can_use_rmsnorm_indexed_scale_shift": "norm.rmsnorm_scale_shift_triton",
"rmsnorm_indexed_scale_shift": "norm.rmsnorm_scale_shift_triton",
"rmsnorm_scale_shift_per_token": "norm.rmsnorm_scale_shift_triton",
"h3_vae_rmsnorm": "norm.h3_vae_fused_norm",
"h3_vae_scale_add_rmsnorm": "norm.h3_vae_fused_norm",
@@ -17,6 +17,10 @@ Unlike the bit-exact ``rmsnorm_scale_shift_bitexact`` (ERNIE), this kernel does
*not* reproduce PyTorch's parallel variance reduction order bit-for-bit, so it
is intended for the request-gated (``quality="lossless"``/``"high"``) fusion
path, matching the existing quality-gated LingBot RMSNorm fusion.
``rmsnorm_indexed_scale_shift`` is the same chain for MiniMax-H3, whose
``scale``/``shift`` rows come from small ``[M, D]`` AdaLN tables picked per
token by an index.
"""
from __future__ import annotations
@@ -143,7 +147,109 @@ def rmsnorm_scale_shift_per_token(
return _rmsnorm_scale_shift_per_token_cuda(x, weight, scale, shift, eps)
@triton.jit
def _rmsnorm_indexed_scale_shift_kernel(
y_ptr,
x_ptr,
w_ptr,
scale_ptr,
shift_ptr,
idx_ptr,
mod_row_stride,
idx_stride,
DIM: tl.constexpr,
EPS: tl.constexpr,
BLOCK_SIZE_DIM: tl.constexpr,
):
row = tl.program_id(0).to(tl.int64)
cols = tl.arange(0, BLOCK_SIZE_DIM)
mask = cols < DIM
mod_ptr = tl.load(idx_ptr + row * idx_stride) * mod_row_stride + cols
x = tl.load(x_ptr + row * DIM + cols, mask=mask, other=0.0).to(tl.float32)
rstd = tl.math.rsqrt(tl.sum(x * x, axis=0) / DIM + EPS)
w = tl.load(w_ptr + cols, mask=mask).to(tl.float32)
scale = tl.load(scale_ptr + mod_ptr, mask=mask).to(tl.float32)
shift = tl.load(shift_ptr + mod_ptr, mask=mask).to(tl.float32)
y = (x * rstd * w) * (1.0 + scale) + shift
tl.store(y_ptr + row * DIM + cols, y, mask=mask)
@register_custom_op(op_name="rmsnorm_indexed_scale_shift_cuda", out_shape="x")
def _rmsnorm_indexed_scale_shift_cuda(
x: torch.Tensor,
weight: torch.Tensor,
shift: torch.Tensor,
scale: torch.Tensor,
indices: torch.Tensor,
eps: float,
) -> torch.Tensor:
out = torch.empty_like(x)
S, D = x.shape
# one row per program: packing more rows spills the fp32 row tiles
# (16 rows of 5376 ran 24x slower on H200)
with torch.get_device_module().device(x.device):
_rmsnorm_indexed_scale_shift_kernel[(S,)](
out,
x,
weight,
scale,
shift,
indices,
scale.stride(0),
indices.stride(0),
DIM=D,
EPS=eps,
BLOCK_SIZE_DIM=triton.next_power_of_2(D),
num_warps=8,
)
return out
def can_use_rmsnorm_indexed_scale_shift(
x: torch.Tensor,
weight: torch.Tensor,
shift: torch.Tensor,
scale: torch.Tensor,
indices: torch.Tensor,
) -> bool:
return (
x.is_cuda
and x.dim() == 2
and x.is_contiguous()
and x.dtype in (torch.float16, torch.bfloat16, torch.float32)
and x.numel() > 0
and weight.is_cuda
and weight.dim() == 1
and weight.numel() == x.shape[1]
and shift.dim() == scale.dim() == 2
and shift.shape == scale.shape
and shift.shape[1] == x.shape[1]
and shift.stride(0) == scale.stride(0)
and shift.stride(1) == scale.stride(1) == 1
and indices.dim() == 1
and indices.numel() == x.shape[0]
and not indices.dtype.is_floating_point
)
def rmsnorm_indexed_scale_shift(
x: torch.Tensor,
weight: torch.Tensor,
shift: torch.Tensor,
scale: torch.Tensor,
indices: torch.Tensor,
eps: float,
) -> torch.Tensor:
"""Fused ``rmsnorm(x) * (1 + scale[indices]) + shift[indices]`` over [T, D]
rows, in fp32 with one rounding: the AdaLN modulation tables are indexed
per row instead of materialized per token."""
return _rmsnorm_indexed_scale_shift_cuda(x, weight, shift, scale, indices, eps)
__all__ = [
"can_use_rmsnorm_indexed_scale_shift",
"can_use_rmsnorm_scale_shift_per_token",
"rmsnorm_indexed_scale_shift",
"rmsnorm_scale_shift_per_token",
]
@@ -0,0 +1,41 @@
"""MiniMax-H3 fused RMSNorm + indexed AdaLN, gated by request quality.
Each H3 block runs ``nn.RMSNorm`` and then the indexed AdaLN modulation
``x * (1 + scale[idx]) + shift[idx]``, rounding to bf16 after the norm and after
every modulation op. The fused kernel keeps the same math and operand precision
in one fp32 pass with a single rounding, so ``quality="lossless"`` and ``"high"``
mount it, while ``"exact"`` keeps the reference chain bit-for-bit.
"""
from __future__ import annotations
import logging
import torch.nn as nn
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
logger = logging.getLogger(__name__)
_FUSION = QualityGatedFusion(
name="MiniMax-H3 fused RMSNorm + AdaLN",
marker_attr="_sgl_minimax_h3_norm_modulate_site",
enabled_attr="_sgl_minimax_h3_norm_modulate_enabled",
)
def mark_minimax_h3_norm_modulate_site(module: nn.Module) -> None:
"""Mark an H3 DiT block; it starts on the reference path."""
_FUSION.mark(module)
def minimax_h3_norm_modulate_active(module: nn.Module) -> bool:
return _FUSION.is_enabled(module)
def mount_minimax_h3_norm_modulate(root: nn.Module) -> bool:
return _FUSION.mount(root, logger=logger)
def unmount_minimax_h3_norm_modulate(root: nn.Module) -> None:
_FUSION.unmount(root)
@@ -25,6 +25,7 @@ from sglang.kernels.ops.activation.activation import (
from sglang.kernels.ops.diffusion import (
can_use_fused_inplace_qknorm_rope,
can_use_mxfp8_swizzled,
can_use_rmsnorm_indexed_scale_shift,
can_use_silu_mul_mxfp8,
fused_inplace_qknorm_rope,
fused_qknorm_rope_out_of_place,
@@ -32,6 +33,9 @@ from sglang.kernels.ops.diffusion import (
indexed_gate_bf16_,
indexed_scale_shift_bf16_,
indexed_scale_shift_mxfp8_,
mark_minimax_h3_norm_modulate_site,
minimax_h3_norm_modulate_active,
rmsnorm_indexed_scale_shift,
silu_mul_mxfp8,
)
from sglang.kernels.ops.layernorm.norm import fused_inplace_qknorm
@@ -392,12 +396,22 @@ def _modulate_rmsnorm_scale_shift(
indices: torch.Tensor,
*,
dtype: torch.dtype,
fused: bool = False,
) -> torch.Tensor:
"""RMSNorm + indexed AdaLN.
Keep ``nn.RMSNorm``; a fused RMSNorm+AdaLN kernel drifted 2-GPU
consistency GT for ~0.3% e2e, so it was removed.
The reference chain rounds after the norm and after every modulation op.
``fused`` (quality lossless / high) runs one fp32 pass with one rounding,
which moves the denoising trajectory at rounding level.
"""
if (
fused
and x.dtype == dtype == _BF16_DTYPE
and norm.weight is not None
and can_use_rmsnorm_indexed_scale_shift(x, norm.weight, shift, scale, indices)
):
eps = norm.eps if norm.eps is not None else torch.finfo(x.dtype).eps
return rmsnorm_indexed_scale_shift(x, norm.weight, shift, scale, indices, eps)
return _modulate_scale_shift(norm(x), shift, scale, indices, dtype=dtype)
@@ -1562,6 +1576,7 @@ class MiniMaxH3DiTBlock(nn.Module):
super().__init__()
self.norm1 = _norm(arch.hidden_size, eps=arch.norm_eps)
self.norm2 = _norm(arch.hidden_size, eps=arch.norm_eps)
mark_minimax_h3_norm_modulate_site(self)
self.attn = MiniMaxH3Attention(
arch,
quant_config,
@@ -1634,6 +1649,7 @@ class MiniMaxH3DiTBlock(nn.Module):
scale_msa,
combined_indices,
dtype=_BF16_DTYPE,
fused=minimax_h3_norm_modulate_active(self),
)
h = self.attn(
h,
@@ -1676,6 +1692,7 @@ class MiniMaxH3DiTBlock(nn.Module):
scale_mlp,
combined_indices,
dtype=_BF16_DTYPE,
fused=minimax_h3_norm_modulate_active(self),
)
h = self.mlp(h)
# `residual` is block-local here (see above), so this stays in-place
@@ -31,6 +31,7 @@ from sglang.kernels.ops.diffusion import (
mount_lingbot_video_rmsnorm,
mount_ltx2_qknorm_split_rope,
mount_ltx2_rms_norm_modulate,
mount_minimax_h3_norm_modulate,
mount_nvfp4_bias_gelu,
mount_qwen_image_added_qkv,
mount_sana_video_linear_attention,
@@ -44,6 +45,7 @@ from sglang.kernels.ops.diffusion import (
unmount_lingbot_video_rmsnorm,
unmount_ltx2_qknorm_split_rope,
unmount_ltx2_rms_norm_modulate,
unmount_minimax_h3_norm_modulate,
unmount_nvfp4_bias_gelu,
unmount_qwen_image_added_qkv,
unmount_sana_video_linear_attention,
@@ -261,6 +263,12 @@ _QUALITY_FUSION_HANDLERS: tuple[
mount_helios_gated_residual,
unmount_helios_gated_residual,
),
(
"lossless",
"MiniMax-H3 fused RMSNorm + AdaLN",
mount_minimax_h3_norm_modulate,
unmount_minimax_h3_norm_modulate,
),
(
# the first attention GEMM takes BF16 inputs where the reference
# promotes them to FP32