[diffusion] perf: fuse MiniMax-H3's RMSNorm and indexed AdaLN under quality lossless (#43327)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-09 22:39:11 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5.5
parent 408f5d057a
commit 121aa8dcc7
7 changed files with 248 additions and 7 deletions
@@ -1014,8 +1014,9 @@ levels:
- `"exact"`: the exact reference path. Output is bit-exact
against the reference implementation and the CI ground truth.
- `"lossless"` (default): includes the global fusion-only tier but does not enable
Cache-DiT or another approximate optimization. The denoise path is the same
as `exact`; VAE decode fuses the ViT decoder's per-head QK RMSNorm and
Cache-DiT or another approximate optimization. Each DiT block runs its
RMSNorm and AdaLN modulation as one kernel that rounds once instead of after
every op, and VAE decode fuses the ViT decoder's per-head QK RMSNorm and
RoPE into one launch and runs its attention on cuDNN SDPA (rounding-level
differences only).
- `"high"`: the audited accelerated path. Quality is guaranteed (the audited
@@ -1064,9 +1065,10 @@ quality-gated VAE fusions.
<Tab title="lossless (default)">
The global fusion-only tier. It does not enable MiniMax-H3 Cache-DiT and
follows the same H3 denoise path as `exact`; only the VAE decoder's fused
QK RMSNorm + RoPE and cuDNN SDPA attention differ.
The global fusion-only tier. It does not enable MiniMax-H3 Cache-DiT. It
differs from `exact` only in rounding: the DiT blocks' fused RMSNorm + AdaLN
modulation, and the VAE decoder's fused QK RMSNorm + RoPE and cuDNN SDPA
attention.
Omitting the field selects this level.
```json Request field
@@ -823,7 +823,13 @@ _EXPORTS: dict[str, str] = {
"mount_lingbot_video_gated_residual": "sites.lingbot_video_gated_residual_site",
"try_lingbot_video_gated_residual": "sites.lingbot_video_gated_residual_site",
"unmount_lingbot_video_gated_residual": "sites.lingbot_video_gated_residual_site",
"mark_minimax_h3_norm_modulate_site": "sites.minimax_h3_norm_modulate_site",
"minimax_h3_norm_modulate_active": "sites.minimax_h3_norm_modulate_site",
"mount_minimax_h3_norm_modulate": "sites.minimax_h3_norm_modulate_site",
"unmount_minimax_h3_norm_modulate": "sites.minimax_h3_norm_modulate_site",
"can_use_rmsnorm_scale_shift_per_token": "norm.rmsnorm_scale_shift_triton",
"can_use_rmsnorm_indexed_scale_shift": "norm.rmsnorm_scale_shift_triton",
"rmsnorm_indexed_scale_shift": "norm.rmsnorm_scale_shift_triton",
"rmsnorm_scale_shift_per_token": "norm.rmsnorm_scale_shift_triton",
"h3_vae_rmsnorm": "norm.h3_vae_fused_norm",
"h3_vae_scale_add_rmsnorm": "norm.h3_vae_fused_norm",
@@ -17,6 +17,10 @@ Unlike the bit-exact ``rmsnorm_scale_shift_bitexact`` (ERNIE), this kernel does
*not* reproduce PyTorch's parallel variance reduction order bit-for-bit, so it
is intended for the request-gated (``quality="lossless"``/``"high"``) fusion
path, matching the existing quality-gated LingBot RMSNorm fusion.
``rmsnorm_indexed_scale_shift`` is the same chain for MiniMax-H3, whose
``scale``/``shift`` rows come from small ``[M, D]`` AdaLN tables picked per
token by an index.
"""
from __future__ import annotations
@@ -143,7 +147,109 @@ def rmsnorm_scale_shift_per_token(
return _rmsnorm_scale_shift_per_token_cuda(x, weight, scale, shift, eps)
@triton.jit
def _rmsnorm_indexed_scale_shift_kernel(
y_ptr,
x_ptr,
w_ptr,
scale_ptr,
shift_ptr,
idx_ptr,
mod_row_stride,
idx_stride,
DIM: tl.constexpr,
EPS: tl.constexpr,
BLOCK_SIZE_DIM: tl.constexpr,
):
row = tl.program_id(0).to(tl.int64)
cols = tl.arange(0, BLOCK_SIZE_DIM)
mask = cols < DIM
mod_ptr = tl.load(idx_ptr + row * idx_stride) * mod_row_stride + cols
x = tl.load(x_ptr + row * DIM + cols, mask=mask, other=0.0).to(tl.float32)
rstd = tl.math.rsqrt(tl.sum(x * x, axis=0) / DIM + EPS)
w = tl.load(w_ptr + cols, mask=mask).to(tl.float32)
scale = tl.load(scale_ptr + mod_ptr, mask=mask).to(tl.float32)
shift = tl.load(shift_ptr + mod_ptr, mask=mask).to(tl.float32)
y = (x * rstd * w) * (1.0 + scale) + shift
tl.store(y_ptr + row * DIM + cols, y, mask=mask)
@register_custom_op(op_name="rmsnorm_indexed_scale_shift_cuda", out_shape="x")
def _rmsnorm_indexed_scale_shift_cuda(
x: torch.Tensor,
weight: torch.Tensor,
shift: torch.Tensor,
scale: torch.Tensor,
indices: torch.Tensor,
eps: float,
) -> torch.Tensor:
out = torch.empty_like(x)
S, D = x.shape
# one row per program: packing more rows spills the fp32 row tiles
# (16 rows of 5376 ran 24x slower on H200)
with torch.get_device_module().device(x.device):
_rmsnorm_indexed_scale_shift_kernel[(S,)](
out,
x,
weight,
scale,
shift,
indices,
scale.stride(0),
indices.stride(0),
DIM=D,
EPS=eps,
BLOCK_SIZE_DIM=triton.next_power_of_2(D),
num_warps=8,
)
return out
def can_use_rmsnorm_indexed_scale_shift(
x: torch.Tensor,
weight: torch.Tensor,
shift: torch.Tensor,
scale: torch.Tensor,
indices: torch.Tensor,
) -> bool:
return (
x.is_cuda
and x.dim() == 2
and x.is_contiguous()
and x.dtype in (torch.float16, torch.bfloat16, torch.float32)
and x.numel() > 0
and weight.is_cuda
and weight.dim() == 1
and weight.numel() == x.shape[1]
and shift.dim() == scale.dim() == 2
and shift.shape == scale.shape
and shift.shape[1] == x.shape[1]
and shift.stride(0) == scale.stride(0)
and shift.stride(1) == scale.stride(1) == 1
and indices.dim() == 1
and indices.numel() == x.shape[0]
and not indices.dtype.is_floating_point
)
def rmsnorm_indexed_scale_shift(
x: torch.Tensor,
weight: torch.Tensor,
shift: torch.Tensor,
scale: torch.Tensor,
indices: torch.Tensor,
eps: float,
) -> torch.Tensor:
"""Fused ``rmsnorm(x) * (1 + scale[indices]) + shift[indices]`` over [T, D]
rows, in fp32 with one rounding: the AdaLN modulation tables are indexed
per row instead of materialized per token."""
return _rmsnorm_indexed_scale_shift_cuda(x, weight, shift, scale, indices, eps)
__all__ = [
"can_use_rmsnorm_indexed_scale_shift",
"can_use_rmsnorm_scale_shift_per_token",
"rmsnorm_indexed_scale_shift",
"rmsnorm_scale_shift_per_token",
]
@@ -0,0 +1,41 @@
"""MiniMax-H3 fused RMSNorm + indexed AdaLN, gated by request quality.
Each H3 block runs ``nn.RMSNorm`` and then the indexed AdaLN modulation
``x * (1 + scale[idx]) + shift[idx]``, rounding to bf16 after the norm and after
every modulation op. The fused kernel keeps the same math and operand precision
in one fp32 pass with a single rounding, so ``quality="lossless"`` and ``"high"``
mount it, while ``"exact"`` keeps the reference chain bit-for-bit.
"""
from __future__ import annotations
import logging
import torch.nn as nn
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
logger = logging.getLogger(__name__)
_FUSION = QualityGatedFusion(
name="MiniMax-H3 fused RMSNorm + AdaLN",
marker_attr="_sgl_minimax_h3_norm_modulate_site",
enabled_attr="_sgl_minimax_h3_norm_modulate_enabled",
)
def mark_minimax_h3_norm_modulate_site(module: nn.Module) -> None:
"""Mark an H3 DiT block; it starts on the reference path."""
_FUSION.mark(module)
def minimax_h3_norm_modulate_active(module: nn.Module) -> bool:
return _FUSION.is_enabled(module)
def mount_minimax_h3_norm_modulate(root: nn.Module) -> bool:
return _FUSION.mount(root, logger=logger)
def unmount_minimax_h3_norm_modulate(root: nn.Module) -> None:
_FUSION.unmount(root)
@@ -25,6 +25,7 @@ from sglang.kernels.ops.activation.activation import (
from sglang.kernels.ops.diffusion import (
can_use_fused_inplace_qknorm_rope,
can_use_mxfp8_swizzled,
can_use_rmsnorm_indexed_scale_shift,
can_use_silu_mul_mxfp8,
fused_inplace_qknorm_rope,
fused_qknorm_rope_out_of_place,
@@ -32,6 +33,9 @@ from sglang.kernels.ops.diffusion import (
indexed_gate_bf16_,
indexed_scale_shift_bf16_,
indexed_scale_shift_mxfp8_,
mark_minimax_h3_norm_modulate_site,
minimax_h3_norm_modulate_active,
rmsnorm_indexed_scale_shift,
silu_mul_mxfp8,
)
from sglang.kernels.ops.layernorm.norm import fused_inplace_qknorm
@@ -392,12 +396,22 @@ def _modulate_rmsnorm_scale_shift(
indices: torch.Tensor,
*,
dtype: torch.dtype,
fused: bool = False,
) -> torch.Tensor:
"""RMSNorm + indexed AdaLN.
Keep ``nn.RMSNorm``; a fused RMSNorm+AdaLN kernel drifted 2-GPU
consistency GT for ~0.3% e2e, so it was removed.
The reference chain rounds after the norm and after every modulation op.
``fused`` (quality lossless / high) runs one fp32 pass with one rounding,
which moves the denoising trajectory at rounding level.
"""
if (
fused
and x.dtype == dtype == _BF16_DTYPE
and norm.weight is not None
and can_use_rmsnorm_indexed_scale_shift(x, norm.weight, shift, scale, indices)
):
eps = norm.eps if norm.eps is not None else torch.finfo(x.dtype).eps
return rmsnorm_indexed_scale_shift(x, norm.weight, shift, scale, indices, eps)
return _modulate_scale_shift(norm(x), shift, scale, indices, dtype=dtype)
@@ -1562,6 +1576,7 @@ class MiniMaxH3DiTBlock(nn.Module):
super().__init__()
self.norm1 = _norm(arch.hidden_size, eps=arch.norm_eps)
self.norm2 = _norm(arch.hidden_size, eps=arch.norm_eps)
mark_minimax_h3_norm_modulate_site(self)
self.attn = MiniMaxH3Attention(
arch,
quant_config,
@@ -1634,6 +1649,7 @@ class MiniMaxH3DiTBlock(nn.Module):
scale_msa,
combined_indices,
dtype=_BF16_DTYPE,
fused=minimax_h3_norm_modulate_active(self),
)
h = self.attn(
h,
@@ -1676,6 +1692,7 @@ class MiniMaxH3DiTBlock(nn.Module):
scale_mlp,
combined_indices,
dtype=_BF16_DTYPE,
fused=minimax_h3_norm_modulate_active(self),
)
h = self.mlp(h)
# `residual` is block-local here (see above), so this stays in-place
@@ -31,6 +31,7 @@ from sglang.kernels.ops.diffusion import (
mount_lingbot_video_rmsnorm,
mount_ltx2_qknorm_split_rope,
mount_ltx2_rms_norm_modulate,
mount_minimax_h3_norm_modulate,
mount_nvfp4_bias_gelu,
mount_qwen_image_added_qkv,
mount_sana_video_linear_attention,
@@ -44,6 +45,7 @@ from sglang.kernels.ops.diffusion import (
unmount_lingbot_video_rmsnorm,
unmount_ltx2_qknorm_split_rope,
unmount_ltx2_rms_norm_modulate,
unmount_minimax_h3_norm_modulate,
unmount_nvfp4_bias_gelu,
unmount_qwen_image_added_qkv,
unmount_sana_video_linear_attention,
@@ -261,6 +263,12 @@ _QUALITY_FUSION_HANDLERS: tuple[
mount_helios_gated_residual,
unmount_helios_gated_residual,
),
(
"lossless",
"MiniMax-H3 fused RMSNorm + AdaLN",
mount_minimax_h3_norm_modulate,
unmount_minimax_h3_norm_modulate,
),
(
# the first attention GEMM takes BF16 inputs where the reference
# promotes them to FP32
@@ -0,0 +1,61 @@
"""MiniMax-H3's fused RMSNorm + indexed AdaLN (quality lossless/high) against the
same math in fp32 with one rounding, and against the reference rounding chain."""
import sys
import pytest
import torch
import torch.nn as nn
from sglang.kernels.ops.diffusion import (
can_use_rmsnorm_indexed_scale_shift,
indexed_scale_shift_bf16_,
rmsnorm_indexed_scale_shift,
)
from sglang.test.ci.ci_register import register_cuda_ci
register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="4-gpu-b200")
EPS = 1e-5
def _inputs(rows: int, hidden: int, seed: int):
g = torch.Generator(device="cuda").manual_seed(seed)
x = (2 * torch.randn(rows, hidden, device="cuda", generator=g)).to(torch.bfloat16)
weight = (1 + 0.1 * torch.randn(hidden, device="cuda", generator=g)).to(
torch.bfloat16
)
# H3's shift/scale are chunks of one [M, 6 * hidden] AdaLN output
adaln = (0.1 * torch.randn(3, 6 * hidden, device="cuda", generator=g)).to(
torch.bfloat16
)
shift, scale = adaln[:, :hidden], adaln[:, hidden : 2 * hidden]
indices = torch.randint(0, 3, (rows,), device="cuda", generator=g)
return x, weight, shift, scale, indices
@pytest.mark.parametrize("hidden", [1024, 5376])
@pytest.mark.parametrize("rows", [1, 37, 1025])
def test_matches_fp32_single_rounding(rows: int, hidden: int) -> None:
x, weight, shift, scale, indices = _inputs(rows, hidden, rows + hidden)
assert can_use_rmsnorm_indexed_scale_shift(x, weight, shift, scale, indices)
xf = x.float()
normed = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + EPS) * weight.float()
ref = normed * (1 + scale[indices].float()) + shift[indices].float()
got = rmsnorm_indexed_scale_shift(x, weight, shift, scale, indices, EPS)
# only the reduction order differs: at most one bf16 rounding step apart
torch.testing.assert_close(got.float(), ref, rtol=2**-7, atol=2**-7)
def test_stays_within_rounding_of_the_reference_chain() -> None:
x, weight, shift, scale, indices = _inputs(1025, 5376, 7)
norm = nn.RMSNorm(5376, eps=EPS, device="cuda", dtype=torch.bfloat16)
with torch.no_grad():
norm.weight.copy_(weight)
chain = indexed_scale_shift_bf16_(norm(x), shift, scale, indices)
got = rmsnorm_indexed_scale_shift(x, weight, shift, scale, indices, EPS)
torch.testing.assert_close(got.float(), chain.float(), rtol=2**-6, atol=2**-6)
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-v"]))