[diffusion] perf: fuse MiniMax-H3's RMSNorm and indexed AdaLN under quality lossless (#43327)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5.5
parent
408f5d057a
commit
121aa8dcc7
@@ -1014,8 +1014,9 @@ levels:
|
||||
- `"exact"`: the exact reference path. Output is bit-exact
|
||||
against the reference implementation and the CI ground truth.
|
||||
- `"lossless"` (default): includes the global fusion-only tier but does not enable
|
||||
Cache-DiT or another approximate optimization. The denoise path is the same
|
||||
as `exact`; VAE decode fuses the ViT decoder's per-head QK RMSNorm and
|
||||
Cache-DiT or another approximate optimization. Each DiT block runs its
|
||||
RMSNorm and AdaLN modulation as one kernel that rounds once instead of after
|
||||
every op, and VAE decode fuses the ViT decoder's per-head QK RMSNorm and
|
||||
RoPE into one launch and runs its attention on cuDNN SDPA (rounding-level
|
||||
differences only).
|
||||
- `"high"`: the audited accelerated path. Quality is guaranteed (the audited
|
||||
@@ -1064,9 +1065,10 @@ quality-gated VAE fusions.
|
||||
|
||||
<Tab title="lossless (default)">
|
||||
|
||||
The global fusion-only tier. It does not enable MiniMax-H3 Cache-DiT and
|
||||
follows the same H3 denoise path as `exact`; only the VAE decoder's fused
|
||||
QK RMSNorm + RoPE and cuDNN SDPA attention differ.
|
||||
The global fusion-only tier. It does not enable MiniMax-H3 Cache-DiT. It
|
||||
differs from `exact` only in rounding: the DiT blocks' fused RMSNorm + AdaLN
|
||||
modulation, and the VAE decoder's fused QK RMSNorm + RoPE and cuDNN SDPA
|
||||
attention.
|
||||
Omitting the field selects this level.
|
||||
|
||||
```json Request field
|
||||
|
||||
@@ -823,7 +823,13 @@ _EXPORTS: dict[str, str] = {
|
||||
"mount_lingbot_video_gated_residual": "sites.lingbot_video_gated_residual_site",
|
||||
"try_lingbot_video_gated_residual": "sites.lingbot_video_gated_residual_site",
|
||||
"unmount_lingbot_video_gated_residual": "sites.lingbot_video_gated_residual_site",
|
||||
"mark_minimax_h3_norm_modulate_site": "sites.minimax_h3_norm_modulate_site",
|
||||
"minimax_h3_norm_modulate_active": "sites.minimax_h3_norm_modulate_site",
|
||||
"mount_minimax_h3_norm_modulate": "sites.minimax_h3_norm_modulate_site",
|
||||
"unmount_minimax_h3_norm_modulate": "sites.minimax_h3_norm_modulate_site",
|
||||
"can_use_rmsnorm_scale_shift_per_token": "norm.rmsnorm_scale_shift_triton",
|
||||
"can_use_rmsnorm_indexed_scale_shift": "norm.rmsnorm_scale_shift_triton",
|
||||
"rmsnorm_indexed_scale_shift": "norm.rmsnorm_scale_shift_triton",
|
||||
"rmsnorm_scale_shift_per_token": "norm.rmsnorm_scale_shift_triton",
|
||||
"h3_vae_rmsnorm": "norm.h3_vae_fused_norm",
|
||||
"h3_vae_scale_add_rmsnorm": "norm.h3_vae_fused_norm",
|
||||
|
||||
@@ -17,6 +17,10 @@ Unlike the bit-exact ``rmsnorm_scale_shift_bitexact`` (ERNIE), this kernel does
|
||||
*not* reproduce PyTorch's parallel variance reduction order bit-for-bit, so it
|
||||
is intended for the request-gated (``quality="lossless"``/``"high"``) fusion
|
||||
path, matching the existing quality-gated LingBot RMSNorm fusion.
|
||||
|
||||
``rmsnorm_indexed_scale_shift`` is the same chain for MiniMax-H3, whose
|
||||
``scale``/``shift`` rows come from small ``[M, D]`` AdaLN tables picked per
|
||||
token by an index.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -143,7 +147,109 @@ def rmsnorm_scale_shift_per_token(
|
||||
return _rmsnorm_scale_shift_per_token_cuda(x, weight, scale, shift, eps)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _rmsnorm_indexed_scale_shift_kernel(
|
||||
y_ptr,
|
||||
x_ptr,
|
||||
w_ptr,
|
||||
scale_ptr,
|
||||
shift_ptr,
|
||||
idx_ptr,
|
||||
mod_row_stride,
|
||||
idx_stride,
|
||||
DIM: tl.constexpr,
|
||||
EPS: tl.constexpr,
|
||||
BLOCK_SIZE_DIM: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
cols = tl.arange(0, BLOCK_SIZE_DIM)
|
||||
mask = cols < DIM
|
||||
mod_ptr = tl.load(idx_ptr + row * idx_stride) * mod_row_stride + cols
|
||||
|
||||
x = tl.load(x_ptr + row * DIM + cols, mask=mask, other=0.0).to(tl.float32)
|
||||
rstd = tl.math.rsqrt(tl.sum(x * x, axis=0) / DIM + EPS)
|
||||
w = tl.load(w_ptr + cols, mask=mask).to(tl.float32)
|
||||
scale = tl.load(scale_ptr + mod_ptr, mask=mask).to(tl.float32)
|
||||
shift = tl.load(shift_ptr + mod_ptr, mask=mask).to(tl.float32)
|
||||
y = (x * rstd * w) * (1.0 + scale) + shift
|
||||
tl.store(y_ptr + row * DIM + cols, y, mask=mask)
|
||||
|
||||
|
||||
@register_custom_op(op_name="rmsnorm_indexed_scale_shift_cuda", out_shape="x")
|
||||
def _rmsnorm_indexed_scale_shift_cuda(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
shift: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
eps: float,
|
||||
) -> torch.Tensor:
|
||||
out = torch.empty_like(x)
|
||||
S, D = x.shape
|
||||
# one row per program: packing more rows spills the fp32 row tiles
|
||||
# (16 rows of 5376 ran 24x slower on H200)
|
||||
with torch.get_device_module().device(x.device):
|
||||
_rmsnorm_indexed_scale_shift_kernel[(S,)](
|
||||
out,
|
||||
x,
|
||||
weight,
|
||||
scale,
|
||||
shift,
|
||||
indices,
|
||||
scale.stride(0),
|
||||
indices.stride(0),
|
||||
DIM=D,
|
||||
EPS=eps,
|
||||
BLOCK_SIZE_DIM=triton.next_power_of_2(D),
|
||||
num_warps=8,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def can_use_rmsnorm_indexed_scale_shift(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
shift: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
) -> bool:
|
||||
return (
|
||||
x.is_cuda
|
||||
and x.dim() == 2
|
||||
and x.is_contiguous()
|
||||
and x.dtype in (torch.float16, torch.bfloat16, torch.float32)
|
||||
and x.numel() > 0
|
||||
and weight.is_cuda
|
||||
and weight.dim() == 1
|
||||
and weight.numel() == x.shape[1]
|
||||
and shift.dim() == scale.dim() == 2
|
||||
and shift.shape == scale.shape
|
||||
and shift.shape[1] == x.shape[1]
|
||||
and shift.stride(0) == scale.stride(0)
|
||||
and shift.stride(1) == scale.stride(1) == 1
|
||||
and indices.dim() == 1
|
||||
and indices.numel() == x.shape[0]
|
||||
and not indices.dtype.is_floating_point
|
||||
)
|
||||
|
||||
|
||||
def rmsnorm_indexed_scale_shift(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
shift: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
eps: float,
|
||||
) -> torch.Tensor:
|
||||
"""Fused ``rmsnorm(x) * (1 + scale[indices]) + shift[indices]`` over [T, D]
|
||||
rows, in fp32 with one rounding: the AdaLN modulation tables are indexed
|
||||
per row instead of materialized per token."""
|
||||
return _rmsnorm_indexed_scale_shift_cuda(x, weight, shift, scale, indices, eps)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"can_use_rmsnorm_indexed_scale_shift",
|
||||
"can_use_rmsnorm_scale_shift_per_token",
|
||||
"rmsnorm_indexed_scale_shift",
|
||||
"rmsnorm_scale_shift_per_token",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
"""MiniMax-H3 fused RMSNorm + indexed AdaLN, gated by request quality.
|
||||
|
||||
Each H3 block runs ``nn.RMSNorm`` and then the indexed AdaLN modulation
|
||||
``x * (1 + scale[idx]) + shift[idx]``, rounding to bf16 after the norm and after
|
||||
every modulation op. The fused kernel keeps the same math and operand precision
|
||||
in one fp32 pass with a single rounding, so ``quality="lossless"`` and ``"high"``
|
||||
mount it, while ``"exact"`` keeps the reference chain bit-for-bit.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import torch.nn as nn
|
||||
|
||||
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_FUSION = QualityGatedFusion(
|
||||
name="MiniMax-H3 fused RMSNorm + AdaLN",
|
||||
marker_attr="_sgl_minimax_h3_norm_modulate_site",
|
||||
enabled_attr="_sgl_minimax_h3_norm_modulate_enabled",
|
||||
)
|
||||
|
||||
|
||||
def mark_minimax_h3_norm_modulate_site(module: nn.Module) -> None:
|
||||
"""Mark an H3 DiT block; it starts on the reference path."""
|
||||
_FUSION.mark(module)
|
||||
|
||||
|
||||
def minimax_h3_norm_modulate_active(module: nn.Module) -> bool:
|
||||
return _FUSION.is_enabled(module)
|
||||
|
||||
|
||||
def mount_minimax_h3_norm_modulate(root: nn.Module) -> bool:
|
||||
return _FUSION.mount(root, logger=logger)
|
||||
|
||||
|
||||
def unmount_minimax_h3_norm_modulate(root: nn.Module) -> None:
|
||||
_FUSION.unmount(root)
|
||||
@@ -25,6 +25,7 @@ from sglang.kernels.ops.activation.activation import (
|
||||
from sglang.kernels.ops.diffusion import (
|
||||
can_use_fused_inplace_qknorm_rope,
|
||||
can_use_mxfp8_swizzled,
|
||||
can_use_rmsnorm_indexed_scale_shift,
|
||||
can_use_silu_mul_mxfp8,
|
||||
fused_inplace_qknorm_rope,
|
||||
fused_qknorm_rope_out_of_place,
|
||||
@@ -32,6 +33,9 @@ from sglang.kernels.ops.diffusion import (
|
||||
indexed_gate_bf16_,
|
||||
indexed_scale_shift_bf16_,
|
||||
indexed_scale_shift_mxfp8_,
|
||||
mark_minimax_h3_norm_modulate_site,
|
||||
minimax_h3_norm_modulate_active,
|
||||
rmsnorm_indexed_scale_shift,
|
||||
silu_mul_mxfp8,
|
||||
)
|
||||
from sglang.kernels.ops.layernorm.norm import fused_inplace_qknorm
|
||||
@@ -392,12 +396,22 @@ def _modulate_rmsnorm_scale_shift(
|
||||
indices: torch.Tensor,
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
fused: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""RMSNorm + indexed AdaLN.
|
||||
|
||||
Keep ``nn.RMSNorm``; a fused RMSNorm+AdaLN kernel drifted 2-GPU
|
||||
consistency GT for ~0.3% e2e, so it was removed.
|
||||
The reference chain rounds after the norm and after every modulation op.
|
||||
``fused`` (quality lossless / high) runs one fp32 pass with one rounding,
|
||||
which moves the denoising trajectory at rounding level.
|
||||
"""
|
||||
if (
|
||||
fused
|
||||
and x.dtype == dtype == _BF16_DTYPE
|
||||
and norm.weight is not None
|
||||
and can_use_rmsnorm_indexed_scale_shift(x, norm.weight, shift, scale, indices)
|
||||
):
|
||||
eps = norm.eps if norm.eps is not None else torch.finfo(x.dtype).eps
|
||||
return rmsnorm_indexed_scale_shift(x, norm.weight, shift, scale, indices, eps)
|
||||
return _modulate_scale_shift(norm(x), shift, scale, indices, dtype=dtype)
|
||||
|
||||
|
||||
@@ -1562,6 +1576,7 @@ class MiniMaxH3DiTBlock(nn.Module):
|
||||
super().__init__()
|
||||
self.norm1 = _norm(arch.hidden_size, eps=arch.norm_eps)
|
||||
self.norm2 = _norm(arch.hidden_size, eps=arch.norm_eps)
|
||||
mark_minimax_h3_norm_modulate_site(self)
|
||||
self.attn = MiniMaxH3Attention(
|
||||
arch,
|
||||
quant_config,
|
||||
@@ -1634,6 +1649,7 @@ class MiniMaxH3DiTBlock(nn.Module):
|
||||
scale_msa,
|
||||
combined_indices,
|
||||
dtype=_BF16_DTYPE,
|
||||
fused=minimax_h3_norm_modulate_active(self),
|
||||
)
|
||||
h = self.attn(
|
||||
h,
|
||||
@@ -1676,6 +1692,7 @@ class MiniMaxH3DiTBlock(nn.Module):
|
||||
scale_mlp,
|
||||
combined_indices,
|
||||
dtype=_BF16_DTYPE,
|
||||
fused=minimax_h3_norm_modulate_active(self),
|
||||
)
|
||||
h = self.mlp(h)
|
||||
# `residual` is block-local here (see above), so this stays in-place
|
||||
|
||||
@@ -31,6 +31,7 @@ from sglang.kernels.ops.diffusion import (
|
||||
mount_lingbot_video_rmsnorm,
|
||||
mount_ltx2_qknorm_split_rope,
|
||||
mount_ltx2_rms_norm_modulate,
|
||||
mount_minimax_h3_norm_modulate,
|
||||
mount_nvfp4_bias_gelu,
|
||||
mount_qwen_image_added_qkv,
|
||||
mount_sana_video_linear_attention,
|
||||
@@ -44,6 +45,7 @@ from sglang.kernels.ops.diffusion import (
|
||||
unmount_lingbot_video_rmsnorm,
|
||||
unmount_ltx2_qknorm_split_rope,
|
||||
unmount_ltx2_rms_norm_modulate,
|
||||
unmount_minimax_h3_norm_modulate,
|
||||
unmount_nvfp4_bias_gelu,
|
||||
unmount_qwen_image_added_qkv,
|
||||
unmount_sana_video_linear_attention,
|
||||
@@ -261,6 +263,12 @@ _QUALITY_FUSION_HANDLERS: tuple[
|
||||
mount_helios_gated_residual,
|
||||
unmount_helios_gated_residual,
|
||||
),
|
||||
(
|
||||
"lossless",
|
||||
"MiniMax-H3 fused RMSNorm + AdaLN",
|
||||
mount_minimax_h3_norm_modulate,
|
||||
unmount_minimax_h3_norm_modulate,
|
||||
),
|
||||
(
|
||||
# the first attention GEMM takes BF16 inputs where the reference
|
||||
# promotes them to FP32
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
"""MiniMax-H3's fused RMSNorm + indexed AdaLN (quality lossless/high) against the
|
||||
same math in fp32 with one rounding, and against the reference rounding chain."""
|
||||
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from sglang.kernels.ops.diffusion import (
|
||||
can_use_rmsnorm_indexed_scale_shift,
|
||||
indexed_scale_shift_bf16_,
|
||||
rmsnorm_indexed_scale_shift,
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="4-gpu-b200")
|
||||
|
||||
EPS = 1e-5
|
||||
|
||||
|
||||
def _inputs(rows: int, hidden: int, seed: int):
|
||||
g = torch.Generator(device="cuda").manual_seed(seed)
|
||||
x = (2 * torch.randn(rows, hidden, device="cuda", generator=g)).to(torch.bfloat16)
|
||||
weight = (1 + 0.1 * torch.randn(hidden, device="cuda", generator=g)).to(
|
||||
torch.bfloat16
|
||||
)
|
||||
# H3's shift/scale are chunks of one [M, 6 * hidden] AdaLN output
|
||||
adaln = (0.1 * torch.randn(3, 6 * hidden, device="cuda", generator=g)).to(
|
||||
torch.bfloat16
|
||||
)
|
||||
shift, scale = adaln[:, :hidden], adaln[:, hidden : 2 * hidden]
|
||||
indices = torch.randint(0, 3, (rows,), device="cuda", generator=g)
|
||||
return x, weight, shift, scale, indices
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hidden", [1024, 5376])
|
||||
@pytest.mark.parametrize("rows", [1, 37, 1025])
|
||||
def test_matches_fp32_single_rounding(rows: int, hidden: int) -> None:
|
||||
x, weight, shift, scale, indices = _inputs(rows, hidden, rows + hidden)
|
||||
assert can_use_rmsnorm_indexed_scale_shift(x, weight, shift, scale, indices)
|
||||
xf = x.float()
|
||||
normed = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + EPS) * weight.float()
|
||||
ref = normed * (1 + scale[indices].float()) + shift[indices].float()
|
||||
got = rmsnorm_indexed_scale_shift(x, weight, shift, scale, indices, EPS)
|
||||
# only the reduction order differs: at most one bf16 rounding step apart
|
||||
torch.testing.assert_close(got.float(), ref, rtol=2**-7, atol=2**-7)
|
||||
|
||||
|
||||
def test_stays_within_rounding_of_the_reference_chain() -> None:
|
||||
x, weight, shift, scale, indices = _inputs(1025, 5376, 7)
|
||||
norm = nn.RMSNorm(5376, eps=EPS, device="cuda", dtype=torch.bfloat16)
|
||||
with torch.no_grad():
|
||||
norm.weight.copy_(weight)
|
||||
chain = indexed_scale_shift_bf16_(norm(x), shift, scale, indices)
|
||||
got = rmsnorm_indexed_scale_shift(x, weight, shift, scale, indices, EPS)
|
||||
torch.testing.assert_close(got.float(), chain.float(), rtol=2**-6, atol=2**-6)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__, "-v"]))
|
||||
Reference in New Issue
Block a user