diff --git a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx index 015ea4301c..44a5ecb38c 100644 --- a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx +++ b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx @@ -1014,8 +1014,9 @@ levels: - `"exact"`: the exact reference path. Output is bit-exact against the reference implementation and the CI ground truth. - `"lossless"` (default): includes the global fusion-only tier but does not enable - Cache-DiT or another approximate optimization. The denoise path is the same - as `exact`; VAE decode fuses the ViT decoder's per-head QK RMSNorm and + Cache-DiT or another approximate optimization. Each DiT block runs its + RMSNorm and AdaLN modulation as one kernel that rounds once instead of after + every op, and VAE decode fuses the ViT decoder's per-head QK RMSNorm and RoPE into one launch and runs its attention on cuDNN SDPA (rounding-level differences only). - `"high"`: the audited accelerated path. Quality is guaranteed (the audited @@ -1064,9 +1065,10 @@ quality-gated VAE fusions. -The global fusion-only tier. It does not enable MiniMax-H3 Cache-DiT and -follows the same H3 denoise path as `exact`; only the VAE decoder's fused -QK RMSNorm + RoPE and cuDNN SDPA attention differ. +The global fusion-only tier. It does not enable MiniMax-H3 Cache-DiT. It +differs from `exact` only in rounding: the DiT blocks' fused RMSNorm + AdaLN +modulation, and the VAE decoder's fused QK RMSNorm + RoPE and cuDNN SDPA +attention. Omitting the field selects this level. ```json Request field diff --git a/python/sglang/kernels/ops/diffusion/__init__.py b/python/sglang/kernels/ops/diffusion/__init__.py index 15d73e4b1f..e242f126af 100644 --- a/python/sglang/kernels/ops/diffusion/__init__.py +++ b/python/sglang/kernels/ops/diffusion/__init__.py @@ -823,7 +823,13 @@ _EXPORTS: dict[str, str] = { "mount_lingbot_video_gated_residual": "sites.lingbot_video_gated_residual_site", "try_lingbot_video_gated_residual": "sites.lingbot_video_gated_residual_site", "unmount_lingbot_video_gated_residual": "sites.lingbot_video_gated_residual_site", + "mark_minimax_h3_norm_modulate_site": "sites.minimax_h3_norm_modulate_site", + "minimax_h3_norm_modulate_active": "sites.minimax_h3_norm_modulate_site", + "mount_minimax_h3_norm_modulate": "sites.minimax_h3_norm_modulate_site", + "unmount_minimax_h3_norm_modulate": "sites.minimax_h3_norm_modulate_site", "can_use_rmsnorm_scale_shift_per_token": "norm.rmsnorm_scale_shift_triton", + "can_use_rmsnorm_indexed_scale_shift": "norm.rmsnorm_scale_shift_triton", + "rmsnorm_indexed_scale_shift": "norm.rmsnorm_scale_shift_triton", "rmsnorm_scale_shift_per_token": "norm.rmsnorm_scale_shift_triton", "h3_vae_rmsnorm": "norm.h3_vae_fused_norm", "h3_vae_scale_add_rmsnorm": "norm.h3_vae_fused_norm", diff --git a/python/sglang/kernels/ops/diffusion/norm/rmsnorm_scale_shift_triton.py b/python/sglang/kernels/ops/diffusion/norm/rmsnorm_scale_shift_triton.py index 89de03a9b9..b3b6fa4cca 100644 --- a/python/sglang/kernels/ops/diffusion/norm/rmsnorm_scale_shift_triton.py +++ b/python/sglang/kernels/ops/diffusion/norm/rmsnorm_scale_shift_triton.py @@ -17,6 +17,10 @@ Unlike the bit-exact ``rmsnorm_scale_shift_bitexact`` (ERNIE), this kernel does *not* reproduce PyTorch's parallel variance reduction order bit-for-bit, so it is intended for the request-gated (``quality="lossless"``/``"high"``) fusion path, matching the existing quality-gated LingBot RMSNorm fusion. + +``rmsnorm_indexed_scale_shift`` is the same chain for MiniMax-H3, whose +``scale``/``shift`` rows come from small ``[M, D]`` AdaLN tables picked per +token by an index. """ from __future__ import annotations @@ -143,7 +147,109 @@ def rmsnorm_scale_shift_per_token( return _rmsnorm_scale_shift_per_token_cuda(x, weight, scale, shift, eps) +@triton.jit +def _rmsnorm_indexed_scale_shift_kernel( + y_ptr, + x_ptr, + w_ptr, + scale_ptr, + shift_ptr, + idx_ptr, + mod_row_stride, + idx_stride, + DIM: tl.constexpr, + EPS: tl.constexpr, + BLOCK_SIZE_DIM: tl.constexpr, +): + row = tl.program_id(0).to(tl.int64) + cols = tl.arange(0, BLOCK_SIZE_DIM) + mask = cols < DIM + mod_ptr = tl.load(idx_ptr + row * idx_stride) * mod_row_stride + cols + + x = tl.load(x_ptr + row * DIM + cols, mask=mask, other=0.0).to(tl.float32) + rstd = tl.math.rsqrt(tl.sum(x * x, axis=0) / DIM + EPS) + w = tl.load(w_ptr + cols, mask=mask).to(tl.float32) + scale = tl.load(scale_ptr + mod_ptr, mask=mask).to(tl.float32) + shift = tl.load(shift_ptr + mod_ptr, mask=mask).to(tl.float32) + y = (x * rstd * w) * (1.0 + scale) + shift + tl.store(y_ptr + row * DIM + cols, y, mask=mask) + + +@register_custom_op(op_name="rmsnorm_indexed_scale_shift_cuda", out_shape="x") +def _rmsnorm_indexed_scale_shift_cuda( + x: torch.Tensor, + weight: torch.Tensor, + shift: torch.Tensor, + scale: torch.Tensor, + indices: torch.Tensor, + eps: float, +) -> torch.Tensor: + out = torch.empty_like(x) + S, D = x.shape + # one row per program: packing more rows spills the fp32 row tiles + # (16 rows of 5376 ran 24x slower on H200) + with torch.get_device_module().device(x.device): + _rmsnorm_indexed_scale_shift_kernel[(S,)]( + out, + x, + weight, + scale, + shift, + indices, + scale.stride(0), + indices.stride(0), + DIM=D, + EPS=eps, + BLOCK_SIZE_DIM=triton.next_power_of_2(D), + num_warps=8, + ) + return out + + +def can_use_rmsnorm_indexed_scale_shift( + x: torch.Tensor, + weight: torch.Tensor, + shift: torch.Tensor, + scale: torch.Tensor, + indices: torch.Tensor, +) -> bool: + return ( + x.is_cuda + and x.dim() == 2 + and x.is_contiguous() + and x.dtype in (torch.float16, torch.bfloat16, torch.float32) + and x.numel() > 0 + and weight.is_cuda + and weight.dim() == 1 + and weight.numel() == x.shape[1] + and shift.dim() == scale.dim() == 2 + and shift.shape == scale.shape + and shift.shape[1] == x.shape[1] + and shift.stride(0) == scale.stride(0) + and shift.stride(1) == scale.stride(1) == 1 + and indices.dim() == 1 + and indices.numel() == x.shape[0] + and not indices.dtype.is_floating_point + ) + + +def rmsnorm_indexed_scale_shift( + x: torch.Tensor, + weight: torch.Tensor, + shift: torch.Tensor, + scale: torch.Tensor, + indices: torch.Tensor, + eps: float, +) -> torch.Tensor: + """Fused ``rmsnorm(x) * (1 + scale[indices]) + shift[indices]`` over [T, D] + rows, in fp32 with one rounding: the AdaLN modulation tables are indexed + per row instead of materialized per token.""" + return _rmsnorm_indexed_scale_shift_cuda(x, weight, shift, scale, indices, eps) + + __all__ = [ + "can_use_rmsnorm_indexed_scale_shift", "can_use_rmsnorm_scale_shift_per_token", + "rmsnorm_indexed_scale_shift", "rmsnorm_scale_shift_per_token", ] diff --git a/python/sglang/kernels/ops/diffusion/sites/minimax_h3_norm_modulate_site.py b/python/sglang/kernels/ops/diffusion/sites/minimax_h3_norm_modulate_site.py new file mode 100644 index 0000000000..8d3e01f0a5 --- /dev/null +++ b/python/sglang/kernels/ops/diffusion/sites/minimax_h3_norm_modulate_site.py @@ -0,0 +1,41 @@ +"""MiniMax-H3 fused RMSNorm + indexed AdaLN, gated by request quality. + +Each H3 block runs ``nn.RMSNorm`` and then the indexed AdaLN modulation +``x * (1 + scale[idx]) + shift[idx]``, rounding to bf16 after the norm and after +every modulation op. The fused kernel keeps the same math and operand precision +in one fp32 pass with a single rounding, so ``quality="lossless"`` and ``"high"`` +mount it, while ``"exact"`` keeps the reference chain bit-for-bit. +""" + +from __future__ import annotations + +import logging + +import torch.nn as nn + +from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion + +logger = logging.getLogger(__name__) + +_FUSION = QualityGatedFusion( + name="MiniMax-H3 fused RMSNorm + AdaLN", + marker_attr="_sgl_minimax_h3_norm_modulate_site", + enabled_attr="_sgl_minimax_h3_norm_modulate_enabled", +) + + +def mark_minimax_h3_norm_modulate_site(module: nn.Module) -> None: + """Mark an H3 DiT block; it starts on the reference path.""" + _FUSION.mark(module) + + +def minimax_h3_norm_modulate_active(module: nn.Module) -> bool: + return _FUSION.is_enabled(module) + + +def mount_minimax_h3_norm_modulate(root: nn.Module) -> bool: + return _FUSION.mount(root, logger=logger) + + +def unmount_minimax_h3_norm_modulate(root: nn.Module) -> None: + _FUSION.unmount(root) diff --git a/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py b/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py index 2a19b8931e..c6a6a391c1 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py @@ -25,6 +25,7 @@ from sglang.kernels.ops.activation.activation import ( from sglang.kernels.ops.diffusion import ( can_use_fused_inplace_qknorm_rope, can_use_mxfp8_swizzled, + can_use_rmsnorm_indexed_scale_shift, can_use_silu_mul_mxfp8, fused_inplace_qknorm_rope, fused_qknorm_rope_out_of_place, @@ -32,6 +33,9 @@ from sglang.kernels.ops.diffusion import ( indexed_gate_bf16_, indexed_scale_shift_bf16_, indexed_scale_shift_mxfp8_, + mark_minimax_h3_norm_modulate_site, + minimax_h3_norm_modulate_active, + rmsnorm_indexed_scale_shift, silu_mul_mxfp8, ) from sglang.kernels.ops.layernorm.norm import fused_inplace_qknorm @@ -392,12 +396,22 @@ def _modulate_rmsnorm_scale_shift( indices: torch.Tensor, *, dtype: torch.dtype, + fused: bool = False, ) -> torch.Tensor: """RMSNorm + indexed AdaLN. - Keep ``nn.RMSNorm``; a fused RMSNorm+AdaLN kernel drifted 2-GPU - consistency GT for ~0.3% e2e, so it was removed. + The reference chain rounds after the norm and after every modulation op. + ``fused`` (quality lossless / high) runs one fp32 pass with one rounding, + which moves the denoising trajectory at rounding level. """ + if ( + fused + and x.dtype == dtype == _BF16_DTYPE + and norm.weight is not None + and can_use_rmsnorm_indexed_scale_shift(x, norm.weight, shift, scale, indices) + ): + eps = norm.eps if norm.eps is not None else torch.finfo(x.dtype).eps + return rmsnorm_indexed_scale_shift(x, norm.weight, shift, scale, indices, eps) return _modulate_scale_shift(norm(x), shift, scale, indices, dtype=dtype) @@ -1562,6 +1576,7 @@ class MiniMaxH3DiTBlock(nn.Module): super().__init__() self.norm1 = _norm(arch.hidden_size, eps=arch.norm_eps) self.norm2 = _norm(arch.hidden_size, eps=arch.norm_eps) + mark_minimax_h3_norm_modulate_site(self) self.attn = MiniMaxH3Attention( arch, quant_config, @@ -1634,6 +1649,7 @@ class MiniMaxH3DiTBlock(nn.Module): scale_msa, combined_indices, dtype=_BF16_DTYPE, + fused=minimax_h3_norm_modulate_active(self), ) h = self.attn( h, @@ -1676,6 +1692,7 @@ class MiniMaxH3DiTBlock(nn.Module): scale_mlp, combined_indices, dtype=_BF16_DTYPE, + fused=minimax_h3_norm_modulate_active(self), ) h = self.mlp(h) # `residual` is block-local here (see above), so this stays in-place diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py index 7012258b9f..f3f20f2ead 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py @@ -31,6 +31,7 @@ from sglang.kernels.ops.diffusion import ( mount_lingbot_video_rmsnorm, mount_ltx2_qknorm_split_rope, mount_ltx2_rms_norm_modulate, + mount_minimax_h3_norm_modulate, mount_nvfp4_bias_gelu, mount_qwen_image_added_qkv, mount_sana_video_linear_attention, @@ -44,6 +45,7 @@ from sglang.kernels.ops.diffusion import ( unmount_lingbot_video_rmsnorm, unmount_ltx2_qknorm_split_rope, unmount_ltx2_rms_norm_modulate, + unmount_minimax_h3_norm_modulate, unmount_nvfp4_bias_gelu, unmount_qwen_image_added_qkv, unmount_sana_video_linear_attention, @@ -261,6 +263,12 @@ _QUALITY_FUSION_HANDLERS: tuple[ mount_helios_gated_residual, unmount_helios_gated_residual, ), + ( + "lossless", + "MiniMax-H3 fused RMSNorm + AdaLN", + mount_minimax_h3_norm_modulate, + unmount_minimax_h3_norm_modulate, + ), ( # the first attention GEMM takes BF16 inputs where the reference # promotes them to FP32 diff --git a/test/registered/kernels/ops/diffusion/test_rmsnorm_indexed_scale_shift.py b/test/registered/kernels/ops/diffusion/test_rmsnorm_indexed_scale_shift.py new file mode 100644 index 0000000000..58412d12bf --- /dev/null +++ b/test/registered/kernels/ops/diffusion/test_rmsnorm_indexed_scale_shift.py @@ -0,0 +1,61 @@ +"""MiniMax-H3's fused RMSNorm + indexed AdaLN (quality lossless/high) against the +same math in fp32 with one rounding, and against the reference rounding chain.""" + +import sys + +import pytest +import torch +import torch.nn as nn + +from sglang.kernels.ops.diffusion import ( + can_use_rmsnorm_indexed_scale_shift, + indexed_scale_shift_bf16_, + rmsnorm_indexed_scale_shift, +) +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="4-gpu-b200") + +EPS = 1e-5 + + +def _inputs(rows: int, hidden: int, seed: int): + g = torch.Generator(device="cuda").manual_seed(seed) + x = (2 * torch.randn(rows, hidden, device="cuda", generator=g)).to(torch.bfloat16) + weight = (1 + 0.1 * torch.randn(hidden, device="cuda", generator=g)).to( + torch.bfloat16 + ) + # H3's shift/scale are chunks of one [M, 6 * hidden] AdaLN output + adaln = (0.1 * torch.randn(3, 6 * hidden, device="cuda", generator=g)).to( + torch.bfloat16 + ) + shift, scale = adaln[:, :hidden], adaln[:, hidden : 2 * hidden] + indices = torch.randint(0, 3, (rows,), device="cuda", generator=g) + return x, weight, shift, scale, indices + + +@pytest.mark.parametrize("hidden", [1024, 5376]) +@pytest.mark.parametrize("rows", [1, 37, 1025]) +def test_matches_fp32_single_rounding(rows: int, hidden: int) -> None: + x, weight, shift, scale, indices = _inputs(rows, hidden, rows + hidden) + assert can_use_rmsnorm_indexed_scale_shift(x, weight, shift, scale, indices) + xf = x.float() + normed = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + EPS) * weight.float() + ref = normed * (1 + scale[indices].float()) + shift[indices].float() + got = rmsnorm_indexed_scale_shift(x, weight, shift, scale, indices, EPS) + # only the reduction order differs: at most one bf16 rounding step apart + torch.testing.assert_close(got.float(), ref, rtol=2**-7, atol=2**-7) + + +def test_stays_within_rounding_of_the_reference_chain() -> None: + x, weight, shift, scale, indices = _inputs(1025, 5376, 7) + norm = nn.RMSNorm(5376, eps=EPS, device="cuda", dtype=torch.bfloat16) + with torch.no_grad(): + norm.weight.copy_(weight) + chain = indexed_scale_shift_bf16_(norm(x), shift, scale, indices) + got = rmsnorm_indexed_scale_shift(x, weight, shift, scale, indices, EPS) + torch.testing.assert_close(got.float(), chain.float(), rtol=2**-6, atol=2**-6) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"]))