diff --git a/python/sglang/kernels/ops/diffusion/README.md b/python/sglang/kernels/ops/diffusion/README.md index c08abc12b1..3c183c9806 100644 --- a/python/sglang/kernels/ops/diffusion/README.md +++ b/python/sglang/kernels/ops/diffusion/README.md @@ -134,6 +134,7 @@ Several norms look interchangeable and are not. Start here. | `flydsl_norm_scale_shift` / `flydsl_fused_residual_norm_scale_shift` | FlyDSL | close | **ROCm gfx950 only** | | `try_fused_scale_residual_norm_scale_shift_nvfp4` | JIT CUDA | matches the selected NVFP4 producer contract | Qwen residual LayerNorm/modulation + FC1 NVFP4 quantization | | `fuse_layernorm_scale_shift_gate_select01_kernel` | Triton | close | per-token select between two modulation rows (Qwen-Image) | +| `indexed_scale_shift_bf16_` / `indexed_scale_shift_fp32_` | Triton | bit-exact vs `x * (1 + scale[idx]) + shift[idx]`, in place | bf16 rows, per-row index into `[n, D]` shift/scale tables; fp32 tables are serialized INT8 MiniMax-H3 | | `norm_infer` / `rms_norm_fn` | Triton (+torch/NPU/MPS fallbacks) | close | the generic entry point; use when nothing above fits | ### Norm variants @@ -158,6 +159,7 @@ Several norms look interchangeable and are not. Start here. | `bias_residual_add` | JIT CUDA | bit-exact `(y + bias[c]) + h` with aten's per-op bf16 rounding, for a conv that skipped its bias pass | channels_last-dense (`C % 8 == 0`) or NCHW/NCDHW contiguous (spatial `% 8 == 0`) bf16; `h` shares `y`'s layout (Qwen-Image 2.1 VAE residual blocks) | | `residual_gate_add` | JIT CUDA (contiguous), Triton (transposed) | bit-exact `residual + update * gate` | contiguous tensors, or a transposed-dense `[B, tokens, hidden]` residual/output with contiguous update and row-broadcast gate (SANA-Video) | | `residual_gate_fp32` | Triton | separate FP32 multiply/add, then cast to input dtype | contiguous `[B,S,D]`, strided `[B or 1,1,D]` gate; Kandinsky6 | +| `indexed_gate_bf16` / `indexed_gate_fp32` (`_` in place) | Triton | bit-exact vs `x + gate[idx] * other`: bf16 rounds the product, fp32 keeps a separate FP32 multiply/add | bf16 rows, per-row index into a `[n, D]` gate table (MiniMax-H3) | These contracts differ: `residual_gate_fp32` does not round the product to BF16/FP16, and disables FMA to preserve the eager FP32 rounding boundary. diff --git a/python/sglang/kernels/ops/diffusion/__init__.py b/python/sglang/kernels/ops/diffusion/__init__.py index dceda5805a..e4d5ee31ac 100644 --- a/python/sglang/kernels/ops/diffusion/__init__.py +++ b/python/sglang/kernels/ops/diffusion/__init__.py @@ -790,6 +790,9 @@ _EXPORTS: dict[str, str] = { "indexed_gate_bf16": "modulate.indexed_modulation_triton", "indexed_gate_bf16_": "modulate.indexed_modulation_triton", "indexed_scale_shift_bf16_": "modulate.indexed_modulation_triton", + "indexed_gate_fp32": "modulate.indexed_modulation_triton", + "indexed_gate_fp32_": "modulate.indexed_modulation_triton", + "indexed_scale_shift_fp32_": "modulate.indexed_modulation_triton", "ltx2_ada_values9": "modulate.ltx2_ada_values_triton", "can_use_modulate_scale_shift_cuda": "modulate.modulate_scale_shift_jit", "modulate_scale_shift": "modulate.modulate_scale_shift_jit", diff --git a/python/sglang/kernels/ops/diffusion/modulate/indexed_modulation_triton.py b/python/sglang/kernels/ops/diffusion/modulate/indexed_modulation_triton.py index 5ebab8178b..54df402fb2 100644 --- a/python/sglang/kernels/ops/diffusion/modulate/indexed_modulation_triton.py +++ b/python/sglang/kernels/ops/diffusion/modulate/indexed_modulation_triton.py @@ -4,7 +4,7 @@ import torch import triton import triton.language as tl -from sglang.kernels.numerics import round_bf16_to_fp32 +from sglang.kernels.numerics import mul_rn_f32, round_bf16_to_fp32 # Column tiles per row. One program per whole row padded to a power of two # masked off 37.5% of MiniMax-H3's 5376-wide rows; 2048-wide tiles run both @@ -165,3 +165,151 @@ def indexed_gate_bf16( indices: torch.Tensor, ) -> torch.Tensor: return _indexed_gate_bf16(torch.empty_like(x), x, gate, other, indices) + + +# FP32 modulation (serialized INT8 H3 keeps its AdaLN tables in FP32). Eager torch +# computes these in FP32 and rounds to BF16 once at the end; mul_rn_f32 keeps the +# product a separate rounding step so the following add cannot contract into an FMA. + + +@triton.jit +def _indexed_scale_shift_fp32_kernel( + output_ptr, + x_ptr, + shift_ptr, + scale_ptr, + indices_ptr, + hidden_size, + stride_x_row, + stride_shift_row, + stride_scale_row, + stride_indices, + BLOCK_N: tl.constexpr, +): + row = tl.program_id(0) + columns = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N) + mask = columns < hidden_size + index = tl.load(indices_ptr + row * stride_indices) + + x = tl.load(x_ptr + row * stride_x_row + columns, mask=mask, other=0.0).to( + tl.float32 + ) + shift = tl.load( + shift_ptr + index * stride_shift_row + columns, mask=mask, other=0.0 + ) + scale = tl.load( + scale_ptr + index * stride_scale_row + columns, mask=mask, other=0.0 + ) + + scaled = mul_rn_f32(x, 1.0 + scale) + tl.store(output_ptr + row * stride_x_row + columns, scaled + shift, mask=mask) + + +@triton.jit +def _indexed_gate_fp32_kernel( + output_ptr, + x_ptr, + gate_ptr, + other_ptr, + indices_ptr, + hidden_size, + stride_output_row, + stride_x_row, + stride_gate_row, + stride_other_row, + stride_indices, + BLOCK_N: tl.constexpr, +): + row = tl.program_id(0) + columns = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N) + mask = columns < hidden_size + index = tl.load(indices_ptr + row * stride_indices) + + x = tl.load(x_ptr + row * stride_x_row + columns, mask=mask, other=0.0).to( + tl.float32 + ) + gate = tl.load(gate_ptr + index * stride_gate_row + columns, mask=mask, other=0.0) + other = tl.load( + other_ptr + row * stride_other_row + columns, mask=mask, other=0.0 + ).to(tl.float32) + + tl.store( + output_ptr + row * stride_output_row + columns, + x + mul_rn_f32(gate, other), + mask=mask, + ) + + +def indexed_scale_shift_fp32_( + x: torch.Tensor, + shift: torch.Tensor, + scale: torch.Tensor, + indices: torch.Tensor, +) -> torch.Tensor: + """In place: x = bf16(x * (1 + scale[idx]) + shift[idx]) with FP32 shift/scale.""" + rows, hidden_size = x.shape + if rows == 0: + return x + block_n, tiles = _row_tiles(hidden_size) + _indexed_scale_shift_fp32_kernel[(rows, tiles)]( + x, + x, + shift, + scale, + indices, + hidden_size, + x.stride(0), + shift.stride(0), + scale.stride(0), + indices.stride(0), + BLOCK_N=block_n, + num_warps=4, + ) + return x + + +def _indexed_gate_fp32( + output: torch.Tensor, + x: torch.Tensor, + gate: torch.Tensor, + other: torch.Tensor, + indices: torch.Tensor, +) -> torch.Tensor: + rows, hidden_size = x.shape + if rows == 0: + return output + block_n, tiles = _row_tiles(hidden_size) + _indexed_gate_fp32_kernel[(rows, tiles)]( + output, + x, + gate, + other, + indices, + hidden_size, + output.stride(0), + x.stride(0), + gate.stride(0), + other.stride(0), + indices.stride(0), + BLOCK_N=block_n, + num_warps=4, + ) + return output + + +def indexed_gate_fp32_( + x: torch.Tensor, + gate: torch.Tensor, + other: torch.Tensor, + indices: torch.Tensor, +) -> torch.Tensor: + return _indexed_gate_fp32(x, x, gate, other, indices) + + +def indexed_gate_fp32( + x: torch.Tensor, + gate: torch.Tensor, + other: torch.Tensor, + indices: torch.Tensor, +) -> torch.Tensor: + return _indexed_gate_fp32(torch.empty_like(x), x, gate, other, indices) diff --git a/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py b/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py index 01002aefd6..ebc75569a6 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py @@ -31,7 +31,10 @@ from sglang.kernels.ops.diffusion import ( fused_qknorm_rope_out_of_place, indexed_gate_bf16, indexed_gate_bf16_, + indexed_gate_fp32, + indexed_gate_fp32_, indexed_scale_shift_bf16_, + indexed_scale_shift_fp32_, indexed_scale_shift_mxfp8_, mark_minimax_h3_norm_modulate_site, minimax_h3_norm_modulate_active, @@ -437,11 +440,15 @@ def _modulate_scale_shift( x.is_cuda and x.dtype == _BF16_DTYPE and dtype == _BF16_DTYPE - and shift.dtype == _BF16_DTYPE - and scale.dtype == _BF16_DTYPE + and shift.dtype == scale.dtype and x.is_contiguous() + and shift.stride(-1) == scale.stride(-1) == 1 ): - return indexed_scale_shift_bf16_(x, shift, scale, indices) + # Serialized INT8 checkpoints keep the AdaLN tables in FP32. + if shift.dtype == _BF16_DTYPE: + return indexed_scale_shift_bf16_(x, shift, scale, indices) + if shift.dtype == _FP32_DTYPE: + return indexed_scale_shift_fp32_(x, shift, scale, indices) return ( x * (1.0 + scale.index_select(0, indices)) + shift.index_select(0, indices) ).to(dtype) @@ -462,14 +469,19 @@ def _modulate_gate( x.is_cuda and x.dtype == _BF16_DTYPE and dtype == _BF16_DTYPE - and gate.dtype == _BF16_DTYPE and other.dtype == _BF16_DTYPE and x.is_contiguous() and other.is_contiguous() + and gate.stride(-1) == 1 ): - if allow_inplace: - return indexed_gate_bf16_(x, gate, other, indices) - return indexed_gate_bf16(x, gate, other, indices) + if gate.dtype == _BF16_DTYPE: + if allow_inplace: + return indexed_gate_bf16_(x, gate, other, indices) + return indexed_gate_bf16(x, gate, other, indices) + if gate.dtype == _FP32_DTYPE: + if allow_inplace: + return indexed_gate_fp32_(x, gate, other, indices) + return indexed_gate_fp32(x, gate, other, indices) return (x + gate.index_select(0, indices) * other).to(dtype) diff --git a/test/registered/kernels/ops/diffusion/test_indexed_modulation.py b/test/registered/kernels/ops/diffusion/test_indexed_modulation.py index b48f79aad4..67d5e7e22b 100644 --- a/test/registered/kernels/ops/diffusion/test_indexed_modulation.py +++ b/test/registered/kernels/ops/diffusion/test_indexed_modulation.py @@ -1,5 +1,6 @@ """MiniMax-H3's indexed scale/shift and gate kernels against the same arithmetic in -torch, byte for byte, across row widths the column tiles split differently.""" +torch, byte for byte, across row widths the column tiles split differently. The fp32 +variants serve serialized INT8 checkpoints, whose AdaLN tables stay in fp32.""" import sys @@ -9,7 +10,10 @@ import torch from sglang.kernels.ops.diffusion import ( indexed_gate_bf16, indexed_gate_bf16_, + indexed_gate_fp32, + indexed_gate_fp32_, indexed_scale_shift_bf16_, + indexed_scale_shift_fp32_, ) from sglang.test.ci.ci_register import register_cuda_ci @@ -57,5 +61,39 @@ def test_indexed_gate_matches_torch(rows: int, hidden: int) -> None: assert torch.equal(indexed_gate_bf16_(x.clone(), gate, other, indices), ref) +def _fp32_table(rows: int, hidden: int, seed: int) -> torch.Tensor: + """A [rows, hidden] slice of a [rows, 6 * hidden] AdaLN table: row stride > width.""" + g = torch.Generator(device="cuda").manual_seed(seed) + table = torch.randn(rows, 6 * hidden, device="cuda", generator=g) + return table[:, 2 * hidden : 3 * hidden] + + +@pytest.mark.parametrize("hidden", HIDDEN) +@pytest.mark.parametrize("rows", ROWS) +def test_indexed_scale_shift_fp32_matches_torch(rows: int, hidden: int) -> None: + x, _, _, _, indices = _inputs(rows, hidden, rows * hidden + 2) + shift = _fp32_table(3, hidden, rows + 3) + scale = 0.1 * _fp32_table(3, hidden, rows + 4) + # A product whose rounding differs once it is contracted into an FMA. + scale[indices[0], 0] = -0.9999999403953552 + shift[indices[0], 0] = 1.234567e-7 + ref = (x.float() * (1.0 + scale[indices]) + shift[indices]).to(torch.bfloat16) + assert torch.equal(indexed_scale_shift_fp32_(x.clone(), shift, scale, indices), ref) + + +@pytest.mark.parametrize("hidden", HIDDEN) +@pytest.mark.parametrize("rows", ROWS) +def test_indexed_gate_fp32_matches_torch(rows: int, hidden: int) -> None: + x, other, _, _, indices = _inputs(rows, hidden, rows * hidden + 5) + gate = _fp32_table(3, hidden, rows + 6) + x[0, 0], other[0, 0] = -0.1484375, 14.25 + gate[indices[0], 0] = 0.01041664183139801 + ref = (x.float() + gate[indices] * other.float()).to(torch.bfloat16) + original = x.clone() + assert torch.equal(indexed_gate_fp32(x, gate, other, indices), ref) + assert torch.equal(x, original) + assert torch.equal(indexed_gate_fp32_(x.clone(), gate, other, indices), ref) + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"]))