[Perf] diffusion: fused indexed modulation for FP32 AdaLN tables (#44180)

Co-authored-by: Mick <mickjagger19@icloud.com>
This commit is contained in:
WenhaoZhang
2026-10-18 20:31:36 +08:00
committed by GitHub
co-authored by Mick
parent 8dee3f6bdb
commit 1fd46ddd56
5 changed files with 212 additions and 9 deletions
@@ -134,6 +134,7 @@ Several norms look interchangeable and are not. Start here.
| `flydsl_norm_scale_shift` / `flydsl_fused_residual_norm_scale_shift` | FlyDSL | close | **ROCm gfx950 only** |
| `try_fused_scale_residual_norm_scale_shift_nvfp4` | JIT CUDA | matches the selected NVFP4 producer contract | Qwen residual LayerNorm/modulation + FC1 NVFP4 quantization |
| `fuse_layernorm_scale_shift_gate_select01_kernel` | Triton | close | per-token select between two modulation rows (Qwen-Image) |
| `indexed_scale_shift_bf16_` / `indexed_scale_shift_fp32_` | Triton | bit-exact vs `x * (1 + scale[idx]) + shift[idx]`, in place | bf16 rows, per-row index into `[n, D]` shift/scale tables; fp32 tables are serialized INT8 MiniMax-H3 |
| `norm_infer` / `rms_norm_fn` | Triton (+torch/NPU/MPS fallbacks) | close | the generic entry point; use when nothing above fits |
### Norm variants
@@ -158,6 +159,7 @@ Several norms look interchangeable and are not. Start here.
| `bias_residual_add` | JIT CUDA | bit-exact `(y + bias[c]) + h` with aten's per-op bf16 rounding, for a conv that skipped its bias pass | channels_last-dense (`C % 8 == 0`) or NCHW/NCDHW contiguous (spatial `% 8 == 0`) bf16; `h` shares `y`'s layout (Qwen-Image 2.1 VAE residual blocks) |
| `residual_gate_add` | JIT CUDA (contiguous), Triton (transposed) | bit-exact `residual + update * gate` | contiguous tensors, or a transposed-dense `[B, tokens, hidden]` residual/output with contiguous update and row-broadcast gate (SANA-Video) |
| `residual_gate_fp32` | Triton | separate FP32 multiply/add, then cast to input dtype | contiguous `[B,S,D]`, strided `[B or 1,1,D]` gate; Kandinsky6 |
| `indexed_gate_bf16` / `indexed_gate_fp32` (`_` in place) | Triton | bit-exact vs `x + gate[idx] * other`: bf16 rounds the product, fp32 keeps a separate FP32 multiply/add | bf16 rows, per-row index into a `[n, D]` gate table (MiniMax-H3) |
These contracts differ: `residual_gate_fp32` does not round the product to
BF16/FP16, and disables FMA to preserve the eager FP32 rounding boundary.
@@ -790,6 +790,9 @@ _EXPORTS: dict[str, str] = {
"indexed_gate_bf16": "modulate.indexed_modulation_triton",
"indexed_gate_bf16_": "modulate.indexed_modulation_triton",
"indexed_scale_shift_bf16_": "modulate.indexed_modulation_triton",
"indexed_gate_fp32": "modulate.indexed_modulation_triton",
"indexed_gate_fp32_": "modulate.indexed_modulation_triton",
"indexed_scale_shift_fp32_": "modulate.indexed_modulation_triton",
"ltx2_ada_values9": "modulate.ltx2_ada_values_triton",
"can_use_modulate_scale_shift_cuda": "modulate.modulate_scale_shift_jit",
"modulate_scale_shift": "modulate.modulate_scale_shift_jit",
@@ -4,7 +4,7 @@ import torch
import triton
import triton.language as tl
from sglang.kernels.numerics import round_bf16_to_fp32
from sglang.kernels.numerics import mul_rn_f32, round_bf16_to_fp32
# Column tiles per row. One program per whole row padded to a power of two
# masked off 37.5% of MiniMax-H3's 5376-wide rows; 2048-wide tiles run both
@@ -165,3 +165,151 @@ def indexed_gate_bf16(
indices: torch.Tensor,
) -> torch.Tensor:
return _indexed_gate_bf16(torch.empty_like(x), x, gate, other, indices)
# FP32 modulation (serialized INT8 H3 keeps its AdaLN tables in FP32). Eager torch
# computes these in FP32 and rounds to BF16 once at the end; mul_rn_f32 keeps the
# product a separate rounding step so the following add cannot contract into an FMA.
@triton.jit
def _indexed_scale_shift_fp32_kernel(
output_ptr,
x_ptr,
shift_ptr,
scale_ptr,
indices_ptr,
hidden_size,
stride_x_row,
stride_shift_row,
stride_scale_row,
stride_indices,
BLOCK_N: tl.constexpr,
):
row = tl.program_id(0)
columns = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N)
mask = columns < hidden_size
index = tl.load(indices_ptr + row * stride_indices)
x = tl.load(x_ptr + row * stride_x_row + columns, mask=mask, other=0.0).to(
tl.float32
)
shift = tl.load(
shift_ptr + index * stride_shift_row + columns, mask=mask, other=0.0
)
scale = tl.load(
scale_ptr + index * stride_scale_row + columns, mask=mask, other=0.0
)
scaled = mul_rn_f32(x, 1.0 + scale)
tl.store(output_ptr + row * stride_x_row + columns, scaled + shift, mask=mask)
@triton.jit
def _indexed_gate_fp32_kernel(
output_ptr,
x_ptr,
gate_ptr,
other_ptr,
indices_ptr,
hidden_size,
stride_output_row,
stride_x_row,
stride_gate_row,
stride_other_row,
stride_indices,
BLOCK_N: tl.constexpr,
):
row = tl.program_id(0)
columns = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N)
mask = columns < hidden_size
index = tl.load(indices_ptr + row * stride_indices)
x = tl.load(x_ptr + row * stride_x_row + columns, mask=mask, other=0.0).to(
tl.float32
)
gate = tl.load(gate_ptr + index * stride_gate_row + columns, mask=mask, other=0.0)
other = tl.load(
other_ptr + row * stride_other_row + columns, mask=mask, other=0.0
).to(tl.float32)
tl.store(
output_ptr + row * stride_output_row + columns,
x + mul_rn_f32(gate, other),
mask=mask,
)
def indexed_scale_shift_fp32_(
x: torch.Tensor,
shift: torch.Tensor,
scale: torch.Tensor,
indices: torch.Tensor,
) -> torch.Tensor:
"""In place: x = bf16(x * (1 + scale[idx]) + shift[idx]) with FP32 shift/scale."""
rows, hidden_size = x.shape
if rows == 0:
return x
block_n, tiles = _row_tiles(hidden_size)
_indexed_scale_shift_fp32_kernel[(rows, tiles)](
x,
x,
shift,
scale,
indices,
hidden_size,
x.stride(0),
shift.stride(0),
scale.stride(0),
indices.stride(0),
BLOCK_N=block_n,
num_warps=4,
)
return x
def _indexed_gate_fp32(
output: torch.Tensor,
x: torch.Tensor,
gate: torch.Tensor,
other: torch.Tensor,
indices: torch.Tensor,
) -> torch.Tensor:
rows, hidden_size = x.shape
if rows == 0:
return output
block_n, tiles = _row_tiles(hidden_size)
_indexed_gate_fp32_kernel[(rows, tiles)](
output,
x,
gate,
other,
indices,
hidden_size,
output.stride(0),
x.stride(0),
gate.stride(0),
other.stride(0),
indices.stride(0),
BLOCK_N=block_n,
num_warps=4,
)
return output
def indexed_gate_fp32_(
x: torch.Tensor,
gate: torch.Tensor,
other: torch.Tensor,
indices: torch.Tensor,
) -> torch.Tensor:
return _indexed_gate_fp32(x, x, gate, other, indices)
def indexed_gate_fp32(
x: torch.Tensor,
gate: torch.Tensor,
other: torch.Tensor,
indices: torch.Tensor,
) -> torch.Tensor:
return _indexed_gate_fp32(torch.empty_like(x), x, gate, other, indices)
@@ -31,7 +31,10 @@ from sglang.kernels.ops.diffusion import (
fused_qknorm_rope_out_of_place,
indexed_gate_bf16,
indexed_gate_bf16_,
indexed_gate_fp32,
indexed_gate_fp32_,
indexed_scale_shift_bf16_,
indexed_scale_shift_fp32_,
indexed_scale_shift_mxfp8_,
mark_minimax_h3_norm_modulate_site,
minimax_h3_norm_modulate_active,
@@ -437,11 +440,15 @@ def _modulate_scale_shift(
x.is_cuda
and x.dtype == _BF16_DTYPE
and dtype == _BF16_DTYPE
and shift.dtype == _BF16_DTYPE
and scale.dtype == _BF16_DTYPE
and shift.dtype == scale.dtype
and x.is_contiguous()
and shift.stride(-1) == scale.stride(-1) == 1
):
return indexed_scale_shift_bf16_(x, shift, scale, indices)
# Serialized INT8 checkpoints keep the AdaLN tables in FP32.
if shift.dtype == _BF16_DTYPE:
return indexed_scale_shift_bf16_(x, shift, scale, indices)
if shift.dtype == _FP32_DTYPE:
return indexed_scale_shift_fp32_(x, shift, scale, indices)
return (
x * (1.0 + scale.index_select(0, indices)) + shift.index_select(0, indices)
).to(dtype)
@@ -462,14 +469,19 @@ def _modulate_gate(
x.is_cuda
and x.dtype == _BF16_DTYPE
and dtype == _BF16_DTYPE
and gate.dtype == _BF16_DTYPE
and other.dtype == _BF16_DTYPE
and x.is_contiguous()
and other.is_contiguous()
and gate.stride(-1) == 1
):
if allow_inplace:
return indexed_gate_bf16_(x, gate, other, indices)
return indexed_gate_bf16(x, gate, other, indices)
if gate.dtype == _BF16_DTYPE:
if allow_inplace:
return indexed_gate_bf16_(x, gate, other, indices)
return indexed_gate_bf16(x, gate, other, indices)
if gate.dtype == _FP32_DTYPE:
if allow_inplace:
return indexed_gate_fp32_(x, gate, other, indices)
return indexed_gate_fp32(x, gate, other, indices)
return (x + gate.index_select(0, indices) * other).to(dtype)