[Perf] diffusion: fused indexed modulation for FP32 AdaLN tables (#44180)
Co-authored-by: Mick <mickjagger19@icloud.com>
This commit is contained in:
@@ -134,6 +134,7 @@ Several norms look interchangeable and are not. Start here.
|
||||
| `flydsl_norm_scale_shift` / `flydsl_fused_residual_norm_scale_shift` | FlyDSL | close | **ROCm gfx950 only** |
|
||||
| `try_fused_scale_residual_norm_scale_shift_nvfp4` | JIT CUDA | matches the selected NVFP4 producer contract | Qwen residual LayerNorm/modulation + FC1 NVFP4 quantization |
|
||||
| `fuse_layernorm_scale_shift_gate_select01_kernel` | Triton | close | per-token select between two modulation rows (Qwen-Image) |
|
||||
| `indexed_scale_shift_bf16_` / `indexed_scale_shift_fp32_` | Triton | bit-exact vs `x * (1 + scale[idx]) + shift[idx]`, in place | bf16 rows, per-row index into `[n, D]` shift/scale tables; fp32 tables are serialized INT8 MiniMax-H3 |
|
||||
| `norm_infer` / `rms_norm_fn` | Triton (+torch/NPU/MPS fallbacks) | close | the generic entry point; use when nothing above fits |
|
||||
|
||||
### Norm variants
|
||||
@@ -158,6 +159,7 @@ Several norms look interchangeable and are not. Start here.
|
||||
| `bias_residual_add` | JIT CUDA | bit-exact `(y + bias[c]) + h` with aten's per-op bf16 rounding, for a conv that skipped its bias pass | channels_last-dense (`C % 8 == 0`) or NCHW/NCDHW contiguous (spatial `% 8 == 0`) bf16; `h` shares `y`'s layout (Qwen-Image 2.1 VAE residual blocks) |
|
||||
| `residual_gate_add` | JIT CUDA (contiguous), Triton (transposed) | bit-exact `residual + update * gate` | contiguous tensors, or a transposed-dense `[B, tokens, hidden]` residual/output with contiguous update and row-broadcast gate (SANA-Video) |
|
||||
| `residual_gate_fp32` | Triton | separate FP32 multiply/add, then cast to input dtype | contiguous `[B,S,D]`, strided `[B or 1,1,D]` gate; Kandinsky6 |
|
||||
| `indexed_gate_bf16` / `indexed_gate_fp32` (`_` in place) | Triton | bit-exact vs `x + gate[idx] * other`: bf16 rounds the product, fp32 keeps a separate FP32 multiply/add | bf16 rows, per-row index into a `[n, D]` gate table (MiniMax-H3) |
|
||||
|
||||
These contracts differ: `residual_gate_fp32` does not round the product to
|
||||
BF16/FP16, and disables FMA to preserve the eager FP32 rounding boundary.
|
||||
|
||||
@@ -790,6 +790,9 @@ _EXPORTS: dict[str, str] = {
|
||||
"indexed_gate_bf16": "modulate.indexed_modulation_triton",
|
||||
"indexed_gate_bf16_": "modulate.indexed_modulation_triton",
|
||||
"indexed_scale_shift_bf16_": "modulate.indexed_modulation_triton",
|
||||
"indexed_gate_fp32": "modulate.indexed_modulation_triton",
|
||||
"indexed_gate_fp32_": "modulate.indexed_modulation_triton",
|
||||
"indexed_scale_shift_fp32_": "modulate.indexed_modulation_triton",
|
||||
"ltx2_ada_values9": "modulate.ltx2_ada_values_triton",
|
||||
"can_use_modulate_scale_shift_cuda": "modulate.modulate_scale_shift_jit",
|
||||
"modulate_scale_shift": "modulate.modulate_scale_shift_jit",
|
||||
|
||||
@@ -4,7 +4,7 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.numerics import round_bf16_to_fp32
|
||||
from sglang.kernels.numerics import mul_rn_f32, round_bf16_to_fp32
|
||||
|
||||
# Column tiles per row. One program per whole row padded to a power of two
|
||||
# masked off 37.5% of MiniMax-H3's 5376-wide rows; 2048-wide tiles run both
|
||||
@@ -165,3 +165,151 @@ def indexed_gate_bf16(
|
||||
indices: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return _indexed_gate_bf16(torch.empty_like(x), x, gate, other, indices)
|
||||
|
||||
|
||||
# FP32 modulation (serialized INT8 H3 keeps its AdaLN tables in FP32). Eager torch
|
||||
# computes these in FP32 and rounds to BF16 once at the end; mul_rn_f32 keeps the
|
||||
# product a separate rounding step so the following add cannot contract into an FMA.
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _indexed_scale_shift_fp32_kernel(
|
||||
output_ptr,
|
||||
x_ptr,
|
||||
shift_ptr,
|
||||
scale_ptr,
|
||||
indices_ptr,
|
||||
hidden_size,
|
||||
stride_x_row,
|
||||
stride_shift_row,
|
||||
stride_scale_row,
|
||||
stride_indices,
|
||||
BLOCK_N: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
columns = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
mask = columns < hidden_size
|
||||
index = tl.load(indices_ptr + row * stride_indices)
|
||||
|
||||
x = tl.load(x_ptr + row * stride_x_row + columns, mask=mask, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
shift = tl.load(
|
||||
shift_ptr + index * stride_shift_row + columns, mask=mask, other=0.0
|
||||
)
|
||||
scale = tl.load(
|
||||
scale_ptr + index * stride_scale_row + columns, mask=mask, other=0.0
|
||||
)
|
||||
|
||||
scaled = mul_rn_f32(x, 1.0 + scale)
|
||||
tl.store(output_ptr + row * stride_x_row + columns, scaled + shift, mask=mask)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _indexed_gate_fp32_kernel(
|
||||
output_ptr,
|
||||
x_ptr,
|
||||
gate_ptr,
|
||||
other_ptr,
|
||||
indices_ptr,
|
||||
hidden_size,
|
||||
stride_output_row,
|
||||
stride_x_row,
|
||||
stride_gate_row,
|
||||
stride_other_row,
|
||||
stride_indices,
|
||||
BLOCK_N: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
columns = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
mask = columns < hidden_size
|
||||
index = tl.load(indices_ptr + row * stride_indices)
|
||||
|
||||
x = tl.load(x_ptr + row * stride_x_row + columns, mask=mask, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
gate = tl.load(gate_ptr + index * stride_gate_row + columns, mask=mask, other=0.0)
|
||||
other = tl.load(
|
||||
other_ptr + row * stride_other_row + columns, mask=mask, other=0.0
|
||||
).to(tl.float32)
|
||||
|
||||
tl.store(
|
||||
output_ptr + row * stride_output_row + columns,
|
||||
x + mul_rn_f32(gate, other),
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
|
||||
def indexed_scale_shift_fp32_(
|
||||
x: torch.Tensor,
|
||||
shift: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""In place: x = bf16(x * (1 + scale[idx]) + shift[idx]) with FP32 shift/scale."""
|
||||
rows, hidden_size = x.shape
|
||||
if rows == 0:
|
||||
return x
|
||||
block_n, tiles = _row_tiles(hidden_size)
|
||||
_indexed_scale_shift_fp32_kernel[(rows, tiles)](
|
||||
x,
|
||||
x,
|
||||
shift,
|
||||
scale,
|
||||
indices,
|
||||
hidden_size,
|
||||
x.stride(0),
|
||||
shift.stride(0),
|
||||
scale.stride(0),
|
||||
indices.stride(0),
|
||||
BLOCK_N=block_n,
|
||||
num_warps=4,
|
||||
)
|
||||
return x
|
||||
|
||||
|
||||
def _indexed_gate_fp32(
|
||||
output: torch.Tensor,
|
||||
x: torch.Tensor,
|
||||
gate: torch.Tensor,
|
||||
other: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
rows, hidden_size = x.shape
|
||||
if rows == 0:
|
||||
return output
|
||||
block_n, tiles = _row_tiles(hidden_size)
|
||||
_indexed_gate_fp32_kernel[(rows, tiles)](
|
||||
output,
|
||||
x,
|
||||
gate,
|
||||
other,
|
||||
indices,
|
||||
hidden_size,
|
||||
output.stride(0),
|
||||
x.stride(0),
|
||||
gate.stride(0),
|
||||
other.stride(0),
|
||||
indices.stride(0),
|
||||
BLOCK_N=block_n,
|
||||
num_warps=4,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
def indexed_gate_fp32_(
|
||||
x: torch.Tensor,
|
||||
gate: torch.Tensor,
|
||||
other: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return _indexed_gate_fp32(x, x, gate, other, indices)
|
||||
|
||||
|
||||
def indexed_gate_fp32(
|
||||
x: torch.Tensor,
|
||||
gate: torch.Tensor,
|
||||
other: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return _indexed_gate_fp32(torch.empty_like(x), x, gate, other, indices)
|
||||
|
||||
@@ -31,7 +31,10 @@ from sglang.kernels.ops.diffusion import (
|
||||
fused_qknorm_rope_out_of_place,
|
||||
indexed_gate_bf16,
|
||||
indexed_gate_bf16_,
|
||||
indexed_gate_fp32,
|
||||
indexed_gate_fp32_,
|
||||
indexed_scale_shift_bf16_,
|
||||
indexed_scale_shift_fp32_,
|
||||
indexed_scale_shift_mxfp8_,
|
||||
mark_minimax_h3_norm_modulate_site,
|
||||
minimax_h3_norm_modulate_active,
|
||||
@@ -437,11 +440,15 @@ def _modulate_scale_shift(
|
||||
x.is_cuda
|
||||
and x.dtype == _BF16_DTYPE
|
||||
and dtype == _BF16_DTYPE
|
||||
and shift.dtype == _BF16_DTYPE
|
||||
and scale.dtype == _BF16_DTYPE
|
||||
and shift.dtype == scale.dtype
|
||||
and x.is_contiguous()
|
||||
and shift.stride(-1) == scale.stride(-1) == 1
|
||||
):
|
||||
return indexed_scale_shift_bf16_(x, shift, scale, indices)
|
||||
# Serialized INT8 checkpoints keep the AdaLN tables in FP32.
|
||||
if shift.dtype == _BF16_DTYPE:
|
||||
return indexed_scale_shift_bf16_(x, shift, scale, indices)
|
||||
if shift.dtype == _FP32_DTYPE:
|
||||
return indexed_scale_shift_fp32_(x, shift, scale, indices)
|
||||
return (
|
||||
x * (1.0 + scale.index_select(0, indices)) + shift.index_select(0, indices)
|
||||
).to(dtype)
|
||||
@@ -462,14 +469,19 @@ def _modulate_gate(
|
||||
x.is_cuda
|
||||
and x.dtype == _BF16_DTYPE
|
||||
and dtype == _BF16_DTYPE
|
||||
and gate.dtype == _BF16_DTYPE
|
||||
and other.dtype == _BF16_DTYPE
|
||||
and x.is_contiguous()
|
||||
and other.is_contiguous()
|
||||
and gate.stride(-1) == 1
|
||||
):
|
||||
if allow_inplace:
|
||||
return indexed_gate_bf16_(x, gate, other, indices)
|
||||
return indexed_gate_bf16(x, gate, other, indices)
|
||||
if gate.dtype == _BF16_DTYPE:
|
||||
if allow_inplace:
|
||||
return indexed_gate_bf16_(x, gate, other, indices)
|
||||
return indexed_gate_bf16(x, gate, other, indices)
|
||||
if gate.dtype == _FP32_DTYPE:
|
||||
if allow_inplace:
|
||||
return indexed_gate_fp32_(x, gate, other, indices)
|
||||
return indexed_gate_fp32(x, gate, other, indices)
|
||||
return (x + gate.index_select(0, indices) * other).to(dtype)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user