[Perf] diffusion: fuse FLUX3 SwiGLU with rowwise FP8 quantization (#42758)

This commit is contained in:
trebladev
2026-10-22 21:17:56 +08:00
committed by GitHub
parent 635836129b
commit 698fa949ec
7 changed files with 617 additions and 3 deletions
@@ -205,6 +205,22 @@ tensor copy at each residual site.
| `vdn_temporal_conv_act`, `vdn_silu_l2norm`, `vdn_linear_epilogue` | Triton | one rounding at the store, within one bf16 ulp of the eager chain; the model's own inference kernels, mounted unconditionally by the VDN-H3 branch |
| `vdn_delta_factors` | JIT CUDA | `(alpha * inv(I + A), B @ inv(I + A))` in one launch; same fp32 accuracy class as the cholesky + solve_triangular chain (cond-dominated); head_dim 128 |
### Rowwise E4M3 producers (SM89+)
| Entry point | Backend | Contract |
|---|---|---|
| `fp8_rowwise` | Triton | Dynamic per-row FP32 scales, E4M3 payload and zero-row padding; matches the eager quantizer |
| `fused_packed_swiglu_fp8_rowwise` | Triton | Packed/row-strided BF16 `[B,T,2*K]` to padded 2D E4M3 + FP32 row scales; preserves both BF16 rounds of `F.silu(gate) * up`; mounted in FLUX3 FP8r MLP output projections |
The SwiGLU producer avoids materializing a BF16 activation. Its measured SM89
`K=9216` path uses nine 1024-element tiles, native BF16 rounding, and paired
FP32-to-E4M3 conversions, retaining one row reduction and zero-row padding.
Other widths/devices retain the full-row implementation. Finite inputs,
`K <= 16384`, and flattenable B/T row strides are required. Padding rows have
zero payload and scale `1e-12`. The local CUDA-graph benchmark is
`test/manual/kernels/diffusion/bench_swiglu_fp8_rowwise.py`; it compares the
existing two kernels and the fusion, both alone and with the same FP8 GEMM.
### MXFP8 producers (online `mxfp8`, cuBLASLt block-scaled GEMM on SM100)
Defined in `sglang.kernels.ops.quantization.mxfp8_swizzled_triton` (`silu_mul_mxfp8` in
@@ -45,6 +45,12 @@ _CUDA_SM10X = frozenset({CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10,
# inventory: callers name the one they want via ``select_kernel``.
# ---------------------------------------------------------------------------
_SPECS: tuple[tuple[str, KernelBackend, str, frozenset], ...] = (
(
"diffusion.fused_packed_swiglu_fp8_rowwise",
KernelBackend.TRITON,
"quantization.swiglu_fp8_rowwise_triton:fused_packed_swiglu_fp8_rowwise",
frozenset({CapabilityRequirement.cuda(min_sm=(8, 9))}),
),
(
"diffusion.residual_gate_fp32",
KernelBackend.TRITON,
@@ -618,6 +624,7 @@ for _op, _backend, _target, _caps in _SPECS:
# then symbol; a new public kernel belongs here and nowhere else.
# ---------------------------------------------------------------------------
_EXPORTS: dict[str, str] = {
"fused_packed_swiglu_fp8_rowwise": "quantization.swiglu_fp8_rowwise_triton",
"apply_matrix_rope": "rope.matrix_rope",
"residual_gate_fp32": "modulate.residual_gate_fp32",
"can_use_fp8_rowwise": "quantization.fp8_rowwise_triton",
@@ -0,0 +1,163 @@
# SPDX-License-Identifier: Apache-2.0
"""Packed BF16 SwiGLU directly to row-scaled E4M3, without a BF16 output."""
import torch
import triton
import triton.language as tl
from sglang.kernels.numerics import round_bf16_to_fp32
@triton.jit
def _swiglu_fp8_rowwise_kernel(
X,
Q,
S,
M: tl.constexpr,
K: tl.constexpr,
STRIDE: tl.constexpr,
BLOCK: tl.constexpr,
):
row = tl.program_id(0).to(tl.int64)
col = tl.arange(0, BLOCK)
mask = (row < M) & (col < K)
gate = tl.load(X + row * STRIDE + col, mask, other=0).to(tl.float32)
up = tl.load(X + row * STRIDE + K + col, mask, other=0).to(tl.float32)
# Reproduce both BF16 rounding boundaries of F.silu(gate) * up.
silu = round_bf16_to_fp32(gate * tl.sigmoid(gate))
value = round_bf16_to_fp32(silu * up)
scale = tl.maximum(tl.max(tl.abs(value), 0) * (1.0 / 448.0), 1.0e-12)
normalized = tl.div_rn(value, scale)
normalized = tl.minimum(tl.maximum(normalized, -448.0), 448.0)
bits = tl.inline_asm_elementwise(
"{ .reg .b16 packed; cvt.rn.satfinite.e4m3x2.f32 packed, $1, $1; cvt.u32.u16 $0, packed; }",
constraints="=r,f",
args=[normalized],
dtype=tl.int32,
is_pure=True,
pack=1,
)
quantized = bits.to(tl.uint8).to(tl.float8e4nv, bitcast=True)
tl.store(Q + row * K + col, quantized, col < K)
tl.store(S + row, scale)
@triton.jit
def _swiglu_fp8_rowwise_tiled_kernel(
X,
Q,
S,
M: tl.constexpr,
K: tl.constexpr,
STRIDE: tl.constexpr,
TILE: tl.constexpr,
):
row = tl.program_id(0).to(tl.int64)
columns = tl.arange(0, TILE)
values = ()
maxima = tl.full((TILE,), 0, tl.float32)
# Keep each tile's BF16-rounded activation in registers, but reduce the
# elementwise maxima first: only one cross-warp reduction per row.
for tile in tl.static_range(triton.cdiv(K, TILE)):
col = columns + tile * TILE
mask = (row < M) & (col < K)
gate = tl.load(X + row * STRIDE + col, mask, other=0).to(tl.float32)
up = tl.load(X + row * STRIDE + K + col, mask, other=0).to(tl.float32)
silu = (gate * tl.sigmoid(gate)).to(tl.bfloat16).to(tl.float32)
value = (silu * up).to(tl.bfloat16).to(tl.float32)
values += (value,)
maxima = tl.maximum(maxima, tl.abs(value))
scale = tl.maximum(tl.max(maxima, 0) * (1.0 / 448.0), 1.0e-12)
for tile in tl.static_range(triton.cdiv(K, TILE)):
col = columns + tile * TILE
normalized = tl.div_rn(values[tile], scale)
normalized = tl.minimum(tl.maximum(normalized, -448.0), 448.0)
# Convert two distinct FP32 elements per instruction; PTX places the
# second source in the low byte. Keep the direct FP32->E4M3 rounding.
bits = tl.inline_asm_elementwise(
"{ .reg .b16 packed; cvt.rn.satfinite.e4m3x2.f32 packed, $3, $2; "
"cvt.u32.u16 $0, packed; shr.b32 $1, $0, 8; }",
constraints="=r,=r,f,f",
args=[normalized],
dtype=tl.int32,
is_pure=True,
pack=2,
)
quantized = bits.to(tl.uint8).to(tl.float8e4nv, bitcast=True)
tl.store(Q + row * K + col, quantized, col < K)
tl.store(S + row, scale)
def fused_packed_swiglu_fp8_rowwise(x: torch.Tensor, row_alignment: int = 16):
"""Quantize ``F.silu(gate) * up`` from packed BF16 ``[B, T, 2*K]``.
Return contiguous E4M3 ``[ceil(B*T/alignment)*alignment, K]`` plus
FP32 row scales. Accept row-strided slices of a packed QKV/MLP projection
when the B/T dimensions can be flattened without a copy. Finite inputs
reproduce both intermediate BF16 rounds and the existing rowwise FP8
contract. Padding rows have zero payload and scale 1e-12. NVIDIA SM89+.
Used by the FLUX3 FP8r MLP output projection.
"""
if not (
x.is_cuda
and torch.version.hip is None
and x.dtype == torch.bfloat16
and x.ndim == 3
and x.stride(-1) == 1
and x.shape[-1] % 2 == 0
and 0 < x.shape[-1] // 2 <= 16384
and x.stride(1) >= x.shape[-1]
and (x.shape[0] <= 1 or x.stride(0) == x.shape[1] * x.stride(1))
and torch.cuda.get_device_capability(x.device) >= (8, 9)
):
raise ValueError(
"requires row-strided packed BF16 [B,T,2*K], K in [1,16384], on SM89+"
)
if (
not isinstance(row_alignment, int)
or isinstance(row_alignment, bool)
or row_alignment <= 0
):
raise ValueError("row_alignment must be a positive integer")
rows = x.shape[0] * x.shape[1]
hidden = x.shape[-1] // 2
padded_rows = triton.cdiv(rows, row_alignment) * row_alignment
q = torch.empty((padded_rows, hidden), device=x.device, dtype=torch.float8_e4m3fn)
scales = torch.empty((padded_rows,), device=x.device, dtype=torch.float32)
if padded_rows:
num_warps = 4 if hidden <= 4096 else 8
# SM89 measurements: extra warps help underfilled grids; eight win
# on the model's large 3072/9216-wide rows without register spills.
if torch.cuda.get_device_capability(x.device) == (8, 9) and hidden in (
3072,
9216,
):
num_warps = 16 if padded_rows <= 64 else 8
with torch.cuda.device(x.device):
if torch.cuda.get_device_capability(x.device) == (8, 9) and hidden == 9216:
# Nine exact-width tiles instead of a 16384-wide row; measured
# on SM89, with unchanged BF16 and E4M3 rounding boundaries.
_swiglu_fp8_rowwise_tiled_kernel[(padded_rows,)](
x,
q,
scales,
rows,
hidden,
x.stride(1),
1024,
num_warps=16,
enable_fp_fusion=False,
)
return q, scales
_swiglu_fp8_rowwise_kernel[(padded_rows,)](
x,
q,
scales,
rows,
hidden,
x.stride(1),
triton.next_power_of_2(hidden),
num_warps=num_warps,
enable_fp_fusion=False,
)
return q, scales
@@ -50,6 +50,7 @@ from sglang.kernels.ops.diffusion import (
fp8_rowwise,
fused_inplace_qknorm_rope,
fused_packed_silu_mul_bitexact,
fused_packed_swiglu_fp8_rowwise,
residual_gate_add,
)
from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
@@ -273,7 +274,7 @@ class Flux3Fp8RowwiseLinear(nn.Module):
parallel linears so the module can replace either kind.
"""
# _scaled_mm needs M padded to a multiple of 16.
# Preserve the checkpoint inference path's row padding convention.
ROW_ALIGNMENT = 16
def __init__(
@@ -325,6 +326,35 @@ class Flux3Fp8RowwiseLinear(nn.Module):
activation, activation_scale, _ = normalize_e4m3fn_to_e4m3fnuz(
activation, activation_scale
)
return self._forward_quantized(activation, activation_scale, leading)
def forward_swiglu(self, packed: torch.Tensor):
"""Fuse packed BF16 SwiGLU with this layer's activation quantization."""
if not (
packed.is_cuda
and torch.version.hip is None
and packed.dtype == torch.bfloat16
and packed.ndim == 3
and packed.shape[-1] == 2 * self.in_features
and 0 < self.in_features <= 16384
and packed.stride(-1) == 1
and packed.stride(1) >= packed.shape[-1]
and (
packed.shape[0] <= 1
or packed.stride(0) == packed.shape[1] * packed.stride(1)
)
and self.weight.dtype == torch.float8_e4m3fn
and torch.cuda.get_device_capability(packed.device) >= (8, 9)
and not torch.compiler.is_compiling()
):
return self(_swiglu(packed))
activation, activation_scale = fused_packed_swiglu_fp8_rowwise(
packed, self.ROW_ALIGNMENT
)
return self._forward_quantized(activation, activation_scale, packed.shape[:-1])
def _forward_quantized(self, activation, activation_scale, leading):
rows = math.prod(leading)
out = torch._scaled_mm(
activation,
self.weight.T,
@@ -528,7 +558,11 @@ class Flux3Block(nn.Module):
q, k, v = q.contiguous(), k.contiguous(), v.contiguous()
attended = self.attn(q, k, v, attn_mask_meta=sp.attn_mask_meta)
attended = attended.reshape(batch, length, self.local_hidden)
out = self.attn_out(attended)[0] + self.mlp_out(_swiglu(mlp))[0]
if isinstance(self.mlp_out, Flux3Fp8RowwiseLinear):
mlp_out = self.mlp_out.forward_swiglu(mlp)[0]
else:
mlp_out = self.mlp_out(_swiglu(mlp))[0]
out = self.attn_out(attended)[0] + mlp_out
if self.tp_size > 1:
# One reduce of the summed partials, not one per branch (the order
# the FLUX 3 reference TP uses).