[Perf] diffusion: fuse FLUX3 SwiGLU with rowwise FP8 quantization (#42758)
This commit is contained in:
@@ -205,6 +205,22 @@ tensor copy at each residual site.
|
||||
| `vdn_temporal_conv_act`, `vdn_silu_l2norm`, `vdn_linear_epilogue` | Triton | one rounding at the store, within one bf16 ulp of the eager chain; the model's own inference kernels, mounted unconditionally by the VDN-H3 branch |
|
||||
| `vdn_delta_factors` | JIT CUDA | `(alpha * inv(I + A), B @ inv(I + A))` in one launch; same fp32 accuracy class as the cholesky + solve_triangular chain (cond-dominated); head_dim 128 |
|
||||
|
||||
### Rowwise E4M3 producers (SM89+)
|
||||
|
||||
| Entry point | Backend | Contract |
|
||||
|---|---|---|
|
||||
| `fp8_rowwise` | Triton | Dynamic per-row FP32 scales, E4M3 payload and zero-row padding; matches the eager quantizer |
|
||||
| `fused_packed_swiglu_fp8_rowwise` | Triton | Packed/row-strided BF16 `[B,T,2*K]` to padded 2D E4M3 + FP32 row scales; preserves both BF16 rounds of `F.silu(gate) * up`; mounted in FLUX3 FP8r MLP output projections |
|
||||
|
||||
The SwiGLU producer avoids materializing a BF16 activation. Its measured SM89
|
||||
`K=9216` path uses nine 1024-element tiles, native BF16 rounding, and paired
|
||||
FP32-to-E4M3 conversions, retaining one row reduction and zero-row padding.
|
||||
Other widths/devices retain the full-row implementation. Finite inputs,
|
||||
`K <= 16384`, and flattenable B/T row strides are required. Padding rows have
|
||||
zero payload and scale `1e-12`. The local CUDA-graph benchmark is
|
||||
`test/manual/kernels/diffusion/bench_swiglu_fp8_rowwise.py`; it compares the
|
||||
existing two kernels and the fusion, both alone and with the same FP8 GEMM.
|
||||
|
||||
### MXFP8 producers (online `mxfp8`, cuBLASLt block-scaled GEMM on SM100)
|
||||
|
||||
Defined in `sglang.kernels.ops.quantization.mxfp8_swizzled_triton` (`silu_mul_mxfp8` in
|
||||
|
||||
@@ -45,6 +45,12 @@ _CUDA_SM10X = frozenset({CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10,
|
||||
# inventory: callers name the one they want via ``select_kernel``.
|
||||
# ---------------------------------------------------------------------------
|
||||
_SPECS: tuple[tuple[str, KernelBackend, str, frozenset], ...] = (
|
||||
(
|
||||
"diffusion.fused_packed_swiglu_fp8_rowwise",
|
||||
KernelBackend.TRITON,
|
||||
"quantization.swiglu_fp8_rowwise_triton:fused_packed_swiglu_fp8_rowwise",
|
||||
frozenset({CapabilityRequirement.cuda(min_sm=(8, 9))}),
|
||||
),
|
||||
(
|
||||
"diffusion.residual_gate_fp32",
|
||||
KernelBackend.TRITON,
|
||||
@@ -618,6 +624,7 @@ for _op, _backend, _target, _caps in _SPECS:
|
||||
# then symbol; a new public kernel belongs here and nowhere else.
|
||||
# ---------------------------------------------------------------------------
|
||||
_EXPORTS: dict[str, str] = {
|
||||
"fused_packed_swiglu_fp8_rowwise": "quantization.swiglu_fp8_rowwise_triton",
|
||||
"apply_matrix_rope": "rope.matrix_rope",
|
||||
"residual_gate_fp32": "modulate.residual_gate_fp32",
|
||||
"can_use_fp8_rowwise": "quantization.fp8_rowwise_triton",
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Packed BF16 SwiGLU directly to row-scaled E4M3, without a BF16 output."""
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.numerics import round_bf16_to_fp32
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _swiglu_fp8_rowwise_kernel(
|
||||
X,
|
||||
Q,
|
||||
S,
|
||||
M: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
STRIDE: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
col = tl.arange(0, BLOCK)
|
||||
mask = (row < M) & (col < K)
|
||||
gate = tl.load(X + row * STRIDE + col, mask, other=0).to(tl.float32)
|
||||
up = tl.load(X + row * STRIDE + K + col, mask, other=0).to(tl.float32)
|
||||
# Reproduce both BF16 rounding boundaries of F.silu(gate) * up.
|
||||
silu = round_bf16_to_fp32(gate * tl.sigmoid(gate))
|
||||
value = round_bf16_to_fp32(silu * up)
|
||||
scale = tl.maximum(tl.max(tl.abs(value), 0) * (1.0 / 448.0), 1.0e-12)
|
||||
normalized = tl.div_rn(value, scale)
|
||||
normalized = tl.minimum(tl.maximum(normalized, -448.0), 448.0)
|
||||
bits = tl.inline_asm_elementwise(
|
||||
"{ .reg .b16 packed; cvt.rn.satfinite.e4m3x2.f32 packed, $1, $1; cvt.u32.u16 $0, packed; }",
|
||||
constraints="=r,f",
|
||||
args=[normalized],
|
||||
dtype=tl.int32,
|
||||
is_pure=True,
|
||||
pack=1,
|
||||
)
|
||||
quantized = bits.to(tl.uint8).to(tl.float8e4nv, bitcast=True)
|
||||
tl.store(Q + row * K + col, quantized, col < K)
|
||||
tl.store(S + row, scale)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _swiglu_fp8_rowwise_tiled_kernel(
|
||||
X,
|
||||
Q,
|
||||
S,
|
||||
M: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
STRIDE: tl.constexpr,
|
||||
TILE: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
columns = tl.arange(0, TILE)
|
||||
values = ()
|
||||
maxima = tl.full((TILE,), 0, tl.float32)
|
||||
# Keep each tile's BF16-rounded activation in registers, but reduce the
|
||||
# elementwise maxima first: only one cross-warp reduction per row.
|
||||
for tile in tl.static_range(triton.cdiv(K, TILE)):
|
||||
col = columns + tile * TILE
|
||||
mask = (row < M) & (col < K)
|
||||
gate = tl.load(X + row * STRIDE + col, mask, other=0).to(tl.float32)
|
||||
up = tl.load(X + row * STRIDE + K + col, mask, other=0).to(tl.float32)
|
||||
silu = (gate * tl.sigmoid(gate)).to(tl.bfloat16).to(tl.float32)
|
||||
value = (silu * up).to(tl.bfloat16).to(tl.float32)
|
||||
values += (value,)
|
||||
maxima = tl.maximum(maxima, tl.abs(value))
|
||||
scale = tl.maximum(tl.max(maxima, 0) * (1.0 / 448.0), 1.0e-12)
|
||||
for tile in tl.static_range(triton.cdiv(K, TILE)):
|
||||
col = columns + tile * TILE
|
||||
normalized = tl.div_rn(values[tile], scale)
|
||||
normalized = tl.minimum(tl.maximum(normalized, -448.0), 448.0)
|
||||
# Convert two distinct FP32 elements per instruction; PTX places the
|
||||
# second source in the low byte. Keep the direct FP32->E4M3 rounding.
|
||||
bits = tl.inline_asm_elementwise(
|
||||
"{ .reg .b16 packed; cvt.rn.satfinite.e4m3x2.f32 packed, $3, $2; "
|
||||
"cvt.u32.u16 $0, packed; shr.b32 $1, $0, 8; }",
|
||||
constraints="=r,=r,f,f",
|
||||
args=[normalized],
|
||||
dtype=tl.int32,
|
||||
is_pure=True,
|
||||
pack=2,
|
||||
)
|
||||
quantized = bits.to(tl.uint8).to(tl.float8e4nv, bitcast=True)
|
||||
tl.store(Q + row * K + col, quantized, col < K)
|
||||
tl.store(S + row, scale)
|
||||
|
||||
|
||||
def fused_packed_swiglu_fp8_rowwise(x: torch.Tensor, row_alignment: int = 16):
|
||||
"""Quantize ``F.silu(gate) * up`` from packed BF16 ``[B, T, 2*K]``.
|
||||
|
||||
Return contiguous E4M3 ``[ceil(B*T/alignment)*alignment, K]`` plus
|
||||
FP32 row scales. Accept row-strided slices of a packed QKV/MLP projection
|
||||
when the B/T dimensions can be flattened without a copy. Finite inputs
|
||||
reproduce both intermediate BF16 rounds and the existing rowwise FP8
|
||||
contract. Padding rows have zero payload and scale 1e-12. NVIDIA SM89+.
|
||||
Used by the FLUX3 FP8r MLP output projection.
|
||||
"""
|
||||
if not (
|
||||
x.is_cuda
|
||||
and torch.version.hip is None
|
||||
and x.dtype == torch.bfloat16
|
||||
and x.ndim == 3
|
||||
and x.stride(-1) == 1
|
||||
and x.shape[-1] % 2 == 0
|
||||
and 0 < x.shape[-1] // 2 <= 16384
|
||||
and x.stride(1) >= x.shape[-1]
|
||||
and (x.shape[0] <= 1 or x.stride(0) == x.shape[1] * x.stride(1))
|
||||
and torch.cuda.get_device_capability(x.device) >= (8, 9)
|
||||
):
|
||||
raise ValueError(
|
||||
"requires row-strided packed BF16 [B,T,2*K], K in [1,16384], on SM89+"
|
||||
)
|
||||
if (
|
||||
not isinstance(row_alignment, int)
|
||||
or isinstance(row_alignment, bool)
|
||||
or row_alignment <= 0
|
||||
):
|
||||
raise ValueError("row_alignment must be a positive integer")
|
||||
rows = x.shape[0] * x.shape[1]
|
||||
hidden = x.shape[-1] // 2
|
||||
padded_rows = triton.cdiv(rows, row_alignment) * row_alignment
|
||||
q = torch.empty((padded_rows, hidden), device=x.device, dtype=torch.float8_e4m3fn)
|
||||
scales = torch.empty((padded_rows,), device=x.device, dtype=torch.float32)
|
||||
if padded_rows:
|
||||
num_warps = 4 if hidden <= 4096 else 8
|
||||
# SM89 measurements: extra warps help underfilled grids; eight win
|
||||
# on the model's large 3072/9216-wide rows without register spills.
|
||||
if torch.cuda.get_device_capability(x.device) == (8, 9) and hidden in (
|
||||
3072,
|
||||
9216,
|
||||
):
|
||||
num_warps = 16 if padded_rows <= 64 else 8
|
||||
with torch.cuda.device(x.device):
|
||||
if torch.cuda.get_device_capability(x.device) == (8, 9) and hidden == 9216:
|
||||
# Nine exact-width tiles instead of a 16384-wide row; measured
|
||||
# on SM89, with unchanged BF16 and E4M3 rounding boundaries.
|
||||
_swiglu_fp8_rowwise_tiled_kernel[(padded_rows,)](
|
||||
x,
|
||||
q,
|
||||
scales,
|
||||
rows,
|
||||
hidden,
|
||||
x.stride(1),
|
||||
1024,
|
||||
num_warps=16,
|
||||
enable_fp_fusion=False,
|
||||
)
|
||||
return q, scales
|
||||
_swiglu_fp8_rowwise_kernel[(padded_rows,)](
|
||||
x,
|
||||
q,
|
||||
scales,
|
||||
rows,
|
||||
hidden,
|
||||
x.stride(1),
|
||||
triton.next_power_of_2(hidden),
|
||||
num_warps=num_warps,
|
||||
enable_fp_fusion=False,
|
||||
)
|
||||
return q, scales
|
||||
@@ -50,6 +50,7 @@ from sglang.kernels.ops.diffusion import (
|
||||
fp8_rowwise,
|
||||
fused_inplace_qknorm_rope,
|
||||
fused_packed_silu_mul_bitexact,
|
||||
fused_packed_swiglu_fp8_rowwise,
|
||||
residual_gate_add,
|
||||
)
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
|
||||
@@ -273,7 +274,7 @@ class Flux3Fp8RowwiseLinear(nn.Module):
|
||||
parallel linears so the module can replace either kind.
|
||||
"""
|
||||
|
||||
# _scaled_mm needs M padded to a multiple of 16.
|
||||
# Preserve the checkpoint inference path's row padding convention.
|
||||
ROW_ALIGNMENT = 16
|
||||
|
||||
def __init__(
|
||||
@@ -325,6 +326,35 @@ class Flux3Fp8RowwiseLinear(nn.Module):
|
||||
activation, activation_scale, _ = normalize_e4m3fn_to_e4m3fnuz(
|
||||
activation, activation_scale
|
||||
)
|
||||
return self._forward_quantized(activation, activation_scale, leading)
|
||||
|
||||
def forward_swiglu(self, packed: torch.Tensor):
|
||||
"""Fuse packed BF16 SwiGLU with this layer's activation quantization."""
|
||||
if not (
|
||||
packed.is_cuda
|
||||
and torch.version.hip is None
|
||||
and packed.dtype == torch.bfloat16
|
||||
and packed.ndim == 3
|
||||
and packed.shape[-1] == 2 * self.in_features
|
||||
and 0 < self.in_features <= 16384
|
||||
and packed.stride(-1) == 1
|
||||
and packed.stride(1) >= packed.shape[-1]
|
||||
and (
|
||||
packed.shape[0] <= 1
|
||||
or packed.stride(0) == packed.shape[1] * packed.stride(1)
|
||||
)
|
||||
and self.weight.dtype == torch.float8_e4m3fn
|
||||
and torch.cuda.get_device_capability(packed.device) >= (8, 9)
|
||||
and not torch.compiler.is_compiling()
|
||||
):
|
||||
return self(_swiglu(packed))
|
||||
activation, activation_scale = fused_packed_swiglu_fp8_rowwise(
|
||||
packed, self.ROW_ALIGNMENT
|
||||
)
|
||||
return self._forward_quantized(activation, activation_scale, packed.shape[:-1])
|
||||
|
||||
def _forward_quantized(self, activation, activation_scale, leading):
|
||||
rows = math.prod(leading)
|
||||
out = torch._scaled_mm(
|
||||
activation,
|
||||
self.weight.T,
|
||||
@@ -528,7 +558,11 @@ class Flux3Block(nn.Module):
|
||||
q, k, v = q.contiguous(), k.contiguous(), v.contiguous()
|
||||
attended = self.attn(q, k, v, attn_mask_meta=sp.attn_mask_meta)
|
||||
attended = attended.reshape(batch, length, self.local_hidden)
|
||||
out = self.attn_out(attended)[0] + self.mlp_out(_swiglu(mlp))[0]
|
||||
if isinstance(self.mlp_out, Flux3Fp8RowwiseLinear):
|
||||
mlp_out = self.mlp_out.forward_swiglu(mlp)[0]
|
||||
else:
|
||||
mlp_out = self.mlp_out(_swiglu(mlp))[0]
|
||||
out = self.attn_out(attended)[0] + mlp_out
|
||||
if self.tp_size > 1:
|
||||
# One reduce of the summed partials, not one per branch (the order
|
||||
# the FLUX 3 reference TP uses).
|
||||
|
||||
Reference in New Issue
Block a user