[diffusion] optimization: fuse Flux3 rowwise FP8 quantization with triton (#41671)

This commit is contained in:
trebladev
2026-10-04 09:01:17 +08:00
committed by GitHub
parent 71dc292b97
commit 295a53c90e
5 changed files with 239 additions and 10 deletions
@@ -49,6 +49,13 @@ _HIP = frozenset({CapabilityRequirement.HIP})
# inventory: callers name the one they want via ``select_kernel``.
# ---------------------------------------------------------------------------
_SPECS: tuple[tuple[str, KernelBackend, str, frozenset, str], ...] = (
(
"diffusion.fp8_rowwise",
KernelBackend.TRITON,
"quantization.fp8_rowwise_triton:fp8_rowwise",
frozenset({CapabilityRequirement.cuda(min_sm=(8, 9))}),
"FLUX.3 dynamic rowwise E4M3 quantization and zero row padding.",
),
(
"diffusion.gelu_tanh_cat",
KernelBackend.JIT,
@@ -555,6 +562,8 @@ for _op, _backend, _target, _caps, _description in _SPECS:
# then symbol; a new public kernel belongs here and nowhere else.
# ---------------------------------------------------------------------------
_EXPORTS: dict[str, str] = {
"can_use_fp8_rowwise": "quantization.fp8_rowwise_triton",
"fp8_rowwise": "quantization.fp8_rowwise_triton",
"can_use_fused_gelu_tanh_cat": "activation.gelu_tanh_cat_jit",
"fused_gelu_tanh_cat": "activation.gelu_tanh_cat_jit",
"load_extension_with_recovery": "sglang.srt.utils.cpp_extension_loader",
@@ -0,0 +1,78 @@
# SPDX-License-Identifier: Apache-2.0
"""FLUX.3 dynamic rowwise E4M3 quantization with optional zero row padding."""
import torch
import triton
import triton.language as tl
def can_use_fp8_rowwise(x: torch.Tensor) -> bool:
return (
x.is_cuda
and torch.version.hip is None
and x.dtype in (torch.bfloat16, torch.float16, torch.float32)
and x.ndim == 2
and x.is_contiguous()
and 0 < x.shape[1] <= 16384
and torch.cuda.get_device_capability(x.device) >= (8, 9)
)
@triton.jit
def _fp8_rowwise_kernel(X, Q, S, M: tl.constexpr, K: tl.constexpr, BLOCK: tl.constexpr):
row = tl.program_id(0)
col = tl.arange(0, BLOCK)
x = tl.load(X + row.to(tl.int64) * K + col, (row < M) & (col < K), other=0).to(
tl.float32
)
amax = tl.max(tl.abs(x), 0)
# aten scalar division multiplies by the rounded FP32 reciprocal.
scale = tl.maximum(amax * (1.0 / 448.0), 1.0e-12)
# Tensor/tensor division must retain IEEE rounding at FP8 boundaries.
q = tl.div_rn(x, scale)
q = tl.minimum(tl.maximum(q, -448.0), 448.0)
# On SM89 Triton's default cast can round through FP16. Convert directly
# from FP32 to avoid double rounding at E4M3 halfway points.
bits = tl.inline_asm_elementwise(
"{ .reg .b16 packed; cvt.rn.satfinite.e4m3x2.f32 packed, $1, $1; cvt.u32.u16 $0, packed; }",
constraints="=r,f",
args=[q],
dtype=tl.int32,
is_pure=True,
pack=1,
)
quantized = bits.to(tl.uint8).to(tl.float8e4nv, bitcast=True)
tl.store(Q + row.to(tl.int64) * K + col, quantized, col < K)
tl.store(S + row, scale)
def fp8_rowwise(x: torch.Tensor, row_alignment: int = 1):
"""Return E4M3 ``[ceil(M/alignment)*alignment, K]`` and FP32 row scales.
Reproduce the FLUX.3 eager chain, including scale >= 1e-12. Padded
rows contain zeros with scale 1e-12. No intermediate FP32 tensor or
padded BF16 copy is materialized.
"""
if not can_use_fp8_rowwise(x):
raise ValueError(
"fp8_rowwise requires contiguous 2D FP32/FP16/BF16 on SM89+, K in [1, 16384]"
)
if not isinstance(row_alignment, int) or row_alignment <= 0:
raise ValueError("row_alignment must be a positive integer")
m, k = x.shape
padded_m = triton.cdiv(m, row_alignment) * row_alignment
q = torch.empty((padded_m, k), dtype=torch.float8_e4m3fn, device=x.device)
scale = torch.empty((padded_m,), dtype=torch.float32, device=x.device)
if padded_m:
with torch.cuda.device(x.device):
_fp8_rowwise_kernel[(padded_m,)](
x,
q,
scale,
m,
k,
triton.next_power_of_2(k),
num_warps=4 if k <= 4096 else 8,
enable_fp_fusion=False,
)
return q, scale
@@ -46,8 +46,10 @@ import torch.nn.functional as F
from sglang.kernels.ops.diffusion import (
BitExactFusionGate,
can_use_fp8_rowwise,
can_use_fused_inplace_qknorm_rope,
can_use_fused_layernorm_modulate,
fp8_rowwise,
fused_inplace_qknorm_rope,
fused_layernorm_modulate_raw,
fused_packed_silu_mul_bitexact,
@@ -328,17 +330,22 @@ class Flux3Fp8RowwiseLinear(nn.Module):
flat = x.reshape(-1, self.in_features).contiguous()
rows = flat.shape[0]
pad = -rows % self.ROW_ALIGNMENT
if pad:
flat = F.pad(flat, (0, 0, 0, pad))
activation, activation_scale = quantize_fp8_rowwise(flat)
if self.weight.dtype == torch.float8_e4m3fnuz:
from sglang.srt.layers.quantization.fp8_utils import (
normalize_e4m3fn_to_e4m3fnuz,
)
if can_use_fp8_rowwise(flat):
activation, activation_scale = fp8_rowwise(flat, self.ROW_ALIGNMENT)
else:
if pad:
flat = F.pad(flat, (0, 0, 0, pad))
activation, activation_scale = quantize_fp8_rowwise(flat)
# ROCm-only: can_use_fp8_rowwise() requires torch.version.hip is None,
# so the fnuz weights only ever reach this branch.
if self.weight.dtype == torch.float8_e4m3fnuz:
from sglang.srt.layers.quantization.fp8_utils import (
normalize_e4m3fn_to_e4m3fnuz,
)
activation, activation_scale, _ = normalize_e4m3fn_to_e4m3fnuz(
activation, activation_scale
)
activation, activation_scale, _ = normalize_e4m3fn_to_e4m3fnuz(
activation, activation_scale
)
out = torch._scaled_mm(
activation,
self.weight.T,