[diffusion] optimization: fuse Flux3 rowwise FP8 quantization with triton (#41671)
This commit is contained in:
@@ -49,6 +49,13 @@ _HIP = frozenset({CapabilityRequirement.HIP})
|
||||
# inventory: callers name the one they want via ``select_kernel``.
|
||||
# ---------------------------------------------------------------------------
|
||||
_SPECS: tuple[tuple[str, KernelBackend, str, frozenset, str], ...] = (
|
||||
(
|
||||
"diffusion.fp8_rowwise",
|
||||
KernelBackend.TRITON,
|
||||
"quantization.fp8_rowwise_triton:fp8_rowwise",
|
||||
frozenset({CapabilityRequirement.cuda(min_sm=(8, 9))}),
|
||||
"FLUX.3 dynamic rowwise E4M3 quantization and zero row padding.",
|
||||
),
|
||||
(
|
||||
"diffusion.gelu_tanh_cat",
|
||||
KernelBackend.JIT,
|
||||
@@ -555,6 +562,8 @@ for _op, _backend, _target, _caps, _description in _SPECS:
|
||||
# then symbol; a new public kernel belongs here and nowhere else.
|
||||
# ---------------------------------------------------------------------------
|
||||
_EXPORTS: dict[str, str] = {
|
||||
"can_use_fp8_rowwise": "quantization.fp8_rowwise_triton",
|
||||
"fp8_rowwise": "quantization.fp8_rowwise_triton",
|
||||
"can_use_fused_gelu_tanh_cat": "activation.gelu_tanh_cat_jit",
|
||||
"fused_gelu_tanh_cat": "activation.gelu_tanh_cat_jit",
|
||||
"load_extension_with_recovery": "sglang.srt.utils.cpp_extension_loader",
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""FLUX.3 dynamic rowwise E4M3 quantization with optional zero row padding."""
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
def can_use_fp8_rowwise(x: torch.Tensor) -> bool:
|
||||
return (
|
||||
x.is_cuda
|
||||
and torch.version.hip is None
|
||||
and x.dtype in (torch.bfloat16, torch.float16, torch.float32)
|
||||
and x.ndim == 2
|
||||
and x.is_contiguous()
|
||||
and 0 < x.shape[1] <= 16384
|
||||
and torch.cuda.get_device_capability(x.device) >= (8, 9)
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fp8_rowwise_kernel(X, Q, S, M: tl.constexpr, K: tl.constexpr, BLOCK: tl.constexpr):
|
||||
row = tl.program_id(0)
|
||||
col = tl.arange(0, BLOCK)
|
||||
x = tl.load(X + row.to(tl.int64) * K + col, (row < M) & (col < K), other=0).to(
|
||||
tl.float32
|
||||
)
|
||||
amax = tl.max(tl.abs(x), 0)
|
||||
# aten scalar division multiplies by the rounded FP32 reciprocal.
|
||||
scale = tl.maximum(amax * (1.0 / 448.0), 1.0e-12)
|
||||
# Tensor/tensor division must retain IEEE rounding at FP8 boundaries.
|
||||
q = tl.div_rn(x, scale)
|
||||
q = tl.minimum(tl.maximum(q, -448.0), 448.0)
|
||||
# On SM89 Triton's default cast can round through FP16. Convert directly
|
||||
# from FP32 to avoid double rounding at E4M3 halfway points.
|
||||
bits = tl.inline_asm_elementwise(
|
||||
"{ .reg .b16 packed; cvt.rn.satfinite.e4m3x2.f32 packed, $1, $1; cvt.u32.u16 $0, packed; }",
|
||||
constraints="=r,f",
|
||||
args=[q],
|
||||
dtype=tl.int32,
|
||||
is_pure=True,
|
||||
pack=1,
|
||||
)
|
||||
quantized = bits.to(tl.uint8).to(tl.float8e4nv, bitcast=True)
|
||||
tl.store(Q + row.to(tl.int64) * K + col, quantized, col < K)
|
||||
tl.store(S + row, scale)
|
||||
|
||||
|
||||
def fp8_rowwise(x: torch.Tensor, row_alignment: int = 1):
|
||||
"""Return E4M3 ``[ceil(M/alignment)*alignment, K]`` and FP32 row scales.
|
||||
|
||||
Reproduce the FLUX.3 eager chain, including scale >= 1e-12. Padded
|
||||
rows contain zeros with scale 1e-12. No intermediate FP32 tensor or
|
||||
padded BF16 copy is materialized.
|
||||
"""
|
||||
if not can_use_fp8_rowwise(x):
|
||||
raise ValueError(
|
||||
"fp8_rowwise requires contiguous 2D FP32/FP16/BF16 on SM89+, K in [1, 16384]"
|
||||
)
|
||||
if not isinstance(row_alignment, int) or row_alignment <= 0:
|
||||
raise ValueError("row_alignment must be a positive integer")
|
||||
m, k = x.shape
|
||||
padded_m = triton.cdiv(m, row_alignment) * row_alignment
|
||||
q = torch.empty((padded_m, k), dtype=torch.float8_e4m3fn, device=x.device)
|
||||
scale = torch.empty((padded_m,), dtype=torch.float32, device=x.device)
|
||||
if padded_m:
|
||||
with torch.cuda.device(x.device):
|
||||
_fp8_rowwise_kernel[(padded_m,)](
|
||||
x,
|
||||
q,
|
||||
scale,
|
||||
m,
|
||||
k,
|
||||
triton.next_power_of_2(k),
|
||||
num_warps=4 if k <= 4096 else 8,
|
||||
enable_fp_fusion=False,
|
||||
)
|
||||
return q, scale
|
||||
@@ -46,8 +46,10 @@ import torch.nn.functional as F
|
||||
|
||||
from sglang.kernels.ops.diffusion import (
|
||||
BitExactFusionGate,
|
||||
can_use_fp8_rowwise,
|
||||
can_use_fused_inplace_qknorm_rope,
|
||||
can_use_fused_layernorm_modulate,
|
||||
fp8_rowwise,
|
||||
fused_inplace_qknorm_rope,
|
||||
fused_layernorm_modulate_raw,
|
||||
fused_packed_silu_mul_bitexact,
|
||||
@@ -328,17 +330,22 @@ class Flux3Fp8RowwiseLinear(nn.Module):
|
||||
flat = x.reshape(-1, self.in_features).contiguous()
|
||||
rows = flat.shape[0]
|
||||
pad = -rows % self.ROW_ALIGNMENT
|
||||
if pad:
|
||||
flat = F.pad(flat, (0, 0, 0, pad))
|
||||
activation, activation_scale = quantize_fp8_rowwise(flat)
|
||||
if self.weight.dtype == torch.float8_e4m3fnuz:
|
||||
from sglang.srt.layers.quantization.fp8_utils import (
|
||||
normalize_e4m3fn_to_e4m3fnuz,
|
||||
)
|
||||
if can_use_fp8_rowwise(flat):
|
||||
activation, activation_scale = fp8_rowwise(flat, self.ROW_ALIGNMENT)
|
||||
else:
|
||||
if pad:
|
||||
flat = F.pad(flat, (0, 0, 0, pad))
|
||||
activation, activation_scale = quantize_fp8_rowwise(flat)
|
||||
# ROCm-only: can_use_fp8_rowwise() requires torch.version.hip is None,
|
||||
# so the fnuz weights only ever reach this branch.
|
||||
if self.weight.dtype == torch.float8_e4m3fnuz:
|
||||
from sglang.srt.layers.quantization.fp8_utils import (
|
||||
normalize_e4m3fn_to_e4m3fnuz,
|
||||
)
|
||||
|
||||
activation, activation_scale, _ = normalize_e4m3fn_to_e4m3fnuz(
|
||||
activation, activation_scale
|
||||
)
|
||||
activation, activation_scale, _ = normalize_e4m3fn_to_e4m3fnuz(
|
||||
activation, activation_scale
|
||||
)
|
||||
out = torch._scaled_mm(
|
||||
activation,
|
||||
self.weight.T,
|
||||
|
||||
Reference in New Issue
Block a user