From 295a53c90ea4adb2588084dc0e3004cd3145e414 Mon Sep 17 00:00:00 2001 From: trebladev <54441939+trebladev@users.noreply.github.com> Date: Sun, 4 Oct 2026 09:01:17 +0800 Subject: [PATCH] [diffusion] optimization: fuse Flux3 rowwise FP8 quantization with triton (#41671) --- .../sglang/kernels/ops/diffusion/__init__.py | 9 ++ .../quantization/fp8_rowwise_triton.py | 78 +++++++++++++ .../runtime/models/dits/flux3.py | 27 +++-- .../benchmark/diffusion/bench_fp8_rowwise.py | 30 +++++ .../kernels/ops/diffusion/test_fp8_rowwise.py | 105 ++++++++++++++++++ 5 files changed, 239 insertions(+), 10 deletions(-) create mode 100644 python/sglang/kernels/ops/diffusion/quantization/fp8_rowwise_triton.py create mode 100644 test/registered/kernels/benchmark/diffusion/bench_fp8_rowwise.py create mode 100644 test/registered/kernels/ops/diffusion/test_fp8_rowwise.py diff --git a/python/sglang/kernels/ops/diffusion/__init__.py b/python/sglang/kernels/ops/diffusion/__init__.py index 9b918fabf5..70faab6b86 100644 --- a/python/sglang/kernels/ops/diffusion/__init__.py +++ b/python/sglang/kernels/ops/diffusion/__init__.py @@ -49,6 +49,13 @@ _HIP = frozenset({CapabilityRequirement.HIP}) # inventory: callers name the one they want via ``select_kernel``. # --------------------------------------------------------------------------- _SPECS: tuple[tuple[str, KernelBackend, str, frozenset, str], ...] = ( + ( + "diffusion.fp8_rowwise", + KernelBackend.TRITON, + "quantization.fp8_rowwise_triton:fp8_rowwise", + frozenset({CapabilityRequirement.cuda(min_sm=(8, 9))}), + "FLUX.3 dynamic rowwise E4M3 quantization and zero row padding.", + ), ( "diffusion.gelu_tanh_cat", KernelBackend.JIT, @@ -555,6 +562,8 @@ for _op, _backend, _target, _caps, _description in _SPECS: # then symbol; a new public kernel belongs here and nowhere else. # --------------------------------------------------------------------------- _EXPORTS: dict[str, str] = { + "can_use_fp8_rowwise": "quantization.fp8_rowwise_triton", + "fp8_rowwise": "quantization.fp8_rowwise_triton", "can_use_fused_gelu_tanh_cat": "activation.gelu_tanh_cat_jit", "fused_gelu_tanh_cat": "activation.gelu_tanh_cat_jit", "load_extension_with_recovery": "sglang.srt.utils.cpp_extension_loader", diff --git a/python/sglang/kernels/ops/diffusion/quantization/fp8_rowwise_triton.py b/python/sglang/kernels/ops/diffusion/quantization/fp8_rowwise_triton.py new file mode 100644 index 0000000000..f382f355d8 --- /dev/null +++ b/python/sglang/kernels/ops/diffusion/quantization/fp8_rowwise_triton.py @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: Apache-2.0 +"""FLUX.3 dynamic rowwise E4M3 quantization with optional zero row padding.""" + +import torch +import triton +import triton.language as tl + + +def can_use_fp8_rowwise(x: torch.Tensor) -> bool: + return ( + x.is_cuda + and torch.version.hip is None + and x.dtype in (torch.bfloat16, torch.float16, torch.float32) + and x.ndim == 2 + and x.is_contiguous() + and 0 < x.shape[1] <= 16384 + and torch.cuda.get_device_capability(x.device) >= (8, 9) + ) + + +@triton.jit +def _fp8_rowwise_kernel(X, Q, S, M: tl.constexpr, K: tl.constexpr, BLOCK: tl.constexpr): + row = tl.program_id(0) + col = tl.arange(0, BLOCK) + x = tl.load(X + row.to(tl.int64) * K + col, (row < M) & (col < K), other=0).to( + tl.float32 + ) + amax = tl.max(tl.abs(x), 0) + # aten scalar division multiplies by the rounded FP32 reciprocal. + scale = tl.maximum(amax * (1.0 / 448.0), 1.0e-12) + # Tensor/tensor division must retain IEEE rounding at FP8 boundaries. + q = tl.div_rn(x, scale) + q = tl.minimum(tl.maximum(q, -448.0), 448.0) + # On SM89 Triton's default cast can round through FP16. Convert directly + # from FP32 to avoid double rounding at E4M3 halfway points. + bits = tl.inline_asm_elementwise( + "{ .reg .b16 packed; cvt.rn.satfinite.e4m3x2.f32 packed, $1, $1; cvt.u32.u16 $0, packed; }", + constraints="=r,f", + args=[q], + dtype=tl.int32, + is_pure=True, + pack=1, + ) + quantized = bits.to(tl.uint8).to(tl.float8e4nv, bitcast=True) + tl.store(Q + row.to(tl.int64) * K + col, quantized, col < K) + tl.store(S + row, scale) + + +def fp8_rowwise(x: torch.Tensor, row_alignment: int = 1): + """Return E4M3 ``[ceil(M/alignment)*alignment, K]`` and FP32 row scales. + + Reproduce the FLUX.3 eager chain, including scale >= 1e-12. Padded + rows contain zeros with scale 1e-12. No intermediate FP32 tensor or + padded BF16 copy is materialized. + """ + if not can_use_fp8_rowwise(x): + raise ValueError( + "fp8_rowwise requires contiguous 2D FP32/FP16/BF16 on SM89+, K in [1, 16384]" + ) + if not isinstance(row_alignment, int) or row_alignment <= 0: + raise ValueError("row_alignment must be a positive integer") + m, k = x.shape + padded_m = triton.cdiv(m, row_alignment) * row_alignment + q = torch.empty((padded_m, k), dtype=torch.float8_e4m3fn, device=x.device) + scale = torch.empty((padded_m,), dtype=torch.float32, device=x.device) + if padded_m: + with torch.cuda.device(x.device): + _fp8_rowwise_kernel[(padded_m,)]( + x, + q, + scale, + m, + k, + triton.next_power_of_2(k), + num_warps=4 if k <= 4096 else 8, + enable_fp_fusion=False, + ) + return q, scale diff --git a/python/sglang/multimodal_gen/runtime/models/dits/flux3.py b/python/sglang/multimodal_gen/runtime/models/dits/flux3.py index c19cc32c99..16a7bcd36b 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/flux3.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/flux3.py @@ -46,8 +46,10 @@ import torch.nn.functional as F from sglang.kernels.ops.diffusion import ( BitExactFusionGate, + can_use_fp8_rowwise, can_use_fused_inplace_qknorm_rope, can_use_fused_layernorm_modulate, + fp8_rowwise, fused_inplace_qknorm_rope, fused_layernorm_modulate_raw, fused_packed_silu_mul_bitexact, @@ -328,17 +330,22 @@ class Flux3Fp8RowwiseLinear(nn.Module): flat = x.reshape(-1, self.in_features).contiguous() rows = flat.shape[0] pad = -rows % self.ROW_ALIGNMENT - if pad: - flat = F.pad(flat, (0, 0, 0, pad)) - activation, activation_scale = quantize_fp8_rowwise(flat) - if self.weight.dtype == torch.float8_e4m3fnuz: - from sglang.srt.layers.quantization.fp8_utils import ( - normalize_e4m3fn_to_e4m3fnuz, - ) + if can_use_fp8_rowwise(flat): + activation, activation_scale = fp8_rowwise(flat, self.ROW_ALIGNMENT) + else: + if pad: + flat = F.pad(flat, (0, 0, 0, pad)) + activation, activation_scale = quantize_fp8_rowwise(flat) + # ROCm-only: can_use_fp8_rowwise() requires torch.version.hip is None, + # so the fnuz weights only ever reach this branch. + if self.weight.dtype == torch.float8_e4m3fnuz: + from sglang.srt.layers.quantization.fp8_utils import ( + normalize_e4m3fn_to_e4m3fnuz, + ) - activation, activation_scale, _ = normalize_e4m3fn_to_e4m3fnuz( - activation, activation_scale - ) + activation, activation_scale, _ = normalize_e4m3fn_to_e4m3fnuz( + activation, activation_scale + ) out = torch._scaled_mm( activation, self.weight.T, diff --git a/test/registered/kernels/benchmark/diffusion/bench_fp8_rowwise.py b/test/registered/kernels/benchmark/diffusion/bench_fp8_rowwise.py new file mode 100644 index 0000000000..4fb5653316 --- /dev/null +++ b/test/registered/kernels/benchmark/diffusion/bench_fp8_rowwise.py @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: Apache-2.0 +import torch +import torch.nn.functional as F + +from sglang.kernels.jit.benchmark import marker +from sglang.kernels.ops.diffusion import fp8_rowwise +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci( + est_time=10, stage="base-b-kernel-benchmark", runner_config="1-gpu-large" +) + + +@marker.parametrize("rows", [32, 340, 2720, 3173], [3173]) +@marker.parametrize("hidden", [3072, 9216], [3072]) +@marker.benchmark("impl", ["eager", "fused"], unit="us") +def benchmark(rows: int, hidden: int, impl: str): + x = torch.randn((rows, hidden), dtype=torch.bfloat16, device="cuda") + + def eager(): + values = F.pad(x, (0, 0, 0, -rows % 16)).float() + scale = (values.abs().amax(1) / 448.0).clamp(min=1e-12) + return (values / scale[:, None]).clamp(-448, 448).to(torch.float8_e4m3fn), scale + + fn = eager if impl == "eager" else lambda: fp8_rowwise(x, 16) + return marker.do_bench(fn, disable_log_bandwidth=True) + + +if __name__ == "__main__": + benchmark.run() diff --git a/test/registered/kernels/ops/diffusion/test_fp8_rowwise.py b/test/registered/kernels/ops/diffusion/test_fp8_rowwise.py new file mode 100644 index 0000000000..896f1b3e04 --- /dev/null +++ b/test/registered/kernels/ops/diffusion/test_fp8_rowwise.py @@ -0,0 +1,105 @@ +# SPDX-License-Identifier: Apache-2.0 +import sys + +import pytest +import torch +import torch.nn.functional as F + +from sglang.kernels.ops.diffusion import can_use_fp8_rowwise, fp8_rowwise +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="4-gpu-b200") + + +def reference(x, alignment=16): + x = F.pad(x, (0, 0, 0, -x.shape[0] % alignment)).float() + s = (x.abs().amax(1) / 448.0).clamp(min=1e-12) + return (x / s[:, None]).clamp(-448, 448).to(torch.float8_e4m3fn), s + + +@pytest.mark.parametrize( + "shape", + [ + (0, 256), + (1, 256), + (17, 33), + (32, 3072), + (340, 3072), + (2720, 3072), + (3173, 3072), + (3173, 9216), + ], +) +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16, torch.float32]) +def test_rowwise_exact(shape, dtype): + if not torch.cuda.is_available(): + pytest.skip("CUDA required") + torch.manual_seed(42) + x = torch.randn(shape, dtype=dtype, device="cuda") + if not can_use_fp8_rowwise(x): + pytest.skip("SM89+ required") + if shape[0]: + x[0].zero_() + if shape[0] > 3: + x[1] *= 1e-13 + x[2] *= 1e3 + x[3, ::2] = -0.0 + q, s = fp8_rowwise(x, 16) + rq, rs = reference(x) + assert torch.equal(s, rs) + assert torch.equal(q.view(torch.uint8), rq.view(torch.uint8)) + + +def test_linear_and_graph(): + if not torch.cuda.is_available(): + pytest.skip("CUDA required") + from sglang.multimodal_gen.runtime.models.dits.flux3 import Flux3Fp8RowwiseLinear + + x = torch.randn((1, 17, 256), device="cuda", dtype=torch.bfloat16) + if not can_use_fp8_rowwise(x.reshape(17, 256)): + pytest.skip("SM89+ required") + w, ws = reference(torch.randn((32, 256), device="cuda", dtype=torch.bfloat16)) + linear = Flux3Fp8RowwiseLinear(w, ws, tuple_output=False) + q, s = reference(x.reshape(17, 256)) + expected = torch._scaled_mm( + q, w.T, s[:, None], ws[None, :], out_dtype=torch.bfloat16, use_fast_accum=True + )[:17].reshape(1, 17, 32) + actual = linear(x) + assert torch.equal(actual, expected) + # Warm up on a side stream before capture, then change input on replay. + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + for _ in range(3): + linear(x) + torch.cuda.current_stream().wait_stream(stream) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = linear(x) + x.normal_() + graph.replay() + assert torch.equal(captured, linear(x)) + + +def test_predicate(): + assert not can_use_fp8_rowwise(torch.empty(4, 32)) + + +def test_fp32_rounding_boundaries(): + if not torch.cuda.is_available(): + pytest.skip("CUDA required") + # Near E4M3 halfway points: an intermediate FP16 cast changes the answer. + x = torch.tensor( + [[448.0, -448.0, 68.02094, -68.02094, 8.502618, -25.005346, 200.04277, -0.0]], + device="cuda", + ) + if not can_use_fp8_rowwise(x): + pytest.skip("SM89+ required") + q, s = fp8_rowwise(x, 16) + rq, rs = reference(x) + assert torch.equal(s, rs) + assert torch.equal(q.view(torch.uint8), rq.view(torch.uint8)) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"]))