From 698fa949ecd3f5662a87edcb4aefcb44884a2776 Mon Sep 17 00:00:00 2001 From: trebladev <54441939+trebladev@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:17:56 +0800 Subject: [PATCH] [Perf] diffusion: fuse FLUX3 SwiGLU with rowwise FP8 quantization (#42758) --- python/sglang/kernels/ops/diffusion/README.md | 16 ++ .../sglang/kernels/ops/diffusion/__init__.py | 7 + .../quantization/swiglu_fp8_rowwise_triton.py | 163 ++++++++++++++++++ .../runtime/models/dits/flux3.py | 38 +++- .../diffusion/bench_swiglu_fp8_rowwise.py | 130 ++++++++++++++ .../kernels/ops/diffusion/test_fp8_rowwise.py | 137 ++++++++++++++- .../multimodal_gen/test_flux3_swiglu_fp8.py | 129 ++++++++++++++ 7 files changed, 617 insertions(+), 3 deletions(-) create mode 100644 python/sglang/kernels/ops/diffusion/quantization/swiglu_fp8_rowwise_triton.py create mode 100644 test/manual/kernels/diffusion/bench_swiglu_fp8_rowwise.py create mode 100644 test/registered/unit/multimodal_gen/test_flux3_swiglu_fp8.py diff --git a/python/sglang/kernels/ops/diffusion/README.md b/python/sglang/kernels/ops/diffusion/README.md index cc6d5d2f50..542b12286c 100644 --- a/python/sglang/kernels/ops/diffusion/README.md +++ b/python/sglang/kernels/ops/diffusion/README.md @@ -205,6 +205,22 @@ tensor copy at each residual site. | `vdn_temporal_conv_act`, `vdn_silu_l2norm`, `vdn_linear_epilogue` | Triton | one rounding at the store, within one bf16 ulp of the eager chain; the model's own inference kernels, mounted unconditionally by the VDN-H3 branch | | `vdn_delta_factors` | JIT CUDA | `(alpha * inv(I + A), B @ inv(I + A))` in one launch; same fp32 accuracy class as the cholesky + solve_triangular chain (cond-dominated); head_dim 128 | +### Rowwise E4M3 producers (SM89+) + +| Entry point | Backend | Contract | +|---|---|---| +| `fp8_rowwise` | Triton | Dynamic per-row FP32 scales, E4M3 payload and zero-row padding; matches the eager quantizer | +| `fused_packed_swiglu_fp8_rowwise` | Triton | Packed/row-strided BF16 `[B,T,2*K]` to padded 2D E4M3 + FP32 row scales; preserves both BF16 rounds of `F.silu(gate) * up`; mounted in FLUX3 FP8r MLP output projections | + +The SwiGLU producer avoids materializing a BF16 activation. Its measured SM89 +`K=9216` path uses nine 1024-element tiles, native BF16 rounding, and paired +FP32-to-E4M3 conversions, retaining one row reduction and zero-row padding. +Other widths/devices retain the full-row implementation. Finite inputs, +`K <= 16384`, and flattenable B/T row strides are required. Padding rows have +zero payload and scale `1e-12`. The local CUDA-graph benchmark is +`test/manual/kernels/diffusion/bench_swiglu_fp8_rowwise.py`; it compares the +existing two kernels and the fusion, both alone and with the same FP8 GEMM. + ### MXFP8 producers (online `mxfp8`, cuBLASLt block-scaled GEMM on SM100) Defined in `sglang.kernels.ops.quantization.mxfp8_swizzled_triton` (`silu_mul_mxfp8` in diff --git a/python/sglang/kernels/ops/diffusion/__init__.py b/python/sglang/kernels/ops/diffusion/__init__.py index 86db6b3142..5559e0bedb 100644 --- a/python/sglang/kernels/ops/diffusion/__init__.py +++ b/python/sglang/kernels/ops/diffusion/__init__.py @@ -45,6 +45,12 @@ _CUDA_SM10X = frozenset({CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10, # inventory: callers name the one they want via ``select_kernel``. # --------------------------------------------------------------------------- _SPECS: tuple[tuple[str, KernelBackend, str, frozenset], ...] = ( + ( + "diffusion.fused_packed_swiglu_fp8_rowwise", + KernelBackend.TRITON, + "quantization.swiglu_fp8_rowwise_triton:fused_packed_swiglu_fp8_rowwise", + frozenset({CapabilityRequirement.cuda(min_sm=(8, 9))}), + ), ( "diffusion.residual_gate_fp32", KernelBackend.TRITON, @@ -618,6 +624,7 @@ for _op, _backend, _target, _caps in _SPECS: # then symbol; a new public kernel belongs here and nowhere else. # --------------------------------------------------------------------------- _EXPORTS: dict[str, str] = { + "fused_packed_swiglu_fp8_rowwise": "quantization.swiglu_fp8_rowwise_triton", "apply_matrix_rope": "rope.matrix_rope", "residual_gate_fp32": "modulate.residual_gate_fp32", "can_use_fp8_rowwise": "quantization.fp8_rowwise_triton", diff --git a/python/sglang/kernels/ops/diffusion/quantization/swiglu_fp8_rowwise_triton.py b/python/sglang/kernels/ops/diffusion/quantization/swiglu_fp8_rowwise_triton.py new file mode 100644 index 0000000000..0e8ae91858 --- /dev/null +++ b/python/sglang/kernels/ops/diffusion/quantization/swiglu_fp8_rowwise_triton.py @@ -0,0 +1,163 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Packed BF16 SwiGLU directly to row-scaled E4M3, without a BF16 output.""" + +import torch +import triton +import triton.language as tl + +from sglang.kernels.numerics import round_bf16_to_fp32 + + +@triton.jit +def _swiglu_fp8_rowwise_kernel( + X, + Q, + S, + M: tl.constexpr, + K: tl.constexpr, + STRIDE: tl.constexpr, + BLOCK: tl.constexpr, +): + row = tl.program_id(0).to(tl.int64) + col = tl.arange(0, BLOCK) + mask = (row < M) & (col < K) + gate = tl.load(X + row * STRIDE + col, mask, other=0).to(tl.float32) + up = tl.load(X + row * STRIDE + K + col, mask, other=0).to(tl.float32) + # Reproduce both BF16 rounding boundaries of F.silu(gate) * up. + silu = round_bf16_to_fp32(gate * tl.sigmoid(gate)) + value = round_bf16_to_fp32(silu * up) + scale = tl.maximum(tl.max(tl.abs(value), 0) * (1.0 / 448.0), 1.0e-12) + normalized = tl.div_rn(value, scale) + normalized = tl.minimum(tl.maximum(normalized, -448.0), 448.0) + bits = tl.inline_asm_elementwise( + "{ .reg .b16 packed; cvt.rn.satfinite.e4m3x2.f32 packed, $1, $1; cvt.u32.u16 $0, packed; }", + constraints="=r,f", + args=[normalized], + dtype=tl.int32, + is_pure=True, + pack=1, + ) + quantized = bits.to(tl.uint8).to(tl.float8e4nv, bitcast=True) + tl.store(Q + row * K + col, quantized, col < K) + tl.store(S + row, scale) + + +@triton.jit +def _swiglu_fp8_rowwise_tiled_kernel( + X, + Q, + S, + M: tl.constexpr, + K: tl.constexpr, + STRIDE: tl.constexpr, + TILE: tl.constexpr, +): + row = tl.program_id(0).to(tl.int64) + columns = tl.arange(0, TILE) + values = () + maxima = tl.full((TILE,), 0, tl.float32) + # Keep each tile's BF16-rounded activation in registers, but reduce the + # elementwise maxima first: only one cross-warp reduction per row. + for tile in tl.static_range(triton.cdiv(K, TILE)): + col = columns + tile * TILE + mask = (row < M) & (col < K) + gate = tl.load(X + row * STRIDE + col, mask, other=0).to(tl.float32) + up = tl.load(X + row * STRIDE + K + col, mask, other=0).to(tl.float32) + silu = (gate * tl.sigmoid(gate)).to(tl.bfloat16).to(tl.float32) + value = (silu * up).to(tl.bfloat16).to(tl.float32) + values += (value,) + maxima = tl.maximum(maxima, tl.abs(value)) + scale = tl.maximum(tl.max(maxima, 0) * (1.0 / 448.0), 1.0e-12) + for tile in tl.static_range(triton.cdiv(K, TILE)): + col = columns + tile * TILE + normalized = tl.div_rn(values[tile], scale) + normalized = tl.minimum(tl.maximum(normalized, -448.0), 448.0) + # Convert two distinct FP32 elements per instruction; PTX places the + # second source in the low byte. Keep the direct FP32->E4M3 rounding. + bits = tl.inline_asm_elementwise( + "{ .reg .b16 packed; cvt.rn.satfinite.e4m3x2.f32 packed, $3, $2; " + "cvt.u32.u16 $0, packed; shr.b32 $1, $0, 8; }", + constraints="=r,=r,f,f", + args=[normalized], + dtype=tl.int32, + is_pure=True, + pack=2, + ) + quantized = bits.to(tl.uint8).to(tl.float8e4nv, bitcast=True) + tl.store(Q + row * K + col, quantized, col < K) + tl.store(S + row, scale) + + +def fused_packed_swiglu_fp8_rowwise(x: torch.Tensor, row_alignment: int = 16): + """Quantize ``F.silu(gate) * up`` from packed BF16 ``[B, T, 2*K]``. + + Return contiguous E4M3 ``[ceil(B*T/alignment)*alignment, K]`` plus + FP32 row scales. Accept row-strided slices of a packed QKV/MLP projection + when the B/T dimensions can be flattened without a copy. Finite inputs + reproduce both intermediate BF16 rounds and the existing rowwise FP8 + contract. Padding rows have zero payload and scale 1e-12. NVIDIA SM89+. + Used by the FLUX3 FP8r MLP output projection. + """ + if not ( + x.is_cuda + and torch.version.hip is None + and x.dtype == torch.bfloat16 + and x.ndim == 3 + and x.stride(-1) == 1 + and x.shape[-1] % 2 == 0 + and 0 < x.shape[-1] // 2 <= 16384 + and x.stride(1) >= x.shape[-1] + and (x.shape[0] <= 1 or x.stride(0) == x.shape[1] * x.stride(1)) + and torch.cuda.get_device_capability(x.device) >= (8, 9) + ): + raise ValueError( + "requires row-strided packed BF16 [B,T,2*K], K in [1,16384], on SM89+" + ) + if ( + not isinstance(row_alignment, int) + or isinstance(row_alignment, bool) + or row_alignment <= 0 + ): + raise ValueError("row_alignment must be a positive integer") + rows = x.shape[0] * x.shape[1] + hidden = x.shape[-1] // 2 + padded_rows = triton.cdiv(rows, row_alignment) * row_alignment + q = torch.empty((padded_rows, hidden), device=x.device, dtype=torch.float8_e4m3fn) + scales = torch.empty((padded_rows,), device=x.device, dtype=torch.float32) + if padded_rows: + num_warps = 4 if hidden <= 4096 else 8 + # SM89 measurements: extra warps help underfilled grids; eight win + # on the model's large 3072/9216-wide rows without register spills. + if torch.cuda.get_device_capability(x.device) == (8, 9) and hidden in ( + 3072, + 9216, + ): + num_warps = 16 if padded_rows <= 64 else 8 + with torch.cuda.device(x.device): + if torch.cuda.get_device_capability(x.device) == (8, 9) and hidden == 9216: + # Nine exact-width tiles instead of a 16384-wide row; measured + # on SM89, with unchanged BF16 and E4M3 rounding boundaries. + _swiglu_fp8_rowwise_tiled_kernel[(padded_rows,)]( + x, + q, + scales, + rows, + hidden, + x.stride(1), + 1024, + num_warps=16, + enable_fp_fusion=False, + ) + return q, scales + _swiglu_fp8_rowwise_kernel[(padded_rows,)]( + x, + q, + scales, + rows, + hidden, + x.stride(1), + triton.next_power_of_2(hidden), + num_warps=num_warps, + enable_fp_fusion=False, + ) + return q, scales diff --git a/python/sglang/multimodal_gen/runtime/models/dits/flux3.py b/python/sglang/multimodal_gen/runtime/models/dits/flux3.py index 3d5af0a02b..4f15fa3a7a 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/flux3.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/flux3.py @@ -50,6 +50,7 @@ from sglang.kernels.ops.diffusion import ( fp8_rowwise, fused_inplace_qknorm_rope, fused_packed_silu_mul_bitexact, + fused_packed_swiglu_fp8_rowwise, residual_gate_add, ) from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz @@ -273,7 +274,7 @@ class Flux3Fp8RowwiseLinear(nn.Module): parallel linears so the module can replace either kind. """ - # _scaled_mm needs M padded to a multiple of 16. + # Preserve the checkpoint inference path's row padding convention. ROW_ALIGNMENT = 16 def __init__( @@ -325,6 +326,35 @@ class Flux3Fp8RowwiseLinear(nn.Module): activation, activation_scale, _ = normalize_e4m3fn_to_e4m3fnuz( activation, activation_scale ) + return self._forward_quantized(activation, activation_scale, leading) + + def forward_swiglu(self, packed: torch.Tensor): + """Fuse packed BF16 SwiGLU with this layer's activation quantization.""" + if not ( + packed.is_cuda + and torch.version.hip is None + and packed.dtype == torch.bfloat16 + and packed.ndim == 3 + and packed.shape[-1] == 2 * self.in_features + and 0 < self.in_features <= 16384 + and packed.stride(-1) == 1 + and packed.stride(1) >= packed.shape[-1] + and ( + packed.shape[0] <= 1 + or packed.stride(0) == packed.shape[1] * packed.stride(1) + ) + and self.weight.dtype == torch.float8_e4m3fn + and torch.cuda.get_device_capability(packed.device) >= (8, 9) + and not torch.compiler.is_compiling() + ): + return self(_swiglu(packed)) + activation, activation_scale = fused_packed_swiglu_fp8_rowwise( + packed, self.ROW_ALIGNMENT + ) + return self._forward_quantized(activation, activation_scale, packed.shape[:-1]) + + def _forward_quantized(self, activation, activation_scale, leading): + rows = math.prod(leading) out = torch._scaled_mm( activation, self.weight.T, @@ -528,7 +558,11 @@ class Flux3Block(nn.Module): q, k, v = q.contiguous(), k.contiguous(), v.contiguous() attended = self.attn(q, k, v, attn_mask_meta=sp.attn_mask_meta) attended = attended.reshape(batch, length, self.local_hidden) - out = self.attn_out(attended)[0] + self.mlp_out(_swiglu(mlp))[0] + if isinstance(self.mlp_out, Flux3Fp8RowwiseLinear): + mlp_out = self.mlp_out.forward_swiglu(mlp)[0] + else: + mlp_out = self.mlp_out(_swiglu(mlp))[0] + out = self.attn_out(attended)[0] + mlp_out if self.tp_size > 1: # One reduce of the summed partials, not one per branch (the order # the FLUX 3 reference TP uses). diff --git a/test/manual/kernels/diffusion/bench_swiglu_fp8_rowwise.py b/test/manual/kernels/diffusion/bench_swiglu_fp8_rowwise.py new file mode 100644 index 0000000000..1ce58c72e8 --- /dev/null +++ b/test/manual/kernels/diffusion/bench_swiglu_fp8_rowwise.py @@ -0,0 +1,130 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Compare packed SwiGLU + FP8 quantization with its standalone fusion. + +Run on an idle GPU, e.g. CUDA_VISIBLE_DEVICES=2 python this_file.py +--output-json logs/swiglu-fp8.json. CUDA graph timing excludes Python overhead. +Each case measures both the producer alone and producer + unchanged FP8 GEMM. +""" + +import argparse +import json +import statistics +from pathlib import Path + +import torch +import triton + +from sglang.kernels.ops.diffusion import ( + fp8_rowwise, + fused_packed_silu_mul_bitexact, + fused_packed_swiglu_fp8_rowwise, +) + + +def benchmark(rows, hidden, rounds, rep_ms): + torch.manual_seed(42) + storage = torch.randn((1, rows, 3 * hidden), device="cuda", dtype=torch.bfloat16) + packed = storage[..., hidden:] + weights, weight_scales = fp8_rowwise( + torch.randn((3072, hidden), device="cuda", dtype=torch.bfloat16), 16 + ) + + def separate(): + values = fused_packed_silu_mul_bitexact(packed) + return fp8_rowwise(values.reshape(rows, hidden), 16) + + def fused(): + return fused_packed_swiglu_fp8_rowwise(packed) + + def gemm(producer): + q, scales = producer() + return torch._scaled_mm( + q, + weights.T, + scales[:, None], + weight_scales[None, :], + out_dtype=torch.bfloat16, + use_fast_accum=True, + ) + + q, scales = separate() + fq, fs = fused() + assert torch.equal(q.view(torch.uint8), fq.view(torch.uint8)) + assert torch.equal(scales, fs) + assert torch.equal(gemm(separate), gemm(fused)) + result = { + "rows": rows, + "hidden": hidden, + "input_row_stride": packed.stride(1), + "padded_rows": q.shape[0], + "bit_exact": True, + } + for name, a, b in [ + ("producer", separate, fused), + ("with_gemm", lambda: gemm(separate), lambda: gemm(fused)), + ]: + for _ in range(10): + a() + b() + torch.cuda.synchronize() + samples = {"separate": [], "fused": []} + # Alternate order to reduce thermal/clock bias. + for i in range(rounds): + order = [("separate", a), ("fused", b)] + if i % 2: + order.reverse() + for key, fn in order: + samples[key].append( + triton.testing.do_bench_cudagraph(fn, rep=rep_ms) * 1000 + ) + medians = {k: statistics.median(v) for k, v in samples.items()} + result[name] = { + "samples_us": samples, + "median_us": medians, + "speedup": medians["separate"] / medians["fused"], + "reduction_pct": 100 * (1 - medians["fused"] / medians["separate"]), + } + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-json", type=Path) + parser.add_argument("--rounds", type=int, default=5) + parser.add_argument("--rep-ms", type=int, default=100) + args = parser.parse_args() + if args.rounds <= 0 or args.rep_ms <= 0: + parser.error("rounds and rep-ms must be positive") + results = [] + for rows, hidden in [ + (1, 9216), + (32, 9216), + (340, 9216), + (2720, 9216), + (3173, 9216), + (3173, 3072), + ]: + r = benchmark(rows, hidden, args.rounds, args.rep_ms) + results.append(r) + print(json.dumps(r), flush=True) + if args.output_json: + args.output_json.parent.mkdir(parents=True, exist_ok=True) + args.output_json.write_text( + json.dumps( + { + "gpu": torch.cuda.get_device_name(), + "torch": torch.__version__, + "triton": triton.__version__, + "mode": "CUDA graph, no profiler", + "rounds": args.rounds, + "rep_ms": args.rep_ms, + "results": results, + }, + indent=2, + ) + + "\n" + ) + + +if __name__ == "__main__": + main() diff --git a/test/registered/kernels/ops/diffusion/test_fp8_rowwise.py b/test/registered/kernels/ops/diffusion/test_fp8_rowwise.py index 896f1b3e04..75f3038c5d 100644 --- a/test/registered/kernels/ops/diffusion/test_fp8_rowwise.py +++ b/test/registered/kernels/ops/diffusion/test_fp8_rowwise.py @@ -5,8 +5,13 @@ import pytest import torch import torch.nn.functional as F -from sglang.kernels.ops.diffusion import can_use_fp8_rowwise, fp8_rowwise +from sglang.kernels.ops.diffusion import ( + can_use_fp8_rowwise, + fp8_rowwise, + fused_packed_swiglu_fp8_rowwise, +) from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="4-gpu-b200") @@ -101,5 +106,135 @@ def test_fp32_rounding_boundaries(): assert torch.equal(q.view(torch.uint8), rq.view(torch.uint8)) +class TestPackedSwiGLUFp8(CustomTestCase): + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available() or not can_use_fp8_rowwise( + torch.empty((1, 1), device="cuda", dtype=torch.bfloat16) + ): + pytest.skip("NVIDIA SM89+ required") + + def assert_matches_reference(self, packed, alignment=16): + gate, up = packed.chunk(2, dim=-1) + values = (F.silu(gate) * up).reshape(-1, gate.shape[-1]) + rq, rs = reference(values, alignment) + q, scales = fused_packed_swiglu_fp8_rowwise(packed, alignment) + self.assertTrue(torch.equal(scales, rs)) + self.assertTrue(torch.equal(q.view(torch.uint8), rq.view(torch.uint8))) + if gate.shape[-1] == 9216 and values.shape[0]: + from sglang.kernels.ops.diffusion.quantization.swiglu_fp8_rowwise_triton import ( + _swiglu_fp8_rowwise_tiled_kernel, + ) + + tq, ts = torch.empty_like(q), torch.empty_like(scales) + _swiglu_fp8_rowwise_tiled_kernel[(q.shape[0],)]( + packed, + tq, + ts, + values.shape[0], + 9216, + packed.stride(1), + 1024, + num_warps=16, + enable_fp_fusion=False, + ) + self.assertTrue(torch.equal(ts, rs)) + self.assertTrue(torch.equal(tq.view(torch.uint8), rq.view(torch.uint8))) + return q, scales + + def test_strided_rounding_and_padding(self): + """Guard row-strided projection slices, BF16 rounds and zero padding.""" + torch.manual_seed(42) + for batch, rows, hidden in [ + (1, 0, 33), + (2, 17, 257), + (1, 32, 9216), + (1, 340, 9216), + (2, 170, 9216), + (1, 3173, 9216), + ]: + with self.subTest(batch=batch, rows=rows, hidden=hidden): + storage = torch.randn( + (batch, rows, 3 * hidden), device="cuda", dtype=torch.bfloat16 + ) + packed = storage[..., hidden:] + if rows: + packed[:, 0].zero_() + if rows > 1: + packed[:, 1] *= 1e-13 + self.assert_matches_reference(packed) + + def test_bf16_gate_values(self): + """Keep SiLU/multiply rounding across every finite BF16 gate encoding.""" + bits = torch.arange(65536, device="cuda", dtype=torch.int32) + gate = bits.to(torch.int16).view(torch.bfloat16) + finite_gate = torch.where(torch.isfinite(gate), gate, 0) + gate = torch.zeros(8 * 9216, device="cuda", dtype=torch.bfloat16) + gate[:65536] = finite_gate + gate = gate.reshape(1, 8, 9216) + up = ( + torch.tensor([0.125, -0.25, 0.5, -1.0], device="cuda", dtype=torch.bfloat16) + .repeat(18432) + .reshape_as(gate) + ) + self.assert_matches_reference(torch.cat((gate, up), dim=-1), 1) + + def test_subnormal_gate_rounding(self): + """Large up values expose lost BF16 subnormals before quantization.""" + gate = torch.tensor( + [1e-38, 1e-39, 1e-40, 1e-41, 1e-42], device="cuda", dtype=torch.bfloat16 + ) + gate = gate[:, None].expand(5, 9216).clone()[None] + self.assert_matches_reference( + torch.cat((gate, torch.full_like(gate, 1e30)), -1) + ) + + def test_graph_replay_and_gemm(self): + """Fresh graph inputs preserve quantized operands and downstream GEMM.""" + packed = torch.randn((1, 17, 27648), device="cuda", dtype=torch.bfloat16)[ + ..., 9216: + ] + w, ws = reference(torch.randn((32, 9216), device="cuda", dtype=torch.bfloat16)) + + def chain(): + q, scales = fused_packed_swiglu_fp8_rowwise(packed) + return ( + q, + scales, + torch._scaled_mm( + q, + w.T, + scales[:, None], + ws[None, :], + out_dtype=torch.bfloat16, + use_fast_accum=True, + ), + ) + + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + for _ in range(3): + chain() + torch.cuda.current_stream().wait_stream(stream) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + q, scales, out = chain() + packed.normal_() + graph.replay() + rq, rs = self.assert_matches_reference(packed) + self.assertTrue(torch.equal(q.view(torch.uint8), rq.view(torch.uint8))) + self.assertTrue(torch.equal(scales, rs)) + expected = torch._scaled_mm( + rq, + w.T, + rs[:, None], + ws[None, :], + out_dtype=torch.bfloat16, + use_fast_accum=True, + ) + self.assertTrue(torch.equal(out, expected)) + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/unit/multimodal_gen/test_flux3_swiglu_fp8.py b/test/registered/unit/multimodal_gen/test_flux3_swiglu_fp8.py new file mode 100644 index 0000000000..d18d53fe49 --- /dev/null +++ b/test/registered/unit/multimodal_gen/test_flux3_swiglu_fp8.py @@ -0,0 +1,129 @@ +# SPDX-License-Identifier: Apache-2.0 +"""FLUX3 packed MLP inputs must preserve SwiGLU rounding through FP8 GEMM.""" + +import unittest +from itertools import product +from unittest.mock import patch + +import torch +import torch.nn.functional as F + +from sglang.kernels.ops.diffusion import BitExactFusionGate +from sglang.multimodal_gen.runtime.models.dits.flux3 import Flux3Fp8RowwiseLinear +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="4-gpu-b200") + + +class TestFlux3SwiGLUFp8(CustomTestCase): + def setUp(self): + super().setUp() + # Unsupported fallback layouts can disable the existing SwiGLU gate; + # isolate that real state machine so other model tests are unaffected. + swiglu_patch = patch( + "sglang.multimodal_gen.runtime.models.dits.flux3._SWIGLU", + BitExactFusionGate("FLUX3 test SwiGLU", per_signature=True), + ) + self.addCleanup(swiglu_patch.stop) + swiglu_patch.start() + + @staticmethod + def _output(result, tuple_output): + if tuple_output: + output, bias = result + assert bias is None + return output + return result + + def _reference(self, layer, packed, tuple_output): + gate, value = packed.chunk(2, dim=-1) + return self._output(layer(F.silu(gate) * value), tuple_output) + + def _assert_matches(self, layer, packed, tuple_output): + actual = self._output(layer.forward_swiglu(packed), tuple_output) + self.assertTrue( + torch.equal(actual, self._reference(layer, packed, tuple_output)) + ) + return actual + + def _cuda_inputs(self, hidden, batch=2, length=17): + if not torch.cuda.is_available() or torch.cuda.get_device_capability() < (8, 9): + self.skipTest("requires NVIDIA SM89+") + generator = torch.Generator(device="cuda").manual_seed(42) + weight = torch.randn(64, hidden, device="cuda", generator=generator).to( + torch.float8_e4m3fn + ) + scales = torch.rand(64, device="cuda", generator=generator) + storage = torch.randn( + batch, + length, + 3 * hidden, + device="cuda", + dtype=torch.bfloat16, + generator=generator, + ) + return weight, scales, storage[..., hidden:] + + def test_cpu_fallback(self): + """Unsupported devices must retain the original quantized linear result.""" + generator = torch.Generator().manual_seed(42) + packed = torch.randn(2, 17, 64, generator=generator).bfloat16() + weight = torch.randn(16, 32, generator=generator).to(torch.float8_e4m3fn) + scales = torch.rand(16, generator=generator) + + # CPU lacks FP8 scaled_mm: emulate only the GEMM dependency, retaining + # the real SwiGLU, padding, quantization and output reshaping paths. + def scaled_mm(a, b, scale_a, scale_b, *, out_dtype, use_fast_accum): + self.assertTrue(use_fast_accum) + return ((a.float() * scale_a) @ (b.float() * scale_b)).to(out_dtype) + + with patch("torch._scaled_mm", scaled_mm), torch.inference_mode(): + for tuple_output in (False, True): + with self.subTest(tuple_output=tuple_output): + layer = Flux3Fp8RowwiseLinear(weight, scales, tuple_output) + actual = self._assert_matches(layer, packed, tuple_output) + self.assertEqual(actual.shape, (2, 17, 16)) + + @torch.inference_mode() + def test_strided_inputs(self): + """Strided rows and row padding preserve the unfused result.""" + for hidden, (batch, length), tuple_output in product( + (3072, 9216), ((1, 1), (2, 17), (1, 340)), (False, True) + ): + with self.subTest( + hidden=hidden, batch=batch, length=length, tuple_output=tuple_output + ): + weight, scales, packed = self._cuda_inputs(hidden, batch, length) + layer = Flux3Fp8RowwiseLinear(weight, scales, tuple_output) + self._assert_matches(layer, packed, tuple_output) + + @torch.inference_mode() + def test_graph_replay(self): + """Graph replay reads updated strided inputs for both output formats.""" + weight, scales, packed = self._cuda_inputs(3072) + for tuple_output in (False, True): + with self.subTest(tuple_output=tuple_output): + layer = Flux3Fp8RowwiseLinear(weight, scales, tuple_output) + self._assert_matches(layer, packed, tuple_output) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = self._output(layer.forward_swiglu(packed), tuple_output) + packed.neg_() + graph.replay() + self.assertTrue( + torch.equal(captured, self._reference(layer, packed, tuple_output)) + ) + + @torch.inference_mode() + def test_transposed_fallback(self): + """Non-flattenable batch/sequence strides preserve output ordering.""" + for hidden in (3072, 9216): + with self.subTest(hidden=hidden): + weight, scales, packed = self._cuda_inputs(hidden, batch=17, length=2) + layer = Flux3Fp8RowwiseLinear(weight, scales, False) + self._assert_matches(layer, packed.transpose(0, 1), False) + + +if __name__ == "__main__": + unittest.main()