From ce617d410bce1a9db48bbada8ee97a1286729e84 Mon Sep 17 00:00:00 2001 From: Tokha233 <61346912+Tokha233@users.noreply.github.com> Date: Sat, 3 Oct 2026 15:23:41 +0800 Subject: [PATCH] [diffusion] fix: fix native FP8 format handling for FLUX 3 row-wise linears (#42255) --- .../runtime/models/dits/flux3.py | 19 +++++ .../test/unit/test_flux3_action.py | 46 ++++++++++++ ...est_modelopt_fp8_layerwise_offload_load.py | 17 ++++- .../test/unit/test_transformer_quant.py | 75 +++++++++++-------- 4 files changed, 123 insertions(+), 34 deletions(-) diff --git a/python/sglang/multimodal_gen/runtime/models/dits/flux3.py b/python/sglang/multimodal_gen/runtime/models/dits/flux3.py index db08b85f0e..c19cc32c99 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/flux3.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/flux3.py @@ -54,6 +54,7 @@ from sglang.kernels.ops.diffusion import ( is_plain_layer_norm, residual_gate_add, ) +from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz from sglang.multimodal_gen.configs.models.dits.flux3 import ( Flux3ArchConfig, Flux3DiTConfig, @@ -304,6 +305,16 @@ class Flux3Fp8RowwiseLinear(nn.Module): raise ValueError( "expected an E4M3 weight with one fp32 scale per output row" ) + if is_fp8_fnuz(): + from sglang.srt.layers.quantization.fp8_utils import ( + normalize_e4m3fn_to_e4m3fnuz, + ) + + # MI300 scaled_mm requires FNUZ. Preserve the dequantized checkpoint + # values and do not mutate checkpoint tensors shared by the loader. + weight, weight_scale, _ = normalize_e4m3fn_to_e4m3fnuz( + weight.clone(), weight_scale + ) self.out_features, self.in_features = weight.shape self.tuple_output = tuple_output # Parameters (not buffers) so that layerwise offload streams them. @@ -320,6 +331,14 @@ class Flux3Fp8RowwiseLinear(nn.Module): if pad: flat = F.pad(flat, (0, 0, 0, pad)) activation, activation_scale = quantize_fp8_rowwise(flat) + if self.weight.dtype == torch.float8_e4m3fnuz: + from sglang.srt.layers.quantization.fp8_utils import ( + normalize_e4m3fn_to_e4m3fnuz, + ) + + activation, activation_scale, _ = normalize_e4m3fn_to_e4m3fnuz( + activation, activation_scale + ) out = torch._scaled_mm( activation, self.weight.T, diff --git a/python/sglang/multimodal_gen/test/unit/test_flux3_action.py b/python/sglang/multimodal_gen/test/unit/test_flux3_action.py index 397be88129..4a4b2e56c6 100644 --- a/python/sglang/multimodal_gen/test/unit/test_flux3_action.py +++ b/python/sglang/multimodal_gen/test/unit/test_flux3_action.py @@ -612,6 +612,52 @@ def test_dit_cached_streams_match_full_forward(): assert full["x_act_cond"].shape == (1, 1, 3) +@pytest.mark.parametrize("fnuz", [False, True]) +@pytest.mark.parametrize("tuple_output", [False, True]) +def test_fp8r_native_format_preserves_values(monkeypatch, fnuz, tuple_output): + from sglang.multimodal_gen.runtime.models.dits import flux3 + + monkeypatch.setattr(flux3, "is_fp8_fnuz", lambda: fnuz, raising=False) + # Include negative zero: reinterpreting it as FNUZ without normalization + # produces NaN. Keep a byte snapshot to detect mutation of the checkpoint. + weight = torch.tensor([[-0.0, 1.0, -2.0, 448.0], [0.0, -1.0, 2.0, -448.0]]).to( + torch.float8_e4m3fn + ) + weight_bytes = weight.view(torch.uint8).clone() + scale = torch.tensor([0.125, 0.25]) + x = torch.tensor([[[-0.0, 1.0, -2.0, 0.5], [0.0, 0.0, 0.0, 0.0]]]) + quantized_x, x_scale = flux3.quantize_fp8_rowwise(x.flatten(0, 1)) + expected_x = quantized_x.float() * x_scale[:, None] + expected_weight = weight.float() * scale[:, None] + expected = (expected_x @ expected_weight.T).bfloat16().reshape(1, 2, 2) + native_dtype = torch.float8_e4m3fnuz if fnuz else torch.float8_e4m3fn + calls = [] + + def scaled_mm(a, b, scale_a, scale_b, *, out_dtype, use_fast_accum): + calls.append(True) + assert a.dtype == b.dtype == native_dtype + assert a.shape == (16, 4) and b.shape == (4, 2) + assert out_dtype == torch.bfloat16 and use_fast_accum + dequant_a = a.float() * scale_a + dequant_b = b.float() * scale_b + torch.testing.assert_close(dequant_a[:2], expected_x, atol=0, rtol=0) + torch.testing.assert_close(dequant_a[2:], torch.zeros(14, 4), atol=0, rtol=0) + torch.testing.assert_close(dequant_b, expected_weight.T, atol=0, rtol=0) + assert torch.isfinite(dequant_a).all() and torch.isfinite(dequant_b).all() + return (dequant_a @ dequant_b).to(out_dtype) + + monkeypatch.setattr(torch, "_scaled_mm", scaled_mm) + layer = flux3.Flux3Fp8RowwiseLinear(weight, scale, tuple_output) + result = layer(x) + if tuple_output: + result, bias = result + assert bias is None + torch.testing.assert_close(result, expected, atol=0, rtol=0) + assert len(calls) == 1 + assert torch.equal(weight.view(torch.uint8), weight_bytes) + torch.testing.assert_close(scale, torch.tensor([0.125, 0.25]), atol=0, rtol=0) + + @pytest.mark.skipif( not torch.cuda.is_available() or torch.cuda.get_device_capability() < (8, 9), reason="needs FP8 scaled_mm", diff --git a/python/sglang/multimodal_gen/test/unit/test_modelopt_fp8_layerwise_offload_load.py b/python/sglang/multimodal_gen/test/unit/test_modelopt_fp8_layerwise_offload_load.py index 62c9f774e5..436d06d9aa 100644 --- a/python/sglang/multimodal_gen/test/unit/test_modelopt_fp8_layerwise_offload_load.py +++ b/python/sglang/multimodal_gen/test/unit/test_modelopt_fp8_layerwise_offload_load.py @@ -8,6 +8,7 @@ from unittest.mock import patch import torch from torch import nn +from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz from sglang.multimodal_gen.runtime.distributed.parallel_state import ( maybe_init_distributed_environment_and_model_parallel, model_parallel_is_initialized, @@ -98,6 +99,9 @@ class TestModelOptFp8LayerwiseOffloadLoad(unittest.TestCase): checkpoint_weight = state_dict["qkv.weight"].clone() checkpoint_scales = state_dict["qkv.weight_scale"].clone() expected_max_scale = checkpoint_scales.max() + fnuz = is_fp8_fnuz() + scale_factor = 2 if fnuz else 1 + expected_dtype = torch.float8_e4m3fnuz if fnuz else torch.float8_e4m3fn with patch( "sglang.multimodal_gen.runtime.layers.quantization." @@ -126,24 +130,29 @@ class TestModelOptFp8LayerwiseOffloadLoad(unittest.TestCase): # consume a channelwise scale, so it preserves the checkpoint's # FP8 shards; the fallback requantizes them to one max scale. weight = model.qkv.weight - self.assertEqual(weight.dtype, torch.float8_e4m3fn) + self.assertEqual(weight.dtype, expected_dtype) self.assertEqual(tuple(weight.shape), (_IN_FEATURES, 2 * _SHARD_OUT)) weight_scale = model.qkv.weight_scale.flatten() if cutlass_supported: expected_scales = torch.repeat_interleave( checkpoint_scales, _SHARD_OUT ) - self.assertTrue(torch.equal(weight.t(), checkpoint_weight)) + torch.testing.assert_close( + weight.t().float() * scale_factor, + checkpoint_weight.float(), + atol=0, + rtol=0, + ) else: expected_scales = expected_max_scale.expand(weight_scale.numel()) torch.testing.assert_close( weight_scale, - expected_scales, + expected_scales * scale_factor, check_device=False, ) torch.testing.assert_close( model.qkv.input_scale.flatten().max(), - torch.tensor(0.5), + torch.tensor(0.5 * scale_factor), check_device=False, ) diff --git a/python/sglang/multimodal_gen/test/unit/test_transformer_quant.py b/python/sglang/multimodal_gen/test/unit/test_transformer_quant.py index 2c03f34035..c3ee016783 100644 --- a/python/sglang/multimodal_gen/test/unit/test_transformer_quant.py +++ b/python/sglang/multimodal_gen/test/unit/test_transformer_quant.py @@ -173,38 +173,53 @@ def _make_quant_config(name: str, **attrs): class TestTransformerQuantHelpers(unittest.TestCase): def test_modelopt_fp8_packed_cutlass_preserves_checkpoint_shard_scales(self): - method = ModelOptFp8LinearMethod( - ModelOptFp8Config(is_checkpoint_fp8_serialized=True) - ) - method.cutlass_fp8_supported = True - layer = torch.nn.Module() - layer.logical_widths = [2, 2, 2] - weight = ( - torch.arange(24, dtype=torch.float32).reshape(6, 4).to(torch.float8_e4m3fn) - ) - layer.register_parameter( - "weight", torch.nn.Parameter(weight.clone(), requires_grad=False) - ) - layer.register_parameter( - "weight_scale", - torch.nn.Parameter( - torch.tensor([0.1, 0.2, 0.3], dtype=torch.float32), - requires_grad=False, - ), - ) - layer.register_parameter( - "input_scale", - torch.nn.Parameter(torch.ones(3, dtype=torch.float32), requires_grad=False), - ) + for fnuz in (False, True): + with ( + self.subTest(fnuz=fnuz), + patch( + "sglang.multimodal_gen.runtime.layers.quantization." + "modelopt_quant.is_fp8_fnuz", + return_value=fnuz, + ), + ): + method = ModelOptFp8LinearMethod( + ModelOptFp8Config(is_checkpoint_fp8_serialized=True) + ) + method.cutlass_fp8_supported = True + layer = torch.nn.Module() + layer.logical_widths = [2, 2, 2] + values = torch.arange(24, dtype=torch.float32).reshape(6, 4) + values[0, 0] = -0.0 + weight = values.to(torch.float8_e4m3fn) + scales = torch.tensor([0.1, 0.2, 0.3], dtype=torch.float32) + layer.register_parameter( + "weight", torch.nn.Parameter(weight.clone(), requires_grad=False) + ) + layer.register_parameter( + "weight_scale", + torch.nn.Parameter(scales.clone(), requires_grad=False), + ) + layer.register_parameter( + "input_scale", + torch.nn.Parameter(torch.ones(3), requires_grad=False), + ) - method.process_weights_after_loading(layer) + method.process_weights_after_loading(layer) - torch.testing.assert_close(layer.weight, weight.t(), rtol=0, atol=0) - torch.testing.assert_close( - layer.weight_scale, - torch.tensor([[0.1], [0.1], [0.2], [0.2], [0.3], [0.3]]), - ) - torch.testing.assert_close(layer.input_scale, torch.tensor(1.0)) + factor = 2 if fnuz else 1 + expected_dtype = torch.float8_e4m3fnuz if fnuz else torch.float8_e4m3fn + self.assertEqual(layer.weight.dtype, expected_dtype) + expected_scales = scales.repeat_interleave(2).view(-1, 1) + torch.testing.assert_close(layer.weight_scale, expected_scales * factor) + torch.testing.assert_close( + layer.input_scale, torch.tensor(float(factor)) + ) + torch.testing.assert_close( + layer.weight.t().float() * layer.weight_scale, + weight.float() * expected_scales, + atol=0, + rtol=0, + ) def test_modelopt_fp8_packed_cutlass_requantizes_incomplete_shard_scales(self): method = ModelOptFp8LinearMethod(