[diffusion] fix: fix native FP8 format handling for FLUX 3 row-wise linears (#42255)

This commit is contained in:
Tokha233
2026-10-03 15:23:41 +08:00
committed by GitHub
parent 00bcc25f6c
commit ce617d410b
4 changed files with 123 additions and 34 deletions
@@ -54,6 +54,7 @@ from sglang.kernels.ops.diffusion import (
is_plain_layer_norm,
residual_gate_add,
)
from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
from sglang.multimodal_gen.configs.models.dits.flux3 import (
Flux3ArchConfig,
Flux3DiTConfig,
@@ -304,6 +305,16 @@ class Flux3Fp8RowwiseLinear(nn.Module):
raise ValueError(
"expected an E4M3 weight with one fp32 scale per output row"
)
if is_fp8_fnuz():
from sglang.srt.layers.quantization.fp8_utils import (
normalize_e4m3fn_to_e4m3fnuz,
)
# MI300 scaled_mm requires FNUZ. Preserve the dequantized checkpoint
# values and do not mutate checkpoint tensors shared by the loader.
weight, weight_scale, _ = normalize_e4m3fn_to_e4m3fnuz(
weight.clone(), weight_scale
)
self.out_features, self.in_features = weight.shape
self.tuple_output = tuple_output
# Parameters (not buffers) so that layerwise offload streams them.
@@ -320,6 +331,14 @@ class Flux3Fp8RowwiseLinear(nn.Module):
if pad:
flat = F.pad(flat, (0, 0, 0, pad))
activation, activation_scale = quantize_fp8_rowwise(flat)
if self.weight.dtype == torch.float8_e4m3fnuz:
from sglang.srt.layers.quantization.fp8_utils import (
normalize_e4m3fn_to_e4m3fnuz,
)
activation, activation_scale, _ = normalize_e4m3fn_to_e4m3fnuz(
activation, activation_scale
)
out = torch._scaled_mm(
activation,
self.weight.T,
@@ -612,6 +612,52 @@ def test_dit_cached_streams_match_full_forward():
assert full["x_act_cond"].shape == (1, 1, 3)
@pytest.mark.parametrize("fnuz", [False, True])
@pytest.mark.parametrize("tuple_output", [False, True])
def test_fp8r_native_format_preserves_values(monkeypatch, fnuz, tuple_output):
from sglang.multimodal_gen.runtime.models.dits import flux3
monkeypatch.setattr(flux3, "is_fp8_fnuz", lambda: fnuz, raising=False)
# Include negative zero: reinterpreting it as FNUZ without normalization
# produces NaN. Keep a byte snapshot to detect mutation of the checkpoint.
weight = torch.tensor([[-0.0, 1.0, -2.0, 448.0], [0.0, -1.0, 2.0, -448.0]]).to(
torch.float8_e4m3fn
)
weight_bytes = weight.view(torch.uint8).clone()
scale = torch.tensor([0.125, 0.25])
x = torch.tensor([[[-0.0, 1.0, -2.0, 0.5], [0.0, 0.0, 0.0, 0.0]]])
quantized_x, x_scale = flux3.quantize_fp8_rowwise(x.flatten(0, 1))
expected_x = quantized_x.float() * x_scale[:, None]
expected_weight = weight.float() * scale[:, None]
expected = (expected_x @ expected_weight.T).bfloat16().reshape(1, 2, 2)
native_dtype = torch.float8_e4m3fnuz if fnuz else torch.float8_e4m3fn
calls = []
def scaled_mm(a, b, scale_a, scale_b, *, out_dtype, use_fast_accum):
calls.append(True)
assert a.dtype == b.dtype == native_dtype
assert a.shape == (16, 4) and b.shape == (4, 2)
assert out_dtype == torch.bfloat16 and use_fast_accum
dequant_a = a.float() * scale_a
dequant_b = b.float() * scale_b
torch.testing.assert_close(dequant_a[:2], expected_x, atol=0, rtol=0)
torch.testing.assert_close(dequant_a[2:], torch.zeros(14, 4), atol=0, rtol=0)
torch.testing.assert_close(dequant_b, expected_weight.T, atol=0, rtol=0)
assert torch.isfinite(dequant_a).all() and torch.isfinite(dequant_b).all()
return (dequant_a @ dequant_b).to(out_dtype)
monkeypatch.setattr(torch, "_scaled_mm", scaled_mm)
layer = flux3.Flux3Fp8RowwiseLinear(weight, scale, tuple_output)
result = layer(x)
if tuple_output:
result, bias = result
assert bias is None
torch.testing.assert_close(result, expected, atol=0, rtol=0)
assert len(calls) == 1
assert torch.equal(weight.view(torch.uint8), weight_bytes)
torch.testing.assert_close(scale, torch.tensor([0.125, 0.25]), atol=0, rtol=0)
@pytest.mark.skipif(
not torch.cuda.is_available() or torch.cuda.get_device_capability() < (8, 9),
reason="needs FP8 scaled_mm",
@@ -8,6 +8,7 @@ from unittest.mock import patch
import torch
from torch import nn
from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
maybe_init_distributed_environment_and_model_parallel,
model_parallel_is_initialized,
@@ -98,6 +99,9 @@ class TestModelOptFp8LayerwiseOffloadLoad(unittest.TestCase):
checkpoint_weight = state_dict["qkv.weight"].clone()
checkpoint_scales = state_dict["qkv.weight_scale"].clone()
expected_max_scale = checkpoint_scales.max()
fnuz = is_fp8_fnuz()
scale_factor = 2 if fnuz else 1
expected_dtype = torch.float8_e4m3fnuz if fnuz else torch.float8_e4m3fn
with patch(
"sglang.multimodal_gen.runtime.layers.quantization."
@@ -126,24 +130,29 @@ class TestModelOptFp8LayerwiseOffloadLoad(unittest.TestCase):
# consume a channelwise scale, so it preserves the checkpoint's
# FP8 shards; the fallback requantizes them to one max scale.
weight = model.qkv.weight
self.assertEqual(weight.dtype, torch.float8_e4m3fn)
self.assertEqual(weight.dtype, expected_dtype)
self.assertEqual(tuple(weight.shape), (_IN_FEATURES, 2 * _SHARD_OUT))
weight_scale = model.qkv.weight_scale.flatten()
if cutlass_supported:
expected_scales = torch.repeat_interleave(
checkpoint_scales, _SHARD_OUT
)
self.assertTrue(torch.equal(weight.t(), checkpoint_weight))
torch.testing.assert_close(
weight.t().float() * scale_factor,
checkpoint_weight.float(),
atol=0,
rtol=0,
)
else:
expected_scales = expected_max_scale.expand(weight_scale.numel())
torch.testing.assert_close(
weight_scale,
expected_scales,
expected_scales * scale_factor,
check_device=False,
)
torch.testing.assert_close(
model.qkv.input_scale.flatten().max(),
torch.tensor(0.5),
torch.tensor(0.5 * scale_factor),
check_device=False,
)
@@ -173,38 +173,53 @@ def _make_quant_config(name: str, **attrs):
class TestTransformerQuantHelpers(unittest.TestCase):
def test_modelopt_fp8_packed_cutlass_preserves_checkpoint_shard_scales(self):
method = ModelOptFp8LinearMethod(
ModelOptFp8Config(is_checkpoint_fp8_serialized=True)
)
method.cutlass_fp8_supported = True
layer = torch.nn.Module()
layer.logical_widths = [2, 2, 2]
weight = (
torch.arange(24, dtype=torch.float32).reshape(6, 4).to(torch.float8_e4m3fn)
)
layer.register_parameter(
"weight", torch.nn.Parameter(weight.clone(), requires_grad=False)
)
layer.register_parameter(
"weight_scale",
torch.nn.Parameter(
torch.tensor([0.1, 0.2, 0.3], dtype=torch.float32),
requires_grad=False,
),
)
layer.register_parameter(
"input_scale",
torch.nn.Parameter(torch.ones(3, dtype=torch.float32), requires_grad=False),
)
for fnuz in (False, True):
with (
self.subTest(fnuz=fnuz),
patch(
"sglang.multimodal_gen.runtime.layers.quantization."
"modelopt_quant.is_fp8_fnuz",
return_value=fnuz,
),
):
method = ModelOptFp8LinearMethod(
ModelOptFp8Config(is_checkpoint_fp8_serialized=True)
)
method.cutlass_fp8_supported = True
layer = torch.nn.Module()
layer.logical_widths = [2, 2, 2]
values = torch.arange(24, dtype=torch.float32).reshape(6, 4)
values[0, 0] = -0.0
weight = values.to(torch.float8_e4m3fn)
scales = torch.tensor([0.1, 0.2, 0.3], dtype=torch.float32)
layer.register_parameter(
"weight", torch.nn.Parameter(weight.clone(), requires_grad=False)
)
layer.register_parameter(
"weight_scale",
torch.nn.Parameter(scales.clone(), requires_grad=False),
)
layer.register_parameter(
"input_scale",
torch.nn.Parameter(torch.ones(3), requires_grad=False),
)
method.process_weights_after_loading(layer)
method.process_weights_after_loading(layer)
torch.testing.assert_close(layer.weight, weight.t(), rtol=0, atol=0)
torch.testing.assert_close(
layer.weight_scale,
torch.tensor([[0.1], [0.1], [0.2], [0.2], [0.3], [0.3]]),
)
torch.testing.assert_close(layer.input_scale, torch.tensor(1.0))
factor = 2 if fnuz else 1
expected_dtype = torch.float8_e4m3fnuz if fnuz else torch.float8_e4m3fn
self.assertEqual(layer.weight.dtype, expected_dtype)
expected_scales = scales.repeat_interleave(2).view(-1, 1)
torch.testing.assert_close(layer.weight_scale, expected_scales * factor)
torch.testing.assert_close(
layer.input_scale, torch.tensor(float(factor))
)
torch.testing.assert_close(
layer.weight.t().float() * layer.weight_scale,
weight.float() * expected_scales,
atol=0,
rtol=0,
)
def test_modelopt_fp8_packed_cutlass_requantizes_incomplete_shard_scales(self):
method = ModelOptFp8LinearMethod(