[diffusion] fix: fix native FP8 format handling for FLUX 3 row-wise linears (#42255)
This commit is contained in:
@@ -54,6 +54,7 @@ from sglang.kernels.ops.diffusion import (
|
||||
is_plain_layer_norm,
|
||||
residual_gate_add,
|
||||
)
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
|
||||
from sglang.multimodal_gen.configs.models.dits.flux3 import (
|
||||
Flux3ArchConfig,
|
||||
Flux3DiTConfig,
|
||||
@@ -304,6 +305,16 @@ class Flux3Fp8RowwiseLinear(nn.Module):
|
||||
raise ValueError(
|
||||
"expected an E4M3 weight with one fp32 scale per output row"
|
||||
)
|
||||
if is_fp8_fnuz():
|
||||
from sglang.srt.layers.quantization.fp8_utils import (
|
||||
normalize_e4m3fn_to_e4m3fnuz,
|
||||
)
|
||||
|
||||
# MI300 scaled_mm requires FNUZ. Preserve the dequantized checkpoint
|
||||
# values and do not mutate checkpoint tensors shared by the loader.
|
||||
weight, weight_scale, _ = normalize_e4m3fn_to_e4m3fnuz(
|
||||
weight.clone(), weight_scale
|
||||
)
|
||||
self.out_features, self.in_features = weight.shape
|
||||
self.tuple_output = tuple_output
|
||||
# Parameters (not buffers) so that layerwise offload streams them.
|
||||
@@ -320,6 +331,14 @@ class Flux3Fp8RowwiseLinear(nn.Module):
|
||||
if pad:
|
||||
flat = F.pad(flat, (0, 0, 0, pad))
|
||||
activation, activation_scale = quantize_fp8_rowwise(flat)
|
||||
if self.weight.dtype == torch.float8_e4m3fnuz:
|
||||
from sglang.srt.layers.quantization.fp8_utils import (
|
||||
normalize_e4m3fn_to_e4m3fnuz,
|
||||
)
|
||||
|
||||
activation, activation_scale, _ = normalize_e4m3fn_to_e4m3fnuz(
|
||||
activation, activation_scale
|
||||
)
|
||||
out = torch._scaled_mm(
|
||||
activation,
|
||||
self.weight.T,
|
||||
|
||||
@@ -612,6 +612,52 @@ def test_dit_cached_streams_match_full_forward():
|
||||
assert full["x_act_cond"].shape == (1, 1, 3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fnuz", [False, True])
|
||||
@pytest.mark.parametrize("tuple_output", [False, True])
|
||||
def test_fp8r_native_format_preserves_values(monkeypatch, fnuz, tuple_output):
|
||||
from sglang.multimodal_gen.runtime.models.dits import flux3
|
||||
|
||||
monkeypatch.setattr(flux3, "is_fp8_fnuz", lambda: fnuz, raising=False)
|
||||
# Include negative zero: reinterpreting it as FNUZ without normalization
|
||||
# produces NaN. Keep a byte snapshot to detect mutation of the checkpoint.
|
||||
weight = torch.tensor([[-0.0, 1.0, -2.0, 448.0], [0.0, -1.0, 2.0, -448.0]]).to(
|
||||
torch.float8_e4m3fn
|
||||
)
|
||||
weight_bytes = weight.view(torch.uint8).clone()
|
||||
scale = torch.tensor([0.125, 0.25])
|
||||
x = torch.tensor([[[-0.0, 1.0, -2.0, 0.5], [0.0, 0.0, 0.0, 0.0]]])
|
||||
quantized_x, x_scale = flux3.quantize_fp8_rowwise(x.flatten(0, 1))
|
||||
expected_x = quantized_x.float() * x_scale[:, None]
|
||||
expected_weight = weight.float() * scale[:, None]
|
||||
expected = (expected_x @ expected_weight.T).bfloat16().reshape(1, 2, 2)
|
||||
native_dtype = torch.float8_e4m3fnuz if fnuz else torch.float8_e4m3fn
|
||||
calls = []
|
||||
|
||||
def scaled_mm(a, b, scale_a, scale_b, *, out_dtype, use_fast_accum):
|
||||
calls.append(True)
|
||||
assert a.dtype == b.dtype == native_dtype
|
||||
assert a.shape == (16, 4) and b.shape == (4, 2)
|
||||
assert out_dtype == torch.bfloat16 and use_fast_accum
|
||||
dequant_a = a.float() * scale_a
|
||||
dequant_b = b.float() * scale_b
|
||||
torch.testing.assert_close(dequant_a[:2], expected_x, atol=0, rtol=0)
|
||||
torch.testing.assert_close(dequant_a[2:], torch.zeros(14, 4), atol=0, rtol=0)
|
||||
torch.testing.assert_close(dequant_b, expected_weight.T, atol=0, rtol=0)
|
||||
assert torch.isfinite(dequant_a).all() and torch.isfinite(dequant_b).all()
|
||||
return (dequant_a @ dequant_b).to(out_dtype)
|
||||
|
||||
monkeypatch.setattr(torch, "_scaled_mm", scaled_mm)
|
||||
layer = flux3.Flux3Fp8RowwiseLinear(weight, scale, tuple_output)
|
||||
result = layer(x)
|
||||
if tuple_output:
|
||||
result, bias = result
|
||||
assert bias is None
|
||||
torch.testing.assert_close(result, expected, atol=0, rtol=0)
|
||||
assert len(calls) == 1
|
||||
assert torch.equal(weight.view(torch.uint8), weight_bytes)
|
||||
torch.testing.assert_close(scale, torch.tensor([0.125, 0.25]), atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available() or torch.cuda.get_device_capability() < (8, 9),
|
||||
reason="needs FP8 scaled_mm",
|
||||
|
||||
@@ -8,6 +8,7 @@ from unittest.mock import patch
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
maybe_init_distributed_environment_and_model_parallel,
|
||||
model_parallel_is_initialized,
|
||||
@@ -98,6 +99,9 @@ class TestModelOptFp8LayerwiseOffloadLoad(unittest.TestCase):
|
||||
checkpoint_weight = state_dict["qkv.weight"].clone()
|
||||
checkpoint_scales = state_dict["qkv.weight_scale"].clone()
|
||||
expected_max_scale = checkpoint_scales.max()
|
||||
fnuz = is_fp8_fnuz()
|
||||
scale_factor = 2 if fnuz else 1
|
||||
expected_dtype = torch.float8_e4m3fnuz if fnuz else torch.float8_e4m3fn
|
||||
|
||||
with patch(
|
||||
"sglang.multimodal_gen.runtime.layers.quantization."
|
||||
@@ -126,24 +130,29 @@ class TestModelOptFp8LayerwiseOffloadLoad(unittest.TestCase):
|
||||
# consume a channelwise scale, so it preserves the checkpoint's
|
||||
# FP8 shards; the fallback requantizes them to one max scale.
|
||||
weight = model.qkv.weight
|
||||
self.assertEqual(weight.dtype, torch.float8_e4m3fn)
|
||||
self.assertEqual(weight.dtype, expected_dtype)
|
||||
self.assertEqual(tuple(weight.shape), (_IN_FEATURES, 2 * _SHARD_OUT))
|
||||
weight_scale = model.qkv.weight_scale.flatten()
|
||||
if cutlass_supported:
|
||||
expected_scales = torch.repeat_interleave(
|
||||
checkpoint_scales, _SHARD_OUT
|
||||
)
|
||||
self.assertTrue(torch.equal(weight.t(), checkpoint_weight))
|
||||
torch.testing.assert_close(
|
||||
weight.t().float() * scale_factor,
|
||||
checkpoint_weight.float(),
|
||||
atol=0,
|
||||
rtol=0,
|
||||
)
|
||||
else:
|
||||
expected_scales = expected_max_scale.expand(weight_scale.numel())
|
||||
torch.testing.assert_close(
|
||||
weight_scale,
|
||||
expected_scales,
|
||||
expected_scales * scale_factor,
|
||||
check_device=False,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
model.qkv.input_scale.flatten().max(),
|
||||
torch.tensor(0.5),
|
||||
torch.tensor(0.5 * scale_factor),
|
||||
check_device=False,
|
||||
)
|
||||
|
||||
|
||||
@@ -173,38 +173,53 @@ def _make_quant_config(name: str, **attrs):
|
||||
|
||||
class TestTransformerQuantHelpers(unittest.TestCase):
|
||||
def test_modelopt_fp8_packed_cutlass_preserves_checkpoint_shard_scales(self):
|
||||
method = ModelOptFp8LinearMethod(
|
||||
ModelOptFp8Config(is_checkpoint_fp8_serialized=True)
|
||||
)
|
||||
method.cutlass_fp8_supported = True
|
||||
layer = torch.nn.Module()
|
||||
layer.logical_widths = [2, 2, 2]
|
||||
weight = (
|
||||
torch.arange(24, dtype=torch.float32).reshape(6, 4).to(torch.float8_e4m3fn)
|
||||
)
|
||||
layer.register_parameter(
|
||||
"weight", torch.nn.Parameter(weight.clone(), requires_grad=False)
|
||||
)
|
||||
layer.register_parameter(
|
||||
"weight_scale",
|
||||
torch.nn.Parameter(
|
||||
torch.tensor([0.1, 0.2, 0.3], dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
),
|
||||
)
|
||||
layer.register_parameter(
|
||||
"input_scale",
|
||||
torch.nn.Parameter(torch.ones(3, dtype=torch.float32), requires_grad=False),
|
||||
)
|
||||
for fnuz in (False, True):
|
||||
with (
|
||||
self.subTest(fnuz=fnuz),
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.layers.quantization."
|
||||
"modelopt_quant.is_fp8_fnuz",
|
||||
return_value=fnuz,
|
||||
),
|
||||
):
|
||||
method = ModelOptFp8LinearMethod(
|
||||
ModelOptFp8Config(is_checkpoint_fp8_serialized=True)
|
||||
)
|
||||
method.cutlass_fp8_supported = True
|
||||
layer = torch.nn.Module()
|
||||
layer.logical_widths = [2, 2, 2]
|
||||
values = torch.arange(24, dtype=torch.float32).reshape(6, 4)
|
||||
values[0, 0] = -0.0
|
||||
weight = values.to(torch.float8_e4m3fn)
|
||||
scales = torch.tensor([0.1, 0.2, 0.3], dtype=torch.float32)
|
||||
layer.register_parameter(
|
||||
"weight", torch.nn.Parameter(weight.clone(), requires_grad=False)
|
||||
)
|
||||
layer.register_parameter(
|
||||
"weight_scale",
|
||||
torch.nn.Parameter(scales.clone(), requires_grad=False),
|
||||
)
|
||||
layer.register_parameter(
|
||||
"input_scale",
|
||||
torch.nn.Parameter(torch.ones(3), requires_grad=False),
|
||||
)
|
||||
|
||||
method.process_weights_after_loading(layer)
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
torch.testing.assert_close(layer.weight, weight.t(), rtol=0, atol=0)
|
||||
torch.testing.assert_close(
|
||||
layer.weight_scale,
|
||||
torch.tensor([[0.1], [0.1], [0.2], [0.2], [0.3], [0.3]]),
|
||||
)
|
||||
torch.testing.assert_close(layer.input_scale, torch.tensor(1.0))
|
||||
factor = 2 if fnuz else 1
|
||||
expected_dtype = torch.float8_e4m3fnuz if fnuz else torch.float8_e4m3fn
|
||||
self.assertEqual(layer.weight.dtype, expected_dtype)
|
||||
expected_scales = scales.repeat_interleave(2).view(-1, 1)
|
||||
torch.testing.assert_close(layer.weight_scale, expected_scales * factor)
|
||||
torch.testing.assert_close(
|
||||
layer.input_scale, torch.tensor(float(factor))
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
layer.weight.t().float() * layer.weight_scale,
|
||||
weight.float() * expected_scales,
|
||||
atol=0,
|
||||
rtol=0,
|
||||
)
|
||||
|
||||
def test_modelopt_fp8_packed_cutlass_requantizes_incomplete_shard_scales(self):
|
||||
method = ModelOptFp8LinearMethod(
|
||||
|
||||
Reference in New Issue
Block a user