[Perf] [AMD] attention: Enable diffusion BF16 attention and MXFP4 compilation (#27702)

Co-authored-by: jacky.cheng <yichiche@amd.com>
This commit is contained in:
Aleksi Vesanto
2026-10-16 00:40:41 +08:00
committed by GitHub
co-authored by jacky.cheng
parent 5b12ec163c
commit 5ef8319f16
4 changed files with 145 additions and 60 deletions
@@ -280,6 +280,61 @@ class AITerImpl(AttentionImpl):
self.softmax_scale = softmax_scale
@torch.compiler.disable
def _forward_fp8(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
) -> torch.Tensor | None:
"""
FP8 FMHA prefill path. Kept out of the compiled graph (the aiter FP8
quant + FMHA kernels are not torch.compile-safe); only invoked when
SGLANG_DIFFUSION_AITER_FP8_ATTN=1.
Returns the attention output, or None if the FP8 FMHA kernel does not
support the current shapes, in which case the caller falls back to the
(compilable) BF16 path.
"""
if query.dtype != _fp8_dtype:
q_fp8, q_scale = _per_tensor_quant_fp8(query)
k_fp8, k_scale = _per_tensor_quant_fp8(key)
v_fp8, v_scale = _per_tensor_quant_fp8(value)
else:
q_fp8, k_fp8, v_fp8 = query, key, value
one = torch.tensor(1.0, dtype=torch.float32, device=query.device)
q_scale = k_scale = v_scale = one
d_q = q_fp8.shape[-1]
d_k = k_fp8.shape[-1]
d_v = v_fp8.shape[-1]
h_q = q_fp8.shape[2]
h_kv = k_fp8.shape[2]
if _can_use_fmha_fp8_prefill(d_q, d_k, d_v, h_q, h_kv):
return _fmha_fp8_prefill_attention(
q_fp8,
k_fp8,
v_fp8,
softmax_scale=self.softmax_scale,
is_causal=self.causal,
q_scale=q_scale,
k_scale=k_scale,
v_scale=v_scale,
)
logger.warning_once(
"FP8 FMHA prefill unsupported for this shape (need gfx950-class AITER, "
"full MHA, q/k/v head_dim=%d; got q=%d, k=%d, v=%d, num_heads=%d, "
"num_kv_heads=%d). Falling back to BF16.",
_FMHA_FP8_HEAD_DIM,
d_q,
d_k,
d_v,
h_q,
h_kv,
)
return None
def forward(
self,
query: torch.Tensor,
@@ -302,44 +357,9 @@ class AITerImpl(AttentionImpl):
Output tensor of shape [batch_size, seq_len, num_heads, head_dim]
"""
if _use_fp8_attn:
if query.dtype != _fp8_dtype:
q_fp8, q_scale = _per_tensor_quant_fp8(query)
k_fp8, k_scale = _per_tensor_quant_fp8(key)
v_fp8, v_scale = _per_tensor_quant_fp8(value)
else:
q_fp8, k_fp8, v_fp8 = query, key, value
one = torch.tensor(1.0, dtype=torch.float32, device=query.device)
q_scale = k_scale = v_scale = one
d_q = q_fp8.shape[-1]
d_k = k_fp8.shape[-1]
d_v = v_fp8.shape[-1]
h_q = q_fp8.shape[2]
h_kv = k_fp8.shape[2]
if _can_use_fmha_fp8_prefill(d_q, d_k, d_v, h_q, h_kv):
return _fmha_fp8_prefill_attention(
q_fp8,
k_fp8,
v_fp8,
softmax_scale=self.softmax_scale,
is_causal=self.causal,
q_scale=q_scale,
k_scale=k_scale,
v_scale=v_scale,
)
logger.warning_once(
"FP8 FMHA prefill unsupported for this shape (need gfx950-class AITER, "
"full MHA, q/k/v head_dim=%d; got q=%d, k=%d, v=%d, num_heads=%d, "
"num_kv_heads=%d). Falling back to BF16.",
_FMHA_FP8_HEAD_DIM,
d_q,
d_k,
d_v,
h_q,
h_kv,
)
output = self._forward_fp8(query, key, value)
if output is not None:
return output
# BF16 path
output = aiter.flash_attn_func(
@@ -17,6 +17,7 @@ from sglang.multimodal_gen.runtime.models.parameter import (
)
from sglang.srt.layers.quantization.utils import is_layer_skipped
from sglang.srt.utils import is_gfx95_supported, is_hip
from sglang.srt.utils.common import direct_register_custom_op
logger = logging.getLogger(__name__)
_is_hip = is_hip()
@@ -40,6 +41,36 @@ if _is_hip:
_MXFP4_MIN_OUTPUT_DIM = 256
if _is_hip and gemm_a4w4 is not None and dynamic_mxfp4_quant is not None:
def _mxfp4_quant_gemm(
x: torch.Tensor,
weight: torch.Tensor,
weight_scale: torch.Tensor,
) -> torch.Tensor:
x_fp4, x_scale = dynamic_mxfp4_quant(x, shuffle=True)
return gemm_a4w4(x_fp4, weight, x_scale, weight_scale)
def _mxfp4_quant_gemm_fake(
x: torch.Tensor,
weight: torch.Tensor,
weight_scale: torch.Tensor,
) -> torch.Tensor:
# weight is [output_size_per_partition, ...]; gemm output is [M, N].
return torch.empty(
(x.shape[0], weight.shape[0]), dtype=x.dtype, device=x.device
)
# Wrap the activation-quant + A4W4 gemm in a single functional custom op.
# This keeps the whole layer torch.compile compatible.
direct_register_custom_op(
op_name="mxfp4_diffusion_quant_gemm",
op_func=_mxfp4_quant_gemm,
mutates_args=[],
fake_impl=_mxfp4_quant_gemm_fake,
)
class Mxfp4Config(QuantizationConfig):
"""
MXFP4 quantization config for diffusion models.
@@ -119,6 +150,18 @@ class Mxfp4LinearMethod(LinearMethodBase):
"""
def __init__(self, quant_config: Mxfp4Config):
# Only supported on ROCm + GFX950
if not is_gfx95_supported():
platform = "unknown"
if _is_hip:
try:
platform = torch.cuda.get_device_properties(0).gcnArchName
except Exception:
platform = "ROCm (unknown arch)"
raise RuntimeError(
f"MXFP4 requires ROCm and MI350+ (gfx95x). "
f"Current platform: {platform}."
)
self.quant_config = quant_config
def create_weights(
@@ -165,18 +208,6 @@ class Mxfp4LinearMethod(LinearMethodBase):
- Packed uint8 (2 FP4 values per byte)
- E8M0 scales (one per 32-element block)
"""
if not is_gfx95_supported():
platform = "unknown"
if _is_hip:
try:
platform = torch.cuda.get_device_properties(0).gcnArchName
except:
platform = "ROCm (unknown arch)"
raise RuntimeError(
f"MXFP4 quantization requires ROCm and MI350+ (gfx95x). "
f"Current platform: {platform}."
)
# Check if weights are already quantized
if layer.weight.dtype not in [torch.bfloat16, torch.float16]:
# Already quantized or unexpected dtype
@@ -215,21 +246,17 @@ class Mxfp4LinearMethod(LinearMethodBase):
x: torch.Tensor,
bias: Optional[torch.Tensor] = None,
) -> torch.Tensor:
if not is_gfx95_supported():
raise RuntimeError(
"MXFP4 inference requires ROCm and MI350+ (gfx95x). "
"Current platform not supported."
)
# Platform support is validated once in __init__; the gemm runs through
# a functional custom op so this stays compilable.
# Handle 3D input tensors [batch, seq, hidden]
original_shape = x.shape
if x.dim() == 3:
x = x.view(-1, x.shape[-1])
x_fp4, x_scale = dynamic_mxfp4_quant(x, shuffle=True)
y = gemm_a4w4(x_fp4, layer.weight, x_scale, layer.weight_scale)
y = torch.ops.sglang.mxfp4_diffusion_quant_gemm(
x, layer.weight, layer.weight_scale
)
if bias is not None:
y = y + bias
@@ -1056,7 +1056,8 @@ def _resolve_quant_config(
# Online-quant convention: for `fp8`, `mxfp4` and `convrot_int8`, a
# no-arg QuantizationConfig() selects the post-load path -- weights
# load in source dtype and are quantized in
# process_weights_after_loading.
# process_weights_after_loading. Only those three configs take
# --quantization-ignored-layers.
quant_cls = get_quantization_config(server_args.quantization)
quant_kwargs = {}
if server_args.quantization in {"fp8", "mxfp4", "convrot_int8"}:
@@ -1488,6 +1488,43 @@ class TestTransformerQuantHelpers(unittest.TestCase):
self.assertIsInstance(quant_config, Fp8Config)
self.assertEqual(quant_config.ignored_layers, ignored_layers)
def _resolve_online_quant_config(self, method, ignored_layers):
return _resolve_quant_config(
hf_config={},
server_args=self._make_server_args(
quantization=method,
quantization_ignored_layers=ignored_layers,
),
safetensors_list=[],
component_model_path="/unused/component/path",
)
def test_online_quant_configs_accepting_ignored_layers_receive_them(self):
# The online methods that honour `ignored_layers` in their
# `get_quant_method` must be handed the CLI patterns.
ignored_layers = ["blocks.0.attn.out_proj", "condition_proj"]
for method in ("fp8", "mxfp4", "convrot_int8"):
with self.subTest(method=method):
quant_config = self._resolve_online_quant_config(method, ignored_layers)
self.assertEqual(quant_config.ignored_layers, ignored_layers)
def test_online_quant_config_without_ignored_layers_param_does_not_raise(self):
# Methods outside the online-quant set must be built without the
# kwarg rather than raising TypeError when the CLI flag is set.
quant_config = self._resolve_online_quant_config(
"mxfp4_npu", ["blocks.0.attn.out_proj"]
)
self.assertIsNotNone(quant_config)
self.assertFalse(hasattr(quant_config, "ignored_layers"))
def test_unset_ignored_layers_leaves_config_default(self):
# Omitting the kwarg when the flag is unset must match the previous
# behaviour of passing ignored_layers=None explicitly.
for ignored_layers in (None, []):
with self.subTest(ignored_layers=ignored_layers):
quant_config = self._resolve_online_quant_config("fp8", ignored_layers)
self.assertEqual(quant_config.ignored_layers, [])
@patch(
"sglang.multimodal_gen.runtime.loader.transformer_load_utils.build_nvfp4_config_from_safetensors_list",
return_value=None,