[Perf] [AMD] attention: Enable diffusion BF16 attention and MXFP4 compilation (#27702)
Co-authored-by: jacky.cheng <yichiche@amd.com>
This commit is contained in:
co-authored by
jacky.cheng
parent
5b12ec163c
commit
5ef8319f16
@@ -280,6 +280,61 @@ class AITerImpl(AttentionImpl):
|
||||
self.softmax_scale = softmax_scale
|
||||
|
||||
@torch.compiler.disable
|
||||
def _forward_fp8(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
) -> torch.Tensor | None:
|
||||
"""
|
||||
FP8 FMHA prefill path. Kept out of the compiled graph (the aiter FP8
|
||||
quant + FMHA kernels are not torch.compile-safe); only invoked when
|
||||
SGLANG_DIFFUSION_AITER_FP8_ATTN=1.
|
||||
|
||||
Returns the attention output, or None if the FP8 FMHA kernel does not
|
||||
support the current shapes, in which case the caller falls back to the
|
||||
(compilable) BF16 path.
|
||||
"""
|
||||
if query.dtype != _fp8_dtype:
|
||||
q_fp8, q_scale = _per_tensor_quant_fp8(query)
|
||||
k_fp8, k_scale = _per_tensor_quant_fp8(key)
|
||||
v_fp8, v_scale = _per_tensor_quant_fp8(value)
|
||||
else:
|
||||
q_fp8, k_fp8, v_fp8 = query, key, value
|
||||
one = torch.tensor(1.0, dtype=torch.float32, device=query.device)
|
||||
q_scale = k_scale = v_scale = one
|
||||
|
||||
d_q = q_fp8.shape[-1]
|
||||
d_k = k_fp8.shape[-1]
|
||||
d_v = v_fp8.shape[-1]
|
||||
h_q = q_fp8.shape[2]
|
||||
h_kv = k_fp8.shape[2]
|
||||
|
||||
if _can_use_fmha_fp8_prefill(d_q, d_k, d_v, h_q, h_kv):
|
||||
return _fmha_fp8_prefill_attention(
|
||||
q_fp8,
|
||||
k_fp8,
|
||||
v_fp8,
|
||||
softmax_scale=self.softmax_scale,
|
||||
is_causal=self.causal,
|
||||
q_scale=q_scale,
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
)
|
||||
|
||||
logger.warning_once(
|
||||
"FP8 FMHA prefill unsupported for this shape (need gfx950-class AITER, "
|
||||
"full MHA, q/k/v head_dim=%d; got q=%d, k=%d, v=%d, num_heads=%d, "
|
||||
"num_kv_heads=%d). Falling back to BF16.",
|
||||
_FMHA_FP8_HEAD_DIM,
|
||||
d_q,
|
||||
d_k,
|
||||
d_v,
|
||||
h_q,
|
||||
h_kv,
|
||||
)
|
||||
return None
|
||||
|
||||
def forward(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
@@ -302,44 +357,9 @@ class AITerImpl(AttentionImpl):
|
||||
Output tensor of shape [batch_size, seq_len, num_heads, head_dim]
|
||||
"""
|
||||
if _use_fp8_attn:
|
||||
if query.dtype != _fp8_dtype:
|
||||
q_fp8, q_scale = _per_tensor_quant_fp8(query)
|
||||
k_fp8, k_scale = _per_tensor_quant_fp8(key)
|
||||
v_fp8, v_scale = _per_tensor_quant_fp8(value)
|
||||
else:
|
||||
q_fp8, k_fp8, v_fp8 = query, key, value
|
||||
one = torch.tensor(1.0, dtype=torch.float32, device=query.device)
|
||||
q_scale = k_scale = v_scale = one
|
||||
|
||||
d_q = q_fp8.shape[-1]
|
||||
d_k = k_fp8.shape[-1]
|
||||
d_v = v_fp8.shape[-1]
|
||||
h_q = q_fp8.shape[2]
|
||||
h_kv = k_fp8.shape[2]
|
||||
|
||||
if _can_use_fmha_fp8_prefill(d_q, d_k, d_v, h_q, h_kv):
|
||||
return _fmha_fp8_prefill_attention(
|
||||
q_fp8,
|
||||
k_fp8,
|
||||
v_fp8,
|
||||
softmax_scale=self.softmax_scale,
|
||||
is_causal=self.causal,
|
||||
q_scale=q_scale,
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
)
|
||||
|
||||
logger.warning_once(
|
||||
"FP8 FMHA prefill unsupported for this shape (need gfx950-class AITER, "
|
||||
"full MHA, q/k/v head_dim=%d; got q=%d, k=%d, v=%d, num_heads=%d, "
|
||||
"num_kv_heads=%d). Falling back to BF16.",
|
||||
_FMHA_FP8_HEAD_DIM,
|
||||
d_q,
|
||||
d_k,
|
||||
d_v,
|
||||
h_q,
|
||||
h_kv,
|
||||
)
|
||||
output = self._forward_fp8(query, key, value)
|
||||
if output is not None:
|
||||
return output
|
||||
|
||||
# BF16 path
|
||||
output = aiter.flash_attn_func(
|
||||
|
||||
@@ -17,6 +17,7 @@ from sglang.multimodal_gen.runtime.models.parameter import (
|
||||
)
|
||||
from sglang.srt.layers.quantization.utils import is_layer_skipped
|
||||
from sglang.srt.utils import is_gfx95_supported, is_hip
|
||||
from sglang.srt.utils.common import direct_register_custom_op
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_is_hip = is_hip()
|
||||
@@ -40,6 +41,36 @@ if _is_hip:
|
||||
_MXFP4_MIN_OUTPUT_DIM = 256
|
||||
|
||||
|
||||
if _is_hip and gemm_a4w4 is not None and dynamic_mxfp4_quant is not None:
|
||||
|
||||
def _mxfp4_quant_gemm(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
x_fp4, x_scale = dynamic_mxfp4_quant(x, shuffle=True)
|
||||
return gemm_a4w4(x_fp4, weight, x_scale, weight_scale)
|
||||
|
||||
def _mxfp4_quant_gemm_fake(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
# weight is [output_size_per_partition, ...]; gemm output is [M, N].
|
||||
return torch.empty(
|
||||
(x.shape[0], weight.shape[0]), dtype=x.dtype, device=x.device
|
||||
)
|
||||
|
||||
# Wrap the activation-quant + A4W4 gemm in a single functional custom op.
|
||||
# This keeps the whole layer torch.compile compatible.
|
||||
direct_register_custom_op(
|
||||
op_name="mxfp4_diffusion_quant_gemm",
|
||||
op_func=_mxfp4_quant_gemm,
|
||||
mutates_args=[],
|
||||
fake_impl=_mxfp4_quant_gemm_fake,
|
||||
)
|
||||
|
||||
|
||||
class Mxfp4Config(QuantizationConfig):
|
||||
"""
|
||||
MXFP4 quantization config for diffusion models.
|
||||
@@ -119,6 +150,18 @@ class Mxfp4LinearMethod(LinearMethodBase):
|
||||
"""
|
||||
|
||||
def __init__(self, quant_config: Mxfp4Config):
|
||||
# Only supported on ROCm + GFX950
|
||||
if not is_gfx95_supported():
|
||||
platform = "unknown"
|
||||
if _is_hip:
|
||||
try:
|
||||
platform = torch.cuda.get_device_properties(0).gcnArchName
|
||||
except Exception:
|
||||
platform = "ROCm (unknown arch)"
|
||||
raise RuntimeError(
|
||||
f"MXFP4 requires ROCm and MI350+ (gfx95x). "
|
||||
f"Current platform: {platform}."
|
||||
)
|
||||
self.quant_config = quant_config
|
||||
|
||||
def create_weights(
|
||||
@@ -165,18 +208,6 @@ class Mxfp4LinearMethod(LinearMethodBase):
|
||||
- Packed uint8 (2 FP4 values per byte)
|
||||
- E8M0 scales (one per 32-element block)
|
||||
"""
|
||||
if not is_gfx95_supported():
|
||||
platform = "unknown"
|
||||
if _is_hip:
|
||||
try:
|
||||
platform = torch.cuda.get_device_properties(0).gcnArchName
|
||||
except:
|
||||
platform = "ROCm (unknown arch)"
|
||||
raise RuntimeError(
|
||||
f"MXFP4 quantization requires ROCm and MI350+ (gfx95x). "
|
||||
f"Current platform: {platform}."
|
||||
)
|
||||
|
||||
# Check if weights are already quantized
|
||||
if layer.weight.dtype not in [torch.bfloat16, torch.float16]:
|
||||
# Already quantized or unexpected dtype
|
||||
@@ -215,21 +246,17 @@ class Mxfp4LinearMethod(LinearMethodBase):
|
||||
x: torch.Tensor,
|
||||
bias: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
|
||||
if not is_gfx95_supported():
|
||||
raise RuntimeError(
|
||||
"MXFP4 inference requires ROCm and MI350+ (gfx95x). "
|
||||
"Current platform not supported."
|
||||
)
|
||||
# Platform support is validated once in __init__; the gemm runs through
|
||||
# a functional custom op so this stays compilable.
|
||||
|
||||
# Handle 3D input tensors [batch, seq, hidden]
|
||||
original_shape = x.shape
|
||||
if x.dim() == 3:
|
||||
x = x.view(-1, x.shape[-1])
|
||||
|
||||
x_fp4, x_scale = dynamic_mxfp4_quant(x, shuffle=True)
|
||||
|
||||
y = gemm_a4w4(x_fp4, layer.weight, x_scale, layer.weight_scale)
|
||||
y = torch.ops.sglang.mxfp4_diffusion_quant_gemm(
|
||||
x, layer.weight, layer.weight_scale
|
||||
)
|
||||
|
||||
if bias is not None:
|
||||
y = y + bias
|
||||
|
||||
@@ -1056,7 +1056,8 @@ def _resolve_quant_config(
|
||||
# Online-quant convention: for `fp8`, `mxfp4` and `convrot_int8`, a
|
||||
# no-arg QuantizationConfig() selects the post-load path -- weights
|
||||
# load in source dtype and are quantized in
|
||||
# process_weights_after_loading.
|
||||
# process_weights_after_loading. Only those three configs take
|
||||
# --quantization-ignored-layers.
|
||||
quant_cls = get_quantization_config(server_args.quantization)
|
||||
quant_kwargs = {}
|
||||
if server_args.quantization in {"fp8", "mxfp4", "convrot_int8"}:
|
||||
|
||||
@@ -1488,6 +1488,43 @@ class TestTransformerQuantHelpers(unittest.TestCase):
|
||||
self.assertIsInstance(quant_config, Fp8Config)
|
||||
self.assertEqual(quant_config.ignored_layers, ignored_layers)
|
||||
|
||||
def _resolve_online_quant_config(self, method, ignored_layers):
|
||||
return _resolve_quant_config(
|
||||
hf_config={},
|
||||
server_args=self._make_server_args(
|
||||
quantization=method,
|
||||
quantization_ignored_layers=ignored_layers,
|
||||
),
|
||||
safetensors_list=[],
|
||||
component_model_path="/unused/component/path",
|
||||
)
|
||||
|
||||
def test_online_quant_configs_accepting_ignored_layers_receive_them(self):
|
||||
# The online methods that honour `ignored_layers` in their
|
||||
# `get_quant_method` must be handed the CLI patterns.
|
||||
ignored_layers = ["blocks.0.attn.out_proj", "condition_proj"]
|
||||
for method in ("fp8", "mxfp4", "convrot_int8"):
|
||||
with self.subTest(method=method):
|
||||
quant_config = self._resolve_online_quant_config(method, ignored_layers)
|
||||
self.assertEqual(quant_config.ignored_layers, ignored_layers)
|
||||
|
||||
def test_online_quant_config_without_ignored_layers_param_does_not_raise(self):
|
||||
# Methods outside the online-quant set must be built without the
|
||||
# kwarg rather than raising TypeError when the CLI flag is set.
|
||||
quant_config = self._resolve_online_quant_config(
|
||||
"mxfp4_npu", ["blocks.0.attn.out_proj"]
|
||||
)
|
||||
self.assertIsNotNone(quant_config)
|
||||
self.assertFalse(hasattr(quant_config, "ignored_layers"))
|
||||
|
||||
def test_unset_ignored_layers_leaves_config_default(self):
|
||||
# Omitting the kwarg when the flag is unset must match the previous
|
||||
# behaviour of passing ignored_layers=None explicitly.
|
||||
for ignored_layers in (None, []):
|
||||
with self.subTest(ignored_layers=ignored_layers):
|
||||
quant_config = self._resolve_online_quant_config("fp8", ignored_layers)
|
||||
self.assertEqual(quant_config.ignored_layers, [])
|
||||
|
||||
@patch(
|
||||
"sglang.multimodal_gen.runtime.loader.transformer_load_utils.build_nvfp4_config_from_safetensors_list",
|
||||
return_value=None,
|
||||
|
||||
Reference in New Issue
Block a user