diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/aiter.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/aiter.py index 2d15e1ab2c..0ad09cf0a5 100755 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/aiter.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/aiter.py @@ -280,6 +280,61 @@ class AITerImpl(AttentionImpl): self.softmax_scale = softmax_scale @torch.compiler.disable + def _forward_fp8( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + ) -> torch.Tensor | None: + """ + FP8 FMHA prefill path. Kept out of the compiled graph (the aiter FP8 + quant + FMHA kernels are not torch.compile-safe); only invoked when + SGLANG_DIFFUSION_AITER_FP8_ATTN=1. + + Returns the attention output, or None if the FP8 FMHA kernel does not + support the current shapes, in which case the caller falls back to the + (compilable) BF16 path. + """ + if query.dtype != _fp8_dtype: + q_fp8, q_scale = _per_tensor_quant_fp8(query) + k_fp8, k_scale = _per_tensor_quant_fp8(key) + v_fp8, v_scale = _per_tensor_quant_fp8(value) + else: + q_fp8, k_fp8, v_fp8 = query, key, value + one = torch.tensor(1.0, dtype=torch.float32, device=query.device) + q_scale = k_scale = v_scale = one + + d_q = q_fp8.shape[-1] + d_k = k_fp8.shape[-1] + d_v = v_fp8.shape[-1] + h_q = q_fp8.shape[2] + h_kv = k_fp8.shape[2] + + if _can_use_fmha_fp8_prefill(d_q, d_k, d_v, h_q, h_kv): + return _fmha_fp8_prefill_attention( + q_fp8, + k_fp8, + v_fp8, + softmax_scale=self.softmax_scale, + is_causal=self.causal, + q_scale=q_scale, + k_scale=k_scale, + v_scale=v_scale, + ) + + logger.warning_once( + "FP8 FMHA prefill unsupported for this shape (need gfx950-class AITER, " + "full MHA, q/k/v head_dim=%d; got q=%d, k=%d, v=%d, num_heads=%d, " + "num_kv_heads=%d). Falling back to BF16.", + _FMHA_FP8_HEAD_DIM, + d_q, + d_k, + d_v, + h_q, + h_kv, + ) + return None + def forward( self, query: torch.Tensor, @@ -302,44 +357,9 @@ class AITerImpl(AttentionImpl): Output tensor of shape [batch_size, seq_len, num_heads, head_dim] """ if _use_fp8_attn: - if query.dtype != _fp8_dtype: - q_fp8, q_scale = _per_tensor_quant_fp8(query) - k_fp8, k_scale = _per_tensor_quant_fp8(key) - v_fp8, v_scale = _per_tensor_quant_fp8(value) - else: - q_fp8, k_fp8, v_fp8 = query, key, value - one = torch.tensor(1.0, dtype=torch.float32, device=query.device) - q_scale = k_scale = v_scale = one - - d_q = q_fp8.shape[-1] - d_k = k_fp8.shape[-1] - d_v = v_fp8.shape[-1] - h_q = q_fp8.shape[2] - h_kv = k_fp8.shape[2] - - if _can_use_fmha_fp8_prefill(d_q, d_k, d_v, h_q, h_kv): - return _fmha_fp8_prefill_attention( - q_fp8, - k_fp8, - v_fp8, - softmax_scale=self.softmax_scale, - is_causal=self.causal, - q_scale=q_scale, - k_scale=k_scale, - v_scale=v_scale, - ) - - logger.warning_once( - "FP8 FMHA prefill unsupported for this shape (need gfx950-class AITER, " - "full MHA, q/k/v head_dim=%d; got q=%d, k=%d, v=%d, num_heads=%d, " - "num_kv_heads=%d). Falling back to BF16.", - _FMHA_FP8_HEAD_DIM, - d_q, - d_k, - d_v, - h_q, - h_kv, - ) + output = self._forward_fp8(query, key, value) + if output is not None: + return output # BF16 path output = aiter.flash_attn_func( diff --git a/python/sglang/multimodal_gen/runtime/layers/quantization/mxfp4.py b/python/sglang/multimodal_gen/runtime/layers/quantization/mxfp4.py index 629a2f9a1d..66c60ea25c 100644 --- a/python/sglang/multimodal_gen/runtime/layers/quantization/mxfp4.py +++ b/python/sglang/multimodal_gen/runtime/layers/quantization/mxfp4.py @@ -17,6 +17,7 @@ from sglang.multimodal_gen.runtime.models.parameter import ( ) from sglang.srt.layers.quantization.utils import is_layer_skipped from sglang.srt.utils import is_gfx95_supported, is_hip +from sglang.srt.utils.common import direct_register_custom_op logger = logging.getLogger(__name__) _is_hip = is_hip() @@ -40,6 +41,36 @@ if _is_hip: _MXFP4_MIN_OUTPUT_DIM = 256 +if _is_hip and gemm_a4w4 is not None and dynamic_mxfp4_quant is not None: + + def _mxfp4_quant_gemm( + x: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + ) -> torch.Tensor: + x_fp4, x_scale = dynamic_mxfp4_quant(x, shuffle=True) + return gemm_a4w4(x_fp4, weight, x_scale, weight_scale) + + def _mxfp4_quant_gemm_fake( + x: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + ) -> torch.Tensor: + # weight is [output_size_per_partition, ...]; gemm output is [M, N]. + return torch.empty( + (x.shape[0], weight.shape[0]), dtype=x.dtype, device=x.device + ) + + # Wrap the activation-quant + A4W4 gemm in a single functional custom op. + # This keeps the whole layer torch.compile compatible. + direct_register_custom_op( + op_name="mxfp4_diffusion_quant_gemm", + op_func=_mxfp4_quant_gemm, + mutates_args=[], + fake_impl=_mxfp4_quant_gemm_fake, + ) + + class Mxfp4Config(QuantizationConfig): """ MXFP4 quantization config for diffusion models. @@ -119,6 +150,18 @@ class Mxfp4LinearMethod(LinearMethodBase): """ def __init__(self, quant_config: Mxfp4Config): + # Only supported on ROCm + GFX950 + if not is_gfx95_supported(): + platform = "unknown" + if _is_hip: + try: + platform = torch.cuda.get_device_properties(0).gcnArchName + except Exception: + platform = "ROCm (unknown arch)" + raise RuntimeError( + f"MXFP4 requires ROCm and MI350+ (gfx95x). " + f"Current platform: {platform}." + ) self.quant_config = quant_config def create_weights( @@ -165,18 +208,6 @@ class Mxfp4LinearMethod(LinearMethodBase): - Packed uint8 (2 FP4 values per byte) - E8M0 scales (one per 32-element block) """ - if not is_gfx95_supported(): - platform = "unknown" - if _is_hip: - try: - platform = torch.cuda.get_device_properties(0).gcnArchName - except: - platform = "ROCm (unknown arch)" - raise RuntimeError( - f"MXFP4 quantization requires ROCm and MI350+ (gfx95x). " - f"Current platform: {platform}." - ) - # Check if weights are already quantized if layer.weight.dtype not in [torch.bfloat16, torch.float16]: # Already quantized or unexpected dtype @@ -215,21 +246,17 @@ class Mxfp4LinearMethod(LinearMethodBase): x: torch.Tensor, bias: Optional[torch.Tensor] = None, ) -> torch.Tensor: - - if not is_gfx95_supported(): - raise RuntimeError( - "MXFP4 inference requires ROCm and MI350+ (gfx95x). " - "Current platform not supported." - ) + # Platform support is validated once in __init__; the gemm runs through + # a functional custom op so this stays compilable. # Handle 3D input tensors [batch, seq, hidden] original_shape = x.shape if x.dim() == 3: x = x.view(-1, x.shape[-1]) - x_fp4, x_scale = dynamic_mxfp4_quant(x, shuffle=True) - - y = gemm_a4w4(x_fp4, layer.weight, x_scale, layer.weight_scale) + y = torch.ops.sglang.mxfp4_diffusion_quant_gemm( + x, layer.weight, layer.weight_scale + ) if bias is not None: y = y + bias diff --git a/python/sglang/multimodal_gen/runtime/loader/transformer_load_utils.py b/python/sglang/multimodal_gen/runtime/loader/transformer_load_utils.py index 54ed9c033f..52ce3057ca 100644 --- a/python/sglang/multimodal_gen/runtime/loader/transformer_load_utils.py +++ b/python/sglang/multimodal_gen/runtime/loader/transformer_load_utils.py @@ -1056,7 +1056,8 @@ def _resolve_quant_config( # Online-quant convention: for `fp8`, `mxfp4` and `convrot_int8`, a # no-arg QuantizationConfig() selects the post-load path -- weights # load in source dtype and are quantized in - # process_weights_after_loading. + # process_weights_after_loading. Only those three configs take + # --quantization-ignored-layers. quant_cls = get_quantization_config(server_args.quantization) quant_kwargs = {} if server_args.quantization in {"fp8", "mxfp4", "convrot_int8"}: diff --git a/python/sglang/multimodal_gen/test/unit/test_transformer_quant.py b/python/sglang/multimodal_gen/test/unit/test_transformer_quant.py index 873cdad415..e6c7ff5be7 100644 --- a/python/sglang/multimodal_gen/test/unit/test_transformer_quant.py +++ b/python/sglang/multimodal_gen/test/unit/test_transformer_quant.py @@ -1488,6 +1488,43 @@ class TestTransformerQuantHelpers(unittest.TestCase): self.assertIsInstance(quant_config, Fp8Config) self.assertEqual(quant_config.ignored_layers, ignored_layers) + def _resolve_online_quant_config(self, method, ignored_layers): + return _resolve_quant_config( + hf_config={}, + server_args=self._make_server_args( + quantization=method, + quantization_ignored_layers=ignored_layers, + ), + safetensors_list=[], + component_model_path="/unused/component/path", + ) + + def test_online_quant_configs_accepting_ignored_layers_receive_them(self): + # The online methods that honour `ignored_layers` in their + # `get_quant_method` must be handed the CLI patterns. + ignored_layers = ["blocks.0.attn.out_proj", "condition_proj"] + for method in ("fp8", "mxfp4", "convrot_int8"): + with self.subTest(method=method): + quant_config = self._resolve_online_quant_config(method, ignored_layers) + self.assertEqual(quant_config.ignored_layers, ignored_layers) + + def test_online_quant_config_without_ignored_layers_param_does_not_raise(self): + # Methods outside the online-quant set must be built without the + # kwarg rather than raising TypeError when the CLI flag is set. + quant_config = self._resolve_online_quant_config( + "mxfp4_npu", ["blocks.0.attn.out_proj"] + ) + self.assertIsNotNone(quant_config) + self.assertFalse(hasattr(quant_config, "ignored_layers")) + + def test_unset_ignored_layers_leaves_config_default(self): + # Omitting the kwarg when the flag is unset must match the previous + # behaviour of passing ignored_layers=None explicitly. + for ignored_layers in (None, []): + with self.subTest(ignored_layers=ignored_layers): + quant_config = self._resolve_online_quant_config("fp8", ignored_layers) + self.assertEqual(quant_config.ignored_layers, []) + @patch( "sglang.multimodal_gen.runtime.loader.transformer_load_utils.build_nvfp4_config_from_safetensors_list", return_value=None,