From 5117bfbaeba655d0a4230b5d1fdb0c7e71f947f8 Mon Sep 17 00:00:00 2001 From: Kolade Fajimi <107228310+koladefaj@users.noreply.github.com> Date: Fri, 16 Oct 2026 05:36:02 +0100 Subject: [PATCH] [Fix] compressed-tensors: Serve NVFP4 W4A4 linears via FP4 Marlin when --fp4-gemm-backend marlin is set on SM100 (#44213) Signed-off-by: kolade fajimi Co-authored-by: Xiaoyu Zhang <1182563586@qq.com> --- .../compressed_tensors/compressed_tensors.py | 26 ++++++++++++++----- .../test_compressed_tensors_fp4_schemes.py | 23 ++++++++++++++++ 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors.py b/python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors.py index 9950075c4b..fbb155ec91 100644 --- a/python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors.py +++ b/python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors.py @@ -67,6 +67,7 @@ from sglang.srt.layers.quantization.compressed_tensors.utils import ( is_activation_quantization_format, should_ignore_layer, ) +from sglang.srt.layers.quantization.fp4_utils import get_fp4_gemm_runner_backend from sglang.srt.layers.quantization.fp8 import Fp8LinearMethod from sglang.srt.layers.quantization.kv_cache import BaseKVCacheMethod from sglang.srt.layers.quantization.unquant import ( @@ -757,15 +758,28 @@ class CompressedTensorsConfig(QuantizationConfig): is_fp4a4_nvfp4_supported = self._check_scheme_supported( CompressedTensorsW4A4Fp4.get_min_capability(), error=False ) - if is_fp4a4_nvfp4_supported: + # FP4 Marlin is weight-only and FlashInfer mm_fp4 has no marlin + # backend, so a marlin request takes the weight-only path on SM100 too. + if ( + is_fp4a4_nvfp4_supported + and not get_fp4_gemm_runner_backend().is_marlin() + ): return CompressedTensorsW4A4Fp4() # note: input_global_scale will be present for w4a4 checkpoints; # it is dropped post loading and activations stay unquantized. - logger.warning_once( - "NVFP4 w4a4 requires compute capability >= 10.0; serving " - "this checkpoint weight-only via FP4 Marlin instead. " - "Activation quantization is ignored." - ) + if is_fp4a4_nvfp4_supported: + logger.warning_once( + "--fp4-gemm-backend marlin is weight-only; serving NVFP4 " + "w4a4 dense linears weight-only via FP4 Marlin, so their " + "activation quantization is ignored. MoE layers follow " + "--moe-runner-backend and stay w4a4." + ) + else: + logger.warning_once( + "NVFP4 w4a4 requires compute capability >= 10.0; serving " + "this checkpoint weight-only via FP4 Marlin instead. " + "Activation quantization is ignored." + ) return CompressedTensorsW4A16Fp4(has_input_global_scale=True) # Activation-quantized float4 that is not the nvfp4 layout: the diff --git a/test/registered/unit/layers/quantization/test_compressed_tensors_fp4_schemes.py b/test/registered/unit/layers/quantization/test_compressed_tensors_fp4_schemes.py index e88102d52c..9417db3db5 100644 --- a/test/registered/unit/layers/quantization/test_compressed_tensors_fp4_schemes.py +++ b/test/registered/unit/layers/quantization/test_compressed_tensors_fp4_schemes.py @@ -158,6 +158,29 @@ class TestFp4SchemeSelection(CustomTestCase): _get_scheme("linear", _make_config(a16), SM90) warn.assert_not_called() + def test_marlin_backend_serves_w4a4_weight_only_on_blackwell(self): + """FlashInfer mm_fp4 has no marlin backend, so --fp4-gemm-backend marlin + must not select the native w4a4 linear on SM100. The MoE follows + --moe-runner-backend instead, which the warning has to say.""" + from sglang.srt.layers.quantization import fp4_utils + + logging.Logger.warning_once.cache_clear() + self.addCleanup(logging.Logger.warning_once.cache_clear) + with ( + mock.patch.object( + fp4_utils, "FP4_GEMM_RUNNER_BACKEND", Fp4GemmRunnerBackend.MARLIN + ), + self.assertLogs(SCHEME_LOGGER, level="WARNING") as captured, + ): + linear = _get_scheme("linear", _make_config("nvfp4"), SM100) + moe = _get_scheme("moe", _make_config("nvfp4"), SM100) + self.assertIsInstance(linear, CompressedTensorsW4A16Fp4) + self.assertTrue(linear.has_input_global_scale) + self.assertIsInstance(moe, CompressedTensorsW4A4Nvfp4MoE) + output = "\n".join(captured.output) + self.assertIn("marlin is weight-only", output) + self.assertIn("MoE layers follow --moe-runner-backend", output) + def test_fp4_weights_with_non_nvfp4_activations_raise(self): """Mismatched activation quantization must not be served weight-only.""" config = _make_config(