diff --git a/python/sglang/srt/layers/quantization/fp4_utils.py b/python/sglang/srt/layers/quantization/fp4_utils.py index 4f6ef82652..0cdafd64bb 100644 --- a/python/sglang/srt/layers/quantization/fp4_utils.py +++ b/python/sglang/srt/layers/quantization/fp4_utils.py @@ -121,6 +121,10 @@ class Fp4GemmRunnerBackend(Enum): def is_flashinfer(self) -> bool: return self.value.startswith("flashinfer_") + def supports_swiglu_fusion(self) -> bool: + """Whether weight preparation builds the fused SwiGLU layout.""" + return not (self.is_marlin() or self.is_flashinfer_trtllm()) + def get_flashinfer_backend(self) -> str: """Get the backend string to pass to FlashInfer's mm_fp4 API. diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index b83c315ed6..75d1277d12 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -197,6 +197,7 @@ from sglang.srt.runtime_context import ( get_device, get_exec, get_forward, + get_lora, get_model, get_parallel, get_platform, @@ -796,6 +797,9 @@ class DeepseekV2MoE(nn.Module): # Flags must be set before weight load so # process_weights_after_loading sees them and builds the # [Up, Gate]-interleaved weight + scale. + from sglang.srt.layers.quantization.fp4_utils import ( + get_fp4_gemm_runner_backend, + ) from sglang.srt.layers.quantization.modelopt_quant import ( ModelOptFp4LinearMethod, ) @@ -803,11 +807,14 @@ class DeepseekV2MoE(nn.Module): fc1_n = self.shared_experts.gate_up_proj.output_size_per_partition if ( get_platform().is_sm100 + # The fused kernel bypasses the LoRA wrappers. + and not (get_lora().enable_lora or get_lora().lora_paths) and isinstance( self.shared_experts.gate_up_proj.quant_method, ModelOptFp4LinearMethod, ) and self.shared_experts.gate_up_proj.quant_method.quant_mode == "w4a4" + and get_fp4_gemm_runner_backend().supports_swiglu_fusion() and isinstance( self.shared_experts.down_proj.quant_method, ModelOptFp4LinearMethod, diff --git a/test/registered/unit/layers/quantization/test_modelopt_nvfp4.py b/test/registered/unit/layers/quantization/test_modelopt_nvfp4.py index 8aeed7ae59..dcc443f18b 100644 --- a/test/registered/unit/layers/quantization/test_modelopt_nvfp4.py +++ b/test/registered/unit/layers/quantization/test_modelopt_nvfp4.py @@ -6,6 +6,7 @@ import torch.nn as nn from sglang.srt.layers.linear import MergedColumnParallelLinear, QKVParallelLinear from sglang.srt.layers.parameter import PerTensorScaleParameter +from sglang.srt.layers.quantization.fp4_utils import Fp4GemmRunnerBackend from sglang.srt.layers.quantization.modelopt_quant import ( ModelOptFp4Config, ModelOptFp4LinearMethod, @@ -18,6 +19,17 @@ register_cpu_ci(est_time=11, suite="base-a-test-cpu") class TestModelOptNvfp4(CustomTestCase): + def test_only_interleaving_backends_support_the_fusion(self): + for backend, expected in ( + (Fp4GemmRunnerBackend.FLASHINFER_CUTEDSL, True), + (Fp4GemmRunnerBackend.FLASHINFER_CUTLASS, True), + (Fp4GemmRunnerBackend.FLASHINFER_CUDNN, True), + (Fp4GemmRunnerBackend.FLASHINFER_TRTLLM, False), + (Fp4GemmRunnerBackend.MARLIN, False), + ): + with self.subTest(backend=backend): + self.assertEqual(backend.supports_swiglu_fusion(), expected) + def _make_layer(self): return MergedColumnParallelLinear( input_size=16,