[Fix] Guard DeepSeek NVFP4 shared-expert fusion for LoRA and FP4 backends (#42203)
This commit is contained in:
@@ -121,6 +121,10 @@ class Fp4GemmRunnerBackend(Enum):
|
||||
def is_flashinfer(self) -> bool:
|
||||
return self.value.startswith("flashinfer_")
|
||||
|
||||
def supports_swiglu_fusion(self) -> bool:
|
||||
"""Whether weight preparation builds the fused SwiGLU layout."""
|
||||
return not (self.is_marlin() or self.is_flashinfer_trtllm())
|
||||
|
||||
def get_flashinfer_backend(self) -> str:
|
||||
"""Get the backend string to pass to FlashInfer's mm_fp4 API.
|
||||
|
||||
|
||||
@@ -197,6 +197,7 @@ from sglang.srt.runtime_context import (
|
||||
get_device,
|
||||
get_exec,
|
||||
get_forward,
|
||||
get_lora,
|
||||
get_model,
|
||||
get_parallel,
|
||||
get_platform,
|
||||
@@ -796,6 +797,9 @@ class DeepseekV2MoE(nn.Module):
|
||||
# Flags must be set before weight load so
|
||||
# process_weights_after_loading sees them and builds the
|
||||
# [Up, Gate]-interleaved weight + scale.
|
||||
from sglang.srt.layers.quantization.fp4_utils import (
|
||||
get_fp4_gemm_runner_backend,
|
||||
)
|
||||
from sglang.srt.layers.quantization.modelopt_quant import (
|
||||
ModelOptFp4LinearMethod,
|
||||
)
|
||||
@@ -803,11 +807,14 @@ class DeepseekV2MoE(nn.Module):
|
||||
fc1_n = self.shared_experts.gate_up_proj.output_size_per_partition
|
||||
if (
|
||||
get_platform().is_sm100
|
||||
# The fused kernel bypasses the LoRA wrappers.
|
||||
and not (get_lora().enable_lora or get_lora().lora_paths)
|
||||
and isinstance(
|
||||
self.shared_experts.gate_up_proj.quant_method,
|
||||
ModelOptFp4LinearMethod,
|
||||
)
|
||||
and self.shared_experts.gate_up_proj.quant_method.quant_mode == "w4a4"
|
||||
and get_fp4_gemm_runner_backend().supports_swiglu_fusion()
|
||||
and isinstance(
|
||||
self.shared_experts.down_proj.quant_method,
|
||||
ModelOptFp4LinearMethod,
|
||||
|
||||
Reference in New Issue
Block a user