minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Update dsv3 quantization auto setting for sm100 (#12778)

Browse Source
This commit is contained in:
Ke Bao
2025-11-06 22:39:13 +08:00
committed by GitHub
parent fc84b0730c
commit cd135bfe30
1 changed files with 22 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/server_args.py
+22 -9
View File
@@ -912,19 +912,32 @@ class ServerArgs:
logger.info( logger.info(
"Enable FlashInfer AllReduce Fusion on sm100 for DeepseekV3ForCausalLM" "Enable FlashInfer AllReduce Fusion on sm100 for DeepseekV3ForCausalLM"
) )
if self.moe_a2a_backend == "none" and self.moe_runner_backend == "auto": quantization_config = getattr(hf_config, "quantization_config", None)
self.moe_runner_backend = "flashinfer_trtllm" quant_method = (
logger.info( quantization_config.get("quant_method")
"Use flashinfer_trtllm as MoE runner backend on sm100 for DeepseekV3ForCausalLM" if quantization_config is not None
) else None
if self.quantization is None: )
# Default DeepSeek V3/R1 native FP8 when not explicitly set, if self.quantization is None:
# Because we need this condition for an assertion in # Default DeepSeek V3/R1 native FP8 when not explicitly set,
# flashinfer_trtllm MoE runner backend. # Because we need this condition for an assertion in
# flashinfer_trtllm MoE runner backend.
if quant_method is None:
self.quantization = "fp8" self.quantization = "fp8"
logger.info( logger.info(
"Quantization not specified, default to fp8 for DeepSeek on sm100" "Quantization not specified, default to fp8 for DeepSeek on sm100"
) )
else:
self.quantization = quant_method
if (
self.moe_a2a_backend == "none"
and self.moe_runner_backend == "auto"
and self.quantization in ["fp8", "modelopt_fp8", "modelopt_fp4"]
):
self.moe_runner_backend = "flashinfer_trtllm"
logger.info(
"Use flashinfer_trtllm as MoE runner backend on sm100 for DeepseekV3ForCausalLM"
)
elif model_arch in ["GptOssForCausalLM"]: elif model_arch in ["GptOssForCausalLM"]:
if ( if (