minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

fix: Llama 4 BF16 load on Blackwell (#12308)

Browse Source
This commit is contained in:
b8zhong
2025-10-28 18:59:01 -07:00
committed by GitHub
parent b48354c537
commit c143f416ce
1 changed files with 5 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/server_args.py
+1
View File
@@ -972,6 +972,7 @@ class ServerArgs:
"Use trtllm_mha as attention backend on sm100 for Llama4 model" "Use trtllm_mha as attention backend on sm100 for Llama4 model"
) )
if is_sm100_supported() and self.moe_runner_backend == "auto": if is_sm100_supported() and self.moe_runner_backend == "auto":
if self.quantization in {"fp8", "modelopt_fp8"}:
self.moe_runner_backend = "flashinfer_trtllm" self.moe_runner_backend = "flashinfer_trtllm"
logger.info( logger.info(
"Use flashinfer_trtllm as MoE runner backend on SM100 for Llama4" "Use flashinfer_trtllm as MoE runner backend on SM100 for Llama4"