minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Fix] A followup fix for TRTLLM BF16 MoE (#15303)

Browse Source
This commit is contained in:
Sam (Kesen Li)
2026-02-26 17:14:20 -08:00
committed by GitHub
parent f0d78f2e20
commit 5194eef88b
1 changed files with 4 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/server_args.py
+4 -3
View File
@@ -2367,8 +2367,9 @@ class ServerArgs:
): ):
self.speculative_draft_model_revision = "main" self.speculative_draft_model_revision = "main"
# Avoid using flashinfer_trtllm for speculative MoE runner backend by default # FlashInfer trtllm moe bf16 only support RenormalizeNaive routing method and Deepseek routing method
# TODO: Remove this block after verifying no accuracy regression with flashinfer_trtllm speculative backend # It is hard to tell the routing method in draft model, and the moe layer in draft model is not the bottleneck among
# end to end, so we just avoid using trtllm_moe for speculative decoding.
from sglang.srt.layers.moe.utils import MoeRunnerBackend from sglang.srt.layers.moe.utils import MoeRunnerBackend
if self.speculative_moe_runner_backend is None: if self.speculative_moe_runner_backend is None:
@@ -2380,7 +2381,7 @@ class ServerArgs:
else: else:
assert not MoeRunnerBackend( assert not MoeRunnerBackend(
self.speculative_moe_runner_backend self.speculative_moe_runner_backend
).is_flashinfer_trtllm(), "Currently speculative MoE runner backend cannot be flashinfer_trtllm for risk in some draft models." ).is_flashinfer_trtllm(), "Currently speculative MoE runner backend doesn't support flashinfer_trtllm, please use triton or auto backend for speculative moe runner instead."
if self.speculative_algorithm == "NEXTN": if self.speculative_algorithm == "NEXTN":
self.speculative_algorithm = "EAGLE" self.speculative_algorithm = "EAGLE"