minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[FIX] trtllm-moe-fp4-renorm for Qwen series models (#14350)

Browse Source
This commit is contained in:
Sam
2025-12-04 12:52:21 -08:00
committed by GitHub
parent d8faf2f361
commit 922756aaa1
2 changed files with 5 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/layers/moe/fused_moe_triton/layer.py
+5 -1
View File
@@ -1126,12 +1126,16 @@ class FlashInferFP4MoE(FusedMoE):
topk_config = topk_output.topk_config topk_config = topk_output.topk_config
hs_fp4, hs_scale_linear = self._quantize_hidden_states_fp4(hidden_states) hs_fp4, hs_scale_linear = self._quantize_hidden_states_fp4(hidden_states)
router_logits = router_logits.to(torch.float32)
routing_method_type = self.routing_method_type routing_method_type = self.routing_method_type
assert ( assert (
routing_method_type is not None routing_method_type is not None
), "flashinfer trtllm moe nvfp4 backend has not been adapted for the current moe layer, you can set routing_method_type (See definition of RoutingMethodType please) for the moe layer explicitly for a quick adaptation." ), "flashinfer trtllm moe nvfp4 backend has not been adapted for the current moe layer, you can set routing_method_type (See definition of RoutingMethodType please) for the moe layer explicitly for a quick adaptation."
# DeepSeekV3 style routing requires float32 router logits,
# see this PR for details: https://github.com/flashinfer-ai/flashinfer/commit/d84e1d560da0a27961c19ca788d96c19cb9dcfb6
if routing_method_type == RoutingMethodType.DeepSeekV3:
router_logits = router_logits.to(torch.float32)
correction_bias = ( correction_bias = (
None None
if topk_config.correction_bias is None if topk_config.correction_bias is None
python/sglang/srt/server_args.py
-1
View File
@@ -1261,7 +1261,6 @@ class ServerArgs:
) )
self.disable_overlap_schedule = True self.disable_overlap_schedule = True
if is_sm100_supported(): if is_sm100_supported():
self.attention_backend = "triton"
quantization_config = getattr(hf_config, "quantization_config", None) quantization_config = getattr(hf_config, "quantization_config", None)
quant_method = ( quant_method = (
quantization_config.get("quant_method") quantization_config.get("quant_method")