From 2cf2920d070f88f0ec40cfdcb1c677291db46541 Mon Sep 17 00:00:00 2001 From: Ziang Li Date: Mon, 13 Jul 2026 07:37:46 -0700 Subject: [PATCH] [FlashInfer v0.6.13] Use CuTe DSL backend for FlashInfer per-token NVFP4 quantization (#28220) --- python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py | 1 + python/sglang/srt/layers/quantization/nvfp4_online.py | 2 +- python/sglang/srt/model_loader/loader.py | 5 +---- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py b/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py index 477263bb6..c814fe270 100644 --- a/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py +++ b/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py @@ -964,6 +964,7 @@ def fused_experts_none_to_flashinfer_trtllm_fp4( 1.0 / (e4m3_max * 6.0), sfLayout=SfLayout.layout_linear, per_token_activation=True, + backend="cute-dsl", ) seq_len, hidden_size = hidden_states.shape diff --git a/python/sglang/srt/layers/quantization/nvfp4_online.py b/python/sglang/srt/layers/quantization/nvfp4_online.py index 1caa48596..3d118f53d 100644 --- a/python/sglang/srt/layers/quantization/nvfp4_online.py +++ b/python/sglang/srt/layers/quantization/nvfp4_online.py @@ -225,7 +225,7 @@ class ModelOptNvFp4OnlineFusedMoEMethod(ModelOptNvFp4FusedMoEMethod): weight.contiguous(), 1.0 / weight_scale_2, sfLayout=SfLayout.layout_linear, - backend="cuda", + backend="cute-dsl", ) rows, cols = weight.shape weight_sf = weight_sf.view(torch.float8_e4m3fn).reshape(rows, cols // 16) diff --git a/python/sglang/srt/model_loader/loader.py b/python/sglang/srt/model_loader/loader.py index e9a8038b1..6dacc3ab3 100644 --- a/python/sglang/srt/model_loader/loader.py +++ b/python/sglang/srt/model_loader/loader.py @@ -817,10 +817,7 @@ class DefaultModelLoader(BaseModelLoader): if is_nvfp4_online: # Scope exact FP4 quantization math to load-time conversion only; # restore the original environment before serving starts. - with temp_set_env( - TRTLLM_DISABLE_FP4_QUANT_FAST_MATH="1", - FLASHINFER_DISABLE_FP4_QUANT_FAST_MATH="1", - ): + with temp_set_env(FLASHINFER_DISABLE_FP4_QUANT_FAST_MATH="1"): model.load_weights(weights) if target_device.type == "cuda": torch.cuda.synchronize()