From 988c6e6aeb5664e6e5a117e19d821318f07149c4 Mon Sep 17 00:00:00 2001 From: Ziang Li Date: Wed, 5 Aug 2026 14:29:32 -0700 Subject: [PATCH] Pin online NVFP4 4over6 quantization settings (#33621) --- python/sglang/srt/model_loader/loader.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/model_loader/loader.py b/python/sglang/srt/model_loader/loader.py index d233612bd..2cc109864 100644 --- a/python/sglang/srt/model_loader/loader.py +++ b/python/sglang/srt/model_loader/loader.py @@ -839,7 +839,13 @@ class DefaultModelLoader(BaseModelLoader): if is_nvfp4_online: # Scope exact FP4 quantization math to load-time conversion only; # restore the original environment before serving starts. - with temp_set_env(FLASHINFER_DISABLE_FP4_QUANT_FAST_MATH="1"): + with temp_set_env( + FLASHINFER_DISABLE_FP4_QUANT_FAST_MATH="1", + FLASHINFER_NVFP4_4OVER6="1", + FLASHINFER_NVFP4_4OVER6_E4M3_USE_256="0", + FLASHINFER_NVFP4_4OVER6_ERR_MODE="MSE", + FLASHINFER_NVFP4_4OVER6_ERR_USE_FAST_MATH="1", + ): model.load_weights(weights) if target_device.type == "cuda": torch.cuda.synchronize()