minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Pin online NVFP4 4over6 quantization settings (#33621)

Browse Source
This commit is contained in:
Ziang Li
2026-08-05 21:29:32 +00:00
committed by GitHub
parent a14c870886
commit 988c6e6aeb
1 changed files with 7 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/model_loader/loader.py
+7 -1
View File
@@ -839,7 +839,13 @@ class DefaultModelLoader(BaseModelLoader):
if is_nvfp4_online: if is_nvfp4_online:
# Scope exact FP4 quantization math to load-time conversion only; # Scope exact FP4 quantization math to load-time conversion only;
# restore the original environment before serving starts. # restore the original environment before serving starts.
with temp_set_env(FLASHINFER_DISABLE_FP4_QUANT_FAST_MATH="1"): with temp_set_env(
FLASHINFER_DISABLE_FP4_QUANT_FAST_MATH="1",
FLASHINFER_NVFP4_4OVER6="1",
FLASHINFER_NVFP4_4OVER6_E4M3_USE_256="0",
FLASHINFER_NVFP4_4OVER6_ERR_MODE="MSE",
FLASHINFER_NVFP4_4OVER6_ERR_USE_FAST_MATH="1",
):
model.load_weights(weights) model.load_weights(weights)
if target_device.type == "cuda": if target_device.type == "cuda":
torch.cuda.synchronize() torch.cuda.synchronize()