Pin online NVFP4 4over6 quantization settings (#33621)

2026-08-05 21:29:32 +00:00
parent a14c870886
commit 988c6e6aeb
1 changed files with 7 additions and 1 deletions
@@ -839,7 +839,13 @@ class DefaultModelLoader(BaseModelLoader):
        if is_nvfp4_online:
            # Scope exact FP4 quantization math to load-time conversion only;
            # restore the original environment before serving starts.
-            with temp_set_env(FLASHINFER_DISABLE_FP4_QUANT_FAST_MATH="1"):
+            with temp_set_env(
+                FLASHINFER_DISABLE_FP4_QUANT_FAST_MATH="1",
+                FLASHINFER_NVFP4_4OVER6="1",
+                FLASHINFER_NVFP4_4OVER6_E4M3_USE_256="0",
+                FLASHINFER_NVFP4_4OVER6_ERR_MODE="MSE",
+                FLASHINFER_NVFP4_4OVER6_ERR_USE_FAST_MATH="1",
+            ):
                model.load_weights(weights)
            if target_device.type == "cuda":
                torch.cuda.synchronize()