diff --git a/python/sglang/srt/layers/quantization/nvfp4_online.py b/python/sglang/srt/layers/quantization/nvfp4_online.py index da44a3dabe..b9aaf212a5 100644 --- a/python/sglang/srt/layers/quantization/nvfp4_online.py +++ b/python/sglang/srt/layers/quantization/nvfp4_online.py @@ -7,6 +7,7 @@ from __future__ import annotations import logging import re import threading +from contextlib import ExitStack, contextmanager from typing import Any, Callable, Dict, List, Optional import torch @@ -27,8 +28,32 @@ from sglang.srt.layers.quantization.utils import ( is_layer_skipped, per_tensor_dequantize, ) +from sglang.srt.utils.common import temp_set_env logger = logging.getLogger(__name__) +_nvfp4_weight_quantization_lock = threading.Lock() +_nvfp4_weight_quantization_users = 0 +_nvfp4_weight_quantization_env = ExitStack() + + +@contextmanager +def _nvfp4_weight_quantization_scope(): + """Share the exact-math override across concurrent weight quantizers.""" + global _nvfp4_weight_quantization_users + with _nvfp4_weight_quantization_lock: + if _nvfp4_weight_quantization_users == 0: + _nvfp4_weight_quantization_env.enter_context( + temp_set_env(FLASHINFER_DISABLE_FP4_QUANT_FAST_MATH="1") + ) + _nvfp4_weight_quantization_users += 1 + # Quantizer work runs outside the lock. + try: + yield + finally: + with _nvfp4_weight_quantization_lock: + _nvfp4_weight_quantization_users -= 1 + if _nvfp4_weight_quantization_users == 0: + _nvfp4_weight_quantization_env.close() class NvFp4OnlineConfig(ModelOptQuantConfig): @@ -254,7 +279,7 @@ class ModelOptNvFp4OnlineFusedMoEMethod(ModelOptNvFp4FusedMoEMethod): scale when multiple shards must share one global scale, for example the gated w1/w3 pair. """ - from flashinfer import SfLayout, nvfp4_quantize + from flashinfer import NVFP44Over6Config, SfLayout, nvfp4_quantize if weight.ndim != 2: raise ValueError( @@ -273,6 +298,11 @@ class ModelOptNvFp4OnlineFusedMoEMethod(ModelOptNvFp4FusedMoEMethod): f"a multiple of 16, got shape {tuple(weight.shape)}." ) + # Pin the weight recipe independently of the activation quantization + # environment, including the E4M3 maximum used by the global scale. + nvfp4_4over6_config = NVFP44Over6Config( + e4m3_max=448, err_mode="MSE", err_use_fast_math=True + ) if weight_scale_2 is None: # weight_scale_2 is the NVFP4 decode scale. FlashInfer consumes its # reciprocal as the global encode scale, matching 448 * 6 / amax. @@ -282,13 +312,7 @@ class ModelOptNvFp4OnlineFusedMoEMethod(ModelOptNvFp4FusedMoEMethod): .amax() .to(device=weight.device, dtype=torch.float32) ) - e4m3_max = ( - 256.0 - if envs.FLASHINFER_NVFP4_4OVER6.get() - and envs.FLASHINFER_NVFP4_4OVER6_E4M3_USE_256.get() - else float(torch.finfo(torch.float8_e4m3fn).max) - ) - fp8_fp4_max = e4m3_max * 6.0 + fp8_fp4_max = nvfp4_4over6_config.e4m3_max * 6.0 weight_scale_2 = torch.where( weight_amax > 0, weight_amax / fp8_fp4_max, @@ -298,12 +322,15 @@ class ModelOptNvFp4OnlineFusedMoEMethod(ModelOptNvFp4FusedMoEMethod): weight_scale_2 = weight_scale_2.to( device=weight.device, dtype=torch.float32 ) - fp4_weight, weight_sf = nvfp4_quantize( - weight.contiguous(), - 1.0 / weight_scale_2, - sfLayout=SfLayout.layout_linear, - backend="cute-dsl", - ) + # FlashInfer still uses an environment flag for exact FP4 math. + with _nvfp4_weight_quantization_scope(): + fp4_weight, weight_sf = nvfp4_quantize( + weight.contiguous(), + 1.0 / weight_scale_2, + sfLayout=SfLayout.layout_linear, + backend="cute-dsl", + nvfp4_4over6=nvfp4_4over6_config, + ) rows, cols = weight.shape weight_sf = weight_sf.view(torch.float8_e4m3fn).reshape(rows, cols // 16) return ( diff --git a/python/sglang/srt/model_loader/loader.py b/python/sglang/srt/model_loader/loader.py index 8c635040bb..a7130ec8d5 100644 --- a/python/sglang/srt/model_loader/loader.py +++ b/python/sglang/srt/model_loader/loader.py @@ -137,7 +137,6 @@ from sglang.srt.utils import ( rank0_log, set_weight_attrs, ) -from sglang.srt.utils.common import temp_set_env if TYPE_CHECKING: from sglang.srt.configs.device_config import DeviceConfig @@ -1059,22 +1058,10 @@ class DefaultModelLoader(BaseModelLoader): ) weights = monokernel_weights_iterator(model=model, weights=weights) - if is_nvfp4_online or is_modelopt_fp4_online: - # Scope exact FP4 quantization math to load-time conversion only; - # restore the original environment before serving starts. - with temp_set_env( - FLASHINFER_DISABLE_FP4_QUANT_FAST_MATH="1", - FLASHINFER_NVFP4_4OVER6="1", - FLASHINFER_NVFP4_4OVER6_E4M3_USE_256="0", - FLASHINFER_NVFP4_4OVER6_ERR_MODE="MSE", - FLASHINFER_NVFP4_4OVER6_ERR_USE_FAST_MATH="1", - ): - model.load_weights(weights) - if target_device.type == "cuda": - torch.cuda.synchronize() - torch.cuda.empty_cache() - else: - model.load_weights(weights) + model.load_weights(weights) + if (is_nvfp4_online or is_modelopt_fp4_online) and target_device.type == "cuda": + torch.cuda.synchronize() + torch.cuda.empty_cache() # Used in tests to verify memory savings when using online quantization. if is_cuda_alike():