[Refactor] nvfp4_online: Use explicit FlashInfer 4over6 config (#41745)

This commit is contained in:
Ziang Li
2026-10-16 20:56:10 -07:00
committed by GitHub
parent 9590dc80cc
commit 29a7e19de1
2 changed files with 45 additions and 31 deletions
@@ -7,6 +7,7 @@ from __future__ import annotations
import logging
import re
import threading
from contextlib import ExitStack, contextmanager
from typing import Any, Callable, Dict, List, Optional
import torch
@@ -27,8 +28,32 @@ from sglang.srt.layers.quantization.utils import (
is_layer_skipped,
per_tensor_dequantize,
)
from sglang.srt.utils.common import temp_set_env
logger = logging.getLogger(__name__)
_nvfp4_weight_quantization_lock = threading.Lock()
_nvfp4_weight_quantization_users = 0
_nvfp4_weight_quantization_env = ExitStack()
@contextmanager
def _nvfp4_weight_quantization_scope():
"""Share the exact-math override across concurrent weight quantizers."""
global _nvfp4_weight_quantization_users
with _nvfp4_weight_quantization_lock:
if _nvfp4_weight_quantization_users == 0:
_nvfp4_weight_quantization_env.enter_context(
temp_set_env(FLASHINFER_DISABLE_FP4_QUANT_FAST_MATH="1")
)
_nvfp4_weight_quantization_users += 1
# Quantizer work runs outside the lock.
try:
yield
finally:
with _nvfp4_weight_quantization_lock:
_nvfp4_weight_quantization_users -= 1
if _nvfp4_weight_quantization_users == 0:
_nvfp4_weight_quantization_env.close()
class NvFp4OnlineConfig(ModelOptQuantConfig):
@@ -254,7 +279,7 @@ class ModelOptNvFp4OnlineFusedMoEMethod(ModelOptNvFp4FusedMoEMethod):
scale when multiple shards must share one global scale, for example the
gated w1/w3 pair.
"""
from flashinfer import SfLayout, nvfp4_quantize
from flashinfer import NVFP44Over6Config, SfLayout, nvfp4_quantize
if weight.ndim != 2:
raise ValueError(
@@ -273,6 +298,11 @@ class ModelOptNvFp4OnlineFusedMoEMethod(ModelOptNvFp4FusedMoEMethod):
f"a multiple of 16, got shape {tuple(weight.shape)}."
)
# Pin the weight recipe independently of the activation quantization
# environment, including the E4M3 maximum used by the global scale.
nvfp4_4over6_config = NVFP44Over6Config(
e4m3_max=448, err_mode="MSE", err_use_fast_math=True
)
if weight_scale_2 is None:
# weight_scale_2 is the NVFP4 decode scale. FlashInfer consumes its
# reciprocal as the global encode scale, matching 448 * 6 / amax.
@@ -282,13 +312,7 @@ class ModelOptNvFp4OnlineFusedMoEMethod(ModelOptNvFp4FusedMoEMethod):
.amax()
.to(device=weight.device, dtype=torch.float32)
)
e4m3_max = (
256.0
if envs.FLASHINFER_NVFP4_4OVER6.get()
and envs.FLASHINFER_NVFP4_4OVER6_E4M3_USE_256.get()
else float(torch.finfo(torch.float8_e4m3fn).max)
)
fp8_fp4_max = e4m3_max * 6.0
fp8_fp4_max = nvfp4_4over6_config.e4m3_max * 6.0
weight_scale_2 = torch.where(
weight_amax > 0,
weight_amax / fp8_fp4_max,
@@ -298,12 +322,15 @@ class ModelOptNvFp4OnlineFusedMoEMethod(ModelOptNvFp4FusedMoEMethod):
weight_scale_2 = weight_scale_2.to(
device=weight.device, dtype=torch.float32
)
fp4_weight, weight_sf = nvfp4_quantize(
weight.contiguous(),
1.0 / weight_scale_2,
sfLayout=SfLayout.layout_linear,
backend="cute-dsl",
)
# FlashInfer still uses an environment flag for exact FP4 math.
with _nvfp4_weight_quantization_scope():
fp4_weight, weight_sf = nvfp4_quantize(
weight.contiguous(),
1.0 / weight_scale_2,
sfLayout=SfLayout.layout_linear,
backend="cute-dsl",
nvfp4_4over6=nvfp4_4over6_config,
)
rows, cols = weight.shape
weight_sf = weight_sf.view(torch.float8_e4m3fn).reshape(rows, cols // 16)
return (
+4 -17
View File
@@ -137,7 +137,6 @@ from sglang.srt.utils import (
rank0_log,
set_weight_attrs,
)
from sglang.srt.utils.common import temp_set_env
if TYPE_CHECKING:
from sglang.srt.configs.device_config import DeviceConfig
@@ -1059,22 +1058,10 @@ class DefaultModelLoader(BaseModelLoader):
)
weights = monokernel_weights_iterator(model=model, weights=weights)
if is_nvfp4_online or is_modelopt_fp4_online:
# Scope exact FP4 quantization math to load-time conversion only;
# restore the original environment before serving starts.
with temp_set_env(
FLASHINFER_DISABLE_FP4_QUANT_FAST_MATH="1",
FLASHINFER_NVFP4_4OVER6="1",
FLASHINFER_NVFP4_4OVER6_E4M3_USE_256="0",
FLASHINFER_NVFP4_4OVER6_ERR_MODE="MSE",
FLASHINFER_NVFP4_4OVER6_ERR_USE_FAST_MATH="1",
):
model.load_weights(weights)
if target_device.type == "cuda":
torch.cuda.synchronize()
torch.cuda.empty_cache()
else:
model.load_weights(weights)
model.load_weights(weights)
if (is_nvfp4_online or is_modelopt_fp4_online) and target_device.type == "cuda":
torch.cuda.synchronize()
torch.cuda.empty_cache()
# Used in tests to verify memory savings when using online quantization.
if is_cuda_alike():