[Refactor] nvfp4_online: Use explicit FlashInfer 4over6 config (#41745)
This commit is contained in:
@@ -7,6 +7,7 @@ from __future__ import annotations
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
import torch
|
||||
@@ -27,8 +28,32 @@ from sglang.srt.layers.quantization.utils import (
|
||||
is_layer_skipped,
|
||||
per_tensor_dequantize,
|
||||
)
|
||||
from sglang.srt.utils.common import temp_set_env
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_nvfp4_weight_quantization_lock = threading.Lock()
|
||||
_nvfp4_weight_quantization_users = 0
|
||||
_nvfp4_weight_quantization_env = ExitStack()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _nvfp4_weight_quantization_scope():
|
||||
"""Share the exact-math override across concurrent weight quantizers."""
|
||||
global _nvfp4_weight_quantization_users
|
||||
with _nvfp4_weight_quantization_lock:
|
||||
if _nvfp4_weight_quantization_users == 0:
|
||||
_nvfp4_weight_quantization_env.enter_context(
|
||||
temp_set_env(FLASHINFER_DISABLE_FP4_QUANT_FAST_MATH="1")
|
||||
)
|
||||
_nvfp4_weight_quantization_users += 1
|
||||
# Quantizer work runs outside the lock.
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
with _nvfp4_weight_quantization_lock:
|
||||
_nvfp4_weight_quantization_users -= 1
|
||||
if _nvfp4_weight_quantization_users == 0:
|
||||
_nvfp4_weight_quantization_env.close()
|
||||
|
||||
|
||||
class NvFp4OnlineConfig(ModelOptQuantConfig):
|
||||
@@ -254,7 +279,7 @@ class ModelOptNvFp4OnlineFusedMoEMethod(ModelOptNvFp4FusedMoEMethod):
|
||||
scale when multiple shards must share one global scale, for example the
|
||||
gated w1/w3 pair.
|
||||
"""
|
||||
from flashinfer import SfLayout, nvfp4_quantize
|
||||
from flashinfer import NVFP44Over6Config, SfLayout, nvfp4_quantize
|
||||
|
||||
if weight.ndim != 2:
|
||||
raise ValueError(
|
||||
@@ -273,6 +298,11 @@ class ModelOptNvFp4OnlineFusedMoEMethod(ModelOptNvFp4FusedMoEMethod):
|
||||
f"a multiple of 16, got shape {tuple(weight.shape)}."
|
||||
)
|
||||
|
||||
# Pin the weight recipe independently of the activation quantization
|
||||
# environment, including the E4M3 maximum used by the global scale.
|
||||
nvfp4_4over6_config = NVFP44Over6Config(
|
||||
e4m3_max=448, err_mode="MSE", err_use_fast_math=True
|
||||
)
|
||||
if weight_scale_2 is None:
|
||||
# weight_scale_2 is the NVFP4 decode scale. FlashInfer consumes its
|
||||
# reciprocal as the global encode scale, matching 448 * 6 / amax.
|
||||
@@ -282,13 +312,7 @@ class ModelOptNvFp4OnlineFusedMoEMethod(ModelOptNvFp4FusedMoEMethod):
|
||||
.amax()
|
||||
.to(device=weight.device, dtype=torch.float32)
|
||||
)
|
||||
e4m3_max = (
|
||||
256.0
|
||||
if envs.FLASHINFER_NVFP4_4OVER6.get()
|
||||
and envs.FLASHINFER_NVFP4_4OVER6_E4M3_USE_256.get()
|
||||
else float(torch.finfo(torch.float8_e4m3fn).max)
|
||||
)
|
||||
fp8_fp4_max = e4m3_max * 6.0
|
||||
fp8_fp4_max = nvfp4_4over6_config.e4m3_max * 6.0
|
||||
weight_scale_2 = torch.where(
|
||||
weight_amax > 0,
|
||||
weight_amax / fp8_fp4_max,
|
||||
@@ -298,12 +322,15 @@ class ModelOptNvFp4OnlineFusedMoEMethod(ModelOptNvFp4FusedMoEMethod):
|
||||
weight_scale_2 = weight_scale_2.to(
|
||||
device=weight.device, dtype=torch.float32
|
||||
)
|
||||
fp4_weight, weight_sf = nvfp4_quantize(
|
||||
weight.contiguous(),
|
||||
1.0 / weight_scale_2,
|
||||
sfLayout=SfLayout.layout_linear,
|
||||
backend="cute-dsl",
|
||||
)
|
||||
# FlashInfer still uses an environment flag for exact FP4 math.
|
||||
with _nvfp4_weight_quantization_scope():
|
||||
fp4_weight, weight_sf = nvfp4_quantize(
|
||||
weight.contiguous(),
|
||||
1.0 / weight_scale_2,
|
||||
sfLayout=SfLayout.layout_linear,
|
||||
backend="cute-dsl",
|
||||
nvfp4_4over6=nvfp4_4over6_config,
|
||||
)
|
||||
rows, cols = weight.shape
|
||||
weight_sf = weight_sf.view(torch.float8_e4m3fn).reshape(rows, cols // 16)
|
||||
return (
|
||||
|
||||
@@ -137,7 +137,6 @@ from sglang.srt.utils import (
|
||||
rank0_log,
|
||||
set_weight_attrs,
|
||||
)
|
||||
from sglang.srt.utils.common import temp_set_env
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.configs.device_config import DeviceConfig
|
||||
@@ -1059,22 +1058,10 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
)
|
||||
|
||||
weights = monokernel_weights_iterator(model=model, weights=weights)
|
||||
if is_nvfp4_online or is_modelopt_fp4_online:
|
||||
# Scope exact FP4 quantization math to load-time conversion only;
|
||||
# restore the original environment before serving starts.
|
||||
with temp_set_env(
|
||||
FLASHINFER_DISABLE_FP4_QUANT_FAST_MATH="1",
|
||||
FLASHINFER_NVFP4_4OVER6="1",
|
||||
FLASHINFER_NVFP4_4OVER6_E4M3_USE_256="0",
|
||||
FLASHINFER_NVFP4_4OVER6_ERR_MODE="MSE",
|
||||
FLASHINFER_NVFP4_4OVER6_ERR_USE_FAST_MATH="1",
|
||||
):
|
||||
model.load_weights(weights)
|
||||
if target_device.type == "cuda":
|
||||
torch.cuda.synchronize()
|
||||
torch.cuda.empty_cache()
|
||||
else:
|
||||
model.load_weights(weights)
|
||||
model.load_weights(weights)
|
||||
if (is_nvfp4_online or is_modelopt_fp4_online) and target_device.type == "cuda":
|
||||
torch.cuda.synchronize()
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
# Used in tests to verify memory savings when using online quantization.
|
||||
if is_cuda_alike():
|
||||
|
||||
Reference in New Issue
Block a user