minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Remove QServe and FBGEMM FP8 quantization (#31109)

Browse Source
This commit is contained in:
Brayden Zhong
2026-07-17 17:10:34 -07:00
committed by GitHub
parent f926c30c57
commit 238b2b2c9c
18 changed files with 1 additions and 2566 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/multimodal_gen/runtime/layers/linear.py
-1
View File
@@ -53,7 +53,6 @@ WEIGHT_LOADER_V2_SUPPORTED = [
"GPTQMarlin24LinearMethod",
"TPUInt8LinearMethod",
"GPTQLinearMethod",
"FBGEMMFp8LinearMethod",
"ModelOptFp8LinearMethod",
"ModelOptFp4LinearMethod",
"ComfyUIFp4LinearMethod",
python/sglang/srt/configs/model_config.py
-3
View File
@@ -1329,7 +1329,6 @@ class ModelConfig:
"fp8",
"compressed_tensors",
"compressed-tensors",
"fbgemm_fp8",
"w8a8_fp8",
"petit_nvfp4",
"quark",
@@ -1350,14 +1349,12 @@ class ModelConfig:
"gptq_marlin_24",
"gptq_marlin",
"awq_marlin",
"fbgemm_fp8",
"compressed_tensors",
"compressed-tensors",
"experts_int8",
"w8a8_int8",
"w8a8_fp8",
"moe_wna16",
"qoq",
"w4afp8",
"petit_nvfp4",
"quark",
python/sglang/srt/layers/linear.py
-1
View File
@@ -66,7 +66,6 @@ WEIGHT_LOADER_V2_SUPPORTED = [
"GPTQMarlin24LinearMethod",
"TPUInt8LinearMethod",
"GPTQLinearMethod",
"FBGEMMFp8LinearMethod",
"GPTQLinearAscendMethod",
"GPTQLinearIntelAMXMethod",
"GPTQMoEAscendMethod",
python/sglang/srt/layers/quantization/__init__.py
-4
View File
@@ -27,7 +27,6 @@ from sglang.srt.layers.quantization.compressed_tensors.compressed_tensors import
CompressedTensorsConfig,
)
from sglang.srt.layers.quantization.fp8 import Fp8Config
from sglang.srt.layers.quantization.fpgemm_fp8 import FBGEMMFp8Config
from sglang.srt.layers.quantization.gguf import GGUFConfig
from sglang.srt.layers.quantization.gptq import (
CPUGPTQConfig,
@@ -49,7 +48,6 @@ from sglang.srt.layers.quantization.npu_mxfp4 import Mxfp4W4A8Config
from sglang.srt.layers.quantization.npu_mxfp4_w4a4 import Mxfp4W4A4Config
from sglang.srt.layers.quantization.nvfp4_online import NvFp4OnlineConfig
from sglang.srt.layers.quantization.petit import PetitNvFp4Config
from sglang.srt.layers.quantization.qoq import QoQConfig
from sglang.srt.layers.quantization.quark.quark import QuarkConfig
from sglang.srt.layers.quantization.quark_int4fp8_moe import QuarkInt4Fp8Config
from sglang.srt.layers.quantization.w4afp8 import W4AFp8Config
@@ -91,10 +89,8 @@ BASE_QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
"gptq_marlin": GPTQMarlinConfig,
"moe_wna16": MoeWNA16Config,
"compressed-tensors": CompressedTensorsConfig,
"qoq": QoQConfig,
"w4afp8": W4AFp8Config,
"petit_nvfp4": PetitNvFp4Config,
"fbgemm_fp8": FBGEMMFp8Config,
"quark": QuarkConfig,
"quark_mxfp4": QuarkConfig,
"auto-round": AutoRoundConfig,
python/sglang/srt/layers/quantization/fpgemm_fp8.py
-202
View File
@@ -1,202 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
import logging
from typing import Any, List, Optional
import torch
from torch.nn import Module
from torch.nn.parameter import Parameter
from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
from sglang.srt.layers.linear import LinearBase
from sglang.srt.layers.parameter import ChannelQuantScaleParameter, ModelWeightParameter
from sglang.srt.layers.quantization.base_config import (
LinearMethodBase,
QuantizationConfig,
QuantizeMethodBase,
)
from sglang.srt.layers.quantization.fp8_utils import (
apply_fp8_linear,
can_auto_enable_marlin_fp8,
cutlass_fp8_supported,
normalize_e4m3fn_to_e4m3fnuz,
)
from sglang.srt.layers.quantization.marlin_utils_fp8 import (
apply_fp8_marlin_linear,
prepare_fp8_layer_for_marlin,
)
from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod
from sglang.srt.layers.quantization.utils import is_layer_skipped
from sglang.srt.utils import get_bool_env_var, is_cuda
_is_cuda = is_cuda()
_is_fp8_fnuz = is_fp8_fnuz()
logger = logging.getLogger(__name__)
class FBGEMMFp8Config(QuantizationConfig):
"""Config class for FBGEMM Fp8."""
def __init__(self, ignore_list: list[str], input_scale_ub: float):
super().__init__()
self.ignore_list = ignore_list if ignore_list else []
self.input_scale_ub = input_scale_ub
# For GPUs that lack FP8 hardware suspport, we can leverage the Marlin
# kernel for fast weight-only FP8 quantization
# self.use_marlin = not marlin_fp8_supported()
self.use_marlin = False
if _is_cuda:
force_marlin = get_bool_env_var("SGLANG_FORCE_FP8_MARLIN")
auto_enable = can_auto_enable_marlin_fp8()
self.use_marlin = force_marlin or auto_enable
@classmethod
def get_name(cls) -> str:
return "fbgemm_fp8"
@classmethod
def get_supported_act_dtypes(cls) -> list[torch.dtype]:
return [torch.bfloat16, torch.float16]
@classmethod
def get_min_capability(cls) -> int:
return 80
@classmethod
def get_config_filenames(cls) -> list[str]:
return []
@classmethod
def from_config(cls, config: dict[str, Any]) -> FBGEMMFp8Config:
ignore_list = cls.get_from_keys(config, ["modules_to_not_convert"])
input_scale_ub = cls.get_from_keys(config, ["activation_scale_ub"])
return cls(ignore_list=ignore_list, input_scale_ub=input_scale_ub)
def get_quant_method(
self, layer: torch.nn.Module, prefix: str
) -> Optional[QuantizeMethodBase]:
if isinstance(layer, LinearBase):
if is_layer_skipped(
prefix=prefix,
ignored_layers=self.ignore_list,
fused_mapping=self.packed_modules_mapping,
):
return UnquantizedLinearMethod()
return FBGEMMFp8LinearMethod(self)
return None
def get_scaled_act_names(self) -> List[str]:
return []
class FBGEMMFp8LinearMethod(LinearMethodBase):
def __init__(self, quant_config: FBGEMMFp8Config):
self.quant_config = quant_config
# self.fp8_linear = Fp8LinearOp(
# act_quant_static=False, act_quant_group_shape=GroupShape.PER_TOKEN)
self.out_dtype = torch.get_default_dtype()
self.cutlass_fp8_supported = cutlass_fp8_supported()
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
):
# maybe_create_device_identity()
weight_loader = extra_weight_attrs.get("weight_loader")
del input_size, output_size
output_size_per_partition = sum(output_partition_sizes)
layer.logical_widths = output_partition_sizes
layer.input_size_per_partition = input_size_per_partition
layer.output_size_per_partition = output_size_per_partition
layer.orig_dtype = params_dtype
# WEIGHT
weight = ModelWeightParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition,
dtype=torch.float8_e4m3fn,
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("weight", weight)
# WEIGHT SCALE
weight_scale = ChannelQuantScaleParameter(
data=torch.empty((sum(output_partition_sizes), 1), dtype=torch.float32),
output_dim=0,
weight_loader=weight_loader,
)
weight_scale[:] = torch.finfo(torch.float32).min
layer.register_parameter("weight_scale", weight_scale)
# INPUT SCALE UPPER BOUND
input_scale_ub = torch.nn.Parameter(
torch.tensor((self.quant_config.input_scale_ub), dtype=torch.float32),
requires_grad=False,
)
layer.input_scale_ub = input_scale_ub
def process_weights_after_loading(self, layer: Module) -> None:
# required by torch.compile
layer.weight_scale = Parameter(layer.weight_scale.data, requires_grad=False)
layer.weight = Parameter(layer.weight.data, requires_grad=False)
weight = layer.weight
if _is_fp8_fnuz:
weight, weight_scale, input_scale = normalize_e4m3fn_to_e4m3fnuz(
weight=weight, weight_scale=layer.weight_scale, input_scale=None
)
if input_scale is not None:
layer.input_scale = Parameter(input_scale, requires_grad=False)
layer.weight_scale = Parameter(weight_scale, requires_grad=False)
layer.weight = Parameter(weight.t(), requires_grad=False)
if self.quant_config.use_marlin:
prepare_fp8_layer_for_marlin(layer)
# Activations not quantized for marlin.
del layer.input_scale_ub
def apply(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: Optional[torch.Tensor] = None,
) -> torch.Tensor:
if self.quant_config.use_marlin:
return apply_fp8_marlin_linear(
input=x,
weight=layer.weight,
weight_scale=layer.weight_scale,
workspace=layer.workspace,
size_n=layer.output_size_per_partition,
size_k=layer.input_size_per_partition,
bias=bias,
)
return apply_fp8_linear(
input=x,
weight=layer.weight,
weight_scale=layer.weight_scale,
input_scale=None,
input_scale_ub=layer.input_scale_ub,
bias=bias,
cutlass_fp8_supported=self.cutlass_fp8_supported,
use_per_token_if_dynamic=False,
)
python/sglang/srt/layers/quantization/qoq.py
-245
View File
@@ -1,245 +0,0 @@
from __future__ import annotations
from typing import Any, Dict, List, Optional
import torch
from torch.nn.parameter import Parameter
from sglang.kernels.ops.quantization.int8_kernel import per_token_quant_int8
from sglang.srt.layers.parameter import (
ChannelQuantScaleParameter,
GroupQuantScaleParameter,
ModelWeightParameter,
)
from sglang.srt.layers.quantization.base_config import (
LinearMethodBase,
QuantizationConfig,
QuantizeMethodBase,
)
from sglang.srt.utils import is_cuda
_is_cuda = is_cuda()
if _is_cuda:
from sgl_kernel import qserve_w4a8_per_chn_gemm, qserve_w4a8_per_group_gemm
QoQ_SUPPORTED_WEIGHT_BITS = [4]
QoQ_SUPPORTED_GROUP_SIZES = [-1, 128]
class QoQConfig(QuantizationConfig):
"""Config class for QoQ Quantization.
- Weight: static, per-channel/group, asymmetric
- Activation: dynamic, per-token, symmetric
Reference: https://arxiv.org/abs/2405.04532
https://github.com/mit-han-lab/omniserve
"""
def __init__(self, weight_bits: int, group_size: int) -> None:
self.weight_bits = weight_bits
self.group_size = group_size
# Verify
if self.weight_bits not in QoQ_SUPPORTED_WEIGHT_BITS:
raise ValueError(
f"QoQ does not support weight_bits = {self.weight_bits}. "
f"Only weight_bits = {QoQ_SUPPORTED_WEIGHT_BITS} "
"are supported."
)
if self.group_size not in QoQ_SUPPORTED_GROUP_SIZES:
raise ValueError(
f"QoQ does not support group_size = {self.group_size}. "
f"Only group_sizes = {QoQ_SUPPORTED_GROUP_SIZES} "
"are supported."
)
# 4 bits packed into 8 bit datatype.
self.pack_factor = 8 // self.weight_bits
def __repr__(self) -> str:
return "QoQConfig(weight_bits={}, group_size={})".format(
self.weight_bits, self.group_size
)
@classmethod
def get_supported_act_dtypes(cls) -> List[torch.dtype]:
return [torch.float16]
@classmethod
def get_min_capability(cls) -> int:
return 80
@classmethod
def get_name(cls) -> str:
return "qoq"
@classmethod
def get_config_filenames(cls) -> List[str]:
"""List of filenames to search for in the model directory."""
return [
"quant_config.json",
"quantize_config.json",
]
@classmethod
def from_config(cls, config: Dict[str, Any]) -> QoQConfig:
weight_bits = cls.get_from_keys(config, ["wbits"])
group_size = cls.get_from_keys(config, ["group_size"])
return cls(weight_bits, group_size)
def get_quant_method(
self,
layer: torch.nn.Module,
prefix: str,
) -> Optional[QuantizeMethodBase]:
from sglang.srt.layers.linear import LinearBase
if isinstance(layer, LinearBase):
return QoQLinearMethod(self)
return None
def get_scaled_act_names(self) -> List[str]:
return []
class QoQLinearMethod(LinearMethodBase):
"""Linear method for QoQ.
Args:
quant_config: The QoQ quantization config.
"""
def __init__(self, quant_config: QoQConfig):
self.quant_config = quant_config
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: List[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
):
weight_loader = extra_weight_attrs.get("weight_loader")
# Validate output_size_per_partition
output_size_per_partition = sum(output_partition_sizes)
if output_size_per_partition % 32 != 0:
raise ValueError(
f"Weight output_size_per_partition = "
f"{output_size_per_partition} is not divisible by 32."
)
# Validate input_size_per_partition
if input_size_per_partition % self.quant_config.pack_factor != 0:
raise ValueError(
f"Weight input_size_per_partition = "
f"{input_size_per_partition} is not divisible by "
f"pack_factor = {self.quant_config.pack_factor}."
)
if (
self.quant_config.group_size != -1
and input_size_per_partition % self.quant_config.group_size != 0
):
raise ValueError(
f"Weight input_size_per_partition = "
f"{input_size_per_partition} is not divisible by "
f"group_size = {self.quant_config.group_size}."
)
qweight = ModelWeightParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition // self.quant_config.pack_factor,
dtype=torch.int8,
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("qweight", qweight)
s1_scales = ChannelQuantScaleParameter(
data=torch.empty(output_size_per_partition, dtype=torch.float16),
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("s1_scales", s1_scales)
if self.quant_config.group_size == -1:
s1_szeros = ChannelQuantScaleParameter(
data=torch.empty(output_size_per_partition, dtype=torch.float16),
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("s1_szeros", s1_szeros)
else:
s2_scales = GroupQuantScaleParameter(
data=torch.empty(
(
input_size_per_partition // self.quant_config.group_size,
output_size_per_partition,
),
dtype=torch.int8,
),
input_dim=0,
output_dim=1,
weight_loader=weight_loader,
)
layer.register_parameter("s2_scales", s2_scales)
s2_zeros = GroupQuantScaleParameter(
data=torch.empty(
(
input_size_per_partition // self.quant_config.group_size,
output_size_per_partition,
),
dtype=torch.int8,
),
input_dim=0,
output_dim=1,
weight_loader=weight_loader,
)
layer.register_parameter("s2_zeros", s2_zeros)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.qweight = Parameter(layer.qweight.data, requires_grad=False)
layer.s1_scales = Parameter(layer.s1_scales.data, requires_grad=False)
if self.quant_config.group_size == -1:
layer.s1_szeros = Parameter(layer.s1_szeros.data, requires_grad=False)
else:
layer.s2_scales = Parameter(layer.s2_scales.data, requires_grad=False)
layer.s2_zeros = Parameter(layer.s2_zeros.data, requires_grad=False)
def apply(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: Optional[torch.Tensor] = None,
):
assert x.dtype == torch.float16, "QoQ only supports float16 input now"
if self.quant_config.group_size == -1:
x_q, x_scale, x_sum = per_token_quant_int8(
x, scale_dtype=x.dtype, cal_sum=True
)
out = qserve_w4a8_per_chn_gemm(
x_q, layer.qweight, layer.s1_scales, x_scale, layer.s1_szeros, x_sum
)
else:
x_q, x_scale = per_token_quant_int8(x, scale_dtype=x.dtype)
out = qserve_w4a8_per_group_gemm(
x_q,
layer.qweight,
layer.s2_zeros,
layer.s2_scales,
layer.s1_scales,
x_scale,
)
if bias is not None:
out = out + bias
return out
python/sglang/srt/server_args.py
-1
View File
@@ -169,7 +169,6 @@ QUANTIZATION_CHOICES = [
"w8a8_int8", # mentioned in quantization.md documentation, supporting compressed-tensors quant_method.
"w8a8_fp8", # mentioned in quantization.md documentation, supporting compressed-tensors quant_method.
"moe_wna16", # custom loading logic for gptq/awq checkpoints (likely untested/unused)
"qoq",
"w4afp8",
"mxfp4", # MOE-only.
"auto-round",