Add Intel Quantization Support in SGLang (#18139)

Signed-off-by: Mengni Wang <mengni.wang@intel.com>
Signed-off-by: WeiweiZhang1 <weiwei1.zhang@intel.com>
Co-authored-by: Peng Zhang <aniz1905@gmail.com>
Co-authored-by: Ma Mingfei <mingfei.ma@intel.com>
Co-authored-by: Weiwei <weiwei1.zhang@intel.com>

docs_new/docs/advanced_features/quantization.mdx

@@ -385,6 +385,34 @@ auto-round \
     --output_dir ./tmp_autoround
 ```
 
+- SGlang API Usage (CPU/CUDA)
+
+```python Example
+from sglang.srt.configs.load_config import LoadConfig
+from sglang.srt.configs.model_config import ModelConfig
+from sglang.srt.model_loader.loader import get_model_loader
+from sglang.srt.configs.device_config import DeviceConfig
+
+# Configure model with inc quantization and saving
+model_config = ModelConfig(
+    model_path="meta-llama/Llama-3.2-3B-Instruct",
+    quantization="auto-round-int8",
+    trust_remote_code=True,
+)
+
+load_config = LoadConfig(
+    inc_save_path="./quantized_model",
+)
+device_config = DeviceConfig(device="cpu")
+
+# Load and quantize the model
+model_loader = get_model_loader(load_config, model_config)
+quantized_model = model_loader.load_model(
+    model_config=model_config,
+    device_config=device_config,
+)
+```
+
 - known issues
 
 Several limitations currently affect offline quantized model loading in sglang, These issues might be resolved in future updates of sglang. If you experience any problems, consider using Hugging Face Transformers as an alternative.
@@ -414,6 +442,10 @@ Several limitations currently affect offline quantized model loading in sglang,
     auto_round:auto_awq and AWQ format:  These work as expected.
     </Accordion>
 
+4. Limited Support for SGlang API Usage
+
+    SGlang API Usage only supports `auto-round-int8` quantization method now, more quantization methods are on the way.
+
 #### Using [GPTQModel](https://github.com/ModelCloud/GPTQModel)
 
 ```bash Command
@@ -857,6 +889,17 @@ sglang serve --model-path Qwen/Qwen3-30B-A3B \
     --quantization quark_mxfp4
 ```
 
+### Intel® Neural Compressor online quantization method
+
+SGLang supports quantization methods based on the advanced algorithm [auto-round](https://github.com/intel/auto-round) in [Intel® Neural Compressor](https://github.com/intel/neural-compressor). You can simply specify `--quantization auto-round-int8` to use this feature. It will quantize the model on the fly to target format. More online quantization methods are on the way.
+
+##### Available Quantization Methods
+
+| Quantization Method | Schemes | Validated Hardware Environment |
+|:--------------------|:--------|:-------------------------------|
+| auto-round-int8     |INT8 per-channel quantized weight <br /> INT8 per-token dynamic quantized activation | Intel Xeon Scalable processor <br /> Nvidia A100 GPU |
+
+
 ## Diffusion Model Quantization on Ascend NPU
 
 SGLang-Diffusion supports MXFP8 quantization for diffusion models (such as Wan2.2) on Ascend A5 NPUs, in both online and offline (ModelSlim) modes. This is separate from the LLM serving path and uses the `sglang serve` / `sglang generate` CLI.

python/pyproject.toml

@@ -139,6 +139,7 @@ fastokens = [
 test = [
   "accelerate",
   "addict",
+  "auto-round>=0.13.1",
   "bitsandbytes",
   "diff-cover",
   "expecttest",

python/sglang/srt/configs/load_config.py

@@ -91,6 +91,11 @@ class LoadConfig:
     # ModelOpt configuration object
     modelopt_config: Optional[ModelOptConfig] = None
 
+    # Inc-related loading options
+    inc_save_path: Optional[str] = None
+    inc_tuning_iters: Optional[int] = 0
+    inc_disable_opt_rtn: Optional[bool] = None
+
     # QuantizedRL-specific options (for FlashRL-style quantization)
     rl_quant_profile: Optional[str] = (
         None  # Path to rollout quantization profile (e.g., /root/profile.7b.pt)

python/sglang/srt/configs/model_config.py

@@ -1187,9 +1187,12 @@ class ModelConfig:
             return True
 
         # Check for HuggingFace quantization config
-        from sglang.srt.utils import has_hf_quant_config
+        quant_cfg = getattr(self.hf_config, "quantization_config", None)
+        if quant_cfg is None:
+            from sglang.srt.utils import has_hf_quant_config
 
-        return has_hf_quant_config(self.model_path)
+            return has_hf_quant_config(self.model_path)
+        return True
 
     def _get_modelopt_quant_type(self) -> str:
         """Extract ModelOpt quantization type from unified quantization flag."""
@@ -1269,6 +1272,7 @@ class ModelConfig:
             "mxfp4",
             "mxfp8",
             "auto-round",
+            "auto-round-int8",
             "quark_int4fp8_moe",
             "quark_mxfp4",
         ]
@@ -1304,6 +1308,7 @@ class ModelConfig:
             "petit_nvfp4": ["modelopt"],
             "w8a8_int8": ["compressed-tensors", "compressed_tensors"],
             "w8a8_fp8": ["compressed-tensors", "compressed_tensors"],
+            "auto-round-int8": ["compressed-tensors", "compressed_tensors"],
         }
         if self.quantization is not None:
             self.quantization = self.quantization.lower()

python/sglang/srt/layers/quantization/__init__.py

@@ -95,6 +95,7 @@ BASE_QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
     "quark": QuarkConfig,
     "quark_mxfp4": QuarkConfig,
     "auto-round": AutoRoundConfig,
+    "auto-round-int8": W8A8Int8Config,
     "modelslim": ModelSlimConfig,
     "quark_int4fp8_moe": QuarkInt4Fp8Config,
 }

python/sglang/srt/model_loader/loader.py

@@ -16,6 +16,7 @@ import math
 import os
 import re
 import socket
+import tempfile
 import threading
 import time
 from abc import ABC, abstractmethod
@@ -2581,6 +2582,110 @@ def load_model_with_cpu_quantization(
     return model.eval()
 
 
+class IncModelLoader(DefaultModelLoader):
+    """
+    Model loader that applies Intel AutoRound quantization
+    """
+
+    def __init__(self, load_config: LoadConfig):
+        super().__init__(load_config)
+
+    def load_model(
+        self,
+        *,
+        model_config: ModelConfig,
+        device_config: DeviceConfig,
+    ) -> nn.Module:
+
+        logger.info("IncModelLoader: Loading model...")
+
+        # Check if model is already quantized
+        if model_config._is_already_quantized():
+            logger.info("Model is already quantized, loading directly...")
+            # Use default loading for pre-quantized models
+            return super().load_model(
+                model_config=model_config, device_config=device_config
+            )
+
+        quant_model = self._autoround_quantization_workflow(model_config, device_config)
+
+        target_device = torch.device(device_config.device)
+
+        # Return autoround model for offline quantization mode
+        if self.load_config.inc_save_path is not None:
+            quant_model.to(target_device)
+            return quant_model.eval()
+
+        model_config.hf_config = quant_model.config
+        quant_config = _get_quantization_config(model_config, self.load_config)
+
+        with set_default_torch_dtype(model_config.dtype):
+            with target_device:
+                model = _initialize_model(
+                    model_config,
+                    self.load_config,
+                    quant_config,
+                )
+
+            self.load_weights_and_postprocess(
+                model, iter(quant_model.state_dict().items()), target_device
+            )
+        return model.eval()
+
+    def _parse_quantization(self, quantization: str):
+        """Map quantization to AutoRound's scheme and format."""
+        AR_QUANT_CFG_CHOICES = {
+            "auto-round-int8": ("INT8", "llm_compressor"),
+        }
+        quant_cfg = AR_QUANT_CFG_CHOICES.get(quantization)
+        if not quant_cfg:
+            raise ValueError(
+                f"Invalid quantization choice: '{quantization}'. "
+                f"Available choices: {list(AR_QUANT_CFG_CHOICES.keys())}"
+            )
+        return quant_cfg
+
+    def _autoround_quantization_workflow(
+        self, model_config: ModelConfig, device_config: DeviceConfig
+    ) -> nn.Module:
+        """Auto-round quantization workflow: quantize, save checkpoint, then return model."""
+        try:
+            from auto_round import AutoRound
+        except ImportError:
+            logger.error(
+                "auto-round library not found. "
+                "Please install it using `pip install auto-round` to use AutoRound quantization."
+            )
+            raise
+
+        scheme, format = self._parse_quantization(model_config.quantization)
+
+        try:
+            autoround = AutoRound(
+                model_config.model_path,
+                scheme=scheme,
+                iters=self.load_config.inc_tuning_iters,
+                disable_opt_rtn=self.load_config.inc_disable_opt_rtn,
+                low_cpu_mem_usage=False,
+            )
+            if self.load_config.inc_save_path is not None:
+                logger.info("Offline quantization mode: Will quantize and save")
+                model, _ = autoround.quantize_and_save(
+                    output_dir=self.load_config.inc_save_path, format=format
+                )
+                return model
+            else:
+                logger.info("Online quantization mode: Will quantize and skip saving")
+                # Use a temporary directory and discard it so nothing is persisted in online mode.
+                with tempfile.TemporaryDirectory() as tmp_save_dir:
+                    model, _ = autoround.quantize_and_save(
+                        output_dir=tmp_save_dir, format=format
+                    )
+                return model
+        except Exception as e:
+            raise ValueError(f"AutoRound quantization failed: {e}")
+
+
 class ModelOptModelLoader(DefaultModelLoader):
     """
     Model loader that applies NVIDIA Model Optimizer quantization
@@ -3102,6 +3207,10 @@ def get_model_loader(
     if load_config.load_format == LoadFormat.DUMMY:
         return DummyModelLoader(load_config)
 
+    if model_config and model_config.quantization in ["auto-round-int8"]:
+        logger.info("Using IncModelLoader due to AutoRound quantization config.")
+        return IncModelLoader(load_config)
+
     # ModelOptModelLoader's local-copy quantize-and-export workflow doesn't apply
     # to non-local loaders. These loaders own their weight transport path and still
     # initialize the model with ModelOpt quantization config where applicable.

python/sglang/srt/server_args.py

@@ -163,6 +163,7 @@ QUANTIZATION_CHOICES = [
     "w4afp8",
     "mxfp4",  # MOE-only.
     "auto-round",
+    "auto-round-int8",
     "compressed-tensors",  # for Ktransformers
     "modelslim",  # for NPU
     "quark",  # AMD Quark quantizer (FP8 / MXFP4 / Int4FP8 etc.)

test/registered/quant/test_autoround_quantization.py

@@ -0,0 +1,92 @@
+"""
+Usage:
+python3 -m unittest test_autoround_quantization
+"""
+
+import os
+import shutil
+import tempfile
+import unittest
+from types import SimpleNamespace
+
+from sglang.srt.configs.device_config import DeviceConfig
+from sglang.srt.configs.load_config import LoadConfig
+from sglang.srt.configs.model_config import ModelConfig
+from sglang.srt.model_loader.loader import get_model_loader
+from sglang.srt.utils import kill_process_tree
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.run_eval import run_eval
+from sglang.test.test_utils import (
+    DEFAULT_MODEL_NAME_FOR_TEST,
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+    DEFAULT_URL_FOR_TEST,
+    CustomTestCase,
+    popen_launch_server,
+)
+
+register_cuda_ci(est_time=120, stage="extra-a", runner_config="1-gpu-large")
+
+
+class TestAutoRoundQuantization(CustomTestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.base_url = DEFAULT_URL_FOR_TEST
+        cls.model = DEFAULT_MODEL_NAME_FOR_TEST
+        cls.output_dir = tempfile.mkdtemp()
+
+    @classmethod
+    def tearDownClass(cls):
+        if os.path.isdir(cls.output_dir):
+            shutil.rmtree(cls.output_dir)
+
+    def test_online_quant(self):
+        process = popen_launch_server(
+            self.model,
+            self.base_url,
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+            other_args=["--trust-remote-code", "--quantization", "auto-round-int8"],
+        )
+
+        try:
+            args = SimpleNamespace(
+                base_url=self.base_url,
+                model=self.model,
+                eval_name="mmlu",
+                num_examples=32,
+                num_threads=32,
+                device="auto",
+            )
+            metrics = run_eval(args)
+            self.assertGreaterEqual(metrics["score"], 0.7)
+        finally:
+            kill_process_tree(process.pid)
+            print(f"[INFO] Server for {self.model} stopped.")
+
+    def test_offline_quant(self):
+        model_config = ModelConfig(
+            model_path=self.model,
+            quantization="auto-round-int8",
+            trust_remote_code=True,
+        )
+        load_config = LoadConfig(
+            inc_save_path=self.output_dir,
+        )
+        device_config = DeviceConfig(device="cuda")
+        model_loader = get_model_loader(load_config, model_config)
+        quantized_model = model_loader.load_model(
+            model_config=model_config,
+            device_config=device_config,
+        )
+        # AutoRound saves the quantized checkpoint into a scheme-derived
+        # subfolder (e.g. "<model>-w8a8/") under the output dir
+        config_found = any(
+            "config.json" in files for _, _, files in os.walk(self.output_dir)
+        )
+        self.assertTrue(
+            config_found,
+            f"No config.json written under {self.output_dir}",
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()