From cb06c4e6ce40ac3c61586597a0b1e5c705ce24f1 Mon Sep 17 00:00:00 2001 From: Zaili Wang <109502517+ZailiWang@users.noreply.github.com> Date: Thu, 2 Jul 2026 13:20:18 +0800 Subject: [PATCH] [CPU] Fix model failures on Xeon (#29497) --- docker/xeon.Dockerfile | 3 +- .../docs/hardware-platforms/cpu_server.mdx | 87 ++++--------------- python/pyproject_cpu.toml | 3 +- .../sglang/srt/layers/quantization/mxfp4.py | 83 ++++++++++++------ .../cuda_graph_buffer_registry.py | 5 +- .../srt/model_executor/runner/eager_runner.py | 3 + 6 files changed, 86 insertions(+), 98 deletions(-) diff --git a/docker/xeon.Dockerfile b/docker/xeon.Dockerfile index 1a6da4b3f..7850e9f8e 100644 --- a/docker/xeon.Dockerfile +++ b/docker/xeon.Dockerfile @@ -42,8 +42,7 @@ RUN source /opt/.venv/bin/activate && \ uv pip install . && \ cd ../sgl-kernel && \ cp pyproject_cpu.toml pyproject.toml && \ - uv pip install . && \ - uv pip install pytest + uv pip install . ENV SGLANG_USE_CPU_ENGINE=1 ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libtcmalloc.so.4:/usr/lib/x86_64-linux-gnu/libtbbmalloc.so:/opt/.venv/lib/libiomp5.so diff --git a/docs_new/docs/hardware-platforms/cpu_server.mdx b/docs_new/docs/hardware-platforms/cpu_server.mdx index 743766656..9de015e73 100644 --- a/docs_new/docs/hardware-platforms/cpu_server.mdx +++ b/docs_new/docs/hardware-platforms/cpu_server.mdx @@ -5,82 +5,31 @@ The document addresses how to set up the [SGLang](https://github.com/sgl-project SGLang is enabled and optimized on the CPUs equipped with Intel® AMX® Instructions, which are 4th generation or newer Intel® Xeon® Scalable Processors. -## Optimized Model List - -A list of popular LLMs are optimized and run efficiently on CPU, +A number of popular LLMs are optimized and run efficiently on CPU, including the most notable open-source models like Llama series, Qwen series, and DeepSeek series like DeepSeek-R1 and DeepSeek-V3.1-Terminus. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Model NameBF16W8A8_INT8FP8
DeepSeek-R1meituan/DeepSeek-R1-Channel-INT8deepseek-ai/DeepSeek-R1
DeepSeek-V3.1-TerminusIntervitensInc/DeepSeek-V3.1-Terminus-Channel-int8deepseek-ai/DeepSeek-V3.1-Terminus
Llama-3.2-3Bmeta-llama/Llama-3.2-3B-InstructRedHatAI/Llama-3.2-3B-quantized.w8a8
Llama-3.1-8Bmeta-llama/Llama-3.1-8B-InstructRedHatAI/Meta-Llama-3.1-8B-quantized.w8a8
QwQ-32BRedHatAI/QwQ-32B-quantized.w8a8
DeepSeek-Distilled-LlamaRedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8
Qwen3-235BQwen/Qwen3-235B-A22B-FP8
- -**Note:** The model identifiers listed in the table above -have been verified on 6th Gen Intel® Xeon® P-core platforms. +Please check the [SGLang Cookbook pages](https://docs.sglang.io/cookbook/intro) +in which the support status and example commands can be found. ## Installation ### Install Using Docker It is recommended to use Docker for setting up the SGLang environment. -A [Dockerfile](https://github.com/sgl-project/sglang/blob/main/docker/xeon.Dockerfile) is provided to facilitate the installation. + +#### Pull from Docker Hub + +Pull the prebuilt docker image of SGLang package releases from `lmsysorg/sglang` repository. +The [CPU image tags](https://hub.docker.com/r/lmsysorg/sglang/tags?name=xeon) end with `xeon` suffix. +The image pulling command is like: + +```bash Command +docker pull lmsysorg/sglang:v0.5.13-xeon +``` + +#### Build from Dockerfile + +A [Dockerfile](https://github.com/sgl-project/sglang/blob/main/docker/xeon.Dockerfile) is provided to facilitate the installation from latest source code. Replace `` below with your [HuggingFace access token](https://huggingface.co/docs/hub/en/security-tokens). ```bash Command @@ -112,7 +61,7 @@ the setup process is as follows: Please install the required packages and libraries beforehand if they are not already present on your system. You can refer to the Ubuntu-based installation commands in -[the Dockerfile](https://github.com/sgl-project/sglang/blob/main/docker/xeon.Dockerfile#L11) +[the Dockerfile](https://github.com/sgl-project/sglang/blob/main/docker/xeon.Dockerfile#L7) for guidance. 1. Install `uv` package manager, then create and activate a virtual environment: diff --git a/python/pyproject_cpu.toml b/python/pyproject_cpu.toml index 0dfbfdc2d..7a76fdd19 100644 --- a/python/pyproject_cpu.toml +++ b/python/pyproject_cpu.toml @@ -49,6 +49,7 @@ dependencies = [ "pybase64", "pydantic", "python-multipart", + "pytest", "pyzmq>=25.1.2", "requests", "scipy", @@ -69,6 +70,7 @@ dependencies = [ "uvicorn", "uvloop", "xgrammar==0.2.1", + "zstandard", ] [project.optional-dependencies] @@ -105,7 +107,6 @@ test = [ "matplotlib", "pandas", "peft>=0.18.0", - "pytest", "sentence_transformers", ] all = [] diff --git a/python/sglang/srt/layers/quantization/mxfp4.py b/python/sglang/srt/layers/quantization/mxfp4.py index 80011b5ff..186e32e93 100644 --- a/python/sglang/srt/layers/quantization/mxfp4.py +++ b/python/sglang/srt/layers/quantization/mxfp4.py @@ -857,6 +857,29 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): layer.w2_weight_bias = Parameter( layer.w2_weight_bias.float(), requires_grad=False ) + return + # Fallback if the TP-sharded layer cannot be AMX-packed + from sglang.srt.layers.quantization.mxfp4_tensor import MXFP4QuantizeUtil + + w13_weight = MXFP4QuantizeUtil.dequantize( + quantized_data=layer.w13_weight, + dtype=torch.bfloat16, + scale=layer.w13_weight_scale, + block_sizes=[32], + ) + w2_weight = MXFP4QuantizeUtil.dequantize( + quantized_data=layer.w2_weight, + dtype=torch.bfloat16, + scale=layer.w2_weight_scale, + block_sizes=[32], + ) + del layer.w13_weight + del layer.w2_weight + del layer.w13_weight_scale + del layer.w2_weight_scale + layer.w13_weight = Parameter(w13_weight, requires_grad=False) + layer.w2_weight = Parameter(w2_weight, requires_grad=False) + return else: from triton_kernels.numerics_details.mxfp import upcast_from_mxfp @@ -1085,32 +1108,42 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): x = dispatch_output.hidden_states topk_output = dispatch_output.topk_output - if use_intel_amx_backend(layer): - from sglang.srt.layers.moe.topk import apply_topk_weights_cpu + if _is_cpu: + if use_intel_amx_backend(layer): + from sglang.srt.layers.moe.topk import apply_topk_weights_cpu - topk_weights, topk_ids, _ = dispatch_output.topk_output - x, topk_weights = apply_topk_weights_cpu( - self.moe_runner_config.apply_router_weight_on_input, topk_weights, x - ) - output = torch.ops.sgl_kernel.fused_experts_cpu( - x, - layer.w13_weight, - layer.w2_weight, - topk_weights, - topk_ids, - False, # inplace See [Note] inplace should be False in fused_experts. - CPUQuantMethod.MXFP4, - layer.w13_weight_scale, # w1_scale - layer.w2_weight_scale, # w2_scale - None, # w1_zp - None, # w2_zp - None, # block_size - getattr(layer, "w13_weight_bias", None), - getattr(layer, "w2_weight_bias", None), - layer.moe_runner_config.gemm1_alpha, - layer.moe_runner_config.gemm1_clamp_limit, - True, # is_vnni - ) + topk_weights, topk_ids, _ = dispatch_output.topk_output + x, topk_weights = apply_topk_weights_cpu( + self.moe_runner_config.apply_router_weight_on_input, topk_weights, x + ) + output = torch.ops.sgl_kernel.fused_experts_cpu( + x, + layer.w13_weight, + layer.w2_weight, + topk_weights, + topk_ids, + False, # inplace See [Note] inplace should be False in fused_experts. + CPUQuantMethod.MXFP4, + layer.w13_weight_scale, # w1_scale + layer.w2_weight_scale, # w2_scale + None, # w1_zp + None, # w2_zp + None, # block_size + getattr(layer, "w13_weight_bias", None), + getattr(layer, "w2_weight_bias", None), + layer.moe_runner_config.gemm1_alpha, + layer.moe_runner_config.gemm1_clamp_limit, + True, # is_vnni + ) + else: + from sglang.srt.layers.moe.fused_moe_native import moe_forward_native + + output = moe_forward_native( + layer, + x, + topk_output, + self.moe_runner_config, + ) return StandardCombineInput(hidden_states=output) if self.use_marlin: diff --git a/python/sglang/srt/model_executor/cuda_graph_buffer_registry.py b/python/sglang/srt/model_executor/cuda_graph_buffer_registry.py index eaaf8b858..a90b9bef9 100644 --- a/python/sglang/srt/model_executor/cuda_graph_buffer_registry.py +++ b/python/sglang/srt/model_executor/cuda_graph_buffer_registry.py @@ -514,6 +514,7 @@ def build_decode_registry( enable_mamba_track: bool = False, is_encoder_decoder: bool = False, encoder_len_fill_value: int = 0, + encoder_lens_dtype: torch.dtype = torch.int32, enable_num_token_non_padded: bool = False, require_gathered_buffer: bool = False, enable_prefill_cp: bool = False, @@ -632,7 +633,7 @@ def build_decode_registry( GraphSlot( "encoder_lens", _bs, - torch.int32, + encoder_lens_dtype, axis="bs", padding_policy=PaddingPolicy.FILL_ONCE, pad_value=encoder_len_fill_value, @@ -898,6 +899,7 @@ def build_eager_registry( enable_mamba_track: bool = False, is_encoder_decoder: bool = False, encoder_len_fill_value: int = 0, + encoder_lens_dtype: torch.dtype = torch.int32, dp_size: int = 1, ) -> CudaGraphBufferRegistry: """One fixed-max input registry for the ``EagerRunner``, serving BOTH eager @@ -924,6 +926,7 @@ def build_eager_registry( enable_mamba_track=enable_mamba_track, is_encoder_decoder=is_encoder_decoder, encoder_len_fill_value=encoder_len_fill_value, + encoder_lens_dtype=encoder_lens_dtype, enable_num_token_non_padded=False, register_global_num_tokens=False, require_gathered_buffer=False, diff --git a/python/sglang/srt/model_executor/runner/eager_runner.py b/python/sglang/srt/model_executor/runner/eager_runner.py index 3137a8eea..a0d9fb9d0 100644 --- a/python/sglang/srt/model_executor/runner/eager_runner.py +++ b/python/sglang/srt/model_executor/runner/eager_runner.py @@ -130,6 +130,9 @@ class EagerRunner(BaseRunner): if is_encoder_decoder else 0 ), + encoder_lens_dtype=( + torch.int64 if torch.device(mr.device).type == "cpu" else torch.int32 + ), dp_size=sa.dp_size, ) # Eager has no capture step, so warm up here (run-once via mr._kernel_warmed_up).