diff --git a/docs_new/cookbook/autoregressive/Qwen/Qwen3.mdx b/docs_new/cookbook/autoregressive/Qwen/Qwen3.mdx index 8c4c140dd..79eb5a446 100644 --- a/docs_new/cookbook/autoregressive/Qwen/Qwen3.mdx +++ b/docs_new/cookbook/autoregressive/Qwen/Qwen3.mdx @@ -26,13 +26,15 @@ SGLang offers multiple installation methods. You can choose the most suitable in Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. +For SGLang CPU installation, please refer to the [CPU version installation guide](../../../docs/hardware-platforms/cpu_server#installation). + ## 3. Model Deployment This section provides deployment configurations optimized for different hardware platforms and use cases. ### 3.1 Basic Configuration -The Qwen3 series offers models in various sizes and architectures, optimized for different hardware platforms including NVIDIA and AMD GPUs. The recommended launch configurations vary by hardware and model size. +The Qwen3 series offers models in various sizes and architectures, optimized for different hardware platforms including NVIDIA GPUs, AMD GPUs, and Intel Xeon CPUs. The recommended launch configurations vary by hardware and model size. **Interactive Command Generator**: Use the configuration selector below to automatically generate the appropriate deployment command for your hardware platform, model size, quantization method, and thinking capabilities. @@ -51,6 +53,7 @@ import { Qwen3Deployment } from "/src/snippets/autoregressive/qwen3-deployment.j - `--speculative-eagle-topk 1`: Top-k sampling for draft tokens - `--speculative-num-draft-tokens 4`: Number of draft tokens per step - `--speculative-draft-model-path`: The path of the draft model weights. This can be a local folder or a Hugging Face repo ID such as [`lmsys/SGLang-EAGLE3-Qwen3-235B-A22B-Instruct-2507-SpecForge-Meituan`](https://huggingface.co/lmsys/SGLang-EAGLE3-Qwen3-235B-A22B-Instruct-2507-SpecForge-Meituan). +- For configuring CPU service, please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. ## 4. Model Invocation diff --git a/docs_new/src/snippets/autoregressive/qwen3-deployment.jsx b/docs_new/src/snippets/autoregressive/qwen3-deployment.jsx index ee99a5793..0d4209c3e 100644 --- a/docs_new/src/snippets/autoregressive/qwen3-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/qwen3-deployment.jsx @@ -9,7 +9,8 @@ export const Qwen3Deployment = () => { b200: { tp: 8, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 4, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 4, ep: 0, bf16: true, fp8: true }, - mi355x: { tp: 4, ep: 0, bf16: true, fp8: true } + mi355x: { tp: 4, ep: 0, bf16: true, fp8: true }, + xeon: { tp: 6, ep: 0, bf16: true, fp8: true } }, '30b': { baseName: '30B-A3B', @@ -19,7 +20,8 @@ export const Qwen3Deployment = () => { b200: { tp: 1, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 1, ep: 0, bf16: true, fp8: true }, - mi355x: { tp: 1, ep: 0, bf16: true, fp8: true } + mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }, + xeon: { tp: 3, ep: 0, bf16: true, fp8: true } }, '32b': { baseName: '32B', @@ -29,7 +31,8 @@ export const Qwen3Deployment = () => { b200: { tp: 1, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 1, ep: 0, bf16: true, fp8: true }, - mi355x: { tp: 1, ep: 0, bf16: true, fp8: true } + mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }, + xeon: { tp: 6, ep: 0, bf16: true, fp8: true } }, '14b': { baseName: '14B', @@ -39,7 +42,8 @@ export const Qwen3Deployment = () => { b200: { tp: 1, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 1, ep: 0, bf16: true, fp8: true }, - mi355x: { tp: 1, ep: 0, bf16: true, fp8: true } + mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }, + xeon: { tp: 3, ep: 0, bf16: true, fp8: true } }, '8b': { baseName: '8B', @@ -49,7 +53,8 @@ export const Qwen3Deployment = () => { b200: { tp: 1, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 1, ep: 0, bf16: true, fp8: true }, - mi355x: { tp: 1, ep: 0, bf16: true, fp8: true } + mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }, + xeon: { tp: 3, ep: 0, bf16: true, fp8: true } }, '4b': { baseName: '4B', @@ -59,7 +64,8 @@ export const Qwen3Deployment = () => { b200: { tp: 1, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 1, ep: 0, bf16: true, fp8: true }, - mi355x: { tp: 1, ep: 0, bf16: true, fp8: true } + mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }, + xeon: { tp: 3, ep: 0, bf16: true, fp8: true } }, '1.7b': { baseName: '1.7B', @@ -69,7 +75,8 @@ export const Qwen3Deployment = () => { b200: { tp: 1, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 1, ep: 0, bf16: true, fp8: true }, - mi355x: { tp: 1, ep: 0, bf16: true, fp8: true } + mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }, + xeon: { tp: 3, ep: 0, bf16: true, fp8: true } }, '0.6b': { baseName: '0.6B', @@ -79,7 +86,8 @@ export const Qwen3Deployment = () => { b200: { tp: 1, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 1, ep: 0, bf16: true, fp8: true }, - mi355x: { tp: 1, ep: 0, bf16: true, fp8: true } + mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }, + xeon: { tp: 3, ep: 0, bf16: true, fp8: true } } }; @@ -94,7 +102,8 @@ export const Qwen3Deployment = () => { { id: 'h200', label: 'H200', default: false }, { id: 'mi300x', label: 'MI300X', default: false }, { id: 'mi325x', label: 'MI325X', default: false }, - { id: 'mi355x', label: 'MI355X', default: false } + { id: 'mi355x', label: 'MI355X', default: false }, + { id: 'xeon', label: 'XEON', default: false } ] }, modelsize: { @@ -261,6 +270,10 @@ export const Qwen3Deployment = () => { let cmd = 'python -m sglang.launch_server \\\n'; cmd += ` --model ${modelName}`; + if (hardware === 'xeon') { + cmd += ` \\\n --device cpu \\\n --disable-overlap-schedule`; + } + if (hwConfig.tp > 1) { cmd += ` \\\n --tp ${hwConfig.tp}`; }