diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-OCR-2.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-OCR-2.mdx index 0ef83c3ff..cfe54990b 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-OCR-2.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-OCR-2.mdx @@ -49,7 +49,7 @@ This section provides deployment configurations optimized for different hardware - **Single GPU Deployment:** DeepSeek-OCR-2 (~3B parameters) fits on a single modern GPU — no tensor parallelism required for low-latency serving. - **High Throughput:** For high-throughput scenarios, use data parallelism with the SGLang Model Gateway. See [DP, DPA and SGLang DP Router](../../../docs/advanced_features/sgl_model_gateway). - **NCCL timeout:** If model loading is slow, increase `--dist-timeout 3600`. -- For configuring CPU service, please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for NUMA binding settings. +- **Xeon CPU service configuration:** Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for NUMA binding settings. ## 4. Model Invocation diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-R1.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-R1.mdx index 35a53ed22..72880aeb6 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-R1.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-R1.mdx @@ -99,7 +99,7 @@ python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-R1 --tp 8 --tru **NCCL timeout:** If model loading is slow, increase: `--dist-timeout 3600`. -For configuring CPU service, please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. +**Xeon CPU service configuration:** Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. ## 4. Model Invocation diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx index 550620fdf..3a17f319f 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx @@ -79,7 +79,7 @@ DeepGEMM is enabled by default on Hopper/Blackwell and can be disabled with `SGL **NCCL timeout:** If model loading is slow and you hit an NCCL timeout, increase it: `--dist-timeout 3600`. -For configuring CPU service, please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. +**Xeon CPU service configuration:** Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. ## 4. Model Invocation diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_1.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_1.mdx index 03e322155..3c721c8d5 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_1.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_1.mdx @@ -77,7 +77,7 @@ DeepGEMM is enabled by default on Hopper/Blackwell and can be disabled with `SGL **NCCL timeout:** If model loading is slow and you hit an NCCL timeout, increase it: `--dist-timeout 3600`. -For configuring CPU service, please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. +**Xeon CPU service configuration:** Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. ## 4. Model Invocation diff --git a/docs_new/cookbook/autoregressive/Llama/Llama3.1.mdx b/docs_new/cookbook/autoregressive/Llama/Llama3.1.mdx index 52c143506..8827abf67 100644 --- a/docs_new/cookbook/autoregressive/Llama/Llama3.1.mdx +++ b/docs_new/cookbook/autoregressive/Llama/Llama3.1.mdx @@ -22,6 +22,8 @@ SGLang offers multiple installation methods. You can choose the most suitable in Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. +For SGLang CPU installation, please refer to the [CPU version installation guide](../../../docs/hardware-platforms/cpu_server#installation). + ## 3. Model Deployment This section provides deployment configurations optimized for different hardware platforms and use cases. @@ -55,6 +57,10 @@ import { Llama31Deployment } from "/src/snippets/autoregressive/llama31-deployme - 70B/8B: Use AMD's optimized `amd/Llama-3.1-{size}-Instruct-FP8-KV` - **Tool Calling**: Enable with `--tool-call-parser llama3` for Instruct models +**Xeon CPU Deployment:** + +- Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. + ## 4. Model Invocation ### 4.1 Basic Usage diff --git a/docs_new/cookbook/autoregressive/Llama/Llama3.3-70B.mdx b/docs_new/cookbook/autoregressive/Llama/Llama3.3-70B.mdx index f865b52fe..549ca1a8d 100644 --- a/docs_new/cookbook/autoregressive/Llama/Llama3.3-70B.mdx +++ b/docs_new/cookbook/autoregressive/Llama/Llama3.3-70B.mdx @@ -24,9 +24,11 @@ For more details, please refer to the [official Llama models repository](https:/ Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. +For SGLang CPU installation, please refer to the [CPU version installation guide](../../../docs/hardware-platforms/cpu_server#installation). + ## 3. Model Deployment -This section provides deployment configurations optimized for AMD GPUs (MI300X, MI325X, MI355X). +This section provides deployment configurations optimized for AMD GPUs (MI300X, MI325X, MI355X) and Intel Xeon CPUs. ### 3.1 Interactive Configuration @@ -45,6 +47,10 @@ import { Llama33Deployment } from "/src/snippets/autoregressive/llama33-70b-depl - **Tool Calling**: Enable with `--tool-call-parser llama3` for function calling support - **Higher Throughput**: Optional TP=2 or TP=4 can be used for increased throughput +**Xeon CPU Deployment:** + +Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. + ## 4. Model Invocation ### 4.1 Basic Usage diff --git a/docs_new/cookbook/autoregressive/Llama/Llama4.mdx b/docs_new/cookbook/autoregressive/Llama/Llama4.mdx index d2631c53d..7c70e05f8 100644 --- a/docs_new/cookbook/autoregressive/Llama/Llama4.mdx +++ b/docs_new/cookbook/autoregressive/Llama/Llama4.mdx @@ -29,6 +29,8 @@ SGLang offers multiple installation methods. You can choose the most suitable in Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. +For SGLang CPU installation, please refer to the [CPU version installation guide](../../../docs/hardware-platforms/cpu_server#installation). + ## 3. Model Deployment This section provides a progressive guide from quick deployment to performance optimization, suitable for users at different levels. @@ -54,6 +56,7 @@ This section provides a progressive guide from quick deployment to performance o - **Multi-Modal:** Add `--enable-multimodal` to enable image input support. - **Hybrid KV Cache:** Set `--swa-full-tokens-ratio` to control the ratio of SWA (local attention) KV tokens to full-attention KV tokens (default: 0.8, range: 0–1). - **EAGLE Speculative Decoding:** Supported for Llama 4 Scout and Maverick via EAGLE3. Enable with the interactive command generator above. +- **Xeon CPU service configuration:** Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. ## 4. Model Invocation diff --git a/docs_new/cookbook/autoregressive/OpenAI/GPT-OSS.mdx b/docs_new/cookbook/autoregressive/OpenAI/GPT-OSS.mdx index 68b79b21a..89d6e546b 100644 --- a/docs_new/cookbook/autoregressive/OpenAI/GPT-OSS.mdx +++ b/docs_new/cookbook/autoregressive/OpenAI/GPT-OSS.mdx @@ -25,6 +25,8 @@ SGLang offers multiple installation methods. You can choose the most suitable in Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. +For SGLang CPU installation, please refer to the [CPU version installation guide](../../../docs/hardware-platforms/cpu_server#installation). + ## 3.Model Deployment This section provides deployment configurations optimized for different hardware platforms and use cases. @@ -47,6 +49,7 @@ import { GPTOSSDeployment } from "/src/snippets/autoregressive/gpt-oss-deploymen - **MCP tool servers:** For production, point SGLang at external MCP SSE servers with `--tool-server ip-1:port-1,ip-2:port-2`. - **Responses API:** GPT-OSS supports OpenAI's Responses API (`client.responses.create`) in addition to the standard Chat Completions API (see section 4.2.4). - **Use Python 3.12** when running the demo Python tool. +- **Xeon CPU service configuration:** Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. ## 4.Model Invocation diff --git a/docs_new/cookbook/autoregressive/Qwen/Qwen2.5-VL.mdx b/docs_new/cookbook/autoregressive/Qwen/Qwen2.5-VL.mdx index b8889f01b..efc8c7d86 100644 --- a/docs_new/cookbook/autoregressive/Qwen/Qwen2.5-VL.mdx +++ b/docs_new/cookbook/autoregressive/Qwen/Qwen2.5-VL.mdx @@ -29,13 +29,15 @@ SGLang offers multiple installation methods. You can choose the most suitable in Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. +For SGLang CPU installation, please refer to the [CPU version installation guide](../../../docs/hardware-platforms/cpu_server#installation). + ## 3. Model Deployment -This section provides deployment configurations optimized for AMD MI300X, MI325X and MI355X hardware platforms and different use cases. +This section provides deployment configurations optimized for AMD MI300X, MI325X and MI355X as well as Intel Xeon CPU hardware platforms and different use cases. ### 3.1 Basic Configuration -The Qwen2.5-VL series offers models in various sizes. The following configurations have been verified on AMD MI300X, MI325X and MI355X GPUs. +The Qwen2.5-VL series offers models in various sizes. The following configurations have been verified on AMD MI300X, MI325X and MI355X GPUs as well as Intel Xeon CPUs. **Interactive Command Generator**: Use the configuration selector below to automatically generate the appropriate deployment command for your hardware platform and model size. @@ -45,6 +47,7 @@ The Qwen2.5-VL series offers models in various sizes. The following configuratio * **Memory Management**: For the 72B model on MI300X/MI325X/MI355X, we have verified successful deployment with `--context-length 128000`. Smaller context lengths can be used to reduce memory usage if needed. * **Multi-GPU Deployment**: Use Tensor Parallelism (`--tp`) to scale across multiple GPUs. For example, use `--tp 8` for the 72B model and `--tp 2` for the 32B model on MI300X/MI325X/MI355X. +* **Xeon CPU service configuration**: Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. ## 4. Model Invocation diff --git a/docs_new/cookbook/autoregressive/Qwen/Qwen3-Coder-Next.mdx b/docs_new/cookbook/autoregressive/Qwen/Qwen3-Coder-Next.mdx index a8908bd7b..241cd256a 100644 --- a/docs_new/cookbook/autoregressive/Qwen/Qwen3-Coder-Next.mdx +++ b/docs_new/cookbook/autoregressive/Qwen/Qwen3-Coder-Next.mdx @@ -48,7 +48,7 @@ This section provides a progressive guide from quick deployment to performance o - **Mamba Radix Cache**: Qwen3-Coder-Next's hybrid Gated Delta Networks architecture supports two mamba scheduling strategies via `--mamba-scheduler-strategy`: - **V1 (`no_buffer`)**: Default. No overlap scheduler, lower memory usage. - **V2 (`extra_buffer`)**: Enables overlap scheduling and branching point caching with `--mamba-scheduler-strategy extra_buffer --page-size 64`. Requires FLA kernel backend. Trades higher mamba state memory for better throughput. Strictly superior in non-KV-cache-bound scenarios; in KV-cache-bound cases, weigh the overlap scheduling benefit against reduced max concurrency. `--page-size` must satisfy `FLA_CHUNK_SIZE % page_size == 0` or `page_size % FLA_CHUNK_SIZE == 0` (`FLA_CHUNK_SIZE` is currently 64). -- For configuring CPU service, please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. +- **Xeon CPU service configuration**: Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. ## 4. Model Invocation diff --git a/docs_new/cookbook/autoregressive/Qwen/Qwen3-Coder.mdx b/docs_new/cookbook/autoregressive/Qwen/Qwen3-Coder.mdx index 2c142bc3b..0b51ab4a7 100644 --- a/docs_new/cookbook/autoregressive/Qwen/Qwen3-Coder.mdx +++ b/docs_new/cookbook/autoregressive/Qwen/Qwen3-Coder.mdx @@ -53,12 +53,12 @@ This section provides deployment configurations verified on AMD MI300X, MI325X, * **NVFP4 Quantization**: Requires `--quantization modelopt_fp4` and uses a different model path (`nvidia/Qwen3-Coder-...`). * **DP Attention**: NVFP4 configuration supports `--enable-dp-attention` for improved throughput. +**Intel Xeon CPU:** +* Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. + **General:** * **Tool Use**: To enable tool calling capabilities, add `--tool-call-parser qwen3_coder` to the launch command. -**Intel Xeon CPU:** -* For configuring CPU service, please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. - ## 4. Model Invocation ### 4.1 Basic Usage diff --git a/docs_new/cookbook/autoregressive/Qwen/Qwen3-Next.mdx b/docs_new/cookbook/autoregressive/Qwen/Qwen3-Next.mdx index be285a346..f6941cb44 100644 --- a/docs_new/cookbook/autoregressive/Qwen/Qwen3-Next.mdx +++ b/docs_new/cookbook/autoregressive/Qwen/Qwen3-Next.mdx @@ -58,7 +58,7 @@ The Qwen3-Next series comes in only one size but offers different thinking modes - **V1 (`no_buffer`)**: Default. No overlap scheduler, lower memory usage. - **V2 (`extra_buffer`)**: Enables overlap scheduling and branching point caching with `--mamba-scheduler-strategy extra_buffer --page-size 64`. Requires FLA kernel backend. Trades higher mamba state memory for better throughput. Strictly superior in non-KV-cache-bound scenarios; in KV-cache-bound cases, weigh the overlap scheduling benefit against reduced max concurrency. `--page-size` must satisfy `FLA_CHUNK_SIZE % page_size == 0` or `page_size % FLA_CHUNK_SIZE == 0` (`FLA_CHUNK_SIZE` is currently 64). -- For configuring CPU service, please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. +- **Xeon CPU service configuration**: Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. ## 4. Model Invocation diff --git a/docs_new/cookbook/autoregressive/Qwen/Qwen3-VL.mdx b/docs_new/cookbook/autoregressive/Qwen/Qwen3-VL.mdx index f8945d403..376a48415 100644 --- a/docs_new/cookbook/autoregressive/Qwen/Qwen3-VL.mdx +++ b/docs_new/cookbook/autoregressive/Qwen/Qwen3-VL.mdx @@ -78,7 +78,7 @@ python -m sglang.launch_server \ --enable-metrics ``` -* For configuring CPU service, please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. +* **Xeon CPU service configuration:** Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. ## 4. Model Invocation diff --git a/docs_new/cookbook/autoregressive/Qwen/Qwen3.5.mdx b/docs_new/cookbook/autoregressive/Qwen/Qwen3.5.mdx index 17f81a90c..5f3a886ea 100644 --- a/docs_new/cookbook/autoregressive/Qwen/Qwen3.5.mdx +++ b/docs_new/cookbook/autoregressive/Qwen/Qwen3.5.mdx @@ -228,7 +228,7 @@ This section provides deployment configurations optimized for different hardware **FP8 KV Cache**: `--kv-cache-dtype fp8_e4m3` quantizes the KV cache to FP8 at runtime. Since these FP8 model checkpoints do not include pre-calibrated KV cache scaling factors, SGLang defaults to a scale of 1.0, which may cause noticeable accuracy degradation on reasoning-heavy tasks. It is not included in the generated commands above; add it manually only if memory constraints require the trade-off. -- For configuring CPU service, please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. +- **Xeon CPU service configuration**: Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. ## 4. Model Invocation diff --git a/docs_new/cookbook/autoregressive/Qwen/Qwen3.6.mdx b/docs_new/cookbook/autoregressive/Qwen/Qwen3.6.mdx index c39fb331e..e48a6520e 100644 --- a/docs_new/cookbook/autoregressive/Qwen/Qwen3.6.mdx +++ b/docs_new/cookbook/autoregressive/Qwen/Qwen3.6.mdx @@ -151,7 +151,7 @@ All Qwen3.6 variants (MoE 35B-A3B and Dense 27B) fit on a single supported GPU a -- For configuring CPU service, please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. +- **Xeon CPU service configuration:** Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. ## 4. Model Invocation diff --git a/docs_new/cookbook/autoregressive/Qwen/Qwen3.mdx b/docs_new/cookbook/autoregressive/Qwen/Qwen3.mdx index 79eb5a446..f58397e7d 100644 --- a/docs_new/cookbook/autoregressive/Qwen/Qwen3.mdx +++ b/docs_new/cookbook/autoregressive/Qwen/Qwen3.mdx @@ -44,16 +44,16 @@ import { Qwen3Deployment } from "/src/snippets/autoregressive/qwen3-deployment.j ### 3.2 Configuration Tips -- **Memory Management** : Set lower `--context-length` to conserve memory. A value of `128000` is sufficient for most scenarios, down from the default 262K. -- **Expert Parallelism** : SGLang supports Expert Parallelism (EP) via `--ep`, allowing experts in MoE models to be deployed on separate GPUs for better throughput. One thing to note is that, for quantized models, you need to set `--ep` to a value that satisfies the requirement: `(moe_intermediate_size / moe_tp_size) % weight_block_size_n == 0, where moe_tp_size is equal to tp_size divided by ep_size.` Note that EP may perform worse in low concurrency scenarios due to additional communication overhead. Check out [Expert Parallelism Deployment](../../../docs/advanced_features/expert_parallelism) for more details. -- **Kernel Tuning** : For MoE Triton kernel tuning on your specific hardware, refer to [fused_moe_triton](https://github.com/sgl-project/sglang/tree/main/benchmark/kernels/fused_moe_triton). -- **Speculative Decoding**: Using Speculative Decoding for latency-sensitive scenarios. +- **Memory Management:** Set lower `--context-length` to conserve memory. A value of `128000` is sufficient for most scenarios, down from the default 262K. +- **Expert Parallelism:** SGLang supports Expert Parallelism (EP) via `--ep`, allowing experts in MoE models to be deployed on separate GPUs for better throughput. One thing to note is that, for quantized models, you need to set `--ep` to a value that satisfies the requirement: `(moe_intermediate_size / moe_tp_size) % weight_block_size_n == 0, where moe_tp_size is equal to tp_size divided by ep_size.` Note that EP may perform worse in low concurrency scenarios due to additional communication overhead. Check out [Expert Parallelism Deployment](../../../docs/advanced_features/expert_parallelism) for more details. +- **Kernel Tuning:** For MoE Triton kernel tuning on your specific hardware, refer to [fused_moe_triton](https://github.com/sgl-project/sglang/tree/main/benchmark/kernels/fused_moe_triton). +- **Speculative Decoding:** Using Speculative Decoding for latency-sensitive scenarios. - `--speculative-algorithm EAGLE3`: Speculative decoding algorithm - `--speculative-num-steps 3`: Number of speculative verification rounds - `--speculative-eagle-topk 1`: Top-k sampling for draft tokens - `--speculative-num-draft-tokens 4`: Number of draft tokens per step - `--speculative-draft-model-path`: The path of the draft model weights. This can be a local folder or a Hugging Face repo ID such as [`lmsys/SGLang-EAGLE3-Qwen3-235B-A22B-Instruct-2507-SpecForge-Meituan`](https://huggingface.co/lmsys/SGLang-EAGLE3-Qwen3-235B-A22B-Instruct-2507-SpecForge-Meituan). -- For configuring CPU service, please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. +- **Xeon CPU service configuration:** Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. ## 4. Model Invocation diff --git a/docs_new/cookbook/autoregressive/Tencent/Hunyuan3-Preview.mdx b/docs_new/cookbook/autoregressive/Tencent/Hunyuan3-Preview.mdx index e22d2b12e..982a4e043 100644 --- a/docs_new/cookbook/autoregressive/Tencent/Hunyuan3-Preview.mdx +++ b/docs_new/cookbook/autoregressive/Tencent/Hunyuan3-Preview.mdx @@ -168,7 +168,7 @@ sglang serve \ Toggle the "Speculative Decoding (MTP)" option in the generator above to add these flags automatically. Tune `num-steps` / `num-draft-tokens` based on acceptance rate in your workload. -For configuring CPU service, please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. +**Xeon CPU service configuration:** Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. ## 4. Model Invocation diff --git a/docs_new/src/snippets/autoregressive/gpt-oss-deployment.jsx b/docs_new/src/snippets/autoregressive/gpt-oss-deployment.jsx index 7cf4e38d4..cca1d32e2 100644 --- a/docs_new/src/snippets/autoregressive/gpt-oss-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/gpt-oss-deployment.jsx @@ -11,7 +11,8 @@ export const GPTOSSDeployment = () => { { id: 'h100', label: 'H100', default: false }, { id: 'mi300x', label: 'MI300X', default: false }, { id: 'mi325x', label: 'MI325X', default: false }, - { id: 'mi355x', label: 'MI355X', default: false } + { id: 'mi355x', label: 'MI355X', default: false }, + { id: 'xeon', label: 'XEON', default: false } ] }, modelsize: { @@ -56,6 +57,18 @@ export const GPTOSSDeployment = () => { } }; + const getDisplayOptions = (values) => ({ + ...options, + quantization: options.quantization, + speculative: { + ...options.speculative, + items: options.speculative.items.map(item => ({ + ...item, + disabled: values.hardware === 'xeon' && item.id === 'enabled' + })) + } + }); + // Initialize state const getInitialState = () => { const initialState = {}; @@ -89,7 +102,13 @@ export const GPTOSSDeployment = () => { }, []); const handleRadioChange = (optionName, value) => { - setValues(prev => ({ ...prev, [optionName]: value })); + setValues(prev => { + const next = { ...prev, [optionName]: value }; + if (optionName === 'hardware' && value === 'xeon') { + next.speculative = 'disabled'; + } + return next; + }); }; // Generate command @@ -106,7 +125,8 @@ export const GPTOSSDeployment = () => { b300: { tp: 8 }, mi300x: { tp: 8 }, mi325x: { tp: 8 }, - mi355x: { tp: 8 } + mi355x: { tp: 8 }, + xeon: { tp: 3 } }, '20b': { baseName: '20b', @@ -116,7 +136,8 @@ export const GPTOSSDeployment = () => { b300: { tp: 1 }, mi300x: { tp: 1 }, mi325x: { tp: 1 }, - mi355x: { tp: 1 } + mi355x: { tp: 1 }, + xeon: { tp: 3 } } }; @@ -146,6 +167,7 @@ export const GPTOSSDeployment = () => { return '# MI300X/MI325X GPUs with MXFP4 quantization: Work In Progress'; } + // AMD MI30x requires SGLANG_USE_AITER=0 due to YaRN RoPE precision issues if (hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x') { cmd += 'SGLANG_USE_AITER=0 '; @@ -159,6 +181,12 @@ export const GPTOSSDeployment = () => { cmd += ` --model ${modelName}`; + if (hardware === 'xeon') { + cmd += ` \\ + --device cpu \\ + --disable-overlap-schedule`; + } + if (hwConfig.tp > 1) { cmd += ` \\\n --tp ${hwConfig.tp}`; } @@ -206,7 +234,7 @@ export const GPTOSSDeployment = () => { return (
- {Object.entries(options).map(([key, option]) => ( + {Object.entries(getDisplayOptions(values)).map(([key, option]) => (
{option.title}
@@ -225,9 +253,10 @@ export const GPTOSSDeployment = () => { ) : ( option.items.map(item => { const isChecked = values[option.name] === item.id; + const isDisabled = Boolean(item.disabled); return ( -