diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-OCR-2.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-OCR-2.mdx index 1b744541f..9140f9c98 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-OCR-2.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-OCR-2.mdx @@ -40,11 +40,13 @@ This section provides deployment configurations optimized for different hardware -**Note**: DeepSeek-OCR-2 has ~3.58B parameters and easily fits on a single modern GPU. For low-latency serving, no model parallelism is needed. For high-throughput requirements, consider using data parallelism with the SGLang Model Gateway — see [DP, DPA and SGLang DP Router](../../../docs/advanced_features/sgl_model_gateway) for more details. +**Note**: DeepSeek-OCR-2 has ~3B parameters and easily fits on a single modern GPU. For low-latency serving, no model parallelism is needed. For high-throughput requirements, consider using data parallelism with the SGLang Model Gateway — see [DP, DPA and SGLang DP Router](../../../docs/advanced_features/sgl_model_gateway) for more details. ### 3.2 Configuration Tips -For more detailed configuration tips, please refer to [DeepSeek V3/V3.1/R1 Usage](../../../docs/basic_usage/deepseek_v3). +- **Single GPU Deployment:** DeepSeek-OCR-2 (~3B parameters) fits on a single modern GPU — no tensor parallelism required for low-latency serving. +- **High Throughput:** For high-throughput scenarios, use data parallelism with the SGLang Model Gateway. See [DP, DPA and SGLang DP Router](../../../docs/advanced_features/sgl_model_gateway). +- **NCCL timeout:** If model loading is slow, increase `--dist-timeout 3600`. ## 4. Model Invocation diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-OCR.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-OCR.mdx index 55b8aca24..abe0802a2 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-OCR.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-OCR.mdx @@ -38,10 +38,6 @@ import { DeepSeekOCRDeployment } from "/src/snippets/autoregressive/deepseek-ocr -### 3.2 Configuration Tips - -For more detailed configuration tips, please refer to [DeepSeek V3/V3.1/R1 Usage](../../../docs/basic_usage/deepseek_v3). - ## 4. Model Invocation ### 4.1 Basic Usage @@ -50,6 +46,50 @@ For basic API usage and request examples, please refer to: - [SGLang Basic Usage Guide](../../../docs/basic_usage/send_request) +### 4.2 OCR-Specific Prompts + +DeepSeek-OCR accepts recommended prompts from the model card: + +```text + +<|grounding|>Convert the document to markdown. +``` + +```text + +Free OCR. +``` + +**OpenAI-compatible image request example:** + +```python Example +import requests + +url = "http://localhost:30000/v1/chat/completions" + +data = { + "model": "deepseek-ai/DeepSeek-OCR", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "\n<|grounding|>Convert the document to markdown." + }, + { + "type": "image_url", + "image_url": {"url": "https://example.com/your_image.jpg"} + }, + ], + } + ], + "max_tokens": 512, +} + +response = requests.post(url, json=data) +print(response.text) +``` ## 5. Benchmark @@ -73,6 +113,7 @@ python3 -m sglang.launch_server \ --model-path deepseek-ai/DeepSeek-OCR \ --tp 1 \ --dtype float16 \ + --trust-remote-code \ --host 0.0.0.0 \ --port 8000 ``` diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-R1.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-R1.mdx index 432f85bbd..27a5e6f96 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-R1.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-R1.mdx @@ -22,6 +22,11 @@ import { DeepSeekR1AdvancedDeployment } from '/src/snippets/autoregressive/deeps - **FP8 (8-bit quantized)**: [deepseek-ai/DeepSeek-R1-0528](https://huggingface.co/deepseek-ai/DeepSeek-R1-0528) - Recommended for H200 and MI300X - **FP4 (4-bit quantized)**: [nvidia/DeepSeek-R1-0528-FP4-v2](https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2) - Recommended for B200 and MI355X +- **BF16 (upcast from FP8)**: [unsloth/DeepSeek-R1-0528-BF16](https://huggingface.co/unsloth/DeepSeek-R1-0528-BF16) +- **INT8 (channel-wise)**: [meituan/DeepSeek-R1-Channel-INT8](https://huggingface.co/meituan/DeepSeek-R1-Channel-INT8) +- **W4A8**: [novita/Deepseek-R1-0528-W4AFP8](https://huggingface.co/novita/Deepseek-R1-0528-W4AFP8) +- **AWQ (4-bit)**: [QuixiAI/DeepSeek-R1-0528-AWQ](https://huggingface.co/QuixiAI/DeepSeek-R1-0528-AWQ) +- **MXFP4**: [amd/DeepSeek-R1-MXFP4](https://huggingface.co/amd/DeepSeek-R1-MXFP4) **License:** To use DeepSeek-R1, you must agree to DeepSeek's Community License. See [LICENSE](https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/main/LICENSE) for details. @@ -50,7 +55,47 @@ Pareto-optimal configurations for B200, H200, MI300X, MI325X, and MI355X hardwar ### 3.3 Configuration Tips -For more detailed configuration tips and advanced tuning, please refer to [DeepSeek V3/V3.1/R1 Usage](../../../docs/basic_usage/deepseek_v3). +DeepSeek-R1 shares the same MoE architecture as DeepSeek-V3, so the same hardware and optimization recommendations apply. + +**Recommended GPU configurations by weight type:** + + + + + + + + + + + + + + + + + + + + + + + + + + +
Weight TypeSupported Hardware
FP8 (recommended)8× H200, 8× B200, 8× MI300X, 2×8× H100/H800/H20
BF16 (upcast from FP8)2×8× H200, 2×8× MI300X, 4×8× H100/H800, 4×8× A100/A800
INT816× A100/A800, 32× L40S, Xeon 6980P CPU, 4× Atlas 800I A3
W4A8 / AWQ / MXFP4 / NVFP48× H20/H100, 4× H200; 8× H100/A100; 8/4× MI355X/MI350X; 8/4× B200
+ +> The official DeepSeek-R1 checkpoint is already in FP8 format — do **not** add `--quantization fp8` when serving it. + +**DeepGEMM precompilation (NVIDIA Hopper / Blackwell):** Precompile GEMM kernels to avoid JIT overhead (~10 min): +```bash +python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-R1 --tp 8 --trust-remote-code +``` + +**Data Parallelism Attention (`--enable-dp-attention`):** Recommended for high-throughput scenarios. Use `--enable-dp-attention --tp 8 --dp 8` on a single 8-GPU node. + +**NCCL timeout:** If model loading is slow, increase: `--dist-timeout 3600`. ## 4. Model Invocation @@ -292,6 +337,40 @@ print(final_response.choices[0].message.content) # Output: "The weather in Beijing is currently 22°C and sunny." ``` +#### 4.2.3 Multi-Token Prediction (EAGLE Speculative Decoding) + +DeepSeek-R1 supports EAGLE-based Multi-Token Prediction (MTP), the same mechanism as DeepSeek-V3. Refer to [DeepSeek-V3 §4.2.3](/cookbook/autoregressive/DeepSeek/DeepSeek-V3#4-2-3-multi-token-prediction-eagle-speculative-decoding) for the complete launch command, flag reference, tuning guidance (`--speculative-num-steps`, `--speculative-eagle-topk`, `--max-running-requests`), and `bench_speculative.py` link. R1's speed benchmark commands that include `--speculative-*` flags use this mechanism. + +#### 4.2.4 Thinking Budget + +Limit the model's thinking token budget using `CustomLogitProcessor`. Launch with `--enable-custom-logit-processor`: + +```shell Command +python3 -m sglang.launch_server \ + --model deepseek-ai/DeepSeek-R1 \ + --tp 8 \ + --port 30000 \ + --reasoning-parser deepseek-r1 \ + --enable-custom-logit-processor +``` + +```python Example +import openai +from sglang.srt.sampling.custom_logit_processor import DeepSeekR1ThinkingBudgetLogitProcessor + +client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="*") +response = client.chat.completions.create( + model="deepseek-ai/DeepSeek-R1", + messages=[{"role": "user", "content": "Is Paris the Capital of France?"}], + max_tokens=1024, + extra_body={ + "custom_logit_processor": DeepSeekR1ThinkingBudgetLogitProcessor().to_str(), + "custom_params": {"thinking_budget": 512}, + }, +) +print(response) +``` + ## 5. Benchmark This section uses **industry-standard configurations** for comparable benchmark results. diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx index b9c26d6bc..56ef88247 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx @@ -35,7 +35,47 @@ import { DeepSeekV3Deployment } from "/src/snippets/autoregressive/deepseek-v3-d ### 3.2 Configuration Tips -For more detailed configuration tips, please refer to [DeepSeek-V3 Usage](../../../docs/basic_usage/deepseek_v3). + +**Recommended GPU configurations by weight type:** + + + + + + + + + + + + + + + + + + + + + + + + + + +
Weight TypeSupported Hardware
FP8 (recommended)8× H200, 8× B200, 8× MI300X, 2×8× H100/H800/H20
BF16 (upcast from FP8)2×8× H200, 2×8× MI300X, 4×8× H100/H800, 4×8× A100/A800
INT816× A100/A800, 32× L40S, Xeon 6980P CPU, 4× Atlas 800I A3
W4A8 / AWQ / MXFP4 / NVFP48× H20/H100, 4× H200; 8× H100/A100; 8/4× MI355X/MI350X; 8/4× B200
+ +> The official DeepSeek-V3 checkpoint is already in FP8 format — do **not** add `--quantization fp8` when serving it. + +**DeepGEMM precompilation (NVIDIA Hopper / Blackwell):** Precompile GEMM kernels before the first server run to avoid JIT overhead (~10 min): +```bash +python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code +``` +DeepGEMM is enabled by default on Hopper/Blackwell and can be disabled with `SGLANG_ENABLE_JIT_DEEPGEMM=0`. + +**Data Parallelism Attention (`--enable-dp-attention`):** Recommended for high-throughput scenarios with large batch sizes. Reduces KV-cache duplication across TP ranks. Use `--enable-dp-attention --tp 8 --dp 8` on a single 8-GPU node. Not recommended for low-latency, small-batch workloads. + +**NCCL timeout:** If model loading is slow and you hit an NCCL timeout, increase it: `--dist-timeout 3600`. ## 4. Model Invocation @@ -64,7 +104,7 @@ python -m sglang.launch_server \ from openai import OpenAI client = OpenAI( - base_url="http://localhost:8000/v1", + base_url="http://localhost:30000/v1", api_key="EMPTY" ) @@ -156,17 +196,35 @@ python -m sglang.launch_server \ --chat-template ./examples/chat_template/tool_chat_template_deepseekv3.jinja \ --tp 8 \ --host 0.0.0.0 \ - --port 8000 + --port 30000 ``` +**Quick Test (curl):** + +```shell Command +curl "http://127.0.0.1:30000/v1/chat/completions" \ + -H "Content-Type: application/json" \ + -d '{ + "temperature": 0, + "max_tokens": 100, + "model": "deepseek-ai/DeepSeek-V3", + "tools": [{"type": "function", "function": {"name": "query_weather", "description": "Get weather of a city", "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}}}], + "messages": [{"role": "user", "content": "How'\''s the weather in Beijing today?"}] + }' +``` + + +Use a low `temperature` (e.g. `0`) for more consistent tool call results. The `--chat-template` flag above provides an improved unified prompt for tool use. + + **Python Example (with Thinking Process):** ```python Example from openai import OpenAI client = OpenAI( - base_url="http://localhost:8000/v1", + base_url="http://localhost:30000/v1", api_key="EMPTY" ) @@ -261,11 +319,8 @@ print() **Output Example:** ```text Output -=============== Thinking ================= -<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather -```json -{"location": "Beijing", "unit": "celsius"} -```<｜tool▁call▁end｜><｜tool▁calls▁end｜> +🔧 Tool Call: get_weather + Arguments: {"location": "Beijing", "unit": "celsius"} ``` **Note:** @@ -316,6 +371,64 @@ print(final_response.choices[0].message.content) # Output: "The weather in Beijing is currently 22°C and sunny." ``` +#### 4.2.3 Multi-Token Prediction (EAGLE Speculative Decoding) + +SGLang implements DeepSeek V3 Multi-Token Prediction (MTP) based on [EAGLE speculative decoding](../../../docs/advanced_features/speculative_decoding#EAGLE-Decoding). With this optimization, decoding speed improves by up to **1.8×** at batch size 1 and **1.5×** at batch size 32 on H200 TP8. + +**Enable with:** + +```shell Command +python3 -m sglang.launch_server \ + --model-path deepseek-ai/DeepSeek-V3-0324 \ + --speculative-algorithm EAGLE \ + --trust-remote-code \ + --tp 8 +``` + +The default configuration is `--speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4`. Find the best values for your workload with [bench_speculative.py](https://github.com/sgl-project/sglang/blob/main/scripts/playground/bench_speculative.py). The minimum viable config is `--speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2`. + + +For large batch sizes (>48), increase `--max-running-requests` beyond the default of 48 for MTP. Also set `--cuda-graph-bs` to include your target batch sizes (default captured sizes for speculative decoding: 48). + + + +The spec-v2 overlap scheduler is enabled by default (`SGLANG_ENABLE_SPEC_V2=True`). It improves performance by overlapping draft and verification stages. Set `SGLANG_ENABLE_SPEC_V2=0` to disable. + + +#### 4.2.4 MLA Optimizations + +DeepSeek V3 uses [Multi-head Latent Attention (MLA)](https://arxiv.org/pdf/2405.04434), an attention mechanism that improves inference efficiency. SGLang implements several optimizations: + +- **Weight Absorption:** Reorders matrix multiplications to improve decoding phase efficiency. +- **MLA Attention Backends:** FA3, Flashinfer, FlashMLA, CutlassMLA, TRTLLM MLA (Blackwell), and Triton. FA3 is the default. +- **FP8 Quantization:** W8A8 FP8 and KV Cache FP8, with BMM operators for weight-absorbed MLA in FP8. +- **CUDA Graph & Torch.compile:** Both MLA and MoE support CUDA Graph and Torch.compile for reduced decoding latency. +- **Chunked Prefix Cache:** Increases throughput for long-sequence chunked prefill (FlashAttention3 backend only). + +Overall, these optimizations achieve up to **7×** output throughput improvement vs. the baseline. + +**Reference:** See [SGLang v0.3 blog](https://lmsys.org/blog/2024-09-04-sglang-v0-3/#deepseek-multi-head-latent-attention-mla-throughput-optimizations) and [Slides](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/lmsys_1st_meetup_deepseek_mla.pdf) for details. + +#### 4.2.5 Multi-Node Deployment + +For multi-node serving and hardware-specific examples: + +- [8× H200 / 4–8× B200](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#using-docker-recommended) +- [8× MI300X](../../../docs/hardware-platforms/amd_gpu#running-deepseek-v3) +- [2×8× H200 with Docker](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-two-h2008-nodes-and-docker) +- [4×8× A100](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-four-a1008-nodes) +- [8× A100 AWQ](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-8-a100a800-with-awq-quantization) +- [16× A100 INT8](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-16-a100a800-with-int8-quantization) +- [32× L40S INT8](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-32-l40s-with-int8-quantization) +- [Xeon 6980P CPU](../../../docs/hardware-platforms/cpu_server#example-running-deepseek-r1) +- [4× Atlas 800I A3 (int8)](../../../docs/hardware-platforms/ascend-npus/ascend_npu_deepseek_example#running-deepseek-with-pd-disaggregation-on-4-x-atlas-800i-a3) + +**Blog references for large-scale deployment:** + +- [Deploying DeepSeek on GB200 NVL72 with PD and Large Scale EP](https://lmsys.org/blog/2025-06-16-gb200-part-1/) ([Part I](https://lmsys.org/blog/2025-06-16-gb200-part-1/), [Part II](https://lmsys.org/blog/2025-09-25-gb200-part-2/)) +- [PD Disaggregation and Large-Scale Expert Parallelism on 96× H100](https://lmsys.org/blog/2025-05-05-large-scale-ep/) +- [Best Practices for Serving DeepSeek-R1 on H20](https://lmsys.org/blog/2025-09-26-sglang-ant-group/) + ## 5. Benchmark ### 5.1 Speed Benchmark diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_1.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_1.mdx index 1b4376a6c..1870e1b1e 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_1.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_1.mdx @@ -31,7 +31,49 @@ import { DeepSeekV31Deployment } from "/src/snippets/autoregressive/deepseek-v31 ### 3.2 Configuration Tips -For more detailed configuration tips, please refer to [DeepSeek V3/V3.1/R1 Usage](../../../docs/basic_usage/deepseek_v3). + +DeepSeek-V3.1 shares the same model architecture as DeepSeek-V3, so the same hardware and optimization recommendations apply. + +**Recommended GPU configurations by weight type:** + + + + + + + + + + + + + + + + + + + + + + + + + + +
Weight TypeSupported Hardware
FP8 (recommended)8× H200, 8× B200, 8× MI300X, 2×8× H100/H800/H20
BF16 (upcast from FP8)2×8× H200, 2×8× MI300X, 4×8× H100/H800, 4×8× A100/A800
INT816× A100/A800, 32× L40S, Xeon 6980P CPU, 4× Atlas 800I A3
W4A8 / AWQ / MXFP4 / NVFP48× H20/H100, 4× H200; 8× H100/A100; 8/4× MI355X/MI350X; 8/4× B200
+ +> The official DeepSeek-V3.1 checkpoint is already in FP8 format — do **not** add `--quantization fp8` when serving it. + +**DeepGEMM precompilation (NVIDIA Hopper / Blackwell):** Precompile GEMM kernels before the first server run to avoid JIT overhead (~10 min): +```bash +python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3.1 --tp 8 --trust-remote-code +``` +DeepGEMM is enabled by default on Hopper/Blackwell and can be disabled with `SGLANG_ENABLE_JIT_DEEPGEMM=0`. + +**Data Parallelism Attention (`--enable-dp-attention`):** Recommended for high-throughput scenarios with large batch sizes. Reduces KV-cache duplication across TP ranks. Use `--enable-dp-attention --tp 8 --dp 8` on a single 8-GPU node. Not recommended for low-latency, small-batch workloads. + +**NCCL timeout:** If model loading is slow and you hit an NCCL timeout, increase it: `--dist-timeout 3600`. ## 4. Model Invocation @@ -148,8 +190,6 @@ Thus, 15% of 240 is 36. DeepSeek-V3.1 and DeepSeek-V3.1-Terminus support tool calling capabilities. Enable the tool call parser: -**Note:** DeepSeek-V3.1-Speciale does **NOT** support tool calling. It is designed exclusively for deep reasoning tasks. - **Deployment Command:** ```shell Command @@ -326,6 +366,10 @@ print(final_response.choices[0].message.content) # Output: "Currently, it is **22°C and sunny** in Beijing." ``` +#### 4.2.3 Multi-Token Prediction (EAGLE Speculative Decoding) + +DeepSeek-V3.1 shares the same architecture as DeepSeek-V3 and supports the same EAGLE-based MTP speculative decoding path. Refer to [DeepSeek-V3 §4.2.3](/cookbook/autoregressive/DeepSeek/DeepSeek-V3#4-2-3-multi-token-prediction-eagle-speculative-decoding) for the full configuration, tuning guidance, and `bench_speculative.py` reference. The `--speculative-num-steps`, `--speculative-eagle-topk`, and `--max-running-requests` recommendations apply equally to V3.1. + ## 5. Benchmark ### 5.1 Speed Benchmark diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx index 8ec0ca505..6c0d5c4d7 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx @@ -24,6 +24,27 @@ SGLang offers multiple installation methods. You can choose the most suitable in Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. +### 2.1 Docker Images + +Pre-built Docker images are available for different hardware platforms: + +```bash Command +# NVIDIA H200 / B200 +docker pull lmsysorg/sglang:latest + +# AMD MI350 / MI355X +docker pull lmsysorg/sglang:v0.5.8-rocm700-mi35x + +# AMD MI300X +# Note: v0.5.8-rocm700-mi30x does not include PR #17504. +# Prefer the newest MI30x ROCm image tag from Docker Hub when available, or build from source. +docker pull lmsysorg/sglang:v0.5.8-rocm700-mi30x + +# Ascend NPU (Atlas 800I A2 / A3) +docker pull lmsysorg/sglang:dsv32-a2 +docker pull lmsysorg/sglang:dsv32-a3 +``` + ## 3. Model Deployment This section provides a progressive guide from quick deployment to performance optimization, suitable for users at different levels. @@ -37,7 +58,18 @@ import { DeepSeekV32Deployment } from "/src/snippets/autoregressive/deepseek-v32 ### 3.2 Configuration Tips -For more detailed configuration tips, please refer to [DeepSeek-V3.2 Usage](../../../docs/basic_usage/deepseek_v32). + +- **Short-sequence MHA prefill (adaptive):** For prefill sequences shorter than 2048 tokens (default threshold), the DSA backend automatically switches to standard MHA (using FlashAttention variable-length on SM90, TRT-LLM ragged MHA on SM100). To extend this to longer sequences set env var `SGLANG_DSA_PREFILL_DENSE_ATTN_KV_LEN_THRESHOLD` to a larger value (potential minor accuracy trade-off). +- **DSA prefill/decode attention kernels (`--dsa-prefill-backend`, `--dsa-decode-backend`):** The `dsa` backend is automatically selected for DeepSeek-V3.2. Available kernels: `flashmla_sparse`, `flashmla_kv`, `flashmla_auto`, `fa3` (Hopper only), `tilelang` (GPU/HPU/NPU), `aiter` (AMD, decode only), `trtllm` (Blackwell only). Defaults: Hopper BF16 KV → `flashmla_sparse` prefill / `fa3` decode; Hopper FP8 KV → `flashmla_kv` both; Blackwell BF16 → `flashmla_sparse` / `trtllm`; Blackwell FP8 → `trtllm` both. +- **Index Cache:** Reuses indexer results across layers for efficiency at negligible accuracy cost. For **GLM-5** specifically, append `--json-model-override-args '{"index_topk_pattern": "FFSFSSSFSSFFFSSSFFFSFSSSSSSFFSFFSFFSSFFFFFFSFFFFFSFFSSSSSSFSFFFSFSSSFSFFSFFSSS"}'` for a better speed/accuracy tradeoff. +- **HiSparse (experimental):** Reduces per-request GPU memory during long-context decode by offloading KV data to CPU pinned memory. Requires PD disaggregation mode (decode instance only). See [HiSparse Guide](../../../docs/advanced_features/hisparse_guide). +- **NVFP4 on Blackwell:** Specify `--quantization modelopt_fp4` and `--moe-runner-backend flashinfer_trtllm` (recommended) / `flashinfer_cutlass` / `flashinfer_cutedsl`. Full example: + ```bash + python -m sglang.launch_server --model nvidia/DeepSeek-V3.2-NVFP4 --tp 4 \ + --quantization modelopt_fp4 --moe-runner-backend flashinfer_trtllm \ + --tool-call-parser deepseekv32 --reasoning-parser deepseek-v3 + ``` +- **NCCL timeout:** Slow model loading → add `--dist-timeout 3600`. ## 4. Model Invocation @@ -131,7 +163,15 @@ The answer is 36. To find 15% of 240, we multiply 240 by 0.15, which equals 36. DeepSeek-V3.2 and DeepSeek-V3.2-Exp support tool calling capabilities. But they use different parameters. Enable the tool call parser: -**Note:** DeepSeek-V3.2-Speciale does **NOT** support tool calling. It is designed exclusively for deep reasoning tasks. +**Note:** DeepSeek-V3.2-Speciale does **NOT** support tool calling. Launch it with reasoning parser only: + +```shell Command +python3 -m sglang.launch_server \ + --model-path deepseek-ai/DeepSeek-V3.2-Speciale \ + --trust-remote-code \ + --tp-size 8 --dp-size 8 --enable-dp-attention \ + --reasoning-parser deepseek-v3 +``` **Deployment Command:** @@ -306,7 +346,110 @@ print(final_response.choices[0].message.content) # Output: "The weather in Beijing is currently 22°C and sunny." ``` -#### 4.2.3 Enabling PP, CP and TP with FP8 KV cache +#### 4.2.3 Multi-Token Prediction (EAGLE Speculative Decoding) + +SGLang implements Multi-Token Prediction (MTP) for DeepSeek V3.2 based on [EAGLE speculative decoding](../../../docs/advanced_features/speculative_decoding#EAGLE-Decoding). This optimization significantly improves decoding speed for small batch sizes. + +**With DP Attention:** + +```bash Command +python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --dp 8 \ + --enable-dp-attention \ + --speculative-algorithm EAGLE \ + --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 +``` + +**With Pure TP:** + +```bash Command +python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 \ + --speculative-algorithm EAGLE \ + --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 +``` + +Find optimal values for your workload with [bench_speculative.py](https://github.com/sgl-project/sglang/blob/main/scripts/playground/bench_speculative.py). The minimum viable config is `--speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2`. + + +`--max-running-requests` defaults to 48 for MTP. Increase it for larger batch sizes. + + + +The spec-v2 overlap scheduler is enabled by default (`SGLANG_ENABLE_SPEC_V2=True`). Set `SGLANG_ENABLE_SPEC_V2=0` to disable. + + +#### 4.2.4 PD Disaggregation + +Prefill-Decode (PD) disaggregation separates prefill and decode stages onto different instances, improving GPU utilization for mixed workloads. + +**Prefill command:** + +```bash Command +python -m sglang.launch_server \ + --model-path deepseek-ai/DeepSeek-V3.2-Exp \ + --disaggregation-mode prefill \ + --host $LOCAL_IP \ + --port $PORT \ + --tp 8 \ + --dp 8 \ + --enable-dp-attention \ + --dist-init-addr ${HOST}:${DIST_PORT} \ + --trust-remote-code \ + --disaggregation-bootstrap-port 8998 \ + --mem-fraction-static 0.9 +``` + +**Decode command:** + +```bash Command +python -m sglang.launch_server \ + --model-path deepseek-ai/DeepSeek-V3.2-Exp \ + --disaggregation-mode decode \ + --host $LOCAL_IP \ + --port $PORT \ + --tp 8 \ + --dp 8 \ + --enable-dp-attention \ + --dist-init-addr ${HOST}:${DIST_PORT} \ + --trust-remote-code \ + --mem-fraction-static 0.9 +``` + +**Router command:** + +```bash Command +python -m sglang_router.launch_router --pd-disaggregation \ + --prefill $PREFILL_ADDR 8998 \ + --decode $DECODE_ADDR \ + --host 127.0.0.1 \ + --port 30000 +``` + +For production deployments (RBG / LWS-based, DeepEP EP parallelism), see [multi_node_deployment docs](../../../docs/references/multi_node_deployment/rbg_pd/deepseekv32_pd). + +#### 4.2.5 DSA Long-Sequence Context Parallel and PP/CP + +SGLang provides two context parallel (CP) modes for long-sequence workloads, controlled with `--dsa-prefill-cp-mode`. + +**In-sequence splitting** (`--dsa-prefill-cp-mode in-seq-split`): Each CP rank handles a uniform shard of the sequence; KV cache is gathered via all-gather. Batch size is restricted to 1 during prefill. See [PR #12065](https://github.com/sgl-project/sglang/pull/12065). + +```bash Command +# In-seq splitting mode — EP + DP, batch size 1 +python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp \ + --tp 8 --ep 8 --dp 2 --enable-dp-attention \ + --enable-dsa-prefill-context-parallel --attn-cp-size 4 \ + --dsa-prefill-cp-mode in-seq-split --max-running-requests 32 +``` + +**Round-robin splitting** (`--dsa-prefill-cp-mode round-robin-split`, default): Distributes tokens by `token_idx % cp_size`. Supports fused MoE, FP8 KV cache, and multi-batch prefill. Cannot be combined with DP attention. See [PR #13959](https://github.com/sgl-project/sglang/pull/13959). + +```bash Command +# Round-robin splitting — FusedMoE + CP8 +python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp \ + --tp 8 --enable-dsa-prefill-context-parallel --attn-cp-size 8 \ + --dsa-prefill-cp-mode round-robin-split --max-running-requests 32 +``` + +**PP + CP (multi-node):** Combines Pipeline Parallelism and Context Parallelism for cross-node scaling. The production-optimized configurations below have been verified on Hopper: We suggested `DP2` + `MTP` for local deployment of agentic workflow with DeepSeek V3.2 on Hopper platform: @@ -595,6 +738,27 @@ python3 -m sglang.test.few_shot_gsm8k --num-questions 200 --port 30000 Output throughput: 965.919 token/s ``` +- **Full GSM8K (1319 questions)** — for a stricter accuracy check, run the full set 8-shot: + +```shell Command +python3 benchmark/gsm8k/bench_sglang.py --num-shots 8 --num-questions 1319 --parallel 1319 +``` + + - 8-shot: + ``` + Accuracy: 0.956 + Invalid: 0.000 + Latency: 25.109 s + Output throughput: 5226.235 token/s + ``` + - 20-shot (long-context; stays close to the 8-shot result): + ``` + Accuracy: 0.956 + Invalid: 0.000 + Latency: 29.545 s + Output throughput: 4418.617 token/s + ``` + #### 5.2.2 MMLU Benchmark - **Benchmark Command:** @@ -622,6 +786,83 @@ python3 benchmark/mmlu/bench_sglang.py --nsub 10 --port 30000 Average accuracy: 0.879 ``` +#### 5.2.3 GPQA-Diamond Benchmark + +- **Benchmark Command:** + +```shell Command +python3 -m sglang.test.run_eval --port 30000 --eval-name gpqa --num-examples 198 --max-tokens 128000 --repeat 8 --thinking-mode deepseek-v3 +``` + +- **Test Results** (model: `deepseek-ai/DeepSeek-V3.2-Exp`, 8×B200): + - Default (`temperature=0`): mean **0.797** over 8 runs — closely matches the official GPQA-Diamond score of **79.9** for DeepSeek-V3.2-Exp reported in its [model card](https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp) + - With `temperature=1.0, top_p=0.95` (as recommended by DeepSeek): + +```shell Command +python3 -m sglang.test.run_eval --port 30000 --eval-name gpqa --num-examples 198 --max-tokens 128000 --repeat 8 --top-p 0.95 --temperature 1.0 --thinking-mode deepseek-v3 +``` + + ``` + Repeat: 8, mean: 0.840 + Scores: ['0.848', '0.808', '0.848', '0.838', '0.879', '0.813', '0.838', '0.848'] + ``` + +#### 5.2.4 AIME 2025 Benchmark + +Results on AIME 2025 (8×B200), evaluated with [NeMo-Skills](https://github.com/NVIDIA/NeMo-Skills.git): + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Modelpass@1 avg-of-4majority@4pass@4
DeepSeek-V3.2-Exp87.50% ± 1.67%90.00%90.00%
DeepSeek-V3.292.50% ± 1.67%94.71%96.67%
DeepSeek-V3.2-Speciale95.00% ± 1.92%95.83%100.00%
+ +**Reproduction.** Install [NeMo-Skills](https://github.com/NVIDIA/NeMo-Skills), launch the server with the tool-call and reasoning parsers, then run `ns eval`: + +```bash Command +pip install git+https://github.com/NVIDIA/NeMo-Skills.git --ignore-installed blinker + +export NEMO_SKILLS_DISABLE_UNCOMMITTED_CHANGES_CHECK=1 +ns prepare_data aime25 + +ns eval \ + --benchmarks=aime25:4 \ + --server_type=sglang \ + --model=deepseek-ai/DeepSeek-V3.2-Exp \ + --server_address=http://localhost:30000/v1 \ + --output_dir=nemo_skills_aime25_output \ + ++chat_template_kwargs.thinking=true \ + ++inference.temperature=1.0 \ + ++inference.top_p=0.95 \ + ++inference.tokens_to_generate=64000 + # Use ++inference.tokens_to_generate=120000 for the DeepSeek-V3.2-Speciale model +``` + ### 5.3 Speed Benchmark on Hopper **Test Environment:** diff --git a/docs_new/cookbook/autoregressive/GLM/GLM-4.5.mdx b/docs_new/cookbook/autoregressive/GLM/GLM-4.5.mdx index fe03bd8da..800e740eb 100644 --- a/docs_new/cookbook/autoregressive/GLM/GLM-4.5.mdx +++ b/docs_new/cookbook/autoregressive/GLM/GLM-4.5.mdx @@ -44,7 +44,8 @@ import { GLM45Deployment } from "/src/snippets/autoregressive/glm-45-deployment. ### 3.2 Configuration Tips -For more detailed configuration tips, please refer to [GLM-4.5/GLM-4.6 Usage](../../../docs/basic_usage/glm45). +- **EAGLE Speculative Decoding:** Supported for GLM-4.5/4.6. Add `--speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4`. The spec-v2 overlap scheduler is enabled by default (`SGLANG_ENABLE_SPEC_V2=True`); set `SGLANG_ENABLE_SPEC_V2=0` to disable. +- **Thinking Budget:** Use `--enable-custom-logit-processor` flag and pass `Glm4MoeThinkingBudgetLogitProcessor` in requests to cap the model's thinking token count (see section 4.2.3). ## 4. Model Invocation @@ -135,6 +136,10 @@ The answer is 36. To find 15% of 240, we multiply 240 by 0.15, which equals 36. #### 4.2.2 Tool Calling + +**Parser names by model:** GLM-4.5 and GLM-4.6 use `--tool-call-parser glm45`. GLM-4.7 and GLM-4.7-Flash use `--tool-call-parser glm47`. All GLM models use `--reasoning-parser glm45` regardless of generation. + + GLM-4.5 supports tool calling capabilities. Enable the tool call parser: ```shell Command @@ -241,6 +246,27 @@ Tool Call: get_weather Arguments: {"location": "Beijing", "unit": "celsius"} ``` +#### 4.2.3 Thinking Budget + +Limit the number of thinking tokens using `CustomLogitProcessor`. Launch with `--enable-custom-logit-processor`: + +```python Example +import openai +from sglang.srt.sampling.custom_logit_processor import Glm4MoeThinkingBudgetLogitProcessor + +client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="*") +response = client.chat.completions.create( + model="zai-org/GLM-4.5", + messages=[{"role": "user", "content": "Is Paris the Capital of France?"}], + max_tokens=1024, + extra_body={ + "custom_logit_processor": Glm4MoeThinkingBudgetLogitProcessor().to_str(), + "custom_params": {"thinking_budget": 512}, + }, +) +print(response) +``` + ## 5. Benchmark This section uses **industry-standard configurations** for comparable benchmark results. diff --git a/docs_new/cookbook/autoregressive/GLM/GLM-4.5V.mdx b/docs_new/cookbook/autoregressive/GLM/GLM-4.5V.mdx index 3bd305d43..25560c1f0 100644 --- a/docs_new/cookbook/autoregressive/GLM/GLM-4.5V.mdx +++ b/docs_new/cookbook/autoregressive/GLM/GLM-4.5V.mdx @@ -42,7 +42,35 @@ import { GLM45VDeployment } from "/src/snippets/autoregressive/glm-45v-deploymen - **TTFT Optimization** : Set `SGLANG_USE_CUDA_IPC_TRANSPORT=1` to use CUDA IPC for transferring multimodal features, which significantly improves TTFT. This consumes additional memory and may require adjusting `--mem-fraction-static` and/or `--max-running-requests`. (additional memory is proportional to image size * number of images in current running requests.) - **TP=8 Configuration**: When using Tensor Parallelism (TP) of 8, the vision attention's 12 heads cannot be evenly divided. You can resolve this by adding `--mm-enable-dp-encoder`. - **Fast Model Loading**: For large models (like the 106B version), you can speed up model loading by using `--model-loader-extra-config='{"enable_multithread_load": "true","num_threads": 64}'`. -- For more detailed configuration tips, please refer to [GLM-4.5V/GLM-4.6V Usage](../../../docs/basic_usage/glmv). +- **Hardware Notes:** + - **H100 (FP8):** Use the FP8 checkpoint for best memory efficiency. + - **A100 / H100 (BF16):** Use standard multimodal parameters to manage throughput and GPU memory usage. + - **H200 / B200:** Runs out of the box, supporting full context length plus concurrent image + video processing. +- **Additional Multimodal Parameters:** + - `--mm-attention-backend fa3`: Specify multimodal attention backend (Flash Attention 3). + - `--keep-mm-feature-on-device`: Retain multimodal feature tensors on GPU after processing to avoid D2H memory copies. + - `SGLANG_USE_CUDA_IPC_TRANSPORT=1`: Use CUDA IPC shared memory for multimodal data transport to significantly improve E2E latency. + +**Example with full multimodal optimizations:** +```bash Command +SGLANG_USE_CUDA_IPC_TRANSPORT=1 \ +SGLANG_VLM_CACHE_SIZE_MB=0 \ +python -m sglang.launch_server \ + --model-path zai-org/GLM-4.5V \ + --host 0.0.0.0 \ + --port 30000 \ + --trust-remote-code \ + --tp-size 8 \ + --enable-cache-report \ + --log-level info \ + --max-running-requests 64 \ + --mem-fraction-static 0.65 \ + --chunked-prefill-size 8192 \ + --attention-backend fa3 \ + --mm-attention-backend fa3 \ + --mm-enable-dp-encoder \ + --enable-metrics +``` ## 4. Model Invocation @@ -470,6 +498,27 @@ print(final_response.choices[0].message.content) # Output: "The weather in Beijing is currently 22°C and sunny." ``` +#### 4.2.4 Thinking Budget + +Beyond enabling/disabling the full reasoning mode (section 4.2.2), you can cap the number of thinking tokens using `CustomLogitProcessor`. Launch with `--enable-custom-logit-processor` and pass `Glm4MoeThinkingBudgetLogitProcessor` in the request: + +```python Example +import openai +from sglang.srt.sampling.custom_logit_processor import Glm4MoeThinkingBudgetLogitProcessor + +client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="*") +response = client.chat.completions.create( + model="zai-org/GLM-4.5V", + messages=[{"role": "user", "content": "Describe this image briefly."}], + max_tokens=1024, + extra_body={ + "custom_logit_processor": Glm4MoeThinkingBudgetLogitProcessor().to_str(), + "custom_params": {"thinking_budget": 512}, + }, +) +print(response) +``` + ## 5. Benchmark ### 5.1 Accuracy Benchmark diff --git a/docs_new/cookbook/autoregressive/GLM/GLM-4.6.mdx b/docs_new/cookbook/autoregressive/GLM/GLM-4.6.mdx index dfefd313a..d7fd1a8a3 100644 --- a/docs_new/cookbook/autoregressive/GLM/GLM-4.6.mdx +++ b/docs_new/cookbook/autoregressive/GLM/GLM-4.6.mdx @@ -38,7 +38,8 @@ import { GLM46Deployment } from "/src/snippets/autoregressive/glm-46-deployment. ### 3.2 Configuration Tips -For more detailed configuration tips, please refer to [GLM-4.5/GLM-4.6 Usage](../../../docs/basic_usage/glm45). +- **EAGLE Speculative Decoding:** Supported for GLM-4.5/4.6. Add `--speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4`. The spec-v2 overlap scheduler is enabled by default (`SGLANG_ENABLE_SPEC_V2=True`); set `SGLANG_ENABLE_SPEC_V2=0` to disable. +- **Thinking Budget:** Use `--enable-custom-logit-processor` flag and pass `Glm4MoeThinkingBudgetLogitProcessor` in requests to cap the model's thinking token count (see section 4.2.3). ## 4. Model Invocation @@ -129,6 +130,10 @@ The answer is 36. To find 15% of 240, we multiply 240 by 0.15, which equals 36. #### 4.2.2 Tool Calling + +**Parser names by model:** GLM-4.5 and GLM-4.6 use `--tool-call-parser glm45`. GLM-4.7 and GLM-4.7-Flash use `--tool-call-parser glm47`. All GLM models use `--reasoning-parser glm45` regardless of generation. + + GLM-4.6 supports tool calling capabilities. Enable the tool call parser: ```shell Command @@ -281,6 +286,27 @@ print(final_response.choices[0].message.content) # Output: "The weather in Beijing is currently 22°C and sunny." ``` +#### 4.2.3 Thinking Budget + +Limit the number of thinking tokens using `CustomLogitProcessor`. Launch with `--enable-custom-logit-processor`: + +```python Example +import openai +from sglang.srt.sampling.custom_logit_processor import Glm4MoeThinkingBudgetLogitProcessor + +client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="*") +response = client.chat.completions.create( + model="zai-org/GLM-4.6", + messages=[{"role": "user", "content": "Is Paris the Capital of France?"}], + max_tokens=1024, + extra_body={ + "custom_logit_processor": Glm4MoeThinkingBudgetLogitProcessor().to_str(), + "custom_params": {"thinking_budget": 512}, + }, +) +print(response) +``` + ## 5. Benchmark This section uses **industry-standard configurations** for comparable benchmark results. diff --git a/docs_new/cookbook/autoregressive/GLM/GLM-4.6V.mdx b/docs_new/cookbook/autoregressive/GLM/GLM-4.6V.mdx index a0fc9d294..2c101cb71 100644 --- a/docs_new/cookbook/autoregressive/GLM/GLM-4.6V.mdx +++ b/docs_new/cookbook/autoregressive/GLM/GLM-4.6V.mdx @@ -10,7 +10,7 @@ GLM-4.6V series model includes two versions: GLM-4.6V (106B), a foundation model Beyond achieves SoTA performance across major multimodal benchmarks at comparable model scales. GLM-4.6V introduces several key features: -- **Native Multimodal Function Calling** Enables native vision-driven tool use. Images, screenshots, and document pages can be passed directly as tool inputs without text conversion, while visual outputs (charts, search images, rendered pages) are interpreted and integrated into the reasoning chain. This closes the loop from perception to understanding to execution. Please refer to this [example](#tool-call-example). +- **Native Multimodal Function Calling** Enables native vision-driven tool use. Images, screenshots, and document pages can be passed directly as tool inputs without text conversion, while visual outputs (charts, search images, rendered pages) are interpreted and integrated into the reasoning chain. This closes the loop from perception to understanding to execution. Please refer to this [example](#4-2-3-tool-calling). - **Interleaved Image-Text Content Generation** Supports high-quality mixed media creation from complex multimodal inputs. GLM-4.6V takes a multimodal context—spanning documents, user inputs, and tool-retrieved images—and synthesizes coherent, interleaved image-text content tailored to the task. During generation it can actively call search and retrieval tools to gather and curate additional text and visuals, producing rich, visually grounded content. - **Multimodal Document Understanding** GLM-4.6V can process up to 128K tokens of multi-document or long-document input, directly interpreting richly formatted pages as images. It understands text, layout, charts, tables, and figures jointly, enabling accurate comprehension of complex, image-heavy documents without requiring prior conversion to plain text. - **Frontend Replication & Visual Editing** Reconstructs pixel-accurate HTML/CSS from UI screenshots and supports natural-language-driven edits. It detects layout, components, and styles visually, generates clean code, and applies iterative visual modifications through simple user instructions. @@ -70,14 +70,56 @@ import { GLM46VDeployment } from "/src/snippets/autoregressive/glm-46v-deploymen - **TTFT Optimization** : Set `SGLANG_USE_CUDA_IPC_TRANSPORT=1` to use CUDA IPC for transferring multimodal features, which significantly improves TTFT. This consumes additional memory and may require adjusting `--mem-fraction-static` and/or `--max-running-requests`. (additional memory is proportional to image size * number of images in current running requests.) - **TP=8 Configuration**: When using Tensor Parallelism (TP) of 8, the vision attention's 12 heads cannot be evenly divided. You can resolve this by adding `--mm-enable-dp-encoder` (which the generator above handles automatically). - **Fast Model Loading**: For large models (like the 106B version), you can speed up model loading by using `--model-loader-extra-config='{"enable_multithread_load": "true","num_threads": 64}'`. -- For more detailed configuration tips, please refer to [GLM-4.5V/GLM-4.6V Usage](../../../docs/basic_usage/glmv). +- **Hardware Notes:** + - **H100 (FP8):** Use the FP8 checkpoint for best memory efficiency. + - **A100 / H100 (BF16):** Use standard multimodal parameters to manage throughput and GPU memory usage. + - **H200 / B200:** Runs out of the box, supporting full context length plus concurrent image + video processing. +- **Additional Multimodal Parameters:** + - `--mm-attention-backend fa3`: Specify multimodal attention backend (Flash Attention 3). + - `--keep-mm-feature-on-device`: Retain multimodal feature tensors on GPU after processing to avoid D2H memory copies. + - `SGLANG_USE_CUDA_IPC_TRANSPORT=1`: Use CUDA IPC shared memory for multimodal data transport to significantly improve E2E latency. -## 4. Example APIs +**Example with full multimodal optimizations:** +```bash Command +SGLANG_USE_CUDA_IPC_TRANSPORT=1 \ +SGLANG_VLM_CACHE_SIZE_MB=0 \ +python -m sglang.launch_server \ + --model-path zai-org/GLM-4.6V \ + --host 0.0.0.0 \ + --port 30000 \ + --trust-remote-code \ + --tp-size 8 \ + --enable-cache-report \ + --log-level info \ + --max-running-requests 64 \ + --mem-fraction-static 0.65 \ + --chunked-prefill-size 8192 \ + --attention-backend fa3 \ + --mm-attention-backend fa3 \ + --mm-enable-dp-encoder \ + --enable-metrics +``` -### Image Input Example +## 4. Model Invocation + +### 4.1 Basic Usage + +For basic API usage and request examples, please refer to: + +- [SGLang Basic Usage Guide](../../../docs/basic_usage/send_request) +- [SGLang OpenAI Vision API Guide](../../../docs/basic_usage/openai_api_vision) + +### 4.2 Advanced Usage + +#### 4.2.1 Multi-Modal Inputs + +GLM-4.6V supports image and video inputs via the OpenAI-compatible API. + +**Image Input:** -#### API Payload ```python Example +import subprocess + curl_command = f""" curl -s http://localhost:{30000}/v1/chat/completions \\ -H "Content-Type: application/json" \\ @@ -90,7 +132,7 @@ curl -s http://localhost:{30000}/v1/chat/completions \\ {{ "type": "image_url", "image_url": {{ - "url": "/home/jobuser/sgl_logo.png" + "url": "https://github.com/sgl-project/sglang/blob/main/examples/assets/example_image.png?raw=true" }} }}, {{ @@ -110,15 +152,15 @@ response = subprocess.check_output(curl_command, shell=True).decode() print(response) ``` -#### API Response ```text Output {"id":"b61596ca71394dd699fd8abd4f650c44","object":"chat.completion","created":1765259019,"model":"default","choices":[{"index":0,"message":{"role":"assistant","content":"The image is a logo featuring the text \"SGL\" (in a bold, orange-brown font) alongside a stylized icon. The icon includes a network-like structure with circular nodes (suggesting connectivity or a tree/graph structure) and a tag with \"\" (a common symbol for coding, web development, or software). The color scheme uses warm orange-brown tones with a black background, giving it a tech-focused, modern aesthetic (likely representing a company, project, or tool related to software, web development, or digital technology).<|begin_of_box|>SGL logo (stylized text + network/coding icon)<|end_of_box|>","reasoning_content":"Okay, let's see. The image has a logo with the text \"SGL\" and a little icon on the left. The icon looks like a network or a tree structure with circles, and there's a tag with \"\" which is a common symbol for coding or web development. The colors are orange and brown tones, with a black background. So probably a logo for a company or project named SGL, maybe related to software, web development, or a tech company.","tool_calls":null},"logprobs":null,"finish_reason":"stop","matched_stop":151336}],"usage":{"prompt_tokens":2222,"total_tokens":2448,"completion_tokens":226,"prompt_tokens_details":null,"reasoning_tokens":0},"metadata":{"weight_version":"default"}} ``` -### Video Input Example +**Video Input:** -#### API Payload ```python Example +import subprocess + curl_command = f""" curl -s http://localhost:{30000}/v1/chat/completions \\ -H "Content-Type: application/json" \\ @@ -131,12 +173,12 @@ curl -s http://localhost:{30000}/v1/chat/completions \\ {{ "type": "video_url", "video_url": {{ - "url": "/home/jobuser/jobs_presenting_ipod.mp4" + "url": "https://github.com/sgl-project/sgl-test-files/raw/refs/heads/main/videos/jobs_presenting_ipod.mp4" }} }}, {{ "type": "text", - "text": "What is the image" + "text": "What is in the video" }} ] }} @@ -151,26 +193,93 @@ response = subprocess.check_output(curl_command, shell=True).decode() print(response) ``` -#### API Response ```text Output {"id":"520e0a079e5d4b17b82a6af619315a97","object":"chat.completion","created":1765259029,"model":"default","choices":[{"index":0,"message":{"role":"assistant","content":"The image is a still from a presentation by a man on a stage. He is pointing to a small pocket on his jeans and asking the audience what the pocket is for. The video is being shared by Evan Carmichael. The man then reveals that the pocket is for an iPod Nano.","reasoning_content":"Based on the visual evidence in the video, here is a breakdown of what is being shown:\n\n* **Subject:** The video features a man on a stage, giving a presentation. He is wearing a black t-shirt and dark jeans.\n* **Action:** The man is pointing to a pocket on his jeans. He is asking the audience a question about the purpose of this pocket.\n* **Context:** The presentation is being filmed, and the video is being shared by \"Evan Carmichael,\" a well-known motivational speaker and content creator. The source of the clip is credited to \"JoshuaG.\"\n* **Reveal:** The man then reveals the answer to his question. He pulls a small, white, rectangular device out of the pocket. He identifies this device as an \"iPod Nano.\"\n\nIn summary, the image is a still from a presentation where a speaker is explaining the purpose of the small pocket found on many pairs of jeans.","tool_calls":null},"logprobs":null,"finish_reason":"stop","matched_stop":151336}],"usage":{"prompt_tokens":30276,"total_tokens":30532,"completion_tokens":256,"prompt_tokens_details":null,"reasoning_tokens":0},"metadata":{"weight_version":"default"}} ``` -### Tool Call Example +#### 4.2.2 Thinking Mode + +GLM-4.6V supports Thinking mode. Enable the reasoning parser during deployment: + +```shell Command +python -m sglang.launch_server \ + --model zai-org/GLM-4.6V \ + --reasoning-parser glm45 \ + --tp 8 \ + --host 0.0.0.0 \ + --port 30000 +``` + +**Streaming with Thinking Process:** -#### API Payload ```python Example from openai import OpenAI -import argparse -import sys -import base64 -def image_to_base64(image_path): - """Convert image file to base64 data URL format for OpenAI API""" - with open(image_path, 'rb') as image_file: - image_data = image_file.read() - base64_string = base64.b64encode(image_data).decode('utf-8') - return f"data:image/png;base64,{base64_string}" +client = OpenAI( + base_url="http://localhost:30000/v1", + api_key="EMPTY" +) + +# Enable streaming to see the thinking process in real-time +response = client.chat.completions.create( + model="zai-org/GLM-4.6V", + messages=[ + {"role": "user", "content": "Solve this problem step by step: What is 15% of 240?"} + ], + temperature=0.7, + max_tokens=2048, + stream=True +) + +# Process the stream +has_thinking = False +has_answer = False +thinking_started = False + +for chunk in response: + if chunk.choices and len(chunk.choices) > 0: + delta = chunk.choices[0].delta + + # Print thinking process + if hasattr(delta, 'reasoning_content') and delta.reasoning_content: + if not thinking_started: + print("=============== Thinking =================", flush=True) + thinking_started = True + has_thinking = True + print(delta.reasoning_content, end="", flush=True) + + # Print answer content + if delta.content: + # Close thinking section and add content header + if has_thinking and not has_answer: + print("\n=============== Content =================", flush=True) + has_answer = True + print(delta.content, end="", flush=True) + +print() +``` + +**Output Example:** + +```text Output +=============== Thinking ================= +To solve this problem, I need to calculate 15% of 240. +Step 1: Convert 15% to decimal: 15% = 0.15 +Step 2: Multiply 240 by 0.15 +Step 3: 240 × 0.15 = 36 +=============== Content ================= + +The answer is 36. To find 15% of 240, we multiply 240 by 0.15, which equals 36. +``` + +**Note:** The reasoning parser captures the model's step-by-step thinking process, allowing you to see how the model arrives at its conclusions. + +#### 4.2.3 Tool Calling + +GLM-4.6V supports tool calling with vision capabilities. Pass tools in your API request: + +```python Example +from openai import OpenAI openai_api_key = "EMPTY" openai_api_base = "http://127.0.0.1:30000/v1" @@ -203,7 +312,7 @@ tools = [ messages = [ { "role": "user", - "content": "Please help me check today’s weather in Beijing, and tell me whether the tool returned an image." + "content": "Please help me check today's weather in Beijing, and tell me whether the tool returned an image." }, { "role": "assistant", @@ -229,7 +338,7 @@ messages = [ { "type": "image_url", "image_url": { - "url": "/home/jobuser/sgl_logo.png" + "url": "https://github.com/sgl-project/sglang/blob/main/examples/assets/example_image.png?raw=true" } } ] @@ -245,7 +354,7 @@ response = client.chat.completions.create( print(response.choices[0].message.content.strip()) ``` -#### Output +**Output Example:** ```text Output The weather in Beijing today (November 7, 2025) is sunny with a temperature of 2°C. @@ -253,6 +362,27 @@ The weather in Beijing today (November 7, 2025) is sunny with a temperature of 2 Yes, the tool returned an image (the SGL logo). ``` +#### 4.2.4 Thinking Budget + +Beyond the reasoning parser, you can cap the number of thinking tokens using `CustomLogitProcessor`. Launch with `--enable-custom-logit-processor` and pass `Glm4MoeThinkingBudgetLogitProcessor` in the request — same as the [GLM-4.6 text model approach](./GLM-4.6#4-2-3-thinking-budget): + +```python Example +import openai +from sglang.srt.sampling.custom_logit_processor import Glm4MoeThinkingBudgetLogitProcessor + +client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="*") +response = client.chat.completions.create( + model="zai-org/GLM-4.6V", + messages=[{"role": "user", "content": "Describe this image briefly."}], + max_tokens=1024, + extra_body={ + "custom_logit_processor": Glm4MoeThinkingBudgetLogitProcessor().to_str(), + "custom_params": {"thinking_budget": 512}, + }, +) +print(response) +``` + ## 5. Benchmark ### 5.1. Text Benchmark: Latency, Throughput and Accuracy diff --git a/docs_new/cookbook/autoregressive/GLM/GLM-4.7-Flash.mdx b/docs_new/cookbook/autoregressive/GLM/GLM-4.7-Flash.mdx index 76bff2893..2fc3b4e01 100644 --- a/docs_new/cookbook/autoregressive/GLM/GLM-4.7-Flash.mdx +++ b/docs_new/cookbook/autoregressive/GLM/GLM-4.7-Flash.mdx @@ -53,7 +53,7 @@ import { GLM47FlashDeployment } from "/src/snippets/autoregressive/glm-47-flash- ### 3.2 Configuration Tips -For more detailed configuration tips, please refer to [GLM-4.7 Usage](../../../docs/basic_usage/glm45). +- **EAGLE Speculative Decoding:** Supported for GLM-4.7-Flash. Add `--speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4`. The spec-v2 overlap scheduler is enabled by default (`SGLANG_ENABLE_SPEC_V2=True`); set `SGLANG_ENABLE_SPEC_V2=0` to disable. Enable via the interactive command generator above. ## 4. Model Invocation @@ -145,6 +145,10 @@ The answer is 36. To find 15% of 240, we multiply 240 by 0.15, which equals 36. #### 4.2.2 Tool Calling + +**Parser names by model:** GLM-4.5 and GLM-4.6 use `--tool-call-parser glm45`. GLM-4.7 and GLM-4.7-Flash use `--tool-call-parser glm47`. All GLM models use `--reasoning-parser glm45` regardless of generation. + + GLM-4.7-Flash supports tool calling capabilities. Enable the tool call parser: ```shell Command diff --git a/docs_new/cookbook/autoregressive/GLM/GLM-4.7.mdx b/docs_new/cookbook/autoregressive/GLM/GLM-4.7.mdx index 98ad0084b..84b3418f9 100644 --- a/docs_new/cookbook/autoregressive/GLM/GLM-4.7.mdx +++ b/docs_new/cookbook/autoregressive/GLM/GLM-4.7.mdx @@ -54,7 +54,8 @@ import { GLM47Deployment } from "/src/snippets/autoregressive/glm-47-deployment. ### 3.2 Configuration Tips -For more detailed configuration tips, please refer to [GLM-4.7 Usage](../../../docs/basic_usage/glm45). +- **EAGLE Speculative Decoding:** Supported for GLM-4.7. Add `--speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4`. The spec-v2 overlap scheduler is enabled by default (`SGLANG_ENABLE_SPEC_V2=True`); set `SGLANG_ENABLE_SPEC_V2=0` to disable. Enable via the interactive command generator above. +- **Thinking Budget:** Use `--enable-custom-logit-processor` flag and pass `Glm4MoeThinkingBudgetLogitProcessor` in requests to cap the model's thinking token count (see section 4.2.3). ## 4. Model Invocation @@ -73,7 +74,7 @@ GLM-4.7 supports Thinking mode by default. Enable the reasoning parser during de ```shell Command python -m sglang.launch_server \ --model zai-org/GLM-4.7 \ - --reasoning-parser glm47 \ + --reasoning-parser glm45 \ --tp 8 \ --host 0.0.0.0 \ --port 8000 @@ -145,12 +146,16 @@ The answer is 36. To find 15% of 240, we multiply 240 by 0.15, which equals 36. #### 4.2.2 Tool Calling + +**Parser names by model:** GLM-4.5 and GLM-4.6 use `--tool-call-parser glm45`. GLM-4.7 and GLM-4.7-Flash use `--tool-call-parser glm47`. All GLM models use `--reasoning-parser glm45` regardless of generation. + + GLM-4.7 supports tool calling capabilities. Enable the tool call parser: ```shell Command python -m sglang.launch_server \ --model zai-org/GLM-4.7 \ - --reasoning-parser glm47 \ + --reasoning-parser glm45 \ --tool-call-parser glm47 \ --tp 8 \ --host 0.0.0.0 \ @@ -297,6 +302,27 @@ print(final_response.choices[0].message.content) # Output: "The weather in Beijing is currently 22°C and sunny." ``` +#### 4.2.3 Thinking Budget + +Limit the number of thinking tokens using `CustomLogitProcessor`. Launch with `--enable-custom-logit-processor`: + +```python Example +import openai +from sglang.srt.sampling.custom_logit_processor import Glm4MoeThinkingBudgetLogitProcessor + +client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="*") +response = client.chat.completions.create( + model="zai-org/GLM-4.7", + messages=[{"role": "user", "content": "Is Paris the Capital of France?"}], + max_tokens=1024, + extra_body={ + "custom_logit_processor": Glm4MoeThinkingBudgetLogitProcessor().to_str(), + "custom_params": {"thinking_budget": 512}, + }, +) +print(response) +``` + ## 5. Benchmark This section uses **industry-standard configurations** for comparable benchmark results. diff --git a/docs_new/cookbook/autoregressive/GLM/GLM-5.1.mdx b/docs_new/cookbook/autoregressive/GLM/GLM-5.1.mdx index f16407106..f40e7aec7 100644 --- a/docs_new/cookbook/autoregressive/GLM/GLM-5.1.mdx +++ b/docs_new/cookbook/autoregressive/GLM/GLM-5.1.mdx @@ -81,7 +81,7 @@ import { GLM51Deployment } from '/src/snippets/autoregressive/glm-51-deployment. - **AMD GPUs**: Both BF16 and FP8 checkpoints are supported on MI300X/MI325X/MI355X at tp=8. Use `--dsa-prefill-backend tilelang --dsa-decode-backend tilelang` for the DSA attention backend. Add `--chunked-prefill-size 131072` and `--watchdog-timeout 1200` (20 minutes for weight loading). FP8 uses approximately half the memory of BF16 (~89 GB/GPU vs ~175 GB/GPU). EAGLE speculative decoding is not currently supported on AMD for GLM-5.1. - **GB300**: Only the FP8 checkpoint is recommended on GB300, with `tp=4`. For high-throughput DP attention on GB300, use `--dp 4`. -- For other configuration tips, please refer to [DeepSeek V3.2 documentation](../../../docs/basic_usage/deepseek_v32). GLM-5.1 and DeepSeek V3.2 share the same model structure, so the optimization techniques between these two models are also common (MTP, DSA kernel, Context Parallel...). +- For other configuration tips (MTP, DSA kernel, Context Parallel, HiSparse, NVFP4, Index Cache), see the [DeepSeek-V3.2 cookbook page](../DeepSeek/DeepSeek-V3_2). GLM-5.1 and DeepSeek-V3.2 share the same model structure, so the optimization techniques are common. - Use `--json-model-override-args '{"index_topk_pattern": "FFSFSSSFSSFFFSSSFFFSFSSSSSSFFSFFSFFSSFFFFFFSFFFFFSFFSSSSSSFSFFFSFSSSFSFFSFFSSS"}'` for GLM-5.1-FP8 if you want to enable the [IndexCache](https://github.com/THUDM/IndexCache) method. This feature is supported through [this PR](https://github.com/sgl-project/sglang/pull/21405) and introduces only a small accuracy loss. However, if you are running rigorous accuracy evaluations, it is not recommended to enable this feature. ## 4. Model Invocation @@ -538,6 +538,10 @@ Max ITL (ms): 1307.02 ### 5.2 Accuracy Benchmark + +The accuracy benchmark results below are shared with GLM-5, as GLM-5.1 was not independently benchmarked at the time of this writing. A separate benchmark run is planned. + + #### 5.2.1 GSM8K Benchmark - Benchmark Command diff --git a/docs_new/cookbook/autoregressive/GLM/GLM-5.mdx b/docs_new/cookbook/autoregressive/GLM/GLM-5.mdx index e1d1f2d3c..444770c7d 100644 --- a/docs_new/cookbook/autoregressive/GLM/GLM-5.mdx +++ b/docs_new/cookbook/autoregressive/GLM/GLM-5.mdx @@ -88,7 +88,7 @@ import { GLM5Deployment } from '/src/snippets/autoregressive/glm-5-deployment.js - **B200 (FP8)**: Use `--ep 1 --attention-backend dsa --dsa-decode-backend trtllm --dsa-prefill-backend trtllm --moe-runner-backend flashinfer_trtllm --enable-flashinfer-allreduce-fusion` for optimized DSA and MoE backends on Blackwell. Also add `--quantization fp8` for FP8 weight quantization. - **AMD GPUs**: Use `--dsa-prefill-backend tilelang --dsa-decode-backend tilelang` for the DSA attention backend. Add `--chunked-prefill-size 131072` and `--watchdog-timeout 1200` (20 minutes for weight loading). EAGLE speculative decoding is not currently supported on AMD for GLM-5. -- For other configuration tips, please refer to [DeepSeek V3.2 documentation](../../../docs/basic_usage/deepseek_v32). GLM-5 and DeepSeek V3.2 share the same model structure, so the optimization techniques between these two models are also common (MTP, DSA kernel, Context Parallel...). +- For other configuration tips (MTP, DSA kernel, Context Parallel, HiSparse, NVFP4, Index Cache), see the [DeepSeek-V3.2 cookbook page](../DeepSeek/DeepSeek-V3_2). GLM-5 and DeepSeek-V3.2 share the same model structure, so the optimization techniques are common. - Use `--json-model-override-args '{"index_topk_pattern": "FFSFSSSFSSFFFSSSFFFSFSSSSSSFFSFFSFFSSFFFFFFSFFFFFSFFSSSSSSFSFFFSFSSSFSFFSFFSSS"}'` for GLM-5-FP8 if you want to enable the [IndexCache](https://github.com/THUDM/IndexCache) method. This feature is supported through [this PR](https://github.com/sgl-project/sglang/pull/21405) and introduces only a small accuracy loss. However, if you are running rigorous accuracy evaluations, it is not recommended to enable this feature. @@ -564,6 +564,10 @@ Max ITL (ms): 1307.02 ### 5.2 Accuracy Benchmark + +The accuracy benchmark results below are shared with GLM-5.1, as GLM-5.1 was not independently benchmarked at the time of this writing. A separate GLM-5.1 benchmark run is planned. + + #### 5.2.1 GSM8K Benchmark - Benchmark Command diff --git a/docs_new/cookbook/autoregressive/GLM/GLM-Glyph.mdx b/docs_new/cookbook/autoregressive/GLM/GLM-Glyph.mdx index 050721fdc..cc676c571 100644 --- a/docs_new/cookbook/autoregressive/GLM/GLM-Glyph.mdx +++ b/docs_new/cookbook/autoregressive/GLM/GLM-Glyph.mdx @@ -45,7 +45,7 @@ This section provides deployment configurations optimized for different hardware ### 3.2 Configuration Tips -For more detailed configuration tips, please refer to [GLM-4.5/GLM-4.6 Usage](../../../docs/basic_usage/glm45). +- **Thinking Budget:** Use `--enable-custom-logit-processor` flag and pass `Glm4MoeThinkingBudgetLogitProcessor` in requests to cap the model's thinking token count. See the [GLM-4.5 cookbook page](/cookbook/autoregressive/GLM/GLM-4.5) for the full Thinking Budget usage example. ## 4. Model Invocation diff --git a/docs_new/cookbook/autoregressive/Llama/Llama4.mdx b/docs_new/cookbook/autoregressive/Llama/Llama4.mdx index 81e7e343a..d2631c53d 100644 --- a/docs_new/cookbook/autoregressive/Llama/Llama4.mdx +++ b/docs_new/cookbook/autoregressive/Llama/Llama4.mdx @@ -41,6 +41,20 @@ This section provides a progressive guide from quick deployment to performance o +### 3.2 Configuration Tips + +- **OOM Mitigation:** Reduce `--context-length` to avoid GPU out-of-memory. Recommended: Scout up to 1M on 8×H100, up to 2.5M on 8×H200; Maverick doesn't need context-length set on 8×H200. With hybrid KV cache enabled, Scout can reach 5M on 8×H100 and 10M on 8×H200. +- **Attention Backend Auto-Selection:** SGLang automatically picks the optimal backend. Manual override with `--attention-backend`: + - Blackwell (B200/GB200): `trtllm_mha` + - Hopper (H100/H200): `fa3` + - AMD GPUs: `aiter` + - Intel XPU: `intel_xpu` + - Other: `triton` +- **Chat Template:** Add `--chat-template llama-4` for chat completion tasks. +- **Multi-Modal:** Add `--enable-multimodal` to enable image input support. +- **Hybrid KV Cache:** Set `--swa-full-tokens-ratio` to control the ratio of SWA (local attention) KV tokens to full-attention KV tokens (default: 0.8, range: 0–1). +- **EAGLE Speculative Decoding:** Supported for Llama 4 Scout and Maverick via EAGLE3. Enable with the interactive command generator above. + ## 4. Model Invocation ### 4.1 Basic Usage @@ -90,12 +104,27 @@ sglang serve \ sglang serve \ --model-path meta-llama/Llama-4-Maverick-17B-128E-Instruct \ --tp 8 \ - --context-length 1000000 \ --trust-remote-code ``` +#### 4.2.3 EAGLE Speculative Decoding + +SGLang supports Llama 4 Maverick (400B) with [EAGLE speculative decoding](../../../docs/advanced_features/speculative_decoding). Enable with the EAGLE3 algorithm and the SGLang EAGLE3 draft model: + +```shell Command +python3 -m sglang.launch_server \ + --model-path meta-llama/Llama-4-Maverick-17B-128E-Instruct \ + --speculative-algorithm EAGLE3 \ + --speculative-draft-model-path lmsys/sglang-EAGLE3-Llama-4-Maverick-17B-128E-Instruct-v1 \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 4 \ + --trust-remote-code \ + --tp 8 +``` + ## 5. Benchmark -### 5.1 Speed Benchmark +### 5.1 Speed Benchmark (Scout) Test Environment: Hardware: AMD MI300x GPU @@ -273,7 +302,7 @@ Max ITL (ms): 226.06 ================================================== ``` -### 5.2 Speed Benchmark +### 5.2 Speed Benchmark (Maverick) Test Environment: Hardware: AMD MI300x GPU @@ -472,3 +501,69 @@ Invalid: 0.000 Latency: 9.739 s Output throughput: 2405.505 token/s ``` + +#### 5.3.2 MMLU Pro with lm-eval + +Accuracy on MMLU Pro matches [Meta's official benchmark numbers](https://ai.meta.com/blog/llama-4-multimodal-intelligence/) on 8×H100 (reproduction details: [PR #5092](https://github.com/sgl-project/sglang/pull/5092)): + + + + + + + + + + + + + + + + + + + + + +
ModelOfficialSGLang
Llama-4-Scout-17B-16E-Instruct74.375.2
Llama-4-Maverick-17B-128E-Instruct80.580.7
+ +**Scout:** + +```bash Command +# Start the server +python -m sglang.launch_server \ + --model-path meta-llama/Llama-4-Scout-17B-16E-Instruct \ + --port 30000 \ + --tp 8 \ + --mem-fraction-static 0.8 \ + --context-length 65536 + +# Run lm_eval +lm_eval --model local-chat-completions \ + --model_args model=meta-llama/Llama-4-Scout-17B-16E-Instruct,base_url=http://localhost:30000/v1/chat/completions,num_concurrent=128,timeout=999999,max_gen_toks=2048 \ + --tasks mmlu_pro \ + --batch_size 128 \ + --apply_chat_template \ + --num_fewshot 0 +``` + +**Maverick:** + +```bash Command +# Start the server +python -m sglang.launch_server \ + --model-path meta-llama/Llama-4-Maverick-17B-128E-Instruct \ + --port 30000 \ + --tp 8 \ + --mem-fraction-static 0.8 \ + --context-length 65536 + +# Run lm_eval +lm_eval --model local-chat-completions \ + --model_args model=meta-llama/Llama-4-Maverick-17B-128E-Instruct,base_url=http://localhost:30000/v1/chat/completions,num_concurrent=128,timeout=999999,max_gen_toks=2048 \ + --tasks mmlu_pro \ + --batch_size 128 \ + --apply_chat_template \ + --num_fewshot 0 +``` diff --git a/docs_new/cookbook/autoregressive/MiniMax/MiniMax-M2.mdx b/docs_new/cookbook/autoregressive/MiniMax/MiniMax-M2.mdx index 11758aa1c..0226d2314 100644 --- a/docs_new/cookbook/autoregressive/MiniMax/MiniMax-M2.mdx +++ b/docs_new/cookbook/autoregressive/MiniMax/MiniMax-M2.mdx @@ -69,6 +69,78 @@ This section provides a progressive guide from quick deployment to performance o +#### 3.1.1 NVIDIA GPU Deployment + +The interactive command generator above covers AMD deployments. For NVIDIA GPUs (H100/H200/B200), use these explicit commands: + +**4-GPU deployment (up to 400K context):** + +```bash Command +python -m sglang.launch_server \ + --model-path MiniMaxAI/MiniMax-M2 \ + --tp-size 4 \ + --tool-call-parser minimax-m2 \ + --reasoning-parser minimax-append-think \ + --host 0.0.0.0 \ + --trust-remote-code \ + --port 30000 \ + --mem-fraction-static 0.85 +``` + +**8-GPU deployment (up to 3M context):** + +```bash Command +python -m sglang.launch_server \ + --model-path MiniMaxAI/MiniMax-M2 \ + --tp-size 8 \ + --ep-size 8 \ + --tool-call-parser minimax-m2 \ + --reasoning-parser minimax-append-think \ + --host 0.0.0.0 \ + --trust-remote-code \ + --port 30000 \ + --mem-fraction-static 0.85 +``` + +### 3.2 System Requirements + +Recommended configurations — actual requirements depend on workload: + + + + + + + + + + + + + + + + + + +
GPUsContext Length Support
4× 96 GB GPUsUp to 400K tokens
8× 144 GB GPUsUp to 3M tokens
+ +### 3.3 Testing Deployment + +After the server starts, verify with: + +```shell Command +curl http://localhost:30000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "MiniMaxAI/MiniMax-M2", + "messages": [ + {"role": "system", "content": [{"type": "text", "text": "You are a helpful assistant."}]}, + {"role": "user", "content": [{"type": "text", "text": "Who won the world series in 2020?"}]} + ] + }' +``` + ## 4. Model Invocation ### 4.1 Basic Usage diff --git a/docs_new/cookbook/autoregressive/OpenAI/GPT-OSS.mdx b/docs_new/cookbook/autoregressive/OpenAI/GPT-OSS.mdx index 30e9f94ca..f5a972a01 100644 --- a/docs_new/cookbook/autoregressive/OpenAI/GPT-OSS.mdx +++ b/docs_new/cookbook/autoregressive/OpenAI/GPT-OSS.mdx @@ -41,7 +41,12 @@ import { GPTOSSDeployment } from "/src/snippets/autoregressive/gpt-oss-deploymen ### 3.2 Configuration Tips -For more detailed configuration tips, please refer to [GPS-OSS Usage](../../../docs/basic_usage/gpt_oss). +- **Demo tool server:** Launch with `--tool-server demo` to enable the built-in web-search (Exa) and Python interpreter tools. +- **Web search tool:** Requires an Exa API key — set `EXA_API_KEY` in your environment. +- **Python tool:** Runs in a Docker sandbox by default. Set `PYTHON_EXECUTION_BACKEND=UV` to run on the host (model-generated code executes locally — use with care). +- **MCP tool servers:** For production, point SGLang at external MCP SSE servers with `--tool-server ip-1:port-1,ip-2:port-2`. +- **Responses API:** GPT-OSS supports OpenAI's Responses API (`client.responses.create`) in addition to the standard Chat Completions API (see section 4.2.3). +- **Use Python 3.12** when running the demo tools. ## 4.Model Invocation @@ -410,6 +415,89 @@ print(final_response.choices[0].message.content) # Output: "The current weather in Beijing is 22 °C and sunny. Let me know if you’d like a forecast for the next few days or any other details!" ``` +#### 4.2.3 EAGLE3 Speculative Decoding + +SGLang supports speculative decoding for GPT-OSS models using the EAGLE3 algorithm. This can significantly improve decoding speed, especially for small batch sizes. + +```shell Command +python3 -m sglang.launch_server \ + --model-path openai/gpt-oss-120b \ + --speculative-algorithm EAGLE3 \ + --speculative-draft-model-path lmsys/EAGLE3-gpt-oss-120b-bf16 \ + --tp 2 +``` + + +The spec-v2 overlap scheduler is enabled by default (`SGLANG_ENABLE_SPEC_V2=True`). It improves performance by overlapping draft and verification stages. Set `SGLANG_ENABLE_SPEC_V2=0` to disable. + + +#### 4.2.4 Responses API and Built-in Tools + +GPT-OSS supports the OpenAI Responses API with built-in tool use (web search and Python interpreter). Set up your environment and launch with `--tool-server demo`: + +```shell Command +export EXA_API_KEY=YOUR_EXA_KEY +# Optional: run Python tool on host instead of Docker (model code executes locally) +export PYTHON_EXECUTION_BACKEND=UV + +python3 -m sglang.launch_server \ + --model-path openai/gpt-oss-120b \ + --tool-server demo \ + --tp 2 +``` + +For production, use external MCP SSE servers instead of `demo`: +```shell Command +mcp run -t sse browser_server.py:mcp +mcp run -t sse python_server.py:mcp + +python -m sglang.launch_server \ + --model-path openai/gpt-oss-120b \ + --tool-server ip-1:port-1,ip-2:port-2 \ + --tp 2 +``` + +**Example using Responses API:** + +```python Example +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:30000/v1", api_key="sk-123456") + +tools = [ + {"type": "code_interpreter"}, + {"type": "web_search_preview"}, +] + +# Configurable reasoning effort: "high", "medium", or "low" +response = client.responses.create( + model="openai/gpt-oss-120b", + instructions="You are a helpful assistant.", + reasoning_effort="high", + input="In one sentence, explain the transformer architecture.", +) +print(response.output_text) + +# Python tool usage +response = client.responses.create( + model="openai/gpt-oss-120b", + instructions="You are a helpful assistant.", + input="Use python tool to calculate the sum of 29138749187 and 29138749187", + tools=tools, +) +print(response.output_text) +# Output: The sum is 58,277,498,374. + +# Web search usage +response = client.responses.create( + model="openai/gpt-oss-120b", + instructions="You are a helpful assistant.", + input="Search the web for the latest news about Nvidia stock price", + tools=tools, +) +print(response.output_text) +``` + ## 5.Benchmark ### 5.1 Speed Benchmark diff --git a/docs_new/cookbook/autoregressive/Qwen/Qwen3-Next.mdx b/docs_new/cookbook/autoregressive/Qwen/Qwen3-Next.mdx index 57bb422b9..18937969c 100644 --- a/docs_new/cookbook/autoregressive/Qwen/Qwen3-Next.mdx +++ b/docs_new/cookbook/autoregressive/Qwen/Qwen3-Next.mdx @@ -566,6 +566,22 @@ SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1 python -m sglang.launch_server --mod ``` +#### 4.2.4 Multi-Token Prediction (NEXTN Speculative Decoding) + +Qwen3-Next ships built-in Multi-Token Prediction (MTP) layers and supports [EAGLE-style speculative decoding](../../../docs/advanced_features/speculative_decoding#EAGLE-Decoding) through the `NEXTN` algorithm. The MTP weights are bundled in the main checkpoint, so no separate draft model is required. + +```shell Command +python3 -m sglang.launch_server \ + --model Qwen/Qwen3-Next-80B-A3B-Instruct \ + --speculative-algorithm NEXTN \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 4 \ + --tp 4 +``` + +Tune `--speculative-num-steps`, `--speculative-eagle-topk`, and `--speculative-num-draft-tokens` for your workload with [bench_speculative.py](https://github.com/sgl-project/sglang/blob/main/scripts/playground/bench_speculative.py). See [PR #10233](https://github.com/sgl-project/sglang/pull/10233) for implementation details. + ## 5. Benchmark ### 5.1 Speed Benchmark diff --git a/docs_new/cookbook/autoregressive/Qwen/Qwen3-VL.mdx b/docs_new/cookbook/autoregressive/Qwen/Qwen3-VL.mdx index 849e648d4..bdbb38743 100644 --- a/docs_new/cookbook/autoregressive/Qwen/Qwen3-VL.mdx +++ b/docs_new/cookbook/autoregressive/Qwen/Qwen3-VL.mdx @@ -48,6 +48,34 @@ import { Qwen3VLDeployment } from "/src/snippets/autoregressive/qwen3-vl-deploym * **Expert Parallelism** : SGLang supports Expert Parallelism (EP) via `--ep`, allowing experts in MoE models to be deployed on separate GPUs for better throughput. One thing to note is that, for quantized models, you need to set `--ep` to a value that satisfies the requirement: `(moe_intermediate_size / moe_tp_size) % weight_block_size_n == 0, where moe_tp_size is equal to tp_size divided by ep_size.` Note that EP may perform worse in low concurrency scenarios due to additional communication overhead. Check out [Expert Parallelism Deployment](../../../docs/advanced_features/expert_parallelism) for more details. * **Kernel Tuning** : For MoE Triton kernel tuning on your specific hardware, refer to [fused_moe_triton](https://github.com/sgl-project/sglang/tree/main/benchmark/kernels/fused_moe_triton). +**Hardware-specific notes:** +- **H100 (FP8):** Use the `Qwen/Qwen3-VL-235B-A22B-Instruct-FP8` checkpoint for best memory efficiency. +- **A100 / H100 (BF16):** Use standard multimodal parameters to manage throughput and GPU memory usage. +- **H200 / B200:** Runs out of the box, supporting full context length plus concurrent image + video processing. + +**Additional multimodal server parameters:** +- `--keep-mm-feature-on-device`: Retain multimodal feature tensors on GPU after processing to avoid device-to-host memory copies, improving performance for high-frequency inference. + +**Example with full multimodal optimizations:** +```bash Command +SGLANG_USE_CUDA_IPC_TRANSPORT=1 \ +SGLANG_VLM_CACHE_SIZE_MB=0 \ +python -m sglang.launch_server \ + --model-path Qwen/Qwen3-VL-235B-A22B-Instruct \ + --host 0.0.0.0 \ + --port 30000 \ + --trust-remote-code \ + --tp-size 8 \ + --enable-cache-report \ + --log-level info \ + --max-running-requests 64 \ + --mem-fraction-static 0.65 \ + --chunked-prefill-size 8192 \ + --attention-backend fa3 \ + --mm-attention-backend fa3 \ + --enable-metrics +``` + ## 4. Model Invocation ### 4.1 Basic Usage @@ -69,7 +97,7 @@ from openai import OpenAI client = OpenAI( api_key="EMPTY", - base_url="http://localhost:8000/v1", + base_url="http://localhost:30000/v1", timeout=3600 ) @@ -129,7 +157,7 @@ from openai import OpenAI client = OpenAI( api_key="EMPTY", - base_url="http://localhost:8000/v1", + base_url="http://localhost:30000/v1", timeout=3600 ) @@ -192,7 +220,7 @@ from openai import OpenAI client = OpenAI( api_key="EMPTY", - base_url="http://localhost:8000/v1", + base_url="http://localhost:30000/v1", timeout=3600 ) @@ -247,7 +275,7 @@ python -m sglang.launch_server \ --reasoning-parser qwen3 \ --tp 8 \ --host 0.0.0.0 \ - --port 8000 + --port 30000 ``` **Streaming with Thinking Process:** @@ -256,7 +284,7 @@ python -m sglang.launch_server \ from openai import OpenAI client = OpenAI( - base_url="http://localhost:8000/v1", + base_url="http://localhost:30000/v1", api_key="EMPTY" ) @@ -325,7 +353,7 @@ python -m sglang.launch_server \ --tool-call-parser qwen \ --tp 8 \ --host 0.0.0.0 \ - --port 8000 + --port 30000 ``` **Python Example (with Thinking Process):** @@ -334,7 +362,7 @@ python -m sglang.launch_server \ from openai import OpenAI client = OpenAI( - base_url="http://localhost:8000/v1", + base_url="http://localhost:30000/v1", api_key="EMPTY" ) @@ -505,7 +533,7 @@ python -m sglang.launch_server \ --model Qwen/Qwen3-VL-235B-A22B-Instruct \ --tp 8 \ --host 0.0.0.0 \ - --port 8000 + --port 30000 ``` - Benchmark Command: @@ -514,7 +542,7 @@ python -m sglang.launch_server \ python3 -m sglang.bench_serving \ --backend sglang-oai-chat \ --host 127.0.0.1 \ - --port 8000 \ + --port 30000 \ --model Qwen/Qwen3-VL-235B-A22B-Instruct \ --dataset-name image \ --image-count 2 \ @@ -577,7 +605,7 @@ SGLANG_USE_CUDA_IPC_TRANSPORT=1 python -m sglang.launch_server \ --model Qwen/Qwen3-VL-235B-A22B-Instruct \ --tp 8 \ --host 0.0.0.0 \ - --port 8000 + --port 30000 ``` - Benchmark Command: @@ -586,7 +614,7 @@ SGLANG_USE_CUDA_IPC_TRANSPORT=1 python -m sglang.launch_server \ python3 -m sglang.bench_serving \ --backend sglang-oai-chat \ --host 127.0.0.1 \ - --port 8000 \ + --port 30000 \ --model Qwen/Qwen3-VL-235B-A22B-Instruct \ --dataset-name image \ --image-count 2 \ @@ -649,7 +677,7 @@ python -m sglang.launch_server \ --model Qwen/Qwen3-VL-235B-A22B-Instruct \ --tp 8 \ --host 0.0.0.0 \ - --port 8000 + --port 30000 ``` - Benchmark Command: @@ -658,7 +686,7 @@ python -m sglang.launch_server \ python3 -m sglang.bench_serving \ --backend sglang-oai-chat \ --host 127.0.0.1 \ - --port 8000 \ + --port 30000 \ --model Qwen/Qwen3-VL-235B-A22B-Instruct \ --dataset-name image \ --image-count 2 \ @@ -723,7 +751,7 @@ uv pip install lmms_eval python3 -m lmms_eval \ --model openai_compatible \ - --model_args "model=Qwen/Qwen3-VL-235B-A22B-Instruct,api_key=EMPTY,base_url=http://127.0.0.1:8000/v1/" \ + --model_args "model=Qwen/Qwen3-VL-235B-A22B-Instruct,api_key=EMPTY,base_url=http://127.0.0.1:30000/v1/" \ --tasks mmmu_val \ --batch_size 128 \ --log_samples \ diff --git a/docs_new/cookbook/autoregressive/Qwen/Qwen3.5.mdx b/docs_new/cookbook/autoregressive/Qwen/Qwen3.5.mdx index 706de5277..5a194539b 100644 --- a/docs_new/cookbook/autoregressive/Qwen/Qwen3.5.mdx +++ b/docs_new/cookbook/autoregressive/Qwen/Qwen3.5.mdx @@ -12,6 +12,11 @@ import { Qwen35Deployment } from '/src/snippets/autoregressive/qwen35-deployment Qwen3.5 features a Gated Delta Networks combined with sparse Mixture-of-Experts architecture (397B total parameters, 17B activated), delivering high-throughput inference with minimal latency. It supports multimodal inputs (text, image, video) and natively handles context lengths of up to 262,144 tokens, extensible to over 1M tokens. +**Architecture details:** +- **Hybrid Attention:** Gated Delta Networks (linear, O(n) complexity) combined with full attention every 4th layer — linear layers provide low-cost long-context processing while periodic full attention ensures high associative recall. +- **MoE routing:** Top-10 active out of 512 routed experts plus a dedicated shared expert for universal features, keeping 17B parameters active from 397B total. +- **Native multimodal:** DeepStack Vision Transformer with Conv3d temporal encoding for image and video understanding without separate visual encoders. + **Key Features:** - **Unified Vision-Language Foundation**: Early fusion training on multimodal tokens achieves cross-generational parity with Qwen3 and outperforms Qwen3-VL models @@ -119,6 +124,8 @@ This section provides deployment configurations optimized for different hardware ### 3.2 Configuration Tips - Speculative decoding (MTP) can significantly reduce latency for interactive use cases. +- **AMD GPUs (MI300X / MI325X / MI355X):** Use `SGLANG_USE_AITER=1` and `--attention-backend triton`. Both the full attention layers and the Gated Delta Net (linear attention) layers use Triton-based kernels on ROCm. Example: `SGLANG_USE_AITER=1 python3 -m sglang.launch_server --model-path Qwen/Qwen3.5-397B-A17B --tp 8 --attention-backend triton --trust-remote-code`. See [AMD's Day-0 support article](https://www.amd.com/en/developer/resources/technical-articles/2026/day-0-support-for-qwen-3-5-on-amd-instinct-gpus.html) for details. +- **Watchdog timeout:** Increase `--watchdog-timeout` to `1200` or higher for this large model, as weight loading can take significant time. - **Mamba Radix Cache**: Qwen3.5's hybrid Gated Delta Networks architecture supports two mamba scheduling strategies via `--mamba-scheduler-strategy`: - **V1 (`no_buffer`)**: Default. No overlap scheduler, lower memory usage. Required for AMD MI GPUs. - **V2 (`extra_buffer`)**: Enables overlap scheduling and branching point caching with `--mamba-scheduler-strategy extra_buffer --page-size 64`. Requires FLA kernel backend (NVIDIA GPUs only). Trades higher mamba state memory for better throughput. Strictly superior in non-KV-cache-bound scenarios; in KV-cache-bound cases, weigh the overlap scheduling benefit against reduced max concurrency. `--page-size` must satisfy `FLA_CHUNK_SIZE % page_size == 0` or `page_size % FLA_CHUNK_SIZE == 0` (`FLA_CHUNK_SIZE` is currently 64). @@ -774,7 +781,22 @@ Latency: 31.784 s Output throughput: 998.166 token/s ``` -#### 5.1.2 MMMU Benchmark +#### 5.1.2 GSM8K with lm-eval (5-shot) + +Evaluate using the industry-standard `lm-eval` harness for reproducible accuracy reporting: + +```bash Command +pip install lm-eval[api] + +lm_eval --model local-completions \ + --model_args '{"base_url": "http://localhost:30000/v1/completions", "model": "Qwen/Qwen3.5-397B-A17B", "num_concurrent": 256, "max_retries": 10, "max_gen_toks": 2048}' \ + --tasks gsm8k \ + --batch_size auto \ + --num_fewshot 5 \ + --trust_remote_code +``` + +#### 5.1.3 MMMU Benchmark - Benchmark Command ```bash Command @@ -842,7 +864,7 @@ SGLANG_USE_CUDA_IPC_TRANSPORT=1 python -m sglang.launch_server \ --port 30000 ``` -#### 5.3.1 Latency Benchmark +#### 5.2.1 Latency Benchmark ```bash Command python3 -m sglang.bench_serving \ @@ -897,7 +919,7 @@ Max ITL (ms): 58.03 ================================================== ``` -#### 5.3.2 Throughput Benchmark +#### 5.2.2 Throughput Benchmark ```bash Command python3 -m sglang.bench_serving \ diff --git a/docs_new/docs.json b/docs_new/docs.json index c55d4b426..3eedb657e 100644 --- a/docs_new/docs.json +++ b/docs_new/docs.json @@ -203,35 +203,39 @@ }, { "source": "/basic_usage/deepseek_ocr.html", - "destination": "/docs/basic_usage/deepseek_ocr" + "destination": "/cookbook/autoregressive/DeepSeek/DeepSeek-OCR" }, { "source": "/basic_usage/deepseek_v3.html", - "destination": "/docs/basic_usage/deepseek_v3" + "destination": "/cookbook/autoregressive/DeepSeek/DeepSeek-V3" }, { "source": "/basic_usage/deepseek_v32.html", - "destination": "/docs/basic_usage/deepseek_v32" + "destination": "/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2" }, { "source": "/basic_usage/glm45.html", - "destination": "/docs/basic_usage/glm45" + "destination": "/cookbook/autoregressive/GLM/GLM-4.5" }, { "source": "/basic_usage/glmv.html", - "destination": "/docs/basic_usage/glmv" + "destination": "/cookbook/autoregressive/GLM/GLM-4.6V" }, { "source": "/basic_usage/gpt_oss.html", - "destination": "/docs/basic_usage/gpt_oss" + "destination": "/cookbook/autoregressive/OpenAI/GPT-OSS" }, { "source": "/basic_usage/llama4.html", - "destination": "/docs/basic_usage/llama4" + "destination": "/cookbook/autoregressive/Llama/Llama4" }, { "source": "/basic_usage/minimax_m2.html", - "destination": "/docs/basic_usage/minimax_m2" + "destination": "/cookbook/autoregressive/MiniMax/MiniMax-M2" + }, + { + "source": "/basic_usage/kimi_k2_5.html", + "destination": "/cookbook/autoregressive/Moonshotai/Kimi-K2.5" }, { "source": "/basic_usage/native_api.html", @@ -263,19 +267,19 @@ }, { "source": "/basic_usage/popular_model_usage.html", - "destination": "/docs/basic_usage/popular_model_usage" + "destination": "/cookbook/autoregressive/intro" }, { "source": "/basic_usage/qwen3.html", - "destination": "/docs/basic_usage/qwen3" + "destination": "/cookbook/autoregressive/Qwen/Qwen3" }, { "source": "/basic_usage/qwen3_5.html", - "destination": "/docs/basic_usage/qwen3_5" + "destination": "/cookbook/autoregressive/Qwen/Qwen3.5" }, { "source": "/basic_usage/qwen3_vl.html", - "destination": "/docs/basic_usage/qwen3_vl" + "destination": "/cookbook/autoregressive/Qwen/Qwen3-VL" }, { "source": "/basic_usage/sampling_params.html", @@ -706,25 +710,7 @@ "docs/basic_usage/ollama_api", "docs/basic_usage/offline_engine_api", "docs/basic_usage/native_api", - "docs/basic_usage/sampling_params", - { - "group": "Popular Model Usage", - "pages": [ - "docs/basic_usage/popular_model_usage", - "docs/basic_usage/deepseek_v3", - "docs/basic_usage/deepseek_v32", - "docs/basic_usage/deepseek_ocr", - "docs/basic_usage/glm45", - "docs/basic_usage/glmv", - "docs/basic_usage/gpt_oss", - "docs/basic_usage/kimi_k2_5", - "docs/basic_usage/minimax_m2", - "docs/basic_usage/qwen3", - "docs/basic_usage/qwen3_5", - "docs/basic_usage/qwen3_vl", - "docs/basic_usage/llama4" - ] - } + "docs/basic_usage/sampling_params" ] }, { diff --git a/docs_new/docs/advanced_features/attention_backend.mdx b/docs_new/docs/advanced_features/attention_backend.mdx index 293859818..56f7d5f77 100644 --- a/docs_new/docs/advanced_features/attention_backend.mdx +++ b/docs_new/docs/advanced_features/attention_backend.mdx @@ -14,7 +14,7 @@ If you don't specify `--attention-backend`, SGLang makes a best effort to automa ## Support Matrix -The support matrix is split into two parts: MHA (standard attention) and MLA (multi-head latent attention). For an explanation of the key differences between MHA and MLA, please see the [SGLang documentation on DeepSeek MLA](../basic_usage/deepseek_v3#multi-head-latent-attention-mla-throughput-optimizations) and the original [DeepSeek MLA paper](https://arxiv.org/pdf/2405.04434). +The support matrix is split into two parts: MHA (standard attention) and MLA (multi-head latent attention). For an explanation of the key differences between MHA and MLA, please see the [SGLang documentation on DeepSeek MLA](/cookbook/autoregressive/DeepSeek/DeepSeek-V3#4-2-4-mla-optimizations) and the original [DeepSeek MLA paper](https://arxiv.org/pdf/2405.04434). ### MHA Backends @@ -289,7 +289,7 @@ Multimodal attention is selected by `--mm-attention-backend`. The "MultiModal" c - FlashAttention 4 supports both prefill and decode on SM90 (Hopper) and SM100 (Blackwell). FA4 MLA supports `page_size = 1`; FA4 MHA requires `page_size = 128`. On SM100, this is auto-enforced by the server; on SM90, users must set `--page-size 128` manually. -- DSA is specifically designed for [DeepSeek V3.2](https://lmsys.org/blog/2025-09-29-deepseek-V32/). See the [DSA Attention Backend](#dsa-attention-backend) section and [DeepSeek V3.2 deployment guide](../basic_usage/deepseek_v32) for details. +- DSA is specifically designed for [DeepSeek V3.2](https://lmsys.org/blog/2025-09-29-deepseek-V32/). See the [DSA Attention Backend](#dsa-attention-backend) section and [DeepSeek V3.2 deployment guide](/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2) for details. @@ -455,7 +455,7 @@ Internally, the DSA backend dispatches to different sub-backends for prefill and -For deployment examples, see the [DeepSeek V3.2 deployment guide](../basic_usage/deepseek_v32). +For deployment examples, see the [DeepSeek V3.2 deployment guide](/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2). ### Hybrid attention (different backends for prefill vs decode) (Experimental) diff --git a/docs_new/docs/advanced_features/speculative_decoding.mdx b/docs_new/docs/advanced_features/speculative_decoding.mdx index 931e5c05a..b213652b7 100644 --- a/docs_new/docs/advanced_features/speculative_decoding.mdx +++ b/docs_new/docs/advanced_features/speculative_decoding.mdx @@ -401,7 +401,7 @@ print(response.choices[0].message.content) ## Multi Token Prediction -We support [MTP (Multi-Token Prediction)](https://arxiv.org/pdf/2404.19737) in SGLang by using speculative decoding. We use `XiaomiMiMo/MiMo-7B-RL` as an example here (for DeepSeek MTP usage, refer to [deepseek_v32 doc](../basic_usage/deepseek_v32#multi-token-prediction)). +We support [MTP (Multi-Token Prediction)](https://arxiv.org/pdf/2404.19737) in SGLang by using speculative decoding. We use `XiaomiMiMo/MiMo-7B-RL` as an example here (for DeepSeek MTP usage, refer to [DeepSeek-V3.2 cookbook §4.2.3](/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2#4-2-3-multi-token-prediction-eagle-speculative-decoding)). ```bash Command python3 -m sglang.launch_server \ diff --git a/docs_new/docs/basic_usage/deepseek_ocr.mdx b/docs_new/docs/basic_usage/deepseek_ocr.mdx deleted file mode 100644 index 97b39e0a0..000000000 --- a/docs_new/docs/basic_usage/deepseek_ocr.mdx +++ /dev/null @@ -1,58 +0,0 @@ ---- -title: "DeepSeek OCR (OCR-1 / OCR-2)" -metatags: - description: "DeepSeek OCR models are multimodal (image + text) models for OCR and document understanding." ---- - -DeepSeek OCR models are multimodal (image + text) models for OCR and document understanding. - -## Launch server - -```shell -python -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-OCR-2 \ - --trust-remote-code \ - --host 0.0.0.0 \ - --port 30000 -``` - -> You can replace `deepseek-ai/DeepSeek-OCR-2` with `deepseek-ai/DeepSeek-OCR`. - -## Prompt examples - -Recommended prompts from the model card: - -``` - -<|grounding|>Convert the document to markdown. -``` - -``` - -Free OCR. -``` - -## OpenAI-compatible request example - -```python -import requests - -url = "http://localhost:30000/v1/chat/completions" - -data = { - "model": "deepseek-ai/DeepSeek-OCR-2", - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": "\n<|grounding|>Convert the document to markdown."}, - {"type": "image_url", "image_url": {"url": "https://example.com/your_image.jpg"}}, - ], - } - ], - "max_tokens": 512, -} - -response = requests.post(url, json=data) -print(response.text) -``` diff --git a/docs_new/docs/basic_usage/deepseek_v3.mdx b/docs_new/docs/basic_usage/deepseek_v3.mdx deleted file mode 100644 index 7d7c93d29..000000000 --- a/docs_new/docs/basic_usage/deepseek_v3.mdx +++ /dev/null @@ -1,375 +0,0 @@ ---- -title: "DeepSeek V3/V3.1/R1 Usage" -metatags: - description: "Deploy DeepSeek V3/R1 with SGLang: MLA optimization, FP8 quantization, multi-node TP, DP attention, MTP speculative decoding. Supports H200, B200, MI300X, A100." ---- -SGLang provides many optimizations specifically designed for the DeepSeek models, making it the inference engine recommended by the official [DeepSeek team](https://github.com/deepseek-ai/DeepSeek-V3/tree/main?tab=readme-ov-file#62-inference-with-sglang-recommended) from Day 0. - -This document outlines current optimizations for DeepSeek. -For an overview of the implemented features see the completed [Roadmap](https://github.com/sgl-project/sglang/issues/2591). - -## Launch DeepSeek V3.1/V3/R1 with SGLang - -To run DeepSeek V3.1/V3/R1 models, the recommended settings are as follows: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Weight TypeConfiguration
Full precision FP8<br>*(recommended)*8 x H200
8 x B200
8 x MI300X
2 x 8 x H100/800/20
Xeon 6980P CPU
Full precision (BF16) (upcast from original FP8)2 x 8 x H200
2 x 8 x MI300X
4 x 8 x H100/800/20
4 x 8 x A100/A800
Quantized weights (INT8)16 x A100/800
32 x L40S
Xeon 6980P CPU
4 x Atlas 800I A3
Quantized weights (W4A8)8 x H20/100, 4 x H200
Quantized weights (AWQ)8 x H100/800/20
8 x A100/A800
Quantized weights (MXFP4)8, 4 x MI355X/350X
Quantized weights (NVFP4)8, 4 x B200
- - - - -The official DeepSeek V3 is already in FP8 format, so you should not run it with any quantization arguments like `--quantization fp8`. - - -Detailed commands for reference: - -- [8 x H200](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#using-docker-recommended) -- [4 x B200, 8 x B200](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-one-b200-node) -- [8 x MI300X](../hardware-platforms/amd_gpu#running-deepseek-v3) -- [2 x 8 x H200](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-two-h2008-nodes-and-docker) -- [4 x 8 x A100](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-four-a1008-nodes) -- [8 x A100 (AWQ)](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-8-a100a800-with-awq-quantization) -- [16 x A100 (INT8)](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-16-a100a800-with-int8-quantization) -- [32 x L40S (INT8)](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-32-l40s-with-int8-quantization) -- [Xeon 6980P CPU](../hardware-platforms/cpu_server#example-running-deepseek-r1) -- [4 x Atlas 800I A3 (int8)](../hardware-platforms/ascend-npus/ascend_npu_deepseek_example#running-deepseek-with-pd-disaggregation-on-4-x-atlas-800i-a3) - -### Download Weights -If you encounter errors when starting the server, ensure the weights have finished downloading. It's recommended to download them beforehand or restart multiple times until all weights are downloaded. Please refer to [DeepSeek V3](https://huggingface.co/deepseek-ai/DeepSeek-V3-Base#61-inference-with-deepseek-infer-demo-example-only) official guide to download the weights. - -### Launch with one node of 8 x H200 -Please refer to [the example](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#installation--launch). - -### Running examples on Multi-Node - -- [Deploying DeepSeek on GB200 NVL72 with PD and Large Scale EP](https://lmsys.org/blog/2025-06-16-gb200-part-1/) ([Part I](https://lmsys.org/blog/2025-06-16-gb200-part-1/), [Part II](https://lmsys.org/blog/2025-09-25-gb200-part-2/)) - Comprehensive guide on GB200 optimizations. - -- [Deploying DeepSeek with PD Disaggregation and Large-Scale Expert Parallelism on 96 H100 GPUs](https://lmsys.org/blog/2025-05-05-large-scale-ep/) - Guide on PD disaggregation and large-scale EP. - -- [Serving with two H20*8 nodes](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-two-h208-nodes). - -- [Best Practices for Serving DeepSeek-R1 on H20](https://lmsys.org/blog/2025-09-26-sglang-ant-group/) - Comprehensive guide on H20 optimizations, deployment and performance. - -- [Serving with two H200*8 nodes and docker](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-two-h2008-nodes-and-docker). - -- [Serving with four A100*8 nodes](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-four-a1008-nodes). - -## Optimizations - -### Multi-head Latent Attention (MLA) Throughput Optimizations - -**Description**: [MLA](https://arxiv.org/pdf/2405.04434) is an innovative attention mechanism introduced by the DeepSeek team, aimed at improving inference efficiency. SGLang has implemented specific optimizations for this, including: - -- **Weight Absorption**: By applying the associative law of matrix multiplication to reorder computation steps, this method balances computation and memory access and improves efficiency in the decoding phase. - -- **MLA Attention Backends**: Currently SGLang supports different optimized MLA attention backends, including [FlashAttention3](https://github.com/Dao-AILab/flash-attention), [Flashinfer](https://docs.flashinfer.ai/api/attention.html#flashinfer-mla), [FlashMLA](https://github.com/deepseek-ai/FlashMLA), [CutlassMLA](https://github.com/sgl-project/sglang/pull/5390), **TRTLLM MLA** (optimized for Blackwell architecture), and [Triton](https://github.com/triton-lang/triton) backends. The default FA3 provides good performance across wide workloads. - -- **FP8 Quantization**: W8A8 FP8 and KV Cache FP8 quantization enables efficient FP8 inference. Additionally, we have implemented Batched Matrix Multiplication (BMM) operator to facilitate FP8 inference in MLA with weight absorption. - -- **CUDA Graph & Torch.compile**: Both MLA and Mixture of Experts (MoE) are compatible with CUDA Graph and Torch.compile, which reduces latency and accelerates decoding speed for small batch sizes. - -- **Chunked Prefix Cache**: Chunked prefix cache optimization can increase throughput by cutting prefix cache into chunks, processing them with multi-head attention and merging their states. Its improvement can be significant when doing chunked prefill on long sequences. Currently this optimization is only available for FlashAttention3 backend. - -Overall, with these optimizations, we have achieved up to **7x** acceleration in output throughput compared to the previous version. - -

- Multi-head Latent Attention for DeepSeek Series Models -

- -**Usage**: MLA optimization is enabled by default. - -**Reference**: Check [Blog](https://lmsys.org/blog/2024-09-04-sglang-v0-3/#deepseek-multi-head-latent-attention-mla-throughput-optimizations) and [Slides](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/lmsys_1st_meetup_deepseek_mla.pdf) for more details. - -### Data Parallelism Attention - -**Description**: This optimization involves data parallelism (DP) for the MLA attention mechanism of DeepSeek Series Models, which allows for a significant reduction in the KV cache size, enabling larger batch sizes. Each DP worker independently handles different types of batches (prefill, decode, idle), which are then synchronized before and after processing through the Mixture-of-Experts (MoE) layer. If you do not use DP attention, KV cache will be duplicated among all TP ranks. - -

- Data Parallelism Attention for DeepSeek Series Models -

- -With data parallelism attention enabled, we have achieved up to **1.9x** decoding throughput improvement compared to the previous version. - -

- Data Parallelism Attention Performance Comparison -

- -**Usage**: -- Append `--enable-dp-attention --tp 8 --dp 8` to the server arguments when using 8 H200 GPUs. This optimization improves peak throughput in high batch size scenarios where the server is limited by KV cache capacity. -- DP and TP attention can be flexibly combined. For example, to deploy DeepSeek-V3/R1 on 2 nodes with 8 H100 GPUs each, you can specify `--enable-dp-attention --tp 16 --dp 2`. This configuration runs attention with 2 DP groups, each containing 8 TP GPUs. - - -Data parallelism attention is not recommended for low-latency, small-batch use cases. It is optimized for high-throughput scenarios with large batch sizes. - - -**Reference**: Check [Blog](https://lmsys.org/blog/2024-12-04-sglang-v0-4/#data-parallelism-attention-for-deepseek-models). - -### Multi-Node Tensor Parallelism - -**Description**: For users with limited memory on a single node, SGLang supports serving DeepSeek Series Models, including DeepSeek V3, across multiple nodes using tensor parallelism. This approach partitions the model parameters across multiple GPUs or nodes to handle models that are too large for one node's memory. - -**Usage**: Check [here](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-two-h2008-nodes-and-docker) for usage examples. - -### Block-wise FP8 - -**Description**: SGLang implements block-wise FP8 quantization with two key optimizations: - -- **Activation**: E4M3 format using per-token-per-128-channel sub-vector scales with online casting. - -- **Weight**: Per-128x128-block quantization for better numerical stability. - -- **DeepGEMM**: The [DeepGEMM](https://github.com/deepseek-ai/DeepGEMM) kernel library optimized for FP8 matrix multiplications. - -**Usage**: The activation and weight optimization above are turned on by default for DeepSeek V3 models. DeepGEMM is enabled by default on NVIDIA Hopper/Blackwell GPUs and disabled by default on other devices. DeepGEMM can also be manually turned off by setting the environment variable `SGLANG_ENABLE_JIT_DEEPGEMM=0`. - - -Before serving the DeepSeek model, precompile the DeepGEMM kernels to improve first-run performance. The precompilation process typically takes around 10 minutes to complete. - - -```bash Command -python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code -``` - -### Multi-token Prediction -**Description**: SGLang implements DeepSeek V3 Multi-Token Prediction (MTP) based on [EAGLE speculative decoding](../advanced_features/speculative_decoding#EAGLE-Decoding). With this optimization, the decoding speed can be improved by **1.8x** for batch size 1 and **1.5x** for batch size 32 respectively on H200 TP8 setting. - -**Usage**: -Add `--speculative-algorithm EAGLE`. Other flags, like `--speculative-num-steps`, `--speculative-eagle-topk` and `--speculative-num-draft-tokens` are optional. For example: -```text Output -python3 -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3-0324 \ - --speculative-algorithm EAGLE \ - --trust-remote-code \ - --tp 8 -``` -- The default configuration for DeepSeek models is `--speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4`. The best configuration for `--speculative-num-steps`, `--speculative-eagle-topk` and `--speculative-num-draft-tokens` can be searched with [bench_speculative.py](https://github.com/sgl-project/sglang/blob/main/scripts/playground/bench_speculative.py) script for given batch size. The minimum configuration is `--speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2`, which can achieve speedup for larger batch sizes. -- Most MLA attention backends fully support MTP usage. See [MLA Backends](../advanced_features/attention_backend#mla-backends) for details. - - -To enable DeepSeek MTP for large batch sizes (>48), you need to adjust some parameters (Reference [this discussion](https://github.com/sgl-project/sglang/issues/4543#issuecomment-2737413756)): -- Adjust `--max-running-requests` to a larger number. The default value is `48` for MTP. For larger batch sizes, you should increase this value beyond the default value. -- Set `--cuda-graph-bs`. It's a list of batch sizes for cuda graph capture. The [default captured batch sizes for speculative decoding](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/server_args.py#L888-L895) is 48. You can customize this by including more batch sizes. - - - -To enable the experimental overlap scheduler for EAGLE speculative decoding, set the environment variable `SGLANG_ENABLE_SPEC_V2=1`. This can improve performance by enabling overlap scheduling between draft and verification stages. - - - -### Reasoning Content for DeepSeek R1 & V3.1 - -See [Reasoning Parser](../advanced_features/separate_reasoning) and [Thinking Parameter for DeepSeek V3.1](./openai_api_completions#Example:-DeepSeek-V3-Models). - - -### Function calling for DeepSeek Models - -Add arguments `--tool-call-parser deepseekv3` and `--chat-template ./examples/chat_template/tool_chat_template_deepseekv3.jinja`(recommended) to enable this feature. For example (running on 1 * H20 node): - -```text Output -python3 -m sglang.launch_server \ - --model deepseek-ai/DeepSeek-V3-0324 \ - --tp 8 \ - --port 30000 \ - --host 0.0.0.0 \ - --mem-fraction-static 0.9 \ - --tool-call-parser deepseekv3 \ - --chat-template ./examples/chat_template/tool_chat_template_deepseekv3.jinja -``` - -Sample Request: - -``` -curl "http://127.0.0.1:30000/v1/chat/completions" \ --H "Content-Type: application/json" \ --d '{"temperature": 0, "max_tokens": 100, "model": "deepseek-ai/DeepSeek-V3-0324", "tools": [{"type": "function", "function": {"name": "query_weather", "description": "Get weather of a city, the user should supply a city first", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city, e.g. Beijing"}}, "required": ["city"]}}}], "messages": [{"role": "user", "content": "How'\''s the weather like in Qingdao today"}]}' -``` - -Expected Response - -```text Output -{"id":"6501ef8e2d874006bf555bc80cddc7c5","object":"chat.completion","created":1745993638,"model":"deepseek-ai/DeepSeek-V3-0324","choices":[{"index":0,"message":{"role":"assistant","content":null,"reasoning_content":null,"tool_calls":[{"id":"0","index":null,"type":"function","function":{"name":"query_weather","arguments":"{\"city\": \"Qingdao\"}"}}]},"logprobs":null,"finish_reason":"tool_calls","matched_stop":null}],"usage":{"prompt_tokens":116,"total_tokens":138,"completion_tokens":22,"prompt_tokens_details":null}} - -``` -Sample Streaming Request: -``` -curl "http://127.0.0.1:30000/v1/chat/completions" \ --H "Content-Type: application/json" \ --d '{"temperature": 0, "max_tokens": 100, "model": "deepseek-ai/DeepSeek-V3-0324","stream":true,"tools": [{"type": "function", "function": {"name": "query_weather", "description": "Get weather of a city, the user should supply a city first", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city, e.g. Beijing"}}, "required": ["city"]}}}], "messages": [{"role": "user", "content": "How'\''s the weather like in Qingdao today"}]}' -``` -Expected Streamed Chunks (simplified for clarity): -```text Output -data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"{\""}}]}}]} -data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"city"}}]}}]} -data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"\":\""}}]}}]} -data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"Q"}}]}}]} -data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"ing"}}]}}]} -data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"dao"}}]}}]} -data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"\"}"}}]}}]} -data: {"choices":[{"delta":{"tool_calls":null}}], "finish_reason": "tool_calls"} -data: [DONE] -``` -The client needs to concatenate all arguments fragments to reconstruct the complete tool call: -```text Output -{"city": "Qingdao"} -``` - - -1. Use a lower `"temperature"` value for better results. -2. To receive more consistent tool call results, it is recommended to use `--chat-template examples/chat_template/tool_chat_template_deepseekv3.jinja`. It provides an improved unified prompt. - - - -### Thinking Budget for DeepSeek R1 - -In SGLang, we can implement thinking budget with `CustomLogitProcessor`. - -Launch a server with `--enable-custom-logit-processor` flag on. - -```text Output -python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-R1 --tp 8 --port 30000 --host 0.0.0.0 --mem-fraction-static 0.9 --disable-cuda-graph --reasoning-parser deepseek-r1 --enable-custom-logit-processor -``` - -Sample Request: - -```python Sample Request -import openai -from rich.pretty import pprint -from sglang.srt.sampling.custom_logit_processor import DeepSeekR1ThinkingBudgetLogitProcessor - - -client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="*") -response = client.chat.completions.create( - model="deepseek-ai/DeepSeek-R1", - messages=[ - { - "role": "user", - "content": "Question: Is Paris the Capital of France?", - } - ], - max_tokens=1024, - extra_body={ - "custom_logit_processor": DeepSeekR1ThinkingBudgetLogitProcessor().to_str(), - "custom_params": { - "thinking_budget": 512, - }, - }, -) -pprint(response) -``` - -## FAQ - -**Q: Model loading is taking too long, and I'm encountering an NCCL timeout. What should I do?** - -A: If you're experiencing extended model loading times and an NCCL timeout, you can try increasing the timeout duration. Add the argument `--dist-timeout 3600` when launching your model. This will set the timeout to one hour, which often resolves the issue. diff --git a/docs_new/docs/basic_usage/deepseek_v32.mdx b/docs_new/docs/basic_usage/deepseek_v32.mdx deleted file mode 100644 index a1cbf0d9c..000000000 --- a/docs_new/docs/basic_usage/deepseek_v32.mdx +++ /dev/null @@ -1,601 +0,0 @@ ---- -title: "DeepSeek V3.2/GLM-5 Usage" -metatags: - description: "Deploy DeepSeek V3.2/GLM-5 with SGLang: DeepSeek Sparse Attention (DSA), long-context optimization, MTP speculative decoding, function calling. Supports H200, B200, MI300X, MI350." ---- -DeepSeek-V3.2 model family equips DeepSeek-V3.1-Terminus with DeepSeek Sparse Attention (DSA) through continued training. With DSA, a fine-grained sparse attention mechanism powered by a lightning indexer, DeepSeek-V3.2 achieves efficiency improvements in long-context scenarios. - - -Note: This document is originally written for the usage of [DeepSeek-V3.2-Exp](https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp) model. The usage of [DeepSeek-V3.2](https://huggingface.co/deepseek-ai/DeepSeek-V3.2) or [DeepSeek-V3.2-Speciale](https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Speciale) is the same as DeepSeek-V3.2-Exp except for the tool call parser. [GLM-5](https://huggingface.co/zai-org/GLM-5) model also applies DSA (DeepSeek Sparse Attention) structure, so it can share most of the usage here, except for the reasoning parser and tool call parser. - - -## Installation - -### Docker - -```bash Command -# H200/B200 -docker pull lmsysorg/sglang:latest - -# MI350/MI355 -docker pull lmsysorg/sglang:v0.5.8-rocm700-mi35x - -# MI300 -# v0.5.8-rocm700-mi30x does not include PR #17504. Prefer the newest MI30x ROCm -# image tag from Docker Hub when available, or build from source (below). -docker pull lmsysorg/sglang:v0.5.8-rocm700-mi30x - - -# NPUs -docker pull lmsysorg/sglang:dsv32-a2 -docker pull lmsysorg/sglang:dsv32-a3 -``` - -### Build From Source - -```bash Command -# Install SGLang -git clone https://github.com/sgl-project/sglang -cd sglang -pip3 install pip --upgrade -pip3 install -e "python" -``` - -## Launch DeepSeek V3.2/GLM-5 with SGLang - -To serve [DeepSeek-V3.2-Exp](https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp) on 8xH200/B200 GPUs: - -```bash Command -# Launch with TP + DP (Recommended) -python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --dp 8 --enable-dp-attention - -# Launch with EP + DP -python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --ep 8 --dp 8 --enable-dp-attention - -# Launch with Pure TP -python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 - -# Launch with TP on MI30x/MI35x -python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --dsa-prefill-backend tilelang --dsa-decode-backend tilelang -``` - -To serve GLM-5, just replace the `--model` argument with `zai-org/GLM-5-FP8`. - -### Configuration Tips -- **DP Attention**: To enable [DP Attention](../advanced_features/dp_dpa_smg_guide), please include `--enable-dp-attention --dp ` in command. DP Attention is better for large concurrency scenarios. -- **TP Attention**: Launching with TP attention is also supported. TP attention is better for low latency scenarios. -- **Short-sequence MHA prefill (adaptive)**: For short prefill sequences (default threshold: **2048 tokens**), the DSA backend uses standard MHA automatically (no extra flags). On H200 (SM90) this path uses the FlashAttention variable-length kernel; on B200 (SM100) it uses TRT-LLM ragged MHA. MHA uses `MHA_ONE_SHOT` for best performance, which computes multi-head attention over all tokens (both cached prefix and newly extended tokens) in a single kernel invocation, avoiding the overhead of chunked KV cache processing. This achieves optimal throughput for short sequences where total sequence length fits within the chunk capacity limit. -- **MHA prefill threshold relaxation**: To apply MHA attention to requests longer than 2048 tokens, please set the flag `SGLANG_DSA_PREFILL_DENSE_ATTN_KV_LEN_THRESHOLD` to a value larger than 2048 (`SGLANG_NSA_PREFILL_DENSE_ATTN_KV_LEN_THRESHOLD` is a deprecated alias). As threshold grows larger, the prefill performance can be improved, but at the cost of potential accuracy drop. -- **Choices of Attention Kernels**: The attention backend is automatically set to `dsa` attention backend for DeepSeek V3.2 model. In this backend, different kernels for sparse prefilling/decoding are implemented, which can be specified by `--dsa-prefill-backend` and `--dsa-decode-backend` server arguments. The choices of dsa prefill/decode attention kernels include: - - `flashmla_sparse`: `flash_mla_sparse_fwd` kernel from `flash_mla` library. Can run on both Hopper and Blackwell GPUs. It requires bf16 q, kv inputs. - - `flashmla_kv`: `flash_mla_with_kvcache` kernel from `flash_mla` library. Can run on both Hopper and Blackwell GPUs. It requires bf16 q, fp8 k_cache inputs. - - `flashmla_auto`: enables automatic selection of either `flashmla_sparse` or `flashmla_kv` kernel for prefill based on KV cache dtype, hardware, and heuristics. With BF16 KV cache, `flashmla_sparse` is always used on both Hopper and Blackwell. With FP8 KV cache: On Hopper (SM90), it unconditionally uses `flashmla_kv`; On Blackwell (SM100), it uses `flashmla_sparse` when `total_kv_tokens < total_q_tokens * 512`, otherwise falls back to `flashmla_kv`. The heuristics may need to be tuned if the performance of either kernel changes significantly. - - `fa3`: `flash_attn_with_kvcache` kernel from `flash_attn` library. Can only run on Hopper GPUs. It requires bf16 q, kv inputs. - - `tilelang`: `tilelang` implementation that can run on GPU, HPU and NPU. - - `aiter`: Aiter kernel on AMD HPUs. Can only be used as decode kernel. - - `trtllm`: `trtllm-mla` sparse kernel from flashinfer library. Only run on blackwell GPUs. It requires q,k,v to be uniformly bf16 or fp8_e4m3 format. - - On the basis of performance benchmarks, the default configuration of DSA kernels on Hopper and Blackwell are set as follows : - - Bfloat 16 kv cache: On Hopper, `flashmla_sparse` prefill attention, `fa3` decode attention; On Blackwell, `flashmla_sparse` prefill attention, `trtllm` decode attention - - Float8_e4m3fn KV cache: On Hopper, `flashmla_kv` prefill attention, `flashmla_kv` decode attention; On Blackwell, `trtllm` prefill attention and `trtllm` decode attention. -- **Index Cache**: Introduce in [this paper](https://arxiv.org/abs/2603.12201), IndexCache improves speed by reusing the result of indexer across different layers, only at cost of negligible accuracy loss. For **GLM-5** model, we recommend appending `--json-model-override-args '{"index_topk_pattern": "FFSFSSSFSSFFFSSSFFFSFSSSSSSFFSFFSFFSSFFFFFFSFFFFFSFFSSSSSSFSFFFSFSSSFSFFSFFSSS"}'` to command for better tradeoff between speedup and performance. - -## Multi-token Prediction -SGLang implements Multi-Token Prediction (MTP) for DeepSeek V3.2 based on [EAGLE speculative decoding](../advanced_features/speculative_decoding#EAGLE-Decoding). With this optimization, the decoding speed can be improved significantly on small batch sizes. Please look at [this PR](https://github.com/sgl-project/sglang/pull/11652) for more information. - -Example usage with DP Attention: -```bash Command -python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --dp 8 --enable-dp-attention --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 -``` - -Example usage with Pure TP: -```bash Command -python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 -``` - -- The best configuration for `--speculative-num-steps`, `--speculative-eagle-topk` and `--speculative-num-draft-tokens` can be searched with [bench_speculative.py](https://github.com/sgl-project/sglang/blob/main/scripts/playground/bench_speculative.py) script for given batch size. The minimum configuration is `--speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2`, which can achieve speedup for larger batch sizes. -- The default value of `--max-running-requests` is set to `48` for MTP. For larger batch sizes, this value should be increased beyond the default value. - - -To enable overlap scheduler for EAGLE speculative decoding, we recommend setting the environment variable `SGLANG_ENABLE_SPEC_V2=1`. This can improve performance by enabling overlap scheduling between draft and verification stages. - - - -## Function Calling and Reasoning Parser -The usage of function calling and reasoning parser is the same as DeepSeek V3.1. Please refer to [Reasoning Parser](../advanced_features/separate_reasoning) and [Tool Parser](../advanced_features/tool_parser) documents. - -To launch `DeepSeek-V3.2-Exp` with function calling and reasoning parser: -> Note: It is recommended to specify the chat-template, ensuring that you are within the sglang's root directory. -```bash Command -python3 -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3.2-Exp \ - --trust-remote-code \ - --tp-size 8 --dp-size 8 --enable-dp-attention \ - --tool-call-parser deepseekv31 \ - --reasoning-parser deepseek-v3 \ - --chat-template ./examples/chat_template/tool_chat_template_deepseekv32.jinja -``` - -To launch `DeepSeek-V3.2` with function calling and reasoning parser: -```bash Command -python3 -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3.2 \ - --trust-remote-code \ - --tp-size 8 --dp-size 8 --enable-dp-attention \ - --tool-call-parser deepseekv32 \ - --reasoning-parser deepseek-v3 -``` - -`DeepSeek-V3.2-Speciale` does not support tool calling, so it can only be launched with the reasoning parser: -```bash Command -python3 -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3.2-Speciale \ - --trust-remote-code \ - --tp-size 8 --dp-size 8 --enable-dp-attention \ - --reasoning-parser deepseek-v3 -``` - -To launch `GLM-5` with function calling and reasoning parser: -```bash Command -python -m sglang.launch_server \ - --model zai-org/GLM-5-FP8 \ - --tp-size 8 --dp-size 8 --enable-dp-attention \ - --tool-call-parser glm47 \ - --reasoning-parser glm45 \ -``` - -## NVFP4 Checkpoint - -To launch deepseek v3.2 [NVFP4 checkpoint](https://huggingface.co/nvidia/DeepSeek-V3.2-NVFP4) on Blackwell devices, the user needs to specify the quantization method as `modelopt_fp4`, and moe runner backend as one of `flashinfer_trtllm`(recommended), `flashinfer_cutlass` and `flashinfer_cutedsl`. Any other usage (parallelism, reasoning parser, ...) is the same as FP8 checkpoint. - -An example launching command can be: -```bash Command -python -m sglang.launch_server --model nvidia/DeepSeek-V3.2-NVFP4 --tp 4 --quantization modelopt_fp4 --moe-runner-backend flashinfer_trtllm --tool-call-parser deepseekv32 --reasoning-parser deepseek-v3 -``` - -## PD Disaggregation - -Prefill Command: -```bash Command -python -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3.2-Exp \ - --disaggregation-mode prefill \ - --host $LOCAL_IP \ - --port $PORT \ - --tp 8 \ - --dp 8 \ - --enable-dp-attention \ - --dist-init-addr ${HOST}:${DIST_PORT} \ - --trust-remote-code \ - --disaggregation-bootstrap-port 8998 \ - --mem-fraction-static 0.9 \ -``` - -Decode command: -```bash Command -python -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3.2-Exp \ - --disaggregation-mode decode \ - --host $LOCAL_IP \ - --port $PORT \ - --tp 8 \ - --dp 8 \ - --enable-dp-attention \ - --dist-init-addr ${HOST}:${DIST_PORT} \ - --trust-remote-code \ - --mem-fraction-static 0.9 \ -``` - -Router command: -```bash Command -python -m sglang_router.launch_router --pd-disaggregation \ - --prefill $PREFILL_ADDR 8998 \ - --decode $DECODE_ADDR \ - --host 127.0.0.1 \ - --port 8000 \ -``` - -If you need more advanced deployment methods or production-ready deployment methods, such as RBG or LWS-based deployment, please refer to [references/multi_node_deployment/rbg_pd/deepseekv32_pd.md](../references/multi_node_deployment/rbg_pd/deepseekv32_pd). Additionally, you can also find startup commands for DeepEP-based EP parallelism in the aforementioned documentation. - - -## Benchmarking Results - -### Accuracy Test with `gsm8k` -A simple accuracy benchmark can be tested with `gsm8k` dataset: -```bash Command -python3 benchmark/gsm8k/bench_sglang.py --num-shots 8 --num-questions 1319 --parallel 1319 -``` - -The result is 0.956, which matches our expectation: -```bash Command -Accuracy: 0.956 -Invalid: 0.000 -Latency: 25.109 s -Output throughput: 5226.235 token/s -``` - -To test long-context accuracy, run gsm8k with `--num-shots 20`. The results are very close to the 8 shots results: -```text Output -Accuracy: 0.956 -Invalid: 0.000 -Latency: 29.545 s -Output throughput: 4418.617 token/s -``` - - -### Accuracy Test with `gpqa-diamond` - -Accuracy benchmark on long context can be tested on GPQA-diamond dataset with long output tokens and thinking enabled: -```bash Command -python3 -m sglang.test.run_eval --port 30000 --eval-name gpqa --num-examples 198 --max-tokens 128000 --repeat 8 --thinking-mode deepseek-v3 -``` - -The mean accuracy over 8 runs shows 0.797, which matches the number 0.799 in official tech report. -```bash Command -Repeat: 8, mean: 0.797 -Scores: ['0.808', '0.798', '0.808', '0.798', '0.783', '0.788', '0.803', '0.793'] -``` - -For DeepSeek V3.2, DeepSeek recommends setting the sampling parameters to temperature = 1.0, top_p = 0.95: - -```bash Command -python3 -m sglang.test.run_eval --port 30000 --eval-name gpqa --num-examples 198 --max-tokens 128000 --repeat 8 --top-p 0.95 --temperature 1.0 --thinking-mode deepseek-v3 - -Repeat: 8, mean: 0.840 -Scores: ['0.848', '0.808', '0.848', '0.838', '0.879', '0.813', '0.838', '0.848'] -``` -which matches the official score, 0.824, as reported in the [DeepSeek-V3.2 technical report](https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/main/assets/paper.pdf). - -### Accuracy Test with `aime 2025` - -Prepare the environment by installing NeMo-Skills in the docker or your own virtual environment: - - ``` - pip install git+https://github.com/NVIDIA/NeMo-Skills.git --ignore-installed blinker - ``` - -Then launch the SGLang server: -```text Output -python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --dp 8 --enable-dp-attention -``` - -**For `DeepSeek-V3.2` and `DeepSeek-V3.2-Speciale`**: - -```text Output -python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V3.2 --trust-remote-code --tp-size 8 --dp-size 8 --enable-dp-attention --tool-call-parser deepseekv32 --reasoning-parser deepseek-v3 -``` - -Run the following script to evaluate AIME 2025: -```text Output -#! /bin/bash -export NEMO_SKILLS_DISABLE_UNCOMMITTED_CHANGES_CHECK=1 - -ns prepare_data aime25 - -PORT=30000 -BACKEND=sglang -MODEL="deepseek-ai/DeepSeek-V3.2-Exp" # Should be changed to the model name -MODEL_NAME="dsv32-fp8" - -echo "Starting AIME25 evaluation with model $MODEL on port $PORT using backend $BACKEND..." -ns eval \ - --benchmarks=aime25:4 \ - --server_type=$BACKEND \ - --model=$MODEL \ - --server_address=http://localhost:${PORT}/v1 \ - --output_dir=nemo_skills_aime25_${MODEL_NAME}_output_${BACKEND}_$(date +%Y%m%d_%H%M%S) \ - ++chat_template_kwargs.thinking=true \ - ++inference.temperature=1.0 \ - ++inference.top_p=0.95 \ - ++inference.tokens_to_generate=64000 - # ++inference.tokens_to_generate=120000 for Speciale model -``` - -Test results (8*B200): - -DeepSeek-V3.2-Exp: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
evaluation_modenum_entriesavg_tokensgen_secondssymbolic_correctno_answer
pass@1[avg-of-4]3015040167387.50% ± 1.67%0.00%
majority@43015040167390.00%0.00%
pass@43015040167390.00%0.00%
- - -DeepSeek-V3.2: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
evaluation_modenum_entriesavg_tokensgen_secondssymbolic_correctno_answer
pass@1[avg-of-4]3013550163292.50% ± 1.67%0.00%
majority@43013550163294.71%0.00%
pass@43013550163296.67%0.00%
- - -DeepSeek-V3.2-Speciale: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
evaluation_modenum_entriesavg_tokensgen_secondssymbolic_correctno_answer
pass@1[avg-of-4]3024155358395.00% ± 1.92%0.00%
majority@43024155358395.83%0.00%
pass@430241553583100.00%0.00%
- - -## DSA long sequence context parallel optimization(experimental) - -**Note: This feature is only verified on Hopper machines** - -For context parallel in DeepSeek V3.2 model, we provide two different modes of splitting tokens, which can be controlled with argument `--dsa-prefill-cp-mode`. - -### In sequence splitting - -The first mode can be enabled by `--dsa-prefill-cp-mode in-seq-split`. This mode implements context parallel for DSA by splitting the sequence uniformly between context parallel ranks. At attention stage, each cp rank computes the indexer results of sharded sequence, and collects the whole kv cache through all gather operator. Add `attn_cp_size` for communication group for context parallel. - -Note that the in-sequence splitting mode has the following restrictions: -- The batch size is restricted to 1 for prefill batches -- `moe_dense_tp_size=1`, `moe_a2a_backend = "deepep"` -- To ensure `cp_size > 1`, the passed in `tp_size` must be larger than `dp_size` - -For more details, please refer to PR https://github.com/sgl-project/sglang/pull/12065. - -Example: -```bash Command -# In-seq splitting mode launched with EP + DP -python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --ep 8 --dp 2 --enable-dp-attention --enable-dsa-prefill-context-parallel --attn-cp-size 4 --dsa-prefill-cp-mode in-seq-split --max-running-requests 32 -``` - -### Round robin splitting (default setting) - -This mode can be enabled by specifying the parameter `--dsa-prefill-cp-mode round-robin-split`, which distributes tokens across ranks based on `token_idx % cp_size`. - -In this scenario, compared to the in-sequence splitting method, it additionally supports the fused MoE backend (the fused MoE backend may deliver better performance than DeepEP in single-machine scenarios), FP8 KV-cache, and multi-batch prefill inference. However, it cannot be enabled with DP attention together. - -For more details, please refer to PR https://github.com/sgl-project/sglang/pull/13959. - -Example usage: -```bash Command -# Launch with FusedMoe + CP8 -python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --enable-dsa-prefill-context-parallel --attn-cp-size 8 --dsa-prefill-cp-mode round-robin-split --max-running-requests 32 -``` -### Pipeline Parallel + Context Parallel (PP + CP) - -This mode combines Pipeline Parallelism (PP) and Context Parallelism (CP) to scale across multiple nodes, which can achieve better throughput and Time To First Token (TTFT). Note that this method has only been tested on H20 96G. - -#### Standard Usage - -To launch with PP=2 and CP (via `round-robin-split` mode) on 2 nodes. This configuration uses the fused MoE kernel by default, which generally provides better performance. - -For related development details, please refer to: -- Fused MoE + CP support: [PR #13959](https://github.com/sgl-project/sglang/pull/13959) -- PP + CP support: [Issue #15358](https://github.com/sgl-project/sglang/issues/15358) and [PR #16380](https://github.com/sgl-project/sglang/pull/16380) - -Node 0: -```bash Command -export SGLANG_PP_LAYER_PARTITION=30,31 -python3 -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3.2-Exp \ - --nnodes 2 --node-rank 0 \ - --dist-init-addr :62001 \ - --tp 8 --pp-size 2 \ - --dp-size 1 --moe-dense-tp-size 1 \ - --enable-dsa-prefill-context-parallel \ - --attn-cp-size 8 \ - --dsa-prefill-cp-mode round-robin-split \ - --trust-remote-code \ - --disable-radix-cache \ - --mem-fraction-static 0.8 \ - --max-running-requests 128 \ - --chunked-prefill-size 16384 \ - --cuda-graph-max-bs 8 \ - --page-size 64 \ - --watchdog-timeout 3600 \ - --host 0.0.0.0 --port 8000 \ - --tool-call-parser deepseekv32 -``` - -Node 1: -```bash Command -export SGLANG_PP_LAYER_PARTITION=30,31 -python3 -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3.2-Exp \ - --nnodes 2 --node-rank 1 \ - --dist-init-addr :62001 \ - --tp 8 --pp-size 2 \ - --dp-size 1 --moe-dense-tp-size 1 \ - --enable-dsa-prefill-context-parallel \ - --attn-cp-size 8 \ - --dsa-prefill-cp-mode round-robin-split \ - --trust-remote-code \ - --disable-radix-cache \ - --mem-fraction-static 0.8 \ - --max-running-requests 128 \ - --chunked-prefill-size 16384 \ - --cuda-graph-max-bs 8 \ - --page-size 64 \ - --watchdog-timeout 3600 \ - --host 0.0.0.0 --port 8000 \ - --tool-call-parser deepseekv32 -``` - -#### PD Disaggregation with PP + CP - -If using PD (Prefill-Decode) Disaggregation, the Prefill nodes can be configured with PP + CP as follows. - -Prefill Node 0: -```bash Command -python -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3.2-Exp \ - --served-model-name deepseek-v32 \ - --nnodes 2 --node-rank 0 \ - --dist-init-addr :20102 \ - --tp 8 --pp-size 2 \ - --dp-size 1 --moe-dense-tp-size 1 \ - --enable-dsa-prefill-context-parallel \ - --attn-cp-size 8 \ - --dsa-prefill-cp-mode round-robin-split \ - --disaggregation-ib-device mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3 \ - --trust-remote-code \ - --disable-radix-cache \ - --max-running-requests 512 \ - --chunked-prefill-size 4096 \ - --context-length 131072 \ - --mem-fraction-static 0.9 \ - --page-size 64 \ - --enable-metrics \ - --collect-tokens-histogram \ - --tokenizer-worker-num 8 \ - --host 0.0.0.0 --port 30000 -``` - -Prefill Node 1: -```bash Command -python -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3.2-Exp \ - --served-model-name deepseek-v32-prefill \ - --nnodes 2 --node-rank 1 \ - --dist-init-addr :20102 \ - --tp 8 --pp-size 2 \ - --dp-size 1 --moe-dense-tp-size 1 \ - --enable-dsa-prefill-context-parallel \ - --attn-cp-size 8 \ - --dsa-prefill-cp-mode round-robin-split \ - --disaggregation-ib-device mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3 \ - --trust-remote-code \ - --disable-radix-cache \ - --max-running-requests 512 \ - --chunked-prefill-size 4096 \ - --context-length 131072 \ - --mem-fraction-static 0.9 \ - --page-size 64 \ - --enable-metrics \ - --collect-tokens-histogram \ - --tokenizer-worker-num 8 \ - --host 0.0.0.0 --port 30000 -``` - -For the Decode nodes, it is recommended to use the **EP mode**. - -## HiSparse: Hierarchical Sparse Attention for DSA (experimental) - -HiSparse reduces per-request GPU memory during decode by keeping only a small "hot" KV buffer on GPU while storing complete KV data in CPU pinned memory. A CUDA kernel dynamically swaps in the top-k most relevant KV entries from host memory on each decode step. This enables significantly higher decode concurrency for long-context DSA models. - -HiSparse currently requires PD disaggregation mode and is enabled on the decode instance only. For detailed design, configuration, and deployment instructions, see the [HiSparse Guide](../advanced_features/hisparse_guide). diff --git a/docs_new/docs/basic_usage/glm45.mdx b/docs_new/docs/basic_usage/glm45.mdx deleted file mode 100644 index 210c85756..000000000 --- a/docs_new/docs/basic_usage/glm45.mdx +++ /dev/null @@ -1,75 +0,0 @@ ---- -title: "Launch GLM-4.5 / GLM-4.6 / GLM-4.7 with SGLang" -metatags: - description: "Deploy GLM-4.5/4.6/4.7 models with SGLang: FP8 inference, EAGLE speculative decoding, function calling support. Optimized for H100/H200 GPUs." ---- -## Launch GLM-4.5 / GLM-4.6 / GLM-4.7 with SGLang - -To serve GLM-4.5 / GLM-4.6 FP8 models on 8xH100/H200 GPUs: - -```bash Command -python3 -m sglang.launch_server --model zai-org/GLM-4.6-FP8 --tp 8 -``` - -### EAGLE Speculative Decoding - -**Description**: SGLang has supported GLM-4.5 / GLM-4.6 models -with [EAGLE speculative decoding](../advanced_features/speculative_decoding#EAGLE-Decoding). - -**Usage**: -Add arguments `--speculative-algorithm`, `--speculative-num-steps`, `--speculative-eagle-topk` and -`--speculative-num-draft-tokens` to enable this feature. For example: - -```bash Command -python3 -m sglang.launch_server \ - --model-path zai-org/GLM-4.6-FP8 \ - --tp-size 8 \ - --tool-call-parser glm45 \ - --reasoning-parser glm45 \ - --speculative-algorithm EAGLE \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --mem-fraction-static 0.9 \ - --served-model-name glm-4.6-fp8 \ - --enable-custom-logit-processor -``` - - -To enable the experimental overlap scheduler for EAGLE speculative decoding, set the environment variable `SGLANG_ENABLE_SPEC_V2=1`. This can improve performance by enabling overlap scheduling between draft and verification stages. - - -### Thinking Budget for GLM-4.5 / GLM-4.6 -**Note**: For GLM-4.7, `--tool-call-parser` should be set to `glm47`, for GLM-4.5 and GLM-4.6, it should be set to `glm45`. - -In SGLang, we can implement thinking budget with `CustomLogitProcessor`. - -Launch a server with `--enable-custom-logit-processor` flag on. - -Sample Request: - -```python Example -import openai -from rich.pretty import pprint -from sglang.srt.sampling.custom_logit_processor import Glm4MoeThinkingBudgetLogitProcessor - - -client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="*") -response = client.chat.completions.create( - model="zai-org/GLM-4.6", - messages=[ - { - "role": "user", - "content": "Question: Is Paris the Capital of France?", - } - ], - max_tokens=1024, - extra_body={ - "custom_logit_processor": Glm4MoeThinkingBudgetLogitProcessor().to_str(), - "custom_params": { - "thinking_budget": 512, - }, - }, -) -pprint(response) -``` diff --git a/docs_new/docs/basic_usage/glmv.mdx b/docs_new/docs/basic_usage/glmv.mdx deleted file mode 100644 index 088ad0b92..000000000 --- a/docs_new/docs/basic_usage/glmv.mdx +++ /dev/null @@ -1,139 +0,0 @@ ---- -title: "GLM-4.6V / GLM-4.5V Usage" -metatags: - description: "Deploy GLM-4.6V/4.5V vision models with SGLang: FP8 and BF16 modes, expert parallelism, video understanding. Supports H100, H200, A100 GPUs." ---- -## Launch commands for SGLang - -Below are suggested launch commands tailored for different hardware / precision modes - -### FP8 (quantised) mode - -For high memory-efficiency and latency optimized deployments (e.g., on H100, H200) where FP8 checkpoint is supported: - -```bash Command -python3 -m sglang.launch_server \ - --model-path zai-org/GLM-4.6V-FP8 \ - --tp 2 \ - --ep 2 \ - --host 0.0.0.0 \ - --port 30000 \ - --keep-mm-feature-on-device -``` - -### Non-FP8 (BF16 / full precision) mode -For deployments on A100/H100 where BF16 is used (or FP8 snapshot not used): -```bash Command -python3 -m sglang.launch_server \ - --model-path zai-org/GLM-4.6V \ - --tp 4 \ - --ep 4 \ - --host 0.0.0.0 \ - --port 30000 -``` - -## Hardware-specific notes / recommendations - -- On H100 with FP8: Use the FP8 checkpoint for best memory efficiency. -- On A100 / H100 with BF16 (non-FP8): It’s recommended to use `--mm-max-concurrent-calls` to control parallel throughput and GPU memory usage during image/video inference. -- On H200 & B200: The model can be run “out of the box”, supporting full context length plus concurrent image + video processing. - -## Sending Image/Video Requests - -### Image input: - -```python Example -import requests - -url = f"http://localhost:30000/v1/chat/completions" - -data = { - "model": "zai-org/GLM-4.6V", - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What’s in this image?"}, - { - "type": "image_url", - "image_url": { - "url": "https://github.com/sgl-project/sglang/blob/main/examples/assets/example_image.png?raw=true" - }, - }, - ], - } - ], - "max_tokens": 300, -} - -response = requests.post(url, json=data) -print(response.text) -``` - -### Video Input: - -```python Example -import requests - -url = f"http://localhost:30000/v1/chat/completions" - -data = { - "model": "zai-org/GLM-4.6V", - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What’s happening in this video?"}, - { - "type": "video_url", - "video_url": { - "url": "https://github.com/sgl-project/sgl-test-files/raw/refs/heads/main/videos/jobs_presenting_ipod.mp4" - }, - }, - ], - } - ], - "max_tokens": 300, -} - -response = requests.post(url, json=data) -print(response.text) -``` - -## Important Server Parameters and Flags - -When launching the model server for **multimodal support**, you can use the following command-line arguments to fine-tune performance and behavior: - -- `--mm-attention-backend`: Specify multimodal attention backend. Eg. `fa3`(Flash Attention 3) -- `--mm-max-concurrent-calls `: Specifies the **maximum number of concurrent asynchronous multimodal data processing calls** allowed on the server. Use this to control parallel throughput and GPU memory usage during image/video inference. -- `--mm-per-request-timeout `: Defines the **timeout duration (in seconds)** for each multimodal request. If a request exceeds this time limit (e.g., for very large video inputs), it will be automatically terminated. -- `--keep-mm-feature-on-device`: Instructs the server to **retain multimodal feature tensors on the GPU** after processing. This avoids device-to-host (D2H) memory copies and improves performance for repeated or high-frequency inference workloads. -- `--mm-enable-dp-encoder`: Placing the ViT in data parallel while keeping the LLM in tensor parallel consistently lowers TTFT and boosts end-to-end throughput. -- `SGLANG_USE_CUDA_IPC_TRANSPORT=1`: Shared memory pool based CUDA IPC for multi-modal data transport. For significantly improving e2e latency. - -### Example usage with the above optimizations: -```bash Command -SGLANG_USE_CUDA_IPC_TRANSPORT=1 \ -SGLANG_VLM_CACHE_SIZE_MB=0 \ -python -m sglang.launch_server \ - --model-path zai-org/GLM-4.6V \ - --host 0.0.0.0 \ - --port 30000 \ - --trust-remote-code \ - --tp-size 8 \ - --enable-cache-report \ - --log-level info \ - --max-running-requests 64 \ - --mem-fraction-static 0.65 \ - --chunked-prefill-size 8192 \ - --attention-backend fa3 \ - --mm-attention-backend fa3 \ - --mm-enable-dp-encoder \ - --enable-metrics -``` - -### Thinking Budget for GLM-4.5V / GLM-4.6V - -In SGLang, we can implement thinking budget with `CustomLogitProcessor`. - -Launch a server with the `--enable-custom-logit-processor` flag. Then, use `Glm4MoeThinkingBudgetLogitProcessor` in the request, similar to the `GLM-4.6` example in [glm45.md](./glm45). diff --git a/docs_new/docs/basic_usage/gpt_oss.mdx b/docs_new/docs/basic_usage/gpt_oss.mdx deleted file mode 100644 index 25c656e91..000000000 --- a/docs_new/docs/basic_usage/gpt_oss.mdx +++ /dev/null @@ -1,181 +0,0 @@ ---- -title: "GPT OSS Usage" -metatags: - description: "Deploy GPT-OSS with SGLang: OpenAI Responses API compatible, built-in tools for web search and Python execution, reasoning levels, MCP tool server support." ---- -Please refer to [#8833](https://github.com/sgl-project/sglang/issues/8833). - -## Responses API & Built-in Tools - -### Responses API - -GPT‑OSS is compatible with the OpenAI Responses API. Use `client.responses.create(...)` with `model`, `instructions`, `input`, and optional `tools` to enable built‑in tool use. You can set reasoning level via `instructions`, e.g., "Reasoning: high" (also supports "medium" and "low") — levels: low (fast), medium (balanced), high (deep). - -### Built-in Tools - -GPT‑OSS can call built‑in tools for web search and Python execution. You can use the demo tool server or connect to external MCP tool servers. - -#### Python Tool - -- Executes short Python snippets for calculations, parsing, and quick scripts. -- By default runs in a Docker-based sandbox. To run on the host, set `PYTHON_EXECUTION_BACKEND=UV` (this executes model-generated code locally; use with care). -- Ensure Docker is available if you are not using the UV backend. It is recommended to run `docker pull python:3.11` in advance. - -#### Web Search Tool - -- Uses the Exa backend for web search. -- Requires an Exa API key; set `EXA_API_KEY` in your environment. Create a key at `https://exa.ai`. - -### Tool & Reasoning Parser - -- We support OpenAI Reasoning and Tool Call parser, as well as our SGLang native api for tool call and reasoning. Refer to [reasoning parser](../advanced_features/separate_reasoning) and [tool call parser](../advanced_features/tool_parser) for more details. - - -## Notes - -- Use **Python 3.12** for the demo tools. And install the required `gpt-oss` packages. -- The default demo integrates the web search tool (Exa backend) and a demo Python interpreter via Docker. -- For search, set `EXA_API_KEY`. For Python execution, either have Docker available or set `PYTHON_EXECUTION_BACKEND=UV`. - -Examples: -```bash Command -export EXA_API_KEY=YOUR_EXA_KEY -# Optional: run Python tool locally instead of Docker (use with care) -export PYTHON_EXECUTION_BACKEND=UV -``` - -Launch the server with the demo tool server: - -```bash Command -python3 -m sglang.launch_server \ - --model-path openai/gpt-oss-120b \ - --tool-server demo \ - --tp 2 -``` - -For production usage, sglang can act as an MCP client for multiple services. An [example tool server](https://github.com/openai/gpt-oss/tree/main/gpt-oss-mcp-server) is provided. Start the servers and point sglang to them: -```bash Command -mcp run -t sse browser_server.py:mcp -mcp run -t sse python_server.py:mcp - -python -m sglang.launch_server ... --tool-server ip-1:port-1,ip-2:port-2 -``` -The URLs should be MCP SSE servers that expose server information and well-documented tools. These tools are added to the system prompt so the model can use them. - -## Speculative Decoding - -SGLang supports speculative decoding for GPT-OSS models using EAGLE3 algorithm. This can significantly improve decoding speed, especially for small batch sizes. - -**Usage**: -Add `--speculative-algorithm EAGLE3` along with the draft model path. -```bash Command -python3 -m sglang.launch_server \ - --model-path openai/gpt-oss-120b \ - --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path lmsys/EAGLE3-gpt-oss-120b-bf16 \ - --tp 2 -``` - - -To enable the experimental overlap scheduler for EAGLE3 speculative decoding, set the environment variable `SGLANG_ENABLE_SPEC_V2=1`. This can improve performance by enabling overlap scheduling between draft and verification stages. - - -### Quick Demo - -```python Example -from openai import OpenAI - -client = OpenAI( - base_url="http://localhost:30000/v1", - api_key="sk-123456" -) - -tools = [ - {"type": "code_interpreter"}, - {"type": "web_search_preview"}, -] - -# Reasoning level example -response = client.responses.create( - model="openai/gpt-oss-120b", - instructions="You are a helpful assistant." - reasoning_effort="high" # Supports high, medium, or low - input="In one sentence, explain the transformer architecture.", -) -print("====== reasoning: high ======") -print(response.output_text) - -# Test python tool -response = client.responses.create( - model="openai/gpt-oss-120b", - instructions="You are a helfpul assistant, you could use python tool to execute code.", - input="Use python tool to calculate the sum of 29138749187 and 29138749187", # 58,277,498,374 - tools=tools -) -print("====== test python tool ======") -print(response.output_text) - -# Test browser tool -response = client.responses.create( - model="openai/gpt-oss-120b", - instructions="You are a helfpul assistant, you could use browser to search the web", - input="Search the web for the latest news about Nvidia stock price", - tools=tools -) -print("====== test browser tool ======") -print(response.output_text) -``` - -Example output: -```text Output -====== test python tool ====== -The sum of 29,138,749,187 and 29,138,749,187 is **58,277,498,374**. -====== test browser tool ====== -**Recent headlines on Nvidia (NVDA) stock** - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Date (2025)SourceKey news pointsStock‑price detail
**May 13**ReutersThe market data page shows Nvidia trading “higher” at **$116.61** with no change from the previous close.**$116.61** – latest trade (delayed ≈ 15 min)【14†L34-L38】
**Aug 18**CNBCMorgan Stanley kept an **overweight** rating and lifted its price target to **$206** (up from $200), implying a 14 % upside from the Friday close. The firm notes Nvidia shares have already **jumped 34 % this year**.No exact price quoted, but the article signals strong upside expectations【9†L27-L31】
**Aug 20**The Motley FoolNvidia is set to release its Q2 earnings on Aug 27. The article lists the **current price of $175.36**, down 0.16 % on the day (as of 3:58 p.m. ET).**$175.36** – current price on Aug 20【10†L12-L15】【10†L53-L57】
- -**What the news tells us** - -* Nvidia’s share price has risen sharply this year – up roughly a third according to Morgan Stanley – and analysts are still raising targets (now $206). -* The most recent market quote (Reuters, May 13) was **$116.61**, but the stock has surged since then, reaching **$175.36** by mid‑August. -* Upcoming earnings on **Aug 27** are a focal point; both the Motley Fool and Morgan Stanley expect the results could keep the rally going. - -**Bottom line:** Nvidia’s stock is on a strong upward trajectory in 2025, with price targets climbing toward $200‑$210 and the market price already near $175 as of late August. - -``` diff --git a/docs_new/docs/basic_usage/kimi_k2_5.mdx b/docs_new/docs/basic_usage/kimi_k2_5.mdx deleted file mode 100644 index d87920198..000000000 --- a/docs_new/docs/basic_usage/kimi_k2_5.mdx +++ /dev/null @@ -1,106 +0,0 @@ ---- -title: "Kimi-K2.5 Usage" -metatags: - description: "Deploy Kimi-K2.5 with SGLang: 1T-parameter multimodal MoE model, 256K context, MLA attention, MoonViT vision encoder, thinking and instant modes, tool calling support." ---- -[Kimi-K2.5](https://huggingface.co/moonshotai/Kimi-K2.5) is Moonshot AI's open-source, native multimodal, agentic MoE. It is a 1T-parameter model (32B active) with 256K context, MLA attention, and a MoonViT vision encoder, supporting both thinking and instant modes. - -In SGLang, Kimi-K2.5 uses the `kimi_k2` reasoning and tool-call parsers for correct thinking and tool handling. - -```{note} Example -Kimi-K2.5 support is in SGLang main and will land in the next release. Use the latest main or a nightly image until then. -``` - -Official deployment guide: [Kimi-K2.5 deployment guide](https://huggingface.co/moonshotai/Kimi-K2.5/blob/main/docs/deploy_guidance) - -## Install (Latest Main) - -```bash Command -uv pip install "sglang @ git+https://github.com/sgl-project/sglang.git#subdirectory=python" -# For CUDA 12: -uv pip install "nvidia-cudnn-cu12==9.16.0.29" -# For CUDA 13: -uv pip install "nvidia-cudnn-cu13==9.16.0.29" -``` - -## Launch Kimi-K2.5 with SGLang - -Example: single node, TP8 on H200. - -```bash Command -python3 -m sglang.launch_server \ - --model-path moonshotai/Kimi-K2.5 \ - --tp 8 \ - --trust-remote-code \ - --tool-call-parser kimi_k2 \ - --reasoning-parser kimi_k2 -``` - -### Parser Requirements - -- `--tool-call-parser kimi_k2`: Required for tool calling. -- `--reasoning-parser kimi_k2`: Required to parse thinking content; thinking mode is enabled by default. - -## Test the Deployment - -Thinking mode is enabled by default. To disable thinking (instant mode), pass `extra_body.chat_template_kwargs.thinking=false`. - -```bash Command -# Thinking mode (default) -curl http://localhost:30000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "moonshotai/Kimi-K2.5", - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Explain mixture-of-experts in one sentence."} - ], - "max_tokens": 256 - }' -``` - -```bash Command -# Instant mode (thinking disabled) -curl http://localhost:30000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "moonshotai/Kimi-K2.5", - "messages": [ - {"role": "user", "content": "Give one sentence on MoE models."} - ], - "max_tokens": 128, - "extra_body": {"chat_template_kwargs": {"thinking": false}} - }' -``` - -## Multimodal Inputs (Image/Video) - -Kimi-K2.5 is multimodal. Image inputs are supported via the OpenAI-compatible vision API. For more details, see `openai_api_vision.ipynb`. - -```bash Command -# Image input (SGLang) -curl http://localhost:30000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "moonshotai/Kimi-K2.5", - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": "Describe this image."}, - { - "type": "image_url", - "image_url": { - "url": "https://github.com/sgl-project/sglang/blob/main/examples/assets/example_image.png?raw=true" - } - } - ] - } - ], - "max_tokens": 256 - }' -``` - - -Video chat is experimental and is only supported in the official Moonshot API for now. - diff --git a/docs_new/docs/basic_usage/llama4.mdx b/docs_new/docs/basic_usage/llama4.mdx deleted file mode 100644 index c68ea27c7..000000000 --- a/docs_new/docs/basic_usage/llama4.mdx +++ /dev/null @@ -1,117 +0,0 @@ ---- -title: "Llama4 Usage" -metatags: - description: "Deploy Llama 4 Scout (109B) and Maverick (400B) with SGLang: up to 10M context, hybrid KV cache, vision support. Optimized for H100/H200 GPUs." ---- -[Llama 4](https://github.com/meta-llama/llama-models/blob/main/models/llama4/MODEL_CARD) is Meta's latest generation of open-source LLM model with industry-leading performance. - -SGLang has supported Llama 4 Scout (109B) and Llama 4 Maverick (400B) since [v0.4.5](https://github.com/sgl-project/sglang/releases/tag/v0.4.5). - -Ongoing optimizations are tracked in the [Roadmap](https://github.com/sgl-project/sglang/issues/5118). - -## Launch Llama 4 with SGLang - -To serve Llama 4 models on 8xH100/H200 GPUs: - -```bash Command -python3 -m sglang.launch_server \ - --model-path meta-llama/Llama-4-Scout-17B-16E-Instruct \ - --tp 8 \ - --context-length 1000000 -``` - -### Configuration Tips - -- **OOM Mitigation**: Adjust `--context-length` to avoid a GPU out-of-memory issue. For the Scout model, we recommend setting this value up to 1M on 8\*H100 and up to 2.5M on 8\*H200. For the Maverick model, we don't need to set context length on 8\*H200. When hybrid kv cache is enabled, `--context-length` can be set up to 5M on 8\*H100 and up to 10M on 8\*H200 for the Scout model. - -- **Attention Backend Auto-Selection**: SGLang automatically selects the optimal attention backend for Llama 4 based on your hardware. You typically don't need to specify `--attention-backend` manually: - - **Blackwell GPUs (B200/GB200)**: `trtllm_mha` - - **Hopper GPUs (H100/H200)**: `fa3` - - **AMD GPUs**: `aiter` - - **Intel XPU**: `intel_xpu` - - **Other platforms**: `triton` (fallback) - - To override the auto-selection, explicitly specify `--attention-backend` with one of the supported backends: `fa3`, `aiter`, `triton`, `trtllm_mha`, or `intel_xpu`. - -- **Chat Template**: Add `--chat-template llama-4` for chat completion tasks. -- **Enable Multi-Modal**: Add `--enable-multimodal` for multi-modal capabilities. -- **Enable Hybrid-KVCache**: Set `--swa-full-tokens-ratio` to adjust the ratio of SWA layer (for Llama4, it's local attention layer) KV tokens / full layer KV tokens. (default: 0.8, range: 0-1) - - -### EAGLE Speculative Decoding -**Description**: SGLang has supported Llama 4 Maverick (400B) with [EAGLE speculative decoding](../advanced_features/speculative_decoding#EAGLE-Decoding). - -**Usage**: -Add arguments `--speculative-draft-model-path`, `--speculative-algorithm`, `--speculative-num-steps`, `--speculative-eagle-topk` and `--speculative-num-draft-tokens` to enable this feature. For example: -```text Output -python3 -m sglang.launch_server \ - --model-path meta-llama/Llama-4-Maverick-17B-128E-Instruct \ - --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path nvidia/Llama-4-Maverick-17B-128E-Eagle3 \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --trust-remote-code \ - --tp 8 \ - --context-length 1000000 -``` - -- **Note** The Llama 4 draft model *nvidia/Llama-4-Maverick-17B-128E-Eagle3* can only recognize conversations in chat mode. - -## Benchmarking Results - -### Accuracy Test with `lm_eval` - -The accuracy on SGLang for both Llama4 Scout and Llama4 Maverick can match the [official benchmark numbers](https://ai.meta.com/blog/llama-4-multimodal-intelligence/). - -Benchmark results on MMLU Pro dataset with 8*H100: - - - - - - - - - - - - - - - - - - - - - - - - - -
Llama-4-Scout-17B-16E-InstructLlama-4-Maverick-17B-128E-Instruct
Official Benchmark74.380.5
SGLang75.280.7
- -Commands: - -```bash Command -# Llama-4-Scout-17B-16E-Instruct model -python -m sglang.launch_server \ - --model-path meta-llama/Llama-4-Scout-17B-16E-Instruct \ - --port 30000 \ - --tp 8 \ - --mem-fraction-static 0.8 \ - --context-length 65536 -lm_eval --model local-chat-completions --model_args model=meta-llama/Llama-4-Scout-17B-16E-Instruct,base_url=http://localhost:30000/v1/chat/completions,num_concurrent=128,timeout=999999,max_gen_toks=2048 --tasks mmlu_pro --batch_size 128 --apply_chat_template --num_fewshot 0 - -# Llama-4-Maverick-17B-128E-Instruct -python -m sglang.launch_server \ - --model-path meta-llama/Llama-4-Maverick-17B-128E-Instruct \ - --port 30000 \ - --tp 8 \ - --mem-fraction-static 0.8 \ - --context-length 65536 -lm_eval --model local-chat-completions --model_args model=meta-llama/Llama-4-Maverick-17B-128E-Instruct,base_url=http://localhost:30000/v1/chat/completions,num_concurrent=128,timeout=999999,max_gen_toks=2048 --tasks mmlu_pro --batch_size 128 --apply_chat_template --num_fewshot 0 -``` - -Details can be seen in [this PR](https://github.com/sgl-project/sglang/pull/5092). diff --git a/docs_new/docs/basic_usage/minimax_m2.mdx b/docs_new/docs/basic_usage/minimax_m2.mdx deleted file mode 100644 index c248b1942..000000000 --- a/docs_new/docs/basic_usage/minimax_m2.mdx +++ /dev/null @@ -1,88 +0,0 @@ ---- -title: "MiniMax M2.5/M2.1/M2 Usage" -metatags: - description: "Deploy MiniMax M2.5/M2.1/M2 with SGLang: 230B MoE model (10B active), up to 3M context, optimized for coding and agentic tasks, tool use support." ---- -[MiniMax-M2.5](https://huggingface.co/MiniMaxAI/MiniMax-M2.5), [MiniMax-M2.1](https://huggingface.co/MiniMaxAI/MiniMax-M2.1), and [MiniMax-M2](https://huggingface.co/MiniMaxAI/MiniMax-M2) are advanced large language models created by [MiniMax](https://www.minimax.io/). - -The MiniMax-M2 series redefines efficiency for agents. These compact, fast, and cost-effective MoE models (230 billion total parameters with 10 billion active parameters) are built for elite performance in coding and agentic tasks, all while maintaining powerful general intelligence. With just 10 billion activated parameters, the MiniMax-M2 series provides sophisticated, end-to-end tool use performance expected from today's leading models, but in a streamlined form factor that makes deployment and scaling easier than ever. - -## Supported Models - -This guide applies to the following models. You only need to update the model name during deployment. The following examples use **MiniMax-M2**: - -- [MiniMaxAI/MiniMax-M2.5](https://huggingface.co/MiniMaxAI/MiniMax-M2.5) -- [MiniMaxAI/MiniMax-M2.1](https://huggingface.co/MiniMaxAI/MiniMax-M2.1) -- [MiniMaxAI/MiniMax-M2](https://huggingface.co/MiniMaxAI/MiniMax-M2) - -## System Requirements - -The following are recommended configurations; actual requirements should be adjusted based on your use case: - -- 4x 96GB GPUs: Supported context length of up to 400K tokens. -- 8x 144GB GPUs: Supported context length of up to 3M tokens. - -## Deployment with Python - -4-GPU deployment command: - -```bash Command -python -m sglang.launch_server \ - --model-path MiniMaxAI/MiniMax-M2 \ - --tp-size 4 \ - --tool-call-parser minimax-m2 \ - --reasoning-parser minimax-append-think \ - --host 0.0.0.0 \ - --trust-remote-code \ - --port 8000 \ - --mem-fraction-static 0.85 -``` - -8-GPU deployment command: - -```bash Command -python -m sglang.launch_server \ - --model-path MiniMaxAI/MiniMax-M2 \ - --tp-size 8 \ - --ep-size 8 \ - --tool-call-parser minimax-m2 \ - --reasoning-parser minimax-append-think \ - --host 0.0.0.0 \ - --trust-remote-code \ - --port 8000 \ - --mem-fraction-static 0.85 -``` - -### AMD GPUs (MI300X/MI325X/MI355X) - -8-GPU deployment command: - -```bash Command -SGLANG_USE_AITER=1 python -m sglang.launch_server \ - --model-path MiniMaxAI/MiniMax-M2.5 \ - --tp-size 8 \ - --ep-size 8 \ - --attention-backend aiter \ - --tool-call-parser minimax-m2 \ - --reasoning-parser minimax-append-think \ - --host 0.0.0.0 \ - --trust-remote-code \ - --port 8000 \ - --mem-fraction-static 0.85 -``` - -## Testing Deployment - -After startup, you can test the SGLang OpenAI-compatible API with the following command: - -```bash Command -curl http://localhost:8000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "MiniMaxAI/MiniMax-M2", - "messages": [ - {"role": "system", "content": [{"type": "text", "text": "You are a helpful assistant."}]}, - {"role": "user", "content": [{"type": "text", "text": "Who won the world series in 2020?"}]} - ] - }' -``` diff --git a/docs_new/docs/basic_usage/overview.mdx b/docs_new/docs/basic_usage/overview.mdx index 6f15d1c33..5004cfd3c 100644 --- a/docs_new/docs/basic_usage/overview.mdx +++ b/docs_new/docs/basic_usage/overview.mdx @@ -8,4 +8,4 @@ description: Core APIs and common usage patterns for SGLang. - [Offline Engine API](./offline_engine_api) - [Native API](./native_api) - [Sampling Parameters](./sampling_params) -- [Popular Model Usage](./popular_model_usage) — DeepSeek, GLM, Qwen, Llama, and more +- [Popular Model Usage](/cookbook/autoregressive/intro) — DeepSeek, GLM, Qwen, Llama, and more diff --git a/docs_new/docs/basic_usage/popular_model_usage.mdx b/docs_new/docs/basic_usage/popular_model_usage.mdx deleted file mode 100644 index 4c5a25e2f..000000000 --- a/docs_new/docs/basic_usage/popular_model_usage.mdx +++ /dev/null @@ -1,17 +0,0 @@ ---- -title: "Popular Model Usage (DeepSeek, GPT-OSS, GLM, Llama, MiniMax, Qwen, and more)" -description: "Documentation for Popular Model Usage (DeepSeek, GPT-OSS, GLM, Llama, MiniMax, Qwen, and more)" ---- -For more usage examples and recipes, visit the [SGLang Cookbook](https://cookbook.sglang.io/). - -- [Deepseek V3](./deepseek_v3) -- [Deepseek V32](./deepseek_v32) -- [Glm45](./glm45) -- [Glmv](./glmv) -- [Gpt Oss](./gpt_oss) -- [Minimax M2](./minimax_m2) -- [Qwen3](./qwen3) -- [Qwen3 5](./qwen3_5) -- [Qwen3 Vl](./qwen3_vl) -- [Deepseek Ocr](./deepseek_ocr) -- [Llama4](./llama4) diff --git a/docs_new/docs/basic_usage/popular_model_usage.rst b/docs_new/docs/basic_usage/popular_model_usage.rst deleted file mode 100644 index 06d426661..000000000 --- a/docs_new/docs/basic_usage/popular_model_usage.rst +++ /dev/null @@ -1,16 +0,0 @@ -Popular Model Usage (DeepSeek, GPT-OSS, GLM, Llama, MiniMax, Qwen, and more) -=============================================================== - -.. toctree:: - :maxdepth: 1 - - deepseek_v3.md - deepseek_v32.md - glm45.md - glmv.md - gpt_oss.md - kimi_k2_5.md - minimax_m2.md - qwen3.md - qwen3_vl.md - llama4.md diff --git a/docs_new/docs/basic_usage/qwen3.mdx b/docs_new/docs/basic_usage/qwen3.mdx deleted file mode 100644 index 4c316dcf8..000000000 --- a/docs_new/docs/basic_usage/qwen3.mdx +++ /dev/null @@ -1,42 +0,0 @@ ---- -title: "Qwen3-Next Usage" -metatags: - description: "Deploy Qwen3-Next with SGLang: 80B hybrid Mamba model, MambaRadixCache prefix caching, EAGLE speculative decoding. Supports H100/H200 GPUs." ---- -SGLang has supported Qwen3-Next-80B-A3B-Instruct and Qwen3-Next-80B-A3B-Thinking since [this PR](https://github.com/sgl-project/sglang/pull/10233). - -## Launch Qwen3-Next with SGLang - -To serve Qwen3-Next models on 4xH100/H200 GPUs: - -```bash Command -python3 -m sglang.launch_server --model Qwen/Qwen3-Next-80B-A3B-Instruct --tp 4 -``` - -### Configuration Tips -- `--max-mamba-cache-size`: Adjust `--max-mamba-cache-size` to increase mamba cache space and max running requests capability. It will decrease KV cache space as a trade-off. You can adjust it according to workload. -- `--mamba-ssm-dtype`: `bfloat16` or `float32`, use `bfloat16` to save mamba cache size and `float32` to get more accurate results. The default setting is `float32`. -- `--mamba-full-memory-ratio`: The ratio of mamba state memory to full kv cache memory. The default is 0.9. - -### Mamba Radix Cache -SGLang supports prefix caching for Qwen3-Next models named `MambaRadixCache`, which improves inference speed by reusing computation results. There are two versions of `MambaRadixCache`: -- `no_buffer`: The default version, which is also other hybrid linear models' choice. When it is enabled, SGLang will automatically close overlap schedule for compatibility reasons. -- `extra_buffer`: An optimized version that is compatible with features like page size > 1, overlap schedule, and speculative decoding. It also supports storing mamba state in branching positions. However, it requires two extra mamba spaces for a ping-pong buffer for each request. To enable it, add the argument `--mamba-scheduler-strategy extra_buffer` when launching the server. - -### EAGLE Speculative Decoding -**Description**: SGLang has supported Qwen3-Next models with [EAGLE speculative decoding](../advanced_features/speculative_decoding#EAGLE-Decoding). - -**Usage**: -Add arguments `--speculative-algorithm`, `--speculative-num-steps`, `--speculative-eagle-topk` and `--speculative-num-draft-tokens` to enable this feature. For example: - -```bash Command -python3 -m sglang.launch_server \ - --model Qwen/Qwen3-Next-80B-A3B-Instruct \ - --tp 4 \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --speculative-algo NEXTN -``` - -Details can be seen in [this PR](https://github.com/sgl-project/sglang/pull/10233). diff --git a/docs_new/docs/basic_usage/qwen3_5.mdx b/docs_new/docs/basic_usage/qwen3_5.mdx deleted file mode 100644 index 88e897d08..000000000 --- a/docs_new/docs/basic_usage/qwen3_5.mdx +++ /dev/null @@ -1,80 +0,0 @@ ---- -title: "Qwen 3.5 Usage" -metatags: - description: "Qwen 3.5 is Alibaba's latest generation LLM featuring a hybrid attention architecture, advanced MoE with shared experts, and native multimodal capabilities." ---- - -Qwen 3.5 is Alibaba's latest generation LLM featuring a hybrid attention architecture, advanced MoE with shared experts, and native multimodal capabilities. - -Key architecture features: -- **Hybrid Attention**: Gated Delta Networks (linear, O(n) complexity) combined with full attention every 4th layer for high associative recall -- **MoE with Shared Experts**: Top-8 active out of 64 routed experts plus a dedicated shared expert for universal features -- **Multimodal**: DeepStack Vision Transformer with Conv3d for native image and video understanding - -## Launch Qwen 3.5 with SGLang - -### Dense Model - -To serve `Qwen/Qwen3.5-397B-A17B` on 8 GPUs: - -```bash -python3 -m sglang.launch_server \ - --model-path Qwen/Qwen3.5-397B-A17B \ - --tp 8 \ - --trust-remote-code -``` - -### AMD GPU (MI300X / MI325X / MI35X) - -On AMD Instinct GPUs, use the `triton` attention backend. Both the full attention layers and the Gated Delta Net (linear attention) layers use Triton-based kernels on ROCm: - -```bash -SGLANG_USE_AITER=1 python3 -m sglang.launch_server \ - --model-path Qwen/Qwen3.5-397B-A17B \ - --tp 8 \ - --attention-backend triton \ - --trust-remote-code -``` - - -Set `SGLANG_USE_AITER=1` to enable AMD's optimized aiter kernels for MoE and GEMM operations. - - -### Configuration Tips - -- `--attention-backend`: Use `triton` on AMD GPUs for Qwen 3.5. The hybrid attention architecture (Gated Delta Networks + full attention) works best with the Triton backend on ROCm. The linear attention (GDN) layers always use Triton kernels internally via the `GDNAttnBackend`. -- `--watchdog-timeout`: Increase to `1200` or higher for this large model, as weight loading takes significant time. -- `--model-loader-extra-config '{"enable_multithread_load": true}'`: Enables parallel weight loading for faster startup. - -### Reasoning and Tool Calling - -Qwen 3.5 supports reasoning and tool calling via the Qwen3 parsers: - -```bash -python3 -m sglang.launch_server \ - --model-path Qwen/Qwen3.5-397B-A17B \ - --tp 8 \ - --trust-remote-code \ - --reasoning-parser qwen3 \ - --tool-call-parser qwen3_coder -``` - -## Accuracy Evaluation - -You can evaluate the model accuracy using `lm-eval`: - -```bash -pip install lm-eval[api] - -lm_eval --model local-completions \ - --model_args '{"base_url": "http://localhost:8000/v1/completions", "model": "Qwen/Qwen3.5-397B-A17B", "num_concurrent": 256, "max_retries": 10, "max_gen_toks": 2048}' \ - --tasks gsm8k \ - --batch_size auto \ - --num_fewshot 5 \ - --trust_remote_code -``` - -## Additional Resources - -- [AMD Day 0 Support for Qwen 3.5 on AMD Instinct GPUs](https://www.amd.com/en/developer/resources/technical-articles/2026/day-0-support-for-qwen-3-5-on-amd-instinct-gpus.html) -- [HuggingFace Model Card](https://huggingface.co/Qwen/Qwen3.5-397B-A17B) diff --git a/docs_new/docs/basic_usage/qwen3_vl.mdx b/docs_new/docs/basic_usage/qwen3_vl.mdx deleted file mode 100644 index a98a5b4e8..000000000 --- a/docs_new/docs/basic_usage/qwen3_vl.mdx +++ /dev/null @@ -1,133 +0,0 @@ ---- -title: "Qwen3-VL Usage" -metatags: - description: "Deploy Qwen3-VL vision models with SGLang: FP8 and BF16 modes, image and video input, expert parallelism. Supports H100, H200, A100 GPUs." ---- -[Qwen3-VL](https://huggingface.co/collections/Qwen/qwen3-vl) -is Alibaba’s latest multimodal large language model with strong text, vision, and reasoning capabilities. -SGLang supports Qwen3-VL Family of models with Image and Video input support. - -## Launch commands for SGLang - -Below are suggested launch commands tailored for different hardware / precision modes - -### FP8 (quantised) mode -For high memory-efficiency and latency optimized deployments (e.g., on H100, H200) where FP8 checkpoint is supported: -```bash Command -python3 -m sglang.launch_server \ - --model-path Qwen/Qwen3-VL-235B-A22B-Instruct-FP8 \ - --tp 8 \ - --ep 8 \ - --host 0.0.0.0 \ - --port 30000 \ - --keep-mm-feature-on-device -``` - -### Non-FP8 (BF16 / full precision) mode -For deployments on A100/H100 where BF16 is used (or FP8 snapshot not used): -```bash Command -python3 -m sglang.launch_server \ - --model-path Qwen/Qwen3-VL-235B-A22B-Instruct \ - --tp 8 \ - --ep 8 \ - --host 0.0.0.0 \ - --port 30000 \ -``` - -## Hardware-specific notes / recommendations - -- On H100 with FP8: Use the FP8 checkpoint for best memory efficiency. -- On A100 / H100 with BF16 (non-FP8): It’s recommended to use `--mm-max-concurrent-calls` to control parallel throughput and GPU memory usage during image/video inference. -- On H200 & B200: The model can be run “out of the box”, supporting full context length plus concurrent image + video processing. - -## Sending Image/Video Requests - -### Image input: - -```python Example -import requests - -url = f"http://localhost:30000/v1/chat/completions" - -data = { - "model": "Qwen/Qwen3-VL-30B-A3B-Instruct", - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What’s in this image?"}, - { - "type": "image_url", - "image_url": { - "url": "https://github.com/sgl-project/sglang/blob/main/examples/assets/example_image.png?raw=true" - }, - }, - ], - } - ], - "max_tokens": 300, -} - -response = requests.post(url, json=data) -print(response.text) -``` - -### Video Input: - -```python Example -import requests - -url = f"http://localhost:30000/v1/chat/completions" - -data = { - "model": "Qwen/Qwen3-VL-30B-A3B-Instruct", - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What’s happening in this video?"}, - { - "type": "video_url", - "video_url": { - "url": "https://github.com/sgl-project/sgl-test-files/raw/refs/heads/main/videos/jobs_presenting_ipod.mp4" - }, - }, - ], - } - ], - "max_tokens": 300, -} - -response = requests.post(url, json=data) -print(response.text) -``` - -## Important Server Parameters and Flags - -When launching the model server for **multimodal support**, you can use the following command-line arguments to fine-tune performance and behavior: - -- `--mm-attention-backend`: Specify multimodal attention backend. Eg. `fa3`(Flash Attention 3) -- `--mm-max-concurrent-calls `: Specifies the **maximum number of concurrent asynchronous multimodal data processing calls** allowed on the server. Use this to control parallel throughput and GPU memory usage during image/video inference. -- `--mm-per-request-timeout `: Defines the **timeout duration (in seconds)** for each multimodal request. If a request exceeds this time limit (e.g., for very large video inputs), it will be automatically terminated. -- `--keep-mm-feature-on-device`: Instructs the server to **retain multimodal feature tensors on the GPU** after processing. This avoids device-to-host (D2H) memory copies and improves performance for repeated or high-frequency inference workloads. -- `SGLANG_USE_CUDA_IPC_TRANSPORT=1`: Shared memory pool based CUDA IPC for multi-modal data transport. For significantly improving e2e latency. - -### Example usage with the above optimizations: -```bash Command -SGLANG_USE_CUDA_IPC_TRANSPORT=1 \ -SGLANG_VLM_CACHE_SIZE_MB=0 \ -python -m sglang.launch_server \ - --model-path Qwen/Qwen3-VL-235B-A22B-Instruct \ - --host 0.0.0.0 \ - --port 30000 \ - --trust-remote-code \ - --tp-size 8 \ - --enable-cache-report \ - --log-level info \ - --max-running-requests 64 \ - --mem-fraction-static 0.65 \ - --chunked-prefill-size 8192 \ - --attention-backend fa3 \ - --mm-attention-backend fa3 \ - --enable-metrics -``` diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_optimization.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_optimization.mdx index 3067889fb..fccd32d9b 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_optimization.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_optimization.mdx @@ -801,5 +801,5 @@ Casts model weight tensors to the Ascend NPU ACL FRACTAL_NZ format (format 29) f - [Ascend NPU Best Practice](/docs/hardware-platforms/ascend-npus/ascend_npu_best_practice) — Complete deployment configurations and benchmark results for all supported models - [Ascend NPU Environment Variables](/docs/hardware-platforms/ascend-npus/ascend_npu_environment_variables) — Reference for all Ascend NPU-related environment variables -- [DeepSeek V3.2 Guide](/docs/basic_usage/deepseek_v32) — Detailed usage guide for DeepSeek V3.2 deployment +- [DeepSeek V3.2 Guide](/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2) — Detailed usage guide for DeepSeek V3.2 deployment - [Expert Parallelism](/docs/advanced_features/expert_parallelism) — DeepEP configuration and tuning guide diff --git a/docs_new/docs/references/multi_node_deployment/multi_node.mdx b/docs_new/docs/references/multi_node_deployment/multi_node.mdx index 645203ea8..7a0193d18 100644 --- a/docs_new/docs/references/multi_node_deployment/multi_node.mdx +++ b/docs_new/docs/references/multi_node_deployment/multi_node.mdx @@ -33,7 +33,7 @@ python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-405B-Instr ## DeepSeek V3/R1 -Please refer to [DeepSeek documents for reference](../../basic_usage/deepseek_v3#running-examples-on-multi-node). +Please refer to [DeepSeek documents for reference](/cookbook/autoregressive/DeepSeek/DeepSeek-V3#4-2-5-multi-node-deployment). ## Multi-Node Inference on SLURM diff --git a/docs_new/docs/supported-models/generative_models.mdx b/docs_new/docs/supported-models/generative_models.mdx index eb83f3e9d..35abca9d1 100644 --- a/docs_new/docs/supported-models/generative_models.mdx +++ b/docs_new/docs/supported-models/generative_models.mdx @@ -41,7 +41,7 @@ in the GitHub search bar. **DeepSeek** (v1, v2, v3/R1) `deepseek-ai/DeepSeek-R1` - Series of advanced reasoning-optimized models (including a 671B MoE) trained with reinforcement learning; top performance on complex reasoning, math, and code tasks. SGLang provides Deepseek v3/R1 model-specific optimizations and Reasoning Parser + Series of advanced reasoning-optimized models (including a 671B MoE) trained with reinforcement learning; top performance on complex reasoning, math, and code tasks. SGLang provides Deepseek v3/R1 model-specific optimizations and Reasoning Parser **Kimi K2** (Thinking, Instruct) @@ -66,7 +66,7 @@ in the GitHub search bar. **Llama** (2, 3.x, 4 series) `meta-llama/Llama-4-Scout-17B-16E-Instruct` - Meta's open LLM series, spanning 7B to 400B parameters (Llama 2, 3, and new Llama 4) with well-recognized performance. SGLang provides Llama-4 model-specific optimizations + Meta's open LLM series, spanning 7B to 400B parameters (Llama 2, 3, and new Llama 4) with well-recognized performance. SGLang provides Llama-4 model-specific optimizations **Mistral** (Mixtral, NeMo, Small3) diff --git a/docs_new/scripts/gen_redirects.py b/docs_new/scripts/gen_redirects.py index 77e2dd7f1..23213faff 100755 --- a/docs_new/scripts/gen_redirects.py +++ b/docs_new/scripts/gen_redirects.py @@ -94,9 +94,20 @@ EXPLICIT = { "/diffusion/reference": "/docs/sglang-diffusion/installation", "/diffusion/support_new_models": "/docs/sglang-diffusion/installation", "/diffusion/usage": "/docs/sglang-diffusion/installation", - # basic_usage dropped pages - "/basic_usage/deepseek_ocr": "/docs/basic_usage/overview", - "/basic_usage/qwen3_5": "/docs/basic_usage/qwen3", + # basic_usage pages migrated to cookbook + "/basic_usage/kimi_k2_5": "/cookbook/autoregressive/Moonshotai/Kimi-K2.5", + "/basic_usage/deepseek_ocr": "/cookbook/autoregressive/DeepSeek/DeepSeek-OCR", + "/basic_usage/deepseek_v3": "/cookbook/autoregressive/DeepSeek/DeepSeek-V3", + "/basic_usage/deepseek_v32": "/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2", + "/basic_usage/glm45": "/cookbook/autoregressive/GLM/GLM-4.5", + "/basic_usage/glmv": "/cookbook/autoregressive/GLM/GLM-4.6V", + "/basic_usage/gpt_oss": "/cookbook/autoregressive/OpenAI/GPT-OSS", + "/basic_usage/llama4": "/cookbook/autoregressive/Llama/Llama4", + "/basic_usage/minimax_m2": "/cookbook/autoregressive/MiniMax/MiniMax-M2", + "/basic_usage/popular_model_usage": "/cookbook/autoregressive/intro", + "/basic_usage/qwen3": "/cookbook/autoregressive/Qwen/Qwen3", + "/basic_usage/qwen3_5": "/cookbook/autoregressive/Qwen/Qwen3.5", + "/basic_usage/qwen3_vl": "/cookbook/autoregressive/Qwen/Qwen3-VL", # advanced_features dropped pages "/advanced_features/adaptive_speculative_decoding": "/docs/advanced_features/speculative_decoding", "/advanced_features/hisparse_guide": "/docs/advanced_features/overview", diff --git a/docs_new/src/snippets/autoregressive/glm-45v-deployment.jsx b/docs_new/src/snippets/autoregressive/glm-45v-deployment.jsx index 8fb67ad25..6d9277dde 100644 --- a/docs_new/src/snippets/autoregressive/glm-45v-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/glm-45v-deployment.jsx @@ -82,6 +82,7 @@ export const GLM45VDeployment = () => { h100: { tp: 4 }, h200: { tp: 4 }, mi300x: { tp: 4 }, + mi325x: { tp: 4 }, mi355x: { tp: 4 } }; diff --git a/docs_new/src/snippets/autoregressive/glm-46v-deployment.jsx b/docs_new/src/snippets/autoregressive/glm-46v-deployment.jsx index 237fc307e..19d48c973 100644 --- a/docs_new/src/snippets/autoregressive/glm-46v-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/glm-46v-deployment.jsx @@ -95,6 +95,7 @@ export const GLM46VDeployment = () => { h200: { tp: 8 }, b200: { tp: 8 }, mi300x: { tp: 8 }, + mi325x: { tp: 8 }, mi355x: { tp: 8 } }, '9b': { @@ -103,6 +104,7 @@ export const GLM46VDeployment = () => { h200: { tp: 1 }, b200: { tp: 1 }, mi300x: { tp: 1 }, + mi325x: { tp: 1 }, mi355x: { tp: 1 } } }; diff --git a/docs_new/src/snippets/autoregressive/glm-47-deployment.jsx b/docs_new/src/snippets/autoregressive/glm-47-deployment.jsx index f90b33c40..777265062 100644 --- a/docs_new/src/snippets/autoregressive/glm-47-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/glm-47-deployment.jsx @@ -138,7 +138,7 @@ export const GLM47Deployment = () => { // Add thinking parser if enabled if (thinking === 'enabled') { - cmd += ` \\\n --reasoning-parser glm47`; + cmd += ` \\\n --reasoning-parser glm45`; } return cmd; diff --git a/docs_new/src/snippets/autoregressive/llama4-maverick-deployment.jsx b/docs_new/src/snippets/autoregressive/llama4-maverick-deployment.jsx index b93fb6ed9..1457f7ccc 100644 --- a/docs_new/src/snippets/autoregressive/llama4-maverick-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/llama4-maverick-deployment.jsx @@ -4,11 +4,37 @@ export const Llama4MaverickDeployment = () => { name: 'hardware', title: 'Hardware Platform', items: [ + { id: 'b200', label: 'B200', default: false }, + { id: 'h200', label: 'H200', default: false }, { id: 'mi300x', label: 'MI300x', default: true }, { id: 'mi325x', label: 'MI325x', default: false }, { id: 'mi355x', label: 'MI355x', default: false } ] }, + quantization: { + name: 'quantization', + title: 'Quantization', + items: [ + { id: 'bf16', label: 'BF16', default: true }, + { id: 'fp8', label: 'FP8', default: false } + ] + }, + toolcall: { + name: 'toolcall', + title: 'Tool Call Parser', + items: [ + { id: 'disabled', label: 'Disabled', default: true }, + { id: 'enabled', label: 'Enabled', default: false } + ] + }, + speculative: { + name: 'speculative', + title: 'Speculative Decoding (EAGLE3)', + items: [ + { id: 'disabled', label: 'Disabled', default: true }, + { id: 'enabled', label: 'Enable EAGLE3', default: false } + ] + }, host: { name: 'host', title: 'Host', @@ -29,9 +55,9 @@ export const Llama4MaverickDeployment = () => { const { hardware, quantization, toolcall, speculative, host, port } = values; let cmd = 'python -m sglang.launch_server \\\n'; - cmd += ` --model-path meta-llama/Llama-4-Scout-17B-16E-Instruct`; + cmd += ` --model-path meta-llama/Llama-4-Maverick-17B-128E-Instruct`; - if (hardware === 'h100' || hardware === 'h200') { + if (hardware === 'h200') { cmd += ` \\\n --tp 8`; } else if (hardware === 'b200') { cmd += ` \\\n --tp 8`; @@ -49,7 +75,7 @@ export const Llama4MaverickDeployment = () => { if (speculative === 'enabled') { cmd += ` \\\n --speculative-algorithm EAGLE3 \\\n`; - cmd += ` --speculative-draft-model-path lmsys/sglang-EAGLE3-Llama-4-Scout-17B-16E-Instruct-v1 \\\n`; + cmd += ` --speculative-draft-model-path lmsys/sglang-EAGLE3-Llama-4-Maverick-17B-128E-Instruct-v1 \\\n`; cmd += ` --speculative-num-steps 3 \\\n`; cmd += ` --speculative-eagle-topk 1 \\\n`; cmd += ` --speculative-num-draft-tokens 4 \\\n`;