From 900aad5f721c0d647882c444e2500bdcb1b20738 Mon Sep 17 00:00:00 2001 From: zijiexia <37504505+zijiexia@users.noreply.github.com> Date: Tue, 21 Apr 2026 00:15:17 -0700 Subject: [PATCH] [Docs] Sync docs_new with legacy docs and update migration redirects (#23337) Co-authored-by: Mingyi --- .pre-commit-config.yaml | 7 + .../DeepSeek/DeepSeek-Math-V2.mdx | 2 +- .../DeepSeek/DeepSeek-OCR-2.mdx | 2 +- .../autoregressive/DeepSeek/DeepSeek-OCR.mdx | 2 +- .../autoregressive/DeepSeek/DeepSeek-R1.mdx | 2 +- .../autoregressive/DeepSeek/DeepSeek-V3.mdx | 2 +- .../autoregressive/DeepSeek/DeepSeek-V3_1.mdx | 2 +- .../autoregressive/DeepSeek/DeepSeek-V3_2.mdx | 2 +- .../autoregressive/Ernie/Ernie4.5.mdx | 2 +- .../autoregressive/FlashLabs/Chroma1.0.mdx | 1 - .../cookbook/autoregressive/GLM/GLM-4.5.mdx | 2 +- .../cookbook/autoregressive/GLM/GLM-4.5V.mdx | 2 +- .../cookbook/autoregressive/GLM/GLM-4.6.mdx | 2 +- .../cookbook/autoregressive/GLM/GLM-4.6V.mdx | 2 +- .../autoregressive/GLM/GLM-4.7-Flash.mdx | 2 +- .../cookbook/autoregressive/GLM/GLM-4.7.mdx | 2 +- .../cookbook/autoregressive/GLM/GLM-5.1.mdx | 2 +- .../cookbook/autoregressive/GLM/GLM-5.mdx | 3 +- .../cookbook/autoregressive/GLM/GLM-Glyph.mdx | 2 +- .../cookbook/autoregressive/GLM/GLM-OCR.mdx | 2 +- .../cookbook/autoregressive/Google/Gemma4.mdx | 3 +- .../autoregressive/InclusionAI/LLaDA-2.1.mdx | 3 +- .../InclusionAI/Ling-2.5-1T.mdx | 5 +- .../InclusionAI/Ring-2.5-1T.mdx | 3 +- .../autoregressive/Llama/Llama3.1.mdx | 2 +- .../autoregressive/Llama/Llama3.3-70B.mdx | 2 +- .../cookbook/autoregressive/Llama/Llama4.mdx | 2 +- .../autoregressive/MiniMax/MiniMax-M2.5.mdx | 3 +- .../autoregressive/MiniMax/MiniMax-M2.7.mdx | 2 +- .../autoregressive/MiniMax/MiniMax-M2.mdx | 2 +- .../autoregressive/Mistral/Devstral-2.mdx | 2 +- .../autoregressive/Mistral/Ministral-3.mdx | 2 +- .../Mistral/Mistral-Small-4.mdx | 4 +- .../autoregressive/Moonshotai/Kimi-K2.5.mdx | 2 +- .../autoregressive/Moonshotai/Kimi-K2.6.mdx | 2 +- .../autoregressive/Moonshotai/Kimi-K2.mdx | 2 +- .../autoregressive/Moonshotai/Kimi-Linear.mdx | 2 +- .../autoregressive/NVIDIA/Nemotron3-Nano.mdx | 2 +- .../autoregressive/NVIDIA/Nemotron3-Super.mdx | 2 +- .../autoregressive/OpenAI/GPT-OSS.mdx | 2 +- .../autoregressive/Qwen/Qwen2.5-VL.mdx | 2 +- .../autoregressive/Qwen/Qwen3-Coder-Next.mdx | 2 +- .../autoregressive/Qwen/Qwen3-Coder.mdx | 2 +- .../autoregressive/Qwen/Qwen3-Next.mdx | 2 +- .../cookbook/autoregressive/Qwen/Qwen3-VL.mdx | 2 +- .../cookbook/autoregressive/Qwen/Qwen3.5.mdx | 2 +- .../cookbook/autoregressive/Qwen/Qwen3.6.mdx | 2 +- .../cookbook/autoregressive/Qwen/Qwen3.mdx | 2 +- .../autoregressive/StepFun/Step3-VL-10B.mdx | 3 +- .../autoregressive/StepFun/Step3.5.mdx | 1 - .../diffusion/Qwen-Image/Qwen-Image.mdx | 6 +- docs_new/cookbook/diffusion/Wan/Wan2.1.mdx | 6 +- docs_new/cookbook/intro copy.mdx | 2 +- docs_new/docs.json | 208 +- .../adaptive_speculative_decoding.mdx | 197 ++ .../advanced_features/attention_backend.mdx | 198 +- .../advanced_features/epd_disaggregation.mdx | 114 + .../advanced_features/expert_parallelism.mdx | 30 +- .../hicache_best_practices.mdx | 21 +- .../hicache_storage_runtime_attach_detach.mdx | 12 +- .../docs/advanced_features/hisparse_guide.mdx | 187 ++ docs_new/docs/advanced_features/lora.ipynb | 22 +- docs_new/docs/advanced_features/lora.mdx | 12 +- .../advanced_features/pd_disaggregation.mdx | 101 +- .../docs/advanced_features/quantization.mdx | 462 +++- docs_new/docs/advanced_features/rfork.mdx | 40 +- .../separate_reasoning.ipynb | 2 +- .../advanced_features/server_arguments.mdx | 984 ++++---- .../docs/advanced_features/sglang_for_rl.mdx | 106 +- .../speculative_decoding.mdx | 911 ++++++- .../structured_outputs.ipynb | 13 +- .../advanced_features/structured_outputs.mdx | 66 +- ...uctured_outputs_for_reasoning_models.ipynb | 2 +- ...tructured_outputs_for_reasoning_models.mdx | 46 +- .../docs/advanced_features/tool_parser.ipynb | 8 +- .../docs/advanced_features/vlm_query.ipynb | 27 +- docs_new/docs/advanced_features/vlm_query.mdx | 42 +- docs_new/docs/basic_usage/deepseek_ocr.mdx | 58 + docs_new/docs/basic_usage/deepseek_v3.mdx | 104 +- docs_new/docs/basic_usage/deepseek_v32.mdx | 94 +- docs_new/docs/basic_usage/glm45.mdx | 1 - docs_new/docs/basic_usage/glmv.mdx | 2 +- docs_new/docs/basic_usage/minimax_m2.mdx | 27 +- docs_new/docs/basic_usage/native_api.ipynb | 32 +- docs_new/docs/basic_usage/native_api.mdx | 56 +- .../docs/basic_usage/offline_engine_api.ipynb | 2 +- .../basic_usage/openai_api_completions.ipynb | 8 +- .../basic_usage/openai_api_embeddings.ipynb | 6 +- .../docs/basic_usage/openai_api_vision.ipynb | 23 +- .../docs/basic_usage/openai_api_vision.mdx | 41 +- .../docs/basic_usage/popular_model_usage.mdx | 5 +- docs_new/docs/basic_usage/qwen3_5.mdx | 80 + docs_new/docs/basic_usage/send_request.ipynb | 10 +- .../docs/developer_guide/bench_serving.mdx | 41 +- .../benchmark_and_profiling.mdx | 92 +- .../developer_guide/contribution_guide.mdx | 81 +- .../development_guide_using_docker.mdx | 2 +- ...s.mdx => development_jit_kernel_guide.mdx} | 691 ++++-- docs_new/docs/developer_guide/overview.mdx | 2 +- .../developer_guide/setup_github_runner.mdx | 10 +- docs_new/docs/get-started/install.mdx | 237 ++ docs_new/docs/get-started/installation.mdx | 256 -- docs_new/docs/get-started/quickstart.mdx | 14 +- docs_new/docs/hardware-platforms/amd-gpus.mdx | 194 -- docs_new/docs/hardware-platforms/amd_gpu.mdx | 196 ++ .../docs/hardware-platforms/apple_metal.mdx | 24 + .../ascend-npus/DeepSeek-Examples.mdx | 309 --- .../ascend-npus/Qwen3.5.mdx | 106 - .../SGLang-installation-with-NPUs-support.mdx | 318 --- .../ascend-npus/ascend_contribution_guide.mdx | 167 ++ .../ascend-npus/ascend_npu.mdx | 294 +++ ...d-NPU.mdx => ascend_npu_best_practice.mdx} | 2192 +++++++++++++---- .../ascend_npu_deepseek_example.mdx | 301 +++ .../ascend_npu_environment_variables.mdx | 149 ++ ...GLM-5.mdx => ascend_npu_glm5_examples.mdx} | 31 +- .../ascend-npus/ascend_npu_quantization.mdx | 257 ++ .../ascend-npus/ascend_npu_quick_start.mdx | 107 + .../ascend_npu_qwen3_5_examples.mdx | 234 ++ ...ples.mdx => ascend_npu_qwen3_examples.mdx} | 120 +- .../ascend_npu_ring_sp_performance.mdx | 110 + ...PU.mdx => ascend_npu_support_features.mdx} | 2153 ++++++---------- ...-NPU.mdx => ascend_npu_support_models.mdx} | 75 +- ...Spore-Models.mdx => mindspore_backend.mdx} | 2 +- .../docs/hardware-platforms/cpu-server.mdx | 355 --- .../docs/hardware-platforms/cpu_server.mdx | 387 +++ .../docs/hardware-platforms/mthreads_gpu.mdx | 29 + .../docs/hardware-platforms/nvidia-gpus.mdx | 2 +- docs_new/docs/hardware-platforms/nvidia.mdx | 102 - .../docs/hardware-platforms/nvidia_jetson.mdx | 82 + docs_new/docs/hardware-platforms/overview.mdx | 8 +- docs_new/docs/hardware-platforms/plugin.mdx | 849 +++++++ docs_new/docs/hardware-platforms/tpu.mdx | 839 +++---- docs_new/docs/hardware-platforms/xpu.mdx | 93 +- .../docs/references/environment_variables.mdx | 379 ++- .../frontend/frontend_tutorial.ipynb | 8 +- .../references/frontend/frontend_tutorial.mdx | 41 +- .../lws_pd/lws_pd_deploy.mdx | 4 +- .../docs/references/production_metrics.mdx | 42 +- .../references/production_request_trace.mdx | 136 +- docs_new/docs/sglang-diffusion/api/cli.mdx | 539 ++-- .../docs/sglang-diffusion/api/openai-api.mdx | 421 ---- .../docs/sglang-diffusion/api/openai_api.mdx | 450 ++++ .../sglang-diffusion/api/post_processing.mdx | 237 ++ ...on-backends.mdx => attention_backends.mdx} | 114 +- .../{cache-dit.mdx => cache_dit.mdx} | 182 +- .../sglang-diffusion/caching-acceleration.mdx | 9 +- .../{ci-performance.mdx => ci_perf.mdx} | 3 +- .../sglang-diffusion/compatibility_matrix.mdx | 631 +++++ .../docs/sglang-diffusion/contributing.mdx | 77 + .../docs/sglang-diffusion/disaggregation.mdx | 361 +++ .../environment-variables.mdx | 140 -- .../environment_variables.mdx | 395 +++ docs_new/docs/sglang-diffusion/index.mdx | 64 +- .../docs/sglang-diffusion/installation.mdx | 186 +- .../performance-optimization.mdx | 62 +- docs_new/docs/sglang-diffusion/profiling.mdx | 1 - .../docs/sglang-diffusion/quantization.mdx | 551 +++++ .../sglang-diffusion/ring_sp_performance.mdx | 158 ++ .../sglang-diffusion/support_new_models.mdx | 601 +++++ .../sglang-diffusion/supported-models.mdx | 350 --- .../{tea-cache.mdx => teacache.mdx} | 2 +- docs_new/docs/supported-models.mdx | 12 +- .../classification-models.mdx | 323 --- .../docs/supported-models/classify_models.mdx | 163 ++ .../diffusion-language-models.mdx | 13 - .../diffusion_language_models.mdx | 133 + ...edding-models.mdx => embedding_models.mdx} | 66 +- ...guage-models.mdx => generative_models.mdx} | 96 +- ...dspore-models.mdx => mindspore_models.mdx} | 99 +- docs_new/docs/supported-models/modelscope.mdx | 13 +- ...els.mdx => multimodal_language_models.mdx} | 87 +- .../supported-models/new-model-support.mdx | 307 --- .../{rerank-models.mdx => rerank_models.mdx} | 51 +- .../{reward-models.mdx => reward_models.mdx} | 0 .../supported-models/support_new_models.mdx | 522 ++++ ...fallback.mdx => transformers_fallback.mdx} | 31 +- docs_new/index.mdx | 2 +- docs_new/sglang-diffusion/intro.mdx | 65 - scripts/ci/check_no_docs_changes.py | 50 + 179 files changed, 16014 insertions(+), 8162 deletions(-) create mode 100644 docs_new/docs/advanced_features/adaptive_speculative_decoding.mdx create mode 100644 docs_new/docs/advanced_features/hisparse_guide.mdx create mode 100644 docs_new/docs/basic_usage/deepseek_ocr.mdx create mode 100644 docs_new/docs/basic_usage/qwen3_5.mdx rename docs_new/docs/developer_guide/{JIT_kernels.mdx => development_jit_kernel_guide.mdx} (62%) create mode 100644 docs_new/docs/get-started/install.mdx delete mode 100644 docs_new/docs/get-started/installation.mdx delete mode 100644 docs_new/docs/hardware-platforms/amd-gpus.mdx create mode 100644 docs_new/docs/hardware-platforms/amd_gpu.mdx create mode 100644 docs_new/docs/hardware-platforms/apple_metal.mdx delete mode 100644 docs_new/docs/hardware-platforms/ascend-npus/DeepSeek-Examples.mdx delete mode 100644 docs_new/docs/hardware-platforms/ascend-npus/Qwen3.5.mdx delete mode 100644 docs_new/docs/hardware-platforms/ascend-npus/SGLang-installation-with-NPUs-support.mdx create mode 100644 docs_new/docs/hardware-platforms/ascend-npus/ascend_contribution_guide.mdx create mode 100644 docs_new/docs/hardware-platforms/ascend-npus/ascend_npu.mdx rename docs_new/docs/hardware-platforms/ascend-npus/{Best-Practice-on-Ascend-NPU.mdx => ascend_npu_best_practice.mdx} (59%) create mode 100644 docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_deepseek_example.mdx create mode 100644 docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_environment_variables.mdx rename docs_new/docs/hardware-platforms/ascend-npus/{GLM-5.mdx => ascend_npu_glm5_examples.mdx} (91%) create mode 100644 docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_quantization.mdx create mode 100644 docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_quick_start.mdx create mode 100644 docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_5_examples.mdx rename docs_new/docs/hardware-platforms/ascend-npus/{Qwen3-Examples.mdx => ascend_npu_qwen3_examples.mdx} (50%) create mode 100644 docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_ring_sp_performance.mdx rename docs_new/docs/hardware-platforms/ascend-npus/{Support-Features-on-Ascend-NPU.mdx => ascend_npu_support_features.mdx} (64%) rename docs_new/docs/hardware-platforms/ascend-npus/{Support-Models-on-Ascend-NPU.mdx => ascend_npu_support_models.mdx} (97%) rename docs_new/docs/hardware-platforms/ascend-npus/{MindSpore-Models.mdx => mindspore_backend.mdx} (97%) delete mode 100644 docs_new/docs/hardware-platforms/cpu-server.mdx create mode 100644 docs_new/docs/hardware-platforms/cpu_server.mdx create mode 100644 docs_new/docs/hardware-platforms/mthreads_gpu.mdx delete mode 100644 docs_new/docs/hardware-platforms/nvidia.mdx create mode 100644 docs_new/docs/hardware-platforms/nvidia_jetson.mdx create mode 100644 docs_new/docs/hardware-platforms/plugin.mdx delete mode 100644 docs_new/docs/sglang-diffusion/api/openai-api.mdx create mode 100644 docs_new/docs/sglang-diffusion/api/openai_api.mdx create mode 100644 docs_new/docs/sglang-diffusion/api/post_processing.mdx rename docs_new/docs/sglang-diffusion/{attention-backends.mdx => attention_backends.mdx} (74%) rename docs_new/docs/sglang-diffusion/{cache-dit.mdx => cache_dit.mdx} (78%) rename docs_new/docs/sglang-diffusion/{ci-performance.mdx => ci_perf.mdx} (95%) create mode 100644 docs_new/docs/sglang-diffusion/compatibility_matrix.mdx create mode 100644 docs_new/docs/sglang-diffusion/contributing.mdx create mode 100644 docs_new/docs/sglang-diffusion/disaggregation.mdx delete mode 100644 docs_new/docs/sglang-diffusion/environment-variables.mdx create mode 100644 docs_new/docs/sglang-diffusion/environment_variables.mdx create mode 100644 docs_new/docs/sglang-diffusion/quantization.mdx create mode 100644 docs_new/docs/sglang-diffusion/ring_sp_performance.mdx create mode 100644 docs_new/docs/sglang-diffusion/support_new_models.mdx delete mode 100644 docs_new/docs/sglang-diffusion/supported-models.mdx rename docs_new/docs/sglang-diffusion/{tea-cache.mdx => teacache.mdx} (98%) delete mode 100644 docs_new/docs/supported-models/classification-models.mdx create mode 100644 docs_new/docs/supported-models/classify_models.mdx delete mode 100644 docs_new/docs/supported-models/diffusion-language-models.mdx create mode 100644 docs_new/docs/supported-models/diffusion_language_models.mdx rename docs_new/docs/supported-models/{embedding-models.mdx => embedding_models.mdx} (82%) rename docs_new/docs/supported-models/{large-language-models.mdx => generative_models.mdx} (76%) rename docs_new/docs/supported-models/{mindspore-models.mdx => mindspore_models.mdx} (54%) rename docs_new/docs/supported-models/{vision-language-models.mdx => multimodal_language_models.mdx} (85%) delete mode 100644 docs_new/docs/supported-models/new-model-support.mdx rename docs_new/docs/supported-models/{rerank-models.mdx => rerank_models.mdx} (83%) rename docs_new/docs/supported-models/{reward-models.mdx => reward_models.mdx} (100%) create mode 100644 docs_new/docs/supported-models/support_new_models.mdx rename docs_new/docs/supported-models/{transformers-fallback.mdx => transformers_fallback.mdx} (64%) delete mode 100644 docs_new/sglang-diffusion/intro.mdx create mode 100755 scripts/ci/check_no_docs_changes.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 40d68b4ad..8118e91c2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -93,6 +93,13 @@ repos: language: system files: ^test/registered/.*\.py$ pass_filenames: false + - id: check-no-docs-changes + name: reject changes under legacy docs/ + entry: python3 scripts/ci/check_no_docs_changes.py + language: system + pass_filenames: false + always_run: true + stages: [pre-commit] - repo: https://github.com/lycheeverse/lychee.git rev: lychee-v0.22.0 hooks: diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-Math-V2.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-Math-V2.mdx index 04b92b4de..bbb08460f 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-Math-V2.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-Math-V2.mdx @@ -26,7 +26,7 @@ To use DeepSeek-Math-V2, you must agree to DeepSeek's Community License. See [LI ## 2. SGLang Installation -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-OCR-2.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-OCR-2.mdx index 53207e200..1b744541f 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-OCR-2.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-OCR-2.mdx @@ -28,7 +28,7 @@ For more details, please refer to the [official DeepSeek-OCR-2 repository](https ## 2. SGLang Installation -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-OCR.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-OCR.mdx index 0964f86ba..55b8aca24 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-OCR.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-OCR.mdx @@ -24,7 +24,7 @@ For more details, please refer to the [official DeepSeek-OCR repository](https:/ ## 2. SGLang Installation -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-R1.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-R1.mdx index 6ced51128..432f85bbd 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-R1.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-R1.mdx @@ -30,7 +30,7 @@ For more details, please refer to the [official DeepSeek-R1 repository](https:// ## 2. SGLang Installation -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx index 56e8b3260..b9c26d6bc 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx @@ -20,7 +20,7 @@ Key highlights include: SGLang offers multiple installation methods. You can choose the most suitable installation method based on your hardware platform and requirements. -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_1.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_1.mdx index 945e0e815..1b4376a6c 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_1.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_1.mdx @@ -16,7 +16,7 @@ metatags: SGLang offers multiple installation methods. You can choose the most suitable installation method based on your hardware platform and requirements. -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx index 594f6f214..0ab97069f 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx @@ -22,7 +22,7 @@ The DeepSeek-V3.2 series includes three model variants, each optimized for diffe SGLang offers multiple installation methods. You can choose the most suitable installation method based on your hardware platform and requirements. -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/Ernie/Ernie4.5.mdx b/docs_new/cookbook/autoregressive/Ernie/Ernie4.5.mdx index 83a38c1e6..c2dc5fda5 100644 --- a/docs_new/cookbook/autoregressive/Ernie/Ernie4.5.mdx +++ b/docs_new/cookbook/autoregressive/Ernie/Ernie4.5.mdx @@ -21,7 +21,7 @@ ERNIE-4.5 delivers advanced features as below: SGLang offers multiple installation methods. You can choose the most suitable installation method based on your hardware platform and requirements. -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/FlashLabs/Chroma1.0.mdx b/docs_new/cookbook/autoregressive/FlashLabs/Chroma1.0.mdx index 3632bc805..8b5d60964 100644 --- a/docs_new/cookbook/autoregressive/FlashLabs/Chroma1.0.mdx +++ b/docs_new/cookbook/autoregressive/FlashLabs/Chroma1.0.mdx @@ -2,7 +2,6 @@ title: Chroma-1.0 metatags: description: "Deploy Chroma-1.0 end-to-end speech conversation model with SGLang - real-time speech generation, voice cloning, and speech reasoning." -tag: NEW --- ## 1. Model Introduction diff --git a/docs_new/cookbook/autoregressive/GLM/GLM-4.5.mdx b/docs_new/cookbook/autoregressive/GLM/GLM-4.5.mdx index cf5bd5dfb..fe03bd8da 100644 --- a/docs_new/cookbook/autoregressive/GLM/GLM-4.5.mdx +++ b/docs_new/cookbook/autoregressive/GLM/GLM-4.5.mdx @@ -28,7 +28,7 @@ Please refer to the [official GLM-4.5 model card](https://huggingface.co/zai-org SGLang offers multiple installation methods. You can choose the most suitable installation method based on your hardware platform and requirements. -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/GLM/GLM-4.5V.mdx b/docs_new/cookbook/autoregressive/GLM/GLM-4.5V.mdx index 8951a1242..3bd305d43 100644 --- a/docs_new/cookbook/autoregressive/GLM/GLM-4.5V.mdx +++ b/docs_new/cookbook/autoregressive/GLM/GLM-4.5V.mdx @@ -22,7 +22,7 @@ GLM-4.5V introduces several key features: SGLang offers multiple installation methods. You can choose the most suitable installation method based on your hardware platform and requirements. -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/GLM/GLM-4.6.mdx b/docs_new/cookbook/autoregressive/GLM/GLM-4.6.mdx index cc21908ab..dfefd313a 100644 --- a/docs_new/cookbook/autoregressive/GLM/GLM-4.6.mdx +++ b/docs_new/cookbook/autoregressive/GLM/GLM-4.6.mdx @@ -22,7 +22,7 @@ For more details, please refer to the [official GLM-4.6 documentation](https://d SGLang offers multiple installation methods. You can choose the most suitable installation method based on your hardware platform and requirements. -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/GLM/GLM-4.6V.mdx b/docs_new/cookbook/autoregressive/GLM/GLM-4.6V.mdx index 4766898d4..a0fc9d294 100644 --- a/docs_new/cookbook/autoregressive/GLM/GLM-4.6V.mdx +++ b/docs_new/cookbook/autoregressive/GLM/GLM-4.6V.mdx @@ -54,7 +54,7 @@ sudo apt install ffmpeg - Want to use the latest development features - Participate in SGLang project development -For general installation instructions, you can also refer to the [official SGLang installation guide](../../../docs/get-started/installation). +For general installation instructions, you can also refer to the [official SGLang installation guide](../../../docs/get-started/install). ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/GLM/GLM-4.7-Flash.mdx b/docs_new/cookbook/autoregressive/GLM/GLM-4.7-Flash.mdx index 6acd54c89..76bff2893 100644 --- a/docs_new/cookbook/autoregressive/GLM/GLM-4.7-Flash.mdx +++ b/docs_new/cookbook/autoregressive/GLM/GLM-4.7-Flash.mdx @@ -37,7 +37,7 @@ Please refer to the [official GLM-4.7-Flash model card](https://huggingface.co/z SGLang offers multiple installation methods. You can choose the most suitable installation method based on your hardware platform and requirements. -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/GLM/GLM-4.7.mdx b/docs_new/cookbook/autoregressive/GLM/GLM-4.7.mdx index 172dcce64..98ad0084b 100644 --- a/docs_new/cookbook/autoregressive/GLM/GLM-4.7.mdx +++ b/docs_new/cookbook/autoregressive/GLM/GLM-4.7.mdx @@ -38,7 +38,7 @@ Please refer to the [official GLM-4.7 model card](https://huggingface.co/zai-org SGLang offers multiple installation methods. You can choose the most suitable installation method based on your hardware platform and requirements. -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/GLM/GLM-5.1.mdx b/docs_new/cookbook/autoregressive/GLM/GLM-5.1.mdx index 97afecca0..cd4083561 100644 --- a/docs_new/cookbook/autoregressive/GLM/GLM-5.1.mdx +++ b/docs_new/cookbook/autoregressive/GLM/GLM-5.1.mdx @@ -16,7 +16,7 @@ tag: NEW ## 2. SGLang Installation -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/GLM/GLM-5.mdx b/docs_new/cookbook/autoregressive/GLM/GLM-5.mdx index 4a51a8078..9cb8af257 100644 --- a/docs_new/cookbook/autoregressive/GLM/GLM-5.mdx +++ b/docs_new/cookbook/autoregressive/GLM/GLM-5.mdx @@ -2,7 +2,6 @@ title: GLM-5 metatags: description: "Deploy GLM-5 with SGLang on NVIDIA H100/H200/B200 and AMD MI300X/MI325X/MI355X — state-of-the-art reasoning, enhanced coding, and robust tool calling capabilities." -tag: NEW --- ## 1. Model Introduction @@ -28,7 +27,7 @@ With advances in both pre-training (28.5T tokens) and post-training via [slime]( ## 2. SGLang Installation -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/GLM/GLM-Glyph.mdx b/docs_new/cookbook/autoregressive/GLM/GLM-Glyph.mdx index 51971f4d5..050721fdc 100644 --- a/docs_new/cookbook/autoregressive/GLM/GLM-Glyph.mdx +++ b/docs_new/cookbook/autoregressive/GLM/GLM-Glyph.mdx @@ -31,7 +31,7 @@ Please refer to the [official Glyph model card](https://huggingface.co/zai-org/G SGLang offers multiple installation methods. You can choose the most suitable installation method based on your hardware platform and requirements. -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/GLM/GLM-OCR.mdx b/docs_new/cookbook/autoregressive/GLM/GLM-OCR.mdx index 774258ea6..4aafb211a 100644 --- a/docs_new/cookbook/autoregressive/GLM/GLM-OCR.mdx +++ b/docs_new/cookbook/autoregressive/GLM/GLM-OCR.mdx @@ -25,7 +25,7 @@ For more details, please refer to the [official GLM-OCR model card](https://hugg SGLang offers multiple installation methods. You can choose the most suitable installation method based on your hardware platform and requirements. -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/Google/Gemma4.mdx b/docs_new/cookbook/autoregressive/Google/Gemma4.mdx index 75bd0804b..f0910da21 100644 --- a/docs_new/cookbook/autoregressive/Google/Gemma4.mdx +++ b/docs_new/cookbook/autoregressive/Google/Gemma4.mdx @@ -2,6 +2,7 @@ title: Gemma 4 metatags: description: "Deploy Gemma 4 with SGLang - Google's next-generation open models with MoE variants and multimodal support for text, vision, and audio." +tag: NEW --- import { Gemma4Deployment } from '/src/snippets/autoregressive/gemma4-deployment.jsx'; @@ -79,7 +80,7 @@ docker pull lmsysorg/sglang:dev-gemma4 # CUDA 12.9 docker pull lmsysorg/sglang:dev-cu13-gemma4 # CUDA 13 ``` -For the full Docker setup and other installation methods, please refer to the [official SGLang installation guide](../../../docs/get-started/installation). +For the full Docker setup and other installation methods, please refer to the [official SGLang installation guide](../../../docs/get-started/install). ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/InclusionAI/LLaDA-2.1.mdx b/docs_new/cookbook/autoregressive/InclusionAI/LLaDA-2.1.mdx index 162bfa761..aab3190ef 100644 --- a/docs_new/cookbook/autoregressive/InclusionAI/LLaDA-2.1.mdx +++ b/docs_new/cookbook/autoregressive/InclusionAI/LLaDA-2.1.mdx @@ -2,7 +2,6 @@ title: LLaDA 2.1 metatags: description: "Deploy LLaDA 2.1 with SGLang - large-scale discrete diffusion language model with parallel token generation, iterative denoising, MoE architecture, and reinforcement learning for reasoning." -tag: NEW --- import { LLaDA21Deployment } from '/src/snippets/autoregressive/llada-21-deployment.jsx'; @@ -64,7 +63,7 @@ Apache 2.0. Please refer to the [official LLaDA2.X repository](https://github.co SGLang offers multiple installation methods. You can choose the most suitable installation method based on your hardware platform and requirements. -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/InclusionAI/Ling-2.5-1T.mdx b/docs_new/cookbook/autoregressive/InclusionAI/Ling-2.5-1T.mdx index d9bbceb39..68ab2a43c 100644 --- a/docs_new/cookbook/autoregressive/InclusionAI/Ling-2.5-1T.mdx +++ b/docs_new/cookbook/autoregressive/InclusionAI/Ling-2.5-1T.mdx @@ -2,7 +2,6 @@ title: Ling-2.5-1T metatags: description: "Deploy Ling-2.5-1T with SGLang - 1T parameter MoE model with 63B active parameters, trillion-scale context length up to 1M tokens, and agentic tool calling capabilities." -tag: NEW --- ## 1. Model Introduction @@ -35,9 +34,9 @@ docker pull lmsysorg/sglang:nightly-dev-20260213-a0ebaa64 docker pull lmsysorg/sglang:nightly-dev-cu13-20260213-a0ebaa64 ``` -For other installation methods, please refer to the [official SGLang installation guide](../../../docs/get-started/installation). +For other installation methods, please refer to the [official SGLang installation guide](../../../docs/get-started/install). -Ling-2.5-1T is also supported via the **nightly PyPI builds**. See the [SGLang Installation (PyPI)](../../../docs/get-started/installation) guide for setup instructions. +Ling-2.5-1T is also supported via the **nightly PyPI builds**. See the [SGLang Installation (PyPI)](../../../docs/get-started/install) guide for setup instructions. ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/InclusionAI/Ring-2.5-1T.mdx b/docs_new/cookbook/autoregressive/InclusionAI/Ring-2.5-1T.mdx index 0cd4d7f6b..ad711e7ee 100644 --- a/docs_new/cookbook/autoregressive/InclusionAI/Ring-2.5-1T.mdx +++ b/docs_new/cookbook/autoregressive/InclusionAI/Ring-2.5-1T.mdx @@ -2,7 +2,6 @@ title: Ring-2.5-1T metatags: description: "Deploy Ring-2.5-1T with SGLang - world's first open-source 1T parameter reasoning model with hybrid linear attention, deep reasoning, and agentic tool calling capabilities." -tag: NEW --- ## 1. Model Introduction @@ -42,7 +41,7 @@ docker pull lmsysorg/sglang:v0.5.9-rocm700-mi30x docker pull lmsysorg/sglang:v0.5.9-rocm700-mi35x ``` -For other installation methods, please refer to the [official SGLang installation guide](../../../docs/get-started/installation). +For other installation methods, please refer to the [official SGLang installation guide](../../../docs/get-started/install). ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/Llama/Llama3.1.mdx b/docs_new/cookbook/autoregressive/Llama/Llama3.1.mdx index 95bdbd2d6..ac7679cf6 100644 --- a/docs_new/cookbook/autoregressive/Llama/Llama3.1.mdx +++ b/docs_new/cookbook/autoregressive/Llama/Llama3.1.mdx @@ -20,7 +20,7 @@ For further details, please refer to the [Llama 3.1 blog](https://ai.meta.com/bl SGLang offers multiple installation methods. You can choose the most suitable installation method based on your hardware platform and requirements. -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/Llama/Llama3.3-70B.mdx b/docs_new/cookbook/autoregressive/Llama/Llama3.3-70B.mdx index b74da9e0e..f865b52fe 100644 --- a/docs_new/cookbook/autoregressive/Llama/Llama3.3-70B.mdx +++ b/docs_new/cookbook/autoregressive/Llama/Llama3.3-70B.mdx @@ -22,7 +22,7 @@ For more details, please refer to the [official Llama models repository](https:/ ## 2. SGLang Installation -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/Llama/Llama4.mdx b/docs_new/cookbook/autoregressive/Llama/Llama4.mdx index bd624433c..81e7e343a 100644 --- a/docs_new/cookbook/autoregressive/Llama/Llama4.mdx +++ b/docs_new/cookbook/autoregressive/Llama/Llama4.mdx @@ -27,7 +27,7 @@ For more details, please refer to the official llama4 Repository:https://www.lla SGLang offers multiple installation methods. You can choose the most suitable installation method based on your hardware platform and requirements. -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/MiniMax/MiniMax-M2.5.mdx b/docs_new/cookbook/autoregressive/MiniMax/MiniMax-M2.5.mdx index c24ff680d..d4c46623f 100644 --- a/docs_new/cookbook/autoregressive/MiniMax/MiniMax-M2.5.mdx +++ b/docs_new/cookbook/autoregressive/MiniMax/MiniMax-M2.5.mdx @@ -2,7 +2,6 @@ title: MiniMax-M2.5 metatags: description: "Deploy MiniMax-M2.5 with SGLang - community contribution guide for MiniMax M2.5 model deployment." -tag: NEW --- import { MiniMaxM25Deployment } from '/src/snippets/autoregressive/minimax-m25-deployment.jsx'; @@ -24,7 +23,7 @@ For more details, please refer to the [official MiniMax-M2.5 announcement](https SGLang offers multiple installation methods. You can choose the most suitable installation method based on your hardware platform and requirements. -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. **For AMD MI300X/MI325X/MI355X GPUs:** diff --git a/docs_new/cookbook/autoregressive/MiniMax/MiniMax-M2.7.mdx b/docs_new/cookbook/autoregressive/MiniMax/MiniMax-M2.7.mdx index 46dc5d044..a0dfeb1fd 100644 --- a/docs_new/cookbook/autoregressive/MiniMax/MiniMax-M2.7.mdx +++ b/docs_new/cookbook/autoregressive/MiniMax/MiniMax-M2.7.mdx @@ -24,7 +24,7 @@ For more details, see the [official MiniMax-M2.7 blog post](https://www.minimax. SGLang offers multiple installation methods. You can choose the most suitable installation method based on your hardware platform and requirements. -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. **Docker Images by Hardware Platform:** diff --git a/docs_new/cookbook/autoregressive/MiniMax/MiniMax-M2.mdx b/docs_new/cookbook/autoregressive/MiniMax/MiniMax-M2.mdx index 4d5e8d00c..11758aa1c 100644 --- a/docs_new/cookbook/autoregressive/MiniMax/MiniMax-M2.mdx +++ b/docs_new/cookbook/autoregressive/MiniMax/MiniMax-M2.mdx @@ -26,7 +26,7 @@ For more details, please refer to the [official Minimax GitHub Repository](https SGLang offers multiple installation methods. You can choose the most suitable installation method based on your hardware platform and requirements. -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. The AMD environment is currently available in SGLang via Docker image install. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. The AMD environment is currently available in SGLang via Docker image install. ### 2.1 AMD Docker #### 2.1.1 Launch docker diff --git a/docs_new/cookbook/autoregressive/Mistral/Devstral-2.mdx b/docs_new/cookbook/autoregressive/Mistral/Devstral-2.mdx index c8c525fb0..845db315d 100644 --- a/docs_new/cookbook/autoregressive/Mistral/Devstral-2.mdx +++ b/docs_new/cookbook/autoregressive/Mistral/Devstral-2.mdx @@ -35,7 +35,7 @@ For enterprises requiring specialized capabilities (increased context, domain-sp SGLang offers multiple installation methods. You can choose the most suitable installation method based on your hardware platform and requirements. -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. Devstral 2 requires a recent `transformers`. Please verify `transformers >= 5.0.0.rc`: diff --git a/docs_new/cookbook/autoregressive/Mistral/Ministral-3.mdx b/docs_new/cookbook/autoregressive/Mistral/Ministral-3.mdx index 8c858e947..dd2dd8eb1 100644 --- a/docs_new/cookbook/autoregressive/Mistral/Ministral-3.mdx +++ b/docs_new/cookbook/autoregressive/Mistral/Ministral-3.mdx @@ -23,7 +23,7 @@ For further details, please refer to the [official documentation](https://github ## 2. SGLang Installation -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/Mistral/Mistral-Small-4.mdx b/docs_new/cookbook/autoregressive/Mistral/Mistral-Small-4.mdx index 7dc4d179e..23ffd4258 100644 --- a/docs_new/cookbook/autoregressive/Mistral/Mistral-Small-4.mdx +++ b/docs_new/cookbook/autoregressive/Mistral/Mistral-Small-4.mdx @@ -43,10 +43,10 @@ With its multimodal capabilities, efficient MoE architecture, and flexible mode SGLang offers multiple installation methods. You can choose the most suitable installation method based on your hardware platform and requirements. -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. -Mistral Small 4 support landed in [sgl-project/sglang#20708](https://github.com/sgl-project/sglang/pull/20708) and has been merged into `main`. A model-specific Docker image is no longer required. Use the standard SGLang installation methods from the [official installation guide](../../../docs/get-started/installation). +Mistral Small 4 support landed in [sgl-project/sglang#20708](https://github.com/sgl-project/sglang/pull/20708) and has been merged into `main`. A model-specific Docker image is no longer required. Use the standard SGLang installation methods from the [official installation guide](../../../docs/get-started/install). --- diff --git a/docs_new/cookbook/autoregressive/Moonshotai/Kimi-K2.5.mdx b/docs_new/cookbook/autoregressive/Moonshotai/Kimi-K2.5.mdx index 162efb738..9b3a37edb 100644 --- a/docs_new/cookbook/autoregressive/Moonshotai/Kimi-K2.5.mdx +++ b/docs_new/cookbook/autoregressive/Moonshotai/Kimi-K2.5.mdx @@ -23,7 +23,7 @@ For details, see [official documentation](https://huggingface.co/moonshotai/Kimi ## 2. SGLang Installation -Refer to the [official SGLang installation guide](../../../docs/get-started/installation). +Refer to the [official SGLang installation guide](../../../docs/get-started/install). ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/Moonshotai/Kimi-K2.6.mdx b/docs_new/cookbook/autoregressive/Moonshotai/Kimi-K2.6.mdx index 38f04897a..6af071ea8 100644 --- a/docs_new/cookbook/autoregressive/Moonshotai/Kimi-K2.6.mdx +++ b/docs_new/cookbook/autoregressive/Moonshotai/Kimi-K2.6.mdx @@ -72,7 +72,7 @@ For details, see [official documentation](https://huggingface.co/moonshotai/Kimi ## 2. SGLang Installation -Refer to the [official SGLang installation guide](../../../docs/get-started/installation). +Refer to the [official SGLang installation guide](../../../docs/get-started/install). ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/Moonshotai/Kimi-K2.mdx b/docs_new/cookbook/autoregressive/Moonshotai/Kimi-K2.mdx index 0d858c56b..abe1abea1 100644 --- a/docs_new/cookbook/autoregressive/Moonshotai/Kimi-K2.mdx +++ b/docs_new/cookbook/autoregressive/Moonshotai/Kimi-K2.mdx @@ -20,7 +20,7 @@ For details, see [official documentation](https://github.com/MoonshotAI/Kimi-K2) ## 2. SGLang Installation -Refer to the [official SGLang installation guide](../../../docs/get-started/installation). +Refer to the [official SGLang installation guide](../../../docs/get-started/install). ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/Moonshotai/Kimi-Linear.mdx b/docs_new/cookbook/autoregressive/Moonshotai/Kimi-Linear.mdx index c0b4abd27..4c9d1158d 100644 --- a/docs_new/cookbook/autoregressive/Moonshotai/Kimi-Linear.mdx +++ b/docs_new/cookbook/autoregressive/Moonshotai/Kimi-Linear.mdx @@ -24,7 +24,7 @@ For more details, please refer to the [official Kimi Linear GitHub Repository]: SGLang offers multiple installation methods. You can choose the most suitable installation method based on your hardware platform and requirements. -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/NVIDIA/Nemotron3-Nano.mdx b/docs_new/cookbook/autoregressive/NVIDIA/Nemotron3-Nano.mdx index 4406459dd..0da99a106 100644 --- a/docs_new/cookbook/autoregressive/NVIDIA/Nemotron3-Nano.mdx +++ b/docs_new/cookbook/autoregressive/NVIDIA/Nemotron3-Nano.mdx @@ -20,7 +20,7 @@ At a high level: ## 2. SGLang Installation -Refer to the [official SGLang installation guide](../../../docs/get-started/installation), or install nightly wheel through: +Refer to the [official SGLang installation guide](../../../docs/get-started/install), or install nightly wheel through: ```bash Command uv pip install sglang==0.5.6.post3.dev1278+gad1b4e472 --extra-index-url https://sgl-project.github.io/whl/nightly/ ``` diff --git a/docs_new/cookbook/autoregressive/NVIDIA/Nemotron3-Super.mdx b/docs_new/cookbook/autoregressive/NVIDIA/Nemotron3-Super.mdx index abcc1f48d..2adebe7a8 100644 --- a/docs_new/cookbook/autoregressive/NVIDIA/Nemotron3-Super.mdx +++ b/docs_new/cookbook/autoregressive/NVIDIA/Nemotron3-Super.mdx @@ -31,7 +31,7 @@ uv pip install 'git+https://github.com/sgl-project/sglang.git#subdirectory=pytho docker pull lmsysorg/sglang:nightly-dev-20260310-0fd9a57d ``` -For the full Docker setup and other installation methods, please refer to the [official SGLang installation guide](../../../docs/get-started/installation). +For the full Docker setup and other installation methods, please refer to the [official SGLang installation guide](../../../docs/get-started/install). ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/OpenAI/GPT-OSS.mdx b/docs_new/cookbook/autoregressive/OpenAI/GPT-OSS.mdx index 791316982..30e9f94ca 100644 --- a/docs_new/cookbook/autoregressive/OpenAI/GPT-OSS.mdx +++ b/docs_new/cookbook/autoregressive/OpenAI/GPT-OSS.mdx @@ -23,7 +23,7 @@ GPT-OSS introduces several groundbreaking innovations: SGLang offers multiple installation methods. You can choose the most suitable installation method based on your hardware platform and requirements. -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. ## 3.Model Deployment diff --git a/docs_new/cookbook/autoregressive/Qwen/Qwen2.5-VL.mdx b/docs_new/cookbook/autoregressive/Qwen/Qwen2.5-VL.mdx index 650e68791..b8889f01b 100644 --- a/docs_new/cookbook/autoregressive/Qwen/Qwen2.5-VL.mdx +++ b/docs_new/cookbook/autoregressive/Qwen/Qwen2.5-VL.mdx @@ -27,7 +27,7 @@ For more details, please refer to the [official Qwen2.5-VL GitHub Repository](ht SGLang offers multiple installation methods. You can choose the most suitable installation method based on your hardware platform and requirements. -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/Qwen/Qwen3-Coder-Next.mdx b/docs_new/cookbook/autoregressive/Qwen/Qwen3-Coder-Next.mdx index 9016a2bd9..001b2fa6e 100644 --- a/docs_new/cookbook/autoregressive/Qwen/Qwen3-Coder-Next.mdx +++ b/docs_new/cookbook/autoregressive/Qwen/Qwen3-Coder-Next.mdx @@ -24,7 +24,7 @@ For more details, please refer to the [Qwen3-Coder-Next model card](https://hugg SGLang offers multiple installation methods. You can choose the most suitable installation method based on your hardware platform and requirements. -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. **Note:** Qwen3-Coder-Next requires SGLang v0.5.8 or later. diff --git a/docs_new/cookbook/autoregressive/Qwen/Qwen3-Coder.mdx b/docs_new/cookbook/autoregressive/Qwen/Qwen3-Coder.mdx index c02d69088..738d4654f 100644 --- a/docs_new/cookbook/autoregressive/Qwen/Qwen3-Coder.mdx +++ b/docs_new/cookbook/autoregressive/Qwen/Qwen3-Coder.mdx @@ -26,7 +26,7 @@ For more details, please refer to the [official Qwen3-Coder GitHub Repository](h SGLang offers multiple installation methods. You can choose the most suitable installation method based on your hardware platform and requirements. -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/Qwen/Qwen3-Next.mdx b/docs_new/cookbook/autoregressive/Qwen/Qwen3-Next.mdx index 44f91fbfe..57bb422b9 100644 --- a/docs_new/cookbook/autoregressive/Qwen/Qwen3-Next.mdx +++ b/docs_new/cookbook/autoregressive/Qwen/Qwen3-Next.mdx @@ -30,7 +30,7 @@ For more details, please refer to the [official Qwen3-Next blog](https://qwen.ai SGLang offers multiple installation methods. You can choose the most suitable installation method based on your hardware platform and requirements. -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/Qwen/Qwen3-VL.mdx b/docs_new/cookbook/autoregressive/Qwen/Qwen3-VL.mdx index 8e7588fe9..849e648d4 100644 --- a/docs_new/cookbook/autoregressive/Qwen/Qwen3-VL.mdx +++ b/docs_new/cookbook/autoregressive/Qwen/Qwen3-VL.mdx @@ -24,7 +24,7 @@ For more details, please refer to the [official Qwen3-VL GitHub Repository](http SGLang offers multiple installation methods. You can choose the most suitable installation method based on your hardware platform and requirements. -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/Qwen/Qwen3.5.mdx b/docs_new/cookbook/autoregressive/Qwen/Qwen3.5.mdx index 3f8349fde..39a41e861 100644 --- a/docs_new/cookbook/autoregressive/Qwen/Qwen3.5.mdx +++ b/docs_new/cookbook/autoregressive/Qwen/Qwen3.5.mdx @@ -104,7 +104,7 @@ docker pull lmsysorg/sglang:v0.5.9-rocm720-mi30x docker pull lmsysorg/sglang:v0.5.9-rocm720-mi35x ``` -For the full Docker setup and other installation methods, please refer to the [official SGLang installation guide](../../../docs/get-started/installation). +For the full Docker setup and other installation methods, please refer to the [official SGLang installation guide](../../../docs/get-started/install). ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/Qwen/Qwen3.6.mdx b/docs_new/cookbook/autoregressive/Qwen/Qwen3.6.mdx index ebf49b82d..cfe09aa83 100644 --- a/docs_new/cookbook/autoregressive/Qwen/Qwen3.6.mdx +++ b/docs_new/cookbook/autoregressive/Qwen/Qwen3.6.mdx @@ -61,7 +61,7 @@ uv pip install 'git+https://github.com/sgl-project/sglang.git#subdirectory=pytho docker pull lmsysorg/sglang:latest ``` -For the full Docker setup and other installation methods, please refer to the [official SGLang installation guide](../../../docs/get-started/installation). +For the full Docker setup and other installation methods, please refer to the [official SGLang installation guide](../../../docs/get-started/install). ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/Qwen/Qwen3.mdx b/docs_new/cookbook/autoregressive/Qwen/Qwen3.mdx index 74b5684f0..8c4c140dd 100644 --- a/docs_new/cookbook/autoregressive/Qwen/Qwen3.mdx +++ b/docs_new/cookbook/autoregressive/Qwen/Qwen3.mdx @@ -24,7 +24,7 @@ For more details, please refer to the [official Qwen3 GitHub Repository](https:/ SGLang offers multiple installation methods. You can choose the most suitable installation method based on your hardware platform and requirements. -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/StepFun/Step3-VL-10B.mdx b/docs_new/cookbook/autoregressive/StepFun/Step3-VL-10B.mdx index 6cf33cb52..845fb94f5 100644 --- a/docs_new/cookbook/autoregressive/StepFun/Step3-VL-10B.mdx +++ b/docs_new/cookbook/autoregressive/StepFun/Step3-VL-10B.mdx @@ -2,7 +2,6 @@ title: Step3-VL-10B metatags: description: "Deploy Step3-VL-10B multimodal model with SGLang - compact 10B dense model with frontier-level vision understanding, complex reasoning, and tool calling capabilities." -tag: NEW --- import { Step3VL10BDeployment } from '/src/snippets/autoregressive/step-3vl-10b-deployment.jsx'; @@ -24,7 +23,7 @@ For more details, please refer to the [Step3-VL-10B model card on Hugging Face]( SGLang offers multiple installation methods. You can choose the most suitable installation method based on your hardware platform and requirements. -Please refer to the [official SGLang installation guide](../../../docs/get-started/installation) for installation instructions. +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. ## 3. Model Deployment diff --git a/docs_new/cookbook/autoregressive/StepFun/Step3.5.mdx b/docs_new/cookbook/autoregressive/StepFun/Step3.5.mdx index edb98a7e5..176c2bf96 100644 --- a/docs_new/cookbook/autoregressive/StepFun/Step3.5.mdx +++ b/docs_new/cookbook/autoregressive/StepFun/Step3.5.mdx @@ -2,7 +2,6 @@ title: Step-3.5 metatags: description: "Deploy Step-3.5 reasoning engine with SGLang. " -tag: NEW --- import { Step35Deployment } from '/src/snippets/autoregressive/step-35-deployment.jsx'; diff --git a/docs_new/cookbook/diffusion/Qwen-Image/Qwen-Image.mdx b/docs_new/cookbook/diffusion/Qwen-Image/Qwen-Image.mdx index 84970ef9c..dc1c0ecc5 100644 --- a/docs_new/cookbook/diffusion/Qwen-Image/Qwen-Image.mdx +++ b/docs_new/cookbook/diffusion/Qwen-Image/Qwen-Image.mdx @@ -32,7 +32,7 @@ Qwen-Image is a text-to-image model. The recommended launch configurations vary ### 3.2 Configuration Tips -Current supported optimization all listed [here](../../../docs/sglang-diffusion/attention-backends#platform-support-matrix). +Current supported optimization all listed [here](../../../docs/sglang-diffusion/attention_backends#platform-support-matrix). - `--vae-path`: Path to a custom VAE model or HuggingFace model ID (e.g., fal/FLUX.2-Tiny-AutoEncoder). If not specified, the VAE will be loaded from the main model path. - `--num-gpus`: Number of GPUs to use @@ -45,7 +45,7 @@ Current supported optimization all listed [here](../../../docs/sglang-diffusion/ ## 4. API Usage -For complete API documentation, please refer to the [official API usage guide](../../../docs/sglang-diffusion/api/openai-api). +For complete API documentation, please refer to the [official API usage guide](../../../docs/sglang-diffusion/api/openai_api). ### 4.1 Generate an Image @@ -72,7 +72,7 @@ with open("output.png", "wb") as f: #### 4.2.1 Cache-DiT Acceleration -SGLang integrates [Cache-DiT](https://github.com/vipshop/cache-dit), a caching acceleration engine for Diffusion Transformers (DiT), to achieve up to 7.4x inference speedup with minimal quality loss. You can set `SGLANG_CACHE_DIT_ENABLED=True` to enable it. For more details, please refer to the SGLang Cache-DiT [documentation](../../../docs/sglang-diffusion/cache-dit). +SGLang integrates [Cache-DiT](https://github.com/vipshop/cache-dit), a caching acceleration engine for Diffusion Transformers (DiT), to achieve up to 7.4x inference speedup with minimal quality loss. You can set `SGLANG_CACHE_DIT_ENABLED=True` to enable it. For more details, please refer to the SGLang Cache-DiT [documentation](../../../docs/sglang-diffusion/cache_dit). **Basic Usage** diff --git a/docs_new/cookbook/diffusion/Wan/Wan2.1.mdx b/docs_new/cookbook/diffusion/Wan/Wan2.1.mdx index f69f3f86c..7cafe28fc 100644 --- a/docs_new/cookbook/diffusion/Wan/Wan2.1.mdx +++ b/docs_new/cookbook/diffusion/Wan/Wan2.1.mdx @@ -43,7 +43,7 @@ The Wan2.1 series offers models in multiple sizes and resolutions, optimized for ### 3.2 Configuration Tips -Current supported optimization options are listed in the [SGLang diffusion support matrix](../../../docs/sglang-diffusion/attention-backends#platform-support-matrix). +Current supported optimization options are listed in the [SGLang diffusion support matrix](../../../docs/sglang-diffusion/attention_backends#platform-support-matrix). - `--vae-path`: Path to a custom VAE model or HuggingFace model ID. If not specified, the VAE will be loaded from the main model path. - `--num-gpus {NUM_GPUS}`: Number of GPUs to use. @@ -58,7 +58,7 @@ Current supported optimization options are listed in the [SGLang diffusion suppo ### 4.1 Basic Usage For more API usage and request examples, please refer to: -[SGLang Diffusion OpenAI API](../../../docs/sglang-diffusion/api/openai-api) +[SGLang Diffusion OpenAI API](../../../docs/sglang-diffusion/api/openai_api) #### 4.1.1 Launch a server and then send requests @@ -104,7 +104,7 @@ sglang generate "${SERVER_ARGS[@]}" "${SAMPLING_ARGS[@]}" #### 4.2.1 Cache-DiT Acceleration -SGLang integrates [Cache-DiT](https://github.com/vipshop/cache-dit), a caching acceleration engine for Diffusion Transformers (DiT), to achieve significant inference speedups with minimal quality loss. You can set `SGLANG_CACHE_DIT_ENABLED=True` to enable it. For more details, please refer to the SGLang Cache-DiT [documentation](../../../docs/sglang-diffusion/cache-dit). +SGLang integrates [Cache-DiT](https://github.com/vipshop/cache-dit), a caching acceleration engine for Diffusion Transformers (DiT), to achieve significant inference speedups with minimal quality loss. You can set `SGLANG_CACHE_DIT_ENABLED=True` to enable it. For more details, please refer to the SGLang Cache-DiT [documentation](../../../docs/sglang-diffusion/cache_dit). **Basic Usage** diff --git a/docs_new/cookbook/intro copy.mdx b/docs_new/cookbook/intro copy.mdx index 59600608f..c080d0f46 100644 --- a/docs_new/cookbook/intro copy.mdx +++ b/docs_new/cookbook/intro copy.mdx @@ -149,7 +149,7 @@ Each recipe provides step-by-step instructions to help you quickly implement SGL ## Reference -- [Installation (PyPI)](../docs/get-started/installation) - Install SGLang via pip or uv (stable and nightly) +- [Installation (PyPI)](../docs/get-started/install) - Install SGLang via pip or uv (stable and nightly) - [Server arguments](./base/reference/server_arguments) - Understanding all the arguments ## 🚀 Quick Start diff --git a/docs_new/docs.json b/docs_new/docs.json index 32d74f732..1ac386fd6 100644 --- a/docs_new/docs.json +++ b/docs_new/docs.json @@ -18,7 +18,7 @@ }, { "source": "/advanced_features/adaptive_speculative_decoding.html", - "destination": "/docs/advanced_features/speculative_decoding" + "destination": "/docs/advanced_features/adaptive_speculative_decoding" }, { "source": "/advanced_features/attention_backend.html", @@ -78,7 +78,7 @@ }, { "source": "/advanced_features/hisparse_guide.html", - "destination": "/docs/advanced_features/overview" + "destination": "/docs/advanced_features/hisparse_guide" }, { "source": "/advanced_features/hyperparameter_tuning.html", @@ -158,7 +158,7 @@ }, { "source": "/basic_usage/deepseek_ocr.html", - "destination": "/docs/basic_usage/overview" + "destination": "/docs/basic_usage/deepseek_ocr" }, { "source": "/basic_usage/deepseek_v3.html", @@ -226,7 +226,7 @@ }, { "source": "/basic_usage/qwen3_5.html", - "destination": "/docs/basic_usage/qwen3" + "destination": "/docs/basic_usage/qwen3_5" }, { "source": "/basic_usage/qwen3_vl.html", @@ -258,7 +258,7 @@ }, { "source": "/developer_guide/development_jit_kernel_guide.html", - "destination": "/docs/developer_guide/JIT_kernels" + "destination": "/docs/developer_guide/development_jit_kernel_guide" }, { "source": "/developer_guide/evaluating_new_models.html", @@ -278,23 +278,23 @@ }, { "source": "/diffusion/api/openai_api.html", - "destination": "/docs/sglang-diffusion/api/openai-api" + "destination": "/docs/sglang-diffusion/api/openai_api" }, { "source": "/diffusion/api/post_processing.html", - "destination": "/docs/sglang-diffusion/installation" + "destination": "/docs/sglang-diffusion/api/post_processing" }, { "source": "/diffusion/ci_perf.html", - "destination": "/docs/sglang-diffusion/ci-performance" + "destination": "/docs/sglang-diffusion/ci_perf" }, { "source": "/diffusion/compatibility_matrix.html", - "destination": "/docs/sglang-diffusion/installation" + "destination": "/docs/sglang-diffusion/compatibility_matrix" }, { "source": "/diffusion/contributing.html", - "destination": "/docs/sglang-diffusion/installation" + "destination": "/docs/sglang-diffusion/contributing" }, { "source": "/diffusion/development.html", @@ -302,15 +302,15 @@ }, { "source": "/diffusion/disaggregation.html", - "destination": "/docs/sglang-diffusion/installation" + "destination": "/docs/sglang-diffusion/disaggregation" }, { "source": "/diffusion/environment_variables.html", - "destination": "/docs/sglang-diffusion/environment-variables" + "destination": "/docs/sglang-diffusion/environment_variables" }, { "source": "/diffusion/index.html", - "destination": "/docs/sglang-diffusion/installation" + "destination": "/docs/sglang-diffusion/index" }, { "source": "/diffusion/installation.html", @@ -318,11 +318,11 @@ }, { "source": "/diffusion/performance/attention_backends.html", - "destination": "/docs/sglang-diffusion/attention-backends" + "destination": "/docs/sglang-diffusion/attention_backends" }, { "source": "/diffusion/performance/cache/cache_dit.html", - "destination": "/docs/sglang-diffusion/cache-dit" + "destination": "/docs/sglang-diffusion/cache_dit" }, { "source": "/diffusion/performance/cache/index.html", @@ -330,7 +330,7 @@ }, { "source": "/diffusion/performance/cache/teacache.html", - "destination": "/docs/sglang-diffusion/tea-cache" + "destination": "/docs/sglang-diffusion/teacache" }, { "source": "/diffusion/performance/index.html", @@ -342,11 +342,11 @@ }, { "source": "/diffusion/performance/ring_sp_performance.html", - "destination": "/docs/sglang-diffusion/performance-optimization" + "destination": "/docs/sglang-diffusion/ring_sp_performance" }, { "source": "/diffusion/quantization.html", - "destination": "/docs/sglang-diffusion/installation" + "destination": "/docs/sglang-diffusion/quantization" }, { "source": "/diffusion/reference.html", @@ -354,7 +354,7 @@ }, { "source": "/diffusion/support_new_models.html", - "destination": "/docs/sglang-diffusion/installation" + "destination": "/docs/sglang-diffusion/support_new_models" }, { "source": "/diffusion/usage.html", @@ -362,87 +362,91 @@ }, { "source": "/get_started/install.html", - "destination": "/docs/get-started/installation" + "destination": "/docs/get-started/install" }, { "source": "/platforms/amd_gpu.html", - "destination": "/docs/hardware-platforms/amd-gpus" + "destination": "/docs/hardware-platforms/amd_gpu" }, { "source": "/platforms/apple_metal.html", - "destination": "/docs/hardware-platforms/overview" + "destination": "/docs/hardware-platforms/apple_metal" }, { "source": "/platforms/ascend/ascend_contribution_guide.html", - "destination": "/docs/hardware-platforms/overview" + "destination": "/docs/hardware-platforms/ascend-npus/ascend_contribution_guide" }, { "source": "/platforms/ascend/ascend_npu.html", - "destination": "/docs/hardware-platforms/ascend-npus/SGLang-installation-with-NPUs-support" + "destination": "/docs/hardware-platforms/ascend-npus/ascend_npu" }, { "source": "/platforms/ascend/ascend_npu_best_practice.html", - "destination": "/docs/hardware-platforms/ascend-npus/Best-Practice-on-Ascend-NPU" + "destination": "/docs/hardware-platforms/ascend-npus/ascend_npu_best_practice" }, { "source": "/platforms/ascend/ascend_npu_deepseek_example.html", - "destination": "/docs/hardware-platforms/ascend-npus/DeepSeek-Examples" + "destination": "/docs/hardware-platforms/ascend-npus/ascend_npu_deepseek_example" }, { "source": "/platforms/ascend/ascend_npu_environment_variables.html", - "destination": "/docs/hardware-platforms/overview" + "destination": "/docs/hardware-platforms/ascend-npus/ascend_npu_environment_variables" }, { "source": "/platforms/ascend/ascend_npu_glm5_examples.html", - "destination": "/docs/hardware-platforms/ascend-npus/GLM-5" + "destination": "/docs/hardware-platforms/ascend-npus/ascend_npu_glm5_examples" }, { "source": "/platforms/ascend/ascend_npu_quantization.html", - "destination": "/docs/hardware-platforms/overview" + "destination": "/docs/hardware-platforms/ascend-npus/ascend_npu_quantization" + }, + { + "source": "/platforms/ascend/ascend_npu_quick_start.html", + "destination": "/docs/hardware-platforms/ascend-npus/ascend_npu_quick_start" }, { "source": "/platforms/ascend/ascend_npu_qwen3_5_examples.html", - "destination": "/docs/hardware-platforms/ascend-npus/Qwen3.5" + "destination": "/docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_5_examples" }, { "source": "/platforms/ascend/ascend_npu_qwen3_examples.html", - "destination": "/docs/hardware-platforms/ascend-npus/Qwen3-Examples" + "destination": "/docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_examples" }, { "source": "/platforms/ascend/ascend_npu_support.html", - "destination": "/docs/hardware-platforms/overview" + "destination": "/docs/hardware-platforms/ascend-npus/ascend_npu_quick_start" }, { "source": "/platforms/ascend/ascend_npu_support_features.html", - "destination": "/docs/hardware-platforms/ascend-npus/Support-Features-on-Ascend-NPU" + "destination": "/docs/hardware-platforms/ascend-npus/ascend_npu_support_features" }, { "source": "/platforms/ascend/ascend_npu_support_models.html", - "destination": "/docs/hardware-platforms/ascend-npus/Support-Models-on-Ascend-NPU" + "destination": "/docs/hardware-platforms/ascend-npus/ascend_npu_support_models" }, { "source": "/platforms/ascend/mindspore_backend.html", - "destination": "/docs/hardware-platforms/overview" + "destination": "/docs/hardware-platforms/ascend-npus/mindspore_backend" }, { "source": "/platforms/ascend_npu_ring_sp_performance.html", - "destination": "/docs/hardware-platforms/overview" + "destination": "/docs/hardware-platforms/ascend-npus/ascend_npu_ring_sp_performance" }, { "source": "/platforms/cpu_server.html", - "destination": "/docs/hardware-platforms/cpu-server" + "destination": "/docs/hardware-platforms/cpu_server" }, { "source": "/platforms/mthreads_gpu.html", - "destination": "/docs/hardware-platforms/overview" + "destination": "/docs/hardware-platforms/mthreads_gpu" }, { "source": "/platforms/nvidia_jetson.html", - "destination": "/docs/hardware-platforms/overview" + "destination": "/docs/hardware-platforms/nvidia_jetson" }, { "source": "/platforms/plugin.html", - "destination": "/docs/hardware-platforms/overview" + "destination": "/docs/hardware-platforms/plugin" }, { "source": "/platforms/tpu.html", @@ -526,7 +530,7 @@ }, { "source": "/supported_models/extending/mindspore_models.html", - "destination": "/docs/supported-models/mindspore-models" + "destination": "/docs/supported-models/mindspore_models" }, { "source": "/supported_models/extending/modelscope.html", @@ -534,11 +538,11 @@ }, { "source": "/supported_models/extending/support_new_models.html", - "destination": "/docs/supported-models/new-model-support" + "destination": "/docs/supported-models/support_new_models" }, { "source": "/supported_models/extending/transformers_fallback.html", - "destination": "/docs/supported-models/transformers-fallback" + "destination": "/docs/supported-models/transformers_fallback" }, { "source": "/supported_models/index.html", @@ -546,11 +550,11 @@ }, { "source": "/supported_models/retrieval_ranking/classify_models.html", - "destination": "/docs/supported-models/classification-models" + "destination": "/docs/supported-models/classify_models" }, { "source": "/supported_models/retrieval_ranking/embedding_models.html", - "destination": "/docs/supported-models/embedding-models" + "destination": "/docs/supported-models/embedding_models" }, { "source": "/supported_models/retrieval_ranking/index.html", @@ -558,7 +562,7 @@ }, { "source": "/supported_models/retrieval_ranking/rerank_models.html", - "destination": "/docs/supported-models/rerank-models" + "destination": "/docs/supported-models/rerank_models" }, { "source": "/supported_models/specialized/index.html", @@ -566,15 +570,15 @@ }, { "source": "/supported_models/specialized/reward_models.html", - "destination": "/docs/supported-models/reward-models" + "destination": "/docs/supported-models/reward_models" }, { "source": "/supported_models/text_generation/diffusion_language_models.html", - "destination": "/docs/supported-models/diffusion-language-models" + "destination": "/docs/supported-models/diffusion_language_models" }, { "source": "/supported_models/text_generation/generative_models.html", - "destination": "/docs/supported-models/large-language-models" + "destination": "/docs/supported-models/generative_models" }, { "source": "/supported_models/text_generation/index.html", @@ -582,7 +586,7 @@ }, { "source": "/supported_models/text_generation/multimodal_language_models.html", - "destination": "/docs/supported-models/vision-language-models" + "destination": "/docs/supported-models/multimodal_language_models" }, { "source": "/supported_models.html", @@ -590,7 +594,7 @@ }, { "source": "/diffusion.html", - "destination": "/docs/sglang-diffusion/installation" + "destination": "/docs/sglang-diffusion/index" } ], "colors": { @@ -626,7 +630,7 @@ "icon": "play", "pages": [ "index", - "docs/get-started/installation", + "docs/get-started/install", "docs/get-started/quickstart", "docs/basic_usage/send_request" ] @@ -660,12 +664,14 @@ "docs/basic_usage/popular_model_usage", "docs/basic_usage/deepseek_v3", "docs/basic_usage/deepseek_v32", + "docs/basic_usage/deepseek_ocr", "docs/basic_usage/glm45", "docs/basic_usage/glmv", "docs/basic_usage/gpt_oss", "docs/basic_usage/kimi_k2_5", "docs/basic_usage/minimax_m2", "docs/basic_usage/qwen3", + "docs/basic_usage/qwen3_5", "docs/basic_usage/qwen3_vl", "docs/basic_usage/llama4" ] @@ -681,7 +687,9 @@ "docs/advanced_features/object_storage", "docs/advanced_features/hyperparameter_tuning", "docs/advanced_features/attention_backend", + "docs/advanced_features/hisparse_guide", "docs/advanced_features/speculative_decoding", + "docs/advanced_features/adaptive_speculative_decoding", "docs/advanced_features/structured_outputs", "docs/advanced_features/structured_outputs_for_reasoning_models", "docs/advanced_features/tool_parser", @@ -723,32 +731,32 @@ { "group": "Text Generation", "pages": [ - "docs/supported-models/large-language-models", - "docs/supported-models/vision-language-models", - "docs/supported-models/diffusion-language-models" + "docs/supported-models/generative_models", + "docs/supported-models/multimodal_language_models", + "docs/supported-models/diffusion_language_models" ] }, { "group": "Retrieval and Ranking", "pages": [ - "docs/supported-models/embedding-models", - "docs/supported-models/rerank-models", - "docs/supported-models/classification-models" + "docs/supported-models/embedding_models", + "docs/supported-models/rerank_models", + "docs/supported-models/classify_models" ] }, { "group": "Specialized Models", "pages": [ - "docs/supported-models/reward-models" + "docs/supported-models/reward_models" ] }, { "group": "Extending SGLang", "pages": [ - "docs/supported-models/new-model-support", - "docs/supported-models/transformers-fallback", + "docs/supported-models/support_new_models", + "docs/supported-models/transformers_fallback", "docs/supported-models/modelscope", - "docs/supported-models/mindspore-models" + "docs/supported-models/mindspore_models" ] } ] @@ -763,7 +771,7 @@ "group": "Development", "pages": [ "docs/developer_guide/development_guide_using_docker", - "docs/developer_guide/JIT_kernels" + "docs/developer_guide/development_jit_kernel_guide" ] }, { @@ -824,30 +832,38 @@ "pages": [ "docs/hardware-platforms/overview", "docs/hardware-platforms/nvidia-gpus", - "docs/hardware-platforms/amd-gpus", + "docs/hardware-platforms/amd_gpu", + "docs/hardware-platforms/apple_metal", { "group": "Ascend NPUs", "pages": [ - "docs/hardware-platforms/ascend-npus/Best-Practice-on-Ascend-NPU", - "docs/hardware-platforms/ascend-npus/DeepSeek-Examples", - "docs/hardware-platforms/ascend-npus/GLM-5", - "docs/hardware-platforms/ascend-npus/MindSpore-Models", - "docs/hardware-platforms/ascend-npus/Qwen3-Examples", - "docs/hardware-platforms/ascend-npus/Qwen3.5", - "docs/hardware-platforms/ascend-npus/SGLang-installation-with-NPUs-support", - "docs/hardware-platforms/ascend-npus/Support-Features-on-Ascend-NPU", - "docs/hardware-platforms/ascend-npus/Support-Models-on-Ascend-NPU" + "docs/hardware-platforms/ascend-npus/ascend_npu_quick_start", + "docs/hardware-platforms/ascend-npus/ascend_npu", + "docs/hardware-platforms/ascend-npus/ascend_npu_support_features", + "docs/hardware-platforms/ascend-npus/ascend_npu_support_models", + "docs/hardware-platforms/ascend-npus/ascend_npu_quantization", + "docs/hardware-platforms/ascend-npus/ascend_npu_deepseek_example", + "docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_examples", + "docs/hardware-platforms/ascend-npus/mindspore_backend", + "docs/hardware-platforms/ascend-npus/ascend_contribution_guide", + "docs/hardware-platforms/ascend-npus/ascend_npu_best_practice", + "docs/hardware-platforms/ascend-npus/ascend_npu_ring_sp_performance", + "docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_5_examples", + "docs/hardware-platforms/ascend-npus/ascend_npu_glm5_examples", + "docs/hardware-platforms/ascend-npus/ascend_npu_environment_variables" ] }, - "docs/hardware-platforms/cpu-server", + "docs/hardware-platforms/cpu_server", { "group": "Edge & Embedded", "pages": [ - "docs/hardware-platforms/nvidia" + "docs/hardware-platforms/nvidia_jetson" ] }, + "docs/hardware-platforms/mthreads_gpu", "docs/hardware-platforms/tpu", - "docs/hardware-platforms/xpu" + "docs/hardware-platforms/xpu", + "docs/hardware-platforms/plugin" ] } ] @@ -925,14 +941,14 @@ ] }, { - "group": "Moonshotai", - "pages": [ - "cookbook/autoregressive/Moonshotai/Kimi-K2.6", - "cookbook/autoregressive/Moonshotai/Kimi-K2.5", - "cookbook/autoregressive/Moonshotai/Kimi-K2", - "cookbook/autoregressive/Moonshotai/Kimi-Linear" - ] - }, + "group": "Moonshotai", + "pages": [ + "cookbook/autoregressive/Moonshotai/Kimi-K2.6", + "cookbook/autoregressive/Moonshotai/Kimi-K2.5", + "cookbook/autoregressive/Moonshotai/Kimi-K2", + "cookbook/autoregressive/Moonshotai/Kimi-Linear" + ] + }, { "group": "MiniMax", "pages": [ @@ -1073,38 +1089,44 @@ "group": "SGLang Diffusion", "icon": "sparkles", "pages": [ - "sglang-diffusion/intro", + "docs/sglang-diffusion/index", "docs/sglang-diffusion/installation", - "docs/sglang-diffusion/supported-models", + "docs/sglang-diffusion/compatibility_matrix", + "docs/sglang-diffusion/disaggregation", + "docs/sglang-diffusion/quantization", { "group": "Usage", "pages": [ "docs/sglang-diffusion/api/cli", - "docs/sglang-diffusion/api/openai-api" + "docs/sglang-diffusion/api/openai_api", + "docs/sglang-diffusion/api/post_processing" ] }, { "group": "Performance Optimization", "pages": [ "docs/sglang-diffusion/performance-optimization", - "docs/sglang-diffusion/attention-backends", + "docs/sglang-diffusion/ring_sp_performance", + "docs/sglang-diffusion/attention_backends", "docs/sglang-diffusion/profiling", - "docs/sglang-diffusion/ci-performance" + "docs/sglang-diffusion/ci_perf" ] }, { "group": "Caching Strategies", "pages": [ "docs/sglang-diffusion/caching-acceleration", - "docs/sglang-diffusion/cache-dit", - "docs/sglang-diffusion/tea-cache" + "docs/sglang-diffusion/cache_dit", + "docs/sglang-diffusion/teacache" ] }, { "group": "References", "pages": [ - "docs/sglang-diffusion/environment-variables", - "docs/sglang-diffusion/supported-models" + "docs/sglang-diffusion/environment_variables", + "docs/sglang-diffusion/compatibility_matrix", + "docs/sglang-diffusion/support_new_models", + "docs/sglang-diffusion/contributing" ] } ] diff --git a/docs_new/docs/advanced_features/adaptive_speculative_decoding.mdx b/docs_new/docs/advanced_features/adaptive_speculative_decoding.mdx new file mode 100644 index 000000000..7a59376fa --- /dev/null +++ b/docs_new/docs/advanced_features/adaptive_speculative_decoding.mdx @@ -0,0 +1,197 @@ +--- +title: "Adaptive Speculative Decoding" +metatags: + description: "Configure adaptive speculative decoding so SGLang can adjust speculative steps and draft tokens at runtime based on acceptance behavior." +--- + +Adaptive speculative decoding lets SGLang adjust `speculative_num_steps/speculative_num_draft_tokens` at runtime instead of keeping a single fixed value for the whole server lifetime. +It is designed for workloads whose accept length changes over time, where one static step count is rarely optimal. + +## Current support + +- Only `--speculative-algorithm EAGLE` +- Only `--speculative-eagle-topk 1` +- If either condition is not met, SGLang falls back to static speculative settings + +## Why adaptive steps help + +`speculative_num_steps` controls how many draft-model autoregressive steps run in each speculative round. In practice, the best value depends on the current workload. + +- If `num_steps` is too small, the draft model could have produced more accepted tokens, but the round stops too early. +- If `num_steps` is too large, the draft model produces many candidate tokens that the target model rejects, so extra draft work is wasted. + +Real traffic often moves between high-acceptance and low-acceptance phases, so one fixed step count is usually a compromise. Adaptive mode tries to follow the workload instead of hard-coding a single global `num_steps`. + +## Design overview + +The adaptive mechanism has three pieces: + +- `AdaptiveSpeculativeParams`: the EMA-based policy +- `SpecRuntimeState`: the per-tier runtime state bundle +- `AdaptiveController`: the coordinator that chooses a tier and activates the matching runtime state + +At startup, SGLang pre-builds one runtime state per candidate tier. By default, the candidate tiers are `candidate_steps = [1, 3, 7]`. + +```mermaid +--- +title: "SpecRuntimeState — speculative_num_steps / speculative_num_draft_tokens" +--- +graph LR + subgraph SR[" "] + direction LR + subgraph D["Draft stage"] + direction TB + d1[attn_backend] + d2[cuda_graph] + end + subgraph V["Verify stage"] + direction TB + v1[attn_backend] + v2[cuda_graph] + end + subgraph E["Extend stage"] + direction TB + e1[attn_backend] + e2[cuda_graph] + end + end +``` + +This matters because `CudaGraphRunner` is shape-dependent. Each candidate tier owns its own graph and backend state, so runtime switching is a reference swap, not an online graph recapture. + +## Runtime flow + +The adaptive update happens after verify and affects the next round, not the current one: + +```mermaid +--- +title: "EAGLEWorker.forward_batch_generation() — decode path" +--- +flowchart TD + A["① draft(batch)
draft model multi-step generation with current tier"] + B["② verify(batch, spec_info)
target model tree verification → produces accept_length_per_req"] + C["③ forward_draft_extend_after_decode(batch)
draft model KV-cache catch-up"] + D["④ adaptive_controller.on_verify_complete(accept_lengths)
update EMA, apply warmup / interval / hysteresis gates
if tier changed, select a pre-built state from pool"] + E["worker.apply_runtime_state(state)"] + A --> B --> C --> D --> E +``` + +> Tier switch happens after the current round completes. Backends and CUDA graphs are never swapped mid-round. + +## How the policy decides + +After each verify pass, SGLang reads the accepted draft length per request, computes the batch average, smooths it with an exponential moving average (EMA), and switches among the pre-built candidate tiers `[1, 3, 7]` by default. + +The decision logic is intentionally conservative: + +- `warmup_batches` skips the first few batches +- `update_interval` avoids switching every batch +- `down_hysteresis` and `up_hysteresis` reduce oscillation + +Conceptually, the policy probes one step beyond the observed acceptance: + +```text +target_steps ≈ clamp(round(ema_accept_len) + 1, min(candidate_steps), max(candidate_steps)) +``` + +So if recent requests consistently accept more drafted tokens, the policy tends to move up. If they start rejecting earlier, it tends to move down. + +## Usage + +`--speculative-adaptive-config` is optional, but the speculative setup still needs to be valid for adaptive mode. + +```bash +python3 -m sglang.launch_server \ + --model meta-llama/Llama-2-7b-chat-hf \ + --speculative-algorithm EAGLE \ + --speculative-draft-model-path lmsys/sglang-EAGLE-llama2-chat-7B \ + --speculative-eagle-topk 1 \ + --speculative-num-steps 3 \ + --speculative-num-draft-tokens 4 \ + --speculative-adaptive +``` + +If you want to override the defaults, add `--speculative-adaptive-config /path/to/adaptive_spec.json`. + +Example config: + +```json +{ + "candidate_steps": [1, 3, 7], + "ema_alpha": 0.2, + "warmup_batches": 10, + "update_interval": 5 +} +``` + +## Config file reference + +The config file is optional. Any omitted keys use defaults. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
KeyDefaultMeaning
candidate_steps[1, 3, 7]Discrete speculative_num_steps tiers that adaptive mode can switch between
ema_alpha0.2EMA smoothing factor for accepted draft length
update_interval5Recompute interval, in verify batches, after warmup
warmup_batches10Number of verify batches to observe before switching
down_hysteresis-0.25Extra margin before moving to a smaller step
up_hysteresis0.0Extra margin before moving to a larger step
+ +The initial `--speculative-num-steps` is snapped to the nearest value in `candidate_steps`. + +## Monitoring + +You can inspect the active tier and acceptance metric via `/server_info`: + +```bash +curl -s http://127.0.0.1:30000/server_info | jq '.internal_states[0] | {speculative_num_steps, avg_spec_accept_length}' +``` + +- `speculative_num_steps` is the current active tier +- `avg_spec_accept_length` helps explain whether the server is likely to move up or down + +## Tuning tips + +- Start with the default candidate tiers `[1, 3, 7]` +- Use fewer tiers if you want lower startup and graph-memory overhead +- Increase `ema_alpha` to react faster, or lower it for more stability +- Increase `warmup_batches` or `update_interval` if tier switching is too noisy +- If your workload is already stable and one static setting is well tuned, adaptive mode may not help much diff --git a/docs_new/docs/advanced_features/attention_backend.mdx b/docs_new/docs/advanced_features/attention_backend.mdx index 2d07fa02c..ed0d9982b 100644 --- a/docs_new/docs/advanced_features/attention_backend.mdx +++ b/docs_new/docs/advanced_features/attention_backend.mdx @@ -14,7 +14,7 @@ If you don't specify `--attention-backend`, SGLang makes a best effort to automa ## Support Matrix -The support matrix is split into two parts: MHA (standard attention) and MLA (multi-head latent attention). For an explanation of the key differences between MHA and MLA, please see the [SGLang documentation on DeepSeek MLA](../basic_usage/deepseek_v3.md#multi-head-latent-attention-mla-throughput-optimizations) and the original [DeepSeek MLA paper](https://arxiv.org/pdf/2405.04434). +The support matrix is split into two parts: MHA (standard attention) and MLA (multi-head latent attention). For an explanation of the key differences between MHA and MLA, please see the [SGLang documentation on DeepSeek MLA](../basic_usage/deepseek_v3#multi-head-latent-attention-mla-throughput-optimizations) and the original [DeepSeek MLA paper](https://arxiv.org/pdf/2405.04434). ### MHA Backends @@ -67,15 +67,15 @@ The support matrix is split into two parts: MHA (standard attention) and MLA (mu 128 ❌ ✅ - ❌ - ❌ + ✅ + ✅ ❌ ✅ **Triton** ❌ - ❌ + ✅ ✅ ✅ ✅ @@ -129,7 +129,7 @@ The support matrix is split into two parts: MHA (standard attention) and MLA (mu ❌ ✅ ✅ - ❌ + ✅ ✅ @@ -147,9 +147,9 @@ The support matrix is split into two parts: MHA (standard attention) and MLA (mu ✅ ❌ ❌ - ❌ + ✅ ❌ - ❌ + ✅ ✅ @@ -258,7 +258,7 @@ The support matrix is split into two parts: MHA (standard attention) and MLA (mu 1 ❌ ✅ - ❌ + ✅ ❌ ❌ @@ -279,10 +279,14 @@ Multimodal attention is selected by `--mm-attention-backend`. The "MultiModal" c -- FlashAttention 4 is prefill-only for now. -- NSA is specifically designed for [DeepSeek V3.2 DSA](https://lmsys.org/blog/2025-09-29-deepseek-V32/). +- FlashAttention 4 supports both prefill and decode on SM90 (Hopper) and SM100 (Blackwell). FA4 MLA supports `page_size = 1`; FA4 MHA requires `page_size = 128`. On SM100, this is auto-enforced by the server; on SM90, users must set `--page-size 128` manually. +- NSA is specifically designed for [DeepSeek V3.2 DSA](https://lmsys.org/blog/2025-09-29-deepseek-V32/). See the [DSA Attention Backend (NSA)](#dsa-attention-backend-nsa) section and [DeepSeek V3.2 deployment guide](../basic_usage/deepseek_v32) for details. + +**FA4 on Hopper (SM90):** FA4 decode speed decreases as sequence length grows due to lack of SplitKV support. At batch=1 compared to FA3 on H100: ~-10% at 2K tokens, ~-18% at 4K, ~-31% at 8K, ~-49% at 16K. Larger batch sizes reduce the gap (e.g., batch=8: ~-2% at 2K, ~-8% at 4K). Blackwell (SM100) is not affected. + + For the KV4 FA4 scenario, FA4 requires using a different --decode-attention-backend to run. Except for trtllm_mha being incompatible with FA4, all other decode backends behave as shown in the table. @@ -291,8 +295,16 @@ For the KV4 FA4 scenario, FA4 requires using a different --decode-attention-back Speculative decoding topk: `topk` is the number of draft tokens sampled per step from the draft model. `topk = 1` follows classic EAGLE; `topk > 1` explores multiple branches and requires backend support in both draft and verification paths. + +**Speculative Decoding V2 (Spec V2):** Spec V2 uses overlap scheduling (`SGLANG_ENABLE_SPEC_V2=True`) that benefits various attention backends. Requires `--speculative-eagle-topk 1` and currently applies to EAGLE and EAGLE3. + +**Verified backends:** TRTLLM MLA, TRTLLM MHA, FA3, Ascend (NPU), Triton. + +**Limited support:** FlashInfer can run under Spec V2, but its plan stream (used for split-KV optimization) introduces a synchronization point that limits overlap benefits. + + -Page size controls how many tokens are grouped into a KV cache block. For the prefix cache to take effect, the number of tokens must fill at least one complete page. For example, if your prompt is only 32 tokens and `page_size = 64`, it won't fill a complete page and cannot be matched in the prefix cache (pages cannot be padded). With 65 tokens and `page_size = 64`, only the first page of 64 tokens will be cached and matched; the remaining 1 token is discarded. Use `page_size = 1` for maximum prefix reuse (token-level matching). +Page size controls how many tokens are grouped into a KV cache block. For the prefix cache to take effect, the number of tokens must fill at least one complete page. For example, if your prompt is only 32 tokens and `page_size = 64`, it won't fill a complete page and cannot be matched in the prefix cache (pages cannot be padded). With 65 tokens and `page_size = 64`, only the first page of 64 tokens will be cached and matched; the remaining 1 token is discarded. Use `page_size = 1` for maximum prefix reuse (token-level matching). Note that higher page sizes generally improve attention kernel performance, so prefer `page_size > 1` when prefix cache reuse is not critical. Many backends that do not natively operate on pages can emulate `page_size > 1` at the wrapper layer by expanding page tables to per-token indices. The "Page Size > 1 (native)" column indicates true in-kernel paging. Some backends require fixed native page sizes and cannot be reduced/emulated differently: TRTLLM MHA (16/32/64), TRTLLM MLA (32/64), FlashMLA (64), Cutlass MLA (128), Ascend (128). @@ -303,6 +315,138 @@ MLA page-size constraints: - Cutlass MLA: page_size = 128. - TRTLLM MLA: page_size ∈ {32, 64}. +### GDN Attention Backends + +GDN (Gated Delta Network) is a linear attention mechanism with O(n) complexity, used in hybrid models that alternate GDN linear attention layers with standard full attention layers. GDN is **not** selected via `--attention-backend`; it is automatically activated when the model architecture requires it (e.g., Qwen 3.5, Qwen 3 Next, Jet Nemotron, Jet VLM). + +The GDN linear attention layers have their own kernel backends, selected via `--linear-attn-backend` (default: `triton`). You can override the kernel per phase with `--linear-attn-decode-backend` and `--linear-attn-prefill-backend`. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
BackendDecodePrefill / ExtendSpec Decoding (Target Verify)
Triton (CUDA)✅✅✅
Triton (AMD/ROCm)✅✅✅
Triton (NPU)✅✅❌
Triton (CPU)✅✅❌
CuTe DSL (CUDA only)✅❌❌
+ + +GDN models are hybrid: the full-attention layers still require a standard `--attention-backend`. Platform constraints for the full-attention backend on hybrid GDN models: +- **Blackwell (e.g., B200)**: `triton`, `trtllm_mha`, or `fa4` only. +- **NPU (Ascend)**: `ascend` only. +- **AMD (ROCm)**: `triton` recommended. +- **Other CUDA (Hopper, Ampere, etc.)**: auto-selection works; no special constraints. + + +### DSA Attention Backend (NSA) + +DSA (Deepseek Sparse Attention) is a native sparse attention mechanism used by [DeepSeek V3.2](https://lmsys.org/blog/2025-09-29-deepseek-V32/). It is activated automatically when the model architecture requires it and is selected via `--attention-backend nsa`. + +Internally, the NSA backend dispatches to different sub-backends for prefill and decode phases. You can override these with `--nsa-prefill-backend` and `--nsa-decode-backend`: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Sub-backendPrefillDecodeNotes
flashmla_sparse✅✅Default prefill on Hopper and Blackwell (bf16)
flashmla_kv✅✅Default decode for FP8 on Blackwell with DP
flashmla_auto✅❌Auto-selects flashmla_sparse or flashmla_kv based on kv_cache_dtype
fa3✅✅Default decode on Hopper (bf16)
trtllm✅✅Default decode on Blackwell (bf16); default for both on Blackwell without DP
tilelang✅✅Default on AMD (ROCm)
aiter✅✅AMD-specific kernel library (requires aiter package)
+ +For deployment examples, see the [DeepSeek V3.2 deployment guide](../basic_usage/deepseek_v32). + ### Hybrid attention (different backends for prefill vs decode) (Experimental) @@ -354,7 +498,7 @@ If the `--attention-backend` argument is not specified, SGLang automatically sel **2. MLA Models (e.g., DeepSeek V3)** - **Hopper**: Defaults to `fa3` (requires CUDA 12.3+). -- **Blackwell**: Defaults to `trtllm_mla`. +- **Blackwell**: Defaults to `flashinfer`; `trtllm_mla` is auto-selected for DeepSeek V3 models specifically. - **Other Architectures**: Defaults to `triton`. @@ -432,8 +576,34 @@ python3 -m sglang.launch_server \ --trust-remote-code ``` +- TRTLLM MHA (Optimized for Blackwell Architecture, e.g., B200) +```bash Command +python3 -m sglang.launch_server \ + --tp 4 \ + --model Qwen/Qwen3.5-35B-A3B-FP8 \ + --attention-backend trtllm_mha \ + --trust-remote-code +``` + +- TRTLLM MHA (XQA backend) (Optimized for SM90 and SM120, e.g., H20, H200, 5090) + Note that TRTLLM XQA backend only works well for pagesize 64. +```bash Command +python3 -m sglang.launch_server \ + --tp 4 \ + --model Qwen/Qwen3.5-35B-A3B-FP8 \ + --decode-attention-backend trtllm_mha \ + --trust-remote-code +``` + - FlashAttention 4 (MHA & MLA) ```bash Command +# FA4 for both prefill and decode on SM90/SM100 +python3 -m sglang.launch_server \ + --model-path Qwen/Qwen3-30B-A3B-Instruct-2507-FP8 \ + --attention-backend fa4 \ + --page-size 128 \ + --trust-remote-code + python3 -m sglang.launch_server \ --tp 8 \ --model deepseek-ai/DeepSeek-R1 \ @@ -497,6 +667,10 @@ To add a new attention backend, you can learn from the existing backends (`python/sglang/srt/layers/attention/triton_backend.py`, `python/sglang/srt/layers/attention/flashattention_backend.py`) and follow the steps below. + +Linear attention kernel backends (GDN, KDA) follow a different pattern. They implement `LinearAttnKernelBase` in `python/sglang/srt/layers/attention/linear/kernels/` and are dispatched by `GDNKernelDispatcher` / `KDAKernelDispatcher` rather than registered via `@register_attention_backend`. + + 1. Run without cuda graph. Support the two forward functions - forward_extend - Will be used for prefill, prefill with KV cache, and target verification diff --git a/docs_new/docs/advanced_features/epd_disaggregation.mdx b/docs_new/docs/advanced_features/epd_disaggregation.mdx index ca30ef2d4..db936196a 100644 --- a/docs_new/docs/advanced_features/epd_disaggregation.mdx +++ b/docs_new/docs/advanced_features/epd_disaggregation.mdx @@ -19,6 +19,81 @@ When launching a language-only model, you must additionally specify the encoder We support multiple encoder transfer backends, including zmq_to_scheduler, zmq_to_tokenizer, and mooncake (the default is zmq_to_scheduler). The backend can be selected using `--encoder-transfer-backend`. +### Encoder transfer with Mooncake + +`--encoder-transfer-backend mooncake` controls **how encoder outputs are transferred** between encoder and language/prefill services. It is an encoder transfer option and can be used independently of the global multimodal embedding cache. + +Example: + +```bash Command +# encoder +python -m sglang.launch_server \ + --model-path Qwen/Qwen3-VL-8B-Instruct \ + --encoder-only \ + --encoder-transfer-backend mooncake \ + --port 30000 + +# language-only server +python -m sglang.launch_server \ + --model-path Qwen/Qwen3-VL-8B-Instruct \ + --language-only \ + --encoder-urls http://127.0.0.1:30000 \ + --encoder-transfer-backend mooncake \ + --port 30002 +``` + +### Global multimodal embedding cache with Mooncake + +SGLang also supports a Mooncake-backed **global multimodal embedding cache** for EPD workloads. When enabled on encoder servers, repeated image inputs can reuse previously computed ViT embeddings across instances instead of running the vision encoder again. + +This feature is useful when: + +- the deployment serves repeated or overlapping image inputs, +- encoder compute is the bottleneck, and +- Mooncake is already available in the cluster. + +At a high level, the encoder checks whether the image embedding already exists in Mooncake. Cache hits are prefetched from the global store, while misses are encoded normally and inserted into the cache in the background. + +To enable it: + +- install and configure Mooncake in the same way as other SGLang Mooncake integrations, +- add `--enable-mm-global-cache` on the encoder server. + +`--enable-mm-global-cache` controls **whether multimodal embeddings are looked up and stored in the global Mooncake cache**. It is separate from `--encoder-transfer-backend`, which only controls encoder output transport. + +For Mooncake deployment and configuration details, see [HiCache best practices](./hicache_best_practices#deployment-with-mooncake) and the [Mooncake backend README](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/mem_cache/storage/mooncake_store/README.md). + +Example: + +```bash Command +# Shared Mooncake configuration +export MOONCAKE_TE_META_DATA_SERVER="http://127.0.0.1:8080/metadata" +export MOONCAKE_MASTER="127.0.0.1:50051" +export MOONCAKE_PROTOCOL="rdma" +export MOONCAKE_GLOBAL_SEGMENT_SIZE="4gb" + +# encoder with global multimodal cache enabled +python -m sglang.launch_server \ + --model-path Qwen/Qwen3-VL-8B-Instruct \ + --encoder-only \ + --enable-mm-global-cache \ + --port 30000 + +# language-only server +python -m sglang.launch_server \ + --model-path Qwen/Qwen3-VL-8B-Instruct \ + --language-only \ + --encoder-urls http://127.0.0.1:30000 \ + --port 30002 +``` + +Notes: + +- This cache is for **multimodal encoder embeddings**, not the language model KV cache. +- The feature currently uses Mooncake as the shared backing store. +- It can be enabled regardless of which `--encoder-transfer-backend` you use. +- It is most relevant for EPD or encoder-disaggregated VLM deployments where the same images are likely to appear across requests or instances. + #### Qwen VL - EP Disaggregation @@ -81,3 +156,42 @@ python -m sglang_router.launch_router \ --port 8000 ``` + +#### gRPC Encoder (EPD) + +You can run the encoder as a gRPC server while keeping prefill/decode as HTTP. +When using gRPC encoders, set `SGLANG_ENCODER_MM_RECEIVER_MODE=grpc` for the +prefill process so it uses the gRPC receiver. + +```bash Command +# gRPC encoder +python -m sglang.launch_server \ + --model-path Qwen/Qwen3-VL-8B-Instruct \ + --encoder-only \ + --grpc-mode \ + --encoder-transfer-backend zmq_to_scheduler \ + --port 30000 + +# prefill (HTTP) - tell it to use gRPC receiver +SGLANG_ENCODER_MM_RECEIVER_MODE=grpc \ +python -m sglang.launch_server \ + --model-path Qwen/Qwen3-VL-8B-Instruct \ + --disaggregation-mode prefill \ + --language-only \ + --encoder-urls grpc://127.0.0.1:30000 \ + --encoder-transfer-backend zmq_to_scheduler \ + --port 30002 + +# decode (HTTP) +python -m sglang.launch_server \ + --model-path Qwen/Qwen3-VL-8B-Instruct \ + --disaggregation-mode decode \ + --port 30003 + +# router +python -m sglang_router.launch_router \ + --pd-disaggregation \ + --prefill http://$PREFILL_HOST:30002 \ + --decode http://$DECODE_HOST:30003 \ + --port 8000 +``` diff --git a/docs_new/docs/advanced_features/expert_parallelism.mdx b/docs_new/docs/advanced_features/expert_parallelism.mdx index aa53d57d2..1e0e19655 100644 --- a/docs_new/docs/advanced_features/expert_parallelism.mdx +++ b/docs_new/docs/advanced_features/expert_parallelism.mdx @@ -42,6 +42,16 @@ SGLang's EP integrates diverse, highly efficient backends for different use case An extension of DeepEP for elastic inference, leveraging RDMA for high-performance data transfers. Elastic EP serving. + + nixl + NIXL-EP, an elastic EP communication library built on NVIDIA's NIXL framework with native RDMA and NVLink support. + Elastic EP serving with fault tolerance and dynamic scaling. + + + mori + MORI-EP, AMD's native all-to-all communication implementation optimized for ROCm. + AMD GPU deployments. + `flashinfer` Flashinfer implementation of all-to-all. @@ -55,9 +65,9 @@ SGLang's EP integrates diverse, highly efficient backends for different use case -DeepEP and Mooncake backends support two modes for token dispatch: `normal` mode (optimized for prefill workloads with high throughput) and `low_latency` mode (optimized for decode workloads with low latency and CUDA Graph compatibility). Users are recommended to set `--deepep-mode auto` to enable automatic dispatch mode switching during runtime. Setting `--deepep-mode normal` or `--deepep-mode low_latency` is useful for debugging or development purposes. +DeepEP and Mooncake backends support two modes for token dispatch: `normal` mode (optimized for prefill workloads with high throughput) and `low_latency` mode (optimized for decode workloads with low latency and CUDA Graph compatibility). MORI backend only supports `normal` mode now. NIXL-EP currently operates in low-latency mode with CUDA Graph support. Users are recommended to set `--deepep-mode auto` to enable automatic dispatch mode switching during runtime. Setting `--deepep-mode normal` or `--deepep-mode low_latency` is useful for debugging or development purposes. -Currently, DeepEP and Mooncake only support cases where `ep_size = tp_size`. For hybrid EP and TP (i.e., `ep_size < tp_size`), only the `none` backend (All-Reduce or All-Gather-based dispatching) is supported. +Currently, DeepEP, Mooncake, NIXL-EP, `ascend_fuseep` and MORI only support cases where `ep_size = tp_size`. For hybrid EP and TP (i.e., `ep_size < tp_size`), only the `none` backend (All-Reduce or All-Gather-based dispatching) is supported. ### Backends for MoE Computation @@ -82,7 +92,7 @@ Currently, DeepEP and Mooncake only support cases where `ep_size = tp_size`. For `triton` - Triton-based implementation for grouped GEMMs. To achieve higher performance, it's highly recommended to create [tuned configurations](https://github.com/sgl-project/sglang/blob/main/benchmark/kernels/fused_moe_triton/README). + Triton-based implementation for grouped GEMMs. To achieve higher performance, it's highly recommended to create tuned configurations. Custom kernel development or scenarios requiring high extensibility with Torch compilation support. @@ -100,6 +110,11 @@ Currently, DeepEP and Mooncake only support cases where `ep_size = tp_size`. For FlashInfer integrated with TensorRT-LLM for accelerated MoE computations, supporting FP4 communication operators and high-performance GEMMs. Blackwell with TRT-LLM. + + flashinfer_trtllm_routed + FlashInfer integrated with TensorRT-LLM for accelerated routed MoE computations, consuming SGLang-computed top-k expert assignments and weights. + Blackwell with TRT-LLM. + `flashinfer_cutlass` FlashInfer combined with CUTLASS for high-performance grouped GEMMs in MoE layers, handling FP4/FP8 quantization efficiently. @@ -242,7 +257,7 @@ For model like `nvidia/DeepSeek-R1-0528-NVFP4-v2`, the target model uses NVFP4 p ## Ascend NPU Guidance ### Guidance on SGLang configuration in Ascend NPU -- `--moe-a2a-backend` only supports deepep and ascend_fuseep backends, +- `--moe-a2a-backend` only supports `deepep` and `ascend_fuseep` backends, - `deepep`: The mechanism is consistent with the above description. @@ -252,12 +267,13 @@ For model like `nvidia/DeepSeek-R1-0528-NVFP4-v2`, the target model uses NVFP4 p - `--deepep-mode`: - - In PD mixed mode, please set `--deepep-mode` auto. + - In PD mixed mode, please set `--deepep-mode auto`. - - In PD Disaggregation Mode, prefill instance sets `--deepep-mode` normal, and decode instance sets `--deepep-mode` low_latency. + - In PD Disaggregation Mode, prefill instance sets `--deepep-mode normal`, and decode instance sets `--deepep-mode low_latency`. ### DeepEP Ascend Introduction -DeepEP Ascend is the adapted version of the DeepEP communication library for Huawei Ascend NPUs, specifically designed for Mixture-of-Experts (MoE) model Expert Parallelism (EP). It supports the Ant-moving Function (Split the sequence length into rounds for streaming batch transmission) to optimize the buffer size occupied during collective communication in prefill stage, especially for long sequences. +DeepEP Ascend is the adapted version of the DeepEP communication library for Huawei Ascend NPUs, specifically designed for Mixture-of-Experts (MoE) model Expert Parallelism (EP). +It supports the Ant-moving Function (Split the sequence length into rounds for streaming batch transmission) to optimize the buffer size occupied during collective communication in prefill stage, especially for long sequences. Ant-moving Function can be enabled for both the dispatch and combine phases via the following environment variables: diff --git a/docs_new/docs/advanced_features/hicache_best_practices.mdx b/docs_new/docs/advanced_features/hicache_best_practices.mdx index d86e676e2..df91d0aac 100644 --- a/docs_new/docs/advanced_features/hicache_best_practices.mdx +++ b/docs_new/docs/advanced_features/hicache_best_practices.mdx @@ -42,6 +42,23 @@ Notes: - `page_first`: Only compatible with `kernel` I/O backend, automatically switches to `layer_first` with `direct` backend - `page_first_direct`: Specifically designed for `direct` I/O backend with optimized memory organization +### Heterogeneous TP Support (GQA/MHA models) + +HiCache storage supports cross-cluster KV reuse when different deployments use different TP sizes (for example, `tp=4` and `tp=8`) and share the same storage backend namespace. + +Use `tp_lcm_size` in `--hicache-storage-backend-extra-config`: + +```bash Command +# Example: heterogeneous TP = {4, 8}, so lcm = 8 +--hicache-storage-backend-extra-config '{"tp_lcm_size": 8}' +``` + +Guidelines: + +- Set `tp_lcm_size` to the least common multiple (LCM) of all TP sizes that will share the same HiCache storage. +- For MHA models with Mooncake and `page_head` layout, HiCache will split head shards based on `tp_lcm_size` to make keys reusable across heterogeneous TP deployments. +- If all clusters use the same TP size, this option is not needed. + ### Prefetch Policies ```bash Command @@ -108,7 +125,7 @@ python3 -m sglang.launch_server \ ### Deployment with HF3FS -Here is an example of deploying DeepSeek-R1 with HiCache-HF3FS. For more details, see the [HF3FS Documentation](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/mem_cache/storage/hf3fs/docs). +Here is an example of deploying DeepSeek-R1 with HiCache-HF3FS. For more details, see the [HF3FS Documentation](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/mem_cache/storage/hf3fs/docs/README.md). ```bash Command python3 -m sglang.launch_server \ @@ -133,7 +150,7 @@ python3 -m sglang.launch_server \ ### Deployment with Mooncake -Here is an example of deploying Qwen3-235B-A22B-Instruct-2507 with Mooncake. For more details, see the [Mooncake Documentation](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/mem_cache/storage/mooncake_store). +Here is an example of deploying Qwen3-235B-A22B-Instruct-2507 with Mooncake. For more details, see the [Mooncake Documentation](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/mem_cache/storage/mooncake_store/README.md). ```bash Command # Set Mooncake environment variables diff --git a/docs_new/docs/advanced_features/hicache_storage_runtime_attach_detach.mdx b/docs_new/docs/advanced_features/hicache_storage_runtime_attach_detach.mdx index beb9aada8..b245bf520 100644 --- a/docs_new/docs/advanced_features/hicache_storage_runtime_attach_detach.mdx +++ b/docs_new/docs/advanced_features/hicache_storage_runtime_attach_detach.mdx @@ -21,8 +21,8 @@ The control path is: 1. **HTTP Server** (`python/sglang/srt/entrypoints/http_server.py`) - Exposes `PUT /hicache/storage-backend`, `DELETE /hicache/storage-backend`, `GET /hicache/storage-backend` -2. **TokenizerManager** (`python/sglang/srt/managers/tokenizer_communicator_mixin.py`) - - Sends the request to the Scheduler via `_Communicator` +2. **TokenizerManager** (`python/sglang/srt/managers/tokenizer_control_mixin.py`) + - Sends the request to the Scheduler via `FanOutCommunicator` 3. **Scheduler** (`python/sglang/srt/managers/scheduler.py`) - Performs a **strict idle check** - Calls `tree_cache.attach_storage_backend(...)` / `detach_storage_backend(...)` @@ -36,11 +36,11 @@ The control path is: *** ## 2. Idle-state requirement (strict) -The Scheduler uses a stricter `_is_idle_for_hicache_storage_op()`: +The Scheduler uses `is_fully_idle()` which checks: -- `_is_no_request()` is true (covers running/overlap/pp/disagg and other active states) -- `waiting_queue` is empty -- `grammar_queue` is empty (if the grammar backend is enabled) +- No running batches (including chunked prefill, overlap, pipeline-parallel, and disaggregation paths) +- No waiting requests in any queue (waiting, grammar, disagg bootstrap/prealloc/transfer/inflight) +- No DLLM staging requests If the condition is not met, attach/detach returns an error like: diff --git a/docs_new/docs/advanced_features/hisparse_guide.mdx b/docs_new/docs/advanced_features/hisparse_guide.mdx new file mode 100644 index 000000000..9ec2e082b --- /dev/null +++ b/docs_new/docs/advanced_features/hisparse_guide.mdx @@ -0,0 +1,187 @@ +--- +title: "HiSparse: Hierarchical Sparse Attention" +metatags: + description: "Use HiSparse hierarchical sparse attention to reduce decode GPU KV memory with CPU pinned host storage and PD disaggregation." +--- + +HiSparse reduces per-request GPU memory consumption during the decode phase by maintaining only a small "hot" KV buffer on GPU while keeping complete KV data in CPU pinned memory. Combined with PD disaggregation, it enables significantly higher decode concurrency. + +> **Prerequisites**: HiSparse only works with models that use **DeepSeek Sparse Attention (DSA)** architectures (e.g., DeepSeek-V3.2, GLM-5). These models natively select a subset of tokens for attention, making it possible to keep only the top-k KV on GPU while storing the full KV in host memory — without accuracy loss. Additionally, HiSparse currently requires **PD disaggregation mode** and is enabled on the **decode instance** only. + +## Why HiSparse? + +In long-context LLM inference, each decoding request holds a full-length KV cache on GPU, limiting the number of concurrent requests a decode instance can serve. HiSparse addresses this by: + +- **Reducing GPU memory per request**: Each request occupies only a fixed-size device buffer (e.g., 4KB tokens) instead of the full sequence length. +- **On-demand swap-in**: A CUDA kernel dynamically loads the top-k most relevant KV entries from host memory based on attention scores. +- **Transparent to prefill**: HiSparse is entirely a decode-side optimization; the prefill instance requires no changes. + +## Design Overview + +### Decode Workflow + +Each decode step follows this flow: + +1. **Forward decode** — generate the next token +2. **Top-k selection** — select the most relevant token positions via attention scores +3. **Swap-in** — the CUDA kernel loads top-k KV entries from host to device buffer: + - *Short sequences* (`seq_len ≤ device_buffer_size`): fast path, all KV already in buffer + - *Long sequences*: hit detection → LRU reordering → miss handling (host → device copy) +4. **Decode attention** — compute attention using the top-k device locations +5. **Eager backup** — asynchronously copy the previous token's KV from device to host + +### PD Disaggregation Integration (Direct-to-Host) + +In PD disaggregation mode, the prefill instance transfers KV cache directly into the decode instance's host pool via RDMA, bypassing the GPU entirely on the decode side. This eliminates the transient GPU memory spike during KV transfer and removes the staging DMA step. + +``` +Prefill GPU ──RDMA──▶ Decode Host Pool (CPU pinned memory) + │ + ▼ + alloc device buffer (4KB) + │ + ▼ + swap-in kernel (on-demand top-k) +``` + +## Server Arguments + + + + + + + + + + + + + + + + + + + + + + + + + + +
ArgumentType / DefaultDescription
--enable-hisparseflag; default: disabledEnable HiSparse on the decode instance
--hisparse-configJSON stringConfiguration for HiSparse (see below)
+ +### HiSparse Config Parameters + +Pass as a JSON string via `--hisparse-config`: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ParameterType / DefaultDescription
top_kintNumber of topk entries
device_buffer_sizeintNumber of token slots in the per-request GPU device buffer
host_to_device_ratiointRatio of logical pool size to device pool size, determining host memory capacity
+ +Example: `--hisparse-config='{"top_k": 2048, "device_buffer_size": 6144, "host_to_device_ratio": 10}'` + +## Deployment + +HiSparse currently requires **PD disaggregation mode** and is enabled only on the **decode instance**. + +### Prefill Instance + +```bash Command +python3 -m sglang.launch_server \ + --model-path /path/to/model \ + --trust-remote-code \ + --port 8000 --host 0.0.0.0 \ + --context-length 81920 \ + --chunked-prefill-size 65536 \ + --tp-size 8 --dp-size 8 --enable-dp-attention \ + --mem-fraction-static 0.85 \ + --disaggregation-mode prefill \ + --disaggregation-ib-device mlx5_0,mlx5_1,mlx5_2,mlx5_3 \ + --nnodes 1 --node-rank 0 +``` + +### Decode Instance (with HiSparse) + +```bash Command +python3 -m sglang.launch_server \ + --model-path /path/to/model \ + --trust-remote-code \ + --port 8000 --host 0.0.0.0 \ + --context-length 81920 \ + --tp-size 8 --dp-size 8 --enable-dp-attention \ + --mem-fraction-static 0.85 \ + --kv-cache-dtype bfloat16 \ + --nsa-decode-backend flashmla_sparse \ + --disaggregation-mode decode \ + --disaggregation-ib-device mlx5_0,mlx5_1,mlx5_2,mlx5_3 \ + --dist-init-addr 127.0.0.1:5757 \ + --nnodes 1 --node-rank 0 \ + --enable-hisparse \ + --hisparse-config='{"top_k": 2048, "device_buffer_size": 6144, "host_to_device_ratio": 10}' +``` + +### Benchmark + +```bash Command +python3 -m sglang.bench_serving \ + --backend sglang \ + --dataset-path /path/to/ShareGPT_V3_unfiltered_cleaned_split.json \ + --dataset-name random \ + --random-input 40000 \ + --random-output 20000 \ + --num-prompts 200 \ + --max-concurrency 200 \ + --request-rate 40 \ + --random-range-ratio 1.0 \ + --host 127.0.0.1 \ + --port 20000 \ + --model /path/to/model \ + --flush-cache \ +``` + +### Key Notes + +- The prefill instance does not need `--enable-hisparse`; it is unaware of HiSparse. +- On the decode instance, the following flags are **required** for HiSparse: + - `--kv-cache-dtype bfloat16` — currently only bfloat16 KV cache is supported (more dtypes planned). + - `--nsa-decode-backend flashmla_sparse` — currently only `flashmla_sparse` backend is supported. + - `--enable-hisparse` — enables HiSparse. + - `--hisparse-config` — HiSparse configuration (top_k, device_buffer_size, host_to_device_ratio). + - `host_to_device_ratio` should be configured based on the host machine's available memory. For example: + - **~1 TB** host memory → `host_to_device_ratio: 5` + - **~2 TB** host memory → `host_to_device_ratio: 10` + +## Acknowledgments + +We would like to thank the SGLang team and community for the implementation and generous support, especially Zhiqiang Xie, Zhangheng Huang, Tingwei Huang, Shangming Cai, Teng Ma, and many others. We also thank the Alibaba Cloud TairKVCache team and the AntGroup SCT Inference team for their valuable contributions. diff --git a/docs_new/docs/advanced_features/lora.ipynb b/docs_new/docs/advanced_features/lora.ipynb index 36fcf7aee..8e6e6d0a0 100644 --- a/docs_new/docs/advanced_features/lora.ipynb +++ b/docs_new/docs/advanced_features/lora.ipynb @@ -102,7 +102,7 @@ "\"\"\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=server_process)" ] }, { @@ -155,12 +155,12 @@ "python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", " --enable-lora \\\n", " --lora-paths lora0=algoprog/fact-generation-llama-3.1-8b-instruct-lora \\\n", - " lora1=Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16 \\\n", + " lora1=Nutanix/Meta-Llama-3.1-8B-Instruct_SFT_lora_4_alpha_16_humaneval_raw_json \\\n", " --max-loras-per-batch 2 \\\n", " --log-level warning \\\n", "\"\"\")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=server_process)" ] }, { @@ -218,7 +218,7 @@ "metadata": {}, "outputs": [], "source": [ - "lora0 = \"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16\" # rank - 4, target modules - q_proj, k_proj, v_proj, o_proj, gate_proj\n", + "lora0 = \"Nutanix/Meta-Llama-3.1-8B-Instruct_SFT_lora_4_alpha_16_humaneval_raw_json\" # rank - 4, target modules - q_proj, k_proj, v_proj, o_proj, gate_proj\n", "lora1 = \"algoprog/fact-generation-llama-3.1-8b-instruct-lora\" # rank - 64, target modules - q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj\n", "lora0_new = \"philschmid/code-llama-3-1-8b-text-to-sql-lora\" # rank - 256, target modules - q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj\n", "\n", @@ -236,7 +236,7 @@ " \"\"\")\n", "\n", "url = f\"http://127.0.0.1:{port}\"\n", - "wait_for_server(url)" + "wait_for_server(url, process=server_process)" ] }, { @@ -404,7 +404,7 @@ "source": [ "### OpenAI-compatible API usage\n", "\n", - "You can use LoRA adapters via the OpenAI-compatible APIs by specifying the adapter in the `model` field using the `base-model:adapter-name` syntax (for example, `qwen/qwen2.5-0.5b-instruct:adapter_a`). For more details and examples, see the “Using LoRA Adapters” section in the OpenAI API documentation: [openai_api_completions](../basic_usage/openai_api_completions).\n" + "You can use LoRA adapters via the OpenAI-compatible APIs by specifying the adapter in the `model` field using the `base-model:adapter-name` syntax (for example, `qwen/qwen2.5-0.5b-instruct:adapter_a`). For more details and examples, see the “Using LoRA Adapters” section in the OpenAI API documentation: [openai_api_completions.ipynb](../basic_usage/openai_api_completions.ipynb).\n" ] }, { @@ -439,7 +439,7 @@ " --max-lora-rank 256 \\\n", " --lora-target-modules all \\\n", " --lora-paths \\\n", - " {\"lora_name\":\"lora0\",\"lora_path\":\"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16\",\"pinned\":true} \\\n", + " {\"lora_name\":\"lora0\",\"lora_path\":\"Nutanix/Meta-Llama-3.1-8B-Instruct_SFT_lora_4_alpha_16_humaneval_raw_json\",\"pinned\":true} \\\n", " {\"lora_name\":\"lora1\",\"lora_path\":\"algoprog/fact-generation-llama-3.1-8b-instruct-lora\"} \\\n", " lora2=philschmid/code-llama-3-1-8b-text-to-sql-lora\n", " --log-level warning\n", @@ -447,7 +447,7 @@ "\n", "\n", "url = f\"http://127.0.0.1:{port}\"\n", - "wait_for_server(url)" + "wait_for_server(url, process=server_process)" ] }, { @@ -581,7 +581,7 @@ "metadata": {}, "outputs": [], "source": [ - "lora0 = \"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16\"\n", + "lora0 = \"Nutanix/Meta-Llama-3.1-8B-Instruct_SFT_lora_4_alpha_16_humaneval_raw_json\"\n", "lora1 = \"algoprog/fact-generation-llama-3.1-8b-instruct-lora\"\n", "lora2 = \"philschmid/code-llama-3-1-8b-text-to-sql-lora\"\n", "\n", @@ -591,7 +591,7 @@ " --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", " --enable-lora \\\n", " --enable-lora-overlap-loading \\\n", - " --lora-paths lora0=Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16 \\\n", + " --lora-paths lora0=Nutanix/Meta-Llama-3.1-8B-Instruct_SFT_lora_4_alpha_16_humaneval_raw_json \\\n", " lora1=algoprog/fact-generation-llama-3.1-8b-instruct-lora \\\n", " lora2=philschmid/code-llama-3-1-8b-text-to-sql-lora \\\n", " --max-lora-rank 256 \\\n", @@ -600,7 +600,7 @@ " \"\"\")\n", "\n", "url = f\"http://127.0.0.1:{port}\"\n", - "wait_for_server(url)" + "wait_for_server(url, process=server_process)" ] }, { diff --git a/docs_new/docs/advanced_features/lora.mdx b/docs_new/docs/advanced_features/lora.mdx index be2ad9d87..7e0f5d95b 100644 --- a/docs_new/docs/advanced_features/lora.mdx +++ b/docs_new/docs/advanced_features/lora.mdx @@ -43,7 +43,7 @@ From client side, the user needs to provide a list of strings as input batch, an **Note:** SGLang supports LoRA adapters through two APIs: -1. **OpenAI-Compatible API** (`/v1/chat/completions`, `/v1/completions`): Use the `model:adapter-name` syntax. See [OpenAI API with LoRA](../basic_usage/openai_api_completions.ipynb#Using-LoRA-Adapters) for examples. +1. **OpenAI-Compatible API** (`/v1/chat/completions`, `/v1/completions`): Use the `model:adapter-name` syntax. See [OpenAI API with LoRA](../basic_usage/openai_api_completions#using-lora-adapters) for examples. 2. **Native API** (`/generate`): Pass `lora_path` in the request body (shown below). @@ -108,7 +108,7 @@ server_process, port = launch_server_cmd( python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ --enable-lora \ --lora-paths lora0=algoprog/fact-generation-llama-3.1-8b-instruct-lora \ - lora1=Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16 \ + lora1=Nutanix/Meta-Llama-3.1-8B-Instruct_SFT_lora_4_alpha_16_humaneval_raw_json \ --max-loras-per-batch 2 \ --log-level warning \ """ @@ -152,7 +152,7 @@ When using dynamic LoRA loading, it's recommended to explicitly specify both `-- ```python Example -lora0 = "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" # rank - 4, target modules - q_proj, k_proj, v_proj, o_proj, gate_proj +lora0 = "Nutanix/Meta-Llama-3.1-8B-Instruct_SFT_lora_4_alpha_16_humaneval_raw_json" # rank - 4, target modules - q_proj, k_proj, v_proj, o_proj, gate_proj lora1 = "algoprog/fact-generation-llama-3.1-8b-instruct-lora" # rank - 64, target modules - q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj lora0_new = "philschmid/code-llama-3-1-8b-text-to-sql-lora" # rank - 256, target modules - q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj @@ -317,7 +317,7 @@ server_process, port = launch_server_cmd( --max-lora-rank 256 \ --lora-target-modules all \ --lora-paths \ - {"lora_name":"lora0","lora_path":"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16","pinned":true} \ + {"lora_name":"lora0","lora_path":"Nutanix/Meta-Llama-3.1-8B-Instruct_SFT_lora_4_alpha_16_humaneval_raw_json","pinned":true} \ {"lora_name":"lora1","lora_path":"algoprog/fact-generation-llama-3.1-8b-instruct-lora"} \ lora2=philschmid/code-llama-3-1-8b-text-to-sql-lora --log-level warning @@ -418,7 +418,7 @@ By using the `--enable-lora-overlap-loading` server argument, the SGLang engine ```python Example -lora0 = "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" +lora0 = "Nutanix/Meta-Llama-3.1-8B-Instruct_SFT_lora_4_alpha_16_humaneval_raw_json" lora1 = "algoprog/fact-generation-llama-3.1-8b-instruct-lora" lora2 = "philschmid/code-llama-3-1-8b-text-to-sql-lora" @@ -429,7 +429,7 @@ server_process, port = launch_server_cmd( --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ --enable-lora \ --enable-lora-overlap-loading \ - --lora-paths lora0=Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16 \ + --lora-paths lora0=Nutanix/Meta-Llama-3.1-8B-Instruct_SFT_lora_4_alpha_16_humaneval_raw_json \ lora1=algoprog/fact-generation-llama-3.1-8b-instruct-lora \ lora2=philschmid/code-llama-3-1-8b-text-to-sql-lora \ --max-lora-rank 256 \ diff --git a/docs_new/docs/advanced_features/pd_disaggregation.mdx b/docs_new/docs/advanced_features/pd_disaggregation.mdx index fa796a4fb..86f4bf025 100644 --- a/docs_new/docs/advanced_features/pd_disaggregation.mdx +++ b/docs_new/docs/advanced_features/pd_disaggregation.mdx @@ -26,7 +26,7 @@ When you need to profile prefill or decode workers in PD disaggregation mode, pl ## Router Integration -For deploying PD disaggregation at scale with load balancing and fault tolerance, SGLang provides a router. The router can distribute requests between prefill and decode instances using various routing policies. For detailed information on setting up routing with PD disaggregation, including configuration options and deployment patterns, see the [SGLang Model Gateway (former Router)](../advanced_features/sgl_model_gateway.md#prefill-decode-disaggregation). +For deploying PD disaggregation at scale with load balancing and fault tolerance, SGLang provides a router. The router can distribute requests between prefill and decode instances using various routing policies. For detailed information on setting up routing with PD disaggregation, including configuration options and deployment patterns, see the [SGLang Model Gateway (former Router)](./sgl_model_gateway#prefill-decode-disaggregation). ## Mooncake @@ -132,11 +132,13 @@ PD Disaggregation with Mooncake supports the following environment variables for #### NVLink Transport Configuration To enable NVLink transport for KV cache transfers with the mooncake backend (recommended for NVL72 deployments), set the following environment variables. Note that auxiliary data transfer will still use TCP as a temporary workaround. -```bash -export SGLANG_MOONCAKE_CUSTOM_MEM_POOL=True +```bash Command +export SGLANG_MOONCAKE_CUSTOM_MEM_POOL=NVLINK export MC_FORCE_MNNVL=True ``` +The `SGLANG_MOONCAKE_CUSTOM_MEM_POOL` environment variable enables the custom memory pool. Supported values are `NVLINK` (or `True`), `BAREX`, and `INTRA_NODE_NVLINK`. + #### Prefill Server Configuration @@ -155,11 +157,11 @@ export MC_FORCE_MNNVL=True - + - + @@ -167,7 +169,12 @@ export MC_FORCE_MNNVL=True - + + + + + +
**`SGLANG_DISAGGREGATION_THREAD_POOL_SIZE`** Controls the total number of worker threads for KVCache transfer operations per TP rankA dynamic value calculated by `int(0.75 * os.cpu_count()) // 8)`, which is limited to be larger than 4 and less than 12 to ensure efficiency and prevent thread race conditionsA dynamic value calculated by int(0.75 * os.cpu_count()) // 8), which is limited to be larger than 4 and less than 12 to ensure efficiency and prevent thread race conditions
**`SGLANG_DISAGGREGATION_QUEUE_SIZE`**Sets the number of parallel transfer queues. KVCache transfer requests from multiple decode instances will be sharded into these queues so that they can share the threads and the transfer bandwidth at the same time. If it is set to `1`, then we transfer requests one by one according to fcfs strategySets the number of parallel transfer queues. KVCache transfer requests from multiple decode instances will be sharded into these queues so that they can share the threads and the transfer bandwidth at the same time. If it is set to 1, then we transfer requests one by one according to fcfs strategy `4`
Timeout (seconds) for receiving destination KV indices during request initialization `300`
SGLANG_DISAGGREGATION_BOOTSTRAP_ENTRY_CLEANUP_INTERVALInterval (seconds) between cleanups of bootstrap entries120
If a greater mean TTFT is acceptable, you can `export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600` (10 minutes) to relax the timeout condition. @@ -209,6 +216,84 @@ Please be aware that this setting will cause prefill instances to take a longer If a greater mean TTFT is acceptable, you can `export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=600` (10 minutes) to relax the timeout condition. +## Heterogeneous TP with GPU Staging Buffer + +When prefill and decode use different tensor parallelism (TP) sizes (e.g., prefill TP=4, decode DP attention with TP=1), the KV cache memory layout differs between the two sides. The **GPU staging buffer** solves this by gathering KV head slices into a contiguous buffer on the prefill side, performing bulk RDMA transfer, then scattering into the correct KV cache pages on the decode side. This provides **2–5x throughput improvement** over the default per-token slice approach at high concurrency and matches homogeneous TP baselines within ~5%. + +Enable the staging buffer when prefill and decode use **different TP sizes** with the **Mooncake** transfer backend. When both sides use the same TP size, staging is automatically bypassed even if enabled. + +> **Note:** The staging buffer is designed for non-MLA models (e.g. GQA, MHA). MLA models (e.g. DeepSeek-V2/V3) should not enable this flag. + +### Environment Variables + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
VariableDescriptionDefault
SGLANG_DISAGG_STAGING_BUFFEREnable GPU staging buffer for heterogeneous TP KV transferFalse
SGLANG_DISAGG_STAGING_BUFFER_SIZE_MBPrefill-side per-worker staging buffer size in MB64
SGLANG_DISAGG_STAGING_POOL_SIZE_MBDecode-side ring buffer pool total size in MB4096
+ +### Usage Example + +```bash Command +# Set staging buffer environment variables on BOTH prefill and decode +export SGLANG_DISAGG_STAGING_BUFFER=1 +export SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB=64 +export SGLANG_DISAGG_STAGING_POOL_SIZE_MB=4096 + +# Prefill with TP=4 +python -m sglang.launch_server \ + --model-path $MODEL_PATH \ + --disaggregation-mode prefill \ + --port 30000 \ + --tp 4 \ + --trust-remote-code \ + --disaggregation-ib-device mlx5_1,mlx5_2 + +# Decode with TP=1 (or DP attention with effective attention TP=1) +python -m sglang.launch_server \ + --model-path $MODEL_PATH \ + --disaggregation-mode decode \ + --port 30001 \ + --tp 4 \ + --dp 4 \ + --enable-dp-attention \ + --trust-remote-code \ + --disaggregation-ib-device mlx5_3,mlx5_4 + +# Router +python -m sglang_router.launch_router \ + --pd-disaggregation \ + --prefill http://127.0.0.1:30000 \ + --decode http://127.0.0.1:30001 \ + --host 0.0.0.0 --port 8000 +``` + ## NIXL ### Requirements @@ -343,8 +428,8 @@ python -m sglang.launch_server \ Use ascend backend with [memfabric_hybrid](https://gitcode.com/Ascend/memfabric_hybrid) and ASCEND_MF_STORE_URL being set -```bash -pip install memfabric-hybrid==1.0.5 +```bash Command +pip install memfabric-hybrid==1.0.0 export ASCEND_MF_STORE_URL="tcp://xxx.xx.xxx.xxx:xxxx" ``` Use mooncake backend, more details can be found in mooncake section. diff --git a/docs_new/docs/advanced_features/quantization.mdx b/docs_new/docs/advanced_features/quantization.mdx index 3d26fc51e..e13ca084e 100644 --- a/docs_new/docs/advanced_features/quantization.mdx +++ b/docs_new/docs/advanced_features/quantization.mdx @@ -20,11 +20,276 @@ or [NeuralMagic](https://huggingface.co/collections/neuralmagic) collections on popular quality validated quantized models. Quantized models must be validated via benchmarks post-quantization to guard against abnormal quantization loss regressions. +## Platform Compatibility + +The following table summarizes quantization method support across NVIDIA and AMD GPUs, Ascend NPUs. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MethodNVIDIA GPUsAMD GPUs (MI300X/MI325X/MI350X)Ascend NPUs (A2/A3)Notes
fp8YesYesWIPAiter or Triton backend on AMD
mxfp4YesYesWIPRequires CDNA3/CDNA4 with MXFP support; uses Aiter
blockwise_int8YesYesNoTriton-based, works on both platforms
w8a8_int8YesYesNo
w8a8_fp8YesYesNoAiter or Triton FP8 on AMD
awqYesYesYesUses Triton dequantize on AMD (vs. optimized CUDA kernels on NVIDIA). Uses CANN kernels on Ascend
gptqYesYesYesUses Triton or vLLM kernels on AMD. Uses CANN kernels on Ascend
compressed-tensorsYesYesPartialAiter paths for FP8/MoE on AMD. Uses CANN kernels on Ascend, FP8 not supported yet
quarkYesYesNoAMD Quark quantization; Aiter GEMM paths on AMD
auto-roundYesYesPartialPlatform-agnostic (Intel auto-round). Uses CANN kernels on Ascend
quark_int4fp8_moeNoYesNoAMD-only; online INT4-to-FP8 MoE quantization (CDNA3/CDNA4)
awq_marlinYesNoNoMarlin kernels are CUDA-only
gptq_marlinYesNoNoMarlin kernels are CUDA-only
ggufYesNoWIPCUDA-only kernels in sgl-kernel
modelopt / modelopt_fp8Yes (Hopper/SM90+)NoNoNVIDIA ModelOpt; requires NVIDIA hardware
modelopt_fp4Yes (Blackwell/SM100+)NoNoNVIDIA ModelOpt; native FP4 on Blackwell (B200, GB200)
petit_nvfp4NoYes (MI250/MI300X/MI325X)NoEnables NVFP4 on ROCm via Petit; use modelopt_fp4 on NVIDIA Blackwell. Auto-selected when loading NVFP4 models on AMD. See LMSYS blog and AMD ROCm blog.
bitsandbytesYesExperimentalNoDepends on bitsandbytes ROCm support
torchao (int4wo, etc.)YesPartialNoint4wo not supported on AMD; other methods may work
modelslimNoNoYesAscend quantization; Uses CANN kernels
+ +On AMD, several of these methods use [Aiter](https://github.com/ROCm/aiter) for acceleration -- set `SGLANG_USE_AITER=1` where noted. See [AMD GPU setup](../hardware-platforms/amd_gpu) for installation and configuration details. + +On Ascend, various layers quantization configurations are supported, see [Ascend NPU quantization](../hardware-platforms/ascend-npus/ascend_npu_quantization) for details. + +## GEMM Backends for FP4/FP8 Quantization + + +Backend selection is supported only for **blockwise FP8** and **NVFP4** GEMM. When running FP8 or FP4 quantized models, you can select the GEMM backend via `--fp8-gemm-backend` and `--fp4-gemm-backend`. + + +### `--fp8-gemm-backend` (Blockwise FP8 GEMM) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
BackendHardwareDescription
autoAllAuto-selects based on hardware
deep_gemmSM90, SM100JIT-compiled; enabled when DeepGEMM is installed
flashinfer_trtllmSM100FlashInfer TensorRT-LLM backend; optimal for low-latency
flashinfer_cutlassSM100/120FlashInfer CUTLASS groupwise FP8 GEMM
flashinfer_deepgemmSM90Uses swapAB optimization for small M dimensions in decoding
cutlassSM90, SM100/120sgl-kernel CUTLASS
tritonAllFallback; widely compatible
aiterROCmAMD AITER backend
+ +**`auto` selection order:** 1) DeepGEMM (SM90/SM100, installed); 2) FlashInfer TRTLLM (SM100, FlashInfer available); 3) CUTLASS (SM90/SM100/120); 4) AITER (AMD); 5) Triton. **Exception:** SM120 always resolves to Triton. + +### `--fp4-gemm-backend` (NVFP4 GEMM) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
BackendHardwareDescription
autoSM100/120Auto-selects: flashinfer_cudnn on SM120; flashinfer_cutlass on SM100
cutlassSM100/120SGLang CUTLASS kernel
flashinfer_cutlassSM100/120FlashInfer CUTLASS backend
flashinfer_cudnnSM100/120 (CUDA 13+, cuDNN 9.15+)FlashInfer cuDNN backend; used on SM120 for performance
flashinfer_trtllmSM100FlashInfer TensorRT-LLM backend
+ +When FlashInfer is unavailable for NVFP4, the SGLang CUTLASS kernel is used as an automatic fallback. + ## Offline Quantization To load already quantized models, simply load the model weights and config. **Again, if the model has been quantized offline, there's no need to add `--quantization` argument when starting the engine. The quantization method will be parsed from the -downloaded Hugging Face config. For example, DeepSeek V3/R1 models are already in FP8, so do not add redundant parameters.** +downloaded Hugging Face or msModelSlim config. For example, DeepSeek V3/R1 models are already in FP8, so do not add redundant parameters.** ```bash Command python3 -m sglang.launch_server \ @@ -194,23 +459,85 @@ python3 -m sglang.launch_server \ #### Using [NVIDIA ModelOpt](https://github.com/NVIDIA/Model-Optimizer) -NVIDIA Model Optimizer (ModelOpt) provides advanced quantization techniques optimized for NVIDIA hardware. SGLang includes a streamlined workflow for quantizing models with ModelOpt and automatically exporting them for deployment. +NVIDIA Model Optimizer (ModelOpt) provides advanced quantization techniques optimized for NVIDIA hardware. + +**Offline vs. Online Quantization:** + +SGLang supports two modes for ModelOpt. + +* **Offline Quantization (pre-quantized):** + * **Usage:** Download a pre-quantized model from Hugging Face or run `hf_ptq.py` once to create a new quantized checkpoint. Then load this quantized checkpoint. + * **Pros:** Fast server startup, quantization can be validated before deployment, efficient resource usage. + * **Cons:** Requires an extra preparation step. + +* **Online Quantization (quant and serve):** + * **Usage:** Load a standard BF16/FP16 model and add a flag. The engine applies quantization *on startup*. + * **Pros:** Convenient (no new checkpoint needed). + * **Cons:** **High startup time**, increases VRAM usage during initialization (risk of OOM). + +The following sections guide you through using the Offline path: loading pre-quantized models or creating your own checkpoints. + +##### Using Pre-Quantized Checkpoints + +If a model is already quantized (e.g., from Hugging Face), you can load it directly. + +* **FP8 Models:** + Use `--quantization modelopt_fp8`. + ```bash Command + python3 -m sglang.launch_server \ + --model-path nvidia/Llama-3.1-8B-Instruct-FP8 \ + --quantization modelopt_fp8 \ + --port 30000 + ``` + +* **FP4 Models:** + Use `--quantization modelopt_fp4`. + ```bash Command + python3 -m sglang.launch_server \ + --model-path nvidia/Llama-3.3-70B-Instruct-NVFP4 \ + --quantization modelopt_fp4 \ + --port 30000 + ``` + +##### Creating Your Own Quantized Checkpoints + +If a pre-quantized checkpoint is not available for your model, you can create one using NVIDIA Model Optimizer's `hf_ptq.py` script. + +**Why quantize?** +- Reduce VRAM usage +- Higher throughput and lower latency +- More flexible deployment (on smaller GPUs) + +**What can be quantized?** +- The entire model +- MLP layers only +- KV cache + +**Key options in `hf_ptq.py`:** + +`--qformat`: Quantization formats `fp8`, `nvfp4`, `nvfp4_mlp_only` + +`--kv_cache_qformat`: KV cache quantization format (default: `fp8`) + +**Note:** The default `kv_cache_qformat` may not be optimal for all use cases. Consider setting this explicitly. + +**Hardware requirements:** Hopper and higher are recommended. Insufficient GPU memory may cause weight offloading, resulting in extremely long quantization time. + +For detailed usage and supported model architectures, see [NVIDIA Model Optimizer LLM PTQ](https://github.com/NVIDIA/Model-Optimizer/tree/main/examples/llm_ptq). + +SGLang includes a streamlined workflow for quantizing models with ModelOpt and automatically exporting them for deployment. ##### Installation -First, install ModelOpt. You can either install it directly or as an optional SGLang dependency: +First, install ModelOpt: ```bash Command -# Option 1: Install ModelOpt directly pip install nvidia-modelopt - -# Option 2: Install SGLang with ModelOpt support (recommended) -pip install sglang[modelopt] ``` ##### Quantization and Export Workflow -SGLang provides an example script that demonstrates the complete ModelOpt quantization and export workflow: +SGLang provides an example script that demonstrates the complete ModelOpt quantization and export workflow. Run from the SGLang repository root (see [modelopt_quantize_and_export.py](https://github.com/sgl-project/sglang/blob/main/examples/usage/modelopt_quantize_and_export.py)): ```bash Command # Quantize and export a model using ModelOpt FP8 quantization @@ -219,7 +546,7 @@ python examples/usage/modelopt_quantize_and_export.py quantize \ --export-dir ./quantized_tinyllama_fp8 \ --quantization-method modelopt_fp8 -# For FP4 quantization +# For FP4 quantization (requires Blackwell GPU) python examples/usage/modelopt_quantize_and_export.py quantize \ --model-path TinyLlama/TinyLlama-1.1B-Chat-v1.0 \ --export-dir ./quantized_tinyllama_fp4 \ @@ -275,25 +602,39 @@ python -m sglang.launch_server \ --port 30000 --host 0.0.0.0 ``` -Or using the Python API: +Or using the Python API (use the same path as `modelopt_export_path` from the quantize step): ```python Example import sglang as sgl -# Deploy exported ModelOpt quantized model -llm = sgl.Engine( - model_path="./quantized_tinyllama_fp8", - quantization="modelopt" -) +def main(): + # Deploy exported ModelOpt quantized model + # Path must match modelopt_export_path from quantize step (e.g., ./exported_model) + llm = sgl.Engine( + model_path="./exported_model", + quantization="modelopt", + ) -# Run inference -prompts = ["Hello, how are you?", "What is the capital of France?"] -sampling_params = {"temperature": 0.8, "top_p": 0.95, "max_new_tokens": 100} -outputs = llm.generate(prompts, sampling_params) + # Run inference + prompts = [ + "Hello, how are you?", + "What is the capital of France?", + ] + sampling_params = { + "temperature": 0.8, + "top_p": 0.95, + "max_new_tokens": 100, + } + + outputs = llm.generate(prompts, sampling_params) + + for i, output in enumerate(outputs): + print(f"Prompt: {prompts[i]}") + print(f"Output: {output['text']}") + +if __name__ == "__main__": + main() -for i, output in enumerate(outputs): - print(f"Prompt: {prompts[i]}") - print(f"Output: {output.outputs[0].text}") ``` ##### Advanced Features @@ -311,7 +652,7 @@ python examples/usage/modelopt_quantize_and_export.py quantize \ # The checkpoint can be reused for future quantization runs and skip calibration ``` -**Export-only Workflow**: If you have a pre-existing fake quantized ModelOpt checkpoint, you can export it directly: +**Export-only Workflow**: If you have a pre-existing fake quantized ModelOpt checkpoint, you can export it directly. See [LoadConfig](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/configs/load_config.py) for the full API: ```python Example from sglang.srt.configs.device_config import DeviceConfig @@ -330,7 +671,7 @@ load_config = LoadConfig( modelopt_export_path="./exported_model", ) -# Load and export the model +# Load and export the model (DeviceConfig defaults to device="cuda") model_loader = get_model_loader(load_config, model_config) model_loader.load_model(model_config=model_config, device_config=DeviceConfig()) ``` @@ -343,6 +684,74 @@ model_loader.load_model(model_config=model_config, device_config=DeviceConfig()) - **Calibration-based**: Uses calibration datasets for optimal quantization quality - **Production Ready**: Enterprise-grade quantization with NVIDIA support +#### Using [ModelSlim](https://gitcode.com/Ascend/msmodelslim) +MindStudio-ModelSlim (msModelSlim) is a model offline quantization compression tool launched by MindStudio and optimized for Ascend hardware. + +- **Installation** + + ```bash Command + # Clone repo and install msmodelslim: + git clone https://gitcode.com/Ascend/msmodelslim.git + cd msmodelslim + bash install.sh + ``` + +- **LLM quantization** + + Download the original floating-point weights of the large model. Taking Qwen3-32B as an example, you can go to [Qwen3-32B](https://huggingface.co/Qwen/Qwen3-32B) to obtain the original model weights. Then install other dependencies (related to the model, refer to the huggingface model card). + > Note: You can find pre-quantized validated models on [modelscope/Eco-Tech](https://modelscope.cn/models/Eco-Tech). + + _Traditional quantification methods require the preparation of calibration data files (```.jsonl``` formats) for calibration in the quantification process._ + ```bash Command + Qwen3-32B/ # floating-point model downloaded from official HF (or modelscope) repo + msmodelslim/ # msmodelslim repo + |----- lab_calib # calibration date folder (put your dataset here in ```.jsonl``` format or use pre-prepared ones) + |----- some file (such as laos_calib.jsonl) + |----- lab_practice # best practice folder with configs for quantization + |----- model folder (such as qwen3_5_moe folder) # folder with quantization configs + |----- quant_config (such as qwen3_5_moe_w8a8.yaml) # quantization config + |----- another folders + output_folder/ # generated by below command + |----- quant_model_weights-00001-of-0001.safetensors # quantized weights + |----- quant_model_description.json # file with description of the quantization methods for each layer (```W4A4_DYNAMIC```, etc.) + |----- another files (such as config.json, tokenizer.json, etc.) + ``` + Run quantization using one-click quantization (recommended): + ```bash Command + msmodelslim quant \ + --model_path ${MODEL_PATH} \ + --save_path ${SAVE_PATH} \ + --device npu:0,1 \ + --model_type Qwen3-32B \ + --quant_type w8a8 \ + --trust_remote_code True + ``` + +- **Usage Example** + ```bash Command + python3 -m sglang.launch_server \ + --model-path $PWD/Qwen3-32B-w8a8 \ + --port 30000 --host 0.0.0.0 + ``` + +- **Available Quantization Methods**: + - [x] ```W4A4_DYNAMIC``` linear with online quantization of activations + - [x] ```W8A8``` linear with offline quantization of activations + - [x] ```W8A8_DYNAMIC``` linear with online quantization of activations + - [x] ```W4A4_DYNAMIC``` MOE with online quantization of activations + - [x] ```W4A8_DYNAMIC``` MOE with online quantization of activations + - [x] ```W8A8_DYNAMIC``` MOE with online quantization of activations + - [ ] ```W4A8``` linear TBD + - [ ] ```W4A16``` linear TBD + - [ ] ```W48A16``` linear TBD + - [ ] ```W4A16``` MoE in progress + - [ ] ```W8A16``` MoE in progress + - [ ] ```KV Cache``` in progress + - [ ] ```Attention``` in progress + + +For more detailed examples of quantization of models, as well as information about their support, see the [examples](https://gitcode.com/Ascend/msmodelslim/blob/master/example/README.md) section in ModelSLim repo. + ## Online Quantization To enable online quantization, you can simply specify `--quantization` in the command line. For example, you can launch the server with the following command to enable `FP8` quantization for model `meta-llama/Meta-Llama-3.1-8B-Instruct`: @@ -381,7 +790,7 @@ python3 -m sglang.launch_server \ ### `quark_int4fp8_moe` online quantization method -SGLang running on AMD GPUs (CDNA3 or CDNA4 architecture) supports the quantization method `--quantization quark_int4fp8_moe`, that will replace [MoE layers](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/layers/moe/fused_moe_triton/layer.py) originally in high precision (bfloat16, float16 or float32) to use weights dynamically quantized to int4, that are upcasted to float8 during inference to run compute in float8 precision with activations dynamically quantized on the fly to float8. +SGLang running on AMD GPUs (CDNA3 or CDNA4 architecture) supports the quantization method `--quantization quark_int4fp8_moe`, that will replace [MoE layers](https://github.com/sgl-project/sglang/blob/v0.4.8/python/sglang/srt/layers/moe/fused_moe_triton/layer.py#L271) originally in high precision (bfloat16, float16 or float32) to use weights dynamically quantized to int4, that are upcasted to float8 during inference to run compute in float8 precision with activations dynamically quantized on the fly to float8. Other layers (e.g. projections in the attention layers) have their weights quantized online to float8 directly. @@ -390,6 +799,9 @@ Other layers (e.g. projections in the attention layers) have their weights quant - [GPTQModel](https://github.com/ModelCloud/GPTQModel) - [LLM Compressor](https://github.com/vllm-project/llm-compressor/) - [NVIDIA Model Optimizer (ModelOpt)](https://github.com/NVIDIA/Model-Optimizer) +- [NVIDIA Model Optimizer LLM PTQ](https://github.com/NVIDIA/Model-Optimizer/tree/main/examples/llm_ptq) +- [Petit: NVFP4 on ROCm](https://github.com/causalflow-ai/petit-kernel) — [LMSYS blog](https://lmsys.org/blog/2025-09-21-petit-amdgpu/), [AMD ROCm blog](https://rocm.blogs.amd.com/artificial-intelligence/fp4-mixed-precision/README.html) - [Torchao: PyTorch Architecture Optimization](https://github.com/pytorch/ao) - [vLLM Quantization](https://docs.vllm.ai/en/latest/quantization/) - [auto-round](https://github.com/intel/auto-round) +- [ModelSlim](https://gitcode.com/Ascend/msmodelslim) diff --git a/docs_new/docs/advanced_features/rfork.mdx b/docs_new/docs/advanced_features/rfork.mdx index d14109bc2..84bfaa8f8 100644 --- a/docs_new/docs/advanced_features/rfork.mdx +++ b/docs_new/docs/advanced_features/rfork.mdx @@ -5,7 +5,7 @@ metatags: --- R-Fork (Tensor Remote Fork) is a novel weight loading methodology that leverages efficient inter-node GPU-to-GPU data transfer path to load tensors from a running SGLang instance to a new instance with zero-copy. It can significantly optimize the SGLang instance boot-up time by reducing model weights loading from several minutes to mere seconds. -To learn more details about R-Fork, please check **[R-Fork blog](https://lmsys.org/blog/2025-12-10-rfork/)** +To learn more details about R-Fork, please check ** R-Fork blog ** ## Usage @@ -27,25 +27,29 @@ To learn more details about R-Fork, please check **[R-Fork blog](https://lmsys.o remote-instance-weight-loader-backend - `nccl` or `transfer_engine`, default value is `nccl` + nccl, transfer_engine, or modelexpress. Default is nccl. remote-instance-weight-loader-seed-instance-ip - IP address of the seed instance who will provide the model weight + IP address of the seed instance who will provide the model weight. Used by nccl and transfer_engine backends. remote-instance-weight-loader-seed-instance-service-port - the port that the seed instance's HTTP server is listening on + the port that the seed instance's HTTP server is listening on. Used by nccl and transfer_engine backends. remote-instance-weight-loader-send-weights-group-ports - the list of available ports on the seed instance that will be used to build NCCL communication groups between seed and client instance. This argument is only needed by `nccl` backend. + the list of available ports on the seed instance that will be used to build NCCL communication groups between seed and client instance. Only needed by nccl backend. remote-instance-weight-loader-start-seed-via-transfer-engine - set to start seed service that supports TransferEngine as backend. It is needed for seed instances when using `transfer_engine` as backend. + set to start seed service that supports TransferEngine as backend. Needed for seed instances when using transfer_engine as backend. - + + modelexpress-config + JSON config for modelexpress backend. Keys: "url" (required, gRPC host:port of ModelExpress server), "model_name" (optional, defaults to --model-path), "source" (optional bool, true for seed mode). + + ### NCCL as backend @@ -80,3 +84,25 @@ python -m sglang.launch_server [args] \ --remote-instance-weight-loader-seed-instance-service-port [seed_instance_service_port] \ --remote-instance-weight-loader-backend transfer_engine ``` + +### ModelExpress as backend + +[ModelExpress](https://github.com/ai-dynamo/modelexpress) is a coordination service that manages P2P weight transfer metadata. It removes the need for direct seed IP/port configuration by providing a centralized registry that seeds publish to and clients discover from. Under the hood it uses TransferEngine (Mooncake) for the actual RDMA data transfer. + +A running ModelExpress server is required. See the [ModelExpress documentation](https://github.com/ai-dynamo/modelexpress) for setup instructions. + +seed instance: +```bash Command +python -m sglang.launch_server [args] \ + --modelexpress-config '{"url": "[modelexpress_grpc_host:port]", "model_name": "[model_name]", "source": true}' +``` + +client instance: +```bash Command +python -m sglang.launch_server [args] \ + --load-format remote_instance \ + --remote-instance-weight-loader-backend modelexpress \ + --modelexpress-config '{"url": "[modelexpress_grpc_host:port]", "model_name": "[model_name]"}' +``` + +The seed publishes its TransferEngine session ID and tensor layout to ModelExpress. The client queries ModelExpress to discover the seed, then pulls weights directly via RDMA. This enables dynamic seed discovery without hardcoding IPs, and supports multiple models through a single ModelExpress instance. diff --git a/docs_new/docs/advanced_features/separate_reasoning.ipynb b/docs_new/docs/advanced_features/separate_reasoning.ipynb index fde97d8a6..6277dd8bd 100644 --- a/docs_new/docs/advanced_features/separate_reasoning.ipynb +++ b/docs_new/docs/advanced_features/separate_reasoning.ipynb @@ -70,7 +70,7 @@ " \"python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-R1-Distill-Qwen-7B --host 0.0.0.0 --reasoning-parser deepseek-r1 --log-level warning\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=server_process)" ] }, { diff --git a/docs_new/docs/advanced_features/server_arguments.mdx b/docs_new/docs/advanced_features/server_arguments.mdx index e183be1ce..71a7c1ac9 100644 --- a/docs_new/docs/advanced_features/server_arguments.mdx +++ b/docs_new/docs/advanced_features/server_arguments.mdx @@ -53,15 +53,12 @@ You can find all arguments by `python3 -m sglang.launch_server --help` ```bash Command python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3-8B-Instruct --chunked-prefill-size 4096 ``` - -- To enable `torch.compile` acceleration, add `--enable-torch-compile`. It accelerates small models on small batch sizes. By default, the cache path is located at `/tmp/torchinductor_root`, you can customize it using environment variable `TORCHINDUCTOR_CACHE_DIR`. For more details, please refer to [PyTorch official documentation](https://pytorch.org/tutorials/recipes/torch_compile_caching_tutorial.html) and [Enabling cache for torch.compile](../references/torch_compile_cache). -- To enable torchao quantization, add `--torchao-config int4wo-128`. It supports other [quantization strategies (INT8/FP8)](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/server_args.py) as well. - To enable fp8 weight quantization, add `--quantization fp8` on a fp16 checkpoint or directly load a fp8 checkpoint without specifying any arguments. -- To enable fp8 kv cache quantization, add `--kv-cache-dtype fp8_e5m2`. -- To enable deterministic inference and batch invariant operations, add `--enable-deterministic-inference`. More details can be found in [deterministic inference document](../advanced_features/deterministic_inference). +- To enable fp8 kv cache quantization, add `--kv-cache-dtype fp8_e4m3` or `--kv-cache-dtype fp8_e5m2`. +- To enable deterministic inference and batch invariant operations, add `--enable-deterministic-inference`. More details can be found in [deterministic inference document](./deterministic_inference). - If the model does not have a chat template in the Hugging Face tokenizer, you can specify a [custom chat template](../references/custom_chat_template). If the tokenizer has multiple named templates (e.g., 'default', 'tool_use'), you can select one using `--hf-chat-template-name tool_use`. - To run tensor parallelism on multiple nodes, add `--nnodes 2`. If you have two nodes with two GPUs on each node and want to run TP=4, let `sgl-dev-0` be the hostname of the first node and `50000` be an available port, you can use the following commands. If you meet deadlock, please try to add `--disable-cuda-graph` - +- (Note: This feature is out of maintenance and might cause error) To enable `torch.compile` acceleration, add `--enable-torch-compile`. It accelerates small models on small batch sizes. By default, the cache path is located at `/tmp/torchinductor_root`, you can customize it using environment variable `TORCHINDUCTOR_CACHE_DIR`. For more details, please refer to [PyTorch official documentation](https://pytorch.org/tutorials/recipes/torch_compile_caching_tutorial.html) and [Enabling cache for torch.compile](../references/torch_compile_cache). ```bash Command # Node 0 python -m sglang.launch_server \ @@ -115,7 +112,7 @@ Please consult the documentation below and [server_args.py](https://github.com/s ` --tokenizer-mode` Tokenizer mode. 'auto' will use the fast tokenizer if available, and 'slow' will always use the slow tokenizer. ` auto` - ` auto`, `slow` + auto, slow ` --tokenizer-worker-num` @@ -131,9 +128,9 @@ Please consult the documentation below and [server_args.py](https://github.com/s ` --load-format` - The format of the model weights to load. "auto" will try to load the weights in the safetensors format and fall back to the pytorch bin format if safetensors format is not available. "pt" will load the weights in the pytorch bin format. "safetensors" will load the weights in the safetensors format. "npcache" will load the weights in pytorch format and store a numpy cache to speed up the loading. "dummy" will initialize the weights with random values, which is mainly for profiling. "gguf" will load the weights in the gguf format. "bitsandbytes" will load the weights using bitsandbytes quantization. "layered" loads weights layer by layer so that one can quantize a layer before loading another to make the peak memory envelope smaller. "flash_rl" will load the weights in flash_rl format. "fastsafetensors" and "private" are also supported. + The format of the model weights to load. "auto" will try to load the weights in the safetensors format and fall back to the pytorch bin format if safetensors format is not available. "pt" will load the weights in the pytorch bin format. "safetensors" will load the weights in the safetensors format. "npcache" will load the weights in pytorch format and store a numpy cache to speed up the loading. "dummy" will initialize the weights with random values, which is mainly for profiling."gguf" will load the weights in the gguf format. "bitsandbytes" will load the weights using bitsandbytes quantization."layered" loads weights layer by layer so that one can quantize a layer before loading another to make the peak memory envelope smaller. "flash_rl" will load the weights in flash_rl format. "fastsafetensors" and "private" are also supported. "runai_streamer" enables direct model loading from object storage and shared file systems. ` auto` - ` auto`, `pt`, `safetensors`, `npcache`, `dummy`, `sharded_state`, `gguf`, `bitsandbytes`, `layered`, `flash_rl`, `remote`, `remote_instance`, `fastsafetensors`, `private` + auto, pt, safetensors, npcache, dummy, sharded_state, gguf, bitsandbytes, layered, flash_rl, remote, remote_instance, fastsafetensors, private, runai_streamer ` --model-loader-extra-config` @@ -269,13 +266,13 @@ Please consult the documentation below and [server_args.py](https://github.com/s `--dtype` Data type for model weights and activations. * "auto" will use FP16 precision for FP32 and FP16 models, and BF16 precision for BF16 models. * "half" for FP16. Recommended for AWQ quantization. * "float16" is the same as "half". * "bfloat16" for a balance between precision and range. * "float" is shorthand for FP32 precision. * "float32" for FP32 precision. `auto` - `auto`, `half`, `float16`, `bfloat16`, `float`, `float32` + auto, half, float16, bfloat16, float, float32 `--quantization` The quantization method. `None` - `awq`, `fp8`, `gptq`, `marlin`, `gptq_marlin`, `awq_marlin`, `bitsandbytes`, `gguf`, `modelopt`, `modelopt_fp8`, `modelopt_fp4`, `petit_nvfp4`, `w8a8_int8`, `w8a8_fp8`, `moe_wna16`, `qoq`, `w4afp8`, `mxfp4`, `auto-round`, `compressed-tensors`, `modelslim`, `quark_int4fp8_moe` + awq, fp8, gptq, marlin, gptq_marlin, awq_marlin, bitsandbytes, gguf, modelopt, modelopt_fp8, modelopt_fp4, petit_nvfp4, w8a8_int8, w8a8_fp8, moe_wna16, qoq, w4afp8, mxfp4, mxfp8, auto-round, compressed-tensors, modelslim, quark_int4fp8_moe `--quantization-param-path` @@ -287,7 +284,7 @@ Please consult the documentation below and [server_args.py](https://github.com/s `--kv-cache-dtype` Data type for kv cache storage. "auto" will use model data type. "bf16" or "bfloat16" for BF16 KV cache. "fp8_e5m2" and "fp8_e4m3" are supported for CUDA 11.8+. "fp4_e2m1" (only mxfp4) is supported for CUDA 12.8+ and PyTorch 2.8.0+ `auto` - `auto`, `fp8_e5m2`, `fp8_e4m3`, `bf16`, `bfloat16`, `fp4_e2m1` + auto, fp8_e5m2, fp8_e4m3, bf16, bfloat16, fp4_e2m1 `--enable-fp32-lm-head` @@ -528,102 +525,114 @@ Please consult the documentation below and [server_args.py](https://github.com/s Type: int - `--pp-max-micro-batch-size` + --attention-context-parallel-size<br>--attn-cp-size + The attention context parallelism size. + 1 + Type: int + + + --moe-data-parallel-size<br>--moe-dp-size + The moe data parallelism size. + 1 + Type: int + + + --pp-max-micro-batch-size The maximum micro batch size in pipeline parallelism. - `None` + None Type: int - `--pp-async-batch-depth` + --pp-async-batch-depth The async batch depth of pipeline parallelism. - `0` + 0 Type: int - `--stream-interval` - The interval (or buffer size) for streaming in terms of the token length. A smaller value makes streaming smoother, while a larger value makes the throughput higher. - `1` + --stream-interval + The interval (or buffer size) for streaming in terms of the token length. A smaller value makes streaming smoother, while a larger value makes the throughput higher + 1 Type: int - `--stream-output` + --incremental-streaming-output Whether to output as a sequence of disjoint segments. - `False` + False bool flag (set to enable) - `--random-seed` + --random-seed The random seed. - `None` + None Type: int - `--constrained-json-whitespace-pattern` - {"(outlines and llguidance backends only) Regex pattern for syntactic whitespaces allowed in JSON constrained output. For example, to allow the model to generate consecutive whitespaces, set the pattern to [\\n\\t ]*"} - `None` + --constrained-json-whitespace-pattern + (outlines and llguidance backends only) Regex pattern for syntactic whitespaces allowed in JSON constrained output. For example, to allow the model to generate consecutive whitespaces, set the pattern to [\n\t ]* + None Type: str - `--constrained-json-disable-any-whitespace` - {"(xgrammar and llguidance backends only) Enforce compact representation in JSON constrained output."} - `False` + --constrained-json-disable-any-whitespace + (xgrammar and llguidance backends only) Enforce compact representation in JSON constrained output. + False bool flag (set to enable) - `--watchdog-timeout` + --watchdog-timeout Set watchdog timeout in seconds. If a forward batch takes longer than this, the server will crash to prevent hanging. - `300` + 300 Type: float - `--soft-watchdog-timeout` + --soft-watchdog-timeout Set soft watchdog timeout in seconds. If a forward batch takes longer than this, the server will dump information for debugging. `None` Type: float - `--dist-timeout` + --dist-timeout Set timeout for torch.distributed initialization. `None` Type: int - `--download-dir` + --download-dir Model download directory for huggingface. - `None` + None Type: str - `--model-checksum` + --model-checksum Model file integrity verification. If provided without value, uses model-path as HF repo ID. Otherwise, provide checksums JSON file path or HuggingFace repo ID. - `None` + None Type: str - `--base-gpu-id` + --base-gpu-id The base GPU ID to start allocating GPUs from. Useful when running multiple instances on the same machine. - `0` + 0 Type: int - `--gpu-id-step` + --gpu-id-step The delta between consecutive GPU IDs that are used. For example, setting it to 2 will use GPU 0,2,4,... - `1` + 1 Type: int - - `--sleep-on-idle` + + --sleep-on-idle Reduce CPU usage when sglang is idle. - `False` + False bool flag (set to enable) - + `--custom-sigquit-handler` Register a custom sigquit handler so you can do additional cleanup after the server is shutdown. This is only available for Engine, not for CLI. `None` Type: str - + ## Logging @@ -665,13 +674,13 @@ Please consult the documentation below and [server_args.py](https://github.com/s `--log-requests-level` 0: Log metadata (no sampling parameters). 1: Log metadata and sampling parameters. 2: Log metadata, sampling parameters and partial input/output. 3: Log every input/output. `2` - `0`, `1`, `2`, `3` + 0, 1, 2, 3 `--log-requests-format` Format for request logging: 'text' (human-readable) or 'json' (structured) `text` - `text`, `json` + text, json `--log-requests-target` @@ -704,96 +713,102 @@ Please consult the documentation below and [server_args.py](https://github.com/s bool flag (set to enable) - `--enable-metrics-for-all-schedulers` - Enable --enable-metrics-for-all-schedulers when you want schedulers on all TP ranks (not just TP 0) to record request metrics separately. This is especially useful when dp_attention is enabled, as otherwise all metrics appear to come from TP 0. + --enable-mfu-metrics + Enable estimated MFU-related prometheus metrics. `False` bool flag (set to enable) - `--tokenizer-metrics-custom-labels-header` + --enable-metrics-for-all-schedulers + Enable --enable-metrics-for-all-schedulers when you want schedulers on all TP ranks (not just TP 0) to record request metrics separately. This is especially useful when dp_attention is enabled, as otherwise all metrics appear to come from TP 0. + False + bool flag (set to enable) + + + --tokenizer-metrics-custom-labels-header Specify the HTTP header for passing custom labels for tokenizer metrics. - `x-custom-labels` + x-custom-labels Type: str - `--tokenizer-metrics-allowed-custom-labels` + --tokenizer-metrics-allowed-custom-labels The custom labels allowed for tokenizer metrics. The labels are specified via a dict in '--tokenizer-metrics-custom-labels-header' field in HTTP requests, e.g., {'label1': 'value1', 'label2': 'value2'} is allowed if '--tokenizer-metrics-allowed-custom-labels label1 label2' is set. `None` List[str] - `--bucket-time-to-first-token` + --bucket-time-to-first-token The buckets of time to first token, specified as a list of floats. `None` List[float] - `--bucket-inter-token-latency` + --bucket-inter-token-latency The buckets of inter-token latency, specified as a list of floats. `None` List[float] - `--bucket-e2e-request-latency` + --bucket-e2e-request-latency The buckets of end-to-end request latency, specified as a list of floats. - `None` + None List[float] - `--collect-tokens-histogram` + --collect-tokens-histogram Collect prompt/generation tokens histogram. - `False` + False bool flag (set to enable) - `--prompt-tokens-buckets` + --prompt-tokens-buckets The buckets rule of prompt tokens. Supports 3 rule types: 'default' uses predefined buckets; 'tse <middle> <base> <count>' generates two sides exponential distributed buckets (e.g., 'tse 1000 2 8' generates buckets [984.0, 992.0, 996.0, 998.0, 1000.0, 1002.0, 1004.0, 1008.0, 1016.0]).); 'custom <value1> <value2> ...' uses custom bucket values (e.g., 'custom 10 50 100 500'). `None` List[str] - `--generation-tokens-buckets` + --generation-tokens-buckets The buckets rule for generation tokens histogram. Supports 3 rule types: 'default' uses predefined buckets; 'tse <middle> <base> <count>' generates two sides exponential distributed buckets (e.g., 'tse 1000 2 8' generates buckets [984.0, 992.0, 996.0, 998.0, 1000.0, 1002.0, 1004.0, 1008.0, 1016.0]).); 'custom <value1> <value2> ...' uses custom bucket values (e.g., 'custom 10 50 100 500'). - `None` + None List[str] - `--gc-warning-threshold-secs` + --gc-warning-threshold-secs The threshold for long GC warning. If a GC takes longer than this, a warning will be logged. Set to 0 to disable. - `0.0` + 0.0 Type: float - `--decode-log-interval` + --decode-log-interval The log interval of decode batch. - `40` + 40 Type: int - `--enable-request-time-stats-logging` + --enable-request-time-stats-logging Enable per request time stats logging - `False` + False bool flag (set to enable) - `--kv-events-config` + --kv-events-config Config in json format for NVIDIA dynamo KV event publishing. Publishing will be enabled if this flag is used. - `None` + None Type: str - `--enable-trace` + --enable-trace Enable opentelemetry trace - `False` + False bool flag (set to enable) - + `--otlp-traces-endpoint` Config opentelemetry collector endpoint if --enable-trace is set. format: <ip>:<port> `localhost:4317` Type: str - + ## RequestMetricsExporter configuration @@ -853,7 +868,7 @@ Please consult the documentation below and [server_args.py](https://github.com/s `--admin-api-key` - Set **admin API key** for administrative/control endpoints (e.g., weights update, cache flush, `/get_server_info`). Endpoints marked as admin-only require `Authorization: Bearer ` when this is set. + Set admin API key for administrative/control endpoints (e.g., weights update, cache flush, /server_info). Endpoints marked as admin-only require Authorization: Bearer <admin_api_key> when this is set. `None` Type: str @@ -871,7 +886,7 @@ Please consult the documentation below and [server_args.py](https://github.com/s `--chat-template` - The buliltin chat template name or the path of the chat template file. This is only used for OpenAI-compatible API server. + The builtin chat template name or the path of the chat template file. This is only used for OpenAI-compatible API server. `None` Type: str @@ -883,7 +898,7 @@ Please consult the documentation below and [server_args.py](https://github.com/s `--completion-template` - The buliltin completion template name or the path of the completion template file. This is only used for OpenAI-compatible API server. only for code completion currently. + The builtin completion template name or the path of the completion template file. This is only used for OpenAI-compatible API server. only for code completion currently. `None` Type: str @@ -903,13 +918,13 @@ Please consult the documentation below and [server_args.py](https://github.com/s `--reasoning-parser` Specify the parser for reasoning models. Supported parsers: [deepseek-r1, deepseek-v3, glm45, gpt-oss, kimi, qwen3, qwen3-thinking, step3]. `None` - `deepseek-r1`, `deepseek-v3`, `glm45`, `gpt-oss`, `kimi`, `qwen3`, `qwen3-thinking`, `step3` + deepseek-r1, deepseek-v3, glm45, gpt-oss, kimi, qwen3, qwen3-thinking, step3 `--tool-call-parser` Specify the parser for handling tool-call interactions. Supported parsers: [deepseekv3, deepseekv31, glm, glm45, glm47, gpt-oss, kimi_k2, llama3, mistral, pythonic, qwen, qwen25, qwen3_coder, step3]. `None` - `deepseekv3`, `deepseekv31`, `glm`, `glm45`, `glm47`, `gpt-oss`, `kimi_k2`, `llama3`, `mistral`, `pythonic`, `qwen`, `qwen25`, `qwen3_coder`, `step3` + deepseekv3, deepseekv31, glm, glm45, glm47, gpt-oss, kimi_k2, llama3, mistral, pythonic, qwen, qwen25, qwen3_coder, step3, gigachat3 `--tool-server` @@ -921,7 +936,7 @@ Please consult the documentation below and [server_args.py](https://github.com/s `--sampling-defaults` Where to get default sampling parameters. 'openai' uses SGLang/OpenAI defaults (temperature=1.0, top_p=1.0, etc.). 'model' uses the model's generation_config.json to get the recommended sampling parameters if available. Default is 'model'. `model` - `openai`, `model` + openai, model @@ -1059,19 +1074,19 @@ Please consult the documentation below and [server_args.py](https://github.com/s `--max-lora-rank` - The maximum LoRA rank that should be supported. If not specified, it will be automatically inferred from the adapters provided in `--lora-paths`. This argument is needed when you expect to dynamically load adapters of larger LoRA rank after server startup. + The maximum LoRA rank that should be supported. If not specified, it will be automatically inferred from the adapters provided in --lora-paths. This argument is needed when you expect to dynamically load adapters of larger LoRA rank after server startup. `None` Type: int `--lora-target-modules` - The union set of all target modules where LoRA should be applied (e.g., `q_proj`, `k_proj`, `gate_proj`). If not specified, it will be automatically inferred from the adapters provided in `--lora-paths`. You can also set it to `all` to enable LoRA for all supported modules; note this may introduce minor performance overhead. + The union set of all target modules where LoRA should be applied (e.g., q_proj, k_proj, gate_proj). If not specified, it will be automatically inferred from the adapters provided in --lora-paths. You can also set it to all to enable LoRA for all supported modules; note this may introduce minor performance overhead. `None` - {"q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj, qkv_proj, gate_up_proj, all"} + q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj, qkv_proj, gate_up_proj, all `--lora-paths` - {"The list of LoRA adapters to load. Each adapter must be specified in one of the following formats: | = | JSON with schema {\"lora_name\": str, \"lora_path\": str, \"pinned\": bool}."} + The list of LoRA adapters to load. Each adapter must be specified in one of the following formats: <PATH> \| <NAME>=<PATH> \| JSON with schema {"lora_name": str, "lora_path": str, "pinned": bool}. `None` Type: List[str] / JSON objects @@ -1083,7 +1098,7 @@ Please consult the documentation below and [server_args.py](https://github.com/s `--max-loaded-loras` - If specified, limits the maximum number of LoRA adapters loaded in CPU memory at a time. Must be ≥ `--max-loras-per-batch`. + If specified, limits the maximum number of LoRA adapters loaded in CPU memory at a time. Must be ≥ --max-loras-per-batch. `None` Type: int @@ -1091,19 +1106,19 @@ Please consult the documentation below and [server_args.py](https://github.com/s `--lora-eviction-policy` LoRA adapter eviction policy when the GPU memory pool is full. `lru` - lru, fifo + lru, fifo `--lora-backend` Choose the kernel backend for multi-LoRA serving. `csgmv` - triton, csgmv, ascend, torch_native + triton, csgmv, ascend, torch_native `--max-lora-chunk-size` - Maximum chunk size for the ChunkedSGMV LoRA backend. Only used when `--lora-backend` is `csgmv`. Larger values may improve performance. + Maximum chunk size for the ChunkedSGMV LoRA backend. Only used when --lora-backend is csgmv. Larger values may improve performance. `16` - 16, 32, 64, 128 + 16, 32, 64, 128 @@ -1129,61 +1144,61 @@ Please consult the documentation below and [server_args.py](https://github.com/s `--attention-backend` Choose the kernels for attention layers. `None` - `triton`, `torch_native`, `flex_attention`, `nsa`, `cutlass_mla`, `fa3`, `fa4`, `flashinfer`, `flashmla`, `trtllm_mla`, `trtllm_mha`, `dual_chunk_flash_attn`, `aiter`, `wave`, `intel_amx`, `ascend` + triton, torch_native, flex_attention, nsa, cutlass_mla, fa3, fa4, flashinfer, flashmla, trtllm_mla, trtllm_mha, dual_chunk_flash_attn, aiter, wave, intel_amx, ascend `--prefill-attention-backend` Choose the kernels for prefill attention layers (have priority over --attention-backend). `None` - `triton`, `torch_native`, `flex_attention`, `nsa`, `cutlass_mla`, `fa3`, `fa4`, `flashinfer`, `flashmla`, `trtllm_mla`, `trtllm_mha`, `dual_chunk_flash_attn`, `aiter`, `wave`, `intel_amx`, `ascend` + triton, torch_native, flex_attention, nsa, cutlass_mla, fa3, fa4, flashinfer, flashmla, trtllm_mla, trtllm_mha, dual_chunk_flash_attn, aiter, wave, intel_amx, ascend `--decode-attention-backend` Choose the kernels for decode attention layers (have priority over --attention-backend). `None` - `triton`, `torch_native`, `flex_attention`, `nsa`, `cutlass_mla`, `fa3`, `fa4`, `flashinfer`, `flashmla`, `trtllm_mla`, `trtllm_mha`, `dual_chunk_flash_attn`, `aiter`, `wave`, `intel_amx`, `ascend` + triton, torch_native, flex_attention, nsa, cutlass_mla, fa3, fa4, flashinfer, flashmla, trtllm_mla, trtllm_mha, dual_chunk_flash_attn, aiter, wave, intel_amx, ascend `--sampling-backend` Choose the kernels for sampling layers. `None` - `flashinfer`, `pytorch`, `ascend` + flashinfer, pytorch, ascend `--grammar-backend` Choose the backend for grammar-guided decoding. `None` - `xgrammar`, `outlines`, `llguidance`, `none` + xgrammar, outlines, llguidance, none `--mm-attention-backend` Set multimodal attention backend. `None` - `sdpa`, `fa3`, `fa4`, `triton_attn`, `ascend_attn`, `aiter_attn` + sdpa, fa3, fa4, triton_attn, ascend_attn, aiter_attn `--nsa-prefill-backend` Choose the NSA backend for the prefill stage (overrides `--attention-backend` when running DeepSeek NSA-style attention). `flashmla_sparse` - `flashmla_sparse`, `flashmla_kv`, `flashmla_auto`, `fa3`, `tilelang`, `aiter` + flashmla_sparse, flashmla_kv, flashmla_auto, fa3, tilelang, aiter, trtllm `--nsa-decode-backend` Choose the NSA backend for the decode stage when running DeepSeek NSA-style attention. Overrides `--attention-backend` for decoding. `fa3` - `flashmla_sparse`, `flashmla_kv`, `fa3`, `tilelang`, `aiter` + flashmla_sparse, flashmla_kv, fa3, tilelang, aiter, trtllm `--fp8-gemm-backend` - Choose the runner backend for Blockwise FP8 GEMM operations. Options: 'auto' (default, auto-selects based on hardware), 'deep_gemm' (JIT-compiled; enabled by default on NVIDIA Hopper (SM90) and Blackwell (SM100) when DeepGEMM is installed), 'flashinfer_trtllm' (optimal for Blackwell and low-latency), 'cutlass' (optimal for Hopper/Blackwell GPUs and high-throughput), 'triton' (fallback, widely compatible), 'aiter' (ROCm only). **NOTE**: This replaces the deprecated environment variables SGLANG_ENABLE_FLASHINFER_FP8_GEMM and SGLANG_SUPPORT_CUTLASS_BLOCK_FP8. + Choose the runner backend for Blockwise FP8 GEMM operations. Options: 'auto' (default, auto-selects based on hardware), 'deep_gemm' (JIT-compiled; enabled by default on NVIDIA Hopper (SM90) and Blackwell (SM100) when DeepGEMM is installed), 'flashinfer_trtllm' (FlashInfer TRTLLM backend; SM100/SM103 only), 'flashinfer_cutlass' (FlashInfer CUTLASS backend, SM120 only), 'flashinfer_deepgemm' (Hopper SM90 only, uses swapAB optimization for small M dimensions in decoding), 'cutlass' (optimal for Hopper/Blackwell GPUs and high-throughput), 'triton' (fallback, widely compatible), 'aiter' (ROCm only). `auto` - `auto`, `deep_gemm`, `flashinfer_trtllm`, `cutlass`, `triton`, `aiter` + auto, deep_gemm, flashinfer_trtllm, flashinfer_cutlass, flashinfer_deepgemm, cutlass, triton, aiter `--fp4-gemm-backend` - Choose the runner backend for NVFP4 GEMM operations. Options: 'auto' (default, auto-selects between flashinfer_cudnn/flashinfer_cutlass based on CUDA/cuDNN version), 'flashinfer_cudnn' (FlashInfer cuDNN backend, optimal on CUDA 13+ with cuDNN 9.15+), 'flashinfer_cutlass' (FlashInfer CUTLASS backend, optimal on CUDA 12), 'flashinfer_trtllm' (FlashInfer TensorRT-LLM backend, requires different weight preparation with shuffling). All backends are from FlashInfer; when FlashInfer is unavailable, sgl-kernel CUTLASS is used as an automatic fallback. **NOTE**: This replaces the deprecated environment variable SGLANG_FLASHINFER_FP4_GEMM_BACKEND. - `auto` - `auto`, `flashinfer_cudnn`, `flashinfer_cutlass`, `flashinfer_trtllm` + Choose the runner backend for NVFP4 GEMM operations. Options: 'flashinfer_cutlass' (default), 'auto' (auto-selects between flashinfer_cudnn/flashinfer_cutlass based on CUDA/cuDNN version), 'flashinfer_cudnn' (FlashInfer cuDNN backend, optimal on CUDA 13+ with cuDNN 9.15+), 'flashinfer_trtllm' (FlashInfer TensorRT-LLM backend, requires different weight preparation with shuffling). All backends are from FlashInfer; when FlashInfer is unavailable, sgl-kernel CUTLASS is used as an automatic fallback. + flashinfer_cutlass + auto, flashinfer_cudnn, flashinfer_cutlass, flashinfer_trtllm `--disable-flashinfer-autotune` @@ -1322,45 +1337,33 @@ Please consult the documentation below and [server_args.py](https://github.com/s - `--speculative-ngram-min-match-window-size` - The minimum window size for pattern matching in ngram speculative decoding. - `1` - Type: int - - - `--speculative-ngram-max-match-window-size` - The maximum window size for pattern matching in ngram speculative decoding. - `12` - Type: int - - - `--speculative-ngram-min-bfs-breadth` + --speculative-ngram-min-bfs-breadth The minimum breadth for BFS (Breadth-First Search) in ngram speculative decoding. `1` Type: int - `--speculative-ngram-max-bfs-breadth` + --speculative-ngram-max-bfs-breadth The maximum breadth for BFS (Breadth-First Search) in ngram speculative decoding. - `10` + 10 Type: int - `--speculative-ngram-match-type` - The match type for cache tree. - `BFS` - `BFS`, `PROB` + --speculative-ngram-match-type + Ngram tree-building mode. BFS selects recency-based expansion and PROB selects frequency-based expansion. This setting is forwarded to the ngram cache implementation. + BFS + BFS, PROB - `--speculative-ngram-branch-length` - The branch length for ngram speculative decoding. - `18` + --speculative-ngram-max-trie-depth + Maximum suffix length stored and matched by the ngram trie. + 18 Type: int - `--speculative-ngram-capacity` + --speculative-ngram-capacity The cache capacity for ngram speculative decoding. - `10000000` + 10000000 Type: int @@ -1419,19 +1422,19 @@ Please consult the documentation below and [server_args.py](https://github.com/s ` --moe-a2a-backend` Select the backend for all-to-all communication for expert parallelism. ` none` - ` none`, `deepep`, `mooncake`, `ascend_fuseep` + none, deepep, mooncake, mori, nixl, ascend_fuseep ` --moe-runner-backend` Choose the runner backend for MoE. ` auto` - ` auto`, `deep_gemm`, `triton`, `triton_kernel`, `flashinfer_trtllm`, `flashinfer_cutlass`, `flashinfer_mxfp4`, `flashinfer_cutedsl`, `cutlass` + auto, deep_gemm, triton, triton_kernel, flashinfer_trtllm, flashinfer_trtllm_routed, flashinfer_cutlass, flashinfer_mxfp4, flashinfer_cutedsl, cutlass ` --flashinfer-mxfp4-moe-precision` - Choose the computation precision of flashinfer mxfp4 moe. + Choose the computation precision of flashinfer mxfp4 moe ` default` - ` default`, `bf16` + default, bf16 ` --enable-flashinfer-allreduce-fusion` @@ -1440,102 +1443,114 @@ Please consult the documentation below and [server_args.py](https://github.com/s bool flag (set to enable) - ` --deepep-mode` - Select the mode when enable DeepEP MoE, could be `normal`, `low_latency` or `auto`. Default is `auto`, which means `low_latency` for decode batch and `normal` for prefill batch. - ` auto` - ` normal`, `low_latency`, `auto` - - - ` --ep-num-redundant-experts` - Allocate this number of redundant experts in expert parallel. - ` 0` - Type: int - - - ` --ep-dispatch-algorithm` - The algorithm to choose ranks for redundant experts in expert parallel. - ` None` - Type: str - - - ` --init-expert-location` - Initial location of EP experts. - ` trivial` - Type: str - - - ` --enable-eplb` - Enable EPLB algorithm. - ` False` + --enable-aiter-allreduce-fusion + Enable aiter allreduce fusion with Residual RMSNorm. + False bool flag (set to enable) - ` --eplb-algorithm` - Chosen EPLB algorithm. - ` auto` + --deepep-mode + Select the mode when enable DeepEP MoE, could be normal, low_latency or auto. Default is auto, which means low_latency for decode batch and normal for prefill batch. + auto + normal, low_latency, auto + + + --ep-num-redundant-experts + Allocate this number of redundant experts in expert parallel. + 0 + Type: int + + + --ep-dispatch-algorithm + The algorithm to choose ranks for redundant experts in expert parallel. + None Type: str - ` --eplb-rebalance-num-iterations` + --init-expert-location + Initial location of EP experts. + trivial + Type: str + + + --enable-eplb + Enable EPLB algorithm + False + bool flag (set to enable) + + + --eplb-algorithm + Chosen EPLB algorithm + auto + Type: str + + + --eplb-rebalance-num-iterations Number of iterations to automatically trigger a EPLB re-balance. - ` 1000` + 1000 Type: int - ` --eplb-rebalance-layers-per-chunk` + --eplb-rebalance-layers-per-chunk Number of layers to rebalance per forward pass. - ` None` + None Type: int - ` --eplb-min-rebalancing-utilization-threshold` + --eplb-min-rebalancing-utilization-threshold Minimum threshold for GPU average utilization to trigger EPLB rebalancing. Must be in the range [0.0, 1.0]. - ` 1.0` + 1.0 Type: float - ` --expert-distribution-recorder-mode` + --expert-distribution-recorder-mode Mode of expert distribution recorder. ` None` Type: str - ` --expert-distribution-recorder-buffer-size` + --expert-distribution-recorder-buffer-size Circular buffer size of expert distribution recorder. Set to -1 to denote infinite buffer. - ` None` + None Type: int - ` --enable-expert-distribution-metrics` - Enable logging metrics for expert balancedness. - ` False` + --enable-expert-distribution-metrics + Enable logging metrics for expert balancedness + False bool flag (set to enable) - ` --deepep-config` + --deepep-config Tuned DeepEP config suitable for your own cluster. It can be either a string with JSON content or a file path. ` None` Type: str - ` --moe-dense-tp-size` + --moe-dense-tp-size TP size for MoE dense MLP layers. This flag is useful when, with large TP size, there are errors caused by weights in MLP layers having dimension smaller than the min dimension GEMM supports. - ` None` + ` none` Type: int - ` --elastic-ep-backend` + --elastic-ep-backend Specify the collective communication backend for elastic EP. Currently supports 'mooncake'. - ` none` - ` none`, `mooncake` + ` None` + none, mooncake - + + --enable-elastic-expert-backup + Enable elastic EP backend to backup expert weights in DRAM feature. Currently supports 'mooncake'. + False + bool flag (set to enable) + + ` --mooncake-ib-device` The InfiniBand devices for Mooncake Backend transfer, accepts multiple comma-separated devices (e.g., --mooncake-ib-device mlx5_0,mlx5_1). Default is None, which triggers automatic device detection when Mooncake Backend is enabled. ` None` Type: str - + ## Mamba Cache @@ -1565,7 +1580,7 @@ Please consult the documentation below and [server_args.py](https://github.com/s `--mamba-ssm-dtype` The data type of the SSM states in mamba cache. `float32` - `float32`, `bfloat16` + float32, bfloat16, float16 `--mamba-full-memory-ratio` @@ -1575,13 +1590,13 @@ Please consult the documentation below and [server_args.py](https://github.com/s `--mamba-scheduler-strategy` - The strategy to use for mamba scheduler. `auto` currently defaults to `no_buffer`. 1. `no_buffer` does not support overlap scheduler due to not allocating extra mamba state buffers. Branching point caching support is feasible but not implemented. 2. `extra_buffer` supports overlap schedule by allocating extra mamba state buffers to track mamba state for caching (mamba state usage per running req becomes `2x` for non-spec; `1+(1/(2+speculative_num_draft_tokens))x` for spec dec (e.g. 1.16x if speculative_num_draft_tokens==4)). 2a. `extra_buffer` is strictly better for non-KV-cache-bound cases; for KV-cache-bound cases, the tradeoff depends on whether enabling overlap outweighs reduced max running requests. 2b. mamba caching at radix cache branching point is strictly better than non-branch but requires kernel support (currently only FLA backend), currently only extra_buffer supports branching. + The strategy to use for mamba scheduler. auto currently defaults to no_buffer. 1. no_buffer does not support overlap scheduler due to not allocating extra mamba state buffers. Branching point caching support is feasible but not implemented. 2. extra_buffer supports overlap schedule by allocating extra mamba state buffers to track mamba state for caching (mamba state usage per running req becomes 2x for non-spec; 1+(1/(2+speculative_num_draft_tokens))x for spec dec (e.g. 1.16x if speculative_num_draft_tokens==4)). 2a. extra_buffer is strictly better for non-KV-cache-bound cases; for KV-cache-bound cases, the tradeoff depends on whether enabling overlap outweighs reduced max running requests. 2b. mamba caching at radix cache branching point is strictly better than non-branch but requires kernel support (currently only FLA backend), currently only extra_buffer supports branching. `auto` - `auto`, `no_buffer`, `extra_buffer` + auto, no_buffer, extra_buffer `--mamba-track-interval` - The interval (in tokens) to track the mamba state during decode. Only used when `--mamba-scheduler-strategy` is `extra_buffer`. Must be divisible by page_size if set, and must be >= speculative_num_draft_tokens when using speculative decoding. + The interval (in tokens) to track the mamba state during decode. Only used when --mamba-scheduler-strategy is extra_buffer. Must be divisible by page_size if set, and must be >= speculative_num_draft_tokens when using speculative decoding. `256` Type: int @@ -1771,6 +1786,7 @@ Please consult the documentation below and [server_args.py](https://github.com/s ## Diffusion LLM + @@ -1802,62 +1818,6 @@ Please consult the documentation below and [server_args.py](https://github.com/s
-## Double Sparsity - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ArgumentDescriptionDefaultsOptions
`--enable-double-sparsity`Enable double sparsity attention`False`bool flag (set to enable)
`--ds-channel-config-path`The path of the double sparsity channel config`None`Type: str
`--ds-heavy-channel-num`The number of heavy channels in double sparsity attention`32`Type: int
`--ds-heavy-token-num`The number of heavy tokens in double sparsity attention`256`Type: int
`--ds-heavy-channel-type`The type of heavy channels in double sparsity attention`qk`Type: str
`--ds-sparse-decode-threshold`The minimum decode sequence length required before the double-sparsity backend switches from the dense fallback to the sparse decode kernel.`4096`Type: int
- ## Offloading @@ -1952,366 +1912,372 @@ Please consult the documentation below and [server_args.py](https://github.com/s - + - + - - + + - - + + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - - - + + + + + + + + + + - - + + - - + + - - + + - - + + - + - - + + - + - - + + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - - - + + + + - - + + - - + + - + - - + + - + - - + + - + - - + + - + - - - - - + + + + + - - + + - + - - + + - + - +
`--disable-radix-cache`--disable-radix-cache Disable RadixAttention for prefix caching.`False`False bool flag (set to enable)
`--cuda-graph-max-bs`
--cuda-graph-max-bs Set the maximum batch size for cuda graph. It will extend the cuda graph capture batch size to this value. `None` Type: int
`--cuda-graph-bs`
--cuda-graph-bs Set the list of batch sizes for cuda graph. `None` List[int]
`--disable-cuda-graph`
--disable-cuda-graph Disable cuda graph.`False`False bool flag (set to enable)
`--disable-cuda-graph-padding`
--disable-cuda-graph-padding Disable cuda graph when padding is needed. Still uses cuda graph when padding is not needed.`False`False bool flag (set to enable)
`--enable-profile-cuda-graph`
--enable-profile-cuda-graph Enable profiling of cuda graph capture.`False`False bool flag (set to enable)
`--enable-cudagraph-gc`
--enable-cudagraph-gc Enable garbage collection during CUDA graph capture. If disabled (default), GC is frozen during capture to speed up the process.`False`False bool flag (set to enable)
`--enable-layerwise-nvtx-marker`
--enable-layerwise-nvtx-marker Enable layerwise NVTX profiling annotations for the model. This adds NVTX markers to every layer for detailed per-layer performance analysis with Nsight Systems.`False`False bool flag (set to enable)
`--enable-nccl-nvls`
--enable-nccl-nvls Enable NCCL NVLS for prefill heavy requests when available.`False`False bool flag (set to enable)
`--enable-symm-mem`
--enable-symm-mem Enable NCCL symmetric memory for fast collectives.`False`False bool flag (set to enable)
`--disable-flashinfer-cutlass-moe-fp4-allgather`
--disable-flashinfer-cutlass-moe-fp4-allgather Disables quantize before all-gather for flashinfer cutlass moe.`False`False bool flag (set to enable)
`--enable-tokenizer-batch-encode`
--enable-tokenizer-batch-encode Enable batch tokenization for improved performance when processing multiple text inputs. Do not use with image inputs, pre-tokenized input_ids, or input_embeds.`False`False bool flag (set to enable)
`--disable-tokenizer-batch-decode`
--disable-tokenizer-batch-decode Disable batch decoding when decoding multiple completions.`False`False bool flag (set to enable)
`--disable-outlines-disk-cache`
--disable-outlines-disk-cache Disable disk cache of outlines to avoid possible crashes related to file system or high concurrency.`False`False bool flag (set to enable)
`--disable-custom-all-reduce`
--disable-custom-all-reduce Disable the custom all-reduce kernel and fall back to NCCL.`False`False bool flag (set to enable)
`--enable-mscclpp`
--enable-mscclpp Enable using mscclpp for small messages for all-reduce kernel and fall back to NCCL.`False`False bool flag (set to enable)
`--enable-torch-symm-mem`
--enable-torch-symm-mem Enable using torch symm mem for all-reduce kernel and fall back to NCCL. Only supports CUDA device SM90 and above. SM90 supports world size 4, 6, 8. SM10 supports world size 6, 8.`False`False bool flag (set to enable)
`--disable-overlap-schedule`
--disable-overlap-schedule Disable the overlap scheduler, which overlaps the CPU scheduler with GPU model worker.`False`False bool flag (set to enable)
`--enable-mixed-chunk`
--enable-mixed-chunk Enabling mixing prefill and decode in a batch when using chunked prefill.`False`False bool flag (set to enable)
`--enable-dp-attention`
--enable-dp-attention Enabling data parallelism for attention and tensor parallelism for FFN. The dp size should be equal to the tp size. Currently DeepSeek-V2 and Qwen 2/3 MoE models are supported.`False`False bool flag (set to enable)
`--enable-dp-lm-head`
--enable-dp-lm-head Enable vocabulary parallel across the attention TP group to avoid all-gather across DP groups, optimizing performance under DP attention.`False`False bool flag (set to enable)
`--enable-two-batch-overlap`
--enable-two-batch-overlap Enabling two micro batches to overlap.`False`False bool flag (set to enable)
`--enable-single-batch-overlap`
--enable-single-batch-overlap Let computation and communication overlap within one micro batch.`False`False bool flag (set to enable)
`--tbo-token-distribution-threshold`
--tbo-token-distribution-threshold The threshold of token distribution between two batches in micro-batch-overlap, determines whether to two-batch-overlap or two-chunk-overlap. Set to 0 denote disable two-chunk-overlap.`0.48`0.48 Type: float
`--enable-torch-compile`
--enable-torch-compile Optimize the model with torch.compile. Experimental feature.`False`False bool flag (set to enable)
`--enable-torch-compile-debug-mode`
--enable-torch-compile-debug-mode Enable debug mode for torch compile.`False`False bool flag (set to enable)
`--enable-piecewise-cuda-graph`Optimize the model with piecewise cuda graph for extend/prefill only. Experimental feature.`False`
--disable-piecewise-cuda-graphDisable piecewise cuda graph for extend/prefill. PCG is enabled by default.Falsebool flag (set to disable)
--enforce-piecewise-cuda-graphEnforce piecewise cuda graph, skipping all auto-disable conditions. For testing only.False bool flag (set to enable)
`--piecewise-cuda-graph-tokens`
--piecewise-cuda-graph-tokens Set the list of tokens when using piecewise cuda graph. `None` Type: JSON list
`--piecewise-cuda-graph-compiler`
--piecewise-cuda-graph-compiler Set the compiler for piecewise cuda graph. Choices are: eager, inductor.`eager``eager`, `inductor`eagereager, inductor
`--torch-compile-max-bs`
--torch-compile-max-bs Set the maximum batch size when using torch compile.`32`32 Type: int
`--piecewise-cuda-graph-max-tokens`
--piecewise-cuda-graph-max-tokens Set the maximum tokens when using piecewise cuda graph.`4096`4096 Type: int
`--torchao-config`
--torchao-config Optimize the model with torchao. Experimental feature. Current choices are: int8dq, int8wo, int4wo-<group_size>, fp8wo, fp8dq-per_tensor, fp8dq-per_row `` Type: str
`--enable-nan-detection`
--enable-nan-detection Enable the NaN detection for debugging purposes.`False`False bool flag (set to enable)
`--enable-p2p-check`
--enable-p2p-check Enable P2P check for GPU access, otherwise the p2p access is allowed by default.`False`False bool flag (set to enable)
`--triton-attention-reduce-in-fp32`
--triton-attention-reduce-in-fp32 Cast the intermediate attention results to fp32 to avoid possible crashes related to fp16. This only affects Triton attention kernels.`False`False bool flag (set to enable)
`--triton-attention-num-kv-splits`
--triton-attention-num-kv-splits The number of KV splits in flash decoding Triton kernel. Larger value is better in longer context scenarios. The default value is 8.`8`8 Type: int
`--triton-attention-split-tile-size`
--triton-attention-split-tile-size The size of split KV tile in flash decoding Triton kernel. Used for deterministic inference. `None` Type: int
`--num-continuous-decode-steps`
--num-continuous-decode-steps Run multiple continuous decoding steps to reduce scheduling overhead. This can potentially increase throughput but may also increase time-to-first-token latency. The default value is 1, meaning only run one decoding step at a time.`1`1 Type: int
`--delete-ckpt-after-loading`
--delete-ckpt-after-loading Delete the model checkpoint after loading the model.`False`False bool flag (set to enable)
`--enable-memory-saver`
--enable-memory-saver Allow saving memory using release_memory_occupation and resume_memory_occupation`False`False bool flag (set to enable)
`--enable-weights-cpu-backup`
--enable-weights-cpu-backup Save model weights to CPU memory during release_weights_occupation and resume_weights_occupation`False`False bool flag (set to enable)
`--enable-draft-weights-cpu-backup`
--enable-draft-weights-cpu-backup Save draft model weights to CPU memory during release_weights_occupation and resume_weights_occupation`False`False bool flag (set to enable)
`--allow-auto-truncate`
--allow-auto-truncate Allow automatically truncating requests that exceed the maximum input length instead of returning an error.`False`False bool flag (set to enable)
`--enable-custom-logit-processor`
--enable-custom-logit-processor Enable users to pass custom logit processors to the server (disabled by default for security)`False`False bool flag (set to enable)
`--flashinfer-mla-disable-ragged`
--flashinfer-mla-disable-ragged Not using ragged prefill wrapper when running flashinfer mla`False`False bool flag (set to enable)
`--disable-shared-experts-fusion`
--disable-shared-experts-fusion Disable shared experts fusion optimization for deepseek v3/r1.`False`False bool flag (set to enable)
`--disable-chunked-prefix-cache`
--disable-chunked-prefix-cache Disable chunked prefix cache feature for deepseek, which should save overhead for short sequences.`False`False bool flag (set to enable)
`--disable-fast-image-processor`
--disable-fast-image-processor Adopt base image processor instead of fast image processor.`False`False bool flag (set to enable)
`--keep-mm-feature-on-device`
--keep-mm-feature-on-device Keep multimodal feature tensors on device after processing to save D2H copy.`False`False bool flag (set to enable)
`--enable-return-hidden-states`
--enable-return-hidden-states Enable returning hidden states with responses.`False`False bool flag (set to enable)
`--enable-return-routed-experts`
--enable-return-routed-experts Enable returning routed experts of each layer with responses.`False`False bool flag (set to enable)
`--scheduler-recv-interval`The interval to poll requests in scheduler. Can be set to >1 to reduce the overhead of this.`1`
--scheduler-recv-intervalThe interval to poll requests in scheduler. Can be set to >1 to reduce the overhead of this.1 Type: int
`--numa-node`
--numa-node Sets the numa node for the subprocesses. i-th element corresponds to i-th subprocess. `None` List[int]
`--enable-deterministic-inference`
--enable-deterministic-inference Enable deterministic inference mode with batch invariant ops.`False`False bool flag (set to enable)
`--rl-on-policy-target`
--rl-on-policy-target The training system that SGLang needs to match for true on-policy. `None``fsdp`fsdp
`--enable-attn-tp-input-scattered`
--enable-attn-tp-input-scattered Allow input of attention to be scattered when only using tensor parallelism, to reduce the computational load of operations such as qkv latent.`False`False bool flag (set to enable)
`--enable-nsa-prefill-context-parallel`
--enable-nsa-prefill-context-parallel Enable context parallelism used in the long sequence prefill phase of DeepSeek v3.2.`False`False bool flag (set to enable)
`--nsa-prefill-cp-mode`Token splitting mode for the prefill phase of DeepSeek v3.2 under context parallelism. Optional values: `in-seq-split` (default), `round-robin-split`. `round-robin-split` distributes tokens across ranks based on `token_idx % cp_size`. It supports multi-batch prefill, fused MoE, and FP8 KV cache.`in-seq-split``in-seq-split`, `round-robin-split`
--nsa-prefill-cp-modeToken splitting mode for the prefill phase of DeepSeek v3.2 under context parallelism. Optional values: round-robin-split(default),in-seq-split. round-robin-split distributes tokens across ranks based on token_idx % cp_size. It supports multi-batch prefill, fused MoE, and FP8 KV cache.in-seq-splitin-seq-split, round-robin-split
`--enable-fused-qk-norm-rope`
--enable-fused-qk-norm-rope Enable fused qk normalization and rope rotary embedding.`False`False bool flag (set to enable)
`--enable-precise-embedding-interpolation`
--enable-precise-embedding-interpolation Enable corner alignment for resize of embeddings grid to ensure more accurate(but slower) evaluation of interpolated embedding values.`False`False bool flag (set to enable)
## Dynamic batch tokenizer @@ -2414,72 +2380,48 @@ Please consult the documentation below and [server_args.py](https://github.com/s - `--disaggregation-mode` + --disaggregation-mode Only used for PD disaggregation. "prefill" for prefill-only server, and "decode" for decode-only server. If not specified, it is not PD disaggregated - `null` - `null`, `prefill`, `decode` + null + null, prefill, decode - `--disaggregation-transfer-backend` + --disaggregation-transfer-backend The backend for disaggregation transfer. Default is mooncake. - `mooncake` - `mooncake`, `nixl`, `ascend`, `fake` + mooncake + mooncake, nixl, ascend, fake - `--disaggregation-bootstrap-port` + --disaggregation-bootstrap-port Bootstrap server port on the prefill server. Default is 8998. - `8998` + 8998 Type: int - `--disaggregation-decode-tp` - Decode tp size. If not set, it matches the tp size of the current engine. This is only set on the prefill server. - `None` - Type: int - - - `--disaggregation-decode-dp` - Decode dp size. If not set, it matches the dp size of the current engine. This is only set on the prefill server. - `None` - Type: int - - - `--disaggregation-prefill-pp` - Prefill pp size. If not set, it is default to 1. This is only set on the decode server. - `1` - Type: int - - - `--disaggregation-ib-device` + --disaggregation-ib-device The InfiniBand devices for disaggregation transfer, accepts single device (e.g., --disaggregation-ib-device mlx5_0) or multiple comma-separated devices (e.g., --disaggregation-ib-device mlx5_0,mlx5_1). Default is None, which triggers automatic device detection when mooncake backend is enabled. - `None` + None Type: str - - `--disaggregation-decode-enable-offload-kvcache` + + --disaggregation-decode-enable-offload-kvcache Enable async KV cache offloading on decode server (PD mode). `False` bool flag (set to enable) - - `--disaggregation-decode-enable-fake-auto` - Auto enable FAKE mode for decode node testing, no need to pass bootstrap_host and bootstrap_room in request. - `False` - bool flag (set to enable) - - - `--num-reserved-decode-tokens` + + --num-reserved-decode-tokens Number of decode tokens that will have memory reserved when adding new request to the running batch. - `512` + 512 Type: int - - `--disaggregation-decode-polling-interval` - The interval to poll requests in decode server. Can be set to >1 to reduce the overhead of this. - `1` + + --disaggregation-decode-polling-interval + The interval to poll requests in decode server. Can be set to >1 to reduce the overhead of this. + 1 Type: int - + ## Encode prefill disaggregation @@ -2544,48 +2486,60 @@ Please consult the documentation below and [server_args.py](https://github.com/s - `--custom-weight-loader` + --custom-weight-loader The custom dataloader which used to update the model. Should be set with a valid import path, such as my_package.weight_load_func - `None` + None List[str] - `--weight-loader-disable-mmap` + --weight-loader-disable-mmap Disable mmap while loading weight using safetensors. `False` bool flag (set to enable) - `--remote-instance-weight-loader-seed-instance-ip` - The ip of the seed instance for loading weights from remote instance. - `None` - Type: str - - - `--remote-instance-weight-loader-seed-instance-service-port` - The service port of the seed instance for loading weights from remote instance. - `None` - Type: int - - - `--remote-instance-weight-loader-send-weights-group-ports` - The communication group ports for loading weights from remote instance. - `None` - Type: JSON list - - - `--remote-instance-weight-loader-backend` - The backend for loading weights from remote instance. Can be 'transfer_engine' or 'nccl'. Default is 'nccl'. - `nccl` - `transfer_engine`, `nccl` - - - `--remote-instance-weight-loader-start-seed-via-transfer-engine` - Start seed server via transfer engine backend for remote instance weight loader. - `False` + --weight-loader-prefetch-checkpoints + Prefetch checkpoint files into OS page cache before loading. Each rank prefetches a fraction of the shards in a background thread, reducing total network I/O on shared filesystems (NFS/Lustre) from N\*checkpoint to 1\*checkpoint. Recommended for models on network storage. + False bool flag (set to enable) - + + --weight-loader-prefetch-num-threads + Number of threads per rank for checkpoint prefetching. + 4 + Type: int + + + --remote-instance-weight-loader-seed-instance-ip + The ip of the seed instance for loading weights from remote instance. + None + Type: str + + + --remote-instance-weight-loader-seed-instance-service-port + The service port of the seed instance for loading weights from remote instance. + None + Type: int + + + --remote-instance-weight-loader-send-weights-group-ports + The communication group ports for loading weights from remote instance. + None + Type: JSON list + + + --remote-instance-weight-loader-backend + The backend for loading weights from remote instance. Can be 'transfer_engine' or 'nccl'. Default is 'nccl'. + nccl + transfer_engine, nccl + + + --remote-instance-weight-loader-start-seed-via-transfer-engine + Start seed server via transfer engine backend for remote instance weight loader. + False + bool flag (set to enable) + + ## For PD-Multiplexing @@ -2670,42 +2624,48 @@ Please consult the documentation below and [server_args.py](https://github.com/s - `--mm-max-concurrent-calls` + --mm-max-concurrent-calls The max concurrent calls for async mm data processing. - `32` + 32 Type: int - - `--mm-per-request-timeout` + + --mm-per-request-timeout The timeout for each multi-modal request in seconds. - `10.0` + 10.0 Type: int - - `--enable-broadcast-mm-inputs-process` + + --enable-broadcast-mm-inputs-process Enable broadcast mm-inputs process in scheduler. - `False` + False bool flag (set to enable) - - `--mm-process-config` - Multimodal preprocessing config, a json config contains keys: `image`, `video`, `audio`. - `{}` + + --mm-process-config + Multimodal preprocessing config, a json config contains keys: image, video, audio. + {} Type: JSON / Dict - - `--mm-enable-dp-encoder` + + --mm-enable-dp-encoder Enabling data parallelism for mm encoder. The dp size will be set to the tp size automatically. - `False` + False bool flag (set to enable) - - `--limit-mm-data-per-request` + + --limit-mm-data-per-request Limit the number of multimodal inputs per request. e.g. '{"image": 1, "video": 1, "audio": 1}' `None` Type: JSON / Dict - + + --enable-mm-global-cache + Enable Mooncake-backed global multimodal embedding cache on encoder servers so repeated images can reuse cached ViT embeddings instead of recomputing them. + False + bool flag (set to enable) + + ## For checkpoint decryption diff --git a/docs_new/docs/advanced_features/sglang_for_rl.mdx b/docs_new/docs/advanced_features/sglang_for_rl.mdx index 31898d34c..7e2f4a1e1 100644 --- a/docs_new/docs/advanced_features/sglang_for_rl.mdx +++ b/docs_new/docs/advanced_features/sglang_for_rl.mdx @@ -105,7 +105,7 @@ Enable memory saver support when launching the server: ## Open-To-Use Refit Functionality -After training completes each step, rollout engines must be refit with new weights. SGLang supports three refit strategies so you can match your infrastructure style (co-located vs disaggregated) and scaling needs. Each strategy maps to a concrete API with clear request schemas. For a deeper dive into SGLang's weight update utilities, see [RL System Deep Thinking: Weight Update Mechanisms](https://github.com/zhaochenyang20/Awesome-ML-SYS-Tutorial/blob/main/rlhf/sys-design/readme-1-EN). +After training completes each step, rollout engines must be refit with new weights. SGLang supports three refit strategies so you can match your infrastructure style (co-located vs disaggregated) and scaling needs. Each strategy maps to a concrete API with clear request schemas. For a deeper dive into SGLang's weight update utilities, see [RL System Deep Thinking: Weight Update Mechanisms](https://github.com/zhaochenyang20/Awesome-ML-SYS-Tutorial/blob/main/rlhf/sys-design/readme-1-EN.md). **How to choose:** @@ -248,6 +248,86 @@ This path trades some I/O overhead for simplicity and flexibility. It integrates **Python Engine API:** `engine.update_weights_from_disk(model_path, load_format=None)` +**Diffusion engine (SGLang-Diffusion):** The diffusion engine exposes the same `POST /update_weights_from_disk` endpoint with the following behavior: + +- **All-or-nothing with rollback:** if any module fails to load, all previously updated modules are rolled back to the original weights by reloading from the original model path. No partial updates are left behind. If rollback itself fails, the exception propagates so the caller knows the model is in an inconsistent state. +- **Offload-aware:** when layerwise offload (`--dit-layerwise-offload`) is enabled, the diffusion offload manager replaces GPU parameters with small `torch.empty((1,))` placeholders while real weights live in consolidated pinned CPU buffers. A naive `param.data.copy_()` would fail with a shape mismatch. Instead, the updater dynamically detects active offload managers and writes new weights directly into their CPU buffers, bypassing the placeholders entirely. For any layer that happens to be prefetched on GPU at update time, the live GPU tensor is also updated so the change takes effect immediately. This requires no extra GPU memory and does not disturb the offload state. +- **DTensor-aware:** parameters distributed via `torch.distributed.tensor` (tensor parallelism) are updated through `distribute_tensor` so that each shard is correctly placed on the right device mesh. + +**Request body:** + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescriptionDefaultsOptions
model_pathThe model path with the new weights.RequiredType: str
flush_cacheFlush TeaCache state after update.TrueType: bool
target_modulesList of module names to update (e.g. ["transformer"]). If omitted, all nn.Module components are updated.NoneType: list[str]
+ +**Response body:** + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescriptionDefaultsOptions
successWhether the update succeeded.-Type: bool
messageStatus / error message.-Type: str
+ +> **Note:** The diffusion engine (SGLang-Diffusion) does not currently support hot refit (updating weights while inference is in progress). The diffusion scheduler processes one request at a time and completes the entire inference before handling the next request, so weight updates and inference never run concurrently. + ### Update Weights from Tensor **When to use:** @@ -280,33 +360,33 @@ This strategy requires the training process and rollout engine to share access t - `serialized_named_tensors` + serialized_named_tensors Per-TP serialized tensor payloads. Required - Type: list[str + Type: list[str|bytes] - `load_format` + load_format Optional load format selector. - `None` - `None`, `direct`, `flattened_bucket`, or a custom loader path string + None + None, direct, flattened_bucket, or a custom loader path string - `flush_cache` + flush_cache Flush KV cache after update. - `True` + True Type: bool - `abort_all_requests` + abort_all_requests Abort all running requests before update. - `False` + False Type: bool - `weight_version` + weight_version Optional version label tracked by the server. - `None` + None Type: str @@ -551,7 +631,7 @@ SGLang exposes explicit pause/resume APIs so you can pause slow requests and con In many RL stacks, rollout and training are implemented with different kernels or batching behavior. Even when weights are identical, token probabilities can drift, silently breaking the on-policy assumption. This is the training–inference mismatch problem. -SGLang supports a deterministic inference mode that reduces non-determinism across batch shapes. This mitigates variance introduced by runtime batching and kernel selection. To further achieve true on-policy training, you need to modify the training engine to use the same deterministic kernels. For implementation details, see these miles examples: [True On-Policy](https://github.com/radixark/miles/tree/main/examples/true_on_policy) and [True On-Policy for VLM](https://github.com/radixark/miles/tree/main/examples/true_on_policy_vlm). For additional context, see the blog post [Let Speed Be With Stability: All-In-One Solution to Training-Inference Mismatch with Miles](https://github.com/zhaochenyang20/Awesome-ML-SYS-Tutorial/blob/main/rlhf/slime/mismatch/blog-en). +SGLang supports a deterministic inference mode that reduces non-determinism across batch shapes. This mitigates variance introduced by runtime batching and kernel selection. To further achieve true on-policy training, you need to modify the training engine to use the same deterministic kernels. For implementation details, see these miles examples: [True On-Policy](https://github.com/radixark/miles/tree/main/examples/true_on_policy) and [True On-Policy for VLM](https://github.com/radixark/miles/tree/main/examples/true_on_policy_vlm). For additional context, see the blog post [Let Speed Be With Stability: All-In-One Solution to Training-Inference Mismatch with Miles](https://github.com/zhaochenyang20/Awesome-ML-SYS-Tutorial/blob/main/rlhf/slime/mismatch/blog-en.md). **Server flag:** diff --git a/docs_new/docs/advanced_features/speculative_decoding.mdx b/docs_new/docs/advanced_features/speculative_decoding.mdx index b5f4f1158..4946012f1 100644 --- a/docs_new/docs/advanced_features/speculative_decoding.mdx +++ b/docs_new/docs/advanced_features/speculative_decoding.mdx @@ -3,7 +3,111 @@ title: "Speculative Decoding" metatags: description: "SGLang EAGLE speculative decoding: EAGLE-2/EAGLE-3, up to 2.4x throughput improvement, draft model configuration, MTP for DeepSeek." --- -SGLang now provides an EAGLE-based (EAGLE-2/EAGLE-3) speculative decoding option. Our implementation aims to maximize speed and efficiency and is considered to be among the fastest in open-source LLM engines. +SGLang provides several speculative decoding options, including EAGLE-2/EAGLE-3, MTP, classic draft-model decoding, and an NGRAM-based variant. Our implementation aims to maximize speed and efficiency and is considered to be among the fastest in open-source LLM engines. + +## Summary + +### Jump to sections + +- [EAGLE Decoding](#eagle-decoding) + - [EAGLE-2 Decoding](#eagle-2-decoding) + - [EAGLE-2 Decoding with torch.compile](#eagle-2-decoding-with-torchcompile) + - [EAGLE-2 Decoding via Frequency-Ranked Speculative Sampling](#eagle-2-decoding-via-frequency-ranked-speculative-sampling) + - [EAGLE-3 Decoding](#eagle-3-decoding) +- [Multi Token Prediction](#multi-token-prediction) +- [Standalone Speculative Decoding (Small Draft Model)](#standalone-speculative-decoding-small-draft-model) +- [Speculative Decoding V2 (Overlap Scheduler)](#speculative-decoding-v2-overlap-scheduler) +- [Ngram Speculative Decoding](#ngram-speculative-decoding) +- [Full Parameter Reference](#full-parameter-reference) +- [OOM Troubleshooting](#oom-troubleshooting) +- [References](#references) + +### Quick guidance + +- **Best speed/quality (recommended)**: Use **EAGLE-3** with `--speculative-algorithm EAGLE3`. +- **Strong default / broad compatibility**: Use **EAGLE-2** with `--speculative-algorithm EAGLE`. +- **Workload acceptance changes over time**: Use [**Adaptive speculative decoding**](./adaptive_speculative_decoding) on top of **EAGLE** with `--speculative-eagle-topk 1`. +- **Lower `lm_head` overhead for EAGLE-2**: Enable **FR-Spec** with `--speculative-token-map`. +- **Model is MTP-enabled**: Use **MTP via speculative decoding** (often with small `speculative_num_steps/topk/num_draft_tokens`, see the example section). +- **You have a smaller draft LLM**: Use **STANDALONE** (`--speculative-algorithm STANDALONE`). +- **No extra model available**: Use **NGRAM** (`--speculative-algorithm NGRAM`, CUDA-only). +- **Want overlap scheduler (experimental)**: Enable **SpecV2** with `SGLANG_ENABLE_SPEC_V2=True` (requires `--speculative-eagle-topk 1`). + +### Method comparison (mini table) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MethodDraft sourceSeparate draft model?How to enableNotes / constraints
EAGLE-2EAGLE draft model (feature drafting + tree)Typically yes--speculative-algorithm EAGLE + --speculative-draft-model-path ...Tune --speculative-num-steps, --speculative-eagle-topk, --speculative-num-draft-tokens
EAGLE-2 + torch.compileSame as EAGLE-2Typically yesAdd --enable-torch-compile (optionally --torch-compile-max-bs)Benefit varies by hardware/model; benchmark to verify
EAGLE-2 + FR-SpecSame as EAGLE-2 + token subsetTypically yesAdd --speculative-token-map ...Reduces lm_head overhead with high-frequency token vocab
EAGLE-3EAGLE3 draft modelYes--speculative-algorithm EAGLE3 + --speculative-draft-model-path ...Best throughput in the benchmark below
MTPBuilt-in multi-token heads (model-specific)Often noSee Multi Token Prediction sectionUses speculative workflow; draft path may be auto-handled for some models
STANDALONESmaller draft LLM (token-level)Yes--speculative-algorithm STANDALONE + --speculative-draft-model-path ...Does not support --enable-dp-attention
SpecV2 (experimental)V2 workers + overlap schedulerN/ASGLANG_ENABLE_SPEC_V2=TrueOnly supports --speculative-eagle-topk 1; applies to EAGLE, EAGLE3, STANDALONE
NGRAMNgram cache from previous tokensNo--speculative-algorithm NGRAMCUDA-only; no --enable-dp-attention; disables overlap scheduler & mixed chunked prefill
### Performance Highlights @@ -37,51 +141,116 @@ For further details please see the [EAGLE3 paper](https://arxiv.org/pdf/2503.018 +--- ## EAGLE Decoding To enable EAGLE speculative decoding the following parameters are relevant: -* `speculative_draft_model_path`: Specifies draft model. This parameter is required. -* `speculative_num_steps`: Depth of autoregressive drafting. Increases speculation range but risks rejection cascades. Default is 5. -* `speculative_eagle_topk`: Branching factor per step. Improves candidate diversity, will lead to higher acceptance rate, but more lead to higher memory/compute consumption. Default is 4. -* `speculative_num_draft_tokens`: Maximum parallel verification capacity. Allows deeper tree evaluation but will lead to higher GPU memory usage. Default is 8. -These parameters are the same for EAGLE-2 and EAGLE-3. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ParameterDescriptionDefault
--speculative-draft-model-pathDraft model path/weights. Typically required for EAGLE/EAGLE3 and STANDALONE. For some MTP-enabled models, this can be omitted.None
--speculative-num-stepsDepth of autoregressive drafting. Increases speculation range but risks rejection cascades.Auto (5 for Llama/Grok; 3 for many other models)
--speculative-eagle-topkBranching factor per step. Improves candidate diversity and acceptance rate, but increases memory/compute consumption.Auto (4 for Llama/Grok; 1 for many other models)
--speculative-num-draft-tokensMaximum parallel verification capacity. Allows deeper tree evaluation but increases GPU memory usage.Auto (8 for Llama/Grok; 4 for many other models). If topk=1, it is adjusted to num_steps + 1.
--speculative-accept-threshold-singleAcceptance threshold for single-token verification. Lower values accept more aggressively.1.0
--speculative-accept-threshold-accAccumulated acceptance threshold across steps.1.0
--speculative-attention-modeAttention mode for speculative operations (prefill or decode), affecting both target verification and draft extension."prefill"
--speculative-draft-attention-backendOverride attention backend for the draft model.None (same as target)
--speculative-draft-model-quantizationQuantization method for the draft model. Use "unquant" to force no quantization even when the target model is quantized.Same as target model
--speculative-draft-model-revisionSpecific revision/commit of the draft model to load.None (auto-set to "main" when --speculative-draft-model-path is set and revision is omitted)
--speculative-draft-load-formatLoad format for the draft model weights.None
+ +These parameters are mostly the same for EAGLE-2 and EAGLE-3. `--speculative-token-map` is ignored for EAGLE-3 models. +For `--speculative-num-steps`, `--speculative-eagle-topk`, and `--speculative-num-draft-tokens`: leave all three unset to use auto-tuning, or set all three explicitly when tuning. +If you use EAGLE with `--speculative-eagle-topk 1` and your acceptance rate varies across requests, see [Adaptive Speculative Decoding](./adaptive_speculative_decoding). You can find the best combinations of these parameters with [bench_speculative.py](https://github.com/sgl-project/sglang/blob/main/scripts/playground/bench_speculative.py). -In the documentation below, we set `--cuda-graph-max-bs` to be a small value for faster engine startup. For your own workloads, please tune the above parameters together with `--cuda-graph-max-bs`, `--max-running-requests`, `--mem-fraction-static` for the best performance. +### EAGLE-2 Decoding -### EAGLE-2 decoding +You can enable EAGLE-2 Decoding by setting `--speculative-algorithm EAGLE` and choosing an appropriate model. -You can enable EAGLE-2 decoding by setting `--speculative-algorithm EAGLE` and choosing an appropriate model. +**Launch the server:** +```bash Command +python3 -m sglang.launch_server \ + --model meta-llama/Llama-2-7b-chat-hf \ + --speculative-algorithm EAGLE \ + --speculative-draft-model-path lmsys/sglang-EAGLE-llama2-chat-7B \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 4 \ + --speculative-num-draft-tokens 16 \ + --mem-fraction-static 0.7 \ + --cuda-graph-max-bs 8 \ + --log-level warning +``` +**Send a request:** ```python Example -from sglang.test.doc_patch import launch_server_cmd -from sglang.utils import wait_for_server, print_highlight, terminate_process - import openai -``` - -```python Example -server_process, port = launch_server_cmd( - """ -python3 -m sglang.launch_server --model meta-llama/Llama-2-7b-chat-hf --speculative-algorithm EAGLE \ - --speculative-draft-model-path lmsys/sglang-EAGLE-llama2-chat-7B --speculative-num-steps 3 \ - --speculative-eagle-topk 4 --speculative-num-draft-tokens 16 --cuda-graph-max-bs 8 --log-level warning -""" -) - -wait_for_server(f"http://localhost:{port}") -``` - - -```python Example -client = openai.Client(base_url=f"http://127.0.0.1:{port}/v1", api_key="None") +client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="None") response = client.chat.completions.create( model="meta-llama/Llama-2-7b-chat-hf", @@ -92,37 +261,37 @@ response = client.chat.completions.create( max_tokens=64, ) -print_highlight(f"Response: {response}") +print(response.choices[0].message.content) ``` - -```python Example -terminate_process(server_process) -``` +--- ### EAGLE-2 Decoding with `torch.compile` -You can also enable `torch.compile` for further optimizations and optionally set `--torch-compile-max-bs`: +You can optionally enable `torch.compile` to apply kernel-level optimizations (operator fusion, autotune) to the draft model. The actual speedup depends on your hardware, model architecture, and batch size. In some configurations (e.g., small draft models on H100 where cuBLAS is already optimal and CUDA graphs are enabled), the benefit may be negligible. We recommend benchmarking with and without this flag on your specific setup to verify whether it helps. +To enable it, add `--enable-torch-compile` and optionally set `--torch-compile-max-bs`: - - -```python Example -server_process, port = launch_server_cmd( - """ -python3 -m sglang.launch_server --model meta-llama/Llama-2-7b-chat-hf --speculative-algorithm EAGLE \ - --speculative-draft-model-path lmsys/sglang-EAGLE-llama2-chat-7B --speculative-num-steps 5 \ - --speculative-eagle-topk 8 --speculative-num-draft-tokens 64 --mem-fraction 0.6 \ - --enable-torch-compile --torch-compile-max-bs 2 --log-level warning -""" -) - -wait_for_server(f"http://localhost:{port}") +```bash Command +python3 -m sglang.launch_server \ + --model meta-llama/Llama-2-7b-chat-hf \ + --speculative-algorithm EAGLE \ + --speculative-draft-model-path lmsys/sglang-EAGLE-llama2-chat-7B \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 4 \ + --speculative-num-draft-tokens 16 \ + --mem-fraction-static 0.7 \ + --enable-torch-compile \ + --torch-compile-max-bs 8 \ + --log-level warning ``` +**Send a request:** ```python Example -client = openai.Client(base_url=f"http://127.0.0.1:{port}/v1", api_key="None") +import openai + +client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="None") response = client.chat.completions.create( model="meta-llama/Llama-2-7b-chat-hf", @@ -133,40 +302,40 @@ response = client.chat.completions.create( max_tokens=64, ) -print_highlight(f"Response: {response}") +print(response.choices[0].message.content) ``` - -```python Example -terminate_process(server_process) -``` +--- ### EAGLE-2 Decoding via Frequency-Ranked Speculative Sampling -By employing a truncated high-frequency token vocabulary in the draft model, Eagle speculative decoding reduces `lm_head` computational overhead while accelerating the pipeline without quality degradation. For more details, checkout [the paper](https://arxiv.org/pdf/arXiv:2502.14856). +By employing a truncated high-frequency token vocabulary in the draft model, EAGLE speculative decoding reduces `lm_head` computational overhead while accelerating the pipeline without quality degradation. For more details, check out [the paper](https://arxiv.org/pdf/2502.14856). -In our implementation, set `--speculative-token-map` to enable the optimization. You can get the high-frequency token in FR-Spec from [this model](https://huggingface.co/thunlp/LLaMA3-Instruct-8B-FR-Spec). Or you can obtain high-frequency token by directly downloading these token from [this repo](https://github.com/thunlp/FR-Spec/tree/main?tab=readme-ov-file#prepare-fr-spec-vocabulary-subset). +In our implementation, set `--speculative-token-map` to enable the optimization. You can get the high-frequency tokens in FR-Spec from [this model](https://huggingface.co/thunlp/LLaMA3-Instruct-8B-FR-Spec). Or you can obtain high-frequency tokens by directly downloading these tokens from [this repo](https://github.com/thunlp/FR-Spec/tree/main?tab=readme-ov-file#prepare-fr-spec-vocabulary-subset). Thanks for the contribution from [Weilin Zhao](https://github.com/Achazwl) and [Zhousx](https://github.com/Zhou-sx). - - -```python Example -server_process, port = launch_server_cmd( - """ -python3 -m sglang.launch_server --model meta-llama/Meta-Llama-3-8B-Instruct --speculative-algorithm EAGLE \ - --speculative-draft-model-path lmsys/sglang-EAGLE-LLaMA3-Instruct-8B --speculative-num-steps 5 \ - --speculative-eagle-topk 8 --speculative-num-draft-tokens 64 --speculative-token-map thunlp/LLaMA3-Instruct-8B-FR-Spec/freq_32768.pt \ - --mem-fraction 0.7 --cuda-graph-max-bs 2 --dtype float16 --log-level warning -""" -) - -wait_for_server(f"http://localhost:{port}") +```bash Command +python3 -m sglang.launch_server \ + --model meta-llama/Meta-Llama-3-8B-Instruct \ + --speculative-algorithm EAGLE \ + --speculative-draft-model-path lmsys/sglang-EAGLE-LLaMA3-Instruct-8B \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 4 \ + --speculative-num-draft-tokens 16 \ + --speculative-token-map thunlp/LLaMA3-Instruct-8B-FR-Spec/freq_32768.pt \ + --mem-fraction-static 0.7 \ + --cuda-graph-max-bs 8 \ + --dtype float16 \ + --log-level warning ``` +**Send a request:** ```python Example -client = openai.Client(base_url=f"http://127.0.0.1:{port}/v1", api_key="None") +import openai + +client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="None") response = client.chat.completions.create( model="meta-llama/Meta-Llama-3-8B-Instruct", @@ -177,36 +346,35 @@ response = client.chat.completions.create( max_tokens=64, ) -print_highlight(f"Response: {response}") +print(response.choices[0].message.content) ``` - -```python Example -terminate_process(server_process) -``` +--- ### EAGLE-3 Decoding You can enable EAGLE-3 decoding by setting `--speculative-algorithm EAGLE3` and choosing an appropriate model. - - -```python Example -server_process, port = launch_server_cmd( - """ -python3 -m sglang.launch_server --model meta-llama/Llama-3.1-8B-Instruct --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path jamesliu1/sglang-EAGLE3-Llama-3.1-Instruct-8B --speculative-num-steps 5 \ - --speculative-eagle-topk 8 --speculative-num-draft-tokens 32 --mem-fraction 0.6 \ - --cuda-graph-max-bs 2 --dtype float16 --log-level warning -""" -) - -wait_for_server(f"http://localhost:{port}") +```bash Command +python3 -m sglang.launch_server \ + --model meta-llama/Meta-Llama-3.1-8B-Instruct \ + --speculative-algorithm EAGLE3 \ + --speculative-draft-model-path jamesliu1/sglang-EAGLE3-Llama-3.1-Instruct-8B \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 4 \ + --speculative-num-draft-tokens 16 \ + --mem-fraction-static 0.7 \ + --cuda-graph-max-bs 8 \ + --dtype float16 \ + --log-level warning ``` +**Send a request:** ```python Example -client = openai.Client(base_url=f"http://127.0.0.1:{port}/v1", api_key="None") +import openai + +client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="None") response = client.chat.completions.create( model="meta-llama/Meta-Llama-3.1-8B-Instruct", @@ -217,37 +385,35 @@ response = client.chat.completions.create( max_tokens=64, ) -print_highlight(f"Response: {response}") +print(response.choices[0].message.content) ``` - -```python Example -terminate_process(server_process) -``` +--- ## Multi Token Prediction -We support [MTP(Multi-Token Prediction)](https://arxiv.org/pdf/2404.19737) in SGLang by using speculative decoding. We use Xiaomi/MiMo-7B-RL model as example here (deepseek mtp usage refer to [deepseek doc](../basic_usage/deepseek_v3#multi-token-prediction)) +We support [MTP (Multi-Token Prediction)](https://arxiv.org/pdf/2404.19737) in SGLang by using speculative decoding. We use `XiaomiMiMo/MiMo-7B-RL` as an example here (for DeepSeek MTP usage, refer to [deepseek_v32 doc](../basic_usage/deepseek_v32#multi-token-prediction)). - - -```python Example -server_process, port = launch_server_cmd( - """ - python3 -m sglang.launch_server --model-path XiaomiMiMo/MiMo-7B-RL --host 0.0.0.0 --trust-remote-code \ - --speculative-algorithm EAGLE --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ - --mem-fraction 0.5 --log-level warning -""" -) - -wait_for_server(f"http://localhost:{port}") +```bash Command +python3 -m sglang.launch_server \ + --model XiaomiMiMo/MiMo-7B-RL \ + --host 0.0.0.0 \ + --trust-remote-code \ + --speculative-algorithm EAGLE \ + --speculative-num-steps 1 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 2 \ + --mem-fraction-static 0.7 \ + --cuda-graph-max-bs 8 \ + --log-level warning ``` +**Send a request:** ```python Example import requests -url = f"http://localhost:{port}/v1/chat/completions" +url = "http://localhost:30000/v1/chat/completions" data = { "model": "XiaomiMiMo/MiMo-7B-RL", @@ -255,24 +421,537 @@ data = { } response = requests.post(url, json=data) -print_highlight(response.json()) +print(response.json()) ``` +--- + +## Standalone Speculative Decoding (Small Draft Model) + +Besides EAGLE/MTP, SGLang also supports **token-level speculative decoding** using a smaller **draft model**. Enable it with `--speculative-algorithm STANDALONE` and provide a draft model via `--speculative-draft-model-path`. + +Relevant parameters: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ParameterDescriptionDefault
--speculative-draft-model-pathDraft model weights (smaller than the target model).None
--speculative-num-stepsDraft depth (how many steps the draft model runs autoregressively).3 (auto default for STANDALONE)
--speculative-eagle-topkBranching factor (token candidates per step).1 (auto default for STANDALONE)
--speculative-num-draft-tokensVerification capacity.4 (auto default for STANDALONE)
--speculative-draft-model-quantizationQuantization for the draft model. Use "unquant" to disable quantization on the draft even when the target is quantized.Same as target
+ +> **Note:** Standalone speculative decoding currently **does not support** `--enable-dp-attention`. + +```bash Command +python3 -m sglang.launch_server \ + --model Qwen/Qwen2.5-7B-Instruct \ + --speculative-algorithm STANDALONE \ + --speculative-draft-model-path Qwen/Qwen2.5-1.5B-Instruct \ + --speculative-num-steps 4 \ + --speculative-eagle-topk 2 \ + --speculative-num-draft-tokens 7 \ + --mem-fraction-static 0.7 \ + --cuda-graph-max-bs 8 \ + --log-level warning +``` + +**Send a request:** ```python Example -terminate_process(server_process) +import openai + +client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="None") + +response = client.chat.completions.create( + model="Qwen/Qwen2.5-7B-Instruct", + messages=[ + {"role": "user", "content": "List 3 countries and their capitals."}, + ], + temperature=0, + max_tokens=64, +) + +print(response.choices[0].message.content) ``` +--- + +## Speculative Decoding V2 (Overlap Scheduler) + +SGLang provides an **experimental Speculative Decoding V2** implementation that enables an overlap scheduler and uses V2 speculative workers (e.g. `StandaloneWorkerV2`, `EAGLEWorkerV2`). + +To enable it, set the environment variable: +- `SGLANG_ENABLE_SPEC_V2=True` + +Notes: +- SpecV2 currently only supports `--speculative-eagle-topk 1`. When SpecV2 is enabled, **set `--speculative-eagle-topk 1` explicitly**. +- If you explicitly set `--speculative-eagle-topk > 1`, the server will error. +- If you omit `--speculative-eagle-topk`, auto-tuning may pick `topk > 1` for some models (e.g. Llama). This is incompatible with SpecV2 and may not always trigger an immediate config error, so set `--speculative-eagle-topk 1` explicitly. +- This applies to `EAGLE`, `EAGLE3`, and `STANDALONE`. + +```bash Command +SGLANG_ENABLE_SPEC_V2=True python3 -m sglang.launch_server \ + --model Qwen/Qwen2.5-7B-Instruct \ + --speculative-algorithm STANDALONE \ + --speculative-draft-model-path Qwen/Qwen2.5-1.5B-Instruct \ + --speculative-num-steps 4 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 5 \ + --mem-fraction-static 0.7 \ + --cuda-graph-max-bs 8 \ + --log-level warning +``` + +**Send a request:** + +```python Example +import openai + +client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="None") + +response = client.chat.completions.create( + model="Qwen/Qwen2.5-7B-Instruct", + messages=[ + {"role": "user", "content": "List 3 countries and their capitals."}, + ], + temperature=0, + max_tokens=64, +) + +print(response.choices[0].message.content) +``` + +--- + +## Ngram Speculative Decoding + +SGLang also supports **ngram-based speculative decoding** (no separate draft model). It retrieves draft tokens from an ngram cache built from previously generated tokens, and then verifies them with the target model. + +Enable it with: +- `--speculative-algorithm NGRAM` + +### Ngram-specific parameters + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ParameterDescriptionDefault
--speculative-num-draft-tokensNumber of draft tokens verified per step. If omitted, defaults to min(--speculative-ngram-max-trie-depth, 12).12 (with default ngram settings)
--speculative-ngram-min-bfs-breadthMinimum BFS breadth.1
--speculative-ngram-max-bfs-breadthMaximum BFS breadth.10
--speculative-ngram-match-typeNgram tree-building mode: "BFS" for recency-based expansion or "PROB" for frequency-based expansion."BFS"
--speculative-ngram-max-trie-depthMaximum suffix length stored and matched by the ngram trie.18
--speculative-ngram-capacityCache capacity (number of entries).10,000,000
+ +Notes: +- Ngram speculative decoding **only supports CUDA**. +- It currently **does not support** `--enable-dp-attention`. +- It disables the overlap scheduler and mixed chunked prefill. +- If `--speculative-ngram-max-bfs-breadth > 1` (thus `speculative_eagle_topk > 1`) and `page_size > 1`, use `--attention-backend flashinfer`; otherwise the server will error. +- Optional: set `SGLANG_NGRAM_FORCE_GREEDY_VERIFY=True` to force greedy verification. + +```bash Command +python3 -m sglang.launch_server \ + --model Qwen/Qwen2.5-7B-Instruct \ + --speculative-algorithm NGRAM \ + --speculative-num-draft-tokens 16 \ + --speculative-ngram-max-bfs-breadth 10 \ + --mem-fraction-static 0.7 \ + --cuda-graph-max-bs 8 \ + --log-level warning +``` + +**Send a request:** + +```python Example +import openai + +client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="None") + +response = client.chat.completions.create( + model="Qwen/Qwen2.5-7B-Instruct", + messages=[ + {"role": "user", "content": "List 3 countries and their capitals."}, + ], + temperature=0, + max_tokens=64, +) + +print(response.choices[0].message.content) +``` + +--- + +## Full Parameter Reference + +Below is a comprehensive list of all speculative decoding parameters available in SGLang: + +### Core parameters + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ParameterTypeDefaultDescription
--speculative-algorithmstrNoneAlgorithm to use: EAGLE, EAGLE3, STANDALONE, NGRAM, NEXTN (alias of EAGLE)
--speculative-draft-model-pathstrNonePath to the draft model weights
--speculative-draft-model-revisionstrNoneSpecific revision/commit of the draft model ("main" is auto-used when draft path is set and revision is omitted)
--speculative-draft-load-formatstrNoneLoad format for draft model weights
--speculative-num-stepsintNone (auto-chosen when omitted)Autoregressive drafting depth
--speculative-eagle-topkintNone (auto-chosen when omitted)Branching factor per drafting step
--speculative-num-draft-tokensintNone (auto-chosen when omitted)Maximum number of draft tokens for verification
--speculative-accept-threshold-singlefloat1.0Single-token acceptance threshold
--speculative-accept-threshold-accfloat1.0Accumulated acceptance threshold
--speculative-token-mapstrNonePath to FR-Spec high-frequency token map
--speculative-attention-modestr"prefill"Attention mode for speculative operations ("prefill" or "decode")
--speculative-draft-attention-backendstrNoneOverride attention backend for the draft model
--speculative-moe-runner-backendstrNoneMoE runner backend for the draft model
--speculative-moe-a2a-backendstrNoneMoE all-to-all backend for the draft model
--speculative-draft-model-quantizationstrSame as targetQuantization for the draft model ("unquant" to disable)
+ +### Ngram-specific parameters + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ParameterTypeDefaultDescription
--speculative-ngram-min-bfs-breadthint1Minimum BFS breadth
--speculative-ngram-max-bfs-breadthint10Maximum BFS breadth
--speculative-ngram-match-typestr"BFS"Ngram tree-building mode: "BFS" for recency-based expansion or "PROB" for frequency-based expansion
--speculative-ngram-max-trie-depthint18Maximum suffix length stored and matched by the ngram trie
--speculative-ngram-capacityint10,000,000Cache capacity
+ +### Environment variables + + + + + + + + + + + + + + + + + + + + + + + + + + +
VariableDefaultDescription
SGLANG_ENABLE_SPEC_V2FalseEnable Speculative Decoding V2 (overlap scheduler)
SGLANG_NGRAM_FORCE_GREEDY_VERIFYFalseForce greedy verification for ngram decoding
+ +### Other related flags + + + + + + + + + + + + + + + + + + + + + + + + + + +
ParameterDescription
--enable-multi-layer-eagleEnable multi-layer EAGLE (auto-enabled for MiMoV2 and Step3p5 models)
--enable-torch-compileEnable torch.compile for kernel-level optimizations
--torch-compile-max-bsMaximum batch size for torch.compile
+ +--- + +## OOM Troubleshooting + +> [!WARNING] +> **Out of Memory (OOM)?** Speculative decoding may increase GPU memory usage because the draft tree, CUDA graphs, and verification-related buffers consume additional VRAM. If you encounter OOM errors, try the following adjustments. + +### Step 1: Lower static memory fraction (most effective) + +```bash Command +--mem-fraction-static 0.5 # when omitted, this value is auto-computed +``` + +- `--mem-fraction-static` controls the memory budget for model weights + KV cache pool. +- Lowering it directly increases dynamic headroom for activations and CUDA graph buffers. +- If omitted, SGLang auto-estimates this value from other settings, and those auto settings can still be too aggressive for some workloads. + +### Step 2: Reduce CUDA graph batch size + +```bash Command +# Fewer CUDA graph captures = less memory reserved +--cuda-graph-max-bs 4 # or even 2 for tight memory situations +``` + +- If omitted, `--cuda-graph-max-bs` is auto-selected based on GPU memory and TP size, and can be much larger on high-memory GPUs. + +### Step 3: Reduce draft tree size + +These three parameters directly control how much memory the draft tree consumes: + +```bash Command +# Before (aggressive, high memory) +--speculative-num-steps 5 --speculative-eagle-topk 8 --speculative-num-draft-tokens 64 + +# After (conservative, lower memory) +--speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 +``` + +### Step 4: Limit concurrent requests + +```bash Command +# Fewer concurrent requests lowers in-flight load and can reduce OOM risk +--max-running-requests 4 +``` + +### Quick OOM recovery recipe + +If you're hitting OOM and just want something that works, start with this minimal configuration and scale up: + +```bash Command +python3 -m sglang.launch_server \ + --model \ + --speculative-algorithm EAGLE \ + --speculative-draft-model-path \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 4 \ + --cuda-graph-max-bs 2 \ + --mem-fraction-static 0.5 \ + --max-running-requests 4 \ + --log-level warning +``` + +Then gradually increase `--speculative-num-draft-tokens`, `--speculative-eagle-topk`, and `--cuda-graph-max-bs`. Increase `--mem-fraction-static` last, only after the run is stable. + +--- + ## References EAGLE process is as follows: -- Within EAGLE the draft model predicts the next feature vector, i.e. the last hidden state of the original LLM, using the feature sequence $(f_1, ..., f_k)$ and the token sequence $(t_2, ..., t_{k+1})$. -- The next token is then sampled from $p_{k+2}=\text{LMHead}(f_{k+1})$. Afterwards, the two sequences are extended in a tree style—branching out multiple potential continuations, with the branching factor per step controlled by the `speculative_eagle_topk` parameter—to ensure a more coherent connection of context, and are given as input again. -- EAGLE-2 additionally uses the draft model to evaluate how probable certain branches in the draft tree are, dynamically stopping the expansion of unlikely branches. After the expansion phase, reranking is employed to select only the top `speculative_num_draft_tokens` final nodes as draft tokens. +- Within EAGLE the draft model predicts the next feature vector, i.e. the last hidden state of the original LLM, using the feature sequence $(f_1, ..., f_k)$ and the token sequence $(t_2, ..., t_{k+1})$. +- The next token is then sampled from $p_{k+2}=\text{LMHead}(f_{k+1})$. Afterwards, the two sequences are extended in a tree style—branching out multiple potential continuations, with the branching factor per step controlled by the `speculative_eagle_topk` parameter—to ensure a more coherent connection of context, and are given as input again. +- In SGLang's EAGLE-2 implementation, the draft tree is expanded for the configured steps and then reranked to select the top `speculative_num_draft_tokens` final nodes as draft tokens. - EAGLE-3 removes the feature prediction objective, incorporates low and mid-layer features, and is trained in an on-policy manner. -This enhances drafting accuracy by operating on the features instead of tokens for more regular inputs and passing the tokens from the next timestep additionally to minimize randomness effects from sampling. Furthermore the dynamic adjustment of the draft tree and selection of reranked final nodes increases acceptance rate of draft tokens further. For more details see [EAGLE-2](https://arxiv.org/abs/2406.16858) and [EAGLE-3](https://arxiv.org/abs/2503.01840) paper. +This enhances drafting accuracy by operating on features instead of tokens for more regular inputs and by additionally passing tokens from the next timestep to reduce sampling randomness. For more details, see the [EAGLE-2](https://arxiv.org/abs/2406.16858) and [EAGLE-3](https://arxiv.org/abs/2503.01840) papers. - -For guidance how to train your own EAGLE model please see the [EAGLE repo](https://github.com/SafeAILab/EAGLE/tree/main?tab=readme-ov-file#train). +For guidance on how to train your own EAGLE model please see the [EAGLE repo](https://github.com/SafeAILab/EAGLE/tree/main?tab=readme-ov-file#train). For EAGLE-3 training specifically, check out [SpecForge](https://github.com/sgl-project/SpecForge), the SGLang team's training framework designed for EAGLE-3 speculative decoding models with seamless porting to SGLang serving. See the [SpecForge documentation](https://docs.sglang.ai/SpecForge/) and [blog post](https://lmsys.org/blog/2025-07-25-spec-forge) for details. diff --git a/docs_new/docs/advanced_features/structured_outputs.ipynb b/docs_new/docs/advanced_features/structured_outputs.ipynb index bd29a0b3e..8902c9497 100644 --- a/docs_new/docs/advanced_features/structured_outputs.ipynb +++ b/docs_new/docs/advanced_features/structured_outputs.ipynb @@ -19,7 +19,7 @@ "- [Outlines](https://github.com/dottxt-ai/outlines): Supports JSON schema and regular expression constraints.\n", "- [Llguidance](https://github.com/guidance-ai/llguidance): Supports JSON schema, regular expression, and EBNF constraints.\n", "\n", - "We suggest using XGrammar for its better performance and utility. XGrammar currently uses the [GGML BNF format](https://github.com/ggerganov/llama.cpp/blob/master/grammars/README). For more details, see [XGrammar technical overview](https://blog.mlc.ai/2024/11/22/achieving-efficient-flexible-portable-structured-generation-with-xgrammar).\n", + "We suggest using XGrammar for its better performance and utility. XGrammar currently uses the [GGML BNF format](https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md). For more details, see [XGrammar technical overview](https://blog.mlc.ai/2024/11/22/achieving-efficient-flexible-portable-structured-generation-with-xgrammar).\n", "\n", "To use Outlines, simply add `--grammar-backend outlines` when launching the server.\n", "To use llguidance, add `--grammar-backend llguidance` when launching the server.\n", @@ -54,7 +54,7 @@ " \"python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --host 0.0.0.0 --log-level warning\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")\n", + "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n", "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")" ] }, @@ -356,8 +356,7 @@ "outputs": [], "source": [ "# Support for XGrammar latest structural tag format\n", - "# https://xgrammar.mlc.ai/docs/tutorials/structural_tag.html\n", - "\n", + "# \n", "response = client.chat.completions.create(\n", " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", " messages=messages,\n", @@ -645,8 +644,7 @@ "outputs": [], "source": [ "# Support for XGrammar latest structural tag format\n", - "# https://xgrammar.mlc.ai/docs/tutorials/structural_tag.html\n", - "\n", + "# \n", "payload = {\n", " \"text\": text,\n", " \"sampling_params\": {\n", @@ -925,8 +923,7 @@ "outputs": [], "source": [ "# Support for XGrammar latest structural tag format\n", - "# https://xgrammar.mlc.ai/docs/tutorials/structural_tag.html\n", - "\n", + "# \n", "sampling_params = {\n", " \"temperature\": 0.8,\n", " \"top_p\": 0.95,\n", diff --git a/docs_new/docs/advanced_features/structured_outputs.mdx b/docs_new/docs/advanced_features/structured_outputs.mdx index cb9b0b464..26e450bfd 100644 --- a/docs_new/docs/advanced_features/structured_outputs.mdx +++ b/docs_new/docs/advanced_features/structured_outputs.mdx @@ -11,7 +11,7 @@ SGLang supports three grammar backends: - [Outlines](https://github.com/dottxt-ai/outlines): Supports JSON schema and regular expression constraints. - [Llguidance](https://github.com/guidance-ai/llguidance): Supports JSON schema, regular expression, and EBNF constraints. -We suggest using XGrammar for its better performance and utility. XGrammar currently uses the [GGML BNF format](https://github.com/ggerganov/llama.cpp/blob/master/grammars/README). For more details, see [XGrammar technical overview](https://blog.mlc.ai/2024/11/22/achieving-efficient-flexible-portable-structured-generation-with-xgrammar). +We suggest using XGrammar for its better performance and utility. XGrammar currently uses the [GGML BNF format](https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md). For more details, see [XGrammar technical overview](https://blog.mlc.ai/2024/11/22/achieving-efficient-flexible-portable-structured-generation-with-xgrammar). To use Outlines, simply add `--grammar-backend outlines` when launching the server. To use llguidance, add `--grammar-backend llguidance` when launching the server. @@ -247,9 +247,9 @@ If a you choose to call a function ONLY reply in the following format: where start_tag => ` a JSON dict with the function argument name as key and function argument value as value. -end_tag => `</function>` +end_tag => `` Here is an example, -<function=example_function_name>{{"example_name": "example_value"}}</function> +{{"example_name": "example_value"}} Reminder: - Function calls MUST follow the specified format - Required parameters MUST be specified @@ -274,17 +274,17 @@ response = client.chat.completions.create( "type": "structural_tag", "structures": [ { - "begin": "<function=get_current_weather>", + "begin": "", "schema": schema_get_current_weather, - "end": "</function>", + "end": "", }, { - "begin": "<function=get_current_date>", + "begin": "", "schema": schema_get_current_date, - "end": "</function>", + "end": "", }, ], - "triggers": ["<function="], + "triggers": ["", + "begin": "", "content": { "type": "json_schema", "json_schema": schema_get_current_weather, }, - "end": "</function>", + "end": "", }, { - "begin": "<function=get_current_date>", + "begin": "", "content": { "type": "json_schema", "json_schema": schema_get_current_date, }, - "end": "</function>", + "end": "", }, ], "at_least_one": False, @@ -506,17 +506,17 @@ payload = { "type": "structural_tag", "structures": [ { - "begin": "<function=get_current_weather>", + "begin": "", "schema": schema_get_current_weather, - "end": "</function>", + "end": "", }, { - "begin": "<function=get_current_date>", + "begin": "", "schema": schema_get_current_date, - "end": "</function>", + "end": "", }, ], - "triggers": ["<function="], + "triggers": ["", + "begin": "", "content": { "type": "json_schema", "json_schema": schema_get_current_weather, }, - "end": "</function>", + "end": "", }, { - "begin": "<function=get_current_date>", + "begin": "", "content": { "type": "json_schema", "json_schema": schema_get_current_date, }, - "end": "</function>", + "end": "", }, ], "at_least_one": False, @@ -727,17 +727,17 @@ sampling_params = { "type": "structural_tag", "structures": [ { - "begin": "<function=get_current_weather>", + "begin": "", "schema": schema_get_current_weather, - "end": "</function>", + "end": "", }, { - "begin": "<function=get_current_date>", + "begin": "", "schema": schema_get_current_date, - "end": "</function>", + "end": "", }, ], - "triggers": ["<function="], + "triggers": ["", + "begin": "", "content": { "type": "json_schema", "json_schema": schema_get_current_weather, }, - "end": "</function>", + "end": "", }, { - "begin": "<function=get_current_date>", + "begin": "", "content": { "type": "json_schema", "json_schema": schema_get_current_date, }, - "end": "</function>", + "end": "", }, ], "at_least_one": False, diff --git a/docs_new/docs/advanced_features/structured_outputs_for_reasoning_models.ipynb b/docs_new/docs/advanced_features/structured_outputs_for_reasoning_models.ipynb index d05f9e950..cfc07fd01 100644 --- a/docs_new/docs/advanced_features/structured_outputs_for_reasoning_models.ipynb +++ b/docs_new/docs/advanced_features/structured_outputs_for_reasoning_models.ipynb @@ -50,7 +50,7 @@ " \"python -m sglang.launch_server --model-path deepseek-ai/DeepSeek-R1-Distill-Qwen-7B --host 0.0.0.0 --reasoning-parser deepseek-r1 --log-level warning\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")\n", + "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n", "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")" ] }, diff --git a/docs_new/docs/advanced_features/structured_outputs_for_reasoning_models.mdx b/docs_new/docs/advanced_features/structured_outputs_for_reasoning_models.mdx index 22541762e..92033300b 100644 --- a/docs_new/docs/advanced_features/structured_outputs_for_reasoning_models.mdx +++ b/docs_new/docs/advanced_features/structured_outputs_for_reasoning_models.mdx @@ -3,17 +3,17 @@ title: "Structured Outputs For Reasoning Models" metatags: description: "SGLang structured outputs for reasoning models: free-form thinking with constrained final output for DeepSeek R1, QwQ models." --- -When working with reasoning models that use special tokens like `<think>...</think>` to denote reasoning sections, you might want to allow free-form text within these sections while still enforcing grammar constraints on the rest of the output. +When working with reasoning models that use special tokens like `...` to denote reasoning sections, you might want to allow free-form text within these sections while still enforcing grammar constraints on the rest of the output. SGLang provides a feature to disable grammar restrictions within reasoning sections. This is particularly useful for models that need to perform complex reasoning steps before providing a structured output. -To enable this feature, use the `--reasoning-parser` flag which decide the think_end_token, such as `</think>`, when launching the server. You can also specify the reasoning parser using the `--reasoning-parser` flag. +To enable this feature, use the `--reasoning-parser` flag which decide the think_end_token, such as ``, when launching the server. You can also specify the reasoning parser using the `--reasoning-parser` flag. ## Supported Models Currently, SGLang supports the following reasoning models: -- [DeepSeek R1 series](https://huggingface.co/collections/deepseek-ai/deepseek-r1-678e1e131c0169c0bc89728d): The reasoning content is wrapped with `<think>` and `</think>` tags. -- [QwQ](https://huggingface.co/Qwen/QwQ-32B): The reasoning content is wrapped with `<think>` and `</think>` tags. +- [DeepSeek R1 series](https://huggingface.co/collections/deepseek-ai/deepseek-r1-678e1e131c0169c0bc89728d): The reasoning content is wrapped with `` and `` tags. +- [QwQ](https://huggingface.co/Qwen/QwQ-32B): The reasoning content is wrapped with `` and `` tags. ## Usage @@ -252,9 +252,9 @@ If a you choose to call a function ONLY reply in the following format: where start_tag => ` a JSON dict with the function argument name as key and function argument value as value. -end_tag => `</function>` +end_tag => `` Here is an example, -<function=example_function_name>{{"example_name": "example_value"}}</function> +{{"example_name": "example_value"}} Reminder: - Function calls MUST follow the specified format - Required parameters MUST be specified @@ -280,17 +280,17 @@ response = client.chat.completions.create( "max_new_tokens": 2048, "structures": [ { - "begin": "<function=get_current_weather>", + "begin": "", "schema": schema_get_current_weather, - "end": "</function>", + "end": "", }, { - "begin": "<function=get_current_date>", + "begin": "", "schema": schema_get_current_date, - "end": "</function>", + "end": "", }, ], - "triggers": ["<function="], + "triggers": ["")[0] +content = response.json()["text"].split("")[1] print_highlight(f"reasoing_content: {reasoing_content}\n\ncontent: {content}") ``` @@ -460,17 +460,17 @@ payload = { "type": "structural_tag", "structures": [ { - "begin": "<function=get_current_weather>", + "begin": "", "schema": schema_get_current_weather, - "end": "</function>", + "end": "", }, { - "begin": "<function=get_current_date>", + "begin": "", "schema": schema_get_current_date, - "end": "</function>", + "end": "", }, ], - "triggers": ["<function="], + "triggers": ["", + "begin": "", "schema": schema_get_current_weather, - "end": "</function>", + "end": "", }, { - "begin": "<function=get_current_date>", + "begin": "", "schema": schema_get_current_date, - "end": "</function>", + "end": "", }, ], - "triggers": ["<function="], + "triggers": [" You can replace `deepseek-ai/DeepSeek-OCR-2` with `deepseek-ai/DeepSeek-OCR`. + +## Prompt examples + +Recommended prompts from the model card: + +``` + +<|grounding|>Convert the document to markdown. +``` + +``` + +Free OCR. +``` + +## OpenAI-compatible request example + +```python +import requests + +url = "http://localhost:30000/v1/chat/completions" + +data = { + "model": "deepseek-ai/DeepSeek-OCR-2", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "\n<|grounding|>Convert the document to markdown."}, + {"type": "image_url", "image_url": {"url": "https://example.com/your_image.jpg"}}, + ], + } + ], + "max_tokens": 512, +} + +response = requests.post(url, json=data) +print(response.text) +``` diff --git a/docs_new/docs/basic_usage/deepseek_v3.mdx b/docs_new/docs/basic_usage/deepseek_v3.mdx index 78fd62d91..7d7c93d29 100644 --- a/docs_new/docs/basic_usage/deepseek_v3.mdx +++ b/docs_new/docs/basic_usage/deepseek_v3.mdx @@ -25,86 +25,125 @@ To run DeepSeek V3.1/V3/R1 models, the recommended settings are as follows: - **Full precision [FP8](https://huggingface.co/deepseek-ai/DeepSeek-R1-0528)** *(recommended)* + Full precision FP8<br>*(recommended)* 8 x H200 + 8 x B200 + 8 x MI300X + 2 x 8 x H100/800/20 + Xeon 6980P CPU - **Full precision ([BF16](https://huggingface.co/unsloth/DeepSeek-R1-0528-BF16))** (upcast from original FP8) + Full precision (BF16) (upcast from original FP8) 2 x 8 x H200 + 2 x 8 x MI300X + 4 x 8 x H100/800/20 + 4 x 8 x A100/A800 - **Quantized weights ([INT8](https://huggingface.co/meituan/DeepSeek-R1-Channel-INT8))** + Quantized weights (INT8) 16 x A100/800 + 32 x L40S + Xeon 6980P CPU + 4 x Atlas 800I A3 - **Quantized weights ([W4A8](https://huggingface.co/novita/Deepseek-R1-0528-W4AFP8))** + Quantized weights (W4A8) 8 x H20/100, 4 x H200 - **Quantized weights ([AWQ](https://huggingface.co/QuixiAI/DeepSeek-R1-0528-AWQ))** + Quantized weights (AWQ) 8 x H100/800/20 + 8 x A100/A800 - **Quantized weights ([MXFP4](https://huggingface.co/amd/DeepSeek-R1-MXFP4-Preview))** + Quantized weights (MXFP4) 8, 4 x MI355X/350X - **Quantized weights ([NVFP4](https://huggingface.co/nvidia/DeepSeek-R1-0528-NVFP4-v2))** + Quantized weights (NVFP4) 8, 4 x B200 + + + The official DeepSeek V3 is already in FP8 format, so you should not run it with any quantization arguments like `--quantization fp8`. - + Detailed commands for reference: - [8 x H200](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#using-docker-recommended) - [4 x B200, 8 x B200](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-one-b200-node) -- [8 x MI300X](../hardware-platforms/amd-gpus#running-deepseek-v3) -- [2 x 8 x H200](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-two-h208-nodes) +- [8 x MI300X](../hardware-platforms/amd_gpu#running-deepseek-v3) +- [2 x 8 x H200](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-two-h2008-nodes-and-docker) - [4 x 8 x A100](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-four-a1008-nodes) - [8 x A100 (AWQ)](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-8-a100a800-with-awq-quantization) - [16 x A100 (INT8)](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-16-a100a800-with-int8-quantization) - [32 x L40S (INT8)](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-32-l40s-with-int8-quantization) -- [Xeon 6980P CPU](../hardware-platforms/cpu-server#example-running-deepseek-v31-terminus) -- [4 x Atlas 800I A3 (int8)](../hardware-platforms/ascend-npus/DeepSeek-Examples#running-deepseek-with-pd-disaggregation-on-4-x-atlas-800i-a3) +- [Xeon 6980P CPU](../hardware-platforms/cpu_server#example-running-deepseek-r1) +- [4 x Atlas 800I A3 (int8)](../hardware-platforms/ascend-npus/ascend_npu_deepseek_example#running-deepseek-with-pd-disaggregation-on-4-x-atlas-800i-a3) ### Download Weights If you encounter errors when starting the server, ensure the weights have finished downloading. It's recommended to download them beforehand or restart multiple times until all weights are downloaded. Please refer to [DeepSeek V3](https://huggingface.co/deepseek-ai/DeepSeek-V3-Base#61-inference-with-deepseek-infer-demo-example-only) official guide to download the weights. @@ -116,7 +155,7 @@ Please refer to [the example](https://github.com/sgl-project/sglang/tree/main/be - [Deploying DeepSeek on GB200 NVL72 with PD and Large Scale EP](https://lmsys.org/blog/2025-06-16-gb200-part-1/) ([Part I](https://lmsys.org/blog/2025-06-16-gb200-part-1/), [Part II](https://lmsys.org/blog/2025-09-25-gb200-part-2/)) - Comprehensive guide on GB200 optimizations. -- [Deploying DeepSeek with PD Disaggregation and Large-Scale Expert Parallelism on 96 H100 GPUs](https://lmsys.org/blog/2025-05-05-deepseek-pd-ep/) - Guide on PD disaggregation and large-scale EP. +- [Deploying DeepSeek with PD Disaggregation and Large-Scale Expert Parallelism on 96 H100 GPUs](https://lmsys.org/blog/2025-05-05-large-scale-ep/) - Guide on PD disaggregation and large-scale EP. - [Serving with two H20*8 nodes](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-two-h208-nodes). @@ -144,9 +183,9 @@ Please refer to [the example](https://github.com/sgl-project/sglang/tree/main/be Overall, with these optimizations, we have achieved up to **7x** acceleration in output throughput compared to the previous version. - - Multi-head Latent Attention for DeepSeek Series Models - +

+ Multi-head Latent Attention for DeepSeek Series Models +

**Usage**: MLA optimization is enabled by default. @@ -156,15 +195,15 @@ Overall, with these optimizations, we have achieved up to **7x** acceleration in **Description**: This optimization involves data parallelism (DP) for the MLA attention mechanism of DeepSeek Series Models, which allows for a significant reduction in the KV cache size, enabling larger batch sizes. Each DP worker independently handles different types of batches (prefill, decode, idle), which are then synchronized before and after processing through the Mixture-of-Experts (MoE) layer. If you do not use DP attention, KV cache will be duplicated among all TP ranks. - - Data Parallelism Attention for DeepSeek Series Models - +

+ Data Parallelism Attention for DeepSeek Series Models +

With data parallelism attention enabled, we have achieved up to **1.9x** decoding throughput improvement compared to the previous version. - - Data Parallelism Attention Performance Comparison - +

+ Data Parallelism Attention Performance Comparison +

**Usage**: - Append `--enable-dp-attention --tp 8 --dp 8` to the server arguments when using 8 H200 GPUs. This optimization improves peak throughput in high batch size scenarios where the server is limited by KV cache capacity. @@ -180,7 +219,7 @@ Data parallelism attention is not recommended for low-latency, small-batch use c **Description**: For users with limited memory on a single node, SGLang supports serving DeepSeek Series Models, including DeepSeek V3, across multiple nodes using tensor parallelism. This approach partitions the model parameters across multiple GPUs or nodes to handle models that are too large for one node's memory. -**Usage**: Check [here](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-2-h208) for usage examples. +**Usage**: Check [here](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-two-h2008-nodes-and-docker) for usage examples. ### Block-wise FP8 @@ -215,7 +254,7 @@ python3 -m sglang.launch_server \ --tp 8 ``` - The default configuration for DeepSeek models is `--speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4`. The best configuration for `--speculative-num-steps`, `--speculative-eagle-topk` and `--speculative-num-draft-tokens` can be searched with [bench_speculative.py](https://github.com/sgl-project/sglang/blob/main/scripts/playground/bench_speculative.py) script for given batch size. The minimum configuration is `--speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2`, which can achieve speedup for larger batch sizes. -- Most MLA attention backends fully support MTP usage. See [MLA Backends](../advanced_features/attention_backend.md#mla-backends) for details. +- Most MLA attention backends fully support MTP usage. See [MLA Backends](../advanced_features/attention_backend#mla-backends) for details. To enable DeepSeek MTP for large batch sizes (>48), you need to adjust some parameters (Reference [this discussion](https://github.com/sgl-project/sglang/issues/4543#issuecomment-2737413756)): @@ -250,10 +289,10 @@ python3 -m sglang.launch_server \ Sample Request: -```text Output +``` curl "http://127.0.0.1:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --d '{"temperature": 0, "max_tokens": 100, "model": "deepseek-ai/DeepSeek-V3-0324", "tools": [{"type": "function", "function": {"name": "query_weather", "description": "Get weather of an city, the user should supply a city first", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city, e.g. Beijing"}}, "required": ["city"]}}}], "messages": [{"role": "user", "content": "Hows the weather like in Qingdao today"}]}' +-d '{"temperature": 0, "max_tokens": 100, "model": "deepseek-ai/DeepSeek-V3-0324", "tools": [{"type": "function", "function": {"name": "query_weather", "description": "Get weather of a city, the user should supply a city first", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city, e.g. Beijing"}}, "required": ["city"]}}}], "messages": [{"role": "user", "content": "How'\''s the weather like in Qingdao today"}]}' ``` Expected Response @@ -263,10 +302,10 @@ Expected Response ``` Sample Streaming Request: -```text Output +``` curl "http://127.0.0.1:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --d '{"temperature": 0, "max_tokens": 100, "model": "deepseek-ai/DeepSeek-V3-0324","stream":true,"tools": [{"type": "function", "function": {"name": "query_weather", "description": "Get weather of an city, the user should supply a city first", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city, e.g. Beijing"}}, "required": ["city"]}}}], "messages": [{"role": "user", "content": "Hows the weather like in Qingdao today"}]}' +-d '{"temperature": 0, "max_tokens": 100, "model": "deepseek-ai/DeepSeek-V3-0324","stream":true,"tools": [{"type": "function", "function": {"name": "query_weather", "description": "Get weather of a city, the user should supply a city first", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city, e.g. Beijing"}}, "required": ["city"]}}}], "messages": [{"role": "user", "content": "How'\''s the weather like in Qingdao today"}]}' ``` Expected Streamed Chunks (simplified for clarity): ```text Output @@ -284,10 +323,11 @@ The client needs to concatenate all arguments fragments to reconstruct the compl ```text Output {"city": "Qingdao"} ``` - + + 1. Use a lower `"temperature"` value for better results. 2. To receive more consistent tool call results, it is recommended to use `--chat-template examples/chat_template/tool_chat_template_deepseekv3.jinja`. It provides an improved unified prompt. - + ### Thinking Budget for DeepSeek R1 @@ -302,7 +342,6 @@ python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-R1 --tp 8 --port 30 Sample Request: - ```python Sample Request import openai from rich.pretty import pprint @@ -328,7 +367,6 @@ response = client.chat.completions.create( ) pprint(response) ``` - ## FAQ diff --git a/docs_new/docs/basic_usage/deepseek_v32.mdx b/docs_new/docs/basic_usage/deepseek_v32.mdx index 3a9b1643b..1077a9956 100644 --- a/docs_new/docs/basic_usage/deepseek_v32.mdx +++ b/docs_new/docs/basic_usage/deepseek_v32.mdx @@ -1,13 +1,12 @@ --- -title: "DeepSeek V3.2 Usage" +title: "DeepSeek V3.2/GLM-5 Usage" metatags: - description: "Deploy DeepSeek V3.2 with SGLang: DeepSeek Sparse Attention (DSA), long-context optimization, MTP speculative decoding, function calling. Supports H200, B200, MI300X, MI350." + description: "Deploy DeepSeek V3.2/GLM-5 with SGLang: DeepSeek Sparse Attention (DSA), long-context optimization, MTP speculative decoding, function calling. Supports H200, B200, MI300X, MI350." --- DeepSeek-V3.2 model family equips DeepSeek-V3.1-Terminus with DeepSeek Sparse Attention (DSA) through continued training. With DSA, a fine-grained sparse attention mechanism powered by a lightning indexer, DeepSeek-V3.2 achieves efficiency improvements in long-context scenarios. -For reporting issues or tracking upcoming features, please refer to this [Roadmap](https://github.com/sgl-project/sglang/issues/11060). -Note: This document is originally written for the usage of [DeepSeek-V3.2-Exp](https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp) model. The usage of [DeepSeek-V3.2](https://huggingface.co/deepseek-ai/DeepSeek-V3.2) or [DeepSeek-V3.2-Speciale](https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Speciale) is the same as DeepSeek-V3.2-Exp except for the tool call parser. +Note: This document is originally written for the usage of [DeepSeek-V3.2-Exp](https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp) model. The usage of [DeepSeek-V3.2](https://huggingface.co/deepseek-ai/DeepSeek-V3.2) or [DeepSeek-V3.2-Speciale](https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Speciale) is the same as DeepSeek-V3.2-Exp except for the tool call parser. [GLM-5](https://huggingface.co/zai-org/GLM-5) model also applies DSA (DeepSeek Sparse Attention) structure, so it can share most of the usage here, except for the reasoning parser and tool call parser. ## Installation @@ -41,7 +40,8 @@ cd sglang pip3 install pip --upgrade pip3 install -e "python" ``` -## Launch DeepSeek V3.2 with SGLang + +## Launch DeepSeek V3.2/GLM-5 with SGLang To serve [DeepSeek-V3.2-Exp](https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp) on 8xH200/B200 GPUs: @@ -59,19 +59,25 @@ python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --nsa-prefill-backend tilelang --nsa-decode-backend tilelang ``` +To serve GLM-5, just replace the `--model` argument with `zai-org/GLM-5-FP8`. + ### Configuration Tips -- **DP Attention (Recommended)**: For DeepSeek V3.2 model, the kernels are customized for the use case of `dp_size=8`, so DP attention (`--dp 8 --enable-dp-attention`) is the recommended configuration for better stability and performance. All test cases use this configuration by default. -- **Pure TP Mode**: Launching with pure TP (without `--dp` and `--enable-dp-attention`) is also supported. Note that this mode has not been fully validated in PD disaggregation scenarios. -- **Short-sequence MHA prefill (adaptive)**: For short prefill sequences (default threshold: **2048 tokens**), the NSA backend uses standard MHA automatically (no extra flags). On H200 (SM90) this path uses the FlashAttention variable-length kernel; on B200 (SM100) it uses TRT-LLM ragged MHA. MHA uses `MHA_ONE_SHOT` for best performance. `MHA_ONE_SHOT` computes multi-head attention over all tokens (both cached prefix and newly extended tokens) in a single kernel invocation, avoiding the overhead of chunked KV cache processing. This achieves optimal throughput for short sequences where total sequence length fits within the chunk capacity limit. +- **DP Attention**: To enable [DP Attention](../advanced_features/dp_dpa_smg_guide), please include `--enable-dp-attention --dp ` in command. DP Attention is better for large concurrency scenarios. +- **TP Attention**: Launching with TP attention is also supported. TP attention is better for low latency scenarios. +- **Short-sequence MHA prefill (adaptive)**: For short prefill sequences (default threshold: **2048 tokens**), the NSA backend uses standard MHA automatically (no extra flags). On H200 (SM90) this path uses the FlashAttention variable-length kernel; on B200 (SM100) it uses TRT-LLM ragged MHA. MHA uses `MHA_ONE_SHOT` for best performance, which computes multi-head attention over all tokens (both cached prefix and newly extended tokens) in a single kernel invocation, avoiding the overhead of chunked KV cache processing. This achieves optimal throughput for short sequences where total sequence length fits within the chunk capacity limit. +- **MHA prefill threshold relaxation**: To apply MHA attention to requests longer than 2048 tokens, please set the flag `SGLANG_NSA_PREFILL_DENSE_ATTN_KV_LEN_THRESHOLD` to a value larger than 2048. As threshold grows larger, the prefill performance can be improved, but at the cost of potential accuracy drop. - **Choices of Attention Kernels**: The attention backend is automatically set to `nsa` attention backend for DeepSeek V3.2 model. In this backend, different kernels for sparse prefilling/decoding are implemented, which can be specified by `--nsa-prefill-backend` and `--nsa-decode-backend` server arguments. The choices of nsa prefill/decode attention kernels include: - `flashmla_sparse`: `flash_mla_sparse_fwd` kernel from `flash_mla` library. Can run on both Hopper and Blackwell GPUs. It requires bf16 q, kv inputs. - `flashmla_kv`: `flash_mla_with_kvcache` kernel from `flash_mla` library. Can run on both Hopper and Blackwell GPUs. It requires bf16 q, fp8 k_cache inputs. + - `flashmla_auto`: enables automatic selection of either `flashmla_sparse` or `flashmla_kv` kernel for prefill based on KV cache dtype, hardware, and heuristics. With BF16 KV cache, `flashmla_sparse` is always used on both Hopper and Blackwell. With FP8 KV cache: On Hopper (SM90), it unconditionally uses `flashmla_kv`; On Blackwell (SM100), it uses `flashmla_sparse` when `total_kv_tokens < total_q_tokens * 512`, otherwise falls back to `flashmla_kv`. The heuristics may need to be tuned if the performance of either kernel changes significantly. - `fa3`: `flash_attn_with_kvcache` kernel from `flash_attn` library. Can only run on Hopper GPUs. It requires bf16 q, kv inputs. - `tilelang`: `tilelang` implementation that can run on GPU, HPU and NPU. - `aiter`: Aiter kernel on AMD HPUs. Can only be used as decode kernel. -- On the basis of performance benchmarks, the default configuration on H200 and B200 are set as follows : - - H200: `flashmla_sparse` prefill attention (short-seq prefill uses MHA via FlashAttention varlen), `fa3` decode attention, `bf16` kv cache dtype. - - B200: `flashmla_auto` prefill attention (short-seq prefill uses MHA via TRT-LLM ragged), `flashmla_kv` decode attention, `fp8_e4m3` kv cache dtype. `flashmla_auto` enables automatic selection of either `flashmla_sparse` or `flashmla_kv` kernel for prefill based on KV cache dtype, hardware, and heuristics. When FP8 KV cache is enabled and `total_kv_tokens < total_q_tokens * 512`, it uses the `flashmla_sparse` kernel; otherwise, it falls back to the `flashmla_kv` kernel. The heuristics may need to be tuned if the performance of either the `flashmla_sparse` or `flashmla_kv` kernel changes significantly. + - `trtllm`: `trtllm-mla` sparse kernel from flashinfer library. Only run on blackwell GPUs. It requires q,k,v to be uniformly bf16 or fp8_e4m3 format. + - On the basis of performance benchmarks, the default configuration of DSA kernels on Hopper and Blackwell are set as follows : + - Bfloat 16 kv cache: On Hopper, `flashmla_sparse` prefill attention, `fa3` decode attention; On Blackwell, `flashmla_sparse` prefill attention, `trtllm` decode attention + - Float8_e4m3fn KV cache: On Hopper, `flashmla_kv` prefill attention, `flashmla_kv` decode attention; On Blackwell, `trtllm` prefill attention and `trtllm` decode attention. +- **Index Cache**: Introduce in [this paper](https://arxiv.org/abs/2603.12201), IndexCache improves speed by reusing the result of indexer across different layers, only at cost of negligible accuracy loss. For **GLM-5** model, we recommend appending `--json-model-override-args '{"index_topk_pattern": "FFSFSSSFSSFFFSSSFFFSFSSSSSSFFSFFSFFSSFFFFFFSFFFFFSFFSSSSSSFSFFFSFSSSFSFFSFFSSS"}'` to command for better tradeoff between speedup and performance. ## Multi-token Prediction SGLang implements Multi-Token Prediction (MTP) for DeepSeek V3.2 based on [EAGLE speculative decoding](../advanced_features/speculative_decoding#EAGLE-Decoding). With this optimization, the decoding speed can be improved significantly on small batch sizes. Please look at [this PR](https://github.com/sgl-project/sglang/pull/11652) for more information. @@ -90,7 +96,7 @@ python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --sp - The default value of `--max-running-requests` is set to `48` for MTP. For larger batch sizes, this value should be increased beyond the default value. -To enable the experimental overlap scheduler for EAGLE speculative decoding, set the environment variable `SGLANG_ENABLE_SPEC_V2=1`. This can improve performance by enabling overlap scheduling between draft and verification stages. +To enable overlap scheduler for EAGLE speculative decoding, we recommend setting the environment variable `SGLANG_ENABLE_SPEC_V2=1`. This can improve performance by enabling overlap scheduling between draft and verification stages. @@ -98,10 +104,7 @@ To enable the experimental overlap scheduler for EAGLE speculative decoding, set The usage of function calling and reasoning parser is the same as DeepSeek V3.1. Please refer to [Reasoning Parser](../advanced_features/separate_reasoning) and [Tool Parser](../advanced_features/tool_parser) documents. To launch `DeepSeek-V3.2-Exp` with function calling and reasoning parser: - -It is recommended to specify the chat-template, ensuring that you are within the sglang's root directory. - - +> Note: It is recommended to specify the chat-template, ensuring that you are within the sglang's root directory. ```bash Command python3 -m sglang.launch_server \ --model-path deepseek-ai/DeepSeek-V3.2-Exp \ @@ -122,7 +125,7 @@ python3 -m sglang.launch_server \ --reasoning-parser deepseek-v3 ``` -`DeepSeek-V3.2-Speciale` doesn't support tool calling, so can only be launched with reasoning parser: +`DeepSeek-V3.2-Speciale` does not support tool calling, so it can only be launched with the reasoning parser: ```bash Command python3 -m sglang.launch_server \ --model-path deepseek-ai/DeepSeek-V3.2-Speciale \ @@ -131,6 +134,23 @@ python3 -m sglang.launch_server \ --reasoning-parser deepseek-v3 ``` +To launch `GLM-5` with function calling and reasoning parser: +```bash Command +python -m sglang.launch_server \ + --model zai-org/GLM-5-FP8 \ + --tp-size 8 --dp-size 8 --enable-dp-attention \ + --tool-call-parser glm47 \ + --reasoning-parser glm45 \ +``` + +## NVFP4 Checkpoint + +To launch deepseek v3.2 [NVFP4 checkpoint](https://huggingface.co/nvidia/DeepSeek-V3.2-NVFP4) on Blackwell devices, the user needs to specify the quantization method as `modelopt_fp4`, and moe runner backend as one of `flashinfer_trtllm`(recommended), `flashinfer_cutlass` and `flashinfer_cutedsl`. Any other usage (parallelism, reasoning parser, ...) is the same as FP8 checkpoint. + +An example launching command can be: +```bash Command +python -m sglang.launch_server --model nvidia/DeepSeek-V3.2-NVFP4 --tp 4 --quantization modelopt_fp4 --moe-runner-backend flashinfer_trtllm --tool-call-parser deepseekv32 --reasoning-parser deepseek-v3 +``` ## PD Disaggregation @@ -174,7 +194,7 @@ python -m sglang_router.launch_router --pd-disaggregation \ --port 8000 \ ``` -If you need more advanced deployment methods or production-ready deployment methods, such as RBG or LWS-based deployment, please refer to [references/multi_node_deployment/rbg_pd/deepseekv32_pd](../references/multi_node_deployment/rbg_pd/deepseekv32_pd). Additionally, you can also find startup commands for DeepEP-based EP parallelism in the aforementioned documentation. +If you need more advanced deployment methods or production-ready deployment methods, such as RBG or LWS-based deployment, please refer to [references/multi_node_deployment/rbg_pd/deepseekv32_pd.md](../references/multi_node_deployment/rbg_pd/deepseekv32_pd). Additionally, you can also find startup commands for DeepEP-based EP parallelism in the aforementioned documentation. ## Benchmarking Results @@ -215,7 +235,7 @@ Repeat: 8, mean: 0.797 Scores: ['0.808', '0.798', '0.808', '0.798', '0.783', '0.788', '0.803', '0.793'] ``` -For Deepseek V3.2, Deepseek recommends setting the sampling parameters to temperature = 1.0, top_p = 0.95: +For DeepSeek V3.2, DeepSeek recommends setting the sampling parameters to temperature = 1.0, top_p = 0.95: ```bash Command python3 -m sglang.test.run_eval --port 30000 --eval-name gpqa --num-examples 198 --max-tokens 128000 --repeat 8 --top-p 0.95 --temperature 1.0 --thinking-mode deepseek-v3 @@ -223,13 +243,13 @@ python3 -m sglang.test.run_eval --port 30000 --eval-name gpqa --num-examples 198 Repeat: 8, mean: 0.840 Scores: ['0.848', '0.808', '0.848', '0.838', '0.879', '0.813', '0.838', '0.848'] ``` -which matches the official score, 0.824, as reported in the [Deepseek-V3.2 technical report](https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/main/assets/paper.pdf). +which matches the official score, 0.824, as reported in the [DeepSeek-V3.2 technical report](https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/main/assets/paper.pdf). ### Accuracy Test with `aime 2025` Prepare the environment by installing NeMo-Skills in the docker or your own virtual environment: - ```text Output + ``` pip install git+https://github.com/NVIDIA/NeMo-Skills.git --ignore-installed blinker ``` @@ -272,7 +292,7 @@ ns eval \ Test results (8*B200): -DeepSeek-V3.2-Exp： +DeepSeek-V3.2-Exp: @@ -420,23 +440,19 @@ DeepSeek-V3.2-Speciale:
- ## DSA long sequence context parallel optimization(experimental) **Note: This feature is only verified on Hopper machines** For context parallel in DeepSeek V3.2 model, we provide two different modes of splitting tokens, which can be controlled with argument `--nsa-prefill-cp-mode`. -### In sequence splitting (default setting) +### In sequence splitting -The first mode can be enabled by `--nsa-prefill-cp-mode in-seq-split`. This mode implements context parallel for DSA by splitting the sequence uniformly between context parallel ranks. At attention stage, each cp rank computes the indexer results of sharded sequence, and collects the whole kv cache through all gather operator. +The first mode can be enabled by `--nsa-prefill-cp-mode in-seq-split`. This mode implements context parallel for DSA by splitting the sequence uniformly between context parallel ranks. At attention stage, each cp rank computes the indexer results of sharded sequence, and collects the whole kv cache through all gather operator. Add `attn_cp_size` for communication group for context parallel. -The communication group for context parallel reuses the one for attention tp, thus `cp_size` equals `atten_tp_size = tp_size / dp_size`. - -Note that in sequence splitting mode has the following restrictions: +Note that the in-sequence splitting mode has the following restrictions: - The batch size is restricted to 1 for prefill batches -- Multi-node/PD disaggregation is still not supported -- `moe_dense_tp_size=1`, `kv_cache_dtype = "bf16"`, `moe_a2a_backend = "deepep"` +- `moe_dense_tp_size=1`, `moe_a2a_backend = "deepep"` - To ensure `cp_size > 1`, the passed in `tp_size` must be larger than `dp_size` For more details, please refer to PR https://github.com/sgl-project/sglang/pull/12065. @@ -444,21 +460,21 @@ For more details, please refer to PR https://github.com/sgl-project/sglang/pull/ Example: ```bash Command # In-seq splitting mode launched with EP + DP -python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --ep 8 --dp 2 --enable-dp-attention --enable-nsa-prefill-context-parallel --nsa-prefill-cp-mode in-seq-split --max-running-requests 32 +python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --ep 8 --dp 2 --enable-dp-attention --enable-nsa-prefill-context-parallel --attn-cp-size 4 --nsa-prefill-cp-mode in-seq-split --max-running-requests 32 ``` -### Round robin splitting +### Round robin splitting (default setting) This mode can be enabled by specifying the parameter `--nsa-prefill-cp-mode round-robin-split`, which distributes tokens across ranks based on `token_idx % cp_size`. -In this scenario, compared with the aforementioned method, it additionally supports the fused MoE backend (the fused MoE backend may deliver better performance than DeepEP in single-machine scenarios), FP8 KV-cache, and multi-batch prefill inference. But it cannot be enabled with dp attention together. +In this scenario, compared to the in-sequence splitting method, it additionally supports the fused MoE backend (the fused MoE backend may deliver better performance than DeepEP in single-machine scenarios), FP8 KV-cache, and multi-batch prefill inference. However, it cannot be enabled with DP attention together. For more details, please refer to PR https://github.com/sgl-project/sglang/pull/13959. Example usage: ```bash Command # Launch with FusedMoe + CP8 -python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --enable-nsa-prefill-context-parallel --nsa-prefill-cp-mode round-robin-split --max-running-requests 32 +python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --enable-nsa-prefill-context-parallel --attn-cp-size 8 --nsa-prefill-cp-mode round-robin-split --max-running-requests 32 ``` ### Pipeline Parallel + Context Parallel (PP + CP) @@ -482,6 +498,7 @@ python3 -m sglang.launch_server \ --tp 8 --pp-size 2 \ --dp-size 1 --moe-dense-tp-size 1 \ --enable-nsa-prefill-context-parallel \ + --attn-cp-size 8 \ --nsa-prefill-cp-mode round-robin-split \ --trust-remote-code \ --disable-radix-cache \ @@ -505,6 +522,7 @@ python3 -m sglang.launch_server \ --tp 8 --pp-size 2 \ --dp-size 1 --moe-dense-tp-size 1 \ --enable-nsa-prefill-context-parallel \ + --attn-cp-size 8 \ --nsa-prefill-cp-mode round-robin-split \ --trust-remote-code \ --disable-radix-cache \ @@ -532,6 +550,7 @@ python -m sglang.launch_server \ --tp 8 --pp-size 2 \ --dp-size 1 --moe-dense-tp-size 1 \ --enable-nsa-prefill-context-parallel \ + --attn-cp-size 8 \ --nsa-prefill-cp-mode round-robin-split \ --disaggregation-ib-device mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3 \ --trust-remote-code \ @@ -557,6 +576,7 @@ python -m sglang.launch_server \ --tp 8 --pp-size 2 \ --dp-size 1 --moe-dense-tp-size 1 \ --enable-nsa-prefill-context-parallel \ + --attn-cp-size 8 \ --nsa-prefill-cp-mode round-robin-split \ --disaggregation-ib-device mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3 \ --trust-remote-code \ @@ -573,3 +593,9 @@ python -m sglang.launch_server \ ``` For the Decode nodes, it is recommended to use the **EP mode**. + +## HiSparse: Hierarchical Sparse Attention for DSA (experimental) + +HiSparse reduces per-request GPU memory during decode by keeping only a small "hot" KV buffer on GPU while storing complete KV data in CPU pinned memory. A CUDA kernel dynamically swaps in the top-k most relevant KV entries from host memory on each decode step. This enables significantly higher decode concurrency for long-context DSA models. + +HiSparse currently requires PD disaggregation mode and is enabled on the decode instance only. For detailed design, configuration, and deployment instructions, see the [HiSparse Guide](../advanced_features/hisparse_guide). diff --git a/docs_new/docs/basic_usage/glm45.mdx b/docs_new/docs/basic_usage/glm45.mdx index 8ccb41492..210c85756 100644 --- a/docs_new/docs/basic_usage/glm45.mdx +++ b/docs_new/docs/basic_usage/glm45.mdx @@ -3,7 +3,6 @@ title: "Launch GLM-4.5 / GLM-4.6 / GLM-4.7 with SGLang" metatags: description: "Deploy GLM-4.5/4.6/4.7 models with SGLang: FP8 inference, EAGLE speculative decoding, function calling support. Optimized for H100/H200 GPUs." --- - ## Launch GLM-4.5 / GLM-4.6 / GLM-4.7 with SGLang To serve GLM-4.5 / GLM-4.6 FP8 models on 8xH100/H200 GPUs: diff --git a/docs_new/docs/basic_usage/glmv.mdx b/docs_new/docs/basic_usage/glmv.mdx index 0427a4ad0..088ad0b92 100644 --- a/docs_new/docs/basic_usage/glmv.mdx +++ b/docs_new/docs/basic_usage/glmv.mdx @@ -136,4 +136,4 @@ python -m sglang.launch_server \ In SGLang, we can implement thinking budget with `CustomLogitProcessor`. -Launch a server with `--enable-custom-logit-processor` flag on. and using `Glm4MoeThinkingBudgetLogitProcessor` in the request likes `GLM-4.6` example in [glm45](./glm45). +Launch a server with the `--enable-custom-logit-processor` flag. Then, use `Glm4MoeThinkingBudgetLogitProcessor` in the request, similar to the `GLM-4.6` example in [glm45.md](./glm45). diff --git a/docs_new/docs/basic_usage/minimax_m2.mdx b/docs_new/docs/basic_usage/minimax_m2.mdx index 14da7051d..c248b1942 100644 --- a/docs_new/docs/basic_usage/minimax_m2.mdx +++ b/docs_new/docs/basic_usage/minimax_m2.mdx @@ -1,16 +1,17 @@ --- -title: "MiniMax M2.1/M2 Usage" +title: "MiniMax M2.5/M2.1/M2 Usage" metatags: - description: "Deploy MiniMax M2.1/M2 with SGLang: 230B MoE model (10B active), up to 3M context, optimized for coding and agentic tasks, tool use support." + description: "Deploy MiniMax M2.5/M2.1/M2 with SGLang: 230B MoE model (10B active), up to 3M context, optimized for coding and agentic tasks, tool use support." --- -[MiniMax-M2.1](https://huggingface.co/MiniMaxAI/MiniMax-M2.1) and [MiniMax-M2](https://huggingface.co/MiniMaxAI/MiniMax-M2) are advanced large language models created by [MiniMax](https://www.minimax.io/). +[MiniMax-M2.5](https://huggingface.co/MiniMaxAI/MiniMax-M2.5), [MiniMax-M2.1](https://huggingface.co/MiniMaxAI/MiniMax-M2.1), and [MiniMax-M2](https://huggingface.co/MiniMaxAI/MiniMax-M2) are advanced large language models created by [MiniMax](https://www.minimax.io/). -MiniMax-M2 series redefines efficiency for agents. It's a compact, fast, and cost-effective MoE model (230 billion total parameters with 10 billion active parameters) built for elite performance in coding and agentic tasks, all while maintaining powerful general intelligence. With just 10 billion activated parameters, MiniMax-M2 provides the sophisticated, end-to-end tool use performance expected from today's leading models, but in a streamlined form factor that makes deployment and scaling easier than ever. +The MiniMax-M2 series redefines efficiency for agents. These compact, fast, and cost-effective MoE models (230 billion total parameters with 10 billion active parameters) are built for elite performance in coding and agentic tasks, all while maintaining powerful general intelligence. With just 10 billion activated parameters, the MiniMax-M2 series provides sophisticated, end-to-end tool use performance expected from today's leading models, but in a streamlined form factor that makes deployment and scaling easier than ever. ## Supported Models This guide applies to the following models. You only need to update the model name during deployment. The following examples use **MiniMax-M2**: +- [MiniMaxAI/MiniMax-M2.5](https://huggingface.co/MiniMaxAI/MiniMax-M2.5) - [MiniMaxAI/MiniMax-M2.1](https://huggingface.co/MiniMaxAI/MiniMax-M2.1) - [MiniMaxAI/MiniMax-M2](https://huggingface.co/MiniMaxAI/MiniMax-M2) @@ -52,6 +53,24 @@ python -m sglang.launch_server \ --mem-fraction-static 0.85 ``` +### AMD GPUs (MI300X/MI325X/MI355X) + +8-GPU deployment command: + +```bash Command +SGLANG_USE_AITER=1 python -m sglang.launch_server \ + --model-path MiniMaxAI/MiniMax-M2.5 \ + --tp-size 8 \ + --ep-size 8 \ + --attention-backend aiter \ + --tool-call-parser minimax-m2 \ + --reasoning-parser minimax-append-think \ + --host 0.0.0.0 \ + --trust-remote-code \ + --port 8000 \ + --mem-fraction-static 0.85 +``` + ## Testing Deployment After startup, you can test the SGLang OpenAI-compatible API with the following command: diff --git a/docs_new/docs/basic_usage/native_api.ipynb b/docs_new/docs/basic_usage/native_api.ipynb index 59ac6ee3a..d3ead5e34 100644 --- a/docs_new/docs/basic_usage/native_api.ipynb +++ b/docs_new/docs/basic_usage/native_api.ipynb @@ -10,7 +10,7 @@ "\n", "- `/generate` (text generation model)\n", "- `/get_model_info`\n", - "- `/get_server_info`\n", + "- `/server_info`\n", "- `/health`\n", "- `/health_generate`\n", "- `/flush_cache`\n", @@ -49,7 +49,7 @@ " \"python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct --host 0.0.0.0 --log-level warning\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=server_process)" ] }, { @@ -57,7 +57,7 @@ "metadata": {}, "source": [ "## Generate (text generation model)\n", - "Generate completions. This is similar to the `/v1/completions` in OpenAI API. Detailed parameters can be found in the [sampling parameters](sampling_params)." + "Generate completions. This is similar to the `/v1/completions` in OpenAI API. Detailed parameters can be found in the [sampling parameters](sampling_params.md)." ] }, { @@ -140,7 +140,7 @@ "metadata": {}, "outputs": [], "source": [ - "url = f\"http://localhost:{port}/get_server_info\"\n", + "url = f\"http://localhost:{port}/server_info\"\n", "\n", "response = requests.get(url)\n", "print_highlight(response.text)" @@ -185,7 +185,15 @@ "source": [ "## Flush Cache\n", "\n", - "Flush the radix cache. It will be automatically triggered when the model weights are updated by the `/update_weights` API." + "Flush the radix cache. It will be automatically triggered when the model weights are updated by the `/update_weights` API.\n", + "\n", + "Parameters:\n", + "- `timeout` (query, float, default `0`, unit: seconds): Wait time for idle state before flushing. `0` means fail fast if not idle. When HiCache async operations are in-flight, a non-zero timeout allows the server to wait until idle before flushing, avoiding unnecessary 400 errors.\n", + "\n", + "```bash\n", + "# With timeout (wait up to 30s for idle state)\n", + "curl -s -X POST \"http://127.0.0.1:30000/flush_cache?timeout=30\"\n", + "```" ] }, { @@ -265,7 +273,7 @@ "source": [ "## Encode (embedding model)\n", "\n", - "Encode text into embeddings. Note that this API is only available for [embedding models](openai_api_embeddings) and will raise an error for generation models.\n", + "Encode text into embeddings. Note that this API is only available for [embedding models](openai_api_embeddings.ipynb) and will raise an error for generation models.\n", "Therefore, we launch a new server to server an embedding model." ] }, @@ -280,7 +288,7 @@ " --host 0.0.0.0 --is-embedding --log-level warning\n", "\"\"\")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=embedding_process)" ] }, { @@ -327,7 +335,7 @@ " --host 0.0.0.0 --disable-radix-cache --chunked-prefill-size -1 --attention-backend triton --is-embedding --log-level warning\n", "\"\"\")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=reranker_process)" ] }, { @@ -393,7 +401,7 @@ " --host 0.0.0.0 --log-level warning\n", "\"\"\")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=score_process)" ] }, { @@ -454,7 +462,7 @@ "python3 -m sglang.launch_server --model-path Skywork/Skywork-Reward-Llama-3.1-8B-v0.2 --host 0.0.0.0 --is-embedding --log-level warning\n", "\"\"\")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=reward_process)" ] }, { @@ -518,7 +526,7 @@ " \"python3 -m sglang.launch_server --model-path Qwen/Qwen1.5-MoE-A2.7B --host 0.0.0.0 --expert-distribution-recorder-mode stat --log-level warning\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=expert_record_server_process)" ] }, { @@ -571,7 +579,7 @@ "python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct\n", "\"\"\")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=tokenizer_free_server_process)" ] }, { diff --git a/docs_new/docs/basic_usage/native_api.mdx b/docs_new/docs/basic_usage/native_api.mdx index af4c76ce9..42c5bd228 100644 --- a/docs_new/docs/basic_usage/native_api.mdx +++ b/docs_new/docs/basic_usage/native_api.mdx @@ -7,7 +7,7 @@ Apart from the OpenAI compatible APIs, the SGLang Runtime also provides its nati - `/generate` (text generation model) - `/get_model_info` -- `/get_server_info` +- `/server_info` - `/health` - `/health_generate` - `/flush_cache` @@ -35,7 +35,7 @@ server_process, port = launch_server_cmd( "python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct --host 0.0.0.0 --log-level warning" ) -wait_for_server(f"http://localhost:{port}") +wait_for_server(f"http://localhost:{port}", process=server_process) ``` ## Generate (text generation model) @@ -96,7 +96,7 @@ Gets the server information including CLI arguments, token limits, and memory po - `get_max_total_num_tokens` ```python Example -url = f"http://localhost:{port}/get_server_info" +url = f"http://localhost:{port}/server_info" response = requests.get(url) print_highlight(response.text) @@ -124,6 +124,14 @@ print_highlight(response.text) Flush the radix cache. It will be automatically triggered when the model weights are updated by the `/update_weights` API. +Parameters: +- `timeout` (query, float, default `0`, unit: seconds): Wait time for idle state before flushing. `0` means fail fast if not idle. When HiCache async operations are in-flight, a non-zero timeout allows the server to wait until idle before flushing, avoiding unnecessary 400 errors. + +```bash Command +# With timeout (wait up to 30s for idle state) +curl -s -X POST "http://127.0.0.1:30000/flush_cache?timeout=30" +``` + ```python Example url = f"http://localhost:{port}/flush_cache" @@ -176,14 +184,12 @@ Encode text into embeddings. Note that this API is only available for [embedding Therefore, we launch a new server to server an embedding model. ```python Example -embedding_process, port = launch_server_cmd( - """ +embedding_process, port = launch_server_cmd(""" python3 -m sglang.launch_server --model-path Alibaba-NLP/gte-Qwen2-1.5B-instruct \ --host 0.0.0.0 --is-embedding --log-level warning -""" -) +""") -wait_for_server(f"http://localhost:{port}") +wait_for_server(f"http://localhost:{port}", process=embedding_process) ``` ```python Example @@ -205,14 +211,12 @@ terminate_process(embedding_process) Rerank a list of documents given a query using a cross-encoder model. Note that this API is only available for cross encoder model like [BAAI/bge-reranker-v2-m3](https://huggingface.co/BAAI/bge-reranker-v2-m3) with `attention-backend` `triton` and `torch_native`. ```python Example -reranker_process, port = launch_server_cmd( - """ +reranker_process, port = launch_server_cmd(""" python3 -m sglang.launch_server --model-path BAAI/bge-reranker-v2-m3 \ --host 0.0.0.0 --disable-radix-cache --chunked-prefill-size -1 --attention-backend triton --is-embedding --log-level warning -""" -) +""") -wait_for_server(f"http://localhost:{port}") +wait_for_server(f"http://localhost:{port}", process=reranker_process) ``` ```python Example @@ -253,14 +257,12 @@ Parameters: The response contains `scores` - a list of probability lists, one per item, each in the order of `label_token_ids`. ```python Example -score_process, port = launch_server_cmd( - """ +score_process, port = launch_server_cmd(""" python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct \ --host 0.0.0.0 --log-level warning -""" -) +""") -wait_for_server(f"http://localhost:{port}") +wait_for_server(f"http://localhost:{port}", process=score_process) ``` ```python Example @@ -297,13 +299,11 @@ SGLang Runtime also supports reward models. Here we use a reward model to classi # Note that SGLang now treats embedding models and reward models as the same type of models. # This will be updated in the future. -reward_process, port = launch_server_cmd( - """ +reward_process, port = launch_server_cmd(""" python3 -m sglang.launch_server --model-path Skywork/Skywork-Reward-Llama-3.1-8B-v0.2 --host 0.0.0.0 --is-embedding --log-level warning -""" -) +""") -wait_for_server(f"http://localhost:{port}") +wait_for_server(f"http://localhost:{port}", process=reward_process) ``` ```python Example @@ -347,7 +347,7 @@ expert_record_server_process, port = launch_server_cmd( "python3 -m sglang.launch_server --model-path Qwen/Qwen1.5-MoE-A2.7B --host 0.0.0.0 --expert-distribution-recorder-mode stat --log-level warning" ) -wait_for_server(f"http://localhost:{port}") +wait_for_server(f"http://localhost:{port}", process=expert_record_server_process) ``` ```python Example @@ -376,13 +376,11 @@ terminate_process(expert_record_server_process) This example demonstrates how to use the /tokenize and /detokenize endpoints together. We first tokenize a string, then detokenize the resulting IDs to reconstruct the original text. This workflow is useful when you need to handle tokenization externally but still leverage the server for detokenization. ```python Example -tokenizer_free_server_process, port = launch_server_cmd( - """ +tokenizer_free_server_process, port = launch_server_cmd(""" python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct -""" -) +""") -wait_for_server(f"http://localhost:{port}") +wait_for_server(f"http://localhost:{port}", process=tokenizer_free_server_process) ``` ```python Example diff --git a/docs_new/docs/basic_usage/offline_engine_api.ipynb b/docs_new/docs/basic_usage/offline_engine_api.ipynb index 9c03e90a7..fe8a9e304 100644 --- a/docs_new/docs/basic_usage/offline_engine_api.ipynb +++ b/docs_new/docs/basic_usage/offline_engine_api.ipynb @@ -66,7 +66,7 @@ "import asyncio\n", "\n", "import sglang as sgl\n", - "import sglang.test.doc_patch\n", + "import sglang.test.doc_patch # noqa: F401\n", "from sglang.utils import async_stream_and_merge, stream_and_merge\n", "\n", "llm = sgl.Engine(model_path=\"qwen/qwen2.5-0.5b-instruct\")" diff --git a/docs_new/docs/basic_usage/openai_api_completions.ipynb b/docs_new/docs/basic_usage/openai_api_completions.ipynb index e64b7a13e..ffa576ae5 100644 --- a/docs_new/docs/basic_usage/openai_api_completions.ipynb +++ b/docs_new/docs/basic_usage/openai_api_completions.ipynb @@ -14,7 +14,7 @@ "- `chat/completions`\n", "- `completions`\n", "\n", - "Check out other tutorials to learn about [vision APIs](openai_api_vision) for vision-language models and [embedding APIs](openai_api_embeddings) for embedding models." + "Check out other tutorials to learn about [vision APIs](openai_api_vision.ipynb) for vision-language models and [embedding APIs](openai_api_embeddings.ipynb) for embedding models." ] }, { @@ -39,7 +39,7 @@ " \"python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct --host 0.0.0.0 --log-level warning\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")\n", + "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n", "print(f\"Server started on http://localhost:{port}\")" ] }, @@ -477,7 +477,7 @@ "source": [ "## Structured Outputs (JSON, Regex, EBNF)\n", "\n", - "For OpenAI compatible structured outputs API, refer to [Structured Outputs](../advanced_features/structured_outputs) for more details.\n" + "For OpenAI compatible structured outputs API, refer to [Structured Outputs](../advanced_features/structured_outputs.ipynb) for more details.\n" ] }, { @@ -496,7 +496,7 @@ " --lora-paths adapter_a=/path/to/adapter_a adapter_b=/path/to/adapter_b\n", "```\n", "\n", - "For more details on LoRA serving configuration, see the [LoRA documentation](../advanced_features/lora).\n", + "For more details on LoRA serving configuration, see the [LoRA documentation](../advanced_features/lora.ipynb).\n", "\n", "**API Call:**\n", "\n", diff --git a/docs_new/docs/basic_usage/openai_api_embeddings.ipynb b/docs_new/docs/basic_usage/openai_api_embeddings.ipynb index 07031d32c..a6c90c06b 100644 --- a/docs_new/docs/basic_usage/openai_api_embeddings.ipynb +++ b/docs_new/docs/basic_usage/openai_api_embeddings.ipynb @@ -9,7 +9,7 @@ "SGLang provides OpenAI-compatible APIs to enable a smooth transition from OpenAI services to self-hosted local models.\n", "A complete reference for the API is available in the [OpenAI API Reference](https://platform.openai.com/docs/guides/embeddings).\n", "\n", - "This tutorial covers the embedding APIs for embedding models. For a list of the supported models see the [corresponding overview page](../supported_models/embedding_models)\n" + "This tutorial covers the embedding APIs for embedding models. For a list of the supported models see the [corresponding overview page](../supported_models/retrieval_ranking/embedding_models.md)\n" ] }, { @@ -35,7 +35,7 @@ " --host 0.0.0.0 --is-embedding --log-level warning\n", "\"\"\")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=embedding_process)" ] }, { @@ -171,7 +171,7 @@ "metadata": {}, "source": [ "## Multi-Modal Embedding Model\n", - "Please refer to [Multi-Modal Embedding Model](../supported_models/embedding_models)" + "Please refer to [Multi-Modal Embedding Model](../supported_models/retrieval_ranking/embedding_models.md)" ] } ], diff --git a/docs_new/docs/basic_usage/openai_api_vision.ipynb b/docs_new/docs/basic_usage/openai_api_vision.ipynb index aea2c7ef8..b6e6a1a24 100644 --- a/docs_new/docs/basic_usage/openai_api_vision.ipynb +++ b/docs_new/docs/basic_usage/openai_api_vision.ipynb @@ -10,7 +10,7 @@ "A complete reference for the API is available in the [OpenAI API Reference](https://platform.openai.com/docs/guides/vision).\n", "This tutorial covers the vision APIs for vision language models.\n", "\n", - "SGLang supports various vision language models such as Llama 3.2, LLaVA-OneVision, Qwen2.5-VL, Gemma3 and [more](../supported_models/multimodal_language_models).\n", + "SGLang supports various vision language models such as Llama 3.2, LLaVA-OneVision, Qwen2.5-VL, Gemma3 and [more](../supported_models/text_generation/multimodal_language_models.md).\n", "\n", "As an alternative to the OpenAI API, you can also use the [SGLang offline engine](https://github.com/sgl-project/sglang/blob/main/examples/runtime/engine/offline_batch_inference_vlm.py)." ] @@ -33,11 +33,16 @@ "from sglang.test.doc_patch import launch_server_cmd\n", "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", "\n", + "example_image_url = \"https://raw.githubusercontent.com/sgl-project/sglang/main/examples/assets/example_image.png\"\n", + "logo_image_url = (\n", + " \"https://raw.githubusercontent.com/sgl-project/sglang/main/assets/logo.png\"\n", + ")\n", + "\n", "vision_process, port = launch_server_cmd(\"\"\"\n", "python3 -m sglang.launch_server --model-path Qwen/Qwen2.5-VL-7B-Instruct --log-level warning\n", "\"\"\")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=vision_process)" ] }, { @@ -73,7 +78,7 @@ " {{\n", " \"type\": \"image_url\",\n", " \"image_url\": {{\n", - " \"url\": \"https://github.com/sgl-project/sglang/blob/main/examples/assets/example_image.png?raw=true\"\n", + " \"url\": \"{example_image_url}\"\n", " }}\n", " }}\n", " ]\n", @@ -117,9 +122,7 @@ " {\"type\": \"text\", \"text\": \"What’s in this image?\"},\n", " {\n", " \"type\": \"image_url\",\n", - " \"image_url\": {\n", - " \"url\": \"https://github.com/sgl-project/sglang/blob/main/examples/assets/example_image.png?raw=true\"\n", - " },\n", + " \"image_url\": {\"url\": example_image_url},\n", " },\n", " ],\n", " }\n", @@ -160,9 +163,7 @@ " },\n", " {\n", " \"type\": \"image_url\",\n", - " \"image_url\": {\n", - " \"url\": \"https://github.com/sgl-project/sglang/blob/main/examples/assets/example_image.png?raw=true\"\n", - " },\n", + " \"image_url\": {\"url\": example_image_url},\n", " },\n", " ],\n", " }\n", @@ -201,13 +202,13 @@ " {\n", " \"type\": \"image_url\",\n", " \"image_url\": {\n", - " \"url\": \"https://github.com/sgl-project/sglang/blob/main/examples/assets/example_image.png?raw=true\",\n", + " \"url\": example_image_url,\n", " },\n", " },\n", " {\n", " \"type\": \"image_url\",\n", " \"image_url\": {\n", - " \"url\": \"https://raw.githubusercontent.com/sgl-project/sglang/main/assets/logo.png\",\n", + " \"url\": logo_image_url,\n", " },\n", " },\n", " {\n", diff --git a/docs_new/docs/basic_usage/openai_api_vision.mdx b/docs_new/docs/basic_usage/openai_api_vision.mdx index 75e394e7d..e27bf160c 100644 --- a/docs_new/docs/basic_usage/openai_api_vision.mdx +++ b/docs_new/docs/basic_usage/openai_api_vision.mdx @@ -7,36 +7,34 @@ SGLang provides OpenAI-compatible APIs to enable a smooth transition from OpenAI A complete reference for the API is available in the [OpenAI API Reference](https://platform.openai.com/docs/guides/vision). This tutorial covers the vision APIs for vision language models. -SGLang supports various vision language models such as Llama 3.2, LLaVA-OneVision, Qwen2.5-VL, Gemma3 and [more](../supported-models). +SGLang supports various vision language models such as Llama 3.2, LLaVA-OneVision, Qwen2.5-VL, Gemma3 and [more](../supported-models/multimodal_language_models). As an alternative to the OpenAI API, you can also use the [SGLang offline engine](https://github.com/sgl-project/sglang/blob/main/examples/runtime/engine/offline_batch_inference_vlm.py). - ## Launch A Server Launch the server in your terminal and wait for it to initialize. - - ```python Example from sglang.test.doc_patch import launch_server_cmd from sglang.utils import wait_for_server, print_highlight, terminate_process -vision_process, port = launch_server_cmd( - """ -python3 -m sglang.launch_server --model-path Qwen/Qwen2.5-VL-7B-Instruct --log-level warning -""" +example_image_url = "https://raw.githubusercontent.com/sgl-project/sglang/main/examples/assets/example_image.png" +logo_image_url = ( + "https://raw.githubusercontent.com/sgl-project/sglang/main/assets/logo.png" ) -wait_for_server(f"http://localhost:{port}") +vision_process, port = launch_server_cmd(""" +python3 -m sglang.launch_server --model-path Qwen/Qwen2.5-VL-7B-Instruct --log-level warning +""") + +wait_for_server(f"http://localhost:{port}", process=vision_process) ``` ## Using cURL Once the server is up, you can send test requests using curl or requests. - - ```python Example import subprocess @@ -56,7 +54,7 @@ curl -s http://localhost:{port}/v1/chat/completions \\ {{ "type": "image_url", "image_url": {{ - "url": "https://github.com/sgl-project/sglang/blob/main/examples/assets/example_image.png?raw=true" + "url": "{example_image_url}" }} }} ] @@ -76,8 +74,6 @@ print_highlight(response) ## Using Python Requests - - ```python Example import requests @@ -92,9 +88,7 @@ data = { {"type": "text", "text": "What’s in this image?"}, { "type": "image_url", - "image_url": { - "url": "https://github.com/sgl-project/sglang/blob/main/examples/assets/example_image.png?raw=true" - }, + "image_url": {"url": example_image_url}, }, ], } @@ -108,8 +102,6 @@ print_highlight(response.text) ## Using OpenAI Python Client - - ```python Example from openai import OpenAI @@ -127,9 +119,7 @@ response = client.chat.completions.create( }, { "type": "image_url", - "image_url": { - "url": "https://github.com/sgl-project/sglang/blob/main/examples/assets/example_image.png?raw=true" - }, + "image_url": {"url": example_image_url}, }, ], } @@ -144,8 +134,6 @@ print_highlight(response.choices[0].message.content) The server also supports multiple images and interleaved text and images if the model supports it. - - ```python Example from openai import OpenAI @@ -160,13 +148,13 @@ response = client.chat.completions.create( { "type": "image_url", "image_url": { - "url": "https://github.com/sgl-project/sglang/blob/main/examples/assets/example_image.png?raw=true", + "url": example_image_url, }, }, { "type": "image_url", "image_url": { - "url": "https://raw.githubusercontent.com/sgl-project/sglang/main/assets/logo.png", + "url": logo_image_url, }, }, { @@ -183,7 +171,6 @@ response = client.chat.completions.create( print_highlight(response.choices[0].message.content) ``` - ```python Example terminate_process(vision_process) ``` diff --git a/docs_new/docs/basic_usage/popular_model_usage.mdx b/docs_new/docs/basic_usage/popular_model_usage.mdx index db61169e7..4c5a25e2f 100644 --- a/docs_new/docs/basic_usage/popular_model_usage.mdx +++ b/docs_new/docs/basic_usage/popular_model_usage.mdx @@ -2,13 +2,16 @@ title: "Popular Model Usage (DeepSeek, GPT-OSS, GLM, Llama, MiniMax, Qwen, and more)" description: "Documentation for Popular Model Usage (DeepSeek, GPT-OSS, GLM, Llama, MiniMax, Qwen, and more)" --- +For more usage examples and recipes, visit the [SGLang Cookbook](https://cookbook.sglang.io/). + - [Deepseek V3](./deepseek_v3) - [Deepseek V32](./deepseek_v32) - [Glm45](./glm45) - [Glmv](./glmv) - [Gpt Oss](./gpt_oss) -- [Kimi K2 5](./kimi_k2_5) - [Minimax M2](./minimax_m2) - [Qwen3](./qwen3) +- [Qwen3 5](./qwen3_5) - [Qwen3 Vl](./qwen3_vl) +- [Deepseek Ocr](./deepseek_ocr) - [Llama4](./llama4) diff --git a/docs_new/docs/basic_usage/qwen3_5.mdx b/docs_new/docs/basic_usage/qwen3_5.mdx new file mode 100644 index 000000000..88e897d08 --- /dev/null +++ b/docs_new/docs/basic_usage/qwen3_5.mdx @@ -0,0 +1,80 @@ +--- +title: "Qwen 3.5 Usage" +metatags: + description: "Qwen 3.5 is Alibaba's latest generation LLM featuring a hybrid attention architecture, advanced MoE with shared experts, and native multimodal capabilities." +--- + +Qwen 3.5 is Alibaba's latest generation LLM featuring a hybrid attention architecture, advanced MoE with shared experts, and native multimodal capabilities. + +Key architecture features: +- **Hybrid Attention**: Gated Delta Networks (linear, O(n) complexity) combined with full attention every 4th layer for high associative recall +- **MoE with Shared Experts**: Top-8 active out of 64 routed experts plus a dedicated shared expert for universal features +- **Multimodal**: DeepStack Vision Transformer with Conv3d for native image and video understanding + +## Launch Qwen 3.5 with SGLang + +### Dense Model + +To serve `Qwen/Qwen3.5-397B-A17B` on 8 GPUs: + +```bash +python3 -m sglang.launch_server \ + --model-path Qwen/Qwen3.5-397B-A17B \ + --tp 8 \ + --trust-remote-code +``` + +### AMD GPU (MI300X / MI325X / MI35X) + +On AMD Instinct GPUs, use the `triton` attention backend. Both the full attention layers and the Gated Delta Net (linear attention) layers use Triton-based kernels on ROCm: + +```bash +SGLANG_USE_AITER=1 python3 -m sglang.launch_server \ + --model-path Qwen/Qwen3.5-397B-A17B \ + --tp 8 \ + --attention-backend triton \ + --trust-remote-code +``` + + +Set `SGLANG_USE_AITER=1` to enable AMD's optimized aiter kernels for MoE and GEMM operations. + + +### Configuration Tips + +- `--attention-backend`: Use `triton` on AMD GPUs for Qwen 3.5. The hybrid attention architecture (Gated Delta Networks + full attention) works best with the Triton backend on ROCm. The linear attention (GDN) layers always use Triton kernels internally via the `GDNAttnBackend`. +- `--watchdog-timeout`: Increase to `1200` or higher for this large model, as weight loading takes significant time. +- `--model-loader-extra-config '{"enable_multithread_load": true}'`: Enables parallel weight loading for faster startup. + +### Reasoning and Tool Calling + +Qwen 3.5 supports reasoning and tool calling via the Qwen3 parsers: + +```bash +python3 -m sglang.launch_server \ + --model-path Qwen/Qwen3.5-397B-A17B \ + --tp 8 \ + --trust-remote-code \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder +``` + +## Accuracy Evaluation + +You can evaluate the model accuracy using `lm-eval`: + +```bash +pip install lm-eval[api] + +lm_eval --model local-completions \ + --model_args '{"base_url": "http://localhost:8000/v1/completions", "model": "Qwen/Qwen3.5-397B-A17B", "num_concurrent": 256, "max_retries": 10, "max_gen_toks": 2048}' \ + --tasks gsm8k \ + --batch_size auto \ + --num_fewshot 5 \ + --trust_remote_code +``` + +## Additional Resources + +- [AMD Day 0 Support for Qwen 3.5 on AMD Instinct GPUs](https://www.amd.com/en/developer/resources/technical-articles/2026/day-0-support-for-qwen-3-5-on-amd-instinct-gpus.html) +- [HuggingFace Model Card](https://huggingface.co/Qwen/Qwen3.5-397B-A17B) diff --git a/docs_new/docs/basic_usage/send_request.ipynb b/docs_new/docs/basic_usage/send_request.ipynb index 412f1fa60..968a23b8d 100644 --- a/docs_new/docs/basic_usage/send_request.ipynb +++ b/docs_new/docs/basic_usage/send_request.ipynb @@ -7,9 +7,9 @@ "# Sending Requests\n", "This notebook provides a quick-start guide to use SGLang in chat completions after installation. Once your server is running, API documentation is available at `http://localhost:30000/docs` (Swagger UI), `http://localhost:30000/redoc` (ReDoc), or `http://localhost:30000/openapi.json` (OpenAPI spec, useful for AI agents). Replace `30000` with your port if using a different one.\n", "\n", - "- For Vision Language Models, see [OpenAI APIs - Vision](openai_api_vision).\n", - "- For Embedding Models, see [OpenAI APIs - Embedding](openai_api_embeddings) and [Encode (embedding model)](native_api#encode-embedding-model).\n", - "- For Reward Models, see [Classify (reward model)](native_api#classify-reward-model)." + "- For Vision Language Models, see [OpenAI APIs - Vision](openai_api_vision.ipynb).\n", + "- For Embedding Models, see [OpenAI APIs - Embedding](openai_api_embeddings.ipynb) and [Encode (embedding model)](native_api.html#Encode-(embedding-model)).\n", + "- For Reward Models, see [Classify (reward model)](native_api.html#Classify-(reward-model))." ] }, { @@ -36,7 +36,7 @@ " --host 0.0.0.0 --log-level warning\n", "\"\"\")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=server_process)" ] }, { @@ -158,7 +158,7 @@ "source": [ "## Using Native Generation APIs\n", "\n", - "You can also use the native `/generate` endpoint with requests, which provides more flexibility. An API reference is available at [Sampling Parameters](sampling_params)." + "You can also use the native `/generate` endpoint with requests, which provides more flexibility. An API reference is available at [Sampling Parameters](sampling_params.md)." ] }, { diff --git a/docs_new/docs/developer_guide/bench_serving.mdx b/docs_new/docs/developer_guide/bench_serving.mdx index c00b72da1..e02a0ff37 100644 --- a/docs_new/docs/developer_guide/bench_serving.mdx +++ b/docs_new/docs/developer_guide/bench_serving.mdx @@ -142,7 +142,7 @@ python3 -m sglang.bench_serving \ - `--output-file FILE.jsonl`: append JSONL results to file; auto-named if unspecified - `--output-details`: include per-request arrays (generated texts, errors, ttfts, itls, input/output lens) -- `--extra-request-body '{"top_p":0.9,"temperature":0.6}'`: merged into payload (sampling params, etc.) +- `--extra-request-body '{"top_p":0.9,"temperature":0.6}'`: merged into payload (sampling params, etc.) - `--disable-ignore-eos`: pass through EOS behavior (varies by backend) - `--warmup-requests N`: run warmup requests with short output first (default 1) - `--flush-cache`: call `/flush_cache` (sglang) before main run @@ -335,7 +335,7 @@ python3 -m sglang.bench_serving \ python3 -m sglang.bench_serving \ --backend sglang \ --host 127.0.0.1 --port 30000 \ - --model mode-name \ + --model model-name \ --dataset-name mooncake \ --mooncake-slowdown-factor 1.0 \ --mooncake-num-rounds 1000 \ @@ -344,6 +344,41 @@ python3 -m sglang.bench_serving \ --random-output-len 256 ``` +10) Fake decode stress testing (PD disaggregation, decode-only): + +When benchmarking pure decode performance in a PD disaggregation setup, you can bypass the prefill node entirely by using `--fake-prefill`. This requires the decode server to be started with `--disaggregation-transfer-backend fake`: + +```bash Command +# Step 1: Start a decode-only server with fake transfer backend +python -m sglang.launch_server \ + --model-path meta-llama/Llama-3.1-8B-Instruct \ + --disaggregation-mode decode \ + --disaggregation-transfer-backend fake \ + --port 30001 + +# Step 2: Run bench_serving with --fake-prefill +python3 -m sglang.bench_serving \ + --backend sglang \ + --host 127.0.0.1 --port 30001 \ + --model meta-llama/Llama-3.1-8B-Instruct \ + --dataset-name random \ + --num-prompts 500 \ + --random-input-len 1024 --random-output-len 256 \ + --fake-prefill +``` + +Similarly, `bench_one_batch_server` also supports `--fake-prefill`: + +```bash Command +python3 -m sglang.bench_one_batch_server \ + --base-url http://127.0.0.1:30001 \ + --model-path meta-llama/Llama-3.1-8B-Instruct \ + --batch-size 32 --input-len 1024 --output-len 256 \ + --fake-prefill +``` + +The `--fake-prefill` flag automatically injects special sentinel values into each request, telling the decode server to skip real KV transfer and generate fake KV data locally. + ### Troubleshooting - All requests failed: verify `--backend`, server URL/port, `--model`, and authentication. Check warmup errors printed by the script. @@ -355,4 +390,4 @@ python3 -m sglang.bench_serving \ ### Notes - The script raises the file descriptor soft limit (`RLIMIT_NOFILE`) to help with many concurrent connections. -- For sglang, `/get_server_info` is queried post-run to report speculative decoding accept length when available. +- For sglang, `/server_info` is queried post-run to report speculative decoding accept length when available. diff --git a/docs_new/docs/developer_guide/benchmark_and_profiling.mdx b/docs_new/docs/developer_guide/benchmark_and_profiling.mdx index 7418aaaae..33c94b5b2 100644 --- a/docs_new/docs/developer_guide/benchmark_and_profiling.mdx +++ b/docs_new/docs/developer_guide/benchmark_and_profiling.mdx @@ -5,28 +5,69 @@ metatags: --- ## Benchmark -- Benchmark the latency of running a single static batch without a server. The arguments are the same as for `launch_server.py`. - Note that this is a simplified test script without a dynamic batching server, so it may run out of memory for a batch size that a real server can handle. A real server truncates the prefill into several batches, while this simplified script does not. - - Without a server (do not need to launch a server) - ```bash Command - python -m sglang.bench_one_batch --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --batch 32 --input-len 256 --output-len 32 - ``` - - With a server (please use `sglang.launch_server` to launch a server first and run the following command.) - ```bash Command - python -m sglang.bench_one_batch_server --base-url http://127.0.0.1:30000 --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --batch-size 32 --input-len 256 --output-len 32 - ``` +SGLang provides four benchmark tools that operate at different levels of the stack. The table below summarizes their key differences: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ToolHTTP ServerSchedulerUse Case
bench_servingYes (async HTTP client to a running server)Yes (indirectly, via server)Realistic online serving benchmarks with latency metrics (TTFT, TPOT, ITL)
bench_one_batch_serverYes (sends HTTP requests to a running server)Yes (indirectly, via server)End-to-end single-batch latency including HTTP and scheduler overhead
bench_offline_throughputNoYes (directly uses Engine in-process)Maximum throughput measurement without HTTP overhead
bench_one_batchNoNo (directly calls ModelRunner)Kernel-level latency profiling of a single static batch
-- Benchmark offline processing. This script will start an offline engine and run the benchmark. +Use `bench_serving` by default unless there are specific needs. + +**`bench_serving`** is an async HTTP load-testing client that sends requests at controlled rates with configurable concurrency to a running server. It measures realistic online serving metrics including time-to-first-token (TTFT), time-per-output-token (TPOT), inter-token latency (ITL), and throughput. Use `num-prompts >= 5 * max-concurrency` to measure steady-state performance. Launch a server with `sglang.launch_server` first. + + ```bash Command + python3 -m sglang.bench_serving --backend sglang --max-concurrency 16 --num-prompts 80 --random-input-len 256 --random-output-len 32 --dataset-name random + ``` + +**`bench_one_batch_server`** sends a single batch as one HTTP request to a running server. Due to only having a single batch, the server is never in a steady-state and metrics will be biased. Launch a server with `sglang.launch_server` first. + + ```bash Command + python3 -m sglang.bench_one_batch_server --base-url http://127.0.0.1:30000 --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --batch-size 32 --input-len 256 --output-len 32 + ``` + +**`bench_offline_throughput`** directly instantiates the `Engine` object in-process (no HTTP server) and submits all requests at once via `engine.generate()`. The engine's scheduler handles batching and execution. This measures maximum achievable throughput without any network overhead. ```bash Command python3 -m sglang.bench_offline_throughput --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --num-prompts 10 ``` -- Benchmark online serving. Please use `sglang.launch_server` to launch a server first and run the following command. +**`bench_one_batch`** is the lowest-level tool. It directly instantiates a `ModelRunner` and calls `extend()` / `decode()` on a fixed static batch, bypassing the scheduler entirely. The prefill and decode phases are run separately, making profiling easier but rendering the metrics unrealistic. Because there is no dynamic batching, it may run out of memory for batch sizes that a real server can handle (a real server chunks prefill into smaller batches). This is best suited for profiling individual kernel performance. ```bash Command - python3 -m sglang.bench_serving --backend sglang --num-prompt 10 + python3 -m sglang.bench_one_batch --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --batch-size 32 --input-len 256 --output-len 32 ``` ## Profile with PyTorch Profiler @@ -46,7 +87,10 @@ python -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct python -m sglang.bench_serving --backend sglang --model meta-llama/Llama-3.1-8B-Instruct --num-prompts 10 --sharegpt-output-len 100 --profile ``` -Please make sure that the `SGLANG_TORCH_PROFILER_DIR` should be set at both server and client side, otherwise the trace file cannot be generated correctly . A secure way will be setting `SGLANG_TORCH_PROFILER_DIR` in the `.*rc` file of shell (e.g. `~/.bashrc` for bash shells). +For `bench_serving --profile`, the output directory is selected on the client side from `--profile-output-dir` or `SGLANG_TORCH_PROFILER_DIR` (fallback: `/tmp`), then sent in the `/start_profile` request. +If you call `/start_profile` directly and do not provide `output_dir`, the server uses its own `SGLANG_TORCH_PROFILER_DIR` (fallback: `/tmp`). + +Setting `SGLANG_TORCH_PROFILER_DIR` on both server and client is still recommended to avoid confusion about where traces are written. For more details, please refer to [Bench Serving Guide](./bench_serving). @@ -147,7 +191,7 @@ curl -X POST http://127.0.0.1:30000/start_profile \ **Parameters:** - `output_dir` (optional): Directory where profile traces will be saved. If not specified, uses `SGLANG_TORCH_PROFILER_DIR` environment variable, or `/tmp` as the default -- `num_steps` (optional): Number of steps to profile. If not specified, profiling continues until manually stopped with `/end_profile` +- `num_steps` (optional): Number of steps to profile. If not specified, profiling continues until manually stopped with `/stop_profile` - `start_step` (optional): Step number at which to start profiling (inclusive). Useful for skipping warmup iterations - `activities` (optional): List of activities to profile, e.g., `["CPU", "GPU"]`. Default is `["CPU", "GPU"]` - `merge_profiles` (optional): Whether to merge distributed traces. Default is `false` @@ -171,17 +215,17 @@ curl -X POST http://127.0.0.1:30000/start_profile \ **Continuous profiling (manual stop):** ```bash Command -# Start profiling without num_steps - must manually stop with /end_profile +# Start profiling without num_steps - must manually stop with /stop_profile curl -X POST http://127.0.0.1:30000/start_profile ``` -#### Using `/end_profile` endpoint +#### Using `/stop_profile` endpoint -The `/end_profile` endpoint stops an ongoing profiling session and saves the trace file. +The `/stop_profile` endpoint stops an ongoing profiling session and saves the trace file. ```bash Command # Stop profiling and save traces -curl -X POST http://127.0.0.1:30000/end_profile +curl -X POST http://127.0.0.1:30000/stop_profile ``` This is only needed when you start profiling without specifying `num_steps`. If `num_steps` is specified, profiling will automatically stop after that many steps. @@ -204,7 +248,7 @@ curl -X POST http://127.0.0.1:30000/start_profile \ python -m sglang.bench_serving --backend sglang --num-prompts 100 # Terminal 2: Stop profiling when done -curl -X POST http://127.0.0.1:30000/end_profile +curl -X POST http://127.0.0.1:30000/stop_profile ``` ### Profiler Trace Merger for Distributed Traces @@ -246,8 +290,8 @@ python -m sglang.profiler \ #### Output Files The profile merger generates: -- Individual rank trace files: `{profile_id}-TP-{tp}-DP-{dp}-PP-{pp}-EP-{ep}.trace.json.gz` -- Merged trace file: `merged-{profile_id}.trace.json.gz` +- Individual rank trace files: `{profile_id}-TP-{tp}-DP-{dp}-PP-{pp}-EP-{ep}.trace.json.gz` +- Merged trace file: `merged-{profile_id}.trace.json.gz` ### Possible PyTorch bugs If in any cases you encounter the following error (for example, using qwen 2.5 VL): @@ -398,10 +442,10 @@ This method allows you to control exactly when profiling starts/stops via HTTP A ```bash Command # Terminal 2: Only needed if num_steps was not specified - curl -X POST http://127.0.0.1:30000/end_profile + curl -X POST http://127.0.0.1:30000/stop_profile ``` -The `--capture-range=cudaProfilerApi` option tells Nsight Systems to only capture data between `cudaProfilerStart()` and `cudaProfilerStop()` calls (triggered by `/start_profile` and `/end_profile`), reducing overhead and file size. The `start_step` parameter skips the first 3 steps to avoid capturing warmup overhead. +The `--capture-range=cudaProfilerApi` option tells Nsight Systems to only capture data between `cudaProfilerStart()` and `cudaProfilerStop()` calls (triggered by `/start_profile` and `/stop_profile`), reducing overhead and file size. The `start_step` parameter skips the first 3 steps to avoid capturing warmup overhead. **Method 2: Simpler approach without `/start_profile` API** diff --git a/docs_new/docs/developer_guide/contribution_guide.mdx b/docs_new/docs/developer_guide/contribution_guide.mdx index a1de8d6c8..9c01980c2 100644 --- a/docs_new/docs/developer_guide/contribution_guide.mdx +++ b/docs_new/docs/developer_guide/contribution_guide.mdx @@ -4,7 +4,7 @@ mode: wide metatags: description: "SGLang contribution guide: source install, pre-commit, unit tests, CI triggers, code style, sgl-kernel updates." --- -Welcome to **SGLang**! We appreciate your interest in contributing. This guide provides a concise overview of how to set up your environment, run tests, build documentation, and open a Pull Request (PR). Whether you're fixing a small bug or developing a major feature, we encourage following these steps for a smooth contribution process. +Welcome to **SGLang**! We appreciate your interest in contributing. This guide provides a concise overview of how to set up your environment, run tests, build documentation, and open a Pull Request (PR). Whether you’re fixing a small bug or developing a major feature, we encourage following these steps for a smooth contribution process. ## Install SGLang from Source @@ -18,7 +18,7 @@ git clone https://github.com//sglang.git ### Build from source -Refer to [Install SGLang from Source](../get-started/installation). +Refer to [Install SGLang from Source](../get-started/install#method-2-from-source). ## Format code with pre-commit @@ -32,17 +32,50 @@ pre-commit run --all-files - **`pre-commit run --all-files`** manually runs all configured checks, applying fixes if possible. If it fails the first time, re-run it to ensure lint errors are fully resolved. Make sure your code passes all checks **before** creating a Pull Request. - **Do not commit** directly to the `main` branch. Always create a new branch (e.g., `feature/my-new-feature`), push your changes, and open a PR from that branch. +- Link checking with lychee is **enforced in CI**. By default, it is not blocking local commits. +- To run local link checks manually, use: `pre-commit run --hook-stage manual lychee --all-files`. ## Run and add unit tests If you add a new feature or fix a bug, please add corresponding unit tests to ensure coverage and prevent regression. -SGLang uses Python's built-in [unittest](https://docs.python.org/3/library/unittest.html) framework. -For detailed instructions on running tests and integrating them into CI, refer to [test/README](https://github.com/sgl-project/sglang/tree/main/test/README). + +### Unit tests (no server required) + +Unit tests live under [`test/registered/unit/`](https://github.com/sgl-project/sglang/tree/main/test/registered/unit), organized to mirror the `python/sglang/srt/` source tree. These tests validate component logic **without** launching a server or loading real model weights. +SGLang uses Python's built-in [unittest](https://docs.python.org/3/library/unittest.html) framework with [pytest](https://docs.pytest.org/) as the test runner. + +**When to add a unit test:** If you modify a file under `python/sglang/srt/`, check whether a corresponding test exists in `test/registered/unit/` and add coverage for your changes. For example: + +``` +srt/mem_cache/radix_cache.py → unit/mem_cache/test_radix_cache.py +srt/sampling/sampling_params.py → unit/sampling/test_sampling_params.py +``` + +**Run unit tests locally:** + +```bash Command +pytest test/registered/unit/ -v # all unit tests +pytest test/registered/unit/mem_cache/ -v # one module +``` + +**Run with coverage:** + +```bash Command +pytest test/registered/unit/ --cov --cov-config=.coveragerc -v +``` + +For conventions on CI registration, test structure, and examples, see [`test/registered/unit/README.md`](https://github.com/sgl-project/sglang/tree/main/test/registered/unit/README.md). + +### E2E tests (server required) + +For tests that require launching a server, refer to [`test/registered/README.md`](https://github.com/sgl-project/sglang/tree/main/test/registered/README.md) for guidance on where to place your test. + +For detailed instructions on running tests and integrating them into CI, refer to [test/README.md](https://github.com/sgl-project/sglang/tree/main/test/README.md). ## Write documentations We recommend new contributors start from writing documentation, which helps you quickly understand SGLang codebase. -For more details, please refer to [docs/README](https://github.com/sgl-project/sglang/blob/main/docs/README.md). +For more details, please refer to [docs/README.md](https://github.com/sgl-project/sglang/tree/main/docs/README.md). ## Test the accuracy If your code changes the model output, please run the accuracy tests. A quick sanity check is the few-shot GSM8K. @@ -56,19 +89,19 @@ python3 -m sglang.test.few_shot_gsm8k --num-questions 200 ``` Please note that the above script is primarily a sanity check, not a rigorous accuracy or speed test. -This test can have significant variance (1%-5%) in accuracy due to batching and the non-deterministic nature of the inference engine. +This test can have significant variance (1%–5%) in accuracy due to batching and the non-deterministic nature of the inference engine. Also, do not rely on the "Latency/Output throughput" from this script, as it is not a proper speed test. GSM8K is too easy for state-of-the-art models nowadays. Please try your own more challenging accuracy tests. You can find additional accuracy eval examples in: -- [test_eval_accuracy_large.py](https://github.com/sgl-project/sglang/blob/main/test/srt/test_eval_accuracy_large.py) -- [test_gpt_oss_1gpu.py](https://github.com/sgl-project/sglang/blob/main/test/srt/test_gpt_oss_1gpu.py) +- [test_eval_accuracy_large.py](https://github.com/sgl-project/sglang/blob/main/test/registered/eval/test_eval_accuracy_large.py) +- [test_gpt_oss_1gpu.py](https://github.com/sgl-project/sglang/blob/main/test/registered/core/test_gpt_oss_1gpu.py) ## Benchmark the speed -Refer to [Benchmark and Profiling](../developer_guide/benchmark_and_profiling). +Refer to [Benchmark and Profiling](./benchmark_and_profiling). ## Requesting a review for merge -You can follow the pull request merge process described in [MAINTAINER](https://github.com/sgl-project/sglang/blob/main/.github/MAINTAINER). +You can follow the pull request merge process described in [MAINTAINER.md](https://github.com/sgl-project/sglang/blob/main/.github/MAINTAINER.md). You will need to work with the Merge Oncall, Codeowner, and other reviewers to get their approvals. Then your PR can be merged. @@ -77,6 +110,8 @@ Then your PR can be merged. We have a lot of open PRs but limited CI machines, so only top and trusted contributors have permission to trigger CI tests. Users with permission are listed in the [CI_PERMISSIONS.json](https://github.com/sgl-project/sglang/blob/main/.github/CI_PERMISSIONS.json) +**PR authors** can always use `/rerun-failed-ci` on their own PRs, even if they are not listed in `CI_PERMISSIONS.json`. + For CI to run on a pull request, it must have the "run-ci" label. Authorized users can add the label or rerun failed tests by commenting on the PR with one of these commands: - `/tag-run-ci-label`: Adds the "run-ci" label. Every future commit will trigger CI. @@ -84,18 +119,17 @@ For CI to run on a pull request, it must have the "run-ci" label. Authorized use - `/tag-and-rerun-ci`: A single command that performs both `/tag-run-ci-label` and `/rerun-failed-ci`. - `/rerun-stage `: Reruns a specific test stage without waiting for its dependencies. This is useful when you want to quickly validate a fix for a specific test failure instead of waiting ~30 minutes for preceding stages to complete. -If you have permission, the [Slash Command Handler](https://github.com/sgl-project/sglang/actions/workflows/slash-command-handler.yml) will run your command and react with a +1 to your comment. It may take up to a few minutes for the reaction to appear. Here's a usage [example](https://github.com/sgl-project/sglang/pull/14253#issuecomment-3599509302). +If you have permission, the [Slash Command Handler](https://github.com/sgl-project/sglang/actions/workflows/slash-command-handler.yml) will run your command and react with a 👍 to your comment. It may take up to a few minutes for the reaction to appear. Here’s a usage [example](https://github.com/sgl-project/sglang/pull/14253#issuecomment-3599509302). To avoid spamming a PR with too many `/rerun-failed-ci` comments, you can also trigger the command by editing an existing comment and adding any suffix (e.g., `/rerun-failed-ci try again`). Example of rerunning a single test stage: `/rerun-stage unit-test-backend-4-gpu`. -If you don't have permission, please ask maintainers to trigger CI for you. +If you don’t have permission and you’re not the PR author, please ask maintainers to trigger CI for you. ### CI rate limits Due to CI scheduling and limited resources, higher-priority PRs may preempt running jobs. In such cases, you may need to rerun the tests. - We apply CI rate limits to prevent abuse and ensure fair usage of our CI resources. Each CI workflow has a default limit defined in its workflow configuration file. For example, in [pr-gate.yml](https://github.com/sgl-project/sglang/blob/main/.github/workflows/pr-gate.yml), the default cooldown period is 120 minutes, and each workflow can override it via the `cool-down-minutes` input parameter: @@ -107,8 +141,7 @@ cool-down-minutes: default: 120 ``` -Users listed in [CI_PERMISSIONS.json](https://github.com/sgl-project/sglang/blob/main/.github/CI_PERMISSIONS.json) may have a per-user cooldown interval. In practice, we use the minimum of the workflow's default window and the user-specific interval. - +Users listed in [CI_PERMISSIONS.json](https://github.com/sgl-project/sglang/blob/main/.github/CI_PERMISSIONS.json) may have a per-user cooldown interval. In practice, we use the minimum of the workflow’s default window and the user-specific interval. ## Code style guidance - Avoid code duplication. If the same code snippet (more than five lines) appears multiple times, extract it into a shared function. @@ -121,28 +154,34 @@ Users listed in [CI_PERMISSIONS.json](https://github.com/sgl-project/sglang/blob - If a single test file run longer than 500 seconds, split it into multiple smaller files (e.g., `test_eagle_infer_a.py`, `test_eagle_infer_b.py`). - If a single job in a github workflow runs longer than 30 mins, split it into smaller jobs/steps. - Reuse server launches in your unit tests to make tests run faster. +- Never use `pickle.loads()`, `pickle.load()`, or `recv_pyobj()` to deserialize untrusted or network-received data. Python's [pickle module is not secure](https://docs.python.org/3/library/pickle.html) — it can execute arbitrary code during deserialization. Use safe serialization formats such as [msgpack](https://github.com/jcrist/msgspec) or JSON instead. - When supporting new hardware or features, follow these guidelines: - Do not drastically change existing code. - Always prefer new files to introduce specific components for your new hardware (e.g., `allocator_ascend.py`). - If you write multiple if/else blocks for new features, ensure the common path (e.g., NVIDIA hardware or the existing code path) is the first branch. ## How to update sgl-kernel -Since sglang and sgl-kernel are separate Python packages, our current GitHub CI infrastructure does not support updating a kernel and using it immediately within the same pull request (PR). -To add a new kernel or modify an existing one in the sgl-kernel package, you must use multiple PRs. +Since sglang and the `sglang-kernel` (prior `sgl-kernel`) distribution are separate Python packages, our current GitHub CI infrastructure does not support updating a kernel and using it immediately within the same pull request (PR). +To add a new kernel or modify an existing one in the `sgl-kernel/` source tree, you must use multiple PRs. Follow these steps: 1. Submit a PR to update the sgl-kernel source code without using it in sglang python package (e.g., [#8884](https://github.com/sgl-project/sglang/pull/8884/files)). -2. Bump the version of sgl-kernel (e.g., [#9220](https://github.com/sgl-project/sglang/pull/9220/files)). - - Once merged, this will trigger an automatic release of the sgl-kernel wheel to PyPI. +2. Bump the version of the kernel package (e.g., [#9220](https://github.com/sgl-project/sglang/pull/9220/files)). + - Once merged, this will trigger an automatic release of the `sglang-kernel` wheel to PyPI. - If not urgent, you can wait for other people to release the wheel. A new version will typically be released within one week. 3. Apply the changes: - - Update the sgl-kernel version in `sglang/python/pyproject.toml` to use the modified kernels. + - Update the `sglang-kernel` version in `sglang/python/pyproject.toml` to use the modified kernels. - Update the related caller code in the sglang to use the new kernel. ## Tips for newcomers -If you want to contribute but don't have a specific idea in mind, pick issues labeled ["good first issue" or "help wanted"](https://github.com/sgl-project/sglang/issues?q=is%3Aissue+label%3A%22good+first+issue%22%2C%22help+wanted%22). These tasks typically have lower complexity and provide an excellent introduction to the codebase. Also check out this [code walk-through](https://github.com/zhaochenyang20/Awesome-ML-SYS-Tutorial/tree/main/sglang/code-walk-through) for a deeper look into SGLang's workflow. +If you want to contribute but don’t have a specific idea in mind, pick issues labeled [“good first issue” or “help wanted”](https://github.com/sgl-project/sglang/issues?q=is%3Aissue+label%3A%22good+first+issue%22%2C%22help+wanted%22). These tasks typically have lower complexity and provide an excellent introduction to the codebase. + +Also check out the following materials as startup guide: +- [Mini-SGLang](https://github.com/sgl-project/mini-sglang) for a quick overview on the structure of sglang. +- [Code Walk-through](https://github.com/zhaochenyang20/Awesome-ML-SYS-Tutorial/tree/main/sglang/code-walk-through) for a deeper look into SGLang’s workflow. +- [GTC-2026 Training Lab](https://drive.google.com/file/d/1mwOZEtipNLJzrflCTodj34KhuOZEoEw5/view?usp=drive_link) for hands-on practices of how to do optimization, benchmarking, or profiling on a launched SGLang instance. If you have any questions or want to start a discussion, please feel free to ask in our [Slack channel](https://slack.sglang.io). diff --git a/docs_new/docs/developer_guide/development_guide_using_docker.mdx b/docs_new/docs/developer_guide/development_guide_using_docker.mdx index d7e167f9b..70e69aa28 100644 --- a/docs_new/docs/developer_guide/development_guide_using_docker.mdx +++ b/docs_new/docs/developer_guide/development_guide_using_docker.mdx @@ -7,7 +7,7 @@ metatags: ## Setup VSCode on a Remote Host (Optional - you can skip this step if you plan to run sglang dev container locally) -1. In the remote host, download `code` from [Https://code.visualstudio.com/docs/?dv=linux64cli](https://code.visualstudio.com/download) and run `code tunnel` in a shell. +1. In the remote host, download `code` from [VSCode](https://code.visualstudio.com/download) and run `code tunnel` in a shell. Example ```bash Command diff --git a/docs_new/docs/developer_guide/JIT_kernels.mdx b/docs_new/docs/developer_guide/development_jit_kernel_guide.mdx similarity index 62% rename from docs_new/docs/developer_guide/JIT_kernels.mdx rename to docs_new/docs/developer_guide/development_jit_kernel_guide.mdx index 27fe52a45..8b92fcd10 100644 --- a/docs_new/docs/developer_guide/JIT_kernels.mdx +++ b/docs_new/docs/developer_guide/development_jit_kernel_guide.mdx @@ -1,266 +1,425 @@ ---- -title: "Development Guide for JIT Kernels" -sidebarTitle: "JIT Kernels" -metatags: - description: "SGLang JIT kernel development: clangd setup, TensorMatcher, LaunchKernel, add_constant example walkthrough." ---- -## Environment Setup - -We strongly recommend using `clangd` as the language server for JIT kernel development. -For Ubuntu/Debian, you can download clangd from [apt.llvm.org](https://apt.llvm.org/). -If you are using VS Code, we recommend installing the `clangd` extension for better IDE integration. - -All JIT-related files are located in `python/sglang/jit_kernel`. -Unlike `sgl-kernel`, which compiles CUDA/C++ binaries ahead of time (AOT), just-in-time (JIT) kernels are compiled at runtime. -Consequently, a static `compile_commands.json` cannot be generated. -To enable code completion with `clangd`, run `python -m sglang.jit_kernel` to generate a `.clangd` configuration file in your current directory. -After generating the file, restart the clangd language server. It should now recognize all JIT kernel files. - -## Code Structure - -### C++ Implementation - -C++ source code is located in `python/sglang/jit_kernel/csrc`. -Reusable functions should be placed in `python/sglang/jit_kernel/include`. - -We use [tvm-ffi](https://github.com/apache/tvm-ffi) for efficient foreign language bindings. -Refer to the [documentation](https://tvm.apache.org/ffi/) for advanced usage, such as exporting C++ objects. -Typically, `tvm::ffi::TensorView` is sufficient for passing PyTorch Tensors from Python. - -### Python Interface - -Python interfaces are defined in `python/sglang/jit_kernel`. -The `load_jit` utility function in `python/sglang/jit_kernel/utils.py` loads and returns the compiled module. -To export a C++ function (e.g., `cpp_func`), pass `cuda_wrappers=[("func", "cpp_func")]` to `load_jit`. -The function can then be called in Python as `module.func`. - -### C++ Utilities - -The following C++ utilities are available: - -#### Integer Range - -Similar to PyTorch, we provide an `irange` function to represent an integer range. - -```C++ Example -#include - -void test() { - for (auto i : host::irange(100)) { // [0, 100) - // do something - } - for (auto i : host::irange(0, 100)) { // [0, 100) - // do something - } -} - -``` - -#### Runtime Checking - -`RuntimeCheck` validates conditions at runtime. It accepts optional arguments for error reporting. -If the check fails, these arguments are output to aid debugging. -`RuntimeDeviceCheck` verifies the status of the last kernel launch. - -```C++ Example -#include -#include - -void test() { - host::RuntimeCheck(1 + 1 == 2, 1 + 1, " != ", 2); - host::RuntimeDeviceCheck(); - // check the provided `cudaError_t` - host::RuntimeDeviceCheck(cudaGetLastError()); -} - -``` - -#### Tensor Checking - -`TensorMatcher` provides a readable way to validate and extract tensor shape information. - -```cpp Example -#include - -void test(const tvm::ffi::TensorView k_cache, const tvm::ffi::TensorView v_cache) { - using namespace host; - - auto D = SymbolicSize{"D"}; // cache dimension - auto N = SymbolicSize{"N"}; // kvcache stride - auto dtype = SymbolicDType{}; - auto device = SymbolicDevice{}; - - TensorMatcher({-1, D}) // - .with_strides({N, 1}) - .with_dtype(dtype) - .with_device(device) - .verify(k_cache) - .verify(v_cache); -} -``` - -Configure the `TensorMatcher` with expected stride, dtype, and device properties before verification. -- If `with_strides` is omitted, the tensor is expected to be contiguous. -- Template arguments in `with_dtype` restrict the allowed data types. -- Template arguments in `with_device` restrict the allowed devices. -- Values passed to `with_xxx` methods enforce equality checks. -- Passing `-1` for size or stride allows matching any value. - -A `Symbolic` variable must resolve to the same value across all verifications. -Use `.unwrap()` to retrieve the matched value after verification. - - -`TensorMatcher` is a temporary expression and should not be stored in a variable. - - - -Add `//` at the end of the `TensorMatcher` chain to enforce proper indentation. - - -#### Kernel Launching - -`LaunchKernel::resolve_device` retrieves the current `cudaStream` from PyTorch. -Kernels can also be launched directly using `LaunchKernel`. - -```cpp Example -#include - -#include - -__global__ void kernel() {} - -void test() { - const auto num_blocks = 1; - const auto num_threads = 32; - const auto dynamic_smem = 0; - - DLDevice dev; // suppose this is initialized properly - host::LaunchKernel(num_blocks, num_threads, dev)(kernel); - - cudaStream_t stream = host::LaunchKernel::resolve_device(dev); - host::LaunchKernel(num_blocks, num_threads, stream, dynamic_smem)(kernel); -} - -``` - -## Add new kernels - -This section walks through a complete, end-to-end example of adding a new JIT kernel to the system. -We use a simple add_constant kernel as a running example, which adds a constant integer value to every element of an input tensor. - -Conceptually, the Python interface looks like this: - -```python Example -def add_constant(src: torch.Tensor, c: int): - return src + c -``` - -### STEP 1: Write the C++ kernel - -Write your CUDA kernel in [jit_kernel/csrc/add_constant.cuh](https://github.com/sgl-project/sglang/blob/main/python/sglang/jit_kernel/csrc/add_constant.cuh). For demonstration purposes, we pass the constant value as a template parameter. - -```cpp Example -#include // For TensorMatcher, SymbolicSize, SymbolicDevice -#include // For LaunchKernel -#include // For div_ceil, RuntimeCheck - -#include -#include - -#include -#include - -namespace { - -template -__global__ void add_constant_kernel(int32_t* dst, const int32_t* src, size_t length) { - size_t idx = blockIdx.x * blockDim.x + threadIdx.x; - if (idx < length) { - dst[idx] = src[idx] + kConstant; - } -} - -constexpr size_t kBlockSize = 256; - -// You can also use struct with static method as an alternative -template -void add_constant(tvm::ffi::TensorView dst, tvm::ffi::TensorView src) { - using namespace host; - - // 1. Validate input tensors - SymbolicSize N = {"num_elements"}; - SymbolicDevice device_; - TensorMatcher({N}) // 1D tensor, must be contiguous - .with_dtype() // must be int32 - .with_device(device_) // must be on CUDA device - .verify(dst) // check tensor dst - .verify(src); // check tensor src - - // 2. Extract required parameters, prepare for kernel launch - const size_t num_elements = N.unwrap(); - const size_t grid_size = div_ceil(num_elements, kBlockSize); - const DLDevice device = device_.unwrap(); - // some extra runtime checks using host::RuntimeCheck - RuntimeCheck(num_elements > 0, "We only support non-empty tensors, got num_elements = ", num_elements); - - // 3. Launch the kernel. Error code will be automatically checked. - LaunchKernel(grid_size, kBlockSize, device /*, dynamic_smem*/)( - // kernel function - add_constant_kernel, - // kernel arguments - static_cast(dst.data_ptr()), - static_cast(src.data_ptr()), - num_elements); -} - -} // namespace - -``` - -### STEP 2: Create Python Interfaces - -Next, expose the kernel through a Python wrapper. -Create a new file at [jit_kernel/add_constant.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/jit_kernel/add_constant.py) and expose the needed interfaces. - -```python Example -from __future__ import annotations - -import functools -from typing import TYPE_CHECKING - -import torch - -from sglang.jit_kernel.utils import load_jit, make_cpp_args - -if TYPE_CHECKING: - from tvm_ffi.module import Module - - -@functools.cache -def _jit_add_constant_module(constant: int) -> Module: - args = make_cpp_args(constant) # pass all the template argument - return load_jit( - "add_constant", - *args, - cuda_files=["add_constant.cuh"], - cuda_wrappers=[("add_constant", f"add_constant<{args}>")], - ) - - -def add_constant(src: torch.Tensor, constant: int) -> torch.Tensor: - dst = torch.empty_like(src) - module = _jit_add_constant_module(constant) - module.add_constant(dst, src) - return dst - -``` - -### STEP 3: Use your kernel - -Finally, import and use the kernel like a regular Python function: - -```python Example -from sglang.jit_kernel.add_constant import add_constant -``` - -For a complete, runnable example, refer to [test_add_constant.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/jit_kernel/test_add_constant.py). +--- +title: "Development Guide for JIT Kernels" +sidebarTitle: "JIT Kernels" +metatags: + description: "SGLang JIT kernel development: clangd setup, TensorMatcher, LaunchKernel, add_constant example walkthrough." +--- +## Environment Setup + +We strongly recommend using `clangd` as the language server for JIT kernel development. +For Ubuntu/Debian, you can download clangd from [apt.llvm.org](https://apt.llvm.org/). +If you are using VS Code, we recommend installing the `clangd` extension for better IDE integration. + +All JIT-related files are located in `python/sglang/jit_kernel`. +Unlike `sgl-kernel`, which compiles CUDA/C++ binaries ahead of time (AOT), just-in-time (JIT) kernels are compiled at runtime. +Consequently, a static `compile_commands.json` cannot be generated. +To enable code completion with `clangd`, run `python -m sglang.jit_kernel` to generate a `.clangd` configuration file in your current directory. +After generating the file, restart the clangd language server. It should now recognize all JIT kernel files. + +## Code Structure + +### C++ Implementation + +C++ source code is located in `python/sglang/jit_kernel/csrc`. +Reusable functions should be placed in `python/sglang/jit_kernel/include`. + +We use [tvm-ffi](https://github.com/apache/tvm-ffi) for efficient foreign language bindings. +Refer to the [documentation](https://tvm.apache.org/ffi/) for advanced usage, such as exporting C++ objects. +Typically, `tvm::ffi::TensorView` is sufficient for passing PyTorch Tensors from Python. + +### Python Interface + +Python interfaces are defined in `python/sglang/jit_kernel`. +The `load_jit` utility function in `python/sglang/jit_kernel/utils.py` loads and returns the compiled module. +To export a C++ function (e.g., `cpp_func`), pass `cuda_wrappers=[("func", "cpp_func")]` to `load_jit`. +The function can then be called in Python as `module.func`. + +For caching compiled modules, prefer `sglang.jit_kernel.utils.cache_once` over `functools.lru_cache`. +`functools.lru_cache` is not compatible with `torch.compile`. + +### C++ Utilities + +The following C++ utilities are available: + +#### Integer Range + +Similar to PyTorch, we provide an `irange` function to represent an integer range. + +```C++ Example +#include + +void test() { + for (auto i : host::irange(100)) { // [0, 100) + // do something + } + for (auto i : host::irange(0, 100)) { // [0, 100) + // do something + } +} + +``` + +#### Runtime Checking + +`RuntimeCheck` validates conditions at runtime. It accepts optional arguments for error reporting. +If the check fails, these arguments are output to aid debugging. +`RuntimeDeviceCheck` verifies the status of the last kernel launch. + +```C++ Example +#include +#include + +void test() { + host::RuntimeCheck(1 + 1 == 2, 1 + 1, " != ", 2); + host::RuntimeDeviceCheck(); + // check the provided `cudaError_t` + host::RuntimeDeviceCheck(cudaGetLastError()); +} + +``` + +#### Tensor Checking + +`TensorMatcher` provides a readable way to validate and extract tensor shape information. + +```cpp Example +#include + +void test(const tvm::ffi::TensorView k_cache, const tvm::ffi::TensorView v_cache) { + using namespace host; + + auto D = SymbolicSize{"D"}; // cache dimension + auto N = SymbolicSize{"N"}; // kvcache stride + auto dtype = SymbolicDType{}; + auto device = SymbolicDevice{}; + + TensorMatcher({-1, D}) // + .with_strides({N, 1}) + .with_dtype(dtype) + .with_device(device) + .verify(k_cache) + .verify(v_cache); +} +``` + +Configure the `TensorMatcher` with expected stride, dtype, and device properties before verification. +- If `with_strides` is omitted, the tensor is expected to be contiguous. +- Template arguments in `with_dtype` restrict the allowed data types. +- Template arguments in `with_device` restrict the allowed devices. +- Values passed to `with_xxx` methods enforce equality checks. +- Passing `-1` for size or stride allows matching any value. + +A `Symbolic` variable must resolve to the same value across all verifications. +Use `.unwrap()` to retrieve the matched value after verification. + +> Note: `TensorMatcher` is a temporary expression and should not be stored in a variable. + +> Tip: Add `//` at the end of the `TensorMatcher` chain to enforce proper indentation. + +#### Kernel Launching + +`LaunchKernel::resolve_device` retrieves the current `cudaStream` from PyTorch. +Kernels can also be launched directly using `LaunchKernel`. + +```cpp Example +#include + +#include + +__global__ void kernel() {} + +void test() { + const auto num_blocks = 1; + const auto num_threads = 32; + const auto dynamic_smem = 0; + + DLDevice dev; // suppose this is initialized properly + host::LaunchKernel(num_blocks, num_threads, dev)(kernel); + + cudaStream_t stream = host::LaunchKernel::resolve_device(dev); + host::LaunchKernel(num_blocks, num_threads, stream, dynamic_smem)(kernel); +} + +``` + +## Add new kernels + +This section walks through a complete, end-to-end example of adding a new JIT kernel to the system. +We use a simple add_constant kernel as a running example, which adds a constant integer value to every element of an input tensor. + +Conceptually, the Python interface looks like this: + +```python Example +def add_constant(src: torch.Tensor, c: int): + return src + c +``` + +### STEP 1: Write the C++ kernel + +Write your CUDA kernel in [jit_kernel/csrc/add_constant.cuh](https://github.com/sgl-project/sglang/blob/main/python/sglang/jit_kernel/csrc/add_constant.cuh). For demonstration purposes, we pass the constant value as a template parameter. + +```cpp Example +#include // For TensorMatcher, SymbolicSize, SymbolicDevice +#include // For LaunchKernel +#include // For div_ceil, RuntimeCheck + +#include +#include + +#include +#include + +namespace { + +template +__global__ void add_constant_kernel(int32_t* dst, const int32_t* src, size_t length) { + size_t idx = blockIdx.x * blockDim.x + threadIdx.x; + if (idx < length) { + dst[idx] = src[idx] + kConstant; + } +} + +constexpr size_t kBlockSize = 256; + +// You can also use struct with static method as an alternative +template +void add_constant(tvm::ffi::TensorView dst, tvm::ffi::TensorView src) { + using namespace host; + + // 1. Validate input tensors + SymbolicSize N = {"num_elements"}; + SymbolicDevice device_; + TensorMatcher({N}) // 1D tensor, must be contiguous + .with_dtype() // must be int32 + .with_device(device_) // must be on CUDA device + .verify(dst) // check tensor dst + .verify(src); // check tensor src + + // 2. Extract required parameters, prepare for kernel launch + const size_t num_elements = N.unwrap(); + const size_t grid_size = div_ceil(num_elements, kBlockSize); + const DLDevice device = device_.unwrap(); + // some extra runtime checks using host::RuntimeCheck + RuntimeCheck(num_elements > 0, "We only support non-empty tensors, got num_elements = ", num_elements); + + // 3. Launch the kernel. Error code will be automatically checked. + LaunchKernel(grid_size, kBlockSize, device /*, dynamic_smem*/)( + // kernel function + add_constant_kernel, + // kernel arguments + static_cast(dst.data_ptr()), + static_cast(src.data_ptr()), + num_elements); +} + +} // namespace + +``` + +### STEP 2: Create Python Interfaces + +Next, expose the kernel through a Python wrapper. +Create a new file at [jit_kernel/add_constant.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/jit_kernel/add_constant.py) and expose the needed interfaces. + +```python Example +from __future__ import annotations +from typing import TYPE_CHECKING + +import torch + +from sglang.jit_kernel.utils import cache_once, load_jit, make_cpp_args + +if TYPE_CHECKING: + from tvm_ffi.module import Module + + +@cache_once +def _jit_add_constant_module(constant: int) -> Module: + args = make_cpp_args(constant) # pass all the template argument + return load_jit( + "add_constant", + *args, + cuda_files=["add_constant.cuh"], + cuda_wrappers=[("add_constant", f"add_constant<{args}>")], + ) + + +def add_constant(src: torch.Tensor, constant: int) -> torch.Tensor: + if not src.is_cuda: + raise RuntimeError("src must be a CUDA tensor") + if src.dtype != torch.int32: + raise RuntimeError(f"Unsupported dtype {src.dtype}. Supported: int32") + dst = torch.empty_like(src) + module = _jit_add_constant_module(constant) + module.add_constant(dst, src) + return dst + +``` + +Keep the Python wrapper thin, but still validate the basic invariants such as device and dtype before dispatch. In the current JIT/FFI path, invalid tensors are not always rejected safely before launch. + +### STEP 3: Use your kernel + +Finally, import and use the kernel like a regular Python function: + +```python Example +from sglang.jit_kernel.add_constant import add_constant +``` + +For a complete, runnable example, refer to [test_add_constant.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/jit_kernel/tests/test_add_constant.py). + +## C++ Include Library Reference + +The JIT kernel framework provides a set of reusable C++ headers in +`python/sglang/jit_kernel/include/sgl_kernel/`. Each header is designed +to be lightweight and self-contained. Below is a summary of each header +and its key APIs. + +### Core Utilities + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
HeaderNamespacePurpose
utils.hhostHost-side essentials: RuntimeCheck, Panic, div_ceil, irange
utils.cuhdevice / hostType aliases (fp16_t, bf16_t, ...), SGL_DEVICE macro, PDL helpers, LaunchKernel, RuntimeDeviceCheck
source_location.h(global)Portable std::source_location wrapper for error reporting
runtime.cuhhost::runtimeCUDA runtime queries: get_blocks_per_sm, get_sm_count, get_cc_major, get_runtime_version, get_available_dynamic_smem_per_block
+ +### Tensor Validation + + + + + + + + + + + + + + + + +
HeaderNamespacePurpose
tensor.hhostTensorMatcher, SymbolicSize, SymbolicDType, SymbolicDevice
+ +### Math & Type System + + + + + + + + + + + + + + + + + + + + + +
HeaderNamespacePurpose
math.cuhdevice::mathmax, min, abs, sqrt, rsqrt, exp, sin, cos, constants
type.cuh(global) / devicedtype_trait<T>, packed_t<T>, device::cast<To>(from)
+ +### Memory Access + + + + + + + + + + + + + + + + + + + + + +
HeaderNamespacePurpose
vec.cuhdeviceAlignedVector<T, N> - vectorized load/store (up to 128-bit; 256-bit requires Blackwell GPUs)
tile.cuhdevice::tileMemory<T> - cooperative tiled memory I/O (thread/warp/CTA)
+ +### Parallel Primitives + + + + + + + + + + + + + + + + + + + + + + + + + + +
HeaderNamespacePurpose
warp.cuhdevice::warpreduce_sum, reduce_max via __shfl_xor_sync
cta.cuhdevice::ctareduce_max across warps via shared memory
atomic.cuhdevice::atomicmax - atomic float max (CUDA + ROCm fallback)
+ +### Reusable Kernel Templates + + + + + + + + + + + + + + + + +
HeaderNamespacePurpose
impl/norm.cuhhost::norm / device::normRMSNorm building blocks (warp & CTA paths, StorageType)
diff --git a/docs_new/docs/developer_guide/overview.mdx b/docs_new/docs/developer_guide/overview.mdx index 3c952d724..cf1ef2dd7 100644 --- a/docs_new/docs/developer_guide/overview.mdx +++ b/docs_new/docs/developer_guide/overview.mdx @@ -5,7 +5,7 @@ description: Contributing to SGLang — development setup, benchmarking, and eva - [Contribution Guide](./contribution_guide) - [Development Guide (Docker)](./development_guide_using_docker) -- [JIT Kernels](./JIT_kernels) +- [JIT Kernels](./development_jit_kernel_guide) - [Benchmark and Profiling](./benchmark_and_profiling) - [Bench Serving](./bench_serving) - [Evaluating New Models](./evaluating_new_models) diff --git a/docs_new/docs/developer_guide/setup_github_runner.mdx b/docs_new/docs/developer_guide/setup_github_runner.mdx index 4c4243623..213a25d47 100644 --- a/docs_new/docs/developer_guide/setup_github_runner.mdx +++ b/docs_new/docs/developer_guide/setup_github_runner.mdx @@ -10,14 +10,14 @@ metatags: **You can mount a folder for the shared huggingface model weights cache. ** The command below uses `/tmp/huggingface` as an example. -```text Output +``` docker pull nvidia/cuda:12.9.1-devel-ubuntu22.04 # Nvidia docker run --shm-size 128g -it -v /tmp/huggingface:/hf_home --gpus all nvidia/cuda:12.9.1-devel-ubuntu22.04 /bin/bash # AMD -docker run --rm --device=/dev/kfd --device=/dev/dri --group-add video --shm-size 128g -it -v /tmp/huggingface:/hf_home lmsysorg/sglang:v0.5.0rc1-rocm630 /bin/bash +docker run --rm --device=/dev/kfd --device=/dev/dri --group-add video --shm-size 128g -it -v /tmp/huggingface:/hf_home lmsysorg/sglang:v0.5.8-rocm700-mi30x /bin/bash # AMD just the last 2 GPUs -docker run --rm --device=/dev/kfd --device=/dev/dri/renderD176 --device=/dev/dri/renderD184 --group-add video --shm-size 128g -it -v /tmp/huggingface:/hf_home lmsysorg/sglang:v0.5.0rc1-rocm630 /bin/bash +docker run --rm --device=/dev/kfd --device=/dev/dri/renderD176 --device=/dev/dri/renderD184 --group-add video --shm-size 128g -it -v /tmp/huggingface:/hf_home lmsysorg/sglang:v0.5.8-rocm700-mi30x /bin/bash ``` ### Step 2: Configure the runner by `config.sh` @@ -30,11 +30,11 @@ pip install --upgrade pip export RUNNER_ALLOW_RUNASROOT=1 ``` -Then follow https://github.com/sgl-project/sglang/settings/actions/runners/new?arch=x64&os=linux to run `config.sh` +Then follow https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/adding-self-hosted-runners to run `config.sh` **Notes** - Do not need to specify the runner group -- Give it a name (e.g., `test-sgl-gpu-0`) and some labels (e.g., `1-gpu-runner`). The labels can be edited later in Github Settings. +- Give it a name (e.g., `test-sgl-gpu-0`) and some labels (e.g., `1-gpu-h100`). The labels can be edited later in Github Settings. - Do not need to change the work folder. ### Step 3: Run the runner by `run.sh` diff --git a/docs_new/docs/get-started/install.mdx b/docs_new/docs/get-started/install.mdx new file mode 100644 index 000000000..5f2e6b1b2 --- /dev/null +++ b/docs_new/docs/get-started/install.mdx @@ -0,0 +1,237 @@ +--- +title: Installation +description: Install SGLang with pip/uv, source, Docker, Kubernetes, and cloud deployment options. +keywords: + - installation + - sglang + - pip + - docker +--- +You can install SGLang using one of the methods below. +This page primarily applies to common NVIDIA GPU platforms. +For other or newer platforms, please refer to the dedicated pages for [AMD GPUs](../hardware-platforms/amd_gpu), [Intel Xeon CPUs](../hardware-platforms/cpu_server), [Google TPU](../hardware-platforms/tpu), [NVIDIA DGX Spark](https://lmsys.org/blog/2025-11-03-gpt-oss-on-nvidia-dgx-spark/), [NVIDIA Jetson](../hardware-platforms/nvidia_jetson), [Ascend NPUs](../hardware-platforms/ascend-npus/ascend_npu), and [Intel XPU](../hardware-platforms/xpu). + +## Method 1: With pip or uv + +It is recommended to use uv for faster installation: + +```bash Command +pip install --upgrade pip +pip install uv +uv pip install sglang +``` + +### For CUDA 13 + +Docker is recommended (see Method 3 note on B300/GB300/CUDA 13). If you do not have Docker access, follow these steps: + +1. Install PyTorch with CUDA 13 support first: +```bash Command +# Replace X.Y.Z with the version by your SGLang install +uv pip install torch==X.Y.Z torchvision torchaudio --index-url https://download.pytorch.org/whl/cu130 +``` + +2. Install sglang: +```bash Command +uv pip install sglang +``` + +3. Install the `sglang-kernel` wheel for CUDA 13 from [the sgl-project whl releases](https://github.com/sgl-project/whl/blob/gh-pages/cu130/sglang-kernel/index.html). Replace `X.Y.Z` with the `sglang-kernel` version required by your SGLang install (you can find this by running `uv pip show sglang-kernel`). Examples: +```bash Command +# x86_64 +uv pip install "https://github.com/sgl-project/whl/releases/download/vX.Y.Z/sglang_kernel-X.Y.Z+cu130-cp310-abi3-manylinux2014_x86_64.whl" + +# aarch64 +uv pip install "https://github.com/sgl-project/whl/releases/download/vX.Y.Z/sglang_kernel-X.Y.Z+cu130-cp310-abi3-manylinux2014_aarch64.whl" +``` + +4. If you encounter `ptxas fatal : Value 'sm_103a' is not defined for option 'gpu-name'` on B300/GB300, fix it with: +```bash Command +export TRITON_PTXAS_PATH=/usr/local/cuda/bin/ptxas +``` + +### **Quick fixes to common problems** +- If you encounter `OSError: CUDA_HOME environment variable is not set`. Please set it to your CUDA install root with either of the following solutions: + 1. Use `export CUDA_HOME=/usr/local/cuda-` to set the `CUDA_HOME` environment variable. + 2. Install FlashInfer first following [FlashInfer installation doc](https://docs.flashinfer.ai/installation.html), then install SGLang as described above. + +## Method 2: From source + +```bash Command +# Use the last release branch +git clone -b v0.5.9 https://github.com/sgl-project/sglang.git +cd sglang + +# Install the python packages +pip install --upgrade pip +pip install -e "python" +``` + +**Quick fixes to common problems** + +- If you want to develop SGLang, you can try the dev docker image. Please refer to [setup docker container](../developer_guide/development_guide_using_docker#setup-docker-container). The docker image is `lmsysorg/sglang:dev`. + +## Method 3: Using docker + +The docker images are available on Docker Hub at [lmsysorg/sglang](https://hub.docker.com/r/lmsysorg/sglang/tags), built from [Dockerfile](https://github.com/sgl-project/sglang/tree/main/docker). +Replace `` below with your huggingface hub [token](https://huggingface.co/docs/hub/en/security-tokens). + +```bash Command +docker run --gpus all \ + --shm-size 32g \ + -p 30000:30000 \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --env "HF_TOKEN=" \ + --ipc=host \ + lmsysorg/sglang:latest \ + python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --host 0.0.0.0 --port 30000 +``` + +For production deployments, use the `runtime` variant which is significantly smaller (~40% reduction) by excluding build tools and development dependencies: + +```bash Command +docker run --gpus all \ + --shm-size 32g \ + -p 30000:30000 \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --env "HF_TOKEN=" \ + --ipc=host \ + lmsysorg/sglang:latest-runtime \ + python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --host 0.0.0.0 --port 30000 +``` + +You can also find the nightly docker images [here](https://hub.docker.com/r/lmsysorg/sglang/tags?name=nightly). + +Notes: +- On B300/GB300 (SM103) or CUDA 13 environment, we recommend using the nightly image at `lmsysorg/sglang:dev-cu13` or stable image at `lmsysorg/sglang:latest-cu130-runtime`. Please, do not re-install the project as editable inside the docker image, since it will override the version of libraries specified by the cu13 docker image. + +## Method 4: Using Kubernetes + +Please check out [OME](https://github.com/sgl-project/ome), a Kubernetes operator for enterprise-grade management and serving of large language models (LLMs). + +
+More + +1. Option 1: For single node serving (typically when the model size fits into GPUs on one node) + + Execute command `kubectl apply -f docker/k8s-sglang-service.yaml`, to create k8s deployment and service, with llama-31-8b as example. + +2. Option 2: For multi-node serving (usually when a large model requires more than one GPU node, such as `DeepSeek-R1`) + + Modify the LLM model path and arguments as necessary, then execute command `kubectl apply -f docker/k8s-sglang-distributed-sts.yaml`, to create two nodes k8s statefulset and serving service. + +
+ +## Method 5: Using docker compose + +
+More + +> This method is recommended if you plan to serve it as a service. +> A better approach is to use the [k8s-sglang-service.yaml](https://github.com/sgl-project/sglang/blob/main/docker/k8s-sglang-service.yaml). + +1. Copy the [compose.yml](https://github.com/sgl-project/sglang/blob/main/docker/compose.yaml) to your local machine +2. Execute the command `docker compose up -d` in your terminal. +
+ +## Method 6: Run on Kubernetes or Clouds with SkyPilot + +
+More + +To deploy on Kubernetes or 12+ clouds, you can use [SkyPilot](https://github.com/skypilot-org/skypilot). + +1. Install SkyPilot and set up Kubernetes cluster or cloud access: see [SkyPilot's documentation](https://skypilot.readthedocs.io/en/latest/getting-started/installation.html). +2. Deploy on your own infra with a single command and get the HTTP API endpoint: +
+SkyPilot YAML: sglang.yaml + +```yaml Config +# sglang.yaml +envs: + HF_TOKEN: null + +resources: + image_id: docker:lmsysorg/sglang:latest + accelerators: A100 + ports: 30000 + +run: | + conda deactivate + python3 -m sglang.launch_server \ + --model-path meta-llama/Llama-3.1-8B-Instruct \ + --host 0.0.0.0 \ + --port 30000 +``` + +
+ +```bash Command +# Deploy on any cloud or Kubernetes cluster. Use --cloud to select a specific cloud provider. +HF_TOKEN= sky launch -c sglang --env HF_TOKEN sglang.yaml + +# Get the HTTP API endpoint +sky status --endpoint 30000 sglang +``` + +3. To further scale up your deployment with autoscaling and failure recovery, check out the [SkyServe + SGLang guide](https://github.com/skypilot-org/skypilot/tree/master/llm/sglang#serving-llama-2-with-sglang-for-more-traffic-using-skyserve). + +
+ +## Method 7: Run on AWS SageMaker + +
+More + +To deploy on SGLang on AWS SageMaker, check out [AWS SageMaker Inference](https://aws.amazon.com/sagemaker/ai/deploy) + +Amazon Web Services provide supports for SGLang containers along with routine security patching. For available SGLang containers, check out [AWS SGLang DLCs](https://github.com/aws/deep-learning-containers/blob/master/available_images.md#sglang-containers) + +To host a model with your own container, follow the following steps: + +1. Build a docker container with [sagemaker.Dockerfile](https://github.com/sgl-project/sglang/blob/main/docker/sagemaker.Dockerfile) alongside the [serve](https://github.com/sgl-project/sglang/blob/main/docker/serve) script. +2. Push your container onto AWS ECR. + +
+Dockerfile Build Script: build-and-push.sh + +```bash Command +#!/bin/bash +AWS_ACCOUNT="" +AWS_REGION="" +REPOSITORY_NAME="" +IMAGE_TAG="" + +ECR_REGISTRY="${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com" +IMAGE_URI="${ECR_REGISTRY}/${REPOSITORY_NAME}:${IMAGE_TAG}" + +echo "Starting build and push process..." + +# Login to ECR +echo "Logging into ECR..." +aws ecr get-login-password --region ${AWS_REGION} | docker login --username AWS --password-stdin ${ECR_REGISTRY} + +# Build the image +echo "Building Docker image..." +docker build -t ${IMAGE_URI} -f sagemaker.Dockerfile . + +echo "Pushing ${IMAGE_URI}" +docker push ${IMAGE_URI} + +echo "Build and push completed successfully!" +``` + +
+ +3. Deploy a model for serving on AWS Sagemaker, refer to [deploy_and_serve_endpoint.py](https://github.com/sgl-project/sglang/blob/main/examples/sagemaker/deploy_and_serve_endpoint.py). For more information, check out [sagemaker-python-sdk](https://github.com/aws/sagemaker-python-sdk). + 1. By default, the model server on SageMaker will run with the following command: `python3 -m sglang.launch_server --model-path opt/ml/model --host 0.0.0.0 --port 8080`. This is optimal for hosting your own model with SageMaker. + 2. To modify your model serving parameters, the [serve](https://github.com/sgl-project/sglang/blob/main/docker/serve) script allows for all available options within `python3 -m sglang.launch_server --help` cli by specifying environment variables with prefix `SM_SGLANG_`. + 3. The serve script will automatically convert all environment variables with prefix `SM_SGLANG_` from `SM_SGLANG_INPUT_ARGUMENT` into `--input-argument` to be parsed into `python3 -m sglang.launch_server` cli. + 4. For example, to run [Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B) with reasoning parser, simply add additional environment variables `SM_SGLANG_MODEL_PATH=Qwen/Qwen3-0.6B` and `SM_SGLANG_REASONING_PARSER=qwen3`. + +
+ +## Common Notes + +- [FlashInfer](https://github.com/flashinfer-ai/flashinfer) is the default attention kernel backend. It only supports sm75 and above. If you encounter any FlashInfer-related issues on sm75+ devices (e.g., T4, A10, A100, L4, L40S, H100), please switch to other kernels by adding `--attention-backend triton --sampling-backend pytorch` and open an issue on GitHub. +- To reinstall flashinfer locally, use the following command: `pip3 install --upgrade flashinfer-python --force-reinstall --no-deps` and then delete the cache with `rm -rf ~/.cache/flashinfer`. diff --git a/docs_new/docs/get-started/installation.mdx b/docs_new/docs/get-started/installation.mdx deleted file mode 100644 index 0402af64a..000000000 --- a/docs_new/docs/get-started/installation.mdx +++ /dev/null @@ -1,256 +0,0 @@ ---- -title: Installation -description: Install SGLang with pip/uv, source, Docker, Kubernetes, and cloud deployment options. -keywords: - - installation - - sglang - - pip - - docker ---- - -You can install SGLang using one of the methods below. -This page primarily applies to common NVIDIA GPU platforms. -For other or newer platforms, please refer to the dedicated pages for [AMD GPUs](../hardware-platforms/amd-gpus), [Intel Xeon CPUs](../hardware-platforms/cpu-server), [Google TPU](../hardware-platforms/tpu), [NVIDIA DGX Spark](https://lmsys.org/blog/2025-11-03-gpt-oss-on-nvidia-dgx-spark/), [NVIDIA Jetson](../hardware-platforms/nvidia), [Ascend NPUs](../hardware-platforms/ascend-npus/SGLang-installation-with-NPUs-support), and [Intel XPU](../hardware-platforms/xpu). - - -## Install methods - - - - It is recommended to use uv for faster installation: - - ```bash - pip install --upgrade pip - pip install uv - uv pip install "sglang" - ``` - - ### Quick fixes to common problems - - - - In some cases (for example, GB200), the command above might install a wrong torch version (for example, the CPU version) due to dependency resolution. Reinstall the correct [PyTorch](https://pytorch.org/get-started/locally/) with the following: - - ```bash - uv pip install "torch" "torchvision" --extra-index-url https://download.pytorch.org/whl/cu129 --force-reinstall - ``` - - - - If you do not have Docker access, install the matching `sgl_kernel` wheel from [the sgl-project whl releases](https://github.com/sgl-project/whl/releases) after installing SGLang. Replace `X.Y.Z` with the `sgl_kernel` version required by your SGLang (you can find this by running `uv pip show sgl_kernel`). - - **x86_64** - - ```bash - uv pip install "https://github.com/sgl-project/whl/releases/download/vX.Y.Z/sgl_kernel-X.Y.Z+cu130-cp310-abi3-manylinux2014_x86_64.whl" - ``` - - **aarch64** - - ```bash - uv pip install "https://github.com/sgl-project/whl/releases/download/vX.Y.Z/sgl_kernel-X.Y.Z+cu130-cp310-abi3-manylinux2014_aarch64.whl" - ``` - - - - Choose one of the following solutions: - - 1. Set `CUDA_HOME` to your CUDA install root: - - ```bash - export CUDA_HOME=/usr/local/cuda- - ``` - - 2. Install FlashInfer first following the [FlashInfer installation doc](https://docs.flashinfer.ai/installation.html), then install SGLang as described above. - - - - - -```bash -git clone https://github.com/sgl-project/sglang.git -cd sglang -pip install --upgrade pip -pip install -e "python" -``` - - ### Quick fixes to common problems - - - - If you want to develop SGLang, try the dev docker image. Refer to [setup docker container](../developer_guide/development_guide_using_docker#setup-docker-container). The docker image is `lmsysorg/sglang:dev`. - - - - - - The docker images are available on Docker Hub at [lmsysorg/sglang](https://hub.docker.com/r/lmsysorg/sglang/tags), built from [Dockerfile](https://github.com/sgl-project/sglang/tree/main/docker). - Replace `` below with your huggingface hub [token](https://huggingface.co/docs/hub/en/security-tokens). - - **Standard image** - - ```bash - docker run --gpus all \ - --shm-size 32g \ - -p 30000:30000 \ - -v ~/.cache/huggingface:/root/.cache/huggingface \ - --env "HF_TOKEN=" \ - --ipc=host \ - lmsysorg/sglang:latest \ - python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --host 0.0.0.0 --port 30000 - ``` - - **Runtime image for production** - - ```bash - docker run --gpus all \ - --shm-size 32g \ - -p 30000:30000 \ - -v ~/.cache/huggingface:/root/.cache/huggingface \ - --env "HF_TOKEN=" \ - --ipc=host \ - lmsysorg/sglang:latest-runtime \ - python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --host 0.0.0.0 --port 30000 - ``` - - You can also find the nightly docker images [here](https://hub.docker.com/r/lmsysorg/sglang/tags?name=nightly). - - - On B300/GB300 (SM103) or CUDA 13 environment, use the nightly image at `lmsysorg/sglang:dev-cu13` or stable image at `lmsysorg/sglang:latest-cu130-runtime`. Do not re-install the project as editable inside the docker image, since it will override the version of libraries specified by the cu13 docker image. - - - - - Please check out [OME](https://github.com/sgl-project/ome), a Kubernetes operator for enterprise-grade management and serving of large language models (LLMs). - - - - For models that fit into GPUs on one node, create the deployment and service with llama-31-8b as example. - - ```bash - kubectl apply -f docker/k8s-sglang-service.yaml - ``` - - - - For larger models (for example, `DeepSeek-R1`), modify the model path and arguments, then create the statefulset and service. - - ```bash - kubectl apply -f docker/k8s-sglang-distributed-sts.yaml - ``` - - - - - - - This method is recommended if you plan to serve it as a service. A better approach is to use the [k8s-sglang-service.yaml](https://github.com/sgl-project/sglang/blob/main/docker/k8s-sglang-service.yaml). - - - 1. Copy the [compose.yml](https://github.com/sgl-project/sglang/blob/main/docker/compose.yaml) to your local machine. - 2. Start the service: - - ```bash - docker compose up -d - ``` - - - - To deploy on Kubernetes or 12+ clouds, you can use [SkyPilot](https://github.com/skypilot-org/skypilot). - - 1. Install SkyPilot and set up Kubernetes cluster or cloud access. See [SkyPilot's documentation](https://skypilot.readthedocs.io/en/latest/getting-started/installation.html). - 2. Deploy on your own infra with a single command and get the HTTP API endpoint: - - **SkyPilot YAML: `sglang.yaml`** - - ```yaml Config - # sglang.yaml - envs: - HF_TOKEN: null - - resources: - image_id: docker:lmsysorg/sglang:latest - accelerators: A100 - ports: 30000 - - run: | - conda deactivate - python3 -m sglang.launch_server \ - --model-path meta-llama/Llama-3.1-8B-Instruct \ - --host 0.0.0.0 \ - --port 30000 - ``` - - ```bash - # Deploy on any cloud or Kubernetes cluster. Use --cloud to select a specific cloud provider. - HF_TOKEN= sky launch -c sglang --env HF_TOKEN sglang.yaml - - # Get the HTTP API endpoint - sky status --endpoint 30000 sglang - ``` - - 3. To scale with autoscaling and failure recovery, check out the [SkyServe + SGLang guide](https://github.com/skypilot-org/skypilot/tree/master/llm/sglang#serving-llama-2-with-sglang-for-more-traffic-using-skyserve). - - - - To deploy on SGLang on AWS SageMaker, check out [AWS SageMaker Inference](https://aws.amazon.com/sagemaker/ai/deploy). - - Amazon Web Services provide supports for SGLang containers along with routine security patching. For available SGLang containers, check out [AWS SGLang DLCs](https://github.com/aws/deep-learning-containers/blob/master/available_images.md#sglang-containers). - - To host a model with your own container, follow the following steps: - - 1. Build a docker container with [sagemaker.Dockerfile](https://github.com/sgl-project/sglang/blob/main/docker/sagemaker.Dockerfile) alongside the [serve](https://github.com/sgl-project/sglang/blob/main/docker/serve) script, then push it to AWS ECR. - - **Dockerfile build script: `build-and-push.sh`** - - ```bash - #!/bin/bash - AWS_ACCOUNT="" - AWS_REGION="" - REPOSITORY_NAME="" - IMAGE_TAG="" - - ECR_REGISTRY="${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com" - IMAGE_URI="${ECR_REGISTRY}/${REPOSITORY_NAME}:${IMAGE_TAG}" - - echo "Starting build and push process..." - - # Login to ECR - echo "Logging into ECR..." - aws ecr get-login-password --region ${AWS_REGION} | docker login --username AWS --password-stdin ${ECR_REGISTRY} - - # Build the image - echo "Building Docker image..." - docker build -t ${IMAGE_URI} -f sagemaker.Dockerfile . - - echo "Pushing ${IMAGE_URI}" - docker push ${IMAGE_URI} - - echo "Build and push completed successfully!" - ``` - - 2. Deploy a model for serving on AWS Sagemaker. Refer to [deploy_and_serve_endpoint.py](https://github.com/sgl-project/sglang/blob/main/examples/sagemaker/deploy_and_serve_endpoint.py). For more information, check out [sagemaker-python-sdk](https://github.com/aws/sagemaker-python-sdk). - - **Default command** - - The model server on SageMaker runs: `python3 -m sglang.launch_server --model-path opt/ml/model --host 0.0.0.0 --port 8080`. - - **Custom arguments** - - The [serve](https://github.com/sgl-project/sglang/blob/main/docker/serve) script exposes all options in `python3 -m sglang.launch_server --help` through environment variables prefixed with `SM_SGLANG_`. - - **Environment variable mapping** - - The serve script converts variables with prefix `SM_SGLANG_` from `SM_SGLANG_INPUT_ARGUMENT` into `--input-argument` for the `python3 -m sglang.launch_server` CLI. - - **Example** - - To run [Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B) with reasoning parser, add `SM_SGLANG_MODEL_PATH=Qwen/Qwen3-0.6B` and `SM_SGLANG_REASONING_PARSER=qwen3`. - - - -## Common notes - -- [FlashInfer](https://github.com/flashinfer-ai/flashinfer) is the default attention kernel backend. It only supports sm75 and above. If you encounter any FlashInfer-related issues on sm75+ devices (for example, T4, A10, A100, L4, L40S, H100), switch to other kernels by adding `--attention-backend triton --sampling-backend pytorch` and open an issue on GitHub. -- To reinstall flashinfer locally, use the following command: `pip3 install --upgrade flashinfer-python --force-reinstall --no-deps` and then delete the cache with `rm -rf ~/.cache/flashinfer`. -- When encountering `ptxas fatal : Value 'sm_103a' is not defined for option 'gpu-name'` on B300/GB300, fix it with `export TRITON_PTXAS_PATH=/usr/local/cuda/bin/ptxas`. diff --git a/docs_new/docs/get-started/quickstart.mdx b/docs_new/docs/get-started/quickstart.mdx index d33a1fb12..706e389b6 100644 --- a/docs_new/docs/get-started/quickstart.mdx +++ b/docs_new/docs/get-started/quickstart.mdx @@ -22,7 +22,7 @@ By the end, you'll have a working SGLang server responding to your prompts. - **OS**: Linux (recommended) -For other platforms, see the dedicated guides for [AMD GPUs](../hardware-platforms/amd-gpus), [Intel Xeon CPUs](../hardware-platforms/cpu-server), [Google TPUs](../hardware-platforms/tpu), [NVIDIA Jetson](../hardware-platforms/nvidia), [Ascend NPUs](../hardware-platforms/ascend-npus/SGLang-installation-with-NPUs-support), and [Intel XPU](../hardware-platforms/xpu). +For other platforms, see the dedicated guides for [AMD GPUs](../hardware-platforms/amd_gpu), [Intel Xeon CPUs](../hardware-platforms/cpu_server), [Google TPUs](../hardware-platforms/tpu), [NVIDIA Jetson](../hardware-platforms/nvidia_jetson), [Ascend NPUs](../hardware-platforms/ascend-npus/ascend_npu), and [Intel XPU](../hardware-platforms/xpu). --- @@ -310,22 +310,22 @@ WIP, TBD linked later ## What's Next? - + Explore the full Chat Completions and Completions APIs, including multi-turn conversations. - + Send image inputs alongside text using OpenAI-compatible vision APIs. - + Fine-tune generation with temperature, top-p, frequency penalty, and more. - + Customize server behavior with advanced launch arguments like tensor parallelism. - + Constrain model output to JSON, regex, or EBNF grammars. - + Use the familiar Ollama CLI and Python library with SGLang as the backend. diff --git a/docs_new/docs/hardware-platforms/amd-gpus.mdx b/docs_new/docs/hardware-platforms/amd-gpus.mdx deleted file mode 100644 index 823f783dd..000000000 --- a/docs_new/docs/hardware-platforms/amd-gpus.mdx +++ /dev/null @@ -1,194 +0,0 @@ ---- -title: "AMD GPUs" ---- -This document describes how run SGLang on AMD GPUs. If you encounter issues or have questions, please [open an issue](https://github.com/sgl-project/sglang/issues). - -## System Configuration - -When using AMD GPUs (such as MI300X), certain system-level optimizations help ensure stable performance. Here we take MI300X as an example. AMD provides official documentation for MI300X optimization and system tuning: - -* [AMD MI300X Tuning Guides](https://rocm.docs.amd.com/en/latest/how-to/tuning-guides/mi300x/index.html) -* [LLM inference performance validation on AMD Instinct MI300X](https://rocm.docs.amd.com/en/latest/how-to/rocm-for-ai/inference/vllm-benchmark.html) -* [AMD Instinct MI300X System Optimization](https://rocm.docs.amd.com/en/latest/how-to/system-optimization/mi300x.html) -* [AMD Instinct MI300X Workload Optimization](https://rocm.docs.amd.com/en/latest/how-to/rocm-for-ai/inference-optimization/workload.html) -* [Supercharge DeepSeek-R1 Inference on AMD Instinct MI300X](https://rocm.blogs.amd.com/artificial-intelligence/DeepSeekR1-Part2/README.html) - - -We strongly recommend reading these docs and guides entirely to fully utilize your system. - - -Below are a few key settings to confirm or enable for SGLang: - -### Update GRUB Settings - -In `/etc/default/grub`, append the following to `GRUB_CMDLINE_LINUX`: - - -```text GRUB Configuration -pci=realloc=off iommu=pt -``` - - -Afterward, run `sudo update-grub` (or your distro's equivalent) and reboot. - -### Disable NUMA Auto-Balancing - - -```bash Disable NUMA -sudo sh -c 'echo 0 > /proc/sys/kernel/numa_balancing' -``` - - -You can automate or verify this change using [this helpful script](https://github.com/ROCm/triton/blob/rocm_env/scripts/amd/env_check.sh). - -Again, please go through the entire documentation to confirm your system is using the recommended configuration. - -## Install SGLang - - - - The docker images are available on Docker Hub at [lmsysorg/sglang](https://hub.docker.com/r/lmsysorg/sglang/tags), built from [rocm.Dockerfile](https://github.com/sgl-project/sglang/tree/main/docker). - - 1. **Build the docker image** - If you use pre-built images, you can skip this step and replace `sglang_image` with the pre-built image names in the steps below. - - - ```bash Build Image - docker build -t sglang_image -f rocm.Dockerfile . - ``` - - - 2. **Create a convenient alias** - - ```bash Create Alias - alias drun='docker run -it --rm --network=host --privileged --device=/dev/kfd --device=/dev/dri \ - --ipc=host --shm-size 16G --group-add video --cap-add=SYS_PTRACE \ - --security-opt seccomp=unconfined \ - -v $HOME/dockerx:/dockerx \ - -v /data:/data' - ``` - - - If you are using RDMA, please note that: - - * `--network host` and `--privileged` are required by RDMA. If you don't need RDMA, you can remove them. - * You may need to set `NCCL_IB_GID_INDEX` if you are using RoCE, for example: `export NCCL_IB_GID_INDEX=3`. - - 3. **Launch the server** - - Replace `` below with your [huggingface hub token](https://huggingface.co/docs/hub/en/security-tokens). - - - - ```bash Launch Server - drun -p 30000:30000 \ - -v ~/.cache/huggingface:/root/.cache/huggingface \ - --env "HF_TOKEN=" \ - sglang_image \ - python3 -m sglang.launch_server \ - --model-path NousResearch/Meta-Llama-3.1-8B \ - --host 0.0.0.0 \ - --port 30000 - ``` - - - 4. **Verify the installation** - You can run a benchmark in another terminal or refer to [other docs](../basic_usage/openai_api_completions) to send requests to the engine. - - - ```bash Run Benchmark - drun sglang_image \ - python3 -m sglang.bench_serving \ - --backend sglang \ - --dataset-name random \ - --num-prompts 4000 \ - --random-input 128 \ - --random-output 128 - ``` - - - With your AMD system properly configured and SGLang installed, you can now fully leverage AMD hardware to power SGLang's machine learning capabilities. - - - - 1. **Clone the repository** - Clone the SGLang repository. - - - ```bash - git clone https://github.com/sgl-project/sglang.git - cd sglang - ``` - - - 2. **Compile sgl-kernel** - Upgrade pip and compile the sgl-kernel for ROCm support. - - - ```bash - pip install --upgrade pip - cd sgl-kernel - python setup_rocm.py install - ``` - - - 3. **Install sglang package** - Install the SGLang Python package with HIP and diffusion support. - - - ```bash - cd .. - rm -rf python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml - pip install -e "python[all_hip]" - ``` - - - - -## Examples - -### Running DeepSeek-V3 - -The only difference when running DeepSeek-V3 is in how you start the server. - - -```bash DeepSeek-V3 -drun -p 30000:30000 \ - -v ~/.cache/huggingface:/root/.cache/huggingface \ - --ipc=host \ - --env "HF_TOKEN=" \ - sglang_image \ - python3 -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3 \ - --tp 8 \ - --trust-remote-code \ - --host 0.0.0.0 \ - --port 30000 -``` - - -[Running DeepSeek-R1 on a single NDv5 MI300X VM](https://techcommunity.microsoft.com/blog/azurehighperformancecomputingblog/running-deepseek-r1-on-a-single-ndv5-mi300x-vm/4372726) could also be a good reference. - -### Running Llama3.1 - -Running Llama3.1 is nearly identical to running DeepSeek-V3. The only difference is in the model specified when starting the server. - - -```bash Llama3.1 -drun -p 30000:30000 \ - -v ~/.cache/huggingface:/root/.cache/huggingface \ - --ipc=host \ - --env "HF_TOKEN=" \ - sglang_image \ - python3 -m sglang.launch_server \ - --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ - --tp 8 \ - --trust-remote-code \ - --host 0.0.0.0 \ - --port 30000 -``` - - -### Warmup Step - -When the server displays `The server is fired up and ready to roll!`, it means the startup is successful. diff --git a/docs_new/docs/hardware-platforms/amd_gpu.mdx b/docs_new/docs/hardware-platforms/amd_gpu.mdx new file mode 100644 index 000000000..4bc551d86 --- /dev/null +++ b/docs_new/docs/hardware-platforms/amd_gpu.mdx @@ -0,0 +1,196 @@ +--- +title: "AMD GPUs" +--- +This document describes how to run SGLang on AMD GPUs. If you encounter issues or have questions, please [open an issue](https://github.com/sgl-project/sglang/issues). + +## System Configuration + +When using AMD GPUs (such as MI300X), certain system-level optimizations help ensure stable performance. Here we take MI300X as an example. AMD provides official documentation for MI300X optimization and system tuning: + +- [AMD MI300X Tuning Guides](https://rocm.docs.amd.com/en/latest/how-to/tuning-guides/mi300x/index.html) +- [LLM inference performance validation on AMD Instinct MI300X](https://rocm.docs.amd.com/en/latest/how-to/rocm-for-ai/inference/vllm-benchmark.html) +- [AMD Instinct MI300X System Optimization](https://rocm.docs.amd.com/en/latest/how-to/system-optimization/mi300x.html) +- [AMD Instinct MI300X Workload Optimization](https://rocm.docs.amd.com/en/latest/how-to/rocm-for-ai/inference-optimization/workload.html) +- [Supercharge DeepSeek-R1 Inference on AMD Instinct MI300X](https://rocm.blogs.amd.com/artificial-intelligence/DeepSeekR1-Part2/README.html) + +**NOTE:** We strongly recommend reading these docs and guides entirely to fully utilize your system. + +Below are a few key settings to confirm or enable for SGLang: + +### Update GRUB Settings + +In `/etc/default/grub`, append the following to `GRUB_CMDLINE_LINUX`: + +```text GRUB Configuration +pci=realloc=off iommu=pt +``` + +Afterward, run `sudo update-grub` (or your distro’s equivalent) and reboot. + +### Disable NUMA Auto-Balancing + +```bash Disable NUMA +sudo sh -c 'echo 0 > /proc/sys/kernel/numa_balancing' +``` + +You can automate or verify this change using [this helpful script](https://github.com/ROCm/triton/blob/rocm_env/scripts/amd/env_check.sh). + +Again, please go through the entire documentation to confirm your system is using the recommended configuration. + +## Install SGLang + +You can install SGLang using one of the methods below. + +### Install from Source + +```bash Command +# Use the last release branch +git clone -b v0.5.9 https://github.com/sgl-project/sglang.git +cd sglang + +# Compile sgl-kernel +pip install --upgrade pip +cd sgl-kernel +python setup_rocm.py install + +# Install sglang python package along with diffusion support +cd .. +rm -rf python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml +pip install -e "python[all_hip]" +``` + +### Install Using Docker (Recommended) + +The docker images are available on Docker Hub at [lmsysorg/sglang](https://hub.docker.com/r/lmsysorg/sglang/tags), built from [rocm.Dockerfile](https://github.com/sgl-project/sglang/tree/main/docker). + +The steps below show how to build and use an image. + +1. Build the docker image. + If you use pre-built images, you can skip this step and replace `sglang_image` with the pre-built image names in the steps below. + + ```bash Command + docker build -t sglang_image -f rocm.Dockerfile . + ``` + +2. Create a convenient alias. + + ```bash Command + alias drun='docker run -it --rm --network=host --privileged --device=/dev/kfd --device=/dev/dri \ + --ipc=host --shm-size 16G --group-add video --cap-add=SYS_PTRACE \ + --security-opt seccomp=unconfined \ + -v $HOME/dockerx:/dockerx \ + -v /data:/data' + ``` + + If you are using RDMA, please note that: + - `--network host` and `--privileged` are required by RDMA. If you don't need RDMA, you can remove them. + - You may need to set `NCCL_IB_GID_INDEX` if you are using RoCE, for example: `export NCCL_IB_GID_INDEX=3`. + +3. Launch the server. + + **NOTE:** Replace `` below with your [huggingface hub token](https://huggingface.co/docs/hub/en/security-tokens). + + ```bash Command + drun -p 30000:30000 \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --env "HF_TOKEN=" \ + sglang_image \ + python3 -m sglang.launch_server \ + --model-path NousResearch/Meta-Llama-3.1-8B \ + --host 0.0.0.0 \ + --port 30000 + ``` + +4. To verify the utility, you can run a benchmark in another terminal or refer to [other docs](../basic_usage/openai_api_completions) to send requests to the engine. + + ```bash Command + drun sglang_image \ + python3 -m sglang.bench_serving \ + --backend sglang \ + --dataset-name random \ + --num-prompts 4000 \ + --random-input 128 \ + --random-output 128 + ``` + +With your AMD system properly configured and SGLang installed, you can now fully leverage AMD hardware to power SGLang’s machine learning capabilities. + +## Quantization on AMD GPUs + +The [Quantization documentation](../advanced_features/quantization#platform-compatibility) has a full compatibility matrix. The short version: FP8, AWQ, MXFP4, W8A8, GPTQ, compressed-tensors, Quark, and **petit_nvfp4** (NVFP4 on ROCm via [Petit](https://github.com/causalflow-ai/petit-kernel)) all work on AMD. Methods that depend on Marlin or NVIDIA-specific kernels (`awq_marlin`, `gptq_marlin`, `gguf`, `modelopt_fp8`, `modelopt_fp4`) do not. + +A few things to keep in mind: + +- FP8 works via Aiter or Triton. Pre-quantized FP8 models like DeepSeek-V3/R1 work out of the box. +- AWQ uses Triton dequantization kernels on AMD. The faster Marlin path is not available. +- MXFP4 requires CDNA3/CDNA4 and `SGLANG_USE_AITER=1`. +- `petit_nvfp4` enables NVFP4 models (e.g., [Llama 3.3 70B FP4](https://huggingface.co/nvidia/Llama-3.3-70B-Instruct-FP4)) on MI250/MI300X via [Petit](https://github.com/causalflow-ai/petit-kernel). Install with `pip install petit-kernel`; no `--quantization` flag needed when loading pre-quantized NVFP4 models. +- `quark_int4fp8_moe` is an AMD-only online quantization method for MoE models on CDNA3/CDNA4. + +Several of these backends are accelerated by [Aiter](https://github.com/ROCm/aiter). Enable it with: + +```bash Command +export SGLANG_USE_AITER=1 +``` + +Example -- serving an AWQ model: + +```bash Command +python3 -m sglang.launch_server \ + --model-path hugging-quants/Mixtral-8x7B-Instruct-v0.1-AWQ-INT4 \ + --trust-remote-code \ + --port 30000 --host 0.0.0.0 +``` + +Example -- FP8 online quantization: + +```bash Command +python3 -m sglang.launch_server \ + --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ + --quantization fp8 \ + --port 30000 --host 0.0.0.0 +``` + +## Examples + +### Running DeepSeek-V3 + +The only difference when running DeepSeek-V3 is in how you start the server. Here's an example command: + +```bash Command +drun -p 30000:30000 \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --ipc=host \ + --env "HF_TOKEN=" \ + sglang_image \ + python3 -m sglang.launch_server \ + --model-path deepseek-ai/DeepSeek-V3 \ # <- here + --tp 8 \ + --trust-remote-code \ + --host 0.0.0.0 \ + --port 30000 +``` + +[Running DeepSeek-R1 on a single NDv5 MI300X VM](https://techcommunity.microsoft.com/blog/azurehighperformancecomputingblog/running-deepseek-r1-on-a-single-ndv5-mi300x-vm/4372726) could also be a good reference. + +### Running Llama3.1 + +Running Llama3.1 is nearly identical to running DeepSeek-V3. The only difference is in the model specified when starting the server, shown by the following example command: + +```bash Command +drun -p 30000:30000 \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --ipc=host \ + --env "HF_TOKEN=" \ + sglang_image \ + python3 -m sglang.launch_server \ + --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ # <- here + --tp 8 \ + --trust-remote-code \ + --host 0.0.0.0 \ + --port 30000 +``` + +### Warmup Step + +When the server displays `The server is fired up and ready to roll!`, it means the startup is successful. diff --git a/docs_new/docs/hardware-platforms/apple_metal.mdx b/docs_new/docs/hardware-platforms/apple_metal.mdx new file mode 100644 index 000000000..d142caf9b --- /dev/null +++ b/docs_new/docs/hardware-platforms/apple_metal.mdx @@ -0,0 +1,24 @@ +--- +title: "Apple Silicon with Metal" +metatags: + description: "Run SGLang on Apple Silicon using the Metal backend." +--- + +This document describes how run SGLang on Apple Silicon using [Metal](https://developer.apple.com/metal/). If you encounter issues or have questions, please [open an issue](https://github.com/sgl-project/sglang/issues). + +## Install SGLang + +You can install SGLang using one of the methods below. + +### Install from Source + +```bash +# Use the default branch +git clone https://github.com/sgl-project/sglang.git +cd sglang + +# Install sglang python package +pip install --upgrade pip +rm -f python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml +uv pip install -e "python[all_mps]" +``` diff --git a/docs_new/docs/hardware-platforms/ascend-npus/DeepSeek-Examples.mdx b/docs_new/docs/hardware-platforms/ascend-npus/DeepSeek-Examples.mdx deleted file mode 100644 index 243a3897b..000000000 --- a/docs_new/docs/hardware-platforms/ascend-npus/DeepSeek-Examples.mdx +++ /dev/null @@ -1,309 +0,0 @@ -## Running DeepSeek-V3 - -### Running DeepSeek in PD mixed mode on 1 x Atlas 800I A3 - -W4A8 Model weights could be found [here](https://modelers.cn/models/Modelers_Park/DeepSeek-R1-0528-w4a8). - - -```shell Launch Server -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 - -#Deepep communication settings -export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 -export HCCL_BUFFSIZE=1600 - -#spec overlap -export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - -#npu acceleration operator -export SGLANG_NPU_USE_MLAPO=1 -export SGLANG_USE_FIA_NZ=1 - -python3 -m sglang.launch_server \ - --model-path ${MODEL_PATH} \ - --tp 16 \ - --trust-remote-code \ - --attention-backend ascend \ - --device npu \ - --quantization modelslim \ - --watchdog-timeout 9000 \ - --cuda-graph-bs 8 16 24 28 32 \ - --mem-fraction-static 0.68 \ - --max-running-requests 128 \ - --context-length 8188 \ - --disable-radix-cache \ - --chunked-prefill-size -1 \ - --max-prefill-tokens 16384 \ - --moe-a2a-backend deepep \ - --deepep-mode auto \ - --enable-dp-attention \ - --dp-size 4 \ - --enable-dp-lm-head \ - --speculative-algorithm NEXTN \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --dtype bfloat16 -``` - - -### Running DeepSeek with PD disaggregation mode on 2 x Atlas 800I A3 - -W4A8 Model weights could be found [here](https://modelers.cn/models/Modelers_Park/DeepSeek-R1-0528-w4a8). - - - - ```shell Command - export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - export STREAMS_PER_DEVICE=32 - - #memfabric config store - export ASCEND_MF_STORE_URL="tcp://:" - - #Deepep communication settings - export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 - export HCCL_BUFFSIZE=1536 - - #npu acceleration operator - export SGLANG_NPU_USE_MLAPO=1 - export SGLANG_USE_FIA_NZ=1 - export TASK_QUEUE_ENABLE=2 - - python -m sglang.launch_server \ - --model-path ${MODEL_PATH} \ - --host $PREFILL_HOST_IP \ - --port 8000 \ - --disaggregation-mode prefill \ - --disaggregation-bootstrap-port 8996 \ - --disaggregation-transfer-backend ascend \ - --trust-remote-code \ - --nnodes 1 \ - --node-rank 0 \ - --tp-size 16 \ - --mem-fraction-static 0.6 \ - --attention-backend ascend \ - --device npu \ - --quantization modelslim \ - --load-balance-method round_robin \ - --max-running-requests 8 \ - --context-length 8192 \ - --disable-radix-cache \ - --chunked-prefill-size -1 \ - --max-prefill-tokens 28680 \ - --moe-a2a-backend deepep \ - --deepep-mode normal \ - --speculative-algorithm NEXTN \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --dp-size 2 \ - --enable-dp-attention \ - --disable-shared-experts-fusion \ - --dtype bfloat16 - ``` - - - - - ```shell Command - export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - export STREAMS_PER_DEVICE=32 - - #memfabric config store - export ASCEND_MF_STORE_URL="tcp://:" - - #Deepep communication settings - export HCCL_BUFFSIZE=720 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=88 - - #spec overlap - export SGLANG_ENABLE_SPEC_V2=1 - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - - #npu acceleration operator - unset TASK_QUEUE_ENABLE - export SGLANG_NPU_USE_MLAPO=1 - export SGLANG_USE_FIA_NZ=1 - export ENABLE_MOE_NZ=1 - - # suggest max-running-requests <= max-cuda-graph-bs * dp_size, Because when this value is exceeded, performance will significantly degrade. - python -m sglang.launch_server \ - --model-path ${MODEL_PATH} \ - --disaggregation-mode decode \ - --host $DECODE_HOST_IP \ - --port 8001 \ - --trust-remote-code \ - --nnodes 1 \ - --node-rank 0 \ - --tp-size 16 \ - --dp-size 16 \ - --mem-fraction-static 0.8 \ - --max-running-requests 352 \ - --attention-backend ascend \ - --device npu \ - --quantization modelslim \ - --prefill-round-robin-balance \ - --moe-a2a-backend deepep \ - --enable-dp-attention \ - --deepep-mode low_latency \ - --enable-dp-lm-head \ - --cuda-graph-bs 8 10 12 14 16 18 20 22 \ - --disaggregation-transfer-backend ascend \ - --watchdog-timeout 9000 \ - --context-length 8192 \ - --speculative-algorithm NEXTN \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --disable-shared-experts-fusion \ - --dtype bfloat16 \ - --tokenizer-worker-num 4 - ``` - - - - - ```shell Command - python -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy cache_aware \ - --prefill http://:8000 8996 \ - --decode http://:8001 \ - --host 127.0.0.1 \ - --port 6688 - ``` - - - -### Running DeepSeek with PD disaggregation on 4 x Atlas 800I A3 - -W8A8 Model weights could be found [here](https://modelers.cn/models/State_Cloud/Deepseek-R1-bf16-hfd-w8a8). - - - - ```shell Command - echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor - sysctl -w vm.swappiness=0 - sysctl -w kernel.numa_balancing=0 - sysctl -w kernel.sched_migration_cost_ns=50000 - export SGLANG_SET_CPU_AFFINITY=1 - unset ASCEND_LAUNCH_BLOCKING - source /usr/local/Ascend/ascend-toolkit/set_env.sh - source /usr/local/Ascend/nnal/atb/set_env.sh - export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - - export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - export STREAMS_PER_DEVICE=32 - - export ASCEND_MF_STORE_URL="tcp://your prefill ip1:24669" - - P_IP=('your prefill ip1' 'your prefill ip2') - - D_IP=('your decode ip1' 'your decode ip2') - - MODEL_PATH=xxx - - export SGLANG_NPU_USE_MLAPO=1 - export SGLANG_USE_FIA_NZ=1 - - LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` - LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - echo "${LOCAL_HOST1}" - echo "${LOCAL_HOST2}" - # prefill - for i in "${!P_IP[@]}"; - do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - export HCCL_BUFFSIZE=1536 - export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 - export TASK_QUEUE_ENABLE=2 - - export HCCL_SOCKET_IFNAME=lo - export GLOO_SOCKET_IFNAME=lo - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \ - --port 8000 --disaggregation-bootstrap-port $((8998+$i)) --trust-remote-code --nnodes 1 --node-rank 0 \ - --tp-size 16 --mem-fraction-static 0.81 --attention-backend ascend --device npu --quantization modelslim \ - --disaggregation-transfer-backend ascend --max-running-requests 8 --context-length 8192 --disable-radix-cache \ - --chunked-prefill-size -1 --max-prefill-tokens 28680 --moe-a2a-backend deepep --deepep-mode normal \ - --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ - --dp-size 2 --enable-dp-attention --disable-shared-experts-fusion --dtype bfloat16 --enable-attn-tp-input-scattered - NODE_RANK=$i - break - fi - done - - # decode - for i in "${!D_IP[@]}"; - do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_ENABLE_SPEC_V2=1 - export HCCL_BUFFSIZE=650 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=78 - export TASK_QUEUE_ENABLE=1 - export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 - export HCCL_SOCKET_IFNAME=xxx - export GLOO_SOCKET_IFNAME=xxx - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ - --port 8001 --trust-remote-code --dist-init-addr ${D_IP[0]}:5000 --nnodes 2 --node-rank $i --tp-size 32 --dp-size 32 \ - --mem-fraction-static 0.815 --max-running-requests 832 --attention-backend ascend --device npu --quantization modelslim \ - --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head --moe-dense-tp 1 \ - --cuda-graph-bs 12 14 16 18 20 22 24 26 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ - --speculative-algorithm NEXTN --speculative-num-steps 2 --speculative-eagle-topk 1 --speculative-num-draft-tokens 3 \ - --tokenizer-worker-num 4 --prefill-round-robin-balance --disable-shared-experts-fusion --dtype bfloat16 \ - --load-balance-method decode_round_robin - NODE_RANK=$i - break - fi - done - ``` - - - - - ```shell Command - export SGLANG_DP_ROUND_ROBIN=1 - python -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy cache_aware \ - --prefill http://P_IP:8000 8998 \ - --prefill http://P_IP:8000 8999 \ - --decode http://D_IP:8001 \ - --host 127.0.0.1 \ - --port 6688 \ - --mini-lb - ``` - - - -### Test GSM8K - - -```python Test GSM8K -from types import SimpleNamespace -from sglang.test.few_shot_gsm8k import run_eval - -def gsm8k(): - args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=32, - host=f"http://127.0.0.1", - port=6688, - ) - metrics = run_eval(args) - print(f"{metrics=}") - print(f"{metrics['accuracy']=}") -if __name__ == "__main__": - gsm8k() -``` - diff --git a/docs_new/docs/hardware-platforms/ascend-npus/Qwen3.5.mdx b/docs_new/docs/hardware-platforms/ascend-npus/Qwen3.5.mdx deleted file mode 100644 index baca8dd9f..000000000 --- a/docs_new/docs/hardware-platforms/ascend-npus/Qwen3.5.mdx +++ /dev/null @@ -1,106 +0,0 @@ -## Environment Preparation - -### Installation - -The dependencies required for the NPU runtime environment have been integrated into a Docker image and uploaded to the quay.io platform. You can directly pull it. - - -```bash Pull and Start Container -#Atlas 800 A3 -docker pull swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:cann8.5.0-a3-qwen3.5 -#Atlas 800 A2 -docker pull swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:cann8.5.0-910b-qwen3.5 - -#start container -docker run -itd --shm-size=16g --privileged=true --name ${NAME} \ ---privileged=true --net=host \ --v /var/queue_schedule:/var/queue_schedule \ --v /etc/ascend_install.info:/etc/ascend_install.info \ --v /usr/local/sbin:/usr/local/sbin \ --v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ --v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ ---device=/dev/davinci0:/dev/davinci0 \ ---device=/dev/davinci1:/dev/davinci1 \ ---device=/dev/davinci2:/dev/davinci2 \ ---device=/dev/davinci3:/dev/davinci3 \ ---device=/dev/davinci4:/dev/davinci4 \ ---device=/dev/davinci5:/dev/davinci5 \ ---device=/dev/davinci6:/dev/davinci6 \ ---device=/dev/davinci7:/dev/davinci7 \ ---device=/dev/davinci8:/dev/davinci8 \ ---device=/dev/davinci9:/dev/davinci9 \ ---device=/dev/davinci10:/dev/davinci10 \ ---device=/dev/davinci11:/dev/davinci11 \ ---device=/dev/davinci12:/dev/davinci12 \ ---device=/dev/davinci13:/dev/davinci13 \ ---device=/dev/davinci14:/dev/davinci14 \ ---device=/dev/davinci15:/dev/davinci15 \ ---device=/dev/davinci_manager:/dev/davinci_manager \ ---device=/dev/hisi_hdc:/dev/hisi_hdc \ ---entrypoint=bash \ -swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:${TAG} -``` - - -## Deployment - -### Single-node Deployment - -- Quantized model `qwen35_w8a8` can be deployed on 1 Atlas 800 A3 (64G × 16) . - -Run the following script to execute online inference. - - -```shell Launch Server -# high performance cpu -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -# bind cpu -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -# cann -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -export STREAMS_PER_DEVICE=32 -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_NPU_USE_MULTI_STREAM=1 -export HCCL_BUFFSIZE=1000 -export HCCL_OP_EXPANSION_MODE=AIV -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo - -python3 -m sglang.launch_server \ - --model-path $MODEL_PATH \ - --attention-backend ascend \ - --device npu \ - --tp-size 16 --nnodes 1 --node-rank 0 \ - --chunked-prefill-size 16384 --max-prefill-tokens 280000 \ - --trust-remote-code \ - --host 127.0.0.1 \ - --mem-fraction-static 0.7 \ - --port 8000 \ - --cuda-graph-bs 16 \ - --quantization modelslim \ - --enable-multimodal \ - --mm-attention-backend ascend_attn \ - --dtype bfloat16 -``` - - -### Prefill-Decode Disaggregation - -Not test yet. - -### Using Benchmark - -Refer to [Benchmark and Profiling](../../developer_guide/benchmark_and_profiling) for details. diff --git a/docs_new/docs/hardware-platforms/ascend-npus/SGLang-installation-with-NPUs-support.mdx b/docs_new/docs/hardware-platforms/ascend-npus/SGLang-installation-with-NPUs-support.mdx deleted file mode 100644 index 23323cbbf..000000000 --- a/docs_new/docs/hardware-platforms/ascend-npus/SGLang-installation-with-NPUs-support.mdx +++ /dev/null @@ -1,318 +0,0 @@ ---- -title: SGLang installation with NPUs support ---- - -You can install SGLang using any of the methods below. Please go through `System Settings` section to ensure the clusters are roaring at max performance. Feel free to leave an issue [here at sglang](https://github.com/sgl-project/sglang/issues) if you encounter any issues or have any problems. - -## Component Version Mapping For SGLang - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ComponentVersionObtain Way
HDK25.3.RC1[](https://hiascend.com/hardware/firmware-drivers/commercial?product=7\&model=33)
CANN8.5.0[Obtain Images](#obtain-cann-image)
Pytorch Adapter7.3.0[](https://gitcode.com/Ascend/pytorch/releases)
MemFabric1.0.5`pip install memfabric-hybrid==1.0.5`
Triton3.2.0`pip install triton-ascend`
Bisheng20251121[](https://sglang-ascend.obs.cn-east-3.myhuaweicloud.com/sglang/triton_ascend/Ascend-BiSheng-toolkit_aarch64_20251121.run)
SGLang NPU KernelNA[](https://github.com/sgl-project/sgl-kernel-npu/releases)
- - - - You can obtain the dependency of a specified version of CANN through an image. - - ```bash - # for Atlas 800I A3 and Ubuntu OS - docker pull quay.io/ascend/cann:8.5.0-a3-ubuntu22.04-py3.11 - # for Atlas 800I A2 and Ubuntu OS - docker pull quay.io/ascend/cann:8.5.0-910b-ubuntu22.04-py3.11 - ``` - - -## Preparing the Running Environment - - - - - - Only `python==3.11` is supported currently. If you don't want to break system pre-installed python, try installing with [conda](https://github.com/conda/conda). - - ```bash - conda create --name sglang_npu python=3.11 - conda activate sglang_npu - ``` - - - Prior to start work with SGLang on Ascend you need to install CANN Toolkit, Kernels operator package and NNAL version 8.3.RC2 or higher, check the [installation guide](https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/83RC1/softwareinst/instg/instg_0008.html?Mode=PmIns\&InstallType=local\&OS=openEuler\&Software=cannToolKit) - - - If you want to use PD disaggregation mode, you need to install MemFabric-Hybrid. MemFabric-Hybrid is a drop-in replacement of Mooncake Transfer Engine that enables KV cache transfer on Ascend NPU clusters. - - ```bash - pip install memfabric-hybrid==1.0.5 - ``` - - - ```bash - PYTORCH_VERSION=2.8.0 - TORCHVISION_VERSION=0.23.0 - TORCH_NPU_VERSION=2.8.0 - pip install torch==$PYTORCH_VERSION torchvision==$TORCHVISION_VERSION --index-url https://download.pytorch.org/whl/cpu - pip install torch_npu==$TORCH_NPU_VERSION - ``` - - If you are using other versions of `torch` and install `torch_npu`, check [installation guide](https://github.com/Ascend/pytorch/blob/master/README) - - - We provide our own implementation of Triton for Ascend. - - ```bash - BISHENG_NAME="Ascend-BiSheng-toolkit_aarch64_20251121.run" - BISHENG_URL="https://sglang-ascend.obs.cn-east-3.myhuaweicloud.com/sglang/triton_ascend/${BISHENG_NAME}" - wget -O "${BISHENG_NAME}" "${BISHENG_URL}" && chmod a+x "${BISHENG_NAME}" && "./${BISHENG_NAME}" --install && rm "${BISHENG_NAME}" - ``` - - ```bash - pip install triton-ascend - ``` - - For installation of Triton on Ascend nightly builds or from sources, follow [installation guide](https://gitcode.com/Ascend/triton-ascend/blob/master/docs/sources/getting-started/installation) - - - - We provide SGL kernels for Ascend NPU, check [installation guide](https://github.com/sgl-project/sgl-kernel-npu/blob/main/python/sgl_kernel_npu/README). - - - We provide a DeepEP-compatible Library as a drop-in replacement of deepseek-ai's DeepEP library, check the [installation guide](https://github.com/sgl-project/sgl-kernel-npu/blob/main/python/deep_ep/README). - - - ```bash - # Use the last release branch - git clone https://github.com/sgl-project/sglang.git - cd sglang - mv python/pyproject_npu.toml python/pyproject.toml - pip install -e python[all_npu] - ``` - - - - - - - ### Obtain Image - - You can download the SGLang image or build an image based on Dockerfile to obtain the Ascend NPU image. - - 1. **Download SGLang image** - - ```bash - dockerhub: docker.io/lmsysorg/sglang:$tag - # Main-based tag, change main to specific version like v0.5.6, - # you can get image for specific version - Atlas 800I A3 : {main}-cann8.5.0-a3 - Atlas 800I A2: {main}-cann8.5.0-910b - ``` - - 2. **Build an image based on Dockerfile** - - ```bash - # Clone the SGLang repository - git clone https://github.com/sgl-project/sglang.git - cd sglang/docker - - # Build the docker image - # If there are network errors, please modify the Dockerfile to use offline dependencies or use a proxy - docker build -t -f npu.Dockerfile . - ``` - - ### Create Docker - - `--privileged` and `--network=host` are required by RDMA, which is typically needed by Ascend NPU clusters. - The following docker command is based on Atlas 800I A3 machines. If you are using Atlas 800I A2, make sure only `davinci[0-7]` are mapped into container. - - ```bash - alias drun='docker run -it --rm --privileged --network=host --ipc=host --shm-size=16g \ - --device=/dev/davinci0 --device=/dev/davinci1 --device=/dev/davinci2 --device=/dev/davinci3 \ - --device=/dev/davinci4 --device=/dev/davinci5 --device=/dev/davinci6 --device=/dev/davinci7 \ - --device=/dev/davinci8 --device=/dev/davinci9 --device=/dev/davinci10 --device=/dev/davinci11 \ - --device=/dev/davinci12 --device=/dev/davinci13 --device=/dev/davinci14 --device=/dev/davinci15 \ - --device=/dev/davinci_manager --device=/dev/hisi_hdc \ - --volume /usr/local/sbin:/usr/local/sbin --volume /usr/local/Ascend/driver:/usr/local/Ascend/driver \ - --volume /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ - --volume /etc/ascend_install.info:/etc/ascend_install.info \ - --volume /var/queue_schedule:/var/queue_schedule --volume ~/.cache/:/root/.cache/' - - # Add HF_TOKEN env for download model by SGLang. - drun --env "HF_TOKEN=" \ - \ - python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --attention-backend ascend - ``` - - - - -## System Settings - - - - The default power scheme on Ascend hardware is `ondemand` which could affect performance, changing it to `performance` is recommended. - - ```bash - echo performance | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor - - # Make sure changes are applied successfully - cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor # shows performance - ``` - - - ```bash - sudo sysctl -w kernel.numa_balancing=0 - # Check - cat /proc/sys/kernel/numa_balancing # shows 0 - ``` - - - ```bash - sudo sysctl -w vm.swappiness=10 - - # Check - cat /proc/sys/vm/swappiness # shows 10 - ``` - - - -## Running SGLang Service - - - - - ### PD Mixed Scene - - ```bash - # Enabling CPU Affinity - export SGLANG_SET_CPU_AFFINITY=1 - python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --attention-backend ascend - ``` - - ### PD Separation Scene - - 1. **Launch Prefill Server** - - ```bash - # Enabling CPU Affinity - export SGLANG_SET_CPU_AFFINITY=1 - - # PIP: recommended to config first Prefill Server IP - # PORT: one free port - # all sglang servers need to be config the same PIP and PORT, - export ASCEND_MF_STORE_URL="tcp://PIP:PORT" - # if you are Atlas 800I A2 hardware and use rdma for kv cache transfer, add this parameter - export ASCEND_MF_TRANSFER_PROTOCOL="device_rdma" - python3 -m sglang.launch_server \ - --model-path meta-llama/Llama-3.1-8B-Instruct \ - --disaggregation-mode prefill \ - --disaggregation-transfer-backend ascend \ - --disaggregation-bootstrap-port 8995 \ - --attention-backend ascend \ - --device npu \ - --base-gpu-id 0 \ - --tp-size 1 \ - --host 127.0.0.1 \ - --port 8000 - ``` - - 2. **Launch Decode Server** - - ```bash - # PIP: recommended to config first Prefill Server IP - # PORT: one free port - # all sglang servers need to be config the same PIP and PORT, - export ASCEND_MF_STORE_URL="tcp://PIP:PORT" - # if you are Atlas 800I A2 hardware and use rdma for kv cache transfer, add this parameter - export ASCEND_MF_TRANSFER_PROTOCOL="device_rdma" - python3 -m sglang.launch_server \ - --model-path meta-llama/Llama-3.1-8B-Instruct \ - --disaggregation-mode decode \ - --disaggregation-transfer-backend ascend \ - --attention-backend ascend \ - --device npu \ - --base-gpu-id 1 \ - --tp-size 1 \ - --host 127.0.0.1 \ - --port 8001 - ``` - - 3. **Launch Router** - - ```bash - python3 -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy cache_aware \ - --prefill http://127.0.0.1:8000 8995 \ - --decode http://127.0.0.1:8001 \ - --host 127.0.0.1 \ - --port 6688 - ``` - - - - ### PD Mixed Scene - - ```bash - python3 -m sglang.launch_server \ - --model-path Qwen3-VL-30B-A3B-Instruct \ - --host 127.0.0.1 \ - --port 8000 \ - --tp 4 \ - --device npu \ - --attention-backend ascend \ - --mm-attention-backend ascend_attn \ - --disable-radix-cache \ - --trust-remote-code \ - --enable-multimodal \ - --sampling-backend ascend - ``` - - - diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_contribution_guide.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_contribution_guide.mdx new file mode 100644 index 000000000..3005aa246 --- /dev/null +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_contribution_guide.mdx @@ -0,0 +1,167 @@ +--- +title: "Contribution Guide" +metatags: + description: "Set up the Ascend NPU development environment, run tests, build documentation, and open SGLang pull requests." +--- + +Welcome to **SGLang**! We appreciate your interest in contributing. This guide provides a concise overview of how to set up your environment, run tests, build documentation, and open a Pull Request (PR). Whether you’re fixing a small bug or developing a major feature, we encourage following these steps for a smooth contribution process. + +## Install SGLang from Source + +### Prepare Environment + +Before contributing, please ensure that your environment is set up correctly. Follow the steps in the [Installation Guide](./ascend_npu) to install the necessary dependencies. We recommend [using docker](./ascend_npu#method-2-using-docker-image) to build the environment. + +### Fork and clone the repository + +**Note**: New contributors do **not** have the write permission to push to the official SGLang repo. Please fork the repository under your GitHub account, then clone your fork locally. + +```bash +git clone https://github.com//sglang.git +# if you are using docker, the environment is already set up. +cd sglang +export PYTHONPATH=$PWD/python:$PYTHONPATH +``` + +## Format code with pre-commit + +We use [pre-commit](https://pre-commit.com/) to maintain consistent code style checks. Before pushing your changes, please run: + +```bash +pip3 install pre-commit +pre-commit install +pre-commit run --all-files +``` + +- **`pre-commit run --all-files`** manually runs all configured checks, applying fixes if possible. If it fails the first time, re-run it to ensure lint errors are fully resolved. Make sure your code passes all checks **before** creating a Pull Request. +- **Do not commit** directly to the `main` branch. Always create a new branch (e.g., `feature/my-new-feature`), push your changes, and open a PR from that branch. + +## Run and add unit tests + +If you add a new feature or fix a bug, please add corresponding unit tests to ensure coverage and prevent regression. +SGLang uses Python's built-in [unittest](https://docs.python.org/3/library/unittest.html) framework. +For detailed instructions on running tests and integrating them into CI, refer to [test/README.md](https://github.com/sgl-project/sglang/tree/main/test/README.md). + +If you need to use model which is not in `python/sglang/test/ascend/test_ascend_utils.py` list. Follow these steps: +1. Register account and upload your model to [modelscope](https://modelscope.cn/models). +2. Make sure your model is pre-cached on the CI server and is on the way "/data/ascend-ci-share-pkking-sglang/modelscope/hub/models/{your_model_repo}/{your_model}". +If this is not the case, use following command on CI server: + ```bash + modelscope download + --model {your_model_repo}/{your_model} + --local_dir /data/ascend-ci-share-pkking-sglang/modelscope/hub/models/{your_model_repo}/{your_model} + ``` + > Note: If you don’t have access to CI server, please ask maintainers (zl19940307@163.com) to download your model. +4. Add model to ```python/sglang/test/ascend/test_ascend_utils.py``` (use docker ```"/root/.cache/modelscope/hub/models/{your_model_repo}/{your_model}"``` path). + +## Write documentations + +We recommend new contributors start from writing documentation, which helps you quickly understand SGLang codebase. +For more details, please refer to [docs/README.md](https://github.com/sgl-project/sglang/tree/main/docs/README.md). + +## Test the accuracy +If your code changes the model output, please run the accuracy tests. A quick sanity check is the few-shot GSM8K. + +``` +# Launch a server +python3 -m sglang.launch_server --model Qwen/Qwen2-7B-Instruct + +# Evaluate +python3 -m sglang.test.few_shot_gsm8k --num-questions 200 +``` + +Please note that the above script is primarily a sanity check, not a rigorous accuracy or speed test. +This test can have significant variance (1%–5%) in accuracy due to batching and the non-deterministic nature of the inference engine. +Also, do not rely on the "Latency/Output throughput" from this script, as it is not a proper speed test. + +GSM8K is too easy for state-of-the-art models nowadays. Please try your own more challenging accuracy tests. +You can find additional accuracy eval examples in: +- [test_eval_accuracy_large.py](https://github.com/sgl-project/sglang/blob/main/test/registered/eval/test_eval_accuracy_large.py) +- [test_gpt_oss_1gpu.py](https://github.com/sgl-project/sglang/blob/main/test/registered/core/test_gpt_oss_1gpu.py) + +## Benchmark the speed +Refer to [Benchmark and Profiling](../../developer_guide/benchmark_and_profiling). + +## Requesting a review for merge +You can follow the pull request merge process described in [MAINTAINER.md](https://github.com/sgl-project/sglang/blob/main/.github/MAINTAINER.md). +You will need to work with the Merge Oncall, Codeowner, and other reviewers to get their approvals. +Then your PR can be merged. + +## How to Trigger CI Tests + +We have a lot of open PRs but limited CI machines, so only top and trusted contributors have permission to trigger CI tests. +Users with permission are listed in the [CI_PERMISSIONS.json](https://github.com/sgl-project/sglang/blob/main/.github/CI_PERMISSIONS.json) + +For CI to run on a pull request, it must have the "run-ci" label. Authorized users can add the label or rerun failed tests by commenting on the PR with one of these commands: + +- `/tag-run-ci-label`: Adds the "run-ci" label. Every future commit will trigger CI. +- `/rerun-failed-ci`: Reruns the failed or flaky tests from the most recent commit. +- `/tag-and-rerun-ci`: A single command that performs both `/tag-run-ci-label` and `/rerun-failed-ci`. +- `/rerun-stage `: Reruns a specific test stage without waiting for its dependencies. This is useful when you want to quickly validate a fix for a specific test failure instead of waiting ~30 minutes for preceding stages to complete. + +If you have permission, the [Slash Command Handler](https://github.com/sgl-project/sglang/actions/workflows/slash-command-handler.yml) will run your command and react with a 👍 to your comment. It may take up to a few minutes for the reaction to appear. Here’s a usage [example](https://github.com/sgl-project/sglang/pull/14253#issuecomment-3599509302). + +To avoid spamming a PR with too many `/rerun-failed-ci` comments, you can also trigger the command by editing an existing comment and adding any suffix (e.g., `/rerun-failed-ci try again`). + +Example of rerunning a single test stage: `/rerun-stage unit-test-backend-4-gpu`. + +If you don’t have permission, please ask maintainers to trigger CI for you. + +### CI rate limits + +Due to CI scheduling and limited resources, higher-priority PRs may preempt running jobs. In such cases, you may need to rerun the tests. + +We apply CI rate limits to prevent abuse and ensure fair usage of our CI resources. + +Each CI workflow has a default limit defined in its workflow configuration file. For example, in [pr-gate.yml](https://github.com/sgl-project/sglang/blob/main/.github/workflows/pr-gate.yml), the default cooldown period is 120 minutes, and each workflow can override it via the `cool-down-minutes` input parameter: + +```yaml +cool-down-minutes: + description: "Cooldown period in minutes for low-permission users; 0 disables rate limiting" + type: number + default: 120 +``` + +Users listed in [CI_PERMISSIONS.json](https://github.com/sgl-project/sglang/blob/main/.github/CI_PERMISSIONS.json) may have a per-user cooldown interval. In practice, we use the minimum of the workflow’s default window and the user-specific interval. + +## Code style guidance +- Avoid code duplication. If the same code snippet (more than five lines) appears multiple times, extract it into a shared function. +- Minimize device synchronization. Reduce expensive CPU-GPU synchronization operations, such as `tensor.item()` or `tensor.cpu()`, whenever possible. Use vectorized code. +- Prioritize extreme efficiency. SGLang is a runtime, and most of your code runs on the critical path for every request. Optimize all minor overheads as much as possible, especially in the model forward code. + - A common pattern is some runtime checks in the model forward pass (e.g., [this](https://github.com/sgl-project/sglang/blob/f1b0eda55c2c4838e8ab90a0fac7fb1e3d7064ab/python/sglang/srt/models/deepseek_v2.py#L486-L491)). These are very likely the same for every layer. Please cache the result as a single boolean value whenever possible. +- Make functions as pure as possible. Avoid in-place modification of arguments. +- Keep files concise. If a file exceeds 2,000 lines of code, split it into multiple smaller files. (e.g., `scheduler.py`, `scheduler_output_processor_mixin.py`) +- Keep tests run fast. + - If a single test file run longer than 500 seconds, split it into multiple smaller files (e.g., `test_eagle_infer_a.py`, `test_eagle_infer_b.py`). + - If a single job in a github workflow runs longer than 30 mins, split it into smaller jobs/steps. + - Reuse server launches in your unit tests to make tests run faster. +- When supporting new hardware or features, follow these guidelines: + - Do not drastically change existing code. + - Always prefer new files to introduce specific components for your new hardware (e.g., `allocator_npu.py`). + - If you write multiple if/else blocks for new features, ensure the common path (e.g., NVIDIA hardware or the existing code path) is the first branch. + +## How to update sgl-kernel +Since sglang and sgl-kernel are separate Python packages, our current GitHub CI infrastructure does not support updating a kernel and using it immediately within the same pull request (PR). +To add a new kernel or modify an existing one in the `sgl-kernel/` source tree, you must use multiple PRs. + +Follow these steps: + +1. Submit a PR to update the sgl-kernel source code without using it in sglang python package (e.g., [#8884](https://github.com/sgl-project/sglang/pull/8884/files)). +2. Bump the version of the kernel package (e.g., [#9220](https://github.com/sgl-project/sglang/pull/9220/files)). + - Once merged, this will trigger an automatic release of the `sglang-kernel` wheel to PyPI. + - If not urgent, you can wait for other people to release the wheel. A new version will typically be released within one week. +3. Apply the changes: + - Update the `sglang-kernel` version in `sglang/python/pyproject.toml` to use the modified kernels. + - Update the related caller code in the sglang to use the new kernel. + +## How to update sgl-kernel-npu + +Sgl-kernel-npu is the kernel package for Ascend NPU and is maintained in the [sgl-kernel-npu](https://github.com/sgl-project/sgl-kernel-npu) repository. if you want to add a new kernel and want to use it in sglang, please follow the steps in [Contribution Guide](https://github.com/sgl-project/sgl-kernel-npu/blob/main/docs/developer_guide/contribution_guide.md). + +## Tips for newcomers + +If you want to contribute but don’t have a specific idea in mind, pick issues labeled [“good first issue” or “help wanted”](https://github.com/sgl-project/sglang/issues?q=is%3Aissue+label%3A%22good+first+issue%22%2C%22help+wanted%22). These tasks typically have lower complexity and provide an excellent introduction to the codebase. Also check out this [code walk-through](https://github.com/zhaochenyang20/Awesome-ML-SYS-Tutorial/tree/main/sglang/code-walk-through) for a deeper look into SGLang’s workflow. + +If you have any questions or want to start a discussion, please feel free to ask in our [Slack channel](https://slack.sglang.io). + +Thank you for your interest in SGLang. Happy coding! diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu.mdx new file mode 100644 index 000000000..3fe88df70 --- /dev/null +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu.mdx @@ -0,0 +1,294 @@ +--- +title: SGLang installation with NPUs support +--- +You can install SGLang using any of the methods below. Please go through `System Settings` section to ensure the clusters are roaring at max performance. Feel free to leave an issue [here at sglang](https://github.com/sgl-project/sglang/issues) if you encounter any issues or have any problems. + +## Component Version Mapping For SGLang + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ComponentVersionObtain Way
HDK25.5.2link
CANN8.5.0Obtain Images
Pytorch Adapter7.3.0link
MemFabric1.0.5`pip install memfabric-hybrid==1.0.5`
Triton3.2.0`pip install triton-ascend`
SGLang NPU KernelNAlink
+ + +### Obtain CANN Image +You can obtain the dependency of a specified version of CANN through an image. +```bash Command +# for Atlas 800I A3 and Ubuntu OS +docker pull quay.io/ascend/cann:8.5.0-a3-ubuntu22.04-py3.11 +# for Atlas 800I A2 and Ubuntu OS +docker pull quay.io/ascend/cann:8.5.0-910b-ubuntu22.04-py3.11 +``` + +## Preparing the Running Environment + +### Method 1: Installing from source with prerequisites + +#### Python Version + +Only `python==3.11` is supported currently. If you don't want to break system pre-installed python, try installing with [conda](https://github.com/conda/conda). + +```bash Command +conda create --name sglang_npu python=3.11 +conda activate sglang_npu +``` + +#### CANN + +Prior to start work with SGLang on Ascend you need to install CANN Toolkit, Kernels operator package and NNAL version 8.5.0, check the [installation guide](https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/850/softwareinst/instg/instg_0008.html?Mode=PmIns&InstallType=local&OS=openEuler&Software=cannToolKit) + +#### MemFabric-Hybrid + +If you want to use PD disaggregation mode, you need to install MemFabric-Hybrid. MemFabric-Hybrid is a drop-in replacement of Mooncake Transfer Engine that enables KV cache transfer on Ascend NPU clusters. + +```bash Command +pip install memfabric-hybrid==1.0.5 +``` + +#### Pytorch and Pytorch Framework Adaptor on Ascend + +```bash Command +PYTORCH_VERSION=2.8.0 +TORCHVISION_VERSION=0.23.0 +TORCH_NPU_VERSION=2.8.0.post2 +pip install torch==$PYTORCH_VERSION torchvision==$TORCHVISION_VERSION --index-url https://download.pytorch.org/whl/cpu +pip install torch_npu==$TORCH_NPU_VERSION +``` + +If you are using other versions of `torch` and install `torch_npu`, check [installation guide](https://github.com/Ascend/pytorch/blob/master/README.md) + +#### Triton on Ascend + +We provide our own implementation of Triton for Ascend. + +```bash Command +pip install triton-ascend +``` +For installation of Triton on Ascend nightly builds or from sources, follow [installation guide](https://gitcode.com/Ascend/triton-ascend/blob/master/docs/sources/getting-started/installation.md) + +#### SGLang Kernels NPU +We provide SGL kernels for Ascend NPU, check [installation guide](https://github.com/sgl-project/sgl-kernel-npu/blob/main/python/sgl_kernel_npu/README.md). + +#### DeepEP-compatible Library +We provide a DeepEP-compatible Library as a drop-in replacement of deepseek-ai's DeepEP library, check the [installation guide](https://github.com/sgl-project/sgl-kernel-npu/blob/main/python/deep_ep/README.md). + +#### Some other dependencies + +```bash Command +# libGL +apt update +apt install libgl1 libglib2.0-0 + +# ensure setuptools contains pkg_resources module +pip install "setuptools<80" +``` + +#### Installing SGLang from source + +```bash Command +# Use the last release branch +git clone https://github.com/sgl-project/sglang.git +cd sglang +mv python/pyproject_npu.toml python/pyproject.toml +pip install -e python[all_npu] +``` + +### Method 2: Using Docker Image +#### Obtain Image +You can download the SGLang image or build an image based on Dockerfile to obtain the Ascend NPU image. +1. Download SGLang image +```angular2html +dockerhub: docker.io/lmsysorg/sglang:$tag +# Main-based tag, change main to specific version like v0.5.6, +# you can get image for specific version +Atlas 800I A3 : {main}-cann8.5.0-a3 +Atlas 800I A2: {main}-cann8.5.0-910b +``` +2. Build an image based on Dockerfile +```bash Command +# Clone the SGLang repository +git clone https://github.com/sgl-project/sglang.git +cd sglang/docker + +# Build the docker image +# If there are network errors, please modify the Dockerfile to use offline dependencies or use a proxy +# is the target architecture of the image, e.g. amd64, arm64 +docker build --build-arg TARGETARCH= -t -f npu.Dockerfile . +``` + +#### Create Docker +__Notice:__ `--privileged` and `--network=host` are required by RDMA, which is typically needed by Ascend NPU clusters. + +__Notice:__ The following docker command is based on Atlas 800I A3 machines. If you are using Atlas 800I A2, make sure only `davinci[0-7]` are mapped into container. + +```bash Command + +alias drun='docker run -it --rm --privileged --network=host --ipc=host --shm-size=16g \ + --device=/dev/davinci0 --device=/dev/davinci1 --device=/dev/davinci2 --device=/dev/davinci3 \ + --device=/dev/davinci4 --device=/dev/davinci5 --device=/dev/davinci6 --device=/dev/davinci7 \ + --device=/dev/davinci8 --device=/dev/davinci9 --device=/dev/davinci10 --device=/dev/davinci11 \ + --device=/dev/davinci12 --device=/dev/davinci13 --device=/dev/davinci14 --device=/dev/davinci15 \ + --device=/dev/davinci_manager --device=/dev/hisi_hdc \ + --volume /usr/local/sbin:/usr/local/sbin --volume /usr/local/Ascend/driver:/usr/local/Ascend/driver \ + --volume /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ + --volume /etc/ascend_install.info:/etc/ascend_install.info \ + --volume /var/queue_schedule:/var/queue_schedule --volume ~/.cache/:/root/.cache/' + +# Add HF_TOKEN env for download model by SGLang. +drun --env "HF_TOKEN=" \ + \ + python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --attention-backend ascend +``` + +## System Settings + +### CPU performance power scheme + +The default power scheme on Ascend hardware is `ondemand` which could affect performance, changing it to `performance` is recommended. + +```bash Command +echo performance | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor + +# Make sure changes are applied successfully +cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor # shows performance +``` + +### Disable NUMA balancing + +```bash Command +sudo sysctl -w kernel.numa_balancing=0 +# Check +cat /proc/sys/kernel/numa_balancing # shows 0 +``` + +### Prevent swapping out system memory + +```bash Command +sudo sysctl -w vm.swappiness=10 + +# Check +cat /proc/sys/vm/swappiness # shows 10 +``` + +## Running SGLang Service +### Running Service For Large Language Models +#### PD Mixed Scene +```bash Command +# Enabling CPU Affinity +export SGLANG_SET_CPU_AFFINITY=1 +python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --attention-backend ascend +``` + +#### PD Disaggregation Scene +1. Launch Prefill Server +```bash Command +# Enabling CPU Affinity +export SGLANG_SET_CPU_AFFINITY=1 + +# PIP: recommended to config first Prefill Server IP +# PORT: one free port +# all sglang servers need to be config the same PIP and PORT, +export ASCEND_MF_STORE_URL="tcp://PIP:PORT" +# if you are Atlas 800I A2 hardware and use rdma for kv cache transfer, add this parameter +export ASCEND_MF_TRANSFER_PROTOCOL="device_rdma" +python3 -m sglang.launch_server \ + --model-path meta-llama/Llama-3.1-8B-Instruct \ + --disaggregation-mode prefill \ + --disaggregation-transfer-backend ascend \ + --disaggregation-bootstrap-port 8995 \ + --attention-backend ascend \ + --device npu \ + --base-gpu-id 0 \ + --tp-size 1 \ + --host 127.0.0.1 \ + --port 8000 +``` + +2. Launch Decode Server +```bash Command +# PIP: recommended to config first Prefill Server IP +# PORT: one free port +# all sglang servers need to be config the same PIP and PORT, +export ASCEND_MF_STORE_URL="tcp://PIP:PORT" +# if you are Atlas 800I A2 hardware and use rdma for kv cache transfer, add this parameter +export ASCEND_MF_TRANSFER_PROTOCOL="device_rdma" +python3 -m sglang.launch_server \ + --model-path meta-llama/Llama-3.1-8B-Instruct \ + --disaggregation-mode decode \ + --disaggregation-transfer-backend ascend \ + --attention-backend ascend \ + --device npu \ + --base-gpu-id 1 \ + --tp-size 1 \ + --host 127.0.0.1 \ + --port 8001 +``` + +3. Launch Router +```bash Command +python3 -m sglang_router.launch_router \ + --pd-disaggregation \ + --policy cache_aware \ + --prefill http://127.0.0.1:8000 8995 \ + --decode http://127.0.0.1:8001 \ + --host 127.0.0.1 \ + --port 6688 +``` + +### Running Service For Multimodal Language Models +#### PD Mixed Scene +```bash Command +python3 -m sglang.launch_server \ + --model-path Qwen3-VL-30B-A3B-Instruct \ + --host 127.0.0.1 \ + --port 8000 \ + --tp 4 \ + --device npu \ + --attention-backend ascend \ + --mm-attention-backend ascend_attn \ + --disable-radix-cache \ + --trust-remote-code \ + --enable-multimodal \ + --sampling-backend ascend +``` diff --git a/docs_new/docs/hardware-platforms/ascend-npus/Best-Practice-on-Ascend-NPU.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_best_practice.mdx similarity index 59% rename from docs_new/docs/hardware-platforms/ascend-npus/Best-Practice-on-Ascend-NPU.mdx rename to docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_best_practice.mdx index 0b4b3e917..ced8ef268 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/Best-Practice-on-Ascend-NPU.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_best_practice.mdx @@ -1,3 +1,8 @@ +--- +title: "Best Practice on Ascend NPU" +metatags: + description: "Documentation for Best Practice on Ascend NPU" +--- This section describes the best practice data of mainstream LLM models such as DeepSeek and Qwen on the Ascend NPU. If you encounter issues or have any questions, please [open an issue](https://github.com/sgl-project/sglang/issues). @@ -33,51 +38,51 @@ you encounter issues or have any questions, please [open an issue](https://githu Deepseek-R1 Atlas 800I A3 32 - PD Separation + PD Disaggregation 6K+1.6K 20ms W8A8 INT8 - [Optimal Configuration](#deepseek-r1-6k-1_6k-20ms-on-a3-32-cards-separation-mode) + Optimal Configuration Deepseek-R1 Atlas 800I A3 32 - PD Separation + PD Disaggregation 3.9K+1K - 20ms + 19ms W8A8 INT8 - [Optimal Configuration](#deepseek-r1-3_9k-1k-20ms-on-a3-32-cards-separation-mode) + Optimal Configuration Deepseek-R1 Atlas 800I A3 32 - PD Separation + PD Disaggregation 3.5K+1.5K - 20ms + 19ms W8A8 INT8 - [Optimal Configuration](#deepseek-r1-3_5k-1_5k-20ms-on-a3-32-cards-separation-mode) + Optimal Configuration Deepseek-R1 Atlas 800I A3 32 - PD Separation + PD Disaggregation 3.5K+1K - 20ms + 19ms W8A8 INT8 - [Optimal Configuration](#deepseek-r1-3_5k-1k-20ms-on-a3-32-cards-separation-mode) + Optimal Configuration - DeepSeek-V3.2-Exp + DeepSeek-V3.2 Atlas 800I A3 32 - PD Separation - 64K+3K - 30ms + PD Disaggregation + 128K+1K + 26ms W8A8 INT8 - [Optimal Configuration](#deepseek-v32-exp-64k-3k-30ms-on-a3-32-cards-separation-mode) + Optimal Configuration @@ -112,11 +117,21 @@ you encounter issues or have any questions, please [open an issue](https://githu Deepseek-R1 Atlas 800I A3 32 - PD Separation + PD Disaggregation 3.5K+1.5K 50ms W8A8 INT8 - [Optimal Configuration](#deepseek-r1-3_5k-1_5k-50ms-on-a3-32-cards-separation-mode) + Optimal Configuration + + + Deepseek-R1 + Atlas 800I A3 + 24 + PD Disaggregation + 2K+2K + 50ms + W8A8 INT8 + Optimal Configuration Deepseek-R1 @@ -126,17 +141,17 @@ you encounter issues or have any questions, please [open an issue](https://githu 2K+2K 50ms W4A8 INT8 - [Optimal Configuration](#deepseek-r1-2k-2k-50ms-on-a3-8-cards-mixed-mode) + Optimal Configuration Deepseek-R1 Atlas 800I A3 16 - PD Separation + PD Disaggregation 2K+2K 50ms W4A8 INT8 - [Optimal Configuration](#deepseek-r1-2k-2k-50ms-on-a3-16-cards-separation-mode) + Optimal Configuration Deepseek-R1 @@ -146,19 +161,19 @@ you encounter issues or have any questions, please [open an issue](https://githu 3.5K+1.5K 50ms W4A8 INT8 - [Optimal Configuration](#deepseek-r1-3_5k-1_5k-50ms-on-a3-8-cards-mixed-mode) + Optimal Configuration - + Deepseek-R1 Atlas 800I A3 16 - PD Separation + PD Disaggregation 3.5K+1.5K 50ms W4A8 INT8 - [Optimal Configuration](#deepseek-r1-3_5k-1_5k-50ms-on-a3-16-cards-separation-mode) + Optimal Configuration - + ## Qwen Series Models @@ -197,7 +212,7 @@ you encounter issues or have any questions, please [open an issue](https://githu 11K+1K 10ms BF16 - [Optimal Configuration](#qwen3-235b-a22b-11k-1k-10ms-on-a3-8-cards-mixed-mode) + Optimal Configuration Qwen3-32B @@ -207,7 +222,7 @@ you encounter issues or have any questions, please [open an issue](https://githu 6K+1.5K 18ms BF16 - [Optimal Configuration](#qwen3-32b-6k-1_5k-18ms-on-a3-4-cards-mixed-mode) + Optimal Configuration Qwen3-32B @@ -217,7 +232,7 @@ you encounter issues or have any questions, please [open an issue](https://githu 4K+1.5K 11ms BF16 - [Optimal Configuration](#qwen3-32b-4k-1_5k-11ms-on-a3-4-cards-mixed-mode) + Optimal Configuration Qwen3-32B @@ -225,9 +240,9 @@ you encounter issues or have any questions, please [open an issue](https://githu 8 PD Mixed 18K+4K - 12ms + 6ms BF16 - [Optimal Configuration](#qwen3-32b-18k-4k-12ms-on-a3-8-cards-mixed-mode) + Optimal Configuration Qwen3-32B @@ -237,7 +252,7 @@ you encounter issues or have any questions, please [open an issue](https://githu 6K+1.5K 18ms W8A8 INT8 - [Optimal Configuration](#qwen3-32b-6k-1_5k-18ms-on-a2-8-cards-mixed-mode) + Optimal Configuration Qwen3-32B @@ -247,9 +262,119 @@ you encounter issues or have any questions, please [open an issue](https://githu 4K+1.5K 11ms BF16 - [Optimal Configuration](#qwen3-32b-4k-1_5k-11ms-on-a2-8-cards-mixed-mode) + Optimal Configuration - + + Qwen3-32B + Atlas 800I A3 + 2 + PD Mixed + 1K+0.3K + 12ms + W8A8 INT8 + Optimal Configuration + + + Qwen3-32B + Atlas 800I A3 + 2 + PD Mixed + 6K+1.5K + 17ms + W8A8 INT8 + Optimal Configuration + + + Qwen3-8B + Atlas 800I A3 + 1 + PD Mixed + 1K+0.3K + 7ms + W8A8 INT8 + Optimal Configuration + + + Qwen3-8B + Atlas 800I A3 + 1 + PD Mixed + 6K+1.5K + 12ms + W8A8 INT8 + Optimal Configuration + + + Qwen3-8B + Atlas 800I A3 + 1 + PD Mixed + 3.5K+1.5K + 5ms + W8A8 INT8 + Optimal Configuration + + + Qwen3-30B-A3B + Atlas 800I A3 + 1 + PD Mixed + 6K+1.5K + 10ms + W8A8 INT8 + Optimal Configuration + + + Qwen3-30B-A3B + Atlas 800I A3 + 1 + PD Mixed + 1K+0.3K + 7ms + W8A8 INT8 + Optimal Configuration + + + Qwen3-Next-A3B-Instruct + Atlas 800I A3 + 2 + PD Mixed + 1K+0.3K + 14.21ms + W8A8 INT8 + Optimal Configuration + + + Qwen3-Next-A3B-Instruct + Atlas 800I A3 + 2 + PD Mixed + 6K+1.5K + 15.62ms + W8A8 INT8 + Optimal Configuration + + + Qwen3-Next-A3B-Instruct + Atlas 800I A3 + 2 + PD Mixed + 3.5K+1.5K + 20ms + W8A8 INT8 + Optimal Configuration + + + Qwen3-14B + Atlas 800I A3 + 1 + PD Mixed + 3.5K+1.5K + 9ms + W8A8 INT8 + Optimal Configuration + + ### High Throughput @@ -282,11 +407,11 @@ you encounter issues or have any questions, please [open an issue](https://githu Qwen3-235B-A22B Atlas 800I A3 24 - PD Separation + PD Disaggregation 3.5K+1.5K 50ms W8A8 INT8 - [Optimal Configuration](#qwen3-235b-a22b-3_5k-1_5k-50ms-on-a3-24-cards-separation-mode) + Optimal Configuration Qwen3-235B-A22B @@ -296,7 +421,7 @@ you encounter issues or have any questions, please [open an issue](https://githu 3.5K+1.5K 50ms W8A8 INT8 - [Optimal Configuration](#qwen3-235b-a22b-3_5k-1_5k-50ms-on-a3-8-cards-mixed-mode) + Optimal Configuration Qwen3-235B-A22B @@ -306,7 +431,7 @@ you encounter issues or have any questions, please [open an issue](https://githu 2K+2K 100ms W8A8 INT8 - [Optimal Configuration](#qwen3-235b-a22b-2k-2k-100ms-on-a3-8-cards-mixed-mode) + Optimal Configuration Qwen3-235B-A22B @@ -316,7 +441,7 @@ you encounter issues or have any questions, please [open an issue](https://githu 2K+2K 50ms W8A8 INT8 - [Optimal Configuration](#qwen3-235b-a22b-2k-2k-50ms-on-a3-8-cards-mixed-mode) + Optimal Configuration Qwen3-235B-A22B @@ -326,7 +451,7 @@ you encounter issues or have any questions, please [open an issue](https://githu 2K+2K 50ms W8A8 INT8 - [Optimal Configuration](#qwen3-235b-a22b-2k-2k-50ms-on-a3-16-cards-mixed-mode) + Optimal Configuration Qwen3-32B @@ -336,7 +461,7 @@ you encounter issues or have any questions, please [open an issue](https://githu 3.5K+1.5K 50ms W8A8 INT8 - [Optimal Configuration](#qwen3-32b-3_5k-1_5k-50ms-on-a3-2-cards-mixed-mode) + Optimal Configuration Qwen3-32B @@ -346,7 +471,7 @@ you encounter issues or have any questions, please [open an issue](https://githu 2K+2K 50ms W8A8 INT8 - [Optimal Configuration](#qwen3-32b-2k-2k-50ms-on-a3-2-cards-mixed-mode) + Optimal Configuration Qwen3-30B-A3B @@ -356,17 +481,17 @@ you encounter issues or have any questions, please [open an issue](https://githu 3.5K+1.5K 50ms W8A8 INT8 - [Optimal Configuration](#qwen3-30b-a3b-3_5k-1_5k-50ms-on-a3-1-card-mixed-mode) + Optimal Configuration Qwen3-Coder-480B-A35B-Instruct Atlas 800I A3 24 - PD Separation + PD Disaggregation 3.5K+1.5K 50ms W8A8 INT8 - [Optimal Configuration](#qwen3-coder-480b-a35b-instruct-3_5k-1_5k-50ms-on-a3-24-cards-separation-mode) + Optimal Configuration Qwen3-Coder-480B-A35B-Instruct @@ -376,7 +501,7 @@ you encounter issues or have any questions, please [open an issue](https://githu 3.5K+1.5K 50ms W8A8 INT8 - [Optimal Configuration](#qwen3-coder-480b-a35b-instruct-3_5k-1_5k-50ms-on-a3-16-cards-mixed-mode) + Optimal Configuration Qwen3-Coder-480B-A35B-Instruct @@ -386,7 +511,7 @@ you encounter issues or have any questions, please [open an issue](https://githu 3.5K+1.5K 50ms W8A8 INT8 - [Optimal Configuration](#qwen3-coder-480b-a35b-instruct-3_5k-1_5k-50ms-on-a3-8-cards-mixed-mode) + Optimal Configuration Qwen3-Next-80B-A3B-Instruct @@ -396,7 +521,7 @@ you encounter issues or have any questions, please [open an issue](https://githu 3.5K+1.5K 50ms W8A8 INT8 - [Optimal Configuration](#qwen3-next-80B-a3b-instruct-3_5k-1_5k-50ms-on-a3-2-cards-mixed-mode) + Optimal Configuration Qwen3-32B @@ -406,7 +531,7 @@ you encounter issues or have any questions, please [open an issue](https://githu 3.5K+1.5K 50ms W8A8 INT8 - [Optimal Configuration](#qwen3-32b-3_5k-1_5k-50ms-on-a2-8-cards-mixed-mode) + Optimal Configuration Qwen3-32B @@ -416,20 +541,40 @@ you encounter issues or have any questions, please [open an issue](https://githu 2K+2K 50ms W8A8 INT8 - [Optimal Configuration](#qwen3-32b-2k-2k-50ms-on-a2-8-cards-mixed-mode) + Optimal Configuration - + + Qwen3-14B + Atlas 800I A3 + 1 + PD Mixed + 3.5K+1.5K + 50ms + W8A8 INT8 + Optimal Configuration + + + Qwen3-8B + Atlas 800I A3 + 1 + PD Mixed + 3.5K+1.5K + 50ms + W8A8 INT8 + Optimal Configuration + + ## Optimal Configuration -### DeepSeek-R1 3_5K-1_5K 50ms on A3 32 Cards Separation Mode +### DeepSeek-R1 3_5K-1_5K 50ms on A3 32 Cards Disaggregation Mode Model: Deepseek R1 Hardware: Atlas 800I A3 32Card -DeployMode: PD Separation +DeployMode: PD Disaggregation Dataset: random @@ -439,7 +584,7 @@ TPOT: 50ms #### Model Deployment -```shell Command +```bash Command echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor sysctl -w vm.swappiness=0 sysctl -w kernel.numa_balancing=0 @@ -452,6 +597,10 @@ export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export STREAMS_PER_DEVICE=32 +export HCCL_OP_EXPANSION_MODE=AIV +export SGLANG_NPU_USE_MLAPO=1 +export SGLANG_USE_FIA_NZ=1 +export SGLANG_NPU_USE_MULTI_STREAM=1 export ASCEND_MF_STORE_URL="tcp://your prefill ip1:24669" @@ -461,9 +610,6 @@ D_IP=('your decode ip1' 'your decode ip2') MODEL_PATH=xxx -export SGLANG_NPU_USE_MLAPO=1 -export SGLANG_USE_FIA_NZ=1 - LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` echo "${LOCAL_HOST1}" @@ -474,19 +620,22 @@ do if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; then echo "${P_IP[$i]}" - export HCCL_BUFFSIZE=1536 + export SGLANG_USE_AG_AFTER_QLORA=1 + export HCCL_BUFFSIZE=800 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export TASK_QUEUE_ENABLE=2 + export SGLANG_NPU_FUSED_MOE_MODE=2 + export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=131072 export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \ --port 8000 --disaggregation-bootstrap-port $((8998+$i)) --trust-remote-code --nnodes 1 --node-rank 0 \ - --tp-size 16 --mem-fraction-static 0.81 --attention-backend ascend --device npu --quantization modelslim \ - --disaggregation-transfer-backend ascend --max-running-requests 8 --context-length 8192 --disable-radix-cache \ - --chunked-prefill-size -1 --max-prefill-tokens 28680 --moe-a2a-backend deepep --deepep-mode normal \ + --tp-size 16 --mem-fraction-static 0.778 --attention-backend ascend --device npu --quantization modelslim \ + --disaggregation-transfer-backend ascend --max-running-requests 16 --disable-radix-cache \ + --chunked-prefill-size -1 --max-prefill-tokens 60000 --moe-a2a-backend ascend_fuseep --deepep-mode normal \ --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ - --dp-size 2 --enable-dp-attention --disable-shared-experts-fusion --dtype bfloat16 --enable-attn-tp-input-scattered + --dp-size 4 --enable-dp-attention --disable-shared-experts-fusion --dtype bfloat16 --enable-attn-tp-input-scattered NODE_RANK=$i break fi @@ -500,20 +649,21 @@ do echo "${D_IP[$i]}" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_SPEC_V2=1 - export HCCL_BUFFSIZE=650 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=78 + export HCCL_BUFFSIZE=600 + export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64 export TASK_QUEUE_ENABLE=1 - export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 + export SGLANG_NPU_FUSED_MOE_MODE=1 + export SGLANG_LM_HEAD_TP=8 export HCCL_SOCKET_IFNAME=xxx export GLOO_SOCKET_IFNAME=xxx python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ --port 8001 --trust-remote-code --dist-init-addr ${D_IP[0]}:5000 --nnodes 2 --node-rank $i --tp-size 32 --dp-size 32 \ - --mem-fraction-static 0.815 --max-running-requests 832 --attention-backend ascend --device npu --quantization modelslim \ - --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head --moe-dense-tp 1 \ - --cuda-graph-bs 12 14 16 18 20 22 24 26 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ - --speculative-algorithm NEXTN --speculative-num-steps 2 --speculative-eagle-topk 1 --speculative-num-draft-tokens 3 \ - --tokenizer-worker-num 4 --prefill-round-robin-balance --disable-shared-experts-fusion --dtype bfloat16 \ - --load-balance-method decode_round_robin + --mem-fraction-static 0.82 --max-running-requests 1024 --attention-backend ascend --device npu --quantization modelslim \ + --moe-a2a-backend ascend_fuseep --enable-dp-attention --deepep-mode low_latency --moe-dense-tp 1 \ + --cuda-graph-bs 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ + --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ + --tokenizer-worker-num 4 --disable-shared-experts-fusion --dtype bfloat16 \ + --load-balance-method round_robin NODE_RANK=$i break fi @@ -542,13 +692,141 @@ We tested it based on the `RANDOM` dataset. python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 768 --random-input-len 3500 --random-output-len 1500 --num-prompts 3072 --random-range-ratio 1 --request-rate 16 ``` -### DeepSeek-R1 6K-1_6K 20ms on A3 32 Cards Separation Mode +### DeepSeek-R1 2K-2K 50ms on A3 24 Cards Disaggregation Mode + +Model: Deepseek R1 + +Hardware: Atlas 800I A3 24Card + +DeployMode: PD Disaggregation + +Dataset: random + +Input Output Length: 2K+2K + +TPOT: 50ms + +#### Model Deployment + +```bash Command +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +unset ASCEND_LAUNCH_BLOCKING +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH + +export SGLANG_SET_CPU_AFFINITY=1 +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export STREAMS_PER_DEVICE=32 +export SGLANG_NPU_USE_MLAPO=1 +export SGLANG_USE_FIA_NZ=1 + +export ASCEND_MF_STORE_URL="tcp://your prefill ip1:24669" + +P_IP=('your prefill ip1') +D_IP=('your decode ip1' 'your decode ip2') + +MODEL_PATH=xxx + +LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` +LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` +echo "${LOCAL_HOST1}" +echo "${LOCAL_HOST2}" +# prefill +for i in "${!P_IP[@]}"; +do + if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; + then + echo "${P_IP[$i]}" + export HCCL_BUFFSIZE=1600 + export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 + export TASK_QUEUE_ENABLE=2 + export SGLANG_USE_AG_AFTER_QLORA=1 + export HCCL_SOCKET_IFNAME=lo + export GLOO_SOCKET_IFNAME=lo + + python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \ + --port 8000 --disaggregation-bootstrap-port $((8998+$i)) --trust-remote-code --nnodes 1 --node-rank 0 \ + --tp-size 16 --mem-fraction-static 0.8 --attention-backend ascend --device npu --quantization modelslim \ + --disaggregation-transfer-backend ascend --max-running-requests 20 --context-length 8192 --disable-radix-cache \ + --chunked-prefill-size -1 --max-prefill-tokens 28680 --moe-a2a-backend deepep --deepep-mode normal \ + --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ + --dp-size 4 --enable-dp-attention --disable-shared-experts-fusion --dtype bfloat16 --enable-attn-tp-input-scattered + NODE_RANK=$i + break + fi +done + +# decode +for i in "${!D_IP[@]}"; +do + if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; + then + echo "${D_IP[$i]}" + export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + export SGLANG_ENABLE_SPEC_V2=1 + export HCCL_BUFFSIZE=800 + export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=102 + export TASK_QUEUE_ENABLE=1 + export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 + export SGLANG_NPU_FUSED_MOE_MODE=1 + export HCCL_SOCKET_IFNAME=xxx + export GLOO_SOCKET_IFNAME=xxx + + python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ + --port 8001 --trust-remote-code --dist-init-addr ${D_IP[0]}:5000 --nnodes 2 --node-rank $i --tp-size 32 --dp-size 32 \ + --mem-fraction-static 0.81 --max-running-requests 1088 --attention-backend ascend --device npu --quantization modelslim \ + --moe-a2a-backend ascend_fuseep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head --moe-dense-tp 1 \ + --cuda-graph-bs 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ + --speculative-algorithm NEXTN --speculative-num-steps 2 --speculative-eagle-topk 1 --speculative-num-draft-tokens 3 \ + --tokenizer-worker-num 4 --disable-shared-experts-fusion --dtype bfloat16 \ + --load-balance-method round_robin + NODE_RANK=$i + break + fi +done + +``` + +```bash Command +python -m sglang_router.launch_router \ + --pd-disaggregation \ + --policy cache_aware \ + --prefill http://P_IP:8000 8998 \ + --prefill http://P_IP:8000 8999 \ + --decode http://D_IP:8001 \ + --host 127.0.0.1 \ + --port 6688 \ + --mini-lb +``` + +#### Benchmark + +We tested it based on the `RANDOM` dataset. + +```bash Command +python -m sglang.bench_serving --dataset-name random --backend sglang \ +--host 127.0.0.1 \ +--port 6688 \ +--max-concurrency 1088 \ +--random-input-len 2048 \ +--random-output-len 2048 \ +--num-prompts 12800 \ +--random-range-ratio 1 \ +--request-rate 24 +``` + +### DeepSeek-R1 6K-1_6K 20ms on A3 32 Cards Disaggregation Mode Model: Deepseek R1 Hardware: Atlas 800I A3 32Card -DeployMode: PD Separation +DeployMode: PD Disaggregation Dataset: random @@ -558,7 +836,7 @@ TPOT: 20ms #### Model Deployment -```shell Command +```bash Command echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor sysctl -w vm.swappiness=0 sysctl -w kernel.numa_balancing=0 @@ -600,7 +878,133 @@ do python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \ --port 8000 --disaggregation-bootstrap-port $((8998+$i)) --trust-remote-code --nnodes 1 --node-rank 0 \ --tp-size 16 --mem-fraction-static 0.81 --attention-backend ascend --device npu --quantization modelslim \ - --disaggregation-transfer-backend ascend --max-running-requests 4 --context-length 8192 --disable-radix-cache \ + --disaggregation-transfer-backend ascend --max-running-requests 4 --disable-radix-cache \ + --chunked-prefill-size -1 --max-prefill-tokens 28680 --moe-a2a-backend deepep --deepep-mode normal \ + --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ + --dp-size 2 --enable-dp-attention --disable-shared-experts-fusion --dtype bfloat16 --enable-attn-tp-input-scattered + NODE_RANK=$i + break + fi +done + +# decode +for i in "${!D_IP[@]}"; +do + if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; + then + echo "${D_IP[$i]}" + export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + export SGLANG_ENABLE_SPEC_V2=1 + export HCCL_BUFFSIZE=650 + export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16 + export TASK_QUEUE_ENABLE=1 + export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 + export HCCL_SOCKET_IFNAME=xxx + export GLOO_SOCKET_IFNAME=xxx + python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ + --port 8001 --trust-remote-code --dist-init-addr DIP1:5000 --nnodes 2 --node-rank $i --tp-size 32 --dp-size 8 \ + --mem-fraction-static 0.75 --max-running-requests 32 --attention-backend ascend --device npu --quantization modelslim \ + --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head --moe-dense-tp 1 \ + --cuda-graph-bs 2 4 6 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 \ + --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ + --tokenizer-worker-num 4 --disable-shared-experts-fusion --dtype bfloat16 \ + --load-balance-method round_robin + NODE_RANK=$i + break + fi +done + +``` + +```shell Command +export SGLANG_DP_ROUND_ROBIN=1 +python -m sglang_router.launch_router \ + --pd-disaggregation \ + --policy cache_aware \ + --prefill http://P_IP:8000 8998 \ + --prefill http://P_IP:8000 8999 \ + --decode http://D_IP:8001 \ + --host 127.0.0.1 \ + --port 6688 \ + --mini-lb +``` + +#### Benchmark + +We tested it based on the `RANDOM` dataset. + +```bash Command +python -m sglang.bench_serving --dataset-name random --backend sglang \ + --host 127.0.0.1 \ + --port 6688 \ + --max-concurrency 32 \ + --random-input-len 6000 \ + --random-output-len 1600 \ + --num-prompts 32 \ + --random-range-ratio 1 \ + --request-rate 16 +``` + +### DeepSeek-R1 3_9K-1K 19ms on A3 32 Cards Disaggregation Mode + +Model: Deepseek R1 + +Hardware: Atlas 800I A3 32Card + +DeployMode: PD Disaggregation + +Dataset: random + +Input Output Length: 3.9K+1K + +TPOT: 19ms + +#### Model Deployment + +```bash Command +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +unset ASCEND_LAUNCH_BLOCKING +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH + +export SGLANG_SET_CPU_AFFINITY=1 +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export STREAMS_PER_DEVICE=32 +export SGLANG_NPU_USE_MLAPO=1 +export SGLANG_USE_FIA_NZ=1 +export ASCEND_MF_STORE_URL="tcp://your prefill ip1:24669" + +P_IP=('your prefill ip1' 'your prefill ip2') +D_IP=('your decode ip1' 'your decode ip2') + +MODEL_PATH=xxx + +LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` +LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` +echo "${LOCAL_HOST1}" +echo "${LOCAL_HOST2}" + +# prefill +for i in "${!P_IP[@]}"; +do + if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; + then + echo "${P_IP[$i]}" + export HCCL_BUFFSIZE=1536 + export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 + export TASK_QUEUE_ENABLE=2 + export HCCL_SOCKET_IFNAME=lo + export GLOO_SOCKET_IFNAME=lo + + python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \ + --port 8000 --disaggregation-bootstrap-port $((8998+$i)) --trust-remote-code --nnodes 1 --node-rank 0 \ + --tp-size 16 --mem-fraction-static 0.81 --attention-backend ascend --device npu --quantization modelslim \ + --disaggregation-transfer-backend ascend --max-running-requests 4 --context-length 8192 --disable-radix-cache \ --chunked-prefill-size -1 --max-prefill-tokens 28680 --moe-a2a-backend deepep --deepep-mode normal \ --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ --dp-size 2 --enable-dp-attention --disable-shared-experts-fusion --dtype bfloat16 --enable-attn-tp-input-scattered @@ -629,16 +1033,15 @@ do --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head --moe-dense-tp 1 \ --cuda-graph-bs 2 4 6 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --tokenizer-worker-num 4 --prefill-round-robin-balance --disable-shared-experts-fusion --dtype bfloat16 \ - --load-balance-method decode_round_robin + --tokenizer-worker-num 4 --disable-shared-experts-fusion --dtype bfloat16 \ + --load-balance-method round_robin NODE_RANK=$i break fi done - ``` -```shell Command +```bash Command export SGLANG_DP_ROUND_ROBIN=1 python -m sglang_router.launch_router \ --pd-disaggregation \ @@ -655,86 +1058,84 @@ python -m sglang_router.launch_router \ We tested it based on the `RANDOM` dataset. -```shell Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 32 --random-input-len 6000 --random-output-len 1600 --num-prompts 32 --random-range-ratio 1 +```bash Command +python -m sglang.bench_serving --dataset-name random --backend sglang \ + --host 127.0.0.1 \ + --port 6688 \ + --max-concurrency 32 \ + --random-input-len 3900 \ + --random-output-len 1024 \ + --num-prompts 32 \ + --random-range-ratio 1 \ + --request-rate 16 ``` -### DeepSeek-R1 3_9K-1K 20ms on A3 32 Cards Separation Mode +### DeepSeek-R1 3_5K-1_5K 19ms on A3 32 Cards Disaggregation Mode Model: Deepseek R1 Hardware: Atlas 800I A3 32Card -DeployMode: PD Separation - -Dataset: random - -Input Output Length: 3.9K+1K - -TPOT: 20ms - -#### Model Deployment - -Please Turn to [DeepSeek-R1 6K-1_6K 20ms on A3 32 Cards Separation Mode](#deepseek-r1-6k-1_6k-20ms-on-a3-32-cards-separation-mode) - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 768 --random-input-len 3900 --random-output-len 1000 --num-prompts 768 --random-range-ratio 1 --request-rate 16 -``` - -### DeepSeek-R1 3_5K-1_5K 20ms on A3 32 Cards Separation Mode - -Model: Deepseek R1 - -Hardware: Atlas 800I A3 32Card - -DeployMode: PD Separation +DeployMode: PD Disaggregation Dataset: random Input Output Length: 3.5K+1.5K -TPOT: 20ms +TPOT: 19ms #### Model Deployment -Please Turn to [DeepSeek-R1 6K-1_6K 20ms on A3 32 Cards Separation Mode](#deepseek-r1-6k-1_6k-20ms-on-a3-32-cards-separation-mode) +Please Turn to [DeepSeek-R1 3_9K-1K 19ms on A3 32 Cards Disaggregation Mode](#deepseek-r1-3_9k-1k-19ms-on-a3-32-cards-disaggregation-mode) #### Benchmark We tested it based on the `RANDOM` dataset. ```bash Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 768 --random-input-len 3500 --random-output-len 1500 --num-prompts 768 --random-range-ratio 1 --request-rate 16 +python -m sglang.bench_serving --dataset-name random --backend sglang \ + --host 127.0.0.1 \ + --port 6688 \ + --max-concurrency 32 \ + --random-input-len 3500 \ + --random-output-len 1500 \ + --num-prompts 32 \ + --random-range-ratio 1 \ + --request-rate 16 ``` -### DeepSeek-R1 3_5K-1K 20ms on A3 32 Cards Separation Mode +### DeepSeek-R1 3_5K-1K 19ms on A3 32 Cards Disaggregation Mode Model: Deepseek R1 Hardware: Atlas 800I A3 32Card -DeployMode: PD Separation +DeployMode: PD Disaggregation Dataset: random Input Output Length: 3.5K+1K -TPOT: 20ms +TPOT: 19ms #### Model Deployment -Please Turn to [DeepSeek-R1 6K-1_6K 20ms on A3 32 Cards Separation Mode](#deepseek-r1-6k-1_6k-20ms-on-a3-32-cards-separation-mode) +Please Turn to [DeepSeek-R1 3_9K-1K 19ms on A3 32 Cards Disaggregation Mode](#deepseek-r1-3_9k-1k-19ms-on-a3-32-cards-disaggregation-mode) #### Benchmark We tested it based on the `RANDOM` dataset. -```shell Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 768 --random-input-len 3500 --random-output-len 1000 --num-prompts 768 --random-range-ratio 1 --request-rate 16 +```bash Command +python -m sglang.bench_serving --dataset-name random --backend sglang \ + --host 127.0.0.1 \ + --port 6688 \ + --max-concurrency 32 \ + --random-input-len 3500 \ + --random-output-len 1024 \ + --num-prompts 32 \ + --random-range-ratio 1 \ + --request-rate 16 ``` ### DeepSeek-R1 2K-2K 50ms on A3 8 Cards Mixed Mode @@ -753,7 +1154,7 @@ TPOT: 50ms #### Model Deployment -```shell Command +```bash Command echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor sysctl -w vm.swappiness=0 sysctl -w kernel.numa_balancing=0 @@ -772,11 +1173,12 @@ export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export STREAMS_PER_DEVICE=32 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 +export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200 export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64 +export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=88 export HCCL_BUFFSIZE=1600 export DEEPEP_NORMAL_LONG_SEQ_ROUND=10 export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=512 @@ -788,7 +1190,6 @@ export SGLANG_NPU_USE_MLAPO=1 export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_USE_FIA_NZ=1 -export ENABLE_MOE_NZ=1 python3 -m sglang.launch_server --model-path ${MODEL_PATH} \ --tp 16 \ @@ -798,32 +1199,31 @@ python3 -m sglang.launch_server --model-path ${MODEL_PATH} \ --quantization modelslim \ --watchdog-timeout 9000 \ --host 127.0.0.1 --port 6699 \ ---cuda-graph-bs 4 8 16 \ ---mem-fraction-static 0.74 \ ---max-running-requests 256 \ +--cuda-graph-bs 4 8 20 21 22 \ +--mem-fraction-static 0.78 \ +--max-running-requests 352 \ --disable-radix-cache --chunked-prefill-size -1 --max-prefill-tokens 1500 \ --moe-a2a-backend deepep --deepep-mode auto \ --enable-dp-attention --dp-size 16 --enable-dp-lm-head \ ---speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ +--speculative-algorithm NEXTN --speculative-num-steps 2 --speculative-eagle-topk 1 --speculative-num-draft-tokens 3 \ --dtype bfloat16 - ``` #### Benchmark We tested it based on the `RANDOM` dataset. -```shell Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6699 --max-concurrency 256 --random-input-len 2048 --random-output-len 2048 --num-prompts 1024 --random-range-ratio 1 +```bash Command +python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6699 --max-concurrency 352 --random-input-len 2048 --random-output-len 2048 --num-prompts 1408 --random-range-ratio 1 ``` -### DeepSeek-R1 2K-2K 50ms on A3 16 Cards Separation Mode +### DeepSeek-R1 2K-2K 50ms on A3 16 Cards Disaggregation Mode Model: Deepseek R1 Hardware: Atlas 800I A3 16Card -DeployMode: PD Separation +DeployMode: PD Disaggregation Dataset: random @@ -833,14 +1233,12 @@ TPOT: 50ms #### Model Deployment -```shell Command +```bash Command echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor sysctl -w vm.swappiness=0 sysctl -w kernel.numa_balancing=0 sysctl -w kernel.sched_migration_cost_ns=50000 -export SGLANG_SET_CPU_AFFINITY=1 - unset https_proxy unset http_proxy unset HTTPS_PROXY @@ -850,6 +1248,7 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH +export SGLANG_SET_CPU_AFFINITY=1 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export STREAMS_PER_DEVICE=32 @@ -876,7 +1275,7 @@ do if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; then echo "${P_IP[$i]}" - export HCCL_BUFFSIZE=1536 + export HCCL_BUFFSIZE=2600 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export TASK_QUEUE_ENABLE=2 @@ -884,11 +1283,11 @@ do export GLOO_SOCKET_IFNAME=lo python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \ --port 8000 --disaggregation-bootstrap-port $((8998+$i)) --trust-remote-code --nnodes 1 --node-rank 0 \ - --tp-size 16 --mem-fraction-static 0.6 --attention-backend ascend --device npu --quantization modelslim \ - --disaggregation-transfer-backend ascend --max-running-requests 8 --context-length 8192 --disable-radix-cache \ - --chunked-prefill-size 32768 --max-prefill-tokens 28680 --moe-a2a-backend deepep --deepep-mode normal \ + --tp-size 16 --mem-fraction-static 0.7 --attention-backend ascend --device npu --quantization modelslim \ + --disaggregation-transfer-backend ascend --max-running-requests 32 --context-length 8192 --disable-radix-cache \ + --chunked-prefill-size -1 --max-prefill-tokens 10240 --moe-a2a-backend deepep --deepep-mode normal \ --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ - --dp-size 2 --enable-dp-attention --disable-shared-experts-fusion --dtype bfloat16 + --dp-size 8 --enable-dp-attention --disable-shared-experts-fusion --dtype bfloat16 NODE_RANK=$i break fi @@ -902,19 +1301,20 @@ do echo "${D_IP[$i]}" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_SPEC_V2=1 - export HCCL_BUFFSIZE=720 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=96 + export HCCL_BUFFSIZE=900 + export SGLANG_DP_ROUND_ROBIN=1 + export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=112 export TASK_QUEUE_ENABLE=1 export HCCL_SOCKET_IFNAME=xxx export GLOO_SOCKET_IFNAME=xxx python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ --port 8001 --trust-remote-code --nnodes 1 --node-rank 0 --tp-size 16 --dp-size 16 \ - --mem-fraction-static 0.8 --max-running-requests 384 --attention-backend ascend --device npu --quantization modelslim \ + --mem-fraction-static 0.8 --max-running-requests 448 --attention-backend ascend --device npu --quantization modelslim \ --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head \ - --cuda-graph-bs 8 10 12 14 16 18 20 22 24 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ + --cuda-graph-bs 2 4 6 8 10 12 14 16 18 20 22 24 26 28 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --prefill-round-robin-balance --disable-shared-experts-fusion --dtype bfloat16 --tokenizer-worker-num 4 \ - --load-balance-method decode_round_robin + --disable-shared-experts-fusion --dtype bfloat16 --tokenizer-worker-num 4 \ + --load-balance-method round_robin NODE_RANK=$i break fi @@ -922,8 +1322,7 @@ done ``` -```shell Command -export SGLANG_DP_ROUND_ROBIN=1 +```bash Command python -m sglang_router.launch_router \ --pd-disaggregation \ --policy cache_aware \ @@ -938,8 +1337,8 @@ python -m sglang_router.launch_router \ We tested it based on the `RANDOM` dataset. -```shell Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 400 --random-input-len 2048 --random-output-len 2048 --num-prompts 3200 --random-range-ratio 1 --request-rate 8 +```bash Command +python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 448 --random-input-len 2048 --random-output-len 2048 --num-prompts 1792 --random-range-ratio 1 --request-rate 32 ``` ### DeepSeek-R1 3_5K-1_5K 50ms on A3 8 Cards Mixed Mode @@ -958,14 +1357,12 @@ TPOT: 50ms #### Model Deployment -```shell Command +```bash Command echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor sysctl -w vm.swappiness=0 sysctl -w kernel.numa_balancing=0 sysctl -w kernel.sched_migration_cost_ns=50000 -export SGLANG_SET_CPU_AFFINITY=1 - unset https_proxy unset http_proxy unset HTTPS_PROXY @@ -981,14 +1378,16 @@ export STREAMS_PER_DEVICE=32 export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=36 -export HCCL_BUFFSIZE=1600 +export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200 +export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=56 +export HCCL_BUFFSIZE=1200 +export DEEPEP_NORMAL_LONG_SEQ_ROUND=10 +export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=512 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export SGLANG_NPU_USE_MLAPO=1 export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_USE_FIA_NZ=1 -export ENABLE_MOE_NZ=1 MODEL_PATH=xxx @@ -1000,32 +1399,31 @@ python3 -m sglang.launch_server --model-path ${MODEL_PATH} \ --quantization modelslim \ --watchdog-timeout 9000 \ --host 127.0.0.1 --port 6699 \ ---cuda-graph-bs 8 16 24 28 32 36 \ ---mem-fraction-static 0.71 \ ---max-running-requests 144 \ ---context-length 8188 --disable-radix-cache --chunked-prefill-size -1 --max-prefill-tokens 9000 \ +--cuda-graph-bs 4 8 12 14 \ +--mem-fraction-static 0.77 \ +--max-running-requests 224 \ +--context-length 8188 --disable-radix-cache --chunked-prefill-size -1 --max-prefill-tokens 3000 \ --moe-a2a-backend deepep --deepep-mode auto \ ---enable-dp-attention --dp-size 4 --enable-dp-lm-head \ +--enable-dp-attention --dp-size 16 --enable-dp-lm-head \ --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ --dtype bfloat16 - ``` #### Benchmark We tested it based on the `RANDOM` dataset. -```shell Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6699 --max-concurrency 144 --random-input-len 3500 --random-output-len 1500 --num-prompts 576 --random-range-ratio 1 +```bash Command +python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6699 --max-concurrency 224 --random-input-len 3500 --random-output-len 1500 --num-prompts 896 --random-range-ratio 1 ``` -### DeepSeek-R1 3_5K-1_5K 50ms on A3 16 Cards Separation Mode +### DeepSeek-R1 3_5K-1_5K 50ms on A3 16 Cards Disaggregation Mode Model: Deepseek R1 Hardware: Atlas 800I A3 16Card -DeployMode: PD Separation +DeployMode: PD Disaggregation Dataset: random @@ -1035,13 +1433,12 @@ TPOT: 50ms #### Model Deployment -```shell Command +```bash Command echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor sysctl -w vm.swappiness=0 sysctl -w kernel.numa_balancing=0 sysctl -w kernel.sched_migration_cost_ns=50000 -export SGLANG_SET_CPU_AFFINITY=1 unset https_proxy unset http_proxy unset HTTPS_PROXY @@ -1051,6 +1448,7 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH +export SGLANG_SET_CPU_AFFINITY=1 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export STREAMS_PER_DEVICE=32 @@ -1077,7 +1475,7 @@ do if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; then echo "${P_IP[$i]}" - export HCCL_BUFFSIZE=1536 + export HCCL_BUFFSIZE=3500 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export TASK_QUEUE_ENABLE=2 @@ -1085,11 +1483,11 @@ do export GLOO_SOCKET_IFNAME=lo python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \ --port 8000 --disaggregation-bootstrap-port $((8998+$i)) --trust-remote-code --nnodes 1 --node-rank 0 \ - --tp-size 16 --mem-fraction-static 0.6 --attention-backend ascend --device npu --quantization modelslim \ - --disaggregation-transfer-backend ascend --max-running-requests 8 --context-length 8192 --disable-radix-cache \ - --chunked-prefill-size -1 --max-prefill-tokens 28680 --moe-a2a-backend deepep --deepep-mode normal \ + --tp-size 16 --mem-fraction-static 0.62 --attention-backend ascend --device npu --quantization modelslim \ + --disaggregation-transfer-backend ascend --max-running-requests 32 --context-length 8192 --disable-radix-cache \ + --chunked-prefill-size -1 --max-prefill-tokens 20480 --moe-a2a-backend deepep --deepep-mode normal \ --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ - --dp-size 2 --enable-dp-attention --disable-shared-experts-fusion --dtype bfloat16 + --dp-size 8 --enable-dp-attention --disable-shared-experts-fusion --dtype bfloat16 NODE_RANK=$i break fi @@ -1103,19 +1501,19 @@ do echo "${D_IP[$i]}" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_SPEC_V2=1 - export HCCL_BUFFSIZE=720 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=96 + export HCCL_BUFFSIZE=800 + export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=78 export TASK_QUEUE_ENABLE=1 export HCCL_SOCKET_IFNAME=xxx export GLOO_SOCKET_IFNAME=xxx python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ --port 8001 --trust-remote-code --nnodes 1 --node-rank 0 --tp-size 16 --dp-size 16 \ - --mem-fraction-static 0.8 --max-running-requests 384 --attention-backend ascend --device npu --quantization modelslim \ + --mem-fraction-static 0.805 --max-running-requests 416 --attention-backend ascend --device npu --quantization modelslim \ --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head \ - --cuda-graph-bs 8 10 12 14 16 18 20 22 24 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ - --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --prefill-round-robin-balance --disable-shared-experts-fusion --dtype bfloat16 --tokenizer-worker-num 4 \ - --load-balance-method decode_round_robin + --cuda-graph-bs 2 4 6 8 10 12 14 16 18 20 22 24 26 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ + --speculative-algorithm NEXTN --speculative-num-steps 2 --speculative-eagle-topk 1 --speculative-num-draft-tokens 3 \ + --disable-shared-experts-fusion --dtype bfloat16 --tokenizer-worker-num 4 \ + --load-balance-method round_robin NODE_RANK=$i break fi @@ -1123,8 +1521,7 @@ done ``` -```shell Command -export SGLANG_DP_ROUND_ROBIN=1 +```bash Command python -m sglang_router.launch_router \ --pd-disaggregation \ --policy cache_aware \ @@ -1139,35 +1536,32 @@ python -m sglang_router.launch_router \ We tested it based on the `RANDOM` dataset. -```shell Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 384 --random-input-len 3500 --random-output-len 1500 --num-prompts 1536 --random-range-ratio 1 +```bash Command +python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 416 --random-input-len 3500 --random-output-len 1500 --num-prompts 1664 --random-range-ratio 1 ``` -### DeepSeek-V3.2-Exp 64K-3K 30ms on A3 32 Cards Separation Mode +### DeepSeek-V3.2 128K-1K 26ms on A3 32 Cards Disaggregation Mode -Model: DeepSeek-V3.2-Exp-W8A8 +Model: DeepSeek-V3.2-W8A8 Hardware: Atlas 800I A3 32Card -DeployMode: PD Separation +DeployMode: PD Disaggregation Dataset: random -Input Output Length: 64K+3K +Input Output Length: 128K+1K -TPOT: 30ms +TPOT: 26ms #### Model Deployment -Deploy Prefill Instance - -```shell Command +```bash Command echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor sysctl -w vm.swappiness=0 sysctl -w kernel.numa_balancing=0 sysctl -w kernel.sched_migration_cost_ns=50000 -export SGLANG_SET_CPU_AFFINITY=1 unset https_proxy unset http_proxy unset HTTPS_PROXY @@ -1179,168 +1573,116 @@ source /usr/local/Ascend/nnal/atb/set_env.sh export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/op_api/lib/:${LD_LIBRARY_PATH} export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH -export ASCEND_HOME_PATH=/usr/local/Ascend/ascend-toolkit/latest - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 - -export HCCL_BUFFSIZE=1024 -export DEEPEP_NORMAL_LONG_SEQ_ROUND=5 -export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=512 - -MODEL_PATH=xxx - -export SGLANG_NPU_USE_MLAPO=1 -export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 -export SGLANG_NPU_USE_MULTI_STREAM=1 -export HCCL_OP_EXPANSION_MODE=AIV - -IPs=('your prefill ip1' 'your prefill ip2') - -# get IP in current node -LOCAL_HOST=`hostname -I|awk -F " " '{print$1}'` -echo "LOCAL_HOST = " ${LOCAL_HOST} -# get node index -for i in "${!IPs[@]}"; -do - echo "LOCAL_HOST=${LOCAL_HOST}, IPs[${i}]=${IPs[$i]}" - if [ "$LOCAL_HOST" == "${IPs[$i]}" ]; then - echo "Node Rank : ${i}" - VC_TASK_INDEX=$i - break - fi -done - -IFNAMES=('xxx' 'xxx') - -export HCCL_SOCKET_IFNAME=${IFNAMES[$VC_TASK_INDEX]} -export GLOO_SOCKET_IFNAME=${HCCL_SOCKET_IFNAME} -echo "HCCL_SOCKET_IFNAME : ${HCCL_SOCKET_IFNAME}" -nnodes=${#IPs[@]} -tp_size=`expr 16 \* ${nnodes}` -export ASCEND_MF_STORE_URL=tcp://${IPs[0]}:24667 - -python3 -m sglang.launch_server --model-path ${MODEL_PATH} \ ---tp $tp_size \ ---trust-remote-code \ ---attention-backend ascend \ ---device npu \ ---watchdog-timeout 9000 \ ---host ${IPs[$VC_TASK_INDEX]} --port 8000 \ ---mem-fraction-static 0.73 \ ---disable-radix-cache --chunked-prefill-size -1 --max-prefill-tokens 68000 \ ---max-running-requests 1 \ ---moe-a2a-backend deepep --deepep-mode normal \ ---quantization modelslim \ ---disaggregation-transfer-backend ascend \ ---disaggregation-mode prefill \ ---disable-cuda-graph \ ---nnodes $nnodes --node-rank $VC_TASK_INDEX \ ---disaggregation-bootstrap-port 8995 \ ---enable-nsa-prefill-context-parallel --moe-dense-tp-size 1 \ ---speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ ---dist-init-addr ${IPs[0]}:10000 -``` - -Deploy Decode Instance - -```shell Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - export SGLANG_SET_CPU_AFFINITY=1 -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/op_api/lib/:${LD_LIBRARY_PATH} -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH -export ASCEND_HOME_PATH=/usr/local/Ascend/ascend-toolkit/latest - export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export STREAMS_PER_DEVICE=32 +export ASCEND_MF_STORE_URL="tcp://your prefill ip1:24670" +P_IP=('your prefill ip1' 'your prefill ip2') +D_IP=('your decode ip1' 'your decode ip2') MODEL_PATH=xxx -export SGLANG_NPU_USE_MULTI_STREAM=1 -export SGLANG_NPU_USE_MLAPO=1 -export HCCL_OP_EXPANSION_MODE=AIV -export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 -export TASK_QUEUE_ENABLE=0 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 +LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` +LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` +echo "${LOCAL_HOST1}" +echo "${LOCAL_HOST2}" -IPs=('your decode ip1' 'your decode ip2') - -export prefill_ip=your prefill ip1 -# get IP in current node -LOCAL_HOST=`hostname -I|awk -F " " '{print$1}'` -echo "LOCAL_HOST = " ${LOCAL_HOST} -# get node index -for i in "${!IPs[@]}"; +# prefill +for i in "${!P_IP[@]}"; do - echo "LOCAL_HOST=${LOCAL_HOST}, IPs[${i}]=${IPs[$i]}" - if [ "$LOCAL_HOST" == "${IPs[$i]}" ]; then - echo "Node Rank : ${i}" - VC_TASK_INDEX=$i - break - fi + if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; + then + echo "${P_IP[$i]}" + export HCCL_BUFFSIZE=1200 + export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 + export TASK_QUEUE_ENABLE=2 + export HCCL_SOCKET_IFNAME=xxx + export GLOO_SOCKET_IFNAME=xxx + + python3 -m sglang.launch_server --model-path ${MODEL_PATH} \ + --tp 32 \ + --trust-remote-code \ + --attention-backend ascend \ + --device npu \ + --watchdog-timeout 9000 \ + --host ${P_IP[$i]} --port 8000 \ + --mem-fraction-static 0.73 \ + --disable-radix-cache --chunked-prefill-size -1 --max-prefill-tokens 68000 \ + --max-running-requests 1 \ + --moe-a2a-backend deepep --deepep-mode normal \ + --quantization modelslim \ + --disaggregation-transfer-backend ascend \ + --disaggregation-mode prefill \ + --disable-cuda-graph \ + --nnodes 2 --node-rank $i \ + --disaggregation-bootstrap-port 8995 \ + --moe-dense-tp-size 1 \ + --enable-nsa-prefill-context-parallel \ + --nsa-prefill-cp-mode in-seq-split \ + --attn-cp-size 32 \ + --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ + --dist-init-addr ${P_IP[0]}:10000 + break + fi done -IFNAMES=('xxx' 'xxx') -export HCCL_SOCKET_IFNAME=${IFNAMES[$VC_TASK_INDEX]} -export GLOO_SOCKET_IFNAME=${HCCL_SOCKET_IFNAME} -nnodes=${#IPs[@]} -tp_size=`expr 16 \* ${nnodes}` -export ASCEND_MF_STORE_URL=tcp://${prefill_ip}:24667 +# decode +for i in "${!D_IP[@]}"; +do + if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; + then + echo "${D_IP[$i]}" + export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + export SGLANG_ENABLE_SPEC_V2=1 -CHUNKED_SIZE=65536 -DP=8 -export HCCL_BUFFSIZE=400 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8 + export TASK_QUEUE_ENABLE=0 + export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 -python3 -m sglang.launch_server --model-path ${MODEL_PATH} \ ---tp $tp_size \ ---dp ${DP} \ ---ep $tp_size \ ---moe-dense-tp-size 1 \ ---enable-dp-attention \ ---enable-dp-lm-head \ ---trust-remote-code \ ---attention-backend ascend \ ---device npu \ ---watchdog-timeout 9000 \ ---host ${IPs[$VC_TASK_INDEX]} --port 8001 \ ---mem-fraction-static 0.79 \ ---disable-radix-cache \ ---chunked-prefill-size -1 --max-prefill-tokens 68000 \ ---max-running-requests 32 \ ---cuda-graph-max-bs 4 \ ---moe-a2a-backend deepep \ ---deepep-mode low_latency \ ---quantization modelslim \ ---speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ ---disaggregation-transfer-backend ascend \ ---disaggregation-mode decode \ ---prefill-round-robin-balance \ ---load-balance-method round_robin \ ---nnodes $nnodes --node-rank $VC_TASK_INDEX \ ---dist-init-addr ${IPs[0]}:10000 --load-balance-method decode_round_robin + export HCCL_SOCKET_IFNAME=xxx + export GLOO_SOCKET_IFNAME=xxx + + DP=8 + export HCCL_BUFFSIZE=400 + export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8 + + python3 -m sglang.launch_server --model-path ${MODEL_PATH} \ + --tp 32 \ + --dp ${DP} \ + --ep 32 \ + --moe-dense-tp-size 1 \ + --enable-dp-attention \ + --enable-dp-lm-head \ + --trust-remote-code \ + --attention-backend ascend \ + --device npu \ + --watchdog-timeout 9000 \ + --host ${D_IP[$i]} --port 8001 \ + --mem-fraction-static 0.79 \ + --disable-radix-cache \ + --chunked-prefill-size -1 --max-prefill-tokens 68000 \ + --max-running-requests 32 \ + --cuda-graph-max-bs 4 \ + --moe-a2a-backend deepep \ + --deepep-mode low_latency \ + --quantization modelslim \ + --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ + --disaggregation-transfer-backend ascend \ + --disaggregation-mode decode \ + --nnodes 2 --node-rank $i \ + --dist-init-addr ${D_IP[0]}:10000 + break + fi +done ``` -```shell Command -export SGLANG_DP_ROUND_ROBIN=1 + +```bash Command python -m sglang_router.launch_router \ --pd-disaggregation \ --policy cache_aware \ - --prefill http://PIP1:8000 8995 \ - --decode http://DIP1:8001 \ + --prefill http://P_IP1:8000 8995 \ + --decode http://D_IP1:8001 \ --host 127.0.0.1 \ --port 6688 \ --mini-lb @@ -1350,17 +1692,17 @@ python -m sglang_router.launch_router \ We tested it based on the `RANDOM` dataset. -```shell Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 32 --random-input-len 64000 --random-output-len 3000 --num-prompts 64 --random-range-ratio 1 +```bash Command +python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 8 --random-input-len 131076 --random-output-len 1024 --num-prompts 8 --random-range-ratio 1 ``` -### Qwen3-235B-A22B 3_5K-1_5K 50ms on A3 24 Cards Separation Mode +### Qwen3-235B-A22B 3_5K-1_5K 50ms on A3 24 Cards Disaggregation Mode Model: Qwen3-235B-A22B-W8A8 Hardware: Atlas 800I A3 24Card -DeployMode: PD Separation +DeployMode: PD Disaggregation Dataset: random @@ -1370,13 +1712,12 @@ TPOT: 50ms #### Model Deployment -```shell Command +```bash Command echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor sysctl -w vm.swappiness=0 sysctl -w kernel.numa_balancing=0 sysctl -w kernel.sched_migration_cost_ns=50000 -export SGLANG_SET_CPU_AFFINITY=1 unset https_proxy unset http_proxy unset HTTPS_PROXY @@ -1384,18 +1725,19 @@ unset HTTP_PROXY unset ASCEND_LAUNCH_BLOCKING source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16 +export SGLANG_SET_CPU_AFFINITY=1 +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_DP_ROUND_ROBIN=1 +export SGLANG_NPU_FUSED_MOE_MODE=2 MODEL_PATH=xxx export ASCEND_MF_STORE_URL="tcp://your prefill ip1:24667" P_IP=('your prefill ip1') D_IP=('your decode ip1' 'your decode ip2') -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_DP_ROUND_ROBIN=1 LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` @@ -1411,6 +1753,7 @@ do echo "${P_IP[$i]}" source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh + export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=188416 export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=1024 export DEEPEP_NORMAL_LONG_SEQ_ROUND=16 export HCCL_BUFFSIZE=4300 @@ -1429,9 +1772,9 @@ do --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ --speculative-draft-model-quantization unquant \ - --max-running-requests 128 --chunked-prefill-size 262144 --max-prefill-tokens 262144 \ + --max-running-requests 128 --chunked-prefill-size 94208 --max-prefill-tokens 262144 \ --enable-dp-attention \ - --moe-a2a-backend deepep --deepep-mode normal --dtype bfloat16 + --moe-a2a-backend ascend_fuseep --dtype bfloat16 NODE_RANK=$i break fi @@ -1445,8 +1788,9 @@ do echo "${D_IP[$i]}" source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=24 - export HCCL_BUFFSIZE=512 + export DP_ROUND_ROBIN=1 + export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=65536 + export HCCL_BUFFSIZE=800 export HCCL_SOCKET_IFNAME=data0.3001 export GLOO_SOCKET_IFNAME=data0.3001 export STREAMS_PER_DEVICE=32 @@ -1461,8 +1805,8 @@ do --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ --dist-init-addr xxx:5000 \ --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ - --prefill-round-robin-balance --enable-dp-lm-head --dtype bfloat16 --tokenizer-worker-num 4 \ - --load-balance-method decode_round_robin + --enable-dp-lm-head --dtype bfloat16 --tokenizer-worker-num 4 \ + --load-balance-method round_robin NODE_RANK=$i break fi @@ -1506,7 +1850,7 @@ TPOT: 50ms #### Model Deployment -```shell Command +```bash Command echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor sysctl -w vm.swappiness=0 sysctl -w kernel.numa_balancing=0 @@ -1535,24 +1879,27 @@ LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` echo "${LOCAL_HOST1}" echo "${LOCAL_HOST2}" -export HCCL_BUFFSIZE=1600 +export HCCL_BUFFSIZE=570 export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo export HCCL_OP_EXPANSION_MODE="AIV" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=2 +export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 +export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100 + +export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=188416 +export SGLANG_NPU_FUSED_MOE_MODE=2 python -m sglang.launch_server --model-path $MODEL_PATH \ --host 127.0.0.1 --port 7439 --trust-remote-code --nnodes 1 --node-rank 0 \ --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 272 --context-length 8192 --dtype bfloat16 \ - --chunked-prefill-size 32768 --max-prefill-tokens 32768 \ + --max-running-requests 432 --context-length 8192 --dtype bfloat16 \ + --chunked-prefill-size 94208 --max-prefill-tokens 458880 --sampling-backend ascend \ --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --disable-radix-cache --moe-a2a-backend deepep --deepep-mode auto --speculative-draft-model-quantization unquant \ - --tp 16 --dp-size 16 --enable-dp-attention --enable-dp-lm-head --mem-fraction-static 0.8 --cuda-graph-bs 3 4 6 8 10 12 13 14 15 16 17 - + --disable-radix-cache --moe-a2a-backend ascend_fuseep --speculative-draft-model-quantization unquant \ + --tp 16 --dp-size 16 --enable-dp-attention --enable-dp-lm-head --mem-fraction-static 0.8 --cuda-graph-bs 1 2 4 8 16 20 24 26 27 ``` #### Benchmark @@ -1579,13 +1926,12 @@ TPOT: 100ms #### Model Deployment -```shell Command +```bash Command echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor sysctl -w vm.swappiness=0 sysctl -w kernel.numa_balancing=0 sysctl -w kernel.sched_migration_cost_ns=50000 -export SGLANG_SET_CPU_AFFINITY=1 unset https_proxy unset http_proxy unset HTTPS_PROXY @@ -1596,12 +1942,12 @@ source /usr/local/Ascend/nnal/atb/set_env.sh source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH +export SGLANG_SET_CPU_AFFINITY=1 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 MODEL_PATH=xxx -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` @@ -1614,6 +1960,7 @@ export GLOO_SOCKET_IFNAME=lo export HCCL_OP_EXPANSION_MODE="AIV" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=144 python -m sglang.launch_server --model-path $MODEL_PATH \ --host 127.0.0.1 --port 7439 --trust-remote-code --nnodes 1 --node-rank 0 \ @@ -1623,7 +1970,7 @@ python -m sglang.launch_server --model-path $MODEL_PATH \ --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ --disable-radix-cache --moe-a2a-backend deepep --deepep-mode auto --speculative-draft-model-quantization unquant \ - --tp 16 --dp-size 16 --enable-dp-attention --enable-dp-lm-head --mem-fraction-static 0.81 --cuda-graph-bs 8 16 20 24 32 36 + --tp 16 --dp-size 16 --enable-dp-attention --enable-dp-lm-head --mem-fraction-static 0.84 --cuda-graph-bs 8 16 20 24 32 36 ``` @@ -1651,14 +1998,12 @@ TPOT: 50ms #### Model Deployment -```shell Command +```bash Command echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor sysctl -w vm.swappiness=0 sysctl -w kernel.numa_balancing=0 sysctl -w kernel.sched_migration_cost_ns=50000 -export SGLANG_SET_CPU_AFFINITY=1 - unset https_proxy unset http_proxy unset HTTPS_PROXY @@ -1669,34 +2014,38 @@ source /usr/local/Ascend/nnal/atb/set_env.sh source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH +export SGLANG_SET_CPU_AFFINITY=1 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 MODEL_PATH=xxx -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` echo "${LOCAL_HOST1}" echo "${LOCAL_HOST2}" -export HCCL_BUFFSIZE=2100 +export HCCL_BUFFSIZE=450 export HCCL_SOCKET_IFNAME=xxx export GLOO_SOCKET_IFNAME=xxx export HCCL_OP_EXPANSION_MODE="AIV" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 +export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100 +export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=147456 +export SGLANG_NPU_FUSED_MOE_MODE=2 python -m sglang.launch_server --model-path $MODEL_PATH \ --host 127.0.0.1 --port 7439 --trust-remote-code --nnodes 1 --node-rank 0 \ --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 480 --context-length 8192 --dtype bfloat16 \ - --chunked-prefill-size -1 --max-prefill-tokens 4096 --speculative-draft-model-quantization unquant \ + --max-running-requests 624 --context-length 8192 --dtype bfloat16 \ + --chunked-prefill-size 73728 --max-prefill-tokens 458880 --speculative-draft-model-quantization unquant \ --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --disable-radix-cache --moe-a2a-backend deepep --deepep-mode auto \ - --tp 16 --dp-size 16 --enable-dp-attention --enable-dp-lm-head --mem-fraction-static 0.75 --cuda-graph-bs 6 8 10 12 15 18 28 30 + --disable-radix-cache --moe-a2a-backend ascend_fuseep \ + --tp 16 --dp-size 16 --enable-dp-attention --enable-dp-lm-head --mem-fraction-static 0.83 --cuda-graph-bs 4 8 16 24 28 29 30 32 34 36 37 38 39 ``` #### Benchmark @@ -1723,14 +2072,12 @@ TPOT: 50ms #### Model Deployment -```shell Command +```bash Command echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor sysctl -w vm.swappiness=0 sysctl -w kernel.numa_balancing=0 sysctl -w kernel.sched_migration_cost_ns=50000 -export SGLANG_SET_CPU_AFFINITY=1 - unset https_proxy unset http_proxy unset HTTPS_PROXY @@ -1740,12 +2087,12 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash +export SGLANG_SET_CPU_AFFINITY=1 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 MODEL_PATH=xxx -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` @@ -2006,7 +2353,7 @@ We tested it based on the `RANDOM` dataset. python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --random-range-ratio 1 --max-concurrency 1 --random-output-len 1500 --random-input-len 4096 --num-prompts 4 ``` -### Qwen3-32B 18K-4K 12ms on A3 8 Cards Mixed Mode +### Qwen3-32B 18K-4K 6ms on A3 8 Cards Mixed Mode Model: Qwen3-32B @@ -2018,7 +2365,7 @@ Dataset: random Input Output Length: 18K+4K -TPOT: 12ms +TPOT: 6ms #### Model Deployment @@ -2231,13 +2578,12 @@ TPOT: 50ms #### Model Deployment -```shell Command +```bash Command echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor sysctl -w vm.swappiness=0 sysctl -w kernel.numa_balancing=0 sysctl -w kernel.sched_migration_cost_ns=50000 -export SGLANG_SET_CPU_AFFINITY=1 unset https_proxy unset http_proxy unset HTTPS_PROXY @@ -2250,7 +2596,10 @@ export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH MODEL_PATH=xxx +export SGLANG_SET_CPU_AFFINITY=1 +export ASCEND_LAUNCH_BLOCKING=0 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` @@ -2263,18 +2612,21 @@ export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo export HCCL_OP_EXPANSION_MODE="AIV" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 +export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200 export SGLANG_ENABLE_SPEC_V2=1 python -m sglang.launch_server --model-path $MODEL_PATH \ --host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \ --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 192 \ + --max-running-requests 162 \ --disable-radix-cache \ --speculative-draft-model-quantization unquant \ --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --chunked-prefill-size -1 --max-prefill-tokens 32768 \ - --tp-size 2 --mem-fraction-static 0.86 --cuda-graph-bs 42 88 96 132 144 156 172 178 192 --dtype bfloat16 + --chunked-prefill-size -1 --max-prefill-tokens 35000 \ + --tp-size 2 --mem-fraction-static 0.87 --cuda-graph-bs 1 5 15 40 70 100 120 130 140 146 150 154 156 158 160 162 \ + --dtype bfloat16 ``` #### Benchmark @@ -2285,13 +2637,13 @@ We tested it based on the `RANDOM` dataset. python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --max-concurrency 156 --random-input-len 3500 --random-output-len 1500 --num-prompts 624 --random-range-ratio 1 ``` -### Qwen3-Coder-480B-A35B-Instruct 3_5K-1_5K 50ms on A3 24 Cards Separation Mode +### Qwen3-Coder-480B-A35B-Instruct 3_5K-1_5K 50ms on A3 24 Cards Disaggregation Mode Model: Qwen3-Coder-480B-A35B-Instruct Hardware: Atlas 800I A3 24Card -DeployMode: PD Separation +DeployMode: PD Disaggregation Dataset: random @@ -2301,13 +2653,12 @@ TPOT: 50ms #### Model Deployment -```shell Command +```bash Command echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor sysctl -w vm.swappiness=0 sysctl -w kernel.numa_balancing=0 sysctl -w kernel.sched_migration_cost_ns=50000 -export SGLANG_SET_CPU_AFFINITY=1 unset https_proxy unset http_proxy unset HTTPS_PROXY @@ -2315,9 +2666,11 @@ unset HTTP_PROXY unset ASCEND_LAUNCH_BLOCKING source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16 +export SGLANG_SET_CPU_AFFINITY=1 +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export STREAMS_PER_DEVICE=32 +export SGLANG_NPU_FUSED_MOE_MODE=2 MODEL_PATH=xxx export ASCEND_MF_STORE_URL="tcp://PIP:24667" @@ -2339,23 +2692,22 @@ do echo "${P_IP[$i]}" source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh - export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=1024 - export DEEPEP_NORMAL_LONG_SEQ_ROUND=16 - export HCCL_BUFFSIZE=4300 + export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=327680 + export HCCL_BUFFSIZE=1550 export TASK_QUEUE_ENABLE=2 export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo - export STREAMS_PER_DEVICE=32 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill \ --host ${P_IP[$i]} --port 8000 --disaggregation-bootstrap-port 8995 --trust-remote-code \ - --nnodes 1 --node-rank $i --tp-size 16 --dp-size 2 --mem-fraction-static 0.6 \ + --nnodes 1 --node-rank $i --tp-size 16 --dp-size 2 --mem-fraction-static 0.7 \ --disable-radix-cache \ - --attention-backend ascend --device npu --quantization modelslim --disaggregation-transfer-backend ascend \ - --max-running-requests 128 --chunked-prefill-size 65536 --max-prefill-tokens 262144 \ + --attention-backend ascend --device npu --quantization modelslim --disaggregation-transfer-backend ascend \ + --max-running-requests 16 --chunked-prefill-size 20480 --max-prefill-tokens 20480 \ --enable-dp-attention \ - --moe-a2a-backend deepep --deepep-mode normal --dtype bfloat16 + --moe-a2a-backend ascend_fuseep --dtype bfloat16 \ + --disable-overlap-schedule NODE_RANK=$i break fi @@ -2368,20 +2720,20 @@ do echo "${D_IP[$i]}" source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=72 - export HCCL_BUFFSIZE=512 + export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=65536 + export HCCL_BUFFSIZE=600 + export SGLANG_NPU_FUSED_MOE_MODE=2 export HCCL_SOCKET_IFNAME=xxx export GLOO_SOCKET_IFNAME=xxx - export STREAMS_PER_DEVICE=32 python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode \ --host ${D_IP[$i]} --port 8001 --trust-remote-code \ - --nnodes 2 --node-rank $i --tp-size 32 --dp-size 4 --mem-fraction-static 0.73 --max-running-requests 384 \ + --nnodes 2 --node-rank $i --tp-size 32 --dp-size 4 --mem-fraction-static 0.75 --max-running-requests 544 \ --attention-backend ascend --device npu --quantization modelslim --enable-dp-attention \ - --moe-a2a-backend ascend_fuseep --cuda-graph-bs 16 32 48 56 64 72 80 88 96 \ + --moe-a2a-backend ascend_fuseep --cuda-graph-bs 16 32 56 72 80 88 96 104 112 120 128 136 \ --dist-init-addr DIP1:5000 \ - --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ - --prefill-round-robin-balance --enable-dp-lm-head --dtype bfloat16 --tokenizer-worker-num 4 --load-balance-method decode_round_robin + --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ + --enable-dp-lm-head --dtype bfloat16 --tokenizer-worker-num 4 --load-balance-method round_robin NODE_RANK=$i break fi @@ -2389,8 +2741,7 @@ done ``` -```shell Command -export SGLANG_DP_ROUND_ROBIN=1 +```bash Command python -m sglang_router.launch_router \ --pd-disaggregation \ --policy cache_aware \ @@ -2425,13 +2776,12 @@ TPOT: 50ms #### Model Deployment -```shell Command +```bash Command echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor sysctl -w vm.swappiness=0 sysctl -w kernel.numa_balancing=0 sysctl -w kernel.sched_migration_cost_ns=50000 -export SGLANG_SET_CPU_AFFINITY=1 unset https_proxy unset http_proxy unset HTTPS_PROXY @@ -2441,16 +2791,14 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash +export SGLANG_SET_CPU_AFFINITY=1 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16 - +export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=72 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 MODEL_PATH=xxx -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` @@ -2508,14 +2856,12 @@ TPOT: 50ms #### Model Deployment -```shell Command +```bash Command echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor sysctl -w vm.swappiness=0 sysctl -w kernel.numa_balancing=0 sysctl -w kernel.sched_migration_cost_ns=50000 -export SGLANG_SET_CPU_AFFINITY=1 - unset https_proxy unset http_proxy unset HTTPS_PROXY @@ -2526,12 +2872,12 @@ source /usr/local/Ascend/nnal/atb/set_env.sh source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH +export SGLANG_SET_CPU_AFFINITY=1 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 MODEL_PATH=xxx -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` @@ -2771,6 +3117,294 @@ We tested it based on the `RANDOM` dataset. python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7339 --random-range-ratio 1 --max-concurrency 1 --random-output-len 1500 --random-input-len 4096 --num-prompts 4 ``` +### Qwen3-32B 1K-0_3K 12ms on A3 2 Cards Mixed Mode + +Model: Qwen3-32B + +Hardware: Atlas 800I A3 2Card + +DeployMode: PD Mixed + +Dataset: random + +Input Output Length: 1K+0.3K + +TPOT: 12ms + +#### Model Deployment + +```bash Command +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +export SGLANG_SET_CPU_AFFINITY=1 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash +export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH + +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True + +MODEL_PATH=xxx + +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 + +LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` +LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` + +echo "${LOCAL_HOST1}" +echo "${LOCAL_HOST2}" + +export HCCL_SOCKET_IFNAME=lo +export GLOO_SOCKET_IFNAME=lo +export HCCL_OP_EXPANSION_MODE="AIV" +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 + +python -m sglang.launch_server --model-path $MODEL_PATH \ + --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ + --attention-backend ascend --device npu --quantization modelslim \ + --max-running-requests 16 \ + --disable-radix-cache \ + --speculative-draft-model-quantization unquant \ + --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ + --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ + --chunked-prefill-size -1 --max-prefill-tokens 16384 \ + --tp-size 4 --mem-fraction-static 0.843 --cuda-graph-bs 1 4 8 16 --dtype bfloat16 +``` + +#### Benchmark + +We tested it based on the `RANDOM` dataset. + +```bash Command +python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7339 --random-range-ratio 1 --max-concurrency 16 --random-output-len 300 --random-input-len 1024 --num-prompts 16 +``` + +### Qwen3-32B 6K-1_5K 17ms on A3 2 Cards Mixed Mode + +Model: Qwen3-32B + +Hardware: Atlas 800I A3 2Card + +DeployMode: PD Mixed + +Dataset: random + +Input Output Length: 6K+1.5K + +TPOT: 17ms + +#### Model Deployment + +```bash Command +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +export SGLANG_SET_CPU_AFFINITY=1 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash +export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH + +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True + +MODEL_PATH=xxx + +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 + +LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` +LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` + +echo "${LOCAL_HOST1}" +echo "${LOCAL_HOST2}" + +export HCCL_SOCKET_IFNAME=lo +export GLOO_SOCKET_IFNAME=lo +export HCCL_OP_EXPANSION_MODE="AIV" +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 + +python -m sglang.launch_server --model-path $MODEL_PATH \ + --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ + --attention-backend ascend --device npu --quantization modelslim \ + --max-running-requests 16 \ + --disable-radix-cache \ + --speculative-draft-model-quantization unquant \ + --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ + --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ + --chunked-prefill-size -1 --max-prefill-tokens 16384 \ + --tp-size 4 --mem-fraction-static 0.843 --cuda-graph-bs 1 4 10 15 16 --dtype bfloat16 +``` + +#### Benchmark + +We tested it based on the `RANDOM` dataset. + +```bash Command +python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7339 --random-range-ratio 1 --max-concurrency 16 --random-output-len 1500 --random-input-len 6144 --num-prompts 16 +``` + +### Qwen3-8B 1K-0_3K 7ms on A3 1 Cards Mixed Mode + +Model: Qwen3-8B + +Hardware: Atlas 800I A3 1Card + +DeployMode: PD Mixed + +Dataset: random + +Input Output Length: 1K+0.3K + +TPOT: 7ms + +#### Model Deployment + +```bash Command +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +export SGLANG_SET_CPU_AFFINITY=1 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash +export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH + +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True + +MODEL_PATH=xxx + +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 + +LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` +LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` + +echo "${LOCAL_HOST1}" +echo "${LOCAL_HOST2}" + +export HCCL_SOCKET_IFNAME=lo +export GLOO_SOCKET_IFNAME=lo +export HCCL_OP_EXPANSION_MODE="AIV" +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 + +python -m sglang.launch_server --model-path $MODEL_PATH \ + --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ + --attention-backend ascend --device npu --quantization modelslim \ + --max-running-requests 16 \ + --disable-radix-cache \ + --speculative-draft-model-quantization unquant \ + --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ + --speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 \ + --chunked-prefill-size -1 --max-prefill-tokens 16384 \ + --tp-size 2 --mem-fraction-static 0.894 --cuda-graph-bs 1 2 4 6 9 10 15 16 --dtype bfloat16 +``` + +#### Benchmark + +We tested it based on the `RANDOM` dataset. + +```bash Command +python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7339 --random-range-ratio 1 --max-concurrency 16 --random-output-len 300 --random-input-len 1024 --num-prompts 16 +``` + +### Qwen3-8B 6K-1_5K 12ms on A3 1 Cards Mixed Mode + +Model: Qwen3-8B + +Hardware: Atlas 800I A3 1Card + +DeployMode: PD Mixed + +Dataset: random + +Input Output Length: 6K+1.5K + +TPOT: 12ms + +#### Model Deployment + +```bash Command +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +export SGLANG_SET_CPU_AFFINITY=1 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash +export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH + +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True + +MODEL_PATH=xxx + +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 + +LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` +LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` + +echo "${LOCAL_HOST1}" +echo "${LOCAL_HOST2}" + +export HCCL_SOCKET_IFNAME=lo +export GLOO_SOCKET_IFNAME=lo +export HCCL_OP_EXPANSION_MODE="AIV" +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 + +python -m sglang.launch_server --model-path $MODEL_PATH \ + --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ + --attention-backend ascend --device npu --quantization modelslim \ + --max-running-requests 16 \ + --disable-radix-cache \ + --speculative-draft-model-quantization unquant \ + --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ + --speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 \ + --chunked-prefill-size -1 --max-prefill-tokens 16384 \ + --tp-size 2 --mem-fraction-static 0.894 --cuda-graph-bs 1 5 15 16 --dtype bfloat16 +``` + +#### Benchmark + +We tested it based on the `RANDOM` dataset. + +```bash Command +python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7339 --random-range-ratio 1 --max-concurrency 16 --random-output-len 1500 --random-input-len 6144 --num-prompts 16 +``` + ### Qwen3-32B 3_5K-1_5K 50ms on A2 8 Cards Mixed Mode Model: Qwen3-32B @@ -2909,3 +3543,715 @@ We tested it based on the `RANDOM` dataset. ```shell Command python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --max-concurrency 120 --random-output-len 2000 --random-input-len 2000 --num-prompts 120 --random-range-ratio 1 ``` + +### Qwen3-30B-A3B 6K-1_5K 10ms on A3 1 Cards Mixed Mode + +Model: Qwen3-30B-A3B + +Hardware: Atlas 800I A3 1Card + +DeployMode: PD Mixed + +Dataset: random + +Input Output Length: 6K+1.5K + +TPOT: 10ms + +#### Model Deployment + +```bash Command +export SGLANG_SET_CPU_AFFINITY=1 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING + +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True + +MODEL_PATH=xxx + +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 + +LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` +LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` + +echo "${LOCAL_HOST1}" +echo "${LOCAL_HOST2}" + +export HCCL_BUFFSIZE=400 +export HCCL_SOCKET_IFNAME=lo +export GLOO_SOCKET_IFNAME=lo +export HCCL_OP_EXPANSION_MODE="AIV" +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 + +python -m sglang.launch_server --model-path $MODEL_PATH \ + --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ + --attention-backend ascend --device npu --quantization modelslim \ + --max-running-requests 16 \ + --disable-radix-cache \ + --speculative-draft-model-quantization unquant \ + --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ + --speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 \ + --chunked-prefill-size -1 --max-prefill-tokens 35000 \ + --tp-size 2 --mem-fraction-static 0.6 --cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 --dtype bfloat16 +``` + +#### Benchmark + +We tested it based on the `RANDOM` dataset. + +```bash Command +python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7339 --random-range-ratio 1 --max-concurrency 16 --random-output-len 1500 --random-input-len 6144 --num-prompts 16 +``` + +### Qwen3-30B-A3B 1K-0_3K 7ms on A3 1 Cards Mixed Mode + +Model: Qwen3-30B-A3B + +Hardware: Atlas 800I A3 1Card + +DeployMode: PD Mixed + +Dataset: random + +Input Output Length: 1K+0.3K + +TPOT: 7ms + +#### Model Deployment + +```bash Command +export SGLANG_SET_CPU_AFFINITY=1 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING + +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True + +MODEL_PATH=xxx + +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 + +LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` +LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` + +echo "${LOCAL_HOST1}" +echo "${LOCAL_HOST2}" + +export HCCL_BUFFSIZE=400 +export HCCL_SOCKET_IFNAME=lo +export GLOO_SOCKET_IFNAME=lo +export HCCL_OP_EXPANSION_MODE="AIV" +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 + +python -m sglang.launch_server --model-path $MODEL_PATH \ + --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ + --attention-backend ascend --device npu --quantization modelslim \ + --max-running-requests 8 \ + --disable-radix-cache \ + --speculative-draft-model-quantization unquant \ + --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ + --speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 \ + --chunked-prefill-size -1 --max-prefill-tokens 35000 \ + --tp-size 2 --mem-fraction-static 0.7 --cuda-graph-bs 1 2 3 4 5 6 7 8 --dtype bfloat16 +``` + +#### Benchmark + +We tested it based on the `RANDOM` dataset. + +```bash Command +python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7339 --random-range-ratio 1 --max-concurrency 8 --random-output-len 300 --random-input-len 1024 --num-prompts 8 +``` + +### Qwen3-Next 1K-0_3K 14_21ms on A3 2 Cards Mixed Mode + +Model: Qwen3-Next-80B-A3B-Instruct + +Hardware: Atlas 800I A3 2Card + +DeployMode: PD Mixed + +Dataset: random + +Input Output Length: 1K+0.3K + +TPOT: 14.21ms + +#### Model Deployment + +```bash Command +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +export SGLANG_SET_CPU_AFFINITY=1 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash +export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH + +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export STREAMS_PER_DEVICE=32 +export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 +export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=330 +export DEEPEP_NORMAL_LONG_SEQ_ROUND=5 +export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3000 +export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1 + +export ASCEND_USE_FIA=1 +export SGLANG_NPU_USE_MULTI_STREAM=1 + +export SGLANG_WARMUP_TIMEOUT=3600 +export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export FORCE_DRAFT_MODEL_NON_QUANT=1 + +MODEL_PATH=xxx + +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 + +LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` +LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` + +echo "${LOCAL_HOST1}" +echo "${LOCAL_HOST2}" + +export HCCL_BUFFSIZE=2000 +export HCCL_SOCKET_IFNAME=lo +export GLOO_SOCKET_IFNAME=lo +export HCCL_OP_EXPANSION_MODE="AIV" +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 + +python3 -m sglang.launch_server --model-path ${MODEL_PATH} \ + --page-size 128 \ + --tp-size 4 \ + --trust-remote-code \ + --attention-backend ascend \ + --device npu \ + --watchdog-timeout 9000 \ + --host 127.0.0.1 --port 6699 \ + --mem-fraction-static 0.75 \ + --disable-radix-cache --max-prefill-tokens 14080 --context-length 26384 \ + --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --speculative-draft-model-quantization unquant \ + --chunked-prefill-size -1 --max-running-requests 312 \ + --cuda-graph-bs 2 4 16 32 48 64 80 96 128 140 156 \ + --mamba-ssm-dtype bfloat16 \ + --base-gpu-id 0 \ + --speculative-draft-model-path /home/weights/Qwen3-Next-80B-A3B-Instruct \ + --quantization modelslim \ + --moe-a2a-backend deepep --deepep-mode auto \ +``` + +#### Benchmark + +We tested it based on the `RANDOM` dataset. + +```bash Command +python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6699 --random-range-ratio 1 --max-concurrency 16 --random-output-len 300 --random-input-len 1024 --num-prompts 16 +``` + +### Qwen3-Next 6K-1_5K 15_62ms on A3 2 Cards Mixed Mode + +Model: Qwen3-Next-80B-A3B-Instruct + +Hardware: Atlas 800I A3 2Card + +DeployMode: PD Mixed + +Dataset: random + +Input Output Length: 6K+1.5K + +TPOT: 15.62ms + +#### Model Deployment + +```bash Command +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +export SGLANG_SET_CPU_AFFINITY=1 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash +export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH + +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export STREAMS_PER_DEVICE=32 +export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 +export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=330 +export DEEPEP_NORMAL_LONG_SEQ_ROUND=5 +export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3000 +export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1 + +export ASCEND_USE_FIA=1 +export SGLANG_NPU_USE_MULTI_STREAM=1 + +export SGLANG_WARMUP_TIMEOUT=3600 +export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export FORCE_DRAFT_MODEL_NON_QUANT=1 + +MODEL_PATH=xxx + +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 + +LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` +LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` + +echo "${LOCAL_HOST1}" +echo "${LOCAL_HOST2}" + +export HCCL_BUFFSIZE=2000 +export HCCL_SOCKET_IFNAME=lo +export GLOO_SOCKET_IFNAME=lo +export HCCL_OP_EXPANSION_MODE="AIV" +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 + +python3 -m sglang.launch_server --model-path ${MODEL_PATH} \ + --page-size 128 \ + --tp-size 4 \ + --trust-remote-code \ + --attention-backend ascend \ + --device npu \ + --watchdog-timeout 9000 \ + --host 127.0.0.1 --port 6699 \ + --mem-fraction-static 0.75 \ + --disable-radix-cache --max-prefill-tokens 14080 --context-length 26384 \ + --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --speculative-draft-model-quantization unquant \ + --chunked-prefill-size -1 --max-running-requests 312 \ + --cuda-graph-bs 2 4 16 32 48 64 80 96 128 140 156 \ + --mamba-ssm-dtype bfloat16 \ + --base-gpu-id 0 \ + --speculative-draft-model-path /home/weights/Qwen3-Next-80B-A3B-Instruct \ + --quantization modelslim \ + --moe-a2a-backend deepep --deepep-mode auto \ +``` + +#### Benchmark + +We tested it based on the `RANDOM` dataset. + +```bash Command +python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6699 --random-range-ratio 1 --max-concurrency 16 --random-output-len 1500 --random-input-len 6144 --num-prompts 16 +``` + +### Qwen3-14B 3_5K-1_5K 9ms on A3 1 Cards Mixed Mode + +Model: Qwen3-14B + +Hardware: Atlas 800I A3 1Card + +DeployMode: PD Mixed + +Dataset: random + +Input Output Length: 3.5K+1.5K + +TPOT: 9ms + +#### Model Deployment + +```bash Command +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash +export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH + +MODEL_PATH=xxx + +export SGLANG_SET_CPU_AFFINITY=1 +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export HCCL_OP_EXPANSION_MODE="AIV" +export STREAMS_PER_DEVICE=32 +export HCCL_SOCKET_IFNAME=lo +export GLOO_SOCKET_IFNAME=lo +export ASCEND_USE_FIA=0 +export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + +LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` +LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` + +echo "${LOCAL_HOST1}" +echo "${LOCAL_HOST2}" + +python -m sglang.launch_server --model-path $MODEL_PATH \ + --host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \ + --attention-backend ascend --device npu --quantization modelslim \ + --disable-radix-cache --mem-fraction-static 0.8 \ + --tp-size 1 --dp-size 1 \ + --sampling-backend ascend --max-running-requests 8 \ + --served-model-name Qwen3-14B \ + --chunked-prefill-size -1 \ + --cuda-graph-bs 8 \ + --dtype bfloat16 \ + --speculative-draft-model-quantization unquant \ + --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ + --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ + --schedule-conservativeness 0.01 +``` + +#### Benchmark + +We tested it based on the `RANDOM` dataset. + +```bash Command +python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --max-concurrency 1 --random-output-len 1500 --random-input-len 3500 --num-prompts 8 --random-range-ratio 1 +``` + +### Qwen3-14B 3_5K-1_5K 50ms on A3 1 Cards Mixed Mode + +Model: Qwen3-14B + +Hardware: Atlas 800I A3 1Card + +DeployMode: PD Mixed + +Dataset: random + +Input Output Length: 3.5K+1.5K + +TPOT: 50ms + +#### Model Deployment + +```bash Command +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash +export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH + +MODEL_PATH=xxx + +export SGLANG_SET_CPU_AFFINITY=1 +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export HCCL_OP_EXPANSION_MODE="AIV" +export STREAMS_PER_DEVICE=32 +export HCCL_SOCKET_IFNAME=lo +export GLOO_SOCKET_IFNAME=lo +export ASCEND_USE_FIA=0 +export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 +export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200 + +LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` +LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` + +echo "${LOCAL_HOST1}" +echo "${LOCAL_HOST2}" + +python -m sglang.launch_server --model-path $MODEL_PATH \ + --host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \ + --attention-backend ascend --device npu --quantization modelslim \ + --disable-radix-cache --mem-fraction-static 0.89 \ + --tp-size 1 --dp-size 2 \ + --sampling-backend ascend --max-running-requests 144 \ + --max-prefill-tokens 12288 \ + --served-model-name Qwen3-14B \ + --chunked-prefill-size -1 \ + --cuda-graph-bs 8 16 32 44 48 50 52 \ + --dtype bfloat16 \ + --speculative-draft-model-quantization unquant \ + --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ + --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ + --schedule-conservativeness 0.01 +``` + +#### Benchmark + +We tested it based on the `RANDOM` dataset. + +```bash Command +python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --max-concurrency 144 --random-output-len 1500 --random-input-len 3500 --num-prompts 576 --random-range-ratio 1 +``` + +### Qwen3-8B 3_5K-1_5K 50ms on A3 1 Cards Mixed Mode + +Model: Qwen3-8B + +Hardware: Atlas 800I A3 1Card + +DeployMode: PD Mixed + +Dataset: random + +Input Output Length: 3.5K+1.5K + +TPOT: 50ms + +#### Model Deployment + +```bash Command +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash +export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH + +MODEL_PATH=xxx + +export SGLANG_SET_CPU_AFFINITY=1 +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export HCCL_OP_EXPANSION_MODE="AIV" +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 +export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=50 +export HCCL_SOCKET_IFNAME=lo +export GLOO_SOCKET_IFNAME=lo + +LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` +LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` + +echo "${LOCAL_HOST1}" +echo "${LOCAL_HOST2}" + +python -m sglang.launch_server --model-path $MODEL_PATH \ + --host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \ + --attention-backend ascend --device npu --quantization modelslim \ + --disable-radix-cache --mem-fraction-static 0.9 \ + --tp-size 1 \ + --max-running-requests 70 \ + --max-prefill-tokens 16384 \ + --served-model-name Qwen3-8B \ + --chunked-prefill-size 16384 \ + --cuda-graph-bs 8 12 24 36 48 51 55 60 63 64 66 68 70 \ + --dtype bfloat16 \ + --speculative-draft-model-quantization unquant \ + --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ + --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 +``` + +#### Benchmark + +We tested it based on the `RANDOM` dataset. + +```bash Command +python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --max-concurrency 64 --random-output-len 1500 --random-input-len 3500 --num-prompts 256 --random-range-ratio 1 +``` + +### Qwen3-8B 3_5K-1_5K 5ms on A3 1 Cards Mixed Mode + +Model: Qwen3-8B + +Hardware: Atlas 800I A3 1Card + +DeployMode: PD Mixed + +Dataset: random + +Input Output Length: 3.5K+1.5K + +TPOT: 5ms + +#### Model Deployment + +```bash Command +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash +export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH + +MODEL_PATH=xxx + +export SGLANG_SET_CPU_AFFINITY=1 +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export HCCL_OP_EXPANSION_MODE="AIV" +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 +export HCCL_SOCKET_IFNAME=lo +export GLOO_SOCKET_IFNAME=lo + +LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` +LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` + +echo "${LOCAL_HOST1}" +echo "${LOCAL_HOST2}" + +python -m sglang.launch_server --model-path $MODEL_PATH \ + --host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \ + --attention-backend ascend --device npu --quantization modelslim \ + --disable-radix-cache --mem-fraction-static 0.894 \ + --tp-size 2 \ + --max-running-requests 1 \ + --max-prefill-tokens 16384 \ + --served-model-name Qwen3-8B \ + --chunked-prefill-size -1 \ + --cuda-graph-bs 1 \ + --dtype bfloat16 \ + --speculative-draft-model-quantization unquant \ + --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ + --speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 +``` + +#### Benchmark + +We tested it based on the `RANDOM` dataset. + +```bash Command +python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --max-concurrency 1 --random-output-len 1500 --random-input-len 3500 --num-prompts 4 --random-range-ratio 1 +``` + +### Qwen3-Next 3_5K-1_5K 20ms on A3 2 Cards Mixed Mode + +Model: Qwen3-Next-80B-A3B-Instruct + +Hardware: Atlas 800I A3 2Card + +DeployMode: PD Mixed + +Dataset: random + +Input Output Length: 3.5K+1.5K + +TPOT: 20ms + +#### Model Deployment + +```bash Command +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash +export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH + +export SGLANG_SET_CPU_AFFINITY=1 +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export STREAMS_PER_DEVICE=32 +export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 +export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=400 +export DEEPEP_NORMAL_LONG_SEQ_ROUND=10 +export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=2048 +export HCCL_OP_EXPANSION_MODE="AIV" +export TASK_QUEUE_ENABLE=1 +export ASCEND_USE_FIA=1 +export SGLANG_NPU_USE_MULTI_STREAM=0 +export SGLANG_WARMUP_TIMEOUT=3600 +export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export FORCE_DRAFT_MODEL_NON_QUANT=1 +export HCCL_BUFFSIZE=2000 +export ZBCCL_LOCAL_MEM_SIZE=60416 +export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 + +export ZBCCL_BOOTSTRAP_URL=tcp://127.0.0.1:24669 +export ZBCCL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True +export ZBCCL_ENABLE_GRAPH=1 + +export HCCL_SOCKET_IFNAME=lo +export GLOO_SOCKET_IFNAME=lo + +MODEL_PATH=xxx + +LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` +LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` + +echo "${LOCAL_HOST1}" +echo "${LOCAL_HOST2}" + +python3 -m sglang.launch_server --model-path ${MODEL_PATH} \ + --page-size 128 \ + --tp-size 4 --dp-size 2 \ + --trust-remote-code \ + --attention-backend ascend \ + --device npu \ + --quantization modelslim \ + --watchdog-timeout 9000 \ + --host 127.0.0.1 --port 6699 \ + --mem-fraction-static 0.85 \ + --disable-radix-cache --max-prefill-tokens 28672 --context-length 26384 --max-total-tokens 122304 \ + --enable-dp-attention --enable-dp-lm-head \ + --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --speculative-draft-model-quantization unquant \ + --chunked-prefill-size -1 --max-running-requests 16 \ + --cuda-graph-bs 2 4 8 \ + --mamba-ssm-dtype bfloat16 \ + --speculative-draft-model-path /path/to/Qwen3-Next-80B-A3B-Instruct +``` + +#### Benchmark + +We tested it based on the `RANDOM` dataset. + +```bash Command +python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6699 --random-range-ratio 1 --max-concurrency 1 --random-output-len 1500 --random-input-len 3500 --num-prompts 1 +``` diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_deepseek_example.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_deepseek_example.mdx new file mode 100644 index 000000000..6bf89d21f --- /dev/null +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_deepseek_example.mdx @@ -0,0 +1,301 @@ +--- +title: "DeepSeek Examples" +metatags: + description: "Examples for running DeepSeek models on Ascend NPUs, including PD mixed mode, PD disaggregation, and SGLang Model Gateway." +--- + +## Running DeepSeek-V3 + +### Running DeepSeek in PD mixed mode on 1 x Atlas 800I A3. + +W4A8 Model weights could be found [here](https://modelers.cn/models/Modelers_Park/DeepSeek-R1-0528-w4a8). + +```shell Launch Server +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export STREAMS_PER_DEVICE=32 + +#Deepep communication settings +export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 +export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 +export HCCL_BUFFSIZE=1600 + +#spec overlap +export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + +#npu acceleration operator +export SGLANG_NPU_USE_MLAPO=1 +export SGLANG_USE_FIA_NZ=1 + +python3 -m sglang.launch_server \ + --model-path ${MODEL_PATH} \ + --tp 16 \ + --trust-remote-code \ + --attention-backend ascend \ + --device npu \ + --quantization modelslim \ + --watchdog-timeout 9000 \ + --cuda-graph-bs 8 16 24 28 32 \ + --mem-fraction-static 0.68 \ + --max-running-requests 128 \ + --context-length 8188 \ + --disable-radix-cache \ + --chunked-prefill-size -1 \ + --max-prefill-tokens 16384 \ + --moe-a2a-backend deepep \ + --deepep-mode auto \ + --enable-dp-attention \ + --dp-size 4 \ + --enable-dp-lm-head \ + --speculative-algorithm NEXTN \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 4 \ + --dtype bfloat16 +``` + +### Running DeepSeek with PD disaggregation mode on 2 x Atlas 800I A3. + +W4A8 Model weights could be found [here](https://modelers.cn/models/Modelers_Park/DeepSeek-R1-0528-w4a8). + +1. Prefill: + +```bash Command +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export STREAMS_PER_DEVICE=32 + +#memfabric config store +export ASCEND_MF_STORE_URL="tcp://:" + +#Deepep communication settings +export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 +export HCCL_BUFFSIZE=1536 + +#npu acceleration operator +export SGLANG_NPU_USE_MLAPO=1 +export SGLANG_USE_FIA_NZ=1 +export TASK_QUEUE_ENABLE=2 + +python -m sglang.launch_server \ + --model-path ${MODEL_PATH} \ + --host $PREFILL_HOST_IP \ + --port 8000 \ + --disaggregation-mode prefill \ + --disaggregation-bootstrap-port 8996 \ + --disaggregation-transfer-backend ascend \ + --trust-remote-code \ + --nnodes 1 \ + --node-rank 0 \ + --tp-size 16 \ + --mem-fraction-static 0.6 \ + --attention-backend ascend \ + --device npu \ + --quantization modelslim \ + --load-balance-method round_robin \ + --max-running-requests 8 \ + --context-length 8192 \ + --disable-radix-cache \ + --chunked-prefill-size -1 \ + --max-prefill-tokens 28680 \ + --moe-a2a-backend deepep \ + --deepep-mode normal \ + --speculative-algorithm NEXTN \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 4 \ + --dp-size 2 \ + --enable-dp-attention \ + --disable-shared-experts-fusion \ + --dtype bfloat16 +``` + +2. Decode: + +```bash Command +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export STREAMS_PER_DEVICE=32 + +#memfabric config store +export ASCEND_MF_STORE_URL="tcp://:" + +#Deepep communication settings +export HCCL_BUFFSIZE=720 +export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=88 + +#spec overlap +export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + +#npu acceleration operator +unset TASK_QUEUE_ENABLE +export SGLANG_NPU_USE_MLAPO=1 +export SGLANG_USE_FIA_NZ=1 + +# suggest max-running-requests <= max-cuda-graph-bs * dp_size, Because when this value is exceeded, performance will significantly degrade. +python -m sglang.launch_server \ + --model-path ${MODEL_PATH} \ + --disaggregation-mode decode \ + --host $DECODE_HOST_IP \ + --port 8001 \ + --trust-remote-code \ + --nnodes 1 \ + --node-rank 0 \ + --tp-size 16 \ + --dp-size 16 \ + --mem-fraction-static 0.8 \ + --max-running-requests 352 \ + --attention-backend ascend \ + --device npu \ + --quantization modelslim \ + --moe-a2a-backend deepep \ + --enable-dp-attention \ + --deepep-mode low_latency \ + --enable-dp-lm-head \ + --cuda-graph-bs 8 10 12 14 16 18 20 22 \ + --disaggregation-transfer-backend ascend \ + --watchdog-timeout 9000 \ + --context-length 8192 \ + --speculative-algorithm NEXTN \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 4 \ + --disable-shared-experts-fusion \ + --dtype bfloat16 \ + --tokenizer-worker-num 4 +``` + +3. SGLang Model Gateway (former Router) + +```bash Command +python -m sglang_router.launch_router \ + --pd-disaggregation \ + --policy cache_aware \ + --prefill http://:8000 8996 \ + --decode http://:8001 \ + --host 127.0.0.1 \ + --port 6688 +``` + +### Running DeepSeek with PD disaggregation on 4 x Atlas 800I A3. + +W8A8 Model weights could be found [here](https://modelers.cn/models/State_Cloud/Deepseek-R1-bf16-hfd-w8a8). + +1. Prefill & Decode: + +```bash Command +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 +export SGLANG_SET_CPU_AFFINITY=1 +unset ASCEND_LAUNCH_BLOCKING +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH + +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export STREAMS_PER_DEVICE=32 + +export ASCEND_MF_STORE_URL="tcp://your prefill ip1:24669" + +P_IP=('your prefill ip1' 'your prefill ip2') + +D_IP=('your decode ip1' 'your decode ip2') + +MODEL_PATH=xxx + +export SGLANG_NPU_USE_MLAPO=1 +export SGLANG_USE_FIA_NZ=1 + +LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` +LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` +echo "${LOCAL_HOST1}" +echo "${LOCAL_HOST2}" +# prefill +for i in "${!P_IP[@]}"; +do + if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; + then + echo "${P_IP[$i]}" + export HCCL_BUFFSIZE=1536 + export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 + export TASK_QUEUE_ENABLE=2 + + export HCCL_SOCKET_IFNAME=lo + export GLOO_SOCKET_IFNAME=lo + python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \ + --port 8000 --disaggregation-bootstrap-port $((8998+$i)) --trust-remote-code --nnodes 1 --node-rank 0 \ + --tp-size 16 --mem-fraction-static 0.81 --attention-backend ascend --device npu --quantization modelslim \ + --disaggregation-transfer-backend ascend --max-running-requests 8 --context-length 8192 --disable-radix-cache \ + --chunked-prefill-size -1 --max-prefill-tokens 28680 --moe-a2a-backend deepep --deepep-mode normal \ + --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ + --dp-size 2 --enable-dp-attention --disable-shared-experts-fusion --dtype bfloat16 --enable-attn-tp-input-scattered + NODE_RANK=$i + break + fi +done + +# decode +for i in "${!D_IP[@]}"; +do + if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; + then + echo "${D_IP[$i]}" + export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + export SGLANG_ENABLE_SPEC_V2=1 + export HCCL_BUFFSIZE=650 + export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=78 + export TASK_QUEUE_ENABLE=1 + export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 + export HCCL_SOCKET_IFNAME=xxx + export GLOO_SOCKET_IFNAME=xxx + python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ + --port 8001 --trust-remote-code --dist-init-addr ${D_IP[0]}:5000 --nnodes 2 --node-rank $i --tp-size 32 --dp-size 32 \ + --mem-fraction-static 0.815 --max-running-requests 832 --attention-backend ascend --device npu --quantization modelslim \ + --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head --moe-dense-tp 1 \ + --cuda-graph-bs 12 14 16 18 20 22 24 26 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ + --speculative-algorithm NEXTN --speculative-num-steps 2 --speculative-eagle-topk 1 --speculative-num-draft-tokens 3 \ + --tokenizer-worker-num 4 --disable-shared-experts-fusion --dtype bfloat16 \ + --load-balance-method decode_round_robin + NODE_RANK=$i + break + fi +done +``` + +2. SGLang Model Gateway (former Router): + +```bash Command +python -m sglang_router.launch_router \ + --pd-disaggregation \ + --policy cache_aware \ + --prefill http://P_IP:8000 8998 \ + --prefill http://P_IP:8000 8999 \ + --decode http://D_IP:8001 \ + --host 127.0.0.1 \ + --port 6688 \ + --mini-lb +``` + +### test gsm8k + +```python Test GSM8K +from types import SimpleNamespace +from sglang.test.few_shot_gsm8k import run_eval + +def gsm8k(): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=32, + host=f"http://127.0.0.1", + port=6688, + ) + metrics = run_eval(args) + print(f"{metrics=}") + print(f"{metrics['accuracy']=}") +if __name__ == "__main__": + gsm8k() +``` diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_environment_variables.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_environment_variables.mdx new file mode 100644 index 000000000..2ea91da4c --- /dev/null +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_environment_variables.mdx @@ -0,0 +1,149 @@ +--- +title: "Environment Variables" +metatags: + description: "Reference commonly used Ascend NPU environment variables for configuring SGLang runtime behavior." +--- +SGLang supports various environment variables related to Ascend NPU that can be used to configure its runtime behavior. +This document provides a list of commonly used environment variables and aims to stay updated over time. + +## Directly Used in SGLang + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Environment VariableDescriptionDefault Value
SGLANG_NPU_USE_MLAPOAdopts the MLAPO fusion operator in attention <br/> preprocessing stage of the MLA model.false
SGLANG_USE_FIA_NZReshapes KV Cache for FIA NZ format.<br/> SGLANG_USE_FIA_NZ must be enabled with SGLANG_NPU_USE_MLAPOfalse
SGLANG_NPU_USE_MULTI_STREAMEnable dual-stream computation of shared experts <br/> and routing experts in DeepSeek models.<br/> Enable dual-stream computation in DeepSeek NSA Indexer.false
SGLANG_NPU_DISABLE_ACL_FORMAT_WEIGHTDisable cast model weight tensor to a specific NPU <br/> ACL format.false
SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANKThe maximum number of dispatched tokens on each rank.128
+ +## Used in DeepEP Ascend + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Environment VariableDescriptionDefault Value
DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENSEnable ant-moving function in dispatch stage. Indicates <br/> the number of tokens transmitted per round on each rank.8192
DEEPEP_NORMAL_LONG_SEQ_ROUNDEnable ant-moving function in dispatch stage. Indicates <br/> the number of rounds transmitted on each rank.1
DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQEnable ant-moving function in combine stage. <br/> The value 0 means disabled.0
MOE_ENABLE_TOPK_NEG_ONENeeds to be enabled when the expert ID to be processed by <br/> DEEPEP contains -1.0
DEEP_NORMAL_MODE_USE_INT8_QUANTQuantizes x to int8 and returns (tensor, scales) in dispatch operator.0
+ +## Others + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Environment VariableDescriptionDefault Value
TASK_QUEUE_ENABLEUsed to control the optimization level of the dispatch queue<br/> about the task_queue operator. Detail1
INF_NAN_MODE_ENABLEControls whether the chip uses saturation mode or INF_NAN mode. Detail1
STREAMS_PER_DEVICEConfigures the maximum number of streams for the stream pool. Detail32
PYTORCH_NPU_ALLOC_CONFControls the behavior of the cache allocator. <br/>This variable changes memory usage and may cause performance fluctuations. Detail
ASCEND_MF_STORE_URLThe address of config store in MemFabric during PD separation, <br/>which is generally set to the IP address of the P primary node<br/> with an arbitrary port number.
ASCEND_LAUNCH_BLOCKINGControls whether synchronous mode is enabled during operator execution. Detail0
HCCL_OP_EXPANSION_MODEConfigures the expansion position for communication algorithm scheduling. Detail
HCCL_BUFFSIZEControls the size of the buffer area for shared data between two NPUs. <br/>The unit is MB, and the value must be greater than or equal to 1. Detail200
HCCL_SOCKET_IFNAMEConfigures the name of the network card used by the Host <br/>during HCCL initialization. Detail
GLOO_SOCKET_IFNAMEConfigures the network interface name for GLOO communication.
diff --git a/docs_new/docs/hardware-platforms/ascend-npus/GLM-5.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_glm5_examples.mdx similarity index 91% rename from docs_new/docs/hardware-platforms/ascend-npus/GLM-5.mdx rename to docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_glm5_examples.mdx index 783cbf122..0f2fd5ce7 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/GLM-5.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_glm5_examples.mdx @@ -1,3 +1,8 @@ +--- +title: "GLM-5 examples" +metatags: + description: "Documentation for GLM-5 examples" +--- ## Introduction The GLM (General Language Model) series is an open-source bilingual large language model family jointly developed by the KEG Laboratory of Tsinghua University and Zhipu AI. This series of models has performed outstandingly in the field of Chinese NLP with its unique unified pre-training framework and bilingual capabilities. [GLM-5](https://huggingface.co/zai-org/GLM-5) adopts the DeepSeek-V3/V3.2 architecture, including the sparse attention (DSA) and multi-token prediction (MTP). Ascend supports GLM-5 with 0Day based on the SGLang inference framework, achieving low-code seamless enablement and compatibility with the mainstream distributed parallel capabilities within the current SGLang framework. We welcome developers to download and experience it. @@ -13,10 +18,9 @@ The GLM (General Language Model) series is an open-source bilingual large langua ### Installation -The dependencies required for the NPU runtime environment have been integrated into a Docker image and uploaded to the quay.io platform. You can directly pull it. +The dependencies required for the NPU runtime environment have been integrated into a Docker image and uploaded to the online platform. You can directly pull it. - -```bash Pull and Start Container +```bash Command #Atlas 800 A3 docker pull swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:cann8.5.0-a3-glm5 #Atlas 800 A2 @@ -31,7 +35,7 @@ docker run -itd --shm-size=16g --privileged=true --name ${NAME} \ -v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ -v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ --device=/dev/davinci0:/dev/davinci0 \ ---device=/dev/davinci1:/dev/avinci1 \ +--device=/dev/davinci1:/dev/davinci1 \ --device=/dev/davinci2:/dev/davinci2 \ --device=/dev/davinci3:/dev/davinci3 \ --device=/dev/davinci4:/dev/davinci4 \ @@ -51,15 +55,18 @@ docker run -itd --shm-size=16g --privileged=true --name ${NAME} \ --entrypoint=bash \ swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:${TAG} ``` - -Note: Using this image, you need to update transformers to main branch - -```shell Update Transformers +### Best Practices +Note: Using this image for **best practices**, you need to update transformers to version 5.3.0 +``` # reinstall transformers -pip install git+https://github.com/huggingface/transformers.git + +# Install transformers version 5.3.0 from PyPI +pip install transformers==5.3.0 + +# Install from GitHub v5.3.0 tag from GitHub +pip install git+https://github.com/huggingface/transformers.git@v5.3.0 ``` - ## Deployment @@ -69,7 +76,6 @@ pip install git+https://github.com/huggingface/transformers.git Run the following script to execute online inference. - ```shell Launch Server # high performance cpu echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor @@ -113,7 +119,6 @@ python3 -m sglang.launch_server \ --quantization modelslim \ --moe-a2a-backend deepep --deepep-mode auto ``` - ### Multi-node Deployment @@ -125,7 +130,6 @@ Modify the IP of 2 nodes, then run the same scripts on two nodes. **node 0/1** - ```shell Launch Multi-node Server echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor sysctl -w vm.swappiness=0 @@ -189,7 +193,6 @@ do done ``` - ### Prefill-Decode Disaggregation diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_quantization.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_quantization.mdx new file mode 100644 index 000000000..5d3c0b77b --- /dev/null +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_quantization.mdx @@ -0,0 +1,257 @@ +--- +title: "Quantization on Ascend" +metatags: + description: "Load, export, and serve quantized models on Ascend NPUs with SGLang." +--- +To load already quantized models, simply load the model weights and config. Again, if the model has been quantized offline, there's no need to add `--quantization` argument when starting the engine. The quantization method will be automatically parsed from the downloaded `quant_model_description.json` or `config.json` config. + +SGLang support **mix-bits** quantization (independently defines and loads each layer depending on the type of quantification specified in the `quant_model_description'.json`). [Advanced mix-bits for MoE](https://github.com/sgl-project/sglang/pull/17361) in progress, will add independent quantization determination for the w13 (up-gate) and w2 (down) layers. + +[ModelSlim on Ascend support](https://github.com/sgl-project/sglang/pull/14504) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Quantization schemeLayer typeA2 SupportedA3 SupportedA5 SupportedDiffusion models
W4A4 dynamicLinear<span style="color: green;">√</span><span style="color: green;">√</span><span style="color: yellow;">TBD</span><span style="color: green;">√</span>
W8A8 staticLinear<span style="color: green;">√</span><span style="color: green;">√</span><span style="color: yellow;">TBD</span><span style="color: green;">√</span>
W8A8 dynamicLinear<span style="color: green;">√</span><span style="color: green;">√</span><span style="color: yellow;">TBD</span><span style="color: green;">√</span>
MXFP8Linear<span style="color: red;">x</span><span style="color: red;">x</span><span style="color: blue;">WIP</span><span style="color: blue;">WIP</span>
W4A4 dynamicMoE<span style="color: green;">√</span><span style="color: green;">√</span><span style="color: yellow;">TBD</span><span style="color: red;">x</span>
W4A8 dynamicMoE<span style="color: green;">√</span><span style="color: green;">√</span><span style="color: yellow;">TBD</span><span style="color: red;">x</span>
W8A8 dynamicMoE<span style="color: green;">√</span><span style="color: green;">√</span><span style="color: yellow;">TBD</span><span style="color: red;">x</span>
MXFP8MoE<span style="color: red;">x</span><span style="color: red;">x</span><span style="color: blue;">WIP</span><span style="color: red;">x</span>
+ +[AWQ on Ascend support](https://github.com/sgl-project/sglang/pull/10158): + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Quantization schemeLayer typeA2 SupportedA3 SupportedA5 Supported
W4A16Linear<span style="color: green;">√</span><span style="color: green;">√</span><span style="color: yellow;">TBD</span>
W8A16Linear<span style="color: green;">√</span><span style="color: green;">√</span><span style="color: yellow;">TBD</span>
W4A16MoE<span style="color: green;">√</span><span style="color: green;">√</span><span style="color: yellow;">TBD</span>
+ +GPTQ on Ascend support + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Quantization schemeLayer typeA2 SupportedA3 SupportedA5 Supported
W4A16Linear<span style="color: green;">√</span><span style="color: green;">√</span><span style="color: yellow;">TBD</span>
W8A16Linear<span style="color: green;">√</span><span style="color: green;">√</span><span style="color: yellow;">TBD</span>
W4A16 MOEMoE<span style="color: green;">√</span><span style="color: green;">√</span><span style="color: yellow;">TBD</span>
W8A16 MOEMoE<span style="color: green;">√</span><span style="color: green;">√</span><span style="color: yellow;">TBD</span>
+ +[Auto-round on Ascend support](https://github.com/sgl-project/sglang/pull/16699) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Quantization schemeLayer typeA2 SupportedA3 SupportedA5 Supported
W4A16Linear<span style="color: green;">√</span><span style="color: green;">√</span><span style="color: yellow;">TBD</span>
W8A16Linear<span style="color: green;">√</span><span style="color: green;">√</span><span style="color: yellow;">TBD</span>
W4A16MoE<span style="color: green;">√</span><span style="color: green;">√</span><span style="color: yellow;">TBD</span>
W8A16MoE<span style="color: green;">√</span><span style="color: green;">√</span><span style="color: yellow;">TBD</span>
+ +Compressed-tensors (LLM Compressor) on Ascend support: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Quantization schemeLayer typeA2 SupportedA3 SupportedA5 Supported
W8A8 dynamicLinear<span style="color: green;">√</span><span style="color: green;">√</span><span style="color: yellow;">TBD</span>
W4A8 dynamic with/without activation clipMoE<span style="color: green;">√</span><span style="color: green;">√</span><span style="color: yellow;">TBD</span>
W4A16 MOEMoE<span style="color: green;">√</span><span style="color: green;">√</span><span style="color: yellow;">TBD</span>
W8A8 dynamicMoE<span style="color: green;">√</span><span style="color: green;">√</span><span style="color: yellow;">TBD</span>
+ +[GGUF on Ascend support](https://github.com/sgl-project/sglang/pull/17883) + +in progress diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_quick_start.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_quick_start.mdx new file mode 100644 index 000000000..7a88a5e93 --- /dev/null +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_quick_start.mdx @@ -0,0 +1,107 @@ +--- +title: "Ascend NPU Quickstart" +metatags: + description: "Quickstart for running SGLang on Ascend NPUs with the official container image, including server launch and test request examples." +--- + +## Prerequisites + +### Supported Devices + +- Atlas 800I A2 inference series (Atlas 800I A2) +- Atlas 800I A3 inference series (Atlas 800I A3) + +## Setup environment using container + +__Notice:__ The following commands are based on Atlas 800I A3 machines. If you are using Atlas 800I A2, some changes are needed. + +- The image tag needs to be `main-cann8.5.0-a3` for Atlas 800I A3 and `main-cann8.5.0-910b` for Atlas 800I A2. +- The device mapping in `docker run` command needs to be changed to `davinci[0-7]` for Atlas 800I A2. + +```shell Command +# For Atlas 800I A3 +export IMAGE=quay.io/ascend/sglang:main-cann8.5.0-a3 + +docker run -it --rm --privileged --network=host --ipc=host --shm-size=16g \ + --device=/dev/davinci0 --device=/dev/davinci1 --device=/dev/davinci2 --device=/dev/davinci3 \ + --device=/dev/davinci4 --device=/dev/davinci5 --device=/dev/davinci6 --device=/dev/davinci7 \ + --device=/dev/davinci8 --device=/dev/davinci9 --device=/dev/davinci10 --device=/dev/davinci11 \ + --device=/dev/davinci12 --device=/dev/davinci13 --device=/dev/davinci14 --device=/dev/davinci15 \ + --device=/dev/davinci_manager \ + --device=/dev/hisi_hdc \ + --volume /usr/local/sbin:/usr/local/sbin \ + --volume /usr/local/Ascend/driver:/usr/local/Ascend/driver \ + --volume /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ + --volume /etc/ascend_install.info:/etc/ascend_install.info \ + --volume /var/queue_schedule:/var/queue_schedule \ + --volume ~/.cache/:/root/.cache/ \ + --entrypoint=bash \ + $IMAGE +``` + +## Usage + +The SGLang server is installed in the container by default. You can use `pip show sglang` to check the version. + +### Start SGLang server + +SGLang will automatically download the model from Hugging Face. + +```shell Command +# Set HF_ENDPOINT to a mirror site if network is not available +export HF_ENDPOINT=https://hf-mirror.com + +# Set your own HF_TOKEN to download restricted models +export HF_TOKEN= + +# Start SGLang server +# It may take several minutes to download the model on the first run +sglang serve --model-path Qwen/Qwen2.5-7B-Instruct --attention-backend ascend & +``` + +If you see output like the following, the server is running. + +```log Output +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on http://127.0.0.1:30000 (Press CTRL+C to quit) +The server is fired up and ready to roll! +``` + +### Send a test request + +You can do inference using the server: + +```shell Command +curl -X POST http://localhost:30000/generate \ + -H "Content-Type: application/json" \ + -d '{ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 16 + } + }' +``` + +If the "text" field in the response contains "Paris", the server is working as expected. + +### Stop server and exit container + +The SGLang server is running as a background process. You can send a `SIGINT` signal to stop it. + +```shell Command +SGLANG_PID=$(pgrep -f "sglang serve") +kill -SIGINT $SGLANG_PID +``` + +The output should be like the following: + +```log Output +INFO: Shutting down +INFO: Waiting for application shutdown. +INFO: Application shutdown complete. +INFO: Finished server process [25310] +``` + +The server has now stopped. You can verify it with `ps -ef | grep sglang`, then exit the container by pressing `Ctrl+D`. diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_5_examples.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_5_examples.mdx new file mode 100644 index 000000000..f9fad5ad3 --- /dev/null +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_5_examples.mdx @@ -0,0 +1,234 @@ +--- +title: "Qwen3.5 examples" +metatags: + description: "Documentation for Qwen3.5 examples" +--- +## Environment Preparation + +### Installation + +The dependencies required for the NPU runtime environment have been integrated into a Docker image and uploaded to the quay.io platform. You can directly pull it. + +```bash Command +#Atlas 800 A3 +docker pull quay.io/ascend/sglang:main-cann8.5.0-a3 +#Atlas 800 A2 +docker pull quay.io/ascend/sglang:main-cann8.5.0-910b + +#start container +docker run -itd --shm-size=16g --privileged=true --name ${NAME} \ +--privileged=true --net=host \ +-v /var/queue_schedule:/var/queue_schedule \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /usr/local/sbin:/usr/local/sbin \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ +--device=/dev/davinci0:/dev/davinci0 \ +--device=/dev/davinci1:/dev/davinci1 \ +--device=/dev/davinci2:/dev/davinci2 \ +--device=/dev/davinci3:/dev/davinci3 \ +--device=/dev/davinci4:/dev/davinci4 \ +--device=/dev/davinci5:/dev/davinci5 \ +--device=/dev/davinci6:/dev/davinci6 \ +--device=/dev/davinci7:/dev/davinci7 \ +--device=/dev/davinci8:/dev/davinci8 \ +--device=/dev/davinci9:/dev/davinci9 \ +--device=/dev/davinci10:/dev/davinci10 \ +--device=/dev/davinci11:/dev/davinci11 \ +--device=/dev/davinci12:/dev/davinci12 \ +--device=/dev/davinci13:/dev/davinci13 \ +--device=/dev/davinci14:/dev/davinci14 \ +--device=/dev/davinci15:/dev/davinci15 \ +--device=/dev/davinci_manager:/dev/davinci_manager \ +--device=/dev/hisi_hdc:/dev/hisi_hdc \ +--entrypoint=bash \ +quay.io/ascend/sglang:${tag} +``` + +## Deployment + +### Single-node Deployment + +Run the following script to execute online inference. + +#### Qwen3.5 397B + +```bash Command +# high performance cpu +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 +# bind cpu +export SGLANG_SET_CPU_AFFINITY=1 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING +# cann +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +export STREAMS_PER_DEVICE=32 +export HCCL_BUFFSIZE=1000 +export HCCL_OP_EXPANSION_MODE=AIV +export HCCL_SOCKET_IFNAME=lo +export GLOO_SOCKET_IFNAME=lo + +python3 -m sglang.launch_server \ + --model-path $MODEL_PATH \ + --attention-backend ascend \ + --device npu \ + --tp-size 16 --nnodes 1 --node-rank 0 \ + --chunked-prefill-size 4096 --max-prefill-tokens 280000 \ + --disable-radix-cache \ + --trust-remote-code \ + --host 127.0.0.1 \ + --mem-fraction-static 0.7 \ + --port 8000 \ + --cuda-graph-bs 16 \ + --quantization modelslim \ + --enable-multimodal \ + --mm-attention-backend ascend_attn \ + --dtype bfloat16 +``` + +#### Qwen3.5 122B + +```bash Command +# high performance cpu +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 +# bind cpu +export SGLANG_SET_CPU_AFFINITY=1 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING +# cann +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +export STREAMS_PER_DEVICE=32 +export HCCL_BUFFSIZE=1000 +export HCCL_OP_EXPANSION_MODE=AIV +export HCCL_SOCKET_IFNAME=lo +export GLOO_SOCKET_IFNAME=lo + +python3 -m sglang.launch_server \ + --model-path $MODEL_PATH \ + --attention-backend ascend \ + --device npu \ + --tp-size 8 --nnodes 1 --node-rank 0 \ + --chunked-prefill-size 4096 --max-prefill-tokens 280000 \ + --disable-radix-cache \ + --trust-remote-code \ + --host 127.0.0.1 \ + --mem-fraction-static 0.7 \ + --port 8000 \ + --cuda-graph-bs 16 \ + --quantization modelslim \ + --enable-multimodal \ + --mm-attention-backend ascend_attn \ + --dtype bfloat16 +``` + +#### Qwen3.5 35B + +```bash Command +# high performance cpu +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 +# bind cpu +export SGLANG_SET_CPU_AFFINITY=1 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING +# cann +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +export STREAMS_PER_DEVICE=32 +export HCCL_BUFFSIZE=1000 +export HCCL_OP_EXPANSION_MODE=AIV +export HCCL_SOCKET_IFNAME=lo +export GLOO_SOCKET_IFNAME=lo + +python3 -m sglang.launch_server \ + --model-path $MODEL_PATH \ + --attention-backend ascend \ + --device npu \ + --tp-size 2 --nnodes 1 --node-rank 0 \ + --chunked-prefill-size 4096 --max-prefill-tokens 280000 \ + --disable-radix-cache \ + --trust-remote-code \ + --host 127.0.0.1 \ + --mem-fraction-static 0.7 \ + --port 8000 \ + --cuda-graph-bs 16 \ + --quantization modelslim \ + --enable-multimodal \ + --mm-attention-backend ascend_attn \ + --dtype bfloat16 +``` + +#### Qwen3.5 27B + +```bash Command +# high performance cpu +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 +# bind cpu +export SGLANG_SET_CPU_AFFINITY=1 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING +# cann +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +export STREAMS_PER_DEVICE=32 +export HCCL_BUFFSIZE=1000 +export HCCL_OP_EXPANSION_MODE=AIV +export HCCL_SOCKET_IFNAME=lo +export GLOO_SOCKET_IFNAME=lo + +python3 -m sglang.launch_server \ + --model-path $MODEL_PATH \ + --attention-backend ascend \ + --device npu \ + --tp-size 2 \ + --chunked-prefill-size -1 --max-prefill-tokens 120000 \ + --disable-radix-cache \ + --trust-remote-code \ + --host 127.0.0.1 \ + --mem-fraction-static 0.8 \ + --port 8000 \ + --cuda-graph-bs 32 \ + --enable-multimodal \ + --mm-attention-backend ascend_attn +``` + +### Prefill-Decode Disaggregation + +Not test yet. + +### Using Benchmark + +Refer to [Benchmark and Profiling](../../developer_guide/benchmark_and_profiling) for details. diff --git a/docs_new/docs/hardware-platforms/ascend-npus/Qwen3-Examples.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_examples.mdx similarity index 50% rename from docs_new/docs/hardware-platforms/ascend-npus/Qwen3-Examples.mdx rename to docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_examples.mdx index 469ea172f..22bcb24bc 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/Qwen3-Examples.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_examples.mdx @@ -1,10 +1,16 @@ -## Running Qwen3 +--- +title: "Qwen3 Examples" +metatags: + description: "Documentation for Qwen3 Examples" +--- +## Qwen3 examples -### Running Qwen3-32B on 1 x Atlas 800I A3 +### Running Qwen3 + +#### Running Qwen3-32B on 1 x Atlas 800I A3. Model weights could be found [here](https://huggingface.co/Qwen/Qwen3-32B) - ```shell Launch Server export SGLANG_SET_CPU_AFFINITY=1 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True @@ -20,15 +26,13 @@ python -m sglang.launch_server \ --model-path Qwen/Qwen3-32B \ --mem-fraction-static 0.8 ``` - -### Running Qwen3-32B on 1 x Atlas 800I A3 with Qwen3-32B-Eagle3 +#### Running Qwen3-32B on 1 x Atlas 800I A3 with Qwen3-32B-Eagle3. Model weights could be found [here](https://huggingface.co/Qwen/Qwen3-32B) Speculative model weights could be found [here](https://huggingface.co/Zhihu-ai/Zhi-Create-Qwen3-32B-Eagle3) - ```shell Launch Server with Eagle3 export SGLANG_SET_CPU_AFFINITY=1 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True @@ -50,13 +54,11 @@ python -m sglang.launch_server \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 2 ``` - -### Running Qwen3-30B-A3B MOE on 1 x Atlas 800I A3 +#### Running Qwen3-30B-A3B MOE on 1 x Atlas 800I A3. Model weights could be found [here](https://huggingface.co/Qwen/Qwen3-30B-A3B) - ```shell Launch Server export SGLANG_SET_CPU_AFFINITY=1 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True @@ -74,13 +76,11 @@ python -m sglang.launch_server \ --model-path Qwen/Qwen3-30B-A3B \ --mem-fraction-static 0.8 ``` - -### Running Qwen3-235B-A22B-Instruct-2507 MOE on 1 x Atlas 800I A3 +#### Running Qwen3-235B-A22B-Instruct-2507 MOE on 1 x Atlas 800I A3. Model weights could be found [here](https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507) - ```shell Launch Server export SGLANG_SET_CPU_AFFINITY=1 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True @@ -98,13 +98,102 @@ python -m sglang.launch_server \ --watchdog-timeout 9000 \ --mem-fraction-static 0.8 ``` - -### Running Qwen3-VL-8B-Instruct on 1 x Atlas 800I A3 +#### Running Qwen3-235B-A22B-Instruct-2507 with 256K long sequence on 2 x Atlas 800I A3 without CP + +This example uses **PD disaggregation** for long-sequence inference and keeps **context parallel disabled**. + +Set the shared environment variables on both nodes first: + +```bash Command +export ASCEND_USE_FIA=1 +export SGLANG_SET_CPU_AFFINITY=1 +export ASCEND_MF_STORE_URL="tcp://:12345" +export HCCL_SOCKET_IFNAME= +export GLOO_SOCKET_IFNAME= + +MODEL_PATH=/root/.cache/modelscope/hub/models/zcgy26/Qwen3-235B-A22B-Instruct-2507-w8a8 +``` + +**Prefill node:** + +```bash Command +export ASCEND_LAUNCH_BLOCKING=1 +export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 +export HCCL_BUFFSIZE=1500 +export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=1024 +export DEEPEP_NORMAL_LONG_SEQ_ROUND=128 +export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1 + +python3 -m sglang.launch_server \ + --model-path ${MODEL_PATH} \ + --disaggregation-mode prefill \ + --disaggregation-transfer-backend ascend \ + --disaggregation-bootstrap-port 8995 \ + --attention-backend ascend \ + --disable-radix-cache \ + --quantization modelslim \ + --chunked-prefill-size -1 \ + --skip-server-warmup \ + --device npu \ + --tp-size 16 \ + --mem-fraction-static 0.45 \ + --max-running-requests 1 \ + --host \ + --port 8000 \ + --dist-init-addr :5000 \ + --nnodes 1 \ + --node-rank 0 \ + --moe-a2a-backend deepep \ + --deepep-mode normal +``` + +**Decode node:** + +```bash Command +export SGLANG_DEEPEP_BF16_DISPATCH=0 +export HCCL_BUFFSIZE=4000 +export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=4096 +export DEEPEP_NORMAL_LONG_SEQ_ROUND=16 + +python3 -m sglang.launch_server \ + --model-path ${MODEL_PATH} \ + --disaggregation-mode decode \ + --disaggregation-transfer-backend ascend \ + --attention-backend ascend \ + --mem-fraction-static 0.8 \ + --disable-cuda-graph \ + --device npu \ + --disable-radix-cache \ + --quantization modelslim \ + --chunked-prefill-size 8192 \ + --skip-server-warmup \ + --tp-size 16 \ + --max-running-requests 1 \ + --host \ + --port 8232 \ + --moe-a2a-backend deepep \ + --deepep-mode low_latency \ + --disable-overlap-schedule +``` + +**Router:** + +```bash Command +python3 -m sglang_router.launch_router \ + --pd-disaggregation \ + --policy cache_aware \ + --prefill http://:8000 8995 \ + --decode http://:8232 \ + --host \ + --port 6689 \ + --prometheus-port 29010 +``` + +#### Running Qwen3-VL-8B-Instruct on 1 x Atlas 800I A3. Model weights could be found [here](https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct) - ```shell Launch Server export SGLANG_SET_CPU_AFFINITY=1 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True @@ -121,4 +210,3 @@ python -m sglang.launch_server \ --model-path Qwen/Qwen3-VL-8B-Instruct \ --mem-fraction-static 0.8 ``` - diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_ring_sp_performance.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_ring_sp_performance.mdx new file mode 100644 index 000000000..2f0385c56 --- /dev/null +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_ring_sp_performance.mdx @@ -0,0 +1,110 @@ +--- +title: "Ascend NPU Ring-SP Performance (Wan2.1-T2V-1.3B)" +metatags: + description: "This page reports Ring-SP performance on Ascend NPU with torchnpu==2.10.0." +--- + +This page reports Ring-SP performance on Ascend NPU with `torch_npu==2.10.0`. + +- Baseline config: `ulysses=1, ring=1` (short: `u1r1`) +- Ring-SP config: `ulysses=1, ring=2` (short: `u1r2`) + +## Benchmark Setup + +- Model: `Wan2.1-T2V-1.3B-Diffusers` +- Prompt: `"a cat is playing piano"` +- Framework command: `sglang generate` +- Runtime: `torch_npu==2.10.0` + +## Generate Commands + +### Baseline (`u1r1`) + +```bash +sglang generate --model-path /nas/disk1/Wan2.1-T2V-1.3B-Diffusers \ + --prompt "a cat is playing piano" --num-gpus 1 --ring-degree 1 \ + --save-output +``` + +### Ring-SP (`u1r2`) + +```bash +sglang generate --model-path /nas/disk1/Wan2.1-T2V-1.3B-Diffusers \ + --prompt "a cat is playing piano" --num-gpus 2 --ring-degree 2 \ + --save-output +``` + +## Benchmarks + +Benchmark Disclaimer + +These numbers are from one fixed setup and one prompt case. Actual performance may vary by model settings, environment, and workload. + +### Stage Time Breakdown + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Stage / Metricu1r2 (s)u1r1 baseline (s)Speedup
InputValidation0.00030.00020.67x
TextEncoding3.59363.58201.00x
LatentPreparation0.00070.00557.86x
TimestepPreparation0.00080.00070.88x
Denoising121.2788239.25801.97x
Decoding13.868516.49691.19x
Total (Pixel data generated)141.86266.501.88x
+ +## Summary + +- With `torch_npu==2.10.0`, Ring-SP (`u1r2`) runs successfully on NPU for this case. +- End-to-end generation time improves from `266.50s` to `141.86s` (`1.88x`). +- The main gain comes from `DenoisingStage` (`1.97x`), while decoding also improves (`1.19x`). diff --git a/docs_new/docs/hardware-platforms/ascend-npus/Support-Features-on-Ascend-NPU.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_support_features.mdx similarity index 64% rename from docs_new/docs/hardware-platforms/ascend-npus/Support-Features-on-Ascend-NPU.mdx rename to docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_support_features.mdx index 7f91e4e68..a8ab43b80 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/Support-Features-on-Ascend-NPU.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_support_features.mdx @@ -1,11 +1,15 @@ +--- +title: "Support Features on Ascend NPU" +metatags: + description: "Documentation for Support Features on Ascend NPU" +--- This section describes the basic functions and features supported by the Ascend NPU.If you encounter issues or have any questions, please [open an issue](https://github.com/sgl-project/sglang/issues). If you want to know the meaning and usage of each parameter, click [Server Arguments](../../advanced_features/server_arguments). - - +## Model and tokenizer @@ -20,8 +24,7 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + @@ -29,102 +32,84 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + - - + - - - + + - - + - - + - - - + + - - + - - + - - + - - + - - + - - + - - - + +
Argument Defaults OptionsA2A3Server supported
`--model-path`
`--model`		 `None` Type: str✅✅A2, A3
`--tokenizer-path` `None` Type: str✅✅A2, A3
`--tokenizer-mode` `auto``auto`, `slow`✅✅auto, slowA2, A3
`--tokenizer-worker-num` `1` Type: int✅✅A2, A3
`--skip-tokenizer-init` `False` bool flag (set to enable)✅✅A2, A3
`--load-format` `auto``auto`, `safetensors`✅✅auto, safetensorsA2, A3
`--model-loader-`
`extra-config`		 {} Type: str✅✅A2, A3
`--trust-remote-code` `False` bool flag (set to enable)✅✅A2, A3
`--context-length` `None` Type: int✅✅A2, A3
`--is-embedding` `False` bool flag (set to enable)✅✅A2, A3
`--enable-multimodal` `None` bool flag (set to enable)✅✅A2, A3
`--revision` `None` Type: str✅✅A2, A3
`--model-impl` `auto``auto`, `sglang`,
`transformers`		✅✅auto, sglang,<br/> transformersA2, A3
- - - - - - +## HTTP server @@ -139,8 +124,7 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + @@ -148,60 +132,48 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + - - + - - + - - + - - + - - + - - - + +
Argument Defaults OptionsA2A3Server supported
`--host` `127.0.0.1` Type: str✅✅A2, A3
`--port` `30000` Type: int✅✅A2, A3
`--skip-server-warmup` `False` bool flag (set to enable)✅✅A2, A3
`--warmups` `None` Type: str✅✅A2, A3
`--nccl-port` `None` Type: int✅✅A2, A3
`--fastapi-root-path` `None` Type: str✅✅A2, A3
`--grpc-mode` `False`bool flag (set to enable)✅✅FalsePlanned
- - - - - - +## Quantization and data type @@ -217,109 +189,80 @@ click [Server Arguments](../../advanced_features/server_arguments). - - - + - - - - + + - - - + - - - + - - - + - - - + - - - + - - - + - - - + - - - + - - - + - - - +
Argument Defaults OptionsA2A3SpecialServer supported
`--dtype` `auto``auto`,
`float16`,
`bfloat16`		✅✅❌auto,<br/> float16,<br/> bfloat16A2, A3
`--quantization` `None` `modelslim`✅✅❌A2, A3
`--quantization-param-path` `None` Type: str❌❌✅Special For GPU
`--kv-cache-dtype` `auto` `auto`✅✅❌A2, A3
`--enable-fp32-lm-head` `False` bool flag
(set to enable)		✅✅❌A2, A3
`--modelopt-quant` `None` Type: str❌❌✅Special For GPU
`--modelopt-checkpoint-`
`restore-path`		 `None` Type: str❌❌✅Special For GPU
`--modelopt-checkpoint-`
`save-path`		 `None` Type: str❌❌✅Special For GPU
`--modelopt-export-path` `None` Type: str❌❌✅Special For GPU
`--quantize-and-serve` `False` bool flag
(set to enable)		❌❌✅Special For GPU
`--rl-quant-profile` `None` Type: str❌❌✅Special For GPU
- - - - - - +## Memory and scheduling @@ -334,8 +277,7 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + @@ -343,130 +285,144 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + - - + - - + - - + - - + - - + - - + - - - + + - - + - - + - - + - - + - - + - - + - - + - + + + + + + + - - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + - +
Argument Defaults OptionsA2A3Server supported
`--mem-fraction-static` `None` Type: float✅✅A2, A3
`--max-running-requests` `None` Type: int✅✅A2, A3
`--prefill-max-requests` `None` Type: int✅✅A2, A3
`--max-queued-requests` `None` Type: int✅✅A2, A3
`--max-total-tokens` `None` Type: int✅✅A2, A3
`--chunked-prefill-size` `None` Type: int✅✅A2, A3
`--max-prefill-tokens` `16384` Type: int✅✅A2, A3
`--schedule-policy` `fcfs``lpm`, `fcfs`✅✅lpm, fcfsA2, A3
`--enable-priority-`
`scheduling`		 `False` bool flag
(set to enable)		✅✅A2, A3
`--schedule-low-priority-`
`values-first`		 `False` bool flag
(set to enable)		✅✅A2, A3
`--priority-scheduling-`
`preemption-threshold`		 `10` Type: int✅✅A2, A3
`--schedule-conservativeness` `1.0` Type: float✅✅A2, A3
`--page-size` `128` Type: int✅✅A2, A3
`--swa-full-tokens-ratio` `0.8` Type: float✅✅Planned
`--disable-hybrid-swa-memory` `False` bool flag
(set to enable)		✅✅Planned
`--abort-on-priority-`
`when-disabled`		--radix-eviction-policylrulru,<br/>lfuA2, A3
--enable-prefill-delayer `False` bool flag
(set to enable)		✅✅A2, A3
--prefill-delayer-max-delay-passes30Type: intA2, A3
--prefill-delayer-token-usage-low-watermarkNoneType: floatA2, A3
--prefill-delayer-forward-passes-bucketsNoneList[float]A2, A3
--prefill-delayer-wait-seconds-bucketsNoneList[float]A2, A3
--abort-on-priority-<br/>when-disabled`False`bool flag
(set to enable)		A2, A3
`--enable-dynamic-chunking` `False` bool flag
(set to enable)		✅✅Experimental
- - - - - - +## Runtime options @@ -481,8 +437,7 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + @@ -490,130 +445,126 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + - - + - - - + + - - - - - + + + + - - + + - - + - - + + - - + - - + + + + + + + + + + + + + + - - + - - + + - - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + - - - + + + + + + + + + + + + + + + - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +
Argument Defaults OptionsA2A3Server supported
`--device` `None` Type: str✅✅A2, A3
`--tensor-parallel-size`
`--tp-size`		 `1` Type: int✅✅A2, A3
`--pipeline-parallel-size`
`--pp-size`		 `1`Type: int✅✅Type: int; Currently 2 not supportedExperimental
`--pp-max-micro-batch-size``None`Type: int✅✅--attention-context-parallel-size<br/>--attn-cp-size1Type: int; must be equal to --tp-sizeA2, A3
`--pp-async-batch-depth``None`--moe-data-parallel-size<br/>--moe-dp-size1 Type: int✅✅Planned
`--stream-interval``1`--pp-max-micro-batch-sizeNone Type: int✅✅Experimental
`--stream-output``False`--pp-async-batch-depthNoneType: intExperimental
--stream-interval1Type: intA2, A3
--incremental-streaming-outputFalse bool flag (set to enable)✅✅A2, A3
`--random-seed``None`--random-seedNone Type: int✅✅A2, A3
`--constrained-json-`
`whitespace-pattern`		--constrained-json-<br/>whitespace-patternNoneType: strA2, A3
--constrained-json-<br/>disable-any-whitespaceFalsebool flag (set to enable)A2, A3
--watchdog-timeout300Type: floatA2, A3
--soft-watchdog-timeout300Type: floatA2, A3
--dist-timeoutNoneType: intA2, A3
--download-dirNoneType: strA2, A3
--model-checksum `None` Type: str✅✅Planned
`--constrained-json-`
`disable-any-whitespace`		`False`
--base-gpu-id0Type: intA2, A3
--gpu-id-step1Type: intA2, A3
--sleep-on-idleFalse bool flag (set to enable)✅✅A2, A3
`--watchdog-timeout``300`Type: float✅✅
`--soft-watchdog-timeout``300`Type: float✅✅
`--dist-timeout``None`Type: int✅✅
`--base-gpu-id``0`Type: int✅✅
`--gpu-id-step``1`Type: int✅✅
`--sleep-on-idle``False`bool flag (set to enable)✅✅
`--custom-sigquit-handler``None`Optional[Callable]✅✅
- - - - - - +## Logging @@ -629,9 +580,7 @@ click [Server Arguments](../../advanced_features/server_arguments). - - - + @@ -639,187 +588,150 @@ click [Server Arguments](../../advanced_features/server_arguments). - - - + - - - + - - - + - - - - + + - - - - + + - - - + - - - + - - - + - - - + - - - + - - - + - - - + - - - + - - - + - - - + - - - + - - - + - - - + - - - + - - - + - - - + - - - + - + + + + + + + + + + + + +
Argument Defaults OptionsA2A3SpecialServer supported
`--log-level` `info` Type: str✅✅❌A2, A3
`--log-level-http` `None` Type: str✅✅❌A2, A3
`--log-requests` `False` bool flag
(set to enable)		✅✅❌A2, A3
`--log-requests-level` `2``0`, `1`, `2`, `3`✅✅❌0, 1, 2, 3A2, A3
`--log-requests-format` texttext, json✅✅❌text, jsonA2, A3
`--crash-dump-folder` `None` Type: str✅✅❌A2, A3
`--enable-metrics` `False` bool flag
(set to enable)		✅✅❌A2, A3
`--enable-metrics-for-`
`all-schedulers`		 `False` bool flag
(set to enable)		✅✅❌A2, A3
`--tokenizer-metrics-`
`custom-labels-header`		 `x-custom-labels` Type: str✅✅❌A2, A3
`--tokenizer-metrics-`
`allowed-custom-labels`		 `None` List[str]✅✅❌A2, A3
`--bucket-time-to-`
`first-token`		 `None` List[float]✅✅❌A2, A3
`--bucket-inter-token-`
`latency`		 `None` List[float]✅✅❌A2, A3
`--bucket-e2e-request-`
`latency`		 `None` List[float]✅✅❌A2, A3
`--collect-tokens-`
`histogram`		 `False` bool flag
(set to enable)		✅✅❌A2, A3
`--prompt-tokens-buckets` `None` List[str]✅✅❌A2, A3
`--generation-tokens-buckets` `None` List[str]✅✅❌A2, A3
`--gc-warning-threshold-secs` `0.0` Type: float✅✅❌A2, A3
`--decode-log-interval` `40` Type: int✅✅❌A2, A3
`--enable-request-time-`
`stats-logging`		 `False` bool flag
(set to enable)		✅✅❌A2, A3
`--kv-events-config` `None` Type: str❌❌✅Special for GPU
`--enable-trace` `False` bool flag
(set to enable)		✅✅❌A2, A3
`--oltp-traces-endpoint` `localhost:4317` Type: str✅✅❌A2, A3
--log-requests-targetNoneType: strA2, A3
--uvicorn-access-log-exclude-prefixes[]List[str]A2, A3
- - - - - - +## RequestMetricsExporter configuration @@ -834,8 +746,7 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + @@ -843,25 +754,18 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + - - +
Argument Defaults OptionsA2A3Server supported
`--export-metrics-to-`
`file`		 `False` bool flag
(set to enable)		✅✅A2, A3
`--export-metrics-to-`
`file-dir`		 `None` Type: str✅✅A2, A3
- - - - - - +## API related @@ -876,8 +780,7 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + @@ -885,81 +788,72 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + - - + - - + - - + - - + - + - - + - - - - - + + + + - + + + + + + + - - - + + - - - - - + + + + - + - - - + + - +
Argument Defaults OptionsA2A3Server supported
`--api-key` `None` Type: str✅✅A2, A3
`--admin-api-key` `None` Type: str✅✅A2, A3
`--served-model-name` `None` Type: str✅✅A2, A3
`--weight-version` `default` Type: str✅✅A2, A3
`--chat-template` `None` Type: str✅✅A2, A3
`--completion-template`--hf-chat-template-name `None` Type: str✅✅A2, A3
`--enable-cache-report``False`bool flag
(set to enable)		✅✅--completion-templateNoneType: strA2, A3
`--reasoning-parser`--enable-cache-reportFalsebool flag<br/> (set to enable)A2, A3
--reasoning-parser `None``deepseek-r1`✅✅deepseek-r1<br/>deepseek-v3<br/>glm45<br/>gpt-oss<br/>kimi<br/>qwen3<br/>qwen3-thinking<br/>step3A2, A3
`--tool-call-parser``None``llama`,`pythonic`✅✅--tool-call-parserNonellama3<br/> pythonic<br/> qwen<br/> qwen3_coderA2, A3
`--sampling-defaults` `model``openai`, `model`✅✅openai, modelA2, A3
- - - - - - +## Data parallelism @@ -974,8 +868,7 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + @@ -983,32 +876,18 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + - - - - - - - - - - - + + +
Argument Defaults OptionsA2A3Server supported
`--data-parallel-size`
`--dp-size`		 `1` Type: int✅✅A2, A3
`--load-balance-method``round_robin``round_robin`,
`total_requests`,
`total_tokens`		✅✅
`--prefill-round-robin-balance``False`bool flag
(set to enable)		✅✅autoauto,<br/> round_robin,<br/> follow_bootstrap_room,<br/> total_requests,<br/> total_tokensA2, A3
- - - - - - +## Multi-node distributed serving @@ -1023,8 +902,7 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + @@ -1032,32 +910,24 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + - - + - - +
Argument Defaults OptionsA2A3Server supported
`--dist-init-addr`
`--nccl-init-addr`		 `None` Type: str✅✅A2, A3
`--nnodes` `1` Type: int✅✅A2, A3
`--node-rank` `0` Type: int✅✅A2, A3
- - - - - - +## Model override args @@ -1072,8 +942,7 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + @@ -1081,25 +950,18 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + - - +
Argument Defaults OptionsA2A3Server supported
`--json-model-override-`
`args`		 `{}` Type: str✅✅A2, A3
`--preferred-sampling-`
`params`		 `None` Type: str✅✅A2, A3
- - - - - - +## LoRA @@ -1115,9 +977,7 @@ click [Server Arguments](../../advanced_features/server_arguments). - - - + @@ -1125,83 +985,66 @@ click [Server Arguments](../../advanced_features/server_arguments). - - - + - + + + + + + + - - - + - + - - - - + + - - - - - - + + + + - - + + - - - + - - + + - - - + - - - - - - + + + + - - - - - - + + + + - + - - - - + + - +
Argument Defaults OptionsA2A3SpecialServer supported
`--enable-lora` `False` Bool flag
(set to enable)		✅✅❌A2, A3
`--max-lora-rank`--enable-lora-overlap-loadingFalseBool flag <br/>(set to enable)A2, A3
--max-lora-rank `None` Type: int✅✅❌A2, A3
`--lora-target-modules`--lora-target-modules `None``all`✅✅❌allA2, A3
`--lora-paths``None`Type: List[str] /
JSON objects		✅✅❌--lora-pathsNoneType: List[str] /<br/> JSON objectsA2, A3
`--max-loras-per-batch``8`--max-loras-per-batch8 Type: int✅✅❌A2, A3
`--max-loaded-loras``None`--max-loaded-lorasNone Type: int✅✅❌A2, A3
`--lora-eviction-policy``lru``lru`,
`fifo`		✅✅❌--lora-eviction-policylrulru,<br/> fifoA2, A3
`--lora-backend``triton``triton`✅✅❌--lora-backendcsgmvtriton,<br/>csgmv,<br/>ascend,<br/>torch_nativeA2, A3
`--max-lora-chunk-size` `16``16`, `32`,
`64`, `128`		❌❌✅16, 32,<br/> 64, 128Special for GPU
- - - - - - +## Kernel Backends (Attention, Sampling, Grammar, GEMM) @@ -1217,9 +1060,7 @@ click [Server Arguments](../../advanced_features/server_arguments). - - - + @@ -1227,91 +1068,66 @@ click [Server Arguments](../../advanced_features/server_arguments). - - - + - - - + - - - + - - - - + + - - - + - - - + - - - - + + - - - - + + - - - - + + - - - +
Argument Defaults OptionsA2A3SpecialServer supported
`--attention-backend` `None` `ascend`✅✅❌A2, A3
`--prefill-attention-backend` `None` `ascend`✅✅❌A2, A3
`--decode-attention-backend` `None` `ascend`✅✅❌A2, A3
`--sampling-backend` `None``pytorch`,
`ascend`		✅✅❌pytorch,<br/>ascendA2, A3
`--grammar-backend` `None` `xgrammar`✅✅❌A2, A3
`--mm-attention-backend` `None` `ascend_attn`✅✅❌A2, A3
`--nsa-prefill-backend` `flashmla_sparse``flashmla_sparse`,
`flashmla_decode`,
`fa3`,
`tilelang`,
`aiter`		❌❌✅flashmla_sparse,<br/> flashmla_decode,<br/>fa3,<br/> tilelang,<br/> aiterSpecial for GPU
`--nsa-decode-backend` `fa3``flashmla_prefill`,
`flashmla_kv`,
`fa3`,
`tilelang`,
`aiter`		❌❌✅flashmla_prefill,<br/> flashmla_kv,<br/> fa3,<br/>tilelang,<br/> aiterSpecial for GPU
`--fp8-gemm-backend` `auto``auto`,
`deep_gemm`,
`flashinfer_trtllm`,
`cutlass`,
`triton`,
`aiter`		❌❌✅auto,<br/> deep_gemm,<br/> flashinfer_trtllm,<br/>flashinfer_cutlass,<br/>flashinfer_deepgemm,<br/>cutlass,<br/> triton,<br/> aiterSpecial for GPU
`--disable-flashinfer-`
`autotune`		 `False` bool flag
(set to enable)		❌❌✅Special for GPU
- - - - - - +## Speculative decoding @@ -1327,141 +1143,104 @@ click [Server Arguments](../../advanced_features/server_arguments). - - - + - - - - + + - - - + - - - - + + - - - - - + + + - - - + - - - + - - - + - - - + - - - + - - - + - - - - + + - - - + - - - + - - - + - - - +
Argument Defaults OptionsA2A3SpecialServer supported
`--speculative-algorithm` `None``EAGLE3`,
`NEXTN`		✅✅❌EAGLE3,<br/> NEXTNA2, A3
`--speculative-draft-model-path`
`--speculative-draft-model`		 `None` Type: str✅✅❌A2, A3
`--speculative-draft-model-`
`revision`		 `None`Type: str✅✅❌Type: str,<br/> branch name,<br/> tag name,<br/> commit idA2, A3
`--speculative-draft-load-format``None``auto`✅✅❌autoauto,<br/> dummyA2, A3
`--speculative-num-steps` `None` Type: int✅✅❌A2, A3
`--speculative-eagle-topk` `None` Type: int✅✅❌A2, A3
`--speculative-num-draft-tokens` `None` Type: int✅✅❌A2, A3
`--speculative-accept-`
`threshold-single`		 `1.0` Type: float❌❌✅Special for GPU
`--speculative-accept-`
`threshold-acc`		 `1.0` Type: float❌❌✅Special for GPU
`--speculative-token-map` `None` Type: str✅✅❌A2, A3
`--speculative-attention-`
`mode`		 `prefill``prefill`,
`decode`		✅✅❌prefill,<br/> decodeA2, A3
`--speculative-moe-runner-`
`backend`		 `None` `auto`✅✅❌A2, A3
`--speculative-moe-a2a-`
`backend`		 `None` `ascend_fuseep`✅✅❌A2, A3
`--speculative-draft-attention-backend` `None` `ascend`✅✅❌A2, A3
`--speculative-draft-model-quantization` `None` `unquant`✅✅❌A2, A3
- - - - - - +## Ngram speculative decoding @@ -1477,9 +1256,7 @@ click [Server Arguments](../../advanced_features/server_arguments). - - - + @@ -1487,67 +1264,48 @@ click [Server Arguments](../../advanced_features/server_arguments). - - - + - - - + - - - + - - - + - - - - + + - + - - - + - - - +
Argument Defaults OptionsA2A3ExperimentalServer supported
`--speculative-ngram-`
`min-match-window-size`		 `1` Type: int❌❌✅Experimental
`--speculative-ngram-`
`max-match-window-size`		 `12` Type: int❌❌✅Experimental
`--speculative-ngram-`
`min-bfs-breadth`		 `1` Type: int❌❌✅Experimental
`--speculative-ngram-`
`max-bfs-breadth`		 `10` Type: int❌❌✅Experimental
`--speculative-ngram-`
`match-type`		 `BFS``BFS`,
`PROB`		❌❌✅BFS,<br/> PROBExperimental. BFS uses recency-based expansion; PROB uses frequency-based expansion.
`--speculative-ngram-`
`branch-length`		--speculative-ngram-<br/>max-trie-depth `18` Type: int❌❌✅Experimental
`--speculative-ngram-`
`capacity`		 `10000000` Type: int❌❌✅Experimental
- - - - - - +## Expert parallelism @@ -1563,9 +1321,7 @@ click [Server Arguments](../../advanced_features/server_arguments). - - - + @@ -1573,171 +1329,132 @@ click [Server Arguments](../../advanced_features/server_arguments). - - - + - - - - + + - - - - + + - - - - + + - - - + - - - - + + - - - + - - - + - - - - + + - - - - + + - - - + - - - - - + + + - - + + - - - + - - + + + + + + + + - - - + - + - - - - + + - - + + - - - + - - + + - - - + - + - - - - + + - + - - - - + + - + - - - + - +
Argument Defaults OptionsA2A3SpecialServer supported
`--expert-parallel-size`
`--ep-size`
`--ep`		 `1` Type: int✅✅❌A2, A3
`--moe-a2a-backend` `none``none`,
`deepep`,
`ascend_fuseep`		✅✅❌none,<br/> deepep,<br/> ascend_fuseep(It is incompatible with eplb)A2, A3
`--moe-runner-backend` `auto``auto`, `triton`✅✅❌auto, tritonA2, A3
`--flashinfer-mxfp4-`
`moe-precision`		 `default``default`,
`bf16`		❌❌✅default,<br/> bf16Special for GPU
`--enable-flashinfer-`
`allreduce-fusion`		 `False` bool flag
(set to enable)		❌❌✅Special for GPU
`--deepep-mode` `auto``normal`,
`low_latency`,
`auto`		✅✅❌normal, <br/>low_latency,<br/> autoA2, A3
`--deepep-config` `None` Type: str❌❌✅Special for GPU
`--ep-num-redundant-experts` `0` Type: int✅✅❌A2, A3
`--ep-dispatch-algorithm` `None`Type: str✅✅❌static,<br/> dynamic,<br/> fakeA2, A3
`--init-expert-location` `trivial`Type: str✅✅❌trivial,<br/> <path.pt>,<br/> <path.json>,<br/> <json_string>A2, A3
`--enable-eplb` `False` bool flag
(set to enable)		✅✅❌A2, A3
`--eplb-algorithm``auto`Type: str✅✅❌deepseekauto,<br/> deepseekA2, A3
`--eplb-rebalance-layers-`
`per-chunk`		`None`--eplb-rebalance-num-iterations1000 Type: int✅✅❌A2, A3
`--eplb-min-rebalancing-`
`utilization-threshold`		`1.0`--eplb-rebalance-layers-<br/>per-chunkNoneType: intA2, A3
--eplb-min-rebalancing-<br/>utilization-threshold1.0 Type: float✅✅❌A2, A3
`--expert-distribution-`
`recorder-mode`		--expert-distribution-<br/>recorder-mode `None`Type: str✅✅❌stat,<br/> stat_approx,<br/> per_pass,<br/> per_tokenA2, A3
`--expert-distribution-`
`recorder-buffer-size`		`None`--expert-distribution-<br/>recorder-buffer-sizeNone Type: int✅✅❌A2, A3
`--enable-expert-distribution-`
`metrics`		`False`--enable-expert-distribution-<br/>metricsFalse bool flag (set to enable)✅✅❌A2, A3
`--moe-dense-tp-size`--moe-dense-tp-size `None`Type: int✅✅❌1A2, A3
`--elastic-ep-backend`--elastic-ep-backend `None``none`, `mooncake`❌❌✅none, mooncakeSpecial for GPU
`--mooncake-ib-device` `None` Type: str❌❌✅Special for GPU
- - - - - - +## Mamba Cache @@ -1752,8 +1469,7 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + @@ -1761,46 +1477,36 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + - - - + + - + - - + - - - + + - - +
Argument Defaults OptionsA2A3Server supported
`--max-mamba-cache-size` `None` Type: int✅✅A2, A3
`--mamba-ssm-dtype` `float32``float32`,
`bfloat16`		✅✅float32,<br/>bfloat16,<br/>float16A2, A3
`--mamba-full-memory-ratio``0.2`0.9 Type: float✅✅A2, A3
`--mamba-scheduler-strategy` `auto``auto`,
`no_buffer`,
`extra_buffer`		✅✅auto,<br/>no_buffer,<br/>extra_bufferA2, A3
`--mamba-track-interval` `256` Type: int✅✅A2, A3
- - - - - - +## Hierarchical cache @@ -1816,101 +1522,68 @@ click [Server Arguments](../../advanced_features/server_arguments). - - - + - - - - + + - - - + - - - + - - - - + + - - - - - - + + + + - - - - - - + + + + - - - - - - + + + + - - - - - - + + + + - - - - - - - - - - + + - - - +
Argument Defaults OptionsA2A3SpecialServer supported
`--enable-hierarchical-`
`cache`		 `False`bool flag
(set to enable)		✅✅❌bool flag<br/> (set to enable).<br/> Currently, mamba cache is not supported.A2, A3
`--hicache-ratio` `2.0` Type: float✅✅❌A2, A3
`--hicache-size` `0` Type: int✅✅❌A2, A3
`--hicache-write-policy` `write_through``write_back`,
`write_through`,
`write_through_selective`		✅✅❌Currently only write_back supportedA2, A3
`--radix-eviction-policy``lru``lru`, `lfu`✅✅❌--hicache-io-backendkernelkernel_ascend,<br/> directA2, A3
`--hicache-io-backend``kernel``kernel_ascend`,
`direct`		✅✅❌--hicache-mem-layoutlayer_firstpage_first_direct,<br/> page_first_kv_splitA2, A3
`--hicache-mem-layout``layer_first``page_first_direct`,
`page_first_kv_split`		✅✅❌--hicache-storage-<br/>backendNonefileA2, A3
`--hicache-storage-`
`backend`		`None``file`✅✅❌--hicache-storage-<br/>prefetch-policybest_effortbest_effort,<br/> wait_complete,<br/> timeoutSpecial for GPU
`--hicache-storage-`
`prefetch-policy`		`best_effort``best_effort`,
`wait_complete`,
`timeout`		❌❌✅
`--hicache-storage-`
`backend-extra-config`		`None`--hicache-storage-<br/>backend-extra-configNone Type: str❌❌✅Special for GPU
- - - - - - +## LMCache @@ -1926,9 +1599,7 @@ click [Server Arguments](../../advanced_features/server_arguments). - - - + @@ -1936,19 +1607,12 @@ click [Server Arguments](../../advanced_features/server_arguments). - - - +
Argument Defaults OptionsA2A3SpecialServer supported
`--enable-lmcache` `False` bool flag
(set to enable)		❌❌✅Special for GPU
- - - - - - +## Offloading (must be used with `--disable-cuda-graph`) @@ -1963,8 +1627,7 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + @@ -1972,46 +1635,36 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + - - - + + - - - + + - - - + + - - - + +
Argument Defaults OptionsA2A3Server supported
`--cpu-offload-gb` `0` Type: int✅✅A2, A3
`--offload-group-size` `-1`Type: int✅✅Type: int (DeepSeek only)A2, A3
`--offload-num-in-group` `1`Type: int✅✅Type: int (DeepSeek only)A2, A3
`--offload-prefetch-step` `1`Type: int✅✅Type: int (DeepSeek only)A2, A3
`--offload-mode` `cpu`Type: str✅✅cpu (DeepSeek only) <br/>meta (DeepSeek only) <br/>sharded_gpu (DeepSeek only)A2, A3
- - - - - - +## Args for multi-item scoring @@ -2026,8 +1679,7 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + @@ -2035,18 +1687,12 @@ click [Server Arguments](../../advanced_features/server_arguments). - - +
Argument Defaults OptionsA2A3Server supported
`--multi-item-scoring-delimiter` `None` Type: int✅✅A2, A3
- - - - - - +## Optimization/debug options @@ -2063,10 +1709,7 @@ click [Server Arguments](../../advanced_features/server_arguments). - - - - + @@ -2074,515 +1717,342 @@ click [Server Arguments](../../advanced_features/server_arguments). - - - - + - - - - + - - - - + - - - - + - - - - + - - - - + - - - - + - - - - + - - - - + - - - - + - - - - + - + - - - - + - + - - - - + - + - - - - + - + - - - - + - + - - - - + - + - - - - + - + - - - - + - + - - - - + - + - - - - + - + - - - - + - - - - - - - - - - - + + - - - - + - + + + + + + + - - - - + - + - - - - - + + - - - - - - - + + + + - - - - - - - + + + + - - - - - - - - - - - + + - - - - + - - + + - - - - + - + - - - - + - + + + + + + + - - - - + - + - - - - + - - - - - - - - - - - + + - - - - + - - + + - - - - + - + + + + + + + - - - - + - + - - - - + - + - - - - + - + - - - - + - + - - - - + - + - - - - + - + + + + + + + + + + + + + - - - - + - + - - - - + - + - - - - + - + - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - - - + - - + + - - - - + + + + + + + - - - - + - - - - + - - - - + - - - - + - - - - +
Argument Defaults OptionsA2A3SpecialPlannedServer supported
`--disable-radix-cache` `False` bool flag
(set to enable)		✅✅❌❌A2, A3
`--cuda-graph-max-bs` `None` Type: int✅✅❌❌A2, A3
`--cuda-graph-bs` `None` List[int]✅✅❌❌A2, A3
`--disable-cuda-graph` `False` bool flag
(set to enable)		✅✅❌❌A2, A3
`--disable-cuda-graph-`
`padding`		 `False` bool flag
(set to enable)		✅✅❌❌A2, A3
`--enable-profile-`
`cuda-graph`		 `False` bool flag
(set to enable)		✅✅❌❌A2, A3
`--enable-cudagraph-gc` `False` bool flag
(set to enable)		✅✅❌❌A2, A3
`--enable-nccl-nvls` `False` bool flag
(set to enable)		❌❌✅❌Special for GPU
`--enable-symm-mem` `False` bool flag
(set to enable)		❌❌✅❌Special for GPU
`--disable-flashinfer-`
`cutlass-moe-fp4-allgather`		 `False` bool flag
(set to enable)		❌❌✅❌Special for GPU
`--enable-tokenizer-`
`batch-encode`		 `False` bool flag
(set to enable)		✅✅❌❌A2, A3
`--disable-tokenizer-`
`batch-encode`		--disable-tokenizer-<br/>batch-decode `False` bool flag
(set to enable)		✅✅❌❌A2, A3
`--disable-outlines-`
`disk-cache`		--disable-custom-<br/>all-reduce `False` bool flag
(set to enable)		✅✅❌❌Special for GPU
`--disable-custom-`
`all-reduce`		--enable-mscclpp `False` bool flag
(set to enable)		✅✅❌❌Special for GPU
`--enable-mscclpp`--enable-torch-<br/>symm-mem `False` bool flag
(set to enable)		❌❌✅❌Special for GPU
`--enable-torch-`
`symm-mem`		--disable-overlap<br/>-schedule `False` bool flag
(set to enable)		❌❌✅❌A2, A3
`--disable-overlap`
`-schedule`		--enable-mixed-<br/>chunk `False` bool flag
(set to enable)		✅✅❌❌A2, A3
`--enable-mixed-`
`chunk`		--enable-dp-attention `False` bool flag
(set to enable)		✅✅❌❌A2, A3
`--enable-dp-attention`--enable-dp-lm-head `False` bool flag
(set to enable)		✅✅❌❌A2, A3
`--enable-dp-lm-head`--enable-two-<br/>batch-overlap `False` bool flag
(set to enable)		✅✅❌❌Planned
`--enable-two-`
`batch-overlap`		--enable-single-<br/>batch-overlap `False` bool flag
(set to enable)		❌❌❌✅A2, A3
`--enable-single-`
`batch-overlap`		`False`bool flag
(set to enable)		✅✅❌❌
`--tbo-token-`
`distribution-threshold`		`0.48`--tbo-token-<br/>distribution-threshold0.48 Type: float❌❌❌✅Planned
`--enable-torch-`
`compile`		--enable-torch-<br/>compileFalsebool flag<br/> (set to enable)A2, A3
--enable-torch-<br/>compile-debug-mode `False` bool flag
(set to enable)		✅✅❌❌A2, A3
`--enable-torch-`
`compile-debug-mode`		--enforce-piecewise-<br/>cuda-graph `False`bool flag
(set to enable)		✅✅❌❌bool flag<br/> (set to enable); <br/> Currently, Llama-3.1-8B-Instruct and Qwen2.5-7B-Instruct models are supported.A2, A3
`--enable-piecewise-`
`cuda-graph`		`False`bool flag
(set to enable)		✅✅❌❌--piecewise-cuda-<br/>graph-tokensNoneType: JSON<br/> listA2, A3
`--piecewise-cuda-`
`graph-tokens`		`None`Type: JSON
list		✅✅❌❌--piecewise-cuda-<br/>graph-compilereagereagerA2, A3
`--piecewise-cuda-`
`graph-compiler`		`eager`["eager", "inductor"]✅✅❌❌
`--torch-compile-max-bs``32`--torch-compile-max-bs32 Type: int✅✅❌❌A2, A3
`--piecewise-cuda-`
`graph-max-tokens`		`4096`--piecewise-cuda-<br/>graph-max-tokensNone Type: int✅✅❌❌A2, A3
`--torchao-config`--torchao-config `` Type: str❌❌✅❌Special for GPU
`--enable-nan-detection`--enable-nan-detectionFalsebool flag<br/> (set to enable)A2, A3
--enable-p2p-check `False` bool flag
(set to enable)		✅✅❌❌Special for GPU
`--enable-p2p-check`--triton-attention-<br/>reduce-in-fp32 `False` bool flag
(set to enable)		❌❌✅❌Special for GPU
`--triton-attention-`
`reduce-in-fp32`		`False`bool flag
(set to enable)		❌❌✅❌
`--triton-attention-`
`num-kv-splits`		`8`--triton-attention-<br/>num-kv-splits8 Type: int❌❌✅❌Special for GPU
`--triton-attention-`
`split-tile-size`		`None`--triton-attention-<br/>split-tile-sizeNone Type: int❌❌✅❌Special for GPU
`--delete-ckpt-`
`after-loading`		--delete-ckpt-<br/>after-loadingFalsebool flag<br/> (set to enable)A2, A3
--enable-memory-saver `False` bool flag
(set to enable)		✅✅❌❌A2, A3
`--enable-memory-saver`--enable-weights-<br/>cpu-backup `False` bool flag
(set to enable)		✅✅❌❌A2, A3
`--enable-weights-`
`cpu-backup`		--enable-draft-weights-<br/>cpu-backup `False` bool flag
(set to enable)		✅✅❌❌A2, A3
`--enable-draft-weights-`
`cpu-backup`		--allow-auto-truncate `False` bool flag
(set to enable)		✅✅❌❌A2, A3
`--allow-auto-truncate`--enable-custom-<br/>logit-processor `False` bool flag
(set to enable)		✅✅❌❌A2, A3
`--enable-custom-`
`logit-processor`		--flashinfer-mla-<br/>disable-ragged `False` bool flag
(set to enable)		✅✅❌❌Special for GPU
`--flashinfer-mla-`
`disable-ragged`		--disable-shared-<br/>experts-fusionTruebool flag
(set to enable)		A2, A3
--disable-chunked-<br/>prefix-cacheTruebool flag
(set to enable)		A2, A3
--disable-fast-<br/>image-processor `False` bool flag
(set to enable)		❌❌✅❌A2, A3
`--disable-shared-`
`experts-fusion`		--keep-mm-feature-<br/>on-device `False` bool flag
(set to enable)		✅✅❌❌A2, A3
`--disable-chunked-`
`prefix-cache`		--enable-return-<br/>hidden-states `False` bool flag
(set to enable)		✅✅❌❌A2, A3
`--disable-fast-`
`image-processor`		--enable-return-<br/>routed-experts `False` bool flag
(set to enable)		✅✅❌❌A2, A3
`--keep-mm-feature-`
`on-device`		`False`bool flag
(set to enable)		✅✅❌❌
`--enable-return-`
`hidden-states`		`False`bool flag
(set to enable)		✅✅❌❌
`--enable-return-`
`routed-experts`		`False`bool flag
(set to enable)		✅✅❌❌
`--scheduler-recv-`
`interval`		`1`--scheduler-recv-<br/>interval1 Type: int✅✅❌❌A2, A3
`--numa-node``None`--numa-nodeNone List[int]✅✅❌❌A2, A3
--enable-deterministic-<br/>inferenceFalsebool flag<br/> (set to enable)Planned
`--rl-on-policy-target` `None` `fsdp`❌❌❌✅Planned
`--enable-layerwise-`
`nvtx-marker`		 `False` bool flag
(set to enable)		❌❌✅❌Special for GPU
`--enable-attn-tp-`
`input-scattered`		 `False` bool flag
(set to enable)		❌❌❌❌Experimental
`--enable-nsa-prefill-`
`context-parallel`		 `False` bool flag
(set to enable)		✅✅❌❌A2, A3
`--enable-fused-qk-`
`norm-rope`		 `False` bool flag
(set to enable)		❌❌✅❌Special for GPU
- - - - - - +## Dynamic batch tokenizer @@ -2597,8 +2067,7 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + @@ -2606,32 +2075,24 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + - - + - - +
Argument Defaults OptionsA2A3Server supported
`--enable-dynamic-`
`batch-tokenizer`		 `False` bool flag
(set to enable)		✅✅A2, A3
`--dynamic-batch-`
`tokenizer-batch-size`		 `32` Type: int✅✅A2, A3
`--dynamic-batch-`
`tokenizer-batch-timeout`		 `0.002` Type: float✅✅A2, A3
- - - - - - +## Debug tensor dumps @@ -2646,8 +2107,7 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + @@ -2655,32 +2115,24 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + - - + - - +
Argument Defaults OptionsA2A3Server supported
`--debug-tensor-dump-`
`output-folder`		 `None` Type: str✅✅A2, A3
`--debug-tensor-dump-`
`layers`		 `None` List[int]✅✅A2, A3
`--debug-tensor-dump-`
`input-file`		 `None` Type: str✅✅A2, A3
- - - - - - +## PD disaggregation @@ -2696,101 +2148,56 @@ click [Server Arguments](../../advanced_features/server_arguments). - - - + - - - - + + - - - + - - - + - - - - - - - - - - - - - - - - - + - - - + - - - - - - + + + + - - - - - - - - - - + + - - - + - - + + - - - +
Argument Defaults OptionsA2A3SpecialServer supported
`--disaggregation-mode` `null``null`,
`prefill`,
`decode`		✅✅❌null,<br/> prefill,<br/> decodeA2, A3
`--disaggregation-transfer-backend` `mooncake` `ascend`✅✅❌A2, A3
`--disaggregation-bootstrap-port` `8998` Type: int✅✅❌A2, A3
`--disaggregation-decode-tp``None`Type: int✅✅❌
`--disaggregation-decode-dp``None`Type: int✅✅❌
`--disaggregation-ib-device`--disaggregation-ib-device `None` Type: str❌❌✅Special for GPU
`--disaggregation-decode-`
`enable-offload-kvcache`		`False`bool flag
(set to enable)		✅✅❌--disaggregation-decode-<br/>enable-offload-kvcacheFalseFalseA2, A3
`--disaggregation-decode-`
`enable-fake-auto`		`False`bool flag
(set to enable)		✅✅❌
`--num-reserved-decode-tokens``512`--num-reserved-decode-tokens512 Type: int✅✅❌A2, A3
`--disaggregation-decode-`
`polling-interval`		`1`--disaggregation-decode-<br/>polling-interval1 Type: int✅✅❌A2, A3
- - - - - - +## Encode prefill disaggregation @@ -2805,48 +2212,44 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + - + - - - + + - + - - - + + - - - - - + + + + + + + + + + - - - + + - +
Argument Defaults OptionsA2A3Server supported
`--encoder-only`--enable-adaptive-dispatch-to-encoder `False`bool flag
(set to enable)		✅✅bool flag<br/> (set to enable adaptively dispatch)A2, A3
`--language-only`--encoder-only `False`bool flag
(set to enable)		✅✅bool flag<br/> (set to launch an encoder-only server)A2, A3
`--encoder-transfer-backend``zmq_to_scheduler``zmq_to_scheduler`,
`zmq_to_tokenizer`,
`mooncake`		✅✅--language-onlyFalsebool flag<br/> (set to load weights for the language model only)A2, A3
--encoder-transfer-backendzmq_to_schedulerzmq_to_scheduler, <br/> zmq_to_tokenizer,<br/> mooncakeA2, A3
`--encoder-urls` `[]`List[str]✅✅List[str]<br/> (List of encoder server urls)A2, A3
- - - - - - +## Custom weight loader @@ -2862,9 +2265,7 @@ click [Server Arguments](../../advanced_features/server_arguments). - - - + @@ -2872,67 +2273,48 @@ click [Server Arguments](../../advanced_features/server_arguments). - - - + - - - + - - - + - - - + - - - + - - - - + + - - - +
Argument Defaults OptionsA2A3SpecialServer supported
`--custom-weight-loader` `None` List[str]✅✅❌A2, A3
`--weight-loader-disable-`
`mmap`		 `False` bool flag
(set to enable)		✅✅❌A2, A3
`--remote-instance-weight-`
`loader-seed-instance-ip`		 `None` Type: str✅✅❌A2, A3
`--remote-instance-weight-`
`loader-seed-instance-service-port`		 `None` Type: int✅✅❌A2, A3
`--remote-instance-weight-`
`loader-send-weights-group-ports`		 `None` Type: JSON
list		✅✅❌A2, A3
`--remote-instance-weight-`
`loader-backend`		 `nccl``transfer_engine`,
`nccl`		✅✅❌transfer_engine, <br/> ncclA2, A3
`--remote-instance-weight-`
`loader-start-seed-via-transfer-engine`		 `False` bool flag
(set to enable)		❌❌✅Special for GPU
- - - - - - +## For PD-Multiplexing @@ -2948,9 +2330,7 @@ click [Server Arguments](../../advanced_features/server_arguments). - - - + @@ -2958,35 +2338,24 @@ click [Server Arguments](../../advanced_features/server_arguments). - - - + - - - + - - - +
Argument Defaults OptionsA2A3SpecialServer supported
`--enable-pdmux` `False` bool flag
(set to enable)		❌❌✅Special for GPU
`--pdmux-config-path` `None` Type: str❌❌✅Special for GPU
`--sm-group-num` `8` Type: int❌❌✅Special for GPU
- - - - - - +## For Multi-Modal @@ -3001,62 +2370,38 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + - - - - - + + + + - - - - - + + + + - + - - + - + - - - - - - - - - - - - - - - - +
Argument Defaults OptionsA2A3Server supported
`--mm-max-concurrent-calls`32Type: int✅✅--enable-broadcast-mm-<br/>inputs-processFalsebool flag<br/> (set to enable)A2, A3
`--mm-per-request-timeout`10.0Type: float✅✅--mm-process-configNoneType: JSON / DictA2, A3
`--enable-broadcast-mm-`
`inputs-process`		--mm-enable-dp-encoder `False` bool flag
(set to enable)		✅✅A2, A3
`--mm-process-config`--limit-mm-data-per-request `None` Type: JSON / Dict✅✅
`--mm-enable-dp-encoder``False`bool flag
(set to enable)		✅✅
`--limit-mm-data-per-request``None`Type: JSON / Dict✅✅A2, A3
- - - - - - +## For checkpoint decryption @@ -3071,8 +2416,7 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + @@ -3080,32 +2424,24 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + - - + - - +
Argument Defaults OptionsA2A3Server supported
`--decrypted-config-file` `None` Type: str✅✅A2, A3
`--decrypted-draft-config-file` `None` Type: str✅✅A2, A3
`--enable-prefix-mm-cache` `False` bool flag
(set to enable)		✅✅A2, A3
- - - - - - +## Forward hooks @@ -3121,64 +2457,20 @@ click [Server Arguments](../../advanced_features/server_arguments). - - - + - - - - - - - - -
Argument Defaults OptionsA2A3PlannedServer supported
`--enable-deterministic-`
`inference`		`False`bool flag
(set to enable)		❌❌✅
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - +
ArgumentDefaultsOptionsA2A3
`--forward-hooks``None`--forward-hooksNone Type: JSON list✅✅A2, A3
- - - - - - +## Configuration file support @@ -3193,27 +2485,20 @@ click [Server Arguments](../../advanced_features/server_arguments). - - + - + - - +
Argument Defaults OptionsA2A3Server supported
`--config`--config `None` Type: str✅✅A2, A3
- - - - - - +## Other Params The following parameters are not supported because the third-party components that depend on are not compatible with the NPU, like Ktransformer, checkpoint-engine etc. @@ -3287,41 +2572,9 @@ The following parameters have some functional deficiencies on community - `--enable-double-sparsity` - `False` - bool flag
(set to enable) - - - `--ds-channel-config-path` - `None` - Type: str - - - `--ds-heavy-channel-num` - `32` - Type: int - - - `--ds-heavy-token-num` - `256` - Type: int - - - `--ds-heavy-channel-type` - `qk` - Type: str - - - `--ds-sparse-decode-`
`threshold` - `4096` - Type: int - - - `--tool-server` - `None` + --tool-server + None Type: str - - diff --git a/docs_new/docs/hardware-platforms/ascend-npus/Support-Models-on-Ascend-NPU.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_support_models.mdx similarity index 97% rename from docs_new/docs/hardware-platforms/ascend-npus/Support-Models-on-Ascend-NPU.mdx rename to docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_support_models.mdx index 6e07d021c..728346d74 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/Support-Models-on-Ascend-NPU.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_support_models.mdx @@ -1,9 +1,13 @@ +--- +title: "Support Models on Ascend NPU" +metatags: + description: "Documentation for Support Models on Ascend NPU" +--- This section describes the models supported on the Ascend NPU, including Large Language Models, Multimodal Language Models, Embedding Models, Reward Models and Rerank Models. Mainstream DeepSeek/Qwen/GLM series are included. You are welcome to enable various models based on your business requirements. - - +## Large Language Models @@ -16,8 +20,8 @@ You are welcome to enable various models based on your business requirements. - - + + @@ -28,19 +32,19 @@ You are welcome to enable various models based on your business requirements. - + - + - + @@ -64,7 +68,7 @@ You are welcome to enable various models based on your business requirements. - + @@ -88,7 +92,7 @@ You are welcome to enable various models based on your business requirements. - + @@ -256,13 +260,13 @@ You are welcome to enable various models based on your business requirements. - + - + @@ -274,7 +278,7 @@ You are welcome to enable various models based on your business requirements. - + @@ -300,12 +304,7 @@ You are welcome to enable various models based on your business requirements.
Models Model FamilyA2A3A2 SupportedA3 Supported
✅
vllm-ascend/DeepSeek-V3.2-Exp-W8A8DeepSeek-V3.2-W8A8 DeepSeek ✅ ✅
vllm-ascend/DeepSeek-R1-0528-W8A8DeepSeek-R1-0528-W8A8 DeepSeek ✅ ✅
vllm-ascend/DeepSeek-V2-Lite-W8A8DeepSeek-V2-Lite-W8A8 DeepSeek ✅ ✅✅
vllm-ascend/Qwen3-235B-A22B-W8A8Qwen3-235B-A22B-W8A8 Qwen ✅ ✅✅
vllm-ascend/QWQ-32B-W8A8QWQ-32B-W8A8 Qwen ✅ ✅✅
Kimi/Kimi-K2-Thinkingmoonshotai/Kimi-K2-Thinking Kimi ✅ ✅
openai/gpt-oss-120beigen-ai-labs/gpt-oss-120b-bf16 GPTOSS ✅ ✅✅
minimax/MiniMax-M2cyankiwi/MiniMax-M2-BF16 MiniMax-M2 ✅ ✅
- - - - - - +## Multimodal Language Models @@ -318,8 +317,8 @@ You are welcome to enable various models based on your business requirements. - - + + @@ -432,7 +431,7 @@ You are welcome to enable various models based on your business requirements. - + @@ -458,12 +457,7 @@ You are welcome to enable various models based on your business requirements.
Models Model Family (Variants)A2A3A2 SupportedA3 Supported
✅
Kimi/Kimi-VL-A3B-Instructmoonshotai/Kimi-VL-A3B-Instruct Kimi-VL (A3B) ✅ ✅
- - - - - - +## Embedding Models @@ -476,8 +470,8 @@ You are welcome to enable various models based on your business requirements. - - + + @@ -520,12 +514,7 @@ You are welcome to enable various models based on your business requirements.
Models Model FamilyA2A3A2 SupportedA3 Supported
- - - - - - +## Reward Models @@ -538,8 +527,8 @@ You are welcome to enable various models based on your business requirements. - - + + @@ -576,12 +565,7 @@ You are welcome to enable various models based on your business requirements.
Models Model FamilyA2A3A2 SupportedA3 Supported
- - - - - - +## Rerank Models @@ -594,8 +578,8 @@ You are welcome to enable various models based on your business requirements. - - + + @@ -607,4 +591,3 @@ You are welcome to enable various models based on your business requirements.
Models Model FamilyA2A3A2 SupportedA3 Supported
- diff --git a/docs_new/docs/hardware-platforms/ascend-npus/MindSpore-Models.mdx b/docs_new/docs/hardware-platforms/ascend-npus/mindspore_backend.mdx similarity index 97% rename from docs_new/docs/hardware-platforms/ascend-npus/MindSpore-Models.mdx rename to docs_new/docs/hardware-platforms/ascend-npus/mindspore_backend.mdx index 63ccfcf86..32adcbb7d 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/MindSpore-Models.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/mindspore_backend.mdx @@ -18,7 +18,7 @@ Currently, the following models are supported: ## Installation -Currently, MindSpore models are provided by an independent package `sgl-mindspore`. Support for MindSpore is built upon current SGLang support for Ascend NPU platform. Please first [install SGLang for Ascend NPU](./SGLang-installation-with-NPUs-support) and then install `sgl-mindspore`: +Currently, MindSpore models are provided by an independent package `sgl-mindspore`. Support for MindSpore is built upon current SGLang support for Ascend NPU platform. Please first [install SGLang for Ascend NPU](./ascend_npu) and then install `sgl-mindspore`: diff --git a/docs_new/docs/hardware-platforms/cpu-server.mdx b/docs_new/docs/hardware-platforms/cpu-server.mdx deleted file mode 100644 index b3fa72e36..000000000 --- a/docs_new/docs/hardware-platforms/cpu-server.mdx +++ /dev/null @@ -1,355 +0,0 @@ ---- -title: "CPU Servers" ---- - -The document addresses how to set up the [SGLang](https://github.com/sgl-project/sglang) environment and run LLM inference on CPU servers. -SGLang is enabled and optimized on the CPUs equipped with Intel® AMX® Instructions, -which are 4th generation or newer Intel® Xeon® Scalable Processors. - -## Optimized Model List - -A list of popular LLMs are optimized and run efficiently on CPU, -including the most notable open-source models like Llama series, Qwen series, -and DeepSeek series like DeepSeek-R1 and DeepSeek-V3.1-Terminus. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Model NameBF16W8A8_INT8FP8
DeepSeek-R1—DeepSeek-R1-Channel-INT8DeepSeek-R1
DeepSeek-V3.1-Terminus—DeepSeek-V3.1-Terminus-Channel-int8DeepSeek-V3.1-Terminus
Llama-3.2-3BLlama-3.2-3B-InstructLlama-3.2-3B-quantized.w8a8—
Llama-3.1-8BLlama-3.1-8B-InstructLlama-3.1-8B-quantized.w8a8—
QwQ-32B—QwQ-32B-quantized.w8a8—
DeepSeek-Distilled-Llama—DeepSeek-R1-Distill-Llama-70B-quantized.w8a8—
Qwen3-235B——Qwen3-235B-A22B-FP8
- -> **Note:** The model identifiers listed in the table above have been verified on 6th Gen Intel® Xeon® P-core platforms. - -## Installation - - - - It is recommended to use Docker for setting up the SGLang environment. - A [Dockerfile](https://github.com/sgl-project/sglang/blob/main/docker/xeon.Dockerfile) is provided to facilitate the installation. - - > **Note:** Replace `` below with your [HuggingFace access token](https://huggingface.co/docs/hub/en/security-tokens). - - - ```bash Clone, Build and Run - # Clone the SGLang repository - git clone https://github.com/sgl-project/sglang.git - cd sglang/docker - - # Build the docker image - docker build -t sglang-cpu:latest -f xeon.Dockerfile . - - # Initiate a docker container - docker run \ - -it \ - --privileged \ - --ipc=host \ - --network=host \ - -v /dev/shm:/dev/shm \ - -v ~/.cache/huggingface:/root/.cache/huggingface \ - -p 30000:30000 \ - -e "HF_TOKEN=" \ - sglang-cpu:latest /bin/bash - ``` - - - - - If you prefer to install SGLang in a bare metal environment, the setup process is as follows. - - Please install the required packages and libraries beforehand if they are not already present on your system. - You can refer to the Ubuntu-based installation commands in - [the Dockerfile](https://github.com/sgl-project/sglang/blob/main/docker/xeon.Dockerfile#L11) for guidance. - - 1. **Install uv and create a virtual environment** - - ```bash Create Virtual Environment - # Taking '/opt' as the example uv env folder, feel free to change it as needed - cd /opt - curl -LsSf https://astral.sh/uv/install.sh | sh - source $HOME/.local/bin/env - uv venv --python 3.12 - source .venv/bin/activate - ``` - - - 2. **Create a config file for torch package indexes** - Create the `uv.toml` config file: - - - ```bash Open Config File - vim .venv/uv.toml - ``` - - - Press `a` to enter insert mode in `vim`, then paste the following content: - - - ```toml - [[index]] - name = "torch" - url = "https://download.pytorch.org/whl/cpu" - - [[index]] - name = "torchvision" - url = "https://download.pytorch.org/whl/cpu" - - [[index]] - name = "torchaudio" - url = "https://download.pytorch.org/whl/cpu" - - [[index]] - name = "triton" - url = "https://download.pytorch.org/whl/cpu" - ``` - - - Save the file (press `Esc`, then type `:x` and hit `Enter`), then set it as the default `uv` config: - - - ```bash Set Config Path - export UV_CONFIG_FILE=/opt/.venv/uv.toml - ``` - - - 3. **Clone SGLang and build packages** - - ```bash Build SGLang - # Clone the SGLang code - git clone https://github.com/sgl-project/sglang.git - cd sglang - git checkout - - # Use dedicated toml file - cd python - cp pyproject_cpu.toml pyproject.toml - # Install SGLang dependent libs, and build SGLang main package - uv pip install --upgrade pip setuptools - uv pip install . - - # Build the CPU backend kernels - cd ../sgl-kernel - cp pyproject_cpu.toml pyproject.toml - uv pip install . - ``` - - - 4. **Set required environment variables** - - ```bash Set Environment Variables - export SGLANG_USE_CPU_ENGINE=1 - - # Set 'LD_LIBRARY_PATH' and 'LD_PRELOAD' to ensure the libs can be loaded by sglang processes - export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu - export LD_PRELOAD=${LD_PRELOAD}:/opt/.venv/lib/libiomp5.so:${LD_LIBRARY_PATH}/libtcmalloc.so.4:${LD_LIBRARY_PATH}/libtbbmalloc.so.2 - ``` - - - > **Note:** The environment variable `SGLANG_USE_CPU_ENGINE=1` is required to enable the SGLang service with the CPU engine. - - > **Note:** If you encounter code compilation issues during the `sgl-kernel` building process, please check your `gcc` and `g++` versions and upgrade them if they are outdated. It is recommended to use `gcc-13` and `g++-13` as they have been verified in the official Docker container. - - > **Note:** The system library path is typically located in one of the following directories: `~/.local/lib/`, `/usr/local/lib/`, `/usr/local/lib64/`, `/usr/lib/`, `/usr/lib64/`, and `/usr/lib/x86_64-linux-gnu/`. In the above example commands, `/usr/lib/x86_64-linux-gnu` is used. Please adjust the path according to your server configuration. - - It is recommended to add the following to your `~/.bashrc` file to avoid setting these variables every time you open a new terminal: - - - ```bash Persist in ~/.bashrc - source .venv/bin/activate - export SGLANG_USE_CPU_ENGINE=1 - export LD_LIBRARY_PATH= - export LD_PRELOAD= - ``` - - - - -## Launch of the Serving Engine - -Example command to launch SGLang serving: - - -```bash Launch Server -python -m sglang.launch_server \ - --model \ - --trust-remote-code \ - --disable-overlap-schedule \ - --device cpu \ - --host 0.0.0.0 \ - --tp 6 -``` - - -> **Note:** For running W8A8 quantized models, please add the flag `--quantization w8a8_int8`. - -> **Note:** The flag `--tp 6` specifies that tensor parallelism will be applied using 6 ranks (TP6). On a CPU platform, a TP rank means a sub-NUMA cluster (SNC). You can get the SNC count using `lscpu`. If the specified TP rank number differs from the total SNC count, the system will automatically utilize the first `n` SNCs — but `n` cannot exceed the total SNC number. -> -> To specify the cores to be used, set the environment variable `SGLANG_CPU_OMP_THREADS_BIND`. For example, to use the first 40 cores of each SNC on a Xeon® 6980P server (which has 43-43-42 cores on the 3 SNCs of a socket): - - -```bash Set Thread Binding -export SGLANG_CPU_OMP_THREADS_BIND="0-39|43-82|86-125|128-167|171-210|214-253" -``` - - -> Please beware that with `SGLANG_CPU_OMP_THREADS_BIND` set, the available memory amounts of the ranks may not be determined in advance. You may need to set `--max-total-tokens` to avoid out-of-memory errors. - -> **Note:** For optimizing decoding with `torch.compile`, add the flag `--enable-torch-compile`. To specify the maximum batch size, set `--torch-compile-max-bs`. For example, `--enable-torch-compile --torch-compile-max-bs 4` uses `torch.compile` with a maximum batch size of 4. The maximum applicable batch size is 16. - -> **Note:** A warmup step is automatically triggered when the service is started. The server is ready when you see the log `The server is fired up and ready to roll!`. - -## Benchmarking with Requests - -You can benchmark the performance via the `bench_serving` script. -Run the command in another terminal. An example command would be: - - -```bash Run Benchmark -python -m sglang.bench_serving \ - --dataset-name random \ - --random-input-len 1024 \ - --random-output-len 1024 \ - --num-prompts 1 \ - --request-rate inf \ - --random-range-ratio 1.0 -``` - - -Detailed parameter descriptions are available via the command: - - -```bash Benchmark Help -python -m sglang.bench_serving -h -``` - - -Additionally, requests can be formatted using -[the OpenAI Completions API](../basic_usage/openai_api_completions) -and sent via the command line (e.g., using `curl`) or through your own scripts. - -## Example Usage Commands - -Large Language Models can range from fewer than 1 billion to several hundred billion parameters. -Dense models larger than 20B are expected to run on flagship 6th Gen Intel® Xeon® processors -with dual sockets and a total of 6 sub-NUMA clusters. Dense models of approximately 10B parameters or fewer, -or MoE (Mixture of Experts) models with fewer than 10B activated parameters, can run on more common -4th generation or newer Intel® Xeon® processors, or utilize a single socket of the flagship 6th Gen Intel® Xeon® processors. - -### Example: Running DeepSeek-V3.1-Terminus - - -```bash W8A8_INT8 -python -m sglang.launch_server \ - --model IntervitensInc/DeepSeek-V3.1-Terminus-Channel-int8 \ - --trust-remote-code \ - --disable-overlap-schedule \ - --device cpu \ - --quantization w8a8_int8 \ - --host 0.0.0.0 \ - --enable-torch-compile \ - --torch-compile-max-bs 4 \ - --tp 6 -``` - -```bash FP8 -python -m sglang.launch_server \ - --model deepseek-ai/DeepSeek-V3.1-Terminus \ - --trust-remote-code \ - --disable-overlap-schedule \ - --device cpu \ - --host 0.0.0.0 \ - --enable-torch-compile \ - --torch-compile-max-bs 4 \ - --tp 6 -``` - - -> **Note:** Please set `--torch-compile-max-bs` to the maximum desired batch size for your deployment, which can be up to 16. The value `4` in the examples is illustrative. - -### Example: Running Llama-3.2-3B - - -```bash BF16 -python -m sglang.launch_server \ - --model meta-llama/Llama-3.2-3B-Instruct \ - --trust-remote-code \ - --disable-overlap-schedule \ - --device cpu \ - --host 0.0.0.0 \ - --enable-torch-compile \ - --torch-compile-max-bs 16 \ - --tp 2 -``` - -```bash W8A8_INT8 -python -m sglang.launch_server \ - --model RedHatAI/Llama-3.2-3B-quantized.w8a8 \ - --trust-remote-code \ - --disable-overlap-schedule \ - --device cpu \ - --quantization w8a8_int8 \ - --host 0.0.0.0 \ - --enable-torch-compile \ - --torch-compile-max-bs 16 \ - --tp 2 -``` - - -> **Note:** The `--torch-compile-max-bs` and `--tp` settings are examples that should be adjusted for your setup. For instance, use `--tp 3` to utilize 1 socket with 3 sub-NUMA clusters on an Intel® Xeon® 6980P server. - -Once the server has been launched, you can test it using the `bench_serving` command or create -your own commands or scripts following [the benchmarking example](#benchmarking-with-requests). diff --git a/docs_new/docs/hardware-platforms/cpu_server.mdx b/docs_new/docs/hardware-platforms/cpu_server.mdx new file mode 100644 index 000000000..8d2d36b61 --- /dev/null +++ b/docs_new/docs/hardware-platforms/cpu_server.mdx @@ -0,0 +1,387 @@ +--- +title: "CPU Servers" +--- +The document addresses how to set up the [SGLang](https://github.com/sgl-project/sglang) environment and run LLM inference on CPU servers. +SGLang is enabled and optimized on the CPUs equipped with Intel® AMX® Instructions, +which are 4th generation or newer Intel® Xeon® Scalable Processors. + +## Optimized Model List + +A list of popular LLMs are optimized and run efficiently on CPU, +including the most notable open-source models like Llama series, Qwen series, +and DeepSeek series like DeepSeek-R1 and DeepSeek-V3.1-Terminus. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Model NameBF16W8A8_INT8FP8
DeepSeek-R1meituan/DeepSeek-R1-Channel-INT8deepseek-ai/DeepSeek-R1
DeepSeek-V3.1-TerminusIntervitensInc/DeepSeek-V3.1-Terminus-Channel-int8deepseek-ai/DeepSeek-V3.1-Terminus
Llama-3.2-3Bmeta-llama/Llama-3.2-3B-InstructRedHatAI/Llama-3.2-3B-quantized.w8a8
Llama-3.1-8Bmeta-llama/Llama-3.1-8B-InstructRedHatAI/Meta-Llama-3.1-8B-quantized.w8a8
QwQ-32BRedHatAI/QwQ-32B-quantized.w8a8
DeepSeek-Distilled-LlamaRedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8
Qwen3-235BQwen/Qwen3-235B-A22B-FP8
+ +**Note:** The model identifiers listed in the table above +have been verified on 6th Gen Intel® Xeon® P-core platforms. + +## Installation + +### Install Using Docker + +It is recommended to use Docker for setting up the SGLang environment. +A [Dockerfile](https://github.com/sgl-project/sglang/blob/main/docker/xeon.Dockerfile) is provided to facilitate the installation. +Replace `` below with your [HuggingFace access token](https://huggingface.co/docs/hub/en/security-tokens). + +```bash Command +# Clone the SGLang repository +git clone https://github.com/sgl-project/sglang.git +cd sglang/docker + +# Build the docker image +docker build -t sglang-cpu:latest -f xeon.Dockerfile . + +# Initiate a docker container +docker run \ + -it \ + --privileged \ + --ipc=host \ + --network=host \ + -v /dev/shm:/dev/shm \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + -p 30000:30000 \ + -e "HF_TOKEN=" \ + sglang-cpu:latest /bin/bash +``` + +### Install From Source + +If you prefer to install SGLang in a bare metal environment, +the setup process is as follows: + +Please install the required packages and libraries beforehand if +they are not already present on your system. +You can refer to the Ubuntu-based installation commands in +[the Dockerfile](https://github.com/sgl-project/sglang/blob/main/docker/xeon.Dockerfile#L11) +for guidance. + +1. Install `uv` package manager, then create and activate a virtual environment: + +```bash Command +# Taking '/opt' as the example uv env folder, feel free to change it as needed +cd /opt +curl -LsSf https://astral.sh/uv/install.sh | sh +source $HOME/.local/bin/env +uv venv --python 3.12 +source .venv/bin/activate +``` + +2. Create a config file to direct the installation channel + (a.k.a. index-url) of `torch` related packages: + +```bash Command +vim .venv/uv.toml +``` + +Press 'a' to enter insert mode of `vim`, paste the following content into the created file + +```file +[[index]] +name = "torch" +url = "https://download.pytorch.org/whl/cpu" + +[[index]] +name = "torchvision" +url = "https://download.pytorch.org/whl/cpu" + +[[index]] +name = "torchaudio" +url = "https://download.pytorch.org/whl/cpu" + +[[index]] +name = "triton" +url = "https://download.pytorch.org/whl/cpu" + +``` + +Save the file (in `vim`, press 'esc' to exit insert mode, then ':x+Enter'), +and set it as the default `uv` config. + +```bash Command +export UV_CONFIG_FILE=/opt/.venv/uv.toml +``` + +3. Clone the `sglang` source code and build the packages + +```bash Command +# Clone the SGLang code +git clone https://github.com/sgl-project/sglang.git +cd sglang +git checkout + +# Use dedicated toml file +cd python +cp pyproject_cpu.toml pyproject.toml +# Install SGLang dependent libs, and build SGLang main package +uv pip install --upgrade pip setuptools +uv pip install . + +# Build the CPU backend kernels +cd ../sgl-kernel +cp pyproject_cpu.toml pyproject.toml +uv pip install . +``` + +4. Set the required environment variables + +```bash Command +export SGLANG_USE_CPU_ENGINE=1 + +# Set 'LD_LIBRARY_PATH' and 'LD_PRELOAD' to ensure the libs can be loaded by sglang processes +export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu +export LD_PRELOAD=${LD_PRELOAD}:/opt/.venv/lib/libiomp5.so:${LD_LIBRARY_PATH}/libtcmalloc.so.4:${LD_LIBRARY_PATH}/libtbbmalloc.so.2 +``` + +Notes: + +- Note that the environment variable `SGLANG_USE_CPU_ENGINE=1` + is required to enable the SGLang service with the CPU engine. + +- If you encounter code compilation issues during the `sgl-kernel` building process, + please check your `gcc` and `g++` versions and upgrade them if they are outdated. + It is recommended to use `gcc-13` and `g++-13` as they have been verified + in the official Docker container. + +- The system library path is typically located in one of the following directories: + `~/.local/lib/`, `/usr/local/lib/`, `/usr/local/lib64/`, `/usr/lib/`, `/usr/lib64/` + and `/usr/lib/x86_64-linux-gnu/`. In the above example commands, `/usr/lib/x86_64-linux-gnu` + is used. Please adjust the path according to your server configuration. + +- It is recommended to add the following to your `~/.bashrc` file to + avoid setting these variables every time you open a new terminal: + + ```bash Command + source .venv/bin/activate + export SGLANG_USE_CPU_ENGINE=1 + export LD_LIBRARY_PATH= + export LD_PRELOAD= + ``` + +## Launch of the Serving Engine + +Example command to launch SGLang serving: + +```bash Launch Server +python -m sglang.launch_server \ + --model \ + --trust-remote-code \ + --disable-overlap-schedule \ + --device cpu \ + --host 0.0.0.0 \ + --tp 6 +``` + +Notes: + +1. For running W8A8 quantized models, please add the flag `--quantization w8a8_int8`. + +2. The flag `--tp 6` specifies that tensor parallelism will be applied using 6 ranks (TP6). + The number of TP specified is how many TP ranks will be used during the execution. + On a CPU platform, a TP rank means a sub-NUMA cluster (SNC). + Usually we can get the SNC information (How many available) from the Operating System with e.g. `lscpu` command. + + If the specified TP rank number differs from the total SNC count, + the system will automatically utilize the first `n` SNCs. + Note that `n` cannot exceed the total SNC number, doing so will result in an error. + + `SGLANG_CPU_OMP_THREADS_BIND` allows explicit control of CPU cores for each tensor parallel (TP) rank. + + **example 1**: Run SGLang service with TP=6, using the first 40 cores of each SNC on a Xeon® 6980P server, + which has 43-43-42 cores on the 3 SNCs of a socket, we should set: + + ```bash Command + export SGLANG_CPU_OMP_THREADS_BIND="0-39|43-82|86-125|128-167|171-210|214-253" + ``` + This configuration is equivalent to: + - rank 0: `numactl -C 0-39 -m 0` + - rank 1: `numactl -C 43-82 -m 1` + - rank 2: `numactl -C 86-125 -m 2` + - rank 3: `numactl -C 128-167 -m 3` + - rank 4: `numactl -C 171-210 -m 4` + - rank 5: `numactl -C 214-253 -m 5` + + + **example 2**: Run SGLang service with TP=2, using 96 cores cross 3 SNCs on a Xeon® 6972P server, + which has 32-32-32 cores on the 3 SNCs in a socket, we should set: + ```bash Command + export SGLANG_CPU_OMP_THREADS_BIND="0-95|96-191" + ``` + This configuration is equivalent to: + - rank 0: `numactl -C 0-95 -m 0-2` + - rank 1: `numactl -C 96-191 -m 3-5` + + Please beware that with SGLANG_CPU_OMP_THREADS_BIND set, + the available memory amounts of the ranks may not be determined in prior. + You may need to set proper `--max-total-tokens` to avoid the out-of-memory error. + +3. For optimizing decoding with torch.compile, please add the flag `--enable-torch-compile`. + To specify the maximum batch size when using `torch.compile`, set the flag `--torch-compile-max-bs`. + For example, `--enable-torch-compile --torch-compile-max-bs 4` means using `torch.compile` + and setting the maximum batch size to 4. + +4. A warmup step is automatically triggered when the service is started. + The server is ready when you see the log `The server is fired up and ready to roll!`. + +## Benchmarking with Requests + +You can benchmark the performance via the `bench_serving` script. +Run the command in another terminal. An example command would be: + +```bash Run Benchmark +python -m sglang.bench_serving \ + --dataset-name random \ + --random-input-len 1024 \ + --random-output-len 1024 \ + --num-prompts 1 \ + --request-rate inf \ + --random-range-ratio 1.0 +``` + +Detailed parameter descriptions are available via the command: + +```bash Benchmark Help +python -m sglang.bench_serving -h +``` + +Additionally, requests can be formatted using +[the OpenAI Completions API](../basic_usage/openai_api_completions) +and sent via the command line (e.g., using `curl`) or through your own scripts. + +## Example Usage Commands + +Large Language Models can range from fewer than 1 billion to several hundred billion parameters. +Dense models larger than 20B are expected to run on flagship 6th Gen Intel® Xeon® processors +with dual sockets and a total of 6 sub-NUMA clusters. Dense models of approximately 10B parameters or fewer, +or MoE (Mixture of Experts) models with fewer than 10B activated parameters, can run on more common +4th generation or newer Intel® Xeon® processors, or utilize a single socket of the flagship 6th Gen Intel® Xeon® processors. + +### Example: Running DeepSeek-V3.1-Terminus + +An example command to launch service of W8A8_INT8 DeepSeek-V3.1-Terminus on a Xeon® 6980P server: + +```bash W8A8_INT8 +python -m sglang.launch_server \ + --model IntervitensInc/DeepSeek-V3.1-Terminus-Channel-int8 \ + --trust-remote-code \ + --disable-overlap-schedule \ + --device cpu \ + --quantization w8a8_int8 \ + --host 0.0.0.0 \ + --enable-torch-compile \ + --torch-compile-max-bs 4 \ + --tp 6 +``` + +Similarly, an example command to launch service of FP8 DeepSeek-V3.1-Terminus would be: + +```bash FP8 +python -m sglang.launch_server \ + --model deepseek-ai/DeepSeek-V3.1-Terminus \ + --trust-remote-code \ + --disable-overlap-schedule \ + --device cpu \ + --host 0.0.0.0 \ + --enable-torch-compile \ + --torch-compile-max-bs 4 \ + --tp 6 +``` + +Note: Please set `--torch-compile-max-bs` to the maximum desired batch size for your deployment, +which can be up to 16. The value `4` in the examples is illustrative. + +### Example: Running Llama-3.2-3B + +An example command to launch service of Llama-3.2-3B with BF16 precision: + +```bash BF16 +python -m sglang.launch_server \ + --model meta-llama/Llama-3.2-3B-Instruct \ + --trust-remote-code \ + --disable-overlap-schedule \ + --device cpu \ + --host 0.0.0.0 \ + --enable-torch-compile \ + --torch-compile-max-bs 16 \ + --tp 2 +``` + +The example command to launch service of W8A8_INT8 version of Llama-3.2-3B: + +```bash W8A8_INT8 +python -m sglang.launch_server \ + --model RedHatAI/Llama-3.2-3B-quantized.w8a8 \ + --trust-remote-code \ + --disable-overlap-schedule \ + --device cpu \ + --quantization w8a8_int8 \ + --host 0.0.0.0 \ + --enable-torch-compile \ + --torch-compile-max-bs 16 \ + --tp 2 +``` + +Note: The `--torch-compile-max-bs` and `--tp` settings are examples that should be adjusted for your setup. +For instance, use `--tp 3` to utilize 1 socket with 3 sub-NUMA clusters on an Intel® Xeon® 6980P server. + +Once the server have been launched, you can test it using the `bench_serving` command or create +your own commands or scripts following [the benchmarking example](#benchmarking-with-requests). diff --git a/docs_new/docs/hardware-platforms/mthreads_gpu.mdx b/docs_new/docs/hardware-platforms/mthreads_gpu.mdx new file mode 100644 index 000000000..a1df3bd05 --- /dev/null +++ b/docs_new/docs/hardware-platforms/mthreads_gpu.mdx @@ -0,0 +1,29 @@ +--- +title: "Moore Threads GPUs" +metatags: + description: "Run SGLang on Moore Threads GPUs." +--- + +This document describes how run SGLang on Moore Threads GPUs. If you encounter issues or have questions, please [open an issue](https://github.com/sgl-project/sglang/issues). + +## Install SGLang + +You can install SGLang using one of the methods below. + +### Install from Source + +```bash +# Use the default branch +git clone https://github.com/sgl-project/sglang.git +cd sglang + +# Compile sgl-kernel +pip install --upgrade pip +cd sgl-kernel +python setup_musa.py install + +# Install sglang python package +cd .. +rm -f python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml +pip install -e "python[all_musa]" +``` diff --git a/docs_new/docs/hardware-platforms/nvidia-gpus.mdx b/docs_new/docs/hardware-platforms/nvidia-gpus.mdx index 6d85d1d07..979bed120 100644 --- a/docs_new/docs/hardware-platforms/nvidia-gpus.mdx +++ b/docs_new/docs/hardware-platforms/nvidia-gpus.mdx @@ -2,4 +2,4 @@ title: NVIDIA GPUs --- -Please refer to the [Installation Guide](/docs/get-started/installation) to get started with SGLang on NVIDIA GPUs. +Please refer to the [Installation Guide](../get-started/install) to get started with SGLang on NVIDIA GPUs. diff --git a/docs_new/docs/hardware-platforms/nvidia.mdx b/docs_new/docs/hardware-platforms/nvidia.mdx deleted file mode 100644 index b4703e81b..000000000 --- a/docs_new/docs/hardware-platforms/nvidia.mdx +++ /dev/null @@ -1,102 +0,0 @@ ---- -title: NVIDIA Jetson Orin -description: Guide for installing and running SGLang on NVIDIA Jetson Orin devices. ---- - -## Prerequisites - -Before starting, ensure the following: - -- [NVIDIA Jetson AGX Orin Devkit](https://www.nvidia.com/en-us/autonomous-machines/embedded-systems/jetson-orin/) is set up with JetPack 6.1 or later. -- CUDA Toolkit and cuDNN are installed. -- Verify that the Jetson AGX Orin is in high-performance mode: - - -```bash -sudo nvpmodel -m 0 -``` - - -## Installing and Running SGLang with Jetson Containers - -1. **Clone the jetson-containers repository** - -```bash -git clone https://github.com/dusty-nv/jetson-containers.git -``` - -2. **Run the installation script** - -```bash -bash jetson-containers/install.sh -``` - -3. **Build the container image** - -```bash -jetson-containers build sglang -``` - -4. **Run the container** - - - - ```bash - jetson-containers run $(autotag sglang) - ``` - - - ```bash - docker run --runtime nvidia -it --rm --network=host IMAGE_NAME - ``` - - - -## Running Inference - -Launch the server: - - -```bash -python -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-R1-Distill-Llama-8B \ - --device cuda \ - --dtype half \ - --attention-backend flashinfer \ - --mem-fraction-static 0.8 \ - --context-length 8192 -``` - - -The quantization and limited context length (`--dtype half` `--context-length 8192`) are due to the limited computational resources in [Nvidia jetson kit](https://www.nvidia.com/en-us/autonomous-machines/embedded-systems/jetson-orin/). A detailed explanation can be found in [Server Arguments](../advanced_features/server_arguments). - -After launching the engine, refer to [Chat completions](../basic_usage/openai_api_completions#Usage) to test the usability. - -## Running Quantization with TorchAO - -TorchAO is suggested to NVIDIA Jetson Orin. - - -```bash -python -m sglang.launch_server \ - --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ - --device cuda \ - --dtype bfloat16 \ - --attention-backend flashinfer \ - --mem-fraction-static 0.8 \ - --context-length 8192 \ - --torchao-config int4wo-128 -``` - - -This enables TorchAO's int4 weight-only quantization with a 128-group size. The usage of `--torchao-config int4wo-128` is also for memory efficiency. - -## Structured Output with XGrammar - -Please refer to [SGLang doc structured output](../advanced_features/structured_outputs). - -Thanks to the support from [Nurgaliyev Shakhizat](https://github.com/shahizat), [Dustin Franklin](https://github.com/dusty-nv) and [Johnny Núñez Cano](https://github.com/johnnynunez). - -## References - -- [NVIDIA Jetson AGX Orin Documentation](https://developer.nvidia.com/embedded/jetson-agx-orin) diff --git a/docs_new/docs/hardware-platforms/nvidia_jetson.mdx b/docs_new/docs/hardware-platforms/nvidia_jetson.mdx new file mode 100644 index 000000000..26f8e58d4 --- /dev/null +++ b/docs_new/docs/hardware-platforms/nvidia_jetson.mdx @@ -0,0 +1,82 @@ +--- +title: NVIDIA Jetson Orin +description: Guide for installing and running SGLang on NVIDIA Jetson Orin devices. +--- +## Prerequisites + +Before starting, ensure the following: + +- [**NVIDIA Jetson AGX Orin Devkit**](https://www.nvidia.com/en-us/autonomous-machines/embedded-systems/jetson-orin/) is set up with **JetPack 6.1** or later. +- **CUDA Toolkit** and **cuDNN** are installed. +- Verify that the Jetson AGX Orin is in **high-performance mode**: +```bash +sudo nvpmodel -m 0 +``` +* * * * * +## Installing and running SGLang with Jetson Containers +Clone the jetson-containers github repository: +```bash +git clone https://github.com/dusty-nv/jetson-containers.git +``` +Run the installation script: +```bash +bash jetson-containers/install.sh +``` +Build the container image: +```bash +jetson-containers build sglang +``` +Run the container: +``` +jetson-containers run $(autotag sglang) +``` +Or you can also manually run a container with this command: +``` +docker run --runtime nvidia -it --rm --network=host IMAGE_NAME +``` +* * * * * + +Running Inference +----------------------------------------- + +Launch the server: +```bash +python -m sglang.launch_server \ + --model-path deepseek-ai/DeepSeek-R1-Distill-Llama-8B \ + --device cuda \ + --dtype half \ + --attention-backend flashinfer \ + --mem-fraction-static 0.8 \ + --context-length 8192 +``` +The quantization and limited context length (`--dtype half --context-length 8192`) are due to the limited computational resources in [Nvidia jetson kit](https://www.nvidia.com/en-us/autonomous-machines/embedded-systems/jetson-orin/). A detailed explanation can be found in [Server Arguments](../advanced_features/server_arguments). + +After launching the engine, refer to [Chat completions](../basic_usage/openai_api_completions#Usage) to test the usability. +* * * * * +Running quantization with TorchAO +------------------------------------- +TorchAO is suggested to NVIDIA Jetson Orin. +```bash Command +python -m sglang.launch_server \ + --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ + --device cuda \ + --dtype bfloat16 \ + --attention-backend flashinfer \ + --mem-fraction-static 0.8 \ + --context-length 8192 \ + --torchao-config int4wo-128 +``` +This enables TorchAO's int4 weight-only quantization with a 128-group size. The usage of `--torchao-config int4wo-128` is also for memory efficiency. + + +* * * * * +Structured output with XGrammar +------------------------------- +Please refer to [SGLang doc structured output](../advanced_features/structured_outputs). +* * * * * + +Thanks to the support from [Nurgaliyev Shakhizat](https://github.com/shahizat), [Dustin Franklin](https://github.com/dusty-nv) and [Johnny Núñez Cano](https://github.com/johnnynunez). + +References +---------- +- [NVIDIA Jetson AGX Orin Documentation](https://developer.nvidia.com/embedded/jetson-agx-orin) diff --git a/docs_new/docs/hardware-platforms/overview.mdx b/docs_new/docs/hardware-platforms/overview.mdx index b53e06fa0..5bb3c46f9 100644 --- a/docs_new/docs/hardware-platforms/overview.mdx +++ b/docs_new/docs/hardware-platforms/overview.mdx @@ -4,9 +4,9 @@ description: Platform-specific guides for running SGLang on GPUs, TPUs, NPUs, CP --- - [NVIDIA GPUs](./nvidia-gpus) -- [AMD GPUs](./amd-gpus) -- [Ascend NPUs](./ascend-npus/SGLang-installation-with-NPUs-support) -- [CPU Server](./cpu-server) -- [NVIDIA (Edge & Embedded)](./nvidia) +- [AMD GPUs](./amd_gpu) +- [Ascend NPUs](./ascend-npus/ascend_npu) +- [CPU Server](./cpu_server) +- [NVIDIA Jetson Orin](./nvidia_jetson) - [TPU](./tpu) - [XPU](./xpu) diff --git a/docs_new/docs/hardware-platforms/plugin.mdx b/docs_new/docs/hardware-platforms/plugin.mdx new file mode 100644 index 000000000..676eb0450 --- /dev/null +++ b/docs_new/docs/hardware-platforms/plugin.mdx @@ -0,0 +1,849 @@ +--- +title: "SGLang Plugin System" +metatags: + description: "Allows hardware vendors and developers to extend SGLang without modifying the main repository code." +--- + +## Overview + +Allows hardware vendors and developers to extend SGLang **without modifying the main repository code**. + +The framework provides two plugin types, both discovered via Python's standard `setuptools` entry_points: + + + + + + + + + + + + + + + + + + + + + + + + + + +
Plugin TypeEntry Point GroupPurpose
Hardware Platform Pluginsglang.srt.platformsRegister a custom hardware platform (device operations, KV cache pools, attention backends, graph capture, compilation backends, etc.)
General Pluginsglang.srt.pluginsInject hooks (before/after/around/replace) into any function/method, or replace entire classes
+ +### Principles + +- **Non-intrusive**: Existing CUDA/ROCm/NPU/XPU code remains unchanged. OOT code paths are added alongside existing hardware-specific logic. +- **Zero configuration**: Plugins are automatically discovered after `pip install`, no sglang code changes required. +- **Environment variable control**: `SGLANG_PLATFORM` selects or validates the active platform plugin; `SGLANG_PLUGINS` (comma-separated) controls which general plugins to load. + +### Current Scope & Future Direction + +The plugin system currently targets **out-of-tree (OOT) hardware platforms** — enabling new devices to integrate with SGLang without any changes to the main repository. The main-repo hardware paths (CUDA, ROCm, NPU, XPU, etc.) continue to use the existing `is_cuda()`/`is_npu()`/… utility functions. + +As the plugin interfaces mature and stabilize, in-tree hardware backends can be gradually migrated to the same plugin architecture. This would replace the scattered `if device == "cuda" … elif device == "npu" …` branches throughout the codebase with a single polymorphic dispatch through the platform interface, making each hardware backend self-contained and the core engine hardware-agnostic. + +## Architecture + +### Platform Hierarchy + +The platform hierarchy uses a DeviceMixin pattern to share device operations between SRT (LLM inference) and Multimodal subsystems: + +``` +DeviceMixin (shared device identity + operations) +├── SRTPlatform(DeviceMixin) # + graph runner, KV pool, … +│ └── MySRTPlatform(SRTPlatform, MyDeviceMixin) # OOT plugin +└── MMPlatform(DeviceMixin) # + attention backend, VAE, … (future) + └── MyMMPlatform(MMPlatform, MyDeviceMixin) # OOT plugin +``` + +Key design points: +- **DeviceMixin** provides platform identity queries (`is_cuda()`, `is_npu()`, etc.) and device operations (`set_device()`, `get_device_name()`, etc.) +- **SRTPlatform** adds SRT-specific factory methods, capability flags, and lifecycle hooks +- OOT plugins implement a **device mixin** (vendor-specific operations) and compose it with **SRTPlatform** via multiple inheritance +- All methods are **instance methods** (not classmethods), called through the `current_platform` singleton +- Device operations and factory methods raise `NotImplementedError` by default (fail-fast) +- Capability flags use safe conservative defaults (`False`/`pass`) +- Methods are annotated `[Active]` (called by SGLang core) or `[Planned]` (reserved for future migration) + +### Platform Discovery (`current_platform`) + +`current_platform` is a **lazy singleton** in `sglang.srt.platforms`. On first access it resolves the active platform through the following priority chain: + +``` +entry_points("sglang.srt.platforms") → Enumerate ALL plugins by name (metadata only) + │ + ├─ SGLANG_PLATFORM set (front-loading filter): + │ ├─ Name not found in discovered → RuntimeError + │ ├─ activate() returns non-None → load that platform + │ └─ activate() returns None → RuntimeError (hardware unavailable) + │ + └─ SGLANG_PLATFORM unset (auto-discover, activate all): + ├─ 0 activated → fallback base SRTPlatform + ├─ 1 activated → use it + └─ N activated → RuntimeError (must set SGLANG_PLATFORM) +``` + +### Plugin Loading Flow + +`load_plugins()` discovers and executes general plugins, then applies all registered hooks. It is called at four points: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Call SiteProcessTiming
cli/serve.py serve()MainBefore prepare_server_args()
launch_server.py __main__MainBefore prepare_server_args()
engine.py _launch_subprocesses()MainBefore server_args.check_server_args()
scheduler.py run_scheduler_process()SubprocessBefore Scheduler() construction
+ +> **Note**: `load_plugins()` is idempotent (guarded by `_plugins_loaded` flag). In spawn'd subprocesses the flag resets, so plugins are correctly re-loaded. + +``` +load_plugins() + ├── _get_excluded_dists() → compute dists to skip (via SGLANG_PLATFORM) + ├── load_plugins_by_group("sglang.srt.plugins", → discover entry_points, filter by SGLANG_PLUGINS + │ excluded_dists=...) skip plugins from unselected platform packages + ├── for each plugin: → set _current_plugin_source context var + │ func() side effects (register hooks with source tracking) + └── HookRegistry.apply_hooks() → monkey-patch targets +``` + +--- + +## Plugin Type 1: Hardware Platform Plugin + +### Description + +A hardware platform plugin registers an `SRTPlatform` subclass that tells SGLang how to interact with a specific hardware backend. + +### Quick Start + +**1. Create a minimal package:** + +``` +my_platform_plugin/ +├── pyproject.toml +└── my_platform_plugin/ + ├── __init__.py # activate() function + ├── device.py # MyDeviceMixin + └── platform.py # MySRTPlatform +``` + +**2. `pyproject.toml`:** + +```toml +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "my-platform-plugin" +version = "0.1.0" + +[project.entry-points."sglang.srt.platforms"] +my_device = "my_platform_plugin:activate" +``` + +**3. `__init__.py`** — activation function: + +```python +def activate(): + """Return fully-qualified class name to activate, or None to skip.""" + if _my_device_is_available(): + return "my_platform_plugin.platform.MySRTPlatform" + return None +``` + +**4. `device.py`** — device mixin: + +```python +from sglang.srt.platforms.device_mixin import DeviceMixin, PlatformEnum + +class MyDeviceMixin(DeviceMixin): + _enum = PlatformEnum.OOT + device_name = "my_device" + device_type = "my_device" # torch device type + + def set_device(self, device) -> None: ... + def get_device_name(self, device_id=0) -> str: ... + def get_device_total_memory(self, device_id=0) -> int: ... + def get_current_memory_usage(self, device=None) -> float: ... + def get_device_capability(self, device_id=0): ... + def get_torch_distributed_backend_str(self) -> str: ... +``` + +**5. `platform.py`** — SRT platform: + +```python +from sglang.srt.platforms.interface import SRTPlatform +from my_platform_plugin.device import MyDeviceMixin + +class MySRTPlatform(SRTPlatform, MyDeviceMixin): + def get_default_attention_backend(self) -> str: ... + def support_cuda_graph(self) -> bool: ... + # ... override other methods as needed +``` + +**6. Install and verify:** + +```bash +pip install -e my_platform_plugin/ +python -c "from sglang.srt.platforms import current_platform; print(current_platform)" +``` + +### Platform Interface Reference + +#### Identity Queries (from DeviceMixin) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MethodDefaultDescription
is_cuda()Based on _enumWhether this is an NVIDIA CUDA platform
is_rocm()Based on _enumWhether this is an AMD ROCm platform
is_npu()Based on _enumWhether this is a Huawei NPU platform
is_cpu()Based on _enumWhether this is a CPU-only platform
is_xpu()Based on _enumWhether this is an Intel XPU platform
is_musa()Based on _enumWhether this is a Moore Threads MUSA platform
is_cuda_alike()CUDA+ROCM+MUSATrue if the hardware supports CUDA-like APIs
is_out_of_tree()True for OOTAutomatically detected based on _enum = PlatformEnum.OOT
+ +#### Device Operations (from DeviceMixin) + +> Methods annotated **[Active]** are called by SGLang core through `current_platform` — OOT implementations take effect immediately. +> Methods annotated **[Planned]** are reserved interfaces — SGLang core still uses hardcoded calls (e.g. `torch.cuda.empty_cache()`). OOT implementations will NOT take effect until the core is migrated in a future PR. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MethodDefaultStatusDescription
get_device(local_rank)raise NotImplementedErrorPlannedReturn torch.device for a given local rank
set_device(device)raise NotImplementedErrorPlannedSet the current device
get_device_name(device_id)raise NotImplementedErrorPlannedGet human-readable device name
get_device_uuid(device_id)raise NotImplementedErrorPlannedGet unique device identifier
get_device_capability(device_id)raise NotImplementedErrorPlannedGet DeviceCapability(major, minor). None if N/A
empty_cache()passPlannedRelease cached device memory
synchronize()passPlannedSynchronize device operations
get_device_total_memory(device_id)raise NotImplementedErrorActiveGet total device memory in bytes
get_available_memory(device_id)raise NotImplementedErrorPlannedReturn (free_bytes, total_bytes)
get_current_memory_usage(device)raise NotImplementedErrorActiveGet current peak memory usage in bytes
get_torch_distributed_backend_str()raise NotImplementedErrorPlannedDistributed backend string (e.g. "nccl", "hccl")
get_communicator_class()NonePlannedPlatform-specific communicator class
inference_mode()torch.inference_mode(True)PlannedReturn inference mode context manager
seed_everything(seed)Set random/np/torch seedsPlannedSet random seeds for reproducibility
verify_quantization(quant)passPlannedValidate quantization method support
get_cpu_architecture()Auto-detect x86/armPlannedDetect CPU architecture (CpuArchEnum)
+ +#### Types (from DeviceMixin) + + + + + + + + + + + + + + + + + + + + + + + + + + +
TypeDescription
PlatformEnumEnumeration of platform types: CUDA, ROCM, CPU, XPU, MUSA, NPU, TPU, MPS, OOT, UNSPECIFIED
CpuArchEnumCPU architecture: X86, ARM, UNSPECIFIED
DeviceCapabilityNamedTuple(major, minor) with comparison support. Methods: as_version_str(), to_int()
+ +#### Capability Flags (from SRTPlatform) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MethodDefaultDescription
support_cuda_graph()FalseWhether device graph capture is supported (plain CUDA graph)
support_piecewise_cuda_graph()FalseWhether piecewise CUDA graph (torch.compile backend) is supported
supports_fp8()FalseWhether FP8 quantization is supported
is_pin_memory_available()TrueWhether pinned memory is available
+ +#### Subsystem Factory Methods (from SRTPlatform) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MethodDefaultDescription
get_default_attention_backend()raise NotImplementedErrorDefault attention backend name
get_graph_runner_cls()raise NotImplementedErrorGraph Runner class
get_mha_kv_pool_cls()raise NotImplementedErrorMHA KV cache pool class
get_mla_kv_pool_cls()raise NotImplementedErrorMLA KV cache pool class
get_nsa_kv_pool_cls()raise NotImplementedErrorNSA KV cache pool class (DeepSeek V3.2)
get_paged_allocator_cls()raise NotImplementedErrorPaged allocator class
get_piecewise_backend_cls()raise NotImplementedErrorPiecewise compilation backend class
get_compile_backend(mode)"inductor"Compilation backend string
get_dispatch_key_name()"native"MultiPlatformOp dispatch key name
+ +#### Lifecycle Hooks (from SRTPlatform) + + + + + + + + + + + + + + + + + + + + + + + + + + +
MethodInvocation TimingPurpose
apply_server_args_defaults(server_args)After ServerArgs parsing, in __post_init__Set platform-specific defaults
init_backend()In each worker, before model constructionOne-time backend initialization
+ +### Environment Variables + + + + + + + + + + + + + + + + + + + + + + +
VariableDescription
SGLANG_PLATFORMSelect the platform plugin by entry_point name (e.g. kunlun, demo_cuda). When set, only the named plugin's activate() is called (front-loading filter) — other plugins are not touched. Additionally, general plugins (sglang.srt.plugins) from unselected platform packages are automatically skipped to avoid importing their dependencies. Required when multiple plugins would activate. Errors if the name is not found or if the plugin's hardware is unavailable.
SGLANG_PLUGINSComma-separated whitelist of general plugin names to load (group: sglang.srt.plugins). If unset, all discovered general plugins are loaded.
+ +--- + +## Plugin Type 2: General Plugin + +### Description + +General function plugins inject behavior into sglang **without requiring a custom platform**. Use cases include: + +- **Observability**: Add logging, metrics, and tracing to any function +- **Behavior modification**: Modify function arguments or return values +- **Performance profiling**: Add timing to critical functions +- **A/B testing**: Replace implementations at runtime + +### Quick Start + +**1. Create a minimal package:** + +``` +my_general_plugin/ +├── pyproject.toml +└── my_general_plugin/ + └── __init__.py # register() function +``` + +**2. `pyproject.toml`:** + +```toml +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "my-general-plugin" +version = "0.1.0" + +[project.entry-points."sglang.srt.plugins"] +my_plugin = "my_general_plugin:register" +``` + +**3. `__init__.py`** — register hooks: + +```python +from sglang.srt.plugins.hook_registry import HookRegistry, HookType + +def register(): + """Entry point called by load_plugins().""" + HookRegistry.register( + "sglang.srt.managers.scheduler.Scheduler.__init__", + my_hook, + HookType.AROUND, + ) + +def my_hook(original_fn, self, *args, **kwargs): + result = original_fn(self, *args, **kwargs) + print(f"Scheduler initialized! gpu_id={self.gpu_id}") + return result +``` + +**4. Install and run:** + +```bash +pip install -e my_general_plugin/ +sglang serve --model-path [options] +# Look for "Scheduler initialized!" in logs +``` + +### Hook Types + +`HookRegistry` supports four hook types: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Hook TypeSignatureDescription
BEFOREfn(*args, **kwargs) -> (args, kwargs) \| NoneRuns before the original. Return None to keep args unchanged, or (args, kwargs) to modify.
AFTERfn(result, *args, **kwargs) -> new_result \| NoneRuns after the original. Return None to keep result, or a new value to replace.
AROUNDfn(original_fn, *args, **kwargs) -> resultWraps the original. You must call original_fn yourself. Full control over execution.
REPLACEfn(*args, **kwargs) -> result or classReplace the original function or class entirely. For class targets, pass a replacement class directly — it is substituted via setattr preserving isinstance()/issubclass() semantics.
+ +> **Note**: Only `REPLACE` accepts a class as the hook. Passing a class to `BEFORE`/`AFTER`/`AROUND` raises `TypeError` at registration time. + +### Registration API + +Hooks can be registered using the **imperative API** or the **decorator API**: + +```python +# --- Imperative API --- +from sglang.srt.plugins.hook_registry import HookRegistry, HookType + +def my_timer(original_fn, *args, **kwargs): + start = time.perf_counter() + result = original_fn(*args, **kwargs) + print(f"Elapsed: {time.perf_counter() - start:.3f}s") + return result + +HookRegistry.register( + "sglang.srt.managers.scheduler.Scheduler.get_next_batch_to_run", + my_timer, + HookType.AROUND, +) + +# --- Decorator API --- +from sglang.srt.plugins.hook_registry import plugin_hook, HookType + +@plugin_hook( + "sglang.srt.managers.scheduler.Scheduler.get_next_batch_to_run", + type=HookType.AROUND, +) +def my_timer(original_fn, *args, **kwargs): + start = time.perf_counter() + result = original_fn(*args, **kwargs) + print(f"Elapsed: {time.perf_counter() - start:.3f}s") + return result + +# --- Class replacement (REPLACE) --- +from sglang.srt.plugins.hook_registry import plugin_hook, HookType +from sglang.srt.managers.scheduler import Scheduler + +@plugin_hook( + "sglang.srt.managers.scheduler.Scheduler", + type=HookType.REPLACE, +) +class MyScheduler(Scheduler): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + print("Enhanced scheduler initialized!") +``` + +### Hook Target Resolution + +Target paths use fully-qualified dotted notation. Both formats are supported: + +- **Dotted**: `sglang.srt.managers.scheduler.Scheduler.__init__` +- **Entry-points style**: `sglang.srt.managers.scheduler:Scheduler.__init__` (colon treated as dot) + +### Common Hook Targets + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
TargetDescription
sglang.srt.server_args.ServerArgs.add_cli_argsAdd custom CLI arguments
sglang.srt.server_args.ServerArgs.__post_init__Modify ServerArgs after parsing
sglang.srt.server_args.ServerArgs.check_server_argsAdd/relax validation
sglang.srt.managers.scheduler.Scheduler.__init__Custom scheduler state
sglang.srt.managers.scheduler.Scheduler.get_next_batch_to_runCustom scheduling policy
sglang.srt.managers.scheduler.Scheduler.run_batchProfiling / inspection
sglang.srt.managers.scheduler.Scheduler.process_batch_resultCustom metrics
sglang.srt.managers.tp_worker.TpModelWorker.__init__Custom worker state
sglang.srt.managers.tp_worker.TpModelWorker.forward_batch_generationForward pass wrapping
+ +--- + +## File Reference + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FileDescription
sglang/srt/platforms/device_mixin.pyPlatformEnum + DeviceMixin base class
sglang/srt/platforms/interface.pySRTPlatform base class (extends DeviceMixin)
sglang/srt/platforms/__init__.pycurrent_platform lazy singleton + discovery logic
sglang/srt/plugins/__init__.pyload_plugins() + load_plugins_by_group()
sglang/srt/plugins/hook_registry.pyHookRegistry, HookType, plugin_hook decorator
diff --git a/docs_new/docs/hardware-platforms/tpu.mdx b/docs_new/docs/hardware-platforms/tpu.mdx index 12e89b2ea..5e2c75bfa 100644 --- a/docs_new/docs/hardware-platforms/tpu.mdx +++ b/docs_new/docs/hardware-platforms/tpu.mdx @@ -2,53 +2,15 @@ title: "TPU" description: "SGLang supports high-performance TPU inference through the SGLang-JAX backend, which is specifically optimized for Google Cloud TPUs. The JAX-based implementation delivers exceptional throughput and low latency for Large Language Model (LLM) serving workloads on TPU hardware." --- - - - SGLang TPU support is implemented via the SGLang-JAX backend, a dedicated JAX-based inference engine maintained as a separate repository at [sgl-project/sglang-jax](https://github.com/sgl-project/sglang-jax). - +SGLang supports high-performance TPU inference through the SGLang-JAX backend, which is specifically optimized for Google Cloud TPUs. The JAX-based implementation delivers exceptional throughput and low latency for Large Language Model (LLM) serving workloads on TPU hardware. For TPU-specific issues or feature requests, please visit the [sglang-jax GitHub issues page](https://github.com/sgl-project/sglang-jax/issues). ---- +**NOTE:** SGLang TPU support is implemented via the SGLang-JAX backend, a dedicated JAX-based inference engine maintained as a separate repository at [https://github.com/sgl-project/sglang-jax](https://github.com/sgl-project/sglang-jax). -# System Requirements +## System Requirements ---- - -## Supported TPU Hardware - - - - **32 GB** HBM Memory — Available on Google Cloud - - - **96 GB per core** HBM Memory — Available on Google Cloud - - - ---- - -## Software Requirements - - - - Version **3.12 or higher** - - - **Latest version** with TPU support - - - Google Cloud TPU VM or compatible TPU runtime. **Optional:** SkyPilot for simplified cloud deployment. - - - ---- - -# Feature Support Matrix - -SGLang-JAX provides comprehensive TPU-optimized features for production LLM serving: - -### Supported Features +### Supported TPU Hardware @@ -58,9 +20,49 @@ SGLang-JAX provides comprehensive TPU-optimized features for production LLM serv - - - + + + + + + + + + + + + + + + + + +
**Feature****Support Status****Description**TPU TypeHBM MemoryAvailability
TPU v6e32 GBGoogle Cloud
TPU v796 GB per coreGoogle Cloud
+ +### Software Requirements + +- **Python:** 3.12 or higher +- **JAX:** Latest version with TPU support +- **Environment:** Google Cloud TPU VM or compatible TPU runtime +- **Optional:** SkyPilot for simplified cloud deployment + +## Feature Support Matrix + +SGLang-JAX provides comprehensive TPU-optimized features for production LLM serving: + + + + + + + + + + + + + + @@ -74,56 +76,54 @@ SGLang-JAX provides comprehensive TPU-optimized features for production LLM serv - + - + - + - + - + - + - + - + - + - + - + - + - +
FeatureSupport StatusDescription
✅ Memory-efficient prefix sharing between requests
FlashAttention Backend ✅ TPU-optimized attention kernel for long sequences
Tensor Parallelism ✅ Distribute models across multiple TPU cores
Paged Attention ✅ Flexible KV cache management with paging
Speculative Decoding (EAGLE/EAGLE3) ✅ 20-40% throughput improvement for compatible models
Chunked Prefill ✅ Mixed prefill-decode batching
OpenAI-Compatible API ✅ Drop-in replacement for OpenAI API
Data Parallel Attention 🚧In development — Attention computation with data parallelismIn development - Attention computation with data parallelism
Quantization 🚧In development — Model quantization for reduced memory usageIn development - Model quantization for reduced memory usage
Multi-LoRA 🚧In development — Serve multiple LoRA adapters simultaneouslyIn development - Serve multiple LoRA adapters simultaneously
---- - ### Attention Backend Comparison @@ -161,58 +161,54 @@ SGLang-JAX provides comprehensive TPU-optimized features for production LLM serv
- - FlashAttention backend is recommended for production workloads due to superior memory efficiency and performance. - +**NOTE:** FlashAttention backend is recommended for production workloads due to superior memory efficiency and performance. ---- - -# Optimized Model List +## Optimized Model List The following models have been tested and optimized for TPU deployment: - - + + - - + + - + - + - + - + - + - + - + - + @@ -222,394 +218,377 @@ The following models have been tested and optimized for TPU deployment:
**Model Family****Performance Status**Model FamilyPerformance Status
Qwen 3Qwen 3 ⭐ Recommended for production
Qwen 3 MoEQwen 3 MoE ⭐ Best performance
Qwen 2Qwen 2 Needs improvement
Qwen 2 MoEQwen 2 MoE Needs improvement
Qwen 1.5Qwen 1.5 Needs improvement
Llama/LLaMALlama/LLaMA Needs improvement
Grok-2Grok-2 Needs improvement
Gemma 2Gemma 2 Verified on TPU
---- +## Installation -# Installation +### Method 1: Using PyPI (Recommended) - - - ```bash - pip install sglang-jax - ``` - - - ```bash - git clone https://github.com/sgl-project/sglang-jax - cd sglang-jax - uv venv --python 3.12 && source .venv/bin/activate - uv pip install -e "python[all]" - ``` - - - - Docker support for TPU is currently under development. Please use PyPI or source installation methods. - - - - SkyPilot provides simplified deployment on Google Cloud TPU: +```bash Command +pip install sglang-jax +``` - 1. **Install SkyPilot and configure GCP access** +### Method 2: From Source - See the [SkyPilot documentation](https://docs.skypilot.co/en/latest/) for setup instructions. +```bash Command +git clone https://github.com/sgl-project/sglang-jax +cd sglang-jax +uv venv --python 3.12 && source .venv/bin/activate +uv pip install -e "python[all]" +``` - 2. **Create a SkyPilot configuration file** +### Method 3: Using Docker - Create a SkyPilot YAML file: `sglang-jax.sky.yaml` +**NOTE:** Docker support for TPU is currently under development. Please use PyPI or source installation methods. - 3. **Launch your TPU cluster** +### Method 4: Cloud TPU with SkyPilot - ```bash - # Standard deployment - sky launch -c sglang-jax sglang-jax.sky.yaml --infra=gcp +[SkyPilot](https://github.com/skypilot-org/skypilot) provides simplified deployment on Google Cloud TPU: - # With spot instances for cost savings - sky launch -c sglang-jax sglang-jax.sky.yaml --infra=gcp --use-spot - ``` - - +1. Install SkyPilot and configure GCP access (see [SkyPilot documentation](https://skypilot.readthedocs.io/)) ---- +2. Create a SkyPilot configuration file: -# Launch the Serving Engine +
+SkyPilot YAML: sglang-jax.sky.yaml - - - ```bash - JAX_COMPILATION_CACHE_DIR=/tmp/jit_cache python3 -u -m sgl_jax.launch_server \ - --model-path Qwen/Qwen-7B-Chat \ - --trust-remote-code \ - --dist-init-addr=0.0.0.0:10011 \ - --nnodes=1 \ - --tp-size=4 \ - --device=tpu \ - --random-seed=3 \ - --node-rank=0 \ - --mem-fraction-static=0.8 \ - --max-prefill-tokens=8192 \ - --download-dir=/tmp \ - --dtype=bfloat16 \ - --skip-server-warmup \ - --host 0.0.0.0 \ - --port 30000 - ``` +```yaml Config +# sglang-jax.sky.yaml +resources: + accelerators: tpu-v6e-4 + accelerator_args: + tpu_vm: True + runtime_version: v2-alpha-tpuv6e - - - - Enables JIT compilation caching to accelerate server startup on subsequent runs. Recommended: `/tmp/jit_cache` - - - Tensor parallelism size; match this to your TPU core count (typically `1`, `4`, or `8`). - - - Specifies TPU device. This is the default for `sglang-jax`. - - - Uses bfloat16 precision, which TPUs are optimized for. - - - Allocates this fraction of TPU HBM for static memory. Adjustable from `0.2` to `0.9`. - - - Maximum number of tokens processed in the prefill phase. - - - - - - For production workloads with optimal throughput: +run: | + git clone https://github.com/sgl-project/sglang-jax.git + cd sglang-jax + uv venv --python 3.12 + source .venv/bin/activate + uv pip install -e "python[all]" +``` - ```bash - python3 -u -m sgl_jax.launch_server \ - --model-path Qwen/Qwen3-8B \ - --trust-remote-code \ - --tp-size=4 \ - --device=tpu \ - --mem-fraction-static=0.8 \ - --chunked-prefill-size=2048 \ - --dtype=bfloat16 \ - --max-running-requests=256 \ - --page-size=128 \ - --attention-backend=fa - ``` - - - Speculative decoding can improve throughput by 20-40% for compatible models: +
- ```bash - python3 -u -m sgl_jax.launch_server \ - --model-path Qwen/Qwen3-32B \ - --trust-remote-code \ - --device=tpu \ - --tp-size=4 \ - --mem-fraction-static=0.8 \ - --max-prefill-tokens=4096 \ - --attention-backend=fa \ - --dtype=bfloat16 \ - --port=30000 \ - --host=0.0.0.0 \ - --disable-overlap-schedule \ - --speculative-algorithm=EAGLE3 \ - --speculative-draft-model-path=AngelSlim/Qwen3-32B_eagle3 \ - --page-size=64 \ - --speculative-eagle-topk=1 \ - --speculative-num-steps=3 \ - --speculative-num-draft-tokens=4 - ``` +3. Launch your TPU cluster: - - Speculative decoding is currently supported for Qwen3 and LLaMA model families. See the [Speculative Decoding documentation](../advanced_features/speculative_decoding) for detailed configuration guidance. - - - - For large models requiring multiple TPU VMs: +```bash Command +# Standard deployment +sky launch -c sglang-jax sglang-jax.sky.yaml --infra=gcp - ```bash - # Node 0 (coordinator) - python3 -m sgl_jax.launch_server \ - --model-path MODEL_PATH \ - --dist-init-addr=NODE0_IP:10011 \ - --nnodes=2 \ - --node-rank=0 \ - --tp-size=8 \ - [other parameters...] +# With spot instances for cost savings +sky launch -c sglang-jax sglang-jax.sky.yaml --infra=gcp --use-spot +``` - # Node 1 (worker) - python3 -m sgl_jax.launch_server \ - --model-path MODEL_PATH \ - --dist-init-addr=NODE0_IP:10011 \ - --nnodes=2 \ - --node-rank=1 \ - --tp-size=8 \ - [other parameters...] - ``` - - +## Launch of the Serving Engine ---- +### Basic Example: Qwen-7B -# Benchmarking with Requests +```bash Command +JAX_COMPILATION_CACHE_DIR=/tmp/jit_cache python3 -u -m sgl_jax.launch_server \ + --model-path Qwen/Qwen-7B-Chat \ + --trust-remote-code \ + --dist-init-addr=0.0.0.0:10011 \ + --nnodes=1 \ + --tp-size=4 \ + --device=tpu \ + --random-seed=3 \ + --node-rank=0 \ + --mem-fraction-static=0.8 \ + --max-prefill-tokens=8192 \ + --download-dir=/tmp \ + --dtype=bfloat16 \ + --skip-server-warmup \ + --host 0.0.0.0 \ + --port 30000 +``` - - - Basic throughput benchmark: +**Key Parameters Explained:** - ```bash - python3 -m sgl_jax.bench_serving \ - --backend sgl-jax \ - --dataset-name random \ - --num-prompts=100 \ - --random-input=512 \ - --random-output=128 \ - --max-concurrency=8 \ - --random-range-ratio=1 \ - --warmup-requests=0 - ``` - - - Measure single-batch latency: +1. `JAX_COMPILATION_CACHE_DIR=/tmp/jit_cache` - Enables JIT compilation caching to accelerate server startup on subsequent runs +2. `--tp-size=4` - Tensor parallelism size; match this to your TPU core count (typically 1, 4, or 8) +3. `--device=tpu` - Specifies TPU device (this is the default for sglang-jax) +4. `--dtype=bfloat16` - Uses bfloat16 precision, which TPUs are optimized for +5. `--mem-fraction-static=0.8` - Allocates 80% of TPU HBM for static memory (adjustable from 0.2 to 0.9) +6. `--max-prefill-tokens=8192` - Maximum number of tokens processed in the prefill phase - ```bash - python3 -m sgl_jax.bench_one_batch_server \ - --base-url http://127.0.0.1:30000 \ - --model-path Qwen/Qwen-7B-Chat \ - --batch-size=32 \ - --input-len=256 \ - --output-len=32 - ``` - - - For systematic performance evaluation across different configurations: +### High-Performance Configuration: Qwen3-8B - ```bash - #!/bin/bash - set -e +For production workloads with optimal throughput: - backend=${1:-sgl-jax} - num_prompts_per_concurrency=3 - input_seq_lens=(1024 4096 8192) - output_seq_lens=(1 1024) - max_concurrencies=(8 16 32 64 128 256) +```bash Command +python3 -u -m sgl_jax.launch_server \ + --model-path Qwen/Qwen3-8B \ + --trust-remote-code \ + --tp-size=4 \ + --device=tpu \ + --mem-fraction-static=0.8 \ + --chunked-prefill-size=2048 \ + --dtype=bfloat16 \ + --max-running-requests=256 \ + --page-size=128 \ + --attention-backend=fa +``` - for input_seq_len in "${input_seq_lens[@]}"; do - for output_seq_len in "${output_seq_lens[@]}"; do - echo "=======================================" - echo "Testing ISL/OSL: $input_seq_len/$output_seq_len" - echo "=======================================" - for max_concurrency in "${max_concurrencies[@]}"; do - num_prompts=$((num_prompts_per_concurrency * max_concurrency)) - python3 -m sgl_jax.bench_serving \ - --backend ${backend} \ - --dataset-name random \ - --num-prompts ${num_prompts} \ - --random-input ${input_seq_len} \ - --random-output ${output_seq_len} \ - --max-concurrency ${max_concurrency} \ - --random-range-ratio 1 \ - --disable-ignore-eos \ - --warmup-requests 0 - done +### Advanced: Speculative Decoding (EAGLE3) + +Speculative decoding can improve throughput by 20-40% for compatible models: + +```bash Command +python3 -u -m sgl_jax.launch_server \ + --model-path Qwen/Qwen3-32B \ + --trust-remote-code \ + --device=tpu \ + --tp-size=4 \ + --mem-fraction-static=0.8 \ + --max-prefill-tokens=4096 \ + --attention-backend=fa \ + --dtype=bfloat16 \ + --port=30000 \ + --host=0.0.0.0 \ + --disable-overlap-schedule \ + --speculative-algorithm=EAGLE3 \ + --speculative-draft-model-path=AngelSlim/Qwen3-32B_eagle3 \ + --page-size=64 \ + --speculative-eagle-topk=1 \ + --speculative-num-steps=3 \ + --speculative-num-draft-tokens=4 +``` + +**NOTE:** Speculative decoding is currently supported for Qwen3 and LLaMA model families. See the [Speculative Decoding documentation](https://github.com/sgl-project/sglang-jax/blob/main/docs/features/speculative_decoding.md) for detailed configuration guidance. + + +### Multi-Node Distributed Serving + +For large models requiring multiple TPU VMs: + +```bash Command +# Node 0 (coordinator) +python3 -m sgl_jax.launch_server \ + --model-path MODEL_PATH \ + --dist-init-addr=NODE0_IP:10011 \ + --nnodes=2 \ + --node-rank=0 \ + --tp-size=8 \ + [other parameters...] + +# Node 1 (worker) +python3 -m sgl_jax.launch_server \ + --model-path MODEL_PATH \ + --dist-init-addr=NODE0_IP:10011 \ + --nnodes=2 \ + --node-rank=1 \ + --tp-size=8 \ + [other parameters...] +``` + +## Benchmarking with Requests + +### Throughput Testing + +Basic throughput benchmark: + +```bash Command +python3 -m sgl_jax.bench_serving \ + --backend sgl-jax \ + --dataset-name random \ + --num-prompts=100 \ + --random-input=512 \ + --random-output=128 \ + --max-concurrency=8 \ + --random-range-ratio=1 \ + --warmup-requests=0 +``` + +### Latency Testing + +Measure single-batch latency: + +```bash Command +python3 -m sgl_jax.bench_one_batch_server \ + --base-url http://127.0.0.1:30000 \ + --model-path Qwen/Qwen-7B-Chat \ + --batch-size=32 \ + --input-len=256 \ + --output-len=32 +``` + +### Comprehensive Benchmark Script + +For systematic performance evaluation across different configurations: + +```bash Command +#!/bin/bash +set -e + +backend=${1:-sgl-jax} +num_prompts_per_concurrency=3 +input_seq_lens=(1024 4096 8192) +output_seq_lens=(1 1024) +max_concurrencies=(8 16 32 64 128 256) + +for input_seq_len in "${input_seq_lens[@]}"; do + for output_seq_len in "${output_seq_lens[@]}"; do + echo "=======================================" + echo "Testing ISL/OSL: $input_seq_len/$output_seq_len" + echo "=======================================" + for max_concurrency in "${max_concurrencies[@]}"; do + num_prompts=$((num_prompts_per_concurrency * max_concurrency)) + python3 -m sgl_jax.bench_serving \ + --backend ${backend} \ + --dataset-name random \ + --num-prompts ${num_prompts} \ + --random-input ${input_seq_len} \ + --random-output ${output_seq_len} \ + --max-concurrency ${max_concurrency} \ + --random-range-ratio 1 \ + --disable-ignore-eos \ + --warmup-requests 0 done done - ``` +done +``` - For detailed help on all benchmark parameters: +For detailed help on all benchmark parameters: - ```bash - python3 -m sgl_jax.bench_serving --help - ``` +```bash Command +python3 -m sgl_jax.bench_serving --help +``` - See the [Benchmark and Profiling Guide](../developer_guide/benchmark_and_profiling) for advanced benchmarking techniques and profiling with JAX Profiler. - - +See the [Benchmark and Profiling Guide](https://github.com/sgl-project/sglang-jax/blob/main/docs/developer_guide/benchmark_and_profiling.md) for advanced benchmarking techniques and profiling with JAX Profiler. ---- +## Performance Optimization -# Performance Optimization +### Memory Optimization - - - **Reduce memory usage:** - - Lower `--mem-fraction-static` (from `0.8` → `0.5` → `0.3`) - - Decrease `--max-prefill-tokens` (from `16384` → `8192` → `4096`) - - Reduce `--max-running-requests` +**Reduce memory usage:** +- Lower `--mem-fraction-static` (from 0.8 → 0.5 → 0.3) +- Decrease `--max-prefill-tokens` (from 16384 → 8192 → 4096) +- Reduce `--max-running-requests` - **Handle OOM errors:** - - Start with conservative memory settings (`--mem-fraction-static=0.5`) - - Gradually increase until you find the optimal balance - - Increase `--page-size` for better memory locality (`1` → `16` → `64` → `128`) - - - To maximize tokens per second: - - Use FlashAttention backend: `--attention-backend=fa` - - Enable speculative decoding (EAGLE3) for Qwen3 models (20-40% improvement) - - Increase `--max-running-requests` to `256+` - - Set `--mem-fraction-static` to `0.8+` (if memory allows) - - Use larger page sizes (`64-128`) - - Enable chunked prefill: `--chunked-prefill-size=2048` - - - To minimize time-to-first-token (TTFT) and inter-token latency: - - Reduce `--page-size` to `1-4` - - Lower `--max-running-requests` (`16-32`) for smaller batches - - Reduce `--chunked-prefill-size` - - Use conservative memory settings to avoid GC pauses - - - **JIT Compilation Cache:** +**Handle OOM errors:** +- Start with conservative memory settings (`--mem-fraction-static=0.5`) +- Gradually increase until you find the optimal balance +- Increase `--page-size` for better memory locality (1 → 16 → 64 → 128) - ```bash - export JAX_COMPILATION_CACHE_DIR=/tmp/jit_cache - ``` +### Throughput Optimization - Always set this environment variable to cache compiled kernels and accelerate server startup. +To maximize tokens per second: - **Data Type Optimization:** Use `--dtype=bfloat16` for TPU native optimization. TPUs are specifically designed for bfloat16 computations. +- Use FlashAttention backend: `--attention-backend=fa` +- Enable speculative decoding (EAGLE3) for Qwen3 models (20-40% improvement) +- Increase `--max-running-requests` to 256+ +- Set `--mem-fraction-static` to 0.8+ (if memory allows) +- Use larger page sizes (64-128) +- Enable chunked prefill: `--chunked-prefill-size=2048` - **Tensor Parallelism:** Match `--tp-size` to your TPU core configuration (`1`, `4`, or `8`) for optimal model distribution. +### Latency Optimization - **Attention Backend:** Always use `--attention-backend=fa` (FlashAttention) for production workloads. - - +To minimize time-to-first-token (TTFT) and inter-token latency: ---- +- Reduce `--page-size` to 1-4 +- Lower `--max-running-requests` (16-32) for smaller batches +- Reduce `--chunked-prefill-size` +- Use conservative memory settings to avoid GC pauses -# Troubleshooting +### TPU-Specific Optimizations - - - If you encounter out-of-memory errors: +1. **JIT Compilation Cache:** + ```bash Command + export JAX_COMPILATION_CACHE_DIR=/tmp/jit_cache + ``` + Always set this environment variable to cache compiled kernels and accelerate server startup. - 1. **Reduce mem-fraction-static** +2. **Data Type Optimization:** + Use `--dtype=bfloat16` for TPU native optimization. TPUs are specifically designed for bfloat16 computations. - Lower `--mem-fraction-static` from `0.8` to `0.5` or lower. +3. **Tensor Parallelism:** + Match `--tp-size` to your TPU core configuration (1, 4, or 8) for optimal model distribution. - 2. **Decrease max-prefill-tokens** +4. **Attention Backend:** + Always use `--attention-backend=fa` (FlashAttention) for production workloads. - Decrease `--max-prefill-tokens` from `8192` to `4096` or `2048`. +## Troubleshooting - 3. **Lower max-running-requests** +### OOM (Out of Memory) Errors - Lower `--max-running-requests` to reduce concurrent batch size. +If you encounter out-of-memory errors: - 4. **Increase page-size** +1. Reduce `--mem-fraction-static` from 0.8 to 0.5 or lower +2. Decrease `--max-prefill-tokens` from 8192 to 4096 or 2048 +3. Lower `--max-running-requests` to reduce concurrent batch size +4. Increase `--page-size` for better memory layout efficiency - Increase `--page-size` for better memory layout efficiency. - - - If the server takes too long to start: +### Compilation Long-Time - Ensure `JAX_COMPILATION_CACHE_DIR` is properly set - Understand that the first run requires JIT compilation — this is normal - Subsequent runs will be significantly faster with cached compilations - Consider using `--skip-server-warmup` to defer compilation until first request - - - If you're not achieving expected throughput: +If the server takes too long to start: - Verify `--tp-size` matches your TPU core configuration - Check that `--attention-backend=fa` is enabled - Increase `--max-running-requests` to enable larger batch formation - Consider enabling speculative decoding for compatible models - Ensure memory settings allow for sufficient batch sizes - - - If clients cannot connect to the server: +1. Ensure `JAX_COMPILATION_CACHE_DIR` is properly set +2. Understand that the first run requires JIT compilation (this is normal) +3. Subsequent runs will be significantly faster with cached compilations +4. Consider using `--skip-server-warmup` to defer compilation until first request - Ensure `--host=0.0.0.0` for external access (not just `127.0.0.1`) - Verify firewall rules allow traffic on the specified port (default: `30000`) - Check that the server process is running: `curl http://localhost:30000/health` - - +### Low Throughput ---- +If you're not achieving expected throughput: -# Advanced Features +1. Verify `--tp-size` matches your TPU core configuration +2. Check that `--attention-backend=fa` is enabled +3. Increase `--max-running-requests` to enable larger batch formation +4. Consider enabling speculative decoding for compatible models +5. Ensure memory settings allow for sufficient batch sizes - - - SGLang-JAX supports EAGLE and EAGLE3 speculative decoding algorithms for Qwen3 and LLaMA model families. Speculative decoding can improve throughput by 20-40% without affecting output quality. +### Connection Issues - See the [Speculative Decoding documentation](../advanced_features/speculative_decoding) for detailed configuration and supported model combinations. - - - Enable mixed prefill-decode batching for better TPU utilization: +If clients cannot connect to the server: - ```bash - --chunked-prefill-size=2048 --enable-mixed-chunk - ``` +1. Ensure `--host=0.0.0.0` for external access (not just `127.0.0.1`) +2. Verify firewall rules allow traffic on the specified port (default: 30000) +3. Check that the server process is running: `curl http://localhost:30000/health` - This allows the scheduler to mix prefill operations with decode operations in the same batch, improving overall throughput. - - - SGLang-JAX supports a plugin-based attention backend system. You can implement custom attention kernels optimized for specific use cases. +## Advanced Features - See the [Attention Backend documentation](https://github.com/sgl-project/sglang-jax/tree/main/docs) for implementation details. - - - Verify your TPU setup before deploying: +### Speculative Decoding - ```bash - python -c "from sgl_jax import check_env; check_env.check_env()" - ``` +SGLang-JAX supports EAGLE and EAGLE3 speculative decoding algorithms for Qwen3 and LLaMA model families. Speculative decoding can improve throughput by 20-40% without affecting output quality. - This command checks: - - Installed package versions - - TPU device availability and specifications - - System resources and configuration - - Compatibility of settings - - +See the [Speculative Decoding documentation](https://github.com/sgl-project/sglang-jax/blob/main/docs/features/speculative_decoding.md) for detailed configuration and supported model combinations. ---- +### Chunked Prefill -# Contributing +Enable mixed prefill-decode batching for better TPU utilization: + +```bash Command +--chunked-prefill-size=2048 --enable-mixed-chunk +``` + +This allows the scheduler to mix prefill operations with decode operations in the same batch, improving overall throughput. + +### Custom Attention Backends + +SGLang-JAX supports a plugin-based attention backend system. You can implement custom attention kernels optimized for specific use cases. + +See the [Attention Backend documentation](https://github.com/sgl-project/sglang-jax/blob/main/docs/features/attention_backend.md) for implementation details. + +### Environment Verification + +Verify your TPU setup before deploying: + +```bash Command +python -c "from sgl_jax import check_env; check_env.check_env()" +``` + +This command checks: +- Installed package versions +- TPU device availability and specifications +- System resources and configuration +- Compatibility of settings + +## Contributing We welcome contributions to improve TPU support in SGLang-JAX! - - Check the [Development Roadmap](https://github.com/sgl-project/sglang-jax) to see planned features and find opportunities to contribute new functionality. - +### Areas for Contribution + +**Check the [Development Roadmap](https://github.com/sgl-project/sglang-jax/issues/190)** to see planned features and find opportunities to contribute new functionality. Current contribution areas include: @@ -619,55 +598,33 @@ Current contribution areas include: - Bug reports and fixes - Benchmark results and performance analysis - - - Visit the sglang-jax repository - - - Read the Contribution Guide - - - Join the SGL-JAX Slack community for discussions - - +### How to Contribute ---- +1. Visit the [sglang-jax repository](https://github.com/sgl-project/sglang-jax) +2. Read the [Contribution Guide](https://github.com/sgl-project/sglang-jax/blob/main/docs/developer_guide/contribution_guide.md) +3. Join the [SGL-JAX Slack community](https://sgl-fru7574.slack.com/archives/C09EBE5HT5X) for discussions +4. Report issues at [sglang-jax/issues](https://github.com/sgl-project/sglang-jax/issues) -## Testing on TPU +### Testing on TPU For contributors who need TPU access for testing: -- Refer to the [TPU Resources Guide](https://cloud.google.com/tpu/docs/managing-tpus-tpu-vm) for information on accessing TPU hardware +- Refer to the [TPU Resources Guide](https://github.com/sgl-project/sglang-jax/blob/main/docs/developer_guide/tpu_resources_guide.md) for information on accessing TPU hardware - Use SkyPilot with spot instances for cost-effective testing -- Follow the [Benchmark and Profiling Guide](../developer_guide/benchmark_and_profiling) for performance validation +- Follow the [Benchmark and Profiling Guide](https://github.com/sgl-project/sglang-jax/blob/main/docs/developer_guide/benchmark_and_profiling.md) for performance validation ---- +## References -# References +### Documentation - - - Source code and issue tracker for the JAX TPU backend. - - - Step-by-step installation instructions. - - - Get up and running quickly with the Qwen model family. - - - Advanced benchmarking techniques and JAX Profiler usage. - - - EAGLE and EAGLE3 speculative decoding configuration. - - - Official JAX documentation and API reference. - - - Google Cloud TPU product documentation. - - - Simplified cloud deployment with SkyPilot. - - +- [SGLang-JAX Repository](https://github.com/sgl-project/sglang-jax) +- [SGLang-JAX Installation Guide](https://github.com/sgl-project/sglang-jax/blob/main/docs/get_started/install.md) +- [Qwen Models Quick Start](https://github.com/sgl-project/sglang-jax/blob/main/docs/basic_usage/qwen.md) +- [Benchmark and Profiling Guide](https://github.com/sgl-project/sglang-jax/blob/main/docs/developer_guide/benchmark_and_profiling.md) +- [Speculative Decoding](https://github.com/sgl-project/sglang-jax/blob/main/docs/features/speculative_decoding.md) + +### External Resources + +- [JAX Documentation](https://jax.readthedocs.io/) +- [Google Cloud TPU Documentation](https://cloud.google.com/tpu/docs) +- [SkyPilot Documentation](https://skypilot.readthedocs.io/) diff --git a/docs_new/docs/hardware-platforms/xpu.mdx b/docs_new/docs/hardware-platforms/xpu.mdx index 97bd3946d..46a8d1fb2 100644 --- a/docs_new/docs/hardware-platforms/xpu.mdx +++ b/docs_new/docs/hardware-platforms/xpu.mdx @@ -2,12 +2,10 @@ title: XPU sidebarTitle: Intel GPUs (XPU) --- - The document addresses how to set up the [SGLang](https://github.com/sgl-project/sglang) environment and run LLM inference on Intel GPU, [see more context about Intel GPU support within PyTorch ecosystem](https://docs.pytorch.org/docs/stable/notes/get_start_xpu.html). -Specifically, SGLang is optimized for: -- [Intel® Arc™ Pro B-Series Graphics](https://www.intel.com/content/www/us/en/ark/products/series/242616/intel-arc-pro-b-series-graphics.html) -- [Intel® Arc™ B-Series Graphics](https://www.intel.com/content/www/us/en/ark/products/series/240391/intel-arc-b-series-graphics.html). +Specifically, SGLang is optimized for [Intel® Arc™ Pro B-Series Graphics](https://www.intel.com/content/www/us/en/ark/products/series/242616/intel-arc-pro-b-series-graphics.html) and [ +Intel® Arc™ B-Series Graphics](https://www.intel.com/content/www/us/en/ark/products/series/240391/intel-arc-b-series-graphics.html). ## Optimized Model List @@ -40,72 +38,45 @@ A list of LLMs have been optimized on Intel GPU, and more are on the way: -The model identifiers listed in the table above have been verified on [Intel® Arc™ B580 Graphics](https://www.intel.com/content/www/us/en/products/sku/241598/intel-arc-b580-graphics/specifications.html). +**Note:** The model identifiers listed in the table above +have been verified on [Intel® Arc™ B580 Graphics](https://www.intel.com/content/www/us/en/products/sku/241598/intel-arc-b580-graphics/specifications.html). ## Installation - +### Install From Source - +Currently SGLang XPU only supports installation from source. Please refer to ["Getting Started on Intel GPU"](https://docs.pytorch.org/docs/stable/notes/get_start_xpu.html) to install XPU dependency. - Currently SGLang XPU only supports installation from source. Please refer to [“Getting Started on Intel GPU”](https://docs.pytorch.org/docs/stable/notes/get_start_xpu.html) to install XPU dependency. +```bash Command +# Create and activate a conda environment +conda create -n sgl-xpu python=3.12 -y +conda activate sgl-xpu - 1. **Creation & Activation** +# Set PyTorch XPU as primary pip install channel to avoid installing the larger CUDA-enabled version and prevent potential runtime issues. +pip3 install torch==2.11.0+xpu torchao torchvision torchaudio==2.11.0+xpu --index-url https://download.pytorch.org/whl/xpu +pip3 install xgrammar --no-deps # xgrammar will introduce CUDA-enabled triton which might conflict with XPU - Create and activate a conda environment. +# Clone the SGLang code +git clone https://github.com/sgl-project/sglang.git +cd sglang +git checkout - ```bash - conda create -n sgl-xpu python=3.12 -y - conda activate sgl-xpu - ``` +# Use dedicated toml file +cd python +cp pyproject_xpu.toml pyproject.toml +# Install SGLang dependent libs, and build SGLang main package +pip install --upgrade pip setuptools +pip install -v . --extra-index-url https://download.pytorch.org/whl/xpu +``` - 2. **Install PyTorch and Dependencies** +### Install Using Docker - Set PyTorch XPU as primary pip install channel to avoid installing the larger CUDA-enabled version and prevent potential runtime issues. +The docker for XPU is under active development. Please stay tuned. - ```bash - pip3 install torch==2.9.0+xpu torchao torchvision torchaudio pytorch-triton-xpu==3.5.0 --index-url https://download.pytorch.org/whl/xpu - pip3 install xgrammar --no-deps # xgrammar will introduce CUDA-enabled triton which might conflict with XPU - ``` - - 3. **Cloning** - - Clone the SGLang code - - ```bash - git clone https://github.com/sgl-project/sglang.git - cd sglang - git checkout - ``` - - 4. **Configure Build File** - - Use dedicated toml file - - ```bash - cd python - cp pyproject_xpu.toml pyproject.toml - ``` - - 5. **Build and Install** - - Install SGLang dependent libs, and build SGLang main package - - ```bash - pip install --upgrade pip setuptools - pip install -v . - ``` - - - - The docker for XPU is under active development. Please stay tuned. - - ## Launch of the Serving Engine Example command to launch SGLang serving: - ```bash python -m sglang.launch_server \ --model \ @@ -117,13 +88,12 @@ python -m sglang.launch_server \ --attention-backend intel_xpu \ # using intel optimized XPU attention backend --page-size \ # intel_xpu attention backend supports [32, 64, 128] ``` - ## Benchmarking with Requests -You can benchmark the performance via the `bench_serving` script. Run the command in another terminal. +You can benchmark the performance via the `bench_serving` script. +Run the command in another terminal. - ```bash python -m sglang.bench_serving \ --dataset-name random \ @@ -133,14 +103,13 @@ python -m sglang.bench_serving \ --request-rate inf \ --random-range-ratio 1.0 ``` - The detail explanations of the parameters can be looked up by the command: - ```bash python -m sglang.bench_serving -h ``` - -Additionally, the requests can be formed with [OpenAI Completions API](../basic_usage/openai_api_completions) and sent via the command line (e.g. using `curl`) or via your own script. +Additionally, the requests can be formed with +[OpenAI Completions API](../basic_usage/openai_api_completions) +and sent via the command line (e.g. using `curl`) or via your own script. diff --git a/docs_new/docs/references/environment_variables.mdx b/docs_new/docs/references/environment_variables.mdx index d9693284a..7f0c99cd3 100644 --- a/docs_new/docs/references/environment_variables.mdx +++ b/docs_new/docs/references/environment_variables.mdx @@ -49,26 +49,46 @@ SGLang supports various environment variables that can be used to configure its `false` - `SGLANG_HEALTH_CHECK_TIMEOUT` + SGLANG_LOG_REQUEST_HEADERS + Comma-separated list of additional HTTP headers to log when --log-requests is enabled. Appends to the default x-smg-routing-key. + Not set + + + SGLANG_HEALTH_CHECK_TIMEOUT Timeout for health check in seconds - `20` + 20 - `SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL` + SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL The interval of passes to collect the metric of selected count of physical experts on each layer and GPU rank. 0 means disabled. - `0` + 0 - `SGLANG_FORWARD_UNKNOWN_TOOLS` + SGLANG_FORWARD_UNKNOWN_TOOLS Forward unknown tool calls to clients instead of dropping them - `false` (drop unknown tools) + false (drop unknown tools) - - `SGLANG_QUEUED_TIMEOUT_MS` - Timeout (in ms) for requests in the waiting queue + + SGLANG_REQ_WAITING_TIMEOUT + Timeout (in seconds) for requests waiting in the queue before being scheduled `-1` - + + SGLANG_REQ_RUNNING_TIMEOUT + Timeout (in seconds) for requests running in the decode batch + `-1` + + + SGLANG_CACHE_DIR + Cache directory for model weights and other data + ~/.cache/sglang + + + SGLANG_PREFETCH_BLOCK_SIZE_MB + Block size (in MB) for sequential checkpoint prefetch reads that warm the OS page cache before workers load weights via mmap + 16 + + ## Performance Tuning @@ -95,17 +115,17 @@ SGLang supports various environment variables that can be used to configure its `SGLANG_ENABLE_TORCH_COMPILE` Enable torch.compile - `true` + false `SGLANG_SET_CPU_AFFINITY` Enable CPU affinity setting (often set to `1` in Docker builds) - `0` + false `SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN` Allows the scheduler to overwrite longer context length requests (often set to `1` in Docker builds) - `0` + false `SGLANG_IS_FLASHINFER_AVAILABLE` @@ -139,7 +159,7 @@ SGLang supports various environment variables that can be used to configure its `SGLANG_DISABLE_FA4_WARMUP` - Disable Flash Attention 4 warmup passes (set to `1`, `true`, `yes`, or `on` to disable) + Disable Flash Attention 4 warmup passes (set to 1, true, yes, or on to disable) `false` @@ -149,7 +169,7 @@ SGLang supports various environment variables that can be used to configure its `SGLANG_SCHEDULER_RECV_SKIPPER_WEIGHT_DEFAULT` - Default weight value for scheduler recv skipper counter (used when forward mode doesn't match specific modes). Only active when `--scheduler-recv-interval > 1`. The counter accumulates weights and triggers request polling when reaching the interval threshold. + Default weight value for scheduler recv skipper counter (used when forward mode doesn't match specific modes). Only active when --scheduler-recv-interval > 1. The counter accumulates weights and triggers request polling when reaching the interval threshold. `1000` @@ -158,7 +178,7 @@ SGLang supports various environment variables that can be used to configure its `1` - `SGLANG_SCHEDULER_RECV_SKIPPER_WEIGHT_VERIFY` + SGLANG_SCHEDULER_RECV_SKIPPER_WEIGHT_TARGET_VERIFY Weight increment for target verify forward mode in scheduler recv skipper. Works with `--scheduler-recv-interval` to control polling frequency during verification phase. `1` @@ -185,9 +205,29 @@ SGLang supports various environment variables that can be used to configure its `SGLANG_SYMM_MEM_PREALLOC_GB_SIZE` Size of preallocated GPU buffer (in GB) for NCCL symmetric memory pool to limit memory fragmentation. Only have an effect when server arg `--enable-symm-mem` is set. - `4` + -1 - + + SGLANG_CUSTOM_ALLREDUCE_ALGO + The algorithm of custom all-reduce. Set to oneshot or 1stage to force use one-shot. Set to twoshot or 2stage to force use two-shot. + `` + + + SGLANG_SKIP_SOFTMAX_PREFILL_THRESHOLD_SCALE_FACTOR + Skip-softmax threshold scale factor for TRT-LLM prefill attention in flashinfer. None means standard attention. See https://arxiv.org/abs/2512.12087 + None + + + SGLANG_SKIP_SOFTMAX_DECODE_THRESHOLD_SCALE_FACTOR + Skip-softmax threshold scale factor for TRT-LLM decode attention in flashinfer. None means standard attention. See https://arxiv.org/abs/2512.12087 + None + + + SGLANG_USE_SGL_FA3_KERNEL + Use sgl-kernel implementation for FlashAttention v3 + true + + @@ -233,16 +273,21 @@ SGLang supports various environment variables that can be used to configure its `~/.cache/deep_gemm` - `SGL_DG_USE_NVRTC` + SGLANG_DG_USE_NVRTC Use NVRTC (instead of Triton) for JIT compilation (Experimental) - `"0"` + "false" - `SGL_USE_DEEPGEMM_BMM` + SGLANG_USE_DEEPGEMM_BMM Use DeepGEMM for Batched Matrix Multiplication (BMM) operations `"false"` - + + SGLANG_JIT_DEEPGEMM_FAST_WARMUP + Precompile less kernels during warmup, which reduces the warmup time from 30min to less than 3min. Might cause performance degradation during runtime. + `"false"` + + ## DeepEP Configuration @@ -289,6 +334,70 @@ SGLang supports various environment variables that can be used to configure its +## MORI Configuration + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Environment VariableDescriptionDefault Value
SGLANG_MORI_DISPATCH_DTYPEOverride MoRI-EP dispatch quantization type. auto uses auto-detection from weight dtype; bf16/fp8/fp4 forces the specified type for all layers"auto"
SGLANG_MORI_FP8_COMBUse FP8 for combine"false"
SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANKMaximum number of dispatch tokens per rank for MORI-EP buffer allocation4096
SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLDThreshold for switching between InterNodeV1 and InterNodeV1LL kernel types. InterNodeV1LL is used if SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK is less than or equal to this threshold; otherwise, InterNodeV1 is used.256
SGLANG_MORI_PREALLOC_MAX_RECV_TOKENSThis argument devives SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK which indicates customized amount of tokens preallocated for a rank, valid range from 1 to world_size*SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK, by default 0 means maximum. Setting a smaller value will reduce memory footprint but too small value could cause buffer overflow.0
SGLANG_MORI_MOE_MAX_INPUT_TOKENSTruncate the dispatch buffer to this many rows before MoE computation, reducing kernel overhead on padding tokens. The value must be >= the actual number of received tokens (totalRecvTokenNum); setting it too small causes incorrect results. 0 disables truncation (use full buffer).0
SGLANG_MORI_QP_PER_TRANSFERNumber of RDMA Queue Pairs (QPs) used per transfer operation1
SGLANG_MORI_POST_BATCH_SIZENumber of RDMA work requests posted in a single batch to each QP-1
SGLANG_MORI_NUM_WORKERSNumber of worker threads in the RDMA executor thread pool1
+ ## NSA Backend Configuration (For DeepSeek V3.2) {/* # Environment variable to control mtp precomputing of metadata for multi-step speculative decoding */} @@ -308,14 +417,24 @@ SGLang supports various environment variables that can be used to configure its - `SGLANG_NSA_FUSE_TOPK` + SGLANG_NSA_FUSE_TOPK Fuse the operation of picking topk logits and picking topk indices from page table - `true` + true - `SGLANG_NSA_ENABLE_MTP_PRECOMPUTE_METADATA` + SGLANG_NSA_ENABLE_MTP_PRECOMPUTE_METADATA Precompute metadata that can be shared among different draft steps when MTP is enabled - `true` + true + + + SGLANG_USE_FUSED_METADATA_COPY + Control whether to use fused metadata copy kernel for cuda graph replay + true + + + SGLANG_NSA_PREFILL_DENSE_ATTN_KV_LEN_THRESHOLD + When the maximum kv len in current prefill batch exceeds this value, the sparse mla kernel will be applied, else it falls back to dense MHA implementation. Default to the index topk of model (2048 for DeepSeek V3.2) + 2048 @@ -338,26 +457,31 @@ SGLang supports various environment variables that can be used to configure its - `SGLANG_DEBUG_MEMORY_POOL` + SGLANG_DEBUG_MEMORY_POOL Enable memory pool debugging `false` - `SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION` + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION Clip max new tokens estimation for memory planning - `4096` + 4096 - `SGLANG_DETOKENIZER_MAX_STATES` + SGLANG_DETOKENIZER_MAX_STATES Maximum states for detokenizer Default value based on system - - `SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK` + + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK Enable checks for memory imbalance across Tensor Parallel ranks - `true` + true - + + SGLANG_MOONCAKE_CUSTOM_MEM_POOL + Configure the custom memory pool type for Mooncake. Supports NVLINK, BAREX, INTRA_NODE_NVLINK. If set to true, it defaults to NVLINK. + None + + ## Model-Specific Options @@ -377,17 +501,17 @@ SGLang supports various environment variables that can be used to configure its - `SGLANG_USE_AITER` + SGLANG_USE_AITER Use AITER optimize implementation `false` - `SGLANG_MOE_PADDING` - Enable MoE padding (sets padding size to 128 if value is `1`, often set to `1` in Docker builds) - `0` + SGLANG_MOE_PADDING + Enable MoE padding (sets padding size to 128 if value is 1, often set to 1 in Docker builds) + `false` - `SGLANG_CUTLASS_MOE` (deprecated) + SGLANG_CUTLASS_MOE (deprecated) Use Cutlass FP8 MoE kernel on Blackwell GPUs (deprecated, use --moe-runner-backend=cutlass) `false` @@ -411,51 +535,41 @@ SGLang supports various environment variables that can be used to configure its - `SGLANG_INT4_WEIGHT` + SGLANG_INT4_WEIGHT Enable INT4 weight quantization - `false` + false - `SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2` - Apply per token group quantization kernel with fused silu and mul and masked m - `false` - - - `SGLANG_FORCE_FP8_MARLIN` + SGLANG_FORCE_FP8_MARLIN Force using FP8 MARLIN kernels even if other FP8 kernels are available - `false` + false - `SGLANG_FLASHINFER_FP4_GEMM_BACKEND` (deprecated) - Select backend for `mm_fp4` on Blackwell GPUs. **DEPRECATED**: Please use `--fp4-gemm-backend` instead. - `` - - - `SGLANG_NVFP4_CKPT_FP8_GEMM_IN_ATTN` + SGLANG_NVFP4_CKPT_FP8_GEMM_IN_ATTN Quantize q_b_proj from BF16 to FP8 when launching DeepSeek NVFP4 checkpoint - `false` + false - `SGLANG_MOE_NVFP4_DISPATCH` + SGLANG_MOE_NVFP4_DISPATCH Use nvfp4 for moe dispatch (on flashinfer_cutlass or flashinfer_cutedsl moe runner backend) - `"false"` + "false" - - `SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE` + + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE Quantize moe of nextn layer from BF16 to FP8 when launching DeepSeek NVFP4 checkpoint `false` - - `SGLANG_ENABLE_FLASHINFER_FP8_GEMM` (deprecated) - Use flashinfer kernels when running blockwise fp8 GEMM on Blackwell GPUs. **DEPRECATED**: Please use `--fp8-gemm-backend=flashinfer_trtllm` instead. + + SGLANG_QUANT_ALLOW_DOWNCASTING + Allow weight dtype downcasting during loading (e.g., fp32 → fp16). By default, SGLang rejects this kind of downcasting when using quantization. `false` - - `SGLANG_SUPPORT_CUTLASS_BLOCK_FP8` (deprecated) - Use Cutlass kernels when running blockwise fp8 GEMM on Hopper or Blackwell GPUs. **DEPRECATED**: Please use `--fp8-gemm-backend=cutlass` instead. - `false` + + SGLANG_FP8_IGNORED_LAYERS + A comma-separated list of layer names to ignore during FP8 quantization. For example: model.layers.0,model.layers.1.,qkv_proj. + "" - + @@ -498,6 +612,45 @@ SGLang supports various environment variables that can be used to configure its +## PD Disaggregation — Staging Buffer (Heterogeneous TP) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Environment VariableDescriptionDefault Value
SGLANG_DISAGG_STAGING_BUFFEREnable GPU staging buffer for heterogeneous TP KV transfer. Required when prefill and decode use different TP/attention-TP sizes. Only for non-MLA models (e.g. GQA, MHA).false
SGLANG_DISAGG_STAGING_BUFFER_SIZE_MBPrefill-side per-worker staging buffer size in MB. Used for gathering KV head slices before bulk RDMA transfer.64
SGLANG_DISAGG_STAGING_POOL_SIZE_MBDecode-side ring buffer pool total size in MB. Shared buffer receiving RDMA data from all prefill ranks. Larger values support higher concurrency.4096
SGLANG_STAGING_USE_TORCHForce using PyTorch gather/scatter fallback instead of Triton fused kernels for staging operations. Useful for debugging.false
+ ## Testing & Debugging (Internal/CI) *These variables are primarily used for internal testing, continuous integration, or debugging.* @@ -517,36 +670,66 @@ SGLang supports various environment variables that can be used to configure its - `SGLANG_IS_IN_CI` + SGLANG_IS_IN_CI Indicates if running in CI environment - `false` + false - `SGLANG_IS_IN_CI_AMD` + SGLANG_IS_IN_CI_AMD Indicates running in AMD CI environment - `0` + false - `SGLANG_TEST_RETRACT` + SGLANG_TEST_RETRACT Enable retract decode testing `false` - - `SGLANG_TEST_RETRACT_NO_PREFILL_BS` + + SGLANG_TEST_RETRACT_NO_PREFILL_BS When SGLANG_TEST_RETRACT is enabled, no prefill is performed if the batch size exceeds SGLANG_TEST_RETRACT_NO_PREFILL_BS. - `2 ** 31` + 2 ** 31 - - `SGLANG_RECORD_STEP_TIME` + + SGLANG_RECORD_STEP_TIME Record step time for profiling `false` - - `SGLANG_TEST_REQUEST_TIME_STATS` + + SGLANG_TEST_REQUEST_TIME_STATS Test request time statistics `false` - + + SGLANG_DEBUG_SYMM_MEM + Enable debug checks that verify tensors passed to NCCL communication ops are allocated in the symmetric memory pool. Logs warnings (rank 0 only) with stack traces for any tensor not in the pool. + `false` + + + SGLANG_KERNEL_API_LOGLEVEL + Controls crash-debug kernel API logging. 0 disables logging, 1 logs API names, 3 logs tensor metadata, 5 adds tensor statistics, and 10 also writes pre-call dump snapshots. + 0 + + + SGLANG_KERNEL_API_LOGDEST + Destination for crash-debug kernel API logs. Use stdout, stderr, or a file path. %i is replaced with the process PID. + stdout + + + SGLANG_KERNEL_API_DUMP_DIR + Output directory for level-10 kernel API input/output dumps. %i is replaced with the process PID. + sglang_kernel_api_dumps + + + SGLANG_KERNEL_API_DUMP_INCLUDE + Comma-separated wildcard patterns for kernel API names to include in level-10 dumps. + Not set + + + SGLANG_KERNEL_API_DUMP_EXCLUDE + Comma-separated wildcard patterns for kernel API names to exclude from level-10 dumps. + Not set + + ## Profiling & Benchmarking @@ -597,9 +780,9 @@ SGLang supports various environment variables that can be used to configure its - - - + + + @@ -610,30 +793,36 @@ SGLang supports various environment variables that can be used to configure its - + - + - + - + - + - + + + + + +
`SGLANG_WAIT_WEIGHTS_READY_TIMEOUT`SGLANG_WAIT_WEIGHTS_READY_TIMEOUT Timeout period for waiting on weights`120`120
`SGLANG_DISABLE_OUTLINES_DISK_CACHE`SGLANG_DISABLE_OUTLINES_DISK_CACHE Disable Outlines disk cache`true`false
`SGLANG_USE_CUSTOM_TRITON_KERNEL_CACHE`SGLANG_USE_CUSTOM_TRITON_KERNEL_CACHE Use SGLang's custom Triton kernel cache implementation for lower overheads (automatically enabled on CUDA)`false`false
SGLANG_HICACHE_DECODE_OFFLOAD_STRIDEDecode-side incremental KV cache offload stride. Rounded down to a multiple of --page-size (min is --page-size). If unset/invalid/<=0, it falls back to --page-size.Not set (uses --page-size)
+ ## Function Calling / Tool Use - - - + + + @@ -644,9 +833,9 @@ SGLang supports various environment variables that can be used to configure its - - - + + +
` SGLANG_TOOL_STRICT_LEVEL`Controls strictness for tool-call parsing and validation: **Level 0** off (no strict validation); **Level 1** function strict (enables structural tag constraints for all tools, even if none have `strict=True`); **Level 2** parameter strict (enforces strict parameter validation for all tools as if all had `strict=True`).` 0`SGLANG_TOOL_STRICT_LEVELControls the strictness level of tool call parsing and validation. <br>Level 0: Off - No strict validation <br>Level 1: Function strict - Enables structural tag constraints for all tools (even if none have strict=True set) <br>Level 2: Parameter strict - Enforces strict parameter validation for all tools, treating them as if they all have strict=True set0
diff --git a/docs_new/docs/references/frontend/frontend_tutorial.ipynb b/docs_new/docs/references/frontend/frontend_tutorial.ipynb index 166f8cacc..9c4da052c 100644 --- a/docs_new/docs/references/frontend/frontend_tutorial.ipynb +++ b/docs_new/docs/references/frontend/frontend_tutorial.ipynb @@ -42,7 +42,7 @@ " \"python -m sglang.launch_server --model-path Qwen/Qwen2.5-7B-Instruct --host 0.0.0.0 --log-level warning\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")\n", + "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n", "print(f\"Server started on http://localhost:{port}\")" ] }, @@ -385,7 +385,7 @@ "## Multi-modal Generation\n", "\n", "You may use SGLang frontend language to define multi-modal prompts.\n", - "See [here](https://docs.sglang.io/supported_models/generative_models.html) for supported models." + "See [here](https://docs.sglang.io/supported_models/text_generation/multimodal_language_models.html) for supported models." ] }, { @@ -398,7 +398,7 @@ " \"python -m sglang.launch_server --model-path Qwen/Qwen2.5-VL-7B-Instruct --host 0.0.0.0 --log-level warning\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")\n", + "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n", "print(f\"Server started on http://localhost:{port}\")" ] }, @@ -430,7 +430,7 @@ " s += assistant(gen(\"answer\", max_tokens=256))\n", "\n", "\n", - "image_url = \"https://github.com/sgl-project/sglang/blob/main/examples/assets/example_image.png?raw=true\"\n", + "image_url = \"https://raw.githubusercontent.com/sgl-project/sglang/main/examples/assets/example_image.png\"\n", "image_bytes, _ = load_image(image_url)\n", "state = image_qa(image_bytes, \"What is in the image?\")\n", "print_highlight(state[\"answer\"])" diff --git a/docs_new/docs/references/frontend/frontend_tutorial.mdx b/docs_new/docs/references/frontend/frontend_tutorial.mdx index 6814095ac..7d09ca0a5 100644 --- a/docs_new/docs/references/frontend/frontend_tutorial.mdx +++ b/docs_new/docs/references/frontend/frontend_tutorial.mdx @@ -5,13 +5,10 @@ metatags: --- SGLang frontend language can be used to define simple and easy prompts in a convenient, structured way. - ## Launch A Server Launch the server in your terminal and wait for it to initialize. - - ```python Example from sglang import assistant_begin, assistant_end from sglang import assistant, function, gen, system, user @@ -26,14 +23,12 @@ server_process, port = launch_server_cmd( "python -m sglang.launch_server --model-path Qwen/Qwen2.5-7B-Instruct --host 0.0.0.0 --log-level warning" ) -wait_for_server(f"http://localhost:{port}") +wait_for_server(f"http://localhost:{port}", process=server_process) print(f"Server started on http://localhost:{port}") ``` Set the default backend. Note: Besides the local server, you may use also `OpenAI` or other API endpoints. - - ```python Example set_default_backend(RuntimeEndpoint(f"http://localhost:{port}")) ``` @@ -42,8 +37,6 @@ set_default_backend(RuntimeEndpoint(f"http://localhost:{port}")) The most simple way of using SGLang frontend language is a simple question answer dialog between a user and an assistant. - - ```python Example @function def basic_qa(s, question): @@ -52,7 +45,6 @@ def basic_qa(s, question): s += assistant(gen("answer", max_tokens=512)) ``` - ```python Example state = basic_qa("List 3 countries and their capitals.") print_highlight(state["answer"]) @@ -62,8 +54,6 @@ print_highlight(state["answer"]) SGLang frontend language can also be used to define multi-turn dialogs. - - ```python Example @function def multi_turn_qa(s): @@ -84,8 +74,6 @@ print_highlight(state["second_answer"]) You may use any Python code within the function to define more complex control flows. - - ```python Example @function def tool_use(s, question): @@ -112,8 +100,6 @@ print_highlight(state["expression"]) Use `fork` to launch parallel prompts. Because `sgl.gen` is non-blocking, the for loop below issues two generation calls in parallel. - - ```python Example @function def tip_suggestion(s): @@ -144,8 +130,6 @@ print_highlight(state["summary"]) Use `regex` to specify a regular expression as a decoding constraint. This is only supported for local models. - - ```python Example @function def regular_expression_gen(s): @@ -165,8 +149,6 @@ print_highlight(state["answer"]) Use `regex` to define a `JSON` decoding schema. - - ```python Example character_regex = ( r"""\{\n""" @@ -202,8 +184,6 @@ print_highlight(state["json_output"]) Use `run_batch` to run a batch of prompts. - - ```python Example @function def text_qa(s, question): @@ -228,8 +208,6 @@ for i, state in enumerate(states): Use `stream` to stream the output to the user. - - ```python Example @function def text_qa(s, question): @@ -247,9 +225,7 @@ for out in state.text_iter(): ## Complex Prompts -You may use `{system|user|assistant}_{begin|end}` to define complex prompts. - - +You may use `{system|user|assistant}_{begin|end}` to define complex prompts. ```python Example @function @@ -269,7 +245,6 @@ state = chat_example() print_highlight(state["answer"]) ``` - ```python Example terminate_process(server_process) ``` @@ -277,28 +252,23 @@ terminate_process(server_process) ## Multi-modal Generation You may use SGLang frontend language to define multi-modal prompts. -See [here](../../supported-models/large-language-models) for supported models. - - +See [here](../../supported-models/multimodal_language_models) for supported models. ```python Example server_process, port = launch_server_cmd( "python -m sglang.launch_server --model-path Qwen/Qwen2.5-VL-7B-Instruct --host 0.0.0.0 --log-level warning" ) -wait_for_server(f"http://localhost:{port}") +wait_for_server(f"http://localhost:{port}", process=server_process) print(f"Server started on http://localhost:{port}") ``` - ```python Example set_default_backend(RuntimeEndpoint(f"http://localhost:{port}")) ``` Ask a question about an image. - - ```python Example @function def image_qa(s, image_file, question): @@ -306,13 +276,12 @@ def image_qa(s, image_file, question): s += assistant(gen("answer", max_tokens=256)) -image_url = "https://github.com/sgl-project/sglang/blob/main/examples/assets/example_image.png?raw=true" +image_url = "https://raw.githubusercontent.com/sgl-project/sglang/main/examples/assets/example_image.png" image_bytes, _ = load_image(image_url) state = image_qa(image_bytes, "What is in the image?") print_highlight(state["answer"]) ``` - ```python Example terminate_process(server_process) ``` diff --git a/docs_new/docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.mdx b/docs_new/docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.mdx index 53778af8d..3596d36fd 100644 --- a/docs_new/docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.mdx +++ b/docs_new/docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.mdx @@ -641,7 +641,7 @@ kubectl apply -f p.yaml kubectl apply -f d.yaml ``` -At this point, we have completed the deployment of the 1P1D SGlang engine part. +At this point, we have completed the deployment of the 1P1D SGLang engine part. To allow our users to directly experience the model API, we still need a load balancer to handle sequential calls between prefill and decode. Different companies implement LBs differently, and the community will also officially release a new LB component written in Rust in the near future. @@ -776,7 +776,7 @@ At this point, select a nodePort:30800 to access: > ], > "max_tokens":221 > }' -{"id":"ccccdd","object":"chat.completion","created":1750252498,"model":"qwen2","choices":[{"index":0,"message":{"role":"assistant","content":"<think>\n嗯，用户问了一个很基础的自我介绍问题"你是谁？"。这可能是第一次互动时的常规开场白，也可能是想确认我的身份和功能范围。\n\n用户没有提供任何背景信息，语气简洁中性。这种场景下新用户的可能性较高，需要给出清晰友好的自我介绍，同时突出实用价值来降低陌生感。\n\n考虑到中文用户，应该用简体中文回复。重点要说明三点：身份归属（深度求索）、功能定位（AI助手）、服务范围（学习/工作/生活）。结尾用开放性问题引导对话很关键——既能了解需求，又能避免让用户面对空白输入框时不知所措。\n\n用波浪线结尾可以软化语气，那个笑脸表情😊刚好能中和AI的机械感。不过要控制表情符号数量，避免显得轻浮。\n</think>\n你好呀！我是你的AI助手，由深度求索公司（DeepSeek）开发的语言模型，名字叫 **DeepSeek-R1**。你可以把我当成一个知识丰富、随叫随到的小帮手～😊\n\n我的任务就是陪你聊天、解答问题、","reasoning_content":null,"tool_calls":null},"logprobs":null,"finish_reason":"length","matched_stop":null}],"usage":{"prompt_tokens":14,"total_tokens":235,"completion_tokens":221,"prompt_tokens_details":null}} +{"id":"ccccdd","object":"chat.completion","created":1750252498,"model":"qwen2","choices":[{"index":0,"message":{"role":"assistant","content":"\n嗯，用户问了一个很基础的自我介绍问题"你是谁？"。这可能是第一次互动时的常规开场白，也可能是想确认我的身份和功能范围。\n\n用户没有提供任何背景信息，语气简洁中性。这种场景下新用户的可能性较高，需要给出清晰友好的自我介绍，同时突出实用价值来降低陌生感。\n\n考虑到中文用户，应该用简体中文回复。重点要说明三点：身份归属（深度求索）、功能定位（AI助手）、服务范围（学习/工作/生活）。结尾用开放性问题引导对话很关键——既能了解需求，又能避免让用户面对空白输入框时不知所措。\n\n用波浪线结尾可以软化语气，那个笑脸表情😊刚好能中和AI的机械感。不过要控制表情符号数量，避免显得轻浮。\n\n你好呀！我是你的AI助手，由深度求索公司（DeepSeek）开发的语言模型，名字叫 **DeepSeek-R1**。你可以把我当成一个知识丰富、随叫随到的小帮手～😊\n\n我的任务就是陪你聊天、解答问题、","reasoning_content":null,"tool_calls":null},"logprobs":null,"finish_reason":"length","matched_stop":null}],"usage":{"prompt_tokens":14,"total_tokens":235,"completion_tokens":221,"prompt_tokens_details":null}} ``` ## FAQ diff --git a/docs_new/docs/references/production_metrics.mdx b/docs_new/docs/references/production_metrics.mdx index 96ff6c899..97de5a708 100644 --- a/docs_new/docs/references/production_metrics.mdx +++ b/docs_new/docs/references/production_metrics.mdx @@ -145,7 +145,8 @@ This section describes how to set up the monitoring stack (Prometheus + Grafana) python -m sglang.launch_server \ --model-path \ --port 30000 \ - --enable-metrics + --enable-metrics \ + --enable-mfu-metrics ``` Replace `` with the actual path to your model (e.g., `meta-llama/Meta-Llama-3.1-8B-Instruct`). Ensure the server is accessible from the monitoring stack (you might need `--host 0.0.0.0` if running in Docker). By default, the metrics endpoint will be available at `http://:30000/metrics`. @@ -179,13 +180,13 @@ This section describes how to set up the monitoring stack (Prometheus + Grafana) To modify Grafana's port to the other one(like 3090) in your Docker Compose file, you need to explicitly specify the port mapping under the grafana service. Option 1: Add GF_SERVER_HTTP_PORT to the environment section: - ```text Output + ``` environment: - GF_AUTH_ANONYMOUS_ENABLED=true - GF_SERVER_HTTP_PORT=3090 # <-- Add this line ``` Option 2: Use port mapping: - ```text Output + ``` grafana: image: grafana/grafana:latest container_name: grafana @@ -232,3 +233,38 @@ python3 -m sglang.bench_serving \ to generate some requests. Then you should be able to see the metrics in the Grafana dashboard. + +## Estimated Performance Metrics (MFU-related) + +SGLang exports the following estimated per-GPU counters that can be used to derive +Model FLOPs Utilization (MFU)-related signals: + +- `sglang:estimated_flops_per_gpu_total`: Estimated floating-point operations. +- `sglang:estimated_read_bytes_per_gpu_total`: Estimated bytes read from memory. +- `sglang:estimated_write_bytes_per_gpu_total`: Estimated bytes written to memory. + +These metrics are available when both `--enable-metrics` and +`--enable-mfu-metrics` are enabled. + +These are cumulative counters. Use Prometheus `rate(...)` to get per-second values. + +### PromQL examples + +Average TFLOPS per GPU: + +```promql +rate(sglang:estimated_flops_per_gpu_total[1m]) / 1e12 +``` + +Average estimated memory bandwidth in GB/s: + +```promql +(rate(sglang:estimated_read_bytes_per_gpu_total[1m]) + + rate(sglang:estimated_write_bytes_per_gpu_total[1m])) / 1e9 +``` + +### Notes + +- These metrics are estimates intended for observability and trend analysis. +- Estimated memory bytes reflect modeled traffic and are not a direct hardware + counter from GPU profilers. diff --git a/docs_new/docs/references/production_request_trace.mdx b/docs_new/docs/references/production_request_trace.mdx index 53f000ac1..d81e41e56 100644 --- a/docs_new/docs/references/production_request_trace.mdx +++ b/docs_new/docs/references/production_request_trace.mdx @@ -3,7 +3,7 @@ title: "Production Request Tracing" metatags: description: "SGLang OpenTelemetry tracing: Jaeger visualization, trace context propagation, PD disaggregation support." --- -SGlang exports request trace data based on the OpenTelemetry Collector. You can enable tracing by adding the `--enable-trace` and configure the OpenTelemetry Collector endpoint using `--otlp-traces-endpoint` when launching the server. +SGLang exports request trace data based on the OpenTelemetry Collector. You can enable tracing by adding the `--enable-trace` and configure the OpenTelemetry Collector endpoint using `--otlp-traces-endpoint` when launching the server. You can find example screenshots of the visualization in https://github.com/sgl-project/sglang/issues/8965. @@ -20,23 +20,23 @@ This section explains how to configure the request tracing and export the trace pip install opentelemetry-sdk opentelemetry-api opentelemetry-exporter-otlp opentelemetry-exporter-otlp-proto-grpc ``` -2. launch opentelemetry collector and jaeger +2. Launch OpenTelemetry collector and Jaeger ```bash Command docker compose -f examples/monitoring/tracing_compose.yaml up -d ``` -3. start your SGLang server with tracing enabled +3. Start your SGLang server with tracing enabled ```bash Command # set env variables export SGLANG_OTLP_EXPORTER_SCHEDULE_DELAY_MILLIS=500 export SGLANG_OTLP_EXPORTER_MAX_EXPORT_BATCH_SIZE=64 # start the prefill and decode server python -m sglang.launch_server --enable-trace --otlp-traces-endpoint 0.0.0.0:4317 - # start the mini lb + # start the model-gate-way python -m sglang_router.launch_router --enable-trace --otlp-traces-endpoint 0.0.0.0:4317 ``` - Replace `0.0.0.0:4317` with the actual endpoint of the opentelemetry collector. If you launched the openTelemetry collector with tracing_compose.yaml, the default receiving port is 4317. + Replace `0.0.0.0:4317` with the actual endpoint of the OpenTelemetry collector. If you launched the openTelemetry collector with tracing_compose.yaml, the default receiving port is 4317. To use the HTTP/protobuf span exporter, set the following environment variable and point to an HTTP endpoint, for example, `http://0.0.0.0:4318/v1/traces`. ```bash Command @@ -44,15 +44,33 @@ This section explains how to configure the request tracing and export the trace ``` -4. raise some requests +4. Raise some requests 5. Observe whether trace data is being exported * Access port 16686 of Jaeger using a web browser to visualize the request traces. * The OpenTelemetry Collector also exports trace data in JSON format to /tmp/otel_trace.json. In a follow-up patch, we will provide a tool to convert this data into a Perfetto-compatible format, enabling visualization of requests in the Perfetto UI. -## How to add Tracing for slices you're interested in? +6. Dynamically adjust trace level + The trace level accepts configurable values from `0` to `3`. The meanings of different trace level values are as follows: + ``` + 0: disable tracing + 1: Trace important slices + 2: Trace all slices except nested ones + 3: Trace all slices + ``` + The trace level can be dynamically set via HTTP API, for example: + ```bash Command + curl http://0.0.0.0:30000/set_trace_level?level=2 + ``` + Replace `0.0.0.0:30000` with your actual server address, and replace `level=2` with the level you want to set. + + **Note**: You must set the parameter `--enable-trace`; otherwise, the trace capability will not be enabled regardless of any dynamic adjustments to the trace level. + +## How to add Tracing for slices you're interested in?(API introduction) We have already inserted instrumentation points in the tokenizer and scheduler main threads. If you wish to trace additional request execution segments or perform finer-grained tracing, please use the APIs from the tracing package as described below. -1. initialization +**All of the following implementations are done in python/sglang/srt/observability/req_time_stats.py. If you want to add another slice, please do it here.** + +1. Initialization Every process involved in tracing during the initialization phase should execute: ```python Example @@ -66,99 +84,53 @@ We have already inserted instrumentation points in the tokenizer and scheduler m ``` The "thread label" can be regarded as the name of the thread, used to distinguish different threads in the visualization view. -2. Mark the beginning and end of a request - ```text Output - trace_req_start(rid, bootstrap_room) - trace_req_finish(rid) - ``` - These two APIs must be called within the same process, for example, in the tokenizer. +2. Create a trace context for a request + Each request needs to call `TraceReqContext()` to initialize a request context, which is used to generate slice spans and record request stage info. You can either store it within the request object or maintain it as a global variable. -3. Add tracing for slice +3. Mark the beginning and end of a request + ``` + trace_ctx.trace_req_start(). + trace_ctx.trace_req_finish() + ``` + trace_req_start() and trace_req_finish() must be called within the same process, for example, in the tokenizer. + +4. Add tracing for a slice * Add slice tracing normally: ```python Example - trace_slice_start("slice A", rid) - trace_slice_end("slice A", rid) + trace_ctx.trace_slice_start(RequestStage.TOKENIZER.stage_name) + trace_ctx.trace_slice_end(RequestStage.TOKENIZER.stage_name) + + or + trace_ctx.trace_slice(slice: TraceSliceContext) ``` - - Use the "anonymous" flag to not specify a slice name at the start of the slice, allowing the slice name to be determined by trace_slice_end. - -Note: Anonymous slices must not be nested. + - The end of the last slice in a thread must be marked with thread_finish_flag=True, or explicitly call trace_ctx.abort(); otherwise, the thread's span will not be properly generated. ```python Example - trace_slice_start("", rid, anonymous = True) - trace_slice_end("slice A", rid) + trace_ctx.slice_end(RequestStage.D.stage_name, thread_finish_flag = True) + trace_ctx.abort() ``` - - In trace_slice_end, use auto_next_anon to automatically create the next anonymous slice, which can reduce the number of instrumentation points needed. - ```python Example - trace_slice_start("", rid, anonymous = True) - trace_slice_end("slice A", rid, auto_next_anon = True) - trace_slice_end("slice B", rid, auto_next_anon = True) - trace_slice_end("slice C", rid, auto_next_anon = True) - trace_slice_end("slice D", rid) - ``` - - The end of the last slice in a thread must be marked with thread_finish_flag=True; otherwise, the thread's span will not be properly generated. - ```python Example - trace_slice_end("slice D", rid, thread_finish_flag = True) - ``` - -4. When the request execution flow transfers to another thread, the trace context needs to be explicitly propagated. - - sender: Execute the following code before sending the request to another thread via ZMQ - ```python Example - trace_context = trace_get_proc_propagate_context(rid) - req.trace_context = trace_context - ``` +5. When the request execution flow transfers to another thread, the thread context needs to be explicitly rebuilt. - receiver: Execute the following code after receiving the request via ZMQ ```python Example - trace_set_proc_propagate_context(rid, req.trace_context) - ``` - -5. When the request execution flow transfers to another node(PD disaggregation), the trace context needs to be explicitly propagated. - - sender: Execute the following code before sending the request to node thread via http - ```python Example - trace_context = trace_get_remote_propagate_context(bootstrap_room_list) - headers = {"trace_context": trace_context} - session.post(url, headers=headers) - ``` - - receiver: Execute the following code after receiving the request via http - ```python Example - trace_set_remote_propagate_context(request.headers['trace_context']) + trace_ctx.rebuild_thread_context() ``` ## How to Extend the Tracing Framework to Support Complex Tracing Scenarios The currently provided tracing package still has potential for further development. If you wish to build more advanced features upon it, you must first understand its existing design principles. -The core of the tracing framework's implementation lies in the design of the span structure and the trace context. To aggregate scattered slices and enable concurrent tracking of multiple requests, we have designed a two-level trace context structure and a four-level span structure: `SglangTraceReqContext`, `SglangTraceThreadContext`. Their relationship is as follows: -```text Output -SglangTraceReqContext (req_id="req-123") -+-- SglangTraceThreadContext(thread_label="scheduler", tp_rank=0) +The core of the tracing framework's implementation lies in the design of the span structure and the trace context. To aggregate scattered slices and enable concurrent tracking of multiple requests, we have designed a three-level trace context structure or span structure: `TraceReqContext`, `TraceThreadContext` and `TraceSliceContext`. Their relationship is as follows: +``` +TraceReqContext (req_id="req-123") +├── TraceThreadContext(thread_label="scheduler", tp_rank=0) +| └── TraceSliceContext(slice_name="prefill") | -+-- SglangTraceThreadContext(thread_label="scheduler", tp_rank=1) +└── TraceThreadContext(thread_label="scheduler", tp_rank=1) + └── TraceSliceContext(slice_name="prefill") ``` -Each traced request maintains a global `SglangTraceReqContext`. For every thread processing the request, a corresponding `SglangTraceThreadContext` is recorded and composed within the `SglangTraceReqContext`. Within each thread, every currently traced slice (possibly nested) is stored in a list. +Each traced request maintains a global `TraceReqContext` and creates a corresponding request span. For every thread that processes the request, a `TraceThreadContext` is recorded and a thread span is created. The `TraceThreadContext` is nested within the `TraceReqContext`, and each currently traced code slice—potentially nested—is stored in its associated `TraceThreadContext`. In addition to the above hierarchy, each slice also records its previous slice via Span.add_link(), which can be used to trace the execution flow. - -When the request execution flow transfers to a new thread, the trace context needs to be explicitly propagated. In the framework, this is represented by `SglangTracePropagateContext`, which contains the context of the request span and the previous slice span. - - -We designed a four-level span structure, consisting of `bootstrap_room_span`, `req_root_span`, `thread_span`, and `slice_span`. Among them, `req_root_span` and `thread_span` correspond to `SglangTraceReqContext` and `SglangTraceThreadContext`, respectively, and `slice_span` is stored within the `SglangTraceThreadContext`. The `bootstrap_room_span` is designed to accommodate the separation of PD-disaggregation. On different nodes, we may want to add certain attributes to the `req_root_span`. However, if the `req_root_span` is shared across all nodes, the Prefill and Decode nodes would not be allowed to add attributes due to the constraints imposed by OpenTelemetry's design. - -```text Output -bootstrap room span -+-- router req root span -| +-- router thread span -| +-- slice span -+-- prefill req root span -| +-- tokenizer thread span -| | +-- slice span -| +-- scheduler thread span -| +-- slice span -+-- decode req root span - +-- tokenizer thread span - | +-- slice span - +-- scheduler thread span - +-- slice span -``` diff --git a/docs_new/docs/sglang-diffusion/api/cli.mdx b/docs_new/docs/sglang-diffusion/api/cli.mdx index 73f913ffd..8cdcfabf5 100644 --- a/docs_new/docs/sglang-diffusion/api/cli.mdx +++ b/docs_new/docs/sglang-diffusion/api/cli.mdx @@ -3,295 +3,255 @@ title: CLI reference sidebarTitle: CLI description: Run one-off generation tasks and launch the HTTP server from the command line. --- +Use the CLI for one-off generation with `sglang generate` or to start a persistent HTTP server with `sglang serve`. -The `sglang` CLI provides two main subcommands for diffusion inference: +### Overlay repos for non-diffusers models -- **`sglang generate`** -- run a one-off generation without a persistent server -- **`sglang serve`** -- launch the OpenAI-compatible HTTP server +If `--model-path` points to a supported non-diffusers source repo, SGLang can resolve it +through a self-hosted overlay repo. -## Prerequisites +SGLang first checks a built-in overlay registry. Concrete built-in mappings can be added over time without changing the CLI surface. -A working SGLang Diffusion installation with the `sglang` CLI available in your `$PATH`. See the [installation guide](../installation) for setup instructions. +Override example: + +```bash Command +export SGLANG_DIFFUSION_MODEL_OVERLAY_REGISTRY='{ + "Wan-AI/Wan2.2-S2V-14B": { + "overlay_repo_id": "your-org/Wan2.2-S2V-14B-overlay", + "overlay_revision": "main" + } +}' + +sglang generate \ + --model-path Wan-AI/Wan2.2-S2V-14B \ + --config configs/wan_s2v.yaml +``` + +The overlay repo should be a complete diffusers-style/componentized repo + +You can also pass the overlay repo itself as `--model-path` if it contains `_overlay/overlay_manifest.json`. + +Notes: +1. `SGLANG_DIFFUSION_MODEL_OVERLAY_REGISTRY` is only an optional override for +development and debugging. It accepts either a JSON object or a path to a JSON +file, and can extend or replace built-in entries for the current process. +2. On the first load, SGLang will: + - download overlay metadata from the overlay repo + - download the required files from the original source repo + - materialize a local standard component repo under `~/.cache/sgl_diffusion/materialized_models/` +3. Later loads reuse the materialized local repo. The materialized repo is what the runtime loads as a normal componentized model directory. + + +## Quick Start + +### Generate + +```bash Command +sglang generate \ + --model-path Qwen/Qwen-Image \ + --prompt "A beautiful sunset over the mountains" \ + --save-output +``` + +### Serve + +```bash Command +sglang serve \ + --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ + --num-gpus 4 \ + --ulysses-degree 2 \ + --ring-degree 2 \ + --port 30010 +``` + +For request and response examples, see [OpenAI-Compatible API](./openai_api). + + +Use `sglang generate --help` and `sglang serve --help` for the full argument list. The CLI help output is the source of truth for exhaustive flags. + + +## Common Options + +### Model and runtime + +- `--model-path {MODEL}`: model path or Hugging Face model ID +- `--lora-path {PATH}` and `--lora-nickname {NAME}`: load a LoRA adapter +- `--num-gpus {N}`: number of GPUs to use +- `--tp-size {N}`: tensor parallelism size, mainly for encoders +- `--sp-degree {N}`: sequence parallelism size +- `--ulysses-degree {N}` and `--ring-degree {N}`: USP parallelism controls +- `--attention-backend {BACKEND}`: attention backend for native SGLang pipelines +- `--attention-backend-config {CONFIG}`: attention backend configuration + +### Sampling and output + +- `--prompt {PROMPT}` and `--negative-prompt {PROMPT}` +- `--image-path {PATH} [{PATH} ...]`: input image(s) for image-to-video or image-to-image generation +- `--num-inference-steps {STEPS}` and `--seed {SEED}` +- `--height {HEIGHT}`, `--width {WIDTH}`, `--num-frames {N}`, `--fps {FPS}` +- `--output-path {PATH}`, `--output-file-name {NAME}`, `--save-output`, `--return-frames` + +For frame interpolation and upscaling, see [Post-Processing](./post_processing). + +### Quantized transformers + +For quantized transformer checkpoints, prefer: + +- `--model-path` for the base pipeline +- `--transformer-path` for a quantized `transformers` transformer component folder +- `--transformer-weights-path` for a quantized safetensors file, directory, or repo + +See [Quantization](../quantization) for supported quantization families and examples. + +## Configuration Files + +Use `--config` to load JSON or YAML configuration. Command-line flags override values from the config file. + +```bash Command +sglang generate --config config.yaml +``` + +Example: + +```yaml Config +model_path: FastVideo/FastHunyuan-diffusers +prompt: A beautiful woman in a red dress walking down a street +output_path: outputs/ +num_gpus: 2 +sp_size: 2 +tp_size: 1 +num_frames: 45 +height: 720 +width: 1280 +num_inference_steps: 6 +seed: 1024 +fps: 24 +precision: bf16 +vae_precision: fp16 +vae_tiling: true +vae_sp: true +enable_torch_compile: false +``` ## Generate -Run a one-off generation task without launching a persistent server. Pass both server arguments and sampling parameters after the `generate` subcommand: +`sglang generate` runs a single generation job and exits when the job finishes. -```bash -SERVER_ARGS=( - --model-path Wan-AI/Wan2.2-T2V-A14B-Diffusers - --text-encoder-cpu-offload - --pin-cpu-memory - --num-gpus 4 - --ulysses-degree=2 - --ring-degree=2 -) - -SAMPLING_ARGS=( - --prompt "A curious raccoon" - --save-output - --output-path outputs - --output-file-name "A curious raccoon.mp4" -) - -sglang generate "${SERVER_ARGS[@]}" "${SAMPLING_ARGS[@]}" -``` - -You can also enable Cache-DiT acceleration via an environment variable: - -```bash -SGLANG_CACHE_DIT_ENABLED=true sglang generate "${SERVER_ARGS[@]}" "${SAMPLING_ARGS[@]}" +```bash Command +sglang generate \ + --model-path Wan-AI/Wan2.2-T2V-A14B-Diffusers \ + --text-encoder-cpu-offload \ + --pin-cpu-memory \ + --num-gpus 4 \ + --ulysses-degree 2 \ + --ring-degree 2 \ + --prompt "A curious raccoon" \ + --save-output \ + --output-path outputs \ + --output-file-name "a-curious-raccoon.mp4" ``` -HTTP server-related arguments are ignored in `generate` mode. The process shuts down automatically once generation completes. +HTTP server-only arguments are ignored by `sglang generate`. +For diffusers pipelines, Cache-DiT can be enabled with `SGLANG_CACHE_DIT_ENABLED=true` or `--cache-dit-config`. See [Cache-DiT](../cache_dit). + ## Serve -Launch the SGLang Diffusion HTTP server and interact through the OpenAI-compatible API. +`sglang serve` starts the HTTP server and keeps the model loaded for repeated requests. -```bash -SERVER_ARGS=( - --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers - --text-encoder-cpu-offload - --pin-cpu-memory - --num-gpus 4 - --ulysses-degree=2 - --ring-degree=2 -) - -sglang serve "${SERVER_ARGS[@]}" -``` - -- `--model-path` -- which model to load (e.g. `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`) -- `--port` -- HTTP port to listen on (default: `30010`) - -For full API usage including image/video generation and LoRA management, see the [OpenAI API documentation](./openai-api). - ---- - -## Supported arguments - -### Server arguments - - - -| Argument | Description | -|:--|:--| -| `--model-path MODEL_PATH` | Path to the model or HuggingFace model ID | -| `--lora-path LORA_PATH` | Path to a LoRA adapter (local or HuggingFace ID). If omitted, LoRA is not applied | -| `--lora-nickname NAME` | Nickname for the LoRA adapter (default: `default`) | -| `--num-gpus NUM` | Number of GPUs to use | -| `--tp-size SIZE` | Tensor parallelism size (encoder only; keep at most 1 when text encoder offload is enabled) | -| `--sp-degree SIZE` | Sequence parallelism size (typically should match the number of GPUs) | -| `--ulysses-degree SIZE` | DeepSpeed-Ulysses-style SP degree in USP | -| `--ring-degree SIZE` | Ring attention-style SP degree in USP | -| `--attention-backend BACKEND` | Attention backend. Native pipelines: `fa`, `torch_sdpa`, `sage_attn`, etc. Diffusers pipelines: `flash`, `_flash_3_hub`, `sage`, `xformers` | -| `--attention-backend-config CONFIG` | Config for the attention backend. Accepts a JSON string, a JSON/YAML file path, or `key=value` pairs | -| `--cache-dit-config PATH` | Path to a Cache-DiT YAML/JSON config (diffusers backend only) | -| `--dit-precision DTYPE` | Precision for the DiT model (`fp32`, `fp16`, `bf16`) | -| `--text-encoder-cpu-offload` | Offload text encoders to CPU | -| `--pin-cpu-memory` | Pin CPU memory for faster transfers | - - - -### Sampling parameters - - - -| Argument | Description | -|:--|:--| -| `--prompt PROMPT` | Text description for the image or video to generate | -| `--negative-prompt PROMPT` | Negative prompt to guide generation away from certain concepts | -| `--num-inference-steps STEPS` | Number of denoising steps | -| `--seed SEED` | Random seed for reproducible generation | - - - - - -| Argument | Description | -|:--|:--| -| `--height HEIGHT` | Height of the generated output | -| `--width WIDTH` | Width of the generated output | -| `--num-frames NUM` | Number of frames to generate (video only) | -| `--fps FPS` | Frames per second for the saved output (video only) | - - - - - -| Argument | Description | -|:--|:--| -| `--save-output` | Save the image or video to disk | -| `--output-path PATH` | Directory to save the generated output | -| `--output-file-name NAME` | File name for the saved output | -| `--return-frames` | Return the raw frames instead of saving | - - - -### Frame interpolation (video only) - -Frame interpolation is a post-processing step that synthesizes new frames between each pair of consecutive generated frames, producing smoother motion without re-running the diffusion model. - -The `--frame-interpolation-exp` flag controls how many rounds of interpolation to apply: each round inserts one new frame into every gap between adjacent frames, so the output frame count follows the formula: - -$$ -\text{output frames} = (N - 1) \times 2^{\text{exp}} + 1 -$$ - -For example, 5 original frames with `exp=1` -> 4 gaps x 1 new frame + 5 originals = **9 frames**; with `exp=2` -> **17 frames**. - -| Argument | Description | -|:--|:--| -| `--enable-frame-interpolation` | Enable frame interpolation. Model weights are downloaded automatically on first use | -| `--frame-interpolation-exp EXP` | Interpolation exponent -- `1` = 2x temporal resolution, `2` = 4x, etc. (default: `1`) | -| `--frame-interpolation-scale SCALE` | RIFE inference scale; use `0.5` for high-resolution inputs to save memory (default: `1.0`) | -| `--frame-interpolation-model-path PATH` | Local directory or HuggingFace repo ID containing RIFE `flownet.pkl` weights (default: `elfgum/RIFE-4.22.lite`, downloaded automatically) | - -**Example** -- generate a 5-frame video and interpolate to 9 frames ($(5 - 1) \times 2^1 + 1 = 9$): - -```bash -sglang generate \ - --model-path Wan-AI/Wan2.2-T2V-A14B-Diffusers \ - --prompt "A dog running through a park" \ - --num-frames 5 \ - --enable-frame-interpolation \ - --frame-interpolation-exp 1 \ - --save-output -``` - ---- - -## Configuration files - -Instead of passing every parameter on the command line, you can use a JSON or YAML config file. Command-line arguments take precedence over config values. - -```bash -sglang generate --config config.json -``` - - - - ```json config.json - { - "model_path": "FastVideo/FastHunyuan-diffusers", - "prompt": "A beautiful woman in a red dress walking down a street", - "output_path": "outputs/", - "num_gpus": 2, - "sp_size": 2, - "tp_size": 1, - "num_frames": 45, - "height": 720, - "width": 1280, - "num_inference_steps": 6, - "seed": 1024, - "fps": 24, - "precision": "bf16", - "vae_precision": "fp16", - "vae_tiling": true, - "vae_sp": true, - "vae_config": { - "load_encoder": false, - "load_decoder": true, - "tile_sample_min_height": 256, - "tile_sample_min_width": 256 - }, - "text_encoder_precisions": ["fp16", "fp16"], - "mask_strategy_file_path": null, - "enable_torch_compile": false - } - ``` - - - ```yaml config.yaml - model_path: "FastVideo/FastHunyuan-diffusers" - prompt: "A beautiful woman in a red dress walking down a street" - output_path: "outputs/" - num_gpus: 2 - sp_size: 2 - tp_size: 1 - num_frames: 45 - height: 720 - width: 1280 - num_inference_steps: 6 - seed: 1024 - fps: 24 - precision: "bf16" - vae_precision: "fp16" - vae_tiling: true - vae_sp: true - vae_config: - load_encoder: false - load_decoder: true - tile_sample_min_height: 256 - tile_sample_min_width: 256 - text_encoder_precisions: - - "fp16" - - "fp16" - mask_strategy_file_path: null - enable_torch_compile: false - ``` - - - -To see all available options: - -```bash -sglang generate --help -``` - ---- - -## Component path overrides - -You can override any pipeline component (e.g. `vae`, `transformer`, `text_encoder`) by specifying a custom checkpoint path with `---path`, where `` matches the key in the model's `model_index.json`. - -### Example: FLUX.2-dev with Tiny AutoEncoder - -Replace the default VAE with a distilled tiny autoencoder for ~3x faster decoding: - -```bash +```bash Command sglang serve \ - --model-path=black-forest-labs/FLUX.2-dev \ - --vae-path=fal/FLUX.2-Tiny-AutoEncoder + --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ + --text-encoder-cpu-offload \ + --pin-cpu-memory \ + --num-gpus 4 \ + --ulysses-degree 2 \ + --ring-degree 2 \ + --port 30010 ``` -You can also use a local path: +### Cloud Storage -```bash +SGLang Diffusion can upload generated images and videos to S3-compatible object storage after generation. + +```bash Command +export SGLANG_CLOUD_STORAGE_TYPE=s3 +export SGLANG_S3_BUCKET_NAME=my-bucket +export SGLANG_S3_ACCESS_KEY_ID=your-access-key +export SGLANG_S3_SECRET_ACCESS_KEY=your-secret-key +export SGLANG_S3_ENDPOINT_URL=https://minio.example.com +``` + +See [Environment Variables](../environment_variables) for the full set of storage options. + +## Component Path Overrides + +Override individual pipeline components such as `vae`, `transformer`, or `text_encoder` with `---path`. + +```bash Command sglang serve \ - --model-path=black-forest-labs/FLUX.2-dev \ - --vae-path=~/.cache/huggingface/hub/models--fal--FLUX.2-Tiny-AutoEncoder/snapshots/.../vae + --model-path black-forest-labs/FLUX.2-dev \ + --vae-path fal/FLUX.2-Tiny-AutoEncoder ``` - -The component key must match the one in the model's `model_index.json` (e.g. `vae`). -The path must be either a HuggingFace repo ID or point to a complete component folder containing `config.json` and safetensors files. - +The component key must match the key in the model's `model_index.json`, and the path must be either a Hugging Face repo ID or a complete component directory. ---- +## Diffusers Backend -## Diffusers backend +Use `--backend diffusers` to force vanilla diffusers pipelines when no native SGLang implementation exists or when a model requires a custom pipeline class. -SGLang Diffusion supports a diffusers backend that runs any diffusers-compatible model through SGLang's infrastructure using vanilla diffusers pipelines. This is useful for models without native SGLang implementations or models with custom pipeline classes. +### Key Options -### Backend arguments + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ArgumentValuesDescription
--backendauto, sglang, diffusersChoose native SGLang, force native, or force diffusers
--diffusers-attention-backendflash, _flash_3_hub, sage, xformers, nativeAttention backend for diffusers pipelines
--trust-remote-codeflagRequired for models with custom pipeline classes
--vae-tiling and --vae-slicingflagLower memory usage for VAE decode
--dit-precision and --vae-precisionfp16, bf16, fp32Precision controls
--enable-torch-compileflagEnable torch.compile
--cache-dit-config{PATH}Cache-DiT config for diffusers pipelines
-| Argument | Values | Description | -|:--|:--|:--| -| `--backend` | `auto` (default), `sglang`, `diffusers` | `auto`: prefer native SGLang, fallback to diffusers. `sglang`: force native (fails if unavailable). `diffusers`: force vanilla diffusers pipeline | -| `--diffusers-attention-backend` | `flash`, `_flash_3_hub`, `sage`, `xformers`, `native` | Attention backend for diffusers pipelines | -| `--trust-remote-code` | flag | Required for models with custom pipeline classes | -| `--vae-tiling` | flag | Enable VAE tiling for large image support (decodes tile-by-tile) | -| `--vae-slicing` | flag | Enable VAE slicing for lower memory usage (decodes slice-by-slice) | -| `--dit-precision` | `fp16`, `bf16`, `fp32` | Precision for the diffusion transformer | -| `--vae-precision` | `fp16`, `bf16`, `fp32` | Precision for the VAE | - -### Example: running Ovis-Image-7B - -[Ovis-Image-7B](https://huggingface.co/AIDC-AI/Ovis-Image-7B) is a 7B text-to-image model optimized for high-quality text rendering. +### Example ```bash sglang generate \ @@ -308,59 +268,4 @@ sglang generate \ --output-file-name ovis_garden.png ``` -### Extra diffusers arguments - -For pipeline-specific parameters not exposed via CLI, use `diffusers_kwargs` in a config file: - -```json config.json -{ - "model_path": "AIDC-AI/Ovis-Image-7B", - "backend": "diffusers", - "prompt": "A beautiful landscape", - "diffusers_kwargs": { - "cross_attention_kwargs": {"scale": 0.5} - } -} -``` - -```bash -sglang generate --config config.json -``` - -### Cache-DiT acceleration - -Users on the diffusers backend can leverage Cache-DiT acceleration by loading custom cache configs from a YAML file. See the [Cache-DiT documentation](../cache-dit) for details. - ---- - -## Cloud storage support - -The server supports automatically uploading generated artifacts to S3-compatible cloud storage (AWS S3, MinIO, Alibaba Cloud OSS, Tencent Cloud COS). - -The workflow is: **Generate -> Upload -> Delete local file**. The API response returns the public URL of the uploaded object. - -1. **Install boto3** - -```bash -pip install boto3 -``` - -2. **Set environment variables** - -```bash -export SGLANG_CLOUD_STORAGE_TYPE=s3 -export SGLANG_S3_BUCKET_NAME=my-bucket -export SGLANG_S3_ACCESS_KEY_ID=your-access-key -export SGLANG_S3_SECRET_ACCESS_KEY=your-secret-key - -# Optional: custom endpoint for MinIO/OSS/COS -export SGLANG_S3_ENDPOINT_URL=https://minio.example.com -``` - -3. **Launch the server** - -```bash -sglang serve --model-path MODEL_PATH -``` - -See the [environment variables reference](../environment-variables) for all storage-related variables. +For pipeline-specific arguments not exposed in the CLI, pass `diffusers_kwargs` in a config file. diff --git a/docs_new/docs/sglang-diffusion/api/openai-api.mdx b/docs_new/docs/sglang-diffusion/api/openai-api.mdx deleted file mode 100644 index 15642149c..000000000 --- a/docs_new/docs/sglang-diffusion/api/openai-api.mdx +++ /dev/null @@ -1,421 +0,0 @@ ---- -title: OpenAI API -sidebarTitle: OpenAI API -description: Image and video generation endpoints with LoRA adapter management. ---- - -The SGLang Diffusion HTTP server implements an OpenAI-compatible API for image and video generation, as well as dynamic LoRA adapter management. - -## Prerequisites - -- Python 3.11+ if you plan to use the OpenAI Python SDK. -- A running SGLang Diffusion server (see the [CLI reference](./cli) for launch instructions). - -## Start the server - -```bash -SERVER_ARGS=( - --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers - --text-encoder-cpu-offload - --pin-cpu-memory - --num-gpus 4 - --ulysses-degree=2 - --ring-degree=2 - --port 30010 -) - -sglang serve "${SERVER_ARGS[@]}" -``` - -- `--model-path` -- path to the model or HuggingFace model ID -- `--port` -- HTTP port to listen on (default: `30000`) - -### Get model information - -**Endpoint:** `GET /models` - -Returns model path, task type, pipeline configuration, and precision settings. - - -```bash curl -curl -sS -X GET "http://localhost:30010/models" -``` - - -**Response:** - -```json -{ - "model_path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", - "task_type": "T2V", - "pipeline_name": "wan_pipeline", - "pipeline_class": "WanPipeline", - "num_gpus": 4, - "dit_precision": "bf16", - "vae_precision": "fp16" -} -``` - ---- - -## Image generation - -The server implements an OpenAI-compatible Images API under the `/v1/images` namespace. - -### Create an image - -**Endpoint:** `POST /v1/images/generations` - - -```python Python -import base64 -from openai import OpenAI - -client = OpenAI(api_key="sk-proj-1234567890", base_url="http://localhost:30010/v1") - -img = client.images.generate( - prompt="A calico cat playing a piano on stage", - size="1024x1024", - n=1, - response_format="b64_json", -) - -image_bytes = base64.b64decode(img.data[0].b64_json) -with open("output.png", "wb") as f: - f.write(image_bytes) -``` - -```bash curl -curl -sS -X POST "http://localhost:30010/v1/images/generations" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer sk-proj-1234567890" \ - -d '{ - "prompt": "A calico cat playing a piano on stage", - "size": "1024x1024", - "n": 1, - "response_format": "b64_json" - }' -``` - - - -If `response_format=url` is used and cloud storage is not configured, the API returns a relative URL like `/v1/images//content`. - - -### Edit an image - -**Endpoint:** `POST /v1/images/edits` - -Accepts a multipart form upload with input images and a text prompt. Returns either a base64-encoded image or a URL. - - - - ```bash - curl -sS -X POST "http://localhost:30010/v1/images/edits" \ - -H "Authorization: Bearer sk-proj-1234567890" \ - -F "image=@local_input_image.png" \ - -F "url=image_url.jpg" \ - -F "prompt=A calico cat playing a piano on stage" \ - -F "size=1024x1024" \ - -F "response_format=b64_json" - ``` - - - ```bash - curl -sS -X POST "http://localhost:30010/v1/images/edits" \ - -H "Authorization: Bearer sk-proj-1234567890" \ - -F "image=@local_input_image.png" \ - -F "url=image_url.jpg" \ - -F "prompt=A calico cat playing a piano on stage" \ - -F "size=1024x1024" \ - -F "response_format=url" - ``` - - - -### Download image content - -When `response_format=url` is used, the API returns a relative URL like `/v1/images//content`. - -**Endpoint:** `GET /v1/images/{image_id}/content` - -```bash -curl -sS -L "http://localhost:30010/v1/images//content" \ - -H "Authorization: Bearer sk-proj-1234567890" \ - -o output.png -``` - ---- - -## Video generation - -The server implements a subset of the OpenAI Videos API under the `/v1/videos` namespace. - -### Create a video - -**Endpoint:** `POST /v1/videos` - - -```python Python -from openai import OpenAI - -client = OpenAI(api_key="sk-proj-1234567890", base_url="http://localhost:30010/v1") - -video = client.videos.create( - prompt="A calico cat playing a piano on stage", - size="1280x720" -) -print(f"Video ID: {video.id}, Status: {video.status}") -``` - -```bash curl -curl -sS -X POST "http://localhost:30010/v1/videos" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer sk-proj-1234567890" \ - -d '{ - "prompt": "A calico cat playing a piano on stage", - "size": "1280x720" - }' -``` - - -### List videos - -**Endpoint:** `GET /v1/videos` - - -```python Python -videos = client.videos.list() -for item in videos.data: - print(item.id, item.status) -``` - -```bash curl -curl -sS -X GET "http://localhost:30010/v1/videos" \ - -H "Authorization: Bearer sk-proj-1234567890" -``` - - -### Download video content - -**Endpoint:** `GET /v1/videos/{video_id}/content` - - -```python Python -import time - -# Poll for completion -while True: - page = client.videos.list() - item = next((v for v in page.data if v.id == video_id), None) - if item and item.status == "completed": - break - time.sleep(5) - -# Download content -resp = client.videos.download_content(video_id=video_id) -with open("output.mp4", "wb") as f: - f.write(resp.read()) -``` - -```bash curl -curl -sS -L "http://localhost:30010/v1/videos//content" \ - -H "Authorization: Bearer sk-proj-1234567890" \ - -o output.mp4 -``` - - ---- - -## LoRA management - -The server supports dynamic loading, merging, and unmerging of LoRA adapters. - - -- **Mutual exclusion:** Only one LoRA can be merged (active) at a time. -- **Switching:** To switch LoRAs, you must first unmerge the current one, then set the new one. -- **Caching:** The server caches loaded LoRA weights in memory. Switching back to a previously loaded LoRA (same path) has negligible cost. - - -### Set LoRA adapter - -Loads one or more LoRA adapters and merges their weights into the model. Supports both single LoRA (backward compatible) and multiple LoRA adapters. - -**Endpoint:** `POST /v1/set_lora` - -**Parameters:** - -| Parameter | Type | Description | -|:--|:--|:--| -| `lora_nickname` | string or list | A unique identifier for the LoRA adapter(s). Required | -| `lora_path` | string or list | Path to `.safetensors` file(s) or HuggingFace repo ID(s). Required for first load; optional when re-activating a cached nickname | -| `target` | string or list | Which transformer(s) to apply the LoRA to: `"all"` (default), `"transformer"`, `"transformer_2"`, `"critic"` | -| `strength` | float or list | LoRA strength for merge (default: `1.0`). Values < 1.0 reduce the effect, > 1.0 amplify it | - - - - ```bash - curl -X POST http://localhost:30010/v1/set_lora \ - -H "Content-Type: application/json" \ - -d '{ - "lora_nickname": "lora_name", - "lora_path": "/path/to/lora.safetensors", - "target": "all", - "strength": 0.8 - }' - ``` - - - ```bash - curl -X POST http://localhost:30010/v1/set_lora \ - -H "Content-Type: application/json" \ - -d '{ - "lora_nickname": ["lora_1", "lora_2"], - "lora_path": ["/path/to/lora1.safetensors", "/path/to/lora2.safetensors"], - "target": ["transformer", "transformer_2"], - "strength": [0.8, 1.0] - }' - ``` - - - ```bash - curl -X POST http://localhost:30010/v1/set_lora \ - -H "Content-Type: application/json" \ - -d '{ - "lora_nickname": ["style_lora", "character_lora"], - "lora_path": ["/path/to/style.safetensors", "/path/to/character.safetensors"], - "target": "all", - "strength": [0.7, 0.9] - }' - ``` - - - - -When using multiple LoRAs: -- All list parameters (`lora_nickname`, `lora_path`, `target`, `strength`) must have the same length. -- If `target` or `strength` is a single value, it will be applied to all LoRAs. -- Multiple LoRAs applied to the same target will be merged in order. - - -### Merge LoRA weights - -Manually merges the currently set LoRA weights into the base model. - -**Endpoint:** `POST /v1/merge_lora_weights` - -| Parameter | Type | Description | -|:--|:--|:--| -| `target` | string | Which transformer(s) to merge: `"all"` (default), `"transformer"`, `"transformer_2"`, `"critic"` | -| `strength` | float | LoRA strength for merge (default: `1.0`) | - -```bash -curl -X POST http://localhost:30010/v1/merge_lora_weights \ - -H "Content-Type: application/json" \ - -d '{"strength": 0.8}' -``` - - -`set_lora` automatically performs a merge, so this endpoint is typically only needed if you have manually unmerged but want to re-apply the same LoRA without calling `set_lora` again. - - -### Unmerge LoRA weights - -Unmerges the currently active LoRA weights from the base model, restoring it to its original state. Call this before setting a different LoRA. - -**Endpoint:** `POST /v1/unmerge_lora_weights` - -```bash -curl -X POST http://localhost:30010/v1/unmerge_lora_weights \ - -H "Content-Type: application/json" -``` - -### List LoRA adapters - -Returns loaded LoRA adapters and current application status per module. - -**Endpoint:** `GET /v1/list_loras` - -```bash -curl -sS -X GET "http://localhost:30010/v1/list_loras" -``` - -**Response:** - -```json -{ - "loaded_adapters": [ - { "nickname": "lora_a", "path": "/weights/lora_a.safetensors" }, - { "nickname": "lora_b", "path": "/weights/lora_b.safetensors" } - ], - "active": { - "transformer": [ - { - "nickname": "lora2", - "path": "tarn59/pixel_art_style_lora_z_image_turbo", - "merged": true, - "strength": 1.0 - } - ] - } -} -``` - -### Example: switching LoRAs - -1. **Set LoRA A** - -```bash -curl -X POST http://localhost:30010/v1/set_lora \ - -d '{"lora_nickname": "lora_a", "lora_path": "path/to/A"}' -``` - -2. **Generate with LoRA A** - -Run your image or video generation requests. - -3. **Unmerge LoRA A** - -```bash -curl -X POST http://localhost:30010/v1/unmerge_lora_weights -``` - -4. **Set LoRA B** - -```bash -curl -X POST http://localhost:30010/v1/set_lora \ - -d '{"lora_nickname": "lora_b", "lora_path": "path/to/B"}' -``` - -5. **Generate with LoRA B** - -Run your image or video generation requests with the new adapter. - ---- - -## Output quality - -Control output quality and compression for both image and video generation through the `output-quality` and `output-compression` parameters. - -### Parameters - -| Parameter | Type | Description | -|:--|:--|:--| -| `output-quality` | string | Preset quality level. Default: `"default"` | -| `output-compression` | integer | Direct compression level override (0-100). When provided, takes precedence over `output-quality` | - -**Quality presets:** - -| Preset | Compression value | -|:--|:--| -| `"maximum"` | 100 | -| `"high"` | 90 | -| `"medium"` | 55 | -| `"low"` | 35 | -| `"default"` | Auto (50 for video, 75 for image) | - - -- When both `output-quality` and `output-compression` are provided, `output-compression` takes precedence. -- Quality settings apply to JPEG and video formats. PNG uses lossless compression and ignores these settings. -- Lower compression values (or `"low"` quality preset) produce smaller files but may show visible artifacts. - diff --git a/docs_new/docs/sglang-diffusion/api/openai_api.mdx b/docs_new/docs/sglang-diffusion/api/openai_api.mdx new file mode 100644 index 000000000..95874f991 --- /dev/null +++ b/docs_new/docs/sglang-diffusion/api/openai_api.mdx @@ -0,0 +1,450 @@ +--- +title: OpenAI API +sidebarTitle: OpenAI API +description: Image and video generation endpoints with LoRA adapter management. +--- +The SGLang diffusion HTTP server implements an OpenAI-compatible API for image and video generation, as well as LoRA adapter management. + +## Prerequisites + +- Python 3.11+ if you plan to use the OpenAI Python SDK. + +## Serve + +Launch the server using the `sglang serve` command. + +### Start the server + +```bash +SERVER_ARGS=( + --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers + --text-encoder-cpu-offload + --pin-cpu-memory + --num-gpus 4 + --ulysses-degree=2 + --ring-degree=2 + --port 30010 +) + +sglang serve "${SERVER_ARGS[@]}" +``` + +- **--model-path**: Path to the model or model ID. +- **--port**: HTTP port to listen on (default: `30000`). + +**Get Model Information** + +**Endpoint:** `GET /models` + +Returns information about the model served by this server, including model path, task type, pipeline configuration, and precision settings. + +**Curl Example:** + +```bash curl +curl -sS -X GET "http://localhost:30010/models" +``` + +**Response Example:** + +```json +{ + "model_path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", + "task_type": "T2V", + "pipeline_name": "wan_pipeline", + "pipeline_class": "WanPipeline", + "num_gpus": 4, + "dit_precision": "bf16", + "vae_precision": "fp16" +} +``` + +--- + +## Endpoints + +### Image Generation + +The server implements an OpenAI-compatible Images API under the `/v1/images` namespace. + +**Create an image** + +**Endpoint:** `POST /v1/images/generations` + +**Python Example (b64_json response):** + +```python Python +import base64 +from openai import OpenAI + +client = OpenAI(api_key="sk-proj-1234567890", base_url="http://localhost:30010/v1") + +img = client.images.generate( + prompt="A calico cat playing a piano on stage", + size="1024x1024", + n=1, + response_format="b64_json", +) + +image_bytes = base64.b64decode(img.data[0].b64_json) +with open("output.png", "wb") as f: + f.write(image_bytes) +``` + +**Curl Example:** + +```bash curl +curl -sS -X POST "http://localhost:30010/v1/images/generations" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-proj-1234567890" \ + -d '{ + "prompt": "A calico cat playing a piano on stage", + "size": "1024x1024", + "n": 1, + "response_format": "b64_json" + }' +``` + +> **Note** +> If `response_format=url` is used and cloud storage is not configured, the API returns +> a relative URL like `/v1/images//content`. + +**Edit an image** + +**Endpoint:** `POST /v1/images/edits` + +This endpoint accepts a multipart form upload with input images and a text prompt. The server can return either a base64-encoded image or a URL to download the image. + +**Curl Example (b64_json response):** + +```bash Command +curl -sS -X POST "http://localhost:30010/v1/images/edits" \ + -H "Authorization: Bearer sk-proj-1234567890" \ + -F "image=@local_input_image.png" \ + -F "url=image_url.jpg" \ + -F "prompt=A calico cat playing a piano on stage" \ + -F "size=1024x1024" \ + -F "response_format=b64_json" +``` + +**Curl Example (URL response):** + +```bash Command +curl -sS -X POST "http://localhost:30010/v1/images/edits" \ + -H "Authorization: Bearer sk-proj-1234567890" \ + -F "image=@local_input_image.png" \ + -F "url=image_url.jpg" \ + -F "prompt=A calico cat playing a piano on stage" \ + -F "size=1024x1024" \ + -F "response_format=url" +``` + +**Download image content** + +When `response_format=url` is used with `POST /v1/images/generations` or `POST /v1/images/edits`, +the API returns a relative URL like `/v1/images//content`. + +**Endpoint:** `GET /v1/images/{image_id}/content` + +**Curl Example:** + +```bash +curl -sS -L "http://localhost:30010/v1/images//content" \ + -H "Authorization: Bearer sk-proj-1234567890" \ + -o output.png +``` + +### Video Generation + +The server implements a subset of the OpenAI Videos API under the `/v1/videos` namespace. + +**Create a video (text-to-video)** + +**Endpoint:** `POST /v1/videos` + +**Python Example:** + +```python Python +from openai import OpenAI + +client = OpenAI(api_key="sk-proj-1234567890", base_url="http://localhost:30010/v1") + +video = client.videos.create( + prompt="A calico cat playing a piano on stage", + size="1280x720" +) +print(f"Video ID: {video.id}, Status: {video.status}") +``` + +**Curl Example:** + +```bash curl +curl -sS -X POST "http://localhost:30010/v1/videos" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-proj-1234567890" \ + -d '{ + "prompt": "A calico cat playing a piano on stage", + "size": "1280x720" + }' +``` + +**Create a video (image-to-video)** + +For I2V or TI2V models (e.g., Wan2.1 I2V, LTX-2.3 two-stage), pass an input image via multipart form upload or a reference URL. + +**Curl Example (multipart form upload):** + +```bash Command +curl -sS -X POST "http://localhost:30010/v1/videos" \ + -H "Authorization: Bearer sk-proj-1234567890" \ + -F "prompt=A cat playing a piano" \ + -F "input_reference=@input_image.png" \ + -F "size=1280x720" +``` + +**Curl Example (reference URL):** + +```bash Command +curl -sS -X POST "http://localhost:30010/v1/videos" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-proj-1234567890" \ + -d '{ + "prompt": "A cat playing a piano", + "reference_url": "https://example.com/input_image.png", + "size": "1280x720" + }' +``` + +**List videos** + +**Endpoint:** `GET /v1/videos` + +**Python Example:** + +```python Python +videos = client.videos.list() +for item in videos.data: + print(item.id, item.status) +``` + +**Curl Example:** + +```bash curl +curl -sS -X GET "http://localhost:30010/v1/videos" \ + -H "Authorization: Bearer sk-proj-1234567890" +``` + +**Download video content** + +**Endpoint:** `GET /v1/videos/{video_id}/content` + +**Python Example:** + +```python Python +import time + +# Poll for completion +while True: + page = client.videos.list() + item = next((v for v in page.data if v.id == video_id), None) + if item and item.status == "completed": + break + time.sleep(5) + +# Download content +resp = client.videos.download_content(video_id=video_id) +with open("output.mp4", "wb") as f: + f.write(resp.read()) +``` + +**Curl Example:** + +```bash curl +curl -sS -L "http://localhost:30010/v1/videos//content" \ + -H "Authorization: Bearer sk-proj-1234567890" \ + -o output.mp4 +``` + +--- + +### LoRA Management + +The server supports dynamic loading, merging, and unmerging of LoRA adapters. + +**Important Notes:** +- Mutual Exclusion: Only one LoRA can be *merged* (active) at a time +- Switching: To switch LoRAs, you must first `unmerge` the current one, then `set` the new one +- Caching: The server caches loaded LoRA weights in memory. Switching back to a previously loaded LoRA (same path) has little cost + +**Set LoRA Adapter** + +Loads one or more LoRA adapters and merges their weights into the model. Supports both single LoRA (backward compatible) and multiple LoRA adapters. + +**Endpoint:** `POST /v1/set_lora` + +**Parameters:** +- `lora_nickname` (string or list of strings, required): A unique identifier for the LoRA adapter(s). Can be a single string or a list of strings for multiple LoRAs +- `lora_path` (string or list of strings/None, optional): Path to the `.safetensors` file(s) or Hugging Face repo ID(s). Required for the first load; optional if re-activating a cached nickname. If a list, must match the length of `lora_nickname` +- `target` (string or list of strings, optional): Which transformer(s) to apply the LoRA to. If a list, must match the length of `lora_nickname`. Valid values: + - `"all"` (default): Apply to all transformers + - `"transformer"`: Apply only to the primary transformer (high noise for Wan2.2) + - `"transformer_2"`: Apply only to transformer_2 (low noise for Wan2.2) + - `"critic"`: Apply only to the critic model +- `strength` (float or list of floats, optional): LoRA strength for merge, default 1.0. If a list, must match the length of `lora_nickname`. Values < 1.0 reduce the effect, values > 1.0 amplify the effect + +**Single LoRA Example:** + +```bash Command +curl -X POST http://localhost:30010/v1/set_lora \ + -H "Content-Type: application/json" \ + -d '{ + "lora_nickname": "lora_name", + "lora_path": "/path/to/lora.safetensors", + "target": "all", + "strength": 0.8 + }' +``` + +**Multiple LoRA Example:** + +```bash Command +curl -X POST http://localhost:30010/v1/set_lora \ + -H "Content-Type: application/json" \ + -d '{ + "lora_nickname": ["lora_1", "lora_2"], + "lora_path": ["/path/to/lora1.safetensors", "/path/to/lora2.safetensors"], + "target": ["transformer", "transformer_2"], + "strength": [0.8, 1.0] + }' +``` + +**Multiple LoRA with Same Target:** + +```bash Command +curl -X POST http://localhost:30010/v1/set_lora \ + -H "Content-Type: application/json" \ + -d '{ + "lora_nickname": ["style_lora", "character_lora"], + "lora_path": ["/path/to/style.safetensors", "/path/to/character.safetensors"], + "target": "all", + "strength": [0.7, 0.9] + }' +``` + +> [!NOTE] +> When using multiple LoRAs: +> - All list parameters (`lora_nickname`, `lora_path`, `target`, `strength`) must have the same length +> - If `target` or `strength` is a single value, it will be applied to all LoRAs +> - Multiple LoRAs applied to the same target will be merged in order + + +**Merge LoRA Weights** + +Manually merges the currently set LoRA weights into the base model. + +> [!NOTE] +> `set_lora` automatically performs a merge, so this is typically only needed if you have manually unmerged but want to re-apply the same LoRA without calling `set_lora` again.* + +**Endpoint:** `POST /v1/merge_lora_weights` + +**Parameters:** +- `target` (string, optional): Which transformer(s) to merge. One of "all" (default), "transformer", "transformer_2", "critic" +- `strength` (float, optional): LoRA strength for merge, default 1.0. Values < 1.0 reduce the effect, values > 1.0 amplify the effect + +**Curl Example:** + +```bash +curl -X POST http://localhost:30010/v1/merge_lora_weights \ + -H "Content-Type: application/json" \ + -d '{"strength": 0.8}' +``` + + +**Unmerge LoRA Weights** + +Unmerges the currently active LoRA weights from the base model, restoring it to its original state. This **must** be called before setting a different LoRA. + +**Endpoint:** `POST /v1/unmerge_lora_weights` + +**Curl Example:** + +```bash +curl -X POST http://localhost:30010/v1/unmerge_lora_weights \ + -H "Content-Type: application/json" +``` + +**List LoRA Adapters** + +Returns loaded LoRA adapters and current application status per module. + +**Endpoint:** `GET /v1/list_loras` + +**Curl Example:** + +```bash +curl -sS -X GET "http://localhost:30010/v1/list_loras" +``` + +**Response Example:** + +```json +{ + "loaded_adapters": [ + { "nickname": "lora_a", "path": "/weights/lora_a.safetensors" }, + { "nickname": "lora_b", "path": "/weights/lora_b.safetensors" } + ], + "active": { + "transformer": [ + { + "nickname": "lora2", + "path": "tarn59/pixel_art_style_lora_z_image_turbo", + "merged": true, + "strength": 1.0 + } + ] + } +} +``` + +Notes: +- If LoRA is not enabled for the current pipeline, the server will return an error. +- `num_lora_layers_with_weights` counts only layers that have LoRA weights applied for the active adapter. + +### Example: Switching LoRAs + +1. Set LoRA A: + ```bash Command + curl -X POST http://localhost:30010/v1/set_lora -d '{"lora_nickname": "lora_a", "lora_path": "path/to/A"}' + ``` +2. Generate with LoRA A... +3. Unmerge LoRA A: + ```bash Command + curl -X POST http://localhost:30010/v1/unmerge_lora_weights + ``` +4. Set LoRA B: + ```bash Command + curl -X POST http://localhost:30010/v1/set_lora -d '{"lora_nickname": "lora_b", "lora_path": "path/to/B"}' + ``` +5. Generate with LoRA B... + +### Adjust Output Quality + +The server supports adjusting output quality and compression levels for both image and video generation through the `output-quality` and `output-compression` parameters. + +#### Parameters + +- **`output-quality`** (string, optional): Preset quality level that automatically sets compression. **Default is `"default"`**. Valid values: + - `"maximum"`: Highest quality (100) + - `"high"`: High quality (90) + - `"medium"`: Medium quality (55) + - `"low"`: Lower quality (35) + - `"default"`: Auto-adjust based on media type (50 for video, 75 for image) + +- **`output-compression`** (integer, optional): Direct compression level override (0-100). **Default is `None`**. When provided (not `None`), takes precedence over `output-quality`. + - `0`: Lowest quality, smallest file size + - `100`: Highest quality, largest file size + +#### Notes + +- **Precedence**: When both `output-quality` and `output-compression` are provided, `output-compression` takes precedence +- **Format Support**: Quality settings apply to JPEG, and video formats. PNG uses lossless compression and ignores these settings +- **File Size vs Quality**: Lower compression values (or "low" quality preset) produce smaller files but may show visible artifacts diff --git a/docs_new/docs/sglang-diffusion/api/post_processing.mdx b/docs_new/docs/sglang-diffusion/api/post_processing.mdx new file mode 100644 index 000000000..132363a5a --- /dev/null +++ b/docs_new/docs/sglang-diffusion/api/post_processing.mdx @@ -0,0 +1,237 @@ +--- +title: "Post-Processing" +metatags: + description: "Use SGLang Diffusion post-processing for frame interpolation and spatial upscaling after generation." +--- + +SGLang diffusion supports optional post-processing steps that run after +generation to improve temporal smoothness (frame interpolation) or spatial +resolution (upscaling). These steps are independent of the diffusion model and +can be combined in a single run. + +When both are enabled, **frame interpolation runs first** (increasing the frame +count), then **upscaling runs on every frame** (increasing the spatial +resolution). + +--- + +## Frame Interpolation (video only) + +Frame interpolation synthesizes new frames between each pair of consecutive +generated frames, producing smoother motion without re-running the diffusion +model. + +The `--frame-interpolation-exp` flag controls how many rounds of interpolation +to apply: each round inserts one new frame into every gap between adjacent +frames, so the output frame count follows the formula: + +> **(N − 1) × 2^exp + 1** +> +> e.g. 5 original frames with `exp=1` → 4 gaps × 1 new frame + 5 originals = **9** frames; +> with `exp=2` → **17** frames. + +### CLI Arguments + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ArgumentDescription
--enable-frame-interpolationEnable frame interpolation. Model weights are downloaded automatically on first use.
--frame-interpolation-exp {EXP}Interpolation exponent — 1 = 2× temporal resolution, 2 = 4×, etc. (default: 1)
--frame-interpolation-scale {SCALE}RIFE inference scale; use 0.5 for high-resolution inputs to save memory (default: 1.0)
--frame-interpolation-model-path {PATH}Local directory or HuggingFace repo ID containing RIFE flownet.pkl weights (default: elfgum/RIFE-4.22.lite, downloaded automatically)
+ +### Supported Models + +Frame interpolation uses the [RIFE](https://github.com/hzwer/Practical-RIFE) +(Real-Time Intermediate Flow Estimation) architecture. Only **RIFE 4.22.lite** +(`IFNet` with 4-scale `IFBlock` backbone) is supported. The network topology is +hard-coded, so custom weights provided via `--frame-interpolation-model-path` +must be a `flownet.pkl` checkpoint that is compatible with this architecture. + +Other RIFE versions (e.g., older `v4.x` variants with different block counts) +or entirely different frame interpolation methods (FILM, AMT, etc.) are **not +supported**. + + + + + + + + + + + + + + + + + + + + + +
WeightHuggingFace RepoDescription
RIFE 4.22.lite *(default)*elfgum/RIFE-4.22.liteLightweight model, downloaded automatically on first use
+ +### Example + +Generate a 5-frame video and interpolate to 9 frames ((5 − 1) × 2¹ + 1 = 9): + +```bash +sglang generate \ + --model-path Wan-AI/Wan2.2-T2V-A14B-Diffusers \ + --prompt "A dog running through a park" \ + --num-frames 5 \ + --enable-frame-interpolation \ + --frame-interpolation-exp 1 \ + --save-output +``` + +--- + +## Upscaling (image and video) + +Upscaling increases the spatial resolution of generated images or video frames +using [Real-ESRGAN](https://github.com/xinntao/Real-ESRGAN). The model weights +are downloaded automatically on first use and cached for subsequent runs. + +### CLI Arguments + + + + + + + + + + + + + + + + + + + + + + + + + + +
ArgumentDescription
--enable-upscalingEnable post-generation upscaling using Real-ESRGAN.
--upscaling-scale {SCALE}Desired upscaling factor (default: 4). The 4× model is used internally; if a different scale is requested, a bicubic resize is applied after the network output.
--upscaling-model-path {PATH}Local .pth file, HuggingFace repo ID, or repo_id:filename for Real-ESRGAN weights (default: ai-forever/Real-ESRGAN with RealESRGAN_x4.pth, downloaded automatically). Use the repo_id:filename format to specify a custom weight file from a HuggingFace repo (e.g. my-org/my-esrgan:weights.pth).
+ +### Supported Models + +Upscaling supports two Real-ESRGAN network architectures. The correct +architecture is **auto-detected** from the checkpoint keys, so you only need to +point `--upscaling-model-path` at a valid `.pth` file: + + + + + + + + + + + + + + + + + + + + + + + + + + +
ArchitectureExample WeightsDescription
RRDBNetRealESRGAN_x4plus.pthHeavier model with higher quality; best for photos
SRVGGNetCompactRealESRGAN_x4.pth *(default)*, realesr-animevideov3.pth, realesr-general-x4v3.pthLightweight model; faster inference, good for video
+ +The default weight file is +[`ai-forever/Real-ESRGAN`](https://huggingface.co/ai-forever/Real-ESRGAN) with +`RealESRGAN_x4.pth` (SRVGGNetCompact, 4× native scale). + +Other super-resolution models (e.g., SwinIR, HAT, BSRGAN) are **not supported** +— only Real-ESRGAN checkpoints using the two architectures above are +compatible. + +### Examples + +Generate a 1024×1024 image and upscale to 4096×4096: + +```bash +sglang generate \ + --model-path black-forest-labs/FLUX.2-dev \ + --prompt "A cat sitting on a windowsill" \ + --output-size 1024x1024 \ + --enable-upscaling \ + --save-output +``` + +Generate a video and upscale each frame by 4×: + +```bash +sglang generate \ + --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ + --prompt "A curious raccoon" \ + --enable-upscaling \ + --upscaling-scale 4 \ + --save-output +``` + +--- + +## Combining Frame Interpolation and Upscaling + +Frame interpolation and upscaling can be combined in a single run. +Interpolation is applied first (increasing the frame count), then upscaling is +applied to every frame (increasing the spatial resolution). + +Example — generate 5 frames, interpolate to 9 frames, and upscale each frame +by 4×: + +```bash +sglang generate \ + --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ + --prompt "A curious raccoon" \ + --num-frames 5 \ + --enable-frame-interpolation \ + --frame-interpolation-exp 1 \ + --enable-upscaling \ + --upscaling-scale 4 \ + --save-output +``` diff --git a/docs_new/docs/sglang-diffusion/attention-backends.mdx b/docs_new/docs/sglang-diffusion/attention_backends.mdx similarity index 74% rename from docs_new/docs/sglang-diffusion/attention-backends.mdx rename to docs_new/docs/sglang-diffusion/attention_backends.mdx index d8a6c21c1..4f474314b 100644 --- a/docs_new/docs/sglang-diffusion/attention-backends.mdx +++ b/docs_new/docs/sglang-diffusion/attention_backends.mdx @@ -2,7 +2,6 @@ title: "Attention Backends" description: "Select and configure attention backends for SGLang diffusion pipelines." --- - This document describes the attention backends available in sglang diffusion (`sglang.multimodal_gen`) and how to select them. ## Overview @@ -16,8 +15,10 @@ When using the diffusers backend, `--attention-backend` is passed through to dif - **CUDA**: prefers FlashAttention (FA3/FA4) when supported; otherwise falls back to PyTorch SDPA. - **ROCm**: uses FlashAttention when available; otherwise falls back to PyTorch SDPA. +- **Intel XPU**: uses XPU Flash Attention backend (fp16/bf16, head sizes 64/96/128/192/256); otherwise falls back to PyTorch SDPA. +- **MUSA**: uses FlashAttention when available; otherwise falls back to PyTorch SDPA. - **MPS**: always uses PyTorch SDPA. -- **NPU**: always uses PyTorch SDPA. +- **NPU**: for ring attention uses FA otherwise uses PyTorch SDPA. ## Backend options @@ -40,22 +41,22 @@ For SGLang-native pipelines, the CLI accepts the lowercase names of `AttentionBa `fa` / `fa3` / `fa4` `FA` - FlashAttention. `fa3/fa4` are normalized to `fa` during argument parsing (`ServerArgs.__post_init__`). + FlashAttention. fa3/fa4 are normalized to fa during argument parsing (ServerArgs.__post_init__). `torch_sdpa` `TORCH_SDPA` - PyTorch `scaled_dot_product_attention`. + PyTorch scaled_dot_product_attention. `sliding_tile_attn` `SLIDING_TILE_ATTN` - Sliding Tile Attention (STA). Requires `st_attn`. Configure via `--attention-backend-config`. + Sliding Tile Attention (STA). Requires st_attn. Configure via --attention-backend-config. `sage_attn` `SAGE_ATTN` - Requires `sageattention`. Upstream SageAttention CUDA extensions target SM80/SM86/SM89/SM90/SM120 (compute capability 8.0/8.6/8.9/9.0/12.0); see upstream `setup.py`: https://github.com/thu-ml/SageAttention/blob/main/setup.py. + Requires sageattention. Upstream SageAttention CUDA extensions target SM80/SM86/SM89/SM90/SM120 (compute capability 8.0/8.6/8.9/9.0/12.0); see upstream setup.py: https://github.com/thu-ml/SageAttention/blob/main/setup.py. `sage_attn_3` @@ -65,24 +66,39 @@ For SGLang-native pipelines, the CLI accepts the lowercase names of `AttentionBa `video_sparse_attn` `VIDEO_SPARSE_ATTN` - Requires `vsa`. Configure `sparsity` via `--attention-backend-config`. + Requires vsa. Configure sparsity via --attention-backend-config. `vmoba_attn` `VMOBA_ATTN` - Requires `kernel.attn.vmoba_attn.vmoba`. Configure via `--attention-backend-config`. + Requires kernel.attn.vmoba_attn.vmoba. Configure via --attention-backend-config. `aiter` `AITER` - Requires `aiter`. + Requires aiter. + aiter_sage + AITER_SAGE + Requires aiter. + + + sla_attn + SLA_ATTN + Sparse Linear Attention. Requires SpargeAttn. Install with pip install git+https://github.com/thu-ml/SpargeAttn.git --no-build-isolation. + + + sage_sla_attn + SAGE_SLA_ATTN + SageAttention + Sparse Linear Attention. Requires SpargeAttn (same install as SLA). + + `sparse_video_gen_2_attn` `SPARSE_VIDEO_GEN_2_ATTN` - Requires `svg`. See installation instructions at https://github.com/svg-project/Sparse-VideoGen. + Requires svg. See installation instructions at https://github.com/svg-project/Sparse-VideoGen. - + ## Selection priority @@ -97,7 +113,7 @@ The selection order in `runtime/layers/attention/selector.py` is: Some backends require additional configuration. You can pass these parameters via `--attention-backend-config`. This argument accepts: - A path to a JSON or YAML configuration file. -- A JSON string (e.g., `'{"sparsity": 0.5}'`). +- A JSON string (e.g., `'{"sparsity": 0.5}'`). - Key-value pairs (e.g., `"sparsity=0.5,enable_x=true"`). ### Supported Configuration Parameters @@ -289,8 +305,10 @@ Some backends require additional configuration. You can pass these parameters vi Backend CUDA ROCm + XPU + MUSA MPS - NPU + NPU Notes @@ -299,9 +317,11 @@ Some backends require additional configuration. You can pass these parameters vi `fa` Yes Yes - No - No - CUDA requires SM80+ and fp16/bf16. FlashAttention is only used when the required runtime is installed; otherwise it falls back to `torch_sdpa`. + ✅ + ✅ + ❌ + ✅ + CUDA requires SM80+ and fp16/bf16. XPU uses its own flash attention backend. FlashAttention is only used when the required runtime is installed; otherwise it falls back to torch_sdpa. No extra installations are required for NPU `torch_sdpa` @@ -309,6 +329,8 @@ Some backends require additional configuration. You can pass these parameters vi Yes Yes Yes + ✅ + ✅ Most compatible option across platforms. @@ -317,7 +339,9 @@ Some backends require additional configuration. You can pass these parameters vi No No No - CUDA-only. Requires `st_attn`. Configure via `--attention-backend-config`. + ❌ + ❌ + CUDA-only. Requires st_attn. Configure via --attention-backend-config. `sage_attn` @@ -325,6 +349,8 @@ Some backends require additional configuration. You can pass these parameters vi No No No + ❌ + ❌ CUDA-only (optional dependency). @@ -333,6 +359,8 @@ Some backends require additional configuration. You can pass these parameters vi No No No + ❌ + ❌ CUDA-only (optional dependency). @@ -341,33 +369,71 @@ Some backends require additional configuration. You can pass these parameters vi No No No - CUDA-only. Requires `vsa`. Configure `sparsity` via `--attention-backend-config`. + ❌ + ❌ + CUDA-only. Requires vsa. Configure sparsity via --attention-backend-config. - `vmoba_attn` + sla_attn Yes No No No - CUDA-only. Requires `kernel.attn.vmoba_attn.vmoba`. Configure via `--attention-backend-config`. + ❌ + ❌ + CUDA-only. Requires SpargeAttn. - `aiter` + sage_sla_attn Yes No No No - Requires `aiter`. + ❌ + ❌ + CUDA-only. Requires SpargeAttn. + vmoba_attn + Yes + No + No + No + ❌ + ❌ + CUDA-only. Requires kernel.attn.vmoba_attn.vmoba. Configure via --attention-backend-config. + + + aiter + ❌ + ✅ + No + No + ❌ + ❌ + Requires aiter. + + + aiter_sage + ❌ + ✅ + No + No + ❌ + ❌ + Requires aiter. + + `sparse_video_gen_2_attn` Yes No No No - CUDA-only. Requires `svg`. + ❌ + ❌ + CUDA-only. Requires svg. - + ## Usage diff --git a/docs_new/docs/sglang-diffusion/cache-dit.mdx b/docs_new/docs/sglang-diffusion/cache_dit.mdx similarity index 78% rename from docs_new/docs/sglang-diffusion/cache-dit.mdx rename to docs_new/docs/sglang-diffusion/cache_dit.mdx index 59bc388d3..2c6c88bc0 100644 --- a/docs_new/docs/sglang-diffusion/cache-dit.mdx +++ b/docs_new/docs/sglang-diffusion/cache_dit.mdx @@ -2,7 +2,6 @@ title: "Cache-DiT Acceleration" description: "Configure Cache-DiT acceleration for diffusion inference." --- - SGLang integrates [Cache-DiT](https://github.com/vipshop/cache-dit), a caching acceleration engine for Diffusion Transformers (DiT), to achieve up to **1.69x inference speedup** with minimal quality loss. ## Overview @@ -33,6 +32,8 @@ flow requires cache-dit >= 1.2.0 (`cache_dit.load_configs`). Define a `cache.yaml` file that contains: +- DBCache + TaylorSeer + ```yaml cache_config: max_warmup_steps: 8 @@ -56,18 +57,54 @@ sglang generate \ --prompt "A beautiful sunset over the mountains" ``` +- DBCache + TaylorSeer + SCM (Step Computation Mask) + +```yaml Config +cache_config: + max_warmup_steps: 8 + warmup_interval: 2 + max_cached_steps: -1 + max_continuous_cached_steps: 2 + Fn_compute_blocks: 1 + Bn_compute_blocks: 0 + residual_diff_threshold: 0.12 + enable_taylorseer: true + taylorseer_order: 1 + # Must set the num_inference_steps for SCM. The SCM will automatically + # generate the steps computation mask based on the num_inference_steps. + # Reference: https://cache-dit.readthedocs.io/en/latest/user_guide/CACHE_API/#scm-steps-computation-masking + num_inference_steps: 28 + steps_computation_mask: fast +``` + +- DBCache + TaylorSeer + SCM (Step Computation Mask) + Cache CFG + +```yaml Config +cache_config: + max_warmup_steps: 8 + warmup_interval: 2 + max_cached_steps: -1 + max_continuous_cached_steps: 2 + Fn_compute_blocks: 1 + Bn_compute_blocks: 0 + residual_diff_threshold: 0.12 + enable_taylorseer: true + taylorseer_order: 1 + num_inference_steps: 28 + steps_computation_mask: fast + enable_sperate_cfg: true # e.g, Qwen-Image, Wan, Chroma, Ovis-Image, etc. +``` + ### Distributed inference - 1D Parallelism Define a parallelism only config yaml `parallel.yaml` file that contains: -```yaml +```yaml Config parallelism_config: ulysses_size: auto - parallel_kwargs: - attention_backend: native - extra_parallel_modules: ["text_encoder", "vae"] + attention_backend: native ``` Then, apply the distributed inference acceleration config from yaml. `ulysses_size: auto` means that cache-dit will auto detect the `world_size` as the ulysses_size. Otherwise, you should manually set it as specific int number, e.g, 4. @@ -87,13 +124,11 @@ sglang generate \ You can also define a 2D parallelism config yaml `parallel_2d.yaml` file that contains: -```yaml +```yaml Config parallelism_config: ulysses_size: auto tp_size: 2 - parallel_kwargs: - attention_backend: native - extra_parallel_modules: ["text_encoder", "vae"] + attention_backend: native ``` Then, apply the 2D parallelism config from yaml. Here `tp_size: 2` means using tensor parallelism with size 2. The `ulysses_size: auto` means that cache-dit will auto detect the `world_size // tp_size` as the ulysses_size. @@ -101,22 +136,66 @@ Then, apply the 2D parallelism config from yaml. Here `tp_size: 2` means using t You can also define a 3D parallelism config yaml `parallel_3d.yaml` file that contains: -```yaml +```yaml Config parallelism_config: ulysses_size: 2 ring_size: 2 tp_size: 2 - parallel_kwargs: - attention_backend: native - extra_parallel_modules: ["text_encoder", "vae"] + attention_backend: native ``` Then, apply the 3D parallelism config from yaml. Here `ulysses_size: 2`, `ring_size: 2`, `tp_size: 2` means using ulysses parallelism with size 2, ring parallelism with size 2 and tensor parallelism with size 2. +- Ulysses Anything Attention + +To enable Ulysses Anything Attention, you can define a parallelism config yaml `parallel_uaa.yaml` file that contains: + +```yaml Config +parallelism_config: + ulysses_size: auto + attention_backend: native + ulysses_anything: true +``` + +- Ulysses FP8 Communication + +For device that don't have NVLink support, you can enable Ulysses FP8 Communication to further reduce the communication overhead. You can define a parallelism config yaml `parallel_fp8.yaml` file that contains: + +```yaml Config +parallelism_config: + ulysses_size: auto + attention_backend: native + ulysses_float8: true +``` + +- Async Ulysses CP + +You can also enable async ulysses CP to overlap the communication and computation. Define a parallelism config yaml `parallel_async.yaml` file that contains: + +```yaml Config +parallelism_config: + ulysses_size: auto + attention_backend: native + ulysses_async: true # Now, only support for FLUX.1, Qwen-Image, Ovis-Image and Z-Image. +``` +Then, apply the config from yaml. Here `ulysses_async: true` means enabling async ulysses CP. + +- TE-P and VAE-P + +You can also specify the extra parallel modules in the yaml config. For example, define a parallelism config yaml `parallel_extra.yaml` file that contains: + +```yaml Config +parallelism_config: + ulysses_size: auto + attention_backend: native + extra_parallel_modules: ["text_encoder", "vae"] +``` + + ### Hybrid Cache and Parallelism Define a hybrid cache and parallel acceleration config yaml `hybrid.yaml` file that contains: -```yaml +```yaml Config cache_config: max_warmup_steps: 8 warmup_interval: 2 @@ -129,9 +208,8 @@ cache_config: taylorseer_order: 1 parallelism_config: ulysses_size: auto - parallel_kwargs: - attention_backend: native - extra_parallel_modules: ["text_encoder", "vae"] + attention_backend: native + extra_parallel_modules: ["text_encoder", "vae"] ``` Then, apply the hybrid cache and parallel acceleration config from yaml. @@ -145,6 +223,72 @@ sglang generate \ --prompt "A beautiful sunset over the mountains" ``` +### Attention Backend + +In some cases, users may want to only specify the attention backend without any other optimization configs. In this case, you can define a yaml file `attention.yaml` that only contains: + +```yaml Config +attention_backend: "flash" # '_flash_3' for Hopper +``` + +### Quantization + +You can also specify the quantization config in the yaml file, required `torchao>=0.16.0`. For example, define a yaml file `quantize.yaml` that contains: + +```yaml Config +quantize_config: # quantization configuration for transformer modules + # float8 (DQ), float8_weight_only, float8_blockwise, int8 (DQ), int8_weight_only, etc. + quant_type: "float8" + # layers to exclude from quantization (transformer). layers that contains any of the + # keywords in the exclude_layers list will be excluded from quantization. This is useful + # for some sensitive layers that are not robust to quantization, e.g., embedding layers. + exclude_layers: + - "embedder" + - "embed" + verbose: false # whether to print verbose logs during quantization +``` +Then, apply the quantization config from yaml. Please also enable torch.compile for better performance if you are using quantization. For example: + +```bash Command +sglang generate \ + --backend diffusers \ + --model-path Qwen/Qwen-Image \ + --warmup \ + --cache-dit-config quantize.yaml \ + --enable-torch-compile \ + --dit-cpu-offload false \ + --text-encoder-cpu-offload false \ + --prompt "A beautiful sunset over the mountains" +``` + +### Combined Configs: Cache + Parallelism + Quantization + +You can also combine all the above configs together in a single yaml file `combined.yaml` that contains: + +```yaml Config +cache_config: + max_warmup_steps: 8 + warmup_interval: 2 + max_cached_steps: -1 + max_continuous_cached_steps: 2 + Fn_compute_blocks: 1 + Bn_compute_blocks: 0 + residual_diff_threshold: 0.12 + enable_taylorseer: true + taylorseer_order: 1 +parallelism_config: + ulysses_size: auto + attention_backend: native + extra_parallel_modules: ["text_encoder", "vae"] +quantize_config: + quant_type: "float8" + exclude_layers: + - "embedder" + - "embed" + verbose: false +``` +Then, apply the combined cache, parallelism and quantization config from yaml. Please also enable torch.compile for better performance if you are using quantization. + ## Advanced Configuration ### DBCache Parameters @@ -364,7 +508,7 @@ sglang generate --model-path Qwen/Qwen-Image \ ## Environment Variables All Cache-DiT parameters can be configured via environment variables. -See [Environment variables](./environment-variables) for the complete list. +See [Environment Variables](./environment_variables) for the complete list. ## Supported Models @@ -430,4 +574,4 @@ acceleration still works. ## References - [Cache-DiT](https://github.com/vipshop/cache-dit) -- [SGLang diffusion](../../sglang-diffusion/intro) +- [SGLang Diffusion](./performance-optimization) diff --git a/docs_new/docs/sglang-diffusion/caching-acceleration.mdx b/docs_new/docs/sglang-diffusion/caching-acceleration.mdx index 0c95559b2..d0f3e73df 100644 --- a/docs_new/docs/sglang-diffusion/caching-acceleration.mdx +++ b/docs_new/docs/sglang-diffusion/caching-acceleration.mdx @@ -2,8 +2,7 @@ title: "Caching Acceleration" description: "Compare caching acceleration strategies for diffusion models." --- - -SGLang provides multiple caching acceleration strategies for Diffusion Transformer (DiT) models. These strategies can significantly reduce inference time by skipping redundant computation. +SGLang provides two complementary caching strategies for Diffusion Transformer (DiT) models. Both reduce denoising cost by skipping redundant computation, but they operate at different levels. ## Overview @@ -40,13 +39,12 @@ SGLang supports two complementary caching approaches: - ## Cache-DiT [Cache-DiT](https://github.com/vipshop/cache-dit) provides block-level caching with advanced strategies like DBCache and TaylorSeer. It can achieve up to **1.69x speedup**. -See [Cache-DiT](./cache-dit) for detailed configuration. +See [cache_dit.md](./cache_dit) for detailed configuration. ### Quick Start @@ -66,7 +64,7 @@ sglang generate --model-path Qwen/Qwen-Image \ TeaCache (Temporal similarity-based caching) accelerates diffusion inference by detecting when consecutive denoising steps are similar enough to skip computation entirely. -See [TeaCache](./tea-cache) for detailed documentation. +See [teacache.md](./teacache) for detailed documentation. ### Quick Overview @@ -82,6 +80,7 @@ See [TeaCache](./tea-cache) for detailed documentation. For Flux and Qwen models, TeaCache is automatically disabled when CFG is enabled. + ## References - [Cache-DiT Repository](https://github.com/vipshop/cache-dit) diff --git a/docs_new/docs/sglang-diffusion/ci-performance.mdx b/docs_new/docs/sglang-diffusion/ci_perf.mdx similarity index 95% rename from docs_new/docs/sglang-diffusion/ci-performance.mdx rename to docs_new/docs/sglang-diffusion/ci_perf.mdx index 7d2f62ecd..2f87e6358 100644 --- a/docs_new/docs/sglang-diffusion/ci-performance.mdx +++ b/docs_new/docs/sglang-diffusion/ci_perf.mdx @@ -2,10 +2,11 @@ title: "CI Performance Baselines" description: "Generate and update diffusion performance baselines used in CI." --- +## Perf Baseline Generation Script `python/sglang/multimodal_gen/test/scripts/gen_perf_baselines.py` starts a local diffusion server, issues requests for selected test cases, aggregates stage/denoise-step/E2E timings from the perf log, and writes the results back to the `scenarios` section of `perf_baselines.json`. -## Usage +### Usage Update a single case: diff --git a/docs_new/docs/sglang-diffusion/compatibility_matrix.mdx b/docs_new/docs/sglang-diffusion/compatibility_matrix.mdx new file mode 100644 index 000000000..43935a98e --- /dev/null +++ b/docs_new/docs/sglang-diffusion/compatibility_matrix.mdx @@ -0,0 +1,631 @@ +--- +title: "Supported Models" +description: "Check model compatibility across diffusion optimizations and backends." +--- +The table below shows every supported model and the optimizations supported for them. + +The symbols used have the following meanings: + +- ✅ = Full compatibility +- ❌ = No compatibility +- ⭕ = Does not apply to this model + +## Models x Optimization + +The `HuggingFace Model ID` can be passed directly to `from_pretrained()` methods, and sglang-diffusion will use the +optimal +default parameters when initializing and generating videos. + +### Video Generation Models + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Model NameHugging Face Model IDResolutionsTeaCacheSliding Tile AttnSage AttnVideo Sparse Attention (VSA)Sparse Linear Attention (SLA)Sage Sparse Linear Attention (SageSLA)Sparse Video Gen 2 (SVG2)
FastWan2.1 T2V 1.3B`FastVideo/FastWan2.1-T2V-1.3B-Diffusers`480p⭕⭕⭕✅❌❌❌
FastWan2.2 TI2V 5B Full Attn`FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers`720p⭕⭕⭕✅❌❌❌
Wan2.2 TI2V 5B`Wan-AI/Wan2.2-TI2V-5B-Diffusers`720p⭕⭕✅⭕❌❌❌
Wan2.2 T2V A14B`Wan-AI/Wan2.2-T2V-A14B-Diffusers`480p<br>720p❌❌✅⭕❌❌❌
Wan2.2 I2V A14B`Wan-AI/Wan2.2-I2V-A14B-Diffusers`480p<br>720p❌❌✅⭕❌❌❌
HunyuanVideo`hunyuanvideo-community/HunyuanVideo`720×1280<br>544×960❌✅✅⭕❌❌✅
FastHunyuan`FastVideo/FastHunyuan-diffusers`720×1280<br>544×960❌✅✅⭕❌❌✅
Wan2.1 T2V 1.3B`Wan-AI/Wan2.1-T2V-1.3B-Diffusers`480p✅✅✅⭕❌❌✅
Wan2.1 T2V 14B`Wan-AI/Wan2.1-T2V-14B-Diffusers`480p, 720p✅✅✅⭕❌❌✅
Wan2.1 I2V 480P`Wan-AI/Wan2.1-I2V-14B-480P-Diffusers`480p✅✅✅⭕❌❌✅
Wan2.1 I2V 720P`Wan-AI/Wan2.1-I2V-14B-720P-Diffusers`720p✅✅✅⭕❌❌✅
TurboWan2.1 T2V 1.3B`IPostYellow/TurboWan2.1-T2V-1.3B-Diffusers`480p✅❌❌❌✅✅⭕
TurboWan2.1 T2V 14B`IPostYellow/TurboWan2.1-T2V-14B-Diffusers`480p✅❌❌❌✅✅⭕
TurboWan2.1 T2V 14B 720P`IPostYellow/TurboWan2.1-T2V-14B-720P-Diffusers`720p✅❌❌❌✅✅⭕
TurboWan2.2 I2V A14B`IPostYellow/TurboWan2.2-I2V-A14B-Diffusers`720p✅❌❌❌✅✅⭕
Wan2.1 Fun 1.3B InPweizhou03/Wan2.1-Fun-1.3B-InP-Diffusers480p✅✅✅⭕❌❌✅
Helios BaseBestWishYsh/Helios-Base720p❌❌❌❌❌❌❌
Helios MidBestWishYsh/Helios-Mid720p❌❌❌❌❌❌❌
Helios DistilledBestWishYsh/Helios-Distilled720p❌❌❌❌❌❌❌
LTX-2 (one and two stages)Lightricks/LTX-2768×512<br>1536×1024❌❌❌❌❌❌❌
LTX-2.3 (one and two stages)Lightricks/LTX-2.3768×512<br>1536×1024❌❌❌❌❌❌❌
+ +**Note**: + +1. Wan2.2 TI2V 5B has some quality issues when performing I2V generation. We are working on fixing this issue. +2. SageSLA is based on SpargeAttn. Install it first with `pip install git+https://github.com/thu-ml/SpargeAttn.git --no-build-isolation` +3. LTX-2 and LTX-2.3 two-stage generation uses `--pipeline-class-name LTX2TwoStagePipeline`. The spatial upsampler and distilled LoRA are auto-resolved from the model snapshot by default, and can still be overridden with `--spatial-upsampler-path` and `--distilled-lora-path`. + - For LTX models, the `Resolutions` column uses output video `width×height` semantics, matching `sglang generate --width ... --height ...`. +4. LTX-2.3 two-stage also supports `--ltx2-two-stage-device-mode {legacy,snapshot,resident}`: + - `snapshot` is the default and recommended mode. + - `resident` usually provides the best latency/throughput but uses much more VRAM. + - `legacy` preserves the historical switching path for fallback/debug. + +### Image Generation Models + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Model NameHuggingFace Model ID
FLUX.1-dev`black-forest-labs/FLUX.1-dev`
FLUX.2-dev`black-forest-labs/FLUX.2-dev`
FLUX.2-dev-NVFP4black-forest-labs/FLUX.2-dev-NVFP4
FLUX.2-Klein-4Bblack-forest-labs/FLUX.2-klein-4B
FLUX.2-Klein-9Bblack-forest-labs/FLUX.2-klein-9B
Z-ImageTongyi-MAI/Z-Image
Z-Image-TurboTongyi-MAI/Z-Image-Turbo
GLM-Imagezai-org/GLM-Image
Qwen ImageQwen/Qwen-Image
Qwen Image 2512Qwen/Qwen-Image-2512
Qwen Image Edit`Qwen/Qwen-Image-Edit`
Qwen Image Edit 2509Qwen/Qwen-Image-Edit-2509
Qwen Image Edit 2511Qwen/Qwen-Image-Edit-2511
Qwen Image LayeredQwen/Qwen-Image-Layered
SD3 Mediumstabilityai/stable-diffusion-3-medium-diffusers
SD3.5 Mediumstabilityai/stable-diffusion-3.5-medium-diffusers
SD3.5 Largestabilityai/stable-diffusion-3.5-large-diffusers
Hunyuan3D-2tencent/Hunyuan3D-2
SANA 1.5 1.6BEfficient-Large-Model/SANA1.5_1.6B_1024px_diffusers
SANA 1.5 4.8BEfficient-Large-Model/SANA1.5_4.8B_1024px_diffusers
SANA 1600M 1024pxEfficient-Large-Model/Sana_1600M_1024px_diffusers
SANA 600M 1024pxEfficient-Large-Model/Sana_600M_1024px_diffusers
SANA 1600M 512pxEfficient-Large-Model/Sana_1600M_512px_diffusers
SANA 600M 512pxEfficient-Large-Model/Sana_600M_512px_diffusers
FireRed-Image-Edit 1.0FireRedTeam/FireRed-Image-Edit-1.0
FireRed-Image-Edit 1.1FireRedTeam/FireRed-Image-Edit-1.1
ERNIE-Imagebaidu/ERNIE-Image
ERNIE-Image-Turbobaidu/ERNIE-Image-Turbo
+ +## Supported Components + +SGLang Diffusion supports overriding individual pipeline components with +`---path`. The value can be either a Hugging Face repo ID or a local +component directory. + +The same overrides can also be provided in config files through +`component_paths.`. + +### Common Syntax + +CLI: + +```bash Command +sglang generate \ + --model-path black-forest-labs/FLUX.2-dev \ + --vae-path black-forest-labs/FLUX.2-small-decoder \ + --transformer-path /models/flux2/transformer +``` + +Config file: + +```yaml Config +model_path: black-forest-labs/FLUX.2-dev +component_paths: + vae: black-forest-labs/FLUX.2-small-decoder + transformer: /models/flux2/transformer +``` + +Use the component name from the pipeline's `model_index.json` or the native pipeline's registered module name: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Component TypeSupported KeysNotes
VAEvae, video_vae, audio_vaevae is the common image-generation override
Transformer / DiTtransformer, video_dit, audio_dittransformer is the standard override for the main denoiser
Text / Preprocesstext_encoder, text_encoder_2, tokenizer, processor, image_processorReplacement encoders often need matching preprocessing assets
Auxiliaryscheduler, spatial_upsampler, vocoder, connectors, dual_tower_bridge, image_encoder, vision_language_encoderOnly valid for pipelines that expose these components
+ +### Known Component Repos + +The table below lists concrete Hugging Face component repos that are already used in SGLang Diffusion docs or tests. It is not an exhaustive catalog of all compatible component repos. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Base ModelOverride KeyExample RepoNotes
black-forest-labs/FLUX.2-devvaeblack-forest-labs/FLUX.2-small-decoderDecoder-only FLUX.2 VAE override
black-forest-labs/FLUX.2-devvaefal/FLUX.2-Tiny-AutoEncoderExisting tested custom VAE path
+ +### VAE + +- `--vae-path` is the common image-generation override. +- `--video-vae-path` and `--audio-vae-path` are only relevant for pipelines with separate video or audio VAEs. + +### Transformer / DiT + +- `--transformer-path` is the standard override for the main denoising transformer. +- For quantized transformers, prefer `--transformer-path` or `--transformer-weights-path`; see `quantization.md`. +- `--video-dit-path` and `--audio-dit-path` are only for pipelines that split denoisers by modality. + +### Text Encoders and Preprocessors + +- `--text-encoder-path` and `--text-encoder-2-path` override primary and secondary text encoders. +- `--tokenizer-path`, `--processor-path`, and `--image-processor-path` are useful when the replacement encoder requires matching preprocessing assets. + +### Auxiliary Components + +- `--scheduler-path` is only relevant when the pipeline exposes a scheduler component. +- `--spatial-upsampler-path` is mainly for two-stage pipelines such as `LTX2TwoStagePipeline`. +- `--vocoder-path`, `--connectors-path`, `--dual-tower-bridge-path`, `--image-encoder-path`, and `--vision-language-encoder-path` are only valid for pipelines that expose those components. + +### Notes + +1. Component overrides are only valid when the target pipeline actually uses + that component. +2. The override key should match the component name in the pipeline's + `model_index.json` or the native pipeline's registered module name. + +## Verified LoRA Examples + +This section lists example LoRAs that have been explicitly tested and verified with each base model in the **SGLang Diffusion** pipeline. + + +LoRAs that are not listed here are not necessarily incompatible. +In practice, most standard LoRAs are expected to work, especially those following common Diffusers or SD-style conventions. +The entries below simply reflect configurations that have been manually validated by the SGLang team. + + +### Verified LoRAs by Base Model + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Base ModelSupported LoRAs
Wan2.2`lightx2v/Wan2.2-Distill-Loras`
`Cseti/wan2.2-14B-Arcane_Jinx-lora-v1`
Wan2.1`lightx2v/Wan2.1-Distill-Loras`
Z-Image-Turbo`tarn59/pixel_art_style_lora_z_image_turbo`
`wcde/Z-Image-Turbo-DeJPEG-Lora`
Qwen-Image`lightx2v/Qwen-Image-Lightning`
`flymy-ai/qwen-image-realism-lora`
`prithivMLmods/Qwen-Image-HeadshotX`
`starsfriday/Qwen-Image-EVA-LoRA`
Qwen-Image-Edit`ostris/qwen_image_edit_inpainting`
`lightx2v/Qwen-Image-Edit-2511-Lightning`
Flux`dvyio/flux-lora-simple-illustration`
`XLabs-AI/flux-furry-lora`
`XLabs-AI/flux-RealismLora`
+ +## Special requirements + +### Sliding Tile Attention + +- Currently, only Hopper GPUs (H100s) are supported. diff --git a/docs_new/docs/sglang-diffusion/contributing.mdx b/docs_new/docs/sglang-diffusion/contributing.mdx new file mode 100644 index 000000000..f447518f6 --- /dev/null +++ b/docs_new/docs/sglang-diffusion/contributing.mdx @@ -0,0 +1,77 @@ +--- +title: "Contributing to SGLang Diffusion" +metatags: + description: "This guide outlines the requirements for contributing to the SGLang Diffusion module (sglang.multimodalgen)." +--- + +This guide outlines the requirements for contributing to the SGLang Diffusion module (`sglang.multimodal_gen`). + +## Contributor Guides + +- [Support New Models](./support_new_models): implementation guide for adding new diffusion pipelines +- [CI Performance](./ci_perf): update and regenerate perf baselines + + +## On AI-Assisted ("Vibe Coding") PRs + +Vibe-coded PRs are welcome — we judge code quality, not how it was produced. The bar is the same for all PRs: + +- **No over-commenting.** If the name says it all, skip the docstring. +- **No over-catching.** Don't guard against errors that virtually never happen in practice. +- **Test before submitting.** AI-generated code can be subtly wrong — verify correctness end-to-end. + +## Commit Message Convention + +We follow a structured commit message format to maintain a clean history. + +**Format:** +```text +[diffusion] : +``` + +**Examples:** +- `[diffusion] cli: add --perf-dump-path argument` +- `[diffusion] scheduler: fix deadlock in batch processing` +- `[diffusion] model: support Stable Diffusion 3.5` + +**Rules:** +- **Prefix**: Always start with `[diffusion]`. +- **Scope** (Optional): `cli`, `scheduler`, `model`, `pipeline`, `docs`, etc. +- **Subject**: Imperative mood, short and clear (e.g., "add feature" not "added feature"). + +## Performance Reporting + +For PRs that impact **latency**, **throughput**, or **memory usage**, you **should** provide a performance comparison report. + +### How to Generate a Report + +1. **Baseline**: run the benchmark (for a single generation task) + ```bash + $ sglang generate --model-path --prompt "A benchmark prompt" --perf-dump-path baseline.json + ``` + +2. **New**: run the same benchmark, without modifying any server_args or sampling_params + ```bash + $ sglang generate --model-path --prompt "A benchmark prompt" --perf-dump-path new.json + ``` + +3. **Compare**: run the compare script, which will print a Markdown table to the console + ```bash + $ python python/sglang/multimodal_gen/benchmarks/compare_perf.py baseline.json new.json [new2.json ...] + ### Performance Comparison Report + ... + ``` +4. **Paste**: paste the table into the PR description + +## CI-Based Change Protection + +Consider adding tests to the `pr-test` or `nightly-test` suites to safeguard your changes, especially for PRs that: + +- support a new model + - add a testcase for this new model to `testcase_configs.py` +- support or fix important features +- significantly improve performance + +Please run the according testcase, then update/add the baseline to `perf_baselines.json` by following the instruction in console if applicable. + +See [test](https://github.com/sgl-project/sglang/tree/main/python/sglang/multimodal_gen/test) for examples diff --git a/docs_new/docs/sglang-diffusion/disaggregation.mdx b/docs_new/docs/sglang-diffusion/disaggregation.mdx new file mode 100644 index 000000000..fcdc4e679 --- /dev/null +++ b/docs_new/docs/sglang-diffusion/disaggregation.mdx @@ -0,0 +1,361 @@ +--- +title: "Disaggregated Diffusion Pipeline" +metatags: + description: "Split SGLang Diffusion pipelines into independent encoder, denoiser, and decoder services for disaggregated serving." +--- + +Split a monolithic text-to-video/image pipeline into independent **Encoder**, **Denoiser**, and **Decoder** roles, each running on its own GPU(s). A central **DiffusionServer** routes requests through the pipeline. + +## Quick Start + +Disaggregation is controlled by a single flag: `--disagg-role`. Each component is launched independently, just like LLM PD disaggregation. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--disagg-roleWhat it runs
monolithic(Default) Standard single-server mode
encoderAll stages with the default RoleType.ENCODER affinity: InputValidationStage, TextEncodingStage (plus ImageEncodingStage / ImageVAEEncodingStage for image-conditioned pipelines), LatentPreparationStage, TimestepPreparationStage, and any model-specific "before denoising" stage (e.g. QwenImageLayeredBeforeDenoisingStage, GlmImageBeforeDenoisingStage).
denoiserDenoisingStage (and its subclasses: CausalDMDDenoisingStage, DmdDenoisingStage, LTX2AVDenoisingStage, LTX2RefinementStage, Hunyuan3DShapeDenoisingStage, ...) — the DiT forward loop plus the scheduler stepping it drives.
decoderDecodingStage (VAE decode) and its subclasses (LTX2AVDecodingStage, HeliosDecodingStage, ...).
serverDiffusionServer head node + HTTP server (no GPU)
+ +> Each stage declares its role via the `role_affinity` property on `PipelineStage` (default `ENCODER`). When `--disagg-role` is not `monolithic`, the pipeline only instantiates stages whose affinity matches, so the above table is the source of truth for what actually runs in each process. + +### Single-Machine Example (Verified) + +The following commands have been tested end-to-end on an 8×H200 machine with +`Wan-AI/Wan2.1-T2V-1.3B-Diffusers`. Each role runs on a separate GPU via +`--base-gpu-id`; the `server` head node requires no GPU. + +```bash +# Terminal 1: Encoder (GPU 0) +sglang serve --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ + --disagg-role encoder \ + --disagg-server-addr tcp://127.0.0.1:19655 \ + --scheduler-port 19000 \ + --num-gpus 1 --base-gpu-id 0 + +# Terminal 2: Denoiser (GPU 1) +sglang serve --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ + --disagg-role denoiser \ + --disagg-server-addr tcp://127.0.0.1:19655 \ + --scheduler-port 19001 \ + --num-gpus 1 --base-gpu-id 1 + +# Terminal 3: Decoder (GPU 2) +sglang serve --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ + --disagg-role decoder \ + --disagg-server-addr tcp://127.0.0.1:19655 \ + --scheduler-port 19002 \ + --num-gpus 1 --base-gpu-id 2 + +# Terminal 4: DiffusionServer head (no GPU, receives HTTP requests) +sglang serve --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ + --disagg-role server \ + --encoder-urls "tcp://127.0.0.1:19000" \ + --denoiser-urls "tcp://127.0.0.1:19001" \ + --decoder-urls "tcp://127.0.0.1:19002" \ + --host 0.0.0.0 --port 22000 \ + --scheduler-port 19655 + +# Send request (video generation) +curl http://127.0.0.1:22000/v1/videos \ + -H "Content-Type: application/json" \ + -d '{"model": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", "prompt": "A curious raccoon exploring a garden, cinematic", "size": "832x480"}' +``` + +> **Tested result (8×H200):** +> Encoder 2.3 s (TextEncoding) → Denoiser 312.8 s (50 steps, layerwise offload) → Decoder 7.1 s (VAE decode). +> Total ~322 s for 81-frame 1024×1024 video. + +> **Tip:** `--base-gpu-id` controls which physical GPU the role uses. +> Encoder and Decoder can share a GPU (e.g. both `--base-gpu-id 0`) to save resources, +> but make sure the combined GPU memory is sufficient. + +### Multi-Machine Example + +The exact same CLI pattern — just replace `127.0.0.1` with actual IPs and add +RDMA flags for direct transfer: + +```bash +# Machine A (10.0.0.1): Encoder +sglang serve --model-path Wan-AI/Wan2.1-T2V-14B-Diffusers \ + --disagg-role encoder \ + --disagg-server-addr tcp://10.0.0.4:19655 \ + --scheduler-port 19000 \ + --num-gpus 1 \ + --disagg-p2p-hostname 10.0.0.1 --disagg-ib-device mlx5_0 + +# Machine B (10.0.0.2): Denoiser (4 GPUs with SP) +sglang serve --model-path Wan-AI/Wan2.1-T2V-14B-Diffusers \ + --disagg-role denoiser \ + --disagg-server-addr tcp://10.0.0.4:19655 \ + --scheduler-port 19001 \ + --num-gpus 4 --denoiser-sp 4 --denoiser-ulysses 2 --denoiser-ring 2 \ + --disagg-p2p-hostname 10.0.0.2 --disagg-ib-device mlx5_0 + +# Machine C (10.0.0.3): Decoder +sglang serve --model-path Wan-AI/Wan2.1-T2V-14B-Diffusers \ + --disagg-role decoder \ + --disagg-server-addr tcp://10.0.0.4:19655 \ + --scheduler-port 19002 \ + --num-gpus 1 \ + --disagg-p2p-hostname 10.0.0.3 --disagg-ib-device mlx5_0 + +# Machine D (10.0.0.4): DiffusionServer head +sglang serve --model-path Wan-AI/Wan2.1-T2V-14B-Diffusers \ + --disagg-role server \ + --encoder-urls "tcp://10.0.0.1:19000" \ + --denoiser-urls "tcp://10.0.0.2:19001" \ + --decoder-urls "tcp://10.0.0.3:19002" \ + --host 0.0.0.0 --port 30000 \ + --scheduler-port 19655 \ + --disagg-dispatch-policy max_free_slots +``` + +> ZMQ handles startup order gracefully — instances and head can start in any order. + +## Multiple Instances per Role + +Use semicolons in `--*-urls` to register multiple instances: + +```bash +# 2 encoders + 2 denoisers (4-GPU SP each) + 1 decoder +sglang serve --model-path ... --disagg-role server \ + --encoder-urls "tcp://10.0.0.1:35000;tcp://10.0.0.2:35000" \ + --denoiser-urls "tcp://10.0.0.3:35000;tcp://10.0.0.4:35000" \ + --decoder-urls "tcp://10.0.0.5:35000" +``` + +## Port Convention + +Result endpoints are derived deterministically from the head node's `--scheduler-port` (default: 5555): + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
SocketPort
DS frontend (ROUTER)scheduler_port
Encoder result (PULL)scheduler_port + 1
Denoiser result (PULL)scheduler_port + 2
Decoder result (PULL)scheduler_port + 3
+ +Role instances derive their result endpoint automatically from `--disagg-server-addr`. No manual endpoint configuration needed. + +## Transfer Mechanism + +Tensor data between roles (encoder→denoiser, denoiser→decoder) is transferred via a P2P transfer engine. The DiffusionServer only routes lightweight control messages (alloc/push/ready); actual tensor data flows directly between instances. + +**mooncake-transfer-engine** is required for disaggregated diffusion. It provides RDMA for direct GPU-to-GPU data movement. + +```bash +pip install mooncake-transfer-engine +``` + +### Transfer Flow + +1. **Sender** (encoder/denoiser) stages tensors: async copy to transfer buffer (GPU or CPU pinned, depending on GPUDirect support), overlapped with metadata JSON serialization. +2. **Sender** sends `transfer_staged` control message to DiffusionServer (metadata only, no tensor data). +3. **DiffusionServer** sends `transfer_alloc` to receiver → receiver allocates buffer slot → replies `transfer_allocated`. +4. **DiffusionServer** sends `transfer_push` to receiver with sender's address info. +5. **Receiver** pulls data via transfer engine (Mooncake RDMA or mock), sends `transfer_ready`. +6. **Receiver** loads tensors async on a dedicated transfer stream, overlapped with the previous request's compute. + +Decoder results (final output) flow back through DiffusionServer as raw ZMQ frames to the HTTP client. + +### RDMA Flags + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FlagDefaultDescription
--disagg-p2p-hostname127.0.0.1RDMA-reachable hostname/IP of this instance
--disagg-ib-deviceNoneInfiniBand device (e.g., mlx5_0, mlx5_roce0)
--disagg-transfer-pool-size256 MiBPinned memory pool per instance
+ +Set `--disagg-p2p-hostname` to the actual IP on each machine. For multi-machine, `--disagg-ib-device` specifies the RDMA NIC. + +## Per-Role Parallelism + + + + + + + + + + + + + + + + + + + + + + + + + + +
FlagDescription
--encoder-tpEncoder tensor parallelism
--denoiser-tp / --denoiser-sp / --denoiser-ulysses / --denoiser-ringDenoiser parallelism
--decoder-tpDecoder tensor parallelism
+ +If not specified, parallelism is auto-derived from `--num-gpus`. + +## Other Options + + + + + + + + + + + + + + + + + + + + + + + + + + +
FlagDefaultDescription
--disagg-timeout600Timeout (seconds) for pending requests
--disagg-dispatch-policyround_robinround_robin or max_free_slots
+ +## Python API + +For programmatic single-machine deployment, `launch_pool_disagg_server()` is available: + +```python +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.launch_server import launch_pool_disagg_server + +server_args = ServerArgs.from_kwargs( + model_path="Wan-AI/Wan2.1-T2V-14B-Diffusers", + denoiser_sp=4, denoiser_ulysses=2, denoiser_ring=2, + disagg_ib_device="mlx5_0", +) + +launch_pool_disagg_server( + server_args, + encoder_gpus=[[0]], + denoiser_gpus=[[1, 2, 3, 4], [5, 6, 7, 8]], + decoder_gpus=[[0]], +) +``` + +## Architecture + +``` +Client ─── HTTP (port 30000) ──► FastAPI Server + │ + ▼ + DiffusionServer (ROUTER, scheduler_port) + ┌───────┼───────┐ + PUSH work │ │ │ PUSH work + ▼ │ ▼ + Encoder[0..N] │ Decoder[0..K] + │ │ ▲ + P2P tensor │ │ │ P2P tensor + transfer ▼ │ │ transfer + Denoiser[0..M] ─────┘ + │ + PULL results ◄────┘ (decoder → DS → client) +``` + +### Request State Machine + +``` +PENDING → ENCODER_WAITING → ENCODER_RUNNING → ENCODER_DONE + │ + DENOISING_WAITING → DENOISING_RUNNING → DENOISING_DONE + │ + DECODER_WAITING → DECODER_RUNNING → DONE +``` + +Any state can transition to `FAILED` or `TIMED_OUT`. diff --git a/docs_new/docs/sglang-diffusion/environment-variables.mdx b/docs_new/docs/sglang-diffusion/environment-variables.mdx deleted file mode 100644 index 603c68405..000000000 --- a/docs_new/docs/sglang-diffusion/environment-variables.mdx +++ /dev/null @@ -1,140 +0,0 @@ ---- -title: "Environment Variables" -description: "Configure SGLang diffusion behavior with environment variables." ---- - -These variables configure caching acceleration for Diffusion Transformer (DiT) models. -SGLang supports multiple caching strategies - see [performance optimization documentation](./performance-optimization) for an overview. - -See [Environment Variables](../references/environment_variables) for a list of all environment variables. - -## Cache-DiT configuration - -See [Cache-DiT documentation](./cache-dit) for detailed configuration. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Environment VariableDefaultDescription
`SGLANG_CACHE_DIT_ENABLED`falseEnable Cache-DiT acceleration
`SGLANG_CACHE_DIT_FN`1First N blocks to always compute
`SGLANG_CACHE_DIT_BN`0Last N blocks to always compute
`SGLANG_CACHE_DIT_WARMUP`4Warmup steps before caching
`SGLANG_CACHE_DIT_RDT`0.24Residual difference threshold
`SGLANG_CACHE_DIT_MC`3Max continuous cached steps
`SGLANG_CACHE_DIT_TAYLORSEER`falseEnable TaylorSeer calibrator
`SGLANG_CACHE_DIT_TS_ORDER`1TaylorSeer order (1 or 2)
`SGLANG_CACHE_DIT_SCM_PRESET`noneSCM preset (none/slow/medium/fast/ultra)
`SGLANG_CACHE_DIT_SCM_POLICY`dynamicSCM caching policy
`SGLANG_CACHE_DIT_SCM_COMPUTE_BINS`not setCustom SCM compute bins
`SGLANG_CACHE_DIT_SCM_CACHE_BINS`not setCustom SCM cache bins
- -## Cloud Storage - -These variables configure S3-compatible cloud storage for automatically uploading generated images and videos. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Environment VariableDefaultDescription
`SGLANG_CLOUD_STORAGE_TYPE`not setSet to `s3` to enable cloud storage
`SGLANG_S3_BUCKET_NAME`not setThe name of the S3 bucket
`SGLANG_S3_ENDPOINT_URL`not setCustom endpoint URL (for MinIO, OSS, etc.)
`SGLANG_S3_REGION_NAME`us-east-1AWS region name
`SGLANG_S3_ACCESS_KEY_ID`not setAWS Access Key ID
`SGLANG_S3_SECRET_ACCESS_KEY`not setAWS Secret Access Key
diff --git a/docs_new/docs/sglang-diffusion/environment_variables.mdx b/docs_new/docs/sglang-diffusion/environment_variables.mdx new file mode 100644 index 000000000..8ade9a7ca --- /dev/null +++ b/docs_new/docs/sglang-diffusion/environment_variables.mdx @@ -0,0 +1,395 @@ +--- +title: "Environment Variables" +description: "Configure SGLang diffusion behavior with environment variables." +--- +## Runtime + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Environment VariableDefaultDescription
SGLANG_DIFFUSION_TARGET_DEVICEcudaTarget device for inference (cuda, rocm, xpu, npu, musa, mps, cpu)
SGLANG_DIFFUSION_ATTENTION_BACKENDnot setOverride attention backend via env var (e.g. fa, torch_sdpa, sage_attn)
SGLANG_DIFFUSION_ATTENTION_CONFIGnot setPath to attention backend configuration file (JSON/YAML)
SGLANG_DIFFUSION_STAGE_LOGGINGfalseEnable per-stage timing logs
SGLANG_DIFFUSION_SERVER_DEV_MODEfalseEnable dev-only HTTP endpoints for debugging
SGLANG_DIFFUSION_TORCH_PROFILER_DIRnot setDirectory for torch profiler traces (absolute path). Enables profiling when set
SGLANG_DIFFUSION_CACHE_ROOT~/.cache/sgl_diffusionRoot directory for cache files
SGLANG_DIFFUSION_CONFIG_ROOT~/.config/sgl_diffusionRoot directory for configuration files
SGLANG_DIFFUSION_LOGGING_LEVELINFODefault logging level
SGLANG_DIFFUSION_WORKER_MULTIPROC_METHODforkMultiprocess context for workers (fork or spawn)
SGLANG_USE_RUNAI_MODEL_STREAMERtrueUse Run:AI model streamer for model loading
+ +## Platform-Specific + +### Apple MPS + + + + + + + + + + + + + + + + + + + + + +
Environment VariableDefaultDescription
SGLANG_USE_MLXnot setSet to 1 to enable MLX fused Metal kernels for norm ops on MPS
+ +### ROCm (AMD GPUs) + + + + + + + + + + + + + + + + + + + + + + + + + + +
Environment VariableDefaultDescription
SGLANG_USE_ROCM_VAEfalseUse AITer GroupNorm in VAE for improved performance on ROCm
SGLANG_USE_ROCM_CUDNN_BENCHMARKfalseEnable MIOpen auto-tuning for VAE conv layers on ROCm
+ +### Quantization + + + + + + + + + + + + + + + + + + + + + +
Environment VariableDefaultDescription
SGLANG_DIFFUSION_FLASHINFER_FP4_GEMM_BACKENDnot setFlashInfer FP4 GEMM backend for generic NVFP4 fallback
+ +## Caching Acceleration + +These variables configure caching acceleration for Diffusion Transformer (DiT) models. +SGLang supports multiple caching strategies - see [caching documentation](./caching-acceleration) for an overview. + +### Cache-DiT Configuration + +See [cache-dit documentation](./cache_dit) for detailed configuration. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Environment VariableDefaultDescription
`SGLANG_CACHE_DIT_ENABLED`falseEnable Cache-DiT acceleration
`SGLANG_CACHE_DIT_FN`1First N blocks to always compute
`SGLANG_CACHE_DIT_BN`0Last N blocks to always compute
`SGLANG_CACHE_DIT_WARMUP`4Warmup steps before caching
`SGLANG_CACHE_DIT_RDT`0.24Residual difference threshold
`SGLANG_CACHE_DIT_MC`3Max continuous cached steps
`SGLANG_CACHE_DIT_TAYLORSEER`falseEnable TaylorSeer calibrator
`SGLANG_CACHE_DIT_TS_ORDER`1TaylorSeer order (1 or 2)
`SGLANG_CACHE_DIT_SCM_PRESET`noneSCM preset (none/slow/medium/fast/ultra)
`SGLANG_CACHE_DIT_SCM_POLICY`dynamicSCM caching policy
`SGLANG_CACHE_DIT_SCM_COMPUTE_BINS`not setCustom SCM compute bins
`SGLANG_CACHE_DIT_SCM_CACHE_BINS`not setCustom SCM cache bins
+ +### Cache-DiT Secondary Transformer + +For dual-transformer models (e.g., Wan2.2 with high/low-noise experts), these variables configure caching for the secondary transformer. Each falls back to its primary counterpart if not set. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Environment VariableDefaultDescription
SGLANG_CACHE_DIT_SECONDARY_FN(from primary)First N blocks to always compute
SGLANG_CACHE_DIT_SECONDARY_BN(from primary)Last N blocks to always compute
SGLANG_CACHE_DIT_SECONDARY_WARMUP(from primary)Warmup steps before caching
SGLANG_CACHE_DIT_SECONDARY_RDT(from primary)Residual difference threshold
SGLANG_CACHE_DIT_SECONDARY_MC(from primary)Max continuous cached steps
SGLANG_CACHE_DIT_SECONDARY_TAYLORSEER(from primary)Enable TaylorSeer calibrator
SGLANG_CACHE_DIT_SECONDARY_TS_ORDER(from primary)TaylorSeer order (1 or 2)
+ +## Cloud Storage + +These variables configure S3-compatible cloud storage for automatically uploading generated images and videos. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Environment VariableDefaultDescription
`SGLANG_CLOUD_STORAGE_TYPE`not setSet to `s3` to enable cloud storage
`SGLANG_S3_BUCKET_NAME`not setThe name of the S3 bucket
`SGLANG_S3_ENDPOINT_URL`not setCustom endpoint URL (for MinIO, OSS, etc.)
`SGLANG_S3_REGION_NAME`us-east-1AWS region name
`SGLANG_S3_ACCESS_KEY_ID`not setAWS Access Key ID
`SGLANG_S3_SECRET_ACCESS_KEY`not setAWS Secret Access Key
+ +## CUDA Crash Debugging + +These variables enable kernel API logging and optional input/output dumps around diffusion CUDA kernel call boundaries. They are useful when tracking down CUDA crashes such as illegal memory access, device-side assert, or shape mismatches in custom kernels. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Environment VariableDefaultDescription
SGLANG_KERNEL_API_LOGLEVEL0Controls crash-debug kernel API logging. 1 logs API names, 3 logs tensor metadata, 5 adds tensor statistics, and 10 also writes dump snapshots.
SGLANG_KERNEL_API_LOGDESTstdoutDestination for crash-debug kernel API logs. Use stdout, stderr, or a file path. %i is replaced with the process PID.
SGLANG_KERNEL_API_DUMP_DIRsglang_kernel_api_dumpsOutput directory for level-10 kernel API dumps. %i is replaced with the process PID.
SGLANG_KERNEL_API_DUMP_INCLUDEnot setComma-separated wildcard patterns for kernel API names to include in level-10 dumps.
SGLANG_KERNEL_API_DUMP_EXCLUDEnot setComma-separated wildcard patterns for kernel API names to exclude from level-10 dumps.
diff --git a/docs_new/docs/sglang-diffusion/index.mdx b/docs_new/docs/sglang-diffusion/index.mdx index 5e4a54d73..570a273d2 100644 --- a/docs_new/docs/sglang-diffusion/index.mdx +++ b/docs_new/docs/sglang-diffusion/index.mdx @@ -2,64 +2,54 @@ title: SGLang Diffusion description: Accelerated image and video generation with diffusion models. --- +SGLang Diffusion is a high-performance inference framework for image and video generation. It provides native SGLang pipelines, diffusers backend support, an OpenAI-compatible server, and an optimized kernel stack built on both precompiled `sgl-kernel` operators and JIT kernels for key inference paths. -SGLang Diffusion is an inference framework for accelerated image and video generation using diffusion models. It provides an end-to-end unified pipeline with optimized kernels and an efficient scheduler loop. +## Key Features -## Key features +- Broad model support across Wan, Hunyuan, Qwen-Image, FLUX, Z-Image, GLM-Image, and more +- Fast inference with `sgl-kernel`, JIT kernels, scheduler improvements, and caching acceleration +- Multiple interfaces: `sglang generate`, `sglang serve`, and an OpenAI-compatible API +- Multi-platform support for NVIDIA, AMD, Intel XPU, Ascend, Apple Silicon, and Moore Threads -* **Broad model support:** Wan series, FastWan series, Hunyuan, Qwen-Image, Qwen-Image-Edit, Flux, Z-Image, GLM-Image, and more -* **Fast inference:** optimized kernels, efficient scheduler loop, and Cache-DiT acceleration -* **Ease of use:** OpenAI-compatible API, CLI, and Python SDK -* **Multi-platform:** NVIDIA GPUs (H100, H200, A100, B200, 4090), AMD GPUs (MI300X, MI325X), and Ascend NPU (A2, A3) - -## Quick start - -1. **Install SGLang Diffusion** +## Quick Start ```bash uv pip install "sglang[diffusion]" --prerelease=allow ``` -See the [installation guide](./installation) for more installation methods and ROCm-specific instructions. - -2. **Run a one-off generation** - ```bash sglang generate --model-path Qwen/Qwen-Image \ --prompt "A beautiful sunset over the mountains" \ --save-output ``` -3. **Serve with the OpenAI-compatible API** - ```bash sglang serve --model-path Qwen/Qwen-Image --port 30010 ``` -## CLI quick reference +## Start Here -### Generate (one-off generation) +- [Installation](./installation): install SGLang Diffusion and platform dependencies +- [Compatibility Matrix](./compatibility_matrix): check model, optimization, and component override support +- [CLI](./api/cli): run one-off generation jobs or launch a persistent server +- [OpenAI-Compatible API](./api/openai_api): send image and video requests to the HTTP server +- [Attention Backends](./attention_backends): choose the best backend for your model and hardware +- [Caching Acceleration](./caching-acceleration): use Cache-DiT or TeaCache to reduce denoising cost +- [Quantization](./quantization): load quantized transformer checkpoints +- [Contributing](./contributing): contribution workflow, adding new models, and CI perf baselines -```bash -sglang generate --model-path --prompt "" --save-output -``` +## Additional Documentation -### Serve (HTTP server) - -```bash -sglang serve --model-path --port 30010 -``` - -### Enable Cache-DiT acceleration - -```bash -SGLANG_CACHE_DIT_ENABLED=true sglang generate --model-path --prompt "" -``` +- [Post-Processing](./api/post_processing): frame interpolation and upscaling +- [Performance Overview](./performance-optimization): overview of attention, caching, and profiling +- [Environment Variables](./environment_variables): platform, caching, storage, and debugging configuration +- [Support New Models](./support_new_models): implementation guide for new diffusion pipelines +- [CI Performance](./ci_perf): performance baseline generation ## References -* [SGLang GitHub](https://github.com/sgl-project/sglang) -* [Cache-DiT](https://github.com/vipshop/cache-dit) -* [FastVideo](https://github.com/hao-ai-lab/FastVideo) -* [xDiT](https://github.com/xdit-project/xDiT) -* [Diffusers](https://github.com/huggingface/diffusers) +- [SGLang GitHub](https://github.com/sgl-project/sglang) +- [Cache-DiT](https://github.com/vipshop/cache-dit) +- [FastVideo](https://github.com/hao-ai-lab/FastVideo) +- [xDiT](https://github.com/xdit-project/xDiT) +- [Diffusers](https://github.com/huggingface/diffusers) diff --git a/docs_new/docs/sglang-diffusion/installation.mdx b/docs_new/docs/sglang-diffusion/installation.mdx index da8d12cec..13210d83b 100644 --- a/docs_new/docs/sglang-diffusion/installation.mdx +++ b/docs_new/docs/sglang-diffusion/installation.mdx @@ -2,109 +2,129 @@ title: Install SGLang Diffusion description: Install SGLang Diffusion on NVIDIA, AMD, MUSA, and Ascend platforms. --- +You can install SGLang-Diffusion using one of the methods below. The standard installation already includes SGLang's optimized kernel stack, including both `sgl-kernel` and JIT kernels used by diffusion workloads. -You can install SGLang Diffusion using one of the methods below. +## Standard Installation (NVIDIA GPUs) -## Standard installation (NVIDIA GPUs) +### Method 1: With pip or uv -**Platform:** NVIDIA GPUs (CUDA) +It is recommended to use uv for a faster installation: - - - Use `uv` for faster installation: +```bash Command +pip install --upgrade pip +pip install uv +uv pip install "sglang[diffusion]" --prerelease=allow +``` - ```bash - pip install --upgrade pip - pip install uv - uv pip install "sglang[diffusion]" --prerelease=allow - ``` - +### Method 2: From source - - ```bash - git clone https://github.com/sgl-project/sglang.git - cd sglang - pip install --upgrade pip - pip install -e "python[diffusion]" - ``` +```bash Command +# Use the latest release branch +git clone https://github.com/sgl-project/sglang.git +cd sglang - Or with `uv`: +# Install the Python packages +pip install --upgrade pip +pip install -e "python[diffusion]" - ```bash - uv pip install -e "python[diffusion]" --prerelease=allow - ``` - +# With uv +uv pip install -e "python[diffusion]" --prerelease=allow +``` - - The Docker images are available on Docker Hub at [lmsysorg/sglang](https://hub.docker.com/r/lmsysorg/sglang/tags), built from the [Dockerfile](https://github.com/sgl-project/sglang/blob/main/docker/Dockerfile). Replace `` below with your HuggingFace Hub [token](https://huggingface.co/docs/hub/en/security-tokens). +### Method 3: Using Docker - ```bash - docker run --gpus all \ - --shm-size 32g \ - -p 30000:30000 \ - -v ~/.cache/huggingface:/root/.cache/huggingface \ - --env "HF_TOKEN=" \ - --ipc=host \ - lmsysorg/sglang:dev \ - zsh -c '\ - echo "Installing diffusion dependencies..." && \ - pip install -e "python[diffusion]" && \ - echo "Starting SGLang-Diffusion..." && \ - sglang generate \ - --model-path black-forest-labs/FLUX.1-dev \ - --prompt "A logo With Bold Large text: SGL Diffusion" \ - --save-output \ - ' - ``` - - +The Docker images are available on Docker Hub at [lmsysorg/sglang](https://hub.docker.com/r/lmsysorg/sglang), built from the [Dockerfile](https://github.com/sgl-project/sglang/blob/main/docker/Dockerfile). +Replace `` below with your HuggingFace Hub [token](https://huggingface.co/docs/hub/en/security-tokens). -## Platform-specific installs +```bash Command +docker run --gpus all \ + --shm-size 32g \ + -p 30000:30000 \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --env "HF_TOKEN=" \ + --ipc=host \ + lmsysorg/sglang:dev \ + zsh -c '\ + echo "Installing diffusion dependencies..." && \ + pip install -e "python[diffusion]" && \ + echo "Starting SGLang-Diffusion..." && \ + sglang generate \ + --model-path black-forest-labs/FLUX.1-dev \ + --prompt "A logo With Bold Large text: SGL Diffusion" \ + --save-output \ + ' +``` -Use the tab that matches your accelerator. +## Platform-Specific: ROCm (AMD GPUs) - - - **Platform:** AMD Instinct GPUs (ROCm) +For AMD Instinct GPUs (e.g., MI300X), you can use the ROCm-enabled Docker image: - For AMD Instinct GPUs (for example, MI300X), use the ROCm-enabled Docker image: +```bash Command +docker run --device=/dev/kfd --device=/dev/dri --ipc=host \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --env HF_TOKEN= \ + lmsysorg/sglang:v0.5.5.post2-rocm700-mi30x \ + sglang generate --model-path black-forest-labs/FLUX.1-dev --prompt "A logo With Bold Large text: SGL Diffusion" --save-output +``` - ```bash - docker run --device=/dev/kfd --device=/dev/dri --ipc=host \ - -v ~/.cache/huggingface:/root/.cache/huggingface \ - --env HF_TOKEN= \ - lmsysorg/sglang:v0.5.9-rocm700-mi30x \ - sglang generate --model-path black-forest-labs/FLUX.1-dev --prompt "A logo With Bold Large text: SGL Diffusion" --save-output - ``` +For detailed ROCm system configuration and installation from source, see [AMD GPUs](../hardware-platforms/amd_gpu). - For detailed ROCm system configuration and installation from source, see [AMD GPUs](../hardware-platforms/amd-gpus). - +## Platform-Specific: MUSA (Moore Threads GPUs) - - **Platform:** Moore Threads GPUs (MUSA) +For Moore Threads GPUs (MTGPU) with the MUSA software stack, please follow the instructions below to install from source: - For Moore Threads GPUs (MTGPU) with the MUSA software stack: +```bash Command +# Clone the repository +git clone https://github.com/sgl-project/sglang.git +cd sglang - ```bash - git clone https://github.com/sgl-project/sglang.git - cd sglang - pip install --upgrade pip - rm -f python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml - pip install -e "python[all_musa]" - ``` - +# Install the Python packages +pip install --upgrade pip +rm -f python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml +pip install -e "python[all_musa]" +``` - - **Platform:** Ascend NPU +## Platform-Specific: Intel XPU - For Ascend NPU, follow the [NPU installation guide](../hardware-platforms/ascend-npus/SGLang-installation-with-NPUs-support). +For Intel Data Center GPU Max or Arc GPUs, follow the [XPU installation guide](../hardware-platforms/xpu) to set up the base environment, then install diffusion dependencies: - Quick test: +```bash Command +pip install -e "python[diffusion]" +``` - ```bash - sglang generate --model-path black-forest-labs/FLUX.1-dev \ - --prompt "A logo With Bold Large text: SGL Diffusion" \ - --save-output - ``` - - +## Platform-Specific: Ascend NPU + +For Ascend NPU, please follow the [NPU installation guide](../hardware-platforms/ascend-npus/ascend_npu). + +Quick test: + +```bash Command +sglang generate --model-path black-forest-labs/FLUX.1-dev \ + --prompt "A logo With Bold Large text: SGL Diffusion" \ + --save-output +``` + +## Platform-Specific: Apple MPS + +For Apple MPS, please follow the instructions below to install from source: + +```bash Command +# Install ffmpeg +brew install ffmpeg + +# Install uv +brew install uv + +# Clone the repository +git clone https://github.com/sgl-project/sglang.git +cd sglang + +# Create and activate a virtual environment +uv venv -p 3.11 sglang-diffusion +source sglang-diffusion/bin/activate + +# Install the Python packages +uv pip install --upgrade pip +rm -f python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml +uv pip install -e "python[all_mps]" +``` diff --git a/docs_new/docs/sglang-diffusion/performance-optimization.mdx b/docs_new/docs/sglang-diffusion/performance-optimization.mdx index a578a3bea..bb5c15283 100644 --- a/docs_new/docs/sglang-diffusion/performance-optimization.mdx +++ b/docs_new/docs/sglang-diffusion/performance-optimization.mdx @@ -2,8 +2,7 @@ title: "Performance Optimization" description: "Optimize SGLang diffusion performance with caching, kernels, and profiling." --- - -SGLang-Diffusion provides multiple performance optimization strategies to accelerate inference. This section covers all available performance tuning options. +This section covers the main performance levers for SGLang Diffusion: attention backends, caching acceleration, and profiling. ## Overview @@ -29,7 +28,7 @@ SGLang-Diffusion provides multiple performance optimization strategies to accele TeaCache Caching - Timestep-level caching using L1 similarity + Timestep-level caching based on temporal similarity Attention Backends @@ -44,60 +43,23 @@ SGLang-Diffusion provides multiple performance optimization strategies to accele -## Caching Strategies +## Start Here -SGLang supports two complementary caching approaches: +- Use [Attention Backends](./attention_backends) to choose the best backend for your model and hardware. +- Use [Caching Acceleration](./caching-acceleration) to reduce denoising cost with Cache-DiT or TeaCache. +- Use [Profiling](./profiling) when you need to diagnose a bottleneck rather than guess. -### Cache-DiT +## Caching at a Glance -[Cache-DiT](https://github.com/vipshop/cache-dit) provides block-level caching with advanced strategies. It can achieve up to **1.69x speedup**. +- [Cache-DiT](./cache_dit) is block-level caching for diffusers pipelines and higher speedup-oriented tuning. +- [TeaCache](./teacache) is timestep-level caching built into SGLang model families. -**Quick Start:** -```bash -SGLANG_CACHE_DIT_ENABLED=true \ -sglang generate --model-path Qwen/Qwen-Image \ - --prompt "A beautiful sunset over the mountains" -``` -**Key Features:** -- **DBCache**: Dynamic block-level caching based on residual differences -- **TaylorSeer**: Taylor expansion-based calibration for optimized caching -- **SCM**: Step-level computation masking for additional speedup +## Current Baseline Snapshot -See [Cache-DiT documentation](./cache-dit) for detailed configuration. +For Ring SP benchmark details, see: -### TeaCache - -TeaCache (Temporal similarity-based caching) accelerates diffusion inference by detecting when consecutive denoising steps are similar enough to skip computation entirely. - -**Quick Overview:** -- Tracks L1 distance between modulated inputs across timesteps -- When accumulated distance is below threshold, reuses cached residual -- Supports CFG with separate positive/negative caches - -**Supported Models:** Wan (wan2.1, wan2.2), Hunyuan (HunyuanVideo), Z-Image - -See [TeaCache documentation](./tea-cache) for detailed configuration. - -## Attention Backends - -Different attention backends offer varying performance characteristics depending on your hardware and model: - -- **FlashAttention**: Fastest on NVIDIA GPUs with fp16/bf16 -- **SageAttention**: Alternative optimized implementation -- **xformers**: Memory-efficient attention -- **SDPA**: PyTorch native scaled dot-product attention - -See [Attention backends](./attention-backends) for platform support and configuration options. - -## Profiling - -To diagnose performance bottlenecks, SGLang-Diffusion supports profiling tools: - -- **PyTorch Profiler**: Built-in Python profiling -- **Nsight Systems**: GPU kernel-level analysis - -See [Profiling guide](./profiling) for detailed instructions. +- [Ring SP Performance](./ring_sp_performance) ## References diff --git a/docs_new/docs/sglang-diffusion/profiling.mdx b/docs_new/docs/sglang-diffusion/profiling.mdx index 767a6cfab..2fb327a2a 100644 --- a/docs_new/docs/sglang-diffusion/profiling.mdx +++ b/docs_new/docs/sglang-diffusion/profiling.mdx @@ -2,7 +2,6 @@ title: "Profiling" description: "Profile SGLang diffusion workloads with PyTorch Profiler and Nsight Systems." --- - This guide covers profiling techniques for multimodal generation pipelines in SGLang. ## PyTorch Profiler diff --git a/docs_new/docs/sglang-diffusion/quantization.mdx b/docs_new/docs/sglang-diffusion/quantization.mdx new file mode 100644 index 000000000..62d336f34 --- /dev/null +++ b/docs_new/docs/sglang-diffusion/quantization.mdx @@ -0,0 +1,551 @@ +--- +title: "Quantization" +metatags: + description: "SGLang-Diffusion supports quantized transformer checkpoints. In most cases, keep the base model and the quantized transformer override separate." +--- + +SGLang-Diffusion supports quantized transformer checkpoints. In most cases, keep +the base model and the quantized transformer override separate. + +## Quick Reference + +Use these paths: + +- `--model-path`: the base or original model +- `--transformer-path`: a quantized transformers-style transformer component directory that already contains its own `config.json` +- `--transformer-weights-path`: quantized transformer weights provided as a single safetensors file, a sharded safetensors directory, a local path, or a Hugging Face repo ID + +Recommended example: + +```bash +sglang generate \ + --model-path black-forest-labs/FLUX.2-dev \ + --transformer-weights-path black-forest-labs/FLUX.2-dev-NVFP4 \ + --prompt "a curious pikachu" +``` + +For quantized transformers-style transformer component folders: + +```bash +sglang generate \ + --model-path /path/to/base-model \ + --transformer-path /path/to/quantized-transformer \ + --prompt "A Logo With Bold Large Text: SGL Diffusion" +``` + +NOTE: Some model-specific integrations also accept a quantized repo or local +directory directly as `--model-path`, but that is a compatibility path. If a +repo contains multiple candidate checkpoints, pass +`--transformer-weights-path` explicitly. + +## Quant Families + +Here, `quant_family` means a checkpoint and loading family with shared CLI +usage and loader behavior. It is not just the numeric precision or a kernel +backend. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
quant_familycheckpoint formcanonical CLIsupported modelsextra dependencyplatform / notes
fp8Quantized transformer component folder, or safetensors with quantization_config metadata--transformer-path or --transformer-weights-pathALLNoneComponent-folder and single-file flows are both supported
modelopt-fp8Converted ModelOpt FP8 transformer directory or repo with config.json--transformer-pathFLUX.1, FLUX.2, Wan2.2NoneSerialized config stays quant_method=modelopt with quant_algo=FP8; dit_layerwise_offload is supported and dit_cpu_offload stays disabled
modelopt-nvfp4Mixed transformer directory/repo with config.json, or raw NVFP4 safetensors export/repo--transformer-path for mixed overrides; --transformer-weights-path for raw exportsFLUX.1, FLUX.2, Wan2.2NoneMixed override repos keep the base model separate; raw exports such as black-forest-labs/FLUX.2-dev-NVFP4 still use the weights-path flow
nunchaku-svdqPre-quantized Nunchaku transformer weights, usually named svdq-{int4\|fp4}_r{rank}-...--transformer-weights-pathModel-specific support such as Qwen-Image, FLUX, and Z-ImagenunchakuSGLang can infer precision and rank from the filename and supports both int4 and nvfp4
msmodelslimPre-quantized msmodelslim transformer weights--model-pathWan2.2 familyNoneCurrently only compatible with the Ascend NPU family and supports both w8a8 and w4a4
+ +## Validated ModelOpt Checkpoints + +This section is the canonical support matrix for the six diffusion ModelOpt +checkpoints currently wired up in SGLang docs and B200 CI coverage. + +Published checkpoints keep the serialized quantization config as +`quant_method=modelopt`; the FP8 vs NVFP4 split below is a documentation label +derived from `quant_algo`. + +Five of the six repos live under `BBuf/*`. The FLUX.2 NVFP4 entry keeps the +official `black-forest-labs/FLUX.2-dev-NVFP4` repo. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Quant AlgoBase ModelPreferred CLIHF RepoCurrent ScopeNotes
FP8black-forest-labs/FLUX.1-dev--transformer-pathBBuf/flux1-dev-modelopt-fp8-sglang-transformersingle-transformer override, deterministic latent/image comparison, H100 benchmark, torch-profiler traceSGLang converter keeps a validated BF16 fallback set for modulation and FF projection layers; use --model-id FLUX.1-dev for local mirrors
FP8black-forest-labs/FLUX.2-dev--transformer-pathBBuf/flux2-dev-modelopt-fp8-sglang-transformersingle-transformer override load and generation pathpublished SGLang-ready transformer override
FP8Wan-AI/Wan2.2-T2V-A14B-Diffusers--transformer-pathBBuf/wan22-t2v-a14b-modelopt-fp8-sglang-transformerprimary transformer quantized, transformer_2 kept BF16primary-transformer-only path; keep transformer_2 on the base checkpoint, and do not describe this as dual-transformer full-model FP8 unless that path is validated separately
NVFP4black-forest-labs/FLUX.1-dev--transformer-pathBBuf/flux1-dev-modelopt-nvfp4-sglang-transformermixed BF16+NVFP4 transformer override, correctness validation, 4x RTX 5090 benchmark, torch-profiler traceuse build_modelopt_nvfp4_transformer.py; validated builder keeps selected FLUX.1 modules in BF16 and sets swap_weight_nibbles=false
NVFP4black-forest-labs/FLUX.2-dev--transformer-weights-pathblack-forest-labs/FLUX.2-dev-NVFP4packed-QKV load pathofficial raw export repo; validated packed export detection and runtime layout handling
NVFP4Wan-AI/Wan2.2-T2V-A14B-Diffusers--transformer-pathBBuf/wan22-t2v-a14b-modelopt-nvfp4-sglang-transformerprimary transformer quantized with ModelOpt NVFP4, transformer_2 kept BF16primary-transformer-only path; keep transformer_2 on the base checkpoint, and current B200/Blackwell bring-up uses SGLANG_DIFFUSION_FLASHINFER_FP4_GEMM_BACKEND=cudnn
+ +These six checkpoints are also the intended case set for the B200 diffusion CI +job (`multimodal-gen-test-1-b200`). + +## ModelOpt FP8 + +### Usage Examples + +Converted ModelOpt FP8 checkpoints should be loaded as transformer component +overrides. If the repo or local directory already contains `config.json`, use +`--transformer-path`. + +```bash +sglang generate \ + --model-path black-forest-labs/FLUX.2-dev \ + --transformer-path BBuf/flux2-dev-modelopt-fp8-sglang-transformer \ + --prompt "A Logo With Bold Large Text: SGL Diffusion" \ + --save-output +``` + +```bash +sglang generate \ + --model-path Wan-AI/Wan2.2-T2V-A14B-Diffusers \ + --transformer-path BBuf/wan22-t2v-a14b-modelopt-fp8-sglang-transformer \ + --prompt "a fox walking through neon rain" \ + --save-output +``` + +### Notes + +- `--transformer-path` is the canonical flag for converted ModelOpt FP8 + transformer component repos or directories that already carry `config.json`. +- If the override repo or local directory contains its own `config.json`, + SGLang reads the quantization config from that override instead of relying on + the base model config. +- `--transformer-weights-path` still works when you intentionally point at raw + weight files or a directory that should be metadata-probed as weights first. +- `dit_layerwise_offload` is supported for ModelOpt FP8 checkpoints. +- `dit_cpu_offload` still stays disabled for ModelOpt FP8 checkpoints. +- The layerwise offload path now preserves the non-contiguous FP8 weight stride + expected by the runtime FP8 GEMM path. +- On disk, the quantization config stays `quant_method=modelopt` with + `quant_algo=FP8`; the `modelopt-fp8` label in this document is a support + family name, not a serialized config key. +- To build the converted checkpoint yourself from a ModelOpt diffusers export, + use `python -m sglang.multimodal_gen.tools.build_modelopt_fp8_transformer`. + +## ModelOpt NVFP4 + +### Usage Examples + +For mixed ModelOpt NVFP4 transformer overrides that already contain +`config.json`, keep the base model and quantized transformer separate and use +`--transformer-path`: + +```bash +sglang generate \ + --model-path black-forest-labs/FLUX.1-dev \ + --transformer-path BBuf/flux1-dev-modelopt-nvfp4-sglang-transformer \ + --prompt "A Logo With Bold Large Text: SGL Diffusion" \ + --save-output +``` + +For raw NVFP4 exports such as the official FLUX.2 release, use +`--transformer-weights-path`: + +```bash +sglang generate \ + --model-path black-forest-labs/FLUX.2-dev \ + --transformer-weights-path black-forest-labs/FLUX.2-dev-NVFP4 \ + --prompt "A Logo With Bold Large Text: SGL Diffusion" \ + --save-output +``` + +SGLang also supports passing the NVFP4 repo or local directory directly as +`--model-path`: + +```bash +sglang generate \ + --model-path black-forest-labs/FLUX.2-dev-NVFP4 \ + --prompt "A Logo With Bold Large Text: SGL Diffusion" \ + --save-output +``` + +For a dual-transformer Wan2.2 export where only the primary `transformer` +was quantized: + +```bash +SGLANG_DIFFUSION_FLASHINFER_FP4_GEMM_BACKEND=cudnn \ +sglang generate \ + --model-path Wan-AI/Wan2.2-T2V-A14B-Diffusers \ + --transformer-path BBuf/wan22-t2v-a14b-modelopt-nvfp4-sglang-transformer \ + --prompt "a fox walking through neon rain" \ + --save-output +``` + +### Notes + +- Use `--transformer-path` for mixed ModelOpt NVFP4 transformer repos or local + directories that already include `config.json`. +- Use `--transformer-weights-path` for raw NVFP4 exports, individual + safetensors files, or repo layouts that should be treated as weights first. +- For dual-transformer pipelines such as `Wan2.2-T2V-A14B-Diffusers`, the + primary `--transformer-path` override targets only `transformer`. Use a + per-component override such as `--transformer-2-path` only when you + intentionally want a non-default `transformer_2`. +- On Blackwell, the validated Wan2.2 ModelOpt NVFP4 path currently prefers + FlashInfer FP4 GEMM via + `SGLANG_DIFFUSION_FLASHINFER_FP4_GEMM_BACKEND=cudnn`. +- This environment-variable override is a current workaround for NVFP4 cases + where the default sglang JIT/CUTLASS `sm100` path rejects a large-M shape at + `can_implement()`. The intended long-term fix is to add a validated CUTLASS + fallback for those shapes rather than rely on the override. +- Direct `--model-path` loading is a compatibility path for FLUX.2 NVFP4-style + repos or local directories. +- If `--transformer-weights-path` is provided explicitly, it takes precedence + over the compatibility `--model-path` flow. +- For local directories, SGLang first looks for `*-mixed.safetensors`, then + falls back to loading from the directory. +- To force the generic diffusion ModelOpt FP4 path onto a specific FlashInfer + backend, set `SGLANG_DIFFUSION_FLASHINFER_FP4_GEMM_BACKEND`. Supported values + include `flashinfer_cudnn`, `flashinfer_cutlass`, and `flashinfer_trtllm`. +- On disk, the quantization config stays `quant_method=modelopt` with + `quant_algo=NVFP4`; the `modelopt-nvfp4` label here is again a documentation + family name rather than a serialized config key. + +## Nunchaku (SVDQuant) + +### Install + +Install the runtime dependency first: + +```bash +pip install nunchaku +``` + +For platform-specific installation methods and troubleshooting, see the +[Nunchaku installation guide](https://nunchaku.tech/docs/nunchaku/installation/installation.html). + +### File Naming and Auto-Detection + +For Nunchaku checkpoints, `--model-path` should still point to the original +base model, while `--transformer-weights-path` points to the quantized +transformer weights. + +If the basename of `--transformer-weights-path` contains the pattern +`svdq-(int4|fp4)_r{rank}`, SGLang will automatically: +- enable SVDQuant +- infer `--quantization-precision` +- infer `--quantization-rank` + +Examples: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
checkpoint name fragmentinferred precisioninferred ranknotes
svdq-int4_r32int432Standard INT4 checkpoint
svdq-int4_r128int4128Higher-quality INT4 checkpoint
svdq-fp4_r32nvfp432fp4 in the filename maps to CLI value nvfp4
svdq-fp4_r128nvfp4128Higher-quality NVFP4 checkpoint
+ +Common filenames: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
filenameprecisionranktypical use
svdq-int4_r32-qwen-image.safetensorsint432Balanced default
svdq-int4_r128-qwen-image.safetensorsint4128Quality-focused
svdq-fp4_r32-qwen-image.safetensorsnvfp432RTX 50-series / NVFP4 path
svdq-fp4_r128-qwen-image.safetensorsnvfp4128Quality-focused NVFP4
svdq-int4_r32-qwen-image-lightningv1.0-4steps.safetensorsint432Lightning 4-step
svdq-int4_r128-qwen-image-lightningv1.1-8steps.safetensorsint4128Lightning 8-step
+ +If your checkpoint name does not follow this convention, pass +`--enable-svdquant`, `--quantization-precision`, and `--quantization-rank` +explicitly. + +### Usage Examples + +Recommended auto-detected flow: + +```bash +sglang generate \ + --model-path Qwen/Qwen-Image \ + --transformer-weights-path /path/to/svdq-int4_r32-qwen-image.safetensors \ + --prompt "a beautiful sunset" \ + --save-output +``` + +Manual override when the filename does not encode the quant settings: + +```bash +sglang generate \ + --model-path Qwen/Qwen-Image \ + --transformer-weights-path /path/to/custom_nunchaku_checkpoint.safetensors \ + --enable-svdquant \ + --quantization-precision int4 \ + --quantization-rank 128 \ + --prompt "a beautiful sunset" \ + --save-output +``` + +### Notes + +- `--transformer-weights-path` is the canonical flag for Nunchaku checkpoints. + Older config names such as `quantized_model_path` are treated as + compatibility aliases. +- Auto-detection only happens when the checkpoint basename matches + `svdq-(int4|fp4)_r{rank}`. +- The CLI values are `int4` and `nvfp4`. In filenames, the NVFP4 variant is + written as `fp4`. +- Lightning checkpoints usually expect matching `--num-inference-steps`, such + as `4` or `8`. +- Current runtime validation only allows Nunchaku on NVIDIA CUDA Ampere (SM8x) + or SM12x GPUs. Hopper (SM90) is currently rejected. + +## [ModelSlim](https://gitcode.com/Ascend/msmodelslim) +MindStudio-ModelSlim (msModelSlim) is a model offline quantization compression tool launched by MindStudio and optimized for Ascend hardware. + +- **Installation** + + ```bash + # Clone repo and install msmodelslim: + git clone https://gitcode.com/Ascend/msmodelslim.git + cd msmodelslim + bash install.sh + ``` + +- **Multimodal_sd quantization** + + Download the original floating-point weights of the large model. Taking Wan2.2-T2V-A14B as an example, you can go to [Wan2.2-T2V-A14B](https://modelscope.cn/models/Wan-AI/Wan2.2-T2V-A14B) to obtain the original model weights. Then install other dependencies (related to the model, refer to the modelscope model card). + > Note: You can find pre-quantized validated models on [modelscope/Eco-Tech](https://modelscope.cn/models/Eco-Tech). + + Run quantization using one-click quantization (recommended): + + ```bash + msmodelslim quant \ + --model_path /path/to/wan2_2_float_weights \ + --save_path /path/to/wan2_2_quantized_weights \ + --device npu \ + --model_type Wan2_2 \ + --quant_type w8a8 \ + --trust_remote_code True + ``` + + For more detailed examples of quantization of models, as well as information about their support, see the [examples](https://gitcode.com/Ascend/msmodelslim/blob/master/example/multimodal_sd/README.md) section in ModelSLim repo. + + > Note: SGLang does not support quantized embeddings, please disable this option when quantizing using msmodelslim. + +- **Auto-Detection and different formats** + + For msmodelslim checkpoints, it's enough to specify only ```--model-path```, the detection of quantization occurs automatically for each layer using parsing of `quant_model_description.json` config. + + In the case of `Wan2.2` only `Diffusers` weights storage format are supported, whereas modelslim saves the quantized model in the original `Wan2.2` format, + for conversion in use `python/sglang/multimodal_gen/tools/wan_repack.py` script: + + ```bash + python wan_repack.py \ + --input-path {path_to_quantized_model} \ + --output-path {path_to_converted_model} + ``` + + After that, please copy all files from original `Diffusers` checkpoint (instead of `transformer`/`tranfsormer_2` folders) + +- **Usage Example** + + With auto-detected flow: + + ```bash + sglang generate \ + --model-path Eco-Tech/Wan2.2-T2V-A14B-Diffusers-w8a8 \ + --prompt "a beautiful sunset" \ + --save-output + ``` + +- **Available Quantization Methods**: + - [x] ```W4A4_DYNAMIC``` linear with online quantization of activations + - [x] ```W8A8``` linear with offline quantization of activations + - [x] ```W8A8_DYNAMIC``` linear with online quantization of activations + - [ ] ```mxfp8``` linear in progress diff --git a/docs_new/docs/sglang-diffusion/ring_sp_performance.mdx b/docs_new/docs/sglang-diffusion/ring_sp_performance.mdx new file mode 100644 index 000000000..eca61967a --- /dev/null +++ b/docs_new/docs/sglang-diffusion/ring_sp_performance.mdx @@ -0,0 +1,158 @@ +--- +title: "Ring SP Benchmark: Wan2.2-TI2V-5B (u1r2 vs Baseline)" +metatags: + description: "Review Ring-SP benchmark results for Wan2.2-TI2V-5B-Diffusers in SGLang Diffusion." +--- + +This page reports Ring-SP performance for `Wan2.2-TI2V-5B-Diffusers` using: + +- Parallel config: `sp=2, ulysses=1, ring=2` (short: `u1r2`) +- Baseline config: `sp=1, ulysses=1, ring=1` (short: `u1r1`) + +## Benchmark Setup + +- Model: `Wan2.2-TI2V-5B-Diffusers` +- GPU: `48G RTX40 series * 2` + +## Online Serving + +### Ring SP (`u1r2`) + +```bash +sglang serve \ + --model-type diffusion \ + --model-path /model/HuggingFace/Wan-AI/Wan2.2-TI2V-5B-Diffusers \ + --num-gpus 2 --sp-degree 2 --ulysses-degree 1 --ring-degree 2 \ + --port 8898 +``` + +### Baseline (`u1r1`) + +```bash +sglang serve \ + --model-type diffusion \ + --model-path /model/HuggingFace/Wan-AI/Wan2.2-TI2V-5B-Diffusers \ + --num-gpus 1 --sp-degree 1 --ulysses-degree 1 --ring-degree 1 \ + --port 8898 +``` + +## Benchmarks + +### Benchmark Disclaimer + +These benchmarks are provided for reference under one specific setup and command configuration. Actual performance may vary with model settings, runtime environment, and request patterns. + +### Stage Time Breakdown + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Stage / Metricu1r2 (s)u1r1 baseline (s)Speedup
InputValidation0.10600.10290.97x
TextEncoding1.39652.22611.59x
LatentPreparation0.00020.00021.00x
TimestepPreparation0.00030.00041.33x
Denoising52.635871.67851.36x
Decoding7.670813.43141.75x
Total63.7490.631.42x
+ +### Memory Usage + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Memory Metricu1r2 (GB)u1r1 baseline (GB)Delta
Peak GPU Memory20.0727.40-7.33
Peak Allocated13.3520.40-7.05
Memory Overhead6.727.00-0.28
Overhead Ratio33.5%25.6%+7.9pp
+ +## Summary + +- End-to-end latency improves from `90.63s` to `63.74s` (`1.42x`). +- Main gains come from `Denoising` (`1.36x`) and `Decoding` (`1.75x`). +- Absolute memory usage drops noticeably on Ring-SP (`Peak GPU Memory -7.33GB`, `Peak Allocated -7.05GB`). +- Overhead ratio rises (`+7.9pp`), so future tuning can focus on reducing communication/runtime overhead while preserving the latency gain. diff --git a/docs_new/docs/sglang-diffusion/support_new_models.mdx b/docs_new/docs/sglang-diffusion/support_new_models.mdx new file mode 100644 index 000000000..766e24b82 --- /dev/null +++ b/docs_new/docs/sglang-diffusion/support_new_models.mdx @@ -0,0 +1,601 @@ +--- +title: "How to Support New Diffusion Models" +metatags: + description: "This document explains how to add support for new diffusion models in SGLang Diffusion." +--- + +This document explains how to add support for new diffusion models in SGLang Diffusion. + +## Architecture Overview + +SGLang Diffusion is engineered for both performance and flexibility, built upon a pipeline architecture. This +design allows developers to construct pipelines for various diffusion models while keeping the core generation +loop standardized for optimization. + +At its core, the architecture revolves around two key concepts, as highlighted in our [blog post](https://lmsys.org/blog/2025-11-07-sglang-diffusion/#architecture): + +- **`ComposedPipeline`**: This class orchestrates a series of `PipelineStage`s to define the complete generation process for a specific model. It acts as the main entry point for a model and manages the data flow between the different stages of the diffusion process. +- **`PipelineStage`**: Each stage is a modular component that encapsulates a function within the diffusion process. Examples include prompt encoding, the denoising loop, or VAE decoding. + +### Two Pipeline Styles + +SGLang Diffusion supports two pipeline composition styles. Both are valid; choose the one that best fits your model. + +#### Style A: Hybrid Monolithic Pipeline (Recommended Default) + +The recommended default for most new models. Uses a three-stage structure: + +``` +BeforeDenoisingStage (model-specific) → DenoisingStage (standard) → DecodingStage (standard) +``` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
StageOwnershipResponsibility
{Model}BeforeDenoisingStageModel-specificAll pre-processing: input validation, text/image encoding, latent preparation, timestep computation
DenoisingStageFramework-standardThe denoising loop (DiT/UNet forward passes), shared across all models
DecodingStageFramework-standardVAE decoding from latent space to pixel space, shared across all models
+ +**Why recommended?** Modern diffusion models often have highly heterogeneous pre-processing requirements — different text encoders, different latent formats, different conditioning mechanisms. The Hybrid approach keeps pre-processing isolated per model, avoids fragile shared stages with excessive conditional logic, and lets developers port Diffusers reference code quickly. + +#### Style B: Modular Composition Style + +Uses the framework's fine-grained standard stages (`TextEncodingStage`, `LatentPreparationStage`, `TimestepPreparationStage`, etc.) to build the pipeline by composition. Convenience methods like `add_standard_t2i_stages()` and `add_standard_ti2i_stages()` make this very concise. + +This style is appropriate when: +- **The new model's pre-processing can largely reuse existing stages** — e.g., a model that uses standard CLIP/T5 text encoding + standard latent preparation with minimal customization. +- **A model-specific optimization needs to be extracted as a standalone stage** — e.g., a specialized encoding or conditioning step that benefits from being a separate stage for profiling, parallelism control, or reuse across multiple pipeline variants. + +#### How to Choose + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
SituationRecommended Style
Model has unique/complex pre-processing (VLM captioning, AR token generation, custom latent packing, etc.)Hybrid — consolidate into a BeforeDenoisingStage
Model fits neatly into standard text-to-image or text+image-to-image patternModular — use add_standard_t2i_stages() / add_standard_ti2i_stages()
Porting a Diffusers pipeline with many custom stepsHybrid — copy the __call__ logic into a single stage
Adding a variant of an existing model that shares most logicModular — reuse existing stages, customize via PipelineConfig callbacks
A specific pre-processing step needs special parallelism or profiling isolationModular — extract that step as a dedicated stage
+ +## Key Components for Implementation + +To add support for a new diffusion model, you will need to define or configure the following components: + +1. **`PipelineConfig`**: A dataclass holding static configurations for your model pipeline — precision settings, model architecture parameters, and callback methods used by the standard `DenoisingStage` and `DecodingStage`. Each model has its own subclass. + +2. **`SamplingParams`**: A dataclass defining runtime generation parameters — `prompt`, `negative_prompt`, `guidance_scale`, `num_inference_steps`, `seed`, `height`, `width`, etc. + +3. **Pre-processing stage(s)**: Either a single model-specific `{Model}BeforeDenoisingStage` (Hybrid style) or a combination of standard stages (Modular style). See [Two Pipeline Styles](#two-pipeline-styles) above. + +4. **`ComposedPipeline`**: A class that wires together your pre-processing stage(s) with the standard `DenoisingStage` and `DecodingStage`. See base definitions: + - [`ComposedPipelineBase`](https://github.com/sgl-project/sglang/blob/main/python/sglang/multimodal_gen/runtime/pipelines_core/composed_pipeline_base.py) + - [`PipelineStage`](https://github.com/sgl-project/sglang/blob/main/python/sglang/multimodal_gen/runtime/pipelines_core/stages/base.py) + - [Central registry](https://github.com/sgl-project/sglang/blob/main/python/sglang/multimodal_gen/registry.py) + +5. **Modules (model components)**: Each pipeline references modules loaded from the model repository (e.g., Diffusers `model_index.json`): + - `text_encoder`: Encodes text prompts into embeddings. + - `tokenizer`: Tokenizes raw text input for the text encoder(s). + - `processor`: Preprocesses images and extracts features; often used in image-to-image tasks. + - `image_encoder`: Specialized image feature extractor. + - `dit/transformer`: The core denoising network (DiT/UNet architecture) operating in latent space. + - `scheduler`: Controls the timestep schedule and denoising dynamics. + - `vae`: Variational Autoencoder for encoding/decoding between pixel space and latent space. + +## Pipeline Stages Reference + +### Core Stages (used by all pipelines) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Stage ClassDescription
DenoisingStageExecutes the main denoising loop, iteratively applying the model (DiT/UNet) to refine the latents.
DecodingStageDecodes the final latent tensor back into pixel space using the VAE.
DmdDenoisingStageA specialized denoising stage for DMD model architectures.
CausalDMDDenoisingStageA specialized causal denoising stage for specific video models.
+ +### Pre-processing Stages (for Modular Composition Style) + +The following fine-grained stages can be composed to build the pre-processing portion of a pipeline. They are best suited for models whose pre-processing largely fits the standard patterns. If your model requires significant customization, consider the Hybrid style with a single `BeforeDenoisingStage` instead. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Stage ClassDescription
InputValidationStageValidates user-provided SamplingParams.
TextEncodingStageEncodes text prompts into embeddings using one or more text encoders.
ImageEncodingStageEncodes input images into embeddings, often used in image-to-image tasks.
ImageVAEEncodingStageEncodes an input image into latent space using the VAE.
TimestepPreparationStagePrepares the scheduler's timesteps for the diffusion process.
LatentPreparationStageCreates the initial noisy latent tensor that will be denoised.
+ +## Implementation Guide + +### Step 1: Obtain and Study the Reference Implementation + +Before writing any code, obtain the model's original implementation or Diffusers pipeline code: +- The model's Diffusers pipeline source (e.g., the `pipeline_*.py` file from the `diffusers` library or HuggingFace repo) +- Or the model's official reference implementation (e.g., from the model author's GitHub repo) +- Or the HuggingFace model ID to look up `model_index.json` and the associated pipeline class + +Once you have the reference code, study it thoroughly: + +1. Find the model's `model_index.json` to identify required modules. +2. Read the Diffusers pipeline's `__call__` method to understand: + - How text prompts are encoded + - How latents are prepared (shape, dtype, scaling) + - How timesteps/sigmas are computed + - What conditioning kwargs the DiT expects + - How the denoising loop works + - How VAE decoding is done + +### Step 2: Evaluate Reuse of Existing Pipelines and Stages + +Before creating any new files, check whether an existing pipeline or stage can be reused or extended. Only create new pipelines/stages when the existing ones would need substantial structural changes or when no architecturally similar implementation exists. + +- **Compare against existing pipelines** (Flux, Wan, Qwen-Image, GLM-Image, HunyuanVideo, LTX, etc.). If the new model shares most of its structure with an existing one, prefer adding a new config variant or reusing existing stages. +- **Check existing stages** in `runtime/pipelines_core/stages/` and `stages/model_specific_stages/`. +- **Check existing model components** — many models share VAEs (e.g., `AutoencoderKL`), text encoders (CLIP, T5), and schedulers. Reuse these directly. + +### Step 3: Implement Model Components + +Adapt the model's core components: + +- **DiT/Transformer**: Implement in [`runtime/models/dits/`](https://github.com/sgl-project/sglang/blob/main/python/sglang/multimodal_gen/runtime/models/dits/) +- **Encoders**: Implement in [`runtime/models/encoders/`](https://github.com/sgl-project/sglang/blob/main/python/sglang/multimodal_gen/runtime/models/encoders/) +- **VAEs**: Implement in [`runtime/models/vaes/`](https://github.com/sgl-project/sglang/blob/main/python/sglang/multimodal_gen/runtime/models/vaes/) +- **Schedulers**: Implement in [`runtime/models/schedulers/`](https://github.com/sgl-project/sglang/blob/main/python/sglang/multimodal_gen/runtime/models/schedulers/) if needed + +Use SGLang's fused kernels where possible (see `LayerNormScaleShift`, `RMSNormScaleShift`, `apply_qk_norm`, etc.). + +**Tensor Parallel (TP) and Sequence Parallel (SP)**: For multi-GPU deployment, it is recommended to add TP/SP support to the DiT model. This can be done incrementally after the single-GPU implementation is verified. Reference implementations: +- **Wan model** (`runtime/models/dits/wanvideo.py`) — Full TP + SP: `ColumnParallelLinear`/`RowParallelLinear` for attention, sequence dimension sharding via `get_sp_world_size()` +- **Qwen-Image model** (`runtime/models/dits/qwen_image.py`) — SP via `USPAttention` (Ulysses + Ring Attention) + +### Step 4: Create Configs + +- **DiT Config**: `configs/models/dits/{model_name}.py` +- **VAE Config**: `configs/models/vaes/{model_name}.py` +- **SamplingParams**: `configs/sample/{model_name}.py` + +### Step 5: Create PipelineConfig + +The `PipelineConfig` provides callbacks that the standard `DenoisingStage` and `DecodingStage` use: + +```python +# python/sglang/multimodal_gen/configs/pipeline_configs/my_model.py + +@dataclass +class MyModelPipelineConfig(ImagePipelineConfig): + task_type: ModelTaskType = ModelTaskType.T2I + vae_precision: str = "bf16" + should_use_guidance: bool = True + dit_config: DiTConfig = field(default_factory=MyModelDitConfig) + vae_config: VAEConfig = field(default_factory=MyModelVAEConfig) + + def get_freqs_cis(self, batch, device, rotary_emb, dtype): + """Prepare rotary position embeddings for the DiT.""" + ... + + def prepare_pos_cond_kwargs(self, batch, latent_model_input, t, **kwargs): + """Build positive conditioning kwargs for each denoising step.""" + return { + "hidden_states": latent_model_input, + "encoder_hidden_states": batch.prompt_embeds[0], + "timestep": t, + } + + def prepare_neg_cond_kwargs(self, batch, latent_model_input, t, **kwargs): + """Build negative conditioning kwargs for CFG.""" + return { + "hidden_states": latent_model_input, + "encoder_hidden_states": batch.negative_prompt_embeds[0], + "timestep": t, + } + + def get_decode_scale_and_shift(self): + """Return (scale, shift) for latent denormalization before VAE decode.""" + ... +``` + +### Step 6: Implement Pre-processing + +Choose based on your model's needs (see [How to Choose](#how-to-choose)): + +#### Option A: BeforeDenoisingStage (Hybrid Style) + +Create a single stage that handles all pre-processing. Best when the model has custom/complex pre-processing logic. + +```python +# python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/my_model.py + +class MyModelBeforeDenoisingStage(PipelineStage): + """Monolithic pre-processing stage for MyModel. + + Consolidates: input validation, text/image encoding, latent + preparation, and timestep computation. + """ + + def __init__(self, vae, text_encoder, tokenizer, transformer, scheduler): + super().__init__() + self.vae = vae + self.text_encoder = text_encoder + self.tokenizer = tokenizer + self.transformer = transformer + self.scheduler = scheduler + + @torch.no_grad() + def forward(self, batch: Req, server_args: ServerArgs) -> Req: + device = get_local_torch_device() + + # 1. Encode prompt (model-specific logic) + prompt_embeds, negative_prompt_embeds = self._encode_prompt(...) + + # 2. Prepare latents + latents = self._prepare_latents(...) + + # 3. Prepare timesteps + timesteps, sigmas = self._prepare_timesteps(...) + + # 4. Populate batch for DenoisingStage + batch.prompt_embeds = [prompt_embeds] + batch.negative_prompt_embeds = [negative_prompt_embeds] + batch.latents = latents + batch.timesteps = timesteps + batch.num_inference_steps = len(timesteps) + batch.sigmas = sigmas.tolist() + batch.generator = generator + batch.raw_latent_shape = latents.shape + return batch +``` + +#### Option B: Standard Stages (Modular Style) + +Skip creating a custom stage entirely — configure via `PipelineConfig` callbacks and use framework helpers. Best when the model fits standard patterns. + +(This option has no separate stage file; the pipeline class in Step 7 calls `add_standard_t2i_stages()` directly.) + +**Key batch fields that `DenoisingStage` expects** (regardless of which option you choose): + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldTypeDescription
batch.latentstorch.TensorInitial noisy latent tensor
batch.timestepstorch.TensorTimestep schedule
batch.num_inference_stepsintNumber of denoising steps
batch.sigmaslist[float]Sigma schedule (must be a Python list, not numpy)
batch.prompt_embedslist[torch.Tensor]Positive prompt embeddings (wrapped in a list)
batch.negative_prompt_embedslist[torch.Tensor]Negative prompt embeddings (wrapped in a list)
batch.generatortorch.GeneratorRNG generator for reproducibility
batch.raw_latent_shapetupleOriginal latent shape before any packing
+ +### Step 7: Define the Pipeline Class + +#### Hybrid Style + +```python +# python/sglang/multimodal_gen/runtime/pipelines/my_model.py + +class MyModelPipeline(LoRAPipeline, ComposedPipelineBase): + pipeline_name = "MyModelPipeline" # Must match model_index.json _class_name + + _required_config_modules = [ + "text_encoder", "tokenizer", "vae", "transformer", "scheduler", + ] + + def create_pipeline_stages(self, server_args: ServerArgs): + # 1. Monolithic pre-processing (model-specific) + self.add_stage( + MyModelBeforeDenoisingStage( + vae=self.get_module("vae"), + text_encoder=self.get_module("text_encoder"), + tokenizer=self.get_module("tokenizer"), + transformer=self.get_module("transformer"), + scheduler=self.get_module("scheduler"), + ), + ) + + # 2. Standard denoising loop (framework-provided) + self.add_stage( + DenoisingStage( + transformer=self.get_module("transformer"), + scheduler=self.get_module("scheduler"), + ), + ) + + # 3. Standard VAE decoding (framework-provided) + self.add_standard_decoding_stage() + + +EntryClass = [MyModelPipeline] +``` + +#### Modular Style + +```python +# python/sglang/multimodal_gen/runtime/pipelines/my_model.py + +class MyModelPipeline(LoRAPipeline, ComposedPipelineBase): + pipeline_name = "MyModelPipeline" + + _required_config_modules = [ + "text_encoder", "tokenizer", "vae", "transformer", "scheduler", + ] + + def create_pipeline_stages(self, server_args: ServerArgs): + # All pre-processing + denoising + decoding in one call + self.add_standard_t2i_stages( + prepare_extra_timestep_kwargs=[prepare_mu], # model-specific hooks + ) + + +EntryClass = [MyModelPipeline] +``` + +### Step 8: Register the Model + +Register your configs in [`registry.py`](https://github.com/sgl-project/sglang/blob/main/python/sglang/multimodal_gen/registry.py): + +```python +register_configs( + model_family="my_model", + sampling_param_cls=MyModelSamplingParams, + pipeline_config_cls=MyModelPipelineConfig, + hf_model_paths=["org/my-model-name"], +) +``` + +The `EntryClass` in your pipeline file is automatically discovered by the registry — no additional registration needed for the pipeline class itself. + +### Step 9: Verify Output Quality + +After implementation, verify that the generated output is not noise. A noisy or garbled output is the most common sign of an incorrect implementation. Common causes include: + +- Incorrect latent scale/shift factors +- Wrong timestep/sigma schedule (order, dtype, or value range) +- Mismatched conditioning kwargs +- Rotary embedding style mismatch (`is_neox_style`) + +Debug by comparing intermediate tensor values against the Diffusers reference pipeline with the same seed. + +## Reference Implementations + +### Hybrid Style + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ModelPipelineBeforeDenoisingStagePipelineConfig
GLM-Imageruntime/pipelines/glm_image.pystages/model_specific_stages/glm_image.pyconfigs/pipeline_configs/glm_image.py
Qwen-Image-Layeredruntime/pipelines/qwen_image.pystages/model_specific_stages/qwen_image_layered.pyconfigs/pipeline_configs/qwen_image.py
+ +### Modular Style + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ModelPipelineNotes
Qwen-Image (T2I)runtime/pipelines/qwen_image.pyUses add_standard_t2i_stages()
Qwen-Image-Editruntime/pipelines/qwen_image.pyUses add_standard_ti2i_stages()
Fluxruntime/pipelines/flux.pyUses add_standard_t2i_stages() with custom prepare_mu
Wanruntime/pipelines/wan_pipeline.pyUses add_standard_ti2v_stages()
+ +## Checklist + +Before submitting your implementation, verify: + +**Common (both styles):** +- [ ] **Pipeline file** at `runtime/pipelines/{model_name}.py` with `EntryClass` +- [ ] **PipelineConfig** at `configs/pipeline_configs/{model_name}.py` +- [ ] **SamplingParams** at `configs/sample/{model_name}.py` +- [ ] **DiT model** at `runtime/models/dits/{model_name}.py` +- [ ] **Model configs** (DiT, VAE) at `configs/models/dits/` and `configs/models/vaes/` +- [ ] **Registry entry** in `registry.py` via `register_configs()` +- [ ] `pipeline_name` matches Diffusers `model_index.json` `_class_name` +- [ ] `_required_config_modules` lists all modules from `model_index.json` +- [ ] `PipelineConfig` callbacks (`prepare_pos_cond_kwargs`, etc.) match the DiT's `forward()` signature +- [ ] Uses framework-standard `DenoisingStage` and `DecodingStage` (not custom denoising loops) +- [ ] **TP/SP support** considered for DiT model (recommended; reference `wanvideo.py` for TP+SP, `qwen_image.py` for USPAttention) +- [ ] **Output quality verified** — generated images/videos are not noise; compared against Diffusers reference output + +**Hybrid style only:** +- [ ] **BeforeDenoisingStage** at `stages/model_specific_stages/{model_name}.py` +- [ ] `BeforeDenoisingStage.forward()` populates all batch fields required by `DenoisingStage` diff --git a/docs_new/docs/sglang-diffusion/supported-models.mdx b/docs_new/docs/sglang-diffusion/supported-models.mdx deleted file mode 100644 index 3af2f89ad..000000000 --- a/docs_new/docs/sglang-diffusion/supported-models.mdx +++ /dev/null @@ -1,350 +0,0 @@ ---- -title: "Supported Models" -description: "Check model compatibility across diffusion optimizations and backends." ---- - -The table below shows every supported model and the optimizations supported for them. - -Key: - -- `Yes` = Compatible -- `No` = Incompatible -- `N/A` = Not applicable - -## Models x Optimization - -The `HuggingFace Model ID` can be passed directly to `from_pretrained()` methods, and sglang-diffusion will use the optimal default parameters when initializing and generating videos. - -### Video Generation Models - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Model NameHuggingFace Model IDResolutionsTeaCacheSliding Tile AttnSage AttnVideo Sparse Attention (VSA)Sparse Linear Attention (SLA)Sage Sparse Linear Attention (SageSLA)Sparse Video Gen 2 (SVG2)
FastWan2.1 T2V 1.3B`FastVideo/FastWan2.1-T2V-1.3B-Diffusers`480pN/AN/AN/AYesNoNoNo
FastWan2.2 TI2V 5B Full Attn`FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers`720pN/AN/AN/AYesNoNoNo
Wan2.2 TI2V 5B`Wan-AI/Wan2.2-TI2V-5B-Diffusers`720pN/AN/AYesN/ANoNoNo
Wan2.2 T2V A14B`Wan-AI/Wan2.2-T2V-A14B-Diffusers`480p, 720pNoNoYesN/ANoNoNo
Wan2.2 I2V A14B`Wan-AI/Wan2.2-I2V-A14B-Diffusers`480p, 720pNoNoYesN/ANoNoNo
HunyuanVideo`hunyuanvideo-community/HunyuanVideo`720x1280, 544x960NoYesYesN/ANoNoYes
FastHunyuan`FastVideo/FastHunyuan-diffusers`720x1280, 544x960NoYesYesN/ANoNoYes
Wan2.1 T2V 1.3B`Wan-AI/Wan2.1-T2V-1.3B-Diffusers`480pYesYesYesN/ANoNoYes
Wan2.1 T2V 14B`Wan-AI/Wan2.1-T2V-14B-Diffusers`480p, 720pYesYesYesN/ANoNoYes
Wan2.1 I2V 480P`Wan-AI/Wan2.1-I2V-14B-480P-Diffusers`480pYesYesYesN/ANoNoYes
Wan2.1 I2V 720P`Wan-AI/Wan2.1-I2V-14B-720P-Diffusers`720pYesYesYesN/ANoNoYes
TurboWan2.1 T2V 1.3B`IPostYellow/TurboWan2.1-T2V-1.3B-Diffusers`480pYesNoNoNoYesYesN/A
TurboWan2.1 T2V 14B`IPostYellow/TurboWan2.1-T2V-14B-Diffusers`480pYesNoNoNoYesYesN/A
TurboWan2.1 T2V 14B 720P`IPostYellow/TurboWan2.1-T2V-14B-720P-Diffusers`720pYesNoNoNoYesYesN/A
TurboWan2.2 I2V A14B`IPostYellow/TurboWan2.2-I2V-A14B-Diffusers`720pYesNoNoNoYesYesN/A
- - -1. Wan2.2 TI2V 5B has known quality issues for some I2V workloads. -2. SageSLA is based on SpargeAttn. Install SpargeAttn first with `pip install git+https://github.com/thu-ml/SpargeAttn.git --no-build-isolation`. - - -### Image Generation Models - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Model NameHuggingFace Model IDResolutions
FLUX.1-dev`black-forest-labs/FLUX.1-dev`Any resolution
FLUX.2-dev`black-forest-labs/FLUX.2-dev`Any resolution
FLUX.2-Klein`black-forest-labs/FLUX.2-klein-4B`Any resolution
Z-Image-Turbo`Tongyi-MAI/Z-Image-Turbo`Any resolution
GLM-Image`zai-org/GLM-Image`Any resolution
Qwen Image`Qwen/Qwen-Image`Any resolution
Qwen Image 2512`Qwen/Qwen-Image-2512`Any resolution
Qwen Image Edit`Qwen/Qwen-Image-Edit`Any resolution
- -## Verified LoRA Examples - -This section lists example LoRAs that have been explicitly tested and verified with each base model in the **SGLang Diffusion** pipeline. - - -LoRAs that are not listed here are not necessarily incompatible. -In practice, most standard LoRAs are expected to work, especially those following common Diffusers or SD-style conventions. -The entries below simply reflect configurations that have been manually validated by the SGLang team. - - -### Verified LoRAs by Base Model - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Base ModelSupported LoRAs
Wan2.2`lightx2v/Wan2.2-Distill-Loras`
`Cseti/wan2.2-14B-Arcane_Jinx-lora-v1`
Wan2.1`lightx2v/Wan2.1-Distill-Loras`
Z-Image-Turbo`tarn59/pixel_art_style_lora_z_image_turbo`
`wcde/Z-Image-Turbo-DeJPEG-Lora`
Qwen-Image`lightx2v/Qwen-Image-Lightning`
`flymy-ai/qwen-image-realism-lora`
`prithivMLmods/Qwen-Image-HeadshotX`
`starsfriday/Qwen-Image-EVA-LoRA`
Qwen-Image-Edit`ostris/qwen_image_edit_inpainting`
`lightx2v/Qwen-Image-Edit-2511-Lightning`
Flux`dvyio/flux-lora-simple-illustration`
`XLabs-AI/flux-furry-lora`
`XLabs-AI/flux-RealismLora`
- -## Special requirements - -### Sliding Tile Attention - -- Currently, only Hopper GPUs (H100s) are supported. diff --git a/docs_new/docs/sglang-diffusion/tea-cache.mdx b/docs_new/docs/sglang-diffusion/teacache.mdx similarity index 98% rename from docs_new/docs/sglang-diffusion/tea-cache.mdx rename to docs_new/docs/sglang-diffusion/teacache.mdx index 78b4d3714..d7f86219f 100644 --- a/docs_new/docs/sglang-diffusion/tea-cache.mdx +++ b/docs_new/docs/sglang-diffusion/teacache.mdx @@ -4,7 +4,7 @@ description: "Configure TeaCache for temporal similarity-based diffusion acceler --- > **Note**: This is one of two caching strategies available in SGLang. -> For an overview of all caching options, see [SGLang diffusion overview](../../sglang-diffusion/intro). +> For an overview of all caching options, see [caching](./caching-acceleration). TeaCache (Temporal similarity-based caching) accelerates diffusion inference by detecting when consecutive denoising steps are similar enough to skip computation entirely. diff --git a/docs_new/docs/supported-models.mdx b/docs_new/docs/supported-models.mdx index d4f2ea757..bcfcf89a7 100644 --- a/docs_new/docs/supported-models.mdx +++ b/docs_new/docs/supported-models.mdx @@ -13,7 +13,7 @@ SGLang supports model families across text generation, retrieval, and reward wor title="Large language models" mode="card" className="max-w-sm mx-auto" - href="./supported-models/large-language-models" + href="./supported-models/generative_models" img="/cards/LLM-card.png" > Production-tuned Llama and Qwen families validated for high-throughput @@ -23,7 +23,7 @@ SGLang supports model families across text generation, retrieval, and reward wor title="Vision language models" mode="card" className="max-w-sm mx-auto" - href="./supported-models/vision-language-models" + href="./supported-models/multimodal_language_models" img="/cards/VLM-card.png" > Vision-text hybrids that stay responsive on multi-GPU setups. @@ -47,7 +47,7 @@ SGLang supports model families across text generation, retrieval, and reward wor title="Embedding models" mode="card" className="max-w-sm mx-auto" - href="./supported-models/embedding-models" + href="./supported-models/embedding_models" img="/cards/Embedding-card.png" > Dense and sparse embeddings optimized with FlashInfer kernels. @@ -56,7 +56,7 @@ SGLang supports model families across text generation, retrieval, and reward wor title="Rerank models" mode="card" className="max-w-sm mx-auto" - href="./supported-models/rerank-models" + href="./supported-models/rerank_models" img="/cards/Rerank-card.png" > Low-latency rerankers for multi-stage retrieval pipelines. @@ -65,7 +65,7 @@ SGLang supports model families across text generation, retrieval, and reward wor title="Classification models" mode="card" className="max-w-sm mx-auto" - href="./supported-models/classification-models" + href="./supported-models/classify_models" img="/cards/Classification-card.png" > Lightweight classifiers covering safety, intent, and context filters. @@ -79,7 +79,7 @@ SGLang supports model families across text generation, retrieval, and reward wor title="Reward models" mode="card" className="max-w-sm mx-auto" - href="./supported-models/reward-models" + href="./supported-models/reward_models" img="/cards/Reward-card.png" > RLHF and reward scoring pipelines optimized for production latency. diff --git a/docs_new/docs/supported-models/classification-models.mdx b/docs_new/docs/supported-models/classification-models.mdx deleted file mode 100644 index 510effe2f..000000000 --- a/docs_new/docs/supported-models/classification-models.mdx +++ /dev/null @@ -1,323 +0,0 @@ ---- -title: Classification Models ---- - -This document describes the `/v1/classify` API endpoint in SGLang, which is compatible with vLLM's classification API format. - -## Overview - -The classification API allows you to classify text inputs using classification models. This implementation follows the same format as vLLM's 0.7.0 classification API. - -## API endpoint - -```text Output -POST /v1/classify -``` - -## Request format - -```json Config -{ - "model": "model_name", - "input": "text to classify" -} -``` - -### Parameters - - - The name of the classification model to use. - - - - The text to classify. - - - - User identifier for tracking. - - - - Request ID for tracking. - - - - Request priority. - - -## Response format - -```json Config -{ - "id": "classify-9bf17f2847b046c7b2d5495f4b4f9682", - "object": "list", - "created": 1745383213, - "model": "jason9693/Qwen2.5-1.5B-apeach", - "data": [ - { - "index": 0, - "label": "Default", - "probs": [0.565970778465271, 0.4340292513370514], - "num_classes": 2 - } - ], - "usage": { - "prompt_tokens": 10, - "total_tokens": 10, - "completion_tokens": 0, - "prompt_tokens_details": null - } -} -``` - -### Response fields - - - Unique identifier for the classification request. - - - - Always `"list"`. - - - - Unix timestamp when the request was created. - - - - The model used for classification. - - - - Array of classification results. - - - - Index of the result. - - - - Predicted class label. - - - - Array of probabilities for each class. - - - - Total number of classes. - - - - - - - Token usage information. - - - - Number of input tokens. - - - - Total number of tokens. - - - - Number of completion tokens (always `0` for classification). - - - - Additional token details (optional). - - - - - -## Example usage - - - - ```bash Command - curl -v "http://127.0.0.1:8000/v1/classify" \ - -H "Content-Type: application/json" \ - -d '{ - "model": "jason9693/Qwen2.5-1.5B-apeach", - "input": "Loved the new café—coffee was great." - }' - ``` - - - ```python Example - import requests - import json - - # Make classification request - response = requests.post( - "http://127.0.0.1:8000/v1/classify", - headers={"Content-Type": "application/json"}, - json={ - "model": "jason9693/Qwen2.5-1.5B-apeach", - "input": "Loved the new café—coffee was great." - } - ) - - # Parse response - result = response.json() - print(json.dumps(result, indent=2)) - ``` - - - - -## Supported models - -The classification API works with any classification model supported by SGLang, including: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ModelType
`LlamaForSequenceClassification`Multi-class classification
`Qwen2ForSequenceClassification`Multi-class classification
`Qwen3ForSequenceClassification`Multi-class classification
`BertForSequenceClassification`Multi-class classification
`Gemma2ForSequenceClassification`Multi-class classification
- - - The API automatically uses the `id2label` mapping from the model's `config.json` file to provide meaningful label names instead of generic class names. If `id2label` is not available, it falls back to `LABEL_0`, `LABEL_1`, etc., or `Class_0`, `Class_1` as a last resort. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ModelType
`InternLM2ForRewardModel`Single reward score
`Qwen2ForRewardModel`Single reward score
`LlamaForSequenceClassificationWithNormal_Weights`Special reward model
- - - The `/classify` endpoint in SGLang was originally designed for reward models but now supports all non-generative models. The `/v1/classify` endpoint provides a standardized vLLM-compatible interface for classification tasks. - - - - - -## Error handling - -The API returns appropriate HTTP status codes and error messages: - - - - - - - - - - - - - - - - - - - - - - -
Status codeMeaning
`400 Bad Request`Invalid request format or missing required fields
`500 Internal Server Error`Server-side processing error
- -Error response format: - -```json Config -{ - "error": "Error message", - "type": "error_type", - "code": 400 -} -``` - -## Implementation details - - - Handles routing and request/response models in - `sgl-model-gateway/src/protocols/spec.rs`. - - - - Implements the actual endpoint in - `python/sglang/srt/entrypoints/http_server.py`. - - - - Handles the classification logic in - `python/sglang/srt/entrypoints/openai/serving_classify.py`. - - -## Testing - -Use the provided test script to verify the implementation: - - -```bash Command -python test_classify_api.py -``` - - -## Compatibility - - - This implementation is compatible with vLLM's classification API format, - allowing seamless migration from vLLM to SGLang for classification tasks. - diff --git a/docs_new/docs/supported-models/classify_models.mdx b/docs_new/docs/supported-models/classify_models.mdx new file mode 100644 index 000000000..8883f4ec0 --- /dev/null +++ b/docs_new/docs/supported-models/classify_models.mdx @@ -0,0 +1,163 @@ +--- +title: Classification Models +--- +This document describes the `/v1/classify` API endpoint implementation in SGLang, which is compatible with vLLM's classification API format. + +## Overview + +The classification API allows you to classify text inputs using classification models. This implementation follows the same format as vLLM's 0.7.0 classification API. + +## API Endpoint + +```text Output +POST /v1/classify +``` + +## Request Format + +```json Config +{ + "model": "model_name", + "input": "text to classify" +} +``` + +### Parameters + +- `model` (string, required): The name of the classification model to use +- `input` (string, required): The text to classify +- `user` (string, optional): User identifier for tracking +- `rid` (string, optional): Request ID for tracking +- `priority` (integer, optional): Request priority + +## Response Format + +```json Config +{ + "id": "classify-9bf17f2847b046c7b2d5495f4b4f9682", + "object": "list", + "created": 1745383213, + "model": "jason9693/Qwen2.5-1.5B-apeach", + "data": [ + { + "index": 0, + "label": "Default", + "probs": [0.565970778465271, 0.4340292513370514], + "num_classes": 2 + } + ], + "usage": { + "prompt_tokens": 10, + "total_tokens": 10, + "completion_tokens": 0, + "prompt_tokens_details": null + } +} +``` + +### Response Fields + +- `id`: Unique identifier for the classification request +- `object`: Always "list" +- `created`: Unix timestamp when the request was created +- `model`: The model used for classification +- `data`: Array of classification results + - `index`: Index of the result + - `label`: Predicted class label + - `probs`: Array of probabilities for each class + - `num_classes`: Total number of classes +- `usage`: Token usage information + - `prompt_tokens`: Number of input tokens + - `total_tokens`: Total number of tokens + - `completion_tokens`: Number of completion tokens (always 0 for classification) + - `prompt_tokens_details`: Additional token details (optional) + +## Example Usage + +### Using curl + +```bash Command +curl -v "http://127.0.0.1:8000/v1/classify" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "jason9693/Qwen2.5-1.5B-apeach", + "input": "Loved the new café—coffee was great." + }' +``` + +### Using Python + +```python Example +import requests +import json + +# Make classification request +response = requests.post( + "http://127.0.0.1:8000/v1/classify", + headers={"Content-Type": "application/json"}, + json={ + "model": "jason9693/Qwen2.5-1.5B-apeach", + "input": "Loved the new café—coffee was great." + } +) + +# Parse response +result = response.json() +print(json.dumps(result, indent=2)) +``` + +## Supported Models + +The classification API works with any classification model supported by SGLang, including: + +### Classification Models (Multi-class) +- `LlamaForSequenceClassification` - Multi-class classification +- `Qwen2ForSequenceClassification` - Multi-class classification +- `Qwen3ForSequenceClassification` - Multi-class classification +- `BertForSequenceClassification` - Multi-class classification +- `Gemma2ForSequenceClassification` - Multi-class classification + +**Label Mapping**: The API automatically uses the `id2label` mapping from the model's `config.json` file to provide meaningful label names instead of generic class names. If `id2label` is not available, it falls back to `LABEL_0`, `LABEL_1`, etc., or `Class_0`, `Class_1` as a last resort. + +### Reward Models (Single score) +- `InternLM2ForRewardModel` - Single reward score +- `Qwen2ForRewardModel` - Single reward score +- `LlamaForSequenceClassificationWithNormal_Weights` - Special reward model + +**Note**: The `/classify` endpoint in SGLang was originally designed for reward models but now supports all non-generative models. Our `/v1/classify` endpoint provides a standardized vLLM-compatible interface for classification tasks. + +## Error Handling + +The API returns appropriate HTTP status codes and error messages: + +- `400 Bad Request`: Invalid request format or missing required fields +- `500 Internal Server Error`: Server-side processing error + +Error response format: +```json Config +{ + "error": "Error message", + "type": "error_type", + "code": 400 +} +``` + +## Implementation Details + +The classification API is implemented using: + +1. **Rust Model Gateway**: Handles routing and request/response models in `sgl-model-gateway/src/protocols/spec.rs` +2. **Python HTTP Server**: Implements the actual endpoint in `python/sglang/srt/entrypoints/http_server.py` +3. **Classification Service**: Handles the classification logic in `python/sglang/srt/entrypoints/openai/serving_classify.py` + +## Testing + +Use the provided test script to verify the implementation: + +```bash Command +python test_classify_api.py +``` + +## Compatibility + +This implementation is compatible with vLLM's classification API format, allowing seamless migration from vLLM to SGLang for classification tasks. diff --git a/docs_new/docs/supported-models/diffusion-language-models.mdx b/docs_new/docs/supported-models/diffusion-language-models.mdx deleted file mode 100644 index e3adf769b..000000000 --- a/docs_new/docs/supported-models/diffusion-language-models.mdx +++ /dev/null @@ -1,13 +0,0 @@ ---- -title: Diffusion language models ---- - -For detailed documentation on diffusion models in SGLang, see the [SGLang Diffusion](/docs/sglang-diffusion/index) section under Docs. - - - Learn about score-based diffusion backbones, supported models, and usage patterns. - diff --git a/docs_new/docs/supported-models/diffusion_language_models.mdx b/docs_new/docs/supported-models/diffusion_language_models.mdx new file mode 100644 index 000000000..4dea53920 --- /dev/null +++ b/docs_new/docs/supported-models/diffusion_language_models.mdx @@ -0,0 +1,133 @@ +--- +title: Diffusion language models +--- +Diffusion language models have shown promise for non-autoregressive text generation with parallel decoding capabilities. Unlike auto-regressive language models, different diffusion language models require different decoding strategies. + +## Example Launch Command + +SGLang supports different DLLM algorithms such as `LowConfidence` and `JointThreshold`. + +```bash Command +python3 -m sglang.launch_server \ + --model-path inclusionAI/LLaDA2.0-mini \ # example HF/local path + --dllm-algorithm LowConfidence \ + --dllm-algorithm-config ./config.yaml \ # Optional. Uses the algorithm's default if not set. + --host 0.0.0.0 \ + --port 30000 +``` + +## Example Configuration File + +Depending on the algorithm selected, the configuration parameters vary. + +LowConfidence Config: + +```yaml Config +# Confidence threshold for accepting predicted tokens +# - Higher values: More conservative, better quality but slower +# - Lower values: More aggressive, faster but potentially lower quality +# Range: 0.0 - 1.0 +threshold: 0.95 + +# Default: 32, for LLaDA2MoeModelLM +block_size: 32 +``` + +JointThreshold Config: + +```yaml Config +# Decoding threshold for Mask-to-Token (M2T) phase +# - Higher values: More conservative, better quality but slower +# - Lower values: More aggressive, faster but potentially lower quality +# Range: 0.0 - 1.0 +threshold: 0.5 +# Decoding threshold for Token-to-Token (T2T) phase +# Range: 0.0 - 1.0 +# Setting to 0.0 allows full editing (recommended for most cases). +edit_threshold: 0.0 +# Max extra T2T steps after all masks are removed. Prevents infinite loops. +max_post_edit_steps: 16 +# 2-gram repetition penalty (default 0). +# An empirical value of 3 is often sufficient to mitigate most repetitions. +penalty_lambda: 0 +``` + +## Example Client Code Snippet + +Just like other supported models, diffusion language models can be used via the REST API or Python client. + +Python client example for making a generation request to the launched server: + +```python Example +import sglang as sgl + +def main(): + llm = sgl.Engine(model_path="inclusionAI/LLaDA2.0-mini", + dllm_algorithm="LowConfidence", + max_running_requests=1, + trust_remote_code=True) + + prompts = [ + "SYSTEMdetailed thinking off<|role_end|>HUMAN Write a brief introduction of the great wall <|role_end|>ASSISTANT" + ] + + sampling_params = { + "temperature": 0, + "max_new_tokens": 1024, + } + + outputs = llm.generate(prompts, sampling_params) + print(outputs) + +if __name__ == '__main__': + main() +``` + +Curl example for making a generation request to the launched server: + +```bash Command +curl -X POST "http://127.0.0.1:30000/generate" \ + -H "Content-Type: application/json" \ + -d '{ + "text": [ + "SYSTEMdetailed thinking off<|role_end|>HUMAN Write the number from 1 to 128 <|role_end|>ASSISTANT", + "SYSTEMdetailed thinking off<|role_end|>HUMAN Write a brief introduction of the great wall <|role_end|>ASSISTANT" + ], + "stream": true, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 1024 + } + }' +``` + +## Supported Models + +Below the supported models are summarized in a table. + + + + + + + + + + + + + + + + + + + + + + + + + + +
Model FamilyExample ModelDescription
LLaDA2.0 (mini, flash)inclusionAI/LLaDA2.0-flashLLaDA2.0-flash is a diffusion language model featuring a 100B Mixture-of-Experts (MoE) architecture.
SDAR (JetLM)JetLM/SDAR-8B-ChatSDAR series diffusion language model (Chat), dense architecture.
SDAR (JetLM)JetLM/SDAR-30B-A3B-ChatSDAR series diffusion language model (Chat), MoE architecture.
diff --git a/docs_new/docs/supported-models/embedding-models.mdx b/docs_new/docs/supported-models/embedding_models.mdx similarity index 82% rename from docs_new/docs/supported-models/embedding-models.mdx rename to docs_new/docs/supported-models/embedding_models.mdx index 1383440f1..3edf83449 100644 --- a/docs_new/docs/supported-models/embedding-models.mdx +++ b/docs_new/docs/supported-models/embedding_models.mdx @@ -2,17 +2,15 @@ title: Embedding models description: Dense and sparse embedding models with FlashInfer acceleration and SGLang's batching infrastructure. --- - SGLang provides robust support for embedding models by integrating efficient serving mechanisms with its flexible programming interface. This integration allows for streamlined handling of embedding tasks, facilitating faster and more accurate retrieval and semantic search operations. SGLang's architecture enables better resource utilization and reduced latency in embedding model deployment. - Embedding models must be launched with the `--is-embedding` flag. Some models - may also require `--trust-remote-code`. +Embedding models are executed with `--is-embedding` flag and some may require `--trust-remote-code` -## Quick start +## Quick Start -1. **Launch the server** +### Launch Server ```bash python3 -m sglang.launch_server \ @@ -22,7 +20,7 @@ python3 -m sglang.launch_server \ --port 30000 ``` -2. **Send a client request** +### Client Request ```python import requests @@ -39,12 +37,11 @@ response = requests.post(url + "/v1/embeddings", json=payload).json() print("Embedding:", response["data"][0]["embedding"]) ``` -## Multimodal embedding example + +## Multimodal Embedding Example For multimodal models like GME that support both text and images: -1. **Launch the server with a multimodal model** - ```bash python3 -m sglang.launch_server \ --model-path Alibaba-NLP/gme-Qwen2-VL-2B-Instruct \ @@ -54,9 +51,7 @@ python3 -m sglang.launch_server \ --port 30000 ``` -2. **Send a multimodal request** - -```python +```python Example import requests url = "http://127.0.0.1:30000" @@ -67,37 +62,41 @@ image_path = "https://huggingface.co/datasets/liuhaotian/llava-bench-in-the-wild payload = { "model": "gme-qwen2-vl", "input": [ - {"text": text_input}, - {"image": image_path} + { + "text": text_input + }, + { + "image": image_path + } ], } response = requests.post(url + "/v1/embeddings", json=payload).json() + print("Embeddings:", [x.get("embedding") for x in response.get("data", [])]) ``` -## Matryoshka embedding example +## Matryoshka Embedding Example -[Matryoshka Embeddings](https://sbert.net/examples/sentence_transformer/training/matryoshka/README.html#matryoshka-embeddings) or [Matryoshka Representation Learning (MRL)](https://arxiv.org/abs/2205.13147) is a technique used in training embedding models. It allows users to trade off between performance and cost. +[Matryoshka Embeddings](https://sbert.net/examples/sentence_transformer/training/matryoshka/README.html#matryoshka-embeddings) or [Matryoshka Representation Learning (MRL)](https://arxiv.org/abs/2205.13147) is a technique used in training embedding models. It allows user to trade off between performance and cost. -1. **Launch a Matryoshka-capable model** +### 1. Launch a Matryoshka‑capable model -If the model config already includes `matryoshka_dimensions` or `is_matryoshka` then no override is needed. Otherwise, use `--json-model-override-args` as below: +If the model config already includes `matryoshka_dimensions` or `is_matryoshka` then no override is needed. Otherwise, you can use `--json-model-override-args` as below: -```bash +```bash Command python3 -m sglang.launch_server \ - --model-path Qwen/Qwen3-Embedding-0.6B \ - --is-embedding \ - --host 0.0.0.0 \ - --port 30000 \ - --json-model-override-args '{"matryoshka_dimensions": [128, 256, 512, 1024, 1536]}' + --model-path Qwen/Qwen3-Embedding-0.6B \ + --is-embedding \ + --host 0.0.0.0 \ + --port 30000 \ + --json-model-override-args '{"matryoshka_dimensions": [128, 256, 512, 1024, 1536]}' ``` - - Setting `"is_matryoshka": true` allows truncating to any dimension. Otherwise, the server validates that the specified dimension in the request is one of `matryoshka_dimensions`. Omitting `dimensions` in a request returns the full vector. - +1. Setting `"is_matryoshka": true` allows truncating to any dimension. Otherwise, the server will validate that the specified dimension in the request is one of `matryoshka_dimensions`. +2. Omitting `dimensions` in a request returns the full vector. -2. **Make requests with different output dimensions** +### 2. Make requests with different output dimensions ```python import requests @@ -115,7 +114,8 @@ response = requests.post(url + "/v1/embeddings", json=payload).json() print("Embedding:", response["data"][0]["embedding"]) ``` -## Supported models + +## Supported Models @@ -126,10 +126,10 @@ print("Embedding:", response["data"][0]["embedding"]) - - + + - + @@ -155,7 +155,7 @@ print("Embedding:", response["data"][0]["embedding"]) - + diff --git a/docs_new/docs/supported-models/large-language-models.mdx b/docs_new/docs/supported-models/generative_models.mdx similarity index 76% rename from docs_new/docs/supported-models/large-language-models.mdx rename to docs_new/docs/supported-models/generative_models.mdx index 8af620259..b5258001b 100644 --- a/docs_new/docs/supported-models/large-language-models.mdx +++ b/docs_new/docs/supported-models/generative_models.mdx @@ -1,19 +1,16 @@ --- title: Large Language Models --- - These models accept text input and produce text output (e.g., chat completions). They are primarily large language models (LLMs), some with mixture-of-experts (MoE) architectures for scaling. ## Example launch Command - ```shell Command python3 -m sglang.launch_server \ --model-path meta-llama/Llama-3.2-1B-Instruct \ # example HF/local path --host 0.0.0.0 \ --port 30000 \ ``` - ## Supported models @@ -44,12 +41,12 @@ in the GitHub search bar. - + - + @@ -58,18 +55,18 @@ in the GitHub search bar. - + - - - + + + - + @@ -83,7 +80,7 @@ in the GitHub search bar. - + @@ -93,7 +90,7 @@ in the GitHub search bar. - + @@ -102,9 +99,9 @@ in the GitHub search bar. - - - + + + @@ -113,7 +110,7 @@ in the GitHub search bar. - + @@ -193,7 +190,7 @@ in the GitHub search bar. - + @@ -209,7 +206,7 @@ in the GitHub search bar. - + @@ -218,48 +215,73 @@ in the GitHub search bar. - - + + - + - + - - + + + + + + + + + + + + - - + + - - + + - - + + + + + + + + + + + + - - - + + + - + - - + + - + + + + + +
ModelExample HF modelModel FamilyExample Model Chat templateNotesDescription
BGE `BAAI/bge-large-en-v1.5` N/ABAAI's text embeddings (requires `--attention-backend triton` or `torch_native`)BAAI's text embeddings (requires attention-backend triton/torch_native)
GME (Multimodal)
**DeepSeek** (v1, v2, v3/R1) `deepseek-ai/DeepSeek-R1`Series of advanced reasoning-optimized models (including a 671B MoE) trained with reinforcement learning; top performance on complex reasoning, math, and code tasks. [SGLang provides Deepseek v3/R1 model-specific optimizations](../basic_usage/deepseek_v3) and [Reasoning Parser](../advanced_features/separate_reasoning.ipynb)Series of advanced reasoning-optimized models (including a 671B MoE) trained with reinforcement learning; top performance on complex reasoning, math, and code tasks. SGLang provides Deepseek v3/R1 model-specific optimizations and Reasoning Parser
**Kimi K2** (Thinking, Instruct) `moonshotai/Kimi-K2-Instruct`Moonshot AI's 1 trillion parameter MoE model (32B active) with 128K–256K context; state-of-the-art agentic intelligence with stable long-horizon agency across 200–300 sequential tool calls. Features MLA attention and native INT4 quantization. [See Reasoning Parser docs](../advanced_features/separate_reasoning.ipynb)Moonshot AI's 1 trillion parameter MoE model (32B active) with 128K–256K context; state-of-the-art agentic intelligence with stable long-horizon agency across 200–300 sequential tool calls. Features MLA attention and native INT4 quantization. See Reasoning Parser docs
**Kimi Linear** (48B-A3B)
**GPT-OSS**`openai/gpt-oss-20b`, `openai/gpt-oss-120b`openai/gpt-oss-20b, openai/gpt-oss-120b OpenAI’s latest GPT-OSS series for complex reasoning, agentic tasks, and versatile developer use cases.
**Qwen** (3, 3MoE, 3Next, 2.5, 2 series)`Qwen/Qwen3-0.6B`, `Qwen/Qwen3-30B-A3B` `Qwen/Qwen3-Next-80B-A3B-Instruct `Alibaba’s latest Qwen3 series for complex reasoning, language understanding, and generation tasks; Support for MoE variants along with previous generation 2.5, 2, etc. [SGLang provides Qwen3 specific reasoning parser](../advanced_features/separate_reasoning.ipynb)Qwen (3.5, 3, 3MoE, 3Next, 2.5, 2 series)Qwen/Qwen3.5-397B-A17B, Qwen/Qwen3-0.6B, Qwen/Qwen3-30B-A3B, Qwen/Qwen3-Next-80B-A3B-InstructAlibaba’s latest Qwen3 series for complex reasoning, language understanding, and generation tasks; Support for MoE variants along with previous generation 2.5, 2, etc. SGLang provides Qwen3 specific reasoning parser
**Llama** (2, 3.x, 4 series) `meta-llama/Llama-4-Scout-17B-16E-Instruct`Meta's open LLM series, spanning 7B to 400B parameters (Llama 2, 3, and new Llama 4) with well-recognized performance. [SGLang provides Llama-4 model-specific optimizations](../basic_usage/llama4.md)Meta's open LLM series, spanning 7B to 400B parameters (Llama 2, 3, and new Llama 4) with well-recognized performance. SGLang provides Llama-4 model-specific optimizations
**Mistral** (Mixtral, NeMo, Small3)
**Phi** (Phi-1.5, Phi-2, Phi-3, Phi-4, Phi-MoE series)`microsoft/Phi-4-multimodal-instruct`, `microsoft/Phi-3.5-MoE-instruct`microsoft/Phi-4-multimodal-instruct, microsoft/Phi-3.5-MoE-instruct Microsoft’s Phi family of small models (1.3B–5.6B); Phi-4-multimodal (5.6B) processes text, images, and speech, Phi-4-mini is a high-accuracy text model and Phi-3.5-MoE is a mixture-of-experts model.
**OLMo** (2, 3)`allenai/OLMo-3-1125-32B`, `allenai/OLMo-3-32B-Think`, `allenai/OLMo-2-1124-7B-Instruct`allenai/OLMo-3-1125-32B, allenai/OLMo-3-32B-Think, allenai/OLMo-2-1124-7B-Instruct Allen AI’s series of Open Language Models designed to enable the science of language models.
Allen AI’s open Mixture-of-Experts model (7B total, 1B active parameters) delivering state-of-the-art results with sparse expert activation.
**MiniMax-M2** (M2, M2.1)`minimax/MiniMax-M2`, `minimax/MiniMax-M2.1`MiniMax’s SOTA LLM for coding & agentic workflows.MiniMax-M2 (M2, M2.1, M2.5)MiniMaxAI/MiniMax-M2.5, MiniMaxAI/MiniMax-M2.1, MiniMaxAI/MiniMax-M2MiniMax's SOTA LLM for coding & agentic workflows.
**StableLM** (3B, 7B)
**Command-(R,A)** (Cohere)`CohereLabs/c4ai-command-r-v01`, `CohereLabs/c4ai-command-r7b-12-2024`, `CohereLabs/c4ai-command-a-03-2025`CohereLabs/c4ai-command-r-v01, CohereLabs/c4ai-command-r7b-12-2024, CohereLabs/c4ai-command-a-03-2025 Cohere’s open conversational LLM (Command series) optimized for long context, retrieval-augmented generation, and tool use.
**Ling** (16.8B–290B)`inclusionAI/Ling-lite`, `inclusionAI/Ling-plus`inclusionAI/Ling-lite, inclusionAI/Ling-plus InclusionAI’s open MoE models. Ling-Lite has 16.8B total / 2.75B active parameters, and Ling-Plus has 290B total / 28.8B active parameters. They are designed for high performance on NLP and complex reasoning tasks.
**GPT-J** (6B) `EleutherAI/gpt-j-6b`EleutherAI's GPT-2-like causal language model (6B) trained on the [Pile](https://pile.eleuther.ai/) dataset.EleutherAI's GPT-2-like causal language model (6B) trained on the Pile dataset.
**Orion** (14B)
**Llama Nemotron Super** (v1, v1.5, NVIDIA)`nvidia/Llama-3_3-Nemotron-Super-49B-v1`, `nvidia/Llama-3_3-Nemotron-Super-49B-v1_5`The [NVIDIA Nemotron](https://www.nvidia.com/en-us/ai-data-science/foundation-models/nemotron/) family of multimodal models provides state-of-the-art reasoning models specifically designed for enterprise-ready AI agents.nvidia/Llama-3_3-Nemotron-Super-49B-v1, nvidia/Llama-3_3-Nemotron-Super-49B-v1_5The NVIDIA Nemotron family of multimodal models provides state-of-the-art reasoning models specifically designed for enterprise-ready AI agents.
**Llama Nemotron Ultra** (v1, NVIDIA) `nvidia/Llama-3_1-Nemotron-Ultra-253B-v1`The [NVIDIA Nemotron](https://www.nvidia.com/en-us/ai-data-science/foundation-models/nemotron/) family of multimodal models provides state-of-the-art reasoning models specifically designed for enterprise-ready AI agents.The NVIDIA Nemotron family of multimodal models provides state-of-the-art reasoning models specifically designed for enterprise-ready AI agents.
**NVIDIA Nemotron Nano 2.0** `nvidia/NVIDIA-Nemotron-Nano-9B-v2`The [NVIDIA Nemotron](https://www.nvidia.com/en-us/ai-data-science/foundation-models/nemotron/) family of multimodal models provides state-of-the-art reasoning models specifically designed for enterprise-ready AI agents. `Nemotron-Nano-9B-v2` is a hybrid Mamba-Transformer language model designed to increase throughput for reasoning workloads while achieving state-of-the-art accuracy compared to similarly-sized models.The NVIDIA Nemotron family of multimodal models provides state-of-the-art reasoning models specifically designed for enterprise-ready AI agents. Nemotron-Nano-9B-v2 is a hybrid Mamba-Transformer language model designed to increase throughput for reasoning workloads while achieving state-of-the-art accuracy compared to similarly-sized models.
**StarCoder2** (3B-15B)`bigcode/starcoder2-7b`NVIDIA Nemotron 3 Super (NVIDIA)nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4The NVIDIA Nemotron 3 Super is a 120B-parameter MoE model (12B active) delivering high-quality reasoning and generation for enterprise AI agents.
NVIDIA Nemotron 3 Nano (NVIDIA)nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16The NVIDIA Nemotron 3 Nano is a compact model designed for efficient edge and enterprise deployment with strong reasoning capabilities.
StarCoder2 (3B-15B)bigcode/starcoder2-7b StarCoder2 is a family of open large language models (LLMs) specialized for code generation and understanding. It is the successor to StarCoder, jointly developed by the BigCode project (a collaboration between Hugging Face, ServiceNow Research, and other contributors).
**Jet-Nemotron**`jet-ai/Jet-Nemotron-2B`Jet-Nemotronjet-ai/Jet-Nemotron-2B Jet-Nemotron is a new family of hybrid-architecture language models that surpass state-of-the-art open-source full-attention language models, while achieving significant efficiency gains.
**Trinity** (Nano, Mini)`arcee-ai/Trinity-Mini`Trinity (Nano, Mini)arcee-ai/Trinity-Mini Arcee's foundational MoE Trinity family of models, open weights under Apache 2.0.
**Falcon-H1** (0.5B–34B)`tiiuae/Falcon-H1-34B-Instruct`LFM2 (350M, 1.2B)LiquidAI/LFM2.5-1.2B-InstructLiquid AI's hybrid attention + short convolution language model.
LFM2-MoE (8B-A1B, 24B-A2B)LiquidAI/LFM2-8B-A1BLiquid AI's Mixture-of-Experts variant with sigmoid routing and top-k expert selection.
Falcon-H1 (0.5B–34B)tiiuae/Falcon-H1-34B-Instruct TII's hybrid Mamba-Transformer architecture combining attention and state-space models for efficient long-context inference.
**Hunyuan-Large** (389B, MoE)`tencent/Tencent-Hunyuan-Large`
Hunyuan-Large (389B, MoE)tencent/Tencent-Hunyuan-Large Tencent's open-source MoE model with 389B total / 52B active parameters, featuring Cross-Layer Attention (CLA) for improved efficiency.
**IBM Granite 4.0 (Hybrid, Dense)**`ibm-granite/granite-4.0-h-micro`, `ibm-granite/granite-4.0-micro`IBM Granite 4.0 micro models: hybrid Mamba–MoE (`h-micro`) and dense (`micro`) variants. Enterprise-focused reasoning modelsibm-granite/granite-4.0-h-micro, ibm-granite/granite-4.0-microIBM Granite 4.0 micro models: hybrid Mamba–MoE (h-micro) and dense (micro) variants. Enterprise-focused reasoning models
Sarvam 2 (30B-A2B, 105B-A10B)sarvamai/sarvam-2Sarvam's Mixture-of-Experts models. The 105B variant uses MLA (Multi-head Latent Attention) and the 30B variant uses GQA, both with 128 routed experts.
diff --git a/docs_new/docs/supported-models/mindspore-models.mdx b/docs_new/docs/supported-models/mindspore_models.mdx similarity index 54% rename from docs_new/docs/supported-models/mindspore-models.mdx rename to docs_new/docs/supported-models/mindspore_models.mdx index f94987038..83ecd3a6f 100644 --- a/docs_new/docs/supported-models/mindspore-models.mdx +++ b/docs_new/docs/supported-models/mindspore_models.mdx @@ -1,50 +1,42 @@ --- title: "MindSpore Models" --- +## Introduction -MindSpore is a high-performance AI framework optimized for [Ascend NPUs](../hardware-platforms/ascend-npus/SGLang-installation-with-NPUs-support). This doc guides users to run MindSpore models in SGLang. +MindSpore is a high-performance AI framework optimized for Ascend NPUs. This doc guides users to run MindSpore models in SGLang. ## Requirements -MindSpore currently only supports Ascend NPU devices. Users need to first install Ascend CANN software packages. The CANN software packages can be downloaded from the [Ascend Official Website](https://www.hiascend.com). The recommended version is 8.3.RC2. +MindSpore currently only supports Ascend NPU devices. Users need to first install Ascend CANN 8.5. +The CANN software packages can be downloaded from the [Ascend Official Website](https://www.hiascend.com). ## Supported Models Currently, the following models are supported: - - - Dense and MoE models - - - DeepSeek V3 and R1 models - - - Additional models are on the way - - +- **Qwen3**: Dense and MoE models +- **DeepSeek V3/R1** +- *More models coming soon...* ## Installation -Currently, MindSpore models are provided by an independent package `sgl-mindspore`. Support for MindSpore is built upon current SGLang support for Ascend NPU platform. Please first [install SGLang for Ascend NPU](../hardware-platforms/ascend-npus/SGLang-installation-with-NPUs-support) and then install `sgl-mindspore`. +> **Note**: Currently, MindSpore models are provided by an independent package `sgl-mindspore`. Support for MindSpore is built upon current SGLang support for Ascend NPU platform. Please first [install SGLang for Ascend NPU](../hardware-platforms/ascend-npus/ascend_npu) and then install `sgl-mindspore`: - ```bash Install git clone https://github.com/mindspore-lab/sgl-mindspore.git cd sgl-mindspore pip install -e . ``` - + ## Run Model Current SGLang-MindSpore supports Qwen3 and DeepSeek V3/R1 models. This doc uses Qwen3-8B as an example. -### Offline Infer +### Offline inference -Use the following script for offline infer: +Use the following script for offline inference: - ```python Offline Infer import sglang as sgl @@ -73,12 +65,13 @@ for prompt, output in zip(prompts, outputs): print(f"Generated: {output['text']}") print("---") ``` - -### Start Server +### Start server - -```bash Single Node +Launch a server with MindSpore backend: + +```bash Command +# Basic server startup python3 -m sglang.launch_server \ --model-path /path/to/your/model \ --host 0.0.0.0 \ @@ -88,7 +81,11 @@ python3 -m sglang.launch_server \ --tp-size 1 \ --dp-size 1 ``` -```bash Multi-Node Distributed + +For distributed server with multiple nodes: + +```bash Command +# Multi-node distributed server python3 -m sglang.launch_server \ --model-path /path/to/your/model \ --host 0.0.0.0 \ @@ -101,15 +98,13 @@ python3 -m sglang.launch_server \ --tp-size 4 \ --dp-size 2 ``` - ## Troubleshooting -### Debug Mode +#### Debug Mode -Enable sglang debug logging by log-level argument: +Enable sglang debug logging by log-level argument. - ```bash Debug Mode python3 -m sglang.launch_server \ --model-path /path/to/your/model \ @@ -119,49 +114,39 @@ python3 -m sglang.launch_server \ --attention-backend ascend \ --log-level DEBUG ``` - -Enable MindSpore info and debug logging by setting environments: +Enable mindspore info and debug logging by setting environments. - -```bash INFO -export GLOG_v=1 +```bash Command +export GLOG_v=1 # INFO +export GLOG_v=0 # DEBUG ``` -```bash DEBUG -export GLOG_v=0 + +#### Explicitly select devices + +Use the following environment variable to explicitly select the devices to use. + +```bash Command +export ASCEND_RT_VISIBLE_DEVICES=4,5,6,7 # to set device ``` - -### Explicitly Select Devices +#### Some communication environment issues -Use the following environment variable to explicitly select the devices to use: +In case of some environment with special communication environment, users need set some environment variables. - -```bash Select Devices -export ASCEND_RT_VISIBLE_DEVICES=4,5,6,7 -``` - - -### Some Communication Environment Issues - -In case of some environment with special communication environment, users need to set some environment variables: - - ```bash Disable LCCL export MS_ENABLE_LCCL=off # current not support LCCL communication mode in SGLang-MindSpore ``` - -### Some Dependencies of Protobuf +#### Some dependencies of protobuf -In case of some environment with special protobuf version, users need to set some environment variables to avoid binary version mismatch: +In case of some environment with special protobuf version, users need set some environment variables to avoid binary version mismatch. - -```bash Fix Protobuf -export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python +```bash Command +export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python # to avoid protobuf binary version mismatch ``` - ## Support +For MindSpore-specific issues: -For MindSpore-specific issues, refer to the [MindSpore documentation](https://www.mindspore.cn/). +- Refer to the [MindSpore documentation](https://www.mindspore.cn/) diff --git a/docs_new/docs/supported-models/modelscope.mdx b/docs_new/docs/supported-models/modelscope.mdx index b9e20936a..2e26eb292 100644 --- a/docs_new/docs/supported-models/modelscope.mdx +++ b/docs_new/docs/supported-models/modelscope.mdx @@ -1,23 +1,21 @@ --- title: "Use Models From ModelScope" --- - To use a model from [ModelScope](https://www.modelscope.cn), set the environment variable `SGLANG_USE_MODELSCOPE`. - ```bash Set Environment Variable export SGLANG_USE_MODELSCOPE=true ``` - We take [Qwen2-7B-Instruct](https://www.modelscope.cn/models/qwen/qwen2-7b-instruct) as an example. -## Launch the Server - - +Launch the Server: ```bash Python python -m sglang.launch_server --model-path qwen/Qwen2-7B-Instruct --port 30000 ``` + +Or start it by docker: + ```bash Docker docker run --gpus all \ -p 30000:30000 \ @@ -27,6 +25,5 @@ docker run --gpus all \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server --model-path Qwen/Qwen2.5-7B-Instruct --host 0.0.0.0 --port 30000 ``` - -ModelScope uses a different cache directory than Hugging Face. You may need to set it manually to avoid running out of disk space. +Note that modelscope uses a different cache directory than huggingface. You may need to set it manually to avoid running out of disk space. diff --git a/docs_new/docs/supported-models/vision-language-models.mdx b/docs_new/docs/supported-models/multimodal_language_models.mdx similarity index 85% rename from docs_new/docs/supported-models/vision-language-models.mdx rename to docs_new/docs/supported-models/multimodal_language_models.mdx index 12265f80c..2686c660d 100644 --- a/docs_new/docs/supported-models/vision-language-models.mdx +++ b/docs_new/docs/supported-models/multimodal_language_models.mdx @@ -1,15 +1,18 @@ +--- +title: "Multimodal Language Models" +metatags: + description: "Documentation for Multimodal Language Models" +--- These models accept multi-modal inputs (e.g., images and text) and generate text output. They augment language models with multimodal encoders. ## Example launch Command - ```bash Launch Server python3 -m sglang.launch_server \ --model-path meta-llama/Llama-3.2-11B-Vision-Instruct \ # example HF/local path --host 0.0.0.0 \ --port 30000 \ ``` - > See the [OpenAI APIs section](../basic_usage/openai_api_vision) for how to send multimodal requests. @@ -19,11 +22,9 @@ Below the supported models are summarized in a table. If you are unsure if a specific architecture is implemented, you can search for it via GitHub. For example, to search for `Qwen2_5_VLForConditionalGeneration`, use the expression: - ```text GitHub Search repo:sgl-project/sglang path:/^python\/sglang\/srt\/models\// Qwen2_5_VLForConditionalGeneration ``` - in the GitHub search bar. @@ -183,17 +184,48 @@ in the GitHub search bar. + Qwen3-ASR (0.6B, 1.7B) + Qwen/Qwen3-ASR-1.7B + Alibaba's automatic speech recognition models supporting 52 languages. Served via the /v1/audio/transcriptions endpoint. + + + Qwen3-Omni Qwen/Qwen3-Omni-30B-A3B-Instruct Alibaba's omni-modal MoE model. Currently supports the Thinker component (multimodal understanding for text, images, audio, and video), while the Talker component (audio generation) is not yet supported. - + + LFM2-VL + LiquidAI/LFM2.5-VL-1.6B + Liquid AI's vision-language model combining a SigLip2 vision encoder (NaFlex variable-resolution) with the LFM2 hybrid attention + short convolution language model. Supports multi-image inputs. + + + -## Video Input Support +## Audio Transcription -SGLang supports video input for Vision-Language Models (VLMs), enabling temporal reasoning tasks such as video question answering, captioning, and holistic scene understanding. Video clips are decoded, key frames are sampled, and the resulting tensors are batched together with the text prompt, allowing multimodal inference to integrate visual and linguistic context. +SGLang supports audio-only ASR models via the OpenAI-compatible `/v1/audio/transcriptions` endpoint. Upload an audio file and receive a transcription. + +### Launch Command + +```bash Command +sglang serve \ + --model-path Qwen/Qwen3-ASR-1.7B \ + --served-model-name qwen3-asr \ + --trust-remote-code \ + --host 0.0.0.0 --port 30000 +``` + +### Example Request + +```bash Command +curl http://localhost:30000/v1/audio/transcriptions \ + -F file=@audio.wav \ + -F model=qwen3-asr \ + -F response_format=verbose_json +``` @@ -205,7 +237,38 @@ SGLang supports video input for Vision-Language Models (VLMs), enabling temporal - + + + + + + + + + + + + + + + +
Model Family Example IdentifierVideo notesNotes
Whisperopenai/whisper-large-v3OpenAI's speech recognition model.
Qwen3-ASR (0.6B, 1.7B)Qwen/Qwen3-ASR-1.7BUse --trust-remote-code. Supports 52 languages.
+ +## Video Input Support + +SGLang supports video input for Vision-Language Models (VLMs), enabling temporal reasoning tasks such as video question answering, captioning, and holistic scene understanding. Video clips are decoded, key frames are sampled, and the resulting tensors are batched together with the text prompt, allowing multimodal inference to integrate visual and linguistic context. + + + + + + + + + + + + @@ -246,7 +309,6 @@ Use `sgl.video(path, num_frames)` when building prompts to attach clips from you Example OpenAI-compatible request that sends a video clip: - ```python Complete Example import requests @@ -274,7 +336,6 @@ data = { response = requests.post(url, json=data) print(response.text) ``` - ## Usage Notes @@ -289,9 +350,9 @@ Use this flag when you have sufficient GPU memory and want to minimize latency f ### Multimodal Inputs Limitation -- **Use `--mm-process-config '{"image":{"max_pixels":1048576},"video":{"fps":3,"max_pixels":602112,"max_frames":60}}'`**: To set `image`, `video`, and `audio` input limits. +- **Use `--mm-process-config '{"image":{"max_pixels":1048576},"video":{"fps":3,"max_pixels":602112,"max_frames":60}}'`**: To set `image`, `video`, and `audio` input limits. -This can reduce GPU memory usage, improve inference speed, and help to avoid OOM, but may impact model performance, thus set a proper value based on your specific use case. Currently, only `qwen_vl` supports this config. Please refer to [qwen_vl processor](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/multimodal/processors/qwen_vl.py) for understanding the meaning of each parameter. +This can reduce GPU memory usage, improve inference speed, and help to avoid OOM, but may impact model performance, thus set a proper value based on your specific use case. The config entries are passed as `images_kwargs`, `videos_kwargs`, and `audio_kwargs` to the HuggingFace processor, so each modality's settings are kept separate and do not collide. Refer to the HuggingFace documentation for your model's processor to understand the available parameters. ### Bidirectional Attention in Multimodal Model Serving **Note for serving the Gemma-3 multimodal model**: @@ -300,7 +361,6 @@ As mentioned in [Welcome Gemma 3: Google's all new multimodal, multilingual, lon ](https://huggingface.co/blog/gemma3#multimodality), Gemma-3 employs bidirectional attention between image tokens during the prefill phase. Currently, SGLang only supports bidirectional attention when using the Triton Attention Backend. Note, however, that SGLang's current bidirectional attention implementation is incompatible with both CUDA Graph and Chunked Prefill. To enable bidirectional attention, you can use the `TritonAttnBackend` while disabling CUDA Graph and Chunked Prefill. Example launch command: - ```bash Bidirectional Attention python -m sglang.launch_server \ --model-path google/gemma-3-4b-it \ @@ -311,6 +371,5 @@ python -m sglang.launch_server \ --disable-cuda-graph \ # Disable Cuda Graph --chunked-prefill-size -1 # Disable Chunked Prefill ``` - If higher serving performance is required and a certain degree of accuracy loss is acceptable, you may choose to use other attention backends, and you can also enable features like CUDA Graph and Chunked Prefill for better performance, but note that the model will fall back to using causal attention instead of bidirectional attention. diff --git a/docs_new/docs/supported-models/new-model-support.mdx b/docs_new/docs/supported-models/new-model-support.mdx deleted file mode 100644 index cbaaedfca..000000000 --- a/docs_new/docs/supported-models/new-model-support.mdx +++ /dev/null @@ -1,307 +0,0 @@ ---- -title: "How to Support New Models" -description: "This document explains how to add support for new language models and multimodal large language models (MLLMs) in SGLang. It also covers how to test new models and register external implementations." ---- - -## How to Support a New Language Model - -To support a new model in SGLang, you only need to add a single file under the [SGLang Models Directory](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/models). You can learn from existing model implementations and create a new file for your model. For most models, you should be able to find a similar model to start with (e.g., starting from Llama). Also refer how to [port a Model from vLLM to SGLang](#port-a-model-from-vllm-to-sglang). - -## How to Support a New Multimodal Large Language Model - -To support a new multimodal large language model (MLLM) in SGLang, there are several key components in addition to the standard LLM support: - -1. **Register your new model as multimodal:** -Extend `is_multimodal_model` in [model\_config.py](https://github.com/sgl-project/sglang/blob/0ab3f437aba729b348a683ab32b35b214456efc7/python/sglang/srt/configs/model_config.py#L561) to return `True` for your model. - -2. **Register a new chat-template:** -Only when your default chat-template is unable to accept images as input, register a new chat template in [conversation.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/conversation.py) and the corresponding matching function. - -3. **Add a multimodal data processor:** -Define a new `Processor` class that inherits from `BaseMultimodalProcessor` and register this processor as your model's dedicated processor. See [multimodal\_processor.py](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/multimodal/processors) for more details. - -4. **Handle multimodal tokens:** -Implement a `pad_input_ids` function for your new model. In this function, multimodal tokens in the prompt should be expanded (if necessary) and padded with multimodal-data-hashes so that SGLang can recognize different multimodal data with `RadixAttention`. - -5. **Handle image feature extraction:** -Implement a `get_image_feature` function for your new model, which extracts image features from raw image data and converts them into the embeddings used by the language model. - -6. **Adapt to vision attention:** -Adapt the multi-headed `Attention` of ViT with SGLang's `VisionAttention`. - -You can refer to [Qwen2VL](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/models/qwen2_vl.py) or other mllm implementations. These models demonstrate how to correctly handle both multimodal and textual inputs. - -## Testing and Debugging - -Please note all your testing and benchmarking results in PR description. - -### Interactive Debugging - -For interactive debugging, compare the outputs of Hugging Face/Transformers and SGLang. The following two commands should give the same text output and very similar prefill logits: - - -```bash Get reference output -python3 scripts/playground/reference_hf.py --model-path [new model] --model-type {text,mllm} -``` -```bash Get SGLang output -python3 -m sglang.bench_one_batch --correct --model [new model] -``` - - -### Add the Model to the Test Suite - -To ensure the new model is well maintained, add it to the test suite by including it in the `ALL_OTHER_MODELS` list in the [test\_generation\_models.py](https://github.com/sgl-project/sglang/blob/main/test/srt/models/test_generation_models.py) file, test the new model on your local machine and report the results on demonstrative benchmarks (GSM8K, MMLU, MMMU, MMMU-Pro, etc.) in your PR. - -For VLMs, also include a test in `test_vision_openai_server_{x}.py` (e.g. [test\_vision\_openai\_server\_a.py](https://github.com/sgl-project/sglang/blob/main/test/srt/test_vision_openai_server_a.py), [test\_vision\_openai\_server\_b.py](https://github.com/sgl-project/sglang/blob/main/test/srt/test_vision_openai_server_b.py)). - -This is an example command to run to test a new model on your local machine: - - -```bash Run Test -ONLY_RUN=Qwen/Qwen2-1.5B python3 -m unittest test_generation_models.TestGenerationModels.test_others -``` - - -### Benchmark - - - - Follow the MMMU benchmark [README](https://github.com/sgl-project/sglang/blob/main/benchmark/mmmu/README) to get SGLang vs. HF Transformer accuracy comparison. The accuracy score from SGLang run should not be much lower than that from HF Transformer run. Similarly, follow the [benchmark and profiling guide](../developer_guide/benchmark_and_profiling) to get performance comparison: TTFT and throughput must meet or exceed baselines (e.g., HF Transformer). - - - If you ran other evals, please note the results in PR description. - - - -## Port a Model from vLLM to SGLang - -The [vLLM Models Directory](https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/models) is a valuable resource, as vLLM covers many models. SGLang reuses vLLM's interface and some layers, making it easier to port models from vLLM to SGLang. - -To port a model from vLLM to SGLang: - -- Compare these two files for guidance: - - [SGLang Llama Implementation](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/models/llama.py) - - [vLLM Llama Implementation](https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/llama.py) -- The major differences include: - - **Replace vLLM's `Attention` with `RadixAttention`** (ensure you pass `layer_id` to `RadixAttention`). - - **Replace vLLM's `LogitsProcessor` with SGLang's `LogitsProcessor`.** - - **Replace the multi-headed `Attention` of ViT with SGLang's `VisionAttention`.** - - **Replace other vLLM layers** (such as `RMSNorm`, `SiluAndMul`) with SGLang layers. - - **Remove `Sample`.** - - **Change the `forward()` functions** and add a `forward_batch()` method. - - **Add `EntryClass`** at the end. - - **Ensure that the new implementation uses only SGLang components** and does not rely on any vLLM components. - -Make sure you add your new model to the supported models list in the supported models documentation. - -## Registering an External Model Implementation - -In addition to the methods above, you can register your new model with the `ModelRegistry` before launching the server. This allows you to integrate your model without modifying the source code. - -For example: - - -```python Register Model -from sglang.srt.models.registry import ModelRegistry -from sglang.srt.entrypoints.http_server import launch_server - -# For a single model, add it to the registry: -ModelRegistry.models[model_name] = model_class - -# For multiple models, you can imitate the import_model_classes() function: -from functools import lru_cache - -@lru_cache() -def import_new_model_classes(): - model_arch_name_to_cls = {} - # Populate model_arch_name_to_cls with your new model classes. - ... - return model_arch_name_to_cls - -ModelRegistry.models.update(import_new_model_classes()) - -# Launch the server with your server arguments: -launch_server(server_args) -``` - - -## Example: Implementing and Serving a Llama Wrapper Model - -Below is an introductory, step-by-step walkthrough on how to implement a new model end-to-end in SGLang and then run it via the [Offline Engine](../basic_usage/offline_engine_api). - -### Implementing Our Model - -To keep things simple, this new model will be a simple wrapper around [Llama 3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct), and our goal will be just to bias the output logits for each `forward` call by taking the square root of each individual logit. - -Let's start by defining our model in a file called `llama_wrapper.py`. The first step is to import the necessary libraries from SRT, which is SGLang's internal backend. - -```python llama_wrapper.py -import torch -from transformers import LlamaConfig -from typing import Optional -from sglang.srt.layers.logits_processor import LogitsProcessorOutput -from sglang.srt.layers.quantization.base_config import QuantizationConfig -from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors - -from sglang.srt.models.llama import LlamaForCausalLM -``` - -Next, we declare a new `class` for our model and have it inherit from `LlamaForCausalLM`, which allows our model to access `LlamaForCausalLM`'s predefined modules and layers, such as `LlamaAttention` and `LlamaMLP`. Note that almost all model implementations take in `config` and `quant_config` as arguments for their `__init__` method; `config` and `quant_config` are passed in via [`model_loader/loader.py`](https://github.com/sgl-project/sglang/blob/bf72b80122fd888bf619d17b96fa3e323ab809fc/python/sglang/srt/model_loader/loader.py#L219). Because we have inherited from `LlamaForCausalLM`, we can pass our parameters directly to its constructor, which will set the member variables for us. - - -```python Class Definition -class LlamaWrapper(LlamaForCausalLM): - def __init__( - self, - config: LlamaConfig, - quant_config: Optional[QuantizationConfig] = None, - prefix: str = "", - ) -> None: - super().__init__(config=config, quant_config=quant_config, prefix=prefix) -``` - - -Now, we want to define the `forward` method, which is what will be called at inference time. Note that the signature for `forward` is essentially the same for any model; you can take a look at the other models defined in the [`models` directory](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/models/) for references. To see where exactly `forward` is called in the SGLang runtime's internals, take a look at [`forward_decode`](https://github.com/sgl-project/sglang/blob/bf72b80122fd888bf619d17b96fa3e323ab809fc/python/sglang/srt/model_executor/model_runner.py#L1705) and [`forward_extend`](https://github.com/sgl-project/sglang/blob/bf72b80122fd888bf619d17b96fa3e323ab809fc/python/sglang/srt/model_executor/model_runner.py#L1724) in the [`ModelRunner` class](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/model_executor/model_runner.py). - - -```python Forward Method Signature - @torch.no_grad() - def forward( - self, - input_ids: torch.Tensor, - positions: torch.Tensor, - forward_batch: ForwardBatch, - pp_proxy_tensors: Optional[PPProxyTensors] = None, - input_embeds: Optional[torch.Tensor] = None, - get_embedding: bool = False, - ) -> LogitsProcessorOutput: -``` - - -We now call the `__call__` method for `self.model` (which is a member variable that `LlamaForCausalLM` defines in its `__init__` method), which eventually calls `LlamaForCausalLM`'s `forward` method. After that, we feed the `hidden_states` into our model's `LogitsProcessor` (again defined in `LlamaForCausalLM`). - - -```python Call Model and LogitsProcessor - hidden_states = self.model( - input_ids, - positions, - forward_batch, - input_embeds, - pp_proxy_tensors=pp_proxy_tensors, - ) - - res: LogitsProcessorOutput = self.logits_processor( - input_ids, - hidden_states, - self.lm_head, - forward_batch, - ) -``` - - -After receiving the logits for the next token, we can finally perform our biasing step. - - -```python Logit Biasing - orig_logits = res.next_token_logits - res.next_token_logits = torch.where( - orig_logits > 0, - orig_logits.sqrt(), - orig_logits - ) - - return res -``` - - -Now, our `LlamaWrapper` model is created and ready to be served! - -### Serving Our Model Via SGLang's Offline Engine - -The next step of this walkthrough involves hosting our new model offline, so that it can be served locally and without an HTTP server. - -First, create a new file called `run.py`. Now, we must ensure that SGLang's `ModelRegistry` can find our model. To do this, we first download the model's configuration and weights from Huggingface. - -```python run.py -import asyncio -from functools import lru_cache -from huggingface_hub import snapshot_download -from llama_wrapper import LlamaWrapper # Make sure to import our new model! -import sglang as sgl -from sglang.srt.models.registry import ModelRegistry - -# Make sure to request access to this model on Huggingface, then export your -# `HF_TOKEN` to download the model snapshot -llama_dir = snapshot_download( - repo_id="meta-llama/Llama-3.1-8B-Instruct", - local_dir="./llama_ckpt", -) -``` - -Now that we have our model on disk, we want to point it to `LlamaWrapper` by changing the `architectures` field in `./llama_ckpt/config.json` to be `LlamaWrapper`. That way, when we pass in the path of our model checkpoint to SGLang, it will know that we want to use "LlamaWrapper" instead of "LlamaForCausalLM" as our model. - -```json ./llama_ckpt/config.json -{ - "architectures": [ - # "LlamaForCausalLM" - "LlamaWrapper" - ], - ... -} -``` - -However, if we don't link our `LlamaWrapper` class to the "LlamaWrapper" registry keyword, then SGLang won't be able to find our model. Thus, to register our `LlamaWrapper`, we want to follow the steps in the above section titled "Registering an External Model Implementation". - - -```python Register LlamaWrapper -@lru_cache() -def import_new_model_classes(): - model_arch_name_to_cls = {"LlamaWrapper": LlamaWrapper} - return model_arch_name_to_cls - -ModelRegistry.models.update(import_new_model_classes()) -``` - - -Lastly, when we create our `Engine`, we just pass in the path to the local model directory. Then, our `LlamaWrapper` is ready to be served; for this walkthrough, we will use SGLang `Engine`'s non-streaming asynchronous generation endpoint. - - -```python Run Model -def main(): - llm = sgl.Engine(model_path="./llama_ckpt") - sampling_params = {"temperature": 0.2, "top_k": 5} - prompts = [ - "Write a short, neutral self-introduction for a fictional character. Hello, my name is", - "Provide a concise factual statement about France's capital city. The capital of France is", - "Explain possible future trends in artificial intelligence. The future of AI is", - ] - - asyncio.run(run_llm(llm, sampling_params, prompts)) - - llm.shutdown() - -async def run_llm( - llm, - sampling_params, - prompts, -) -> None: - outputs = await llm.async_generate(prompts, sampling_params) - - for prompt, output in zip(prompts, outputs): - print(f"\nPrompt: {prompt}") - print(f"Generated text: {output['text']}") - -if __name__ == "__main__": - main() -``` - - -Now, when we call `python run.py`, we will get the outputs of our newly created model! - -## Documentation - -Add to table of supported models in [generative\_models](/docs/supported-models/large-language-models) or [multimodal\_language\_models](/docs/supported-models/vision-language-models). - ---- - -By following these guidelines, you can add support for new language models and multimodal large language models in SGLang and ensure they are thoroughly tested and easily integrated into the system. diff --git a/docs_new/docs/supported-models/rerank-models.mdx b/docs_new/docs/supported-models/rerank_models.mdx similarity index 83% rename from docs_new/docs/supported-models/rerank-models.mdx rename to docs_new/docs/supported-models/rerank_models.mdx index dab840c6c..034896dc8 100644 --- a/docs_new/docs/supported-models/rerank-models.mdx +++ b/docs_new/docs/supported-models/rerank_models.mdx @@ -1,9 +1,9 @@ --- title: Rerank models --- - SGLang offers comprehensive support for rerank models by incorporating optimized serving frameworks with a flexible programming interface. This setup enables efficient processing of cross-encoder reranking tasks, improving the accuracy and relevance of search result ordering. SGLang’s design ensures high throughput and low latency during reranker model deployment, making it ideal for semantic-based result refinement in large-scale retrieval systems. + Rerank models in SGLang fall into two categories: - **Cross-encoder rerank models**: run with `--is-embedding` (embedding runner). @@ -12,15 +12,40 @@ Rerank models in SGLang fall into two categories: - Multimodal (e.g. Qwen3-VL-Reranker): also supports image/video content Some models may require `--trust-remote-code`. - + ## Supported rerank models -| Model Family (Rerank) | Example HuggingFace Identifier | Chat Template | Description | -|------------------------------------------------|--------------------------------------|---------------|----------------------------------------------------------------------------------------------------------------------------------| -| **BGE-Reranker (BgeRerankModel)** | `BAAI/bge-reranker-v2-m3` | N/A | Currently only support `attention-backend` `triton` and `torch_native`. High-performance cross-encoder reranker model from BAAI. Suitable for reranking search results based on semantic relevance. | -| **Qwen3-Reranker (decoder-only yes/no)** | `Qwen/Qwen3-Reranker-8B` | `examples/chat_template/qwen3_reranker.jinja` | Decoder-only reranker using next-token logprob scoring for labels (yes/no). Launch **without** `--is-embedding`. | -| **Qwen3-VL-Reranker (multimodal yes/no)** | `Qwen/Qwen3-VL-Reranker-2B` | `examples/chat_template/qwen3_vl_reranker.jinja` | Multimodal decoder-only reranker supporting text, images, and videos. Uses yes/no logprob scoring. Launch **without** `--is-embedding`. | +
Model FamilyExample IdentifierVideo notes
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Model Family (Rerank)Example HuggingFace IdentifierChat TemplateDescription
BGE-Reranker (BgeRerankModel)BAAI/bge-reranker-v2-m3N/ACurrently only support attention-backend triton and torch_native. High-performance cross-encoder reranker model from BAAI. Suitable for reranking search results based on semantic relevance.
Qwen3-Reranker (decoder-only yes/no)Qwen/Qwen3-Reranker-8Bexamples/chat_template/qwen3_reranker.jinjaDecoder-only reranker using next-token logprob scoring for labels (yes/no). Launch without --is-embedding.
Qwen3-VL-Reranker (multimodal yes/no)Qwen/Qwen3-VL-Reranker-2Bexamples/chat_template/qwen3_vl_reranker.jinjaMultimodal decoder-only reranker supporting text, images, and videos. Uses yes/no logprob scoring. Launch without --is-embedding.
## Cross-Encoder Rerank (embedding runner) @@ -88,7 +113,9 @@ python3 -m sglang.launch_server \ --chat-template examples/chat_template/qwen3_reranker.jinja ``` + Qwen3-Reranker uses decoder-only logprob scoring (yes/no). Do NOT launch it with `--is-embedding`. + ### Example Client Request (supports optional instruct, top_n, and return_documents) @@ -160,6 +187,7 @@ Example (with `top_n: 2`): ### Common Pitfalls +- **`--chat-template` is required.** Without `--chat-template examples/chat_template/qwen3_reranker.jinja`, the server does not recognize the model as a decoder-only reranker and returns a 400 error: `"This model does not appear to be an embedding model by default. Please add `--is-embedding`..."`. The fix is to add the chat template flag, NOT `--is-embedding`. - If you launch Qwen3-Reranker with `--is-embedding`, `/v1/rerank` cannot compute yes/no logprob scores. Relaunch **without** `--is-embedding`. - If you see a validation error like "score should be a valid number" and the backend returned a list, upgrade to a version that coerces `embedding[0]` into `score` for rerank responses. @@ -179,8 +207,9 @@ python3 -m sglang.launch_server \ --chat-template examples/chat_template/qwen3_vl_reranker.jinja ``` + Qwen3-VL-Reranker uses decoder-only logprob scoring (yes/no) like Qwen3-Reranker. Do NOT launch it with `--is-embedding`. - + ### Text-Only Reranking (backward compatible) @@ -295,9 +324,9 @@ for item in results: ### Request Parameters (Multimodal) - `query` (required): Can be a string (text-only) or a list of content parts: - - `{"type": "text", "text": "..."}` for text - - `{"type": "image_url", "image_url": {"url": "..."}}` for images - - `{"type": "video_url", "video_url": {"url": "..."}}` for videos + - `{"type": "text", "text": "..."}` for text + - `{"type": "image_url", "image_url": {"url": "..."}}` for images + - `{"type": "video_url", "video_url": {"url": "..."}}` for videos - `documents` (required): List where each document can be a string or list of content parts (same format as query) - `instruct` (optional): Instruction text for the reranker - `top_n` (optional): Maximum number of documents to return diff --git a/docs_new/docs/supported-models/reward-models.mdx b/docs_new/docs/supported-models/reward_models.mdx similarity index 100% rename from docs_new/docs/supported-models/reward-models.mdx rename to docs_new/docs/supported-models/reward_models.mdx diff --git a/docs_new/docs/supported-models/support_new_models.mdx b/docs_new/docs/supported-models/support_new_models.mdx new file mode 100644 index 000000000..ba7836732 --- /dev/null +++ b/docs_new/docs/supported-models/support_new_models.mdx @@ -0,0 +1,522 @@ +--- +title: "How to Support New Models" +description: "This document explains how to add support for new language models and multimodal large language models (MLLMs) in SGLang. It also covers how to test new models and register external implementations." +--- +This document explains how to add support for new language models and multimodal large language models (MLLMs) in +SGLang. It also covers how to test new models and register external implementations. + +## How to Support a New Language Model + +To support a new model in SGLang, you only need to add a single file under +the [SGLang Models Directory](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/models). You can learn +from existing model implementations and create a new file for your model. For most models, you should be able to find a +similar model to start with (e.g., starting from Llama). Also refer how +to [port a Model from vLLM to SGLang](#port-a-model-from-vllm-to-sglang) + +## How to Support a New Multimodal Large Language Model + +To support a new multimodal large language model (MLLM) in SGLang, there are several key components in addition to the +standard LLM support: + +1. **Register your new model as multimodal**: + Extend `is_multimodal_model` + in [model_config.py](https://github.com/sgl-project/sglang/blob/0ab3f437aba729b348a683ab32b35b214456efc7/python/sglang/srt/configs/model_config.py#L561) + to return `True` for your model. + +2. **Register a new chat-template**: + Only when your default chat-template is unable to accept images as input: Register a new chat template in [conversation.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/parser/conversation.py) and the corresponding matching function. + +3. **Multimodal Data Processor**: + Define a new `Processor` class that inherits from `BaseMultimodalProcessor` and register this processor as your + model’s dedicated processor. + See [multimodal_processor.py](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/multimodal/processors) + for more details. + +4. **Handle Multimodal Tokens**: + Implement a `pad_input_ids` function for your new model. In this function, multimodal tokens in the prompt should be + expanded (if necessary) and padded with multimodal-data-hashes so that SGLang can recognize different multimodal data + with `RadixAttention`. + +5. **Handle Image Feature Extraction**: + Implement a `get_image_feature` function for your new model, which extracts image features from raw image data and converts them into the embeddings used by the language model. + +6. **Adapt to Vision Attention**: + Adapt the multi-headed `Attention` of ViT with SGLang’s `VisionAttention`. + +You can refer to [Qwen2VL](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/models/qwen2_vl.py) or +other mllm implementations. These models demonstrate how to correctly handle both multimodal and textual inputs. + +## Testing and Debugging + +Please note all your testing and benchmarking results in PR description. + +### Interactive Debugging + +For interactive debugging, compare the outputs of Hugging Face/Transformers and SGLang. The following two commands +should give the same text output and very similar prefill logits: + +- Get the reference output: + ```bash Command + python3 scripts/playground/reference_hf.py --model-path [new model] --model-type {text,vlm} + ``` +- Get the SGLang output: + ```bash Command + python3 -m sglang.bench_one_batch --correct --model [new model] + ``` + +### Add the Model to the Test Suite + +To ensure the new model is well maintained, add it to the test suite by including it in the `ALL_OTHER_MODELS` list in +the [test_generation_models.py](https://github.com/sgl-project/sglang/blob/main/test/registered/models/test_generation_models.py) +file, test the new model on your local machine and report the results on demonstrative benchmarks (GSM8K, MMLU, MMMU, +MMMU-Pro, etc.) in your PR. \\ +For VLMs, also include a test in `test_vision_openai_server_{x}.py` (e.g. [test_vision_openai_server_a.py](https://github.com/sgl-project/sglang/blob/main/test/registered/vlm/test_vision_openai_server_a.py)). + +This is an example command to run to test a new model on your local machine: + +```bash Run Test +ONLY_RUN=Qwen/Qwen2-1.5B python3 -m unittest test_generation_models.TestGenerationModels.test_others +``` + +### Benchmark + +- **(Required) MMMU**: follow MMMU benchmark [README.md](https://github.com/sgl-project/sglang/blob/main/benchmark/mmmu/README.md) to get SGLang vs. HF Transformer accuracy comparison. The accuracy score from SGLang run should not be much lower than that from HF Transformer run. Similarly, follow https://docs.sglang.io/developer_guide/benchmark_and_profiling.html to get performance comparison: TTFT and throughput must meet or exceed baselines (e.g., HF Transformer). +- **(Optional) Other evals**: If you ran other evals, please note the results in PR description. + +## Port a Model from vLLM to SGLang + +The [vLLM Models Directory](https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/models) is a valuable +resource, as vLLM covers many models. SGLang reuses vLLM’s interface and some layers, making it easier to port models +from vLLM to SGLang. + +To port a model from vLLM to SGLang: + +- Compare these two files for guidance: + - [SGLang Llama Implementation](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/models/llama.py) + - [vLLM Llama Implementation](https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/llama.py) +- The major differences include: + - **Replace vLLM’s `Attention` with `RadixAttention`** (ensure you pass `layer_id` to `RadixAttention`). + - **Replace vLLM’s `LogitsProcessor` with SGLang’s `LogitsProcessor`.** + - **Replace the multi-headed `Attention` of ViT with SGLang’s `VisionAttention`.** + - **Replace other vLLM layers** (such as `RMSNorm`, `SiluAndMul`) with SGLang layers. + - **Remove `Sample`.** + - **Change the `forward()` functions** and add a `forward_batch()` method. + - **Add `EntryClass`** at the end. + - **Ensure that the new implementation uses only SGLang components** and does not rely on any vLLM components. + +Note: make sure you add your new model to the supported models list in the supported models documentation. + +## Registering an External Model Implementation + +In addition to the methods above, you can register your new model with the `ModelRegistry` before launching the server. +This allows you to integrate your model without modifying the source code. + +For example: + +```python Register Model +from sglang.srt.models.registry import ModelRegistry +from sglang.srt.entrypoints.http_server import launch_server + +# For a single model, add it to the registry: +ModelRegistry.models[model_name] = model_class + +# For multiple models, you can imitate the import_model_classes() function: +from functools import lru_cache + +@lru_cache() +def import_new_model_classes(): + model_arch_name_to_cls = {} + # Populate model_arch_name_to_cls with your new model classes. + ... + return model_arch_name_to_cls + +ModelRegistry.models.update(import_new_model_classes()) + +# Launch the server with your server arguments: +launch_server(server_args) +``` + +## Example: Implementing and Serving a Llama Wrapper Model + +Below is an introductory, step-by-step walkthrough on how to implement a new model end-to-end in SGLang and then run it via the [Offline Engine](../basic_usage/offline_engine_api). + +### Implementing Our Model + +To keep things simple, this new model will be a simple wrapper around [Llama 3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct), and our goal will be just to bias the output logits for each `forward` call by taking the square root of each individual logit. + +Let's start by defining our model in a file called `llama_wrapper.py`. +The first step is to import the necessary libraries from SRT, which is SGLang's internal backend. + +```python Example +# In the file `llama_wrapper.py` + +import torch +from transformers import LlamaConfig +from typing import Optional +from sglang.srt.layers.logits_processor import LogitsProcessorOutput +from sglang.srt.layers.quantization.base_config import QuantizationConfig +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors + +from sglang.srt.models.llama import LlamaForCausalLM +``` + +Next, we declare a new `class` for our model and have it inherit from `LlamaForCausalLM`, which allows our model to access `LlamaForCausalLM`'s predefined modules and layers, such as `LlamaAttention` and `LlamaMLP`. +Note that almost all model implementations take in `config` and `quant_config` as arguments for their `__init__` method; `config` and `quant_config` are passed in via [`model_loader/loader.py`](https://github.com/sgl-project/sglang/blob/bf72b80122fd888bf619d17b96fa3e323ab809fc/python/sglang/srt/model_loader/loader.py#L219). +Because we have inherited from `LlamaForCausalLM`, we can pass our parameters directly to its constructor, which will set the member variables for us. + +```python Class Definition +class LlamaWrapper(LlamaForCausalLM): + def __init__( + self, + config: LlamaConfig, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__(config=config, quant_config=quant_config, prefix=prefix) +``` + +Now, we want to define the `forward` method, which is what will be called at inference time. +Note that the signature for `forward` is essentially the same for any model; you can take a look at the other models defined in the [`models` directory](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/models/) for references. +To see where exactly `forward` is called in the SGLang runtime's internals, take a look at [`forward_decode`](https://github.com/sgl-project/sglang/blob/bf72b80122fd888bf619d17b96fa3e323ab809fc/python/sglang/srt/model_executor/model_runner.py#L1705) and [`forward_extend`](https://github.com/sgl-project/sglang/blob/bf72b80122fd888bf619d17b96fa3e323ab809fc/python/sglang/srt/model_executor/model_runner.py#L1724) in the [`ModelRunner` class](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/model_executor/model_runner.py). + +```python Forward Method Signature + @torch.no_grad() + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + forward_batch: ForwardBatch, + pp_proxy_tensors: Optional[PPProxyTensors] = None, + input_embeds: Optional[torch.Tensor] = None, + get_embedding: bool = False, + ) -> LogitsProcessorOutput: +``` + +We now call the `__call__` method for `self.model` (which is a member variable that `LlamaForCausalLM` defines in its `__init__` method), which eventually calls `LlamaForCausalLM`'s `forward` method. +After that, we feed the `hidden_states` into our model's `LogitsProcessor` (again defined in `LlamaForCausalLM`). + +```python Call Model and LogitsProcessor + hidden_states = self.model( + input_ids, + positions, + forward_batch, + input_embeds, + pp_proxy_tensors=pp_proxy_tensors, + ) + + res: LogitsProcessorOutput = self.logits_processor( + input_ids, + hidden_states, + self.lm_head, + forward_batch, + ) +``` + +After receiving the logits for the next token, we can finally perform our biasing step. + +```python Logit Biasing + orig_logits = res.next_token_logits + res.next_token_logits = torch.where( + orig_logits > 0, + orig_logits.sqrt(), + orig_logits + ) + + return res +``` + +Now, our `LlamaWrapper` model is created and ready to be served! + +### Serving Our Model Via SGLang's Offline Engine + +The next step of this walkthrough involves hosting our new model offline, so that it can be served locally and without an HTTP server. + +First, create a new file called `run.py`. +Now, we must ensure that SGLang's `ModelRegistry` can find our model. +To do this, we first download the model's configuration and weights from Huggingface. + +```python Example +# In the file `run.py` + +import asyncio +from functools import lru_cache +from huggingface_hub import snapshot_download +from llama_wrapper import LlamaWrapper # Make sure to import our new model! +import sglang as sgl +from sglang.srt.models.registry import ModelRegistry + +# Make sure to request access to this model on Huggingface, then export your +# `HF_TOKEN` to download the model snapshot +llama_dir = snapshot_download( + repo_id="meta-llama/Llama-3.1-8B-Instruct", + local_dir="./llama_ckpt", +) +``` + +Now that we have our model on disk, we want to point it to `LlamaWrapper` by changing the `architectures` field in `./llama_ckpt/config.json` to be `LlamaWrapper`. +That way, when we pass in the path of our model checkpoint to SGLang, it will know that we want to use "LlamaWrapper" instead of "LlamaForCausalLM" as our model. + +```python Example +{ + "architectures": [ + # "LlamaForCausalLM" + "LlamaWrapper" + ], + ... +} +``` + +However, if we don't link our `LlamaWrapper` class to the "LlamaWrapper" registry keyword, then SGLang won't be able to find our model. +Thus, to register our `LlamaWrapper`, we want to follow the steps in the above section titled "Registering an External Model Implementation". + +```python Register LlamaWrapper +@lru_cache() +def import_new_model_classes(): + model_arch_name_to_cls = {"LlamaWrapper": LlamaWrapper} + return model_arch_name_to_cls + +ModelRegistry.models.update(import_new_model_classes()) +``` + +Lastly, when we create our `Engine`, we just pass in the path to the local model directory. +Then, our `LlamaWrapper` is ready to be served; for this walkthrough, we will use SGLang `Engine`'s non-streaming asynchronous generation endpoint. + +```python Example +def main(): + llm = sgl.Engine(model_path="./llama_ckpt") + sampling_params = {"temperature": 0.2, "top_k": 5} + prompts = [ + "Write a short, neutral self-introduction for a fictional character. Hello, my name is", + "Provide a concise factual statement about France’s capital city. The capital of France is", + "Explain possible future trends in artificial intelligence. The future of AI is", + ] + + asyncio.run(run_llm(llm, sampling_params, prompts)) + + llm.shutdown() + +async def run_llm( + llm, + sampling_params, + prompts, +) -> None: + outputs = await llm.async_generate(prompts, sampling_params) + + for prompt, output in zip(prompts, outputs): + print(f"\nPrompt: {prompt}") + print(f"Generated text: {output['text']}") + +if __name__ == "__main__": + main() +``` + +Now, when we call `python run.py`, we will get the outputs of our newly created model! + +## Serving External Models via the Standard CLI + +The previous sections show how to register a model programmatically via `ModelRegistry` and serve it through the Offline Engine. Similar to vLLM model plugin, there is an alternative that lets you keep using the standard `python -m sglang.launch_server` CLI without modifying any SGLang source code: you can register your model using the `SGLANG_EXTERNAL_MODEL_PACKAGE` environment variable. + +### The `EntryClass` Variable + +When SGLang scans a model package, it looks for the variable `EntryClass` at the module level of your Python file. The [model registry](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/models/registry.py) imports your file, checks for `EntryClass`, and registers the class assigned to it. If you are using a model based on HuggingFace, the name of this class needs to match the `"architectures"` field in your model's `config.json`. + +For example, if you are implementing a Llama wrapper, add this line at the end of your model file: + +```python Example +# This is what "Add EntryClass at the end" means +EntryClass = LlamaWrapper +``` + +### Example: Text-Only Model + +Using the same Llama wrapper from the previous section, here is how to package and serve it via the CLI. + +1. Create your project + +``` +sglang_custom_project/ +|----setup.py +|----custom_llm/ + |----__init__.py + |----llama_wrapper.py +``` + +Write the `setup.py`: + +```python Example +# sglang_custom_project/setup.py + +from setuptools import setup, find_packages +setup( + name="sglang-custom-plugins", + version="0.1", + packages=find_packages(), +) +``` + +2. Write your model code + +Inside `llama_wrapper.py`, write your model and include `EntryClass`: + +```python Example +# sglang_custom_project/custom_llm/llama_wrapper.py + +import torch +from typing import Optional +from sglang.srt.layers.logits_processor import LogitsProcessorOutput +from sglang.srt.layers.quantization.base_config import QuantizationConfig +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors +from sglang.srt.models.llama import LlamaForCausalLM + +class LlamaWrapper(LlamaForCausalLM): + def __init__(self, config, quant_config: Optional[QuantizationConfig] = None, + prefix: str = "") -> None: + super().__init__(config=config, quant_config=quant_config, prefix=prefix) + @torch.no_grad() + def forward(self, input_ids, positions, forward_batch, + pp_proxy_tensors=None, input_embeds=None, get_embedding=False): + hidden_states = self.model( + input_ids, positions, forward_batch, input_embeds, + pp_proxy_tensors=pp_proxy_tensors, + ) + res: LogitsProcessorOutput = self.logits_processor( + input_ids, hidden_states, self.lm_head, forward_batch, + ) + + orig = res.next_token_logits + res.next_token_logits = torch.where(orig > 0, orig.sqrt(), orig) + return res + +# Don't forget to add EntryClass +EntryClass = LlamaWrapper +``` + +3. Install your package + +Run this inside your `sglang_custom_project` directory to install your code into the active Python environment: + +```bash Command +pip install -e . +``` + +4. Update your `config.json` + +Update the `config.json` under your HuggingFace model checkpoint directory so the `architectures` field matches your class name: + +```json Config +{ + "architectures": ["LlamaWrapper"], + ... +} +``` + +5. Launch the server + +Set the environment variable before running the CLI: + +```bash Command +export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_llm +python -m sglang.launch_server \ + --model-path /path/to/Llama-3.1-8B-Instruct \ + --port 8000 +``` + +The `SGLANG_EXTERNAL_MODEL_PACKAGE` should be the parent folder name containing your model-related code. In this example, it should be `custom_llm`. + +### Example: Multimodal Model + +If you are working with multimodal models, setting `SGLANG_EXTERNAL_MODEL_PACKAGE` alone is not enough. SGLang also needs to recognize your architecture as multimodal to enable the image/video processing pipelines, and it needs a custom processor. + +You can handle this by setting two additional environment variables: + +- `SGLANG_EXTERNAL_MM_MODEL_ARCH`: Adds your architecture name to SGLang's internal list of multimodal models. +- `SGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE`: Tells SGLang where to find your custom processor class. + +For example, let's build a custom model based on Qwen2-VL-Instruct that takes the square root of the logits. + +Create the project: + +``` +sglang_custom_project_vl/ +|----setup.py +|----custom_vlm/ + |----__init__.py + |----qwenvl_wrapper.py +``` + +Write `setup.py`: + +```python Example +# sglang_custom_project_vl/setup.py + +from setuptools import setup, find_packages +setup( + name="sglang-custom-plugins-vl", + version="0.1", + packages=find_packages(), +) +``` + +Write the model in `qwenvl_wrapper.py`: + +```python Example +# sglang_custom_project_vl/custom_vlm/qwenvl_wrapper.py +import torch +from sglang.srt.models.qwen2_vl import Qwen2VLForConditionalGeneration +from sglang.srt.multimodal.processors.qwen_vl import QwenVLImageProcessor + +class CustomQwen2VL(Qwen2VLForConditionalGeneration): + def forward(self, input_ids, positions, forward_batch, + input_embeds=None, get_embedding=False): + res = super().forward( + input_ids, positions, forward_batch, + input_embeds=input_embeds, get_embedding=get_embedding + ) + if not get_embedding: + orig = res.next_token_logits + res.next_token_logits = torch.where(orig > 0, orig.sqrt(), orig) + return res + +class CustomQwen2VLProcessor(QwenVLImageProcessor): + models = [CustomQwen2VL] + + def __init__(self, hf_config, server_args, _processor, *args, **kwargs): + super().__init__(hf_config, server_args, _processor, *args, **kwargs) + +EntryClass = CustomQwen2VL +``` + +**Note:** you don't need a separate `EntryClass` for the custom processor as long as you associate the processor with the specific model class. + +Install the package, update `config.json`, and launch: + +```bash Command +pip install -e . +``` + +```json Config +{ + "architectures": ["CustomQwen2VL"], + ... +} +``` + +```bash Command +export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_vlm +export SGLANG_EXTERNAL_MM_MODEL_ARCH=CustomQwen2VL +export SGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE=custom_vlm + +python -m sglang.launch_server \ + --model-path /path/to/Qwen2-VL-2B-Instruct \ + --port 8000 \ + --enable-multimodal +``` + +## Documentation + +Add to table of supported models in [generative_models.md](./generative_models) or [multimodal_language_models.md](./multimodal_language_models) + +--- + +By following these guidelines, you can add support for new language models and multimodal large language models in +SGLang and ensure they are thoroughly tested and easily integrated into the system. diff --git a/docs_new/docs/supported-models/transformers-fallback.mdx b/docs_new/docs/supported-models/transformers_fallback.mdx similarity index 64% rename from docs_new/docs/supported-models/transformers-fallback.mdx rename to docs_new/docs/supported-models/transformers_fallback.mdx index f43708ea3..8db61f528 100644 --- a/docs_new/docs/supported-models/transformers-fallback.mdx +++ b/docs_new/docs/supported-models/transformers_fallback.mdx @@ -1,14 +1,12 @@ --- title: "Transformers Fallback in SGLang" --- - `sglang` can fall back to using models that are available in `transformers`. This works for most decoder-style language models and support for vision-language models is coming soon! -## Example Launch Command +## Example launch Command By default, we will use sglang implementation if it is available. Otherwise, we will fall back to transformers one. However, you can switch the implementation by setting `--model-impl` to `transformers`. - ```shell Launch Server python3 -m sglang.launch_server \ --model-path meta-llama/Llama-3.2-1B-Instruct \ @@ -16,27 +14,27 @@ python3 -m sglang.launch_server \ --port 30000 \ --model-impl transformers ``` - -## Supported Features +## Supported features ### Quantization -Transformers fallback has supported most of available quantization in SGLang (except GGUF). See the [Quantization page](../advanced_features/quantization) for more information about supported quantization in SGLang. +Transformers fall back has supported most of available quantization in SGLang (except GGUF). See [Quantization page](../advanced_features/quantization) for more information about supported quantization in SGLang. -### Remote Code +### Remote code This fallback also means that any model on the hub that can be used in `transformers` with `trust_remote_code=True` that correctly implements attention can be used in production! A model just needs the following two things: - -```python Required Implementation +```python Example from transformers import PreTrainedModel from torch import nn class MyAttention(nn.Module): + def forward(self, hidden_states, **kwargs): # <- kwargs are required + ... attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] attn_output, attn_weights = attention_interface( @@ -51,20 +49,11 @@ class MyAttention(nn.Module): class MyModel(PreTrainedModel): _supports_attention_backend = True ``` - Here is what happens in the background: -1. **Load the config** - -The config is loaded. - -2. **Load the model class** - -`MyModel` python class is loaded from the `auto_map`, and we check that the model `_supports_attention_backend`. - -3. **Use the TransformersModel backend** - -The `TransformersModel` backend is used. See `/srt/models/transformers`, which leverages `self.config._attn_implementation = "sglang"`, thus the need to use `ALL_ATTENTION_FUNCTIONS`. +1. The config is loaded +2. `MyModel` python class is loaded from the `auto_map`, and we check that the model `_supports_attention_backend`. +3. The `TransformersModel` backend is used. See `/srt/models/transformers`, which leverages `self.config._attn_implementation = "sglang"`, thus the need to use `ALL_ATTENTION_FUNCTIONS`. That's it! diff --git a/docs_new/index.mdx b/docs_new/index.mdx index be0eb1f3c..cfcd262ae 100644 --- a/docs_new/index.mdx +++ b/docs_new/index.mdx @@ -61,7 +61,7 @@ SGLang is an inference framework meant for production level serving. It is designed to deliver low-latency and high-throughput inference across a wide range of setups, from a single GPU to large distributed clusters. - + Install SGLang with pip, from source, or via Docker on your preferred hardware platform. diff --git a/docs_new/sglang-diffusion/intro.mdx b/docs_new/sglang-diffusion/intro.mdx deleted file mode 100644 index c38b7650d..000000000 --- a/docs_new/sglang-diffusion/intro.mdx +++ /dev/null @@ -1,65 +0,0 @@ ---- -title: SGLang Diffusion -description: Accelerated image and video generation with diffusion models. ---- - -SGLang Diffusion is an inference framework for accelerated image and video generation using diffusion models. It provides an end-to-end unified pipeline with optimized kernels and an efficient scheduler loop. - -## Key features - -* **Broad model support:** Wan series, FastWan series, Hunyuan, Qwen-Image, Qwen-Image-Edit, Flux, Z-Image, GLM-Image, and more -* **Fast inference:** optimized kernels, efficient scheduler loop, and Cache-DiT acceleration -* **Ease of use:** OpenAI-compatible API, CLI, and Python SDK -* **Multi-platform:** NVIDIA GPUs (H100, H200, A100, B200, 4090), AMD GPUs (MI300X, MI325X), and Ascend NPU (A2, A3) - -## Quick start - -1. **Install SGLang Diffusion** - -```bash -uv pip install "sglang[diffusion]" --prerelease=allow -``` - -See the [installation guide](../docs/sglang-diffusion/installation) for more installation methods and ROCm-specific instructions. - -2. **Run a one-off generation** - -```bash -sglang generate --model-path Qwen/Qwen-Image \ - --prompt "A beautiful sunset over the mountains" \ - --save-output -``` - -3. **Serve with the OpenAI-compatible API** - -```bash -sglang serve --model-path Qwen/Qwen-Image --port 30010 -``` - -## CLI quick reference - -### Generate (one-off generation) - -```bash -sglang generate --model-path --prompt "" --save-output -``` - -### Serve (HTTP server) - -```bash -sglang serve --model-path --port 30010 -``` - -### Enable Cache-DiT acceleration - -```bash -SGLANG_CACHE_DIT_ENABLED=true sglang generate --model-path --prompt "" -``` - -## References - -* [SGLang GitHub](https://github.com/sgl-project/sglang) -* [Cache-DiT](https://github.com/vipshop/cache-dit) -* [FastVideo](https://github.com/hao-ai-lab/FastVideo) -* [xDiT](https://github.com/xdit-project/xDiT) -* [Diffusers](https://github.com/huggingface/diffusers) diff --git a/scripts/ci/check_no_docs_changes.py b/scripts/ci/check_no_docs_changes.py new file mode 100755 index 000000000..fb302439b --- /dev/null +++ b/scripts/ci/check_no_docs_changes.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Reject staged changes under the legacy docs/ tree.""" + +from __future__ import annotations + +import subprocess +import sys + +ERROR_MESSAGE = """\ +Changes under the legacy docs/ directory are not allowed. + +The documentation has been migrated. Please make documentation updates in the +corresponding location under docs_new/ instead. +""" + + +def staged_paths() -> list[str]: + result = subprocess.run( + [ + "git", + "diff", + "--cached", + "--name-only", + "--diff-filter=ACMRDTUXB", + ], + check=True, + capture_output=True, + text=True, + ) + return [line.strip() for line in result.stdout.splitlines() if line.strip()] + + +def main() -> int: + paths = sys.argv[1:] or staged_paths() + docs_paths = sorted( + path for path in paths if path == "docs" or path.startswith("docs/") + ) + + if not docs_paths: + return 0 + + print(ERROR_MESSAGE, file=sys.stderr) + print("Detected legacy docs/ changes:", file=sys.stderr) + for path in docs_paths: + print(f" - {path}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main())