[NPU][DOC] Restructure ascend-npus docs into layered navigation (#32857)
This commit is contained in:
amote-i
@@ -425,7 +425,7 @@ For multi-node serving and hardware-specific examples:
|
||||
- [16× A100 INT8](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-16-a100a800-with-int8-quantization)
|
||||
- [32× L40S INT8](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-32-l40s-with-int8-quantization)
|
||||
- [Xeon 6980P CPU](../../../docs/hardware-platforms/cpu_server#example-running-deepseek-v3-1-terminus)
|
||||
- [4× Atlas 800I A3 (int8)](../../../docs/hardware-platforms/ascend-npus/model-tutorials/deepseek_r1#multi-node-pd-disaggregation-deployment)
|
||||
- [4× Atlas 800I A3 (int8)](../../../docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_r1#multi-node-pd-disaggregation-deployment)
|
||||
|
||||
**Blog references for large-scale deployment:**
|
||||
|
||||
|
||||
+292
-70
@@ -435,55 +435,55 @@
|
||||
},
|
||||
{
|
||||
"source": "/platforms/ascend/ascend_contribution_guide.html",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/ascend_contribution_guide"
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/development/contribution_guide"
|
||||
},
|
||||
{
|
||||
"source": "/platforms/ascend/ascend_npu.html",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/ascend_npu"
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/getting-started/installation"
|
||||
},
|
||||
{
|
||||
"source": "/platforms/ascend/ascend_npu_best_practice.html",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/best_practice"
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_235b_a22b"
|
||||
},
|
||||
{
|
||||
"source": "/platforms/ascend/ascend_npu_deepseek_example.html",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-tutorials/deepseek_v3_2"
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v3_2"
|
||||
},
|
||||
{
|
||||
"source": "/platforms/ascend/ascend_npu_environment_variables.html",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/ascend_npu_environment_variables"
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/reference/environment_variables"
|
||||
},
|
||||
{
|
||||
"source": "/platforms/ascend/ascend_npu_glm5_examples.html",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-tutorials/glm_5_1"
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_1"
|
||||
},
|
||||
{
|
||||
"source": "/platforms/ascend/ascend_npu_quantization.html",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/ascend_npu_quantization"
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/optimization/quantization"
|
||||
},
|
||||
{
|
||||
"source": "/platforms/ascend/ascend_npu_quick_start.html",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/ascend_npu_quick_start"
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/getting-started/quick_start"
|
||||
},
|
||||
{
|
||||
"source": "/platforms/ascend/ascend_npu_qwen3_5_examples.html",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_5_397b"
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_5_397b"
|
||||
},
|
||||
{
|
||||
"source": "/platforms/ascend/ascend_npu_qwen3_examples.html",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_235b_a22b"
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_235b_a22b"
|
||||
},
|
||||
{
|
||||
"source": "/platforms/ascend/ascend_npu_support.html",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/ascend_npu_quick_start"
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/getting-started/quick_start"
|
||||
},
|
||||
{
|
||||
"source": "/platforms/ascend/ascend_npu_support_features.html",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/ascend_npu_support_features"
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/reference/support_features"
|
||||
},
|
||||
{
|
||||
"source": "/platforms/ascend/ascend_npu_support_models.html",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/ascend_npu_support_models"
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/reference/support_models"
|
||||
},
|
||||
{
|
||||
"source": "/platforms/ascend/mindspore_backend.html",
|
||||
@@ -491,7 +491,199 @@
|
||||
},
|
||||
{
|
||||
"source": "/platforms/ascend_npu_ring_sp_performance.html",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/ascend_npu_ring_sp_performance"
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/optimization/ring_sp_performance"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/ascend_npu",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/getting-started/installation"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/ascend_npu_reference",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/reference/glossary"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/ascend_npu_optimization",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/ascend_npu_quick_start",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/getting-started/quick_start"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/ascend_npu_support_features",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/reference/support_features"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/ascend_npu_support_models",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/reference/support_models"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/ascend_npu_environment_variables",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/reference/environment_variables"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/ascend_npu_quantization",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/optimization/quantization"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/evaluation/accuracy_evaluation"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/evaluation/performance_testing"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/ascend_npu_ring_sp_performance",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/optimization/ring_sp_performance"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/ascend_npu_profiling",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/optimization/profiling"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/ascend_npu_operator_development",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/development/operator_development"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/ascend_npu_operator_performance_optimizing",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/development/operator_performance_optimizing"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/ascend_contribution_guide",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/development/contribution_guide"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/ascend_npu_support_new_models",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/development/support_new_models"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/ascend_npu_faq",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/faq"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/model-tutorials/deepseek_r1",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_r1"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/model-tutorials/deepseek_v3_2",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v3_2"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/model-tutorials/glm_5_1",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_1"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/model-tutorials/glm_5_2",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_2"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/model-tutorials/kimi_k2_6",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/kimi_k2_6"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/model-tutorials/minimax_m2_5",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/minimax_m2_5"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/model-tutorials/mimo_v2_flash",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/mimo_v2_flash"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3-8b",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_8b"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_32b",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_32b"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_30b_a3b",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_30b_a3b"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_235b_a22b",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_235b_a22b"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_5_397b",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_5_397b"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_6_27b",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_6_27b"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_6_35b_a3b",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_6_35b_a3b"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_next_80b_a3b_instruct",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_next_80b_a3b_instruct"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/best_practice/deepseek_r1",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/best_practice/deepseek_v3_2",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v3_2"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/best_practice/glm5_1",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/best_practice/kimi_k2_6",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/best_practice/minimax_m2_5",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/best_practice/mimo_v2_flash",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/best_practice/qwen3-8b",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_8b"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/best_practice/qwen3_32b",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_32b"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/best_practice/qwen3_30b_a3b",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_30b_a3b"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/best_practice/qwen3_235b_a22b",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_235b_a22b"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/best_practice/qwen3_5_397b",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_5_397b"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_27b",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_27b"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_35b_a3b",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_35b_a3b"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/best_practice/qwen3_next_80b_a3b_instruct",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_next_80b_a3b_instruct"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/model-tutorials",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_r1"
|
||||
},
|
||||
{
|
||||
"source": "/docs/hardware-platforms/ascend-npus/best_practice",
|
||||
"destination": "/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1"
|
||||
},
|
||||
{
|
||||
"source": "/platforms/cpu_server.html",
|
||||
@@ -887,63 +1079,93 @@
|
||||
{
|
||||
"group": "Ascend NPUs",
|
||||
"pages": [
|
||||
"docs/hardware-platforms/ascend-npus/ascend_npu_quick_start",
|
||||
"docs/hardware-platforms/ascend-npus/ascend_npu",
|
||||
"docs/hardware-platforms/ascend-npus/ascend_npu_support_features",
|
||||
"docs/hardware-platforms/ascend-npus/ascend_npu_support_models",
|
||||
"docs/hardware-platforms/ascend-npus/ascend_npu_quantization",
|
||||
{
|
||||
"group": "Getting Started",
|
||||
"pages": [
|
||||
"docs/hardware-platforms/ascend-npus/getting-started/quick_start",
|
||||
"docs/hardware-platforms/ascend-npus/getting-started/installation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"group": "Reference",
|
||||
"pages": [
|
||||
"docs/hardware-platforms/ascend-npus/reference/support_models",
|
||||
"docs/hardware-platforms/ascend-npus/reference/support_features",
|
||||
"docs/hardware-platforms/ascend-npus/reference/environment_variables",
|
||||
"docs/hardware-platforms/ascend-npus/reference/glossary"
|
||||
]
|
||||
},
|
||||
{
|
||||
"group": "Model Deployment",
|
||||
"pages": [
|
||||
{
|
||||
"group": "Model Tutorials",
|
||||
"pages": [
|
||||
"docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_r1",
|
||||
"docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v3_2",
|
||||
"docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_1",
|
||||
"docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_2",
|
||||
"docs/hardware-platforms/ascend-npus/model-deployment/tutorials/kimi_k2_6",
|
||||
"docs/hardware-platforms/ascend-npus/model-deployment/tutorials/minimax_m2_5",
|
||||
"docs/hardware-platforms/ascend-npus/model-deployment/tutorials/mimo_v2_flash",
|
||||
"docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_8b",
|
||||
"docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_32b",
|
||||
"docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_30b_a3b",
|
||||
"docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_235b_a22b",
|
||||
"docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_5_397b",
|
||||
"docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_6_27b",
|
||||
"docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_6_35b_a3b",
|
||||
"docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_next_80b_a3b_instruct"
|
||||
]
|
||||
},
|
||||
{
|
||||
"group": "Best Practices",
|
||||
"pages": [
|
||||
"docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1",
|
||||
"docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v3_2",
|
||||
"docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1",
|
||||
"docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6",
|
||||
"docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5",
|
||||
"docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash",
|
||||
"docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_8b",
|
||||
"docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_32b",
|
||||
"docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_30b_a3b",
|
||||
"docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_235b_a22b",
|
||||
"docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_5_397b",
|
||||
"docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_27b",
|
||||
"docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_35b_a3b",
|
||||
"docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_next_80b_a3b_instruct"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"group": "Evaluation",
|
||||
"pages": [
|
||||
"docs/hardware-platforms/ascend-npus/evaluation/accuracy_evaluation",
|
||||
"docs/hardware-platforms/ascend-npus/evaluation/performance_testing"
|
||||
]
|
||||
},
|
||||
{
|
||||
"group": "Optimization",
|
||||
"pages": [
|
||||
"docs/hardware-platforms/ascend-npus/optimization/parameter_tuning",
|
||||
"docs/hardware-platforms/ascend-npus/optimization/profiling",
|
||||
"docs/hardware-platforms/ascend-npus/optimization/quantization",
|
||||
"docs/hardware-platforms/ascend-npus/optimization/ring_sp_performance"
|
||||
]
|
||||
},
|
||||
{
|
||||
"group": "Development",
|
||||
"pages": [
|
||||
"docs/hardware-platforms/ascend-npus/development/contribution_guide",
|
||||
"docs/hardware-platforms/ascend-npus/development/support_new_models",
|
||||
"docs/hardware-platforms/ascend-npus/development/operator_development",
|
||||
"docs/hardware-platforms/ascend-npus/development/operator_performance_optimizing"
|
||||
]
|
||||
},
|
||||
"docs/hardware-platforms/ascend-npus/mindspore_backend",
|
||||
"docs/hardware-platforms/ascend-npus/ascend_contribution_guide",
|
||||
"docs/hardware-platforms/ascend-npus/ascend_npu_support_new_models",
|
||||
{
|
||||
"group": "Model Tutorials",
|
||||
"pages": [
|
||||
"docs/hardware-platforms/ascend-npus/model-tutorials/deepseek_r1",
|
||||
"docs/hardware-platforms/ascend-npus/model-tutorials/deepseek_v3_2",
|
||||
"docs/hardware-platforms/ascend-npus/model-tutorials/glm_5_2",
|
||||
"docs/hardware-platforms/ascend-npus/model-tutorials/glm_5_1",
|
||||
"docs/hardware-platforms/ascend-npus/model-tutorials/kimi_k2_6",
|
||||
"docs/hardware-platforms/ascend-npus/model-tutorials/minimax_m2_5",
|
||||
"docs/hardware-platforms/ascend-npus/model-tutorials/mimo_v2_flash",
|
||||
"docs/hardware-platforms/ascend-npus/model-tutorials/qwen3-8b",
|
||||
"docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_32b",
|
||||
"docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_30b_a3b",
|
||||
"docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_235b_a22b",
|
||||
"docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_5_397b",
|
||||
"docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_6_27b",
|
||||
"docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_6_35b_a3b",
|
||||
"docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_next_80b_a3b_instruct"
|
||||
]
|
||||
},
|
||||
{
|
||||
"group": "Best Practice",
|
||||
"pages": [
|
||||
"docs/hardware-platforms/ascend-npus/best_practice/deepseek_r1",
|
||||
"docs/hardware-platforms/ascend-npus/best_practice/deepseek_v3_2",
|
||||
"docs/hardware-platforms/ascend-npus/best_practice/glm5_1",
|
||||
"docs/hardware-platforms/ascend-npus/best_practice/kimi_k2_6",
|
||||
"docs/hardware-platforms/ascend-npus/best_practice/minimax_m2_5",
|
||||
"docs/hardware-platforms/ascend-npus/best_practice/mimo_v2_flash",
|
||||
"docs/hardware-platforms/ascend-npus/best_practice/qwen3-8b",
|
||||
"docs/hardware-platforms/ascend-npus/best_practice/qwen3_32b",
|
||||
"docs/hardware-platforms/ascend-npus/best_practice/qwen3_30b_a3b",
|
||||
"docs/hardware-platforms/ascend-npus/best_practice/qwen3_235b_a22b",
|
||||
"docs/hardware-platforms/ascend-npus/best_practice/qwen3_5_397b",
|
||||
"docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_27b",
|
||||
"docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_35b_a3b",
|
||||
"docs/hardware-platforms/ascend-npus/best_practice/qwen3_next_80b_a3b_instruct"
|
||||
]
|
||||
},
|
||||
"docs/hardware-platforms/ascend-npus/ascend_npu_optimization",
|
||||
"docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation",
|
||||
"docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing",
|
||||
"docs/hardware-platforms/ascend-npus/ascend_npu_ring_sp_performance",
|
||||
"docs/hardware-platforms/ascend-npus/ascend_npu_environment_variables",
|
||||
"docs/hardware-platforms/ascend-npus/ascend_npu_faq",
|
||||
"docs/hardware-platforms/ascend-npus/ascend_npu_operator_performance_optimizing",
|
||||
"docs/hardware-platforms/ascend-npus/ascend_npu_profiling",
|
||||
"docs/hardware-platforms/ascend-npus/ascend_npu_operator_development",
|
||||
"docs/hardware-platforms/ascend-npus/ascend_npu_reference"
|
||||
"docs/hardware-platforms/ascend-npus/faq"
|
||||
]
|
||||
},
|
||||
"docs/hardware-platforms/cpu_server",
|
||||
|
||||
@@ -202,7 +202,7 @@ The following table summarizes quantization method support across NVIDIA and AMD
|
||||
|
||||
On AMD, several of these methods use [Aiter](https://github.com/ROCm/aiter) for acceleration -- set `SGLANG_USE_AITER=1` where noted. See [AMD GPU setup](../hardware-platforms/amd_gpu) for installation and configuration details.
|
||||
|
||||
On Ascend, various layers quantization configurations are supported, see [Ascend NPU quantization](../hardware-platforms/ascend-npus/ascend_npu_quantization) for details.
|
||||
On Ascend, various layers quantization configurations are supported, see [Ascend NPU quantization](../hardware-platforms/ascend-npus/optimization/quantization) for details.
|
||||
|
||||
## GEMM Backends for FP4/FP8 Quantization
|
||||
|
||||
@@ -946,7 +946,7 @@ sglang generate \
|
||||
--save-output
|
||||
```
|
||||
|
||||
For the full quantization + format conversion workflow and a complete list of supported schemes, see [Diffusion Quantization on Ascend NPU](../hardware-platforms/ascend-npus/ascend_npu_quantization#diffusion-model-quantization-on-ascend-npu) and [SGLang-Diffusion Quantization](../sglang-diffusion/quantization#modelslim).
|
||||
For the full quantization + format conversion workflow and a complete list of supported schemes, see [Diffusion Quantization on Ascend NPU](../hardware-platforms/ascend-npus/optimization/quantization#diffusion-model-quantization-on-ascend-npu) and [SGLang-Diffusion Quantization](../sglang-diffusion/quantization#modelslim).
|
||||
|
||||
## Reference
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ keywords:
|
||||
---
|
||||
You can install SGLang using one of the methods below.
|
||||
This page primarily applies to common NVIDIA GPU platforms.
|
||||
For other or newer platforms, please refer to the dedicated pages for [AMD GPUs](../hardware-platforms/amd_gpu), [Apple Metal](../hardware-platforms/apple_metal), [Intel Xeon CPUs](../hardware-platforms/cpu_server), [Google TPU](../hardware-platforms/tpu), [NVIDIA DGX Spark](https://lmsys.org/blog/2025-11-03-gpt-oss-on-nvidia-dgx-spark/), [NVIDIA Jetson](../hardware-platforms/nvidia_jetson), [Ascend NPUs](../hardware-platforms/ascend-npus/ascend_npu), and [Intel XPU](../hardware-platforms/xpu).
|
||||
For other or newer platforms, please refer to the dedicated pages for [AMD GPUs](../hardware-platforms/amd_gpu), [Apple Metal](../hardware-platforms/apple_metal), [Intel Xeon CPUs](../hardware-platforms/cpu_server), [Google TPU](../hardware-platforms/tpu), [NVIDIA DGX Spark](https://lmsys.org/blog/2025-11-03-gpt-oss-on-nvidia-dgx-spark/), [NVIDIA Jetson](../hardware-platforms/nvidia_jetson), [Ascend NPUs](../hardware-platforms/ascend-npus/getting-started/installation), and [Intel XPU](../hardware-platforms/xpu).
|
||||
|
||||
<Note>
|
||||
Prerequisites: Python 3.10 or higher.
|
||||
|
||||
@@ -22,7 +22,7 @@ By the end, you'll have a working SGLang server responding to your prompts.
|
||||
- **OS**: Linux (recommended)
|
||||
|
||||
<Note>
|
||||
For other platforms, see the dedicated guides for [AMD GPUs](../hardware-platforms/amd_gpu), [Intel Xeon CPUs](../hardware-platforms/cpu_server), [Google TPUs](../hardware-platforms/tpu), [NVIDIA Jetson](../hardware-platforms/nvidia_jetson), [Ascend NPUs](../hardware-platforms/ascend-npus/ascend_npu), and [Intel XPU](../hardware-platforms/xpu).
|
||||
For other platforms, see the dedicated guides for [AMD GPUs](../hardware-platforms/amd_gpu), [Intel Xeon CPUs](../hardware-platforms/cpu_server), [Google TPUs](../hardware-platforms/tpu), [NVIDIA Jetson](../hardware-platforms/nvidia_jetson), [Ascend NPUs](../hardware-platforms/ascend-npus/getting-started/installation), and [Intel XPU](../hardware-platforms/xpu).
|
||||
</Note>
|
||||
|
||||
---
|
||||
|
||||
+3
-3
@@ -11,7 +11,7 @@ Welcome to **SGLang**! We appreciate your interest in contributing. This guide p
|
||||
|
||||
### Prepare Environment
|
||||
|
||||
Before contributing, please ensure that your environment is set up correctly. Follow the steps in the [Installation Guide](./ascend_npu) to install the necessary dependencies. We recommend [using docker](./ascend_npu#method-2-using-docker-image) to build the environment.
|
||||
Before contributing, please ensure that your environment is set up correctly. Follow the steps in the [Installation Guide](../getting-started/installation) to install the necessary dependencies. We recommend [using docker](../getting-started/installation#method-2-using-docker-image) to build the environment.
|
||||
|
||||
### Fork and clone the repository
|
||||
|
||||
@@ -126,7 +126,7 @@ You can find additional accuracy eval examples in:
|
||||
|
||||
## Benchmark the speed
|
||||
|
||||
Refer to [Benchmark and Profiling](../../developer_guide/benchmark_and_profiling).
|
||||
Refer to [Benchmark and Profiling](../../../developer_guide/benchmark_and_profiling).
|
||||
|
||||
## Requesting a review for merge
|
||||
|
||||
@@ -196,7 +196,7 @@ Users listed in [CI_PERMISSIONS.json](https://github.com/sgl-project/sglang/blob
|
||||
|
||||
Sgl-kernel-npu is the separate kernel package for Ascend NPU, containing both Ascend C and Triton operators. It is maintained in the [sgl-kernel-npu](https://github.com/sgl-project/sgl-kernel-npu) repository.
|
||||
|
||||
For detailed guidance on developing and integrating operators (Ascend C directory structure, PyTorch op registration, build, test, and code style), see the [Ascend NPU Operator Development Guide](./ascend_npu_operator_development).
|
||||
For detailed guidance on developing and integrating operators (Ascend C directory structure, PyTorch op registration, build, test, and code style), see the [Ascend NPU Operator Development Guide](./operator_development).
|
||||
|
||||
### Multi-PR workflow
|
||||
|
||||
+2
-2
@@ -1,5 +1,5 @@
|
||||
---
|
||||
title: "Ascend NPU Operator Development Guide"
|
||||
title: "Operator Development"
|
||||
description: "How to develop custom operators (Ascend C / Triton) for Ascend NPU and integrate them into the SGLang inference engine."
|
||||
---
|
||||
|
||||
@@ -523,4 +523,4 @@ submitting a PR.
|
||||
- [Ascend C Kernel Development Guide](https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/850alpha001/opdevg/Ascendcopdevg/atlas_ascendc_10_0001.html)
|
||||
- [PyTorch Custom Ops Schema Reference](https://github.com/pytorch/pytorch/blob/main/aten/src/ATen/native/README.md#func)
|
||||
- [helloworld Example Operator](https://github.com/sgl-project/sgl-kernel-npu/tree/main/csrc/helloworld)
|
||||
- [SGLang Contribution Guide](/docs/hardware-platforms/ascend-npus/ascend_contribution_guide)
|
||||
- [SGLang Contribution Guide](/docs/hardware-platforms/ascend-npus/development/contribution_guide)
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
---
|
||||
title: "Operator Performance Optimizing Guidance"
|
||||
title: "Operator Performance Optimization"
|
||||
metatags:
|
||||
description: "Operator Performance Optimizing Guidance for NPU"
|
||||
---
|
||||
+2
-2
@@ -103,7 +103,7 @@ To port a model from vLLM to SGLang:
|
||||
`sglang/srt/hardware_backend/npu/`.
|
||||
|
||||
Note: make sure you add your new model to the supported models list in the
|
||||
[supported models documentation](https://github.com/sgl-project/sglang/blob/main/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_support_models.mdx).
|
||||
[supported models documentation](https://github.com/sgl-project/sglang/blob/main/docs_new/docs/hardware-platforms/ascend-npus/reference/support_models.mdx).
|
||||
|
||||
## Registering an External Model Implementation
|
||||
|
||||
@@ -521,7 +521,7 @@ Add to table of supported models in [generative_models.mdx](/docs/supported-mode
|
||||
|
||||
<Tip>
|
||||
For NPU-adapted models, also add entries to the NPU support models table in
|
||||
[ascend_npu_support_models.mdx](./ascend_npu_support_models).
|
||||
[reference/support_models.mdx](../reference/support_models).
|
||||
</Tip>
|
||||
|
||||
---
|
||||
@@ -1,10 +1,10 @@
|
||||
---
|
||||
title: "Disaggregation of Diffusion Pipeline on Ascend NPU"
|
||||
title: "Disaggregation of Diffusion Pipeline"
|
||||
---
|
||||
|
||||
## Quick Start
|
||||
|
||||
Please follow the [NPU installation guide](../ascend_npu.mdx) first.
|
||||
Please follow the [NPU installation guide](../getting-started/installation) first.
|
||||
|
||||
Then install Mooncake from sources.
|
||||
|
||||
|
||||
+2
-2
@@ -1,5 +1,5 @@
|
||||
---
|
||||
title: "Ascend NPU Accuracy Evaluation"
|
||||
title: "Accuracy Evaluation"
|
||||
metatags:
|
||||
description: "Complete guide for SGLang model accuracy evaluation on Ascend NPU using EvalScope, covering text/multimodal scenarios."
|
||||
---
|
||||
@@ -101,7 +101,7 @@ export HF_ENDPOINT=https://hf-mirror.com
|
||||
sglang serve --model-path /home/weights/Qwen2.5-7B-Instruct --attention-backend ascend --host 0.0.0.0 --port 30000 &
|
||||
```
|
||||
|
||||
For more details of SGLang server, refer to the [Ascend NPU Quick Start](/docs/hardware-platforms/ascend-npus/ascend_npu_quick_start)
|
||||
For more details of SGLang server, refer to the [Ascend NPU Quick Start](/docs/hardware-platforms/ascend-npus/getting-started/quick_start)
|
||||
|
||||
#### Execute Accuracy Evaluation
|
||||
|
||||
+3
-3
@@ -1,5 +1,5 @@
|
||||
---
|
||||
title: "Ascend NPU Performance Testing"
|
||||
title: "Performance Testing"
|
||||
metatags:
|
||||
description: "Performance testing for SGLang on Ascend NPUs for text generation, multimodal, and embedding models in both online and offline serving modes."
|
||||
---
|
||||
@@ -12,7 +12,7 @@ This page walks through performance testing your SGLang deployment on Ascend NPU
|
||||
|
||||
### 1.1 Start SGLang server
|
||||
|
||||
Launch the server with the appropriate flags for each model type. Make sure SGLang is installed first — see [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/ascend_npu_quick_start) for environment setup.
|
||||
Launch the server with the appropriate flags for each model type. Make sure SGLang is installed first — see [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/getting-started/quick_start) for environment setup.
|
||||
|
||||
<Tabs>
|
||||
<Tab title="Text Generation">
|
||||
@@ -765,6 +765,6 @@ python -m sglang.bench_offline_throughput \
|
||||
## See also
|
||||
|
||||
- [Bench Serving Guide](/docs/developer_guide/bench_serving) — all backends, datasets, and advanced options for `bench_serving`
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/ascend_npu_quick_start) — environment setup for Ascend NPUs
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/getting-started/quick_start) — environment setup for Ascend NPUs
|
||||
- [Evalscope Performance Testing Guide](https://evalscope.readthedocs.io/en/latest/user_guides/stress_test/quick_start.html) — full Evalscope documentation
|
||||
- [AISBench Documentation](https://ais-bench-benchmark.readthedocs.io/en/latest/) — full AISBench documentation
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
---
|
||||
title: "Ascend NPU Troubleshooting and FAQ"
|
||||
title: "Troubleshooting and FAQ"
|
||||
metatags:
|
||||
description: "Troubleshooting and frequently asked questions for Ascend NPU"
|
||||
---
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
---
|
||||
title: SGLang installation with NPUs support
|
||||
title: Installation
|
||||
description: "Complete installation guide for SGLang on Ascend NPUs, including component version mapping, environment setup, and launching inference services."
|
||||
---
|
||||
You can install SGLang using any of the methods below. Please go through `System Settings` section to ensure the clusters are operating at optimal performance. Feel free to leave an issue [here at sglang](https://github.com/sgl-project/sglang/issues) if you encounter any issues or have any problems.
|
||||
+3
-3
@@ -1,5 +1,5 @@
|
||||
---
|
||||
title: "Ascend NPU Quickstart"
|
||||
title: "Quickstart"
|
||||
metatags:
|
||||
description: "Quickstart for running SGLang on Ascend NPUs with the official container image, including server launch and test request examples."
|
||||
---
|
||||
@@ -7,7 +7,7 @@ metatags:
|
||||
<Note>
|
||||
This page covers only the simplest deployment flow using the official container image. For the complete installation guide
|
||||
across all scenarios (source install, Docker build, system settings, version mapping, etc.),
|
||||
see [SGLang installation with NPUs support](/docs/hardware-platforms/ascend-npus/ascend_npu).
|
||||
see [SGLang installation with NPUs support](/docs/hardware-platforms/ascend-npus/getting-started/installation).
|
||||
</Note>
|
||||
|
||||
## Prerequisites
|
||||
@@ -18,7 +18,7 @@ see [SGLang installation with NPUs support](/docs/hardware-platforms/ascend-npus
|
||||
- Atlas 800I A3 inference series (Atlas 800I A3)
|
||||
|
||||
To identify your device, run `npu-smi info -l`: A3 reports `Chip Count: 2` per NPU, while A2 reports `Chip Count: 1` per NPU.
|
||||
For hardware details, see the [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/ascend_npu_reference#hardware).
|
||||
For hardware details, see the [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware).
|
||||
|
||||
### Docker
|
||||
|
||||
@@ -18,7 +18,7 @@ Currently, the following models are supported:
|
||||
## Installation
|
||||
|
||||
<Note>
|
||||
Currently, MindSpore models are provided by an independent package `sgl-mindspore`. Support for MindSpore is built upon current SGLang support for Ascend NPU platform. Please first [install SGLang for Ascend NPU](./ascend_npu) and then install `sgl-mindspore`:
|
||||
Currently, MindSpore models are provided by an independent package `sgl-mindspore`. Support for MindSpore is built upon current SGLang support for Ascend NPU platform. Please first [install SGLang for Ascend NPU](./getting-started/installation) and then install `sgl-mindspore`:
|
||||
</Note>
|
||||
|
||||
<CodeGroup>
|
||||
|
||||
+2
-2
@@ -5,9 +5,9 @@ metatags:
|
||||
---
|
||||
|
||||
<Note>
|
||||
This page focuses on optimal configuration and benchmark results for DeepSeek-R1 on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [DeepSeek-R1 Model Tutorial](/docs/hardware-platforms/ascend-npus/model-tutorials/deepseek_r1).
|
||||
This page focuses on optimal configuration and benchmark results for DeepSeek-R1 on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [DeepSeek-R1 Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_r1).
|
||||
|
||||
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/ascend_npu_reference#hardware) for details.
|
||||
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details.
|
||||
</Note>
|
||||
|
||||
### Low Latency
|
||||
+2
-2
@@ -5,9 +5,9 @@ metatags:
|
||||
---
|
||||
|
||||
<Note>
|
||||
This page focuses on optimal configuration and benchmark results for DeepSeek-V3.2 on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [DeepSeek-V3.2 Model Tutorial](/docs/hardware-platforms/ascend-npus/model-tutorials/deepseek_v3_2).
|
||||
This page focuses on optimal configuration and benchmark results for DeepSeek-V3.2 on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [DeepSeek-V3.2 Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v3_2).
|
||||
|
||||
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/ascend_npu_reference#hardware) for details.
|
||||
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details.
|
||||
</Note>
|
||||
|
||||
### Low Latency
|
||||
+2
-2
@@ -5,9 +5,9 @@ metatags:
|
||||
---
|
||||
|
||||
<Note>
|
||||
This page focuses on optimal configuration and benchmark results for GLM-5.1 on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [GLM-5.1 Model Tutorial](/docs/hardware-platforms/ascend-npus/model-tutorials/glm_5_1).
|
||||
This page focuses on optimal configuration and benchmark results for GLM-5.1 on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [GLM-5.1 Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_1).
|
||||
|
||||
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/ascend_npu_reference#hardware) for details.
|
||||
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details.
|
||||
</Note>
|
||||
|
||||
### Low Latency
|
||||
+2
-2
@@ -5,9 +5,9 @@ metatags:
|
||||
---
|
||||
|
||||
<Note>
|
||||
This page focuses on optimal configuration and benchmark results for Kimi-K2.6 on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [Kimi-K2.6 Model Tutorial](/docs/hardware-platforms/ascend-npus/model-tutorials/kimi_k2_6).
|
||||
This page focuses on optimal configuration and benchmark results for Kimi-K2.6 on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [Kimi-K2.6 Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/kimi_k2_6).
|
||||
|
||||
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/ascend_npu_reference#hardware) for details.
|
||||
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details.
|
||||
</Note>
|
||||
|
||||
### Low Latency
|
||||
+2
-2
@@ -5,9 +5,9 @@ metatags:
|
||||
---
|
||||
|
||||
<Note>
|
||||
This page focuses on optimal configuration and benchmark results for MiMo-V2-Flash on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [MiMo-V2-Flash Model Tutorial](/docs/hardware-platforms/ascend-npus/model-tutorials/mimo_v2_flash).
|
||||
This page focuses on optimal configuration and benchmark results for MiMo-V2-Flash on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [MiMo-V2-Flash Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/mimo_v2_flash).
|
||||
|
||||
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/ascend_npu_reference#hardware) for details.
|
||||
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details.
|
||||
</Note>
|
||||
|
||||
### Low Latency
|
||||
+2
-2
@@ -5,9 +5,9 @@ metatags:
|
||||
---
|
||||
|
||||
<Note>
|
||||
This page focuses on optimal configuration and benchmark results for MiniMax-M2.5 on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [MiniMax-M2.5 Model Tutorial](/docs/hardware-platforms/ascend-npus/model-tutorials/minimax_m2_5).
|
||||
This page focuses on optimal configuration and benchmark results for MiniMax-M2.5 on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [MiniMax-M2.5 Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/minimax_m2_5).
|
||||
|
||||
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/ascend_npu_reference#hardware) for details.
|
||||
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details.
|
||||
</Note>
|
||||
|
||||
### Low Latency
|
||||
+2
-2
@@ -5,9 +5,9 @@ metatags:
|
||||
---
|
||||
|
||||
<Note>
|
||||
This page focuses on optimal configuration and benchmark results for Qwen3-235B-A22B on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [Qwen3-235B-A22B Model Tutorial](/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_235b_a22b).
|
||||
This page focuses on optimal configuration and benchmark results for Qwen3-235B-A22B on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [Qwen3-235B-A22B Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_235b_a22b).
|
||||
|
||||
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/ascend_npu_reference#hardware) for details.
|
||||
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details.
|
||||
</Note>
|
||||
|
||||
### Low Latency
|
||||
+2
-2
@@ -5,9 +5,9 @@ metatags:
|
||||
---
|
||||
|
||||
<Note>
|
||||
This page focuses on optimal configuration and benchmark results for Qwen3-30B-A3B on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [Qwen3-30B-A3B Model Tutorial](/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_30b_a3b).
|
||||
This page focuses on optimal configuration and benchmark results for Qwen3-30B-A3B on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [Qwen3-30B-A3B Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_30b_a3b).
|
||||
|
||||
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/ascend_npu_reference#hardware) for details.
|
||||
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details.
|
||||
</Note>
|
||||
|
||||
### Low Latency
|
||||
+2
-2
@@ -5,9 +5,9 @@ metatags:
|
||||
---
|
||||
|
||||
<Note>
|
||||
This page focuses on optimal configuration and benchmark results for Qwen3-32B on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [Qwen3-32B Model Tutorial](/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_32b).
|
||||
This page focuses on optimal configuration and benchmark results for Qwen3-32B on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [Qwen3-32B Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_32b).
|
||||
|
||||
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/ascend_npu_reference#hardware) for details.
|
||||
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details.
|
||||
</Note>
|
||||
|
||||
### Low Latency
|
||||
+3
-3
@@ -1,13 +1,13 @@
|
||||
---
|
||||
title: "Qwen3.5-397B"
|
||||
title: "Qwen3.5-397B-A17B"
|
||||
metatags:
|
||||
description: "Best Practice for Qwen3.5-397B on Ascend NPU"
|
||||
---
|
||||
|
||||
<Note>
|
||||
This page focuses on optimal configuration and benchmark results for Qwen3.5-397B on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [Qwen3.5-397B Model Tutorial](/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_5_397b).
|
||||
This page focuses on optimal configuration and benchmark results for Qwen3.5-397B on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [Qwen3.5-397B Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_5_397b).
|
||||
|
||||
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/ascend_npu_reference#hardware) for details.
|
||||
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details.
|
||||
</Note>
|
||||
|
||||
### Low Latency
|
||||
+2
-2
@@ -5,9 +5,9 @@ metatags:
|
||||
---
|
||||
|
||||
<Note>
|
||||
This page focuses on optimal configuration and benchmark results for Qwen3.6-27B on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [Qwen3.6-27B Model Tutorial](/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_6_27b).
|
||||
This page focuses on optimal configuration and benchmark results for Qwen3.6-27B on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [Qwen3.6-27B Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_6_27b).
|
||||
|
||||
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/ascend_npu_reference#hardware) for details.
|
||||
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details.
|
||||
</Note>
|
||||
|
||||
### High Throughput
|
||||
+2
-2
@@ -5,9 +5,9 @@ metatags:
|
||||
---
|
||||
|
||||
<Note>
|
||||
This page focuses on optimal configuration and benchmark results for Qwen3.6-35B-A3B on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [Qwen3.6-35B-A3B Model Tutorial](/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_6_35b_a3b).
|
||||
This page focuses on optimal configuration and benchmark results for Qwen3.6-35B-A3B on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [Qwen3.6-35B-A3B Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_6_35b_a3b).
|
||||
|
||||
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/ascend_npu_reference#hardware) for details.
|
||||
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details.
|
||||
|
||||
Use image **SGLang `>= v0.5.14`** for these NEXTN configurations. Without `--dataset-path`, `bench_serving --dataset-name random` downloads ShareGPT from Hugging Face; in offline environments, pass a local dataset path (for example a ShareGPT JSON file).
|
||||
</Note>
|
||||
+2
-2
@@ -5,9 +5,9 @@ metatags:
|
||||
---
|
||||
|
||||
<Note>
|
||||
This page focuses on optimal configuration and benchmark results for Qwen3-8B on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [Qwen3-8B Model Tutorial](/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3-8b).
|
||||
This page focuses on optimal configuration and benchmark results for Qwen3-8B on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [Qwen3-8B Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_8b).
|
||||
|
||||
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/ascend_npu_reference#hardware) for details.
|
||||
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details.
|
||||
</Note>
|
||||
|
||||
### Low Latency
|
||||
+2
-2
@@ -5,9 +5,9 @@ metatags:
|
||||
---
|
||||
|
||||
<Note>
|
||||
This page focuses on optimal configuration and benchmark results for Qwen3-Next-80B-A3B-Instruct on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [Qwen3-Next-80B-A3B-Instruct Model Tutorial](/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_next_80b_a3b_instruct).
|
||||
This page focuses on optimal configuration and benchmark results for Qwen3-Next-80B-A3B-Instruct on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [Qwen3-Next-80B-A3B-Instruct Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_next_80b_a3b_instruct).
|
||||
|
||||
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/ascend_npu_reference#hardware) for details.
|
||||
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details.
|
||||
</Note>
|
||||
|
||||
### Low Latency
|
||||
+14
-14
@@ -38,12 +38,12 @@ v0.5.13 or a later version.
|
||||
<Note>
|
||||
The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment
|
||||
mode, and workload. For parameter details, see
|
||||
[Feature descriptions](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-descriptions); for
|
||||
[Feature descriptions](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-descriptions); for
|
||||
recommended configurations for each deployment scenario, see [Best practices](#best-practices).
|
||||
</Note>
|
||||
|
||||
For feature compatibility and conflict information between features,
|
||||
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-compatibility).
|
||||
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-compatibility).
|
||||
|
||||
## Prerequisites
|
||||
|
||||
@@ -51,10 +51,10 @@ see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_opti
|
||||
|
||||
Before following this tutorial, complete the environment setup in the documents below:
|
||||
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/ascend_npu_quick_start) — the fastest way to get started.
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/getting-started/quick_start) — the fastest way to get started.
|
||||
It walks you through launching the official container image, starting the SGLang server, and sending a test request.
|
||||
Recommended if you are new to SGLang on Ascend.
|
||||
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/ascend_npu) — the full installation guide.
|
||||
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/getting-started/installation) — the full installation guide.
|
||||
It covers the component version mapping (CANN, TorchNPU, Triton, kernels, etc.), building from source or from a
|
||||
Dockerfile, and recommended system settings (CPU power scheme, NUMA, swap). Use it when you need to install or customize
|
||||
the environment instead of using the official image.
|
||||
@@ -79,7 +79,7 @@ This is the minimum recommended configuration. For optimized configurations,
|
||||
see [Best practices](#best-practices), which may require additional cards or nodes.
|
||||
|
||||
For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3),
|
||||
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/ascend_npu_reference#hardware).
|
||||
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware).
|
||||
|
||||
## Installation
|
||||
|
||||
@@ -91,7 +91,7 @@ The dependencies required for the NPU runtime environment have been integrated i
|
||||
online platform. You can directly pull it.
|
||||
|
||||
Both **stable releases** and **daily builds** are available. The following command is based on the stable release tag.
|
||||
For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/ascend_npu_faq#8-docker-image-versions-stable-release-vs-daily-build).
|
||||
For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build).
|
||||
|
||||
<Tabs>
|
||||
<Tab title="Atlas 800I A3">
|
||||
@@ -171,14 +171,14 @@ quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b
|
||||
Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios
|
||||
with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized
|
||||
deployment commands and benchmark data, see
|
||||
[DeepSeek-R1 Best Practice — W4A8 8P PD Mixed On A3](/docs/hardware-platforms/ascend-npus/best_practice/deepseek_r1#single-node-pd-mixed).
|
||||
[DeepSeek-R1 Best Practice — W4A8 8P PD Mixed On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1#single-node-pd-mixed).
|
||||
|
||||
### Multi-node PD disaggregation deployment
|
||||
|
||||
PD disaggregation splits the prefill and decode stages onto separate nodes, reducing interference and improving
|
||||
throughput for high-concurrency scenarios. This scenario is already covered in the best practice. For the complete, optimized
|
||||
deployment commands and benchmark data, see
|
||||
[DeepSeek-R1 Best Practice — W8A8 32P PD Disaggregation On A3](/docs/hardware-platforms/ascend-npus/best_practice/deepseek_r1#pd-disaggregation).
|
||||
[DeepSeek-R1 Best Practice — W8A8 32P PD Disaggregation On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1#pd-disaggregation).
|
||||
|
||||
## Functional verification
|
||||
|
||||
@@ -206,28 +206,28 @@ Expected result: an HTTP 200 response with the generated text containing "Paris"
|
||||
|
||||
Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For more
|
||||
testing examples (Health Check, Generate, Chat Completions, and port usage guidance),
|
||||
see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service).
|
||||
see [Testing the Service](/docs/hardware-platforms/ascend-npus/getting-started/installation#testing-the-service).
|
||||
|
||||
## Accuracy evaluation
|
||||
|
||||
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation).
|
||||
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/accuracy_evaluation).
|
||||
|
||||
## Performance
|
||||
|
||||
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing).
|
||||
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/performance_testing).
|
||||
|
||||
## Best practices
|
||||
|
||||
### Best practice configuration reference
|
||||
|
||||
For complete optimal configurations with deployment scripts and benchmark commands, see the
|
||||
[DeepSeek-R1 Best Practice](/docs/hardware-platforms/ascend-npus/best_practice/deepseek_r1) page.
|
||||
[DeepSeek-R1 Best Practice](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1) page.
|
||||
|
||||
## Performance tuning
|
||||
|
||||
For the full list of supported features, see [Supported features](#supported-features). For detailed optimization
|
||||
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization).
|
||||
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning).
|
||||
|
||||
## FAQ
|
||||
|
||||
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/ascend_npu_faq).
|
||||
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/faq).
|
||||
+13
-13
@@ -37,12 +37,12 @@ recommended to use v0.5.13 or a later version.
|
||||
<Note>
|
||||
The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment
|
||||
mode, and workload. For parameter details, see
|
||||
[Feature descriptions](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-descriptions); for
|
||||
[Feature descriptions](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-descriptions); for
|
||||
recommended configurations for each deployment scenario, see [Best practices](#best-practices).
|
||||
</Note>
|
||||
|
||||
For feature compatibility and conflict information between features,
|
||||
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-compatibility).
|
||||
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-compatibility).
|
||||
|
||||
## Prerequisites
|
||||
|
||||
@@ -50,10 +50,10 @@ see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_opti
|
||||
|
||||
Before following this tutorial, complete the environment setup in the documents below:
|
||||
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/ascend_npu_quick_start) — the fastest way to get started.
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/getting-started/quick_start) — the fastest way to get started.
|
||||
It walks you through launching the official container image, starting the SGLang server, and sending a test request.
|
||||
Recommended if you are new to SGLang on Ascend.
|
||||
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/ascend_npu) — the full installation guide.
|
||||
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/getting-started/installation) — the full installation guide.
|
||||
It covers the component version mapping (CANN, TorchNPU, Triton, kernels, etc.), building from source or from a
|
||||
Dockerfile, and recommended system settings (CPU power scheme, NUMA, swap). Use it when you need to install or customize
|
||||
the environment instead of using the official image.
|
||||
@@ -74,7 +74,7 @@ This is the minimum recommended configuration. For optimized configurations,
|
||||
see [Best practices](#best-practices), which may require additional cards or nodes.
|
||||
|
||||
For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3),
|
||||
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/ascend_npu_reference#hardware).
|
||||
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware).
|
||||
|
||||
## Installation
|
||||
|
||||
@@ -86,7 +86,7 @@ The dependencies required for the NPU runtime environment have been integrated i
|
||||
online platform. You can directly pull it.
|
||||
|
||||
Both **stable releases** and **daily builds** are available. The following command is based on the stable release tag.
|
||||
For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/ascend_npu_faq#8-docker-image-versions-stable-release-vs-daily-build).
|
||||
For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build).
|
||||
|
||||
<Tabs>
|
||||
<Tab title="Atlas 800I A3">
|
||||
@@ -166,7 +166,7 @@ quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b
|
||||
PD disaggregation splits the prefill and decode stages onto separate nodes, reducing interference and improving
|
||||
throughput for high-concurrency scenarios. This scenario is already covered in the best practice. For the complete, optimized
|
||||
deployment commands and benchmark data, see
|
||||
[DeepSeek-V3.2 Best Practice — PD Disaggregation On A3](/docs/hardware-platforms/ascend-npus/best_practice/deepseek_v3_2#pd-disaggregation).
|
||||
[DeepSeek-V3.2 Best Practice — PD Disaggregation On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v3_2#pd-disaggregation).
|
||||
|
||||
## Functional verification
|
||||
|
||||
@@ -194,28 +194,28 @@ Expected result: an HTTP 200 response with the generated text containing "Paris"
|
||||
|
||||
Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For more
|
||||
testing examples (Health Check, Generate, Chat Completions, and port usage guidance),
|
||||
see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service).
|
||||
see [Testing the Service](/docs/hardware-platforms/ascend-npus/getting-started/installation#testing-the-service).
|
||||
|
||||
## Accuracy evaluation
|
||||
|
||||
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation).
|
||||
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/accuracy_evaluation).
|
||||
|
||||
## Performance
|
||||
|
||||
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing).
|
||||
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/performance_testing).
|
||||
|
||||
## Best practices
|
||||
|
||||
### Best practice configuration reference
|
||||
|
||||
For complete optimal configurations with deployment scripts and benchmark commands, see the
|
||||
[DeepSeek-V3.2 Best Practice](/docs/hardware-platforms/ascend-npus/best_practice/deepseek_v3_2) page.
|
||||
[DeepSeek-V3.2 Best Practice](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v3_2) page.
|
||||
|
||||
## Performance tuning
|
||||
|
||||
For the full list of supported features, see [Supported features](#supported-features). For detailed optimization
|
||||
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization).
|
||||
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning).
|
||||
|
||||
## FAQ
|
||||
|
||||
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/ascend_npu_faq).
|
||||
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/faq).
|
||||
+14
-14
@@ -39,12 +39,12 @@ v0.5.13 or a later version.
|
||||
<Note>
|
||||
The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment
|
||||
mode, and workload. For parameter details, see
|
||||
[Feature descriptions](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-descriptions); for
|
||||
[Feature descriptions](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-descriptions); for
|
||||
recommended configurations for each deployment scenario, see [Best practices](#best-practices).
|
||||
</Note>
|
||||
|
||||
For feature compatibility and conflict information between features,
|
||||
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-compatibility).
|
||||
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-compatibility).
|
||||
|
||||
## Prerequisites
|
||||
|
||||
@@ -52,10 +52,10 @@ see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_opti
|
||||
|
||||
Before following this tutorial, complete the environment setup in the documents below:
|
||||
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/ascend_npu_quick_start) — the fastest way to get started.
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/getting-started/quick_start) — the fastest way to get started.
|
||||
It walks you through launching the official container image, starting the SGLang server, and sending a test request.
|
||||
Recommended if you are new to SGLang on Ascend.
|
||||
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/ascend_npu) — the full installation guide.
|
||||
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/getting-started/installation) — the full installation guide.
|
||||
It covers the component version mapping (CANN, TorchNPU, Triton, kernels, etc.), building from source or from a
|
||||
Dockerfile, and recommended system settings (CPU power scheme, NUMA, swap). Use it when you need to install or customize
|
||||
the environment instead of using the official image.
|
||||
@@ -78,7 +78,7 @@ This is the minimum recommended configuration. For optimized configurations,
|
||||
see [Best practices](#best-practices), which may require additional cards or nodes.
|
||||
|
||||
For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3),
|
||||
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/ascend_npu_reference#hardware).
|
||||
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware).
|
||||
|
||||
## Installation
|
||||
|
||||
@@ -90,7 +90,7 @@ The dependencies required for the NPU runtime environment have been integrated i
|
||||
online platform. You can directly pull it.
|
||||
|
||||
Both **stable releases** and **daily builds** are available. The following command is based on the stable release tag.
|
||||
For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/ascend_npu_faq#8-docker-image-versions-stable-release-vs-daily-build).
|
||||
For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build).
|
||||
|
||||
<Tabs>
|
||||
<Tab title="Atlas 800I A3">
|
||||
@@ -171,14 +171,14 @@ Multi-node deployment distributes the model across multiple Atlas 800I A3 nodes
|
||||
prefill and decode on the same nodes (PD mixed mode), suitable for scenarios that need more device memory than a single
|
||||
node can provide. This scenario is already covered in the best practice. For the complete, optimized
|
||||
deployment commands and benchmark data, see
|
||||
[GLM-5.1 Best Practice — Multi-node PD Mixed On A3](/docs/hardware-platforms/ascend-npus/best_practice/glm5_1#multi-node-pd-mixed).
|
||||
[GLM-5.1 Best Practice — Multi-node PD Mixed On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1#multi-node-pd-mixed).
|
||||
|
||||
### Multi-node PD disaggregation deployment
|
||||
|
||||
PD disaggregation splits the prefill and decode stages onto separate nodes, reducing interference and improving
|
||||
throughput for high-concurrency scenarios. This scenario is already covered in the best practice. For the complete, optimized
|
||||
deployment commands and benchmark data, see
|
||||
[GLM-5.1 Best Practice — PD Disaggregation On A3](/docs/hardware-platforms/ascend-npus/best_practice/glm5_1#pd-disaggregation).
|
||||
[GLM-5.1 Best Practice — PD Disaggregation On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1#pd-disaggregation).
|
||||
|
||||
## Functional verification
|
||||
|
||||
@@ -206,28 +206,28 @@ Expected result: an HTTP 200 response with the generated text containing "Paris"
|
||||
|
||||
Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For more
|
||||
testing examples (Health Check, Generate, Chat Completions, and port usage guidance),
|
||||
see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service).
|
||||
see [Testing the Service](/docs/hardware-platforms/ascend-npus/getting-started/installation#testing-the-service).
|
||||
|
||||
## Accuracy evaluation
|
||||
|
||||
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation).
|
||||
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/accuracy_evaluation).
|
||||
|
||||
## Performance
|
||||
|
||||
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing).
|
||||
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/performance_testing).
|
||||
|
||||
## Best practices
|
||||
|
||||
### Best practice configuration reference
|
||||
|
||||
For complete optimal configurations with deployment scripts and benchmark commands, see the
|
||||
[GLM-5.1 Best Practice](/docs/hardware-platforms/ascend-npus/best_practice/glm5_1) page.
|
||||
[GLM-5.1 Best Practice](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1) page.
|
||||
|
||||
## Performance tuning
|
||||
|
||||
For the full list of supported features, see [Supported features](#supported-features). For detailed optimization
|
||||
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization).
|
||||
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning).
|
||||
|
||||
## FAQ
|
||||
|
||||
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/ascend_npu_faq).
|
||||
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/faq).
|
||||
+11
-11
@@ -35,12 +35,12 @@ multi-node deployment, prefill-decode disaggregation, feature configuration, and
|
||||
<Note>
|
||||
The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment
|
||||
mode, and workload. For parameter details, see
|
||||
[Feature descriptions](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-descriptions); for
|
||||
[Feature descriptions](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-descriptions); for
|
||||
recommended configurations for each deployment scenario, see [Best practices](#best-practices).
|
||||
</Note>
|
||||
|
||||
For feature compatibility and conflict information between features,
|
||||
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-compatibility).
|
||||
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-compatibility).
|
||||
|
||||
## Prerequisites
|
||||
|
||||
@@ -48,10 +48,10 @@ see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_opti
|
||||
|
||||
Before following this tutorial, complete the environment setup in the documents below:
|
||||
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/ascend_npu_quick_start) — the fastest way to get started.
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/getting-started/quick_start) — the fastest way to get started.
|
||||
It walks you through launching the official container image, starting the SGLang server, and sending a test request.
|
||||
Recommended if you are new to SGLang on Ascend.
|
||||
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/ascend_npu) — the full installation guide.
|
||||
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/getting-started/installation) — the full installation guide.
|
||||
It covers the component version mapping (CANN, TorchNPU, Triton, kernels, etc.), building from source or from a
|
||||
Dockerfile, and recommended system settings (CPU power scheme, NUMA, swap). Use it when you need to install or customize
|
||||
the environment instead of using the official image.
|
||||
@@ -74,7 +74,7 @@ This is the minimum recommended configuration. For optimized configurations,
|
||||
see [Best practices](#best-practices), which may require additional cards or nodes.
|
||||
|
||||
For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3),
|
||||
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/ascend_npu_reference#hardware).
|
||||
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware).
|
||||
|
||||
## Installation
|
||||
|
||||
@@ -489,15 +489,15 @@ Expected result: an HTTP 200 response with the generated text containing "Paris"
|
||||
|
||||
Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For more
|
||||
testing examples (Health Check, Generate, Chat Completions, and port usage guidance),
|
||||
see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service).
|
||||
see [Testing the Service](/docs/hardware-platforms/ascend-npus/getting-started/installation#testing-the-service).
|
||||
|
||||
## Accuracy evaluation
|
||||
|
||||
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation).
|
||||
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/accuracy_evaluation).
|
||||
|
||||
## Performance
|
||||
|
||||
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing).
|
||||
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/performance_testing).
|
||||
|
||||
## Best practices
|
||||
|
||||
@@ -509,13 +509,13 @@ ready-to-use scripts for single-node, multi-node, and PD disaggregation, each em
|
||||
combinations and tuning parameters (e.g., DeepEP mode, speculative decoding, overlap schedule).
|
||||
|
||||
For the full catalog of optimization features and their parameter and compatibility details,
|
||||
see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization).
|
||||
see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning).
|
||||
|
||||
## Performance tuning
|
||||
|
||||
For the full list of supported features, see [Supported features](#supported-features). For detailed optimization
|
||||
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization).
|
||||
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning).
|
||||
|
||||
## FAQ
|
||||
|
||||
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/ascend_npu_faq).
|
||||
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/faq).
|
||||
+15
-15
@@ -40,12 +40,12 @@ v0.5.13 or a later version.
|
||||
<Note>
|
||||
The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment
|
||||
mode, and workload. For parameter details, see
|
||||
[Feature descriptions](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-descriptions); for
|
||||
[Feature descriptions](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-descriptions); for
|
||||
recommended configurations for each deployment scenario, see [Best practices](#best-practices).
|
||||
</Note>
|
||||
|
||||
For feature compatibility and conflict information between features,
|
||||
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-compatibility).
|
||||
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-compatibility).
|
||||
|
||||
## Prerequisites
|
||||
|
||||
@@ -53,10 +53,10 @@ see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_opti
|
||||
|
||||
Before following this tutorial, complete the environment setup in the documents below:
|
||||
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/ascend_npu_quick_start) — the fastest way to get started.
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/getting-started/quick_start) — the fastest way to get started.
|
||||
It walks you through launching the official container image, starting the SGLang server, and sending a test request.
|
||||
Recommended if you are new to SGLang on Ascend.
|
||||
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/ascend_npu) — the full installation guide.
|
||||
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/getting-started/installation) — the full installation guide.
|
||||
It covers the component version mapping (CANN, TorchNPU, Triton, kernels, etc.), building from source or from a
|
||||
Dockerfile, and recommended system settings (CPU power scheme, NUMA, swap). Use it when you need to install or customize
|
||||
the environment instead of using the official image.
|
||||
@@ -80,7 +80,7 @@ This is the minimum recommended configuration. For optimized configurations,
|
||||
see [Best practices](#best-practices), which may require additional cards or nodes.
|
||||
|
||||
For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3),
|
||||
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/ascend_npu_reference#hardware).
|
||||
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware).
|
||||
|
||||
## Installation
|
||||
|
||||
@@ -92,7 +92,7 @@ The dependencies required for the NPU runtime environment have been integrated i
|
||||
online platform. You can directly pull it.
|
||||
|
||||
Both **stable releases** and **daily builds** are available. The following command is based on the stable release tag.
|
||||
For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/ascend_npu_faq#8-docker-image-versions-stable-release-vs-daily-build).
|
||||
For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build).
|
||||
|
||||
<Tabs>
|
||||
<Tab title="Atlas 800I A3">
|
||||
@@ -172,7 +172,7 @@ quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b
|
||||
Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios
|
||||
with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized
|
||||
deployment commands and benchmark data, see
|
||||
[Kimi K2.6 Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/best_practice/kimi_k2_6#single-node-pd-mixed).
|
||||
[Kimi K2.6 Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6#single-node-pd-mixed).
|
||||
|
||||
### Multi-node online deployment
|
||||
|
||||
@@ -180,14 +180,14 @@ Multi-node deployment distributes the model across multiple Atlas 800I A3 nodes
|
||||
prefill and decode on the same nodes (PD mixed mode), suitable for scenarios that need more device memory than a single
|
||||
node can provide. This scenario is already covered in the best practice. For the complete, optimized
|
||||
deployment commands and benchmark data, see
|
||||
[Kimi-K2.6 Best Practice — Multi-node On A3](/docs/hardware-platforms/ascend-npus/best_practice/kimi_k2_6#multi-node-pd-mixed).
|
||||
[Kimi-K2.6 Best Practice — Multi-node On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6#multi-node-pd-mixed).
|
||||
|
||||
### Multi-node PD disaggregation deployment
|
||||
|
||||
PD disaggregation splits the prefill and decode stages onto separate nodes, reducing interference and improving
|
||||
throughput for high-concurrency scenarios. This scenario is already covered in the best practice. For the complete, optimized
|
||||
deployment commands and benchmark data, see
|
||||
[Kimi-K2.6 Best Practice — PD Disaggregation On A3](/docs/hardware-platforms/ascend-npus/best_practice/kimi_k2_6#pd-disaggregation).
|
||||
[Kimi-K2.6 Best Practice — PD Disaggregation On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6#pd-disaggregation).
|
||||
|
||||
## Functional verification
|
||||
|
||||
@@ -242,28 +242,28 @@ Expected result: an HTTP 200 response with a description of the image.
|
||||
|
||||
Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For more
|
||||
testing examples (Health Check, Generate, Chat Completions, and port usage guidance),
|
||||
see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service).
|
||||
see [Testing the Service](/docs/hardware-platforms/ascend-npus/getting-started/installation#testing-the-service).
|
||||
|
||||
## Accuracy evaluation
|
||||
|
||||
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation).
|
||||
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/accuracy_evaluation).
|
||||
|
||||
## Performance
|
||||
|
||||
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing).
|
||||
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/performance_testing).
|
||||
|
||||
## Best practices
|
||||
|
||||
### Best practice configuration reference
|
||||
|
||||
For complete optimal configurations with deployment scripts and benchmark commands, see the
|
||||
[Kimi-K2.6 Best Practice](/docs/hardware-platforms/ascend-npus/best_practice/kimi_k2_6) page.
|
||||
[Kimi-K2.6 Best Practice](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6) page.
|
||||
|
||||
## Performance tuning
|
||||
|
||||
For the full list of supported features, see [Supported features](#supported-features). For detailed optimization
|
||||
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization).
|
||||
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning).
|
||||
|
||||
## FAQ
|
||||
|
||||
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/ascend_npu_faq).
|
||||
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/faq).
|
||||
+13
-13
@@ -35,12 +35,12 @@ v0.5.13 or a later version.
|
||||
<Note>
|
||||
The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment
|
||||
mode, and workload. For parameter details, see
|
||||
[Feature descriptions](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-descriptions); for
|
||||
[Feature descriptions](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-descriptions); for
|
||||
recommended configurations for each deployment scenario, see [Best practices](#best-practices).
|
||||
</Note>
|
||||
|
||||
For feature compatibility and conflict information between features,
|
||||
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-compatibility).
|
||||
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-compatibility).
|
||||
|
||||
## Prerequisites
|
||||
|
||||
@@ -48,10 +48,10 @@ see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_opti
|
||||
|
||||
Before following this tutorial, complete the environment setup in the documents below:
|
||||
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/ascend_npu_quick_start) — the fastest way to get started.
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/getting-started/quick_start) — the fastest way to get started.
|
||||
It walks you through launching the official container image, starting the SGLang server, and sending a test request.
|
||||
Recommended if you are new to SGLang on Ascend.
|
||||
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/ascend_npu) — the full installation guide.
|
||||
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/getting-started/installation) — the full installation guide.
|
||||
It covers the component version mapping (CANN, TorchNPU, Triton, kernels, etc.), building from source or from a
|
||||
Dockerfile, and recommended system settings (CPU power scheme, NUMA, swap). Use it when you need to install or customize
|
||||
the environment instead of using the official image.
|
||||
@@ -72,7 +72,7 @@ This is the minimum recommended configuration. For optimized configurations,
|
||||
see [Best practices](#best-practices), which may require additional cards or nodes.
|
||||
|
||||
For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3),
|
||||
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/ascend_npu_reference#hardware).
|
||||
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware).
|
||||
|
||||
## Installation
|
||||
|
||||
@@ -84,7 +84,7 @@ The dependencies required for the NPU runtime environment have been integrated i
|
||||
online platform. You can directly pull it.
|
||||
|
||||
Both **stable releases** and **daily builds** are available. The following command is based on the stable release tag.
|
||||
For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/ascend_npu_faq#8-docker-image-versions-stable-release-vs-daily-build).
|
||||
For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build).
|
||||
|
||||
<Tabs>
|
||||
<Tab title="Atlas 800I A3">
|
||||
@@ -164,7 +164,7 @@ quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b
|
||||
PD disaggregation splits the prefill and decode stages onto separate nodes, reducing interference and improving
|
||||
throughput for high-concurrency scenarios. This scenario is already covered in the best practice. For the complete, optimized
|
||||
deployment commands and benchmark data, see
|
||||
[MiMo-V2-Flash Best Practice — W8A8 24P PD Disaggregation On A3](/docs/hardware-platforms/ascend-npus/best_practice/mimo_v2_flash#pd-disaggregation).
|
||||
[MiMo-V2-Flash Best Practice — W8A8 24P PD Disaggregation On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash#pd-disaggregation).
|
||||
|
||||
## Functional verification
|
||||
|
||||
@@ -192,28 +192,28 @@ Expected result: an HTTP 200 response with the generated text containing "Paris"
|
||||
|
||||
Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For more
|
||||
testing examples (Health Check, Generate, Chat Completions, and port usage guidance),
|
||||
see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service).
|
||||
see [Testing the Service](/docs/hardware-platforms/ascend-npus/getting-started/installation#testing-the-service).
|
||||
|
||||
## Accuracy evaluation
|
||||
|
||||
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation).
|
||||
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/accuracy_evaluation).
|
||||
|
||||
## Performance
|
||||
|
||||
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing).
|
||||
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/performance_testing).
|
||||
|
||||
## Best practices
|
||||
|
||||
### Best practice configuration reference
|
||||
|
||||
For complete optimal configurations with deployment scripts and benchmark commands, see the
|
||||
[MiMo-V2-Flash Best Practice](/docs/hardware-platforms/ascend-npus/best_practice/mimo_v2_flash) page.
|
||||
[MiMo-V2-Flash Best Practice](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash) page.
|
||||
|
||||
## Performance tuning
|
||||
|
||||
For the full list of supported features, see [Supported features](#supported-features). For detailed optimization
|
||||
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization).
|
||||
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning).
|
||||
|
||||
## FAQ
|
||||
|
||||
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/ascend_npu_faq).
|
||||
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/faq).
|
||||
+13
-13
@@ -35,12 +35,12 @@ v0.5.13 or a later version.
|
||||
<Note>
|
||||
The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment
|
||||
mode, and workload. For parameter details, see
|
||||
[Feature descriptions](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-descriptions); for
|
||||
[Feature descriptions](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-descriptions); for
|
||||
recommended configurations for each deployment scenario, see [Best practices](#best-practices).
|
||||
</Note>
|
||||
|
||||
For feature compatibility and conflict information between features,
|
||||
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-compatibility).
|
||||
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-compatibility).
|
||||
|
||||
## Prerequisites
|
||||
|
||||
@@ -48,10 +48,10 @@ see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_opti
|
||||
|
||||
Before following this tutorial, complete the environment setup in the documents below:
|
||||
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/ascend_npu_quick_start) — the fastest way to get started.
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/getting-started/quick_start) — the fastest way to get started.
|
||||
It walks you through launching the official container image, starting the SGLang server, and sending a test request.
|
||||
Recommended if you are new to SGLang on Ascend.
|
||||
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/ascend_npu) — the full installation guide.
|
||||
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/getting-started/installation) — the full installation guide.
|
||||
It covers the component version mapping (CANN, TorchNPU, Triton, kernels, etc.), building from source or from a
|
||||
Dockerfile, and recommended system settings (CPU power scheme, NUMA, swap). Use it when you need to install or customize
|
||||
the environment instead of using the official image.
|
||||
@@ -72,7 +72,7 @@ This is the minimum recommended configuration. For optimized configurations,
|
||||
see [Best practices](#best-practices), which may require additional cards or nodes.
|
||||
|
||||
For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3),
|
||||
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/ascend_npu_reference#hardware).
|
||||
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware).
|
||||
|
||||
## Installation
|
||||
|
||||
@@ -84,7 +84,7 @@ The dependencies required for the NPU runtime environment have been integrated i
|
||||
online platform. You can directly pull it.
|
||||
|
||||
Both **stable releases** and **daily builds** are available. The following command is based on the stable release tag.
|
||||
For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/ascend_npu_faq#8-docker-image-versions-stable-release-vs-daily-build).
|
||||
For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build).
|
||||
|
||||
<Tabs>
|
||||
<Tab title="Atlas 800I A3">
|
||||
@@ -164,7 +164,7 @@ quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b
|
||||
Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios
|
||||
with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized
|
||||
deployment commands and benchmark data, see
|
||||
[MiniMax-M2.5 Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/best_practice/minimax_m2_5#single-node-pd-mixed).
|
||||
[MiniMax-M2.5 Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5#single-node-pd-mixed).
|
||||
|
||||
## Functional verification
|
||||
|
||||
@@ -192,28 +192,28 @@ Expected result: an HTTP 200 response with the generated text containing "Paris"
|
||||
|
||||
Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For more
|
||||
testing examples (Health Check, Generate, Chat Completions, and port usage guidance),
|
||||
see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service).
|
||||
see [Testing the Service](/docs/hardware-platforms/ascend-npus/getting-started/installation#testing-the-service).
|
||||
|
||||
## Accuracy evaluation
|
||||
|
||||
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation).
|
||||
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/accuracy_evaluation).
|
||||
|
||||
## Performance
|
||||
|
||||
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing).
|
||||
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/performance_testing).
|
||||
|
||||
## Best practices
|
||||
|
||||
### Best practice configuration reference
|
||||
|
||||
For complete optimal configurations with deployment scripts and benchmark commands, see the
|
||||
[MiniMax-M2.5 Best Practice](/docs/hardware-platforms/ascend-npus/best_practice/minimax_m2_5) page.
|
||||
[MiniMax-M2.5 Best Practice](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5) page.
|
||||
|
||||
## Performance tuning
|
||||
|
||||
For the full list of supported features, see [Supported features](#supported-features). For detailed optimization
|
||||
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization).
|
||||
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning).
|
||||
|
||||
## FAQ
|
||||
|
||||
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/ascend_npu_faq).
|
||||
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/faq).
|
||||
+13
-13
@@ -39,12 +39,12 @@ v0.5.13 or a later version.
|
||||
<Note>
|
||||
The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment
|
||||
mode, and workload. For parameter details, see
|
||||
[Feature descriptions](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-descriptions); for
|
||||
[Feature descriptions](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-descriptions); for
|
||||
recommended configurations for each deployment scenario, see [Best practices](#best-practices).
|
||||
</Note>
|
||||
|
||||
For feature compatibility and conflict information between features,
|
||||
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-compatibility).
|
||||
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-compatibility).
|
||||
|
||||
## Prerequisites
|
||||
|
||||
@@ -52,10 +52,10 @@ see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_opti
|
||||
|
||||
Before following this tutorial, complete the environment setup in the documents below:
|
||||
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/ascend_npu_quick_start) — the fastest way to get started.
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/getting-started/quick_start) — the fastest way to get started.
|
||||
It walks you through launching the official container image, starting the SGLang server, and sending a test request.
|
||||
Recommended if you are new to SGLang on Ascend.
|
||||
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/ascend_npu) — the full installation guide.
|
||||
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/getting-started/installation) — the full installation guide.
|
||||
It covers the component version mapping (CANN, TorchNPU, Triton, kernels, etc.), building from source or from a
|
||||
Dockerfile, and recommended system settings (CPU power scheme, NUMA, swap). Use it when you need to install or customize
|
||||
the environment instead of using the official image.
|
||||
@@ -80,7 +80,7 @@ This is the minimum recommended configuration. For optimized configurations,
|
||||
see [Best practices](#best-practices), which may require additional cards or nodes.
|
||||
|
||||
For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3),
|
||||
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/ascend_npu_reference#hardware).
|
||||
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware).
|
||||
|
||||
## Installation
|
||||
|
||||
@@ -92,7 +92,7 @@ The dependencies required for the NPU runtime environment have been integrated i
|
||||
online platform. You can directly pull it.
|
||||
|
||||
Both **stable releases** and **daily builds** are available. The following command is based on the stable release tag.
|
||||
For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/ascend_npu_faq#8-docker-image-versions-stable-release-vs-daily-build).
|
||||
For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build).
|
||||
|
||||
<Tabs>
|
||||
<Tab title="Atlas 800I A3">
|
||||
@@ -172,7 +172,7 @@ quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b
|
||||
Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios
|
||||
with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized
|
||||
deployment commands and benchmark data, see
|
||||
[Qwen3-235B-A22B Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/best_practice/qwen3_235b_a22b#single-node-pd-mixed).
|
||||
[Qwen3-235B-A22B Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_235b_a22b#single-node-pd-mixed).
|
||||
|
||||
### Multi-node PD disaggregation deployment
|
||||
|
||||
@@ -418,28 +418,28 @@ Expected result: an HTTP 200 response with the generated text containing "Paris"
|
||||
|
||||
Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For more
|
||||
testing examples (Health Check, Generate, Chat Completions, and port usage guidance),
|
||||
see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service).
|
||||
see [Testing the Service](/docs/hardware-platforms/ascend-npus/getting-started/installation#testing-the-service).
|
||||
|
||||
## Accuracy evaluation
|
||||
|
||||
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation).
|
||||
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/accuracy_evaluation).
|
||||
|
||||
## Performance
|
||||
|
||||
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing).
|
||||
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/performance_testing).
|
||||
|
||||
## Best practices
|
||||
|
||||
### Best practice configuration reference
|
||||
|
||||
For complete optimal configurations with deployment scripts and benchmark commands, see the
|
||||
[Qwen3-235B-A22B Best Practice](/docs/hardware-platforms/ascend-npus/best_practice/qwen3_235b_a22b) page.
|
||||
[Qwen3-235B-A22B Best Practice](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_235b_a22b) page.
|
||||
|
||||
## Performance tuning
|
||||
|
||||
For the full list of supported features, see [Supported features](#supported-features). For detailed optimization
|
||||
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization).
|
||||
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning).
|
||||
|
||||
## FAQ
|
||||
|
||||
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/ascend_npu_faq).
|
||||
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/faq).
|
||||
+13
-13
@@ -33,12 +33,12 @@ v0.5.13 or a later version.
|
||||
<Note>
|
||||
The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment
|
||||
mode, and workload. For parameter details, see
|
||||
[Feature descriptions](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-descriptions); for
|
||||
[Feature descriptions](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-descriptions); for
|
||||
recommended configurations for each deployment scenario, see [Best practices](#best-practices).
|
||||
</Note>
|
||||
|
||||
For feature compatibility and conflict information between features,
|
||||
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-compatibility).
|
||||
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-compatibility).
|
||||
|
||||
## Prerequisites
|
||||
|
||||
@@ -46,10 +46,10 @@ see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_opti
|
||||
|
||||
Before following this tutorial, complete the environment setup in the documents below:
|
||||
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/ascend_npu_quick_start) — the fastest way to get started.
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/getting-started/quick_start) — the fastest way to get started.
|
||||
It walks you through launching the official container image, starting the SGLang server, and sending a test request.
|
||||
Recommended if you are new to SGLang on Ascend.
|
||||
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/ascend_npu) — the full installation guide.
|
||||
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/getting-started/installation) — the full installation guide.
|
||||
It covers the component version mapping (CANN, TorchNPU, Triton, kernels, etc.), building from source or from a
|
||||
Dockerfile, and recommended system settings (CPU power scheme, NUMA, swap). Use it when you need to install or customize
|
||||
the environment instead of using the official image.
|
||||
@@ -71,7 +71,7 @@ This is the minimum recommended configuration. For optimized configurations,
|
||||
see [Best practices](#best-practices), which may require additional cards or nodes.
|
||||
|
||||
For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3),
|
||||
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/ascend_npu_reference#hardware).
|
||||
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware).
|
||||
|
||||
## Installation
|
||||
|
||||
@@ -83,7 +83,7 @@ The dependencies required for the NPU runtime environment have been integrated i
|
||||
online platform. You can directly pull it.
|
||||
|
||||
Both **stable releases** and **daily builds** are available. The following command is based on the stable release tag.
|
||||
For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/ascend_npu_faq#8-docker-image-versions-stable-release-vs-daily-build).
|
||||
For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build).
|
||||
|
||||
<Tabs>
|
||||
<Tab title="Atlas 800I A3">
|
||||
@@ -163,7 +163,7 @@ quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b
|
||||
Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios
|
||||
with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized
|
||||
deployment commands and benchmark data, see
|
||||
[Qwen3-30B-A3B Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/best_practice/qwen3_30b_a3b#single-node-pd-mixed).
|
||||
[Qwen3-30B-A3B Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_30b_a3b#single-node-pd-mixed).
|
||||
|
||||
## Functional verification
|
||||
|
||||
@@ -191,28 +191,28 @@ Expected result: an HTTP 200 response with the generated text containing "Paris"
|
||||
|
||||
Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For more
|
||||
testing examples (Health Check, Generate, Chat Completions, and port usage guidance),
|
||||
see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service).
|
||||
see [Testing the Service](/docs/hardware-platforms/ascend-npus/getting-started/installation#testing-the-service).
|
||||
|
||||
## Accuracy evaluation
|
||||
|
||||
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation).
|
||||
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/accuracy_evaluation).
|
||||
|
||||
## Performance
|
||||
|
||||
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing).
|
||||
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/performance_testing).
|
||||
|
||||
## Best practices
|
||||
|
||||
### Best practice configuration reference
|
||||
|
||||
For complete optimal configurations with deployment scripts and benchmark commands, see the
|
||||
[Qwen3-30B-A3B Best Practice](/docs/hardware-platforms/ascend-npus/best_practice/qwen3_30b_a3b) page.
|
||||
[Qwen3-30B-A3B Best Practice](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_30b_a3b) page.
|
||||
|
||||
## Performance tuning
|
||||
|
||||
For the full list of supported features, see [Supported features](#supported-features). For detailed optimization
|
||||
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization).
|
||||
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning).
|
||||
|
||||
## FAQ
|
||||
|
||||
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/ascend_npu_faq).
|
||||
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/faq).
|
||||
+13
-13
@@ -31,12 +31,12 @@ version.
|
||||
<Note>
|
||||
The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment
|
||||
mode, and workload. For parameter details, see
|
||||
[Feature descriptions](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-descriptions); for
|
||||
[Feature descriptions](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-descriptions); for
|
||||
recommended configurations for each deployment scenario, see [Best practices](#best-practices).
|
||||
</Note>
|
||||
|
||||
For feature compatibility and conflict information between features,
|
||||
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-compatibility).
|
||||
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-compatibility).
|
||||
|
||||
## Prerequisites
|
||||
|
||||
@@ -44,10 +44,10 @@ see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_opti
|
||||
|
||||
Before following this tutorial, complete the environment setup in the documents below:
|
||||
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/ascend_npu_quick_start) — the fastest way to get started.
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/getting-started/quick_start) — the fastest way to get started.
|
||||
It walks you through launching the official container image, starting the SGLang server, and sending a test request.
|
||||
Recommended if you are new to SGLang on Ascend.
|
||||
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/ascend_npu) — the full installation guide.
|
||||
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/getting-started/installation) — the full installation guide.
|
||||
It covers the component version mapping (CANN, TorchNPU, Triton, kernels, etc.), building from source or from a
|
||||
Dockerfile, and recommended system settings (CPU power scheme, NUMA, swap). Use it when you need to install or customize
|
||||
the environment instead of using the official image.
|
||||
@@ -71,7 +71,7 @@ This is the minimum recommended configuration. For optimized configurations,
|
||||
see [Best practices](#best-practices), which may require additional cards or nodes.
|
||||
|
||||
For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3),
|
||||
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/ascend_npu_reference#hardware).
|
||||
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware).
|
||||
|
||||
## Installation
|
||||
|
||||
@@ -83,7 +83,7 @@ The dependencies required for the NPU runtime environment have been integrated i
|
||||
online platform. You can directly pull it.
|
||||
|
||||
Both **stable releases** and **daily builds** are available. The following command is based on the stable release tag.
|
||||
For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/ascend_npu_faq#8-docker-image-versions-stable-release-vs-daily-build).
|
||||
For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build).
|
||||
|
||||
<Tabs>
|
||||
<Tab title="Atlas 800I A3">
|
||||
@@ -163,7 +163,7 @@ quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b
|
||||
Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios
|
||||
with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized
|
||||
deployment commands and benchmark data, see
|
||||
[Qwen3-32B Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/best_practice/qwen3_32b#single-node-pd-mixed).
|
||||
[Qwen3-32B Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_32b#single-node-pd-mixed).
|
||||
|
||||
## Functional verification
|
||||
|
||||
@@ -191,28 +191,28 @@ Expected result: an HTTP 200 response with the generated text containing "Paris"
|
||||
|
||||
Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For more
|
||||
testing examples (Health Check, Generate, Chat Completions, and port usage guidance),
|
||||
see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service).
|
||||
see [Testing the Service](/docs/hardware-platforms/ascend-npus/getting-started/installation#testing-the-service).
|
||||
|
||||
## Accuracy evaluation
|
||||
|
||||
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation).
|
||||
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/accuracy_evaluation).
|
||||
|
||||
## Performance
|
||||
|
||||
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing).
|
||||
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/performance_testing).
|
||||
|
||||
## Best practices
|
||||
|
||||
### Best practice configuration reference
|
||||
|
||||
For complete optimal configurations with deployment scripts and benchmark commands, see the
|
||||
[Qwen3-32B Best Practice](/docs/hardware-platforms/ascend-npus/best_practice/qwen3_32b) page.
|
||||
[Qwen3-32B Best Practice](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_32b) page.
|
||||
|
||||
## Performance tuning
|
||||
|
||||
For the full list of supported features, see [Supported features](#supported-features). For detailed optimization
|
||||
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization).
|
||||
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning).
|
||||
|
||||
## FAQ
|
||||
|
||||
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/ascend_npu_faq).
|
||||
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/faq).
|
||||
+13
-13
@@ -36,12 +36,12 @@ use v0.5.13 or a later version.
|
||||
<Note>
|
||||
The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment
|
||||
mode, and workload. For parameter details, see
|
||||
[Feature descriptions](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-descriptions); for
|
||||
[Feature descriptions](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-descriptions); for
|
||||
recommended configurations for each deployment scenario, see [Best practices](#best-practices).
|
||||
</Note>
|
||||
|
||||
For feature compatibility and conflict information between features,
|
||||
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-compatibility).
|
||||
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-compatibility).
|
||||
|
||||
## Prerequisites
|
||||
|
||||
@@ -49,10 +49,10 @@ see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_opti
|
||||
|
||||
Before following this tutorial, complete the environment setup in the documents below:
|
||||
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/ascend_npu_quick_start) — the fastest way to get started.
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/getting-started/quick_start) — the fastest way to get started.
|
||||
It walks you through launching the official container image, starting the SGLang server, and sending a test request.
|
||||
Recommended if you are new to SGLang on Ascend.
|
||||
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/ascend_npu) — the full installation guide.
|
||||
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/getting-started/installation) — the full installation guide.
|
||||
It covers the component version mapping (CANN, TorchNPU, Triton, kernels, etc.), building from source or from a
|
||||
Dockerfile, and recommended system settings (CPU power scheme, NUMA, swap). Use it when you need to install or customize
|
||||
the environment instead of using the official image.
|
||||
@@ -73,7 +73,7 @@ This is the minimum recommended configuration. For optimized configurations,
|
||||
see [Best practices](#best-practices), which may require additional cards or nodes.
|
||||
|
||||
For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3),
|
||||
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/ascend_npu_reference#hardware).
|
||||
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware).
|
||||
|
||||
## Installation
|
||||
|
||||
@@ -85,7 +85,7 @@ The dependencies required for the NPU runtime environment have been integrated i
|
||||
online platform. You can directly pull it.
|
||||
|
||||
Both **stable releases** and **daily builds** are available. The following command is based on the stable release tag.
|
||||
For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/ascend_npu_faq#8-docker-image-versions-stable-release-vs-daily-build).
|
||||
For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build).
|
||||
|
||||
<Tabs>
|
||||
<Tab title="Atlas 800I A3">
|
||||
@@ -165,7 +165,7 @@ quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b
|
||||
Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios
|
||||
with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized
|
||||
deployment commands and benchmark data, see
|
||||
[Qwen3.5-397B Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/best_practice/qwen3_5_397b#single-node-pd-mixed).
|
||||
[Qwen3.5-397B Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_5_397b#single-node-pd-mixed).
|
||||
|
||||
## Functional verification
|
||||
|
||||
@@ -220,28 +220,28 @@ Expected result: an HTTP 200 response with a description of the image.
|
||||
|
||||
Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For more
|
||||
testing examples (Health Check, Generate, Chat Completions, and port usage guidance),
|
||||
see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service).
|
||||
see [Testing the Service](/docs/hardware-platforms/ascend-npus/getting-started/installation#testing-the-service).
|
||||
|
||||
## Accuracy evaluation
|
||||
|
||||
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation).
|
||||
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/accuracy_evaluation).
|
||||
|
||||
## Performance
|
||||
|
||||
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing).
|
||||
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/performance_testing).
|
||||
|
||||
## Best practices
|
||||
|
||||
### Best practice configuration reference
|
||||
|
||||
For complete optimal configurations with deployment scripts and benchmark commands, see the
|
||||
[Qwen3.5-397B Best Practice](/docs/hardware-platforms/ascend-npus/best_practice/qwen3_5_397b) page.
|
||||
[Qwen3.5-397B Best Practice](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_5_397b) page.
|
||||
|
||||
## Performance tuning
|
||||
|
||||
For the full list of supported features, see [Supported features](#supported-features). For detailed optimization
|
||||
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization).
|
||||
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning).
|
||||
|
||||
## FAQ
|
||||
|
||||
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/ascend_npu_faq).
|
||||
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/faq).
|
||||
+13
-13
@@ -33,12 +33,12 @@ v0.5.13 or a later version.
|
||||
<Note>
|
||||
The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment
|
||||
mode, and workload. For parameter details, see
|
||||
[Feature descriptions](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-descriptions); for
|
||||
[Feature descriptions](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-descriptions); for
|
||||
recommended configurations for each deployment scenario, see [Best practices](#best-practices).
|
||||
</Note>
|
||||
|
||||
For feature compatibility and conflict information between features,
|
||||
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-compatibility).
|
||||
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-compatibility).
|
||||
|
||||
## Prerequisites
|
||||
|
||||
@@ -46,10 +46,10 @@ see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_opti
|
||||
|
||||
Before following this tutorial, complete the environment setup in the documents below:
|
||||
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/ascend_npu_quick_start) — the fastest way to get started.
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/getting-started/quick_start) — the fastest way to get started.
|
||||
It walks you through launching the official container image, starting the SGLang server, and sending a test request.
|
||||
Recommended if you are new to SGLang on Ascend.
|
||||
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/ascend_npu) — the full installation guide.
|
||||
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/getting-started/installation) — the full installation guide.
|
||||
It covers the component version mapping (CANN, TorchNPU, Triton, kernels, etc.), building from source or from a
|
||||
Dockerfile, and recommended system settings (CPU power scheme, NUMA, swap). Use it when you need to install or customize
|
||||
the environment instead of using the official image.
|
||||
@@ -69,7 +69,7 @@ This is the minimum recommended configuration. For optimized configurations,
|
||||
see [Best practices](#best-practices), which may require additional cards or nodes.
|
||||
|
||||
For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3),
|
||||
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/ascend_npu_reference#hardware).
|
||||
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware).
|
||||
|
||||
## Installation
|
||||
|
||||
@@ -81,7 +81,7 @@ The dependencies required for the NPU runtime environment have been integrated i
|
||||
online platform. You can directly pull it.
|
||||
|
||||
Both **stable releases** and **daily builds** are available. The following command is based on the stable release tag.
|
||||
For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/ascend_npu_faq#8-docker-image-versions-stable-release-vs-daily-build).
|
||||
For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build).
|
||||
|
||||
<Tabs>
|
||||
<Tab title="Atlas 800I A3">
|
||||
@@ -161,7 +161,7 @@ quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b
|
||||
Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios
|
||||
with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized
|
||||
deployment commands and benchmark data, see
|
||||
[Qwen3.6-27B Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_27b#single-node-pd-mixed).
|
||||
[Qwen3.6-27B Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_27b#single-node-pd-mixed).
|
||||
|
||||
## Functional verification
|
||||
|
||||
@@ -216,28 +216,28 @@ Expected result: an HTTP 200 response with a description of the image.
|
||||
|
||||
Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For more
|
||||
testing examples (Health Check, Generate, Chat Completions, and port usage guidance),
|
||||
see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service).
|
||||
see [Testing the Service](/docs/hardware-platforms/ascend-npus/getting-started/installation#testing-the-service).
|
||||
|
||||
## Accuracy evaluation
|
||||
|
||||
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation).
|
||||
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/accuracy_evaluation).
|
||||
|
||||
## Performance
|
||||
|
||||
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing).
|
||||
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/performance_testing).
|
||||
|
||||
## Best practices
|
||||
|
||||
### Best practice configuration reference
|
||||
|
||||
For complete optimal configurations with deployment scripts and benchmark commands, see the
|
||||
[Qwen3.6-27B Best Practice](/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_27b) page.
|
||||
[Qwen3.6-27B Best Practice](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_27b) page.
|
||||
|
||||
## Performance tuning
|
||||
|
||||
For the full list of supported features, see [Supported features](#supported-features). For detailed optimization
|
||||
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization).
|
||||
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning).
|
||||
|
||||
## FAQ
|
||||
|
||||
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/ascend_npu_faq).
|
||||
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/faq).
|
||||
+13
-13
@@ -34,12 +34,12 @@ in this version. To use the latest features (e.g., speculative decoding / NEXTN,
|
||||
<Note>
|
||||
The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment
|
||||
mode, and workload. For parameter details, see
|
||||
[Feature descriptions](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-descriptions); for
|
||||
[Feature descriptions](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-descriptions); for
|
||||
recommended configurations for each deployment scenario, see [Best practices](#best-practices).
|
||||
</Note>
|
||||
|
||||
For feature compatibility and conflict information between features,
|
||||
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-compatibility).
|
||||
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-compatibility).
|
||||
|
||||
## Prerequisites
|
||||
|
||||
@@ -47,10 +47,10 @@ see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_opti
|
||||
|
||||
Before following this tutorial, complete the environment setup in the documents below:
|
||||
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/ascend_npu_quick_start) — the fastest way to get started.
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/getting-started/quick_start) — the fastest way to get started.
|
||||
It walks you through launching the official container image, starting the SGLang server, and sending a test request.
|
||||
Recommended if you are new to SGLang on Ascend.
|
||||
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/ascend_npu) — the full installation guide.
|
||||
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/getting-started/installation) — the full installation guide.
|
||||
It covers the component version mapping (CANN, TorchNPU, Triton, kernels, etc.), building from source or from a
|
||||
Dockerfile, and recommended system settings (CPU power scheme, NUMA, swap). Use it when you need to install or customize
|
||||
the environment instead of using the official image.
|
||||
@@ -70,7 +70,7 @@ This is the minimum recommended configuration. For optimized configurations,
|
||||
see [Best practices](#best-practices), which may require additional cards or nodes.
|
||||
|
||||
For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3),
|
||||
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/ascend_npu_reference#hardware).
|
||||
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware).
|
||||
|
||||
## Installation
|
||||
|
||||
@@ -82,7 +82,7 @@ The dependencies required for the NPU runtime environment have been integrated i
|
||||
online platform. You can directly pull it.
|
||||
|
||||
Both **stable releases** and **daily builds** are available. The following command is based on the stable release tag.
|
||||
For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/ascend_npu_faq#8-docker-image-versions-stable-release-vs-daily-build).
|
||||
For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build).
|
||||
|
||||
<Tabs>
|
||||
<Tab title="Atlas 800I A3">
|
||||
@@ -162,7 +162,7 @@ quay.io/ascend/sglang:v0.5.14-cann9.0.0-910b
|
||||
Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios
|
||||
with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized
|
||||
deployment commands and benchmark data, see
|
||||
[Qwen3.6-35B-A3B Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_35b_a3b#single-node-pd-mixed).
|
||||
[Qwen3.6-35B-A3B Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_35b_a3b#single-node-pd-mixed).
|
||||
|
||||
## Functional verification
|
||||
|
||||
@@ -217,28 +217,28 @@ Expected result: an HTTP 200 response with a description of the image.
|
||||
|
||||
Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For more
|
||||
testing examples (Health Check, Generate, Chat Completions, and port usage guidance),
|
||||
see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service).
|
||||
see [Testing the Service](/docs/hardware-platforms/ascend-npus/getting-started/installation#testing-the-service).
|
||||
|
||||
## Accuracy evaluation
|
||||
|
||||
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation).
|
||||
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/accuracy_evaluation).
|
||||
|
||||
## Performance
|
||||
|
||||
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing).
|
||||
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/performance_testing).
|
||||
|
||||
## Best practices
|
||||
|
||||
### Best practice configuration reference
|
||||
|
||||
For complete optimal configurations with deployment scripts and benchmark commands, see the
|
||||
[Qwen3.6-35B-A3B Best Practice](/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_35b_a3b) page.
|
||||
[Qwen3.6-35B-A3B Best Practice](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_35b_a3b) page.
|
||||
|
||||
## Performance tuning
|
||||
|
||||
For the full list of supported features, see [Supported features](#supported-features). For detailed optimization
|
||||
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization).
|
||||
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning).
|
||||
|
||||
## FAQ
|
||||
|
||||
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/ascend_npu_faq).
|
||||
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/faq).
|
||||
+13
-13
@@ -33,12 +33,12 @@ version.
|
||||
<Note>
|
||||
The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment
|
||||
mode, and workload. For parameter details, see
|
||||
[Feature descriptions](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-descriptions); for
|
||||
[Feature descriptions](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-descriptions); for
|
||||
recommended configurations for each deployment scenario, see [Best practices](#best-practices).
|
||||
</Note>
|
||||
|
||||
For feature compatibility and conflict information between features,
|
||||
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-compatibility).
|
||||
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-compatibility).
|
||||
|
||||
## Prerequisites
|
||||
|
||||
@@ -46,10 +46,10 @@ see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_opti
|
||||
|
||||
Before following this tutorial, complete the environment setup in the documents below:
|
||||
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/ascend_npu_quick_start) — the fastest way to get started.
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/getting-started/quick_start) — the fastest way to get started.
|
||||
It walks you through launching the official container image, starting the SGLang server, and sending a test request.
|
||||
Recommended if you are new to SGLang on Ascend.
|
||||
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/ascend_npu) — the full installation guide.
|
||||
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/getting-started/installation) — the full installation guide.
|
||||
It covers the component version mapping (CANN, TorchNPU, Triton, kernels, etc.), building from source or from a
|
||||
Dockerfile, and recommended system settings (CPU power scheme, NUMA, swap). Use it when you need to install or customize
|
||||
the environment instead of using the official image.
|
||||
@@ -72,7 +72,7 @@ This is the minimum recommended configuration. For optimized configurations,
|
||||
see [Best practices](#best-practices), which may require additional cards or nodes.
|
||||
|
||||
For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3),
|
||||
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/ascend_npu_reference#hardware).
|
||||
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware).
|
||||
|
||||
## Installation
|
||||
|
||||
@@ -84,7 +84,7 @@ The dependencies required for the NPU runtime environment have been integrated i
|
||||
online platform. You can directly pull it.
|
||||
|
||||
Both **stable releases** and **daily builds** are available. The following command is based on the stable release tag.
|
||||
For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/ascend_npu_faq#8-docker-image-versions-stable-release-vs-daily-build).
|
||||
For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build).
|
||||
|
||||
<Tabs>
|
||||
<Tab title="Atlas 800I A3">
|
||||
@@ -164,7 +164,7 @@ quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b
|
||||
Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios
|
||||
with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized
|
||||
deployment commands and benchmark data, see
|
||||
[Qwen3-8B Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/best_practice/qwen3-8b#single-node-pd-mixed).
|
||||
[Qwen3-8B Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_8b#single-node-pd-mixed).
|
||||
|
||||
## Functional verification
|
||||
|
||||
@@ -192,28 +192,28 @@ Expected result: an HTTP 200 response with the generated text containing "Paris"
|
||||
|
||||
Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For more
|
||||
testing examples (Health Check, Generate, Chat Completions, and port usage guidance),
|
||||
see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service).
|
||||
see [Testing the Service](/docs/hardware-platforms/ascend-npus/getting-started/installation#testing-the-service).
|
||||
|
||||
## Accuracy evaluation
|
||||
|
||||
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation).
|
||||
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/accuracy_evaluation).
|
||||
|
||||
## Performance
|
||||
|
||||
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing).
|
||||
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/performance_testing).
|
||||
|
||||
## Best practices
|
||||
|
||||
### Best practice configuration reference
|
||||
|
||||
For complete optimal configurations with deployment scripts and benchmark commands, see the
|
||||
[Qwen3-8B Best Practice](/docs/hardware-platforms/ascend-npus/best_practice/qwen3-8b) page.
|
||||
[Qwen3-8B Best Practice](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_8b) page.
|
||||
|
||||
## Performance tuning
|
||||
|
||||
For the full list of supported features, see [Supported features](#supported-features). For detailed optimization
|
||||
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization).
|
||||
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning).
|
||||
|
||||
## FAQ
|
||||
|
||||
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/ascend_npu_faq).
|
||||
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/faq).
|
||||
+13
-13
@@ -35,12 +35,12 @@ v0.5.13 or a later version.
|
||||
<Note>
|
||||
The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment
|
||||
mode, and workload. For parameter details, see
|
||||
[Feature descriptions](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-descriptions); for
|
||||
[Feature descriptions](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-descriptions); for
|
||||
recommended configurations for each deployment scenario, see [Best practices](#best-practices).
|
||||
</Note>
|
||||
|
||||
For feature compatibility and conflict information between features,
|
||||
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-compatibility).
|
||||
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-compatibility).
|
||||
|
||||
## Prerequisites
|
||||
|
||||
@@ -48,10 +48,10 @@ see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_opti
|
||||
|
||||
Before following this tutorial, complete the environment setup in the documents below:
|
||||
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/ascend_npu_quick_start) — the fastest way to get started.
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/getting-started/quick_start) — the fastest way to get started.
|
||||
It walks you through launching the official container image, starting the SGLang server, and sending a test request.
|
||||
Recommended if you are new to SGLang on Ascend.
|
||||
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/ascend_npu) — the full installation guide.
|
||||
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/getting-started/installation) — the full installation guide.
|
||||
It covers the component version mapping (CANN, TorchNPU, Triton, kernels, etc.), building from source or from a
|
||||
Dockerfile, and recommended system settings (CPU power scheme, NUMA, swap). Use it when you need to install or customize
|
||||
the environment instead of using the official image.
|
||||
@@ -74,7 +74,7 @@ This is the minimum recommended configuration. For optimized configurations,
|
||||
see [Best practices](#best-practices), which may require additional cards or nodes.
|
||||
|
||||
For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3),
|
||||
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/ascend_npu_reference#hardware).
|
||||
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware).
|
||||
|
||||
## Installation
|
||||
|
||||
@@ -86,7 +86,7 @@ The dependencies required for the NPU runtime environment have been integrated i
|
||||
online platform. You can directly pull it.
|
||||
|
||||
Both **stable releases** and **daily builds** are available. The following command is based on the stable release tag.
|
||||
For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/ascend_npu_faq#8-docker-image-versions-stable-release-vs-daily-build).
|
||||
For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build).
|
||||
|
||||
<Tabs>
|
||||
<Tab title="Atlas 800I A3">
|
||||
@@ -166,7 +166,7 @@ quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b
|
||||
Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios
|
||||
with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized
|
||||
deployment commands and benchmark data, see
|
||||
[Qwen3-Next-80B-A3B-Instruct Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/best_practice/qwen3_next_80b_a3b_instruct#single-node-pd-mixed).
|
||||
[Qwen3-Next-80B-A3B-Instruct Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_next_80b_a3b_instruct#single-node-pd-mixed).
|
||||
|
||||
## Functional verification
|
||||
|
||||
@@ -194,28 +194,28 @@ Expected result: an HTTP 200 response with the generated text containing "Paris"
|
||||
|
||||
Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For more
|
||||
testing examples (Health Check, Generate, Chat Completions, and port usage guidance),
|
||||
see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service).
|
||||
see [Testing the Service](/docs/hardware-platforms/ascend-npus/getting-started/installation#testing-the-service).
|
||||
|
||||
## Accuracy evaluation
|
||||
|
||||
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation).
|
||||
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/accuracy_evaluation).
|
||||
|
||||
## Performance
|
||||
|
||||
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing).
|
||||
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/performance_testing).
|
||||
|
||||
## Best practices
|
||||
|
||||
### Best practice configuration reference
|
||||
|
||||
For complete optimal configurations with deployment scripts and benchmark commands, see the
|
||||
[Qwen3-Next-80B-A3B-Instruct Best Practice](/docs/hardware-platforms/ascend-npus/best_practice/qwen3_next_80b_a3b_instruct) page.
|
||||
[Qwen3-Next-80B-A3B-Instruct Best Practice](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_next_80b_a3b_instruct) page.
|
||||
|
||||
## Performance tuning
|
||||
|
||||
For the full list of supported features, see [Supported features](#supported-features). For detailed optimization
|
||||
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization).
|
||||
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning).
|
||||
|
||||
## FAQ
|
||||
|
||||
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/ascend_npu_faq).
|
||||
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/faq).
|
||||
+6
-6
@@ -1,13 +1,13 @@
|
||||
---
|
||||
title: "Ascend NPU Optimization"
|
||||
title: "Parameter Tuning"
|
||||
metatags:
|
||||
description: "Learn how to optimize SGLang for Ascend NPU using the best practice configurations for mainstream LLM models"
|
||||
---
|
||||
|
||||
This guide explains the role of each parameter used in SGLang deployments on Ascend NPU. It uses
|
||||
the [DeepSeek-V3.2 best practice configuration](/docs/hardware-platforms/ascend-npus/best_practice/deepseek_v3_2#pd-disaggregation)
|
||||
the [DeepSeek-V3.2 best practice configuration](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v3_2#pd-disaggregation)
|
||||
as the reference example. For a complete list of tested deployment configurations, see the
|
||||
[Ascend NPU Best Practice](/docs/hardware-platforms/ascend-npus/best_practice) page.
|
||||
[Ascend NPU Best Practice](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1) page.
|
||||
|
||||
<Note>
|
||||
Parameters in this guide fall into two categories:
|
||||
@@ -749,7 +749,7 @@ Reduces model weight and activation precision (e.g., W8A8, W4A8, W4A16) to decre
|
||||
On Ascend, the natively supported and highly optimized quantization method is ModelSlim (supporting W4A4, W8A8, W4A8 dynamic/static).
|
||||
Support for other community formats (such as AWQ, GPTQ, Auto-round, and Compressed-tensors) depends on the availability
|
||||
of specific Ascend custom kernels in your environment;
|
||||
see [Quantization on Ascend](/docs/hardware-platforms/ascend-npus/ascend_npu_quantization) for details and compatibility matrices.
|
||||
see [Quantization on Ascend](/docs/hardware-platforms/ascend-npus/optimization/quantization) for details and compatibility matrices.
|
||||
|
||||
### Chunked Prefill (`--chunked-prefill-size`)
|
||||
|
||||
@@ -797,7 +797,7 @@ Casts model weight tensors to the Ascend NPU ACL FRACTAL_NZ format (format 29) f
|
||||
|
||||
## See Also
|
||||
|
||||
- [Ascend NPU Best Practice](/docs/hardware-platforms/ascend-npus/best_practice) — Complete deployment configurations and benchmark results for all supported models
|
||||
- [Ascend NPU Environment Variables](/docs/hardware-platforms/ascend-npus/ascend_npu_environment_variables) — Reference for all Ascend NPU-related environment variables
|
||||
- [Ascend NPU Best Practice](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1) — Complete deployment configurations and benchmark results for all supported models
|
||||
- [Ascend NPU Environment Variables](/docs/hardware-platforms/ascend-npus/reference/environment_variables) — Reference for all Ascend NPU-related environment variables
|
||||
- [DeepSeek V3.2 Guide](/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2) — Detailed usage guide for DeepSeek V3.2 deployment
|
||||
- [Expert Parallelism](/docs/advanced_features/expert_parallelism) — DeepEP configuration and tuning guide
|
||||
+5
-5
@@ -1,5 +1,5 @@
|
||||
---
|
||||
title: "Ascend NPU Performance Profiling Guide"
|
||||
title: "Performance Profiling"
|
||||
metatags:
|
||||
description: "Ascend NPU performance profiling guide: use SGLang's built-in PyTorch Profiler for operator-level performance analysis of Ascend NPU inference services."
|
||||
---
|
||||
@@ -535,11 +535,11 @@ If the original data is still needed, back it up before running `analyse()`.
|
||||
|
||||
- [SGLang Benchmark and Profiling](/docs/developer_guide/benchmark_and_profiling)
|
||||
— General SGLang profiling guide
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/ascend_npu_quick_start)
|
||||
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/getting-started/quick_start)
|
||||
— Ascend NPU environment setup
|
||||
- [Ascend NPU Optimization](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization)
|
||||
- [Ascend NPU Optimization](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning)
|
||||
— Ascend NPU optimization parameters
|
||||
- [Ascend NPU Performance Testing](/docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing)
|
||||
- [Ascend NPU Performance Testing](/docs/hardware-platforms/ascend-npus/evaluation/performance_testing)
|
||||
— Ascend NPU performance benchmarking
|
||||
- [Ascend NPU Environment Variables](/docs/hardware-platforms/ascend-npus/ascend_npu_environment_variables)
|
||||
- [Ascend NPU Environment Variables](/docs/hardware-platforms/ascend-npus/reference/environment_variables)
|
||||
— Environment variable reference
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
---
|
||||
title: "Ascend NPU Ring-SP Performance (Wan2.1-T2V-1.3B)"
|
||||
title: "Ring-SP Performance (Wan2.1-T2V-1.3B)"
|
||||
metatags:
|
||||
description: "This page reports Ring-SP performance on Ascend NPU with torchnpu==2.10.0."
|
||||
---
|
||||
+8
-8
@@ -1,5 +1,5 @@
|
||||
---
|
||||
title: "Ascend NPU Reference"
|
||||
title: "Glossary"
|
||||
metatags:
|
||||
description: "Hardware concepts, communication libraries, deployment terminology, and common abbreviations for deploying SGLang on Ascend NPUs."
|
||||
---
|
||||
@@ -86,7 +86,7 @@ PD disaggregation separates inference into two stages running on different nodes
|
||||
| **2P1D** | 2 prefill nodes + 1 decode node |
|
||||
| **1P2D** | 1 prefill node + 2 decode nodes |
|
||||
|
||||
You will see these in [Best Practice](/docs/hardware-platforms/ascend-npus/best_practice)
|
||||
You will see these in [Best Practice](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1)
|
||||
section headings, e.g., `W8A8 2P1D 32P` means "W8A8 quantization, 2 prefill nodes +
|
||||
1 decode node, 32 cards total."
|
||||
|
||||
@@ -140,9 +140,9 @@ section headings, e.g., `W8A8 2P1D 32P` means "W8A8 quantization, 2 prefill node
|
||||
|
||||
## Where to go next
|
||||
|
||||
- [Quickstart](/docs/hardware-platforms/ascend-npus/ascend_npu_quick_start) — launch your first server
|
||||
- [Installation Guide](/docs/hardware-platforms/ascend-npus/ascend_npu) — full installation with component version mapping
|
||||
- [Supported Features](/docs/hardware-platforms/ascend-npus/ascend_npu_support_features) — per-parameter Ascend support status
|
||||
- [Supported Models](/docs/hardware-platforms/ascend-npus/ascend_npu_support_models) — models verified on Ascend NPU
|
||||
- [Model Tutorials](/docs/hardware-platforms/ascend-npus/model-tutorials) — step-by-step deployment guides
|
||||
- [Best Practice](/docs/hardware-platforms/ascend-npus/best_practice) — benchmark configurations and results
|
||||
- [Quickstart](/docs/hardware-platforms/ascend-npus/getting-started/quick_start) — launch your first server
|
||||
- [Installation Guide](/docs/hardware-platforms/ascend-npus/getting-started/installation) — full installation with component version mapping
|
||||
- [Supported Features](/docs/hardware-platforms/ascend-npus/reference/support_features) — per-parameter Ascend support status
|
||||
- [Supported Models](/docs/hardware-platforms/ascend-npus/reference/support_models) — models verified on Ascend NPU
|
||||
- [Model Tutorials](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_r1) — step-by-step deployment guides
|
||||
- [Best Practice](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1) — benchmark configurations and results
|
||||
+2
-2
@@ -1,5 +1,5 @@
|
||||
---
|
||||
title: "Support Features on Ascend NPU"
|
||||
title: "Supported Features"
|
||||
metatags:
|
||||
description: "Documentation for Support Features on Ascend NPU"
|
||||
---
|
||||
@@ -7,7 +7,7 @@ This section describes the basic functions and features supported by the Ascend
|
||||
questions, please [open an issue](https://github.com/sgl-project/sglang/issues).
|
||||
|
||||
If you want to know the meaning and usage of each parameter,
|
||||
click [Server Arguments](../../advanced_features/server_arguments).
|
||||
click [Server Arguments](../../../advanced_features/server_arguments).
|
||||
|
||||
## Model and tokenizer
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
---
|
||||
title: "Support Models on Ascend NPU"
|
||||
title: "Supported Models"
|
||||
metatags:
|
||||
description: "Documentation for Support Models on Ascend NPU"
|
||||
---
|
||||
@@ -5,7 +5,7 @@ description: Platform-specific guides for running SGLang on GPUs, TPUs, NPUs, CP
|
||||
|
||||
- [NVIDIA GPUs](./nvidia-gpus)
|
||||
- [AMD GPUs](./amd_gpu)
|
||||
- [Ascend NPUs](./ascend-npus/ascend_npu)
|
||||
- [Ascend NPUs](./ascend-npus/getting-started/installation)
|
||||
- [CPU Server](./cpu_server)
|
||||
- [NVIDIA Jetson Orin](./nvidia_jetson)
|
||||
- [TPU](./tpu)
|
||||
|
||||
@@ -95,7 +95,7 @@ pip install -e "python[diffusion]"
|
||||
|
||||
## Platform-Specific: Ascend NPU
|
||||
|
||||
For Ascend NPU, please follow the [NPU installation guide](../hardware-platforms/ascend-npus/ascend_npu).
|
||||
For Ascend NPU, please follow the [NPU installation guide](../hardware-platforms/ascend-npus/getting-started/installation).
|
||||
|
||||
Quick test:
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ Currently, the following models are supported:
|
||||
|
||||
## Installation
|
||||
|
||||
> **Note**: Currently, MindSpore models are provided by an independent package `sgl-mindspore`. Support for MindSpore is built upon current SGLang support for Ascend NPU platform. Please first [install SGLang for Ascend NPU](../hardware-platforms/ascend-npus/ascend_npu) and then install `sgl-mindspore`:
|
||||
> **Note**: Currently, MindSpore models are provided by an independent package `sgl-mindspore`. Support for MindSpore is built upon current SGLang support for Ascend NPU platform. Please first [install SGLang for Ascend NPU](../hardware-platforms/ascend-npus/getting-started/installation) and then install `sgl-mindspore`:
|
||||
|
||||
```bash Install
|
||||
git clone https://github.com/mindspore-lab/sgl-mindspore.git
|
||||
|
||||
Reference in New Issue
Block a user