diff --git a/docs/platforms/ascend_npu_best_practice.md b/docs/platforms/ascend_npu_best_practice.md index fd9f11926..aba6d2012 100644 --- a/docs/platforms/ascend_npu_best_practice.md +++ b/docs/platforms/ascend_npu_best_practice.md @@ -148,7 +148,7 @@ do --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head --moe-dense-tp 1 \ --cuda-graph-bs 12 14 16 18 20 22 24 26 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ --speculative-algorithm NEXTN --speculative-num-steps 2 --speculative-eagle-topk 1 --speculative-num-draft-tokens 3 \ - --tokenizer-worker-num 4 --prefill-round-robin-balance --disable-shared-experts-fusion --dtype bfloat16 \ + --tokenizer-worker-num 4 --disable-shared-experts-fusion --dtype bfloat16 \ --load-balance-method decode_round_robin NODE_RANK=$i break @@ -265,7 +265,7 @@ do --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head --moe-dense-tp 1 \ --cuda-graph-bs 2 4 6 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --tokenizer-worker-num 4 --prefill-round-robin-balance --disable-shared-experts-fusion --dtype bfloat16 \ + --tokenizer-worker-num 4 --disable-shared-experts-fusion --dtype bfloat16 \ --load-balance-method decode_round_robin NODE_RANK=$i break @@ -549,8 +549,8 @@ do --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head \ --cuda-graph-bs 8 10 12 14 16 18 20 22 24 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --prefill-round-robin-balance --disable-shared-experts-fusion --dtype bfloat16 --tokenizer-worker-num 4 \ - --load-balance-method decode_round_robin + --disable-shared-experts-fusion --dtype bfloat16 --tokenizer-worker-num 4 \ + --load-balance-method decode_round_robin NODE_RANK=$i break fi @@ -750,8 +750,8 @@ do --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head \ --cuda-graph-bs 8 10 12 14 16 18 20 22 24 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --prefill-round-robin-balance --disable-shared-experts-fusion --dtype bfloat16 --tokenizer-worker-num 4 \ - --load-balance-method decode_round_robin + --disable-shared-experts-fusion --dtype bfloat16 --tokenizer-worker-num 4 \ + --load-balance-method decode_round_robin NODE_RANK=$i break fi @@ -964,7 +964,6 @@ python3 -m sglang.launch_server --model-path ${MODEL_PATH} \ --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ --disaggregation-transfer-backend ascend \ --disaggregation-mode decode \ ---prefill-round-robin-balance \ --load-balance-method round_robin \ --nnodes $nnodes --node-rank $VC_TASK_INDEX \ --dist-init-addr ${IPs[0]}:10000 --load-balance-method decode_round_robin @@ -1097,7 +1096,7 @@ do --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ --dist-init-addr xxx:5000 \ --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ - --prefill-round-robin-balance --enable-dp-lm-head --dtype bfloat16 --tokenizer-worker-num 4 \ + --enable-dp-lm-head --dtype bfloat16 --tokenizer-worker-num 4 \ --load-balance-method decode_round_robin NODE_RANK=$i break @@ -2016,8 +2015,8 @@ do --attention-backend ascend --device npu --quantization modelslim --enable-dp-attention \ --moe-a2a-backend ascend_fuseep --cuda-graph-bs 16 32 48 56 64 72 80 88 96 \ --dist-init-addr DIP1:5000 \ - --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ - --prefill-round-robin-balance --enable-dp-lm-head --dtype bfloat16 --tokenizer-worker-num 4 --load-balance-method decode_round_robin + --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ + --enable-dp-lm-head --dtype bfloat16 --tokenizer-worker-num 4 --load-balance-method decode_round_robin NODE_RANK=$i break fi diff --git a/docs/platforms/ascend_npu_deepseek_example.md b/docs/platforms/ascend_npu_deepseek_example.md index 267d6ce76..cdecb544c 100644 --- a/docs/platforms/ascend_npu_deepseek_example.md +++ b/docs/platforms/ascend_npu_deepseek_example.md @@ -143,7 +143,6 @@ python -m sglang.launch_server \ --attention-backend ascend \ --device npu \ --quantization modelslim \ - --prefill-round-robin-balance \ --moe-a2a-backend deepep \ --enable-dp-attention \ --deepep-mode low_latency \ @@ -252,7 +251,7 @@ do --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head --moe-dense-tp 1 \ --cuda-graph-bs 12 14 16 18 20 22 24 26 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ --speculative-algorithm NEXTN --speculative-num-steps 2 --speculative-eagle-topk 1 --speculative-num-draft-tokens 3 \ - --tokenizer-worker-num 4 --prefill-round-robin-balance --disable-shared-experts-fusion --dtype bfloat16 \ + --tokenizer-worker-num 4 --disable-shared-experts-fusion --dtype bfloat16 \ --load-balance-method decode_round_robin NODE_RANK=$i break diff --git a/docs/platforms/ascend_npu_glm5_examples.md b/docs/platforms/ascend_npu_glm5_examples.md index e72250366..f748b6408 100644 --- a/docs/platforms/ascend_npu_glm5_examples.md +++ b/docs/platforms/ascend_npu_glm5_examples.md @@ -15,7 +15,7 @@ The GLM (General Language Model) series is an open-source bilingual large langua ### Installation -The dependencies required for the NPU runtime environment have been integrated into a Docker image and uploaded to the Ascend platform. You can directly pull it. +The dependencies required for the NPU runtime environment have been integrated into a Docker image and uploaded to the online platform. You can directly pull it. ```{code-block} bash #Atlas 800 A3 diff --git a/docs/platforms/ascend_npu_support_features.md b/docs/platforms/ascend_npu_support_features.md index ee243b798..80bf6ce89 100644 --- a/docs/platforms/ascend_npu_support_features.md +++ b/docs/platforms/ascend_npu_support_features.md @@ -72,6 +72,11 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen | `--swa-full-tokens-ratio` | `0.8` | Type: float | A2, A3 | | `--disable-hybrid-swa-memory` | `False` | bool flag
(set to enable) | A2, A3 | | `--radix-eviction-policy` | `lru` | `lru`,
`lfu` | A2, A3 | +| `--enable-prefill-delayer` | `False` | bool flag
(set to enable) | A2, A3 | +| `--prefill-delayer-max-delay-passes` | `30` | Type: int | A2, A3 | +| `--prefill-delayer-token-usage-low-watermark` | `None` | Type: float | A2, A3 | +| `--prefill-delayer-forward-passes-buckets` | `None` | List[float] | A2, A3 | +| `--prefill-delayer-wait-seconds-buckets` | `None` | List[float] | A2, A3 | | `--abort-on-priority-`
`when-disabled` | `False` | bool flag
(set to enable) | A2, A3 | | `--enable-dynamic-chunking` | `False` | bool flag
(set to enable) | A2, A3 | @@ -82,6 +87,8 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen | `--device` | `None` | Type: str | A2, A3 | | `--tensor-parallel-size`
`--tp-size` | `1` | Type: int | A2, A3 | | `--pipeline-parallel-size`
`--pp-size` | `1` | Type: int | A2, A3 | +| `--attention-context-parallel-size`
`--attn-cp-size` | `1` | Type: int | A2, A3 | +| `--moe-data-parallel-size`
`--moe-dp-size` | `1` | Type: int | A2, A3 | | `--pp-max-micro-batch-size` | `None` | Type: int | A2, A3 | | `--pp-async-batch-depth` | `None` | Type: int | A2, A3 | | `--stream-interval` | `1` | Type: int | A2, A3 | @@ -92,6 +99,8 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen | `--watchdog-timeout` | `300` | Type: float | A2, A3 | | `--soft-watchdog-timeout` | `300` | Type: float | A2, A3 | | `--dist-timeout` | `None` | Type: int | A2, A3 | +| `--download-dir` | `None` | Type: str | A2, A3 | +| `--model-checksum` | `None` | Type: str | A2, A3 | | `--base-gpu-id` | `0` | Type: int | A2, A3 | | `--gpu-id-step` | `1` | Type: int | A2, A3 | | `--sleep-on-idle` | `False` | bool flag (set to enable) | A2, A3 | @@ -142,6 +151,7 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen | `--served-model-name` | `None` | Type: str | A2, A3 | | `--weight-version` | `default` | Type: str | A2, A3 | | `--chat-template` | `None` | Type: str | A2, A3 | +| `--hf-chat-template-name` | `None` | Type: str | A2, A3 | | `--completion-template` | `None` | Type: str | A2, A3 | | `--enable-cache-report` | `False` | bool flag
(set to enable) | A2, A3 | | `--reasoning-parser` | `None` | `deepseek-r1`
`deepseek-v3`
`glm45`
`gpt-oss`
`kimi`
`qwen3`
`qwen3-thinking`
`step3` | A2, A3 | @@ -154,7 +164,6 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen |----------------------------------------|---------------|-----------------------------------------------------------|:----------------:| | `--data-parallel-size`
`--dp-size` | `1` | Type: int | A2, A3 | | `--load-balance-method` | `auto` | `auto`,
`round_robin`,
`follow_bootstrap_room`,
`total_requests`,
`total_tokens` | A2, A3 | -| `--prefill-round-robin-balance` | `False` | bool flag
(set to enable) | A2, A3 | ## Multi-node distributed serving @@ -389,7 +398,6 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen | `--disaggregation-bootstrap-port` | `8998` | Type: int | A2, A3 | | `--disaggregation-ib-device` | `None` | Type: str | Special for GPU | | `--disaggregation-decode-`
`enable-offload-kvcache` | `False` | bool flag
(set to enable) | A2, A3 | -| `--disaggregation-decode-`
`enable-fake-auto` | `False` | bool flag
(set to enable) | A2, A3 | | `--num-reserved-decode-tokens` | `512` | Type: int | A2, A3 | | `--disaggregation-decode-`
`polling-interval` | `1` | Type: int | A2, A3 | @@ -464,7 +472,7 @@ NPU, like Ktransformer, checkpoint-engine etc. | `--kt-weight-path` | `None` | Type: str | | `--kt-method` | `AMXINT4` | Type: str | | `--kt-cpuinfer` | `None` | Type: int | -| `--kt-threadpool-count` | 2 | Type: int | +| `--kt-threadpool-count` | `2` | Type: int | | `--kt-num-gpu-experts` | `None` | Type: int | | `--kt-max-deferred-`
`experts-per-token` | `None` | Type: int | diff --git a/docs/platforms/ascend_npu_support_models.md b/docs/platforms/ascend_npu_support_models.md index 674436537..3e4b03a3d 100644 --- a/docs/platforms/ascend_npu_support_models.md +++ b/docs/platforms/ascend_npu_support_models.md @@ -48,9 +48,9 @@ You are welcome to enable various models based on your business requirements. | baidu/ERNIE-4.5-21B-A3B-PT | ERNIE-4.5 (4.5, 4.5MoE series) | **√** | **√** | | OpenBMB/MiniCPM3-4B | MiniCPM (v3, 4B) | **√** | **√** | | Kimi/Kimi-K2-Thinking | Kimi | **√** | **√** | -| openai/gpt-oss-120b | GPTOSS | **√** | **√** | +| eigen-ai-labs/gpt-oss-120b-bf16 | GPTOSS | **√** | **√** | | allenai/OLMo-2-1124-7B-Instruct | OLMo | **√** | **√** | -| minimax/MiniMax-M2 | MiniMax-M2 | **√** | **√** | +| cyankiwi/MiniMax-M2-BF16 | MiniMax-M2 | **√** | **√** | | upstage/SOLAR-10.7B-Instruct-v1.0 | Solar | **√** | **√** | | bigcode/starcoder2-7b | StarCoder2 | **√** | **√** | | arcee-ai/Trinity-Mini | Trinity (Nano, Mini) | **√** | **√** |