diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_support_features.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_support_features.mdx index 408f62f21..63efb4f1c 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_support_features.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_support_features.mdx @@ -43,7 +43,7 @@ click [Server Arguments](../../advanced_features/server_arguments). `--tokenizer-mode` `auto` - auto, slow + `auto`, `slow` A2, A3 @@ -61,7 +61,7 @@ click [Server Arguments](../../advanced_features/server_arguments). `--load-format` `auto` - auto, safetensors, gguf + `auto`, `safetensors`, `gguf` A2, A3 @@ -103,7 +103,7 @@ click [Server Arguments](../../advanced_features/server_arguments). `--model-impl` `auto` - auto, sglang,
transformers + `auto`, `sglang`,
`transformers` A2, A3 @@ -167,7 +167,7 @@ click [Server Arguments](../../advanced_features/server_arguments). `--grpc-mode` `False` - False + `False` Planned @@ -248,7 +248,7 @@ click [Server Arguments](../../advanced_features/server_arguments). `--dtype` `auto` - auto,
float16,
bfloat16 + `auto`,
`float16`,
`bfloat16` A2, A3 @@ -378,7 +378,7 @@ click [Server Arguments](../../advanced_features/server_arguments). `--schedule-policy` `fcfs` - lpm, fcfs + `lpm`, `fcfs` A2, A3 @@ -436,43 +436,43 @@ click [Server Arguments](../../advanced_features/server_arguments). Planned - --radix-eviction-policy - lru - lru,
lfu + `--radix-eviction-policy` + `lru` + `lru`,
`lfu` A2, A3 - --enable-prefill-delayer + `--enable-prefill-delayer` `False` bool flag
(set to enable) A2, A3 - --prefill-delayer-max-delay-passes - 30 + `--prefill-delayer-max-delay-passes` + `30` Type: int A2, A3 - --prefill-delayer-token-usage-low-watermark - None + `--prefill-delayer-token-usage-low-watermark` + `None` Type: float A2, A3 - --prefill-delayer-forward-passes-buckets - None + `--prefill-delayer-forward-passes-buckets` + `None` List[float] A2, A3 - --prefill-delayer-wait-seconds-buckets - None + `--prefill-delayer-wait-seconds-buckets` + `None` List[float] A2, A3 - --abort-on-priority-
when-disabled + `--abort-on-priority-`
`when-disabled` `False` bool flag
(set to enable) A2, A3 @@ -520,132 +520,132 @@ click [Server Arguments](../../advanced_features/server_arguments). `--pipeline-parallel-size`
`--pp-size` `1` - Type: int; Currently 2 not supported + Type: int; Currently `2` not supported Experimental - --attention-context-parallel-size
--attn-cp-size - 1 + `--attention-context-parallel-size`
`--attn-cp-size` + `1` Type: int; must be equal to --tp-size A2, A3 - --moe-data-parallel-size
--moe-dp-size - 1 + `--moe-data-parallel-size`
`--moe-dp-size` + `1` Type: int Planned - --pp-max-micro-batch-size - None + `--pp-max-micro-batch-size` + `None` Type: int Experimental - --pp-async-batch-depth - None + `--pp-async-batch-depth` + `None` Type: int Experimental - --stream-interval - 1 + `--stream-interval` + `1` Type: int A2, A3 - --incremental-streaming-output - False + `--incremental-streaming-output` + `False` bool flag (set to enable) A2, A3 - --stream-response-default-include-usage - False + `--stream-response-default-include-usage` + `False` bool flag (set to enable) A2, A3 - --enable-streaming-session - False + `--enable-streaming-session` + `False` bool flag (set to enable) A2, A3 - --random-seed - None + `--random-seed` + `None` Type: int A2, A3 - --constrained-json-
whitespace-pattern - None + `--constrained-json-`
`whitespace-pattern` + `None` Type: str A2, A3 - --constrained-json-
disable-any-whitespace - False + `--constrained-json-`
`disable-any-whitespace` + `False` bool flag (set to enable) A2, A3 - --watchdog-timeout - 300 + `--watchdog-timeout` + `300` Type: float A2, A3 - --soft-watchdog-timeout - 300 + `--soft-watchdog-timeout` + `300` Type: float A2, A3 - --dist-timeout - None + `--dist-timeout` + `None` Type: int A2, A3 - --download-dir - None + `--download-dir` + `None` Type: str A2, A3 - --model-checksum + `--model-checksum` `None` Type: str Planned - --base-gpu-id - 0 + `--base-gpu-id` + `0` Type: int A2, A3 - --gpu-id-step - 1 + `--gpu-id-step` + `1` Type: int A2, A3 - --sleep-on-idle - False + `--sleep-on-idle` + `False` bool flag (set to enable) A2, A3 - --use-ray - False + `--use-ray` + `False` bool flag (set to enable) A2, A3 - --custom-sigquit-handler - None + `--custom-sigquit-handler` + `None` Only for engine A2, A3 @@ -693,13 +693,13 @@ click [Server Arguments](../../advanced_features/server_arguments). `--log-requests-level` `2` - 0, 1, 2, 3 + `0`, `1`, `2`, `3` A2, A3 `--log-requests-format` text - text, json + `text`, `json` A2, A3 @@ -817,14 +817,14 @@ click [Server Arguments](../../advanced_features/server_arguments). A2, A3 - --log-requests-target - None + `--log-requests-target` + `None` Type: str A2, A3 - --uvicorn-access-log-exclude-prefixes - [] + `--uvicorn-access-log-exclude-prefixes` + `[]` List[str] A2, A3 @@ -915,45 +915,45 @@ click [Server Arguments](../../advanced_features/server_arguments). A2, A3 - --hf-chat-template-name + `--hf-chat-template-name` `None` Type: str A2, A3 - --completion-template - None + `--completion-template` + `None` Type: str A2, A3 - --file-storage-path - sglang_storage + `--file-storage-path` + `sglang_storage` Type: str Unused reserved parameter - --enable-cache-report - False + `--enable-cache-report` + `False` bool flag
(set to enable) A2, A3 - --reasoning-parser + `--reasoning-parser` `None` - deepseek-r1
deepseek-v3
glm45
gpt-oss
kimi
qwen3
qwen3-thinking
step3 + `deepseek-r1`
`deepseek-v3`
`glm45`
`gpt-oss`
`kimi`
`qwen3`
`qwen3-thinking`
`step3` A2, A3 - --tool-call-parser - None - llama3
pythonic
qwen
qwen3_coder + `--tool-call-parser` + `None` + `llama3`
`pythonic`
`qwen`
`qwen3_coder` A2, A3 `--sampling-defaults` `model` - openai, model + `openai`, `model` A2, A3 @@ -986,8 +986,8 @@ click [Server Arguments](../../advanced_features/server_arguments). `--load-balance-method` - auto - auto,
round_robin,
follow_bootstrap_room,
total_requests,
total_tokens + `auto` + `auto`,
`round_robin`,
`follow_bootstrap_room`,
`total_requests`,
`total_tokens` A2, A3 @@ -1094,75 +1094,75 @@ click [Server Arguments](../../advanced_features/server_arguments). A2, A3 - --enable-lora-overlap-loading - False + `--enable-lora-overlap-loading` + `False` Bool flag
(set to enable) A2, A3 - --max-lora-rank + `--max-lora-rank` `None` Type: int A2, A3 - --lora-target-modules + `--lora-target-modules` `None` - all + `all` A2, A3 - --lora-paths - None + `--lora-paths` + `None` Type: List[str] /
JSON objects A2, A3 - --max-loras-per-batch - 8 + `--max-loras-per-batch` + `8` Type: int A2, A3 - --max-loaded-loras - None + `--max-loaded-loras` + `None` Type: int A2, A3 - --lora-eviction-policy - lru - lru,
fifo + `--lora-eviction-policy` + `lru` + `lru`,
`fifo` A2, A3 - --lora-backend - csgmv - triton,
csgmv,
ascend,
torch_native + `--lora-backend` + `csgmv` + `triton`,
`csgmv`,
`ascend`,
`torch_native` A2, A3 - --experts-shared-outer-loras - None + `--experts-shared-outer-loras` + `None` Type: bool A2, A3 - --lora-use-virtual-experts - False + `--lora-use-virtual-experts` + `False` bool flag
(set to enable) A2, A3 - --lora-strict-loading - False + `--lora-strict-loading` + `False` Type: bool A2, A3 `--max-lora-chunk-size` `16` - 16, 32,
64, 128 + `16`, `32`,
`64`, `128` Special for GPU @@ -1209,7 +1209,7 @@ click [Server Arguments](../../advanced_features/server_arguments). `--sampling-backend` `None` - pytorch,
ascend + `pytorch`,
`ascend` A2, A3 @@ -1227,19 +1227,19 @@ click [Server Arguments](../../advanced_features/server_arguments). `--dsa-prefill-backend` `flashmla_sparse` - flashmla_sparse,
flashmla_decode,
fa3,
tilelang,
aiter + `flashmla_sparse`,
`flashmla_decode`,
`fa3`,
`tilelang`,
`aiter` Special for GPU `--dsa-decode-backend` `fa3` - flashmla_prefill,
flashmla_kv,
fa3,
tilelang,
aiter + `flashmla_prefill`,
`flashmla_kv`,
`fa3`,
`tilelang`,
`aiter` Special for GPU `--fp8-gemm-backend` `auto` - auto,
deep_gemm,
flashinfer_trtllm,
flashinfer_cutlass,
flashinfer_deepgemm,
cutlass,
triton,
aiter + `auto`,
`deep_gemm`,
`flashinfer_trtllm`,
`flashinfer_cutlass`,
`flashinfer_deepgemm`,
`cutlass`,
`triton`,
`aiter` Special for GPU @@ -1274,7 +1274,7 @@ click [Server Arguments](../../advanced_features/server_arguments). `--speculative-algorithm` `None` - EAGLE3,
NEXTN + `EAGLE3`,
`NEXTN` A2, A3 @@ -1286,13 +1286,13 @@ click [Server Arguments](../../advanced_features/server_arguments). `--speculative-draft-model-`
`revision` `None` - Type: str,
branch name,
tag name,
commit id + Type: str,
`branch name`,
`tag name`,
`commit id` A2, A3 `--speculative-draft-load-format` - auto - auto,
dummy + `auto` + `auto`,
`dummy` A2, A3 @@ -1334,7 +1334,7 @@ click [Server Arguments](../../advanced_features/server_arguments). `--speculative-attention-`
`mode` `prefill` - prefill,
decode + `prefill`,
`decode` A2, A3 @@ -1411,11 +1411,11 @@ click [Server Arguments](../../advanced_features/server_arguments). `--speculative-ngram-`
`match-type` `BFS` - BFS,
PROB - Experimental. BFS uses recency-based expansion; PROB uses frequency-based expansion. + `BFS`,
`PROB` + Experimental. `BFS` uses recency-based expansion; `PROB` uses frequency-based expansion. - --speculative-ngram-
max-trie-depth + `--speculative-ngram-`
`max-trie-depth` `18` Type: int Experimental @@ -1476,19 +1476,19 @@ click [Server Arguments](../../advanced_features/server_arguments). `--moe-a2a-backend` `none` - none,
deepep,
ascend_fuseep(It is incompatible with eplb) + `none`,
`deepep`,
`ascend_fuseep`(It is incompatible with eplb) A2, A3 `--moe-runner-backend` `auto` - auto, triton + `auto`, `triton` A2, A3 `--flashinfer-mxfp4-`
`moe-precision` `default` - default,
bf16 + `default`,
`bf16` Special for GPU @@ -1500,7 +1500,7 @@ click [Server Arguments](../../advanced_features/server_arguments). `--deepep-mode` `auto` - normal,
low_latency,
auto + `normal`,
`low_latency`,
`auto` A2, A3 @@ -1518,13 +1518,13 @@ click [Server Arguments](../../advanced_features/server_arguments). `--ep-dispatch-algorithm` `None` - static,
dynamic,
fake + `static`,
`dynamic`,
`fake` A2, A3 `--init-expert-location` `trivial` - trivial,
<path.pt>,
<path.json>,
<json_string> + `trivial`,
`<path.pt>`,
`<path.json>`,
`<json_string>` A2, A3 @@ -1535,56 +1535,56 @@ click [Server Arguments](../../advanced_features/server_arguments). `--eplb-algorithm` - deepseek - auto,
deepseek + `deepseek` + `auto`,
`deepseek` A2, A3 - --eplb-rebalance-num-iterations - 1000 + `--eplb-rebalance-num-iterations` + `1000` Type: int A2, A3 - --eplb-rebalance-layers-
per-chunk - None + `--eplb-rebalance-layers-`
`per-chunk` + `None` Type: int A2, A3 - --eplb-min-rebalancing-
utilization-threshold - 1.0 + `--eplb-min-rebalancing-`
`utilization-threshold` + `1.0` Type: float A2, A3 - --expert-distribution-
recorder-mode + `--expert-distribution-`
`recorder-mode` `None` - stat,
stat_approx,
per_pass,
per_token + `stat`,
`stat_approx`,
`per_pass`,
`per_token` A2, A3 - --expert-distribution-
recorder-buffer-size - None + `--expert-distribution-`
`recorder-buffer-size` + `None` Type: int A2, A3 - --enable-expert-distribution-
metrics - False + `--enable-expert-distribution-`
`metrics` + `False` bool flag (set to enable) A2, A3 - --moe-dense-tp-size + `--moe-dense-tp-size` `None` - 1 + `1` A2, A3 - --elastic-ep-backend + `--elastic-ep-backend` `None` - none, mooncake + `none`, `mooncake` Special for GPU @@ -1624,19 +1624,19 @@ click [Server Arguments](../../advanced_features/server_arguments). `--mamba-ssm-dtype` `float32` - float32,
bfloat16,
float16 + `float32`,
`bfloat16`,
`float16` A2, A3 `--mamba-full-memory-ratio` - 0.9 + `0.9` Type: float A2, A3 `--mamba-scheduler-strategy` `auto` - auto,
no_buffer,
extra_buffer + `auto`,
`no_buffer`,
`extra_buffer` A2, A3 @@ -1689,36 +1689,36 @@ click [Server Arguments](../../advanced_features/server_arguments). `--hicache-write-policy` `write_through` - Currently only write_back supported + Currently only `write_back` supported A2, A3 - --hicache-io-backend - kernel - kernel_ascend,
direct + `--hicache-io-backend` + `kernel` + `kernel_ascend`,
`direct` A2, A3 - --hicache-mem-layout - layer_first - page_first_direct,
page_first_kv_split + `--hicache-mem-layout` + `layer_first` + `page_first_direct`,
`page_first_kv_split` A2, A3 - --hicache-storage-
backend - None - file + `--hicache-storage-`
`backend` + `None` + `file` A2, A3 - --hicache-storage-
prefetch-policy - timeout - best_effort,
wait_complete,
timeout + `--hicache-storage-`
`prefetch-policy` + `timeout` + `best_effort`,
`wait_complete`,
`timeout` Special for GPU - --hicache-storage-
backend-extra-config - None + `--hicache-storage-`
`backend-extra-config` + `None` Type: str Special for GPU @@ -1840,7 +1840,7 @@ click [Server Arguments](../../advanced_features/server_arguments). `--offload-mode` `cpu` - cpu (DeepSeek only)
meta (DeepSeek only)
sharded_gpu (DeepSeek only, only support tp=1 dp>1) + `cpu` (DeepSeek only)
`meta` (DeepSeek only)
`sharded_gpu` (DeepSeek only, only support tp=1 dp>1) A2, A3 @@ -1934,254 +1934,248 @@ click [Server Arguments](../../advanced_features/server_arguments). A2, A3 - --disable-tokenizer-
batch-decode + `--disable-tokenizer-`
`batch-decode` `False` bool flag
(set to enable) A2, A3 - --disable-custom-
all-reduce + `--disable-custom-`
`all-reduce` `False` bool flag
(set to enable) Special for GPU - --enable-mscclpp + `--enable-mscclpp` `False` bool flag
(set to enable) Special for GPU - --enable-torch-
symm-mem + `--enable-torch-`
`symm-mem` `False` bool flag
(set to enable) Special for GPU - --disable-overlap
-schedule + `--disable-overlap`
`-schedule` `False` bool flag
(set to enable) A2, A3 - --enable-mixed-
chunk + `--enable-mixed-`
`chunk` `False` bool flag
(set to enable) A2, A3 - --enable-dp-attention + `--enable-dp-attention` `False` bool flag
(set to enable) A2, A3 - --enable-dp-attention-local-control-broadcast + `--enable-dp-attention-local-control-broadcast` `False` bool flag
(set to enable) A2, A3 - --enable-dp-lm-head + `--enable-dp-lm-head` `False` bool flag
(set to enable) A2, A3 - --enable-two-
batch-overlap + `--enable-two-`
`batch-overlap` `False` bool flag
(set to enable) Planned - --enable-single-
batch-overlap + `--enable-single-`
`batch-overlap` `False` bool flag
(set to enable) A2, A3 - --tbo-token-
distribution-threshold - 0.48 + `--tbo-token-`
`distribution-threshold` + `0.48` Type: float Planned - --enable-torch-
compile - False - bool flag
(set to enable) - A2, A3 - - - --enable-torch-
compile-debug-mode + `--enable-torch-`
`compile` `False` bool flag
(set to enable) A2, A3 - --enforce-piecewise-
cuda-graph + `--enable-torch-`
`compile-debug-mode` + `False` + bool flag
(set to enable) + A2, A3 + + + `--enforce-piecewise-`
`cuda-graph` `False` bool flag
(set to enable);
Currently, Llama-3.1-8B-Instruct and Qwen2.5-7B-Instruct models are supported. A2, A3 - --piecewise-cuda-
graph-tokens - None + `--piecewise-cuda-`
`graph-tokens` + `None` Type: JSON
list A2, A3 - --piecewise-cuda-
graph-compiler - eager - eager + `--piecewise-cuda-`
`graph-compiler` + `eager` + `eager` A2, A3 - --torch-compile-max-bs - 32 + `--torch-compile-max-bs` + `32` Type: int A2, A3 - --piecewise-cuda-
graph-max-tokens - None + `--piecewise-cuda-`
`graph-max-tokens` + `None` Type: int A2, A3 - --torchao-config + `--torchao-config` `` Type: str Special for GPU - --enable-nan-detection - False - bool flag
(set to enable) - A2, A3 - - - --enable-p2p-check + `--enable-p2p-check` `False` bool flag
(set to enable) Special for GPU - --triton-attention-
reduce-in-fp32 + `--triton-attention-`
`reduce-in-fp32` `False` bool flag
(set to enable) Special for GPU - --triton-attention-
num-kv-splits - 8 + `--triton-attention-`
`num-kv-splits` + `8` Type: int Special for GPU - --triton-attention-
split-tile-size - None + `--triton-attention-`
`split-tile-size` + `None` Type: int Special for GPU - --delete-ckpt-
after-loading - False - bool flag
(set to enable) - A2, A3 - - - --enable-memory-saver + `--delete-ckpt-`
`after-loading` `False` bool flag
(set to enable) A2, A3 - --enable-weights-
cpu-backup + `--enable-memory-saver` `False` bool flag
(set to enable) A2, A3 - --enable-draft-weights-
cpu-backup + `--enable-weights-`
`cpu-backup` `False` bool flag
(set to enable) A2, A3 - --allow-auto-truncate + `--enable-draft-weights-`
`cpu-backup` `False` bool flag
(set to enable) A2, A3 - --enable-custom-
logit-processor + `--allow-auto-truncate` `False` bool flag
(set to enable) A2, A3 - --flashinfer-mla-
disable-ragged + `--enable-custom-`
`logit-processor` + `False` + bool flag
(set to enable) + A2, A3 + + + `--flashinfer-mla-`
`disable-ragged` `False` bool flag
(set to enable) Special for GPU - --disable-shared-
experts-fusion - True + `--disable-shared-`
`experts-fusion` + `True` bool flag
(set to enable) A2, A3 - --enforce-shared-experts-fusion - False - bool flag
(set to enable) - A2, A3 - - - --disable-chunked-
prefix-cache - True - bool flag
(set to enable) - A2, A3 - - - --disable-fast-
image-processor + `--enforce-shared-experts-fusion` `False` bool flag
(set to enable) A2, A3 - --keep-mm-feature-
on-device + `--disable-chunked-`
`prefix-cache` + `True` + bool flag
(set to enable) + A2, A3 + + + `--disable-fast-`
`image-processor` `False` bool flag
(set to enable) A2, A3 - --enable-return-
hidden-states + `--keep-mm-feature-`
`on-device` `False` bool flag
(set to enable) A2, A3 - --enable-return-
routed-experts + `--enable-return-`
`hidden-states` `False` bool flag
(set to enable) A2, A3 - --scheduler-recv-
interval - 1 + `--enable-return-`
`routed-experts` + `False` + bool flag
(set to enable) + A2, A3 + + + `--scheduler-recv-`
`interval` + `1` Type: int A2, A3 - --numa-node - None + `--numa-node` + `None` List[int] A2, A3 - --enable-deterministic-
inference - False + `--enable-deterministic-`
`inference` + `False` bool flag
(set to enable) Planned @@ -2345,7 +2339,7 @@ click [Server Arguments](../../advanced_features/server_arguments). `--disaggregation-mode` `null` - null,
prefill,
decode + `null`,
`prefill`,
`decode` A2, A3 @@ -2361,26 +2355,26 @@ click [Server Arguments](../../advanced_features/server_arguments). A2, A3 - --disaggregation-ib-device + `--disaggregation-ib-device` `None` Type: str Special for GPU - --disaggregation-decode-
enable-offload-kvcache - False - False + `--disaggregation-decode-`
`enable-offload-kvcache` + `False` + bool flag
(set to enable) A2, A3 - --num-reserved-decode-tokens - 512 + `--num-reserved-decode-tokens` + `512` Type: int A2, A3 - --disaggregation-decode-
polling-interval - 1 + `--disaggregation-decode-`
`polling-interval` + `1` Type: int A2, A3 @@ -2407,27 +2401,27 @@ click [Server Arguments](../../advanced_features/server_arguments). - --enable-adaptive-dispatch-to-encoder + `--enable-adaptive-dispatch-to-encoder` `False` bool flag
(set to enable adaptively dispatch) A2, A3 - --encoder-only + `--encoder-only` `False` bool flag
(set to launch an encoder-only server) A2, A3 - --language-only - False + `--language-only` + `False` bool flag
(set to load weights for the language model only) A2, A3 - --encoder-transfer-backend - zmq_to_scheduler - zmq_to_scheduler,
zmq_to_tokenizer,
mooncake + `--encoder-transfer-backend` + `zmq_to_scheduler` + `zmq_to_scheduler`,
`zmq_to_tokenizer`,
`mooncake` A2, A3 @@ -2504,7 +2498,7 @@ click [Server Arguments](../../advanced_features/server_arguments). `--remote-instance-weight-`
`loader-backend` `nccl` - transfer_engine,
nccl + `transfer_engine`,
`nccl` Special for GPU @@ -2577,25 +2571,25 @@ click [Server Arguments](../../advanced_features/server_arguments). - --enable-broadcast-mm-
inputs-process - False - bool flag
(set to enable) - A2, A3 - - - --mm-process-config - None - Type: JSON / Dict - A2, A3 - - - --mm-enable-dp-encoder + `--enable-broadcast-mm-`
`inputs-process` `False` bool flag
(set to enable) A2, A3 - --limit-mm-data-per-request + `--mm-process-config` + `None` + Type: JSON / Dict + A2, A3 + + + `--mm-enable-dp-encoder` + `False` + bool flag
(set to enable) + A2, A3 + + + `--limit-mm-data-per-request` `None` Type: JSON / Dict A2, A3 @@ -2664,8 +2658,8 @@ click [Server Arguments](../../advanced_features/server_arguments). - --forward-hooks - None + `--forward-hooks` + `None` Type: JSON list A2, A3 @@ -2692,7 +2686,7 @@ click [Server Arguments](../../advanced_features/server_arguments). - --config + `--config` `None` Type: str A2, A3 @@ -2774,8 +2768,8 @@ The following parameters have some functional deficiencies on community - --tool-server - None + `--tool-server` + `None` Type: str