diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-R1.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-R1.mdx index 72880aeb6..19c4cac29 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-R1.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-R1.mdx @@ -79,7 +79,7 @@ DeepSeek-R1 shares the same MoE architecture as DeepSeek-V3, so the same hardwar INT8 - 16× A100/A800, 32× L40S, Xeon 6980P CPU, 4× Atlas 800I A3 + 16× A100/A800, 32× L40S, Xeon 6980P CPU, 4× Ascend A3 Series W4A8 / AWQ / MXFP4 / NVFP4 diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx index 27c65ca95..ec65a5866 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx @@ -58,7 +58,7 @@ import { DeepSeekV3Deployment } from "/src/snippets/autoregressive/deepseek-v3-d INT8 - 16× A100/A800, 32× L40S, Xeon 6980P CPU, 4× Atlas 800I A3 + 16× A100/A800, 32× L40S, Xeon 6980P CPU, 4× Ascend A3 Series W4A8 / AWQ / MXFP4 / NVFP4 @@ -425,7 +425,7 @@ For multi-node serving and hardware-specific examples: - [16× A100 INT8](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-16-a100a800-with-int8-quantization) - [32× L40S INT8](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-32-l40s-with-int8-quantization) - [Xeon 6980P CPU](../../../docs/hardware-platforms/cpu_server#example-running-deepseek-v3-1-terminus) -- [4× Atlas 800I A3 (int8)](../../../docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_r1#multi-node-pd-disaggregation-deployment) +- [4× Ascend A3 Series (int8)](../../../docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_r1#multi-node-pd-disaggregation-deployment) **Blog references for large-scale deployment:** diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_1.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_1.mdx index 3c721c8d5..883ac1077 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_1.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_1.mdx @@ -56,7 +56,7 @@ DeepSeek-V3.1 shares the same model architecture as DeepSeek-V3, so the same har INT8 - 16× A100/A800, 32× L40S, Xeon 6980P CPU, 4× Atlas 800I A3 + 16× A100/A800, 32× L40S, Xeon 6980P CPU, 4× Ascend A3 Series W4A8 / AWQ / MXFP4 / NVFP4 diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx index 29d33e1d2..f0c4a7df4 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx @@ -40,7 +40,7 @@ docker pull lmsysorg/sglang:v0.5.8-rocm700-mi35x # Prefer the newest MI30x ROCm image tag from Docker Hub when available, or build from source. docker pull lmsysorg/sglang:v0.5.8-rocm700-mi30x -# Ascend NPU (Atlas 800I A2 / A3) +# Ascend NPU (A2/A3 Series) docker pull lmsysorg/sglang:dsv32-a2 docker pull lmsysorg/sglang:dsv32-a3 ``` diff --git a/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx b/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx index 4d55c6182..e316b9b01 100644 --- a/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx +++ b/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx @@ -56,7 +56,7 @@ For host and platform setup, see the -Pick your hardware, then the deployment shape and operating point. Node count follows the hardware recipe (B200 2×8, GB200 4×4, H100 4×8, B300 1×8, H200 2×8 — 4×8 on Unified High-Throughput, GB300 2×4, MI350X/MI355X 1×8, Atlas 800I A3 4×8 — 32 cards / 64 dies), so it is not a separate choice. If you serve the NVFP4 checkpoint (`nvidia/Kimi-K3-NVFP4`, the **Quantization** row in the panel below), use the `lmsysorg/sglang:dev-dev-kimi-k3-nvfp4` image. +Pick your hardware, then the deployment shape and operating point. Node count follows the hardware recipe (B200 2×8, GB200 4×4, H100 4×8, B300 1×8, H200 2×8 — 4×8 on Unified High-Throughput, GB300 2×4, MI350X/MI355X 1×8, Ascend A3 Series 4×8 — 32 cards / 64 dies), so it is not a separate choice. If you serve the NVFP4 checkpoint (`nvidia/Kimi-K3-NVFP4`, the **Quantization** row in the panel below), use the `lmsysorg/sglang:dev-dev-kimi-k3-nvfp4` image. **PD Mode** — `Unified` serves prefill and decode together. `Prefill` / `Decode` split them into dedicated pools (see [PD disaggregation](#3-4-pd-disaggregation)); `Prefill` ships two strategies, both chunked at 16k. On the 8-GPU platforms (B300 1×8, GB300 2×4), `Default` is TP8 and `Long-Context` is `--pp-size 8 --tp-size 1`. On the 16-GPU platforms (B200 2×8, GB200 4×4), both are `--pp-size 16 --tp-size 1` and differ only in `--mem-fraction-static` (0.85 vs 0.90) — deep PP is the throughput shape there, not just the long-context one (see [Deep PP](#deep-pp-for-prefill)). @@ -71,7 +71,7 @@ Pick your hardware, then the deployment shape and operating point. Node count fo **Spec Decode** — layers onto the strategy without changing it, on every platform except B200. DSPARK proposes 7 draft tokens per step (tune in the Playground) and requires `pp_size == 1`, so on B200 it also drops the pipeline and re-lays the same 16 GPUs flat: PP2 × TP8 → TP16, PP2 × DCPEP8 → DCPEP16. DFLASH has no published draft checkpoint. The win is largest on short interactive traffic and fades as the prompt grows. -`--mamba-full-memory-ratio` is the one sizing flag, computed live: set your average request length in the [Mamba ratio calculator](#mamba-ratio-calculator); everything else follows the panels, and the result is pinned into the command. (The Atlas 800I A3 uses `--max-mamba-cache-size` instead.) +`--mamba-full-memory-ratio` is the one sizing flag, computed live: set your average request length in the [Mamba ratio calculator](#mamba-ratio-calculator); everything else follows the panels, and the result is pinned into the command. (The Ascend A3 Series uses `--max-mamba-cache-size` instead.) import { Deployment } from "/src/snippets/_deployment.jsx"; @@ -146,7 +146,7 @@ not been re-measured on any cell — re-measure before you rely on one. ## 2. Configuration Tips -**Memory: two pools, one flag.** K3 splits static memory into a worst-case-reserved **KDA state pool** (it sets the concurrency ceiling) and a paged **MLA KV pool**, divided by `--mamba-full-memory-ratio`. The command panel pins that flag to the [calculator](#mamba-ratio-calculator)'s output — set your average request length there; every other calculator input follows the panels. (On the Atlas 800I A3: `--max-mamba-cache-size`, no calculator.) After boot, read back `max_total_num_tokens` (the KV side) and the admitted-request cap (the state side). +**Memory: two pools, one flag.** K3 splits static memory into a worst-case-reserved **KDA state pool** (it sets the concurrency ceiling) and a paged **MLA KV pool**, divided by `--mamba-full-memory-ratio`. The command panel pins that flag to the [calculator](#mamba-ratio-calculator)'s output — set your average request length there; every other calculator input follows the panels. (On the Ascend A3 Series: `--max-mamba-cache-size`, no calculator.) After boot, read back `max_total_num_tokens` (the KV side) and the admitted-request cap (the state side). Capacity levers, all in the Playground. Each trades precision or cache behavior for capacity — re-verify accuracy on your workload: @@ -179,7 +179,7 @@ Speculation: DSPARK holds block size + 1 (= 8) intermediate states per request | H200 2×8 (4×8 on Unified High-Throughput) | TP16/EP16 + symm-mem, Marlin + FlashMLA; High-Throughput widens to TP32/EP32 over 4 nodes at mem-frac 0.90 with `extra_buffer_lazy` | same block on every node; export the cross-node NIC (`GLOO_SOCKET_IFNAME` / `NCCL_SOCKET_IFNAME`, `SGLANG_HOST_IP`); keep `NCCL_MNNVL_ENABLE=1 NCCL_CUMEM_ENABLE=1` | | H100 4×8 | TP32/EP32, Marlin + FlashMLA | SM90a build of the K3 image; pin NCCL/Gloo to the same NIC on all nodes; least post-weight headroom (80 GB) | | MI350X/MI355X 1×8 | TP8 ROCm/AITER | AITER A8W4 FlyDSL MoE, Triton attention (`SGLANG_MLA_DECODE_TUNE=1` for gfx950 MLA decode geometry), graph bs up to 256, fp8 kvcache; DSPARK supported. Activation-quant and fused-KDA-decode knobs: [AMD ROCm/AITER environment](#amd-env) | -| Atlas 800I A3 4×8 (32 cards / 64 dies) | TP64/DP4 + DeepEP | PD-mixed `Unified` only; DSPARK baked in; pin `GLOO`/`HCCL_SOCKET_IFNAME` on every node | +| Ascend A3 Series 4×8 (32 cards / 64 dies) | TP64/DP4 + DeepEP | PD-mixed `Unified` only; DSPARK baked in; pin `GLOO`/`HCCL_SOCKET_IFNAME` on every node | **DCP notes** — the DCP cells are Balanced and High-Throughput on every Blackwell platform, in both the `Unified` and `Decode` roles: @@ -239,7 +239,7 @@ Pending update... ### 3.2 Tool Calling -Enable the `kimi_k3` tool-call parser (toggle **Tool Call Parser** in the **Parsers** card of the [Playground above](#playground)) to surface structured tool calls via `message.tool_calls`. Because K3 is a thinking model, the follow-up turn may put text in `reasoning_content` as well as `content` — print both. (Not yet supported on the Atlas 800I A3.) +Enable the `kimi_k3` tool-call parser (toggle **Tool Call Parser** in the **Parsers** card of the [Playground above](#playground)) to surface structured tool calls via `message.tool_calls`. Because K3 is a thinking model, the follow-up turn may put text in `reasoning_content` as well as `content` — print both. (Not yet supported on the Ascend A3 Series.) diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx index ac2decb20..30cd5eac5 100644 --- a/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx +++ b/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx @@ -286,7 +286,7 @@ row back to Auto. candidates through the target `lm_head`, including quantized heads, so it runs on the NVFP4 checkpoint (whose head is NVFP4-packed; the BF16 and FP8 checkpoints keep a dense head). - The Ascend comparison in #35629 used a 910C with BF16 target weights, + The Ascend comparison in #35629 used an A3 Series device with BF16 target weights, `--tp-size 2 --attention-backend ascend --mamba-ssm-dtype bfloat16 --mamba-radix-cache-strategy extra_buffer`, and disabled RadixCache for both baseline and DFlash2 to exclude cache warm-up and prefix reuse. The DFlash2 diff --git a/docs/cookbook/diffusion/FLUX/FLUX.mdx b/docs/cookbook/diffusion/FLUX/FLUX.mdx index 38f37de33..3c14f419f 100644 --- a/docs/cookbook/diffusion/FLUX/FLUX.mdx +++ b/docs/cookbook/diffusion/FLUX/FLUX.mdx @@ -34,7 +34,7 @@ This section provides deployment configurations optimized for different hardware FLUX models are optimized for high-quality image generation. The recommended launch configurations vary by hardware and model version. -**Interactive Command Generator**: Use the configuration selector below to automatically generate the appropriate deployment command for your hardware platform and model version. SGLang supports serving FLUX on NVIDIA B200, H200, H100, and AMD MI355X, MI325X, MI300X GPUs and Ascend A2, A3 NPUs. +**Interactive Command Generator**: Use the configuration selector below to automatically generate the appropriate deployment command for your hardware platform and model version. SGLang supports serving FLUX on NVIDIA B200, H200, H100, and AMD MI355X, MI325X, MI300X GPUs and Ascend A2/A3 Series NPUs. @@ -277,11 +277,11 @@ Test Environment: ============================================================ ``` - + **Server Command**: ```shell Command - #One A3 card has 2 npu chips + #One A3 Series card has 2 npu chips sglang serve --tp-size 2 --sp-degree 1 --model-path black-forest-labs/FLUX.1-dev --num-gpus 2 ``` @@ -363,11 +363,11 @@ Test Environment: ============================================================ ``` - + **Server Command** : ```shell Command - #One A3 card has 2 npu chips + #One A3 Series card has 2 npu chips sglang serve --tp-size 2 --sp-degree 1 --model-path black-forest-labs/FLUX.1-dev --num-gpus 2 ``` diff --git a/docs/cookbook/diffusion/Qwen-Image/Qwen-Image.mdx b/docs/cookbook/diffusion/Qwen-Image/Qwen-Image.mdx index 9a0fac5c6..87d6e1320 100644 --- a/docs/cookbook/diffusion/Qwen-Image/Qwen-Image.mdx +++ b/docs/cookbook/diffusion/Qwen-Image/Qwen-Image.mdx @@ -27,7 +27,7 @@ This section provides deployment configurations optimized for different hardware ### 3.1 Basic Configuration -Qwen-Image is a text-to-image model. The recommended launch configurations vary by hardware. SGLang supports serving Qwen-Image on NVIDIA B200, B300, H200, H100, AMD MI300X, MI325X, MI355X GPUs and Ascend A2, A3 NPUs. +Qwen-Image is a text-to-image model. The recommended launch configurations vary by hardware. SGLang supports serving Qwen-Image on NVIDIA B200, B300, H200, H100, AMD MI300X, MI325X, MI355X GPUs and Ascend A2/A3 Series NPUs. **Interactive Command Generator**: Use the configuration selector below to automatically generate the appropriate deployment command for your hardware platform. @@ -311,11 +311,11 @@ Test Environment: ============================================================ ``` - + **Server Command**: ```shell Command - #One A3 card has 2 npu chips + #One A3 Series card has 2 npu chips sglang serve --tp-size 2 --sp-degree 1 --model-path Qwen/Qwen-Image --num-gpus 2 ``` @@ -391,7 +391,7 @@ Test Environment: ============================================================ ``` - + **Benchmark Command**: ```shell Command diff --git a/docs/cookbook/diffusion/Wan/Wan2.1.mdx b/docs/cookbook/diffusion/Wan/Wan2.1.mdx index bd1e85726..a517902c9 100644 --- a/docs/cookbook/diffusion/Wan/Wan2.1.mdx +++ b/docs/cookbook/diffusion/Wan/Wan2.1.mdx @@ -27,7 +27,7 @@ This section provides deployment configurations optimized for different hardware ### 3.1 Basic Configuration -The Wan2.1 series offers models in multiple sizes and resolutions. SGLang supports Wan2.1 deployment on NVIDIA B200, B300, H200, H100, and AMD MI300X, MI325X, MI355X GPUs and Ascend A2, A3 NPUs. The recommended launch configurations vary by hardware, model size, and memory headroom. +The Wan2.1 series offers models in multiple sizes and resolutions. SGLang supports Wan2.1 deployment on NVIDIA B200, B300, H200, H100, and AMD MI300X, MI325X, MI355X GPUs and Ascend A2/A3 Series NPUs. The recommended launch configurations vary by hardware, model size, and memory headroom. **Interactive Command Generator**: Use the configuration selector below to automatically generate an appropriate deployment command for your model variant and options. @@ -217,11 +217,11 @@ You can use the built-in SGLang diffusion benchmark script to evaluate Wan2.1 pe ``` - + **Server Command**: ```bash Command - #One A3 card has 2 npu chips. Benchmark was did with two A3 cards + #One A3 Series card has 2 npu chips. Benchmark was done with two A3 Series cards sglang serve \ --model-path /models/Wan-AI/Wan2.1-T2V-14B-Diffusers/ \ --tp-size 2 \ @@ -321,11 +321,11 @@ You can use the built-in SGLang diffusion benchmark script to evaluate Wan2.1 pe ``` - + **Server Command**: ```bash Command - #One A3 card has 2 npu chips. Benchmark was did with two Atlas 3 cards + #One A3 Series card has 2 npu chips. Benchmark was done with two A3 Series cards SGLANG_CACHE_DIT_FN=2 \ SGLANG_CACHE_DIT_BN=1 \ SGLANG_CACHE_DIT_WARMUP=4 \ diff --git a/docs/cookbook/diffusion/Wan/Wan2.2.mdx b/docs/cookbook/diffusion/Wan/Wan2.2.mdx index 6dad57d92..d2b8f6c9f 100644 --- a/docs/cookbook/diffusion/Wan/Wan2.2.mdx +++ b/docs/cookbook/diffusion/Wan/Wan2.2.mdx @@ -35,7 +35,7 @@ This section provides deployment configurations optimized for different hardware The Wan2.2 series offers models in various sizes, architectures and input types, optimized for different hardware platforms. The recommended launch configurations vary by hardware and model size. -**Interactive Command Generator**: Use the configuration selector below to automatically generate the appropriate deployment command for your hardware platform, model size. SGLang supports serving Wan2.2 on NVIDIA B200, H200, AMD MI300X, MI325X, MI355X GPUs and Ascend A2, A3 NPUs. +**Interactive Command Generator**: Use the configuration selector below to automatically generate the appropriate deployment command for your hardware platform, model size. SGLang supports serving Wan2.2 on NVIDIA B200, H200, AMD MI300X, MI325X, MI355X GPUs and Ascend A2/A3 Series NPUs. @@ -297,10 +297,10 @@ Test Environment: ``` - + **Server Command**: ```shell Command - #One A3 card has 2 npu chips. Using four A3 cards in benchmarking + #One A3 Series card has 2 npu chips. Using four A3 Series cards in benchmarking sglang serve \ --model-path /models/Wan-AI/Wan2.2-T2V-A14B-Diffusers/ \ --tp-size 2 \ @@ -399,11 +399,11 @@ Test Environment: ``` - + **Server Command**: ```shell Command - #One A3 card has 2 npu chips. Using four A3 cards in benchmarking + #One A3 Series card has 2 npu chips. Using four A3 Series cards in benchmarking SGLANG_CACHE_DIT_FN=2 \ SGLANG_CACHE_DIT_BN=1 \ SGLANG_CACHE_DIT_WARMUP=4 \ diff --git a/docs/cookbook/diffusion/Z-Image/Z-Image-Turbo.mdx b/docs/cookbook/diffusion/Z-Image/Z-Image-Turbo.mdx index e4e5b7e12..91cdba231 100644 --- a/docs/cookbook/diffusion/Z-Image/Z-Image-Turbo.mdx +++ b/docs/cookbook/diffusion/Z-Image/Z-Image-Turbo.mdx @@ -249,11 +249,11 @@ Test Environment: ============================================================ ``` - + **Server Command**: ```shell Command - #One A3 card has 2 npu chips + #One A3 Series card has 2 npu chips sglang serve --model-path Tongyi-MAI/Z-Image-Turbo --tp-size 2 --sp-degree 1 --num-gpus 2 ``` @@ -329,7 +329,7 @@ Test Environment: ============================================================ ``` - + **Benchmark Command**: ```shell Command diff --git a/docs/docs/advanced_features/quantization.mdx b/docs/docs/advanced_features/quantization.mdx index befcb4d93..b56d15507 100644 --- a/docs/docs/advanced_features/quantization.mdx +++ b/docs/docs/advanced_features/quantization.mdx @@ -30,7 +30,7 @@ The following table summarizes quantization method support across NVIDIA and AMD Method NVIDIA GPUs AMD GPUs (MI300X/MI325X/MI350X) - Ascend NPUs (A2/A3/A5) + Ascend NPUs (A2/A3 Series, 950PR/DT Series) Notes @@ -46,22 +46,22 @@ The following table summarizes quantization method support across NVIDIA and AMD mxfp4 Yes Yes - Yes (A5) - On GPU: requires CDNA3/CDNA4 with MXFP support (uses Aiter). On Ascend NPU (A5): W4A4 MXFP4 for Qwen3 dense and MoE LLMs (MXFP4 weights + activations) — dense models support online dual-level MXFP4; offline W4A4_MXFP4 dense and MoE checkpoints (single-level) are auto-detected via modelslim. On Intel GPUs (XPU): W4A16 MoE experts on Xe2/BMG via sgl-kernel-xpu, enabled automatically with --device xpu (see Intel GPUs (XPU)) + Yes (950PR/DT Series) + On GPU: requires CDNA3/CDNA4 with MXFP support (uses Aiter). On Ascend NPU (950PR/DT Series): W4A4 MXFP4 for Qwen3 dense and MoE LLMs (MXFP4 weights + activations) — dense models support online dual-level MXFP4; offline W4A4_MXFP4 dense and MoE checkpoints (single-level) are auto-detected via modelslim. On Intel GPUs (XPU): W4A16 MoE experts on Xe2/BMG via sgl-kernel-xpu, enabled automatically with --device xpu (see Intel GPUs (XPU)) mxfp8 No No - Yes (A5 for Diffusion, LLM Dense Linear and LLM MoE) - Ascend NPU only; online + offline MXFP8 for Diffusion models (e.g., Wan2.2), LLM Dense Linear, and LLM MoE (FusedMoE, e.g. Qwen3-30B-A3B) on A5 series; uses CANN npu_dynamic_mx_quant / npu_quant_matmul (dense) and npu_grouped_matmul_swiglu_quant_v2 / npu_grouped_matmul (MoE) kernels + Yes (950PR/DT Series for Diffusion, LLM Dense Linear and LLM MoE) + Ascend NPU only; online + offline MXFP8 for Diffusion models (e.g., Wan2.2), LLM Dense Linear, and LLM MoE (FusedMoE, e.g. Qwen3-30B-A3B) on 950PR/DT Series; uses CANN npu_dynamic_mx_quant / npu_quant_matmul (dense) and npu_grouped_matmul_swiglu_quant_v2 / npu_grouped_matmul (MoE) kernels mxfp_w4a8 No No - Yes (A5) - Ascend NPU only; online W4A8 for Qwen3 dense LLM (MXFP4 weights + MXFP8 activations) on A5 series; offline W4A8_MXFP dense and MoE checkpoints are auto-detected via modelslim + Yes (950PR/DT Series) + Ascend NPU only; online W4A8 for Qwen3 dense LLM (MXFP4 weights + MXFP8 activations) on 950PR/DT Series; offline W4A8_MXFP dense and MoE checkpoints are auto-detected via modelslim blockwise_int8 @@ -923,9 +923,9 @@ SGLang supports quantization methods based on the advanced algorithm [auto-round ## Diffusion Model Quantization on Ascend NPU -SGLang-Diffusion supports MXFP8 quantization for diffusion models (such as Wan2.2) on Ascend A5 NPUs, in both online and offline (ModelSlim) modes. This is separate from the LLM serving path and uses the `sglang serve` / `sglang generate` CLI. +SGLang-Diffusion supports MXFP8 quantization for diffusion models (such as Wan2.2) on Ascend 950PR/DT Series NPUs, in both online and offline (ModelSlim) modes. This is separate from the LLM serving path and uses the `sglang serve` / `sglang generate` CLI. -**Requirements:** Ascend A5, CANN ≥ 8.0.RC3 +**Requirements:** Ascend 950PR/DT Series, CANN ≥ 8.0.RC3 ### Online MXFP8 diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx index 36f76a01f..7f45aea5c 100644 --- a/docs/docs/advanced_features/server_arguments.mdx +++ b/docs/docs/advanced_features/server_arguments.mdx @@ -1824,7 +1824,7 @@ Combining `--enable-response-store` with `--disaggregation-mode=prefill` or `dec `--deepep-dispatcher-output-dtype` - Select DeepEP dispather output dtype, could be bf16, fp8, int8 (only Ascend A2/A3 NPU), nvfp4 or auto. Default is auto, which follows a priority order (server argument → deprecated env var → input_global_scale check → dispatcher_output_dtype from quant_config → flashinfer/cutlass backend → NPU BF16 default → GPU FP8 default). + Select DeepEP dispatcher output dtype, could be bf16, fp8, int8 (only Ascend A2/A3 Series NPUs), nvfp4 or auto. Default is auto, which follows a priority order (server argument → deprecated env var → input_global_scale check → dispatcher_output_dtype from quant_config → flashinfer/cutlass backend → NPU BF16 default → GPU FP8 default). `auto` auto, bf16, fp8, int8, nvfp4 diff --git a/docs/docs/hardware-platforms/ascend-npus/development/operator_performance_optimizing.mdx b/docs/docs/hardware-platforms/ascend-npus/development/operator_performance_optimizing.mdx index 49c2748bc..7ab0d2def 100644 --- a/docs/docs/hardware-platforms/ascend-npus/development/operator_performance_optimizing.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/development/operator_performance_optimizing.mdx @@ -165,7 +165,7 @@ Must be vectorized. ### Tips -1. Ascend 910 series usually has only 40 or 48 vector cores. If the number of grids exceeds 40 or 48 vector cores, the grids will be delivered in a queue, resulting in a long waiting time. Therefore, the number of cores for high-performance implementation does not exceed the number of vector cores. +1. A2/A3 Series products usually have only 40 or 48 vector cores. If the number of grids exceeds 40 or 48 vector cores, the grids will be delivered in a queue, resulting in a long waiting time. Therefore, the number of cores for high-performance implementation does not exceed the number of vector cores. 2. Try to use up all the UB as much as possible. Move a large block size at a time to ensure that the bound is in the MTE. No Redundant Copy. 3. If the offset is a negative number, the current triton-ascend considers it as a discrete memory access scenario. As a result, the performance severely deteriorates, and the data is read from the entire DMA block instead of being read in scalar mode. 4. The UB of the Ascend hardware requires that the size of the tail axis of the tensor can be exactly divided by 32bytes. If the length of the tail axis is insufficient, the length of the tail axis is automatically supplemented. For example, the performance deteriorates exponentially due to automatic supplementation for the Tensor whose shape is (2048, 3). In this situation, you can perform the transposition operation to change the alignment axis to a lower dimension. In addition, the transposition operation is affected by the automatic supplement rule. Therefore, special skills are also required to avoid supplementation. diff --git a/docs/docs/hardware-platforms/ascend-npus/evaluation/accuracy_evaluation.mdx b/docs/docs/hardware-platforms/ascend-npus/evaluation/accuracy_evaluation.mdx index 265fb3315..6406b1723 100644 --- a/docs/docs/hardware-platforms/ascend-npus/evaluation/accuracy_evaluation.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/evaluation/accuracy_evaluation.mdx @@ -23,7 +23,7 @@ Ensure sufficient disk space before proceeding. The Docker image requires at lea First, launch the SGLang environment using the provided container image: - + ```shell Command export IMAGE=quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 @@ -46,7 +46,7 @@ docker run -it --rm --privileged --network=host --ipc=host --shm-size=16g \ ``` - + ```shell Command export IMAGE=quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 @@ -214,8 +214,8 @@ For more details, refer to the [EvalScope documentation](https://evalscope.readt ### SGLang Server Startup Failure -1. Verify device mapping: A2 uses `davinci[0-7]`, A3 uses `davinci[0-15]` -2. Confirm image tag matches device type: A2 uses `...-910b`, A3 uses `...-a3` +1. Verify device mapping: A2 Series uses `davinci[0-7]`, A3 Series uses `davinci[0-15]` +2. Confirm image tag matches device type: A2 Series uses `...-910b`, A3 Series uses `...-a3` 3. Check NPU status with `npu-smi info` 4. First run requires model download; set `HF_ENDPOINT=https://hf-mirror.com` if network access is restricted diff --git a/docs/docs/hardware-platforms/ascend-npus/evaluation/performance_testing.mdx b/docs/docs/hardware-platforms/ascend-npus/evaluation/performance_testing.mdx index 251fb5fdb..d4d16bb11 100644 --- a/docs/docs/hardware-platforms/ascend-npus/evaluation/performance_testing.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/evaluation/performance_testing.mdx @@ -6,7 +6,7 @@ metatags: This page walks through performance testing your SGLang deployment on Ascend NPUs. We cover three model types — text generation (`Qwen/Qwen2.5-7B-Instruct`), multimodal vision (`Qwen/Qwen2.5-VL-7B-Instruct`), and embedding (`Qwen/Qwen3-Embedding-8B`) — in both online and offline serving modes. You can use [Evalscope](https://evalscope.readthedocs.io/en/latest/), [AISBench](https://ais-bench-benchmark.readthedocs.io/en/latest/), or SGLang's built-in benchmarking tools. -The benchmark output examples in this guide are for illustration only. Actual performance depends on your hardware (e.g., Atlas 800I A2 vs A3), model version, SGLang version, and deployment configuration. Always run benchmarks on your own hardware to obtain accurate performance data. +The benchmark output examples in this guide are for illustration only. Actual performance depends on your hardware (e.g., Ascend A2 Series vs Ascend A3 Series), model version, SGLang version, and deployment configuration. Always run benchmarks on your own hardware to obtain accurate performance data. ## 1. Prepare diff --git a/docs/docs/hardware-platforms/ascend-npus/getting-started/installation.mdx b/docs/docs/hardware-platforms/ascend-npus/getting-started/installation.mdx index 54b7e3f4b..2de76d297 100644 --- a/docs/docs/hardware-platforms/ascend-npus/getting-started/installation.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/getting-started/installation.mdx @@ -137,7 +137,7 @@ Ensure sufficient disk space before pulling images. Each Docker image requires a You can obtain the dependency of a specified version of CANN through an image. - + ```bash Command # CANN 9.0.0 @@ -148,7 +148,7 @@ docker pull quay.io/ascend/cann:9.1.0-a3-ubuntu22.04-py3.12 ``` - + ```bash Command # CANN 9.0.0 @@ -298,7 +298,7 @@ SGLang images for the CANN 9.1.0 set have not been published yet. For now, pleas - + ```bash Command # Stable release @@ -309,7 +309,7 @@ docker pull quay.io/ascend/sglang:main-cann9.0.0-a3 ``` - + ```bash Command # Stable release @@ -332,7 +332,7 @@ cd sglang/docker # Build the docker image # Replace with the target architecture, e.g., amd64, arm64. # Optional build arguments: -# --build-arg DEVICE_TYPE=910b # Required for Atlas 800I A2 +# --build-arg DEVICE_TYPE=910b # Required for Ascend A2 Series # --build-arg APTMIRROR= # Use a custom APT mirror to improve download speed # If there are network errors, please modify the Dockerfile to add ARG HTTP_PROXY/HTTPS_PROXY and set them as ENV. docker build --build-arg TARGETARCH= -t -f npu.Dockerfile . @@ -343,7 +343,7 @@ docker build --build-arg TARGETARCH= -t -f npu.Dockerfile __Notice:__ `--privileged` and `--network=host` are required by RDMA, which is typically needed by Ascend NPU clusters. - + ```bash Command # Create a shortcut 'drun' to launch a privileged Docker container @@ -366,7 +366,7 @@ drun --env "HF_TOKEN=" \ ``` - + ```bash Command # Create a shortcut 'drun' to launch a privileged Docker container @@ -444,7 +444,7 @@ python3 -m sglang.launch_server \ 1. Launch Prefill Server - + ```bash Command # Enabling CPU Affinity @@ -468,7 +468,7 @@ python3 -m sglang.launch_server \ ``` - + ```bash Command # Enabling CPU Affinity @@ -498,7 +498,7 @@ python3 -m sglang.launch_server \ 2. Launch Decode Server - + ```bash Command # PREFILL_IP: IP address of the first Prefill Server @@ -518,7 +518,7 @@ python3 -m sglang.launch_server \ ``` - + ```bash Command # PREFILL_IP: IP address of the first Prefill Server diff --git a/docs/docs/hardware-platforms/ascend-npus/getting-started/quick_start.mdx b/docs/docs/hardware-platforms/ascend-npus/getting-started/quick_start.mdx index 6f82fcf38..65a696cd6 100644 --- a/docs/docs/hardware-platforms/ascend-npus/getting-started/quick_start.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/getting-started/quick_start.mdx @@ -14,10 +14,10 @@ see [SGLang installation with NPUs support](/docs/hardware-platforms/ascend-npus ### Supported Devices -- Atlas 800I A2 inference series (Atlas 800I A2) -- Atlas 800I A3 inference series (Atlas 800I A3) +- Ascend A2 Series Products +- Ascend A3 Series Products -To identify your device, run `npu-smi info -l`: A3 reports `Chip Count: 2` per NPU, while A2 reports `Chip Count: 1` per NPU. +To identify your device, run `npu-smi info -l`: A3 Series reports `Chip Count: 2` per NPU, while A2 Series reports `Chip Count: 1` per NPU. For hardware details, see the [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware). ### Docker @@ -48,7 +48,7 @@ If you have already downloaded model weights to a local path (e.g., `/path/to/mo adding `--volume /path/to/model:/path/to/model` to the `docker run` command below. - + ```shell Command # Choose one (uncomment the line you want): @@ -73,7 +73,7 @@ docker run -it --rm --privileged --network=host --ipc=host --shm-size=16g \ ``` - + ```shell Command # Choose one (uncomment the line you want): diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1.mdx index 3a5e16157..87960b283 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1.mdx @@ -7,25 +7,25 @@ metatags: This page focuses on optimal configuration and benchmark results for DeepSeek-R1 on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [DeepSeek-R1 Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_r1). -On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. +On the A3 Series, each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. ### Low Latency | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| DeepSeek-R1 | Atlas 800I A3 | 32 | PD Disaggregation | 3.5k+1.5k | 20ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-w8a8-2p1d-32p-in3k5-out1k5-20ms) | -| DeepSeek-R1 | Atlas 800I A3 | 32 | PD Disaggregation | 3.5k+1k | 19.0ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-w8a8-2p1d-32p-in3k5-out1k-19-0ms) | -| DeepSeek-R1 | Atlas 800I A3 | 32 | PD Disaggregation | 3.9k+1k | 19.0ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-w8a8-2p1d-32p-in3k9-out1k-19-0ms) | -| DeepSeek-R1 | Atlas 800I A3 | 32 | PD Disaggregation | 6k+1.6k | 20.5ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-w8a8-2p1d-32p-in6k-out1k6-20-5ms) | +| DeepSeek-R1 | Ascend A3 Series Products | 32 | PD Disaggregation | 3.5k+1.5k | 20ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-w8a8-2p1d-32p-in3k5-out1k5-20ms) | +| DeepSeek-R1 | Ascend A3 Series Products | 32 | PD Disaggregation | 3.5k+1k | 19.0ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-w8a8-2p1d-32p-in3k5-out1k-19-0ms) | +| DeepSeek-R1 | Ascend A3 Series Products | 32 | PD Disaggregation | 3.9k+1k | 19.0ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-w8a8-2p1d-32p-in3k9-out1k-19-0ms) | +| DeepSeek-R1 | Ascend A3 Series Products | 32 | PD Disaggregation | 6k+1.6k | 20.5ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-w8a8-2p1d-32p-in6k-out1k6-20-5ms) | ### High Throughput | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| DeepSeek-R1 | Atlas 800I A3 | 16 | PD Disaggregation | 3.5k+1.5k | 50ms | W4A8 INT8 | [Optimal Configuration](#deepseek-r1-w4a8-1p1d-16p-in3k5-out1k5-50ms) | -| DeepSeek-R1 | Atlas 800I A3 | 8 | PD Mixed | 3.5k+1.5k | 50ms | W4A8 INT8 | [Optimal Configuration](#deepseek-r1-w4a8-8p-in3k5-out1k5-50ms) | -| DeepSeek-R1 | Atlas 800I A3 | 32 | PD Disaggregation | 3.5k+1.5k | 50ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-w8a8-2p1d-32p-in3k5-out1k5-50ms) | +| DeepSeek-R1 | Ascend A3 Series Products | 16 | PD Disaggregation | 3.5k+1.5k | 50ms | W4A8 INT8 | [Optimal Configuration](#deepseek-r1-w4a8-1p1d-16p-in3k5-out1k5-50ms) | +| DeepSeek-R1 | Ascend A3 Series Products | 8 | PD Mixed | 3.5k+1.5k | 50ms | W4A8 INT8 | [Optimal Configuration](#deepseek-r1-w4a8-8p-in3k5-out1k5-50ms) | +| DeepSeek-R1 | Ascend A3 Series Products | 32 | PD Disaggregation | 3.5k+1.5k | 50ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-w8a8-2p1d-32p-in3k5-out1k5-50ms) | ## Optimal Configuration @@ -33,7 +33,7 @@ On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend **Model**: DeepSeek-R1 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 16 @@ -233,7 +233,7 @@ python -m sglang.bench_serving \ **Model**: DeepSeek-R1 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 8 @@ -341,7 +341,7 @@ python -m sglang.bench_serving \ **Model**: DeepSeek-R1 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 32 @@ -544,7 +544,7 @@ python -m sglang.bench_serving \ **Model**: DeepSeek-R1 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 32 @@ -751,7 +751,7 @@ python -m sglang.bench_serving \ **Model**: DeepSeek-R1 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 32 @@ -954,7 +954,7 @@ python -m sglang.bench_serving \ **Model**: DeepSeek-R1 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 32 @@ -1157,7 +1157,7 @@ python -m sglang.bench_serving \ **Model**: DeepSeek-R1 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 32 diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v3_2.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v3_2.mdx index bf08bdec5..6f3d6120a 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v3_2.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v3_2.mdx @@ -7,21 +7,21 @@ metatags: This page focuses on optimal configuration and benchmark results for DeepSeek-V3.2 on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [DeepSeek-V3.2 Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v3_2). -On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. +On the A3 Series, each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. ### Low Latency | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| DeepSeek-V3.2 | Atlas 800I A3 | 32 | PD Disaggregation | 128k+1k | 26ms | W8A8 INT8 | [Optimal Configuration](#deepseek-v3-2-w8a8-1p1d-32p-in128k-out1k-26ms) | -| DeepSeek-V3.2 | Atlas 800I A3 | 32 | PD Disaggregation | 128k+1k | 26ms | W8A8 INT8 | [Optimal Configuration](#deepseek-v3-2-w8a8-1p1d-32p-in128k-out1k-bs8) | +| DeepSeek-V3.2 | Ascend A3 Series Products | 32 | PD Disaggregation | 128k+1k | 26ms | W8A8 INT8 | [Optimal Configuration](#deepseek-v3-2-w8a8-1p1d-32p-in128k-out1k-26ms) | +| DeepSeek-V3.2 | Ascend A3 Series Products | 32 | PD Disaggregation | 128k+1k | 26ms | W8A8 INT8 | [Optimal Configuration](#deepseek-v3-2-w8a8-1p1d-32p-in128k-out1k-bs8) | ### High Throughput | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| DeepSeek-V3.2 | Atlas 800I A3 | 32 | PD Disaggregation | 128k+1k | 107ms | W8A8 INT8 | [Optimal Configuration](#deepseek-v3-2-w8a8-1p1d-32p-in128k-out1k-bs16) | +| DeepSeek-V3.2 | Ascend A3 Series Products | 32 | PD Disaggregation | 128k+1k | 107ms | W8A8 INT8 | [Optimal Configuration](#deepseek-v3-2-w8a8-1p1d-32p-in128k-out1k-bs16) | ## Optimal Configuration @@ -31,7 +31,7 @@ On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend **Model**: DeepSeek-V3.2 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 32 @@ -229,7 +229,7 @@ python -m sglang.bench_serving \ **Model**: DeepSeek-V3.2 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 32 @@ -426,7 +426,7 @@ python -m sglang.bench_serving \ **Model**: DeepSeek-V3.2 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 32 diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash.mdx index c4e5fed01..cdf073e43 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash.mdx @@ -7,16 +7,16 @@ metatags: This page focuses on optimal configuration and benchmark results for DeepSeek-V4-Flash on the Ascend NPU. -On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. +On the A3 Series, each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. ### High Throughput | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| DeepSeek-V4-Flash | Atlas 800I A3 | 16 | PD Disaggregation | 8k+1k | 50ms | W8A8 INT8 | [Optimal Configuration](#deepseek-v4-flash-w8a8-1p1d-16p-in8k-out1k-50ms) | -| DeepSeek-V4-Flash | Atlas 800I A3 | 8 | PD Mixed | 32k+1k | 50ms | W8A8 INT8 | [Optimal Configuration](#deepseek-v4-flash-w8a8-8p-in32k-out1k-50ms) | -| DeepSeek-V4-Flash | Atlas 800I A3 | 8 | PD Mixed | 8k+1k | 50ms | W8A8 INT8 | [Optimal Configuration](#deepseek-v4-flash-w8a8-8p-in8k-out1k-50ms) | +| DeepSeek-V4-Flash | Ascend A3 Series Products | 16 | PD Disaggregation | 8k+1k | 50ms | W8A8 INT8 | [Optimal Configuration](#deepseek-v4-flash-w8a8-1p1d-16p-in8k-out1k-50ms) | +| DeepSeek-V4-Flash | Ascend A3 Series Products | 8 | PD Mixed | 32k+1k | 50ms | W8A8 INT8 | [Optimal Configuration](#deepseek-v4-flash-w8a8-8p-in32k-out1k-50ms) | +| DeepSeek-V4-Flash | Ascend A3 Series Products | 8 | PD Mixed | 8k+1k | 50ms | W8A8 INT8 | [Optimal Configuration](#deepseek-v4-flash-w8a8-8p-in8k-out1k-50ms) | ## Optimal Configuration @@ -26,7 +26,7 @@ On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend **Model**: DeepSeek-V4-Flash -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 16 @@ -237,7 +237,7 @@ python -m sglang.bench_serving \ **Model**: DeepSeek-V4-Flash -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 8 @@ -363,7 +363,7 @@ python -m sglang.bench_serving \ **Model**: DeepSeek-V4-Flash -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 8 diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1.mdx index d6a713988..2835b6cf3 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1.mdx @@ -7,26 +7,26 @@ metatags: This page focuses on optimal configuration and benchmark results for GLM-5.1 on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [GLM-5.1 Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_1). -On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. +On the A3 Series, each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. ### Low Latency | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | TTFT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| GLM-5.1 | Atlas 800I A3 | 32 | PD Disaggregation | 65k+1.5k (90% prefix cache hit rate) | 25ms | - | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in65k-out1k5-prefix90-25ms) | +| GLM-5.1 | Ascend A3 Series Products | 32 | PD Disaggregation | 65k+1.5k (90% prefix cache hit rate) | 25ms | - | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in65k-out1k5-prefix90-25ms) | ### High Throughput | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | TTFT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| GLM-5.1 | Atlas 800I A3 | 16 | PD Mixed | 3.5k+1.5k | 50ms | - | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-16p-in3k5-out1k5-50ms) | -| GLM-5.1 | Atlas 800I A3 | 32 | PD Disaggregation | 128k+1k | 56.4ms | 13.1s | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in128k-out1k-56-4ms) | -| GLM-5.1 | Atlas 800I A3 | 32 | PD Disaggregation | 16k+1k | 50ms | - | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in16k-out1k-50ms) | -| GLM-5.1 | Atlas 800I A3 | 32 | PD Disaggregation | 64k+1k | 55.2ms | 7.58s | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in64k-out1k-55-2ms) | -| GLM-5.1 | Atlas 800I A3 | 32 | PD Disaggregation | 64k+1k | 50ms | - | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in64k-out1k-50ms) | -| GLM-5.1 | Atlas 800I A3 | 48 | PD Disaggregation | 128k+1k (90% prefix cache hit rate) | 50ms | - | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-2p1d-48p-in128k-out1k-prefix90-50ms) | -| GLM-5.1 | Atlas 800I A3 | 48 | PD Disaggregation | 64k+1k (90% prefix cache hit rate) | 50ms | - | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-4p1d-48p-in64k-out1k-prefix90-50ms) | +| GLM-5.1 | Ascend A3 Series Products | 16 | PD Mixed | 3.5k+1.5k | 50ms | - | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-16p-in3k5-out1k5-50ms) | +| GLM-5.1 | Ascend A3 Series Products | 32 | PD Disaggregation | 128k+1k | 56.4ms | 13.1s | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in128k-out1k-56-4ms) | +| GLM-5.1 | Ascend A3 Series Products | 32 | PD Disaggregation | 16k+1k | 50ms | - | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in16k-out1k-50ms) | +| GLM-5.1 | Ascend A3 Series Products | 32 | PD Disaggregation | 64k+1k | 55.2ms | 7.58s | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in64k-out1k-55-2ms) | +| GLM-5.1 | Ascend A3 Series Products | 32 | PD Disaggregation | 64k+1k | 50ms | - | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in64k-out1k-50ms) | +| GLM-5.1 | Ascend A3 Series Products | 48 | PD Disaggregation | 128k+1k (90% prefix cache hit rate) | 50ms | - | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-2p1d-48p-in128k-out1k-prefix90-50ms) | +| GLM-5.1 | Ascend A3 Series Products | 48 | PD Disaggregation | 64k+1k (90% prefix cache hit rate) | 50ms | - | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-4p1d-48p-in64k-out1k-prefix90-50ms) | ## Optimal Configuration @@ -36,7 +36,7 @@ On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend **Model**: GLM-5.1 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 16 @@ -155,7 +155,7 @@ python -m sglang.bench_serving \ **Model**: GLM-5.1 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 32 @@ -363,7 +363,7 @@ python -m sglang.bench_serving \ **Model**: GLM-5.1 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 32 @@ -565,7 +565,7 @@ python -m sglang.bench_serving \ **Model**: GLM-5.1 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 32 @@ -774,7 +774,7 @@ python -m sglang.bench_serving \ **Model**: GLM-5.1 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 32 @@ -973,7 +973,7 @@ python -m sglang.bench_serving \ **Model**: GLM-5.1 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 32 @@ -1180,7 +1180,7 @@ python -m sglang.bench_serving \ **Model**: GLM-5.1 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 48 @@ -1390,7 +1390,7 @@ python -m sglang.bench_serving \ **Model**: GLM-5.1 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 48 diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_2.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_2.mdx index deacbeeb9..2e631442a 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_2.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_2.mdx @@ -7,14 +7,14 @@ metatags: This page focuses on optimal configuration and benchmark results for GLM-5.2 on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [GLM-5.2 Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_2). -On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. +On the A3 Series, each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. ### High Throughput | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| GLM-5.2 | Atlas 800I A3 | 32 | PD Disaggregation | 16k+1k | 50ms | W4A8 INT8 | [Optimal Configuration](#glm-5-2-w4a8-3p1d-32p-in16k-out1k-50ms) | +| GLM-5.2 | Ascend A3 Series Products | 32 | PD Disaggregation | 16k+1k | 50ms | W4A8 INT8 | [Optimal Configuration](#glm-5-2-w4a8-3p1d-32p-in16k-out1k-50ms) | ## Optimal Configuration @@ -24,7 +24,7 @@ On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend **Model**: GLM-5.2 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 32 diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6.mdx index c19432fcb..4fc9f3bf6 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6.mdx @@ -7,27 +7,27 @@ metatags: This page focuses on optimal configuration and benchmark results for Kimi-K2.6 on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [Kimi-K2.6 Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/kimi_k2_6). -On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. +On the A3 Series, each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. ### Low Latency | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | TTFT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| Kimi-K2.6 | Atlas 800I A3 | 8 | PD Mixed | 3.5k+1.5k | 20ms | - | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-8p-in3k5-out1k5-20ms) | +| Kimi-K2.6 | Ascend A3 Series Products | 8 | PD Mixed | 3.5k+1.5k | 20ms | - | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-8p-in3k5-out1k5-20ms) | ### High Throughput | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | TTFT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| Kimi-K2.6 | Atlas 800I A3 | 16 | PD Mixed | 64k+1k | 100ms | - | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-16p-in64k-out1k-100ms) | -| Kimi-K2.6 | Atlas 800I A3 | 16 | PD Disaggregation | 128k+1k | 100ms | - | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-1p1d-16p-in128k-out1k-100ms) | -| Kimi-K2.6 | Atlas 800I A3 | 16 | PD Disaggregation | 128k+1k (90% prefix cache hit rate) | 100ms | 5s | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-1p1d-16p-in128k-out1k-prefix90-100ms) | -| Kimi-K2.6 | Atlas 800I A3 | 16 | PD Disaggregation | 64k+1.5k | 100ms | - | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-1p1d-16p-in64k-out1k5-100ms) | -| Kimi-K2.6 | Atlas 800I A3 | 16 | PD Disaggregation | 64k+1.5k (90% prefix cache hit rate) | 100ms | 3s | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-1p1d-16p-in64k-out1k5-prefix90-100ms) | -| Kimi-K2.6 | Atlas 800I A3 | 8 | PD Mixed | 1024x1024 (30)+1024 | 50ms | - | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-8p-in1024x1024-30-out1024-50ms) | -| Kimi-K2.6 | Atlas 800I A3 | 8 | PD Mixed | 1080p_30+256 | 50ms | - | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-8p-in1080p-30-out256-50ms) | -| Kimi-K2.6 | Atlas 800I A3 | 8 | PD Mixed | 3.5k+1.5k | 50ms | - | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-8p-in3k5-out1k5-50ms) | +| Kimi-K2.6 | Ascend A3 Series Products | 16 | PD Mixed | 64k+1k | 100ms | - | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-16p-in64k-out1k-100ms) | +| Kimi-K2.6 | Ascend A3 Series Products | 16 | PD Disaggregation | 128k+1k | 100ms | - | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-1p1d-16p-in128k-out1k-100ms) | +| Kimi-K2.6 | Ascend A3 Series Products | 16 | PD Disaggregation | 128k+1k (90% prefix cache hit rate) | 100ms | 5s | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-1p1d-16p-in128k-out1k-prefix90-100ms) | +| Kimi-K2.6 | Ascend A3 Series Products | 16 | PD Disaggregation | 64k+1.5k | 100ms | - | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-1p1d-16p-in64k-out1k5-100ms) | +| Kimi-K2.6 | Ascend A3 Series Products | 16 | PD Disaggregation | 64k+1.5k (90% prefix cache hit rate) | 100ms | 3s | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-1p1d-16p-in64k-out1k5-prefix90-100ms) | +| Kimi-K2.6 | Ascend A3 Series Products | 8 | PD Mixed | 1024x1024 (30)+1024 | 50ms | - | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-8p-in1024x1024-30-out1024-50ms) | +| Kimi-K2.6 | Ascend A3 Series Products | 8 | PD Mixed | 1080p_30+256 | 50ms | - | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-8p-in1080p-30-out256-50ms) | +| Kimi-K2.6 | Ascend A3 Series Products | 8 | PD Mixed | 3.5k+1.5k | 50ms | - | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-8p-in3k5-out1k5-50ms) | ## Optimal Configuration @@ -37,7 +37,7 @@ On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend **Model**: Kimi-K2.6 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 16 @@ -162,7 +162,7 @@ python -m sglang.bench_serving \ **Model**: Kimi-K2.6 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 16 @@ -358,7 +358,7 @@ python -m sglang.bench_serving \ **Model**: Kimi-K2.6 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 16 @@ -561,7 +561,7 @@ python -m sglang.bench_serving \ **Model**: Kimi-K2.6 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 16 @@ -755,7 +755,7 @@ python -m sglang.bench_serving \ **Model**: Kimi-K2.6 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 16 @@ -955,7 +955,7 @@ python -m sglang.bench_serving \ **Model**: Kimi-K2.6 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 8 @@ -1068,7 +1068,7 @@ python -m sglang.bench_serving \ **Model**: Kimi-K2.6 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 8 @@ -1179,7 +1179,7 @@ python -m sglang.bench_serving \ **Model**: Kimi-K2.6 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 8 @@ -1294,7 +1294,7 @@ python -m sglang.bench_serving \ **Model**: Kimi-K2.6 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 8 diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash.mdx index 1b7bf98d4..326b3aec8 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash.mdx @@ -7,22 +7,22 @@ metatags: This page focuses on optimal configuration and benchmark results for MiMo-V2-Flash on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [MiMo-V2-Flash Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/mimo_v2_flash). -On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. +On the A3 Series, each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. ### Low Latency | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | TTFT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| MiMo-V2-Flash | Atlas 800I A3 | 12 | PD Disaggregation | 16k+1k | 20ms | - | W8A8 INT8 | [Optimal Configuration](#mimo-v2-flash-1p1d-12p-in16k-out1k-tpot-20ms) | -| MiMo-V2-Flash | Atlas 800I A3 | 12 | PD Disaggregation | 32k+1k | 20ms | - | W8A8 INT8 | [Optimal Configuration](#mimo-v2-flash-1p1d-12p-in32k-out1k-tpot-20ms) | +| MiMo-V2-Flash | Ascend A3 Series Products | 12 | PD Disaggregation | 16k+1k | 20ms | - | W8A8 INT8 | [Optimal Configuration](#mimo-v2-flash-1p1d-12p-in16k-out1k-tpot-20ms) | +| MiMo-V2-Flash | Ascend A3 Series Products | 12 | PD Disaggregation | 32k+1k | 20ms | - | W8A8 INT8 | [Optimal Configuration](#mimo-v2-flash-1p1d-12p-in32k-out1k-tpot-20ms) | ### High Throughput | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | TTFT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| MiMo-V2-Flash | Atlas 800I A3 | 12 | PD Disaggregation | 16k+1 | - | 5s | W8A8 INT8 | [Optimal Configuration](#mimo-v2-flash-1p1d-12p-in16k-out1-ttft-5s) | -| MiMo-V2-Flash | Atlas 800I A3 | 12 | PD Disaggregation | 32k+1 | - | 5s | W8A8 INT8 | [Optimal Configuration](#mimo-v2-flash-1p1d-12p-in32k-out1-ttft-5s) | +| MiMo-V2-Flash | Ascend A3 Series Products | 12 | PD Disaggregation | 16k+1 | - | 5s | W8A8 INT8 | [Optimal Configuration](#mimo-v2-flash-1p1d-12p-in16k-out1-ttft-5s) | +| MiMo-V2-Flash | Ascend A3 Series Products | 12 | PD Disaggregation | 32k+1 | - | 5s | W8A8 INT8 | [Optimal Configuration](#mimo-v2-flash-1p1d-12p-in32k-out1-ttft-5s) | ## Optimal Configuration @@ -32,7 +32,7 @@ On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend **Model**: MiMo-V2-Flash -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 12 @@ -217,7 +217,7 @@ python -m sglang.bench_serving \ **Model**: MiMo-V2-Flash -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 12 @@ -402,7 +402,7 @@ python -m sglang.bench_serving \ **Model**: MiMo-V2-Flash -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 12 @@ -587,7 +587,7 @@ python -m sglang.bench_serving \ **Model**: MiMo-V2-Flash -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 12 diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5.mdx index 83415f179..8c5b96446 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5.mdx @@ -7,23 +7,23 @@ metatags: This page focuses on optimal configuration and benchmark results for MiniMax-M2.5 on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [MiniMax-M2.5 Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/minimax_m2_5). -On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. +On the A3 Series, each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. ### Low Latency | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| MiniMax-M2.5 | Atlas 800I A3 | 8 | PD Mixed | 128k+1k (90% prefix cache hit rate) | 24.44ms | W8A8 INT8 | [Optimal Configuration](#minimax-m2-5-w8a8-8p-in128k-out1k-prefix90-24-44ms) | -| MiniMax-M2.5 | Atlas 800I A3 | 8 | PD Mixed | 3.5k+1.5k | 20ms | W8A8 INT8 | [Optimal Configuration](#minimax-m2-5-w8a8-8p-in3k5-out1k5-20ms) | +| MiniMax-M2.5 | Ascend A3 Series Products | 8 | PD Mixed | 128k+1k (90% prefix cache hit rate) | 24.44ms | W8A8 INT8 | [Optimal Configuration](#minimax-m2-5-w8a8-8p-in128k-out1k-prefix90-24-44ms) | +| MiniMax-M2.5 | Ascend A3 Series Products | 8 | PD Mixed | 3.5k+1.5k | 20ms | W8A8 INT8 | [Optimal Configuration](#minimax-m2-5-w8a8-8p-in3k5-out1k5-20ms) | ### High Throughput | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| MiniMax-M2.5 | Atlas 800I A3 | 4 | PD Mixed | 32k+1k | 50ms | W8A8 INT8 | [Optimal Configuration](#minimax-m2-5-w8a8-4p-in32k-out1k-50ms) | -| MiniMax-M2.5 | Atlas 800I A3 | 4 | PD Mixed | 64k+1k (90% prefix cache hit rate) | 50ms | W8A8 INT8 | [Optimal Configuration](#minimax-m2-5-w8a8-4p-in64k-out1k-prefix90-50ms) | -| MiniMax-M2.5 | Atlas 800I A3 | 8 | PD Mixed | 3.5k+1.5k | 50ms | W8A8 INT8 | [Optimal Configuration](#minimax-m2-5-w8a8-8p-in3k5-out1k5-50ms) | +| MiniMax-M2.5 | Ascend A3 Series Products | 4 | PD Mixed | 32k+1k | 50ms | W8A8 INT8 | [Optimal Configuration](#minimax-m2-5-w8a8-4p-in32k-out1k-50ms) | +| MiniMax-M2.5 | Ascend A3 Series Products | 4 | PD Mixed | 64k+1k (90% prefix cache hit rate) | 50ms | W8A8 INT8 | [Optimal Configuration](#minimax-m2-5-w8a8-4p-in64k-out1k-prefix90-50ms) | +| MiniMax-M2.5 | Ascend A3 Series Products | 8 | PD Mixed | 3.5k+1.5k | 50ms | W8A8 INT8 | [Optimal Configuration](#minimax-m2-5-w8a8-8p-in3k5-out1k5-50ms) | ## Optimal Configuration @@ -33,7 +33,7 @@ On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend **Model**: MiniMax-M2.5 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 4 @@ -145,7 +145,7 @@ python -m sglang.bench_serving \ **Model**: MiniMax-M2.5 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 4 @@ -254,7 +254,7 @@ python -m sglang.bench_serving \ **Model**: MiniMax-M2.5 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 8 @@ -366,7 +366,7 @@ python -m sglang.bench_serving \ **Model**: MiniMax-M2.5 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 8 @@ -473,7 +473,7 @@ python -m sglang.bench_serving \ **Model**: MiniMax-M2.5 -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 8 diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_235b_a22b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_235b_a22b.mdx index 846f0107a..0448dfd03 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_235b_a22b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_235b_a22b.mdx @@ -7,20 +7,20 @@ metatags: This page focuses on optimal configuration and benchmark results for Qwen3-235B-A22B on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [Qwen3-235B-A22B Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_235b_a22b). -On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. +On the A3 Series, each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. ### Low Latency | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| Qwen3-235B-A22B | Atlas 800I A3 | 8 | PD Mixed | 11k+1.5k | 8ms | BF16 | [Optimal Configuration](#qwen3-235b-a22b-bf16-8p-in11k-out1k5-8ms) | +| Qwen3-235B-A22B | Ascend A3 Series Products | 8 | PD Mixed | 11k+1.5k | 8ms | BF16 | [Optimal Configuration](#qwen3-235b-a22b-bf16-8p-in11k-out1k5-8ms) | ### High Throughput | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| Qwen3-235B-A22B | Atlas 800I A3 | 8 | PD Mixed | 3.5k+1.5k | 50.1ms | W8A8 INT8 | [Optimal Configuration](#qwen3-235b-a22b-w8a8-8p-in3k5-out1k5-50-1ms) | +| Qwen3-235B-A22B | Ascend A3 Series Products | 8 | PD Mixed | 3.5k+1.5k | 50.1ms | W8A8 INT8 | [Optimal Configuration](#qwen3-235b-a22b-w8a8-8p-in3k5-out1k5-50-1ms) | ## Optimal Configuration @@ -30,7 +30,7 @@ On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend **Model**: Qwen3-235B-A22B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 8 @@ -128,7 +128,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3-235B-A22B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 8 diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_30b_a3b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_30b_a3b.mdx index 8f2d87a30..7aa70cd25 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_30b_a3b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_30b_a3b.mdx @@ -7,22 +7,22 @@ metatags: This page focuses on optimal configuration and benchmark results for Qwen3-30B-A3B on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [Qwen3-30B-A3B Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_30b_a3b). -On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. +On the A3 Series, each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. ### Low Latency | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| Qwen3-30B-A3B | Atlas 800I A3 | 1 | PD Mixed | 3.5k+1.5k | 10ms | W8A8 INT8 | [Optimal Configuration](#qwen3-30b-a3b-w8a8-1p-in3k5-out1k5-10ms) | -| Qwen3-30B-A3B | Atlas 800I A3 | 1 | PD Mixed | 6k+1.5k | 10.25ms | W8A8 INT8 | [Optimal Configuration](#qwen3-30b-a3b-w8a8-1p-in6k-out1k5-bs16) | +| Qwen3-30B-A3B | Ascend A3 Series Products | 1 | PD Mixed | 3.5k+1.5k | 10ms | W8A8 INT8 | [Optimal Configuration](#qwen3-30b-a3b-w8a8-1p-in3k5-out1k5-10ms) | +| Qwen3-30B-A3B | Ascend A3 Series Products | 1 | PD Mixed | 6k+1.5k | 10.25ms | W8A8 INT8 | [Optimal Configuration](#qwen3-30b-a3b-w8a8-1p-in6k-out1k5-bs16) | ### High Throughput | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| Qwen3-30B-A3B | Atlas 800I A3 | 1 | PD Mixed | 1k+100 | 10000ms | BF16 | [Optimal Configuration](#qwen3-30b-a3b-bf16-1p-in1k-out100) | -| Qwen3-30B-A3B | Atlas 800I A3 | 1 | PD Mixed | 3.5k+1.5k | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-30b-a3b-w8a8-1p-in3k5-out1k5-50ms) | +| Qwen3-30B-A3B | Ascend A3 Series Products | 1 | PD Mixed | 1k+100 | 10000ms | BF16 | [Optimal Configuration](#qwen3-30b-a3b-bf16-1p-in1k-out100) | +| Qwen3-30B-A3B | Ascend A3 Series Products | 1 | PD Mixed | 3.5k+1.5k | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-30b-a3b-w8a8-1p-in3k5-out1k5-50ms) | ## Optimal Configuration @@ -32,7 +32,7 @@ On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend **Model**: Qwen3-30B-A3B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 1 @@ -138,7 +138,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3-30B-A3B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 1 @@ -239,7 +239,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3-30B-A3B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 1 @@ -339,7 +339,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3-30B-A3B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 1 diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_32b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_32b.mdx index 9f9be89c0..2a42f7d8e 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_32b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_32b.mdx @@ -7,21 +7,21 @@ metatags: This page focuses on optimal configuration and benchmark results for Qwen3-32B on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [Qwen3-32B Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_32b). -On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. +On the A3 Series, each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. ### Low Latency | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| Qwen3-32B | Atlas 800I A3 | 8 | PD Mixed | 18k+4k | 6ms | BF16 | [Optimal Configuration](#qwen3-32b-bf16-8p-in18k-out4k-6ms) | +| Qwen3-32B | Ascend A3 Series Products | 8 | PD Mixed | 18k+4k | 6ms | BF16 | [Optimal Configuration](#qwen3-32b-bf16-8p-in18k-out4k-6ms) | ### High Throughput | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| Qwen3-32B | Atlas 800I A3 | 2 | PD Mixed | 3.5k+1.5k | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-32b-w8a8-2p-in3k5-out1k5-50ms) | -| Qwen3-32B | Atlas 800I A2 | 2 | PD Mixed | 3.5k+1.5k | 55ms | W8A8 INT8 | [Optimal Configuration](#qwen3-32b-w8a8-2p-in3k5-out1k5-55ms-a2) | +| Qwen3-32B | Ascend A3 Series Products | 2 | PD Mixed | 3.5k+1.5k | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-32b-w8a8-2p-in3k5-out1k5-50ms) | +| Qwen3-32B | Ascend A2 Series Products | 2 | PD Mixed | 3.5k+1.5k | 55ms | W8A8 INT8 | [Optimal Configuration](#qwen3-32b-w8a8-2p-in3k5-out1k5-55ms-a2-series) | ## Optimal Configuration @@ -31,7 +31,7 @@ On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend **Model**: Qwen3-32B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 8 @@ -128,7 +128,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3-32B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 2 @@ -223,11 +223,11 @@ python -m sglang.bench_serving \ --seed 1 ``` -### Qwen3-32B W8A8 2P IN3K5 OUT1K5 55ms A2 +### Qwen3-32B W8A8 2P IN3K5 OUT1K5 55ms A2 Series **Model**: Qwen3-32B -**Hardware**: Atlas 800I A2 +**Hardware**: Ascend A2 Series Products **Cards**: 2 diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_5_397b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_5_397b.mdx index f7736146f..cfd82ab23 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_5_397b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_5_397b.mdx @@ -9,28 +9,28 @@ This page focuses on optimal configuration and benchmark results for Qwen3.5-397 setup, model weight download, feature configuration, and deployment instructions, etc., see the [Qwen3.5-397B-A17B Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_5_397b). -On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. +On the A3 Series, each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. ### Low Latency | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 128k+1k | 20ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in128k-out1k-20ms) | -| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 16k+1k | 20ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in16k-out1k-20ms) | -| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 3.5k+1.5k | 22.2ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in3k5-out1k5-22-2ms) | -| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 64k+1k | 20ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in64k-out1k-20ms) | +| Qwen3.5-397B-A17B | Ascend A3 Series Products | 8 | PD Mixed | 128k+1k | 20ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in128k-out1k-20ms) | +| Qwen3.5-397B-A17B | Ascend A3 Series Products | 8 | PD Mixed | 16k+1k | 20ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in16k-out1k-20ms) | +| Qwen3.5-397B-A17B | Ascend A3 Series Products | 8 | PD Mixed | 3.5k+1.5k | 22.2ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in3k5-out1k5-22-2ms) | +| Qwen3.5-397B-A17B | Ascend A3 Series Products | 8 | PD Mixed | 64k+1k | 20ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in64k-out1k-20ms) | ### High Throughput | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 128k+1k | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in128k-out1k-50ms) | -| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 128k+1k (90% prefix cache hit rate) | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in128k-out1k-prefix90-50ms) | -| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 16k+1k | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in16k-out1k-50ms) | -| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 3.5k+1.5k | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in3k5-out1k5-50ms) | -| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 64k+1k | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in64k-out1k-50ms) | -| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 64k+1k (90% prefix cache hit rate) | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in64k-out1k-prefix90-50ms) | +| Qwen3.5-397B-A17B | Ascend A3 Series Products | 8 | PD Mixed | 128k+1k | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in128k-out1k-50ms) | +| Qwen3.5-397B-A17B | Ascend A3 Series Products | 8 | PD Mixed | 128k+1k (90% prefix cache hit rate) | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in128k-out1k-prefix90-50ms) | +| Qwen3.5-397B-A17B | Ascend A3 Series Products | 8 | PD Mixed | 16k+1k | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in16k-out1k-50ms) | +| Qwen3.5-397B-A17B | Ascend A3 Series Products | 8 | PD Mixed | 3.5k+1.5k | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in3k5-out1k5-50ms) | +| Qwen3.5-397B-A17B | Ascend A3 Series Products | 8 | PD Mixed | 64k+1k | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in64k-out1k-50ms) | +| Qwen3.5-397B-A17B | Ascend A3 Series Products | 8 | PD Mixed | 64k+1k (90% prefix cache hit rate) | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in64k-out1k-prefix90-50ms) | ## Optimal Configuration @@ -40,7 +40,7 @@ On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend **Model**: Qwen3.5-397B-A17B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 8 @@ -154,7 +154,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3.5-397B-A17B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 8 @@ -267,7 +267,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3.5-397B-A17B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 8 @@ -379,7 +379,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3.5-397B-A17B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 8 @@ -497,7 +497,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3.5-397B-A17B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 8 @@ -615,7 +615,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3.5-397B-A17B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 8 @@ -732,7 +732,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3.5-397B-A17B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 8 @@ -849,7 +849,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3.5-397B-A17B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 8 @@ -967,7 +967,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3.5-397B-A17B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 8 @@ -1085,7 +1085,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3.5-397B-A17B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 8 diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_27b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_27b.mdx index a481effa1..289913e11 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_27b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_27b.mdx @@ -7,21 +7,21 @@ metatags: This page focuses on optimal configuration and benchmark results for Qwen3.6-27B on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [Qwen3.6-27B Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_6_27b). -On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. +On the A3 Series, each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. ### High Throughput | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| Qwen3.6-27B | Atlas 800I A3 | 1 | PD Mixed | 1024x1024 (30)+1024 | 50ms | BF16 | [Optimal Configuration](#qwen3-6-27b-1p-in1024x1024-30-out1024-50ms) | -| Qwen3.6-27B | Atlas 800I A3 | 1 | PD Mixed | 1080p_30+256 | 50ms | BF16 | [Optimal Configuration](#qwen3-6-27b-1p-in1080p-30-out256-50ms) | -| Qwen3.6-27B | Atlas 800I A3 | 1 | PD Mixed | 64k+1k (90% prefix cache hit rate) | 50ms | BF16 | [Optimal Configuration](#qwen3-6-27b-1p-in64k-out1k-prefix90-50ms) | -| Qwen3.6-27B | Atlas 800I A3 | 1 | PD Mixed | 3.5k+1.5k | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-1p-in3k5-out1k5-50ms) | -| Qwen3.6-27B | Atlas 800I A3 | 1 | PD Mixed | 64k+1k | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-1p-in64k-out1k-50ms) | -| Qwen3.6-27B | Atlas 800I A3 | 2 | PD Mixed | 128k+1k | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-2p-in128k-out1k-50ms) | -| Qwen3.6-27B | Atlas 800I A3 | 2 | PD Mixed | 16k+1k | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-2p-in16k-out1k-50ms) | -| Qwen3.6-27B | Atlas 800I A3 | 2 | PD Mixed | 64k+1k | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-2p-in64k-out1k-50ms) | +| Qwen3.6-27B | Ascend A3 Series Products | 1 | PD Mixed | 1024x1024 (30)+1024 | 50ms | BF16 | [Optimal Configuration](#qwen3-6-27b-1p-in1024x1024-30-out1024-50ms) | +| Qwen3.6-27B | Ascend A3 Series Products | 1 | PD Mixed | 1080p_30+256 | 50ms | BF16 | [Optimal Configuration](#qwen3-6-27b-1p-in1080p-30-out256-50ms) | +| Qwen3.6-27B | Ascend A3 Series Products | 1 | PD Mixed | 64k+1k (90% prefix cache hit rate) | 50ms | BF16 | [Optimal Configuration](#qwen3-6-27b-1p-in64k-out1k-prefix90-50ms) | +| Qwen3.6-27B | Ascend A3 Series Products | 1 | PD Mixed | 3.5k+1.5k | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-1p-in3k5-out1k5-50ms) | +| Qwen3.6-27B | Ascend A3 Series Products | 1 | PD Mixed | 64k+1k | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-1p-in64k-out1k-50ms) | +| Qwen3.6-27B | Ascend A3 Series Products | 2 | PD Mixed | 128k+1k | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-2p-in128k-out1k-50ms) | +| Qwen3.6-27B | Ascend A3 Series Products | 2 | PD Mixed | 16k+1k | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-2p-in16k-out1k-50ms) | +| Qwen3.6-27B | Ascend A3 Series Products | 2 | PD Mixed | 64k+1k | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-2p-in64k-out1k-50ms) | ## Optimal Configuration @@ -31,7 +31,7 @@ On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend **Model**: Qwen3.6-27B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 1 @@ -134,7 +134,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3.6-27B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 1 @@ -235,7 +235,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3.6-27B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 1 @@ -335,7 +335,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3.6-27B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 1 @@ -433,7 +433,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3.6-27B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 1 @@ -525,7 +525,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3.6-27B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 2 @@ -619,7 +619,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3.6-27B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 2 @@ -717,7 +717,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3.6-27B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 2 diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_35b_a3b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_35b_a3b.mdx index 037ff132c..de3268731 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_35b_a3b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_35b_a3b.mdx @@ -7,27 +7,27 @@ metatags: This page focuses on optimal configuration and benchmark results for Qwen3.6-35B-A3B on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [Qwen3.6-35B-A3B Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_6_35b_a3b). -On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. +On the A3 Series, each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. ### Low Latency | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 254k+1k | 16.1ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in254k-out1k) | +| Qwen3.6-35B-A3B | Ascend A3 Series Products | 1 | PD Mixed | 254k+1k | 16.1ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in254k-out1k) | ### High Throughput | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 1024x1024 (30)+1024 | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in1024x1024-30-out1024-50ms) | -| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 1080p_30+256 | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in1080p-30-out256-50ms) | -| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 128k+1k | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in128k-out1k-50ms) | -| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 128k+1k (90% prefix cache hit rate) | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in128k-out1k-prefix90-50ms) | -| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 3.5k+1.5k | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in3k5-out1k5-50ms) | -| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 64k+1k | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in64k-out1k-50ms) | -| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 64k+1k (90% prefix cache hit rate) | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in64k-out1k-prefix90-50ms) | -| Qwen3.6-35B-A3B | Atlas 800I A3 | 2 | PD Mixed | 984k+1k | 40.91ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-2p-in984k-out1k) | +| Qwen3.6-35B-A3B | Ascend A3 Series Products | 1 | PD Mixed | 1024x1024 (30)+1024 | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in1024x1024-30-out1024-50ms) | +| Qwen3.6-35B-A3B | Ascend A3 Series Products | 1 | PD Mixed | 1080p_30+256 | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in1080p-30-out256-50ms) | +| Qwen3.6-35B-A3B | Ascend A3 Series Products | 1 | PD Mixed | 128k+1k | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in128k-out1k-50ms) | +| Qwen3.6-35B-A3B | Ascend A3 Series Products | 1 | PD Mixed | 128k+1k (90% prefix cache hit rate) | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in128k-out1k-prefix90-50ms) | +| Qwen3.6-35B-A3B | Ascend A3 Series Products | 1 | PD Mixed | 3.5k+1.5k | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in3k5-out1k5-50ms) | +| Qwen3.6-35B-A3B | Ascend A3 Series Products | 1 | PD Mixed | 64k+1k | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in64k-out1k-50ms) | +| Qwen3.6-35B-A3B | Ascend A3 Series Products | 1 | PD Mixed | 64k+1k (90% prefix cache hit rate) | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in64k-out1k-prefix90-50ms) | +| Qwen3.6-35B-A3B | Ascend A3 Series Products | 2 | PD Mixed | 984k+1k | 40.91ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-2p-in984k-out1k) | ## Optimal Configuration @@ -37,7 +37,7 @@ On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend **Model**: Qwen3.6-35B-A3B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 1 @@ -142,7 +142,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3.6-35B-A3B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 1 @@ -244,7 +244,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3.6-35B-A3B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 1 @@ -344,7 +344,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3.6-35B-A3B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 1 @@ -446,7 +446,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3.6-35B-A3B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 1 @@ -543,7 +543,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3.6-35B-A3B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 1 @@ -642,7 +642,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3.6-35B-A3B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 1 @@ -741,7 +741,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3.6-35B-A3B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 1 @@ -844,7 +844,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3.6-35B-A3B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 2 diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_8b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_8b.mdx index 2ae6bf832..2b125e621 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_8b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_8b.mdx @@ -7,21 +7,21 @@ metatags: This page focuses on optimal configuration and benchmark results for Qwen3-8B on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [Qwen3-8B Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_8b). -On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. +On the A3 Series, each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. ### Low Latency | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| Qwen3-8B | Atlas 800I A3 | 1 | PD Mixed | 3.5k+1.5k | 5ms | W8A8 INT8 | [Optimal Configuration](#qwen3-8b-w8a8-1p-in3k5-out1k5-5ms) | -| Qwen3-8B | Atlas 800I A3 | 1 | PD Mixed | 6k+1.5k | 11.79ms | W8A8 INT8 | [Optimal Configuration](#qwen3-8b-w8a8-1p-in6k-out1k5-bs16) | +| Qwen3-8B | Ascend A3 Series Products | 1 | PD Mixed | 3.5k+1.5k | 5ms | W8A8 INT8 | [Optimal Configuration](#qwen3-8b-w8a8-1p-in3k5-out1k5-5ms) | +| Qwen3-8B | Ascend A3 Series Products | 1 | PD Mixed | 6k+1.5k | 11.79ms | W8A8 INT8 | [Optimal Configuration](#qwen3-8b-w8a8-1p-in6k-out1k5-bs16) | ### High Throughput | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| Qwen3-8B | Atlas 800I A3 | 1 | PD Mixed | 3.5k+1.5k | 37ms | W8A8 INT8 | [Optimal Configuration](#qwen3-8b-w8a8-1p-in3k5-out1k5-37ms) | +| Qwen3-8B | Ascend A3 Series Products | 1 | PD Mixed | 3.5k+1.5k | 37ms | W8A8 INT8 | [Optimal Configuration](#qwen3-8b-w8a8-1p-in3k5-out1k5-37ms) | ## Optimal Configuration @@ -31,7 +31,7 @@ On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend **Model**: Qwen3-8B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 1 @@ -129,7 +129,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3-8B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 1 @@ -227,7 +227,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3-8B -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 1 diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_next_80b_a3b_instruct.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_next_80b_a3b_instruct.mdx index d147cc65e..bcf516b92 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_next_80b_a3b_instruct.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_next_80b_a3b_instruct.mdx @@ -7,21 +7,21 @@ metatags: This page focuses on optimal configuration and benchmark results for Qwen3-Next-80B-A3B-Instruct on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [Qwen3-Next-80B-A3B-Instruct Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_next_80b_a3b_instruct). -On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. +On the A3 Series, each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. ### Low Latency | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| Qwen3-Next-80B-A3B-Instruct | Atlas 800I A3 | 1 | PD Mixed | 3.5k+1.5k | 20ms | W8A8 INT8 | [Optimal Configuration](#qwen3-next-80b-a3b-instruct-w8a8-1p-in3k5-out1k5-20ms) | -| Qwen3-Next-80B-A3B-Instruct | Atlas 800I A3 | 2 | PD Mixed | 6k+1.5k | 15.62ms | W8A8 INT8 | [Optimal Configuration](#qwen3-next-80b-a3b-instruct-w8a8-2p-in6k-out1k5-bs16) | +| Qwen3-Next-80B-A3B-Instruct | Ascend A3 Series Products | 1 | PD Mixed | 3.5k+1.5k | 20ms | W8A8 INT8 | [Optimal Configuration](#qwen3-next-80b-a3b-instruct-w8a8-1p-in3k5-out1k5-20ms) | +| Qwen3-Next-80B-A3B-Instruct | Ascend A3 Series Products | 2 | PD Mixed | 6k+1.5k | 15.62ms | W8A8 INT8 | [Optimal Configuration](#qwen3-next-80b-a3b-instruct-w8a8-2p-in6k-out1k5-bs16) | ### High Throughput | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| Qwen3-Next-80B-A3B-Instruct | Atlas 800I A3 | 2 | PD Mixed | 3.5k+1.5k | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-next-80b-a3b-instruct-w8a8-2p-in3k5-out1k5-50ms) | +| Qwen3-Next-80B-A3B-Instruct | Ascend A3 Series Products | 2 | PD Mixed | 3.5k+1.5k | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-next-80b-a3b-instruct-w8a8-2p-in3k5-out1k5-50ms) | ## Optimal Configuration @@ -31,7 +31,7 @@ On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend **Model**: Qwen3-Next-80B-A3B-Instruct -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 1 @@ -145,7 +145,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3-Next-80B-A3B-Instruct -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 2 @@ -262,7 +262,7 @@ python -m sglang.bench_serving \ **Model**: Qwen3-Next-80B-A3B-Instruct -**Hardware**: Atlas 800I A3 +**Hardware**: Ascend A3 Series Products **Cards**: 2 diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_r1.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_r1.mdx index 1c27bd712..15a893cdc 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_r1.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_r1.mdx @@ -68,17 +68,17 @@ For multi-node deployment, download the weights to a shared directory accessible - [DeepSeek-R1-0528-W8A8](https://modelers.cn/models/Modelers_Park/DeepSeek-R1-0528-w8a8) (Quantized version, 647GB) -The W4A8 variant (376GB) can be deployed on 8 × 64GB of device memory (`--tp-size 8`), which corresponds to one full A2 -node or 8 dies on A3 (4 cards). +The W4A8 variant (376GB) can be deployed on 8 × 64GB of device memory (`--tp-size 8`), which corresponds to one full A2 Series +node or 8 dies on A3 Series (4 cards). The W8A8 variant (647GB) can be deployed on 16 × 64GB of device memory (`--tp-size 16`), which corresponds to one full -A3 node (8 cards, 16 dies) or two A2 nodes. +A3 Series node (8 cards, 16 dies) or two A2 Series nodes. This is the minimum recommended configuration. For optimized configurations, see [Best practices](#best-practices), which may require additional cards or nodes. -For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3), +For the hardware specifications (memory per die, dies per card, and the difference between A2 Series and A3 Series), see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware). ## Installation @@ -94,7 +94,7 @@ Both **stable releases** and **daily builds** are available. The following comma For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build). - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 @@ -129,7 +129,7 @@ quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 ``` - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 @@ -171,14 +171,14 @@ quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized deployment commands and benchmark data, see -[DeepSeek-R1 Best Practice — W4A8 8P PD Mixed On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1#single-node-pd-mixed). +[DeepSeek-R1 Best Practice — W4A8 8P PD Mixed On A3 Series](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1#single-node-pd-mixed). ### Multi-node PD disaggregation deployment PD disaggregation splits the prefill and decode stages onto separate nodes, reducing interference and improving throughput for high-concurrency scenarios. This scenario is already covered in the best practice. For the complete, optimized deployment commands and benchmark data, see -[DeepSeek-R1 Best Practice — W8A8 32P PD Disaggregation On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1#pd-disaggregation). +[DeepSeek-R1 Best Practice — W8A8 32P PD Disaggregation On A3 Series](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1#pd-disaggregation). ## Functional verification diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v3_2.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v3_2.mdx index 544e54885..c3a0592fb 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v3_2.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v3_2.mdx @@ -67,13 +67,13 @@ For multi-node deployment, download the weights to a shared directory accessible The W8A8 variant (694.47GB) can be deployed on 16 × 64GB of device memory (`--tp-size 16`), which corresponds to one full -A3 node (8 cards, 16 dies) or two A2 nodes. +A3 Series node (8 cards, 16 dies) or two A2 Series nodes. This is the minimum recommended configuration. For optimized configurations, see [Best practices](#best-practices), which may require additional cards or nodes. -For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3), +For the hardware specifications (memory per die, dies per card, and the difference between A2 Series and A3 Series), see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware). ## Installation @@ -89,7 +89,7 @@ Both **stable releases** and **daily builds** are available. The following comma For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build). - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 @@ -124,7 +124,7 @@ quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 ``` - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 @@ -166,7 +166,7 @@ quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 PD disaggregation splits the prefill and decode stages onto separate nodes, reducing interference and improving throughput for high-concurrency scenarios. This scenario is already covered in the best practice. For the complete, optimized deployment commands and benchmark data, see -[DeepSeek-V3.2 Best Practice — PD Disaggregation On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v3_2#pd-disaggregation). +[DeepSeek-V3.2 Best Practice — PD Disaggregation On A3 Series](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v3_2#pd-disaggregation). ## Functional verification diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v4_flash.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v4_flash.mdx index 2a334e454..4620ccc91 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v4_flash.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v4_flash.mdx @@ -65,14 +65,14 @@ For multi-node deployment, download the weights to a shared directory accessible - [DeepSeek-V4-Flash-w8a8-mtp](https://www.modelscope.cn/models/Eco-Tech/DeepSeek-V4-Flash-w8a8-mtp) (W8A8 quantized version, 300GB) -The W8A8 variant (300GB) can be deployed on 8 × 64GB of device memory (`--tp-size 8`), which corresponds to one full A2 -node or 8 dies on A3 (4 cards). +The W8A8 variant (300GB) can be deployed on 8 × 64GB of device memory (`--tp-size 8`), which corresponds to one full A2 Series +node or 8 dies on A3 Series (4 cards). This is the minimum recommended configuration. For optimized configurations, see [Best practices](#best-practices), which may require additional cards or nodes. -For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3), +For the hardware specifications (memory per die, dies per card, and the difference between A2 Series and A3 Series), see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware). ## Installation @@ -88,7 +88,7 @@ Both **stable releases** and **daily builds** are available. The following comma For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build). - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 @@ -123,7 +123,7 @@ quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 ``` - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 @@ -163,8 +163,8 @@ quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 ### Single-node PD mixed deployment Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios -with limited hardware resources. In such scenarios, DeepSeek-V4-Flash can be launched with half of an Atlas 800I A3 -node (4 cards, 8 dies) or a full Atlas 800I A2 node (8 cards). The configuration below was validated on Atlas 800I A3 +with limited hardware resources. In such scenarios, DeepSeek-V4-Flash can be launched with half of an Ascend A3 Series +node (4 cards, 8 dies) or a full Ascend A2 Series node (8 cards). The configuration below was validated on an Ascend A3 Series node (4 cards, 8 dies) with `--tp-size 8`. ```bash Command @@ -226,14 +226,14 @@ python3 -m sglang.launch_server \ ``` For the complete benchmark data and additional single-node configurations, see -[DeepSeek-V4-Flash Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash#single-node-pd-mixed). +[DeepSeek-V4-Flash Best Practice — PD Mixed On A3 Series](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash#single-node-pd-mixed). ### Multi-node PD disaggregation deployment PD disaggregation splits the prefill and decode stages onto separate nodes, reducing interference and improving throughput for high-concurrency scenarios. This scenario is already covered in the best practice. For the complete, optimized deployment commands and benchmark data, see -[DeepSeek-V4-Flash Best Practice — PD Disaggregation On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash#pd-disaggregation). +[DeepSeek-V4-Flash Best Practice — PD Disaggregation On A3 Series](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash#pd-disaggregation). ## Functional verification diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_1.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_1.mdx index 29e59b653..84a3121f7 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_1.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_1.mdx @@ -71,13 +71,13 @@ For multi-node deployment, download the weights to a shared directory accessible We recommend deploying the W4A8 variant for reduced resource usage and higher throughput. -It (420.17GB) can be deployed on 8 × 64GB of device memory (`--tp-size 8`), which corresponds to one full A2 node or 8 dies on A3 (4 cards). +It (420.17GB) can be deployed on 8 × 64GB of device memory (`--tp-size 8`), which corresponds to one full A2 Series node or 8 dies on A3 Series (4 cards). This is the minimum recommended configuration. For optimized configurations, see [Best practices](#best-practices), which may require additional cards or nodes. -For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3), +For the hardware specifications (memory per die, dies per card, and the difference between A2 Series and A3 Series), see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware). ## Installation @@ -93,7 +93,7 @@ Both **stable releases** and **daily builds** are available. The following comma For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build). - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 @@ -128,7 +128,7 @@ quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 ``` - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 @@ -167,18 +167,18 @@ quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 ### Multi-node PD mixed deployment -Multi-node deployment distributes the model across multiple Atlas 800I A3 nodes using tensor parallelism while keeping +Multi-node deployment distributes the model across multiple Ascend A3 Series nodes using tensor parallelism while keeping prefill and decode on the same nodes (PD mixed mode), suitable for scenarios that need more device memory than a single node can provide. This scenario is already covered in the best practice. For the complete, optimized deployment commands and benchmark data, see -[GLM-5.1 Best Practice — Multi-node PD Mixed On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1#multi-node-pd-mixed). +[GLM-5.1 Best Practice — Multi-node PD Mixed On A3 Series](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1#multi-node-pd-mixed). ### Multi-node PD disaggregation deployment PD disaggregation splits the prefill and decode stages onto separate nodes, reducing interference and improving throughput for high-concurrency scenarios. This scenario is already covered in the best practice. For the complete, optimized deployment commands and benchmark data, see -[GLM-5.1 Best Practice — PD Disaggregation On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1#pd-disaggregation). +[GLM-5.1 Best Practice — PD Disaggregation On A3 Series](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1#pd-disaggregation). ## Functional verification diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_2.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_2.mdx index e29cd64d5..4a18b869c 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_2.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_2.mdx @@ -63,13 +63,13 @@ For multi-node deployment, download the weights to a shared directory accessible We recommend deploying the W8A8 variant for reduced resource usage and higher throughput. -It (774.08GB) can be deployed on 16 × 64GB of device memory (`--tp-size 16`), which corresponds to one full A3 node (8 cards, 16 dies) or two A2 nodes. +It (774.08GB) can be deployed on 16 × 64GB of device memory (`--tp-size 16`), which corresponds to one full A3 Series node (8 cards, 16 dies) or two A2 Series nodes. This is the minimum recommended configuration. For optimized configurations, see [Best practices](#best-practices), which may require additional cards or nodes. -For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3), +For the hardware specifications (memory per die, dies per card, and the difference between A2 Series and A3 Series), see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware). ## Installation @@ -82,7 +82,7 @@ The dependencies required for the NPU runtime environment have been integrated i online platform. You can directly pull it. - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 @@ -117,7 +117,7 @@ quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 ``` - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 @@ -164,7 +164,7 @@ quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 ### Single-node deployment -Quantized model `GLM-5.2-w8a8` can be deployed on one Atlas 800I A3 node. +Quantized model `GLM-5.2-w8a8` can be deployed on one Ascend A3 Series node. Run the following script to execute online inference. @@ -229,7 +229,7 @@ python3 -m sglang.launch_server \ ### Multi-node deployment -Quantized model `GLM-5.2-w8a8` can be deployed on two Atlas 800I A3 nodes. +Quantized model `GLM-5.2-w8a8` can be deployed on two Ascend A3 Series nodes. Modify the IP addresses of the two nodes, then run the same script on both nodes. diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/hy3.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/hy3.mdx index b7909ed90..c2ac0d561 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/hy3.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/hy3.mdx @@ -13,8 +13,8 @@ parameters, developed by the Tencent Hy Team. It features 192 experts with top-8 outperforms similar-size models and rivals flagship open-source models with 2–5× the parameters, with notable improvements in agentic workflows, long-context tasks, and production reliability. -This document demonstrates the deployment of Hy3 on Ascend NPUs using SGLang, including single-node (Atlas 800I A3) and -multi-node (Atlas 800I A2) PD mixed mode and speculative decoding. +This document demonstrates the deployment of Hy3 on Ascend NPUs using SGLang, including single-node (Ascend A3 Series) and +multi-node (Ascend A2 Series) PD mixed mode and speculative decoding. ## Supported features @@ -60,12 +60,12 @@ Hy3 has 295B total parameters. If you need to download model weights, check the The BF16 variant (597.60GB) can be deployed on 16 × 64GB of device memory (`--tp-size 16`), which corresponds to one full -Atlas 800I A3 node (8 cards × 2 dies) or two Atlas 800I A2 nodes (8 cards × 1 die each). +Ascend A3 Series node (8 cards × 2 dies) or two Ascend A2 Series nodes (8 cards × 1 die each). It is recommended to download the model weights to a shared directory accessible from within the container. -For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3), +For the hardware specifications (memory per die, dies per card, and the difference between A2 Series and A3 Series), see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware). ## Installation @@ -84,7 +84,7 @@ For details, see - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 @@ -119,7 +119,7 @@ quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 ``` - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 @@ -160,7 +160,7 @@ quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 ### Single-node online deployment Single-node deployment completes both prefill and decode within the same node (PD mixed mode). Hy3 with 295B -parameters requires all 16 logical NPUs on an Atlas 800I A3 server (8 physical NPUs × 2 logical cores each, 64 GB HBM +parameters requires all 16 logical NPUs on an Ascend A3 Series server (8 physical NPUs × 2 logical cores each, 64 GB HBM per logical NPU). Set the following environment variables before launching the server: @@ -216,10 +216,10 @@ python3 -m sglang.launch_server \ --speculative-num-draft-tokens 3 ``` -### Multi-node PD mixed deployment on Atlas 800I A2 +### Multi-node PD mixed deployment on Ascend A2 Series -Each Atlas 800I A2 node provides 8 NPUs (Ascend 910B, 64 GB HBM per NPU). Hy3 requires all 16 logical NPUs across -**2 × Atlas 800I A2** nodes in PD mixed mode (prefill and decode are colocated on the same 2-node cluster). +Each Ascend A2 Series node provides 8 NPUs (64 GB HBM per NPU). Hy3 requires all 16 logical NPUs across +**2 × Ascend A2 Series** nodes in PD mixed mode (prefill and decode are colocated on the same 2-node cluster). On each node, set the following environment variables before launching the server: @@ -359,7 +359,7 @@ For performance data and benchmark commands, see [Performance Testing on Ascend ### Best practice configuration reference Hy3 has no standalone best practice page yet. The [Single-node online deployment](#single-node-online-deployment) and -[Multi-node PD mixed deployment on Atlas 800I A2](#multi-node-pd-mixed-deployment-on-atlas-800i-a2) sections above provide +[Multi-node PD mixed deployment on Ascend A2 Series](#multi-node-pd-mixed-deployment-on-ascend-a2-series) sections above provide the ready-to-use scripts for the supported PD mixed cases. For the full catalog of optimization features and their parameter and compatibility details, diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/kimi_k2_6.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/kimi_k2_6.mdx index 0b7837b06..7d8bec441 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/kimi_k2_6.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/kimi_k2_6.mdx @@ -73,13 +73,13 @@ For multi-node deployment, download the weights to a shared directory accessible We recommend deploying the W4A8 variant for reduced resource usage and higher throughput. -It (535.91GB) can be deployed on 16 × 64GB of device memory (`--tp-size 16`), which corresponds to one full A3 node (8 cards, 16 dies) or two A2 nodes. +It (535.91GB) can be deployed on 16 × 64GB of device memory (`--tp-size 16`), which corresponds to one full A3 Series node (8 cards, 16 dies) or two A2 Series nodes. This is the minimum recommended configuration. For optimized configurations, see [Best practices](#best-practices), which may require additional cards or nodes. -For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3), +For the hardware specifications (memory per die, dies per card, and the difference between A2 Series and A3 Series), see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware). ## Installation @@ -95,7 +95,7 @@ Both **stable releases** and **daily builds** are available. The following comma For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build). - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 @@ -130,7 +130,7 @@ quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 ``` - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 @@ -172,22 +172,22 @@ quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized deployment commands and benchmark data, see -[Kimi K2.6 Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6#single-node-pd-mixed). +[Kimi K2.6 Best Practice — PD Mixed On A3 Series](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6#single-node-pd-mixed). ### Multi-node online deployment -Multi-node deployment distributes the model across multiple Atlas 800I A3 nodes using tensor parallelism while keeping +Multi-node deployment distributes the model across multiple Ascend A3 Series nodes using tensor parallelism while keeping prefill and decode on the same nodes (PD mixed mode), suitable for scenarios that need more device memory than a single node can provide. This scenario is already covered in the best practice. For the complete, optimized deployment commands and benchmark data, see -[Kimi-K2.6 Best Practice — Multi-node On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6#multi-node-pd-mixed). +[Kimi-K2.6 Best Practice — Multi-node On A3 Series](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6#multi-node-pd-mixed). ### Multi-node PD disaggregation deployment PD disaggregation splits the prefill and decode stages onto separate nodes, reducing interference and improving throughput for high-concurrency scenarios. This scenario is already covered in the best practice. For the complete, optimized deployment commands and benchmark data, see -[Kimi-K2.6 Best Practice — PD Disaggregation On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6#pd-disaggregation). +[Kimi-K2.6 Best Practice — PD Disaggregation On A3 Series](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6#pd-disaggregation). ## Functional verification diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/mimo_v2_flash.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/mimo_v2_flash.mdx index f6ff9e4c5..a497850d5 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/mimo_v2_flash.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/mimo_v2_flash.mdx @@ -64,14 +64,14 @@ For multi-node deployment, download the weights to a shared directory accessible - [MiMo-V2-Flash-W8A8](https://www.modelscope.cn/models/iridiumine/MiMo-V2-Flash-W8A8) (Quantized version, 311.5GB) -The W8A8 variant (311.5GB) can be deployed on 8 × 64GB of device memory (`--tp-size 8`), which corresponds to one full A2 -node or 8 dies on A3 (4 cards). +The W8A8 variant (311.5GB) can be deployed on 8 × 64GB of device memory (`--tp-size 8`), which corresponds to one full A2 Series +node or 8 dies on A3 Series (4 cards). This is the minimum recommended configuration. For optimized configurations, see [Best practices](#best-practices), which may require additional cards or nodes. -For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3), +For the hardware specifications (memory per die, dies per card, and the difference between A2 Series and A3 Series), see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware). ## Installation @@ -87,7 +87,7 @@ Both **stable releases** and **daily builds** are available. The following comma For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build). - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 @@ -122,7 +122,7 @@ quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 ``` - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 @@ -164,7 +164,7 @@ quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 PD disaggregation splits the prefill and decode stages onto separate nodes, reducing interference and improving throughput for high-concurrency scenarios. This scenario is already covered in the best practice. For the complete, optimized deployment commands and benchmark data, see -[MiMo-V2-Flash Best Practice — W8A8 24P PD Disaggregation On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash#pd-disaggregation). +[MiMo-V2-Flash Best Practice — W8A8 24P PD Disaggregation On A3 Series](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash#pd-disaggregation). ## Functional verification diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/minimax_m2_5.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/minimax_m2_5.mdx index b8fd627de..01d1f1605 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/minimax_m2_5.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/minimax_m2_5.mdx @@ -64,14 +64,14 @@ Before downloading model weights, check the model size to reserve enough disk sp - [MiniMax-M2.5-eagle-model](https://www.modelscope.cn/models/sgl-npu/MiniMax-M2.5-eagel-model-0318) (EAGLE3 draft model for speculative decoding) -The W8A8 variant (230.82GB) can be deployed on 8 × 64GB of device memory (`--tp-size 8`), which corresponds to one full A2 -node or 8 dies on A3 (4 cards). +The W8A8 variant (230.82GB) can be deployed on 8 × 64GB of device memory (`--tp-size 8`), which corresponds to one full A2 Series +node or 8 dies on A3 Series (4 cards). This is the minimum recommended configuration. For optimized configurations, see [Best practices](#best-practices), which may require additional cards or nodes. -For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3), +For the hardware specifications (memory per die, dies per card, and the difference between A2 Series and A3 Series), see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware). ## Installation @@ -87,7 +87,7 @@ Both **stable releases** and **daily builds** are available. The following comma For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build). - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 @@ -122,7 +122,7 @@ quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 ``` - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 @@ -164,7 +164,7 @@ quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized deployment commands and benchmark data, see -[MiniMax-M2.5 Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5#single-node-pd-mixed). +[MiniMax-M2.5 Best Practice — PD Mixed On A3 Series](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5#single-node-pd-mixed). ## Functional verification diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_235b_a22b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_235b_a22b.mdx index b826ab416..a9af4e443 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_235b_a22b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_235b_a22b.mdx @@ -71,15 +71,15 @@ For multi-node deployment, download the weights to a shared directory accessible The BF16 variant (470.21GB) can be deployed on 16 × 64GB of device memory (`--tp-size 16`), which corresponds to one full -A3 node (8 cards, 16 dies) or two A2 nodes. +A3 Series node (8 cards, 16 dies) or two A2 Series nodes. -The W8A8 variant (236.80GB) can be deployed on 8 × 64GB (`--tp-size 8`), which corresponds to one full A2 node or 8 dies on A3 (4 cards). +The W8A8 variant (236.80GB) can be deployed on 8 × 64GB (`--tp-size 8`), which corresponds to one full A2 Series node or 8 dies on A3 Series (4 cards). This is the minimum recommended configuration. For optimized configurations, see [Best practices](#best-practices), which may require additional cards or nodes. -For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3), +For the hardware specifications (memory per die, dies per card, and the difference between A2 Series and A3 Series), see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware). ## Installation @@ -95,7 +95,7 @@ Both **stable releases** and **daily builds** are available. The following comma For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build). - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 @@ -130,7 +130,7 @@ quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 ``` - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 @@ -172,13 +172,13 @@ quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized deployment commands and benchmark data, see -[Qwen3-235B-A22B Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_235b_a22b#single-node-pd-mixed). +[Qwen3-235B-A22B Best Practice — PD Mixed On A3 Series](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_235b_a22b#single-node-pd-mixed). ### Multi-node PD disaggregation deployment -#### 256K long-sequence PD disaggregation on 2 x Atlas 800I A3 (without CP) +#### 256K long-sequence PD disaggregation on 2 x Ascend A3 Series nodes (without CP) -This configuration uses PD disaggregation for **256K long-sequence inference** on 2 x Atlas 800I A3 with context parallel disabled. +This configuration uses PD disaggregation for **256K long-sequence inference** on 2 x Ascend A3 Series nodes with context parallel disabled. The following command is based on the **W8A8** quantized model. 1. Set the shared environment variables on both prefill and decode nodes: @@ -291,7 +291,7 @@ python3 -m sglang_router.launch_router \ --prometheus-port 29010 ``` -#### Prefill Context Parallel (PCP) on 2 x Atlas 800I A3 +#### Prefill Context Parallel (PCP) on 2 x Ascend A3 Series nodes This configuration enables **Prefill Context Parallel** (`--enable-prefill-context-parallel`) to split the context across CP ranks during prefill, reducing per-device memory pressure and improving TTFT for long sequences. diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_30b_a3b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_30b_a3b.mdx index 0c71b5731..fc96c6a67 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_30b_a3b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_30b_a3b.mdx @@ -64,13 +64,13 @@ For multi-node deployment, download the weights to a shared directory accessible - [Qwen3-a3B_eagle3](https://www.modelscope.cn/models/vllm-ascend/Qwen3-a3B_eagle3) (EAGLE3 draft model for speculative decoding) -Both variants fit within a single 64GB die. For single-node deployment, `--tp-size 1` is sufficient on either A2 or A3. +Both variants fit within a single 64GB die. For single-node deployment, `--tp-size 1` is sufficient on either A2 Series or A3 Series. This is the minimum recommended configuration. For optimized configurations, see [Best practices](#best-practices), which may require additional cards or nodes. -For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3), +For the hardware specifications (memory per die, dies per card, and the difference between A2 Series and A3 Series), see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware). ## Installation @@ -86,7 +86,7 @@ Both **stable releases** and **daily builds** are available. The following comma For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build). - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 @@ -121,7 +121,7 @@ quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 ``` - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 @@ -163,7 +163,7 @@ quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized deployment commands and benchmark data, see -[Qwen3-30B-A3B Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_30b_a3b#single-node-pd-mixed). +[Qwen3-30B-A3B Best Practice — PD Mixed On A3 Series](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_30b_a3b#single-node-pd-mixed). ## Functional verification diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_32b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_32b.mdx index 543e00d53..77b6a95fe 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_32b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_32b.mdx @@ -62,7 +62,7 @@ For multi-node deployment, download the weights to a shared directory accessible - [Eagle3-Qwen3-32B-zh](https://www.modelscope.cn/models/Zjcxy-SmartAI/Eagle3-Qwen3-32B-zh) (EAGLE3 draft model for speculative decoding) -The BF16 variant (65.54GB) can be deployed on 2 × 64GB of device memory (`--tp-size 2`), which corresponds to 2 cards on A2 or 1 card (2 dies) on A3. +The BF16 variant (65.54GB) can be deployed on 2 × 64GB of device memory (`--tp-size 2`), which corresponds to 2 cards on A2 Series or 1 card (2 dies) on A3 Series. The W8A8 variant (42.77GB) fits within a single die. @@ -70,7 +70,7 @@ The W8A8 variant (42.77GB) fits within a single die. This is the minimum recommended configuration. For optimized configurations, see [Best practices](#best-practices), which may require additional cards or nodes. -For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3), +For the hardware specifications (memory per die, dies per card, and the difference between A2 Series and A3 Series), see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware). ## Installation @@ -86,7 +86,7 @@ Both **stable releases** and **daily builds** are available. The following comma For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build). - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 @@ -121,7 +121,7 @@ quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 ``` - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 @@ -163,7 +163,7 @@ quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized deployment commands and benchmark data, see -[Qwen3-32B Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_32b#single-node-pd-mixed). +[Qwen3-32B Best Practice — PD Mixed On A3 Series](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_32b#single-node-pd-mixed). ## Functional verification diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_5_397b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_5_397b.mdx index cdd8cbfef..21fd78ddd 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_5_397b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_5_397b.mdx @@ -65,14 +65,14 @@ For multi-node deployment, download the weights to a shared directory accessible - [Eco-Tech/Qwen3.5-397B-A17B-w4a8-mtp](https://www.modelscope.cn/models/Eco-Tech/Qwen3.5-397B-A17B-w4a8-mtp) (W4A8 quantized version with MTP, 235.88GB) -The W4A8 variant (235.88GB) can be deployed on 8 × 64GB of device memory (`--tp-size 8`), which corresponds to one full A2 -node or 8 dies on A3 (4 cards). +The W4A8 variant (235.88GB) can be deployed on 8 × 64GB of device memory (`--tp-size 8`), which corresponds to one full A2 Series +node or 8 dies on A3 Series (4 cards). This is the minimum recommended configuration. For optimized configurations, see [Best practices](#best-practices), which may require additional cards or nodes. -For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3), +For the hardware specifications (memory per die, dies per card, and the difference between A2 Series and A3 Series), see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware). ## Installation @@ -88,7 +88,7 @@ Both **stable releases** and **daily builds** are available. The following comma For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build). - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 @@ -123,7 +123,7 @@ quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 ``` - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 @@ -165,7 +165,7 @@ quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized deployment commands and benchmark data, see -[Qwen3.5-397B Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_5_397b#single-node-pd-mixed). +[Qwen3.5-397B Best Practice — PD Mixed On A3 Series](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_5_397b#single-node-pd-mixed). ## Functional verification diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_6_27b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_6_27b.mdx index ad1d6c29c..96d914aa9 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_6_27b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_6_27b.mdx @@ -62,13 +62,13 @@ For multi-node deployment, download the weights to a shared directory accessible - [Eco-Tech/Qwen3.6-27B-w8a8](https://www.modelscope.cn/models/Eco-Tech/Qwen3.6-27B-w8a8) (W8A8 quantized version, 36.45GB) -The W8A8 variant (36.45GB) fits within a single 64GB die. For single-node deployment, `--tp-size 1` is sufficient on either A2 or A3. +The W8A8 variant (36.45GB) fits within a single 64GB die. For single-node deployment, `--tp-size 1` is sufficient on either A2 Series or A3 Series. This is the minimum recommended configuration. For optimized configurations, see [Best practices](#best-practices), which may require additional cards or nodes. -For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3), +For the hardware specifications (memory per die, dies per card, and the difference between A2 Series and A3 Series), see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware). ## Installation @@ -84,7 +84,7 @@ Both **stable releases** and **daily builds** are available. The following comma For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build). - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 @@ -119,7 +119,7 @@ quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 ``` - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 @@ -161,7 +161,7 @@ quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized deployment commands and benchmark data, see -[Qwen3.6-27B Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_27b#single-node-pd-mixed). +[Qwen3.6-27B Best Practice — PD Mixed On A3 Series](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_27b#single-node-pd-mixed). ## Functional verification diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_6_35b_a3b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_6_35b_a3b.mdx index ef794bbf0..70370e652 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_6_35b_a3b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_6_35b_a3b.mdx @@ -62,13 +62,13 @@ For multi-node deployment, download the weights to a shared directory accessible - [Eco-Tech/Qwen3.6-35B-A3B-w8a8](https://www.modelscope.cn/models/Eco-Tech/Qwen3.6-35B-A3B-w8a8) (W8A8 quantized version, 39.81GB) -The W8A8 variant (39.81GB) fits within a single 64GB die. For single-node deployment, `--tp-size 1` is sufficient on either A2 or A3. +The W8A8 variant (39.81GB) fits within a single 64GB die. For single-node deployment, `--tp-size 1` is sufficient on either A2 Series or A3 Series. This is the minimum recommended configuration. For optimized configurations, see [Best practices](#best-practices), which may require additional cards or nodes. -For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3), +For the hardware specifications (memory per die, dies per card, and the difference between A2 Series and A3 Series), see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware). ## Installation @@ -84,7 +84,7 @@ Both **stable releases** and **daily builds** are available. The following comma For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build). - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 @@ -119,7 +119,7 @@ quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 ``` - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 @@ -161,7 +161,7 @@ quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized deployment commands and benchmark data, see -[Qwen3.6-35B-A3B Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_35b_a3b#single-node-pd-mixed). +[Qwen3.6-35B-A3B Best Practice — PD Mixed On A3 Series](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_35b_a3b#single-node-pd-mixed). ## Functional verification diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_8_max.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_8_max.mdx index 89728e74f..e64e5e6cd 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_8_max.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_8_max.mdx @@ -66,7 +66,7 @@ For multi-node deployment, download the weights to a shared directory accessible We recommend deploying the W4A8 variant for reduced resource usage and higher throughput. -The validated configuration in this tutorial deploys the W4A8 variant on 4 Atlas 800I A3 nodes (`--tp-size 64`, 64 dies +The validated configuration in this tutorial deploys the W4A8 variant on 4 Ascend A3 Series nodes (`--tp-size 64`, 64 dies in total). The BF16 checkpoint alone weighs approximately 4.8TB and requires additional nodes. @@ -86,7 +86,7 @@ The following command is based on the daily build tag, which contains the latest For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build). - + ```bash Command docker pull quay.io/ascend/sglang:main-cann9.0.0-a3 @@ -133,7 +133,7 @@ quay.io/ascend/sglang:main-cann9.0.0-a3 ### Multi-node online deployment -Multi-node deployment distributes the model across multiple Atlas 800I A3 nodes using tensor parallelism while keeping +Multi-node deployment distributes the model across multiple Ascend A3 Series nodes using tensor parallelism while keeping prefill and decode on the same nodes (PD mixed mode), suitable for scenarios that need more device memory than a single node can provide. The validated configuration deploys the W4A8 quantized checkpoint on 4 nodes with `--tp-size 64` (16 dies per node), DP attention (`--dp-size 4`), and DeepEP in `auto` mode. diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_8b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_8b.mdx index 8b5533162..375ae88cf 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_8b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_8b.mdx @@ -65,13 +65,13 @@ For multi-node deployment, download the weights to a shared directory accessible We recommend deploying the W8A8 variant for reduced resource usage and higher throughput. -It (11.27GB) fits within a single 64GB die, so `--tp-size 1` is sufficient on either A2 or A3. +It (11.27GB) fits within a single 64GB die, so `--tp-size 1` is sufficient on either A2 Series or A3 Series. This is the minimum recommended configuration. For optimized configurations, see [Best practices](#best-practices), which may require additional cards or nodes. -For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3), +For the hardware specifications (memory per die, dies per card, and the difference between A2 Series and A3 Series), see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware). ## Installation @@ -87,7 +87,7 @@ Both **stable releases** and **daily builds** are available. The following comma For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build). - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 @@ -122,7 +122,7 @@ quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 ``` - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 @@ -164,7 +164,7 @@ quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized deployment commands and benchmark data, see -[Qwen3-8B Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_8b#single-node-pd-mixed). +[Qwen3-8B Best Practice — PD Mixed On A3 Series](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_8b#single-node-pd-mixed). ## Functional verification diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_next_80b_a3b_instruct.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_next_80b_a3b_instruct.mdx index 699eece1e..be6c47b99 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_next_80b_a3b_instruct.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_next_80b_a3b_instruct.mdx @@ -66,14 +66,14 @@ For multi-node deployment, download the weights to a shared directory accessible We recommend deploying the W8A8 variant as the main model for reduced resource usage and higher throughput. -It (84.90GB) can be deployed on 2 × 64GB (`--tp-size 2`), which corresponds to 2 cards on A2 or 1 card (2 dies) on A3. +It (84.90GB) can be deployed on 2 × 64GB (`--tp-size 2`), which corresponds to 2 cards on A2 Series or 1 card (2 dies) on A3 Series. The BF16 weights serve as the EAGLE3 draft model (set `--speculative-draft-model-path` to the BF16 weight path). This is the minimum recommended configuration. For optimized configurations, see [Best practices](#best-practices), which may require additional cards or nodes. -For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3), +For the hardware specifications (memory per die, dies per card, and the difference between A2 Series and A3 Series), see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware). ## Installation @@ -89,7 +89,7 @@ Both **stable releases** and **daily builds** are available. The following comma For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build). - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 @@ -124,7 +124,7 @@ quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 ``` - + ```bash Command docker pull quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 @@ -166,7 +166,7 @@ quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized deployment commands and benchmark data, see -[Qwen3-Next-80B-A3B-Instruct Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_next_80b_a3b_instruct#single-node-pd-mixed). +[Qwen3-Next-80B-A3B-Instruct Best Practice — PD Mixed On A3 Series](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_next_80b_a3b_instruct#single-node-pd-mixed). ## Functional verification diff --git a/docs/docs/hardware-platforms/ascend-npus/optimization/quantization.mdx b/docs/docs/hardware-platforms/ascend-npus/optimization/quantization.mdx index 17048ebf9..95f64e3a3 100644 --- a/docs/docs/hardware-platforms/ascend-npus/optimization/quantization.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/optimization/quantization.mdx @@ -13,9 +13,9 @@ SGLang supports **mix-bits** quantization (independently defines and loads each Quantization scheme Layer type - A2 Supported - A3 Supported - Ascend 950 Products Supported + Ascend A2 Series Products Supported + Ascend A3 Series Products Supported + Ascend 950PR/DT Series Products Supported Diffusion models @@ -133,9 +133,9 @@ SGLang supports **mix-bits** quantization (independently defines and loads each Quantization scheme Layer type - A2 Supported - A3 Supported - Ascend 950 Products Supported + Ascend A2 Series Products Supported + Ascend A3 Series Products Supported + Ascend 950PR/DT Series Products Supported @@ -169,9 +169,9 @@ GPTQ on Ascend support Quantization scheme Layer type - A2 Supported - A3 Supported - Ascend 950 Products Supported + Ascend A2 Series Products Supported + Ascend A3 Series Products Supported + Ascend 950PR/DT Series Products Supported @@ -212,9 +212,9 @@ GPTQ on Ascend support Quantization scheme Layer type - A2 Supported - A3 Supported - Ascend 950 Products Supported + Ascend A2 Series Products Supported + Ascend A3 Series Products Supported + Ascend 950PR/DT Series Products Supported @@ -255,9 +255,9 @@ Compressed-tensors (LLM Compressor) on Ascend support: Quantization scheme Layer type - A2 Supported - A3 Supported - Ascend 950 Products Supported + Ascend A2 Series Products Supported + Ascend A3 Series Products Supported + Ascend 950PR/DT Series Products Supported @@ -298,9 +298,9 @@ Compressed-tensors (LLM Compressor) on Ascend support: Quantization type Layer type - A2 Supported - A3 Supported - Ascend 950 Products Supported + Ascend A2 Series Products Supported + Ascend A3 Series Products Supported + Ascend 950PR/DT Series Products Supported @@ -351,7 +351,7 @@ python3 -m sglang.launch_server \ **MXFP8 for LLM dense models (e.g., Qwen3 / Qwen3.5):** -LLM dense W8A8 MXFP8 Linear support on Ascend was added in [PR #22352](https://github.com/sgl-project/sglang/pull/22352). Requires Ascend 950 Products or newer (`npu_dynamic_mx_quant` is not available on A2 / A3). +LLM dense W8A8 MXFP8 Linear support on Ascend was added in [PR #22352](https://github.com/sgl-project/sglang/pull/22352). Requires Ascend 950PR/DT Series or newer (`npu_dynamic_mx_quant` is not available on A2/A3 Series). - Online MXFP8 quantization (BF16/FP16 weights → MXFP8 at load time): @@ -381,7 +381,7 @@ python3 -m sglang.launch_server \ **MXFP8 for LLM MoE models (e.g. Qwen3-30B-A3B / Qwen3.5 MoE):** -LLM MoE W8A8 MXFP8 (FusedMoE) support builds on the dense MXFP8 path. Requires Ascend A5 series or newer — the fused MoE MX kernels (`npu_grouped_matmul_swiglu_quant_v2`, `npu_dynamic_mx_quant`) are A5-only. +LLM MoE W8A8 MXFP8 (FusedMoE) support builds on the dense MXFP8 path. Requires Ascend 950PR/DT Series or newer — the fused MoE MX kernels (`npu_grouped_matmul_swiglu_quant_v2`, `npu_dynamic_mx_quant`) are only available on the 950PR/DT Series. - Online MXFP8 quantization (BF16/FP16 expert weights → MXFP8 at load time): @@ -410,11 +410,11 @@ python3 -m sglang.launch_server \ > - Offline path: `ModelSlimMXFP8MoEScheme` (one instance per weight group) loads `float8_e4m3fn` expert weights + uint8 (e8m0, exponent + 127) block scales. The scale is reshaped `[E, N, K/32] → [E, N, K/64, 2]` (contiguous pairing, matching `npu_dynamic_mx_quant`) then transposed. > - Forward: `AscendTPDispatcher` runs `npu_moe_init_routing_v2(quant_mode=3)`, which fuses the per-token MX activation quant into routing (e4m3 payload + e8m0 block scale, reshaped to the pair-split layout). `AscendRunnerCore` then runs gmm1 `npu_grouped_matmul_swiglu_quant_v2` (cumulative `group_list`; fuses gate/up + swiglu + requant, so no separate activation step) → gmm2 `npu_grouped_matmul` (count `group_list`). The UE8M0 (`float8_e8m0fnu`) scale dtypes are passed explicitly; the e4m3 `x`/`weight` dtypes are left implicit. > - **Router gate**: msmodelslim may also quantize `mlp.gate` (`W8A8_MXFP8`). The gate is a `ReplicatedLinear`, so its quantization must be **description-driven**: for the offline `modelslim` path the gate is passed the quant config and dequantized correctly; the online path keeps it in BF16. Loading a quantized gate as BF16 without its block scale scrambles routing and produces garbage output. -> - Where the activation quant happens depends on the dispatcher. On `ascend_tp` it is fused into routing as described above. DeepEP has no MXFP8 dispatch dtype, so it keeps dispatching BF16 and gmm1 quantizes the hidden states itself via `npu_dynamic_mx_quant` before the fused kernel — the two paths reach the same gmm1 input. Only the `ascend_tp` path has been validated end-to-end on Ascend A5. +> - Where the activation quant happens depends on the dispatcher. On `ascend_tp` it is fused into routing as described above. DeepEP has no MXFP8 dispatch dtype, so it keeps dispatching BF16 and gmm1 quantizes the hidden states itself via `npu_dynamic_mx_quant` before the fused kernel — the two paths reach the same gmm1 input. Only the `ascend_tp` path has been validated end-to-end on the Ascend 950PR/DT Series. **MXFP4 W4A8 for LLM dense models (e.g., Qwen3 / Qwen3.5):** -LLM dense W4A8 (MXFP4 4-bit weights + MXFP8 8-bit activations) Linear support was added in [PR #23650](https://github.com/sgl-project/sglang/pull/23650). Requires Ascend 950 Products or newer. +LLM dense W4A8 (MXFP4 4-bit weights + MXFP8 8-bit activations) Linear support was added in [PR #23650](https://github.com/sgl-project/sglang/pull/23650). Requires Ascend 950PR/DT Series or newer. - Online W4A8 quantization (BF16/FP16 weights → MXFP4 at load time): @@ -436,7 +436,7 @@ python3 -m sglang.launch_server \ **ModelSlim W4A8 MXFP4 for LLM MoE models:** -SGLang auto-detects offline ModelSlim `W4A8_MXFP` MoE checkpoints from `quant_model_description.json`; do not pass `--quantization`. This path requires Ascend A5 or newer. +SGLang auto-detects offline ModelSlim `W4A8_MXFP` MoE checkpoints from `quant_model_description.json`; do not pass `--quantization`. This path requires Ascend 950PR/DT Series or newer. ```bash Command MODEL_PATH=/path/to/w4a8-mxfp4-moe-model @@ -453,7 +453,7 @@ python3 -m sglang.launch_server \ **MXFP4 W4A4 for LLM dense models (e.g. Qwen3 / Qwen3.5):** -LLM dense W4A4 (MXFP4 4-bit weights + 4-bit activations) Linear support was added in [PR #23795](https://github.com/sgl-project/sglang/pull/23795). Requires Ascend A5 series (Ascend 950) or newer — the dual-level online path uses the `DualLevelQuantBatchMatmul` op, which A2/A3 lack. On the Ascend NPU backend `--quantization mxfp4` selects this W4A4 path (on GPU the same flag selects the upstream OCP MXFP4 MoE config instead). +LLM dense W4A4 (MXFP4 4-bit weights + 4-bit activations) Linear support was added in [PR #23795](https://github.com/sgl-project/sglang/pull/23795). Requires Ascend 950PR/DT Series or newer — the dual-level online path uses the `DualLevelQuantBatchMatmul` op, which A2/A3 Series lack. On the Ascend NPU backend `--quantization mxfp4` selects this W4A4 path (on GPU the same flag selects the upstream OCP MXFP4 MoE config instead). - Online W4A4 quantization (BF16/FP16 weights → dual-level MXFP4 at load time): @@ -472,11 +472,11 @@ python3 -m sglang.launch_server \ > - **Online** (`NPUDualLevelMXFP4LinearMethod`) uses **dual-level** MXFP4: both weights and activations are quantized with a fine FP8 (E4M3) L0 block scale plus a coarser L1 scale via `npu_dynamic_dual_level_mx_quant`, and the matmul runs via `npu_dual_level_quant_matmul` (weight in FRACTAL_NZ). Dual-level captures per-block dynamic range far better than a single UE8M0 (power-of-2) scale, which is what made an earlier single-level RTN online path degenerate (greedy decoding could loop without emitting EOS). > - **Offline** (`ModelSlimMXFP4Scheme` → `NPUSingleLevelMXFP4OfflineLinearMethod`) is **single-level**: msmodelslim's `W4A4_MXFP4` checkpoint ships single-level UE8M0 block scales (block_size = 32), so the matmul runs via `npu_quant_matmul(..., x1_dtype=x2_dtype=torch_npu.float4_e2m1fn_x2, group_sizes=[1, 1, 32])`. The online and offline paths therefore use different matmul kernels — they no longer share the matmul path. > - As with W4A8, the packed-FP4 dtype passed to the NPU ops (`dst_type` / `x2_dtype`) must be resolved from `torch_npu.float4_e2m1fn_x2` (an int enum), not the `torch.float4_e2m1fn_x2` dtype object, which recent op-plugin builds reject. -> - Validated end-to-end on Ascend A5 hardware. +> - Validated end-to-end on Ascend 950PR/DT Series hardware. **ModelSlim W4A4 MXFP4 for LLM MoE models:** -SGLang auto-detects offline ModelSlim `W4A4_MXFP4` MoE checkpoints from `quant_model_description.json`; do not pass `--quantization`. This path requires Ascend 950 products or newer. +SGLang auto-detects offline ModelSlim `W4A4_MXFP4` MoE checkpoints from `quant_model_description.json`; do not pass `--quantization`. This path requires Ascend 950PR/DT Series or newer. ```bash Command MODEL_PATH=/path/to/w4a4-mxfp4-moe-model @@ -493,9 +493,9 @@ python3 -m sglang.launch_server \ ## Diffusion Model Quantization on Ascend NPU -SGLang-Diffusion supports MXFP8 online and offline quantization for diffusion models (such as Wan2.2) on Ascend NPUs. MXFP8 requires Ascend 950 Products; the ModelSlim W8A8/W4A4 schemes work on A2/A3. +SGLang-Diffusion supports MXFP8 online and offline quantization for diffusion models (such as Wan2.2) on Ascend NPUs. MXFP8 requires Ascend 950PR/DT Series; the ModelSlim W8A8/W4A4 schemes work on A2/A3 Series. -**Requirements for MXFP8:** CANN ≥ 8.0.RC3, Ascend 950 Products +**Requirements for MXFP8:** CANN ≥ 8.0.RC3, Ascend 950PR/DT Series @@ -504,8 +504,8 @@ SGLang-Diffusion supports MXFP8 online and offline quantization for diffusion mo - - + + diff --git a/docs/docs/hardware-platforms/ascend-npus/reference/glossary.mdx b/docs/docs/hardware-platforms/ascend-npus/reference/glossary.mdx index fc72acc58..c2e9cd746 100644 --- a/docs/docs/hardware-platforms/ascend-npus/reference/glossary.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/reference/glossary.mdx @@ -13,23 +13,23 @@ you will encounter throughout the Ascend NPU documentation. Refer back here when SGLang supports the following Ascend inference hardware: -| Hardware | Chip | Devices | Dies per card | Memory configuration | -| ------------------ | ----------- | ------- | ------------- | ------------------------------------ | -| Atlas 800I A2 (A2) | Ascend 910B | 8 | 1 | 8(cards) × 1(die/card) × 64(GB/die) | -| Atlas 800I A3 (A3) | Ascend 910C | 16 | 2 | 8(cards) × 2(dies/card) × 64(GB/die) | +| Hardware | Devices | Dies per card | Memory configuration | +| ------------------------- | ------- | ------------- | ------------------------------------ | +| Ascend A2 Series Products | 8 | 1 | 8(cards) × 1(die/card) × 64(GB/die) | +| Ascend A3 Series Products | 16 | 2 | 8(cards) × 2(dies/card) × 64(GB/die) | -Throughout these docs, **A2** and **A3** are used as shorthand for the hardware above. -Docker image tags use `910b` for A2 and `a3` for A3. +Throughout these docs, **A2 Series** and **A3 Series** are used as shorthand for the hardware above. +Docker image tags use `910b` for the A2 Series and `a3` for the A3 Series. For example, `cann9.0.0-910b-v0.5.16` and `cann9.0.0-a3-v0.5.16`. -On A3, each card has 2 dies, giving 16 devices vs 8 on A2. -Benchmark pages use "Cards" to refer to physical cards, so `Cards: 4` on A3 means `--tp-size 8`. +On the A3 Series, each card has 2 dies, giving 16 devices vs 8 on the A2 Series. +Benchmark pages use "Cards" to refer to physical cards, so `Cards: 4` on the A3 Series means `--tp-size 8`. -From a deployment perspective, the two key differences between A2 and A3 are: +From a deployment perspective, the two key differences between the A2 Series and the A3 Series are: 1. dies per card — which drives both total memory and `--tp-size` configuration -2. PD disaggregation — A2 requires setting `export ASCEND_MF_TRANSFER_PROTOCOL="device_rdma"`, while A3 uses the default protocol. +2. PD disaggregation — the A2 Series requires setting `export ASCEND_MF_TRANSFER_PROTOCOL="device_rdma"`, while the A3 Series uses the default protocol. ### NPU @@ -37,10 +37,10 @@ From a deployment perspective, the two key differences between A2 and A3 are: **NPU** stands for Neural Processing Unit. Each NPU device is a single `davinci` core. The terms "NPU" and "davinci" are used interchangeably in commands and error logs. -On A2, devices are numbered `/dev/davinci0` through `/dev/davinci7` (8 devices). -On A3, devices are numbered `/dev/davinci0` through `/dev/davinci15` (16 devices). +On the A2 Series, devices are numbered `/dev/davinci0` through `/dev/davinci7` (8 devices). +On the A3 Series, devices are numbered `/dev/davinci0` through `/dev/davinci15` (16 devices). -On either an A2 or A3 server, run `npu-smi info` to view NPU information such as device +On either an A2 Series or an A3 Series server, run `npu-smi info` to view NPU information such as device health, memory usage, and chip status. If the command is not found or reports no devices, the driver is likely not installed. Follow the [Ascend driver installation guide](https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/900/softwareinst/instg/instg_0005.html?OS=openEuler&InstallType=local) @@ -63,7 +63,7 @@ to install it. | **W4A8** | 4-bit weights, 8-bit activations | | **W4A16** | 4-bit weights, 16-bit activations | | **BF16** | Brain Floating Point 16 — 16-bit format optimized for ML workloads | -| **FP8** | 8-bit Floating Point — not supported on A2/A3 | +| **FP8** | 8-bit Floating Point — not supported on the A2/A3 Series | | **INT8** | 8-bit Integer quantization | To apply quantization, use `--quantization modelslim` for W8A8 INT8, or load a pre-quantized diff --git a/docs/docs/hardware-platforms/ascend-npus/reference/support_features.mdx b/docs/docs/hardware-platforms/ascend-npus/reference/support_features.mdx index d54a7f70b..e9e355f3b 100644 --- a/docs/docs/hardware-platforms/ascend-npus/reference/support_features.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/reference/support_features.mdx @@ -32,97 +32,97 @@ click [Server Arguments](../../../advanced_features/server_arguments). - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - +
quant_type in JSON Scheme class ModeA2/A3 SupportedAscend 950 Products SupportedAscend A2/A3 Series Products SupportedAscend 950PR/DT Series Products Supported Trigger
`--model-path`
`--model`		 `None` Type: strA2, A3A2/A3 Series
`--tokenizer-path` `None` Type: strA2, A3A2/A3 Series
`--tokenizer-mode` `auto` `auto`, `slow`A2, A3A2/A3 Series
`--tokenizer-backend` `huggingface` `huggingface`, `fastokens`A2, A3A2/A3 Series
`--tokenizer-worker-num` `1` Type: intA2, A3A2/A3 Series
`--detokenizer-worker-num` `1` Type: intA2, A3A2/A3 Series
`--skip-tokenizer-init` `False` bool flag (set to enable)A2, A3A2/A3 Series
`--load-format` `auto` `auto`, `safetensors`, `gguf`A2, A3A2/A3 Series
`--model-loader-extra-config` `{}` Type: strA2, A3A2/A3 Series
`--trust-remote-code` `False` bool flag (set to enable)A2, A3A2/A3 Series
`--context-length` `None` Type: intA2, A3A2/A3 Series
`--is-embedding` `False` bool flag (set to enable)A2, A3A2/A3 Series
`--enable-multimodal` `None` bool flag (set to enable)A2, A3A2/A3 Series
`--revision` `None` Type: strA2, A3A2/A3 Series
`--model-impl` `auto` `auto`, `sglang`,
`transformers`		A2, A3A2/A3 Series
`--model-config-parser` `auto` Type: strA2, A3A2/A3 Series
@@ -150,37 +150,37 @@ click [Server Arguments](../../../advanced_features/server_arguments). `--host` `127.0.0.1` Type: str - A2, A3 + A2/A3 Series `--port` `30000` Type: int - A2, A3 + A2/A3 Series `--skip-server-warmup` `False` bool flag (set to enable) - A2, A3 + A2/A3 Series `--warmups` `None` Type: str - A2, A3 + A2/A3 Series `--nccl-port` `None` Type: int - A2, A3 + A2/A3 Series `--fastapi-root-path` `None` Type: str - A2, A3 + A2/A3 Series `--grpc-mode` @@ -214,31 +214,31 @@ click [Server Arguments](../../../advanced_features/server_arguments). `--ssl-keyfile` `None` Type: str - A2, A3 + A2/A3 Series `--ssl-certfile` `None` Type: str - A2, A3 + A2/A3 Series `--ssl-keyfile-password` `None` Type: str - A2, A3 + A2/A3 Series `--enable-ssl-refresh` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--enable-http2` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series @@ -267,13 +267,13 @@ click [Server Arguments](../../../advanced_features/server_arguments). `--dtype` `auto` `auto`,
`float16`,
`bfloat16` - A2, A3 + A2/A3 Series `--quantization` `None` `modelslim` - A2, A3 + A2/A3 Series `--quantization-param-path` @@ -285,13 +285,13 @@ click [Server Arguments](../../../advanced_features/server_arguments). `--kv-cache-dtype` `auto` `auto` - A2, A3 + A2/A3 Series `--enable-fp32-lm-head` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--modelopt-quant` @@ -355,169 +355,169 @@ click [Server Arguments](../../../advanced_features/server_arguments). `--mem-fraction-static` `None` Type: float - A2, A3 + A2/A3 Series `--max-running-requests` `None` Type: int - A2, A3 + A2/A3 Series `--prefill-max-requests` `None` Type: int - A2, A3 + A2/A3 Series `--max-queued-requests` `None` Type: int - A2, A3 + A2/A3 Series `--max-total-tokens` `None` Type: int - A2, A3 + A2/A3 Series `--chunked-prefill-size` `None` Type: int - A2, A3 + A2/A3 Series `--max-prefill-tokens` `16384` Type: int - A2, A3 + A2/A3 Series `--schedule-policy` `fcfs` `lpm`,
`fcfs`,
`random` - A2, A3 + A2/A3 Series `--enable-priority-scheduling` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--disable-priority-preemption` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--default-priority-value` `None` Type: int - A2, A3 + A2/A3 Series `--schedule-low-priority-values-first` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--priority-scheduling-preemption-threshold` `10` Type: int - A2, A3 + A2/A3 Series `--retraction-policy` `length` `length`,
`priority` - A2, A3 + A2/A3 Series `--schedule-conservativeness` `1.0` Type: float - A2, A3 + A2/A3 Series `--page-size` `128` Type: int - A2, A3 + A2/A3 Series `--swa-full-tokens-ratio` `0.8` Type: float - A2, A3 + A2/A3 Series `--disable-hybrid-swa-memory` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--radix-eviction-policy` `lru` `lru`,
`lfu` - A2, A3 + A2/A3 Series `--enable-prefill-delayer` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--prefill-delayer-max-delay-passes` `30` Type: int - A2, A3 + A2/A3 Series `--prefill-delayer-token-usage-low-watermark` `None` Type: float - A2, A3 + A2/A3 Series `--prefill-delayer-forward-passes-buckets` `None` List[float] - A2, A3 + A2/A3 Series `--prefill-delayer-wait-seconds-buckets` `None` List[float] - A2, A3 + A2/A3 Series `--prefill-delayer-queue-min-ratio` `None` Type: float - A2, A3 + A2/A3 Series `--prefill-delayer-max-delay-ms` `None` Type: float - A2, A3 + A2/A3 Series `--abort-on-priority-when-disabled` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--enable-dynamic-chunking` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series @@ -545,151 +545,151 @@ click [Server Arguments](../../../advanced_features/server_arguments). `--device` `None` Type: str - A2, A3 + A2/A3 Series `--tensor-parallel-size`
`--tp-size` `1` Type: int - A2, A3 + A2/A3 Series `--pipeline-parallel-size`
`--pp-size` `1` Type: int; Currently `2` not supported; Cannot be used together with TP - A2, A3 + A2/A3 Series `--attention-context-parallel-size`
`--attn-cp-size` `1` Type: int; must be equal to --tp-size - A2, A3 + A2/A3 Series `--moe-data-parallel-size`
`--moe-dp-size` `1` Type: int - A2, A3 + A2/A3 Series `--pp-max-micro-batch-size` `None` Type: int - A2, A3 + A2/A3 Series `--pp-async-batch-depth` `0` Type: int - A2, A3 + A2/A3 Series `--stream-interval` `1` Type: int - A2, A3 + A2/A3 Series `--batch-notify-size` `16` Type: int - A2, A3 + A2/A3 Series `--incremental-streaming-output` `False` bool flag (set to enable) - A2, A3 + A2/A3 Series `--stream-response-default-include-usage` `False` bool flag (set to enable) - A2, A3 + A2/A3 Series `--enable-streaming-session` `False` bool flag (set to enable) - A2, A3 + A2/A3 Series `--enable-session-radix-cache` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--random-seed` `None` Type: int - A2, A3 + A2/A3 Series `--constrained-json-whitespace-pattern` `None` Type: str - A2, A3 + A2/A3 Series `--constrained-json-disable-any-whitespace` `False` bool flag (set to enable) - A2, A3 + A2/A3 Series `--watchdog-timeout` `300` Type: float - A2, A3 + A2/A3 Series `--soft-watchdog-timeout` `300` Type: float - A2, A3 + A2/A3 Series `--dist-timeout` `None` Type: int - A2, A3 + A2/A3 Series `--download-dir` `None` Type: str - A2, A3 + A2/A3 Series `--model-checksum` `None` Type: str - A2, A3 + A2/A3 Series `--base-gpu-id` `0` Type: int - A2, A3 + A2/A3 Series `--gpu-id-step` `1` Type: int - A2, A3 + A2/A3 Series `--sleep-on-idle` `False` bool flag (set to enable) - A2, A3 + A2/A3 Series `--load-snapshot-publish-interval` `15` Type: int - A2, A3 + A2/A3 Series `--use-ray` @@ -701,7 +701,7 @@ click [Server Arguments](../../../advanced_features/server_arguments). `--custom-sigquit-handler` `None` Only for engine - A2, A3 + A2/A3 Series @@ -730,121 +730,121 @@ click [Server Arguments](../../../advanced_features/server_arguments). `--log-level` `info` Type: str - A2, A3 + A2/A3 Series `--log-level-http` `None` Type: str - A2, A3 + A2/A3 Series `--log-requests` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--log-requests-level` `2` `0`, `1`, `2`, `3` - A2, A3 + A2/A3 Series `--log-requests-format` text `text`, `json` - A2, A3 + A2/A3 Series `--crash-dump-folder` `None` Type: str - A2, A3 + A2/A3 Series `--enable-metrics` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--enable-mfu-metrics` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--enable-metrics-for-all-schedulers` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--tokenizer-metrics-custom-labels-header` `x-custom-labels` Type: str - A2, A3 + A2/A3 Series `--tokenizer-metrics-allowed-custom-labels` `None` List[str] - A2, A3 + A2/A3 Series `--extra-metric-labels` `None` Type: JSON/Dict - A2, A3 + A2/A3 Series `--bucket-time-to-first-token` `None` List[float] - A2, A3 + A2/A3 Series `--bucket-inter-token-latency` `None` List[float] - A2, A3 + A2/A3 Series `--bucket-e2e-request-latency` `None` List[float] - A2, A3 + A2/A3 Series `--prompt-tokens-buckets` `None` List[str] - A2, A3 + A2/A3 Series `--generation-tokens-buckets` `None` List[str] - A2, A3 + A2/A3 Series `--gc-warning-threshold-secs` `0.0` Type: float - A2, A3 + A2/A3 Series `--decode-log-interval` `40` Type: int - A2, A3 + A2/A3 Series `--enable-request-time-stats-logging` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--kv-events-config` @@ -856,49 +856,49 @@ click [Server Arguments](../../../advanced_features/server_arguments). `--enable-forward-pass-metrics` `False` bool flag (set to enable) - A2, A3 + A2/A3 Series `--forward-pass-metrics-worker-id` `` Type: str - A2, A3 + A2/A3 Series `--forward-pass-metrics-ipc-name` `None` Type: str - A2, A3 + A2/A3 Series `--enable-trace` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--trace-modules` `request` Type: str - A2, A3 + A2/A3 Series `--otlp-traces-endpoint` `localhost:4317` Type: str - A2, A3 + A2/A3 Series `--log-requests-target` `None` Type: str - A2, A3 + A2/A3 Series `--uvicorn-access-log-exclude-prefixes` `[]` List[str] - A2, A3 + A2/A3 Series @@ -926,13 +926,13 @@ click [Server Arguments](../../../advanced_features/server_arguments). `--export-metrics-to-file` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--export-metrics-to-file-dir` `None` Type: str - A2, A3 + A2/A3 Series @@ -960,43 +960,43 @@ click [Server Arguments](../../../advanced_features/server_arguments). `--api-key` `None` Type: str - A2, A3 + A2/A3 Series `--admin-api-key` `None` Type: str - A2, A3 + A2/A3 Series `--served-model-name` `None` Type: str - A2, A3 + A2/A3 Series `--weight-version` `default` Type: str - A2, A3 + A2/A3 Series `--chat-template` `None` Type: str - A2, A3 + A2/A3 Series `--hf-chat-template-name` `None` Type: str - A2, A3 + A2/A3 Series `--completion-template` `None` Type: str - A2, A3 + A2/A3 Series `--file-storage-path` @@ -1008,55 +1008,55 @@ click [Server Arguments](../../../advanced_features/server_arguments). `--enable-cache-report` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--reasoning-parser` `None` `deepseek-r1`
`deepseek-v3`
`glm45`
`gpt-oss`
`kimi`
`qwen3`
`qwen3-thinking`
`step3` - A2, A3 + A2/A3 Series `--default-chat-template-kwargs` `None` Type: JSON / Dict - A2, A3 + A2/A3 Series `--strip-thinking-cache` `False` bool flag (set to enable) - A2, A3 + A2/A3 Series `--enable-strict-thinking` `False` bool flag (set to enable) - A2, A3 + A2/A3 Series `--tool-call-parser` `None` `llama3`
`pythonic`
`qwen`
`qwen3_coder` - A2, A3 + A2/A3 Series `--sampling-defaults` `model` `openai`, `model` - A2, A3 + A2/A3 Series `--asr-max-buffer-seconds` `60` Type: int - A2, A3 + A2/A3 Series `--asr-max-concurrent-sessions` `32` Type: int - A2, A3 + A2/A3 Series @@ -1084,13 +1084,13 @@ click [Server Arguments](../../../advanced_features/server_arguments). `--data-parallel-size`
`--dp-size` `1` Type: int - A2, A3 + A2/A3 Series `--load-balance-method` `auto` `auto`,
`round_robin`,
`follow_bootstrap_room`,
`total_requests`,
`total_tokens` - A2, A3 + A2/A3 Series @@ -1118,19 +1118,19 @@ click [Server Arguments](../../../advanced_features/server_arguments). `--dist-init-addr`
`--nccl-init-addr` `None` Type: str - A2, A3 + A2/A3 Series `--nnodes` `1` Type: int - A2, A3 + A2/A3 Series `--node-rank` `0` Type: int - A2, A3 + A2/A3 Series @@ -1158,13 +1158,13 @@ click [Server Arguments](../../../advanced_features/server_arguments). `--json-model-override-args` `{}` Type: str - A2, A3 + A2/A3 Series `--preferred-sampling-params` `None` Type: str - A2, A3 + A2/A3 Series @@ -1193,61 +1193,61 @@ click [Server Arguments](../../../advanced_features/server_arguments). `--enable-lora` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--enable-lora-overlap-loading` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--max-lora-rank` `None` Type: int - A2, A3 + A2/A3 Series `--lora-target-modules` `None` `all` - A2, A3 + A2/A3 Series `--lora-paths` `None` Type: List[str] /
JSON objects - A2, A3 + A2/A3 Series `--max-loras-per-batch` `8` Type: int - A2, A3 + A2/A3 Series `--max-loaded-loras` `None` Type: int - A2, A3 + A2/A3 Series `--lora-eviction-policy` `lru` `lru`,
`fifo` - A2, A3 + A2/A3 Series `--lora-backend` `csgmv` `triton`,
`csgmv`,
`ascend`,
`torch_native` - A2, A3 + A2/A3 Series `--experts-shared-outer-loras` `None` Type: bool - A2, A3 + A2/A3 Series `--lora-use-virtual-experts` @@ -1265,7 +1265,7 @@ click [Server Arguments](../../../advanced_features/server_arguments). `--lora-drain-wait-threshold` `0.0` Type: float - A2, A3 + A2/A3 Series `--max-lora-chunk-size` @@ -1300,43 +1300,43 @@ click [Server Arguments](../../../advanced_features/server_arguments). `--attention-backend` `None` `ascend` - A2, A3 + A2/A3 Series `--prefill-attention-backend` `None` `ascend` - A2, A3 + A2/A3 Series `--decode-attention-backend` `None` `ascend` - A2, A3 + A2/A3 Series `--sampling-backend` `None` `pytorch`,
`ascend` - A2, A3 + A2/A3 Series `--grammar-backend` `None` `xgrammar`,
`outlines`,
`llguidance` - A2, A3 + A2/A3 Series `--radix-cache-backend` `None` Type: str - A2, A3 + A2/A3 Series `--mm-attention-backend` `None` `ascend_attn` - A2, A3 + A2/A3 Series `--dsa-prefill-backend` @@ -1399,109 +1399,109 @@ non-default speculative acceptance thresholds or deterministic inference. `--speculative-algorithm` `None` `EAGLE`,
`EAGLE3`,
`NEXTN` - A2, A3 + A2/A3 Series `--speculative-draft-model-path`
`--speculative-draft-model` `None` Type: str - A2, A3 + A2/A3 Series `--speculative-draft-model-revision` `None` Type: str,
`branch name`,
`tag name`,
`commit id` - A2, A3 + A2/A3 Series `--speculative-draft-load-format` `auto` `auto`,
`dummy` - A2, A3 + A2/A3 Series `--speculative-num-steps` `None` Type: int - A2, A3 + A2/A3 Series `--speculative-eagle-topk` `None` `1` (the only supported value on Ascend NPU) - A2, A3 + A2/A3 Series `--speculative-num-draft-tokens` `None` Type: int - A2, A3 + A2/A3 Series `--speculative-dflash-block-size` `None` Type: int - A2, A3 + A2/A3 Series `--speculative-accept-threshold-single` `1.0` Type: float - A2, A3 + A2/A3 Series `--speculative-accept-threshold-acc` `1.0` Type: float - A2, A3 + A2/A3 Series `--speculative-use-rejection-sampling` `False` bool flag
(requires `--speculative-eagle-topk 1`) - A2, A3 + A2/A3 Series `--speculative-token-map` `None` Type: str; requires `--tp-size=1` - A2, A3 + A2/A3 Series `--speculative-attention-mode` `prefill` `prefill`,
`decode` - A2, A3 + A2/A3 Series `--speculative-moe-runner-backend` `None` `auto` - A2, A3 + A2/A3 Series `--speculative-moe-a2a-backend` `None` `ascend_fuseep` (the only supported value on Ascend NPU) - A2, A3 + A2/A3 Series `--speculative-draft-attention-backend` `None` `ascend` - A2, A3 + A2/A3 Series `--speculative-draft-window-size` `None` Type: int - A2, A3 + A2/A3 Series `--speculative-draft-model-quantization` `None` `unquant` (the only supported value for speculative decoding on Ascend NPU) - A2, A3 + A2/A3 Series @@ -1590,19 +1590,19 @@ non-default speculative acceptance thresholds or deterministic inference. `--speculative-adaptive` `False` bool flag (set to enable) - A2, A3 + A2/A3 Series `--speculative-adaptive-config` `None` Type: str - A2, A3 + A2/A3 Series `--speculative-skip-dp-mlp-sync` `False` bool flag (set to enable) - A2, A3 + A2/A3 Series @@ -1631,19 +1631,19 @@ non-default speculative acceptance thresholds or deterministic inference. `--expert-parallel-size`
`--ep-size`
`--ep` `1` Type: int - A2, A3 + A2/A3 Series `--moe-a2a-backend` `none` `none`,
`deepep`,
`ascend_fuseep`(It is incompatible with eplb) - A2, A3 + A2/A3 Series `--moe-runner-backend` `auto` `auto`, `triton` - A2, A3 + A2/A3 Series `--flashinfer-mxfp4-moe-precision` @@ -1661,20 +1661,20 @@ non-default speculative acceptance thresholds or deterministic inference. `--deepep-mode` `auto` `normal`,
`low_latency`,
`auto` - A2, A3 + A2/A3 Series `--fuseep-mode` `2` `1`,
`2` - A2, A3 + A2/A3 Series `--deepep-dispatcher-output-dtype` `auto` `auto`,
`bf16`,
`int8`
(When enabling DeepEP for a quantized model, set the dispatcher output dtype according to your model’s quantization. If the value is int8, you must also set the environment variable:DEEP_NORMAL_MODE_USE_INT8_QUANT=1) - A2, A3 + A2/A3 Series `--deepep-config` @@ -1686,73 +1686,73 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD `--ep-num-redundant-experts` `0` Type: int - A2, A3 + A2/A3 Series `--ep-dispatch-algorithm` `None` `static`,
`dynamic`,
`fake` - A2, A3 + A2/A3 Series `--init-expert-location` `trivial` `trivial`,
``,
``,
`` - A2, A3 + A2/A3 Series `--enable-eplb` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--eplb-algorithm` `deepseek` `auto`,
`deepseek` - A2, A3 + A2/A3 Series `--eplb-rebalance-num-iterations` `1000` Type: int - A2, A3 + A2/A3 Series `--eplb-rebalance-layers-per-chunk` `None` Type: int - A2, A3 + A2/A3 Series `--eplb-min-rebalancing-utilization-threshold` `1.0` Type: float - A2, A3 + A2/A3 Series `--expert-distribution-recorder-mode` `None` `stat`,
`stat_approx`,
`per_pass`,
`per_token` - A2, A3 + A2/A3 Series `--expert-distribution-recorder-buffer-size` `None` Type: int - A2, A3 + A2/A3 Series `--expert-balancedness-report-mode` off off, server_log, prometheus, both - A2, A3 + A2/A3 Series `--moe-dense-tp-size` `None` `1` - A2, A3 + A2/A3 Series `--elastic-ep-backend` @@ -1770,7 +1770,7 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD `--enable-waterfill` `False` bool flag (set to enable) - A2, A3 + A2/A3 Series @@ -1798,31 +1798,31 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD `--max-mamba-cache-size` `None` Type: int - A2, A3 + A2/A3 Series `--mamba-ssm-dtype` `float32` `float32`,
`bfloat16`,
`float16` - A2, A3 + A2/A3 Series `--mamba-full-memory-ratio` `0.9` Type: float - A2, A3 + A2/A3 Series `--mamba-radix-cache-strategy` `auto` `auto`,
`no_buffer`,
`extra_buffer` - A2, A3 + A2/A3 Series `--mamba-track-interval` `256` Type: int - A2, A3 + A2/A3 Series @@ -1851,43 +1851,43 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD `--enable-hierarchical-cache` `False` bool flag
(set to enable).
Currently, mamba cache is not supported. - A2, A3 + A2/A3 Series `--hicache-ratio` `2.0` Type: float - A2, A3 + A2/A3 Series `--hicache-size` `0` Type: int - A2, A3 + A2/A3 Series `--hicache-write-policy` `write_through` `write_back`,
`write_through`,
`write_through_selective` - A2, A3 + A2/A3 Series `--hicache-io-backend` `kernel` `kernel_ascend`,
`direct` - A2, A3 + A2/A3 Series `--hicache-mem-layout` `layer_first` `page_first_direct`,
`page_first_kv_split` - A2, A3 + A2/A3 Series `--hicache-storage-backend` `None` `file` - A2, A3 + A2/A3 Series `--hicache-storage-prefetch-policy` @@ -1962,13 +1962,13 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD `--dllm-algorithm` `None` Type: str - A2, A3 + A2/A3 Series `--dllm-algorithm-config` `None` Type: str - A2, A3 + A2/A3 Series @@ -1996,31 +1996,31 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD `--cpu-offload-gb` `0` Type: int - A2, A3 + A2/A3 Series `--offload-group-size` `-1` Type: int (DeepSeek only) - A2, A3 + A2/A3 Series `--offload-num-in-group` `1` Type: int (DeepSeek only) - A2, A3 + A2/A3 Series `--offload-prefetch-step` `1` Type: int (DeepSeek only) - A2, A3 + A2/A3 Series `--offload-mode` `cpu` `cpu` (DeepSeek only)
`meta` (DeepSeek only)
`sharded_gpu` (DeepSeek only, only support tp=1 dp>1) - A2, A3 + A2/A3 Series @@ -2050,91 +2050,91 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD `--disable-radix-cache` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--cuda-graph-config` `None` Type: JSON - A2, A3 + A2/A3 Series `--cuda-graph-backend-decode` `None` `full`,
`disabled` - A2, A3 + A2/A3 Series `--cuda-graph-backend-prefill` `None` `disabled`, `tc_piecewise`
(`tc_piecewise` currently supports Llama-3.1-8B-Instruct and Qwen2.5-7B-Instruct) - A2, A3 + A2/A3 Series `--cuda-graph-max-bs-decode` `None` Type: int - A2, A3 + A2/A3 Series `--cuda-graph-max-bs-prefill` `None` Type: int - A2, A3 + A2/A3 Series `--cuda-graph-bs-decode` `None` Type: List[int] - A2, A3 + A2/A3 Series `--cuda-graph-bs-prefill` `None` Type: List[int] - A2, A3 + A2/A3 Series `--disable-prefill-cuda-graph` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--disable-decode-cuda-graph` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--cuda-graph-bs-decode` `None` List[int] - A2, A3 + A2/A3 Series `--disable-cuda-graph` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--disable-cuda-graph-padding` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--enable-profile-cuda-graph` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--enable-cudagraph-gc` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--enable-nccl-nvls` @@ -2158,19 +2158,19 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD `--enable-tokenizer-batch-encode` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--disable-tokenizer-batch-decode` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--disable-custom-all-reduce` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--enable-mscclpp` @@ -2182,7 +2182,7 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD `--pre-warm-nccl` `False` bool flag (set to enable) - A2, A3 + A2/A3 Series `--enable-torch-symm-mem` @@ -2194,31 +2194,31 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD `--disable-overlap-schedule` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--enable-mixed-chunk` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--enable-dp-attention` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--enable-dp-attention-local-control-broadcast` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--enable-dp-lm-head` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--enable-two-batch-overlap` @@ -2242,37 +2242,37 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD `--enable-torch-compile` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--enable-torch-compile-debug-mode` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--cuda-graph-bs-prefill` `None` Type: JSON
list - A2, A3 + A2/A3 Series `--cuda-graph-tc-compiler` `eager` `eager` - A2, A3 + A2/A3 Series `--torch-compile-max-bs` `32` Type: int - A2, A3 + A2/A3 Series `--cuda-graph-max-bs-prefill` `None` Type: int - A2, A3 + A2/A3 Series `--enable-p2p-check` @@ -2302,37 +2302,37 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD `--delete-ckpt-after-loading` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--enable-memory-saver` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--enable-weights-cpu-backup` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--enable-draft-weights-cpu-backup` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--allow-auto-truncate` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--enable-custom-logit-processor` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--flashinfer-mla-disable-ragged` @@ -2344,79 +2344,79 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD `--disable-shared-experts-fusion` `True` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--enforce-shared-experts-fusion` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--disable-chunked-prefix-cache` `True` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--image-processor-backend` `auto` `auto`, `torchvision`, `pil` - A2, A3 + A2/A3 Series `--disable-fast-image-processor` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--keep-mm-feature-on-device` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--enable-return-hidden-states` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--enable-return-routed-experts` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--scheduler-recv-interval` `1` Type: int - A2, A3 + A2/A3 Series `--numa-node` `None` List[int] - A2, A3 + A2/A3 Series `--enable-deterministic-inference` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--rl-on-policy-target` `None` `fsdp` - A2, A3 + A2/A3 Series `--disable-attn-tp-gather` `False` bool flag (set to enable) - A2, A3 + A2/A3 Series `--enable-layerwise-nvtx-marker` @@ -2434,13 +2434,13 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD `--enable-prefill-cp` `False` bool flag (set to enable) - A2, A3 + A2/A3 Series `--cp-strategy` `None` `zigzag` - A2, A3 + A2/A3 Series `--enable-fused-qk-norm-rope` @@ -2452,13 +2452,13 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD `--enable-precise-embedding-interpolation` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--gc-threshold` `None` List[int] - A2, A3 + A2/A3 Series @@ -2486,19 +2486,19 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD `--enable-dynamic-batch-tokenizer` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--dynamic-batch-tokenizer-batch-size` `32` Type: int - A2, A3 + A2/A3 Series `--dynamic-batch-tokenizer-batch-timeout` `0.002` Type: float - A2, A3 + A2/A3 Series @@ -2526,19 +2526,19 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD `--debug-tensor-dump-output-folder` `None` Type: str - A2, A3 + A2/A3 Series `--debug-tensor-dump-layers` `None` List[int] - A2, A3 + A2/A3 Series `--debug-tensor-dump-input-file` `None` Type: str - A2, A3 + A2/A3 Series @@ -2567,19 +2567,19 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD `--disaggregation-mode` `null` `null`,
`prefill`,
`decode` - A2, A3 + A2/A3 Series `--disaggregation-transfer-backend` `mooncake` `ascend` (default `mooncake` not supported on NPU, must be specified manually) - A2, A3 + A2/A3 Series `--disaggregation-bootstrap-port` `8998` Type: int - A2, A3 + A2/A3 Series `--disaggregation-ib-device` @@ -2591,25 +2591,25 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD `--disaggregation-decode-enable-offload-kvcache` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--num-reserved-decode-tokens` `512` Type: int - A2, A3 + A2/A3 Series `--disaggregation-decode-polling-interval` `1` Type: int - A2, A3 + A2/A3 Series `--optimistic-prefill-attempts` `0` Type: int - A2, A3 + A2/A3 Series @@ -2637,43 +2637,43 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD `--enable-adaptive-dispatch-to-encoder` `False` bool flag
(set to enable adaptive dispatch) - A2, A3 + A2/A3 Series `--encoder-only` `False` bool flag
(set to launch an encoder-only server) - A2, A3 + A2/A3 Series `--language-only` `False` bool flag
(set to load weights for the language model only) - A2, A3 + A2/A3 Series `--encoder-transfer-backend` `zmq_to_scheduler` `zmq_to_scheduler`,
`zmq_to_tokenizer`,
`mooncake` - A2, A3 + A2/A3 Series `--encoder-urls` `[]` List[str]
(List of encoder server urls) - A2, A3 + A2/A3 Series `--encoder-bootstrap-port` `8997` Type: int - A2, A3 + A2/A3 Series `--encoder-register-urls` `[]` List[str] - A2, A3 + A2/A3 Series @@ -2702,55 +2702,55 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD `--custom-weight-loader` `None` List[str] - A2, A3 + A2/A3 Series `--weight-loader-disable-mmap` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--weight-loader-prefetch-checkpoints` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--weight-loader-prefetch-num-threads` `4` Type: int - A2, A3 + A2/A3 Series `--remote-instance-weight-loader-seed-instance-ip` `None` Type: str - A2, A3 + A2/A3 Series `--remote-instance-weight-loader-seed-instance-service-port` `None` Type: int - A2, A3 + A2/A3 Series `--remote-instance-weight-loader-send-weights-group-ports` `None` Type: JSON
list - A2, A3 + A2/A3 Series `--remote-instance-weight-loader-backend` `nccl` `transfer_engine`,
`nccl`,
`modelexpress` - A2, A3 + A2/A3 Series `--weight-loader-drop-cache-after-load` `False` bool flag (set to enable) - A2, A3 + A2/A3 Series `--remote-instance-weight-loader-start-seed-via-transfer-engine` @@ -2825,37 +2825,37 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD `--enable-broadcast-mm-inputs-process` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--mm-process-config` `None` Type: JSON / Dict - A2, A3 + A2/A3 Series `--mm-enable-dp-encoder` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series `--mm-processor-worker-num` `0` Type: int - A2, A3 + A2/A3 Series `--mm-io-worker-num` `0` Type: int - A2, A3 + A2/A3 Series `--limit-mm-data-per-request` `None` Type: JSON / Dict - A2, A3 + A2/A3 Series @@ -2883,19 +2883,19 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD `--decrypted-config-file` `None` Type: str - A2, A3 + A2/A3 Series `--decrypted-draft-config-file` `None` Type: str - A2, A3 + A2/A3 Series `--enable-prefix-mm-cache` `False` bool flag
(set to enable) - A2, A3 + A2/A3 Series @@ -2924,13 +2924,13 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD `--forward-hooks` `None` Type: JSON list - A2, A3 + A2/A3 Series `--enable-quant-communications` `False` bool flag (set to enable)
(When using DeepEP with a quantized model, you must set --deepep-dispatcher-output-dtype) - A2, A3 + A2/A3 Series @@ -2958,7 +2958,7 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD `--msprobe-dump-config` `None` Type: str - A2, A3 + A2/A3 Series @@ -2986,7 +2986,7 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD `--config` `None` Type: str - A2, A3 + A2/A3 Series diff --git a/docs/docs/hardware-platforms/ascend-npus/reference/support_models.mdx b/docs/docs/hardware-platforms/ascend-npus/reference/support_models.mdx index 4d07ed1d4..40b68c17b 100644 --- a/docs/docs/hardware-platforms/ascend-npus/reference/support_models.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/reference/support_models.mdx @@ -20,8 +20,8 @@ You are welcome to enable various models based on your business requirements. Models Model Family - A2 Supported - A3 Supported + Ascend A2 Series Products Supported + Ascend A3 Series Products Supported @@ -461,8 +461,8 @@ You are welcome to enable various models based on your business requirements. Models Model Family (Variants) - A2 Supported - A3 Supported + Ascend A2 Series Products Supported + Ascend A3 Series Products Supported @@ -644,8 +644,8 @@ You are welcome to enable various models based on your business requirements. Models Model Family - A2 Supported - A3 Supported + Ascend A2 Series Products Supported + Ascend A3 Series Products Supported @@ -695,8 +695,8 @@ You are welcome to enable various models based on your business requirements. Models Model Family - A2 Supported - A3 Supported + Ascend A2 Series Products Supported + Ascend A3 Series Products Supported @@ -740,8 +740,8 @@ You are welcome to enable various models based on your business requirements. Models Model Family - A2 Supported - A3 Supported + Ascend A2 Series Products Supported + Ascend A3 Series Products Supported diff --git a/docs/docs/sglang-diffusion/models_with_ar.mdx b/docs/docs/sglang-diffusion/models_with_ar.mdx index 8b734f82e..010883045 100644 --- a/docs/docs/sglang-diffusion/models_with_ar.mdx +++ b/docs/docs/sglang-diffusion/models_with_ar.mdx @@ -116,7 +116,7 @@ export HCCL_NPU_SOCKET_PORT_RANGE="24200-24399" ## Best practices -GLM-Image example for Ascend A3 2 cards (4 devices) +GLM-Image example for Ascend A3 Series, 2 cards (4 devices) ```bash # Terminal 1 : server with AR model export HCCL_IF_BASE_PORT=23000 diff --git a/docs/docs/sglang-diffusion/quantization.mdx b/docs/docs/sglang-diffusion/quantization.mdx index ec372d063..3ce08e7af 100644 --- a/docs/docs/sglang-diffusion/quantization.mdx +++ b/docs/docs/sglang-diffusion/quantization.mdx @@ -1182,8 +1182,8 @@ sglang generate \ ``` The selected SRT backend must provide an MXFP8 kernel on NVIDIA or ROCm. On -Ascend, the hardware requirement remains A5 series or newer; -`npu_dynamic_mx_quant` is not available on A2/A3. +Ascend, the hardware requirement remains 950PR/DT Series or newer; +`npu_dynamic_mx_quant` is not available on A2/A3 Series. ## MXFP8 Offline Quantization (msmodelslim) @@ -1216,8 +1216,8 @@ sglang generate \ --save-output ``` -> **Hardware requirement:** Ascend A5 series or newer. `npu_dynamic_dual_level_mx_quant` -> and `npu_dual_level_quant_matmul` are not available on A2/A3. +> **Hardware requirement:** Ascend 950PR/DT Series or newer. `npu_dynamic_dual_level_mx_quant` +> and `npu_dual_level_quant_matmul` are not available on A2/A3 Series. > > **Note:** Online MXFP4 weight quantization is experimental. The offline msmodelslim > flow uses pre-quantized weights and may produce different numerical results. diff --git a/docs/src/snippets/_deployment.jsx b/docs/src/snippets/_deployment.jsx index f4ab16400..8febe6c2b 100644 --- a/docs/src/snippets/_deployment.jsx +++ b/docs/src/snippets/_deployment.jsx @@ -147,10 +147,10 @@ export const Deployment = ({ config, benchmarks }) => { { id: "mi355x", label: "MI355X", vram: "288GB", multiNodeDockerFlags: [...AMD_RDMA_DOCKER_FLAGS] }, ], - // Atlas 800I A3 (910C): 1 card = 2 dies, so --tp-size is 2× the card + // Ascend A3 Series: 1 card = 2 dies, so --tp-size is 2× the card // count (32 cards -> --tp-size 64). npu: [ - { id: "a3", label: "Atlas 800I A3", vram: "64GB/die" }, + { id: "a3", label: "Ascend A3 Series", vram: "64GB/die" }, ], }; @@ -839,7 +839,7 @@ export const Deployment = ({ config, benchmarks }) => { : vendorOf(sel.hw) === "npu" ? [ // NPU: --privileged grants the davinci devices (16 dies on an - // 8-card Atlas 800I A3 node); the host CANN driver/firmware/state + // 8-card Ascend A3 Series node); the host CANN driver/firmware/state // must be mounted in. "docker run --privileged --shm-size=16g", " --device=/dev/davinci0 --device=/dev/davinci1 --device=/dev/davinci2 --device=/dev/davinci3", diff --git a/docs/src/snippets/_playground.jsx b/docs/src/snippets/_playground.jsx index 062b2370c..2fcf21377 100644 --- a/docs/src/snippets/_playground.jsx +++ b/docs/src/snippets/_playground.jsx @@ -772,7 +772,7 @@ export const Playground = ({ config }) => { && (!mmOpt.excludesStrategy || !mmOpt.excludesStrategy.includes(base.strategy)); const backendIsMega = slotDisplay("backend") === "megamoe"; // `ep.showWhen` (function of base) drops the whole EP select on bases - // where EP is not a supported lever (e.g. the single-shape A3 recipe). + // where EP is not a supported lever (e.g. the single-shape A3 Series recipe). const epShown = !!fc.ep && !(typeof fc.ep.showWhen === "function" && !fc.ep.showWhen(base)); return ( diff --git a/docs/src/snippets/configs/moonshotai/kimi-k3.jsx b/docs/src/snippets/configs/moonshotai/kimi-k3.jsx index d8c347131..561382eb2 100644 --- a/docs/src/snippets/configs/moonshotai/kimi-k3.jsx +++ b/docs/src/snippets/configs/moonshotai/kimi-k3.jsx @@ -198,7 +198,7 @@ export const config = { disableReason: (s) => (s.hw === "a3" ? "Only Modelslim (W4A8) is supported on this recipe." : ""), }, { - // A3 only (NPU W4A8 checkpoint); hidden on the GPU recipes. + // A3 Series only (NPU W4A8 checkpoint); hidden on the GPU recipes. id: "modelslim", label: "Modelslim (W4A8)", subtitle: "ModelScope NPU checkpoint", @@ -513,7 +513,7 @@ export const config = { ]; }, }, { - // A3 only: 64 ranks (4 nodes × 8 cards × 2 dies); hidden on the GPU recipes. + // A3 Series only: 64 ranks (4 nodes × 8 cards × 2 dies); hidden on the GPU recipes. value: 64, hide: { hw: ["b300", "gb300", "b200", "gb200", "h200", "h100", "mi350x", "mi355x"] }, }, @@ -719,7 +719,7 @@ export const config = { // EAGLE --speculative-num-steps N (chain; topk>1 is a tree) // Only DSPARK is selectable today, so only its form is emitted. id: "proposedDraftTokens", title: "Proposed Draft Tokens", - // The A3 recipe pins the shipped block size (7). + // The A3 Series recipe pins the shipped block size (7). showWhen: (b) => b.spec === "dspark" && b.hw !== "a3", control: "slider", stripPrefixes: [ @@ -743,7 +743,7 @@ export const config = { // Spec-only, so gate the row on DSPARK; every DSPARK recipe (except the PD // prefill role) turns it on in the base, so this row derives to On and // exists mainly as the opt-out. - // Needs the Triton linear-attn decode backend (the K3 default); the A3 + // Needs the Triton linear-attn decode backend (the K3 default); the A3 Series // script never sets it. id: "replaySsm", title: "ReplaySSM (spec)", showWhen: (b) => b.spec === "dspark" && b.hw !== "a3", @@ -766,7 +766,7 @@ export const config = { // without --speculative-dspark-sps-table-path (every step still // verifies full width); fails fast with ReplaySSM or DCP > 1. id: "raggedVerify", title: "Ragged Verify Mode (spec)", - // The A3 recipe pins static. + // The A3 Series recipe pins static. showWhen: (b) => b.spec === "dspark" && b.hw !== "a3", stripEnv: ["SGLANG_RAGGED_VERIFY_MODE"], options: [ diff --git a/docs/src/snippets/diffusion/flux-deployment.jsx b/docs/src/snippets/diffusion/flux-deployment.jsx index 400f3d2ce..24e4b7e00 100644 --- a/docs/src/snippets/diffusion/flux-deployment.jsx +++ b/docs/src/snippets/diffusion/flux-deployment.jsx @@ -14,8 +14,8 @@ export const FluxDeployment = () => { { id: 'mi355x', label: 'MI355X', default: false }, { id: 'mi325x', label: 'MI325X', default: false }, { id: 'mi300x', label: 'MI300X', default: false }, - { id: 'a2', label: 'A2', default: false }, - { id: 'a3', label: 'A3', default: false } + { id: 'a2', label: 'A2 Series', default: false }, + { id: 'a3', label: 'A3 Series', default: false } ] }, version: { @@ -51,7 +51,7 @@ export const FluxDeployment = () => { } if (hardware === 'a3') { - return `#One A3 card has 2 npu chips + return `#One A3 Series card has 2 npu chips sglang serve \\ --tp-size 2 \\ --model-path ${config.repoId} \\ @@ -133,7 +133,7 @@ sglang serve \\ useEffect(() => { const isAscend = values.hardware === 'a2' || values.hardware === 'a3'; - const targetTabName = isAscend ? 'Ascend A3' : 'NVIDIA B200'; + const targetTabName = isAscend ? 'Ascend A3 Series' : 'NVIDIA B200'; const allTabs = document.querySelectorAll('button, [role="tab"]'); allTabs.forEach((tab) => { diff --git a/docs/src/snippets/diffusion/qwen-image-deployment.jsx b/docs/src/snippets/diffusion/qwen-image-deployment.jsx index 7b2b1d7cf..2ced24128 100644 --- a/docs/src/snippets/diffusion/qwen-image-deployment.jsx +++ b/docs/src/snippets/diffusion/qwen-image-deployment.jsx @@ -14,8 +14,8 @@ export const QwenImageDeployment = () => { { id: 'mi300x', label: 'MI300X', default: false }, { id: 'mi325x', label: 'MI325X', default: false }, { id: 'mi355x', label: 'MI355X', default: false }, - { id: 'a2', label: 'A2', default: false }, - { id: 'a3', label: 'A3', default: false } + { id: 'a2', label: 'A2 Series', default: false }, + { id: 'a3', label: 'A3 Series', default: false } ] }, precision: { @@ -42,7 +42,7 @@ export const QwenImageDeployment = () => { } if (values.hardware === 'a3') { - return `#One A3 card has 2 npu chips + return `#One A3 Series card has 2 npu chips sglang serve \\ --model-path Qwen/Qwen-Image \\ --tp-size 1 \\ @@ -131,8 +131,8 @@ sglang serve \\ useEffect(() => { let targetTabName = 'AMD MI300X'; - if (values.hardware === 'a2') targetTabName = 'Ascend A2'; - if (values.hardware === 'a3') targetTabName = 'Ascend A3'; + if (values.hardware === 'a2') targetTabName = 'Ascend A2 Series'; + if (values.hardware === 'a3') targetTabName = 'Ascend A3 Series'; const allTabs = document.querySelectorAll('button, [role="tab"]'); diff --git a/docs/src/snippets/diffusion/wan21-deployment.jsx b/docs/src/snippets/diffusion/wan21-deployment.jsx index 8073aa4f4..f7e2f6e81 100644 --- a/docs/src/snippets/diffusion/wan21-deployment.jsx +++ b/docs/src/snippets/diffusion/wan21-deployment.jsx @@ -47,8 +47,8 @@ export const Wan21Deployment = () => { { id: 'mi300x', label: 'MI300X', default: false }, { id: 'mi325x', label: 'MI325X', default: false }, { id: 'mi355x', label: 'MI355X', default: false }, - { id: 'a2', label: 'A2', default: false }, - { id: 'a3', label: 'A3', default: false } + { id: 'a2', label: 'A2 Series', default: false }, + { id: 'a3', label: 'A3 Series', default: false } ], }, task: { @@ -119,7 +119,7 @@ export const Wan21Deployment = () => { useEffect(() => { const isAscend = values.hardware === 'a2' || values.hardware === 'a3'; - const targetTabName = isAscend ? 'Ascend A3' : 'NVIDIA B200'; + const targetTabName = isAscend ? 'Ascend A3 Series' : 'NVIDIA B200'; const allTabs = document.querySelectorAll('button, [role="tab"]'); @@ -182,7 +182,7 @@ export const Wan21Deployment = () => { if (hardware === 'a2' || hardware === 'a3') { const comment = hardware === 'a3' - ? '#One A3 card has 2 npu chips\n' + ? '#One A3 Series card has 2 npu chips\n' : ''; const isBestPractice = bestPractice === 'on'; let command; diff --git a/docs/src/snippets/diffusion/wan22-deployment.jsx b/docs/src/snippets/diffusion/wan22-deployment.jsx index c854eefb5..4eefd681b 100644 --- a/docs/src/snippets/diffusion/wan22-deployment.jsx +++ b/docs/src/snippets/diffusion/wan22-deployment.jsx @@ -11,8 +11,8 @@ { id: 'mi300x', label: 'MI300X', default: false }, { id: 'mi325x', label: 'MI325X', default: false }, { id: 'mi355x', label: 'MI355X', default: false }, - { id: 'a2', label: 'A2', default: false }, - { id: 'a3', label: 'A3', default: false } + { id: 'a2', label: 'A2 Series', default: false }, + { id: 'a3', label: 'A3 Series', default: false } ], }, task: { @@ -90,7 +90,7 @@ useEffect(() => { const isAscend = values.hardware === 'a2' || values.hardware === 'a3'; - const targetTabName = isAscend ? 'Ascend A3' : 'NVIDIA B200'; + const targetTabName = isAscend ? 'Ascend A3 Series' : 'NVIDIA B200'; const allTabs = document.querySelectorAll('button, [role="tab"]'); @@ -138,7 +138,7 @@ if (hardware === 'a2' || hardware === 'a3') { const comment = hardware === 'a3' - ? '#One A3 card has 2 npu chips\n' + ? '#One A3 Series card has 2 npu chips\n' : ''; const isBestPractice = bestPractice === 'on'; let command; diff --git a/docs/src/snippets/diffusion/zimage-turbo-deployment.jsx b/docs/src/snippets/diffusion/zimage-turbo-deployment.jsx index e66748b15..3acead1b7 100644 --- a/docs/src/snippets/diffusion/zimage-turbo-deployment.jsx +++ b/docs/src/snippets/diffusion/zimage-turbo-deployment.jsx @@ -13,8 +13,8 @@ export const ZImageTurboDeployment = () => { { id: 'b200', label: 'B200', default: true }, { id: 'h200', label: 'H200', default: false }, { id: 'h100', label: 'H100', default: false }, - { id: 'a2', label: 'A2', default: false }, - { id: 'a3', label: 'A3', default: false } + { id: 'a2', label: 'A2 Series', default: false }, + { id: 'a3', label: 'A3 Series', default: false } ] } }, @@ -29,7 +29,7 @@ export const ZImageTurboDeployment = () => { } if (hardware === 'a3') { - return `#One A3 card has 2 npu chips + return `#One A3 Series card has 2 npu chips sglang serve \\ --model-path Tongyi-MAI/Z-Image-Turbo \\ --tp-size 2 \\ @@ -112,7 +112,7 @@ sglang serve \\ useEffect(() => { const isAscend = values.hardware === 'a2' || values.hardware === 'a3'; - const targetTabName = isAscend ? 'Ascend A2 / A3' : 'AMD MI300X'; + const targetTabName = isAscend ? 'Ascend A2/A3 Series' : 'AMD MI300X'; const allTabs = document.querySelectorAll('button, [role="tab"]'); allTabs.forEach((tab) => {