From edb1b3f8f5ab066af1e9b6ee8e8738fadcfa77e7 Mon Sep 17 00:00:00 2001 From: jianzhao-xu <978716854@qq.com> Date: Thu, 14 May 2026 11:28:16 +0800 Subject: [PATCH] [NPU] add Ascend NPU Accuracy Evaluation and Faq docs (#24777) --- .codespellrc | 2 +- docs_new/docs.json | 4 +- .../ascend_npu_accuracy_evaluation.mdx | 470 ++++++++++++++++++ .../ascend-npus/ascend_npu_faq.mdx | 223 +++++++++ 4 files changed, 697 insertions(+), 2 deletions(-) create mode 100644 docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation.mdx create mode 100644 docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_faq.mdx diff --git a/.codespellrc b/.codespellrc index e27616e21..c7d18ae09 100644 --- a/.codespellrc +++ b/.codespellrc @@ -1,3 +1,3 @@ [codespell] -ignore-words-list = ans, als, hel, boostrap, childs, te, vas, hsa, ment, cann, thi, makro, wil, rouge, PRIS, ather, MIS, medias, allready, inout, nd, fo, visibles, nothink, renderD, ond +ignore-words-list = ans, als, hel, boostrap, childs, te, vas, hsa, ment, cann, thi, makro, wil, rouge, PRIS, ather, MIS, medias, allready, inout, nd, fo, visibles, nothink, renderD, ond, tbe skip = *.json, *.jsonl, *.patch, *.txt, *.lock diff --git a/docs_new/docs.json b/docs_new/docs.json index 09d3ab57c..50d0540ed 100644 --- a/docs_new/docs.json +++ b/docs_new/docs.json @@ -898,11 +898,13 @@ "docs/hardware-platforms/ascend-npus/ascend_npu_support_new_models", "docs/hardware-platforms/ascend-npus/ascend_npu_best_practice", "docs/hardware-platforms/ascend-npus/ascend_npu_optimization", + "docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation", "docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing", "docs/hardware-platforms/ascend-npus/ascend_npu_ring_sp_performance", "docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_5_examples", "docs/hardware-platforms/ascend-npus/ascend_npu_glm5_examples", - "docs/hardware-platforms/ascend-npus/ascend_npu_environment_variables" + "docs/hardware-platforms/ascend-npus/ascend_npu_environment_variables", + "docs/hardware-platforms/ascend-npus/ascend_npu_faq" ] }, "docs/hardware-platforms/cpu_server", diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation.mdx new file mode 100644 index 000000000..38ae57679 --- /dev/null +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation.mdx @@ -0,0 +1,470 @@ +--- +title: "Ascend NPU Accuracy Evaluation" +metatags: + description: "Complete guide for SGLang model accuracy evaluation on Ascend NPU using EvalScope and AISBench, covering text/multimodal scenarios." +--- + +# Ascend NPU Accuracy Evaluation + +This document describes how to perform accuracy evaluation for SGLang models running on Ascend NPU using two tools: **EvalScope** and **AISBench**. The following scenarios are covered: + +- **Online Testing**: Evaluate via API interface after starting SGLang server +- **Text Models**: Using Qwen2.5-7B-Instruct as example +- **Multimodal Models**: Using Qwen2.5-VL-7B-Instruct as example + +--- + +## Environment Setup + +First, launch the SGLang environment using the provided container image: + +```shell Command +# Atlas 800I A3 environment +export IMAGE=quay.io/ascend/sglang:main-cann8.5.0-a3 + +docker run -it --rm --privileged --network=host --ipc=host --shm-size=16g \ + --device=/dev/davinci0 --device=/dev/davinci1 --device=/dev/davinci2 --device=/dev/davinci3 \ + --device=/dev/davinci4 --device=/dev/davinci5 --device=/dev/davinci6 --device=/dev/davinci7 \ + --device=/dev/davinci8 --device=/dev/davinci9 --device=/dev/davinci10 --device=/dev/davinci11 \ + --device=/dev/davinci12 --device=/dev/davinci13 --device=/dev/davinci14 --device=/dev/davinci15 \ + --device=/dev/davinci_manager \ + --device=/dev/hisi_hdc \ + --volume /usr/local/sbin:/usr/local/sbin \ + --volume /usr/local/Ascend/driver:/usr/local/Ascend/driver \ + --volume /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ + --volume /etc/ascend_install.info:/etc/ascend_install.info \ + --volume /var/queue_schedule:/var/queue_schedule \ + --volume ~/.cache/:/root/.cache/ \ + --entrypoint=bash \ + $IMAGE +``` + +**For Atlas 800I A2 users**: Replace the image tag with `main-cann8.5.0-910b` and adjust device mappings from `/dev/davinci[8-15]` to `/dev/davinci[0-7]`. + +--- + +## Using EvalScope + +[EvalScope](https://github.com/modelscope/evalscope) is a comprehensive model evaluation framework from ModelScope, supporting both accuracy evaluation and performance stress testing. + +### Install EvalScope + +```shell Command +# Method1 Installing via pip +pip install evalscope + +# Method2 Installing from source +git clone https://github.com/modelscope/evalscope.git +cd evalscope/ +pip install -e . +``` + +### Online Text Model Testing + +This section covers online evaluation scenarios where the SGLang server is already running. + +#### Start SGLang Server + +```shell Command +# Set HuggingFace mirror (if network access is restricted) +export HF_ENDPOINT=https://hf-mirror.com + +# Start text model server +sglang serve --model-path /home/weights/Qwen2.5-7B-Instruct --attention-backend ascend --host 0.0.0.0 --port 30000 & +``` +For more details of SGLang server, refer to the [Ascend NPU Quick Start](/docs/hardware-platforms/ascend-npus/ascend_npu_quick_start) + +#### Execute Accuracy Evaluation + +EvalScope connects to the SGLang server via OpenAI-compatible API. The following example uses the GSM8K dataset: + +```shell Command +evalscope eval \ + --model /home/weights/Qwen2.5-7B-Instruct \ + --api-url http://localhost:30000/v1 \ + --api-key EMPTY \ + --eval-type server \ + --datasets gsm8k \ + --limit 10 +``` + +Upon completion, results similar to the following will be displayed: + +``` ++---------------------+-----------+----------+----------+-------+---------+---------+ +| Model | Dataset | Metric | Subset | Num | Score | Cat.0 | ++=====================+===========+==========+==========+=======+=========+=========+ +| Qwen2.5-7B-Instruct | gsm8k | mean_acc | main | 5 | 1.0 | default | ++---------------------+-----------+----------+----------+-------+---------+---------+ +``` + +> **Note**: Output format may vary slightly across different EvalScope versions. The above example is from EvalScope 1.6.x. Ensure the `--model` parameter matches the model name returned by the SGLang server's `/v1/models` endpoint. When starting the server with an HF path (e.g., `Qwen/Qwen2.5-7B-Instruct`), use that path directly. For local paths, pass the full path or the model name returned by `/v1/models`. + +#### Common Datasets for Online Evaluation + +```shell Command +# MMLU +evalscope eval \ + --model /home/weights/Qwen2.5-7B-Instruct \ + --api-url http://localhost:30000/v1 \ + --api-key EMPTY \ + --eval-type server \ + --datasets mmlu + +# CEval (Chinese evaluation) +evalscope eval \ + --model /home/weights/Qwen2.5-7B-Instruct \ + --api-url http://localhost:30000/v1 \ + --api-key EMPTY \ + --eval-type server \ + --datasets ceval + +# MATH-500 +evalscope eval \ + --model /home/weights/Qwen2.5-7B-Instruct \ + --api-url http://localhost:30000/v1 \ + --api-key EMPTY \ + --eval-type server \ + --datasets math + +# HumanEval (code generation) +evalscope eval \ + --model /home/weights/Qwen2.5-7B-Instruct \ + --api-url http://localhost:30000/v1 \ + --api-key EMPTY \ + --eval-type server \ + --datasets humaneval +``` + +### Online Multimodal Model Testing + +#### Start Multimodal Model Server + +```shell Command +# Start multimodal model server (Qwen2.5-VL-7B-Instruct) +# Multimodal models require both --attention-backend and --mm-attention-backend +sglang serve --model-path /home/weights/Qwen2.5-VL-7B-Instruct \ + --attention-backend ascend \ + --mm-attention-backend ascend_attn \ + --host 0.0.0.0 --port 30000 & +``` + +#### Execute Multimodal Accuracy Evaluation + +```shell Command +# MMBench (multimodal evaluation) +evalscope eval \ + --model /home/weights/Qwen2.5-VL-7B-Instruct \ + --api-url http://localhost:30000/v1 \ + --api-key EMPTY \ + --eval-type server \ + --datasets mmbench + +# MMMU (multimodal comprehensive understanding) +evalscope eval \ + --model /home/weights/Qwen2.5-VL-7B-Instruct \ + --api-url http://localhost:30000/v1 \ + --api-key EMPTY \ + --eval-type server \ + --datasets mmmu + +# HallusionBench (hallucination evaluation) +evalscope eval \ + --model /home/weights/Qwen2.5-VL-7B-Instruct \ + --api-url http://localhost:30000/v1 \ + --api-key EMPTY \ + --eval-type server \ + --datasets hallusionbench +``` + +For more details, refer to the [EvalScope documentation](https://evalscope.readthedocs.io/). + +--- + +## Using AISBench + +[AISBench](https://github.com/AISBench/benchmark) is an official benchmark testing tool from Ascend, supporting accuracy and performance evaluation across multiple datasets. + +### Install AISBench + +```shell Command +# Install from source (recommended to use Gitee mirror) +git clone https://github.com/AISBench/benchmark.git +cd benchmark/ + +# Install core package (use Aliyun mirror if network access is restricted) +pip3 install -e ./ --use-pep517 + +# If dependency installation times out, use Aliyun mirror: +# pip3 install -r requirements/runtime.txt -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com + +# Verify installation +ais_bench -h +``` + +> **Note**: When using `pip install -e` (development mode), the `ais_bench` command may not be in PATH. Use `python3 -m ais_bench.benchmark.cli.main` as an alternative. + +### Configuration File Setup + +Each model task, dataset task, and result presentation task corresponds to a configuration file. You need to modify the content of these configuration files before running the command. The paths of these configuration files can be queried by adding `--search` to the original AISBench command. For example: +``` +ais_bench --models vllm_api_general_chat --datasets gsm8k_gen_0_shot_cot_chat_prompt.py --mode all --dump-eval-details --merge-ds --search +``` +Executing the query command will yield the following results: +``` +╒═════════════╤══════════════════════════════════╤═════════════════════════════════════════════════════════════════════════════════════════════════════════╕ +│ Task Type │ Task Name │ Config File Path │ +╞═════════════╪══════════════════════════════════╪═════════════════════════════════════════════════════════════════════════════════════════════════════════╡ +│ --models │ vllm_api_general_chat │ /home/code/benchmark/ais_bench/benchmark/configs/models/vllm_api/vllm_api_general_chat.py │ +├─────────────┼──────────────────────────────────┼─────────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│ --datasets │ gsm8k_gen_0_shot_cot_chat_prompt │ /home/code/benchmark/ais_bench/benchmark/configs/datasets/gsm8k/gsm8k_gen_0_shot_cot_chat_prompt.py │ +╘═════════════╧══════════════════════════════════╧═════════════════════════════════════════════════════════════════════════════════════════════════════════╛ +``` +For online text models, edit `benchmark/ais_bench/benchmark/configs/models/vllm_api/vllm_api_general_chat.py`: + +```python +from ais_bench.benchmark.models import VLLMCustomAPIChat +from ais_bench.benchmark.utils.model_postprocessors import extract_non_reasoning_content + +models = [ + dict( + attr="service", # Backend type identifier + type=VLLMCustomAPIChat, + abbr='vllm-api-general-chat', + path="/home/weights/Qwen2.5-7B-Instruct", # Path to model vocabulary file (usually not required for accuracy testing) + model="/home/weights/Qwen2.5-7B-Instruct", # Model name on server (empty string auto-detects) + request_rate=0, # Request frequency; sends all at once if <0.1 + retry=2, # Maximum retry attempts per request + host_ip="localhost", # Inference service IP + host_port=30000, # Inference service port + max_out_len=512, # Maximum output tokens + batch_size=1, # Maximum request concurrency + trust_remote_code=False, # Whether tokenizer trusts remote code + generation_kwargs=dict( # Inference parameters (passed directly to requests) + temperature=0.6, + top_k=10, + top_p=0.95, + seed=None, + repetition_penalty=1.03, + ), + pred_postprocessor=dict(type=extract_non_reasoning_content) + ) +] +``` + +> **Note**: SGLang server defaults to port `30000` and is compatible with OpenAI API format, so AISBench's `VLLMCustomAPIChat` can connect directly to SGLang. +> +> **Important**: The sum of `max_out_len` and input token count must not exceed the SGLang server's `max_model_len` (default 32768 for Qwen2.5-7B). We recommend setting `max_out_len` to `512` or `1024` to avoid `400` errors caused by exceeding the context window. + +### Download Datasets + +AISBench supports multiple common datasets that must be downloaded to a specified path before use. + +```shell Command +# C-Eval +cd ais_bench/datasets +mkdir ceval/ && mkdir ceval/formal_ceval +cd ceval/formal_ceval +wget https://www.modelscope.cn/datasets/opencompass/ceval-exam/resolve/master/ceval-exam.zip +unzip ceval-exam.zip && rm ceval-exam.zip +cd ../../.. + +# MMLU +cd ais_bench/datasets +wget http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/mmlu.zip +unzip mmlu.zip && rm mmlu.zip +cd ../.. + +# GSM8K +cd ais_bench/datasets +wget http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/gsm8k.zip +unzip gsm8k.zip && rm gsm8k.zip +cd ../.. + +# GPQA +cd ais_bench/datasets +wget http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/gpqa.zip +unzip gpqa.zip && rm gpqa.zip +cd ../.. + +# MATH-500 +cd ais_bench/datasets +wget http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/math.zip +unzip math.zip && rm math.zip +cd ../.. + +# AIME 2024 +cd ais_bench/datasets +mkdir aime/ && cd aime/ +wget http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/aime.zip +unzip aime.zip && rm aime.zip +cd ../../.. + +# MMStar +cd ais_bench/datasets +mkdir mmstar +cd mmstar +wget https://www.modelscope.cn/datasets/evalscope/MMStar/resolve/master/MMStar.tsv +cd ../.. + +# MMMU +cd ais_bench/datasets +git lfs install +git clone https://www.modelscope.cn/datasets/AI-ModelScope/MMMU.git mmmu +cd ../.. +``` + +### Online Text Model Testing + +#### Start SGLang Server + +```shell Command +sglang serve --model-path /home/weights/Qwen2.5-7B-Instruct \ + --attention-backend ascend \ + --host 0.0.0.0 --port 30000 & +``` + +#### Execute Accuracy Evaluation + +```shell Command +# Run C-Eval dataset +ais_bench --models vllm_api_general_chat --datasets ceval_gen_0_shot_cot_chat_prompt.py --mode all --dump-eval-details --merge-ds + +# Run MMLU dataset +ais_bench --models vllm_api_general_chat --datasets mmlu_gen_0_shot_cot_chat_prompt.py --mode all --dump-eval-details --merge-ds + +# Run GSM8K dataset +ais_bench --models vllm_api_general_chat --datasets gsm8k_gen_0_shot_cot_chat_prompt.py --mode all --dump-eval-details --merge-ds + +# Run GPQA dataset +ais_bench --models vllm_api_general_chat --datasets gpqa_gen_0_shot_str.py --mode all --dump-eval-details --merge-ds + +# Run MATH-500 dataset +ais_bench --models vllm_api_general_chat --datasets math500_gen_0_shot_cot_chat_prompt.py --mode all --dump-eval-details --merge-ds + +# Run AIME 2024 dataset +ais_bench --models vllm_api_general_chat --datasets aime2024_gen_0_shot_chat_prompt.py --mode all --dump-eval-details --merge-ds +``` + +After execution, results are saved in `outputs/default//` with the following structure: + +``` +outputs/default/20250628_151326/ +├── configs # Configuration files +├── logs # Execution logs +│ ├── eval # Accuracy evaluation logs +│ └── infer # Inference process logs +├── predictions # Inference results (JSON) +├── results # Raw accuracy scores (JSON) +└── summary # Final result summary + ├── summary_20250628_151326.csv + ├── summary_20250628_151326.md + └── summary_20250628_151326.txt +``` + +### Online Multimodal Model Testing + +#### Configuration File + +Edit multimodal model configuration file (e.g., `vllm_api_stream_chat_mutiturn.py`): + +```python +from ais_bench.benchmark.models import VLLMCustomAPIChat +from ais_bench.benchmark.utils.postprocess.model_postprocessors import extract_non_reasoning_content + +models = [ + dict( + attr="service", + type=VLLMCustomAPIChat, + abbr="vllm-multiturn-api-chat-stream", + path="/home/weights/Qwen2.5-VL-7B-Instruct", + model="/home/weights/Qwen2.5-VL-7B-Instruct", + stream=True, + request_rate=0, + retry=2, + api_key="", + host_ip="localhost", + host_port=30000, + url="", + max_out_len=512, + batch_size=1, + trust_remote_code=False, + generation_kwargs=dict( + temperature=0.01, + ignore_eos=False, + ), + pred_postprocessor=dict(type=extract_non_reasoning_content), + ) +] +``` + +#### Start Multimodal Server and Execute Evaluation + +```shell Command +# Start multimodal server +sglang serve --model-path Qwen/Qwen2.5-VL-7B-Instruct \ + --attention-backend ascend \ + --mm-attention-backend ascend_attn \ + --host 0.0.0.0 --port 30000 & + +# Run MMStar dataset +ais_bench --models vllm_api_stream_chat_mutiturn --datasets mmstar_gen --mode all --dump-eval-details --merge-ds + +# Run MMMU dataset +ais_bench --models vllm_api_stream_chat_mutiturn --datasets mmmu_gen --mode all --dump-eval-details --merge-ds +``` + +For more details, refer to the [AISBench documentation](https://yh-ais-bench-benchmark.readthedocs.io). + +--- + +## Troubleshooting + +### SGLang Server Startup Failure + +1. Verify device mapping: A2 uses `davinci[0-7]`, A3 uses `davinci[0-15]` +2. Confirm image tag matches device type: A2 uses `...-910b`, A3 uses `...-a3` +3. Check NPU status with `npu-smi info` +4. First run requires model download; set `HF_ENDPOINT=https://hf-mirror.com` if network access is restricted + +### EvalScope Connection Failure to Server + +1. Confirm SGLang server started successfully (look for `Application startup complete` in logs) +2. Verify `--api-url` points to the correct port (SGLang defaults to `30000`) +3. Ensure URL ends with `/v1`, e.g., `http://localhost:30000/v1` + +### EvalScope SSL certificate verification failed +When using EvalScope commands without specifying a dataset or model path, it will attempt to download automatically, which may encounter an SSL certificate verification error: +``` + File "/usr/local/python3.11.14/lib/python3.11/site-packages/requests/sessions.py", line 605, in get + return self.request("GET", url, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/python3.11.14/lib/python3.11/site-packages/requests/sessions.py", line 592, in request + resp = self.send(prep, **send_kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/python3.11.14/lib/python3.11/site-packages/requests/sessions.py", line 706, in send + r = adapter.send(request, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/python3.11.14/lib/python3.11/site-packages/requests/adapters.py", line 676, in send + raise SSLError(e, request=request) +requests.exceptions.SSLError: HTTPSConnectionPool(host='www.modelscope.cn', port=443): Max retries exceeded with url: /api/v1/datasets/AI-ModelScope/gsm8k (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1016)'))) +[ERROR] 2026-05-13-02:20:01 (PID:876, Device:-1, RankID:-1) ERR99999 UNKNOWN application exception +``` +You can navigate to `/usr/local/python3.11.14/lib/python3.11/site-packages/requests/sessions.py`, find the `class Session` definition, and set `self.verify` to `False` to resolve this. + +### Download Dataset Error +For this error +``` +root@localhost:/home/# wget https://www.modelscope.cn/datasets/evalscope/MMStar/resolve/master/MMStar.tsv +--2026-05-12 12:08:01-- https://www.modelscope.cn/datasets/evalscope/MMStar/resolve/master/MMStar.tsv +Connecting to 141.5.152.215:6688... connected. +ERROR: cannot verify www.modelscope.cn's certificate, issued by ‘CN=Huawei Web Secure Internet Gateway CA V2,OU=IT,O=Huawei,L=Shenzhen,ST=GuangDong,C=CN’: + Self-signed certificate encountered. +To connect to www.modelscope.cn insecurely, use `--no-check-certificate'. +``` +You can add `--no-check-certificate' +``` +wget https://www.modelscope.cn/datasets/evalscope/MMStar/resolve/master/MMStar.tsv --no-check-certificate +``` + +For additional assistance, refer to [SGLang GitHub Issues](https://github.com/sgl-project/sglang/issues). diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_faq.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_faq.mdx new file mode 100644 index 000000000..7da46609f --- /dev/null +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_faq.mdx @@ -0,0 +1,223 @@ +--- +title: "Ascend NPU Troubleshooting and FAQ" +metatags: + description: "Troubleshooting and frequently asked questions for Ascend NPU" +--- + +Contributions are welcome. Feel free to add more. + +## 1. Context corruption with GLOO op.preamble.length <= op.nbytes in PD disaggregation + +### Error message + +```text highlight=2,5-7 +[2026-04-07 13:24:13 TP0] Decode batch, #running-req: 10, #token: 485248, token usage: 0.94, pre-allocated usage: 0.51, #prealloc-req: 1, #transfer-req: 12, #retracted-req: 0, npu graph: True, gen throughput (token/s): 259.82, #queue-req: 0 +[2026-04-07 13:24:13 TP0] Context corruption detected: Request 3b5dcfe1575d4e1f9b18c953de878a93 (bootstrap_room=7451500070298748792) received metadata from bootstrap_room=4125156593077881415. Metadata buffer index: 1. This indicates metadata buffer index collision. +[2026-04-07 13:24:13] INFO: 127.0.0.1:59272 - "POST /v1/chat/completions HTTP/1.1" 200 OK +[2026-04-07 13:24:13] INFO: 127.0.0.1:34000 - "POST /v1/chat/completions HTTP/1.1" 200 OK +terminate called after throwing an instance of 'gloo::EnforceNotMet' + what(): [enforce fail at /pytorch/third_party/gloo/gloo/transport/tcp/pair.cc:456] op.preamble.length <= op.nbytes. 4 vs 3 +Fatal Python error: Aborted + +Thread 0x0000fff873f6f120 (most recent call first): + File "/usr/local/python3.11.14/lib/python3.11/site-packages/sglang/srt/disaggregation/mooncake/conn.py", line 1499 in heartbeat_checker +``` + +### Cause + +(Possibly, not precisely located) High-concurrency long sequences fill up the transfer buffer, causing buffer index collision and data corruption. + +### Solution + +1. Disable overlap on the Prefill node with `--disable-overlap-schedule`. The Prefill node in PD disaggregation must not enable overlap, otherwise it causes timing issues that lead to out-of-order reception on the Decode node. +2. Even with overlap disabled, multi-Prefill node high-concurrency long-sequence scenarios may still encounter this issue with low probability. This is a known issue pending resolution. + +## 2. Graph mode aclnnInplaceFillScalar error + +### Error message + +```text highlight=2 +(SGLangEngine pid=3872176) [rank0]:[E414 12:14:41.204711510 compiler_depend.ts:444] operator():build/CMakeFiles/torch_npu.dir/compiler_depend.ts:26 NPU function error: + call aclnnInplaceFillScalar failed, error code is 507000 +(SGLangEngine pid=3872176) [ERROR] 2026-04-14-12:14:41 (PID:3874122, Device:0, RankID:-1) ERR00100 PTA call acl api failed +(SGLangEngine pid=3872176) [Error]: An internal error occurs in the runtime module on the host. +(SGLangEngine pid=3872176) Rectify the fault based on the error information in the ascend log. +(SGLangEngine pid=3872176) [PID: 3874122] 2026-04-14-12:14:41.897.548 AclNN_Runtime_Error(EZ9903): aclrtLaunchKerneWithHostArgs failed, return: 507000 +(SGLangEngine pid=3872176) Solution: In this scenario, collect the plog when the fault occurs and locate the fault based on the plog. +(SGLangEngine pid=3872176) TraceBack (most recent call last): +(SGLangEngine pid=3872176) Check kernel task failed, stream_id=2028, task_id=48, retCode=0x7080005.[FUNC:LaunchKernel][FILE:context.cc][LINE:1585] +(SGLangEngine pid=3872176) rtsLaunchKernelWithHostArgs execution failed, reason=kernel type error[FUNC:FuncErrorReason][FILE:error_message_manage.cc][LINE:61] +(SGLangEngine pid=3872176) rtsLaunchKernelWithHostArgs failed, runtime result = 507000.[FUNC:ReportCallError][FILE:Log_inner.cpp][LINE:148] +(SGLangEngine pid=3872176) aclrtLaunchKerneWWithHostArgs failed, return: 507000 +(SGLangEngine pid=3872176) Launch kernel failed. +(SGLangEngine pid=3872176) #### KernelLaunch failed: /home/850b160/cann-8.5.8/opp/built-in/op_impl/ai_core/tbe//kernel/ascend910_93/ops_legacy/fill/Fill_41dadce325bOf810d03359af2a38990b_high_performance.o +(SGLangEngine pid=3872176) Kernel Run failed. opType: 18, Fill +(SGLangEngine pid=3872176) launch failed for Fill, errno:361001. +(SGLangEngine pid=3872176) +(SGLangEngine pid=3872176) Exception raised from operator() at build/CMakeFiles/torch_npu.dir/compiler depend.ts:26 (most recent call first): +(SGLangEngine pid=3872176) frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >)+ 0xb0 (0xffff806848c0 in /root/anaconda3/envs/slime_re/Lib/python3.11/site-packages/torch/lib/libc10.so) +(SGLangEngine pid=3872176) frame #1: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string, std::allocator > const&) + 0x68(0xffff8062c140 in /root/anaconda3/envs/slime_re/Tib/puthon3.11/site-packages/torch/lib/libc10.so) +(SGLangEngine pid=3872176) frame #2: + 0x110e2b4 (0xffff6d44e2b4 in /root/anaconda3/envs/slime_re/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so) +(SGLangEngine pid=3872176) frame #3: + 0x29f0894(0xffff6ed30894 in /root/anaconda3/envs/slime_re/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so) +(SGLangEngine pid=3872176) frame #4: + 0x9cc708(0xffff6cd0c700 in /root/anaconda3/envs/slime_re/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so) +(SGLangEngine pid=3872176) frame #5: + 0x9cd2dc (0xffff6cd0d2dc in /root/anaconda3/envs/slime_re/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so) +``` + +### Cause + +Too many captured graphs cause a conflict in the graph mode update stream. Each graph is placed on a separate stream, but the number of streams is limited. If too many graphs are captured, conflicts occur. + +### Solution + +- CANN 8.5 + PTA 2.8 should have resolved this issue. +- If your versions do not match, reduce the number of captured graphs to 10 or fewer. + +## 3. alloc_extend_kernel error + +### Error message + +``` +[root@os-node-created-6z9tp pd_7p1d_tp2_20260414_030537]# grep -nR -E "aivec error|ACL" *,log +prefill 1.log:5739:EZ9999[PID: 164243] 2026-04-14-06:40:35.033.745 (EZ9999): The error from device(chipId:0, dieId:1), serial number is 1, there is an exception of aivec error, core id is 23, error code = 0, dump info: pc start: 0x12420156a000, current: 0x12420156ad74, vec error info: 0xdb1751f50e, mte error info: 0x98f6388707, ifu error info: 0x212c93fc00000, ccu error info: 0x998e981c00000000, cube error info: 0, biu error info: 0, aic error mask: 0x6500020bd00028c, para base: 0x12c100dccc00.[FUNC:PrintCoreInfo][FILE:device error core proc.cc][LINE:347] +prefill_1.log:5747:EZ9999[PID: 164242] 2026-04-14-06:40:35.034.456 (EZ9999): The error from device(chipId:0, dieId:0), serial number is 1, there is an exception of aivec error, core id is 45, error code = O, dump info: pc start: 0x12400156a000, current: 0x12400156ad74, vec error info: 0x7304583407, mte error info: 0x27e2b1c765, ifu error info: 0x212c93fc00000, ccu error info: 0x6e0836f300000000, cube error info: 0, biu error info: 0O, aic error mask: 0x6500020bd00028c, para base: 0x12c100dccc0O.[FUNC:PrintCoreInfo][FILE:device_error_core_proc.cc][LINE:347] +prefill_1.Log:5784:RuntimeError: ACL stream synchronize failed, error code:507035 +prefill_1.Log:5816:RuntimeError: ACL stream synchronize failed, error code:507035 +prefill 2.log:2566:EZ9999[PID: 163475] 2026-04-14-05:28:43.282.016 (EZ9999): The error from device(chipId:1, dieId:1), serial number is 1, there is an exception of aivec error, core id is 20, error code = 0, dump info: pc start: 0x124601550000, current: 0x124601550d74, vec error info: 0x9b07e496e3, mte error info: 0xc2a6806828, ifu error info: 0x212c93f200000, ccu error info: 0xaa64401100000000, cube error info: O, biu error info: 0, aic error mask: 0x6500020bd00028c, para base: 0x12c100c72400.[FUNC:PrintCoreInfo][FILE:device_error_core proc.cc][LINE:347] +prefill 2.log:2574:EZ9999[PID: 163474] 2026-04-14-05:28:43.282.809 (EZ9999): The error from device(chipId:1, dieId:0), serial number is 1, there is an exception of aivec error, core id is 22, error code = 0, dump info: pc start: 0x124401550000, current: 0x124401550d74, vec error info: 0xa80496049a, mte error info: 0xa0770e2bf2, ifu error info: 0x212c93f200000, ccu error info: 0x1083000000000000, cube error info: O, biu error info: 0O, aic error mask: 0x6500020bd00028c, para base: 0x12c100c72400.[FUNC:PrintCoreInfo][FILE:device_error_core_proc.cc][LINE:347] +prefill 2.log:2611:RuntimeError: ACL stream synchronize failed, error code:507035 +prefill 2.log:2644:RuntimeError: ACL stream synchronize failed, error code:507035 +prefill 4.log:1323:EZ9999[PID: 163478] 2026-04-14-05:08:15.363.509 (EZ9999): The error from device(chipId:3, dieId:0), serial number is 1, there is an exception of aivec error, core id is 5, error code = 0, dump info: pc start: 0x124c00dff000, current: 0x124c00dff860, vec error info: 0xf311fb8727, mte error info: 0x45418486a, ifu error info: 0x212c93fa00000, ccu error info: 0x17a9996f00000000, cube error info: 0, biu error info: 0, aic error mask: 0x6500020bd00028c, para base: 0x12c100d40c00.[FUNC:PrintCoreInfo][FILE:device_error_core_proc.cc][LINE:347] +prefill 4.log:1331:EZ9999[PID: 163479] 2026-04-14-05:08:15.364.016 (EZ9999): The error from device(chipId:3, dieId:1), serial number is 1, there is an exception of aivec error, core id is 40, error code = 0, dump info: pc start: 0x124e00dff000, current: 0x124e00dff860, vec error info: 0xf11a9cf188, mte error info: 0xfb7710514a, ifu error info: 0x212c93fa00000, ccu error info: 0x1f5828a900000000, cube error info: 0, biu error info: O, aic error mask: 0x6500020bd00028c, para base: 0x12c100d40c00.[FUNC:PrintCoreInfo][FILE:device_error_core_proc.cc][LINE:347] +prefill 4.Log:1368:RuntimeError: ACL stream synchronize failed, error code:507035 +prefill 4.log:1400:RuntimeError: ACL stream synchronize failed, error code:507035 +prefill 7.log:3017:EZ9999[PID: 164628] 2026-04-14-05:34:12.369.755 (EZ9999): The error from device(chipId:6, dieId:0), serial number is 1, there is an exception of aivec error, core id is 33, error code = 0, dump info: pc start: 0x1258015ae000, current: 0x1258015aed74, vec error info: 0x740053222c, mte error info: 0x97103df5a0, ifu error info: 0x212c93f400000, ccu error info: 0x391c89ab00000000, cube error info: 0, biu error info: 0O, aic error mask: 0x6500020bd00028c, para base: 0x12c10Ocd6400.[FUNC:PrintCoreInfo][FILE:device_error_core_proc.cc][LINE:347] +prefill 7.log:3025:EZ9999[PID: 164629] 2026-04-14-05:34:12.369.742 (EZ9999): The error from device(chipId:6, dieId:1), serial number is 1, there is an exception of aivec error, core id is 42, error code = 0, dump info: pc start: 0x125a015ae000, current: 0x125a015aed74, vec error info: 0xb91ae07b35, mte error info: 0xfc000670ef, ifu error info: 0x212c93f400000, ccu error info: 0x57c069b000000000, cube error info: 0, biu error info: O, aic error mask: 0x6500020bd00028c, para base: 0x12c10Ocd6400.[FUNC:PrintCoreInfo][FILE:device_error_core_proc.cc][LINE:347] +prefill 7.log:3062:RuntimeError: ACL stream synchronize failed, error code:507035 +prefill 7.log:3094:RuntimeError: ACL stream synchronize failed, error code:507035 +``` + +### Error plog + +```text highlight=21 +[ERROR] RUNTIME(164629,):2026-04-14-05:34:12.353.461 [stars engine.cc:1534]170327 ProcLogicCaReport:Task run failed, device id=13, stream id=43, task id=13145, sqe type=0(ffts), errType=0x1(task exception), sqSwStatus=0 +[ERROR] RUNTIME(164629,):2026-04-14-05:34:12.369.720 device error core proc.cc:3211170327 AddExceptionReqInfo:add error register: core id=42, stream id=43, task id=13145 +[ERROR] RUNTIME(164629,):2026-04-14-05:34:12.369.730 device error core proc.cc:3471170327 PrintCorelnfo:The error from device(chipld:6, dield:1), serial number is 1, there is an exception of aivec error, core id is 42, error code = O, dump info: pc start: 0x125a015ae000,current: 0x125a015aed74, vec error info: 0xb91ae07b35, mte error info: 0xfc000670ef, ifu error info: 0x212c93f400000, ccu error info: 0x57c069b000000000, cube error info: 0, biu error info: 0, aic error mask: 0x6500020bd00028c, para base: 0x12c100cd6400. +[ERROR] RUNTIME(164629,):2026-04-14-05:34:12.369.774 device error core proc.cc:3601170327 PrintCorelnto:The extend into: errcode:(0, 0x8000, 0) errorStr: When the D-cache reads and writes data to the UB, the response value returned bv the bus is a non-zero value. fixp_error0 info: 0x670ef, fixp error1 info: 0xfc, fsmId:0, tslot:2, thread:0, ctxid:0, blk:2, sublk:0, subErrType:4. +[ERROR] RUNTIME(164629,):2026-04-14-05:34:12.369.787 device error core proc.cc:4341170327 ProcessStarsCoreErrorInfo:devId=13, streamId=43, taskId=13145, MTE errorCode=0. +[ERROR] RUNTIME(164629,):2026-04-14-05:34:12.369.795 davinci task.cc:2011170327 SetStarsResultForDavinciTask:AIV Kernel happen error, retCode=0x31. +[ERROR] RUNTIME(164629,):2026-04-14-05:34:12.381.644 davinci kernel task.cc:15821170327 PreCheckTaskErr:Kernel task happen error retCode=0x31, vector core exception1. +[ERROR] RUNTIME(164629,):2026-04-14-05:34:12.381.705 davinci kernel task.cc:14241170327 GetArasInfo:[AIC INFO] aras(0 to 9) after execute:0x3fffffb9000, 0, 0, 0x12c9323ff600, 0x12c93231ee00, 0x12c93f1d8800,0x12c93f3ff600,0x12c958200000,0x100000003, 0xaaaa00000001. +[ERROR] RUNTIME(164629,):2026-04-14-05:34:12.381.710 [davinci kernel task.cc:1427]170327 GetArgsInfo:tilingKey = 0, print 1 Times totalLen=(10*8), argsSize=80, blockDim=3 +[ERROR] RUNTIME(164629,):2026-04-14-05:34:12.381.717 [davinci kernel task.cc:1468]170327 PrintErrorInfoForDavinciTask:[AIC INFO] after execute:arqs print end +[ERROR] RUNTIME(164629,):2026-04-14-05:34:12.381.751 davinci kernel task.cc:14981170327 PrintErrorInfoForDavinciTask:[DFX INFO]Aicore kernel execute failed, device id=13, stream id=43, report stream id=43, task id=13145, flip num=56, fault kernel_name=alloc_extend _kernel_18, fault kernel info ext=alloc_extend_kernel, program id=141, hash=14069671779787989248. +[ERROR] IDEDD(164629,):2026-04-14-05:34:12.381.823 [dump manager.cpp:41][tid:170327] An exception callback message is received. +[ERROR] IDEDD(164629,):2026-04-14-05:34:12.381.971 [kernel info collector.cpp:384][tid:170327] Get error register information. coreNum=0 +[ERROR] IDEDD(164629,):2026-04-14-05:34:12.381.981 kernel info collector.cpp:4771tid:1703271 It is Non-SuperKernel. functionCount=1, qlobalCount=1 +[ERROR] IDEDD(164629,):2026-04-14-05:34:12.381.987 [dump args.cpp:668][tid:170327] In arqAddr[0x12c100cd6400]|arqSize[80]dfxAddr[(nil)]|dfxSize[0] has invalid attribute. +[ERROR] IDEDD(164629,):2026-04-14-05:34:12.383.894 [dump_printf.cpp:1118][tid:170327] infoAddr is null +[ERROR] RUNTIME(164629,):2026-04-14-05:34:12.383.908 [stream.cc:1332]170327 GetError:Stream Synchronize failed, stream id=43, retCode=0x31, [vector core exception]. +[ERROR] RUNTIME(164629,):2026-04-14-05:34:12.383.911 [stream.cc:1335]170327 GetError:AIV Kernel happen error, retCode=0x31. +[ERROR] RUNTIME(164629,):2026-04-14-05:34:12.383.929 [stream.cc:1335]170327 GetError:[AIC_INFO] after execute:args print end +[ERROR] RUNTIME(164629,):2026-04-14-05:34:12.383.936 stream.cc:13351170327 GetError: DFX INFO1Aicore kernel execute failed, device id=13, stream id=43, report stream id=43, task id=13145, tlip num=56, + fault kernel_name=alloc_extend_kernel_18, ault kernel info ext=alloc_extend_kernel, program id=141, hash=14069671779787989248. +[ERROR] RUNTIME(164629,):2026-04-14-05:34:12.383.943 [stream.cc:3549]170327 EnterFailureAbort:stream id=43 enter failure abort. +[ERROR] RUNTIME(164629,):2026-04-14-05:34:12.383.973 [stars_engine.cc:1427]170327 StarsResumeRtsa:stop scheduling in abort failure mode: stream id=43, sq id=6,sq head=801, task id=13145, taskType=66. +[ERROR] RUNTIME(164629,):2026-04-14-05:34:12.383.972 [stream.cc:1463]164629 SynchronizeExecutedTask:context is abort, status=0x715005e. +[ERROR] RUNTIME(164629,):2026-04-14-05:34:12.383.978 [stream.cc:1516]164629 Synchronizelmpl:failed, stream_id=43, error=0x715005e +[ERROR] RUNTIME(164629,):2026-04-14-05:34:12.383.982 [api error.cc:1015]164629 StreamSynchronize:Stream synchronize failed, stream_id=43, timeout=-1ms. +[ERROR] RUNTIME(164629,):2026-04-14-05:34:12.383.991 [apic stream.cc:154]164629 rtStreamSynchronize:ErrCode=507035, desc=[vector core exception], InnerCode=0x715005e +[ERROR] RUNTIME(164629,):2026-04-14-05:34:12.383.997 [error_message_manage.cc:61]164629 FuncErrorReason:rtStreamSynchronize execution failed, reason=vector core exception +[ERROR] ASCENDCL(164629,):2026-04-14-05:34:12.384.269 [stream.cpp:140]164629 acIrtSynchronizeStreamImpl:synchronize stream failed, runtime result = 507035 +``` + +### Cause + +The `alloc_extend_kernel` operator appears to have a memory allocation issue. Pending resolution. + +### Solution + +Modify `sglang/srt/hardware_backend/npu/allocator_npu.py` to comment out the affected branch and use the else branch instead. + +```text highlight=15-36 + def alloc_extend( + self, + prefix_lens: torch.Tensor, + prefix_lens_cpu: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_cpu: torch.Tensor, + last_loc: torch.Tensor, + extend_num_tokens: int, + num_new_pages: int = None, + ): + ... + if num_new_pages_item > len(self.free_pages): + return None + + # if num_new_pages_item < 200: + # from sgl_kernel_npu.mem_cache.allocator import alloc_extend_kernel + + # out_indices = torch.empty( + # (extend_num_tokens,), + # dtype=torch.int64, + # device=self.device, + # ) + # max_num_extend_tokens = next_power_of_2(extend_num_tokens) + # bs = prefix_lens.shape[0] + # alloc_extend_kernel[(bs,)]( + # prefix_lens, + # seq_lens, + # last_loc, + # self.free_pages, + # out_indices, + # next_power_of_2(bs), + # self.page_size, + # max_num_extend_tokens, + # ) + + # else: + out_indices = torch.empty( + (extend_num_tokens,), + dtype=torch.int32, + device=self.device, + ) + ... +``` + +## 4. Out of NPU memory + +```text highlight=7 + File "/home/code/sglang/python/sglang/srt/model_executor/pool_configurator.py", line 175, in calculate_pool_sizes + return MemoryPoolConfig(max_total_num_tokens=max_total_num_tokens) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "", line 8, in __init__ + File "/home/code/sglang/python/sglang/srt/model_executor/pool_configurator.py", line 44, in __post_init__ + raise RuntimeError(msg) +RuntimeError: Not enough memory. Please try to increase --mem-fraction-static. +``` + +### Solution + +First, use the `npu-smi info` command to check the NPU memory usage. + +If the NPUs are occupied by other processes, use `--base-gpu-id` to specify the starting device index. + +If the NPUs are not occupied, you can use `--tp` to deploy across multiple devices, or reduce the KV cache memory usage by decreasing the `--mem-fraction-static` value. For detailed tuning guidance, see [Hyperparameter Tuning](/docs/advanced_features/hyperparameter_tuning). + +## 5. How to update sgl-kernel-npu + +### Solution +``` +git clone https://github.com/sgl-project/sgl-kernel-npu.git + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +cd sgl-kernel-npu +# Building Project +bash build.sh + +pip install output/sgl_kernel_npu*.whl --force-reinstall + +# (Optional) Confirm whether the import can be successfully +python -c "import sgl_kernel_npu; print(sgl_kernel_npu.__path__)" + +rm -rf sgl-kernel-npu +```