minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

test(npu): remove obsolete npu pr nightly cases, move accuracy cases to full (#37990)

Browse Source 
Co-authored-by: Sugar920 <Sugar920@users.noreply.github.com>
Co-authored-by: Claude Code <noreply@anthropic.com>
This commit is contained in:
Sugar920
2026-09-05 20:58:52 +08:00
committed by GitHub
co-authored by Sugar920 Claude Code
parent a18106bbc3
commit 4b802c052b
60 changed files with 283 additions and 3134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/_npu-pr-test-stage.yml
+19 -7
View File
@@ -186,8 +186,6 @@ jobs:
}
- name: Install dependencies
# Only PR jobs install dependencies
if: ${{ inputs.is_nightly_pipeline_job != true }}
env:
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
@@ -195,17 +193,31 @@ jobs:
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
run: |
# speed up by using infra cache services
set -euo pipefail
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
pip config set global.index-url http://${CACHING_URL}/pypi/simple
pip config set global.trusted-host "${CACHING_URL}"
if [[ "${{ inputs.is_nightly_pipeline_job }}" != "true" ]]; then
bash scripts/ci/npu/npu_ci_install_dependency.sh ${{ inputs.npu_device_type }}
# copy required file from our daily cache
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
# copy gsm8k dataset
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
else
sglang_pkg_path=/sgl-workspace/sglang/python
ascend_test_util_path=${sglang_pkg_path}/sglang/test/ascend
mkdir -p ${ascend_test_util_path}
mv ${ascend_test_util_path} ${ascend_test_util_path}_bak
cp -r $(pwd)/python/sglang/test/ascend ${ascend_test_util_path}
pip install sentence_transformers zss "wandb>=0.16.0" tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 "sacrebleu>=1.5.0" pytablewriter "peft>=0.2.0" "accelerate>=0.29.1"
pip install jsonlines "evaluate>=0.4.0" numexpr dotenv
hf download MMMU/MMMU --repo-type dataset
fi
# install sglang_router
apt-get install -y libssl-dev
.github/workflows/full-test-npu.yml
+234 -245
View File
@@ -18,16 +18,16 @@ on:
required: false
type: string
default: 'all'
image_a2:
description: 'The a2 running docker image of the test task.'
required: false
type: string
default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-910b'
image_a3:
description: 'The a3 running docker image of the test task.'
description: 'The a3 running docker image of the test task. Must be an image with sglang preinstalled (e.g. lmsysorg/sglang:*a3*).'
required: false
type: string
default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-a3-ubuntu22.04-py3.11'
skip_install_flag:
description: 'Indicates whether to skip the installation of sglang, defaulting to false.'
required: false
type: string
default: 'false'
default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3'
concurrency:
group: full-test-npu-${{ inputs.ref || github.ref }}
@@ -39,8 +39,8 @@ jobs:
outputs:
ref: ${{ steps.set-vars.outputs.ref }}
job_filter: ${{ steps.set-vars.outputs.job_filter }}
image_a2: ${{ steps.set-vars.outputs.image_a2 }}
image_a3: ${{ steps.set-vars.outputs.image_a3 }}
skip_install_flag: ${{ steps.set-vars.outputs.skip_install_flag }}
steps:
# When triggered by PR, no inputs parameters are used. The latest community code is tested by default.
- name: Set image config
@@ -58,16 +58,16 @@ jobs:
echo "job_filter=${{ inputs.job_filter }}" >> $GITHUB_OUTPUT
fi
if [ -z "${{ inputs.image_a3 }}" ]; then
echo "image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.5.0-a3-ubuntu22.04-py3.11" >> $GITHUB_OUTPUT
if [ -z "${{ inputs.image_a2 }}" ]; then
echo "image_a2=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-910b" >> $GITHUB_OUTPUT
else
echo "image_a3=${{ inputs.image_a3 }}" >> $GITHUB_OUTPUT
echo "image_a2=${{ inputs.image_a2 }}" >> $GITHUB_OUTPUT
fi
if [ -z "${{ inputs.skip_install_flag }}" ]; then
echo "skip_install_flag=false" >> $GITHUB_OUTPUT
if [ -z "${{ inputs.image_a3 }}" ]; then
echo "image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3" >> $GITHUB_OUTPUT
else
echo "skip_install_flag=${{ inputs.skip_install_flag }}" >> $GITHUB_OUTPUT
echo "image_a3=${{ inputs.image_a3 }}" >> $GITHUB_OUTPUT
fi
nighly-test-npu:
@@ -79,269 +79,258 @@ jobs:
ref: ${{ needs.set-image-config.outputs.ref }}
job_filter: ${{ needs.set-image-config.outputs.job_filter }}
image_a3: ${{ needs.set-image-config.outputs.image_a3 }}
skip_install_flag: ${{ needs.set-image-config.outputs.skip_install_flag }}
secrets: inherit
full-1-npu-a3:
needs: [set-image-config]
full-4-npu-a2:
name: full-4-npu-a2
if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
runs-on: linux-aarch64-a3-2
container:
image: ${{ needs.set-image-config.outputs.image_a3 }}
steps:
- name: Checkout code
uses: actions/checkout@v4
needs: [set-image-config]
uses: ./.github/workflows/_npu-single-node-test-stage.yml
with:
ref: ${{ needs.set-image-config.outputs.ref || github.ref }}
runner: linux-aarch64-a2-4
test_type: 'perf'
test_suite: full-4-npu-a2
is_nightly_pipeline_job: true
skip_pr_test_health_check: 'true'
image: ${{ needs.set-image-config.outputs.image_a2 }}
device_type_for_deps: '910b'
- name: Install dependencies
env:
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
run: |
# speed up by using infra cache services
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
pip config set global.index-url http://${CACHING_URL}/pypi/simple
pip config set global.trusted-host "${CACHING_URL}"
if [ ${{ needs.set-image-config.outputs.skip_install_flag }} != "true" ];then
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
fi
# copy required file from our daily cache
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
# copy gsm8k dataset
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
- name: Print Log Information
run: |
bash scripts/ci/npu/npu_log_print.sh
- name: Run test
timeout-minutes: 240
env:
SGLANG_USE_MODELSCOPE: true
SGLANG_IS_IN_CI: true
HF_ENDPOINT: https://hf-mirror.com
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
STREAMS_PER_DEVICE: 32
run: |
pip install sglang_router
hf download lmms-lab/MMMU --repo-type dataset
pip install sentence_transformers torchaudio==2.8.0
pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
cd ./lmms-eval
nohup pip install . > lmmslog.txt 2>&1 &
sleep 120
export PYTHONPATH=$PYTHONPATH:$(pwd)
cd ../
cd test
python3 run_suite.py --hw npu --suite full-1-npu-a3 --nightly --continue-on-error --timeout-per-file 3600
full-1-npu-a3:
name: full-1-npu-a3
if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
needs: [set-image-config]
uses: ./.github/workflows/_npu-pr-test-stage.yml
with:
self_name: full-1-npu-a3
runner_config: linux-aarch64-a3-2
image: ${{ needs.set-image-config.outputs.image_a3 }}
run_timeout_minutes: '240'
timeout_per_file: '3600'
ref: ${{ needs.set-image-config.outputs.ref }}
is_nightly_pipeline_job: true
skip_pr_test_health_check: 'true'
secrets: inherit
full-2-npu-a3:
needs: [set-image-config]
name: full-2-npu-a3
if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
runs-on: linux-aarch64-a3-2
container:
image: ${{ needs.set-image-config.outputs.image_a3 }}
steps:
- name: Checkout code
uses: actions/checkout@v4
needs: [set-image-config]
uses: ./.github/workflows/_npu-pr-test-stage.yml
with:
ref: ${{ needs.set-image-config.outputs.ref || github.ref }}
- name: Install dependencies
env:
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
run: |
# speed up by using infra cache services
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
pip config set global.index-url http://${CACHING_URL}/pypi/simple
pip config set global.trusted-host "${CACHING_URL}"
if [ ${{ needs.set-image-config.outputs.skip_install_flag }} != "true" ];then
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
fi
# copy required file from our daily cache
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
# copy gsm8k dataset
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
- name: Print Log Information
run: |
bash scripts/ci/npu/npu_log_print.sh
- name: Run test
timeout-minutes: 240
env:
SGLANG_USE_MODELSCOPE: true
SGLANG_IS_IN_CI: true
HF_ENDPOINT: https://hf-mirror.com
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
STREAMS_PER_DEVICE: 32
run: |
pip install sglang_router
hf download lmms-lab/MMMU --repo-type dataset
pip install sentence_transformers torchaudio==2.8.0
pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
cd ./lmms-eval
nohup pip install . > lmmslog.txt 2>&1 &
sleep 120
export PYTHONPATH=$PYTHONPATH:$(pwd)
cd ../
cd test
python3 run_suite.py --hw npu --suite full-2-npu-a3 --nightly --continue-on-error --timeout-per-file 3600
self_name: full-2-npu-a3
runner_config: linux-aarch64-a3-2
image: ${{ needs.set-image-config.outputs.image_a3 }}
run_timeout_minutes: '240'
timeout_per_file: '3600'
ref: ${{ needs.set-image-config.outputs.ref }}
is_nightly_pipeline_job: true
skip_pr_test_health_check: 'true'
secrets: inherit
full-4-npu-a3:
needs: [set-image-config]
name: full-4-npu-a3
if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
runs-on: linux-aarch64-a3-4
container:
image: ${{ needs.set-image-config.outputs.image_a3 }}
steps:
- name: Checkout code
uses: actions/checkout@v4
needs: [set-image-config]
uses: ./.github/workflows/_npu-pr-test-stage.yml
with:
ref: ${{ needs.set-image-config.outputs.ref || github.ref }}
self_name: full-4-npu-a3
runner_config: linux-aarch64-a3-4
image: ${{ needs.set-image-config.outputs.image_a3 }}
run_timeout_minutes: '240'
timeout_per_file: '3600'
ref: ${{ needs.set-image-config.outputs.ref }}
is_nightly_pipeline_job: true
skip_pr_test_health_check: 'true'
secrets: inherit
- name: Install dependencies
env:
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
run: |
# speed up by using infra cache services
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
pip config set global.index-url http://${CACHING_URL}/pypi/simple
pip config set global.trusted-host "${CACHING_URL}"
if [ ${{ needs.set-image-config.outputs.skip_install_flag }} != "true" ];then
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
fi
# copy required file from our daily cache
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
# copy gsm8k dataset
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
- name: Print Log Information
run: |
bash scripts/ci/npu/npu_log_print.sh
- name: Run test
timeout-minutes: 240
env:
SGLANG_USE_MODELSCOPE: true
SGLANG_IS_IN_CI: true
HF_ENDPOINT: https://hf-mirror.com
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
STREAMS_PER_DEVICE: 32
run: |
pip install sglang_router
hf download lmms-lab/MMMU --repo-type dataset
pip install sentence_transformers torchaudio==2.8.0
pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
cd ./lmms-eval
nohup pip install . > lmmslog.txt 2>&1 &
sleep 120
export PYTHONPATH=$PYTHONPATH:$(pwd)
cd ../
cd test
python3 run_suite.py --hw npu --suite full-4-npu-a3 --nightly --continue-on-error --timeout-per-file 3600
full-8-npu-a3:
name: full-8-npu-a3
if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
needs: [set-image-config]
uses: ./.github/workflows/_npu-pr-test-stage.yml
with:
self_name: full-8-npu-a3
runner_config: linux-aarch64-a3-8
image: ${{ needs.set-image-config.outputs.image_a3 }}
run_timeout_minutes: '240'
timeout_per_file: '3600'
ref: ${{ needs.set-image-config.outputs.ref }}
is_nightly_pipeline_job: true
skip_pr_test_health_check: 'true'
secrets: inherit
full-16-npu-a3:
needs: [set-image-config]
name: full-16-npu-a3
if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
runs-on: linux-aarch64-a3-16
container:
image: ${{ needs.set-image-config.outputs.image_a3 }}
steps:
- name: Checkout code
uses: actions/checkout@v4
needs: [set-image-config]
uses: ./.github/workflows/_npu-pr-test-stage.yml
with:
ref: ${{ needs.set-image-config.outputs.ref || github.ref }}
self_name: full-16-npu-a3
runner_config: linux-aarch64-a3-16
image: ${{ needs.set-image-config.outputs.image_a3 }}
run_timeout_minutes: '240'
timeout_per_file: '3600'
ref: ${{ needs.set-image-config.outputs.ref }}
is_nightly_pipeline_job: true
skip_pr_test_health_check: 'true'
secrets: inherit
- name: Install dependencies
env:
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
run: |
# speed up by using infra cache services
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
pip config set global.index-url http://${CACHING_URL}/pypi/simple
pip config set global.trusted-host "${CACHING_URL}"
full-acc-2-npu-a3:
name: full-acc-2-npu-a3
if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
needs: [set-image-config]
uses: ./.github/workflows/_npu-single-node-test-stage.yml
with:
runner: linux-aarch64-a3-2-
test_type: 'accuracy'
test_suite: full-acc-2-npu-a3
image: ${{ needs.set-image-config.outputs.image_a3 }}
is_nightly_pipeline_job: true
install_sglang_deps: true
device_type_for_deps: 'a3'
skip_pr_test_health_check: 'true'
secrets: inherit
if [ ${{ needs.set-image-config.outputs.skip_install_flag }} != "true" ];then
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
fi
full-acc-4-npu-a3:
name: full-acc-4-npu-a3
if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
needs: [set-image-config]
uses: ./.github/workflows/_npu-single-node-test-stage.yml
with:
runner: linux-aarch64-a3-4-
test_type: 'accuracy'
test_suite: full-acc-4-npu-a3
image: ${{ needs.set-image-config.outputs.image_a3 }}
is_nightly_pipeline_job: true
install_sglang_deps: true
device_type_for_deps: 'a3'
skip_pr_test_health_check: 'true'
secrets: inherit
# copy required file from our daily cache
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
# copy gsm8k dataset
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
full-acc-16-npu-a3:
name: full-acc-16-npu-a3
if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
needs: [set-image-config]
uses: ./.github/workflows/_npu-single-node-test-stage.yml
with:
runner: linux-aarch64-a3-16-
test_type: 'accuracy'
test_suite: full-acc-16-npu-a3
image: ${{ needs.set-image-config.outputs.image_a3 }}
is_nightly_pipeline_job: true
install_sglang_deps: true
device_type_for_deps: 'a3'
skip_pr_test_health_check: 'true'
secrets: inherit
- name: Print Log Information
run: |
bash scripts/ci/npu/npu_log_print.sh
full-perf-2-npu-a3:
name: full-perf-2-npu-a3
if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
needs: [set-image-config]
uses: ./.github/workflows/_npu-pr-test-stage.yml
with:
self_name: full-perf-2-npu-a3
runner_config: linux-aarch64-a3-800t-2
image: ${{ needs.set-image-config.outputs.image_a3 }}
run_timeout_minutes: '240'
timeout_per_file: '3600'
ref: ${{ needs.set-image-config.outputs.ref }}
is_nightly_pipeline_job: true
skip_pr_test_health_check: 'true'
secrets: inherit
- name: Run test
timeout-minutes: 240
env:
SGLANG_USE_MODELSCOPE: true
SGLANG_IS_IN_CI: true
HF_ENDPOINT: https://hf-mirror.com
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
STREAMS_PER_DEVICE: 32
run: |
pip install sglang_router
hf download lmms-lab/MMMU --repo-type dataset
pip install sentence_transformers torchaudio==2.8.0
pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
cd ./lmms-eval
nohup pip install . > lmmslog.txt 2>&1 &
sleep 120
export PYTHONPATH=$PYTHONPATH:$(pwd)
cd ../
cd test
python3 run_suite.py --hw npu --suite full-16-npu-a3 --nightly --continue-on-error --timeout-per-file 3600
full-perf-4-npu-a3:
name: full-perf-4-npu-a3
if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
needs: [set-image-config]
uses: ./.github/workflows/_npu-pr-test-stage.yml
with:
self_name: full-perf-4-npu-a3
runner_config: linux-aarch64-a3-800t-4
image: ${{ needs.set-image-config.outputs.image_a3 }}
run_timeout_minutes: '240'
timeout_per_file: '3600'
ref: ${{ needs.set-image-config.outputs.ref }}
is_nightly_pipeline_job: true
skip_pr_test_health_check: 'true'
secrets: inherit
full-perf-8-npu-a3:
name: full-perf-8-npu-a3
if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
needs: [set-image-config]
uses: ./.github/workflows/_npu-pr-test-stage.yml
with:
self_name: full-perf-8-npu-a3
runner_config: linux-aarch64-a3-800t-8
image: ${{ needs.set-image-config.outputs.image_a3 }}
run_timeout_minutes: '240'
timeout_per_file: '3600'
ref: ${{ needs.set-image-config.outputs.ref }}
is_nightly_pipeline_job: true
skip_pr_test_health_check: 'true'
secrets: inherit
full-perf-16-npu-a3:
name: full-perf-16-npu-a3
if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
needs: [set-image-config]
uses: ./.github/workflows/_npu-pr-test-stage.yml
with:
self_name: full-perf-16-npu-a3
runner_config: linux-aarch64-a3-800t-16
image: ${{ needs.set-image-config.outputs.image_a3 }}
run_timeout_minutes: '240'
timeout_per_file: '3600'
ref: ${{ needs.set-image-config.outputs.ref }}
is_nightly_pipeline_job: true
skip_pr_test_health_check: 'true'
secrets: inherit
full-poc-multi-node-mix-tests:
name: multi-node-mix-poc
if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
needs: [set-image-config]
strategy:
fail-fast: false
max-parallel: 1
matrix:
test_config:
# kimi_k2_6 performance tests
- name: kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25
node_size: 2
test_case: test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py
test_type: 'accuracy'
uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml
with:
runner: linux-amd64-cpu-4
test_type: ${{ matrix.test_config.test_type || 'perf' }}
test_config_name: ${{ matrix.test_config.name }}
node_size: ${{ matrix.test_config.node_size }}
test_case: ${{ matrix.test_config.test_case }}
image: ${{ needs.set-image-config.outputs.image_a3 }}
install_sglang_from_source: false
prefill_decode_deployment: 'mix'
transformers_version: ''
check-all-jobs:
if: github.repository == 'sgl-project/sglang' && always()
needs:
- nighly-test-npu
- full-4-npu-a2
- full-1-npu-a3
- full-2-npu-a3
- full-4-npu-a3
- full-8-npu-a3
- full-16-npu-a3
- full-acc-2-npu-a3
- full-acc-4-npu-a3
- full-acc-16-npu-a3
- full-perf-2-npu-a3
- full-perf-4-npu-a3
- full-perf-8-npu-a3
- full-perf-16-npu-a3
- full-poc-multi-node-mix-tests
runs-on: ubuntu-latest
container:
image: docker.m.daocloud.io/ubuntu:22.04
.github/workflows/nightly-test-npu.yml
+3 -84
View File
@@ -122,22 +122,6 @@ jobs:
# Write to GITHUB_OUTPUT and print to the log in one command.
echo "run_start_metadata=${RUN_START_METADATA}" | tee -a $GITHUB_OUTPUT
nightly-1-npu-a2:
name: nightly-1-npu-a2
if: ${{ !cancelled() }}
needs: [set-image-config]
uses: ./.github/workflows/_npu-single-node-test-stage.yml
with:
runner: linux-aarch64-a2-4
test_type: 'perf'
test_suite: nightly-1-npu-a2
is_nightly_pipeline_job: true
skip_pr_test_health_check: 'true'
image: ${{ needs.set-image-config.outputs.image_a2 }}
install_sglang_deps: false
device_type_for_deps: '910b'
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
nightly-1-npu-a3:
name: nightly-1-npu-a3
if: ${{ !cancelled() }}
@@ -247,22 +231,6 @@ jobs:
device_type_for_deps: 'a3'
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
nightly-perf-8-npu-a3:
name: nightly-perf-8-npu-a3
if: ${{ !cancelled() }}
needs: [set-image-config]
uses: ./.github/workflows/_npu-single-node-test-stage.yml
with:
runner: linux-aarch64-a3-800t-8
test_type: 'perf'
test_suite: nightly-perf-8-npu-a3
is_nightly_pipeline_job: true
skip_pr_test_health_check: 'true'
image: ${{ needs.set-image-config.outputs.image_a3 }}
install_sglang_deps: false
device_type_for_deps: 'a3'
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
nightly-perf-16-npu-a3:
name: nightly-perf-16-npu-a3
if: ${{ !cancelled() }}
@@ -298,22 +266,6 @@ jobs:
device_type_for_deps: 'a3'
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
nightly-acc-4-npu-a3:
name: nightly-acc-4-npu-a3
if: ${{ !cancelled() }}
needs: [set-image-config]
uses: ./.github/workflows/_npu-single-node-test-stage.yml
with:
runner: linux-aarch64-a3-4-
test_type: 'accuracy'
test_suite: nightly-acc-4-npu-a3
is_nightly_pipeline_job: true
skip_pr_test_health_check: 'true'
image: ${{ needs.set-image-config.outputs.image_a3 }}
install_sglang_deps: false
device_type_for_deps: 'a3'
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
nightly-acc-16-npu-a3:
name: nightly-acc-16-npu-a3
if: ${{ !cancelled() }}
@@ -333,7 +285,7 @@ jobs:
nightly-poc-multi-node-tests:
name: multi-node-poc
if: ${{ !cancelled() }}
needs: [set-image-config, nightly-perf-2-npu-a3, nightly-perf-4-npu-a3, nightly-perf-16-npu-a3, nightly-acc-2-npu-a3, nightly-acc-4-npu-a3, nightly-acc-16-npu-a3]
needs: [set-image-config, nightly-perf-2-npu-a3, nightly-perf-4-npu-a3, nightly-perf-16-npu-a3, nightly-acc-2-npu-a3, nightly-acc-16-npu-a3]
strategy:
fail-fast: false
max-parallel: 1
@@ -354,21 +306,6 @@ jobs:
test_case: test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py
test_type: 'accuracy'
prefill_decode_deployment: 'separation'
# mimo_v2_flash performance tests
- name: test_npu_mimo_v2_flash_1p1d_12p_in16k_out1_ttft_5s
prefill_size: 1
decode_size: 1
router_size: 1
test_case: test/registered/npu/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1_ttft_5s.py
test_type: 'perf'
prefill_decode_deployment: 'separation'
- name: test_npu_mimo_v2_flash_1p1d_12p_in16k_out1k_tpot_20ms
prefill_size: 1
decode_size: 1
router_size: 1
test_case: test/registered/npu/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1k_tpot_20ms.py
test_type: 'perf'
prefill_decode_deployment: 'separation'
# deepseek_v4_flash performance tests
- name: deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms
prefill_size: 1
@@ -395,21 +332,12 @@ jobs:
nightly-poc-multi-node-mix-tests:
name: multi-node-mix-poc
if: ${{ !cancelled() }}
needs: [set-image-config, nightly-perf-2-npu-a3, nightly-perf-4-npu-a3, nightly-perf-16-npu-a3, nightly-acc-2-npu-a3, nightly-acc-4-npu-a3, nightly-acc-16-npu-a3, nightly-poc-multi-node-tests]
needs: [set-image-config, nightly-perf-2-npu-a3, nightly-perf-4-npu-a3, nightly-perf-16-npu-a3, nightly-acc-2-npu-a3, nightly-acc-16-npu-a3, nightly-poc-multi-node-tests]
strategy:
fail-fast: false
max-parallel: 1
matrix:
test_config:
# kimi_k2_6 performance tests
- name: kimi_k2_6_w4a8_16p_in64k_out1k_100ms
node_size: 2
test_case: test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms.py
test_type: 'perf'
- name: kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25
node_size: 2
test_case: test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py
test_type: 'accuracy'
# glm_5_2 accuracy tests
- name: glm_5_2_w4a8_16p_gpqa
node_size: 2
@@ -436,7 +364,6 @@ jobs:
check-all-jobs:
if: ${{ !cancelled() }}
needs:
- nightly-1-npu-a2
- nightly-1-npu-a3
- nightly-2-npu-a3
- nightly-4-npu-a3
@@ -444,10 +371,8 @@ jobs:
- nightly-16-npu-a3
- nightly-perf-2-npu-a3
- nightly-perf-4-npu-a3
- nightly-perf-8-npu-a3
- nightly-perf-16-npu-a3
- nightly-acc-2-npu-a3
- nightly-acc-4-npu-a3
- nightly-acc-16-npu-a3
- nightly-poc-multi-node-tests
- nightly-poc-multi-node-mix-tests
@@ -455,7 +380,6 @@ jobs:
steps:
- name: Generate results table
run: |
single_result_a2="${{ needs.nightly-1-npu-a2.result }}"
multi_result="${{ needs.nightly-poc-multi-node-tests.result }}"
mix_result="${{ needs.nightly-poc-multi-node-mix-tests.result }}"
@@ -469,10 +393,8 @@ jobs:
"${{ needs.nightly-16-npu-a3.result }}" \
"${{ needs.nightly-perf-2-npu-a3.result }}" \
"${{ needs.nightly-perf-4-npu-a3.result }}" \
"${{ needs.nightly-perf-8-npu-a3.result }}" \
"${{ needs.nightly-perf-16-npu-a3.result }}" \
"${{ needs.nightly-acc-2-npu-a3.result }}" \
"${{ needs.nightly-acc-4-npu-a3.result }}" \
"${{ needs.nightly-acc-16-npu-a3.result }}"; do
if [ "${r}" != "success" ] && [ "${r}" != "skipped" ]; then
single_result="failure"
@@ -493,7 +415,6 @@ jobs:
echo "" >> $GITHUB_STEP_SUMMARY
echo "| Group | Status |" >> $GITHUB_STEP_SUMMARY
echo "|-------|--------|" >> $GITHUB_STEP_SUMMARY
echo "| nightly-1-npu-a2 | $(group_icon ${single_result_a2}) ${single_result_a2} |" >> $GITHUB_STEP_SUMMARY
for entry in \
"nightly-1-npu-a3:${{ needs.nightly-1-npu-a3.result }}" \
"nightly-2-npu-a3:${{ needs.nightly-2-npu-a3.result }}" \
@@ -502,10 +423,8 @@ jobs:
"nightly-16-npu-a3:${{ needs.nightly-16-npu-a3.result }}" \
"nightly-perf-2-npu-a3:${{ needs.nightly-perf-2-npu-a3.result }}" \
"nightly-perf-4-npu-a3:${{ needs.nightly-perf-4-npu-a3.result }}" \
"nightly-perf-8-npu-a3:${{ needs.nightly-perf-8-npu-a3.result }}" \
"nightly-perf-16-npu-a3:${{ needs.nightly-perf-16-npu-a3.result }}" \
"nightly-acc-2-npu-a3:${{ needs.nightly-acc-2-npu-a3.result }}" \
"nightly-acc-4-npu-a3:${{ needs.nightly-acc-4-npu-a3.result }}" \
"nightly-acc-16-npu-a3:${{ needs.nightly-acc-16-npu-a3.result }}"; do
suite="${entry%%:*}"
r="${entry##*:}"
@@ -516,7 +435,7 @@ jobs:
echo "" >> $GITHUB_STEP_SUMMARY
FAIL=0
for r in "${single_result_a2}" "${single_result}" "${multi_result}" "${mix_result}"; do
for r in "${single_result}" "${multi_result}" "${mix_result}"; do
if [ "${r}" != "success" ] && [ "${r}" != "skipped" ]; then FAIL=1; fi
done
exit $FAIL
test/registered/npu/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py
+1 -1
View File
@@ -8,7 +8,7 @@ from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=4800,
suite="nightly-acc-16-npu-a3",
suite="full-acc-16-npu-a3",
nightly=True,
)
test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py
+1 -1
View File
@@ -8,7 +8,7 @@ from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=6500,
suite="nightly-acc-2-npu-a3",
suite="full-acc-2-npu-a3",
nightly=True,
)
test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_gsm8k.py
+1 -1
View File
@@ -6,7 +6,7 @@ from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
from sglang.test.ascend.e2e.test_npu_performance_utils import GLM_4_7_FLASH_MODEL_PATH
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=6500, suite="nightly-acc-2-npu-a3", nightly=True)
register_npu_ci(est_time=6500, suite="full-acc-2-npu-a3", nightly=True)
ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py
+1 -1
View File
@@ -12,7 +12,7 @@ from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=7200,
suite="nightly-acc-16-npu-a3",
suite="full-acc-16-npu-a3",
nightly=True,
)
test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py
+1 -1
View File
@@ -12,7 +12,7 @@ from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=4800,
suite="nightly-acc-16-npu-a3",
suite="full-acc-16-npu-a3",
nightly=True,
)
test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py
+1 -1
View File
@@ -11,7 +11,7 @@ from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3700,
suite="nightly-acc-2-npu-a3",
suite="full-acc-2-npu-a3",
nightly=True,
)
test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py
+1 -1
View File
@@ -11,7 +11,7 @@ from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3700,
suite="nightly-acc-2-npu-a3",
suite="full-acc-2-npu-a3",
nightly=True,
)
test/registered/npu/accuracy/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py
+1 -1
View File
@@ -11,7 +11,7 @@ from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=2800,
suite="nightly-acc-2-npu-a3",
suite="full-acc-2-npu-a3",
nightly=True,
)
test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py
+1 -1
View File
@@ -11,7 +11,7 @@ from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=4800,
suite="nightly-acc-16-npu-a3",
suite="full-acc-16-npu-a3",
nightly=True,
)
test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py
+1 -1
View File
@@ -11,7 +11,7 @@ from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=4800,
suite="nightly-acc-4-npu-a3",
suite="full-acc-4-npu-a3",
nightly=True,
)
test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa_a2.py
+1 -1
View File
@@ -11,7 +11,7 @@ from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=4800,
suite="nightly-1-npu-a2",
suite="full-4-npu-a2",
nightly=True,
)
test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py
+1 -1
View File
@@ -8,7 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=2800, suite="nightly-acc-2-npu-a3", nightly=True)
register_npu_ci(est_time=2800, suite="full-acc-2-npu-a3", nightly=True)
QWEN3_5_9B_ENVS = {
"SGLANG_SET_CPU_AFFINITY": "1",
test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py
+1 -1
View File
@@ -11,7 +11,7 @@ from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=4800,
suite="nightly-acc-4-npu-a3",
suite="full-acc-4-npu-a3",
nightly=True,
)
test/registered/npu/accuracy/qwen3_vl_30b_a3b/test_npu_qwen3_vl_30b_a3b_bf16_2p_gsm8k.py
+1 -1
View File
@@ -8,7 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=4800, suite="nightly-acc-4-npu-a3", nightly=True)
register_npu_ci(est_time=4800, suite="full-acc-4-npu-a3", nightly=True)
QWEN3_VL_30B_A3B_ENVS = {
"SGLANG_SET_CPU_AFFINITY": "1",
test/registered/npu/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py
+1 -1
View File
@@ -10,7 +10,7 @@ from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=8400,
suite="nightly-acc-2-npu-a3",
suite="full-acc-2-npu-a3",
nightly=True,
)
test/registered/npu/accuracy/qwen3_vl_8b/test_npu_qwen3_vl_8b_bf16_2p_gsm8k.py
+1 -1
View File
@@ -8,7 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=4800, suite="nightly-acc-4-npu-a3", nightly=True)
register_npu_ci(est_time=4800, suite="full-acc-4-npu-a3", nightly=True)
QWEN3_VL_8B_ENVS = {
"SGLANG_SET_CPU_AFFINITY": "1",
test/registered/npu/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py
+1 -1
View File
@@ -9,7 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=12000, suite="nightly-acc-2-npu-a3", nightly=True)
register_npu_ci(est_time=12000, suite="full-acc-2-npu-a3", nightly=True)
_is_pr_pipeline = os.environ.get("GITHUB_EVENT_NAME") == "pull_request"
test/registered/npu/embedding_models/test_npu_bge_large_en_v1_5.py
-111
View File
@@ -1,111 +0,0 @@
import multiprocessing as mp
import unittest
from typing import Optional
import torch
from transformers import AutoConfig, AutoTokenizer
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.runners import HFRunner, SRTRunner
from sglang.test.test_utils import CustomTestCase, get_similarities
register_npu_ci(
est_time=400,
suite="full-1-npu-a3",
nightly=True,
disabled="embeddings are not all close",
)
DEFAULT_PROMPTS = [
"The capital of the United Kingdom is",
"Today is a sunny day and I like",
"AI is a field of computer science focused on",
]
MODELS = [
("/root/.cache/modelscope/hub/models/bge-large-en-v1.5", 1, 1e-5),
]
TORCH_DTYPES = [torch.float16]
class TestEmbeddingModels(CustomTestCase):
@classmethod
def setUpClass(cls):
mp.set_start_method("spawn", force=True)
def _truncate_prompts(self, prompts, model_path):
config = AutoConfig.from_pretrained(model_path)
max_length = getattr(config, "max_position_embeddings", 2048)
tokenizer = AutoTokenizer.from_pretrained(model_path)
truncated_prompts = []
for prompt in prompts:
tokens = tokenizer(prompt, return_tensors="pt", truncation=False)
if len(tokens.input_ids[0]) > max_length:
truncated_text = tokenizer.decode(
tokens.input_ids[0][: max_length - 1], skip_special_tokens=True
)
truncated_prompts.append(truncated_text)
else:
truncated_prompts.append(prompt)
return truncated_prompts
def assert_close_prefill_logits(
self,
prompts,
model_path,
tp_size,
torch_dtype,
prefill_tolerance,
matryoshka_dim: Optional[int] = None,
) -> None:
truncated_prompts = self._truncate_prompts(prompts, model_path)
with HFRunner(
model_path,
torch_dtype=torch_dtype,
model_type="embedding",
matryoshka_dim=matryoshka_dim,
) as hf_runner:
hf_outputs = hf_runner.forward(truncated_prompts)
attention_backend = "ascend"
with SRTRunner(
model_path,
tp_size=tp_size,
torch_dtype=torch_dtype,
model_type="embedding",
attention_backend=attention_backend,
json_model_override_args=(
{"matryoshka_dimensions": [matryoshka_dim]} if matryoshka_dim else None
),
) as srt_runner:
srt_outputs = srt_runner.forward(
truncated_prompts, dimensions=matryoshka_dim
)
for i in range(len(prompts)):
hf_logits = torch.Tensor(hf_outputs.embed_logits[i])
srt_logits = torch.Tensor(srt_outputs.embed_logits[i])
similarity = torch.tensor(get_similarities(hf_logits, srt_logits))
print("similarity diff", abs(similarity - 1))
if len(prompts[i]) <= 1000:
assert torch.all(abs(similarity - 1) < prefill_tolerance), (
"embeddings are not all close"
)
def test_prefill_logits(self):
models_to_test = MODELS
for model, tp_size, prefill_tolerance in models_to_test:
for torch_dtype in TORCH_DTYPES:
self.assert_close_prefill_logits(
DEFAULT_PROMPTS, model, tp_size, torch_dtype, prefill_tolerance
)
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_ascend_dbrx_instruct.py
-26
View File
@@ -1,26 +0,0 @@
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=400, suite="full-8-npu-a3", nightly=True)
class TestDbrx(GSM8KAscendMixin, CustomTestCase):
model = "/root/.cache/modelscope/hub/models/AI-ModelScope/dbrx-instruct"
accuracy = 0.735
other_args = [
"--trust-remote-code",
"--mem-fraction-static",
"0.8",
"--attention-backend",
"ascend",
"--disable-cuda-graph",
"--tp-size",
"8",
]
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_baichuan2_13b_chat.py
-37
View File
@@ -1,37 +0,0 @@
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import BAICHUAN2_13B_CHAT_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)
class TestBaichuan(GSM8KAscendMixin, CustomTestCase):
"""Testcase: Verify that the inference accuracy of the baichuan-inc/Baichuan2-13B-Chat model on the GSM8K dataset is no less than 0.48.
[Test Category] Model
[Test Target] baichuan-inc/Baichuan2-13B-Chat
"""
model = BAICHUAN2_13B_CHAT_WEIGHTS_PATH
accuracy = 0.48
other_args = [
"--trust-remote-code",
"--mem-fraction-static",
"0.8",
"--attention-backend",
"ascend",
"--disable-cuda-graph",
"--max-running-requests",
"128",
"--disable-radix-cache",
"--chunked-prefill-size",
"-1",
]
gsm8k_num_shots = 1
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_c4ai_command_r_v01.py
-40
View File
@@ -1,40 +0,0 @@
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import (
C4AI_COMMAND_R_V01_CHAT_TEMPLATE_PATH,
C4AI_COMMAND_R_V01_WEIGHTS_PATH,
)
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=400, suite="full-2-npu-a3", nightly=True)
class TestC4AI(GSM8KAscendMixin, CustomTestCase):
"""Testcase: Verify that the inference accuracy of the CohereForAI/c4ai-command-r-v01 model on the GSM8K dataset is no less than 0.55.
[Test Category] Model
[Test Target] CohereForAI/c4ai-command-r-v01
"""
model = C4AI_COMMAND_R_V01_WEIGHTS_PATH
accuracy = 0.55
other_args = [
"--trust-remote-code",
"--mem-fraction-static",
"0.8",
"--attention-backend",
"ascend",
"--disable-cuda-graph",
"--chat-template",
C4AI_COMMAND_R_V01_CHAT_TEMPLATE_PATH,
"--tp-size",
"2",
"--dtype",
"bfloat16",
]
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_chatglm2_6b.py
-33
View File
@@ -1,33 +0,0 @@
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import CHATGLM2_6B_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)
class TestChatGlm2(GSM8KAscendMixin, CustomTestCase):
"""Testcase: Verify that the inference accuracy of the ZhipuAI/chatglm2-6b model on the GSM8K dataset is no less than 0.25.
[Test Category] Model
[Test Target] ZhipuAI/chatglm2-6b
"""
model = CHATGLM2_6B_WEIGHTS_PATH
accuracy = 0.25
other_args = [
"--trust-remote-code",
"--mem-fraction-static",
"0.8",
"--attention-backend",
"ascend",
"--disable-cuda-graph",
"--dtype",
"bfloat16",
]
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_exaone_3.py
-34
View File
@@ -1,34 +0,0 @@
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import EXAONE_3_5_7_8B_INSTRUCT_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)
class TestEXAONE(GSM8KAscendMixin, CustomTestCase):
"""Testcase: Verify that the inference accuracy of the LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct model on the GSM8K dataset is no less than 0.8.
[Test Category] Model
[Test Target] LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct
"""
model = EXAONE_3_5_7_8B_INSTRUCT_WEIGHTS_PATH
# Allow 1% tolerance for the accuracy threshold
accuracy = round(0.8 * 0.99, 3)
other_args = [
"--trust-remote-code",
"--mem-fraction-static",
"0.8",
"--attention-backend",
"ascend",
"--disable-cuda-graph",
"--dtype",
"bfloat16",
]
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_glm4_9b_chat.py
-27
View File
@@ -1,27 +0,0 @@
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import GLM_4_9B_CHAT_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(
est_time=400,
suite="full-1-npu-a3",
nightly=True,
)
class TestGLM49BChat(GSM8KAscendMixin, CustomTestCase):
"""Testcase: Verify that the inference accuracy of the ZhipuAI/glm-4-9b-chat model on the GSM8K dataset is no less than 0.77.
[Test Category] Model
[Test Target] ZhipuAI/glm-4-9b-chat
"""
model = GLM_4_9B_CHAT_WEIGHTS_PATH
accuracy = 0.77
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_granite_3_0_3b_a800m.py
-25
View File
@@ -1,25 +0,0 @@
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import (
GRANITE_3_0_3B_A800M_INSTRUCT_WEIGHTS_PATH,
)
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)
class TestGranite(GSM8KAscendMixin, CustomTestCase):
"""Testcase: Verify that the inference accuracy of the ibm-granite/granite-3.0-3b-a800m-instruct model on the GSM8K dataset is no less than 0.38.
[Test Category] Model
[Test Target] ibm-granite/granite-3.0-3b-a800m-instruct
"""
model = GRANITE_3_0_3B_A800M_INSTRUCT_WEIGHTS_PATH
accuracy = 0.38
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_granite_3_1_8b.py
-23
View File
@@ -1,23 +0,0 @@
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import GRANITE_3_1_8B_INSTRUCT_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)
class TestGranite(GSM8KAscendMixin, CustomTestCase):
"""Testcase: Verify that the inference accuracy of the ibm-granite/granite-3.1-8b-instruct model on the GSM8K dataset is no less than 0.695.
[Test Category] Model
[Test Target] ibm-granite/granite-3.1-8b-instruct
"""
model = GRANITE_3_1_8B_INSTRUCT_WEIGHTS_PATH
accuracy = 0.695
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_grok_2.py
-33
View File
@@ -1,33 +0,0 @@
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(
est_time=400,
suite="full-16-npu-a3",
nightly=True,
)
class TestGrok2(GSM8KAscendMixin, CustomTestCase):
model = "/root/.cache/modelscope/hub/models/huihui-ai/grok-2"
accuracy = 0.91
other_args = [
"--trust-remote-code",
"--mem-fraction-static",
"0.8",
"--attention-backend",
"ascend",
"--disable-radix-cache",
"--disable-cuda-graph",
"--tokenizer-path",
"/root/.cache/modelscope/hub/models/huihui-ai/grok-2/tokenizer.tok.json",
"--tp-size",
"16",
]
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_internlm2_7b.py
-29
View File
@@ -1,29 +0,0 @@
import os
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import INTERNLM2_7B_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(
est_time=400,
suite="full-1-npu-a3",
nightly=True,
)
class TestInternlm2(GSM8KAscendMixin, CustomTestCase):
"""Testcase: Verify that the inference accuracy of the Shanghai_AI_Laboratory/internlm2-7b model on the GSM8K dataset is no less than 0.585.
[Test Category] Model
[Test Target] Shanghai_AI_Laboratory/internlm2-7b
"""
os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
model = INTERNLM2_7B_WEIGHTS_PATH
accuracy = 0.585
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_llama_2_7b.py
-23
View File
@@ -1,23 +0,0 @@
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import LLAMA_2_7B_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)
class TestLlama(GSM8KAscendMixin, CustomTestCase):
"""Testcase: Verify that the inference accuracy of the LLM-Research/Llama-2-7B model on the GSM8K dataset is no less than 0.18.
[Test Category] Model
[Test Target] LLM-Research/Llama-2-7B
"""
model = LLAMA_2_7B_WEIGHTS_PATH
accuracy = 0.18
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_llama_2_7b_communications_compression.py
-37
View File
@@ -1,37 +0,0 @@
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import LLAMA_2_7B_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=400, suite="full-2-npu-a3", nightly=True)
class TestLlama(GSM8KAscendMixin, CustomTestCase):
"""Testcase: Verify that the inference accuracy of the LLM-Research/Llama-2-7B model on the GSM8K dataset with tp communications quantization is no less than 0.18.
[Test Category] Model
[Test Target] LLM-Research/Llama-2-7B
"""
model = LLAMA_2_7B_WEIGHTS_PATH
accuracy = 0.18
other_args = [
"--trust-remote-code",
"--mem-fraction-static",
0.8,
"--max-running-requests",
32,
"--attention-backend",
"ascend",
"--cuda-graph-max-bs-decode",
32,
"--tp-size",
2,
"--enable-quant-communications",
]
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_minicpm3_4b.py
-37
View File
@@ -1,37 +0,0 @@
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import MINICPM3_4B_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)
class TestMiniCPM3(GSM8KAscendMixin, CustomTestCase):
"""Testcase: Verify that the inference accuracy of the OpenBMB/MiniCPM3-4B model on the GSM8K dataset is no less than 0.69.
[Test Category] Model
[Test Target] OpenBMB/MiniCPM3-4B
"""
model = MINICPM3_4B_WEIGHTS_PATH
accuracy = 0.69
other_args = [
"--trust-remote-code",
"--mem-fraction-static",
"0.8",
"--attention-backend",
"ascend",
"--disable-cuda-graph",
"--disable-radix-cache",
"--disable-overlap-schedule",
"--max-running-requests",
"128",
"--chunked-prefill-size",
"-1",
]
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_mistral_7b.py
-23
View File
@@ -1,23 +0,0 @@
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import MISTRAL_7B_INSTRUCT_V0_2_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)
class TestMistral7B(GSM8KAscendMixin, CustomTestCase):
"""Testcase: Verify that the inference accuracy of the mistralai/Mistral-7B-Instruct-v0.2 model on the GSM8K dataset is no less than 0.375.
[Test Category] Model
[Test Target] mistralai/Mistral-7B-Instruct-v0.2
"""
model = MISTRAL_7B_INSTRUCT_V0_2_WEIGHTS_PATH
accuracy = 0.375
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_persimmon_8b_chat.py
-29
View File
@@ -1,29 +0,0 @@
import os
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import PERSIMMON_8B_CHAT_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(
est_time=400,
suite="full-1-npu-a3",
nightly=True,
)
class TestPersimmon8BChat(GSM8KAscendMixin, CustomTestCase):
"""Testcase: Verify that the inference accuracy of the Howeee/persimmon-8b-chat model on the GSM8K dataset is no less than 0.17.
[Test Category] Model
[Test Target] Howeee/persimmon-8b-chat
"""
os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
model = PERSIMMON_8B_CHAT_WEIGHTS_PATH
accuracy = 0.17
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_smollm_1_7b.py
-33
View File
@@ -1,33 +0,0 @@
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import SMOLLM_1_7B_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)
class TestSmolLM(GSM8KAscendMixin, CustomTestCase):
"""Testcase: Verify that the inference accuracy of the HuggingFaceTB/SmolLM-1.7B model on the GSM8K dataset is no less than 0.05.
[Test Category] Model
[Test Target] HuggingFaceTB/SmolLM-1.7B
"""
model = SMOLLM_1_7B_WEIGHTS_PATH
accuracy = 0.05
other_args = [
"--trust-remote-code",
"--mem-fraction-static",
"0.8",
"--attention-backend",
"ascend",
"--disable-cuda-graph",
"--dtype",
"bfloat16",
]
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_stablelm_2_1_6b.py
-34
View File
@@ -1,34 +0,0 @@
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import STABLELM_2_1_6B_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)
class TestStablelm(GSM8KAscendMixin, CustomTestCase):
"""Testcase: Verify that the inference accuracy of the stabilityai/stablelm-2-1_6b model on the GSM8K dataset is no less than 0.195.
[Test Category] Model
[Test Target] stabilityai/stablelm-2-1_6b
"""
model = STABLELM_2_1_6B_WEIGHTS_PATH
accuracy = 0.195
other_args = [
"--trust-remote-code",
"--mem-fraction-static",
"0.8",
"--attention-backend",
"ascend",
"--disable-cuda-graph",
"--tp-size",
1,
"--enable-torch-compile",
]
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms.py
-113
View File
@@ -1,113 +0,0 @@
import unittest
from sglang.test.ascend.e2e.test_npu_multi_node_utils import NIC_NAME
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
KIMI_K2_6_EAGLE3_MODEL_PATH,
KIMI_K2_6_W4A8_MODEL_PATH,
TestNpuPerfMultiNodePdMixTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=1800,
suite="full-8-npu-a3",
nightly=True,
disabled="Currently it is executed by the npu performance workflow.",
)
ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"SGLANG_SET_CPU_AFFINITY": "1",
"STREAMS_PER_DEVICE": "32",
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "64",
"DEEPEP_HCCL_BUFFSIZE": "4400",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"HCCL_SOCKET_IFNAME": NIC_NAME,
"GLOO_SOCKET_IFNAME": NIC_NAME,
}
OTHER_ARGS = [
"--trust-remote-code",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--dtype",
"bfloat16",
"--tp-size",
32,
"--nnodes",
2,
"--mem-fraction-static",
0.68,
"--max-running-requests",
32,
"--chunked-prefill-size",
32768,
"--context-length",
75000,
"--enable-multimodal",
"--mm-attention-backend",
"ascend_attn",
"--sampling-backend",
"ascend",
"--enable-dp-attention",
"--dp-size",
32,
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"auto",
"--cuda-graph-bs",
1,
"--disable-radix-cache",
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
KIMI_K2_6_EAGLE3_MODEL_PATH,
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--speculative-draft-model-quantization",
"unquant",
"--reasoning-parser",
"kimi_k2",
"--tool-call-parser",
"kimi_k2",
]
MODEL_CONFIG = {
"model_path": KIMI_K2_6_W4A8_MODEL_PATH,
"other_args": OTHER_ARGS,
"node_envs": ENVS,
}
class TestNPUKimiK2_6_W4A8_16P_In64k_Out1k_100ms(TestNpuPerfMultiNodePdMixTestCaseBase):
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
model_config = MODEL_CONFIG
dataset_name = "random"
max_concurrency = 32
num_prompts = 32
input_len = 64000
output_len = 1000
random_range_ratio = 1
seed = 1
tpot = 100
output_token_throughput = 160
def test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms(self):
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms.py
-126
View File
@@ -1,126 +0,0 @@
import unittest
from sglang.test.ascend.e2e.test_npu_multi_node_utils import NIC_NAME
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
KIMI_K2_6_EAGLE3_MODEL_PATH,
KIMI_K2_6_W4A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=1200, suite="nightly-perf-16-npu-a3", nightly=True)
KIMI_K2_6_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"SGLANG_SET_CPU_AFFINITY": "1",
"HCCL_SOCKET_IFNAME": NIC_NAME,
"GLOO_SOCKET_IFNAME": NIC_NAME,
"STREAMS_PER_DEVICE": "32",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "96",
"DEEPEP_HCCL_BUFFSIZE": "1200",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_NPU_USE_MLAPO": "1",
"SGLANG_NPU_USE_MULTI_STREAM": "1",
"SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "64",
}
KIMI_K2_6_OTHER_ARGS = [
"--trust-remote-code",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--dtype",
"bfloat16",
"--tp-size",
16,
"--mem-fraction-static",
0.865,
"--max-running-requests",
80,
"--chunked-prefill-size",
32768,
"--context-length",
6144,
"--max-prefill-tokens",
65536,
"--max-total-tokens",
32256,
"--enable-multimodal",
"--mm-attention-backend",
"ascend_attn",
"--sampling-backend",
"ascend",
"--enable-dp-attention",
"--dp-size",
16,
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"auto",
"--cuda-graph-bs-decode",
1,
2,
3,
4,
5,
"--disable-radix-cache",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
KIMI_K2_6_EAGLE3_MODEL_PATH,
"--speculative-num-steps",
4,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
5,
"--speculative-draft-model-quantization",
"unquant",
"--prefill-delayer-max-delay-passes",
200,
"--enable-prefill-delayer",
"--reasoning-parser",
"kimi_k2",
"--tool-call-parser",
"kimi_k2",
]
class TestKimiK25W4A8(
TestNpuPerformanceTestCaseBase,
):
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
max_attempts = 5
model = KIMI_K2_6_W4A8_MODEL_PATH
other_args = KIMI_K2_6_OTHER_ARGS
envs = KIMI_K2_6_ENVS
backend = "sglang"
dataset_name = "random"
max_concurrency = 64
num_prompts = 256
input_len = 3500
output_len = 1500
random_range_ratio = 1
warmup_requests = 0
seed = 1
tpot = 20
output_token_throughput = 1900
def test_kimi_k2_6_w4a8(self):
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1_ttft_5s.py
-191
View File
@@ -1,191 +0,0 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
MIMO_V2_FLASH_MODEL_PATH,
TestNpuPerfMultiNodePdSepTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
PREFILL_ENVS = {
"SGLANG_SET_CPU_AFFINITY": "1",
"STREAMS_PER_DEVICE": "32",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "32",
"DEEPEP_HCCL_BUFFSIZE": "1024",
"HCCL_OP_EXPANSION_MODE": "AIV",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"SGLANG_NPU_PROFILING": "0",
"SGLANG_NPU_PROFILING_STAGE": "prefill",
"DEEPEP_NORMAL_LONG_SEQ_ROUND": "32",
"DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "3584",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "3600",
"SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "3600",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0",
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"SGLANG_DEEPEP_BF16_DISPATCH": "0",
"SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK": "1",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_CONNECT_TIMEOUT": "1800",
"ASCEND_USE_FIA": "1",
}
DECODE_ENVS = {
"SGLANG_SET_CPU_AFFINITY": "1",
"STREAMS_PER_DEVICE": "32",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "128",
"DEEPEP_HCCL_BUFFSIZE": "800",
"HCCL_OP_EXPANSION_MODE": "AIV",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"SGLANG_NPU_PROFILING": "0",
"SGLANG_NPU_PROFILING_STAGE": "prefill",
"DEEPEP_NORMAL_LONG_SEQ_ROUND": "32",
"DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "3584",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "3600",
"SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "3600",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0",
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"SGLANG_DEEPEP_BF16_DISPATCH": "0",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_CONNECT_TIMEOUT": "1800",
"SGLANG_PROFILE_WITH_STACK": "True",
"ASCEND_USE_FIA": "1",
}
PREFILL_ARGS = [
"--attention-backend",
"ascend",
"--device",
"npu",
"--tp-size",
8,
"--nnodes",
"1",
"--node-rank",
"0",
"--chunked-prefill-size",
8192,
"--trust-remote-code",
"--max-running-requests",
64,
"--mem-fraction-static",
0.8,
"--swa-full-tokens-ratio",
0.3,
"--disaggregation-mode",
"prefill",
"--disaggregation-transfer-backend",
"ascend",
"--disable-radix-cache",
"--disable-cuda-graph",
"--disable-piecewise-cuda-graph",
"--dp-size",
"2",
]
DECODE_ARGS = [
"--attention-backend",
"ascend",
"--device",
"npu",
"--tp-size",
16,
"--nnodes",
"1",
"--node-rank",
"0",
"--trust-remote-code",
"--max-running-requests",
64,
"--mem-fraction-static",
0.8,
"--swa-full-tokens-ratio",
0.3,
"--cuda-graph-bs",
1,
2,
4,
8,
12,
16,
20,
24,
28,
32,
"--disaggregation-mode",
"decode",
"--disaggregation-transfer-backend",
"ascend",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--enable-multi-layer-eagle",
"--disable-radix-cache",
"--dp-size",
"2",
"--enable-dp-attention",
"--enable-dp-lm-head",
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"low_latency",
]
ROUTER_ARGS = [
"--health-check-interval-secs",
"3600",
"--mini-lb",
]
MODEL_CONFIG = {
"model_path": MIMO_V2_FLASH_MODEL_PATH,
"prefill_args": PREFILL_ARGS,
"decode_args": DECODE_ARGS,
"prefill_envs": PREFILL_ENVS,
"decode_envs": DECODE_ENVS,
"router_args": ROUTER_ARGS,
"router_envs": {},
}
class TestNPUMimo_v2_flash_1P1D_16p_In16k_Out1_TTFT_5s(
TestNpuPerfMultiNodePdSepTestCaseBase
):
"""Test NPU performance for mimo_v2_flash 1P+1D 16p: input_len=16000, output_len=1, 0 cache, TTFT=5s"""
model_config = MODEL_CONFIG
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
dataset_name = "random"
max_concurrency = 64
num_prompts = 128
request_rate = 0.4
input_len = 16000
output_len = 1
random_range_ratio = 1
seed = 1
ttft = 5000
def test_npu_mimo_v2_flash_1p1d_16p_in16k_out1_ttft_5s(self):
"""Run NPU performance test for 1P+1D 16p with 16k input, 1 output, 0 cache, TTFT=5s"""
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1k_tpot_20ms.py
-187
View File
@@ -1,187 +0,0 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
MIMO_V2_FLASH_MODEL_PATH,
TestNpuPerfMultiNodePdSepTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
PREFILL_ENVS = {
"SGLANG_SET_CPU_AFFINITY": "1",
"STREAMS_PER_DEVICE": "32",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "32",
"DEEPEP_HCCL_BUFFSIZE": "1024",
"HCCL_OP_EXPANSION_MODE": "AIV",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"SGLANG_NPU_PROFILING": "0",
"SGLANG_NPU_PROFILING_STAGE": "prefill",
"DEEPEP_NORMAL_LONG_SEQ_ROUND": "32",
"DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "3584",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "3600",
"SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "3600",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0",
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"SGLANG_DEEPEP_BF16_DISPATCH": "0",
"SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK": "1",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_CONNECT_TIMEOUT": "1800",
"ASCEND_USE_FIA": "1",
}
DECODE_ENVS = {
"SGLANG_SET_CPU_AFFINITY": "1",
"STREAMS_PER_DEVICE": "32",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "128",
"DEEPEP_HCCL_BUFFSIZE": "800",
"HCCL_OP_EXPANSION_MODE": "AIV",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"SGLANG_NPU_PROFILING": "0",
"SGLANG_NPU_PROFILING_STAGE": "prefill",
"DEEPEP_NORMAL_LONG_SEQ_ROUND": "32",
"DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "3584",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "3600",
"SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "3600",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0",
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"SGLANG_DEEPEP_BF16_DISPATCH": "0",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_CONNECT_TIMEOUT": "1800",
"SGLANG_PROFILE_WITH_STACK": "True",
"ASCEND_USE_FIA": "1",
}
PREFILL_ARGS = [
"--attention-backend",
"ascend",
"--device",
"npu",
"--tp-size",
8,
"--nnodes",
"1",
"--node-rank",
"0",
"--chunked-prefill-size",
8192,
"--trust-remote-code",
"--max-running-requests",
64,
"--mem-fraction-static",
0.8,
"--swa-full-tokens-ratio",
0.3,
"--disaggregation-mode",
"prefill",
"--disaggregation-transfer-backend",
"ascend",
"--disable-radix-cache",
"--disable-cuda-graph",
"--disable-piecewise-cuda-graph",
"--dp-size",
"2",
]
DECODE_ARGS = [
"--attention-backend",
"ascend",
"--device",
"npu",
"--tp-size",
16,
"--nnodes",
"1",
"--node-rank",
"0",
"--trust-remote-code",
"--max-running-requests",
32,
"--mem-fraction-static",
0.8,
"--swa-full-tokens-ratio",
0.3,
"--cuda-graph-bs",
1,
2,
4,
8,
12,
16,
"--disaggregation-mode",
"decode",
"--disaggregation-transfer-backend",
"ascend",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--enable-multi-layer-eagle",
"--disable-radix-cache",
"--dp-size",
"2",
"--enable-dp-attention",
"--enable-dp-lm-head",
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"low_latency",
]
ROUTER_ARGS = [
"--health-check-interval-secs",
"3600",
"--mini-lb",
]
MODEL_CONFIG = {
"model_path": MIMO_V2_FLASH_MODEL_PATH,
"prefill_args": PREFILL_ARGS,
"decode_args": DECODE_ARGS,
"prefill_envs": PREFILL_ENVS,
"decode_envs": DECODE_ENVS,
"router_args": ROUTER_ARGS,
"router_envs": {},
}
class TestNPUMimo_v2_flash_1P1D_16p_In16k_Out1k_TTFT_5s(
TestNpuPerfMultiNodePdSepTestCaseBase
):
"""Test NPU performance for mimo_v2_flash 1P+1D 16p: input_len=16000, output_len=1000, 0 cache, TPOT=20ms"""
model_config = MODEL_CONFIG
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
dataset_name = "random"
max_concurrency = 32
num_prompts = 128
request_rate = float("inf")
input_len = 16000
output_len = 1000
random_range_ratio = 1
seed = 1
tpot = 20
def test_npu_mimo_v2_flash_1p1d_16p_in16k_out1k_tpot_20ms(self):
"""Run NPU performance test for 1P+1D 16p with 16k input, 1k output, 0 cache, TPOT=20ms"""
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py
-118
View File
@@ -1,118 +0,0 @@
import os
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
BENCHMARK_TOOL_DEFAULT,
MINIMAX_M2_5_EAGLE3_MODEL_PATH,
MINIMAX_M2_5_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=3600, suite="nightly-perf-8-npu-a3", nightly=True)
MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"TASK_QUEUE_ENABLE": "1",
"ASCEND_USE_FIA": "1",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "140000",
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"DEEPEP_HCCL_BUFFSIZE": "1024",
"SGLANG_EXTERNAL_MODEL_PACKAGE": "custom_eagle3",
"SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "999999999",
"PYTHONPATH": f"{MINIMAX_M2_5_EAGLE3_MODEL_PATH}:{os.environ.get('PYTHONPATH', '')}",
}
MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_OTHER_ARGS = [
"--tp-size",
8,
"--mem-fraction-static",
0.63,
"--max-running-requests",
24,
"--reasoning-parser",
"minimax-append-think",
"--tool-call-parser",
"minimax-m2",
"--enable-prefill-delayer",
"--prefill-delayer-max-delay-passes",
30,
"--prefill-max-requests",
10,
"--chunked-prefill-size",
67072,
"--max-prefill-token",
67000,
"--cuda-graph-bs",
2,
4,
8,
12,
16,
18,
20,
22,
24,
"--moe-a2a-backend",
"ascend_fuseep",
"--fuseep-mode",
2,
"--deepep-mode",
"auto",
"--quantization",
"modelslim",
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
MINIMAX_M2_5_EAGLE3_MODEL_PATH,
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--speculative-draft-model-quantization",
"unquant",
"--dtype",
"bfloat16",
"--trust-remote-code",
"--reasoning-parser",
"minimax-append-think",
"--tool-call-parser",
"minimax-m2",
]
class TestNPUMiniMaxM2_5W8A8_4P_In64k_Out1k_Prefix90_50ms(
TestNpuPerformanceTestCaseBase
):
"""MiniMax-M2.5-w8a8 4p (4 cards) 64k input 1k output with 90% prefix cache performance test"""
benchmark_tool = BENCHMARK_TOOL_DEFAULT
model = MINIMAX_M2_5_W8A8_MODEL_PATH
other_args = MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_OTHER_ARGS
envs = MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS
dataset_name = "generated-shared-prefix"
max_concurrency = 24
num_prompts = 96
input_len = 65536
output_len = 1024
random_range_ratio = 1
seed = 1
repeat_rate = 0.9
tpot = 50
output_token_throughput = 390.5859
request_rate = float("inf")
def test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms(self):
"""Run MiniMax-M2.5-w8a8 4p 64k/1k prefix90 performance test"""
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms.py
-117
View File
@@ -1,117 +0,0 @@
import os
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
MINIMAX_M2_5_EAGLE3_MODEL_PATH,
MINIMAX_M2_5_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="nightly-perf-16-npu-a3",
nightly=True,
)
MINIMAX_M2_5_HIGH_THROUGHPUT_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"TASK_QUEUE_ENABLE": "1",
"DEEPEP_HCCL_BUFFSIZE": "1024",
"ASCEND_USE_FIA": "1",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "204800",
"PYTHONPATH": f"{MINIMAX_M2_5_EAGLE3_MODEL_PATH}:{os.environ.get('PYTHONPATH', '')}",
"SGLANG_EXTERNAL_MODEL_PACKAGE": "custom_eagle3",
}
MINIMAX_M2_5_HIGH_THROUGHPUT_OTHER_ARGS = [
"--tp-size",
16,
"--enable-dp-attention",
"--dp-size",
16,
"--mem-fraction-static",
0.75,
"--max-running-requests",
320,
"--disable-radix-cache",
"--reasoning-parser",
"minimax-append-think",
"--tool-call-parser",
"minimax-m2",
"--prefill-delayer-max-delay-passes",
500,
"--enable-prefill-delayer",
"--chunked-prefill-size",
196608,
"--max-prefill-token",
8192,
"--cuda-graph-bs",
1,
2,
4,
8,
12,
16,
20,
"--moe-a2a-backend",
"ascend_fuseep",
"--fuseep-mode",
2,
"--quantization",
"modelslim",
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
MINIMAX_M2_5_EAGLE3_MODEL_PATH,
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--speculative-draft-model-quantization",
"unquant",
"--dtype",
"bfloat16",
"--reasoning-parser",
"minimax-append-think",
"--tool-call-parser",
"minimax-m2",
]
class TestNPUMiniMaxM2_5_W8A8_8P_In3k5_Out1k5_HighThroughput(
TestNpuPerformanceTestCaseBase
):
"""Test NPU performance for MiniMax-M2.5-w8a8 8p single node high throughput in3k5 out1k5"""
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = MINIMAX_M2_5_W8A8_MODEL_PATH
other_args = MINIMAX_M2_5_HIGH_THROUGHPUT_OTHER_ARGS
envs = MINIMAX_M2_5_HIGH_THROUGHPUT_ENVS
dataset_name = "random"
max_concurrency = 320
num_prompts = 1280
input_len = 3500
output_len = 1500
random_range_ratio = 1
seed = 1
tpot = 50
output_token_throughput = 5717.58
def test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_high_throughput(self):
"""Run NPU performance test for MiniMax-M2.5-w8a8 high throughput"""
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py
-105
View File
@@ -1,105 +0,0 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
QWEN3_8B_EAGLE_MODEL_PATH,
QWEN3_8B_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=3600, suite="nightly-perf-2-npu-a3", nightly=True)
QWEN3_8B_ENVS = {
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "50",
"SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "128",
}
QWEN3_8B_OTHER_ARGS = [
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--max-running-requests",
70,
"--max-prefill-tokens",
16384,
"--disable-radix-cache",
"--chunked-prefill-size",
16384,
"--tp-size",
1,
"--mem-fraction-static",
0.85,
"--cuda-graph-bs",
8,
12,
24,
36,
48,
51,
55,
60,
63,
64,
66,
68,
70,
"--dtype",
"bfloat16",
"--speculative-draft-model-quantization",
"unquant",
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
QWEN3_8B_EAGLE_MODEL_PATH,
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen",
]
class TestQwen8B(TestNpuPerformanceTestCaseBase):
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_8B_W8A8_MODEL_PATH
other_args = QWEN3_8B_OTHER_ARGS
envs = QWEN3_8B_ENVS
dataset_name = "random"
max_concurrency = 64
num_prompts = 256
input_len = 3500
output_len = 1500
random_range_ratio = 1
seed = 1
tpot = 37
output_token_throughput = 1586
def test_qwen3_8b(self):
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16.py
-93
View File
@@ -1,93 +0,0 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
QWEN3_8B_EAGLE_MODEL_PATH,
QWEN3_8B_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="nightly-perf-2-npu-a3",
nightly=True,
)
QWEN3_8B_ENVS = {
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
}
QWEN3_8B_OTHER_ARGS = [
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--max-running-requests",
16,
"--max-prefill-tokens",
16384,
"--disable-radix-cache",
"--chunked-prefill-size",
-1,
"--tp-size",
2,
"--mem-fraction-static",
0.894,
"--cuda-graph-bs",
1,
5,
15,
16,
"--dtype",
"bfloat16",
"--speculative-draft-model-quantization",
"unquant",
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
QWEN3_8B_EAGLE_MODEL_PATH,
"--speculative-num-steps",
4,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
5,
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen",
]
class TestQwen8B(TestNpuPerformanceTestCaseBase):
max_attempts = 5
model = QWEN3_8B_W8A8_MODEL_PATH
other_args = QWEN3_8B_OTHER_ARGS
envs = QWEN3_8B_ENVS
dataset_name = "random"
max_concurrency = 16
num_prompts = 16
input_len = 6144
output_len = 1500
random_range_ratio = 1
tpot = 11.79
output_token_throughput = 1040.96
def test_qwen3_8b(self):
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py
-122
View File
@@ -1,122 +0,0 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
QWEN3_235B_A22B_EAGLE_MODEL_PATH,
QWEN3_235B_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=3600, suite="nightly-perf-16-npu-a3", nightly=True)
QWEN3_235B_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"DEEPEP_HCCL_BUFFSIZE": "570",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "50",
"SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "16",
"SGLANG_NPU_PROFILING": "0",
"SGLANG_NPU_PROFILING_BS": "27",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "188416",
}
QWEN3_235B_OTHER_ARGS = [
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--max-running-requests",
432,
"--context-length",
8192,
"--dtype",
"bfloat16",
"--chunked-prefill-size",
94208,
"--max-prefill-tokens",
458880,
"--sampling-backend",
"ascend",
"--ep-dispatch-algorithm",
"static",
"--init-expert-location",
"/root/.cache/modelscope/hub/models/hot_map/235B_3_5k_decode.pt",
"--disable-radix-cache",
"--moe-a2a-backend",
"ascend_fuseep",
"--fuseep-mode",
2,
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
QWEN3_235B_A22B_EAGLE_MODEL_PATH,
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--speculative-draft-model-quantization",
"unquant",
"--tp",
"16",
"--dp-size",
"16",
"--enable-dp-attention",
"--enable-dp-lm-head",
"--mem-fraction-static",
"0.8",
"--cuda-graph-bs",
"1",
"2",
"4",
"8",
"16",
"20",
"24",
"26",
"27",
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen25",
]
class TestQwen235B(TestNpuPerformanceTestCaseBase):
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_235B_W8A8_MODEL_PATH
other_args = QWEN3_235B_OTHER_ARGS
envs = QWEN3_235B_ENVS
dataset_name = "random"
max_concurrency = 432
num_prompts = int(max_concurrency) * 4
input_len = 3500
output_len = 1500
random_range_ratio = 1
seed = 1
tpot = 50.1
output_token_throughput = 6189
def test_qwen3_235b(self):
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py
-110
View File
@@ -1,110 +0,0 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
QWEN3_30B_A3B_W8A8_VLLM_MODEL_PATH,
QWEN3_A3B_EAGLE_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=3600, suite="nightly-perf-2-npu-a3", nightly=True)
QWEN3_30B_A3B_ENVS = {
"ASCEND_LAUNCH_BLOCKING": "0",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200",
"SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "128",
"DEEPEP_HCCL_BUFFSIZE": "400",
}
QWEN3_30B_A3B_OTHER_ARGS = [
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--max-running-requests",
162,
"--disable-radix-cache",
"--speculative-draft-model-quantization",
"unquant",
"--chunked-prefill-size",
-1,
"--max-prefill-tokens",
35000,
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
QWEN3_A3B_EAGLE_MODEL_PATH,
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--tp-size",
2,
"--mem-fraction-static",
0.87,
"--cuda-graph-bs",
1,
5,
15,
40,
70,
100,
120,
130,
140,
146,
150,
154,
156,
158,
160,
162,
"--dtype",
"bfloat16",
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen",
]
class TestQwen30B(TestNpuPerformanceTestCaseBase):
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_30B_A3B_W8A8_VLLM_MODEL_PATH
other_args = QWEN3_30B_A3B_OTHER_ARGS
envs = QWEN3_30B_A3B_ENVS
dataset_name = "random"
max_concurrency = 160
num_prompts = int(max_concurrency) * 4
input_len = 3500
output_len = 1500
random_range_ratio = 1
seed = 1
tpot = 50
output_token_throughput = 3200
def test_qwen3_30b(self):
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py
-94
View File
@@ -1,94 +0,0 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
QWEN3_32B_EAGLE_MODEL_PATH,
QWEN3_32B_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="nightly-perf-16-npu-a3",
nightly=True,
)
QWEN3_32B_ENVS = {
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200",
}
QWEN3_32B_OTHER_ARGS = [
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--max-running-requests",
1,
"--disable-radix-cache",
"--speculative-draft-model-quantization",
"unquant",
"--chunked-prefill-size",
-1,
"--max-prefill-tokens",
65536,
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
QWEN3_32B_EAGLE_MODEL_PATH,
"--speculative-num-steps",
4,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
5,
"--tp-size",
16,
"--mem-fraction-static",
0.72,
"--cuda-graph-bs",
1,
"--dtype",
"bfloat16",
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen",
]
class TestQwen32B(TestNpuPerformanceTestCaseBase):
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_32B_MODEL_PATH
other_args = QWEN3_32B_OTHER_ARGS
envs = QWEN3_32B_ENVS
dataset_name = "random"
max_concurrency = 1
num_prompts = 1
input_len = 18000
output_len = 4000
random_range_ratio = 1
seed = 1
tpot = 6
output_token_throughput = 171
def test_qwen3_32b(self):
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py
-107
View File
@@ -1,107 +0,0 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
QWEN3_32B_EAGLE_MODEL_PATH,
QWEN3_32B_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=3600, suite="nightly-perf-4-npu-a3", nightly=True)
QWEN3_32B_ENVS = {
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100",
"SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "128",
"SGLANG_NPU_USE_DEEPGEMM": "1",
}
QWEN3_32B_OTHER_ARGS = [
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--max-running-requests",
101,
"--disable-radix-cache",
"--speculative-draft-model-quantization",
"unquant",
"--chunked-prefill-size",
-1,
"--max-prefill-tokens",
35000,
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
QWEN3_32B_EAGLE_MODEL_PATH,
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--tp-size",
4,
"--mem-fraction-static",
0.845,
"--cuda-graph-bs",
16,
32,
64,
72,
88,
90,
92,
94,
96,
97,
98,
99,
100,
101,
"--dtype",
"bfloat16",
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen",
]
class TestQwen32B(TestNpuPerformanceTestCaseBase):
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_32B_W8A8_MODEL_PATH
other_args = QWEN3_32B_OTHER_ARGS
envs = QWEN3_32B_ENVS
dataset_name = "random"
max_concurrency = 100
num_prompts = 400
input_len = 3584
output_len = 1536
random_range_ratio = 1
seed = 1
tpot = 50
output_token_throughput = 1600
def test_qwen3_32b(self):
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py
-110
View File
@@ -1,110 +0,0 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
DEFAULT_URL_FOR_TEST,
QWEN3_32B_EAGLE_MODEL_PATH,
QWEN3_32B_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="nightly-1-npu-a2",
nightly=True,
)
QWEN3_32B_ENVS = {
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100",
"SGLANG_NPU_USE_DEEPGEMM": "1",
}
QWEN3_32B_OTHER_ARGS = [
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--max-running-requests",
101,
"--disable-radix-cache",
"--speculative-draft-model-quantization",
"unquant",
"--chunked-prefill-size",
-1,
"--max-prefill-tokens",
35000,
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
QWEN3_32B_EAGLE_MODEL_PATH,
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--tp-size",
4,
"--mem-fraction-static",
0.845,
"--cuda-graph-bs",
16,
32,
64,
72,
88,
90,
92,
94,
96,
97,
98,
99,
100,
101,
"--dtype",
"bfloat16",
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen",
]
class TestQwen32B(TestNpuPerformanceTestCaseBase):
base_url = DEFAULT_URL_FOR_TEST
dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_32B_W8A8_MODEL_PATH
other_args = QWEN3_32B_OTHER_ARGS
envs = QWEN3_32B_ENVS
dataset_name = "random"
max_concurrency = 100
num_prompts = 400
input_len = 3584
output_len = 1536
random_range_ratio = 1
seed = 1
tpot = 55
output_token_throughput = 1500
def test_qwen3_32b(self):
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in64k_out1k_50ms.py
-100
View File
@@ -1,100 +0,0 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
QWEN3_6_27B_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
QWEN3_6_27B_64K_1K_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "30",
"ASCEND_USE_FIA": "1",
}
QWEN3_6_27B_64K_1K_OTHER_ARGS = [
"--tp-size",
4,
"--nnodes",
1,
"--attention-backend",
"ascend",
"--device",
"npu",
"--chunked-prefill-size",
-1,
"--max-prefill-tokens",
50000,
"--disable-radix-cache",
"--trust-remote-code",
"--max-running-requests",
28,
"--max-mamba-cache-size",
50,
"--mem-fraction-static",
0.7,
"--cuda-graph-bs",
2,
4,
6,
"--enable-multimodal",
"--quantization",
"modelslim",
"--mm-attention-backend",
"ascend_attn",
"--dtype",
"bfloat16",
"--mamba-ssm-dtype",
"bfloat16",
"--speculative-algorithm",
"NEXTN",
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
]
class TestNPUQwen3_6_27B_2P_In64k_Out1k_50ms(TestNpuPerformanceTestCaseBase):
"""Test NPU performance for Qwen3.6-27B-w8a8 2p in64k out1k 50ms"""
benchmark_tool = BENCHMARK_TOOL_DEFAULT
aisbench_dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_6_27B_W8A8_MODEL_PATH
other_args = QWEN3_6_27B_64K_1K_OTHER_ARGS
envs = QWEN3_6_27B_64K_1K_ENVS
dataset_name = "random"
max_concurrency = 6
num_prompts = 24
input_len = 64000
output_len = 1000
random_range_ratio = 1
seed = 1
tpot = 50
output_token_throughput = 122.6
def test_npu_qwen3_6_27b_2p_in64k_out1k_50ms(self):
"""Run NPU performance test for Qwen3.6-27B-w8a8 in64k out1k 50ms"""
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16.py
-110
View File
@@ -1,110 +0,0 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
QWEN3_NEXT_80B_A3B_MODEL_PATH,
QWEN3_NEXT_80B_A3B_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=3600, suite="nightly-perf-4-npu-a3", nightly=True)
QWEN3_NEXT_80B_A3B_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "400",
"DEEPEP_NORMAL_LONG_SEQ_ROUND": "10",
"DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "2048",
"HCCL_OP_EXPANSION_MODE": "AIV",
"TASK_QUEUE_ENABLE": "1",
"ASCEND_USE_FIA": "1",
"SGLANG_NPU_USE_MULTI_STREAM": "0",
"SGLANG_WARMUP_TIMEOUT": "3600",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"FORCE_DRAFT_MODEL_NON_QUANT": "1",
"DEEPEP_HCCL_BUFFSIZE": "2000",
"ZBCCL_LOCAL_MEM_SIZE": "60416",
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0",
"ZBCCL_BOOTSTRAP_URL": "tcp://127.0.0.1:24669",
"ZBCCL_NPU_ALLOC_CONF": "use_vmm_for_static_memory:True",
"ZBCCL_ENABLE_GRAPH": "1",
}
QWEN3_NEXT_80B_A3B_OTHER_ARGS = [
"--trust-remote-code",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--page-size",
128,
"--tp-size",
4,
"--watchdog-timeout",
9000,
"--mem-fraction-static",
0.85,
"--disable-radix-cache",
"--max-prefill-tokens",
28672,
"--context-length",
81920,
"--max-total-tokens",
122304,
"--dp-size",
2,
"--enable-dp-attention",
"--enable-dp-lm-head",
"--speculative-algorithm",
"NEXTN",
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--speculative-draft-model-quantization",
"unquant",
"--chunked-prefill-size",
-1,
"--max-running-requests",
16,
"--cuda-graph-bs",
2,
4,
8,
"--mamba-ssm-dtype",
"bfloat16",
"--speculative-draft-model-path",
QWEN3_NEXT_80B_A3B_MODEL_PATH,
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen3_coder",
]
class TestQwen3Next80BA3B(TestNpuPerformanceTestCaseBase):
max_attempts = 5
model = QWEN3_NEXT_80B_A3B_W8A8_MODEL_PATH
other_args = QWEN3_NEXT_80B_A3B_OTHER_ARGS
envs = QWEN3_NEXT_80B_A3B_ENVS
dataset_name = "random"
max_concurrency = 16
num_prompts = 16
input_len = 6144
output_len = 1500
random_range_ratio = 1
tpot = 15.62
def test_qwen3_next_80b_a3b(self):
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/rerank_models/test_npu_bge_reranker_v2_m3.py
-98
View File
@@ -1,98 +0,0 @@
import multiprocessing as mp
import unittest
import torch
from sglang.test.ascend.test_ascend_utils import BGE_RERANKER_V2_M3_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.runners import TEST_RERANK_QUERY_DOCS, HFRunner, SRTRunner
from sglang.test.test_utils import CustomTestCase
register_npu_ci(
est_time=400,
suite="full-1-npu-a3",
nightly=True,
)
MODELS = [
(BGE_RERANKER_V2_M3_WEIGHTS_PATH, 1, 1e-2),
]
ATTENTION_BACKEND = ["ascend"]
TORCH_DTYPES = [torch.bfloat16]
class TestBgeReranker(CustomTestCase):
"""Testcase: This test case validates that the cross-encoder scores from the BAAI/bge-reranker-v2-m3 model in the
SGLang framework are less than 1e-2 different from the Hugging Face implementation.
[Test Category] Model
[Test Target] BAAI/bge-reranker-v2-m3
"""
@classmethod
def setUpClass(cls):
mp.set_start_method("spawn", force=True)
def assert_close_prefill_logits(
self,
prompts,
model_path,
tp_size,
torch_dtype,
score_tolerance,
attention_backend,
) -> None:
with HFRunner(
model_path,
torch_dtype=torch_dtype,
model_type="cross_encoder",
) as hf_runner:
hf_scores = hf_runner.forward(prompts).scores
with SRTRunner(
model_path,
tp_size=tp_size,
torch_dtype=torch_dtype,
model_type="cross_encoder",
attention_backend=attention_backend,
chunked_prefill_size=-1,
disable_radix_cache=True,
) as srt_runner:
srt_scores = srt_runner.forward(prompts).scores
for i in range(len(srt_scores)):
score_difference = abs(hf_scores[i] - srt_scores[i])
assert score_difference < score_tolerance, (
"cross encoder scores are not all close"
)
def preprocess_prompts(self, prompt):
processed_prompts = []
query = prompt["query"]
documents = prompt["documents"]
for document in documents:
processed_prompts.append([query, document])
return processed_prompts
def test_prefill_logits(self):
models_to_test = MODELS
for model, tp_size, prefill_tolerance in models_to_test:
for attention_backend in ATTENTION_BACKEND:
for queryDocs in TEST_RERANK_QUERY_DOCS:
prompts = self.preprocess_prompts(queryDocs)
for torch_dtype in TORCH_DTYPES:
self.assert_close_prefill_logits(
prompts,
model,
tp_size,
torch_dtype,
prefill_tolerance,
attention_backend,
)
if __name__ == "__main__":
unittest.main()
test/registered/npu/reward_models/test_npu_internlm2_7b_reward.py
-65
View File
@@ -1,65 +0,0 @@
import os
os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
import multiprocessing as mp
import unittest
import torch
from sglang.test.ascend.test_ascend_utils import INTERNLM2_7B_REWARD_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.runners import SRTRunner
from sglang.test.test_utils import CustomTestCase
register_npu_ci(
est_time=400,
suite="full-4-npu-a3",
nightly=True,
)
PROMPT = (
"What is the range of the numeric output of a sigmoid node in a neural network?"
)
RESPONSE1 = "The output of a sigmoid node is bounded between -1 and 1."
RESPONSE2 = "The output of a sigmoid node is bounded between 0 and 1."
CONVS = [
[{"role": "user", "content": PROMPT}, {"role": "assistant", "content": RESPONSE1}],
[{"role": "user", "content": PROMPT}, {"role": "assistant", "content": RESPONSE2}],
]
class TestInternlm2(CustomTestCase):
"""Testcase: This test case verifies that the Shanghai_AI_Laboratory/internlm2-7b-reward model can successfully generate reward
scores for different conversational responses using the SGLang framework, without comparing to a reference implementation.
[Test Category] Model
[Test Target] Shanghai_AI_Laboratory/internlm2-7b-reward
"""
model_path = INTERNLM2_7B_REWARD_WEIGHTS_PATH
torch_dtype = torch.float16
@classmethod
def setUpClass(cls):
mp.set_start_method("spawn", force=True)
def test_assert_close_reward_scores(self):
with SRTRunner(
self.model_path,
torch_dtype=self.torch_dtype,
model_type="reward",
trust_remote_code=True,
disable_cuda_graph=True,
tp_size=4,
mem_fraction_static=0.8,
) as srt_runner:
prompts = srt_runner.tokenizer.apply_chat_template(CONVS, tokenize=False)
srt_outputs = srt_runner.forward(prompts)
srt_scores = torch.tensor(srt_outputs.scores)
print(f"accuracy: {srt_scores}")
self.assertIsInstance(srt_scores, torch.Tensor)
if __name__ == "__main__":
unittest.main()
test/registered/npu/reward_models/test_npu_llama_3_1_8b_v0_2.py
-86
View File
@@ -1,86 +0,0 @@
import multiprocessing as mp
import unittest
import torch
from sglang.test.ascend.test_ascend_utils import (
SKYWORK_REWARD_LLAMA_3_1_8B_V0_2_WEIGHTS_PATH,
)
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.runners import HFRunner, SRTRunner
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)
MODELS = [
(SKYWORK_REWARD_LLAMA_3_1_8B_V0_2_WEIGHTS_PATH, 1, 4e-2),
]
TORCH_DTYPES = [torch.float16]
PROMPT = (
"What is the range of the numeric output of a sigmoid node in a neural network?"
)
RESPONSE1 = "The output of a sigmoid node is bounded between -1 and 1."
RESPONSE2 = "The output of a sigmoid node is bounded between 0 and 1."
CONVS = [
[{"role": "user", "content": PROMPT}, {"role": "assistant", "content": RESPONSE1}],
[{"role": "user", "content": PROMPT}, {"role": "assistant", "content": RESPONSE2}],
]
class TestLlama(CustomTestCase):
"""Testcase: This test case validates that the reward scores from the Skywork/Skywork-Reward-Llama-3.1-8B-v0.2 model
in the SGLang framework are less than 4e-2 different from the Hugging Face implementation.
[Test Category] Model
[Test Target] Skywork/Skywork-Reward-Llama-3.1-8B-v0.2
"""
@classmethod
def setUpClass(cls):
mp.set_start_method("spawn", force=True)
def assert_close_reward_scores(
self,
convs,
model_path,
tp_size,
torch_dtype,
tolerance,
) -> None:
with HFRunner(
model_path,
torch_dtype=torch_dtype,
model_type="reward",
) as hf_runner:
hf_outputs = hf_runner.forward(convs)
with SRTRunner(
model_path,
tp_size=tp_size,
torch_dtype=torch_dtype,
model_type="reward",
) as srt_runner:
prompts = srt_runner.tokenizer.apply_chat_template(convs, tokenize=False)
srt_outputs = srt_runner.forward(prompts)
hf_scores = torch.tensor(hf_outputs.scores)
srt_scores = torch.tensor(srt_outputs.scores)
print(f"{hf_scores=}")
print(f"{srt_scores=}")
assert torch.all(abs(hf_scores - srt_scores) < tolerance), (
"reward scores are not all close"
)
def test_reward_scores(self):
for model, tp_size, tolerance in MODELS:
for torch_dtype in TORCH_DTYPES:
self.assert_close_reward_scores(
CONVS, model, tp_size, torch_dtype, tolerance
)
if __name__ == "__main__":
unittest.main()
test/registered/npu/vlm_models/test_npu_deepseek_vl2.py
-29
View File
@@ -1,29 +0,0 @@
import unittest
from sglang.test.ascend.test_ascend_utils import DEEPSEEK_VL2_WEIGHTS_PATH
from sglang.test.ascend.vlm_utils import TestVLMModels
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=400,
suite="full-4-npu-a3",
nightly=True,
)
class TestDeepseekVl2(TestVLMModels):
"""Testcase: Verify that the inference accuracy of the deepseek-ai/deepseek-vl2 model on the MMMU dataset is no less than 0.2.
[Test Category] Model
[Test Target] deepseek-ai/deepseek-vl2
"""
model = DEEPSEEK_VL2_WEIGHTS_PATH
mmmu_accuracy = 0.2
def test_vlm_mmmu_benchmark(self):
self._run_vlm_mmmu_test()
if __name__ == "__main__":
unittest.main()
test/registered/npu/vlm_models/test_npu_llama_3_2_11b_vision_instruct.py
-36
View File
@@ -1,36 +0,0 @@
import unittest
from sglang.test.ascend.test_ascend_utils import (
LLAMA_3_2_11B_VISION_INSTRUCT_WEIGHTS_PATH,
)
from sglang.test.ascend.vlm_utils import TestVLMModels
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)
class TestLlama3211BVisionInstruct(TestVLMModels):
"""Testcase: Verify that the inference accuracy of the LLM-Research/Llama-3.2-11B-Vision-Instruct model on the MMMU dataset is no less than 0.2.
[Test Category] Model
[Test Target] LLM-Research/Llama-3.2-11B-Vision-Instruct
"""
model = LLAMA_3_2_11B_VISION_INSTRUCT_WEIGHTS_PATH
mmmu_accuracy = 0.2
other_args = [
"--trust-remote-code",
"--mem-fraction-static",
"0.8",
"--attention-backend",
"ascend",
"--disable-cuda-graph",
"--disable-radix-cache",
]
def test_vlm_mmmu_benchmark(self):
self._run_vlm_mmmu_test()
if __name__ == "__main__":
unittest.main()
test/registered/npu/vlm_models/test_npu_minicpm_v_2_6.py
-25
View File
@@ -1,25 +0,0 @@
import unittest
from sglang.test.ascend.test_ascend_utils import MINICPM_V_2_6_WEIGHTS_PATH
from sglang.test.ascend.vlm_utils import TestVLMModels
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=400, suite="full-4-npu-a3", nightly=True)
class TestMiniCPMModelsV(TestVLMModels):
"""Testcase: Verify that the inference accuracy of the openbmb/MiniCPM-V-2_6 model on the MMMU dataset is no less than 0.2.
[Test Category] Model
[Test Target] openbmb/MiniCPM-V-2_6
"""
model = MINICPM_V_2_6_WEIGHTS_PATH
mmmu_accuracy = 0.2
def test_vlm_mmmu_benchmark(self):
self._run_vlm_mmmu_test()
if __name__ == "__main__":
unittest.main()
test/run_suite.py
+8 -3
View File
@@ -153,25 +153,30 @@ NIGHTLY_SUITES = {
],
HWBackend.CPU: [],
HWBackend.NPU: [
"nightly-1-npu-a2",
"nightly-1-npu-a3",
"nightly-2-npu-a3",
"nightly-4-npu-a3",
"nightly-8-npu-a3",
"nightly-16-npu-a3",
"nightly-acc-2-npu-a3",
"nightly-acc-4-npu-a3",
"nightly-acc-8-npu-a3",
"nightly-acc-16-npu-a3",
"nightly-perf-2-npu-a3",
"nightly-perf-4-npu-a3",
"nightly-perf-8-npu-a3",
"nightly-perf-16-npu-a3",
"full-4-npu-a2",
"full-1-npu-a3",
"full-2-npu-a3",
"full-4-npu-a3",
"full-8-npu-a3",
"full-16-npu-a3",
"full-acc-2-npu-a3",
"full-acc-4-npu-a3",
"full-acc-16-npu-a3",
"full-perf-2-npu-a3",
"full-perf-4-npu-a3",
"full-perf-8-npu-a3",
"full-perf-16-npu-a3",
],
HWBackend.XPU: [
"nightly-xpu-1-gpu",