diff --git a/.github/workflows/_npu-pr-test-stage.yml b/.github/workflows/_npu-pr-test-stage.yml index d36933d8d..6a062cb47 100644 --- a/.github/workflows/_npu-pr-test-stage.yml +++ b/.github/workflows/_npu-pr-test-stage.yml @@ -186,8 +186,6 @@ jobs: } - name: Install dependencies - # Only PR jobs install dependencies - if: ${{ inputs.is_nightly_pipeline_job != true }} env: TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu" PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" @@ -195,17 +193,31 @@ jobs: GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/" RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082" run: | - # speed up by using infra cache services + set -euo pipefail + cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp + cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp + CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local" sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list pip config set global.index-url http://${CACHING_URL}/pypi/simple pip config set global.trusted-host "${CACHING_URL}" - bash scripts/ci/npu/npu_ci_install_dependency.sh ${{ inputs.npu_device_type }} - # copy required file from our daily cache - cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp - # copy gsm8k dataset - cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp + if [[ "${{ inputs.is_nightly_pipeline_job }}" != "true" ]]; then + bash scripts/ci/npu/npu_ci_install_dependency.sh ${{ inputs.npu_device_type }} + + else + sglang_pkg_path=/sgl-workspace/sglang/python + ascend_test_util_path=${sglang_pkg_path}/sglang/test/ascend + mkdir -p ${ascend_test_util_path} + mv ${ascend_test_util_path} ${ascend_test_util_path}_bak + cp -r $(pwd)/python/sglang/test/ascend ${ascend_test_util_path} + + pip install sentence_transformers zss "wandb>=0.16.0" tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap + pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 "sacrebleu>=1.5.0" pytablewriter "peft>=0.2.0" "accelerate>=0.29.1" + pip install jsonlines "evaluate>=0.4.0" numexpr dotenv + + hf download MMMU/MMMU --repo-type dataset + fi # install sglang_router apt-get install -y libssl-dev diff --git a/.github/workflows/full-test-npu.yml b/.github/workflows/full-test-npu.yml index 42360ff67..9f38e5771 100644 --- a/.github/workflows/full-test-npu.yml +++ b/.github/workflows/full-test-npu.yml @@ -18,16 +18,16 @@ on: required: false type: string default: 'all' + image_a2: + description: 'The a2 running docker image of the test task.' + required: false + type: string + default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-910b' image_a3: - description: 'The a3 running docker image of the test task.' + description: 'The a3 running docker image of the test task. Must be an image with sglang preinstalled (e.g. lmsysorg/sglang:*a3*).' required: false type: string - default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-a3-ubuntu22.04-py3.11' - skip_install_flag: - description: 'Indicates whether to skip the installation of sglang, defaulting to false.' - required: false - type: string - default: 'false' + default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3' concurrency: group: full-test-npu-${{ inputs.ref || github.ref }} @@ -39,8 +39,8 @@ jobs: outputs: ref: ${{ steps.set-vars.outputs.ref }} job_filter: ${{ steps.set-vars.outputs.job_filter }} + image_a2: ${{ steps.set-vars.outputs.image_a2 }} image_a3: ${{ steps.set-vars.outputs.image_a3 }} - skip_install_flag: ${{ steps.set-vars.outputs.skip_install_flag }} steps: # When triggered by PR, no inputs parameters are used. The latest community code is tested by default. - name: Set image config @@ -58,16 +58,16 @@ jobs: echo "job_filter=${{ inputs.job_filter }}" >> $GITHUB_OUTPUT fi - if [ -z "${{ inputs.image_a3 }}" ]; then - echo "image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.5.0-a3-ubuntu22.04-py3.11" >> $GITHUB_OUTPUT + if [ -z "${{ inputs.image_a2 }}" ]; then + echo "image_a2=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-910b" >> $GITHUB_OUTPUT else - echo "image_a3=${{ inputs.image_a3 }}" >> $GITHUB_OUTPUT + echo "image_a2=${{ inputs.image_a2 }}" >> $GITHUB_OUTPUT fi - if [ -z "${{ inputs.skip_install_flag }}" ]; then - echo "skip_install_flag=false" >> $GITHUB_OUTPUT + if [ -z "${{ inputs.image_a3 }}" ]; then + echo "image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3" >> $GITHUB_OUTPUT else - echo "skip_install_flag=${{ inputs.skip_install_flag }}" >> $GITHUB_OUTPUT + echo "image_a3=${{ inputs.image_a3 }}" >> $GITHUB_OUTPUT fi nighly-test-npu: @@ -79,269 +79,258 @@ jobs: ref: ${{ needs.set-image-config.outputs.ref }} job_filter: ${{ needs.set-image-config.outputs.job_filter }} image_a3: ${{ needs.set-image-config.outputs.image_a3 }} - skip_install_flag: ${{ needs.set-image-config.outputs.skip_install_flag }} secrets: inherit - full-1-npu-a3: - needs: [set-image-config] + full-4-npu-a2: + name: full-4-npu-a2 if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }} - runs-on: linux-aarch64-a3-2 - container: + needs: [set-image-config] + uses: ./.github/workflows/_npu-single-node-test-stage.yml + with: + runner: linux-aarch64-a2-4 + test_type: 'perf' + test_suite: full-4-npu-a2 + is_nightly_pipeline_job: true + skip_pr_test_health_check: 'true' + image: ${{ needs.set-image-config.outputs.image_a2 }} + device_type_for_deps: '910b' + + full-1-npu-a3: + name: full-1-npu-a3 + if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }} + needs: [set-image-config] + uses: ./.github/workflows/_npu-pr-test-stage.yml + with: + self_name: full-1-npu-a3 + runner_config: linux-aarch64-a3-2 image: ${{ needs.set-image-config.outputs.image_a3 }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ needs.set-image-config.outputs.ref || github.ref }} - - - name: Install dependencies - env: - TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu" - PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/" - run: | - # speed up by using infra cache services - CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local" - sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list - pip config set global.index-url http://${CACHING_URL}/pypi/simple - pip config set global.trusted-host "${CACHING_URL}" - - if [ ${{ needs.set-image-config.outputs.skip_install_flag }} != "true" ];then - bash scripts/ci/npu/npu_ci_install_dependency.sh a3 - fi - - # copy required file from our daily cache - cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp - # copy gsm8k dataset - cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp - - - name: Print Log Information - run: | - bash scripts/ci/npu/npu_log_print.sh - - - name: Run test - timeout-minutes: 240 - env: - SGLANG_USE_MODELSCOPE: true - SGLANG_IS_IN_CI: true - HF_ENDPOINT: https://hf-mirror.com - TORCH_EXTENSIONS_DIR: /tmp/torch_extensions - PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True" - STREAMS_PER_DEVICE: 32 - run: | - pip install sglang_router - hf download lmms-lab/MMMU --repo-type dataset - pip install sentence_transformers torchaudio==2.8.0 - pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap - pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1 - pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv - git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git - cd ./lmms-eval - nohup pip install . > lmmslog.txt 2>&1 & - sleep 120 - export PYTHONPATH=$PYTHONPATH:$(pwd) - cd ../ - cd test - python3 run_suite.py --hw npu --suite full-1-npu-a3 --nightly --continue-on-error --timeout-per-file 3600 + run_timeout_minutes: '240' + timeout_per_file: '3600' + ref: ${{ needs.set-image-config.outputs.ref }} + is_nightly_pipeline_job: true + skip_pr_test_health_check: 'true' + secrets: inherit full-2-npu-a3: - needs: [set-image-config] + name: full-2-npu-a3 if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }} - runs-on: linux-aarch64-a3-2 - container: + needs: [set-image-config] + uses: ./.github/workflows/_npu-pr-test-stage.yml + with: + self_name: full-2-npu-a3 + runner_config: linux-aarch64-a3-2 image: ${{ needs.set-image-config.outputs.image_a3 }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ needs.set-image-config.outputs.ref || github.ref }} - - - name: Install dependencies - env: - TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu" - PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/" - run: | - # speed up by using infra cache services - CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local" - sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list - pip config set global.index-url http://${CACHING_URL}/pypi/simple - pip config set global.trusted-host "${CACHING_URL}" - - if [ ${{ needs.set-image-config.outputs.skip_install_flag }} != "true" ];then - bash scripts/ci/npu/npu_ci_install_dependency.sh a3 - fi - - # copy required file from our daily cache - cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp - # copy gsm8k dataset - cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp - - - name: Print Log Information - run: | - bash scripts/ci/npu/npu_log_print.sh - - - name: Run test - timeout-minutes: 240 - env: - SGLANG_USE_MODELSCOPE: true - SGLANG_IS_IN_CI: true - HF_ENDPOINT: https://hf-mirror.com - TORCH_EXTENSIONS_DIR: /tmp/torch_extensions - PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True" - STREAMS_PER_DEVICE: 32 - run: | - pip install sglang_router - hf download lmms-lab/MMMU --repo-type dataset - pip install sentence_transformers torchaudio==2.8.0 - pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap - pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1 - pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv - git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git - cd ./lmms-eval - nohup pip install . > lmmslog.txt 2>&1 & - sleep 120 - export PYTHONPATH=$PYTHONPATH:$(pwd) - cd ../ - cd test - python3 run_suite.py --hw npu --suite full-2-npu-a3 --nightly --continue-on-error --timeout-per-file 3600 + run_timeout_minutes: '240' + timeout_per_file: '3600' + ref: ${{ needs.set-image-config.outputs.ref }} + is_nightly_pipeline_job: true + skip_pr_test_health_check: 'true' + secrets: inherit full-4-npu-a3: - needs: [set-image-config] + name: full-4-npu-a3 if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }} - runs-on: linux-aarch64-a3-4 - container: + needs: [set-image-config] + uses: ./.github/workflows/_npu-pr-test-stage.yml + with: + self_name: full-4-npu-a3 + runner_config: linux-aarch64-a3-4 image: ${{ needs.set-image-config.outputs.image_a3 }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ needs.set-image-config.outputs.ref || github.ref }} + run_timeout_minutes: '240' + timeout_per_file: '3600' + ref: ${{ needs.set-image-config.outputs.ref }} + is_nightly_pipeline_job: true + skip_pr_test_health_check: 'true' + secrets: inherit - - name: Install dependencies - env: - TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu" - PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/" - run: | - # speed up by using infra cache services - CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local" - sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list - pip config set global.index-url http://${CACHING_URL}/pypi/simple - pip config set global.trusted-host "${CACHING_URL}" - - if [ ${{ needs.set-image-config.outputs.skip_install_flag }} != "true" ];then - bash scripts/ci/npu/npu_ci_install_dependency.sh a3 - fi - - # copy required file from our daily cache - cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp - # copy gsm8k dataset - cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp - - - name: Print Log Information - run: | - bash scripts/ci/npu/npu_log_print.sh - - - name: Run test - timeout-minutes: 240 - env: - SGLANG_USE_MODELSCOPE: true - SGLANG_IS_IN_CI: true - HF_ENDPOINT: https://hf-mirror.com - TORCH_EXTENSIONS_DIR: /tmp/torch_extensions - PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True" - STREAMS_PER_DEVICE: 32 - run: | - pip install sglang_router - hf download lmms-lab/MMMU --repo-type dataset - pip install sentence_transformers torchaudio==2.8.0 - pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap - pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1 - pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv - git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git - cd ./lmms-eval - nohup pip install . > lmmslog.txt 2>&1 & - sleep 120 - export PYTHONPATH=$PYTHONPATH:$(pwd) - cd ../ - cd test - python3 run_suite.py --hw npu --suite full-4-npu-a3 --nightly --continue-on-error --timeout-per-file 3600 + full-8-npu-a3: + name: full-8-npu-a3 + if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }} + needs: [set-image-config] + uses: ./.github/workflows/_npu-pr-test-stage.yml + with: + self_name: full-8-npu-a3 + runner_config: linux-aarch64-a3-8 + image: ${{ needs.set-image-config.outputs.image_a3 }} + run_timeout_minutes: '240' + timeout_per_file: '3600' + ref: ${{ needs.set-image-config.outputs.ref }} + is_nightly_pipeline_job: true + skip_pr_test_health_check: 'true' + secrets: inherit full-16-npu-a3: - needs: [set-image-config] + name: full-16-npu-a3 if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }} - runs-on: linux-aarch64-a3-16 - container: + needs: [set-image-config] + uses: ./.github/workflows/_npu-pr-test-stage.yml + with: + self_name: full-16-npu-a3 + runner_config: linux-aarch64-a3-16 image: ${{ needs.set-image-config.outputs.image_a3 }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ needs.set-image-config.outputs.ref || github.ref }} + run_timeout_minutes: '240' + timeout_per_file: '3600' + ref: ${{ needs.set-image-config.outputs.ref }} + is_nightly_pipeline_job: true + skip_pr_test_health_check: 'true' + secrets: inherit - - name: Install dependencies - env: - TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu" - PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/" - run: | - # speed up by using infra cache services - CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local" - sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list - pip config set global.index-url http://${CACHING_URL}/pypi/simple - pip config set global.trusted-host "${CACHING_URL}" + full-acc-2-npu-a3: + name: full-acc-2-npu-a3 + if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }} + needs: [set-image-config] + uses: ./.github/workflows/_npu-single-node-test-stage.yml + with: + runner: linux-aarch64-a3-2- + test_type: 'accuracy' + test_suite: full-acc-2-npu-a3 + image: ${{ needs.set-image-config.outputs.image_a3 }} + is_nightly_pipeline_job: true + install_sglang_deps: true + device_type_for_deps: 'a3' + skip_pr_test_health_check: 'true' + secrets: inherit - if [ ${{ needs.set-image-config.outputs.skip_install_flag }} != "true" ];then - bash scripts/ci/npu/npu_ci_install_dependency.sh a3 - fi + full-acc-4-npu-a3: + name: full-acc-4-npu-a3 + if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }} + needs: [set-image-config] + uses: ./.github/workflows/_npu-single-node-test-stage.yml + with: + runner: linux-aarch64-a3-4- + test_type: 'accuracy' + test_suite: full-acc-4-npu-a3 + image: ${{ needs.set-image-config.outputs.image_a3 }} + is_nightly_pipeline_job: true + install_sglang_deps: true + device_type_for_deps: 'a3' + skip_pr_test_health_check: 'true' + secrets: inherit - # copy required file from our daily cache - cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp - # copy gsm8k dataset - cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp + full-acc-16-npu-a3: + name: full-acc-16-npu-a3 + if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }} + needs: [set-image-config] + uses: ./.github/workflows/_npu-single-node-test-stage.yml + with: + runner: linux-aarch64-a3-16- + test_type: 'accuracy' + test_suite: full-acc-16-npu-a3 + image: ${{ needs.set-image-config.outputs.image_a3 }} + is_nightly_pipeline_job: true + install_sglang_deps: true + device_type_for_deps: 'a3' + skip_pr_test_health_check: 'true' + secrets: inherit - - name: Print Log Information - run: | - bash scripts/ci/npu/npu_log_print.sh + full-perf-2-npu-a3: + name: full-perf-2-npu-a3 + if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }} + needs: [set-image-config] + uses: ./.github/workflows/_npu-pr-test-stage.yml + with: + self_name: full-perf-2-npu-a3 + runner_config: linux-aarch64-a3-800t-2 + image: ${{ needs.set-image-config.outputs.image_a3 }} + run_timeout_minutes: '240' + timeout_per_file: '3600' + ref: ${{ needs.set-image-config.outputs.ref }} + is_nightly_pipeline_job: true + skip_pr_test_health_check: 'true' + secrets: inherit - - name: Run test - timeout-minutes: 240 - env: - SGLANG_USE_MODELSCOPE: true - SGLANG_IS_IN_CI: true - HF_ENDPOINT: https://hf-mirror.com - TORCH_EXTENSIONS_DIR: /tmp/torch_extensions - PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True" - STREAMS_PER_DEVICE: 32 - run: | - pip install sglang_router - hf download lmms-lab/MMMU --repo-type dataset - pip install sentence_transformers torchaudio==2.8.0 - pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap - pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1 - pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv - git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git - cd ./lmms-eval - nohup pip install . > lmmslog.txt 2>&1 & - sleep 120 - export PYTHONPATH=$PYTHONPATH:$(pwd) - cd ../ - cd test - python3 run_suite.py --hw npu --suite full-16-npu-a3 --nightly --continue-on-error --timeout-per-file 3600 + full-perf-4-npu-a3: + name: full-perf-4-npu-a3 + if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }} + needs: [set-image-config] + uses: ./.github/workflows/_npu-pr-test-stage.yml + with: + self_name: full-perf-4-npu-a3 + runner_config: linux-aarch64-a3-800t-4 + image: ${{ needs.set-image-config.outputs.image_a3 }} + run_timeout_minutes: '240' + timeout_per_file: '3600' + ref: ${{ needs.set-image-config.outputs.ref }} + is_nightly_pipeline_job: true + skip_pr_test_health_check: 'true' + secrets: inherit + + full-perf-8-npu-a3: + name: full-perf-8-npu-a3 + if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }} + needs: [set-image-config] + uses: ./.github/workflows/_npu-pr-test-stage.yml + with: + self_name: full-perf-8-npu-a3 + runner_config: linux-aarch64-a3-800t-8 + image: ${{ needs.set-image-config.outputs.image_a3 }} + run_timeout_minutes: '240' + timeout_per_file: '3600' + ref: ${{ needs.set-image-config.outputs.ref }} + is_nightly_pipeline_job: true + skip_pr_test_health_check: 'true' + secrets: inherit + + full-perf-16-npu-a3: + name: full-perf-16-npu-a3 + if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }} + needs: [set-image-config] + uses: ./.github/workflows/_npu-pr-test-stage.yml + with: + self_name: full-perf-16-npu-a3 + runner_config: linux-aarch64-a3-800t-16 + image: ${{ needs.set-image-config.outputs.image_a3 }} + run_timeout_minutes: '240' + timeout_per_file: '3600' + ref: ${{ needs.set-image-config.outputs.ref }} + is_nightly_pipeline_job: true + skip_pr_test_health_check: 'true' + secrets: inherit + + full-poc-multi-node-mix-tests: + name: multi-node-mix-poc + if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }} + needs: [set-image-config] + strategy: + fail-fast: false + max-parallel: 1 + matrix: + test_config: + # kimi_k2_6 performance tests + - name: kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25 + node_size: 2 + test_case: test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py + test_type: 'accuracy' + uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml + with: + runner: linux-amd64-cpu-4 + test_type: ${{ matrix.test_config.test_type || 'perf' }} + test_config_name: ${{ matrix.test_config.name }} + node_size: ${{ matrix.test_config.node_size }} + test_case: ${{ matrix.test_config.test_case }} + image: ${{ needs.set-image-config.outputs.image_a3 }} + install_sglang_from_source: false + prefill_decode_deployment: 'mix' + transformers_version: '' check-all-jobs: if: github.repository == 'sgl-project/sglang' && always() needs: - nighly-test-npu + - full-4-npu-a2 - full-1-npu-a3 - full-2-npu-a3 - full-4-npu-a3 + - full-8-npu-a3 - full-16-npu-a3 + - full-acc-2-npu-a3 + - full-acc-4-npu-a3 + - full-acc-16-npu-a3 + - full-perf-2-npu-a3 + - full-perf-4-npu-a3 + - full-perf-8-npu-a3 + - full-perf-16-npu-a3 + - full-poc-multi-node-mix-tests runs-on: ubuntu-latest container: image: docker.m.daocloud.io/ubuntu:22.04 diff --git a/.github/workflows/nightly-test-npu.yml b/.github/workflows/nightly-test-npu.yml index cd29a6c16..1f8c27142 100644 --- a/.github/workflows/nightly-test-npu.yml +++ b/.github/workflows/nightly-test-npu.yml @@ -122,22 +122,6 @@ jobs: # Write to GITHUB_OUTPUT and print to the log in one command. echo "run_start_metadata=${RUN_START_METADATA}" | tee -a $GITHUB_OUTPUT - nightly-1-npu-a2: - name: nightly-1-npu-a2 - if: ${{ !cancelled() }} - needs: [set-image-config] - uses: ./.github/workflows/_npu-single-node-test-stage.yml - with: - runner: linux-aarch64-a2-4 - test_type: 'perf' - test_suite: nightly-1-npu-a2 - is_nightly_pipeline_job: true - skip_pr_test_health_check: 'true' - image: ${{ needs.set-image-config.outputs.image_a2 }} - install_sglang_deps: false - device_type_for_deps: '910b' - run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }} - nightly-1-npu-a3: name: nightly-1-npu-a3 if: ${{ !cancelled() }} @@ -247,22 +231,6 @@ jobs: device_type_for_deps: 'a3' run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }} - nightly-perf-8-npu-a3: - name: nightly-perf-8-npu-a3 - if: ${{ !cancelled() }} - needs: [set-image-config] - uses: ./.github/workflows/_npu-single-node-test-stage.yml - with: - runner: linux-aarch64-a3-800t-8 - test_type: 'perf' - test_suite: nightly-perf-8-npu-a3 - is_nightly_pipeline_job: true - skip_pr_test_health_check: 'true' - image: ${{ needs.set-image-config.outputs.image_a3 }} - install_sglang_deps: false - device_type_for_deps: 'a3' - run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }} - nightly-perf-16-npu-a3: name: nightly-perf-16-npu-a3 if: ${{ !cancelled() }} @@ -298,22 +266,6 @@ jobs: device_type_for_deps: 'a3' run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }} - nightly-acc-4-npu-a3: - name: nightly-acc-4-npu-a3 - if: ${{ !cancelled() }} - needs: [set-image-config] - uses: ./.github/workflows/_npu-single-node-test-stage.yml - with: - runner: linux-aarch64-a3-4- - test_type: 'accuracy' - test_suite: nightly-acc-4-npu-a3 - is_nightly_pipeline_job: true - skip_pr_test_health_check: 'true' - image: ${{ needs.set-image-config.outputs.image_a3 }} - install_sglang_deps: false - device_type_for_deps: 'a3' - run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }} - nightly-acc-16-npu-a3: name: nightly-acc-16-npu-a3 if: ${{ !cancelled() }} @@ -333,7 +285,7 @@ jobs: nightly-poc-multi-node-tests: name: multi-node-poc if: ${{ !cancelled() }} - needs: [set-image-config, nightly-perf-2-npu-a3, nightly-perf-4-npu-a3, nightly-perf-16-npu-a3, nightly-acc-2-npu-a3, nightly-acc-4-npu-a3, nightly-acc-16-npu-a3] + needs: [set-image-config, nightly-perf-2-npu-a3, nightly-perf-4-npu-a3, nightly-perf-16-npu-a3, nightly-acc-2-npu-a3, nightly-acc-16-npu-a3] strategy: fail-fast: false max-parallel: 1 @@ -354,21 +306,6 @@ jobs: test_case: test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py test_type: 'accuracy' prefill_decode_deployment: 'separation' - # mimo_v2_flash performance tests - - name: test_npu_mimo_v2_flash_1p1d_12p_in16k_out1_ttft_5s - prefill_size: 1 - decode_size: 1 - router_size: 1 - test_case: test/registered/npu/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1_ttft_5s.py - test_type: 'perf' - prefill_decode_deployment: 'separation' - - name: test_npu_mimo_v2_flash_1p1d_12p_in16k_out1k_tpot_20ms - prefill_size: 1 - decode_size: 1 - router_size: 1 - test_case: test/registered/npu/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1k_tpot_20ms.py - test_type: 'perf' - prefill_decode_deployment: 'separation' # deepseek_v4_flash performance tests - name: deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms prefill_size: 1 @@ -395,21 +332,12 @@ jobs: nightly-poc-multi-node-mix-tests: name: multi-node-mix-poc if: ${{ !cancelled() }} - needs: [set-image-config, nightly-perf-2-npu-a3, nightly-perf-4-npu-a3, nightly-perf-16-npu-a3, nightly-acc-2-npu-a3, nightly-acc-4-npu-a3, nightly-acc-16-npu-a3, nightly-poc-multi-node-tests] + needs: [set-image-config, nightly-perf-2-npu-a3, nightly-perf-4-npu-a3, nightly-perf-16-npu-a3, nightly-acc-2-npu-a3, nightly-acc-16-npu-a3, nightly-poc-multi-node-tests] strategy: fail-fast: false max-parallel: 1 matrix: test_config: - # kimi_k2_6 performance tests - - name: kimi_k2_6_w4a8_16p_in64k_out1k_100ms - node_size: 2 - test_case: test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms.py - test_type: 'perf' - - name: kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25 - node_size: 2 - test_case: test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py - test_type: 'accuracy' # glm_5_2 accuracy tests - name: glm_5_2_w4a8_16p_gpqa node_size: 2 @@ -436,7 +364,6 @@ jobs: check-all-jobs: if: ${{ !cancelled() }} needs: - - nightly-1-npu-a2 - nightly-1-npu-a3 - nightly-2-npu-a3 - nightly-4-npu-a3 @@ -444,10 +371,8 @@ jobs: - nightly-16-npu-a3 - nightly-perf-2-npu-a3 - nightly-perf-4-npu-a3 - - nightly-perf-8-npu-a3 - nightly-perf-16-npu-a3 - nightly-acc-2-npu-a3 - - nightly-acc-4-npu-a3 - nightly-acc-16-npu-a3 - nightly-poc-multi-node-tests - nightly-poc-multi-node-mix-tests @@ -455,7 +380,6 @@ jobs: steps: - name: Generate results table run: | - single_result_a2="${{ needs.nightly-1-npu-a2.result }}" multi_result="${{ needs.nightly-poc-multi-node-tests.result }}" mix_result="${{ needs.nightly-poc-multi-node-mix-tests.result }}" @@ -469,10 +393,8 @@ jobs: "${{ needs.nightly-16-npu-a3.result }}" \ "${{ needs.nightly-perf-2-npu-a3.result }}" \ "${{ needs.nightly-perf-4-npu-a3.result }}" \ - "${{ needs.nightly-perf-8-npu-a3.result }}" \ "${{ needs.nightly-perf-16-npu-a3.result }}" \ "${{ needs.nightly-acc-2-npu-a3.result }}" \ - "${{ needs.nightly-acc-4-npu-a3.result }}" \ "${{ needs.nightly-acc-16-npu-a3.result }}"; do if [ "${r}" != "success" ] && [ "${r}" != "skipped" ]; then single_result="failure" @@ -493,7 +415,6 @@ jobs: echo "" >> $GITHUB_STEP_SUMMARY echo "| Group | Status |" >> $GITHUB_STEP_SUMMARY echo "|-------|--------|" >> $GITHUB_STEP_SUMMARY - echo "| nightly-1-npu-a2 | $(group_icon ${single_result_a2}) ${single_result_a2} |" >> $GITHUB_STEP_SUMMARY for entry in \ "nightly-1-npu-a3:${{ needs.nightly-1-npu-a3.result }}" \ "nightly-2-npu-a3:${{ needs.nightly-2-npu-a3.result }}" \ @@ -502,10 +423,8 @@ jobs: "nightly-16-npu-a3:${{ needs.nightly-16-npu-a3.result }}" \ "nightly-perf-2-npu-a3:${{ needs.nightly-perf-2-npu-a3.result }}" \ "nightly-perf-4-npu-a3:${{ needs.nightly-perf-4-npu-a3.result }}" \ - "nightly-perf-8-npu-a3:${{ needs.nightly-perf-8-npu-a3.result }}" \ "nightly-perf-16-npu-a3:${{ needs.nightly-perf-16-npu-a3.result }}" \ "nightly-acc-2-npu-a3:${{ needs.nightly-acc-2-npu-a3.result }}" \ - "nightly-acc-4-npu-a3:${{ needs.nightly-acc-4-npu-a3.result }}" \ "nightly-acc-16-npu-a3:${{ needs.nightly-acc-16-npu-a3.result }}"; do suite="${entry%%:*}" r="${entry##*:}" @@ -516,7 +435,7 @@ jobs: echo "" >> $GITHUB_STEP_SUMMARY FAIL=0 - for r in "${single_result_a2}" "${single_result}" "${multi_result}" "${mix_result}"; do + for r in "${single_result}" "${multi_result}" "${mix_result}"; do if [ "${r}" != "success" ] && [ "${r}" != "skipped" ]; then FAIL=1; fi done exit $FAIL diff --git a/test/registered/npu/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py b/test/registered/npu/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py index b6ac070b7..19e537ddc 100644 --- a/test/registered/npu/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py +++ b/test/registered/npu/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py @@ -8,7 +8,7 @@ from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( est_time=4800, - suite="nightly-acc-16-npu-a3", + suite="full-acc-16-npu-a3", nightly=True, ) diff --git a/test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py b/test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py index f3c135924..670274c63 100644 --- a/test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py +++ b/test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py @@ -8,7 +8,7 @@ from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( est_time=6500, - suite="nightly-acc-2-npu-a3", + suite="full-acc-2-npu-a3", nightly=True, ) diff --git a/test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_gsm8k.py b/test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_gsm8k.py index df8b7be29..a0a14628e 100644 --- a/test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_gsm8k.py +++ b/test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_gsm8k.py @@ -6,7 +6,7 @@ from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( from sglang.test.ascend.e2e.test_npu_performance_utils import GLM_4_7_FLASH_MODEL_PATH from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci(est_time=6500, suite="nightly-acc-2-npu-a3", nightly=True) +register_npu_ci(est_time=6500, suite="full-acc-2-npu-a3", nightly=True) ENVS = { "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", diff --git a/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py b/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py index 955f656f2..617b5ccc9 100644 --- a/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py +++ b/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py @@ -12,7 +12,7 @@ from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( est_time=7200, - suite="nightly-acc-16-npu-a3", + suite="full-acc-16-npu-a3", nightly=True, ) diff --git a/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py b/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py index 21eba280f..825b28f66 100644 --- a/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py +++ b/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py @@ -12,7 +12,7 @@ from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( est_time=4800, - suite="nightly-acc-16-npu-a3", + suite="full-acc-16-npu-a3", nightly=True, ) diff --git a/test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py b/test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py index abae77090..b7e342248 100644 --- a/test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py +++ b/test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py @@ -11,7 +11,7 @@ from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( est_time=3700, - suite="nightly-acc-2-npu-a3", + suite="full-acc-2-npu-a3", nightly=True, ) diff --git a/test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py b/test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py index e31bfe394..534d7488e 100644 --- a/test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py +++ b/test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py @@ -11,7 +11,7 @@ from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( est_time=3700, - suite="nightly-acc-2-npu-a3", + suite="full-acc-2-npu-a3", nightly=True, ) diff --git a/test/registered/npu/accuracy/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py b/test/registered/npu/accuracy/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py index 7e4e4d0de..54f0ddc81 100644 --- a/test/registered/npu/accuracy/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py +++ b/test/registered/npu/accuracy/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py @@ -11,7 +11,7 @@ from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( est_time=2800, - suite="nightly-acc-2-npu-a3", + suite="full-acc-2-npu-a3", nightly=True, ) diff --git a/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py b/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py index e8776f041..437907bfe 100644 --- a/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py +++ b/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py @@ -11,7 +11,7 @@ from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( est_time=4800, - suite="nightly-acc-16-npu-a3", + suite="full-acc-16-npu-a3", nightly=True, ) diff --git a/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py b/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py index b1ee89074..7fc8e9fec 100644 --- a/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py +++ b/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py @@ -11,7 +11,7 @@ from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( est_time=4800, - suite="nightly-acc-4-npu-a3", + suite="full-acc-4-npu-a3", nightly=True, ) diff --git a/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa_a2.py b/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa_a2.py index b46999437..a0aaae51a 100644 --- a/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa_a2.py +++ b/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa_a2.py @@ -11,7 +11,7 @@ from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( est_time=4800, - suite="nightly-1-npu-a2", + suite="full-4-npu-a2", nightly=True, ) diff --git a/test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py b/test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py index e1b3eb4d9..1aa88934e 100644 --- a/test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py +++ b/test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py @@ -8,7 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( ) from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci(est_time=2800, suite="nightly-acc-2-npu-a3", nightly=True) +register_npu_ci(est_time=2800, suite="full-acc-2-npu-a3", nightly=True) QWEN3_5_9B_ENVS = { "SGLANG_SET_CPU_AFFINITY": "1", diff --git a/test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py b/test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py index 3cfbba230..7acc6e50d 100644 --- a/test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py +++ b/test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py @@ -11,7 +11,7 @@ from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( est_time=4800, - suite="nightly-acc-4-npu-a3", + suite="full-acc-4-npu-a3", nightly=True, ) diff --git a/test/registered/npu/accuracy/qwen3_vl_30b_a3b/test_npu_qwen3_vl_30b_a3b_bf16_2p_gsm8k.py b/test/registered/npu/accuracy/qwen3_vl_30b_a3b/test_npu_qwen3_vl_30b_a3b_bf16_2p_gsm8k.py index 068e360e6..066dd31a1 100644 --- a/test/registered/npu/accuracy/qwen3_vl_30b_a3b/test_npu_qwen3_vl_30b_a3b_bf16_2p_gsm8k.py +++ b/test/registered/npu/accuracy/qwen3_vl_30b_a3b/test_npu_qwen3_vl_30b_a3b_bf16_2p_gsm8k.py @@ -8,7 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( ) from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci(est_time=4800, suite="nightly-acc-4-npu-a3", nightly=True) +register_npu_ci(est_time=4800, suite="full-acc-4-npu-a3", nightly=True) QWEN3_VL_30B_A3B_ENVS = { "SGLANG_SET_CPU_AFFINITY": "1", diff --git a/test/registered/npu/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py b/test/registered/npu/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py index 7dafc8a55..47d17e00d 100644 --- a/test/registered/npu/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py +++ b/test/registered/npu/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py @@ -10,7 +10,7 @@ from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( est_time=8400, - suite="nightly-acc-2-npu-a3", + suite="full-acc-2-npu-a3", nightly=True, ) diff --git a/test/registered/npu/accuracy/qwen3_vl_8b/test_npu_qwen3_vl_8b_bf16_2p_gsm8k.py b/test/registered/npu/accuracy/qwen3_vl_8b/test_npu_qwen3_vl_8b_bf16_2p_gsm8k.py index 59c77f45e..ae7487597 100644 --- a/test/registered/npu/accuracy/qwen3_vl_8b/test_npu_qwen3_vl_8b_bf16_2p_gsm8k.py +++ b/test/registered/npu/accuracy/qwen3_vl_8b/test_npu_qwen3_vl_8b_bf16_2p_gsm8k.py @@ -8,7 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( ) from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci(est_time=4800, suite="nightly-acc-4-npu-a3", nightly=True) +register_npu_ci(est_time=4800, suite="full-acc-4-npu-a3", nightly=True) QWEN3_VL_8B_ENVS = { "SGLANG_SET_CPU_AFFINITY": "1", diff --git a/test/registered/npu/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py b/test/registered/npu/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py index 854f495e1..ca4195c31 100644 --- a/test/registered/npu/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py +++ b/test/registered/npu/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py @@ -9,7 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( ) from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci(est_time=12000, suite="nightly-acc-2-npu-a3", nightly=True) +register_npu_ci(est_time=12000, suite="full-acc-2-npu-a3", nightly=True) _is_pr_pipeline = os.environ.get("GITHUB_EVENT_NAME") == "pull_request" diff --git a/test/registered/npu/embedding_models/test_npu_bge_large_en_v1_5.py b/test/registered/npu/embedding_models/test_npu_bge_large_en_v1_5.py deleted file mode 100644 index d5a6e4e39..000000000 --- a/test/registered/npu/embedding_models/test_npu_bge_large_en_v1_5.py +++ /dev/null @@ -1,111 +0,0 @@ -import multiprocessing as mp -import unittest -from typing import Optional - -import torch -from transformers import AutoConfig, AutoTokenizer - -from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.runners import HFRunner, SRTRunner -from sglang.test.test_utils import CustomTestCase, get_similarities - -register_npu_ci( - est_time=400, - suite="full-1-npu-a3", - nightly=True, - disabled="embeddings are not all close", -) - -DEFAULT_PROMPTS = [ - "The capital of the United Kingdom is", - "Today is a sunny day and I like", - "AI is a field of computer science focused on", -] - -MODELS = [ - ("/root/.cache/modelscope/hub/models/bge-large-en-v1.5", 1, 1e-5), -] -TORCH_DTYPES = [torch.float16] - - -class TestEmbeddingModels(CustomTestCase): - @classmethod - def setUpClass(cls): - mp.set_start_method("spawn", force=True) - - def _truncate_prompts(self, prompts, model_path): - config = AutoConfig.from_pretrained(model_path) - max_length = getattr(config, "max_position_embeddings", 2048) - - tokenizer = AutoTokenizer.from_pretrained(model_path) - - truncated_prompts = [] - for prompt in prompts: - tokens = tokenizer(prompt, return_tensors="pt", truncation=False) - if len(tokens.input_ids[0]) > max_length: - truncated_text = tokenizer.decode( - tokens.input_ids[0][: max_length - 1], skip_special_tokens=True - ) - truncated_prompts.append(truncated_text) - else: - truncated_prompts.append(prompt) - return truncated_prompts - - def assert_close_prefill_logits( - self, - prompts, - model_path, - tp_size, - torch_dtype, - prefill_tolerance, - matryoshka_dim: Optional[int] = None, - ) -> None: - truncated_prompts = self._truncate_prompts(prompts, model_path) - - with HFRunner( - model_path, - torch_dtype=torch_dtype, - model_type="embedding", - matryoshka_dim=matryoshka_dim, - ) as hf_runner: - hf_outputs = hf_runner.forward(truncated_prompts) - - attention_backend = "ascend" - with SRTRunner( - model_path, - tp_size=tp_size, - torch_dtype=torch_dtype, - model_type="embedding", - attention_backend=attention_backend, - json_model_override_args=( - {"matryoshka_dimensions": [matryoshka_dim]} if matryoshka_dim else None - ), - ) as srt_runner: - srt_outputs = srt_runner.forward( - truncated_prompts, dimensions=matryoshka_dim - ) - - for i in range(len(prompts)): - hf_logits = torch.Tensor(hf_outputs.embed_logits[i]) - srt_logits = torch.Tensor(srt_outputs.embed_logits[i]) - - similarity = torch.tensor(get_similarities(hf_logits, srt_logits)) - print("similarity diff", abs(similarity - 1)) - - if len(prompts[i]) <= 1000: - assert torch.all(abs(similarity - 1) < prefill_tolerance), ( - "embeddings are not all close" - ) - - def test_prefill_logits(self): - models_to_test = MODELS - - for model, tp_size, prefill_tolerance in models_to_test: - for torch_dtype in TORCH_DTYPES: - self.assert_close_prefill_logits( - DEFAULT_PROMPTS, model, tp_size, torch_dtype, prefill_tolerance - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/llm_models/test_ascend_dbrx_instruct.py b/test/registered/npu/llm_models/test_ascend_dbrx_instruct.py deleted file mode 100644 index 3ae7bb19f..000000000 --- a/test/registered/npu/llm_models/test_ascend_dbrx_instruct.py +++ /dev/null @@ -1,26 +0,0 @@ -import unittest - -from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin -from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.test_utils import CustomTestCase - -register_npu_ci(est_time=400, suite="full-8-npu-a3", nightly=True) - - -class TestDbrx(GSM8KAscendMixin, CustomTestCase): - model = "/root/.cache/modelscope/hub/models/AI-ModelScope/dbrx-instruct" - accuracy = 0.735 - other_args = [ - "--trust-remote-code", - "--mem-fraction-static", - "0.8", - "--attention-backend", - "ascend", - "--disable-cuda-graph", - "--tp-size", - "8", - ] - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/llm_models/test_npu_baichuan2_13b_chat.py b/test/registered/npu/llm_models/test_npu_baichuan2_13b_chat.py deleted file mode 100644 index 19cc857d0..000000000 --- a/test/registered/npu/llm_models/test_npu_baichuan2_13b_chat.py +++ /dev/null @@ -1,37 +0,0 @@ -import unittest - -from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin -from sglang.test.ascend.test_ascend_utils import BAICHUAN2_13B_CHAT_WEIGHTS_PATH -from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.test_utils import CustomTestCase - -register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) - - -class TestBaichuan(GSM8KAscendMixin, CustomTestCase): - """Testcase: Verify that the inference accuracy of the baichuan-inc/Baichuan2-13B-Chat model on the GSM8K dataset is no less than 0.48. - - [Test Category] Model - [Test Target] baichuan-inc/Baichuan2-13B-Chat - """ - - model = BAICHUAN2_13B_CHAT_WEIGHTS_PATH - accuracy = 0.48 - other_args = [ - "--trust-remote-code", - "--mem-fraction-static", - "0.8", - "--attention-backend", - "ascend", - "--disable-cuda-graph", - "--max-running-requests", - "128", - "--disable-radix-cache", - "--chunked-prefill-size", - "-1", - ] - gsm8k_num_shots = 1 - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/llm_models/test_npu_c4ai_command_r_v01.py b/test/registered/npu/llm_models/test_npu_c4ai_command_r_v01.py deleted file mode 100644 index 1af86dfbf..000000000 --- a/test/registered/npu/llm_models/test_npu_c4ai_command_r_v01.py +++ /dev/null @@ -1,40 +0,0 @@ -import unittest - -from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin -from sglang.test.ascend.test_ascend_utils import ( - C4AI_COMMAND_R_V01_CHAT_TEMPLATE_PATH, - C4AI_COMMAND_R_V01_WEIGHTS_PATH, -) -from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.test_utils import CustomTestCase - -register_npu_ci(est_time=400, suite="full-2-npu-a3", nightly=True) - - -class TestC4AI(GSM8KAscendMixin, CustomTestCase): - """Testcase: Verify that the inference accuracy of the CohereForAI/c4ai-command-r-v01 model on the GSM8K dataset is no less than 0.55. - - [Test Category] Model - [Test Target] CohereForAI/c4ai-command-r-v01 - """ - - model = C4AI_COMMAND_R_V01_WEIGHTS_PATH - accuracy = 0.55 - other_args = [ - "--trust-remote-code", - "--mem-fraction-static", - "0.8", - "--attention-backend", - "ascend", - "--disable-cuda-graph", - "--chat-template", - C4AI_COMMAND_R_V01_CHAT_TEMPLATE_PATH, - "--tp-size", - "2", - "--dtype", - "bfloat16", - ] - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/llm_models/test_npu_chatglm2_6b.py b/test/registered/npu/llm_models/test_npu_chatglm2_6b.py deleted file mode 100644 index 92a430db3..000000000 --- a/test/registered/npu/llm_models/test_npu_chatglm2_6b.py +++ /dev/null @@ -1,33 +0,0 @@ -import unittest - -from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin -from sglang.test.ascend.test_ascend_utils import CHATGLM2_6B_WEIGHTS_PATH -from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.test_utils import CustomTestCase - -register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) - - -class TestChatGlm2(GSM8KAscendMixin, CustomTestCase): - """Testcase: Verify that the inference accuracy of the ZhipuAI/chatglm2-6b model on the GSM8K dataset is no less than 0.25. - - [Test Category] Model - [Test Target] ZhipuAI/chatglm2-6b - """ - - model = CHATGLM2_6B_WEIGHTS_PATH - accuracy = 0.25 - other_args = [ - "--trust-remote-code", - "--mem-fraction-static", - "0.8", - "--attention-backend", - "ascend", - "--disable-cuda-graph", - "--dtype", - "bfloat16", - ] - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/llm_models/test_npu_exaone_3.py b/test/registered/npu/llm_models/test_npu_exaone_3.py deleted file mode 100644 index f658afce7..000000000 --- a/test/registered/npu/llm_models/test_npu_exaone_3.py +++ /dev/null @@ -1,34 +0,0 @@ -import unittest - -from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin -from sglang.test.ascend.test_ascend_utils import EXAONE_3_5_7_8B_INSTRUCT_WEIGHTS_PATH -from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.test_utils import CustomTestCase - -register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) - - -class TestEXAONE(GSM8KAscendMixin, CustomTestCase): - """Testcase: Verify that the inference accuracy of the LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct model on the GSM8K dataset is no less than 0.8. - - [Test Category] Model - [Test Target] LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct - """ - - model = EXAONE_3_5_7_8B_INSTRUCT_WEIGHTS_PATH - # Allow 1% tolerance for the accuracy threshold - accuracy = round(0.8 * 0.99, 3) - other_args = [ - "--trust-remote-code", - "--mem-fraction-static", - "0.8", - "--attention-backend", - "ascend", - "--disable-cuda-graph", - "--dtype", - "bfloat16", - ] - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/llm_models/test_npu_glm4_9b_chat.py b/test/registered/npu/llm_models/test_npu_glm4_9b_chat.py deleted file mode 100644 index 4864ca0ff..000000000 --- a/test/registered/npu/llm_models/test_npu_glm4_9b_chat.py +++ /dev/null @@ -1,27 +0,0 @@ -import unittest - -from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin -from sglang.test.ascend.test_ascend_utils import GLM_4_9B_CHAT_WEIGHTS_PATH -from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.test_utils import CustomTestCase - -register_npu_ci( - est_time=400, - suite="full-1-npu-a3", - nightly=True, -) - - -class TestGLM49BChat(GSM8KAscendMixin, CustomTestCase): - """Testcase: Verify that the inference accuracy of the ZhipuAI/glm-4-9b-chat model on the GSM8K dataset is no less than 0.77. - - [Test Category] Model - [Test Target] ZhipuAI/glm-4-9b-chat - """ - - model = GLM_4_9B_CHAT_WEIGHTS_PATH - accuracy = 0.77 - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/llm_models/test_npu_granite_3_0_3b_a800m.py b/test/registered/npu/llm_models/test_npu_granite_3_0_3b_a800m.py deleted file mode 100644 index 9552e3ad9..000000000 --- a/test/registered/npu/llm_models/test_npu_granite_3_0_3b_a800m.py +++ /dev/null @@ -1,25 +0,0 @@ -import unittest - -from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin -from sglang.test.ascend.test_ascend_utils import ( - GRANITE_3_0_3B_A800M_INSTRUCT_WEIGHTS_PATH, -) -from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.test_utils import CustomTestCase - -register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) - - -class TestGranite(GSM8KAscendMixin, CustomTestCase): - """Testcase: Verify that the inference accuracy of the ibm-granite/granite-3.0-3b-a800m-instruct model on the GSM8K dataset is no less than 0.38. - - [Test Category] Model - [Test Target] ibm-granite/granite-3.0-3b-a800m-instruct - """ - - model = GRANITE_3_0_3B_A800M_INSTRUCT_WEIGHTS_PATH - accuracy = 0.38 - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/llm_models/test_npu_granite_3_1_8b.py b/test/registered/npu/llm_models/test_npu_granite_3_1_8b.py deleted file mode 100644 index 1ef751dd0..000000000 --- a/test/registered/npu/llm_models/test_npu_granite_3_1_8b.py +++ /dev/null @@ -1,23 +0,0 @@ -import unittest - -from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin -from sglang.test.ascend.test_ascend_utils import GRANITE_3_1_8B_INSTRUCT_WEIGHTS_PATH -from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.test_utils import CustomTestCase - -register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) - - -class TestGranite(GSM8KAscendMixin, CustomTestCase): - """Testcase: Verify that the inference accuracy of the ibm-granite/granite-3.1-8b-instruct model on the GSM8K dataset is no less than 0.695. - - [Test Category] Model - [Test Target] ibm-granite/granite-3.1-8b-instruct - """ - - model = GRANITE_3_1_8B_INSTRUCT_WEIGHTS_PATH - accuracy = 0.695 - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/llm_models/test_npu_grok_2.py b/test/registered/npu/llm_models/test_npu_grok_2.py deleted file mode 100644 index 398c4c3eb..000000000 --- a/test/registered/npu/llm_models/test_npu_grok_2.py +++ /dev/null @@ -1,33 +0,0 @@ -import unittest - -from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin -from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.test_utils import CustomTestCase - -register_npu_ci( - est_time=400, - suite="full-16-npu-a3", - nightly=True, -) - - -class TestGrok2(GSM8KAscendMixin, CustomTestCase): - model = "/root/.cache/modelscope/hub/models/huihui-ai/grok-2" - accuracy = 0.91 - other_args = [ - "--trust-remote-code", - "--mem-fraction-static", - "0.8", - "--attention-backend", - "ascend", - "--disable-radix-cache", - "--disable-cuda-graph", - "--tokenizer-path", - "/root/.cache/modelscope/hub/models/huihui-ai/grok-2/tokenizer.tok.json", - "--tp-size", - "16", - ] - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/llm_models/test_npu_internlm2_7b.py b/test/registered/npu/llm_models/test_npu_internlm2_7b.py deleted file mode 100644 index 3812e5234..000000000 --- a/test/registered/npu/llm_models/test_npu_internlm2_7b.py +++ /dev/null @@ -1,29 +0,0 @@ -import os -import unittest - -from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin -from sglang.test.ascend.test_ascend_utils import INTERNLM2_7B_WEIGHTS_PATH -from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.test_utils import CustomTestCase - -register_npu_ci( - est_time=400, - suite="full-1-npu-a3", - nightly=True, -) - - -class TestInternlm2(GSM8KAscendMixin, CustomTestCase): - """Testcase: Verify that the inference accuracy of the Shanghai_AI_Laboratory/internlm2-7b model on the GSM8K dataset is no less than 0.585. - - [Test Category] Model - [Test Target] Shanghai_AI_Laboratory/internlm2-7b - """ - - os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python" - model = INTERNLM2_7B_WEIGHTS_PATH - accuracy = 0.585 - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/llm_models/test_npu_llama_2_7b.py b/test/registered/npu/llm_models/test_npu_llama_2_7b.py deleted file mode 100644 index 579aea704..000000000 --- a/test/registered/npu/llm_models/test_npu_llama_2_7b.py +++ /dev/null @@ -1,23 +0,0 @@ -import unittest - -from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin -from sglang.test.ascend.test_ascend_utils import LLAMA_2_7B_WEIGHTS_PATH -from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.test_utils import CustomTestCase - -register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) - - -class TestLlama(GSM8KAscendMixin, CustomTestCase): - """Testcase: Verify that the inference accuracy of the LLM-Research/Llama-2-7B model on the GSM8K dataset is no less than 0.18. - - [Test Category] Model - [Test Target] LLM-Research/Llama-2-7B - """ - - model = LLAMA_2_7B_WEIGHTS_PATH - accuracy = 0.18 - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/llm_models/test_npu_llama_2_7b_communications_compression.py b/test/registered/npu/llm_models/test_npu_llama_2_7b_communications_compression.py deleted file mode 100644 index 0c8374d54..000000000 --- a/test/registered/npu/llm_models/test_npu_llama_2_7b_communications_compression.py +++ /dev/null @@ -1,37 +0,0 @@ -import unittest - -from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin -from sglang.test.ascend.test_ascend_utils import LLAMA_2_7B_WEIGHTS_PATH -from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.test_utils import CustomTestCase - -register_npu_ci(est_time=400, suite="full-2-npu-a3", nightly=True) - - -class TestLlama(GSM8KAscendMixin, CustomTestCase): - """Testcase: Verify that the inference accuracy of the LLM-Research/Llama-2-7B model on the GSM8K dataset with tp communications quantization is no less than 0.18. - - [Test Category] Model - [Test Target] LLM-Research/Llama-2-7B - """ - - model = LLAMA_2_7B_WEIGHTS_PATH - accuracy = 0.18 - other_args = [ - "--trust-remote-code", - "--mem-fraction-static", - 0.8, - "--max-running-requests", - 32, - "--attention-backend", - "ascend", - "--cuda-graph-max-bs-decode", - 32, - "--tp-size", - 2, - "--enable-quant-communications", - ] - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/llm_models/test_npu_minicpm3_4b.py b/test/registered/npu/llm_models/test_npu_minicpm3_4b.py deleted file mode 100644 index 362bbb9b3..000000000 --- a/test/registered/npu/llm_models/test_npu_minicpm3_4b.py +++ /dev/null @@ -1,37 +0,0 @@ -import unittest - -from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin -from sglang.test.ascend.test_ascend_utils import MINICPM3_4B_WEIGHTS_PATH -from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.test_utils import CustomTestCase - -register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) - - -class TestMiniCPM3(GSM8KAscendMixin, CustomTestCase): - """Testcase: Verify that the inference accuracy of the OpenBMB/MiniCPM3-4B model on the GSM8K dataset is no less than 0.69. - - [Test Category] Model - [Test Target] OpenBMB/MiniCPM3-4B - """ - - model = MINICPM3_4B_WEIGHTS_PATH - accuracy = 0.69 - other_args = [ - "--trust-remote-code", - "--mem-fraction-static", - "0.8", - "--attention-backend", - "ascend", - "--disable-cuda-graph", - "--disable-radix-cache", - "--disable-overlap-schedule", - "--max-running-requests", - "128", - "--chunked-prefill-size", - "-1", - ] - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/llm_models/test_npu_mistral_7b.py b/test/registered/npu/llm_models/test_npu_mistral_7b.py deleted file mode 100644 index 5c5e41b0a..000000000 --- a/test/registered/npu/llm_models/test_npu_mistral_7b.py +++ /dev/null @@ -1,23 +0,0 @@ -import unittest - -from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin -from sglang.test.ascend.test_ascend_utils import MISTRAL_7B_INSTRUCT_V0_2_WEIGHTS_PATH -from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.test_utils import CustomTestCase - -register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) - - -class TestMistral7B(GSM8KAscendMixin, CustomTestCase): - """Testcase: Verify that the inference accuracy of the mistralai/Mistral-7B-Instruct-v0.2 model on the GSM8K dataset is no less than 0.375. - - [Test Category] Model - [Test Target] mistralai/Mistral-7B-Instruct-v0.2 - """ - - model = MISTRAL_7B_INSTRUCT_V0_2_WEIGHTS_PATH - accuracy = 0.375 - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/llm_models/test_npu_persimmon_8b_chat.py b/test/registered/npu/llm_models/test_npu_persimmon_8b_chat.py deleted file mode 100644 index 21d9722c8..000000000 --- a/test/registered/npu/llm_models/test_npu_persimmon_8b_chat.py +++ /dev/null @@ -1,29 +0,0 @@ -import os -import unittest - -from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin -from sglang.test.ascend.test_ascend_utils import PERSIMMON_8B_CHAT_WEIGHTS_PATH -from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.test_utils import CustomTestCase - -register_npu_ci( - est_time=400, - suite="full-1-npu-a3", - nightly=True, -) - - -class TestPersimmon8BChat(GSM8KAscendMixin, CustomTestCase): - """Testcase: Verify that the inference accuracy of the Howeee/persimmon-8b-chat model on the GSM8K dataset is no less than 0.17. - - [Test Category] Model - [Test Target] Howeee/persimmon-8b-chat - """ - - os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python" - model = PERSIMMON_8B_CHAT_WEIGHTS_PATH - accuracy = 0.17 - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/llm_models/test_npu_smollm_1_7b.py b/test/registered/npu/llm_models/test_npu_smollm_1_7b.py deleted file mode 100644 index dbe1bcc1c..000000000 --- a/test/registered/npu/llm_models/test_npu_smollm_1_7b.py +++ /dev/null @@ -1,33 +0,0 @@ -import unittest - -from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin -from sglang.test.ascend.test_ascend_utils import SMOLLM_1_7B_WEIGHTS_PATH -from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.test_utils import CustomTestCase - -register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) - - -class TestSmolLM(GSM8KAscendMixin, CustomTestCase): - """Testcase: Verify that the inference accuracy of the HuggingFaceTB/SmolLM-1.7B model on the GSM8K dataset is no less than 0.05. - - [Test Category] Model - [Test Target] HuggingFaceTB/SmolLM-1.7B - """ - - model = SMOLLM_1_7B_WEIGHTS_PATH - accuracy = 0.05 - other_args = [ - "--trust-remote-code", - "--mem-fraction-static", - "0.8", - "--attention-backend", - "ascend", - "--disable-cuda-graph", - "--dtype", - "bfloat16", - ] - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/llm_models/test_npu_stablelm_2_1_6b.py b/test/registered/npu/llm_models/test_npu_stablelm_2_1_6b.py deleted file mode 100644 index f61f3ac88..000000000 --- a/test/registered/npu/llm_models/test_npu_stablelm_2_1_6b.py +++ /dev/null @@ -1,34 +0,0 @@ -import unittest - -from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin -from sglang.test.ascend.test_ascend_utils import STABLELM_2_1_6B_WEIGHTS_PATH -from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.test_utils import CustomTestCase - -register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) - - -class TestStablelm(GSM8KAscendMixin, CustomTestCase): - """Testcase: Verify that the inference accuracy of the stabilityai/stablelm-2-1_6b model on the GSM8K dataset is no less than 0.195. - - [Test Category] Model - [Test Target] stabilityai/stablelm-2-1_6b - """ - - model = STABLELM_2_1_6B_WEIGHTS_PATH - accuracy = 0.195 - other_args = [ - "--trust-remote-code", - "--mem-fraction-static", - "0.8", - "--attention-backend", - "ascend", - "--disable-cuda-graph", - "--tp-size", - 1, - "--enable-torch-compile", - ] - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms.py b/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms.py deleted file mode 100644 index 693ddc1bb..000000000 --- a/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms.py +++ /dev/null @@ -1,113 +0,0 @@ -import unittest - -from sglang.test.ascend.e2e.test_npu_multi_node_utils import NIC_NAME -from sglang.test.ascend.e2e.test_npu_performance_utils import ( - AISBENCHMARK_DATASET_DEFAULT, - BENCHMARK_TOOL_DEFAULT, - KIMI_K2_6_EAGLE3_MODEL_PATH, - KIMI_K2_6_W4A8_MODEL_PATH, - TestNpuPerfMultiNodePdMixTestCaseBase, -) -from sglang.test.ci.ci_register import register_npu_ci - -register_npu_ci( - est_time=1800, - suite="full-8-npu-a3", - nightly=True, - disabled="Currently it is executed by the npu performance workflow.", -) - -ENVS = { - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", - "SGLANG_SET_CPU_AFFINITY": "1", - "STREAMS_PER_DEVICE": "32", - "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", - "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "64", - "DEEPEP_HCCL_BUFFSIZE": "4400", - "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", - "HCCL_SOCKET_IFNAME": NIC_NAME, - "GLOO_SOCKET_IFNAME": NIC_NAME, -} - -OTHER_ARGS = [ - "--trust-remote-code", - "--attention-backend", - "ascend", - "--device", - "npu", - "--quantization", - "modelslim", - "--dtype", - "bfloat16", - "--tp-size", - 32, - "--nnodes", - 2, - "--mem-fraction-static", - 0.68, - "--max-running-requests", - 32, - "--chunked-prefill-size", - 32768, - "--context-length", - 75000, - "--enable-multimodal", - "--mm-attention-backend", - "ascend_attn", - "--sampling-backend", - "ascend", - "--enable-dp-attention", - "--dp-size", - 32, - "--moe-a2a-backend", - "deepep", - "--deepep-mode", - "auto", - "--cuda-graph-bs", - 1, - "--disable-radix-cache", - "--speculative-algorithm", - "EAGLE3", - "--speculative-draft-model-path", - KIMI_K2_6_EAGLE3_MODEL_PATH, - "--speculative-num-steps", - 3, - "--speculative-eagle-topk", - 1, - "--speculative-num-draft-tokens", - 4, - "--speculative-draft-model-quantization", - "unquant", - "--reasoning-parser", - "kimi_k2", - "--tool-call-parser", - "kimi_k2", -] - -MODEL_CONFIG = { - "model_path": KIMI_K2_6_W4A8_MODEL_PATH, - "other_args": OTHER_ARGS, - "node_envs": ENVS, -} - - -class TestNPUKimiK2_6_W4A8_16P_In64k_Out1k_100ms(TestNpuPerfMultiNodePdMixTestCaseBase): - benchmark_tool = BENCHMARK_TOOL_DEFAULT - dataset_type = AISBENCHMARK_DATASET_DEFAULT - model_config = MODEL_CONFIG - dataset_name = "random" - max_concurrency = 32 - num_prompts = 32 - input_len = 64000 - output_len = 1000 - random_range_ratio = 1 - seed = 1 - tpot = 100 - output_token_throughput = 160 - - def test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms(self): - self.run_throughput() - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms.py b/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms.py deleted file mode 100644 index 85f27ada5..000000000 --- a/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms.py +++ /dev/null @@ -1,126 +0,0 @@ -import unittest - -from sglang.test.ascend.e2e.test_npu_multi_node_utils import NIC_NAME -from sglang.test.ascend.e2e.test_npu_performance_utils import ( - AISBENCHMARK_DATASET_DEFAULT, - BENCHMARK_TOOL_DEFAULT, - KIMI_K2_6_EAGLE3_MODEL_PATH, - KIMI_K2_6_W4A8_MODEL_PATH, - TestNpuPerformanceTestCaseBase, -) -from sglang.test.ci.ci_register import register_npu_ci - -register_npu_ci(est_time=1200, suite="nightly-perf-16-npu-a3", nightly=True) - -KIMI_K2_6_ENVS = { - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", - "SGLANG_SET_CPU_AFFINITY": "1", - "HCCL_SOCKET_IFNAME": NIC_NAME, - "GLOO_SOCKET_IFNAME": NIC_NAME, - "STREAMS_PER_DEVICE": "32", - "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", - "SGLANG_ENABLE_SPEC_V2": "1", - "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", - "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", - "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "96", - "DEEPEP_HCCL_BUFFSIZE": "1200", - "HCCL_OP_EXPANSION_MODE": "AIV", - "SGLANG_NPU_USE_MLAPO": "1", - "SGLANG_NPU_USE_MULTI_STREAM": "1", - "SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "64", -} - -KIMI_K2_6_OTHER_ARGS = [ - "--trust-remote-code", - "--attention-backend", - "ascend", - "--device", - "npu", - "--quantization", - "modelslim", - "--dtype", - "bfloat16", - "--tp-size", - 16, - "--mem-fraction-static", - 0.865, - "--max-running-requests", - 80, - "--chunked-prefill-size", - 32768, - "--context-length", - 6144, - "--max-prefill-tokens", - 65536, - "--max-total-tokens", - 32256, - "--enable-multimodal", - "--mm-attention-backend", - "ascend_attn", - "--sampling-backend", - "ascend", - "--enable-dp-attention", - "--dp-size", - 16, - "--moe-a2a-backend", - "deepep", - "--deepep-mode", - "auto", - "--cuda-graph-bs-decode", - 1, - 2, - 3, - 4, - 5, - "--disable-radix-cache", - "--model-loader-extra-config", - '{"enable_multithread_load": true}', - "--speculative-algorithm", - "EAGLE3", - "--speculative-draft-model-path", - KIMI_K2_6_EAGLE3_MODEL_PATH, - "--speculative-num-steps", - 4, - "--speculative-eagle-topk", - 1, - "--speculative-num-draft-tokens", - 5, - "--speculative-draft-model-quantization", - "unquant", - "--prefill-delayer-max-delay-passes", - 200, - "--enable-prefill-delayer", - "--reasoning-parser", - "kimi_k2", - "--tool-call-parser", - "kimi_k2", -] - - -class TestKimiK25W4A8( - TestNpuPerformanceTestCaseBase, -): - benchmark_tool = BENCHMARK_TOOL_DEFAULT - dataset_type = AISBENCHMARK_DATASET_DEFAULT - max_attempts = 5 - model = KIMI_K2_6_W4A8_MODEL_PATH - other_args = KIMI_K2_6_OTHER_ARGS - envs = KIMI_K2_6_ENVS - backend = "sglang" - dataset_name = "random" - max_concurrency = 64 - num_prompts = 256 - input_len = 3500 - output_len = 1500 - random_range_ratio = 1 - warmup_requests = 0 - seed = 1 - tpot = 20 - output_token_throughput = 1900 - - def test_kimi_k2_6_w4a8(self): - self.run_throughput() - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1_ttft_5s.py b/test/registered/npu/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1_ttft_5s.py deleted file mode 100644 index 504ce0c82..000000000 --- a/test/registered/npu/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1_ttft_5s.py +++ /dev/null @@ -1,191 +0,0 @@ -import unittest - -from sglang.test.ascend.e2e.test_npu_performance_utils import ( - AISBENCHMARK_DATASET_DEFAULT, - BENCHMARK_TOOL_DEFAULT, - MIMO_V2_FLASH_MODEL_PATH, - TestNpuPerfMultiNodePdSepTestCaseBase, -) -from sglang.test.ci.ci_register import register_npu_ci - -register_npu_ci( - est_time=3600, - suite="", - nightly=True, - disabled="performance testcase", -) - -PREFILL_ENVS = { - "SGLANG_SET_CPU_AFFINITY": "1", - "STREAMS_PER_DEVICE": "32", - "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "32", - "DEEPEP_HCCL_BUFFSIZE": "1024", - "HCCL_OP_EXPANSION_MODE": "AIV", - "HCCL_SOCKET_IFNAME": "lo", - "GLOO_SOCKET_IFNAME": "lo", - "SGLANG_NPU_PROFILING": "0", - "SGLANG_NPU_PROFILING_STAGE": "prefill", - "DEEPEP_NORMAL_LONG_SEQ_ROUND": "32", - "DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "3584", - "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "3600", - "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "3600", - "SGLANG_ENABLE_SPEC_V2": "1", - "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0", - "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", - "SGLANG_DEEPEP_BF16_DISPATCH": "0", - "SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK": "1", - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", - "HCCL_CONNECT_TIMEOUT": "1800", - "ASCEND_USE_FIA": "1", -} - -DECODE_ENVS = { - "SGLANG_SET_CPU_AFFINITY": "1", - "STREAMS_PER_DEVICE": "32", - "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "128", - "DEEPEP_HCCL_BUFFSIZE": "800", - "HCCL_OP_EXPANSION_MODE": "AIV", - "HCCL_SOCKET_IFNAME": "lo", - "GLOO_SOCKET_IFNAME": "lo", - "SGLANG_NPU_PROFILING": "0", - "SGLANG_NPU_PROFILING_STAGE": "prefill", - "DEEPEP_NORMAL_LONG_SEQ_ROUND": "32", - "DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "3584", - "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "3600", - "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "3600", - "SGLANG_ENABLE_SPEC_V2": "1", - "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0", - "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", - "SGLANG_DEEPEP_BF16_DISPATCH": "0", - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", - "HCCL_CONNECT_TIMEOUT": "1800", - "SGLANG_PROFILE_WITH_STACK": "True", - "ASCEND_USE_FIA": "1", -} - -PREFILL_ARGS = [ - "--attention-backend", - "ascend", - "--device", - "npu", - "--tp-size", - 8, - "--nnodes", - "1", - "--node-rank", - "0", - "--chunked-prefill-size", - 8192, - "--trust-remote-code", - "--max-running-requests", - 64, - "--mem-fraction-static", - 0.8, - "--swa-full-tokens-ratio", - 0.3, - "--disaggregation-mode", - "prefill", - "--disaggregation-transfer-backend", - "ascend", - "--disable-radix-cache", - "--disable-cuda-graph", - "--disable-piecewise-cuda-graph", - "--dp-size", - "2", -] - -DECODE_ARGS = [ - "--attention-backend", - "ascend", - "--device", - "npu", - "--tp-size", - 16, - "--nnodes", - "1", - "--node-rank", - "0", - "--trust-remote-code", - "--max-running-requests", - 64, - "--mem-fraction-static", - 0.8, - "--swa-full-tokens-ratio", - 0.3, - "--cuda-graph-bs", - 1, - 2, - 4, - 8, - 12, - 16, - 20, - 24, - 28, - 32, - "--disaggregation-mode", - "decode", - "--disaggregation-transfer-backend", - "ascend", - "--speculative-algorithm", - "EAGLE", - "--speculative-num-steps", - 3, - "--speculative-eagle-topk", - 1, - "--speculative-num-draft-tokens", - 4, - "--enable-multi-layer-eagle", - "--disable-radix-cache", - "--dp-size", - "2", - "--enable-dp-attention", - "--enable-dp-lm-head", - "--moe-a2a-backend", - "deepep", - "--deepep-mode", - "low_latency", -] - -ROUTER_ARGS = [ - "--health-check-interval-secs", - "3600", - "--mini-lb", -] - -MODEL_CONFIG = { - "model_path": MIMO_V2_FLASH_MODEL_PATH, - "prefill_args": PREFILL_ARGS, - "decode_args": DECODE_ARGS, - "prefill_envs": PREFILL_ENVS, - "decode_envs": DECODE_ENVS, - "router_args": ROUTER_ARGS, - "router_envs": {}, -} - - -class TestNPUMimo_v2_flash_1P1D_16p_In16k_Out1_TTFT_5s( - TestNpuPerfMultiNodePdSepTestCaseBase -): - """Test NPU performance for mimo_v2_flash 1P+1D 16p: input_len=16000, output_len=1, 0 cache, TTFT=5s""" - - model_config = MODEL_CONFIG - benchmark_tool = BENCHMARK_TOOL_DEFAULT - dataset_type = AISBENCHMARK_DATASET_DEFAULT - dataset_name = "random" - max_concurrency = 64 - num_prompts = 128 - request_rate = 0.4 - input_len = 16000 - output_len = 1 - random_range_ratio = 1 - seed = 1 - ttft = 5000 - - def test_npu_mimo_v2_flash_1p1d_16p_in16k_out1_ttft_5s(self): - """Run NPU performance test for 1P+1D 16p with 16k input, 1 output, 0 cache, TTFT=5s""" - self.run_throughput() - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1k_tpot_20ms.py b/test/registered/npu/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1k_tpot_20ms.py deleted file mode 100644 index ddaeaf7c4..000000000 --- a/test/registered/npu/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1k_tpot_20ms.py +++ /dev/null @@ -1,187 +0,0 @@ -import unittest - -from sglang.test.ascend.e2e.test_npu_performance_utils import ( - AISBENCHMARK_DATASET_DEFAULT, - BENCHMARK_TOOL_DEFAULT, - MIMO_V2_FLASH_MODEL_PATH, - TestNpuPerfMultiNodePdSepTestCaseBase, -) -from sglang.test.ci.ci_register import register_npu_ci - -register_npu_ci( - est_time=3600, - suite="", - nightly=True, - disabled="performance testcase", -) - -PREFILL_ENVS = { - "SGLANG_SET_CPU_AFFINITY": "1", - "STREAMS_PER_DEVICE": "32", - "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "32", - "DEEPEP_HCCL_BUFFSIZE": "1024", - "HCCL_OP_EXPANSION_MODE": "AIV", - "HCCL_SOCKET_IFNAME": "lo", - "GLOO_SOCKET_IFNAME": "lo", - "SGLANG_NPU_PROFILING": "0", - "SGLANG_NPU_PROFILING_STAGE": "prefill", - "DEEPEP_NORMAL_LONG_SEQ_ROUND": "32", - "DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "3584", - "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "3600", - "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "3600", - "SGLANG_ENABLE_SPEC_V2": "1", - "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0", - "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", - "SGLANG_DEEPEP_BF16_DISPATCH": "0", - "SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK": "1", - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", - "HCCL_CONNECT_TIMEOUT": "1800", - "ASCEND_USE_FIA": "1", -} - -DECODE_ENVS = { - "SGLANG_SET_CPU_AFFINITY": "1", - "STREAMS_PER_DEVICE": "32", - "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "128", - "DEEPEP_HCCL_BUFFSIZE": "800", - "HCCL_OP_EXPANSION_MODE": "AIV", - "HCCL_SOCKET_IFNAME": "lo", - "GLOO_SOCKET_IFNAME": "lo", - "SGLANG_NPU_PROFILING": "0", - "SGLANG_NPU_PROFILING_STAGE": "prefill", - "DEEPEP_NORMAL_LONG_SEQ_ROUND": "32", - "DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "3584", - "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "3600", - "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "3600", - "SGLANG_ENABLE_SPEC_V2": "1", - "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0", - "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", - "SGLANG_DEEPEP_BF16_DISPATCH": "0", - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", - "HCCL_CONNECT_TIMEOUT": "1800", - "SGLANG_PROFILE_WITH_STACK": "True", - "ASCEND_USE_FIA": "1", -} - -PREFILL_ARGS = [ - "--attention-backend", - "ascend", - "--device", - "npu", - "--tp-size", - 8, - "--nnodes", - "1", - "--node-rank", - "0", - "--chunked-prefill-size", - 8192, - "--trust-remote-code", - "--max-running-requests", - 64, - "--mem-fraction-static", - 0.8, - "--swa-full-tokens-ratio", - 0.3, - "--disaggregation-mode", - "prefill", - "--disaggregation-transfer-backend", - "ascend", - "--disable-radix-cache", - "--disable-cuda-graph", - "--disable-piecewise-cuda-graph", - "--dp-size", - "2", -] - -DECODE_ARGS = [ - "--attention-backend", - "ascend", - "--device", - "npu", - "--tp-size", - 16, - "--nnodes", - "1", - "--node-rank", - "0", - "--trust-remote-code", - "--max-running-requests", - 32, - "--mem-fraction-static", - 0.8, - "--swa-full-tokens-ratio", - 0.3, - "--cuda-graph-bs", - 1, - 2, - 4, - 8, - 12, - 16, - "--disaggregation-mode", - "decode", - "--disaggregation-transfer-backend", - "ascend", - "--speculative-algorithm", - "EAGLE", - "--speculative-num-steps", - 3, - "--speculative-eagle-topk", - 1, - "--speculative-num-draft-tokens", - 4, - "--enable-multi-layer-eagle", - "--disable-radix-cache", - "--dp-size", - "2", - "--enable-dp-attention", - "--enable-dp-lm-head", - "--moe-a2a-backend", - "deepep", - "--deepep-mode", - "low_latency", -] - -ROUTER_ARGS = [ - "--health-check-interval-secs", - "3600", - "--mini-lb", -] - -MODEL_CONFIG = { - "model_path": MIMO_V2_FLASH_MODEL_PATH, - "prefill_args": PREFILL_ARGS, - "decode_args": DECODE_ARGS, - "prefill_envs": PREFILL_ENVS, - "decode_envs": DECODE_ENVS, - "router_args": ROUTER_ARGS, - "router_envs": {}, -} - - -class TestNPUMimo_v2_flash_1P1D_16p_In16k_Out1k_TTFT_5s( - TestNpuPerfMultiNodePdSepTestCaseBase -): - """Test NPU performance for mimo_v2_flash 1P+1D 16p: input_len=16000, output_len=1000, 0 cache, TPOT=20ms""" - - model_config = MODEL_CONFIG - benchmark_tool = BENCHMARK_TOOL_DEFAULT - dataset_type = AISBENCHMARK_DATASET_DEFAULT - dataset_name = "random" - max_concurrency = 32 - num_prompts = 128 - request_rate = float("inf") - input_len = 16000 - output_len = 1000 - random_range_ratio = 1 - seed = 1 - tpot = 20 - - def test_npu_mimo_v2_flash_1p1d_16p_in16k_out1k_tpot_20ms(self): - """Run NPU performance test for 1P+1D 16p with 16k input, 1k output, 0 cache, TPOT=20ms""" - self.run_throughput() - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py b/test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py deleted file mode 100644 index e128cb39b..000000000 --- a/test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py +++ /dev/null @@ -1,118 +0,0 @@ -import os -import unittest - -from sglang.test.ascend.e2e.test_npu_performance_utils import ( - BENCHMARK_TOOL_DEFAULT, - MINIMAX_M2_5_EAGLE3_MODEL_PATH, - MINIMAX_M2_5_W8A8_MODEL_PATH, - TestNpuPerformanceTestCaseBase, -) -from sglang.test.ci.ci_register import register_npu_ci - -register_npu_ci(est_time=3600, suite="nightly-perf-8-npu-a3", nightly=True) - -MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS = { - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", - "STREAMS_PER_DEVICE": "32", - "HCCL_SOCKET_IFNAME": "lo", - "GLOO_SOCKET_IFNAME": "lo", - "TASK_QUEUE_ENABLE": "1", - "ASCEND_USE_FIA": "1", - "SGLANG_SET_CPU_AFFINITY": "1", - "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", - "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "140000", - "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", - "DEEPEP_HCCL_BUFFSIZE": "1024", - "SGLANG_EXTERNAL_MODEL_PACKAGE": "custom_eagle3", - "SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "999999999", - "PYTHONPATH": f"{MINIMAX_M2_5_EAGLE3_MODEL_PATH}:{os.environ.get('PYTHONPATH', '')}", -} - -MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_OTHER_ARGS = [ - "--tp-size", - 8, - "--mem-fraction-static", - 0.63, - "--max-running-requests", - 24, - "--reasoning-parser", - "minimax-append-think", - "--tool-call-parser", - "minimax-m2", - "--enable-prefill-delayer", - "--prefill-delayer-max-delay-passes", - 30, - "--prefill-max-requests", - 10, - "--chunked-prefill-size", - 67072, - "--max-prefill-token", - 67000, - "--cuda-graph-bs", - 2, - 4, - 8, - 12, - 16, - 18, - 20, - 22, - 24, - "--moe-a2a-backend", - "ascend_fuseep", - "--fuseep-mode", - 2, - "--deepep-mode", - "auto", - "--quantization", - "modelslim", - "--speculative-algorithm", - "EAGLE3", - "--speculative-draft-model-path", - MINIMAX_M2_5_EAGLE3_MODEL_PATH, - "--speculative-num-steps", - 3, - "--speculative-eagle-topk", - 1, - "--speculative-num-draft-tokens", - 4, - "--speculative-draft-model-quantization", - "unquant", - "--dtype", - "bfloat16", - "--trust-remote-code", - "--reasoning-parser", - "minimax-append-think", - "--tool-call-parser", - "minimax-m2", -] - - -class TestNPUMiniMaxM2_5W8A8_4P_In64k_Out1k_Prefix90_50ms( - TestNpuPerformanceTestCaseBase -): - """MiniMax-M2.5-w8a8 4p (4 cards) 64k input 1k output with 90% prefix cache performance test""" - - benchmark_tool = BENCHMARK_TOOL_DEFAULT - model = MINIMAX_M2_5_W8A8_MODEL_PATH - other_args = MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_OTHER_ARGS - envs = MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS - dataset_name = "generated-shared-prefix" - max_concurrency = 24 - num_prompts = 96 - input_len = 65536 - output_len = 1024 - random_range_ratio = 1 - seed = 1 - repeat_rate = 0.9 - tpot = 50 - output_token_throughput = 390.5859 - request_rate = float("inf") - - def test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms(self): - """Run MiniMax-M2.5-w8a8 4p 64k/1k prefix90 performance test""" - self.run_throughput() - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms.py deleted file mode 100644 index d60fb67c7..000000000 --- a/test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms.py +++ /dev/null @@ -1,117 +0,0 @@ -import os -import unittest - -from sglang.test.ascend.e2e.test_npu_performance_utils import ( - AISBENCHMARK_DATASET_DEFAULT, - BENCHMARK_TOOL_DEFAULT, - MINIMAX_M2_5_EAGLE3_MODEL_PATH, - MINIMAX_M2_5_W8A8_MODEL_PATH, - TestNpuPerformanceTestCaseBase, -) -from sglang.test.ci.ci_register import register_npu_ci - -register_npu_ci( - est_time=3600, - suite="nightly-perf-16-npu-a3", - nightly=True, -) - -MINIMAX_M2_5_HIGH_THROUGHPUT_ENVS = { - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", - "STREAMS_PER_DEVICE": "32", - "HCCL_SOCKET_IFNAME": "lo", - "GLOO_SOCKET_IFNAME": "lo", - "TASK_QUEUE_ENABLE": "1", - "DEEPEP_HCCL_BUFFSIZE": "1024", - "ASCEND_USE_FIA": "1", - "SGLANG_SET_CPU_AFFINITY": "1", - "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", - "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "204800", - "PYTHONPATH": f"{MINIMAX_M2_5_EAGLE3_MODEL_PATH}:{os.environ.get('PYTHONPATH', '')}", - "SGLANG_EXTERNAL_MODEL_PACKAGE": "custom_eagle3", -} - -MINIMAX_M2_5_HIGH_THROUGHPUT_OTHER_ARGS = [ - "--tp-size", - 16, - "--enable-dp-attention", - "--dp-size", - 16, - "--mem-fraction-static", - 0.75, - "--max-running-requests", - 320, - "--disable-radix-cache", - "--reasoning-parser", - "minimax-append-think", - "--tool-call-parser", - "minimax-m2", - "--prefill-delayer-max-delay-passes", - 500, - "--enable-prefill-delayer", - "--chunked-prefill-size", - 196608, - "--max-prefill-token", - 8192, - "--cuda-graph-bs", - 1, - 2, - 4, - 8, - 12, - 16, - 20, - "--moe-a2a-backend", - "ascend_fuseep", - "--fuseep-mode", - 2, - "--quantization", - "modelslim", - "--speculative-algorithm", - "EAGLE3", - "--speculative-draft-model-path", - MINIMAX_M2_5_EAGLE3_MODEL_PATH, - "--speculative-num-steps", - 3, - "--speculative-eagle-topk", - 1, - "--speculative-num-draft-tokens", - 4, - "--speculative-draft-model-quantization", - "unquant", - "--dtype", - "bfloat16", - "--reasoning-parser", - "minimax-append-think", - "--tool-call-parser", - "minimax-m2", -] - - -class TestNPUMiniMaxM2_5_W8A8_8P_In3k5_Out1k5_HighThroughput( - TestNpuPerformanceTestCaseBase -): - """Test NPU performance for MiniMax-M2.5-w8a8 8p single node high throughput in3k5 out1k5""" - - benchmark_tool = BENCHMARK_TOOL_DEFAULT - dataset_type = AISBENCHMARK_DATASET_DEFAULT - model = MINIMAX_M2_5_W8A8_MODEL_PATH - other_args = MINIMAX_M2_5_HIGH_THROUGHPUT_OTHER_ARGS - envs = MINIMAX_M2_5_HIGH_THROUGHPUT_ENVS - dataset_name = "random" - max_concurrency = 320 - num_prompts = 1280 - input_len = 3500 - output_len = 1500 - random_range_ratio = 1 - seed = 1 - tpot = 50 - output_token_throughput = 5717.58 - - def test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_high_throughput(self): - """Run NPU performance test for MiniMax-M2.5-w8a8 high throughput""" - self.run_throughput() - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py deleted file mode 100644 index fcab24202..000000000 --- a/test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py +++ /dev/null @@ -1,105 +0,0 @@ -import unittest - -from sglang.test.ascend.e2e.test_npu_performance_utils import ( - AISBENCHMARK_DATASET_DEFAULT, - BENCHMARK_TOOL_DEFAULT, - QWEN3_8B_EAGLE_MODEL_PATH, - QWEN3_8B_W8A8_MODEL_PATH, - TestNpuPerformanceTestCaseBase, -) -from sglang.test.ci.ci_register import register_npu_ci - -register_npu_ci(est_time=3600, suite="nightly-perf-2-npu-a3", nightly=True) - -QWEN3_8B_ENVS = { - "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", - "HCCL_SOCKET_IFNAME": "lo", - "GLOO_SOCKET_IFNAME": "lo", - "HCCL_OP_EXPANSION_MODE": "AIV", - "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", - "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", - "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "50", - "SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "128", -} - -QWEN3_8B_OTHER_ARGS = [ - "--trust-remote-code", - "--nnodes", - "1", - "--node-rank", - "0", - "--attention-backend", - "ascend", - "--device", - "npu", - "--quantization", - "modelslim", - "--max-running-requests", - 70, - "--max-prefill-tokens", - 16384, - "--disable-radix-cache", - "--chunked-prefill-size", - 16384, - "--tp-size", - 1, - "--mem-fraction-static", - 0.85, - "--cuda-graph-bs", - 8, - 12, - 24, - 36, - 48, - 51, - 55, - 60, - 63, - 64, - 66, - 68, - 70, - "--dtype", - "bfloat16", - "--speculative-draft-model-quantization", - "unquant", - "--speculative-algorithm", - "EAGLE3", - "--speculative-draft-model-path", - QWEN3_8B_EAGLE_MODEL_PATH, - "--speculative-num-steps", - 3, - "--speculative-eagle-topk", - 1, - "--speculative-num-draft-tokens", - 4, - "--reasoning-parser", - "qwen3", - "--tool-call-parser", - "qwen", -] - - -class TestQwen8B(TestNpuPerformanceTestCaseBase): - benchmark_tool = BENCHMARK_TOOL_DEFAULT - dataset_type = AISBENCHMARK_DATASET_DEFAULT - model = QWEN3_8B_W8A8_MODEL_PATH - other_args = QWEN3_8B_OTHER_ARGS - envs = QWEN3_8B_ENVS - dataset_name = "random" - max_concurrency = 64 - num_prompts = 256 - input_len = 3500 - output_len = 1500 - random_range_ratio = 1 - seed = 1 - tpot = 37 - output_token_throughput = 1586 - - def test_qwen3_8b(self): - self.run_throughput() - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16.py b/test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16.py deleted file mode 100644 index 7e92f9d19..000000000 --- a/test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16.py +++ /dev/null @@ -1,93 +0,0 @@ -import unittest - -from sglang.test.ascend.e2e.test_npu_performance_utils import ( - QWEN3_8B_EAGLE_MODEL_PATH, - QWEN3_8B_W8A8_MODEL_PATH, - TestNpuPerformanceTestCaseBase, -) -from sglang.test.ci.ci_register import register_npu_ci - -register_npu_ci( - est_time=3600, - suite="nightly-perf-2-npu-a3", - nightly=True, -) - -QWEN3_8B_ENVS = { - "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", - "HCCL_SOCKET_IFNAME": "lo", - "GLOO_SOCKET_IFNAME": "lo", - "HCCL_OP_EXPANSION_MODE": "AIV", - "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", -} - -QWEN3_8B_OTHER_ARGS = [ - "--trust-remote-code", - "--nnodes", - "1", - "--node-rank", - "0", - "--attention-backend", - "ascend", - "--device", - "npu", - "--quantization", - "modelslim", - "--max-running-requests", - 16, - "--max-prefill-tokens", - 16384, - "--disable-radix-cache", - "--chunked-prefill-size", - -1, - "--tp-size", - 2, - "--mem-fraction-static", - 0.894, - "--cuda-graph-bs", - 1, - 5, - 15, - 16, - "--dtype", - "bfloat16", - "--speculative-draft-model-quantization", - "unquant", - "--speculative-algorithm", - "EAGLE3", - "--speculative-draft-model-path", - QWEN3_8B_EAGLE_MODEL_PATH, - "--speculative-num-steps", - 4, - "--speculative-eagle-topk", - 1, - "--speculative-num-draft-tokens", - 5, - "--reasoning-parser", - "qwen3", - "--tool-call-parser", - "qwen", -] - - -class TestQwen8B(TestNpuPerformanceTestCaseBase): - max_attempts = 5 - model = QWEN3_8B_W8A8_MODEL_PATH - other_args = QWEN3_8B_OTHER_ARGS - envs = QWEN3_8B_ENVS - dataset_name = "random" - max_concurrency = 16 - num_prompts = 16 - input_len = 6144 - output_len = 1500 - random_range_ratio = 1 - tpot = 11.79 - output_token_throughput = 1040.96 - - def test_qwen3_8b(self): - self.run_throughput() - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py deleted file mode 100644 index 2b7575a3b..000000000 --- a/test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py +++ /dev/null @@ -1,122 +0,0 @@ -import unittest - -from sglang.test.ascend.e2e.test_npu_performance_utils import ( - AISBENCHMARK_DATASET_DEFAULT, - BENCHMARK_TOOL_DEFAULT, - QWEN3_235B_A22B_EAGLE_MODEL_PATH, - QWEN3_235B_W8A8_MODEL_PATH, - TestNpuPerformanceTestCaseBase, -) -from sglang.test.ci.ci_register import register_npu_ci - -register_npu_ci(est_time=3600, suite="nightly-perf-16-npu-a3", nightly=True) - -QWEN3_235B_ENVS = { - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", - "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", - "DEEPEP_HCCL_BUFFSIZE": "570", - "HCCL_SOCKET_IFNAME": "lo", - "GLOO_SOCKET_IFNAME": "lo", - "HCCL_OP_EXPANSION_MODE": "AIV", - "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", - "SGLANG_ENABLE_SPEC_V2": "1", - "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", - "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "50", - "SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "16", - "SGLANG_NPU_PROFILING": "0", - "SGLANG_NPU_PROFILING_BS": "27", - "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "188416", -} - -QWEN3_235B_OTHER_ARGS = [ - "--trust-remote-code", - "--nnodes", - "1", - "--node-rank", - "0", - "--attention-backend", - "ascend", - "--device", - "npu", - "--quantization", - "modelslim", - "--max-running-requests", - 432, - "--context-length", - 8192, - "--dtype", - "bfloat16", - "--chunked-prefill-size", - 94208, - "--max-prefill-tokens", - 458880, - "--sampling-backend", - "ascend", - "--ep-dispatch-algorithm", - "static", - "--init-expert-location", - "/root/.cache/modelscope/hub/models/hot_map/235B_3_5k_decode.pt", - "--disable-radix-cache", - "--moe-a2a-backend", - "ascend_fuseep", - "--fuseep-mode", - 2, - "--speculative-algorithm", - "EAGLE3", - "--speculative-draft-model-path", - QWEN3_235B_A22B_EAGLE_MODEL_PATH, - "--speculative-num-steps", - "3", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "4", - "--speculative-draft-model-quantization", - "unquant", - "--tp", - "16", - "--dp-size", - "16", - "--enable-dp-attention", - "--enable-dp-lm-head", - "--mem-fraction-static", - "0.8", - "--cuda-graph-bs", - "1", - "2", - "4", - "8", - "16", - "20", - "24", - "26", - "27", - "--reasoning-parser", - "qwen3", - "--tool-call-parser", - "qwen25", -] - - -class TestQwen235B(TestNpuPerformanceTestCaseBase): - benchmark_tool = BENCHMARK_TOOL_DEFAULT - dataset_type = AISBENCHMARK_DATASET_DEFAULT - model = QWEN3_235B_W8A8_MODEL_PATH - other_args = QWEN3_235B_OTHER_ARGS - envs = QWEN3_235B_ENVS - dataset_name = "random" - max_concurrency = 432 - num_prompts = int(max_concurrency) * 4 - input_len = 3500 - output_len = 1500 - random_range_ratio = 1 - seed = 1 - tpot = 50.1 - output_token_throughput = 6189 - - def test_qwen3_235b(self): - self.run_throughput() - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py deleted file mode 100644 index 6cf927abc..000000000 --- a/test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py +++ /dev/null @@ -1,110 +0,0 @@ -import unittest - -from sglang.test.ascend.e2e.test_npu_performance_utils import ( - AISBENCHMARK_DATASET_DEFAULT, - BENCHMARK_TOOL_DEFAULT, - QWEN3_30B_A3B_W8A8_VLLM_MODEL_PATH, - QWEN3_A3B_EAGLE_MODEL_PATH, - TestNpuPerformanceTestCaseBase, -) -from sglang.test.ci.ci_register import register_npu_ci - -register_npu_ci(est_time=3600, suite="nightly-perf-2-npu-a3", nightly=True) - -QWEN3_30B_A3B_ENVS = { - "ASCEND_LAUNCH_BLOCKING": "0", - "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", - "HCCL_SOCKET_IFNAME": "lo", - "GLOO_SOCKET_IFNAME": "lo", - "HCCL_OP_EXPANSION_MODE": "AIV", - "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", - "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", - "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200", - "SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "128", - "DEEPEP_HCCL_BUFFSIZE": "400", -} - -QWEN3_30B_A3B_OTHER_ARGS = [ - "--trust-remote-code", - "--nnodes", - "1", - "--node-rank", - "0", - "--attention-backend", - "ascend", - "--device", - "npu", - "--quantization", - "modelslim", - "--max-running-requests", - 162, - "--disable-radix-cache", - "--speculative-draft-model-quantization", - "unquant", - "--chunked-prefill-size", - -1, - "--max-prefill-tokens", - 35000, - "--speculative-algorithm", - "EAGLE3", - "--speculative-draft-model-path", - QWEN3_A3B_EAGLE_MODEL_PATH, - "--speculative-num-steps", - 3, - "--speculative-eagle-topk", - 1, - "--speculative-num-draft-tokens", - 4, - "--tp-size", - 2, - "--mem-fraction-static", - 0.87, - "--cuda-graph-bs", - 1, - 5, - 15, - 40, - 70, - 100, - 120, - 130, - 140, - 146, - 150, - 154, - 156, - 158, - 160, - 162, - "--dtype", - "bfloat16", - "--reasoning-parser", - "qwen3", - "--tool-call-parser", - "qwen", -] - - -class TestQwen30B(TestNpuPerformanceTestCaseBase): - benchmark_tool = BENCHMARK_TOOL_DEFAULT - dataset_type = AISBENCHMARK_DATASET_DEFAULT - model = QWEN3_30B_A3B_W8A8_VLLM_MODEL_PATH - other_args = QWEN3_30B_A3B_OTHER_ARGS - envs = QWEN3_30B_A3B_ENVS - dataset_name = "random" - max_concurrency = 160 - num_prompts = int(max_concurrency) * 4 - input_len = 3500 - output_len = 1500 - random_range_ratio = 1 - seed = 1 - tpot = 50 - output_token_throughput = 3200 - - def test_qwen3_30b(self): - self.run_throughput() - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py b/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py deleted file mode 100644 index 5b96b211a..000000000 --- a/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py +++ /dev/null @@ -1,94 +0,0 @@ -import unittest - -from sglang.test.ascend.e2e.test_npu_performance_utils import ( - AISBENCHMARK_DATASET_DEFAULT, - BENCHMARK_TOOL_DEFAULT, - QWEN3_32B_EAGLE_MODEL_PATH, - QWEN3_32B_MODEL_PATH, - TestNpuPerformanceTestCaseBase, -) -from sglang.test.ci.ci_register import register_npu_ci - -register_npu_ci( - est_time=3600, - suite="nightly-perf-16-npu-a3", - nightly=True, -) - -QWEN3_32B_ENVS = { - "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", - "HCCL_SOCKET_IFNAME": "lo", - "GLOO_SOCKET_IFNAME": "lo", - "HCCL_OP_EXPANSION_MODE": "AIV", - "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", - "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", - "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200", -} - -QWEN3_32B_OTHER_ARGS = [ - "--trust-remote-code", - "--nnodes", - "1", - "--node-rank", - "0", - "--attention-backend", - "ascend", - "--device", - "npu", - "--max-running-requests", - 1, - "--disable-radix-cache", - "--speculative-draft-model-quantization", - "unquant", - "--chunked-prefill-size", - -1, - "--max-prefill-tokens", - 65536, - "--speculative-algorithm", - "EAGLE3", - "--speculative-draft-model-path", - QWEN3_32B_EAGLE_MODEL_PATH, - "--speculative-num-steps", - 4, - "--speculative-eagle-topk", - 1, - "--speculative-num-draft-tokens", - 5, - "--tp-size", - 16, - "--mem-fraction-static", - 0.72, - "--cuda-graph-bs", - 1, - "--dtype", - "bfloat16", - "--reasoning-parser", - "qwen3", - "--tool-call-parser", - "qwen", -] - - -class TestQwen32B(TestNpuPerformanceTestCaseBase): - benchmark_tool = BENCHMARK_TOOL_DEFAULT - dataset_type = AISBENCHMARK_DATASET_DEFAULT - model = QWEN3_32B_MODEL_PATH - other_args = QWEN3_32B_OTHER_ARGS - envs = QWEN3_32B_ENVS - dataset_name = "random" - max_concurrency = 1 - num_prompts = 1 - input_len = 18000 - output_len = 4000 - random_range_ratio = 1 - seed = 1 - tpot = 6 - output_token_throughput = 171 - - def test_qwen3_32b(self): - self.run_throughput() - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py deleted file mode 100644 index 00773e982..000000000 --- a/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py +++ /dev/null @@ -1,107 +0,0 @@ -import unittest - -from sglang.test.ascend.e2e.test_npu_performance_utils import ( - AISBENCHMARK_DATASET_DEFAULT, - BENCHMARK_TOOL_DEFAULT, - QWEN3_32B_EAGLE_MODEL_PATH, - QWEN3_32B_W8A8_MODEL_PATH, - TestNpuPerformanceTestCaseBase, -) -from sglang.test.ci.ci_register import register_npu_ci - -register_npu_ci(est_time=3600, suite="nightly-perf-4-npu-a3", nightly=True) - -QWEN3_32B_ENVS = { - "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", - "HCCL_SOCKET_IFNAME": "lo", - "GLOO_SOCKET_IFNAME": "lo", - "HCCL_OP_EXPANSION_MODE": "AIV", - "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", - "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", - "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100", - "SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "128", - "SGLANG_NPU_USE_DEEPGEMM": "1", -} - -QWEN3_32B_OTHER_ARGS = [ - "--trust-remote-code", - "--nnodes", - "1", - "--node-rank", - "0", - "--attention-backend", - "ascend", - "--device", - "npu", - "--quantization", - "modelslim", - "--max-running-requests", - 101, - "--disable-radix-cache", - "--speculative-draft-model-quantization", - "unquant", - "--chunked-prefill-size", - -1, - "--max-prefill-tokens", - 35000, - "--speculative-algorithm", - "EAGLE3", - "--speculative-draft-model-path", - QWEN3_32B_EAGLE_MODEL_PATH, - "--speculative-num-steps", - 3, - "--speculative-eagle-topk", - 1, - "--speculative-num-draft-tokens", - 4, - "--tp-size", - 4, - "--mem-fraction-static", - 0.845, - "--cuda-graph-bs", - 16, - 32, - 64, - 72, - 88, - 90, - 92, - 94, - 96, - 97, - 98, - 99, - 100, - 101, - "--dtype", - "bfloat16", - "--reasoning-parser", - "qwen3", - "--tool-call-parser", - "qwen", -] - - -class TestQwen32B(TestNpuPerformanceTestCaseBase): - benchmark_tool = BENCHMARK_TOOL_DEFAULT - dataset_type = AISBENCHMARK_DATASET_DEFAULT - model = QWEN3_32B_W8A8_MODEL_PATH - other_args = QWEN3_32B_OTHER_ARGS - envs = QWEN3_32B_ENVS - dataset_name = "random" - max_concurrency = 100 - num_prompts = 400 - input_len = 3584 - output_len = 1536 - random_range_ratio = 1 - seed = 1 - tpot = 50 - output_token_throughput = 1600 - - def test_qwen3_32b(self): - self.run_throughput() - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py b/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py deleted file mode 100644 index 7742c900f..000000000 --- a/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py +++ /dev/null @@ -1,110 +0,0 @@ -import unittest - -from sglang.test.ascend.e2e.test_npu_performance_utils import ( - AISBENCHMARK_DATASET_DEFAULT, - DEFAULT_URL_FOR_TEST, - QWEN3_32B_EAGLE_MODEL_PATH, - QWEN3_32B_W8A8_MODEL_PATH, - TestNpuPerformanceTestCaseBase, -) -from sglang.test.ci.ci_register import register_npu_ci - -register_npu_ci( - est_time=3600, - suite="nightly-1-npu-a2", - nightly=True, -) - -QWEN3_32B_ENVS = { - "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", - "HCCL_SOCKET_IFNAME": "lo", - "GLOO_SOCKET_IFNAME": "lo", - "HCCL_OP_EXPANSION_MODE": "AIV", - "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", - "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", - "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100", - "SGLANG_NPU_USE_DEEPGEMM": "1", -} - -QWEN3_32B_OTHER_ARGS = [ - "--trust-remote-code", - "--nnodes", - "1", - "--node-rank", - "0", - "--attention-backend", - "ascend", - "--device", - "npu", - "--quantization", - "modelslim", - "--max-running-requests", - 101, - "--disable-radix-cache", - "--speculative-draft-model-quantization", - "unquant", - "--chunked-prefill-size", - -1, - "--max-prefill-tokens", - 35000, - "--speculative-algorithm", - "EAGLE3", - "--speculative-draft-model-path", - QWEN3_32B_EAGLE_MODEL_PATH, - "--speculative-num-steps", - 3, - "--speculative-eagle-topk", - 1, - "--speculative-num-draft-tokens", - 4, - "--tp-size", - 4, - "--mem-fraction-static", - 0.845, - "--cuda-graph-bs", - 16, - 32, - 64, - 72, - 88, - 90, - 92, - 94, - 96, - 97, - 98, - 99, - 100, - 101, - "--dtype", - "bfloat16", - "--reasoning-parser", - "qwen3", - "--tool-call-parser", - "qwen", -] - - -class TestQwen32B(TestNpuPerformanceTestCaseBase): - base_url = DEFAULT_URL_FOR_TEST - dataset_type = AISBENCHMARK_DATASET_DEFAULT - model = QWEN3_32B_W8A8_MODEL_PATH - other_args = QWEN3_32B_OTHER_ARGS - envs = QWEN3_32B_ENVS - dataset_name = "random" - max_concurrency = 100 - num_prompts = 400 - input_len = 3584 - output_len = 1536 - random_range_ratio = 1 - seed = 1 - tpot = 55 - output_token_throughput = 1500 - - def test_qwen3_32b(self): - self.run_throughput() - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in64k_out1k_50ms.py b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in64k_out1k_50ms.py deleted file mode 100644 index c501be3d1..000000000 --- a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in64k_out1k_50ms.py +++ /dev/null @@ -1,100 +0,0 @@ -import unittest - -from sglang.test.ascend.e2e.test_npu_performance_utils import ( - AISBENCHMARK_DATASET_DEFAULT, - BENCHMARK_TOOL_DEFAULT, - QWEN3_6_27B_W8A8_MODEL_PATH, - TestNpuPerformanceTestCaseBase, -) -from sglang.test.ci.ci_register import register_npu_ci - -register_npu_ci( - est_time=3600, - suite="", - nightly=True, - disabled="performance testcase", -) - -QWEN3_6_27B_64K_1K_ENVS = { - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", - "STREAMS_PER_DEVICE": "32", - "HCCL_SOCKET_IFNAME": "lo", - "GLOO_SOCKET_IFNAME": "lo", - "HCCL_OP_EXPANSION_MODE": "AIV", - "SGLANG_SET_CPU_AFFINITY": "1", - "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", - "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", - "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "30", - "ASCEND_USE_FIA": "1", -} - -QWEN3_6_27B_64K_1K_OTHER_ARGS = [ - "--tp-size", - 4, - "--nnodes", - 1, - "--attention-backend", - "ascend", - "--device", - "npu", - "--chunked-prefill-size", - -1, - "--max-prefill-tokens", - 50000, - "--disable-radix-cache", - "--trust-remote-code", - "--max-running-requests", - 28, - "--max-mamba-cache-size", - 50, - "--mem-fraction-static", - 0.7, - "--cuda-graph-bs", - 2, - 4, - 6, - "--enable-multimodal", - "--quantization", - "modelslim", - "--mm-attention-backend", - "ascend_attn", - "--dtype", - "bfloat16", - "--mamba-ssm-dtype", - "bfloat16", - "--speculative-algorithm", - "NEXTN", - "--speculative-num-steps", - 3, - "--speculative-eagle-topk", - 1, - "--speculative-num-draft-tokens", - 4, -] - - -class TestNPUQwen3_6_27B_2P_In64k_Out1k_50ms(TestNpuPerformanceTestCaseBase): - """Test NPU performance for Qwen3.6-27B-w8a8 2p in64k out1k 50ms""" - - benchmark_tool = BENCHMARK_TOOL_DEFAULT - aisbench_dataset_type = AISBENCHMARK_DATASET_DEFAULT - model = QWEN3_6_27B_W8A8_MODEL_PATH - other_args = QWEN3_6_27B_64K_1K_OTHER_ARGS - envs = QWEN3_6_27B_64K_1K_ENVS - dataset_name = "random" - max_concurrency = 6 - num_prompts = 24 - input_len = 64000 - output_len = 1000 - random_range_ratio = 1 - seed = 1 - tpot = 50 - output_token_throughput = 122.6 - - def test_npu_qwen3_6_27b_2p_in64k_out1k_50ms(self): - """Run NPU performance test for Qwen3.6-27B-w8a8 in64k out1k 50ms""" - self.run_throughput() - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16.py b/test/registered/npu/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16.py deleted file mode 100644 index 598d23d92..000000000 --- a/test/registered/npu/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16.py +++ /dev/null @@ -1,110 +0,0 @@ -import unittest - -from sglang.test.ascend.e2e.test_npu_performance_utils import ( - QWEN3_NEXT_80B_A3B_MODEL_PATH, - QWEN3_NEXT_80B_A3B_W8A8_MODEL_PATH, - TestNpuPerformanceTestCaseBase, -) -from sglang.test.ci.ci_register import register_npu_ci - -register_npu_ci(est_time=3600, suite="nightly-perf-4-npu-a3", nightly=True) - -QWEN3_NEXT_80B_A3B_ENVS = { - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", - "STREAMS_PER_DEVICE": "32", - "HCCL_SOCKET_IFNAME": "lo", - "GLOO_SOCKET_IFNAME": "lo", - "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", - "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "400", - "DEEPEP_NORMAL_LONG_SEQ_ROUND": "10", - "DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "2048", - "HCCL_OP_EXPANSION_MODE": "AIV", - "TASK_QUEUE_ENABLE": "1", - "ASCEND_USE_FIA": "1", - "SGLANG_NPU_USE_MULTI_STREAM": "0", - "SGLANG_WARMUP_TIMEOUT": "3600", - "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", - "FORCE_DRAFT_MODEL_NON_QUANT": "1", - "DEEPEP_HCCL_BUFFSIZE": "2000", - "ZBCCL_LOCAL_MEM_SIZE": "60416", - "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0", - "ZBCCL_BOOTSTRAP_URL": "tcp://127.0.0.1:24669", - "ZBCCL_NPU_ALLOC_CONF": "use_vmm_for_static_memory:True", - "ZBCCL_ENABLE_GRAPH": "1", -} - -QWEN3_NEXT_80B_A3B_OTHER_ARGS = [ - "--trust-remote-code", - "--attention-backend", - "ascend", - "--device", - "npu", - "--quantization", - "modelslim", - "--page-size", - 128, - "--tp-size", - 4, - "--watchdog-timeout", - 9000, - "--mem-fraction-static", - 0.85, - "--disable-radix-cache", - "--max-prefill-tokens", - 28672, - "--context-length", - 81920, - "--max-total-tokens", - 122304, - "--dp-size", - 2, - "--enable-dp-attention", - "--enable-dp-lm-head", - "--speculative-algorithm", - "NEXTN", - "--speculative-num-steps", - 3, - "--speculative-eagle-topk", - 1, - "--speculative-num-draft-tokens", - 4, - "--speculative-draft-model-quantization", - "unquant", - "--chunked-prefill-size", - -1, - "--max-running-requests", - 16, - "--cuda-graph-bs", - 2, - 4, - 8, - "--mamba-ssm-dtype", - "bfloat16", - "--speculative-draft-model-path", - QWEN3_NEXT_80B_A3B_MODEL_PATH, - "--reasoning-parser", - "qwen3", - "--tool-call-parser", - "qwen3_coder", -] - - -class TestQwen3Next80BA3B(TestNpuPerformanceTestCaseBase): - max_attempts = 5 - model = QWEN3_NEXT_80B_A3B_W8A8_MODEL_PATH - other_args = QWEN3_NEXT_80B_A3B_OTHER_ARGS - envs = QWEN3_NEXT_80B_A3B_ENVS - dataset_name = "random" - max_concurrency = 16 - num_prompts = 16 - input_len = 6144 - output_len = 1500 - random_range_ratio = 1 - tpot = 15.62 - - def test_qwen3_next_80b_a3b(self): - self.run_throughput() - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/rerank_models/test_npu_bge_reranker_v2_m3.py b/test/registered/npu/rerank_models/test_npu_bge_reranker_v2_m3.py deleted file mode 100644 index d395bae96..000000000 --- a/test/registered/npu/rerank_models/test_npu_bge_reranker_v2_m3.py +++ /dev/null @@ -1,98 +0,0 @@ -import multiprocessing as mp -import unittest - -import torch - -from sglang.test.ascend.test_ascend_utils import BGE_RERANKER_V2_M3_WEIGHTS_PATH -from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.runners import TEST_RERANK_QUERY_DOCS, HFRunner, SRTRunner -from sglang.test.test_utils import CustomTestCase - -register_npu_ci( - est_time=400, - suite="full-1-npu-a3", - nightly=True, -) - -MODELS = [ - (BGE_RERANKER_V2_M3_WEIGHTS_PATH, 1, 1e-2), -] -ATTENTION_BACKEND = ["ascend"] -TORCH_DTYPES = [torch.bfloat16] - - -class TestBgeReranker(CustomTestCase): - """Testcase: This test case validates that the cross-encoder scores from the BAAI/bge-reranker-v2-m3 model in the - SGLang framework are less than 1e-2 different from the Hugging Face implementation. - - [Test Category] Model - [Test Target] BAAI/bge-reranker-v2-m3 - """ - - @classmethod - def setUpClass(cls): - mp.set_start_method("spawn", force=True) - - def assert_close_prefill_logits( - self, - prompts, - model_path, - tp_size, - torch_dtype, - score_tolerance, - attention_backend, - ) -> None: - with HFRunner( - model_path, - torch_dtype=torch_dtype, - model_type="cross_encoder", - ) as hf_runner: - hf_scores = hf_runner.forward(prompts).scores - - with SRTRunner( - model_path, - tp_size=tp_size, - torch_dtype=torch_dtype, - model_type="cross_encoder", - attention_backend=attention_backend, - chunked_prefill_size=-1, - disable_radix_cache=True, - ) as srt_runner: - srt_scores = srt_runner.forward(prompts).scores - - for i in range(len(srt_scores)): - score_difference = abs(hf_scores[i] - srt_scores[i]) - - assert score_difference < score_tolerance, ( - "cross encoder scores are not all close" - ) - - def preprocess_prompts(self, prompt): - processed_prompts = [] - query = prompt["query"] - documents = prompt["documents"] - for document in documents: - processed_prompts.append([query, document]) - - return processed_prompts - - def test_prefill_logits(self): - models_to_test = MODELS - - for model, tp_size, prefill_tolerance in models_to_test: - for attention_backend in ATTENTION_BACKEND: - for queryDocs in TEST_RERANK_QUERY_DOCS: - prompts = self.preprocess_prompts(queryDocs) - for torch_dtype in TORCH_DTYPES: - self.assert_close_prefill_logits( - prompts, - model, - tp_size, - torch_dtype, - prefill_tolerance, - attention_backend, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/reward_models/test_npu_internlm2_7b_reward.py b/test/registered/npu/reward_models/test_npu_internlm2_7b_reward.py deleted file mode 100644 index a0877d30f..000000000 --- a/test/registered/npu/reward_models/test_npu_internlm2_7b_reward.py +++ /dev/null @@ -1,65 +0,0 @@ -import os - -os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python" -import multiprocessing as mp -import unittest - -import torch - -from sglang.test.ascend.test_ascend_utils import INTERNLM2_7B_REWARD_WEIGHTS_PATH -from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.runners import SRTRunner -from sglang.test.test_utils import CustomTestCase - -register_npu_ci( - est_time=400, - suite="full-4-npu-a3", - nightly=True, -) - -PROMPT = ( - "What is the range of the numeric output of a sigmoid node in a neural network?" -) -RESPONSE1 = "The output of a sigmoid node is bounded between -1 and 1." -RESPONSE2 = "The output of a sigmoid node is bounded between 0 and 1." - -CONVS = [ - [{"role": "user", "content": PROMPT}, {"role": "assistant", "content": RESPONSE1}], - [{"role": "user", "content": PROMPT}, {"role": "assistant", "content": RESPONSE2}], -] - - -class TestInternlm2(CustomTestCase): - """Testcase: This test case verifies that the Shanghai_AI_Laboratory/internlm2-7b-reward model can successfully generate reward - scores for different conversational responses using the SGLang framework, without comparing to a reference implementation. - - [Test Category] Model - [Test Target] Shanghai_AI_Laboratory/internlm2-7b-reward - """ - - model_path = INTERNLM2_7B_REWARD_WEIGHTS_PATH - torch_dtype = torch.float16 - - @classmethod - def setUpClass(cls): - mp.set_start_method("spawn", force=True) - - def test_assert_close_reward_scores(self): - with SRTRunner( - self.model_path, - torch_dtype=self.torch_dtype, - model_type="reward", - trust_remote_code=True, - disable_cuda_graph=True, - tp_size=4, - mem_fraction_static=0.8, - ) as srt_runner: - prompts = srt_runner.tokenizer.apply_chat_template(CONVS, tokenize=False) - srt_outputs = srt_runner.forward(prompts) - srt_scores = torch.tensor(srt_outputs.scores) - print(f"accuracy: {srt_scores}") - self.assertIsInstance(srt_scores, torch.Tensor) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/reward_models/test_npu_llama_3_1_8b_v0_2.py b/test/registered/npu/reward_models/test_npu_llama_3_1_8b_v0_2.py deleted file mode 100644 index 14118d683..000000000 --- a/test/registered/npu/reward_models/test_npu_llama_3_1_8b_v0_2.py +++ /dev/null @@ -1,86 +0,0 @@ -import multiprocessing as mp -import unittest - -import torch - -from sglang.test.ascend.test_ascend_utils import ( - SKYWORK_REWARD_LLAMA_3_1_8B_V0_2_WEIGHTS_PATH, -) -from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.runners import HFRunner, SRTRunner -from sglang.test.test_utils import CustomTestCase - -register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) - -MODELS = [ - (SKYWORK_REWARD_LLAMA_3_1_8B_V0_2_WEIGHTS_PATH, 1, 4e-2), -] -TORCH_DTYPES = [torch.float16] - -PROMPT = ( - "What is the range of the numeric output of a sigmoid node in a neural network?" -) -RESPONSE1 = "The output of a sigmoid node is bounded between -1 and 1." -RESPONSE2 = "The output of a sigmoid node is bounded between 0 and 1." - -CONVS = [ - [{"role": "user", "content": PROMPT}, {"role": "assistant", "content": RESPONSE1}], - [{"role": "user", "content": PROMPT}, {"role": "assistant", "content": RESPONSE2}], -] - - -class TestLlama(CustomTestCase): - """Testcase: This test case validates that the reward scores from the Skywork/Skywork-Reward-Llama-3.1-8B-v0.2 model - in the SGLang framework are less than 4e-2 different from the Hugging Face implementation. - - [Test Category] Model - [Test Target] Skywork/Skywork-Reward-Llama-3.1-8B-v0.2 - """ - - @classmethod - def setUpClass(cls): - mp.set_start_method("spawn", force=True) - - def assert_close_reward_scores( - self, - convs, - model_path, - tp_size, - torch_dtype, - tolerance, - ) -> None: - with HFRunner( - model_path, - torch_dtype=torch_dtype, - model_type="reward", - ) as hf_runner: - hf_outputs = hf_runner.forward(convs) - - with SRTRunner( - model_path, - tp_size=tp_size, - torch_dtype=torch_dtype, - model_type="reward", - ) as srt_runner: - prompts = srt_runner.tokenizer.apply_chat_template(convs, tokenize=False) - srt_outputs = srt_runner.forward(prompts) - - hf_scores = torch.tensor(hf_outputs.scores) - srt_scores = torch.tensor(srt_outputs.scores) - print(f"{hf_scores=}") - print(f"{srt_scores=}") - - assert torch.all(abs(hf_scores - srt_scores) < tolerance), ( - "reward scores are not all close" - ) - - def test_reward_scores(self): - for model, tp_size, tolerance in MODELS: - for torch_dtype in TORCH_DTYPES: - self.assert_close_reward_scores( - CONVS, model, tp_size, torch_dtype, tolerance - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/vlm_models/test_npu_deepseek_vl2.py b/test/registered/npu/vlm_models/test_npu_deepseek_vl2.py deleted file mode 100644 index 27ae7dedc..000000000 --- a/test/registered/npu/vlm_models/test_npu_deepseek_vl2.py +++ /dev/null @@ -1,29 +0,0 @@ -import unittest - -from sglang.test.ascend.test_ascend_utils import DEEPSEEK_VL2_WEIGHTS_PATH -from sglang.test.ascend.vlm_utils import TestVLMModels -from sglang.test.ci.ci_register import register_npu_ci - -register_npu_ci( - est_time=400, - suite="full-4-npu-a3", - nightly=True, -) - - -class TestDeepseekVl2(TestVLMModels): - """Testcase: Verify that the inference accuracy of the deepseek-ai/deepseek-vl2 model on the MMMU dataset is no less than 0.2. - - [Test Category] Model - [Test Target] deepseek-ai/deepseek-vl2 - """ - - model = DEEPSEEK_VL2_WEIGHTS_PATH - mmmu_accuracy = 0.2 - - def test_vlm_mmmu_benchmark(self): - self._run_vlm_mmmu_test() - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/vlm_models/test_npu_llama_3_2_11b_vision_instruct.py b/test/registered/npu/vlm_models/test_npu_llama_3_2_11b_vision_instruct.py deleted file mode 100644 index 12c8b3200..000000000 --- a/test/registered/npu/vlm_models/test_npu_llama_3_2_11b_vision_instruct.py +++ /dev/null @@ -1,36 +0,0 @@ -import unittest - -from sglang.test.ascend.test_ascend_utils import ( - LLAMA_3_2_11B_VISION_INSTRUCT_WEIGHTS_PATH, -) -from sglang.test.ascend.vlm_utils import TestVLMModels -from sglang.test.ci.ci_register import register_npu_ci - -register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) - - -class TestLlama3211BVisionInstruct(TestVLMModels): - """Testcase: Verify that the inference accuracy of the LLM-Research/Llama-3.2-11B-Vision-Instruct model on the MMMU dataset is no less than 0.2. - - [Test Category] Model - [Test Target] LLM-Research/Llama-3.2-11B-Vision-Instruct - """ - - model = LLAMA_3_2_11B_VISION_INSTRUCT_WEIGHTS_PATH - mmmu_accuracy = 0.2 - other_args = [ - "--trust-remote-code", - "--mem-fraction-static", - "0.8", - "--attention-backend", - "ascend", - "--disable-cuda-graph", - "--disable-radix-cache", - ] - - def test_vlm_mmmu_benchmark(self): - self._run_vlm_mmmu_test() - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/vlm_models/test_npu_minicpm_v_2_6.py b/test/registered/npu/vlm_models/test_npu_minicpm_v_2_6.py deleted file mode 100644 index 846c38fc9..000000000 --- a/test/registered/npu/vlm_models/test_npu_minicpm_v_2_6.py +++ /dev/null @@ -1,25 +0,0 @@ -import unittest - -from sglang.test.ascend.test_ascend_utils import MINICPM_V_2_6_WEIGHTS_PATH -from sglang.test.ascend.vlm_utils import TestVLMModels -from sglang.test.ci.ci_register import register_npu_ci - -register_npu_ci(est_time=400, suite="full-4-npu-a3", nightly=True) - - -class TestMiniCPMModelsV(TestVLMModels): - """Testcase: Verify that the inference accuracy of the openbmb/MiniCPM-V-2_6 model on the MMMU dataset is no less than 0.2. - - [Test Category] Model - [Test Target] openbmb/MiniCPM-V-2_6 - """ - - model = MINICPM_V_2_6_WEIGHTS_PATH - mmmu_accuracy = 0.2 - - def test_vlm_mmmu_benchmark(self): - self._run_vlm_mmmu_test() - - -if __name__ == "__main__": - unittest.main() diff --git a/test/run_suite.py b/test/run_suite.py index 53fdc3726..665de6773 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -153,25 +153,30 @@ NIGHTLY_SUITES = { ], HWBackend.CPU: [], HWBackend.NPU: [ - "nightly-1-npu-a2", "nightly-1-npu-a3", "nightly-2-npu-a3", "nightly-4-npu-a3", "nightly-8-npu-a3", "nightly-16-npu-a3", "nightly-acc-2-npu-a3", - "nightly-acc-4-npu-a3", "nightly-acc-8-npu-a3", "nightly-acc-16-npu-a3", "nightly-perf-2-npu-a3", "nightly-perf-4-npu-a3", - "nightly-perf-8-npu-a3", "nightly-perf-16-npu-a3", + "full-4-npu-a2", "full-1-npu-a3", "full-2-npu-a3", "full-4-npu-a3", "full-8-npu-a3", "full-16-npu-a3", + "full-acc-2-npu-a3", + "full-acc-4-npu-a3", + "full-acc-16-npu-a3", + "full-perf-2-npu-a3", + "full-perf-4-npu-a3", + "full-perf-8-npu-a3", + "full-perf-16-npu-a3", ], HWBackend.XPU: [ "nightly-xpu-1-gpu",