From e76d0acdc923d992bbda20d4b2bc51db9ac314a7 Mon Sep 17 00:00:00 2001 From: Sugar920 <121632458+Sugar920@users.noreply.github.com> Date: Wed, 5 Aug 2026 01:17:58 +0800 Subject: [PATCH] migrate NPU PR/nightly test cases to a3-560T (#33346) Co-authored-by: Claude Co-authored-by: sglang-npu-bot --- .github/workflows/pr-test-npu.yml | 286 ++++++------------ .../development/contribution_guide.mdx | 2 +- .../sglang/test/ascend/gsm8k_ascend_mixin.py | 8 + .../test/ascend/npu_eval_accuracy_kit.py | 22 ++ .../HiCache/test_npu_hicache_mha.py | 43 ++- .../HiCache/test_npu_hicache_mla.py | 36 ++- .../backends/test_npu_sampling_backend.py | 6 +- .../dp_attn/test_npu_dp_attention.py | 6 +- .../test_npu_compile_graph_tp1_bf16.py | 38 +-- .../test_npu_piecewise_graph_prefill.py | 2 +- .../test_npu_load_balance_method.py | 4 + ...u_load_balance_method_pd_disaggregation.py | 4 + .../expert_parallelism/test_npu_deepep.py | 36 ++- ...b_min_rebalancing_utilization_threshold.py | 4 + .../test_npu_moe_dense_tp_size.py | 4 + .../quant/test_npu_autoround_dense.py | 38 +-- .../quant/test_npu_autoround_moe.py | 38 +-- .../basic_function/quant/test_npu_gptq_moe.py | 38 +-- .../runtime_opts/test_npu_mla_fia_w8a8int8.py | 38 +-- .../runtime_opts/test_npu_mla_w8a8int8.py | 36 ++- .../runtime_opts/test_npu_tp1_bf16.py | 38 +-- .../runtime_opts/test_npu_tp2_bf16.py | 38 +-- .../runtime_opts/test_npu_tp2_fia_bf16.py | 38 +-- .../runtime_opts/test_npu_tp4_bf16.py | 36 ++- .../test_npu_basic_sanity_eagle3.py | 4 +- .../test_npu_speculative_attention_mode.py | 31 +- ...npu_speculative_draft_attention_backend.py | 4 + .../test_npu_speculative_moe_a2a_backend.py | 39 +-- .../test_npu_speculative_token_map.py | 60 ++-- ...u_llama_2_7b_communications_compression.py | 2 +- ...pu_qwen3_8b_communications_quantization.py | 2 +- .../test_vision_backend_selection.py | 3 +- test/run_suite.py | 5 +- 33 files changed, 502 insertions(+), 487 deletions(-) diff --git a/.github/workflows/pr-test-npu.yml b/.github/workflows/pr-test-npu.yml index 641670ec3..3387c6b78 100644 --- a/.github/workflows/pr-test-npu.yml +++ b/.github/workflows/pr-test-npu.yml @@ -130,116 +130,10 @@ jobs: cd test python3 run_suite.py --hw npu --suite stage-a-unit-test-npu - stage-b-test-1-npu-a2: - needs: [check-changes, pr-gate, set-image-config] - if: needs.check-changes.outputs.main_package == 'true' - runs-on: linux-aarch64-a2-1 - strategy: - fail-fast: false - matrix: - part: [ 0, 1 ] - container: - image: ${{ needs.set-image-config.outputs.CANN_image_910b }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.ref }} - - - name: Mark repository safe - run: | - git config --system --add safe.directory ${GITHUB_WORKSPACE} - - - name: Install dependencies - env: - TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu" - PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/" - RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082" - run: | - # speed up by using infra cache services - CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local" - sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list - pip config set global.index-url http://${CACHING_URL}/pypi/simple - pip config set global.trusted-host "${CACHING_URL}" - - bash scripts/ci/npu/npu_ci_install_dependency.sh 910b - # copy required file from our daily cache - cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp - # copy gsm8k dataset - cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp - - - name: Run test - timeout-minutes: 60 - env: - SGLANG_USE_MODELSCOPE: true - SGLANG_IS_IN_CI: true - HF_ENDPOINT: https://hf-mirror.com - TORCH_EXTENSIONS_DIR: /tmp/torch_extensions - PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True" - STREAMS_PER_DEVICE: 32 - run: | - cd test - python3 run_suite.py --hw npu --suite stage-b-test-1-npu-a2 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 - - stage-b-test-2-npu-a2: - needs: [check-changes, pr-gate, set-image-config] - if: needs.check-changes.outputs.main_package == 'true' - runs-on: linux-aarch64-a2-2 - strategy: - fail-fast: true - matrix: - part: [0, 1] - container: - image: ${{ needs.set-image-config.outputs.CANN_image_910b }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.ref }} - - - name: Mark repository safe - run: | - git config --system --add safe.directory ${GITHUB_WORKSPACE} - - - name: Install dependencies - env: - TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu" - PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/" - RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082" - run: | - # speed up by using infra cache services - CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local" - sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list - pip config set global.index-url http://${CACHING_URL}/pypi/simple - pip config set global.trusted-host "${CACHING_URL}" - - bash scripts/ci/npu/npu_ci_install_dependency.sh 910b - # copy required file from our daily cache - cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp - # copy gsm8k dataset - cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp - - - name: Run test - timeout-minutes: 60 - env: - SGLANG_USE_MODELSCOPE: true - SGLANG_IS_IN_CI: true - HF_ENDPOINT: https://hf-mirror.com - TORCH_EXTENSIONS_DIR: /tmp/torch_extensions - PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True" - STREAMS_PER_DEVICE: 32 - run: | - cd test - python3 run_suite.py --hw npu --suite stage-b-test-2-npu-a2 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 - stage-b-test-1-npu-a3: needs: [check-changes, pr-gate, set-image-config] if: needs.check-changes.outputs.main_package == 'true' - runs-on: linux-aarch64-a3-2 + runs-on: linux-aarch64-a3-2- container: image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} steps: @@ -289,7 +183,7 @@ jobs: stage-b-test-2-npu-a3: needs: [check-changes, pr-gate, set-image-config] if: needs.check-changes.outputs.main_package == 'true' - runs-on: linux-aarch64-a3-2 + runs-on: linux-aarch64-a3-2- container: image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} steps: @@ -339,7 +233,7 @@ jobs: stage-b-test-4-npu-a3: needs: [check-changes, pr-gate, set-image-config] if: needs.check-changes.outputs.main_package == 'true' - runs-on: linux-aarch64-a3-4 + runs-on: linux-aarch64-a3-4- strategy: fail-fast: false matrix: @@ -393,7 +287,7 @@ jobs: stage-b-test-8-npu-a3: needs: [check-changes, pr-gate, set-image-config] if: needs.check-changes.outputs.main_package == 'true' - runs-on: linux-aarch64-a3-8 + runs-on: linux-aarch64-a3-8- container: image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} steps: @@ -442,7 +336,7 @@ jobs: stage-b-test-16-npu-a3: needs: [check-changes, pr-gate, set-image-config] if: needs.check-changes.outputs.main_package == 'true' - runs-on: linux-aarch64-a3-16 + runs-on: linux-aarch64-a3-16- container: image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} steps: @@ -493,7 +387,7 @@ jobs: multimodal-gen-test-1-npu-a3: needs: [check-changes, pr-gate, set-image-config] if: needs.check-changes.outputs.multimodal_gen == 'true' - runs-on: linux-aarch64-a3-2 + runs-on: linux-aarch64-a3-800t-2 container: image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} steps: @@ -550,7 +444,7 @@ jobs: multimodal-gen-test-2-npu-a3: needs: [check-changes, pr-gate, set-image-config] if: needs.check-changes.outputs.multimodal_gen == 'true' - runs-on: linux-aarch64-a3-16 + runs-on: linux-aarch64-a3-800t-2 container: image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} steps: @@ -615,105 +509,107 @@ jobs: test_config: # qwen3_6_27b performance tests - name: qwen3_6_27b_w8a8_1p_in64k_out1k_50ms - runner: linux-aarch64-a3-2 + runner: linux-aarch64-a3-800t-2 test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py test_type: 'perf' - - name: qwen3_6_27b_1p_in1024x1024_30_out1024_50ms - runner: linux-aarch64-a3-2 - test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py - test_type: 'perf' + # - name: qwen3_6_27b_1p_in1024x1024_30_out1024_50ms + # runner: linux-aarch64-a3-800t-2 + # test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py + # test_type: 'perf' - # qwen3_8b performance tests - - name: qwen3_8b_w8a8_1p_in3k5_out1k5_50ms - runner: linux-aarch64-a3-2 - test_case: test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py - test_type: 'perf' + # # qwen3_8b performance tests + # - name: qwen3_8b_w8a8_1p_in3k5_out1k5_50ms + # runner: linux-aarch64-a3-800t-2 + # test_case: test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py + # test_type: 'perf' - # qwen3_30b_a3b performance tests - - name: qwen3_30b_w8a8_1p_in3k5_out1k5_50ms - runner: linux-aarch64-a3-2 - test_case: test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py - test_type: 'perf' + # # qwen3_30b_a3b performance tests + # - name: qwen3_30b_w8a8_1p_in3k5_out1k5_50ms + # runner: linux-aarch64-a3-800t-2 + # test_case: test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py + # test_type: 'perf' - # qwen3_6_35b_a3b performance tests - - name: qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms - runner: linux-aarch64-a3-2 - test_case: test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms.py - test_type: 'perf' + # # qwen3_6_35b_a3b performance tests + # - name: qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms + # runner: linux-aarch64-a3-800t-2 + # test_case: test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms.py + # test_type: 'perf' - # qwen3_vl_8b_thinking accuracy tests - - name: qwen3_vl_8b_thinking_1p_mmmu - runner: linux-aarch64-a3-2 - test_case: test/registered/npu/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py - test_type: 'accuracy' + # # qwen3_vl_8b_thinking accuracy tests + # - name: qwen3_vl_8b_thinking_1p_mmmu + # runner: linux-aarch64-a3-2- + # test_case: test/registered/npu/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py + # test_type: 'accuracy' - # qwen3_32b performance tests - - name: qwen3_32b_w8a8_2p_in3k5_out1k5_50ms - runner: linux-aarch64-a3-4 - test_case: test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py - test_type: 'perf' + # # qwen3_32b performance tests + # - name: qwen3_32b_w8a8_2p_in3k5_out1k5_50ms + # runner: linux-aarch64-a3-800t-4 + # test_case: test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py + # test_type: 'perf' - # qwen3_next_80b_a3b performance tests - - name: qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16 - runner: linux-aarch64-a3-4 - test_case: test/registered/npu/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16.py - test_type: 'perf' + # # qwen3_next_80b_a3b performance tests + # - name: qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16 + # runner: linux-aarch64-a3-800t-4 + # test_case: test/registered/npu/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16.py + # test_type: 'perf' - # minimax_m2_5 performance tests - - name: minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms - runner: linux-aarch64-a3-8 - test_case: test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py - test_type: 'perf' + # # minimax_m2_5 performance tests + # - name: minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms + # runner: linux-aarch64-a3-800t-8 + # test_case: test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py + # test_type: 'perf' - # deepseek_v4_flash performance tests - - name: deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms - runner: linux-aarch64-a3-16 - test_case: test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py - test_type: 'perf' + # # deepseek_v4_flash performance tests + # - name: deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms + # runner: linux-aarch64-a3-800t-16 + # test_case: test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py + # test_type: 'perf' - # kimi_k2_6 performance tests - - name: kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms - runner: linux-aarch64-a3-16 - test_case: test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms.py - test_type: 'perf' + # # kimi_k2_6 performance tests + # - name: kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms + # runner: linux-aarch64-a3-800t-16 + # test_case: test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms.py + # test_type: 'perf' - # qwen3_235b performance tests - - name: qwen3_235b_w8a8_8p_in3k5_out1k5_50ms - runner: linux-aarch64-a3-16 - test_case: test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py - test_type: 'perf' + # # qwen3_235b performance tests + # - name: qwen3_235b_w8a8_8p_in3k5_out1k5_50ms + # runner: linux-aarch64-a3-800t-16 + # test_case: test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py + # test_type: 'perf' + + # # qwen3_5_397b performance tests + # - name: qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms + # runner: linux-aarch64-a3-800t-16 + # test_case: test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py + # test_type: 'perf' - # qwen3_5_397b performance tests - - name: qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms - runner: linux-aarch64-a3-16 - test_case: test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py - test_type: 'perf' # NPU accuracy tests - - name: glm4_7_flash_1p_gsm8k - runner: linux-aarch64-a3-2 - test_case: test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_gsm8k.py - test_type: 'accuracy' - - name: qwen3_vl_30b_a3b_bf16_2p_gsm8k - runner: linux-aarch64-a3-4 - test_case: test/registered/npu/accuracy/qwen3_vl_30b_a3b/test_npu_qwen3_vl_30b_a3b_bf16_2p_gsm8k.py - test_type: 'accuracy' + # - name: glm4_7_flash_1p_gsm8k + # runner: linux-aarch64-a3-2- + # test_case: test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_gsm8k.py + # test_type: 'accuracy' + # - name: qwen3_vl_30b_a3b_bf16_2p_gsm8k + # runner: linux-aarch64-a3-4- + # test_case: test/registered/npu/accuracy/qwen3_vl_30b_a3b/test_npu_qwen3_vl_30b_a3b_bf16_2p_gsm8k.py + # test_type: 'accuracy' - name: glm5_top64_pruned_bf16_8p_gsm8k - runner: linux-aarch64-a3-16 + runner: linux-aarch64-a3-16- test_case: test/registered/npu/accuracy/glm5_top64_pruned/test_npu_glm5_top64_pruned_bf16_8p_gsm8k.py test_type: 'accuracy' - - name: moonshotai_moonlight_16b_a3b_bf16_1p_gsm8k - runner: linux-aarch64-a3-2 - test_case: test/registered/npu/accuracy/moonshotai_moonlight_16b_a3b/test_npu_moonlight_16b_a3b_bf16_1p_gsm8k.py - test_type: 'accuracy' - - name: qwen3_5_9b_bf16_1p_gsm8k - runner: linux-aarch64-a3-2 - test_case: test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py - test_type: 'accuracy' - - name: qwen3_vl_8b_bf16_2p_gsm8k - runner: linux-aarch64-a3-4 - test_case: test/registered/npu/accuracy/qwen3_vl_8b/test_npu_qwen3_vl_8b_bf16_2p_gsm8k.py - test_type: 'accuracy' + # - name: moonshotai_moonlight_16b_a3b_bf16_1p_gsm8k + # runner: linux-aarch64-a3-2- + # test_case: test/registered/npu/accuracy/moonshotai_moonlight_16b_a3b/test_npu_moonlight_16b_a3b_bf16_1p_gsm8k.py + # test_type: 'accuracy' + # - name: qwen3_5_9b_bf16_1p_gsm8k + # runner: linux-aarch64-a3-2- + # test_case: test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py + # test_type: 'accuracy' + # - name: qwen3_vl_8b_bf16_2p_gsm8k + # runner: linux-aarch64-a3-4- + # test_case: test/registered/npu/accuracy/qwen3_vl_8b/test_npu_qwen3_vl_8b_bf16_2p_gsm8k.py + # test_type: 'accuracy' + uses: ./.github/workflows/nightly-test-npu-e2e-single-node.yml with: runner: ${{ matrix.test_config.runner }} @@ -732,9 +628,7 @@ jobs: check-changes, stage-a-unit-test-npu, - stage-b-test-1-npu-a2, stage-b-test-1-npu-a3, - stage-b-test-2-npu-a2, stage-b-test-2-npu-a3, stage-b-test-4-npu-a3, stage-b-test-8-npu-a3, diff --git a/docs/docs/hardware-platforms/ascend-npus/development/contribution_guide.mdx b/docs/docs/hardware-platforms/ascend-npus/development/contribution_guide.mdx index 25bb50ff1..aaf9bdc13 100644 --- a/docs/docs/hardware-platforms/ascend-npus/development/contribution_guide.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/development/contribution_guide.mdx @@ -66,7 +66,7 @@ See [`test_npu_sampling_backend.py`](https://github.com/sgl-project/sglang/tree/ 4. Register your test with `register_npu_ci()`: ```python from sglang.test.ci.ci_register import register_npu_ci - register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a2", nightly=False) + register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False) register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) ``` diff --git a/python/sglang/test/ascend/gsm8k_ascend_mixin.py b/python/sglang/test/ascend/gsm8k_ascend_mixin.py index 95f5713c5..fcf012ea3 100644 --- a/python/sglang/test/ascend/gsm8k_ascend_mixin.py +++ b/python/sglang/test/ascend/gsm8k_ascend_mixin.py @@ -66,6 +66,14 @@ class GSM8KAscendMixin(ABC): kill_process_tree(cls.process.pid) def test_gsm8k(self): + from sglang.test.ascend.npu_eval_accuracy_kit import ( + _is_pr_pipeline, + run_npu_pr_smoke, + ) + + if _is_pr_pipeline: + run_npu_pr_smoke(self.base_url) + return accuracy_threshold = getattr(self, "accuracy", 0.00) output_throughput_threshold = getattr(self, "output_throughput", 0.00) diff --git a/python/sglang/test/ascend/npu_eval_accuracy_kit.py b/python/sglang/test/ascend/npu_eval_accuracy_kit.py index 42b54fab5..575a6057d 100644 --- a/python/sglang/test/ascend/npu_eval_accuracy_kit.py +++ b/python/sglang/test/ascend/npu_eval_accuracy_kit.py @@ -16,16 +16,38 @@ Usage:: gsm8k_accuracy_thres = 0.6 """ +import os + +import requests + from sglang.test.kits.eval_accuracy_kit import GSM8KMixin _NPU_ACCURACY_TOLERANCE = 0.99 _NPU_MAX_ACCURACY_ATTEMPTS = 3 +_is_pr_pipeline = os.environ.get("GITHUB_EVENT_NAME") == "pull_request" + + +def run_npu_pr_smoke(base_url): + """Send a single inference request for PR pipeline smoke test.""" + response = requests.post( + f"{base_url}/generate", + json={ + "text": "The capital of France is", + "sampling_params": {"temperature": 0, "max_new_tokens": 32}, + }, + ) + assert response.status_code == 200 + assert "Paris" in response.text + class NPUGSM8KMixin(GSM8KMixin): """NPU GSM8K accuracy mixin — 1% tolerance + up to 3 retries.""" def test_gsm8k(self): + if _is_pr_pipeline: + run_npu_pr_smoke(self.base_url) + return threshold = self.gsm8k_score_threshold if threshold != threshold: # NaN → legacy alias threshold = self.gsm8k_accuracy_thres diff --git a/test/registered/npu/basic_function/HiCache/test_npu_hicache_mha.py b/test/registered/npu/basic_function/HiCache/test_npu_hicache_mha.py index 829a5fee4..b83fdaf70 100644 --- a/test/registered/npu/basic_function/HiCache/test_npu_hicache_mha.py +++ b/test/registered/npu/basic_function/HiCache/test_npu_hicache_mha.py @@ -3,6 +3,7 @@ from types import SimpleNamespace from urllib.parse import urlparse from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke from sglang.test.ci.ci_register import register_npu_ci from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k from sglang.test.test_utils import ( @@ -12,7 +13,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a2", nightly=False) +register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False) register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) TEST_MODEL_MATRIX = { @@ -45,33 +46,31 @@ class TestAscendMhaHicache(CustomTestCase): def test_a_gsm8k(self): for model in self.models: with self.subTest(model=model): - print(f"##=== Testing accuracy: {model} ===##") - process = popen_launch_server( model, self.base_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - *self.common_args, - ], + other_args=[*self.common_args], ) - try: - args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=1319, - max_new_tokens=512, - parallel=128, - host=f"http://{self.url.hostname}", - port=int(self.url.port), - ) - - metrics = run_eval_few_shot_gsm8k(args) - self.assertGreaterEqual( - metrics["accuracy"], - TEST_MODEL_MATRIX[model]["accuracy"], - ) + if _is_pr_pipeline: + run_npu_pr_smoke(self.base_url) + else: + print(f"##=== Testing accuracy: {model} ===##") + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) finally: kill_process_tree(process.pid) diff --git a/test/registered/npu/basic_function/HiCache/test_npu_hicache_mla.py b/test/registered/npu/basic_function/HiCache/test_npu_hicache_mla.py index 140d590dd..9448071ed 100644 --- a/test/registered/npu/basic_function/HiCache/test_npu_hicache_mla.py +++ b/test/registered/npu/basic_function/HiCache/test_npu_hicache_mla.py @@ -3,6 +3,7 @@ from types import SimpleNamespace from urllib.parse import urlparse from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke from sglang.test.ci.ci_register import register_npu_ci from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k from sglang.test.test_utils import ( @@ -47,8 +48,6 @@ class TestAscendMlaHicache(CustomTestCase): def test_a_gsm8k(self): for model in self.models: with self.subTest(model=model): - print(f"##=== Testing accuracy: {model} ===##") - process = popen_launch_server( model, self.base_url, @@ -59,21 +58,26 @@ class TestAscendMlaHicache(CustomTestCase): ) try: - args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=1319, - max_new_tokens=512, - parallel=128, - host=f"http://{self.url.hostname}", - port=int(self.url.port), - ) + if _is_pr_pipeline: + run_npu_pr_smoke(self.base_url) + else: + print(f"##=== Testing accuracy: {model} ===##") - metrics = run_eval_few_shot_gsm8k(args) - self.assertGreaterEqual( - metrics["accuracy"], - TEST_MODEL_MATRIX[model]["accuracy"], - ) + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) finally: kill_process_tree(process.pid) diff --git a/test/registered/npu/basic_function/backends/test_npu_sampling_backend.py b/test/registered/npu/basic_function/backends/test_npu_sampling_backend.py index 7da4595f2..8b278d3f5 100644 --- a/test/registered/npu/basic_function/backends/test_npu_sampling_backend.py +++ b/test/registered/npu/basic_function/backends/test_npu_sampling_backend.py @@ -4,6 +4,7 @@ from types import SimpleNamespace import requests from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke from sglang.test.ci.ci_register import register_npu_ci from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( @@ -13,7 +14,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a2", nightly=False) +register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False) register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) @@ -41,6 +42,9 @@ class TestAscendSamplingBackend(CustomTestCase): kill_process_tree(cls.process.pid) def test_mmlu(self): + if _is_pr_pipeline: + run_npu_pr_smoke(self.base_url) + return args = SimpleNamespace( base_url=self.base_url, model=self.model, diff --git a/test/registered/npu/basic_function/dp_attn/test_npu_dp_attention.py b/test/registered/npu/basic_function/dp_attn/test_npu_dp_attention.py index aed6429a8..a93a94312 100644 --- a/test/registered/npu/basic_function/dp_attn/test_npu_dp_attention.py +++ b/test/registered/npu/basic_function/dp_attn/test_npu_dp_attention.py @@ -7,6 +7,7 @@ from sglang.srt.utils import kill_process_tree from sglang.test.ascend.npu_eval_accuracy_kit import NPUGSM8KMixin from sglang.test.ascend.test_ascend_utils import ( DEEPSEEK_V2_LITE_W8A8_WEIGHTS_PATH, + IMAGES_EXAMPLE_PATH, KIMI_VL_A3B_INSTRUCT_WEIGHTS_PATH, QWEN3_32B_WEIGHTS_PATH, ) @@ -16,7 +17,6 @@ from sglang.test.kits.json_constrained_kit import JSONConstrainedMixin from sglang.test.kits.radix_cache_server_kit import run_radix_attention_test from sglang.test.kits.regex_constrained_kit import RegexConstrainedMixin from sglang.test.test_utils import ( - DEFAULT_IMAGE_URL, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, @@ -74,7 +74,7 @@ class TestDPAttentionMixedChunk( CustomTestCase, NPUGSM8KMixin, ): - gsm8k_accuracy_thres = 0.35 + gsm8k_accuracy_thres = 0.34 @classmethod def setUpClass(cls): @@ -151,7 +151,7 @@ class TestDPAttentionDP2TP2VLM(CustomTestCase): def setUpClass(cls): cls.model = KIMI_VL_A3B_INSTRUCT_WEIGHTS_PATH cls.base_url = DEFAULT_URL_FOR_TEST - cls.image_url = DEFAULT_IMAGE_URL + cls.image_url = IMAGES_EXAMPLE_PATH cls.process = popen_launch_server( cls.model, cls.base_url, diff --git a/test/registered/npu/basic_function/optimization_debug/test_npu_compile_graph_tp1_bf16.py b/test/registered/npu/basic_function/optimization_debug/test_npu_compile_graph_tp1_bf16.py index 2a94826d6..9255b5337 100644 --- a/test/registered/npu/basic_function/optimization_debug/test_npu_compile_graph_tp1_bf16.py +++ b/test/registered/npu/basic_function/optimization_debug/test_npu_compile_graph_tp1_bf16.py @@ -4,6 +4,7 @@ from types import SimpleNamespace from urllib.parse import urlparse from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke from sglang.test.ci.ci_register import register_npu_ci from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k from sglang.test.test_utils import ( @@ -13,7 +14,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a2", nightly=False) +register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False) register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) TEST_MODEL_MATRIX = { @@ -49,8 +50,6 @@ class TestAscendTp1Bf16(CustomTestCase): def test_a_gsm8k(self): for model in self.models: with self.subTest(model=model): - print(f"##=== Testing accuracy: {model} ===##") - process = popen_launch_server( model, self.base_url, @@ -61,21 +60,26 @@ class TestAscendTp1Bf16(CustomTestCase): ) try: - args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=1319, - max_new_tokens=512, - parallel=32, - host=f"http://{self.url.hostname}", - port=int(self.url.port), - ) + if _is_pr_pipeline: + run_npu_pr_smoke(self.base_url) + else: + print(f"##=== Testing accuracy: {model} ===##") - metrics = run_eval_few_shot_gsm8k(args) - self.assertGreaterEqual( - metrics["accuracy"], - TEST_MODEL_MATRIX[model]["accuracy"], - ) + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=32, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) finally: kill_process_tree(process.pid) diff --git a/test/registered/npu/basic_function/optimization_debug/test_npu_piecewise_graph_prefill.py b/test/registered/npu/basic_function/optimization_debug/test_npu_piecewise_graph_prefill.py index 52be19f03..b4b620c9f 100644 --- a/test/registered/npu/basic_function/optimization_debug/test_npu_piecewise_graph_prefill.py +++ b/test/registered/npu/basic_function/optimization_debug/test_npu_piecewise_graph_prefill.py @@ -12,7 +12,7 @@ from sglang.test.test_utils import ( run_bench_one_batch, ) -register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a2", nightly=False) +register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False) register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) diff --git a/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method.py b/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method.py index 49144015f..7f2a60492 100644 --- a/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method.py +++ b/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method.py @@ -6,6 +6,7 @@ from urllib.parse import urlparse import requests from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke from sglang.test.ascend.test_ascend_utils import DEEPSEEK_R1_0528_W8A8_WEIGHTS_PATH from sglang.test.ci.ci_register import register_npu_ci from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k @@ -72,6 +73,9 @@ class TestDPAttentionRoundBinLoadBalance(CustomTestCase): kill_process_tree(cls.process.pid) def test_gsm8k(self): + if _is_pr_pipeline: + run_npu_pr_smoke(self.base_url) + return args = SimpleNamespace( num_shots=5, data_path=None, diff --git a/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method_pd_disaggregation.py b/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method_pd_disaggregation.py index 6d5a5a44e..6a4075cd4 100644 --- a/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method_pd_disaggregation.py +++ b/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method_pd_disaggregation.py @@ -11,6 +11,7 @@ from urllib.parse import urlparse import requests from sglang.test.ascend.disaggregation_utils import TestDisaggregationBase +from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke from sglang.test.ascend.test_ascend_utils import ( QWEN3_30B_A3B_INSTRUCT_2507_WEIGHTS_PATH, ) @@ -134,6 +135,9 @@ class BaseTestNPULoadBalanceMethodDPDisaggregation(TestDisaggregationBase): ) def test_gsm8k(self): + if _is_pr_pipeline: + run_npu_pr_smoke(self.lb_url) + return args = SimpleNamespace( num_shots=5, data_path=None, diff --git a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep.py b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep.py index a56b71300..949701643 100644 --- a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep.py +++ b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep.py @@ -4,6 +4,7 @@ from types import SimpleNamespace from urllib.parse import urlparse from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke from sglang.test.ci.ci_register import register_npu_ci from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k from sglang.test.test_utils import ( @@ -67,8 +68,6 @@ class TestAscendDeepEP(CustomTestCase): def test_a_gsm8k(self): for model in self.models: with self.subTest(model=model): - print(f"##=== Testing accuracy: {model} ===##") - process = popen_launch_server( model, self.base_url, @@ -79,21 +78,26 @@ class TestAscendDeepEP(CustomTestCase): ) try: - args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=500, - max_new_tokens=512, - parallel=128, - host=f"http://{self.url.hostname}", - port=int(self.url.port), - ) + if _is_pr_pipeline: + run_npu_pr_smoke(self.base_url) + else: + print(f"##=== Testing accuracy: {model} ===##") - metrics = run_eval_few_shot_gsm8k(args) - self.assertGreaterEqual( - metrics["accuracy"], - TEST_MODEL_MATRIX[model]["accuracy"], - ) + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=500, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) finally: kill_process_tree(process.pid) diff --git a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_eplb_min_rebalancing_utilization_threshold.py b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_eplb_min_rebalancing_utilization_threshold.py index 7b0cabd67..057a09b51 100644 --- a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_eplb_min_rebalancing_utilization_threshold.py +++ b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_eplb_min_rebalancing_utilization_threshold.py @@ -3,6 +3,7 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke from sglang.test.ascend.test_ascend_utils import QWEN3_30B_A3B_W8A8_WEIGHTS_PATH from sglang.test.ci.ci_register import register_npu_ci from sglang.test.run_eval import run_eval @@ -94,6 +95,9 @@ class TestEplbMinRebalancingUtilizationThresholdBase(CustomTestCase): os.remove("./rebalance_err_log.txt") def test_gsm8k(self): + if _is_pr_pipeline: + run_npu_pr_smoke(self.base_url) + return args = SimpleNamespace( max_tokens=512, base_url=DEFAULT_URL_FOR_TEST, diff --git a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_moe_dense_tp_size.py b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_moe_dense_tp_size.py index cd6ecad7f..1e1364a7c 100644 --- a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_moe_dense_tp_size.py +++ b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_moe_dense_tp_size.py @@ -2,6 +2,7 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke from sglang.test.ascend.test_ascend_utils import DEEPSEEK_CODER_V2_LITE_WEIGHTS_PATH from sglang.test.ci.ci_register import register_npu_ci from sglang.test.run_eval import run_eval @@ -61,6 +62,9 @@ class TestAscendMoeDenseTPSize(CustomTestCase): kill_process_tree(cls.process.pid) def test_gsm8k(self): + if _is_pr_pipeline: + run_npu_pr_smoke(DEFAULT_URL_FOR_TEST) + return args = SimpleNamespace( max_tokens=512, base_url=DEFAULT_URL_FOR_TEST, diff --git a/test/registered/npu/basic_function/quant/test_npu_autoround_dense.py b/test/registered/npu/basic_function/quant/test_npu_autoround_dense.py index 87575681b..f684e5990 100644 --- a/test/registered/npu/basic_function/quant/test_npu_autoround_dense.py +++ b/test/registered/npu/basic_function/quant/test_npu_autoround_dense.py @@ -4,6 +4,7 @@ from types import SimpleNamespace from urllib.parse import urlparse from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke from sglang.test.ascend.test_ascend_utils import QWEN3_8B_INT4_AUTOROUND_WEIGHTS_PATH from sglang.test.ci.ci_register import register_npu_ci from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k @@ -14,7 +15,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a2", nightly=False) +register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False) register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) logger = logging.getLogger(__name__) @@ -46,8 +47,6 @@ class TestAscendAutoRoundDense(CustomTestCase): def test_a_gsm8k(self): for model in self.models: with self.subTest(model=model): - logger.info(f"##=== Testing accuracy: {model} ===##") - process = popen_launch_server( model, self.base_url, @@ -58,21 +57,26 @@ class TestAscendAutoRoundDense(CustomTestCase): ) try: - args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=1319, - max_new_tokens=512, - parallel=128, - host=f"http://{self.url.hostname}", - port=int(self.url.port), - ) + if _is_pr_pipeline: + run_npu_pr_smoke(self.base_url) + else: + logger.info(f"##=== Testing accuracy: {model} ===##") - metrics = run_eval_few_shot_gsm8k(args) - self.assertGreaterEqual( - metrics["accuracy"], - TEST_MODEL_MATRIX[model]["accuracy"], - ) + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) finally: kill_process_tree(process.pid) diff --git a/test/registered/npu/basic_function/quant/test_npu_autoround_moe.py b/test/registered/npu/basic_function/quant/test_npu_autoround_moe.py index f771c67c6..a9641ec69 100644 --- a/test/registered/npu/basic_function/quant/test_npu_autoround_moe.py +++ b/test/registered/npu/basic_function/quant/test_npu_autoround_moe.py @@ -4,6 +4,7 @@ from types import SimpleNamespace from urllib.parse import urlparse from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke from sglang.test.ascend.test_ascend_utils import ( QWEN3_30B_A3B_INSTRUCT_2507_INT4_AUTOROUND_WEIGHTS_PATH, ) @@ -15,7 +16,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a2", nightly=False) +register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False) register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) logger = logging.getLogger(__name__) @@ -47,8 +48,6 @@ class TestAscendAutoRoundMoE(CustomTestCase): def test_a_gsm8k(self): for model in self.models: with self.subTest(model=model): - logger.info(f"##=== Testing accuracy: {model} ===##") - process = popen_launch_server( model, self.base_url, @@ -59,21 +58,26 @@ class TestAscendAutoRoundMoE(CustomTestCase): ) try: - args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=1319, - max_new_tokens=512, - parallel=128, - host=f"http://{self.url.hostname}", - port=int(self.url.port), - ) + if _is_pr_pipeline: + run_npu_pr_smoke(self.base_url) + else: + logger.info(f"##=== Testing accuracy: {model} ===##") - metrics = run_eval_few_shot_gsm8k(args) - self.assertGreaterEqual( - metrics["accuracy"], - TEST_MODEL_MATRIX[model]["accuracy"], - ) + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) finally: kill_process_tree(process.pid) diff --git a/test/registered/npu/basic_function/quant/test_npu_gptq_moe.py b/test/registered/npu/basic_function/quant/test_npu_gptq_moe.py index 686f5daa1..b305f2872 100644 --- a/test/registered/npu/basic_function/quant/test_npu_gptq_moe.py +++ b/test/registered/npu/basic_function/quant/test_npu_gptq_moe.py @@ -4,6 +4,7 @@ from types import SimpleNamespace from urllib.parse import urlparse from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke from sglang.test.ascend.test_ascend_utils import ( QWEN3_30B_A3B_GPTQ_2507_INT4_WEIGHTS_PATH, ) @@ -16,7 +17,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a2", nightly=False) +register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False) register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) logger = logging.getLogger(__name__) @@ -48,8 +49,6 @@ class TestAscendGPTQMoEInt4(CustomTestCase): def test_a_gsm8k(self): for model in self.models: with self.subTest(model=model): - logger.info(f"##=== Testing accuracy: {model} ===##") - process = popen_launch_server( model, self.base_url, @@ -60,21 +59,26 @@ class TestAscendGPTQMoEInt4(CustomTestCase): ) try: - args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=1319, - max_new_tokens=512, - parallel=128, - host=f"http://{self.url.hostname}", - port=int(self.url.port), - ) + if _is_pr_pipeline: + run_npu_pr_smoke(self.base_url) + else: + logger.info(f"##=== Testing accuracy: {model} ===##") - metrics = run_eval_few_shot_gsm8k(args) - self.assertGreaterEqual( - metrics["accuracy"], - TEST_MODEL_MATRIX[model]["accuracy"], - ) + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) finally: kill_process_tree(process.pid) diff --git a/test/registered/npu/basic_function/runtime_opts/test_npu_mla_fia_w8a8int8.py b/test/registered/npu/basic_function/runtime_opts/test_npu_mla_fia_w8a8int8.py index 0b4902837..75d4bcc1a 100644 --- a/test/registered/npu/basic_function/runtime_opts/test_npu_mla_fia_w8a8int8.py +++ b/test/registered/npu/basic_function/runtime_opts/test_npu_mla_fia_w8a8int8.py @@ -4,6 +4,7 @@ from types import SimpleNamespace from urllib.parse import urlparse from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke from sglang.test.ci.ci_register import register_npu_ci from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k from sglang.test.test_utils import ( @@ -13,7 +14,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a2", nightly=False) +register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a3", nightly=False) register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True) TEST_MODEL_MATRIX = { @@ -48,8 +49,6 @@ class TestAscendMlaW8A8Int8(CustomTestCase): os.environ["ASCEND_USE_FIA"] = "true" for model in self.models: with self.subTest(model=model): - print(f"##=== Testing accuracy: {model} ===##") - process = popen_launch_server( model, self.base_url, @@ -60,21 +59,26 @@ class TestAscendMlaW8A8Int8(CustomTestCase): ) try: - args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=1319, - max_new_tokens=512, - parallel=128, - host=f"http://{self.url.hostname}", - port=int(self.url.port), - ) + if _is_pr_pipeline: + run_npu_pr_smoke(self.base_url) + else: + print(f"##=== Testing accuracy: {model} ===##") - metrics = run_eval_few_shot_gsm8k(args) - self.assertGreaterEqual( - metrics["accuracy"], - TEST_MODEL_MATRIX[model]["accuracy"], - ) + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) finally: kill_process_tree(process.pid) diff --git a/test/registered/npu/basic_function/runtime_opts/test_npu_mla_w8a8int8.py b/test/registered/npu/basic_function/runtime_opts/test_npu_mla_w8a8int8.py index c50bee071..94c76b258 100644 --- a/test/registered/npu/basic_function/runtime_opts/test_npu_mla_w8a8int8.py +++ b/test/registered/npu/basic_function/runtime_opts/test_npu_mla_w8a8int8.py @@ -3,6 +3,7 @@ from types import SimpleNamespace from urllib.parse import urlparse from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke from sglang.test.ci.ci_register import register_npu_ci from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k from sglang.test.test_utils import ( @@ -46,8 +47,6 @@ class TestAscendMlaW8A8Int8(CustomTestCase): def test_a_gsm8k(self): for model in self.models: with self.subTest(model=model): - print(f"##=== Testing accuracy: {model} ===##") - process = popen_launch_server( model, self.base_url, @@ -58,21 +57,26 @@ class TestAscendMlaW8A8Int8(CustomTestCase): ) try: - args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=1319, - max_new_tokens=512, - parallel=128, - host=f"http://{self.url.hostname}", - port=int(self.url.port), - ) + if _is_pr_pipeline: + run_npu_pr_smoke(self.base_url) + else: + print(f"##=== Testing accuracy: {model} ===##") - metrics = run_eval_few_shot_gsm8k(args) - self.assertGreaterEqual( - metrics["accuracy"], - TEST_MODEL_MATRIX[model]["accuracy"], - ) + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) finally: kill_process_tree(process.pid) diff --git a/test/registered/npu/basic_function/runtime_opts/test_npu_tp1_bf16.py b/test/registered/npu/basic_function/runtime_opts/test_npu_tp1_bf16.py index b01510dc7..e9740342e 100644 --- a/test/registered/npu/basic_function/runtime_opts/test_npu_tp1_bf16.py +++ b/test/registered/npu/basic_function/runtime_opts/test_npu_tp1_bf16.py @@ -3,6 +3,7 @@ from types import SimpleNamespace from urllib.parse import urlparse from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke from sglang.test.ci.ci_register import register_npu_ci from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k from sglang.test.test_utils import ( @@ -12,7 +13,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a2", nightly=False) +register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False) register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) TEST_MODEL_MATRIX = { @@ -43,8 +44,6 @@ class TestAscendTp1Bf16(CustomTestCase): def test_a_gsm8k(self): for model in self.models: with self.subTest(model=model): - print(f"##=== Testing accuracy: {model} ===##") - process = popen_launch_server( model, self.base_url, @@ -55,21 +54,26 @@ class TestAscendTp1Bf16(CustomTestCase): ) try: - args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=1319, - max_new_tokens=512, - parallel=128, - host=f"http://{self.url.hostname}", - port=int(self.url.port), - ) + if _is_pr_pipeline: + run_npu_pr_smoke(self.base_url) + else: + print(f"##=== Testing accuracy: {model} ===##") - metrics = run_eval_few_shot_gsm8k(args) - self.assertGreaterEqual( - metrics["accuracy"], - TEST_MODEL_MATRIX[model]["accuracy"], - ) + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) finally: kill_process_tree(process.pid) diff --git a/test/registered/npu/basic_function/runtime_opts/test_npu_tp2_bf16.py b/test/registered/npu/basic_function/runtime_opts/test_npu_tp2_bf16.py index 8f85a16c0..2ec3d16dd 100644 --- a/test/registered/npu/basic_function/runtime_opts/test_npu_tp2_bf16.py +++ b/test/registered/npu/basic_function/runtime_opts/test_npu_tp2_bf16.py @@ -3,6 +3,7 @@ from types import SimpleNamespace from urllib.parse import urlparse from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke from sglang.test.ci.ci_register import register_npu_ci from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k from sglang.test.test_utils import ( @@ -12,7 +13,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a2", nightly=False) +register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a3", nightly=False) register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True) TEST_MODEL_MATRIX = { @@ -45,8 +46,6 @@ class TestAscendTp2Bf16(CustomTestCase): def test_a_gsm8k(self): for model in self.models: with self.subTest(model=model): - print(f"##=== Testing accuracy: {model} ===##") - process = popen_launch_server( model, self.base_url, @@ -57,21 +56,26 @@ class TestAscendTp2Bf16(CustomTestCase): ) try: - args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=1319, - max_new_tokens=512, - parallel=128, - host=f"http://{self.url.hostname}", - port=int(self.url.port), - ) + if _is_pr_pipeline: + run_npu_pr_smoke(self.base_url) + else: + print(f"##=== Testing accuracy: {model} ===##") - metrics = run_eval_few_shot_gsm8k(args) - self.assertGreaterEqual( - metrics["accuracy"], - TEST_MODEL_MATRIX[model]["accuracy"], - ) + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) finally: kill_process_tree(process.pid) diff --git a/test/registered/npu/basic_function/runtime_opts/test_npu_tp2_fia_bf16.py b/test/registered/npu/basic_function/runtime_opts/test_npu_tp2_fia_bf16.py index 54f3db7d7..821f09abe 100644 --- a/test/registered/npu/basic_function/runtime_opts/test_npu_tp2_fia_bf16.py +++ b/test/registered/npu/basic_function/runtime_opts/test_npu_tp2_fia_bf16.py @@ -4,6 +4,7 @@ from types import SimpleNamespace from urllib.parse import urlparse from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke from sglang.test.ci.ci_register import register_npu_ci from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k from sglang.test.test_utils import ( @@ -13,7 +14,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a2", nightly=False) +register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a3", nightly=False) register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True) TEST_MODEL_MATRIX = { @@ -48,8 +49,6 @@ class TestAscendTp2Bf16(CustomTestCase): os.environ["ASCEND_USE_FIA"] = "true" for model in self.models: with self.subTest(model=model): - print(f"##=== Testing accuracy: {model} ===##") - process = popen_launch_server( model, self.base_url, @@ -60,21 +59,26 @@ class TestAscendTp2Bf16(CustomTestCase): ) try: - args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=1319, - max_new_tokens=512, - parallel=128, - host=f"http://{self.url.hostname}", - port=int(self.url.port), - ) + if _is_pr_pipeline: + run_npu_pr_smoke(self.base_url) + else: + print(f"##=== Testing accuracy: {model} ===##") - metrics = run_eval_few_shot_gsm8k(args) - self.assertGreaterEqual( - metrics["accuracy"], - TEST_MODEL_MATRIX[model]["accuracy"], - ) + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) finally: kill_process_tree(process.pid) diff --git a/test/registered/npu/basic_function/runtime_opts/test_npu_tp4_bf16.py b/test/registered/npu/basic_function/runtime_opts/test_npu_tp4_bf16.py index e33b928c2..b94887783 100644 --- a/test/registered/npu/basic_function/runtime_opts/test_npu_tp4_bf16.py +++ b/test/registered/npu/basic_function/runtime_opts/test_npu_tp4_bf16.py @@ -3,6 +3,7 @@ from types import SimpleNamespace from urllib.parse import urlparse from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke from sglang.test.ci.ci_register import register_npu_ci from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k from sglang.test.test_utils import ( @@ -48,8 +49,6 @@ class TestAscendTp4Bf16(CustomTestCase): def test_a_gsm8k(self): for model in self.models: with self.subTest(model=model): - print(f"##=== Testing accuracy: {model} ===##") - process = popen_launch_server( model, self.base_url, @@ -60,21 +59,26 @@ class TestAscendTp4Bf16(CustomTestCase): ) try: - args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=1319, - max_new_tokens=512, - parallel=128, - host=f"http://{self.url.hostname}", - port=int(self.url.port), - ) + if _is_pr_pipeline: + run_npu_pr_smoke(self.base_url) + else: + print(f"##=== Testing accuracy: {model} ===##") - metrics = run_eval_few_shot_gsm8k(args) - self.assertGreaterEqual( - metrics["accuracy"], - TEST_MODEL_MATRIX[model]["accuracy"], - ) + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) finally: kill_process_tree(process.pid) diff --git a/test/registered/npu/basic_function/speculative_inference/test_npu_basic_sanity_eagle3.py b/test/registered/npu/basic_function/speculative_inference/test_npu_basic_sanity_eagle3.py index 88a793696..7254361fb 100644 --- a/test/registered/npu/basic_function/speculative_inference/test_npu_basic_sanity_eagle3.py +++ b/test/registered/npu/basic_function/speculative_inference/test_npu_basic_sanity_eagle3.py @@ -1,6 +1,7 @@ import unittest from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.npu_eval_accuracy_kit import NPUGSM8KMixin from sglang.test.ascend.test_ascend_utils import ( QWEN3_8B_EAGLE3_WEIGHTS_PATH, QWEN3_8B_WEIGHTS_PATH, @@ -9,7 +10,6 @@ from sglang.test.ci.ci_register import register_npu_ci from sglang.test.kits.basic_api_contract_kit import BasicAPIContractMixin from sglang.test.kits.basic_decode_correctness_kit import BasicDecodeCorrectnessMixin from sglang.test.kits.basic_scheduler_stress_kit import BasicSchedulerStressMixin -from sglang.test.kits.eval_accuracy_kit import GSM8KMixin from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -25,7 +25,7 @@ class TestBasicSanityEagle3( BasicAPIContractMixin, BasicDecodeCorrectnessMixin, BasicSchedulerStressMixin, - GSM8KMixin, + NPUGSM8KMixin, CustomTestCase, ): served_model_name = QWEN3_8B_WEIGHTS_PATH diff --git a/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_attention_mode.py b/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_attention_mode.py index 3159b33b1..c6cec152a 100644 --- a/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_attention_mode.py +++ b/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_attention_mode.py @@ -3,6 +3,7 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke from sglang.test.ascend.test_ascend_utils import ( QWEN3_32B_EAGLE3_WEIGHTS_PATH, QWEN3_32B_W8A8_MINDIE_WEIGHTS_PATH, @@ -103,12 +104,15 @@ class TestNpuSpeculativeAttentionMode(CustomTestCase): ) try: - metrics = self._run_gsm8k_eval() - self.assertGreaterEqual( - metrics["score"], - 0.83, - f"GSM8K score {metrics['score']} below threshold 0.83", - ) + if _is_pr_pipeline: + run_npu_pr_smoke(DEFAULT_URL_FOR_TEST) + else: + metrics = self._run_gsm8k_eval() + self.assertGreaterEqual( + metrics["score"], + 0.83, + f"GSM8K score {metrics['score']} below threshold 0.83", + ) finally: kill_process_tree(process.pid) @@ -164,12 +168,15 @@ class TestNpuSpeculativeAttentionMode(CustomTestCase): ) try: - metrics = self._run_gsm8k_eval() - self.assertGreaterEqual( - metrics["score"], - 0.83, - f"GSM8K score {metrics['score']} below threshold 0.83", - ) + if _is_pr_pipeline: + run_npu_pr_smoke(DEFAULT_URL_FOR_TEST) + else: + metrics = self._run_gsm8k_eval() + self.assertGreaterEqual( + metrics["score"], + 0.83, + f"GSM8K score {metrics['score']} below threshold 0.83", + ) finally: kill_process_tree(process.pid) diff --git a/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_draft_attention_backend.py b/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_draft_attention_backend.py index e1d14eee0..c61021bb9 100644 --- a/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_draft_attention_backend.py +++ b/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_draft_attention_backend.py @@ -4,6 +4,7 @@ from types import SimpleNamespace from urllib.parse import urlparse from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke from sglang.test.ascend.test_ascend_utils import ( DEEPSEEK_R1_0528_W4A8_PER_CHANNEL_WEIGHTS_PATH, ) @@ -87,6 +88,9 @@ class TestAscendSpeculativeDraftAttentionAndMoeRunner(CustomTestCase): kill_process_tree(cls.process.pid) def test_a_gsm8k(self): + if _is_pr_pipeline: + run_npu_pr_smoke(self.base_url) + return args = SimpleNamespace( base_url=self.base_url, eval_name="gsm8k", diff --git a/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_moe_a2a_backend.py b/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_moe_a2a_backend.py index e0b7d2f84..e02518785 100644 --- a/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_moe_a2a_backend.py +++ b/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_moe_a2a_backend.py @@ -4,6 +4,7 @@ from types import SimpleNamespace from urllib.parse import urlparse from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke from sglang.test.ascend.test_ascend_utils import ( DEEPSEEK_R1_0528_W8A8_WEIGHTS_PATH, ) @@ -64,7 +65,6 @@ class TestAscendDistTimeout(CustomTestCase): ] def test_a_gsm8k(self): - print(f"##=== Testing accuracy: {self.model} ===##") process = popen_launch_server( self.model, self.base_url, @@ -74,23 +74,28 @@ class TestAscendDistTimeout(CustomTestCase): ) try: - args = SimpleNamespace( - base_url=self.base_url, - eval_name="gsm8k", - api="completion", - num_examples=1319, - num_threads=128, - max_tokens=512, - num_shots=5, - temperature=0.0, - ) + if _is_pr_pipeline: + run_npu_pr_smoke(self.base_url) + else: + print(f"##=== Testing accuracy: {self.model} ===##") - metrics = run_eval(args) - self.assertGreaterEqual( - metrics["score"], - self.accuracy, - f"GSM8K score {metrics['score']} below threshold {self.accuracy}", - ) + args = SimpleNamespace( + base_url=self.base_url, + eval_name="gsm8k", + api="completion", + num_examples=1319, + num_threads=128, + max_tokens=512, + num_shots=5, + temperature=0.0, + ) + + metrics = run_eval(args) + self.assertGreaterEqual( + metrics["score"], + self.accuracy, + f"GSM8K score {metrics['score']} below threshold {self.accuracy}", + ) finally: kill_process_tree(process.pid) diff --git a/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_token_map.py b/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_token_map.py index d52a5d20a..8d0400b28 100644 --- a/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_token_map.py +++ b/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_token_map.py @@ -3,6 +3,7 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke from sglang.test.ascend.test_ascend_utils import ( FR_SPEC_TOKEN_MAP_PATH, LLAMA_3_8B_EAGLE_WEIGHTS_PATH, @@ -77,24 +78,26 @@ class TestNpuSpeculativeTokenMap(CustomTestCase): env=env, ) try: - eval_args = SimpleNamespace( - base_url=DEFAULT_URL_FOR_TEST, - eval_name="gsm8k", - api="completion", - num_examples=1319, - num_threads=128, - max_tokens=512, - num_shots=5, - temperature=0.0, - ) - metrics = run_eval(eval_args) - self.assertGreaterEqual(metrics["score"], 0.83) + if _is_pr_pipeline: + run_npu_pr_smoke(DEFAULT_URL_FOR_TEST) + else: + eval_args = SimpleNamespace( + base_url=DEFAULT_URL_FOR_TEST, + eval_name="gsm8k", + api="completion", + num_examples=1319, + num_threads=128, + max_tokens=512, + num_shots=5, + temperature=0.0, + ) + metrics = run_eval(eval_args) + self.assertGreaterEqual(metrics["score"], 0.83) finally: kill_process_tree(process.pid) def test_eagle_with_valid_token_map_gsm8k(self): """EAGLE (EAGLE-2) with valid token map; GSM8K accuracy should meet threshold.""" - args = [ "--trust-remote-code", "--attention-backend", @@ -135,20 +138,23 @@ class TestNpuSpeculativeTokenMap(CustomTestCase): env=env, ) try: - eval_args = SimpleNamespace( - base_url=DEFAULT_URL_FOR_TEST, - eval_name="gsm8k", - api="completion", - num_examples=1319, - num_threads=128, - max_tokens=512, - num_shots=5, - temperature=0.0, - ) - metrics = run_eval(eval_args) - self.assertGreaterEqual( - metrics["score"], 0.75 - ) # adjust threshold as needed + if _is_pr_pipeline: + run_npu_pr_smoke(DEFAULT_URL_FOR_TEST) + else: + eval_args = SimpleNamespace( + base_url=DEFAULT_URL_FOR_TEST, + eval_name="gsm8k", + api="completion", + num_examples=1319, + num_threads=128, + max_tokens=512, + num_shots=5, + temperature=0.0, + ) + metrics = run_eval(eval_args) + self.assertGreaterEqual( + metrics["score"], 0.75 + ) # adjust threshold as needed finally: kill_process_tree(process.pid) diff --git a/test/registered/npu/llm_models/test_npu_llama_2_7b_communications_compression.py b/test/registered/npu/llm_models/test_npu_llama_2_7b_communications_compression.py index ebc4472fb..e7b7fe361 100644 --- a/test/registered/npu/llm_models/test_npu_llama_2_7b_communications_compression.py +++ b/test/registered/npu/llm_models/test_npu_llama_2_7b_communications_compression.py @@ -5,7 +5,7 @@ from sglang.test.ascend.test_ascend_utils import LLAMA_2_7B_WEIGHTS_PATH from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase -register_npu_ci(est_time=400, suite="nightly-2-npu-a3") +register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True) class TestLlama(GSM8KAscendMixin, CustomTestCase): diff --git a/test/registered/npu/llm_models/test_npu_qwen3_8b_communications_quantization.py b/test/registered/npu/llm_models/test_npu_qwen3_8b_communications_quantization.py index 5f8df9550..6f8d0f5b3 100644 --- a/test/registered/npu/llm_models/test_npu_qwen3_8b_communications_quantization.py +++ b/test/registered/npu/llm_models/test_npu_qwen3_8b_communications_quantization.py @@ -5,7 +5,7 @@ from sglang.test.ascend.test_ascend_utils import QWEN3_8B_WEIGHTS_PATH from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase -register_npu_ci(est_time=400, suite="nightly-2-npu-a3") +register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True) class TestQwen38BCommQuantization(GSM8KAscendMixin, CustomTestCase): diff --git a/test/registered/unit/layers/attention/test_vision_backend_selection.py b/test/registered/unit/layers/attention/test_vision_backend_selection.py index 50f241a55..b7e6296fa 100644 --- a/test/registered/unit/layers/attention/test_vision_backend_selection.py +++ b/test/registered/unit/layers/attention/test_vision_backend_selection.py @@ -11,7 +11,8 @@ from sglang.srt.layers.attention import vision from sglang.test.ci.ci_register import register_cpu_ci, register_npu_ci register_cpu_ci(est_time=2, suite="base-a-test-cpu") -register_npu_ci(est_time=2, suite="stage-b-test-1-npu-a2") +register_npu_ci(est_time=2, suite="stage-b-test-1-npu-a3") +register_npu_ci(est_time=2, suite="nightly-1-npu-a3", nightly=True) @pytest.fixture diff --git a/test/run_suite.py b/test/run_suite.py index 7d0a73c9b..de4143366 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -100,9 +100,10 @@ PER_COMMIT_SUITES = { HWBackend.NPU: [ "base-a-test-1-gpu-small", "stage-a-unit-test-npu", - "stage-b-test-1-npu-a2", - "stage-b-test-2-npu-a2", + "stage-b-test-1-npu-a3", + "stage-b-test-2-npu-a3", "stage-b-test-4-npu-a3", + "stage-b-test-8-npu-a3", "stage-b-test-16-npu-a3", ], HWBackend.XPU: [