From 00a9a81b67774e8374172646c092e14999472703 Mon Sep 17 00:00:00 2001 From: pllimax Date: Wed, 16 Sep 2026 20:20:37 +0800 Subject: [PATCH] [NPU][CI] Fail fast and speed up long-running qwen3.6 accuracy cases (#39813) --- .github/workflows/nightly-test-npu.yml | 2 +- python/sglang/test/ascend/e2e/test_npu_accuracy_utils.py | 9 ++++++++- .../accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py | 1 + ...est_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py | 3 ++- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nightly-test-npu.yml b/.github/workflows/nightly-test-npu.yml index 623b89ce8..72e01e46b 100644 --- a/.github/workflows/nightly-test-npu.yml +++ b/.github/workflows/nightly-test-npu.yml @@ -480,7 +480,7 @@ jobs: nightly-poc-multi-node-mix-tests-cann910: name: multi-node-mix-poc-cann910 - if: ${{ !cancelled() }} + if: false needs: [set-image-config, nightly-poc-multi-node-tests-cann910] strategy: fail-fast: false diff --git a/python/sglang/test/ascend/e2e/test_npu_accuracy_utils.py b/python/sglang/test/ascend/e2e/test_npu_accuracy_utils.py index 5e5375555..0178cb962 100644 --- a/python/sglang/test/ascend/e2e/test_npu_accuracy_utils.py +++ b/python/sglang/test/ascend/e2e/test_npu_accuracy_utils.py @@ -314,6 +314,9 @@ class TestNpuAccuracyTestCaseBase(CustomTestCase): server_timeout = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH envs = None accuracy = 0.1 + # Per-case evalscope accuracy retry count. None uses the dataset default + # (get_max_retries); set to 1 to fail fast on first measurement. + max_retries = None test_type = "accuracy" @classmethod @@ -478,7 +481,11 @@ class TestNpuAccuracyTestCaseBase(CustomTestCase): port = parsed_url.port if self.benchmark_tool == EVALSCOPE: model_name = os.path.basename(self.model) - max_retries = get_max_retries(self.datasets) + max_retries = ( + self.max_retries + if self.max_retries is not None + else get_max_retries(self.datasets) + ) best_metrics = None for attempt in range(max_retries): metrics = run_evalscope( diff --git a/test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py b/test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py index ae048613f..6cf18a387 100644 --- a/test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py +++ b/test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py @@ -91,6 +91,7 @@ class TestNPUQwen3_6_27B_1P_In64k_Out1k_Prefix90_gpqa(TestNpuAccuracyTestCaseBas envs = QWEN3_6_27B_64K_PREFIX_ENVS other_args = QWEN3_6_27B_64K_PREFIX_OTHER_ARGS accuracy = 0.878 + max_retries = 1 datasets = ["gpqa_diamond"] few_shot_num = 0 eval_batch_size = 64 diff --git a/test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py b/test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py index f1b2624b1..e93d8a53f 100644 --- a/test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py +++ b/test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py @@ -86,9 +86,10 @@ class TestNPUQwen3_6_27B_1P_In3k5_Out1k5_gpqa(TestNpuAccuracyTestCaseBase): envs = QWEN3_6_27B_3K5_1K5_ENVS other_args = QWEN3_6_27B_3K5_1K5_OTHER_ARGS accuracy = 0.878 + max_retries = 1 datasets = ["gpqa_diamond"] few_shot_num = 0 - eval_batch_size = 8 + eval_batch_size = 16 generation_config = { "max_tokens": 81920, "temperature": 1.0,