[NPU][CI] Fail fast and speed up long-running qwen3.6 accuracy cases (#39813)
This commit is contained in:
@@ -480,7 +480,7 @@ jobs:
|
|||||||
|
|
||||||
nightly-poc-multi-node-mix-tests-cann910:
|
nightly-poc-multi-node-mix-tests-cann910:
|
||||||
name: multi-node-mix-poc-cann910
|
name: multi-node-mix-poc-cann910
|
||||||
if: ${{ !cancelled() }}
|
if: false
|
||||||
needs: [set-image-config, nightly-poc-multi-node-tests-cann910]
|
needs: [set-image-config, nightly-poc-multi-node-tests-cann910]
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
|
|||||||
@@ -314,6 +314,9 @@ class TestNpuAccuracyTestCaseBase(CustomTestCase):
|
|||||||
server_timeout = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
|
server_timeout = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
|
||||||
envs = None
|
envs = None
|
||||||
accuracy = 0.1
|
accuracy = 0.1
|
||||||
|
# Per-case evalscope accuracy retry count. None uses the dataset default
|
||||||
|
# (get_max_retries); set to 1 to fail fast on first measurement.
|
||||||
|
max_retries = None
|
||||||
test_type = "accuracy"
|
test_type = "accuracy"
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@@ -478,7 +481,11 @@ class TestNpuAccuracyTestCaseBase(CustomTestCase):
|
|||||||
port = parsed_url.port
|
port = parsed_url.port
|
||||||
if self.benchmark_tool == EVALSCOPE:
|
if self.benchmark_tool == EVALSCOPE:
|
||||||
model_name = os.path.basename(self.model)
|
model_name = os.path.basename(self.model)
|
||||||
max_retries = get_max_retries(self.datasets)
|
max_retries = (
|
||||||
|
self.max_retries
|
||||||
|
if self.max_retries is not None
|
||||||
|
else get_max_retries(self.datasets)
|
||||||
|
)
|
||||||
best_metrics = None
|
best_metrics = None
|
||||||
for attempt in range(max_retries):
|
for attempt in range(max_retries):
|
||||||
metrics = run_evalscope(
|
metrics = run_evalscope(
|
||||||
|
|||||||
@@ -91,6 +91,7 @@ class TestNPUQwen3_6_27B_1P_In64k_Out1k_Prefix90_gpqa(TestNpuAccuracyTestCaseBas
|
|||||||
envs = QWEN3_6_27B_64K_PREFIX_ENVS
|
envs = QWEN3_6_27B_64K_PREFIX_ENVS
|
||||||
other_args = QWEN3_6_27B_64K_PREFIX_OTHER_ARGS
|
other_args = QWEN3_6_27B_64K_PREFIX_OTHER_ARGS
|
||||||
accuracy = 0.878
|
accuracy = 0.878
|
||||||
|
max_retries = 1
|
||||||
datasets = ["gpqa_diamond"]
|
datasets = ["gpqa_diamond"]
|
||||||
few_shot_num = 0
|
few_shot_num = 0
|
||||||
eval_batch_size = 64
|
eval_batch_size = 64
|
||||||
|
|||||||
+2
-1
@@ -86,9 +86,10 @@ class TestNPUQwen3_6_27B_1P_In3k5_Out1k5_gpqa(TestNpuAccuracyTestCaseBase):
|
|||||||
envs = QWEN3_6_27B_3K5_1K5_ENVS
|
envs = QWEN3_6_27B_3K5_1K5_ENVS
|
||||||
other_args = QWEN3_6_27B_3K5_1K5_OTHER_ARGS
|
other_args = QWEN3_6_27B_3K5_1K5_OTHER_ARGS
|
||||||
accuracy = 0.878
|
accuracy = 0.878
|
||||||
|
max_retries = 1
|
||||||
datasets = ["gpqa_diamond"]
|
datasets = ["gpqa_diamond"]
|
||||||
few_shot_num = 0
|
few_shot_num = 0
|
||||||
eval_batch_size = 8
|
eval_batch_size = 16
|
||||||
generation_config = {
|
generation_config = {
|
||||||
"max_tokens": 81920,
|
"max_tokens": 81920,
|
||||||
"temperature": 1.0,
|
"temperature": 1.0,
|
||||||
|
|||||||
Reference in New Issue
Block a user