diff --git a/python/sglang/test/ascend/e2e/test_npu_accuracy_utils.py b/python/sglang/test/ascend/e2e/test_npu_accuracy_utils.py index 8a8d632a8..98bf2d76f 100644 --- a/python/sglang/test/ascend/e2e/test_npu_accuracy_utils.py +++ b/python/sglang/test/ascend/e2e/test_npu_accuracy_utils.py @@ -227,17 +227,24 @@ def run_evalscope( if "accuracy" not in metrics: accuracy_patterns = [ + # Add adaptation for evalscope 1.11+ table format + r"Accuracy\s*[↑↓]?\s*│\s*[^│]*│\s*\d+\s*│\s*([\d.]+)%?\s*│", r"mean_acc\s*.*?│\s*\d+\s*│\s*([\d.]+)\s*│", r"│\s+([\d.]+)\s+│\s+\S+\s+│\s*$", r"accuracy\s*[:=]?\s*([\d.]+)", + # Keep compatibility with legacy evalscope 1.10 table format r"Accuracy\s*[:=]?\s*([\d.]+)", r"score\s*[:=]?\s*([\d.]+)", ] for pattern in accuracy_patterns: - matches = re.findall(pattern, full_output) + matches = list(re.finditer(pattern, full_output)) if matches: - final_accuracy = float(matches[-1]) + final_accuracy = float(matches[-1].group(1)) + # evalscope 1.11+ reports accuracy as a percentage (e.g. 66.67%); + # normalize it to a 0-1 fraction to compare against the baseline. + if "%" in matches[-1].group(0): + final_accuracy /= 100.0 metrics["accuracy"] = final_accuracy logger.info(f"The Final Accuracy from output: {final_accuracy}") break @@ -307,7 +314,6 @@ class TestNpuAccuracyTestCaseBase(CustomTestCase): other_args = None server_timeout = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH envs = None - n_runs = 3 accuracy = 0.1 test_type = "accuracy" @@ -504,67 +510,6 @@ class TestNpuAccuracyTestCaseBase(CustomTestCase): ) assert_metrics(self, best_metrics) - def run_accuracy_multiple(self, n_runs=None): - if n_runs is None: - n_runs = self.n_runs - - parsed_url = urlparse(self.base_url) - host = parsed_url.hostname - port = parsed_url.port - - if self.benchmark_tool != EVALSCOPE: - raise Exception( - "run_accuracy_multiple only supports evalscope benchmark tool" - ) - - model_name = os.path.basename(self.model) - all_metrics = [] - - for i in range(n_runs): - logger.info(f"=== Accuracy run {i + 1}/{n_runs} ===") - metrics = run_evalscope( - host=host, - port=port, - model=model_name, - datasets=self.datasets, - dataset_args=self.dataset_args, - eval_batch_size=self.eval_batch_size, - limit=self.limit, - generation_config=self.generation_config, - dataset_dir=self.dataset_dir, - stream=self.stream, - timeout=self.timeout, - eval_type=self.eval_type, - ) - all_metrics.append(metrics) - if metrics and "accuracy" in metrics: - logger.info(f"Run {i + 1} accuracy: {metrics['accuracy']}") - else: - logger.warning(f"Run {i + 1} failed to get accuracy metric") - - valid_metrics = [m for m in all_metrics if m and "accuracy" in m] - if not valid_metrics: - raise Exception("No valid accuracy metrics obtained from any run") - - avg_accuracy = sum(float(m["accuracy"]) for m in valid_metrics) / len( - valid_metrics - ) - - logger.info("=" * 60) - logger.info("Multiple Run Accuracy Results:") - for i, m in enumerate(valid_metrics): - logger.info(f" Run {i + 1}: {m['accuracy']}") - logger.info(f" Average: {avg_accuracy}") - logger.info("=" * 60) - - avg_metrics = {"accuracy": avg_accuracy} - dump_metric( - "accuracy_avg", - avg_accuracy, - labels={"test_case": self.__class__.__name__, "type": "accuracy"}, - ) - assert_metrics(self, avg_metrics) - class TestNpuAccuracyMultiNodePdMixTestCaseBase(CustomTestCase): model_config = None diff --git a/test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py b/test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py index ac127ba43..2000d1c11 100644 --- a/test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py +++ b/test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py @@ -159,7 +159,7 @@ GLM_5_1_PD_SEP_MODEL_CONFIG = { "decode_args": GLM_5_1_PD_SEP_DECODE_ARGS, "prefill_envs": GLM_5_1_PD_SEP_PREFILL_ENVS, "decode_envs": GLM_5_1_PD_SEP_DECODE_ENVS, - "router_args": ["--policy", "round_robin"], + "router_args": ["--policy", "round_robin", "--request-timeout-secs", 7200], "router_envs": {}, } diff --git a/test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py b/test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py index b69a2a499..3cfbba230 100644 --- a/test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py +++ b/test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py @@ -112,7 +112,7 @@ class TestQwen3Next80BA3B_aime25(TestNpuAccuracyTestCaseBase): max_concurrency = 16 def test_aime25(self): - self.run_accuracy_multiple(n_runs=3) + self.run_accuracy() if __name__ == "__main__": diff --git a/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms.py b/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms.py index b3fad7885..290e52aa1 100644 --- a/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms.py +++ b/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms.py @@ -44,7 +44,7 @@ OTHER_ARGS = [ "--nnodes", 2, "--mem-fraction-static", - 0.662, + 0.68, "--max-running-requests", 32, "--chunked-prefill-size",