From 18359aadc8a456397fb102455edaf64b5ec1192e Mon Sep 17 00:00:00 2001 From: Kangyan-Zhou Date: Wed, 22 Apr 2026 22:30:54 -0700 Subject: [PATCH] [CI] Lower GSM8K baselines for B200 nightly after eval unification (#22136) Co-authored-by: Claude Opus 4.6 (1M context) --- python/sglang/test/accuracy_test_runner.py | 6 ++++++ .../backends/test_flashinfer_trtllm_gen_moe_backend.py | 2 +- test/registered/perf/test_dpsk_v3_fp4_4gpu_perf.py | 5 ++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/python/sglang/test/accuracy_test_runner.py b/python/sglang/test/accuracy_test_runner.py index ac9da2382..7c91095a8 100644 --- a/python/sglang/test/accuracy_test_runner.py +++ b/python/sglang/test/accuracy_test_runner.py @@ -30,6 +30,7 @@ class AccuracyTestParams: top_p: Optional[float] = None top_k: Optional[int] = None repeat: Optional[int] = None + api: Optional[str] = None # "chat" or "completion"; defaults to "chat" in run_eval @dataclass @@ -87,6 +88,7 @@ def _run_simple_eval( top_p: Optional[float] = None, top_k: Optional[int] = None, repeat: Optional[int] = None, + api: Optional[str] = None, ) -> Tuple[bool, Optional[str], Optional[dict]]: """Run evaluation using simple_eval backend (run_eval.py). @@ -111,6 +113,9 @@ def _run_simple_eval( num_threads=num_threads or 1024, ) + if api is not None: + args.api = api + if max_tokens is not None: args.max_tokens = max_tokens @@ -489,6 +494,7 @@ def run_accuracy_test( top_p=params.top_p, top_k=params.top_k, repeat=params.repeat, + api=params.api, ) if not success: diff --git a/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py b/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py index 9c7260b19..0515f5cdd 100644 --- a/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py +++ b/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py @@ -59,7 +59,7 @@ class FlashinferTrtllmGenMoeBackendFP8Base: ) metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["score"], 0.93) + self.assertGreater(metrics["score"], 0.89) class FlashinferTrtllmGenMoeBackendBF16Base: diff --git a/test/registered/perf/test_dpsk_v3_fp4_4gpu_perf.py b/test/registered/perf/test_dpsk_v3_fp4_4gpu_perf.py index e2e9ddd5e..36e5ac3ca 100644 --- a/test/registered/perf/test_dpsk_v3_fp4_4gpu_perf.py +++ b/test/registered/perf/test_dpsk_v3_fp4_4gpu_perf.py @@ -63,7 +63,10 @@ class TestDeepseekR1FP4Unified(unittest.TestCase): models=variants, test_name="DeepSeek-V3-0324-FP4 Unified", accuracy_params=AccuracyTestParams( - dataset="gsm8k", baseline_accuracy=0.935 + dataset="gsm8k", + baseline_accuracy=0.935, + num_examples=200, + api="completion", ), performance_params=PerformanceTestParams( profile_dir="performance_profiles_deepseek_v3_fp4",