Fix Mistral GSM8K chat eval (#27757)

2026-07-09 21:08:48 -07:00
parent a38cfc6768
commit b76dd0be69
10 changed files with 429 additions and 131 deletions
@@ -26,23 +26,23 @@ NIGHTLY_EVAL_SERVER_TIMEOUT = 1800
 register_cuda_ci(est_time=3600, suite="nightly-eval-text-2-gpu", nightly=True)

 MODEL_SCORE_THRESHOLDS = {
-    # Thresholds set at 5% below reported GSM8K (5-shot/CoT) scores
-    "meta-llama/Llama-3.1-8B-Instruct": 0.80,  # 84.5% - 5%
-    "mistralai/Mistral-7B-Instruct-v0.3": 0.47,  # 52.1% - 5%
-    "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": 0.81,  # 86.4% - 5%
-    "google/gemma-2-27b-it": 0.81,  # 85.5% measured - 5%
-    "meta-llama/Llama-3.1-70B-Instruct": 0.89,  # 94.1% - 5%
-    "mistralai/Mixtral-8x7B-Instruct-v0.1": 0.69,  # 74.4% - 5%
-    "Qwen/Qwen2-57B-A14B-Instruct": 0.76,  # 80.7% - 5% (official A14B score; 88.2% was the 72B)
-    "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8": 0.80,  # 84.5% - 5%
-    "neuralmagic/Mistral-7B-Instruct-v0.3-FP8": 0.47,  # 52.1% - 5%
-    "neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8": 0.81,  # 86.4% - 5%
-    "zai-org/GLM-4.5-Air-FP8": 0.80,  # ~85%  - 5%
-    "neuralmagic/gemma-2-2b-it-FP8": 0.53,  # 58.4% measured - 5%
-    "neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8": 0.89,  # 94.1% - 5%
-    "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8": 0.69,  # 74.4% - 5%
-    "neuralmagic/Qwen2-72B-Instruct-FP8": 0.86,  # 91.1% - 5%
-    "neuralmagic/Qwen2-57B-A14B-Instruct-FP8": 0.76,  # 80.7% - 5% (official A14B score)
+    # sgl-eval (zero-shot chat, \boxed{}, math_verify grading). Thresholds are
+    # measured_score - 0.05, baselined on H100 2-GPU over the full 1319 split.
+    "meta-llama/Llama-3.1-8B-Instruct": 0.77,  # 81.05% measured - 5%
+    "Qwen/Qwen3-8B": 0.76,  # 81.43% measured - 5%
+    "Qwen/Qwen3-4B": 0.77,  # 82.41% measured - 5%
+    "meta-llama/Llama-3.1-70B-Instruct": 0.90,  # 94.77% measured - 5%
+    "mistralai/Mixtral-8x7B-Instruct-v0.1": 0.39,  # 43.52% measured - 5%
+    "Qwen/Qwen2-57B-A14B-Instruct": 0.46,  # 50.87% measured - 5%
+    "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8": 0.77,  # 82.34% measured - 5%
+    "neuralmagic/Mistral-7B-Instruct-v0.3-FP8": 0.23,  # 27.82% measured - 5%
+    "neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8": 0.80,  # 84.91% measured - 5%
+    "zai-org/GLM-4.5-Air-FP8": 0.73,  # 77.48% measured - 5%
+    "neuralmagic/gemma-2-2b-it-FP8": 0.02,  # 6.52% measured - 5%
+    "neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8": 0.89,  # 94.01% measured - 5%
+    "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8": 0.35,  # 40.33% measured - 5%
+    "neuralmagic/Qwen2-72B-Instruct-FP8": 0.83,  # 87.64% measured - 5%
+    "neuralmagic/Qwen2-57B-A14B-Instruct-FP8": 0.40,  # 44.66% measured - 5%
 }


@@ -91,6 +91,7 @@ class TestNightlyGsm8KEval(unittest.TestCase):
                        base_url=self.base_url,
                        model=model_setup.model_path,
                        eval_name="gsm8k",
+                        api="sgl_eval",
                        num_examples=None,
                        num_threads=1024,
                    )
@@ -5,8 +5,10 @@ import unittest
 from typing import List, Tuple

 from sglang.test.ci.ci_register import register_cpu_ci
-from sglang.test.simple_eval_gsm8k import get_one_example
-from sglang.test.simple_eval_mixed_prefix_gsm8k import MixedPrefixGSM8KEval
+from sglang.test.simple_eval_mixed_prefix_gsm8k import (
+    MixedPrefixGSM8KEval,
+    get_one_example,
+)
 from sglang.test.test_utils import CustomTestCase

 register_cpu_ci(est_time=5, suite="base-c-test-cpu")
@@ -0,0 +1,172 @@
+import json
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import patch
+
+from sglang.test.ci.ci_register import register_cpu_ci
+from sglang.test.run_eval import _run_sgl_eval
+from sglang.test.test_utils import CustomTestCase
+
+register_cpu_ci(est_time=5, suite="base-b-test-cpu")
+
+
+def _write_fake_metrics(out_parent: Path, eval_name: str, payload: dict) -> None:
+    run_dir = out_parent / f"sgl_eval_{eval_name}_20260101-000000"
+    run_dir.mkdir(parents=True, exist_ok=True)
+    (run_dir / "metrics.json").write_text(json.dumps(payload))
+
+
+class TestRunSglEval(CustomTestCase):
+    """sgl-eval is a black box, so these mock subprocess.run and assert the shim
+    builds the CLI and parses metrics.json's aggregate.score (not top-level)."""
+
+    def _args(self, out_dir: str, **overrides):
+        defaults = dict(
+            base_url="http://127.0.0.1:30000",
+            model="test-model",
+            num_examples=7,
+            num_threads=8,
+            temperature=0.0,
+            sgl_eval_out_dir=out_dir,
+        )
+        defaults.update(overrides)
+        return SimpleNamespace(**defaults)
+
+    def _fake_run_factory(self, out_dir: Path, eval_name: str, payload: dict):
+        def fake_run(cmd, **kwargs):
+            _write_fake_metrics(out_dir, eval_name, payload)
+            return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="")
+
+        return fake_run
+
+    def test_parses_aggregate_score_and_maps_latency(self):
+        payload = {
+            "name": "gsm8k",
+            "model": "test-model",
+            "num_examples": 7,
+            "n_repeats": 1,
+            "latency_seconds": 12.5,
+            "output_throughput_tps": 34.0,
+            "aggregate": {"score": 0.75, "no_answer": 0.1},
+        }
+        with tempfile.TemporaryDirectory() as td:
+            out_dir = Path(td)
+            args = self._args(td)
+            with patch(
+                "sglang.test.run_eval.subprocess.run",
+                side_effect=self._fake_run_factory(out_dir, "gsm8k", payload),
+            ):
+                metrics = _run_sgl_eval("gsm8k", args)
+
+        self.assertAlmostEqual(metrics["score"], 0.75)
+        self.assertAlmostEqual(metrics["latency"], 12.5)
+        self.assertAlmostEqual(metrics["output_throughput"], 34.0)
+        self.assertEqual(metrics["no_answer"], 0.1)
+        self.assertTrue(metrics["sgl_eval_metrics_path"].endswith("metrics.json"))
+
+    def test_builds_cli_with_required_flags(self):
+        captured = {}
+
+        def fake_run(cmd, **kwargs):
+            captured["cmd"] = cmd
+            out_dir = Path(captured["cmd"][captured["cmd"].index("--out-dir") + 1])
+            _write_fake_metrics(
+                out_dir,
+                "gsm8k",
+                {
+                    "name": "gsm8k",
+                    "model": "test-model",
+                    "latency_seconds": 1.0,
+                    "output_throughput_tps": 1.0,
+                    "aggregate": {"score": 0.5},
+                },
+            )
+            return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="")
+
+        with tempfile.TemporaryDirectory() as td:
+            args = self._args(td)
+            with patch("sglang.test.run_eval.subprocess.run", side_effect=fake_run):
+                _run_sgl_eval("gsm8k", args)
+
+        cmd = captured["cmd"]
+        self.assertEqual(cmd[0:3], ["sgl-eval", "run", "gsm8k"])
+        self.assertIn("--base-url", cmd)
+        self.assertIn("http://127.0.0.1:30000/v1", cmd)
+        self.assertIn("--model", cmd)
+        self.assertIn("test-model", cmd)
+        self.assertIn("--num-threads", cmd)
+        self.assertIn("8", cmd)
+        self.assertIn("--temperature", cmd)
+        self.assertIn("0.0", cmd)
+        self.assertIn("--num-examples", cmd)
+        self.assertIn("7", cmd)
+
+    def test_omits_num_examples_when_none(self):
+        captured = {}
+
+        def fake_run(cmd, **kwargs):
+            captured["cmd"] = cmd
+            out_dir = Path(cmd[cmd.index("--out-dir") + 1])
+            _write_fake_metrics(
+                out_dir,
+                "gsm8k",
+                {
+                    "model": "test-model",
+                    "latency_seconds": 1.0,
+                    "output_throughput_tps": 1.0,
+                    "aggregate": {"score": 0.5},
+                },
+            )
+            return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="")
+
+        with tempfile.TemporaryDirectory() as td:
+            args = self._args(td, num_examples=None)
+            with patch("sglang.test.run_eval.subprocess.run", side_effect=fake_run):
+                _run_sgl_eval("gsm8k", args)
+
+        self.assertNotIn("--num-examples", captured["cmd"])
+
+    def test_raises_on_nonzero_exit(self):
+        def fake_run(cmd, **kwargs):
+            return subprocess.CompletedProcess(cmd, 2, stdout="", stderr="boom")
+
+        with tempfile.TemporaryDirectory() as td:
+            args = self._args(td)
+            with patch("sglang.test.run_eval.subprocess.run", side_effect=fake_run):
+                with self.assertRaises(RuntimeError) as cm:
+                    _run_sgl_eval("gsm8k", args)
+            self.assertIn("exit code 2", str(cm.exception))
+
+    def test_raises_when_metrics_json_missing(self):
+        def fake_run(cmd, **kwargs):
+            return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="")
+
+        with tempfile.TemporaryDirectory() as td:
+            args = self._args(td)
+            with patch("sglang.test.run_eval.subprocess.run", side_effect=fake_run):
+                with self.assertRaises(FileNotFoundError):
+                    _run_sgl_eval("gsm8k", args)
+
+    def test_raises_when_aggregate_score_missing(self):
+        payload = {
+            "name": "gsm8k",
+            "latency_seconds": 1.0,
+            "output_throughput_tps": 1.0,
+            "aggregate": {"no_answer": 0.5},  # no score key
+        }
+        with tempfile.TemporaryDirectory() as td:
+            out_dir = Path(td)
+            args = self._args(td)
+            with patch(
+                "sglang.test.run_eval.subprocess.run",
+                side_effect=self._fake_run_factory(out_dir, "gsm8k", payload),
+            ):
+                with self.assertRaises(KeyError):
+                    _run_sgl_eval("gsm8k", args)
+
+
+if __name__ == "__main__":
+    unittest.main()