fix(precision): do not promote failed runs to the comparison baseline (#28190)

Co-authored-by: Alison Shao <54658187+alisonshao@users.noreply.github.com>

python/sglang/test/precision_baseline_store.py

@@ -102,6 +102,10 @@ def _select_latest_run(
     # when a signature is given, mismatched (incl. legacy unsigned) rows are
     # skipped — fetch then returns None and the caller establishes a fresh one
     # instead of erroring on incompatible tensors.
+    # A failed run must not become the next comparison baseline, or a persistent
+    # regression is masked: today's regressed tensors (uploaded as "failed")
+    # would be selected as the reference next run. Prefer non-failed rows and
+    # fall back to a failed one only when no usable baseline exists.
     candidates: list[tuple[tuple[int, int], dict[str, Any]]] = []
     for idx, row in enumerate(rows):
         if row.get("model") != model:
@@ -117,7 +121,9 @@ def _select_latest_run(
     if not candidates:
         return None
     candidates.sort(key=lambda kv: kv[0])
-    return candidates[-1][1]["run_path"]
+    usable = [c for c in candidates if c[1].get("pass_label") != "failed"]
+    chosen = usable or candidates
+    return chosen[-1][1]["run_path"]
 
 
 def fetch_latest_baseline(

test/registered/unit/test_precision_baseline_store.py

@@ -157,6 +157,72 @@ class TestSelectLatestRun(CustomTestCase):
             hfs._select_latest_run(rows, model="org/m", capture_signature="zzz")
         )
 
+    def test_prefers_older_passed_over_newer_failed(self):
+        # A failed run must not shadow an older good baseline, or a persistent
+        # regression is masked after one night.
+        rows = [
+            {
+                "model": "org/m",
+                "run_path": "good",
+                "pass_label": "passed",
+                "push_index": 1,
+            },
+            {
+                "model": "org/m",
+                "run_path": "bad",
+                "pass_label": "failed",
+                "push_index": 2,
+            },
+        ]
+        self.assertEqual(hfs._select_latest_run(rows, model="org/m"), "good")
+
+    def test_prefers_baseline_established_over_newer_failed(self):
+        rows = [
+            {
+                "model": "org/m",
+                "run_path": "seed",
+                "pass_label": "baseline_established",
+                "push_index": 1,
+            },
+            {
+                "model": "org/m",
+                "run_path": "bad",
+                "pass_label": "failed",
+                "push_index": 2,
+            },
+        ]
+        self.assertEqual(hfs._select_latest_run(rows, model="org/m"), "seed")
+
+    def test_falls_back_to_failed_when_only_failed(self):
+        rows = [
+            {
+                "model": "org/m",
+                "run_path": "bad1",
+                "pass_label": "failed",
+                "push_index": 1,
+            },
+            {
+                "model": "org/m",
+                "run_path": "bad2",
+                "pass_label": "failed",
+                "push_index": 2,
+            },
+        ]
+        self.assertEqual(hfs._select_latest_run(rows, model="org/m"), "bad2")
+
+    def test_missing_pass_label_treated_as_usable(self):
+        # Legacy rows without pass_label stay usable as baselines.
+        rows = [
+            {"model": "org/m", "run_path": "legacy", "push_index": 1},
+            {
+                "model": "org/m",
+                "run_path": "bad",
+                "pass_label": "failed",
+                "push_index": 2,
+            },
+        ]
+        self.assertEqual(hfs._select_latest_run(rows, model="org/m"), "legacy")
+
 
 class TestReadManifest(CustomTestCase):
     @patch("sglang.test.precision_baseline_store.hf_hub_download")