[XPU][CI] Fix empty nightly dashboard (#37800)

.github/workflows/nightly-test-intel.yml

@@ -59,6 +59,17 @@ jobs:
         env:
           GITHUB_WORKSPACE: ${{ github.workspace }}
 
+      - name: Overlay in-tree sglang test helpers onto image site-packages
+        run: |
+          # Bridges the up-to-24h window between merge and nightly image rebuild.
+          dest=$(docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
+                 "import sglang, os; print(os.path.dirname(sglang.__file__))")
+          docker exec ci_sglang_xpu cp \
+            /sglang-checkout/python/sglang/test/ci/ci_utils.py \
+            "$dest/test/ci/ci_utils.py"
+          docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
+            "import sglang.test.ci.ci_utils; print('overlay ok')"
+
       - name: HF login + install run_suite extras
         timeout-minutes: 10
         run: |
@@ -101,6 +112,7 @@ jobs:
             cd /sglang-checkout/test &&
             OLMOCR_BENCH_DIR=/sglang-checkout/olmOCR-bench/bench_data \
             GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \
+            SGLANG_IS_IN_CI=true \
             SGLANG_TEST_METRICS_FILE=/sglang-checkout/nightly-xpu-1-gpu-metrics.jsonl \
             python3 run_suite.py --hw xpu --suite nightly-xpu-1-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
           " || TEST_EXIT_CODE=$?
@@ -154,6 +166,17 @@ jobs:
         env:
           GITHUB_WORKSPACE: ${{ github.workspace }}
 
+      - name: Overlay in-tree sglang test helpers onto image site-packages
+        run: |
+          # Bridges the up-to-24h window between merge and nightly image rebuild.
+          dest=$(docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
+                 "import sglang, os; print(os.path.dirname(sglang.__file__))")
+          docker exec ci_sglang_xpu cp \
+            /sglang-checkout/python/sglang/test/ci/ci_utils.py \
+            "$dest/test/ci/ci_utils.py"
+          docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
+            "import sglang.test.ci.ci_utils; print('overlay ok')"
+
       - name: HF login + install run_suite extras
         timeout-minutes: 10
         run: |
@@ -185,6 +208,7 @@ jobs:
             source /opt/venv/bin/activate &&
             cd /sglang-checkout/test &&
             GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \
+            SGLANG_IS_IN_CI=true \
             SGLANG_TEST_METRICS_FILE=/sglang-checkout/nightly-xpu-2-gpu-metrics.jsonl \
             python3 run_suite.py --hw xpu --suite nightly-xpu-2-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
           " || TEST_EXIT_CODE=$?
@@ -238,6 +262,17 @@ jobs:
         env:
           GITHUB_WORKSPACE: ${{ github.workspace }}
 
+      - name: Overlay in-tree sglang test helpers onto image site-packages
+        run: |
+          # Bridges the up-to-24h window between merge and nightly image rebuild.
+          dest=$(docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
+                 "import sglang, os; print(os.path.dirname(sglang.__file__))")
+          docker exec ci_sglang_xpu cp \
+            /sglang-checkout/python/sglang/test/ci/ci_utils.py \
+            "$dest/test/ci/ci_utils.py"
+          docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
+            "import sglang.test.ci.ci_utils; print('overlay ok')"
+
       - name: HF login + install run_suite extras
         timeout-minutes: 10
         run: |
@@ -269,6 +304,7 @@ jobs:
             source /opt/venv/bin/activate &&
             cd /sglang-checkout/test &&
             GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \
+            SGLANG_IS_IN_CI=true \
             SGLANG_TEST_METRICS_FILE=/sglang-checkout/nightly-xpu-4-gpu-metrics.jsonl \
             python3 run_suite.py --hw xpu --suite nightly-xpu-4-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
           " || TEST_EXIT_CODE=$?
@@ -322,6 +358,17 @@ jobs:
         env:
           GITHUB_WORKSPACE: ${{ github.workspace }}
 
+      - name: Overlay in-tree sglang test helpers onto image site-packages
+        run: |
+          # Bridges the up-to-24h window between merge and nightly image rebuild.
+          dest=$(docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
+                 "import sglang, os; print(os.path.dirname(sglang.__file__))")
+          docker exec ci_sglang_xpu cp \
+            /sglang-checkout/python/sglang/test/ci/ci_utils.py \
+            "$dest/test/ci/ci_utils.py"
+          docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
+            "import sglang.test.ci.ci_utils; print('overlay ok')"
+
       - name: HF login + install run_suite extras
         timeout-minutes: 10
         run: |
@@ -348,16 +395,27 @@ jobs:
       - name: Nightly Test (8-GPU XPU)
         timeout-minutes: 480
         run: |
-          touch github_summary.md
+          touch github_summary.md nightly-xpu-8-gpu-metrics.jsonl
           docker exec ci_sglang_xpu bash -c "
             source /opt/venv/bin/activate &&
             cd /sglang-checkout/test &&
             GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \
+            SGLANG_IS_IN_CI=true \
+            SGLANG_TEST_METRICS_FILE=/sglang-checkout/nightly-xpu-8-gpu-metrics.jsonl \
             python3 run_suite.py --hw xpu --suite nightly-xpu-8-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
           " || TEST_EXIT_CODE=$?
           echo "$(<github_summary.md)" >> $GITHUB_STEP_SUMMARY || true
           exit ${TEST_EXIT_CODE:-0}
 
+      - name: Upload per-model metrics jsonl
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: nightly-xpu-8-gpu-metrics
+          path: nightly-xpu-8-gpu-metrics.jsonl
+          if-no-files-found: warn
+          retention-days: 30
+
       - name: Cleanup container
         if: always()
         run: |

.github/workflows/xpu-ci-job-monitor.yml

@@ -288,10 +288,10 @@ jobs:
           with open(snap_path) as f:
               snapshot = json.load(f)
 
-          # Only look at nightly-test-intel jobs
+          # Snapshot writer emits `workflow` (filename), not `workflow_path`.
           nightly = [
               j for j in snapshot.get("jobs", [])
-              if j.get("workflow_path", "").endswith("nightly-test-intel.yml")
+              if (j.get("workflow") or "").endswith("nightly-test-intel.yml")
           ]
 
           # status_summary style counts, per job name

python/sglang/test/ci/ci_utils.py

@@ -417,8 +417,28 @@ def run_unittest_files(
 
         if not file_passed:
             success = False
-            if not continue_on_error:
+
-                break
+        # Flush per-file so a SIGKILL mid-suite still surfaces completed files.
+        metrics_path = os.environ.get("SGLANG_TEST_METRICS_FILE")
+        if metrics_path and filename in file_elapsed:
+            try:
+                incremental_record = {
+                    "kind": "file",
+                    "test_file": os.path.basename(filename),
+                    "status": "pass" if file_passed else "fail",
+                    "duration": round(file_elapsed[filename], 2),
+                }
+                if not file_passed:
+                    reason = next((r for f, r in failed_tests if f == filename), None)
+                    if reason:
+                        incremental_record["error"] = reason
+                with open(metrics_path, "a") as f:
+                    f.write(json.dumps(incremental_record) + "\n")
+            except OSError:
+                pass
+
+        if not file_passed and not continue_on_error:
+            break
 
     if fork_worker is not None:
         fork_worker.close()
@@ -484,27 +504,4 @@ def run_unittest_files(
             summary += f"- ✗ Still failed: {', '.join(failed_after_retry)}\n"
         write_github_step_summary(summary)
 
-    # Fully guarded auto-record for SGLANG_TEST_METRICS_FILE: unset (the default)
-    # means zero delta for every non-XPU-nightly suite. OSError is swallowed so
-    # a bad filesystem cannot turn a passing run red. Any new test file added
-    # to run_suite.py is picked up here without per-test wiring.
-    metrics_path = os.environ.get("SGLANG_TEST_METRICS_FILE")
-    if metrics_path:
-        passed_set = set(passed_tests)
-        failed_reasons = dict(failed_tests)
-        try:
-            with open(metrics_path, "a") as f:
-                for fname, elapsed in file_elapsed.items():
-                    record = {
-                        "kind": "file",
-                        "test_file": os.path.basename(fname),
-                        "status": "pass" if fname in passed_set else "fail",
-                        "duration": round(elapsed, 2),
-                    }
-                    if fname in failed_reasons:
-                        record["error"] = failed_reasons[fname]
-                    f.write(json.dumps(record) + "\n")
-        except OSError:
-            pass
-
     return 0 if success else -1

test/registered/xpu/test_deepseek_ocr_2_olmbench.py

@@ -27,6 +27,7 @@ from sglang.test.test_utils import (
     popen_launch_server,
     write_github_step_summary,
 )
+from sglang.test.xpu.test_xpu_utils import write_results_to_github_step_summary
 
 register_xpu_ci(est_time=7200, suite="nightly-xpu-1-gpu", nightly=True)
 
@@ -93,78 +94,94 @@ class TestDeepSeekOCR2OlmBenchXPU(CustomTestCase):
             kill_process_tree(cls.process.pid)
 
     def test_olmocr_bench(self):
-        if not self.bench_dir.exists():
+        model_metrics = {
-            self.fail(
+            "client": "olmOCR-bench",
-                f"olmOCR-bench data not found at {self.bench_dir}. Download it first:\n"
+            "accuracy_threshold": self.accuracy,
-                "  hf download --repo-type dataset allenai/olmOCR-bench "
+        }
-                "--local-dir ./olmOCR-bench"
-            )
-
-        port = urlparse(self.base_url).port
-        cmd = [
-            sys.executable,
-            str(_REPO_ROOT / "benchmark" / "ocr" / "bench_sglang.py"),
-            "--port",
-            str(port),
-            "--split",
-            self.split,
-            "--concurrency",
-            str(self.concurrency),
-            "--model",
-            self.model,
-            *(["--max-samples", str(self.max_samples)] if self.max_samples > 0 else []),
-            "--bench-dir",
-            str(self.bench_dir),
-            "--output-dir",
-            str(self.output_dir),
-        ]
-
         try:
-            subprocess.run(cmd, check=True, cwd=str(_REPO_ROOT))
+            if not self.bench_dir.exists():
-        except subprocess.CalledProcessError as e:
+                self.fail(
-            self.fail(f"olmOCR-bench run failed for {self.model}: {e}")
+                    f"olmOCR-bench data not found at {self.bench_dir}. Download it first:\n"
+                    "  hf download --repo-type dataset allenai/olmOCR-bench "
+                    "--local-dir ./olmOCR-bench"
+                )
 
-        summary_path = self.output_dir / "summary.json"
+            port = urlparse(self.base_url).port
-        if not summary_path.exists():
+            cmd = [
-            self.fail(f"Benchmark produced no summary at {summary_path}")
+                sys.executable,
+                str(_REPO_ROOT / "benchmark" / "ocr" / "bench_sglang.py"),
+                "--port",
+                str(port),
+                "--split",
+                self.split,
+                "--concurrency",
+                str(self.concurrency),
+                "--model",
+                self.model,
+                *(
+                    ["--max-samples", str(self.max_samples)]
+                    if self.max_samples > 0
+                    else []
+                ),
+                "--bench-dir",
+                str(self.bench_dir),
+                "--output-dir",
+                str(self.output_dir),
+            ]
 
-        with open(summary_path, encoding="utf-8") as f:
+            try:
-            results = json.load(f)
+                subprocess.run(cmd, check=True, cwd=str(_REPO_ROOT))
+            except subprocess.CalledProcessError as e:
+                self.fail(f"olmOCR-bench run failed for {self.model}: {e}")
 
-        total_tests = sum(r.get("total_tests", 0) for r in results.values())
+            summary_path = self.output_dir / "summary.json"
-        total_passed = sum(r.get("total_passed", 0) for r in results.values())
+            if not summary_path.exists():
-        total_errored = sum(r.get("error_samples", 0) for r in results.values())
+                self.fail(f"Benchmark produced no summary at {summary_path}")
-        score = total_passed / total_tests if total_tests else 0.0
 
-        lines = [
+            with open(summary_path, encoding="utf-8") as f:
-            f"## DeepSeek-OCR-2 olmOCR-bench (XPU, concurrency {self.concurrency})",
+                results = json.load(f)
-            "",
+
-            "| Split | Tests | Passed | Score | Errored |",
+            total_tests = sum(r.get("total_tests", 0) for r in results.values())
-            "| --- | ---: | ---: | ---: | ---: |",
+            total_passed = sum(r.get("total_passed", 0) for r in results.values())
-        ]
+            total_errored = sum(r.get("error_samples", 0) for r in results.values())
-        for split, r in results.items():
+            score = total_passed / total_tests if total_tests else 0.0
+            model_metrics["accuracy"] = score
+            model_metrics["num_prompts"] = total_tests
+
+            lines = [
+                f"## DeepSeek-OCR-2 olmOCR-bench (XPU, concurrency {self.concurrency})",
+                "",
+                "| Split | Tests | Passed | Score | Errored |",
+                "| --- | ---: | ---: | ---: | ---: |",
+            ]
+            for split, r in results.items():
+                lines.append(
+                    f"| {split} | {r.get('total_tests', 0)} | "
+                    f"{r.get('total_passed', 0)} | {r.get('overall_score', 0.0):.1f}% | "
+                    f"{r.get('error_samples', 0)} |"
+                )
             lines.append(
-                f"| {split} | {r.get('total_tests', 0)} | "
+                f"| **TOTAL** | {total_tests} | {total_passed} | "
-                f"{r.get('total_passed', 0)} | {r.get('overall_score', 0.0):.1f}% | "
+                f"**{100.0 * score:.1f}%** | {total_errored} |"
-                f"{r.get('error_samples', 0)} |"
             )
-        lines.append(
+            write_github_step_summary("\n".join(lines) + "\n")
-            f"| **TOTAL** | {total_tests} | {total_passed} | "
-            f"**{100.0 * score:.1f}%** | {total_errored} |"
-        )
-        write_github_step_summary("\n".join(lines) + "\n")
 
-        # Guard against a silent empty run before comparing the score.
+            # Guard against a silent empty run before comparing the score.
-        self.assertGreater(
+            self.assertGreater(
-            total_tests, 0, f"olmOCR-bench scored 0 tests for {self.model}"
+                total_tests, 0, f"olmOCR-bench scored 0 tests for {self.model}"
-        )
+            )
-        self.assertGreaterEqual(
+            self.assertGreaterEqual(
-            score,
+                score,
-            self.accuracy,
+                self.accuracy,
-            f"olmOCR-bench aggregate for {self.model} is {100.0 * score:.1f}%, "
+                f"olmOCR-bench aggregate for {self.model} is {100.0 * score:.1f}%, "
-            f"below the {100.0 * self.accuracy:.0f}% threshold "
+                f"below the {100.0 * self.accuracy:.0f}% threshold "
-            f"({total_errored} samples errored)",
+                f"({total_errored} samples errored)",
-        )
+            )
+        except Exception as e:
+            model_metrics["error"] = str(e)
+            raise
+        finally:
+            write_results_to_github_step_summary({self.model: model_metrics})
 
 
 if __name__ == "__main__":