[XPU][CI] Fix empty nightly dashboard (#37800)

.github/workflows/nightly-test-intel.yml

@@ -59,6 +59,17 @@ jobs:
         env:
           GITHUB_WORKSPACE: ${{ github.workspace }}
 
+      - name: Overlay in-tree sglang test helpers onto image site-packages
+        run: |
+          # Bridges the up-to-24h window between merge and nightly image rebuild.
+          dest=$(docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
+                 "import sglang, os; print(os.path.dirname(sglang.__file__))")
+          docker exec ci_sglang_xpu cp \
+            /sglang-checkout/python/sglang/test/ci/ci_utils.py \
+            "$dest/test/ci/ci_utils.py"
+          docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
+            "import sglang.test.ci.ci_utils; print('overlay ok')"
+
       - name: HF login + install run_suite extras
         timeout-minutes: 10
         run: |
@@ -101,6 +112,7 @@ jobs:
             cd /sglang-checkout/test &&
             OLMOCR_BENCH_DIR=/sglang-checkout/olmOCR-bench/bench_data \
             GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \
+            SGLANG_IS_IN_CI=true \
             SGLANG_TEST_METRICS_FILE=/sglang-checkout/nightly-xpu-1-gpu-metrics.jsonl \
             python3 run_suite.py --hw xpu --suite nightly-xpu-1-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
           " || TEST_EXIT_CODE=$?
@@ -154,6 +166,17 @@ jobs:
         env:
           GITHUB_WORKSPACE: ${{ github.workspace }}
 
+      - name: Overlay in-tree sglang test helpers onto image site-packages
+        run: |
+          # Bridges the up-to-24h window between merge and nightly image rebuild.
+          dest=$(docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
+                 "import sglang, os; print(os.path.dirname(sglang.__file__))")
+          docker exec ci_sglang_xpu cp \
+            /sglang-checkout/python/sglang/test/ci/ci_utils.py \
+            "$dest/test/ci/ci_utils.py"
+          docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
+            "import sglang.test.ci.ci_utils; print('overlay ok')"
+
       - name: HF login + install run_suite extras
         timeout-minutes: 10
         run: |
@@ -185,6 +208,7 @@ jobs:
             source /opt/venv/bin/activate &&
             cd /sglang-checkout/test &&
             GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \
+            SGLANG_IS_IN_CI=true \
             SGLANG_TEST_METRICS_FILE=/sglang-checkout/nightly-xpu-2-gpu-metrics.jsonl \
             python3 run_suite.py --hw xpu --suite nightly-xpu-2-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
           " || TEST_EXIT_CODE=$?
@@ -238,6 +262,17 @@ jobs:
         env:
           GITHUB_WORKSPACE: ${{ github.workspace }}
 
+      - name: Overlay in-tree sglang test helpers onto image site-packages
+        run: |
+          # Bridges the up-to-24h window between merge and nightly image rebuild.
+          dest=$(docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
+                 "import sglang, os; print(os.path.dirname(sglang.__file__))")
+          docker exec ci_sglang_xpu cp \
+            /sglang-checkout/python/sglang/test/ci/ci_utils.py \
+            "$dest/test/ci/ci_utils.py"
+          docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
+            "import sglang.test.ci.ci_utils; print('overlay ok')"
+
       - name: HF login + install run_suite extras
         timeout-minutes: 10
         run: |
@@ -269,6 +304,7 @@ jobs:
             source /opt/venv/bin/activate &&
             cd /sglang-checkout/test &&
             GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \
+            SGLANG_IS_IN_CI=true \
             SGLANG_TEST_METRICS_FILE=/sglang-checkout/nightly-xpu-4-gpu-metrics.jsonl \
             python3 run_suite.py --hw xpu --suite nightly-xpu-4-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
           " || TEST_EXIT_CODE=$?
@@ -322,6 +358,17 @@ jobs:
         env:
           GITHUB_WORKSPACE: ${{ github.workspace }}
 
+      - name: Overlay in-tree sglang test helpers onto image site-packages
+        run: |
+          # Bridges the up-to-24h window between merge and nightly image rebuild.
+          dest=$(docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
+                 "import sglang, os; print(os.path.dirname(sglang.__file__))")
+          docker exec ci_sglang_xpu cp \
+            /sglang-checkout/python/sglang/test/ci/ci_utils.py \
+            "$dest/test/ci/ci_utils.py"
+          docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
+            "import sglang.test.ci.ci_utils; print('overlay ok')"
+
       - name: HF login + install run_suite extras
         timeout-minutes: 10
         run: |
@@ -348,16 +395,27 @@ jobs:
       - name: Nightly Test (8-GPU XPU)
         timeout-minutes: 480
         run: |
-          touch github_summary.md
+          touch github_summary.md nightly-xpu-8-gpu-metrics.jsonl
           docker exec ci_sglang_xpu bash -c "
             source /opt/venv/bin/activate &&
             cd /sglang-checkout/test &&
             GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \
+            SGLANG_IS_IN_CI=true \
+            SGLANG_TEST_METRICS_FILE=/sglang-checkout/nightly-xpu-8-gpu-metrics.jsonl \
             python3 run_suite.py --hw xpu --suite nightly-xpu-8-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
           " || TEST_EXIT_CODE=$?
           echo "$(<github_summary.md)" >> $GITHUB_STEP_SUMMARY || true
           exit ${TEST_EXIT_CODE:-0}
 
+      - name: Upload per-model metrics jsonl
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: nightly-xpu-8-gpu-metrics
+          path: nightly-xpu-8-gpu-metrics.jsonl
+          if-no-files-found: warn
+          retention-days: 30
+
       - name: Cleanup container
         if: always()
         run: |

.github/workflows/xpu-ci-job-monitor.yml

@@ -288,10 +288,10 @@ jobs:
           with open(snap_path) as f:
               snapshot = json.load(f)
 
-          # Only look at nightly-test-intel jobs
+          # Snapshot writer emits `workflow` (filename), not `workflow_path`.
           nightly = [
               j for j in snapshot.get("jobs", [])
-              if j.get("workflow_path", "").endswith("nightly-test-intel.yml")
+              if (j.get("workflow") or "").endswith("nightly-test-intel.yml")
           ]
 
           # status_summary style counts, per job name

python/sglang/test/ci/ci_utils.py

@@ -417,7 +417,27 @@ def run_unittest_files(
 
         if not file_passed:
             success = False
-            if not continue_on_error:
+
+        # Flush per-file so a SIGKILL mid-suite still surfaces completed files.
+        metrics_path = os.environ.get("SGLANG_TEST_METRICS_FILE")
+        if metrics_path and filename in file_elapsed:
+            try:
+                incremental_record = {
+                    "kind": "file",
+                    "test_file": os.path.basename(filename),
+                    "status": "pass" if file_passed else "fail",
+                    "duration": round(file_elapsed[filename], 2),
+                }
+                if not file_passed:
+                    reason = next((r for f, r in failed_tests if f == filename), None)
+                    if reason:
+                        incremental_record["error"] = reason
+                with open(metrics_path, "a") as f:
+                    f.write(json.dumps(incremental_record) + "\n")
+            except OSError:
+                pass
+
+        if not file_passed and not continue_on_error:
             break
 
     if fork_worker is not None:
@@ -484,27 +504,4 @@ def run_unittest_files(
             summary += f"- ✗ Still failed: {', '.join(failed_after_retry)}\n"
         write_github_step_summary(summary)
 
-    # Fully guarded auto-record for SGLANG_TEST_METRICS_FILE: unset (the default)
-    # means zero delta for every non-XPU-nightly suite. OSError is swallowed so
-    # a bad filesystem cannot turn a passing run red. Any new test file added
-    # to run_suite.py is picked up here without per-test wiring.
-    metrics_path = os.environ.get("SGLANG_TEST_METRICS_FILE")
-    if metrics_path:
-        passed_set = set(passed_tests)
-        failed_reasons = dict(failed_tests)
-        try:
-            with open(metrics_path, "a") as f:
-                for fname, elapsed in file_elapsed.items():
-                    record = {
-                        "kind": "file",
-                        "test_file": os.path.basename(fname),
-                        "status": "pass" if fname in passed_set else "fail",
-                        "duration": round(elapsed, 2),
-                    }
-                    if fname in failed_reasons:
-                        record["error"] = failed_reasons[fname]
-                    f.write(json.dumps(record) + "\n")
-        except OSError:
-            pass
-
     return 0 if success else -1

test/registered/xpu/test_deepseek_ocr_2_olmbench.py

@@ -27,6 +27,7 @@ from sglang.test.test_utils import (
     popen_launch_server,
     write_github_step_summary,
 )
+from sglang.test.xpu.test_xpu_utils import write_results_to_github_step_summary
 
 register_xpu_ci(est_time=7200, suite="nightly-xpu-1-gpu", nightly=True)
 
@@ -93,6 +94,11 @@ class TestDeepSeekOCR2OlmBenchXPU(CustomTestCase):
             kill_process_tree(cls.process.pid)
 
     def test_olmocr_bench(self):
+        model_metrics = {
+            "client": "olmOCR-bench",
+            "accuracy_threshold": self.accuracy,
+        }
+        try:
             if not self.bench_dir.exists():
                 self.fail(
                     f"olmOCR-bench data not found at {self.bench_dir}. Download it first:\n"
@@ -112,7 +118,11 @@ class TestDeepSeekOCR2OlmBenchXPU(CustomTestCase):
                 str(self.concurrency),
                 "--model",
                 self.model,
-            *(["--max-samples", str(self.max_samples)] if self.max_samples > 0 else []),
+                *(
+                    ["--max-samples", str(self.max_samples)]
+                    if self.max_samples > 0
+                    else []
+                ),
                 "--bench-dir",
                 str(self.bench_dir),
                 "--output-dir",
@@ -135,6 +145,8 @@ class TestDeepSeekOCR2OlmBenchXPU(CustomTestCase):
             total_passed = sum(r.get("total_passed", 0) for r in results.values())
             total_errored = sum(r.get("error_samples", 0) for r in results.values())
             score = total_passed / total_tests if total_tests else 0.0
+            model_metrics["accuracy"] = score
+            model_metrics["num_prompts"] = total_tests
 
             lines = [
                 f"## DeepSeek-OCR-2 olmOCR-bench (XPU, concurrency {self.concurrency})",
@@ -165,6 +177,11 @@ class TestDeepSeekOCR2OlmBenchXPU(CustomTestCase):
                 f"below the {100.0 * self.accuracy:.0f}% threshold "
                 f"({total_errored} samples errored)",
             )
+        except Exception as e:
+            model_metrics["error"] = str(e)
+            raise
+        finally:
+            write_results_to_github_step_summary({self.model: model_metrics})
 
 
 if __name__ == "__main__":