xpu: record per-model metrics to jsonl for nightly dashboard (#36699)

Co-authored-by: arathi-hlab <arathi-hlab@users.noreply.github.com>

python/sglang/srt/environ.py

@@ -408,6 +408,14 @@ class Envs:
     # KL tests: skip the cache-hit count assertion (e.g. when alloc failure reduces hits)
     SGLANG_TEST_SKIP_CACHE_HIT_ASSERT = EnvBool(False)
 
+    # ===================================================================
+    # CI reporting: per-model metrics jsonl for nightly XPU dashboard
+    # ===================================================================
+    # When set, XPU nightly tests append one JSON record per model to this file
+    # so xpu-ci-job-monitor.yml can render per-model ref/actual/status/duration
+    # tables. Unset (the default) is a full no-op — pre-existing CI unaffected.
+    SGLANG_TEST_METRICS_FILE = EnvStr(None)
+
     # ===================================================================
     # PD and scripted-runtime tests
     # ===================================================================

python/sglang/test/ci/ci_utils.py

@@ -484,4 +484,27 @@ def run_unittest_files(
             summary += f"- ✗ Still failed: {', '.join(failed_after_retry)}\n"
         write_github_step_summary(summary)
 
+    # Fully guarded auto-record for SGLANG_TEST_METRICS_FILE: unset (the default)
+    # means zero delta for every non-XPU-nightly suite. OSError is swallowed so
+    # a bad filesystem cannot turn a passing run red. Any new test file added
+    # to run_suite.py is picked up here without per-test wiring.
+    metrics_path = os.environ.get("SGLANG_TEST_METRICS_FILE")
+    if metrics_path:
+        passed_set = set(passed_tests)
+        failed_reasons = dict(failed_tests)
+        try:
+            with open(metrics_path, "a") as f:
+                for fname, elapsed in file_elapsed.items():
+                    record = {
+                        "kind": "file",
+                        "test_file": os.path.basename(fname),
+                        "status": "pass" if fname in passed_set else "fail",
+                        "duration": round(elapsed, 2),
+                    }
+                    if fname in failed_reasons:
+                        record["error"] = failed_reasons[fname]
+                    f.write(json.dumps(record) + "\n")
+        except OSError:
+            pass
+
     return 0 if success else -1

python/sglang/test/xpu/simple_eval_gsm8k_xpu_mixin.py

@@ -84,6 +84,9 @@ class SimpleEvalGSM8KXPUMixin(ABC):
             "client": "simple_eval_gsm8k",
             "accuracy_threshold": getattr(self, "accuracy", "N/A"),
             "output_throughput_threshold": getattr(self, "output_throughput", "N/A"),
+            "num_prompts": self.num_examples,
+            "num_threads": self.num_threads,
+            "max_tokens": self.max_tokens,
         }
 
         try:

python/sglang/test/xpu/test_xpu_utils.py

@@ -5,11 +5,16 @@ so XPU and Ascend nightly runs render the same Markdown table in
 `$GITHUB_STEP_SUMMARY`.
 """
 
+import json
+import os
+import sys
+
+from sglang.srt.environ import envs
 from sglang.test.test_utils import is_in_ci, write_github_step_summary
 
 HEADER = """
-| Model | Server | Client | Output Throughput | Expected Output Throughput | Accuracy | Expected Accuracy | Status |
-| ----- | ------ | ------ | ----------------- | -------------------------- | -------- | ----------------- | ------ |
+| Model | Server | Client | Prompts | Output Throughput | Expected Output Throughput | Accuracy | Expected Accuracy | Status |
+| ----- | ------ | ------ | ------- | ----------------- | -------------------------- | -------- | ----------------- | ------ |
 """
 
 _HEADER_WRITTEN = False
@@ -40,11 +45,52 @@ def write_results_to_github_step_summary(results: dict):
         output_throughput_threshold = metrics.get("output_throughput_threshold", "N/A")
         server = metrics.get("server", "N/A")
         client = metrics.get("client", "N/A")
+        num_prompts = metrics.get("num_prompts", "N/A")
         error = metrics.get("error", "")
         status = "PASS" if error == "" else f"FAIL: {error}"
         summary += (
-            f"| {model} | {server} | {client} | {output_throughput} "
-            f"| {output_throughput_threshold} | {accuracy} "
-            f"| {accuracy_threshold} | {status} |\n"
+            f"| {model} | {server} | {client} | {num_prompts} "
+            f"| {output_throughput} | {output_throughput_threshold} "
+            f"| {accuracy} | {accuracy_threshold} | {status} |\n"
         )
     write_github_step_summary(summary)
+    _append_metric_records(results)
+
+
+def _append_metric_records(results: dict) -> None:
+    """Append one JSON record per model to `SGLANG_TEST_METRICS_FILE`, if set.
+
+    Consumed by the nightly XPU dashboard step in xpu-ci-job-monitor.yml to
+    render per-model ref/actual/status/duration tables. Errors are swallowed
+    so a broken write never turns a passing test red.
+    """
+    path = envs.SGLANG_TEST_METRICS_FILE.get()
+    if not path:
+        return
+    # sys.argv[0] is the test script path when a unittest file is run via
+    # `python3 test_foo.py`; renderer groups rich records to the file they came
+    # from so file-level fallback rows don't double-count them.
+    test_file = os.path.basename(sys.argv[0]) if sys.argv and sys.argv[0] else ""
+    try:
+        with open(path, "a") as f:
+            for model, metrics in results.items():
+                record = {
+                    "kind": "model",
+                    "test_file": test_file,
+                    "model": model,
+                    "accuracy": metrics.get("accuracy"),
+                    "accuracy_threshold": metrics.get("accuracy_threshold"),
+                    "output_throughput": metrics.get("output_throughput"),
+                    "output_throughput_threshold": metrics.get(
+                        "output_throughput_threshold"
+                    ),
+                    "latency": metrics.get("latency"),
+                    "num_prompts": metrics.get("num_prompts"),
+                    "num_threads": metrics.get("num_threads"),
+                    "max_tokens": metrics.get("max_tokens"),
+                    "error": metrics.get("error", ""),
+                    "status": "pass" if not metrics.get("error") else "fail",
+                }
+                f.write(json.dumps(record) + "\n")
+    except OSError:
+        pass