From 783af667fbfac46de4c3e914122e7f8daf0cb23d Mon Sep 17 00:00:00 2001 From: ashwini rathi Date: Tue, 1 Sep 2026 07:50:56 +0530 Subject: [PATCH] xpu: record per-model metrics to jsonl for nightly dashboard (#36699) Co-authored-by: arathi-hlab --- .github/workflows/nightly-test-intel.yml | 36 +- .github/workflows/xpu-ci-job-monitor.yml | 363 ++++++++++++++++++ python/sglang/srt/environ.py | 8 + python/sglang/test/ci/ci_utils.py | 23 ++ .../test/xpu/simple_eval_gsm8k_xpu_mixin.py | 3 + python/sglang/test/xpu/test_xpu_utils.py | 56 ++- 6 files changed, 481 insertions(+), 8 deletions(-) diff --git a/.github/workflows/nightly-test-intel.yml b/.github/workflows/nightly-test-intel.yml index 2d16a5df9..a15dcd2ff 100644 --- a/.github/workflows/nightly-test-intel.yml +++ b/.github/workflows/nightly-test-intel.yml @@ -95,17 +95,27 @@ jobs: - name: Nightly Test (1-GPU XPU) timeout-minutes: 240 run: | - touch github_summary.md + touch github_summary.md nightly-xpu-1-gpu-metrics.jsonl docker exec ci_sglang_xpu bash -c " source /opt/venv/bin/activate && cd /sglang-checkout/test && OLMOCR_BENCH_DIR=/sglang-checkout/olmOCR-bench/bench_data \ GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \ + SGLANG_TEST_METRICS_FILE=/sglang-checkout/nightly-xpu-1-gpu-metrics.jsonl \ python3 run_suite.py --hw xpu --suite nightly-xpu-1-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} " || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} + - name: Upload per-model metrics jsonl + if: always() + uses: actions/upload-artifact@v4 + with: + name: nightly-xpu-1-gpu-metrics + path: nightly-xpu-1-gpu-metrics.jsonl + if-no-files-found: warn + retention-days: 30 + - name: Cleanup container if: always() run: | @@ -170,16 +180,26 @@ jobs: - name: Nightly Test (2-GPU XPU) timeout-minutes: 240 run: | - touch github_summary.md + touch github_summary.md nightly-xpu-2-gpu-metrics.jsonl docker exec ci_sglang_xpu bash -c " source /opt/venv/bin/activate && cd /sglang-checkout/test && GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \ + SGLANG_TEST_METRICS_FILE=/sglang-checkout/nightly-xpu-2-gpu-metrics.jsonl \ python3 run_suite.py --hw xpu --suite nightly-xpu-2-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} " || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} + - name: Upload per-model metrics jsonl + if: always() + uses: actions/upload-artifact@v4 + with: + name: nightly-xpu-2-gpu-metrics + path: nightly-xpu-2-gpu-metrics.jsonl + if-no-files-found: warn + retention-days: 30 + - name: Cleanup container if: always() run: | @@ -244,16 +264,26 @@ jobs: - name: Nightly Test (4-GPU XPU) timeout-minutes: 480 run: | - touch github_summary.md + touch github_summary.md nightly-xpu-4-gpu-metrics.jsonl docker exec ci_sglang_xpu bash -c " source /opt/venv/bin/activate && cd /sglang-checkout/test && GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \ + SGLANG_TEST_METRICS_FILE=/sglang-checkout/nightly-xpu-4-gpu-metrics.jsonl \ python3 run_suite.py --hw xpu --suite nightly-xpu-4-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} " || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} + - name: Upload per-model metrics jsonl + if: always() + uses: actions/upload-artifact@v4 + with: + name: nightly-xpu-4-gpu-metrics + path: nightly-xpu-4-gpu-metrics.jsonl + if-no-files-found: warn + retention-days: 30 + - name: Cleanup container if: always() run: | diff --git a/.github/workflows/xpu-ci-job-monitor.yml b/.github/workflows/xpu-ci-job-monitor.yml index 460a64e92..340943766 100644 --- a/.github/workflows/xpu-ci-job-monitor.yml +++ b/.github/workflows/xpu-ci-job-monitor.yml @@ -254,3 +254,366 @@ jobs: --hours ${{ inputs.hours || '24' }} \ --input-data-file ci-data/actions-job-snapshot.json \ --summary + + # Colorful nightly XPU status dashboard, rendered into the workflow summary. + # Success-rate thresholds: >=90% healthy, >=70% degraded, else critical. + nightly-xpu-status-dashboard: + name: Nightly XPU Status Dashboard + if: ${{ !inputs.job_filter }} + needs: [fetch-actions-data, nightly-reports] + runs-on: ubuntu-latest + env: + HEALTHY_THRESHOLD: '90' + DEGRADED_THRESHOLD: '70' + WINDOW_HOURS: ${{ inputs.hours || '24' }} + steps: + - name: Download Actions data snapshot + uses: actions/download-artifact@v4 + with: + name: actions-job-snapshot + path: ci-data + + - name: Render colorful nightly status dashboard + run: | + python - <<'PY' + import json, os, sys + from datetime import datetime, timezone + + snap_path = "ci-data/actions-job-snapshot.json" + out = os.environ.get("GITHUB_STEP_SUMMARY", "/dev/stdout") + healthy = float(os.environ["HEALTHY_THRESHOLD"]) + degraded = float(os.environ["DEGRADED_THRESHOLD"]) + window_hours = os.environ.get("WINDOW_HOURS", "24") + + with open(snap_path) as f: + snapshot = json.load(f) + + # Only look at nightly-test-intel jobs + nightly = [ + j for j in snapshot.get("jobs", []) + if j.get("workflow_path", "").endswith("nightly-test-intel.yml") + ] + + # status_summary style counts, per job name + per_job = {} + for j in nightly: + name = j.get("job_name") or j.get("name", "unknown") + d = per_job.setdefault(name, {"success": 0, "failure": 0, "cancelled": 0, "skipped": 0, "in_progress": 0, "other": 0}) + status = j.get("status", "") + conclusion = j.get("conclusion", "") + if status == "completed": + if conclusion == "success": + d["success"] += 1 + elif conclusion == "failure": + d["failure"] += 1 + elif conclusion == "skipped": + d["skipped"] += 1 + elif conclusion == "cancelled": + d["cancelled"] += 1 + else: + d["other"] += 1 + elif status in ("in_progress", "queued", "waiting"): + d["in_progress"] += 1 + else: + d["other"] += 1 + + def badge(label, value, color): + # shields.io renders inline in GitHub markdown summaries. + label = label.replace("-", "--").replace(" ", "_") + value = str(value).replace("-", "--").replace(" ", "_") + return f"![{label}](https://img.shields.io/badge/{label}-{value}-{color})" + + def health(rate): + if rate is None: + return ("gray", "UNKNOWN", "⚪") # white circle + if rate >= healthy: + return ("brightgreen", "HEALTHY", "\U0001F7E2") # green circle + if rate >= degraded: + return ("yellow", "DEGRADED", "\U0001F7E1") # yellow circle + return ("red", "CRITICAL", "\U0001F534") # red circle + + totals = {"success": 0, "failure": 0, "cancelled": 0, "skipped": 0, "in_progress": 0, "other": 0} + for counts in per_job.values(): + for k, v in counts.items(): + totals[k] += v + finished = totals["success"] + totals["failure"] + overall_rate = (totals["success"] / finished * 100.0) if finished else None + fleet_color, fleet_label, fleet_dot = health(overall_rate) + + now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + lines = [] + lines.append("## Nightly XPU Status Dashboard") + lines.append("") + lines.append(f"_Window: last **{window_hours}h** · Snapshot rendered at **{now}**_") + lines.append("") + lines.append("### Fleet health") + lines.append("") + lines.append( + " ".join([ + badge("fleet", fleet_label, fleet_color), + badge("success_rate", f"{overall_rate:.1f}%25" if overall_rate is not None else "n%2Fa", fleet_color), + badge("healthy_threshold", f"{int(healthy)}%25", "brightgreen"), + badge("degraded_threshold", f"{int(degraded)}%25", "yellow"), + ]) + ) + lines.append("") + lines.append( + f"**{fleet_dot} Overall:** " + f"{totals['success']} passed · {totals['failure']} failed · " + f"{totals['cancelled']} cancelled · {totals['skipped']} skipped · " + f"{totals['in_progress']} in flight" + ) + lines.append("") + lines.append("### Per-model / per-job breakdown") + lines.append("") + lines.append("| Status | Job | Success rate | Passed | Failed | Skipped | Cancelled | In-flight |") + lines.append("|:------:|:----|-------------:|-------:|-------:|--------:|----------:|----------:|") + + def sort_key(item): + name, c = item + finished = c["success"] + c["failure"] + rate = (c["success"] / finished) if finished else 2.0 # untested last + return (rate, name) + + for name, c in sorted(per_job.items(), key=sort_key): + finished = c["success"] + c["failure"] + rate = (c["success"] / finished * 100.0) if finished else None + color, label, dot = health(rate) + rate_txt = f"{rate:.1f}%" if rate is not None else "n/a" + rate_badge = badge("rate", f"{rate:.1f}%25" if rate is not None else "n%2Fa", color) + lines.append( + f"| {dot} `{label}` | `{name}` | {rate_badge} {rate_txt} | " + f"{c['success']} | {c['failure']} | {c['skipped']} | {c['cancelled']} | {c['in_progress']} |" + ) + + lines.append("") + lines.append("Thresholds: " + f"\U0001F7E2 healthy ≥ {int(healthy)}% · " + f"\U0001F7E1 degraded ≥ {int(degraded)}% · " + f"\U0001F534 critical below") + lines.append("") + + with open(out, "a") as f: + f.write("\n".join(lines) + "\n") + + # Fail the step if fleet is CRITICAL so oncall sees a red X on the run. + if overall_rate is not None and overall_rate < degraded: + print(f"::error::Nightly XPU fleet is CRITICAL: success rate {overall_rate:.1f}% < {int(degraded)}%") + sys.exit(1) + PY + + # ------------------------------------------------------------------- + # Intel(XPU) Nightly report — per-model 1x/2x/4x/8x tables. + # Downloads the per-model metrics jsonl artifacts uploaded by + # nightly-test-intel.yml (SGLANG_TEST_METRICS_FILE side-writer) and renders + # one Markdown table per GPU-count suite. 8x is future-ready: it renders + # "No metrics recorded" until an 8-GPU suite is enabled upstream. + # ------------------------------------------------------------------- + nightly-xpu-per-model-report: + name: Intel(XPU) Nightly report + # Fork PRs don't get secrets and can't read cross-workflow artifacts; + # rest is same gating as the fetch-actions-data job above. + if: >- + github.event_name != 'pull_request' || + (github.event.pull_request.head.repo.full_name == github.repository && + contains(github.event.pull_request.labels.*.name, 'run-ci')) + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Resolve latest Nightly Test (Intel) run id + id: source-run + run: | + # Pick the most recent completed run of nightly-test-intel.yml on the + # default branch. workflow_dispatch reruns are included; skipped or + # in-progress runs are not. + RUN_ID=$(gh run list \ + --repo "${GITHUB_REPOSITORY}" \ + --workflow nightly-test-intel.yml \ + --branch main \ + --status completed \ + --limit 1 \ + --json databaseId --jq '.[0].databaseId // empty') + if [ -z "$RUN_ID" ]; then + echo "No completed Nightly Test (Intel) run found on main." + echo "run_id=" >> "$GITHUB_OUTPUT" + else + echo "Latest completed nightly run: $RUN_ID" + echo "run_id=$RUN_ID" >> "$GITHUB_OUTPUT" + fi + + - name: Download per-model metrics artifacts + if: steps.source-run.outputs.run_id != '' + uses: actions/download-artifact@v4 + with: + pattern: nightly-xpu-*-metrics + path: metrics/ + run-id: ${{ steps.source-run.outputs.run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + repository: ${{ github.repository }} + # Keep artifact names as subdirs so we can split by GPU-count suite. + merge-multiple: false + continue-on-error: true + + - name: Render per-model 1x / 2x / 4x / 8x tables + env: + SOURCE_RUN_ID: ${{ steps.source-run.outputs.run_id }} + run: | + python - <<'PY' + import glob + import json + import os + + SUITES = [ + ("1x", "nightly-xpu-1-gpu-metrics"), + ("2x", "nightly-xpu-2-gpu-metrics"), + ("4x", "nightly-xpu-4-gpu-metrics"), + # 8x is here for future readiness. Until nightly-test-intel.yml + # adds an 8-GPU suite + upload step, its artifact directory is + # missing and the section renders "No metrics recorded". + ("8x", "nightly-xpu-8-gpu-metrics"), + ] + + def load(subdir: str): + path = os.path.join("metrics", subdir) + records = [] + for f in sorted(glob.glob(os.path.join(path, "*.jsonl"))): + with open(f) as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + records.append(json.loads(line)) + except json.JSONDecodeError: + # Skip malformed rows so one bad line doesn't + # blank the whole table. + continue + return records + + def fmt(value, precision: int = 2) -> str: + if value is None or value == "" or value == "N/A": + return "N/A" + try: + return f"{float(value):.{precision}f}" + except (TypeError, ValueError): + return str(value) + + out = os.environ.get("GITHUB_STEP_SUMMARY", "/dev/stdout") + source_run = os.environ.get("SOURCE_RUN_ID", "") + repo = os.environ.get("GITHUB_REPOSITORY", "") + + lines = ["", "## Intel(XPU) Nightly report", ""] + if source_run and repo: + lines.append( + f"Source: [Intel(XPU) Nightly report - run #{source_run}]" + f"(https://github.com/{repo}/actions/runs/{source_run})" + ) + lines.append("") + elif not source_run: + lines.append("_No completed Nightly Test (Intel) run found; nothing to render._") + lines.append("") + + def rows_for(records): + # Group records by test_file. Two record shapes: + # kind=="model": rich per-model row emitted by tests that call + # write_results_to_github_step_summary. + # kind=="file": file-level fallback emitted by run_unittest_files + # for every test file in the suite. + # If a test_file has any model rows, we show those and drop the + # matching file row so a test does not double-count. Test files + # that never emit a model row fall back to the file row - that + # is what makes newly-added tests appear automatically. + by_file_model = {} + by_file_file = {} + orphans_model = [] + for r in records: + kind = r.get("kind") + tf = r.get("test_file", "") or "" + if kind == "model": + if tf: + by_file_model.setdefault(tf, []).append(r) + else: + orphans_model.append(r) + else: + # "file" or legacy records without kind + if tf and "model" not in r: + by_file_file[tf] = r + else: + orphans_model.append(r) + + rendered = [] + for tf in sorted(set(by_file_model) | set(by_file_file)): + if tf in by_file_model: + for r in by_file_model[tf]: + rendered.append(("model", r)) + else: + rendered.append(("file", by_file_file[tf])) + for r in orphans_model: + rendered.append(("model", r)) + return rendered + + any_fail = 0 + for label, subdir in SUITES: + records = load(subdir) + lines.append(f"### {label} ({subdir[:-len('-metrics')]})") + lines.append("") + if not records: + lines.append("_No metrics recorded for this suite._") + lines.append("") + continue + + lines.append( + "| Model / Test file | Prompts | Ref accuracy | Actual accuracy " + "| Ref throughput | Actual throughput | Duration | Status |" + ) + lines.append( + "|-------------------|--------:|-------------:|----------------:" + "|---------------:|------------------:|---------:|:------:|" + ) + for row_kind, r in rows_for(records): + status = str(r.get("status", "")).lower() + is_pass = status == "pass" + if not is_pass: + any_fail += 1 + status_cell = "PASS" if is_pass else (status.upper() or "N/A") + if row_kind == "model": + duration = r.get("latency") + lines.append( + f"| `{r.get('model','?')}` " + f"| {r.get('num_prompts','N/A')} " + f"| {fmt(r.get('accuracy_threshold'), 3)} " + f"| {fmt(r.get('accuracy'), 3)} " + f"| {fmt(r.get('output_throughput_threshold'), 2)} " + f"| {fmt(r.get('output_throughput'), 2)} " + f"| {fmt(duration, 2)} " + f"| {status_cell} |" + ) + else: + tf = r.get("test_file", "?") + lines.append( + f"| {tf} " + f"| N/A | N/A | N/A | N/A | N/A " + f"| {fmt(r.get('duration'), 2)} " + f"| {status_cell} |" + ) + lines.append("") + + lines.append( + "Rows with a model name are per-model records emitted by " + "`write_results_to_github_step_summary`; rows with just a test " + "file name are auto-recorded by `run_unittest_files`. Ref columns " + "are the accuracy / throughput thresholds declared by each test; " + "Actual is what nightly measured. Any new test added to " + "run_suite.py shows up here on the next nightly." + ) + lines.append("") + + with open(out, "a") as f: + f.write("\n".join(lines) + "\n") + + # Surface — but don't hard-fail — the report step: nightly-test-intel + # already flags model failures. This job is a rendering step. + if any_fail: + print(f"::warning::{any_fail} nightly XPU model row(s) marked FAIL in the per-model tables") + PY diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index b04f162a1..d42e6b60a 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -408,6 +408,14 @@ class Envs: # KL tests: skip the cache-hit count assertion (e.g. when alloc failure reduces hits) SGLANG_TEST_SKIP_CACHE_HIT_ASSERT = EnvBool(False) + # =================================================================== + # CI reporting: per-model metrics jsonl for nightly XPU dashboard + # =================================================================== + # When set, XPU nightly tests append one JSON record per model to this file + # so xpu-ci-job-monitor.yml can render per-model ref/actual/status/duration + # tables. Unset (the default) is a full no-op — pre-existing CI unaffected. + SGLANG_TEST_METRICS_FILE = EnvStr(None) + # =================================================================== # PD and scripted-runtime tests # =================================================================== diff --git a/python/sglang/test/ci/ci_utils.py b/python/sglang/test/ci/ci_utils.py index 50a7da70c..8b87d8b84 100644 --- a/python/sglang/test/ci/ci_utils.py +++ b/python/sglang/test/ci/ci_utils.py @@ -484,4 +484,27 @@ def run_unittest_files( summary += f"- ✗ Still failed: {', '.join(failed_after_retry)}\n" write_github_step_summary(summary) + # Fully guarded auto-record for SGLANG_TEST_METRICS_FILE: unset (the default) + # means zero delta for every non-XPU-nightly suite. OSError is swallowed so + # a bad filesystem cannot turn a passing run red. Any new test file added + # to run_suite.py is picked up here without per-test wiring. + metrics_path = os.environ.get("SGLANG_TEST_METRICS_FILE") + if metrics_path: + passed_set = set(passed_tests) + failed_reasons = dict(failed_tests) + try: + with open(metrics_path, "a") as f: + for fname, elapsed in file_elapsed.items(): + record = { + "kind": "file", + "test_file": os.path.basename(fname), + "status": "pass" if fname in passed_set else "fail", + "duration": round(elapsed, 2), + } + if fname in failed_reasons: + record["error"] = failed_reasons[fname] + f.write(json.dumps(record) + "\n") + except OSError: + pass + return 0 if success else -1 diff --git a/python/sglang/test/xpu/simple_eval_gsm8k_xpu_mixin.py b/python/sglang/test/xpu/simple_eval_gsm8k_xpu_mixin.py index 9b6e1a936..6ed2e284d 100644 --- a/python/sglang/test/xpu/simple_eval_gsm8k_xpu_mixin.py +++ b/python/sglang/test/xpu/simple_eval_gsm8k_xpu_mixin.py @@ -84,6 +84,9 @@ class SimpleEvalGSM8KXPUMixin(ABC): "client": "simple_eval_gsm8k", "accuracy_threshold": getattr(self, "accuracy", "N/A"), "output_throughput_threshold": getattr(self, "output_throughput", "N/A"), + "num_prompts": self.num_examples, + "num_threads": self.num_threads, + "max_tokens": self.max_tokens, } try: diff --git a/python/sglang/test/xpu/test_xpu_utils.py b/python/sglang/test/xpu/test_xpu_utils.py index b9b23e518..6dcda4459 100644 --- a/python/sglang/test/xpu/test_xpu_utils.py +++ b/python/sglang/test/xpu/test_xpu_utils.py @@ -5,11 +5,16 @@ so XPU and Ascend nightly runs render the same Markdown table in `$GITHUB_STEP_SUMMARY`. """ +import json +import os +import sys + +from sglang.srt.environ import envs from sglang.test.test_utils import is_in_ci, write_github_step_summary HEADER = """ -| Model | Server | Client | Output Throughput | Expected Output Throughput | Accuracy | Expected Accuracy | Status | -| ----- | ------ | ------ | ----------------- | -------------------------- | -------- | ----------------- | ------ | +| Model | Server | Client | Prompts | Output Throughput | Expected Output Throughput | Accuracy | Expected Accuracy | Status | +| ----- | ------ | ------ | ------- | ----------------- | -------------------------- | -------- | ----------------- | ------ | """ _HEADER_WRITTEN = False @@ -40,11 +45,52 @@ def write_results_to_github_step_summary(results: dict): output_throughput_threshold = metrics.get("output_throughput_threshold", "N/A") server = metrics.get("server", "N/A") client = metrics.get("client", "N/A") + num_prompts = metrics.get("num_prompts", "N/A") error = metrics.get("error", "") status = "PASS" if error == "" else f"FAIL: {error}" summary += ( - f"| {model} | {server} | {client} | {output_throughput} " - f"| {output_throughput_threshold} | {accuracy} " - f"| {accuracy_threshold} | {status} |\n" + f"| {model} | {server} | {client} | {num_prompts} " + f"| {output_throughput} | {output_throughput_threshold} " + f"| {accuracy} | {accuracy_threshold} | {status} |\n" ) write_github_step_summary(summary) + _append_metric_records(results) + + +def _append_metric_records(results: dict) -> None: + """Append one JSON record per model to `SGLANG_TEST_METRICS_FILE`, if set. + + Consumed by the nightly XPU dashboard step in xpu-ci-job-monitor.yml to + render per-model ref/actual/status/duration tables. Errors are swallowed + so a broken write never turns a passing test red. + """ + path = envs.SGLANG_TEST_METRICS_FILE.get() + if not path: + return + # sys.argv[0] is the test script path when a unittest file is run via + # `python3 test_foo.py`; renderer groups rich records to the file they came + # from so file-level fallback rows don't double-count them. + test_file = os.path.basename(sys.argv[0]) if sys.argv and sys.argv[0] else "" + try: + with open(path, "a") as f: + for model, metrics in results.items(): + record = { + "kind": "model", + "test_file": test_file, + "model": model, + "accuracy": metrics.get("accuracy"), + "accuracy_threshold": metrics.get("accuracy_threshold"), + "output_throughput": metrics.get("output_throughput"), + "output_throughput_threshold": metrics.get( + "output_throughput_threshold" + ), + "latency": metrics.get("latency"), + "num_prompts": metrics.get("num_prompts"), + "num_threads": metrics.get("num_threads"), + "max_tokens": metrics.get("max_tokens"), + "error": metrics.get("error", ""), + "status": "pass" if not metrics.get("error") else "fail", + } + f.write(json.dumps(record) + "\n") + except OSError: + pass