diff --git a/.github/workflows/nightly-test-intel.yml b/.github/workflows/nightly-test-intel.yml index d27965679..15e496ba5 100644 --- a/.github/workflows/nightly-test-intel.yml +++ b/.github/workflows/nightly-test-intel.yml @@ -59,6 +59,17 @@ jobs: env: GITHUB_WORKSPACE: ${{ github.workspace }} + - name: Overlay in-tree sglang test helpers onto image site-packages + run: | + # Bridges the up-to-24h window between merge and nightly image rebuild. + dest=$(docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \ + "import sglang, os; print(os.path.dirname(sglang.__file__))") + docker exec ci_sglang_xpu cp \ + /sglang-checkout/python/sglang/test/ci/ci_utils.py \ + "$dest/test/ci/ci_utils.py" + docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \ + "import sglang.test.ci.ci_utils; print('overlay ok')" + - name: HF login + install run_suite extras timeout-minutes: 10 run: | @@ -101,6 +112,7 @@ jobs: cd /sglang-checkout/test && OLMOCR_BENCH_DIR=/sglang-checkout/olmOCR-bench/bench_data \ GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \ + SGLANG_IS_IN_CI=true \ SGLANG_TEST_METRICS_FILE=/sglang-checkout/nightly-xpu-1-gpu-metrics.jsonl \ python3 run_suite.py --hw xpu --suite nightly-xpu-1-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} " || TEST_EXIT_CODE=$? @@ -154,6 +166,17 @@ jobs: env: GITHUB_WORKSPACE: ${{ github.workspace }} + - name: Overlay in-tree sglang test helpers onto image site-packages + run: | + # Bridges the up-to-24h window between merge and nightly image rebuild. + dest=$(docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \ + "import sglang, os; print(os.path.dirname(sglang.__file__))") + docker exec ci_sglang_xpu cp \ + /sglang-checkout/python/sglang/test/ci/ci_utils.py \ + "$dest/test/ci/ci_utils.py" + docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \ + "import sglang.test.ci.ci_utils; print('overlay ok')" + - name: HF login + install run_suite extras timeout-minutes: 10 run: | @@ -185,6 +208,7 @@ jobs: source /opt/venv/bin/activate && cd /sglang-checkout/test && GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \ + SGLANG_IS_IN_CI=true \ SGLANG_TEST_METRICS_FILE=/sglang-checkout/nightly-xpu-2-gpu-metrics.jsonl \ python3 run_suite.py --hw xpu --suite nightly-xpu-2-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} " || TEST_EXIT_CODE=$? @@ -238,6 +262,17 @@ jobs: env: GITHUB_WORKSPACE: ${{ github.workspace }} + - name: Overlay in-tree sglang test helpers onto image site-packages + run: | + # Bridges the up-to-24h window between merge and nightly image rebuild. + dest=$(docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \ + "import sglang, os; print(os.path.dirname(sglang.__file__))") + docker exec ci_sglang_xpu cp \ + /sglang-checkout/python/sglang/test/ci/ci_utils.py \ + "$dest/test/ci/ci_utils.py" + docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \ + "import sglang.test.ci.ci_utils; print('overlay ok')" + - name: HF login + install run_suite extras timeout-minutes: 10 run: | @@ -269,6 +304,7 @@ jobs: source /opt/venv/bin/activate && cd /sglang-checkout/test && GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \ + SGLANG_IS_IN_CI=true \ SGLANG_TEST_METRICS_FILE=/sglang-checkout/nightly-xpu-4-gpu-metrics.jsonl \ python3 run_suite.py --hw xpu --suite nightly-xpu-4-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} " || TEST_EXIT_CODE=$? @@ -322,6 +358,17 @@ jobs: env: GITHUB_WORKSPACE: ${{ github.workspace }} + - name: Overlay in-tree sglang test helpers onto image site-packages + run: | + # Bridges the up-to-24h window between merge and nightly image rebuild. + dest=$(docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \ + "import sglang, os; print(os.path.dirname(sglang.__file__))") + docker exec ci_sglang_xpu cp \ + /sglang-checkout/python/sglang/test/ci/ci_utils.py \ + "$dest/test/ci/ci_utils.py" + docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \ + "import sglang.test.ci.ci_utils; print('overlay ok')" + - name: HF login + install run_suite extras timeout-minutes: 10 run: | @@ -348,16 +395,27 @@ jobs: - name: Nightly Test (8-GPU XPU) timeout-minutes: 480 run: | - touch github_summary.md + touch github_summary.md nightly-xpu-8-gpu-metrics.jsonl docker exec ci_sglang_xpu bash -c " source /opt/venv/bin/activate && cd /sglang-checkout/test && GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \ + SGLANG_IS_IN_CI=true \ + SGLANG_TEST_METRICS_FILE=/sglang-checkout/nightly-xpu-8-gpu-metrics.jsonl \ python3 run_suite.py --hw xpu --suite nightly-xpu-8-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} " || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} + - name: Upload per-model metrics jsonl + if: always() + uses: actions/upload-artifact@v4 + with: + name: nightly-xpu-8-gpu-metrics + path: nightly-xpu-8-gpu-metrics.jsonl + if-no-files-found: warn + retention-days: 30 + - name: Cleanup container if: always() run: | diff --git a/.github/workflows/xpu-ci-job-monitor.yml b/.github/workflows/xpu-ci-job-monitor.yml index 340943766..7303a74e7 100644 --- a/.github/workflows/xpu-ci-job-monitor.yml +++ b/.github/workflows/xpu-ci-job-monitor.yml @@ -288,10 +288,10 @@ jobs: with open(snap_path) as f: snapshot = json.load(f) - # Only look at nightly-test-intel jobs + # Snapshot writer emits `workflow` (filename), not `workflow_path`. nightly = [ j for j in snapshot.get("jobs", []) - if j.get("workflow_path", "").endswith("nightly-test-intel.yml") + if (j.get("workflow") or "").endswith("nightly-test-intel.yml") ] # status_summary style counts, per job name diff --git a/python/sglang/test/ci/ci_utils.py b/python/sglang/test/ci/ci_utils.py index 787a8e3b2..755f85d01 100644 --- a/python/sglang/test/ci/ci_utils.py +++ b/python/sglang/test/ci/ci_utils.py @@ -417,8 +417,28 @@ def run_unittest_files( if not file_passed: success = False - if not continue_on_error: - break + + # Flush per-file so a SIGKILL mid-suite still surfaces completed files. + metrics_path = os.environ.get("SGLANG_TEST_METRICS_FILE") + if metrics_path and filename in file_elapsed: + try: + incremental_record = { + "kind": "file", + "test_file": os.path.basename(filename), + "status": "pass" if file_passed else "fail", + "duration": round(file_elapsed[filename], 2), + } + if not file_passed: + reason = next((r for f, r in failed_tests if f == filename), None) + if reason: + incremental_record["error"] = reason + with open(metrics_path, "a") as f: + f.write(json.dumps(incremental_record) + "\n") + except OSError: + pass + + if not file_passed and not continue_on_error: + break if fork_worker is not None: fork_worker.close() @@ -484,27 +504,4 @@ def run_unittest_files( summary += f"- ✗ Still failed: {', '.join(failed_after_retry)}\n" write_github_step_summary(summary) - # Fully guarded auto-record for SGLANG_TEST_METRICS_FILE: unset (the default) - # means zero delta for every non-XPU-nightly suite. OSError is swallowed so - # a bad filesystem cannot turn a passing run red. Any new test file added - # to run_suite.py is picked up here without per-test wiring. - metrics_path = os.environ.get("SGLANG_TEST_METRICS_FILE") - if metrics_path: - passed_set = set(passed_tests) - failed_reasons = dict(failed_tests) - try: - with open(metrics_path, "a") as f: - for fname, elapsed in file_elapsed.items(): - record = { - "kind": "file", - "test_file": os.path.basename(fname), - "status": "pass" if fname in passed_set else "fail", - "duration": round(elapsed, 2), - } - if fname in failed_reasons: - record["error"] = failed_reasons[fname] - f.write(json.dumps(record) + "\n") - except OSError: - pass - return 0 if success else -1 diff --git a/test/registered/xpu/test_deepseek_ocr_2_olmbench.py b/test/registered/xpu/test_deepseek_ocr_2_olmbench.py index 48ddc1b9a..a6c1c800b 100644 --- a/test/registered/xpu/test_deepseek_ocr_2_olmbench.py +++ b/test/registered/xpu/test_deepseek_ocr_2_olmbench.py @@ -27,6 +27,7 @@ from sglang.test.test_utils import ( popen_launch_server, write_github_step_summary, ) +from sglang.test.xpu.test_xpu_utils import write_results_to_github_step_summary register_xpu_ci(est_time=7200, suite="nightly-xpu-1-gpu", nightly=True) @@ -93,78 +94,94 @@ class TestDeepSeekOCR2OlmBenchXPU(CustomTestCase): kill_process_tree(cls.process.pid) def test_olmocr_bench(self): - if not self.bench_dir.exists(): - self.fail( - f"olmOCR-bench data not found at {self.bench_dir}. Download it first:\n" - " hf download --repo-type dataset allenai/olmOCR-bench " - "--local-dir ./olmOCR-bench" - ) - - port = urlparse(self.base_url).port - cmd = [ - sys.executable, - str(_REPO_ROOT / "benchmark" / "ocr" / "bench_sglang.py"), - "--port", - str(port), - "--split", - self.split, - "--concurrency", - str(self.concurrency), - "--model", - self.model, - *(["--max-samples", str(self.max_samples)] if self.max_samples > 0 else []), - "--bench-dir", - str(self.bench_dir), - "--output-dir", - str(self.output_dir), - ] - + model_metrics = { + "client": "olmOCR-bench", + "accuracy_threshold": self.accuracy, + } try: - subprocess.run(cmd, check=True, cwd=str(_REPO_ROOT)) - except subprocess.CalledProcessError as e: - self.fail(f"olmOCR-bench run failed for {self.model}: {e}") + if not self.bench_dir.exists(): + self.fail( + f"olmOCR-bench data not found at {self.bench_dir}. Download it first:\n" + " hf download --repo-type dataset allenai/olmOCR-bench " + "--local-dir ./olmOCR-bench" + ) - summary_path = self.output_dir / "summary.json" - if not summary_path.exists(): - self.fail(f"Benchmark produced no summary at {summary_path}") + port = urlparse(self.base_url).port + cmd = [ + sys.executable, + str(_REPO_ROOT / "benchmark" / "ocr" / "bench_sglang.py"), + "--port", + str(port), + "--split", + self.split, + "--concurrency", + str(self.concurrency), + "--model", + self.model, + *( + ["--max-samples", str(self.max_samples)] + if self.max_samples > 0 + else [] + ), + "--bench-dir", + str(self.bench_dir), + "--output-dir", + str(self.output_dir), + ] - with open(summary_path, encoding="utf-8") as f: - results = json.load(f) + try: + subprocess.run(cmd, check=True, cwd=str(_REPO_ROOT)) + except subprocess.CalledProcessError as e: + self.fail(f"olmOCR-bench run failed for {self.model}: {e}") - total_tests = sum(r.get("total_tests", 0) for r in results.values()) - total_passed = sum(r.get("total_passed", 0) for r in results.values()) - total_errored = sum(r.get("error_samples", 0) for r in results.values()) - score = total_passed / total_tests if total_tests else 0.0 + summary_path = self.output_dir / "summary.json" + if not summary_path.exists(): + self.fail(f"Benchmark produced no summary at {summary_path}") - lines = [ - f"## DeepSeek-OCR-2 olmOCR-bench (XPU, concurrency {self.concurrency})", - "", - "| Split | Tests | Passed | Score | Errored |", - "| --- | ---: | ---: | ---: | ---: |", - ] - for split, r in results.items(): + with open(summary_path, encoding="utf-8") as f: + results = json.load(f) + + total_tests = sum(r.get("total_tests", 0) for r in results.values()) + total_passed = sum(r.get("total_passed", 0) for r in results.values()) + total_errored = sum(r.get("error_samples", 0) for r in results.values()) + score = total_passed / total_tests if total_tests else 0.0 + model_metrics["accuracy"] = score + model_metrics["num_prompts"] = total_tests + + lines = [ + f"## DeepSeek-OCR-2 olmOCR-bench (XPU, concurrency {self.concurrency})", + "", + "| Split | Tests | Passed | Score | Errored |", + "| --- | ---: | ---: | ---: | ---: |", + ] + for split, r in results.items(): + lines.append( + f"| {split} | {r.get('total_tests', 0)} | " + f"{r.get('total_passed', 0)} | {r.get('overall_score', 0.0):.1f}% | " + f"{r.get('error_samples', 0)} |" + ) lines.append( - f"| {split} | {r.get('total_tests', 0)} | " - f"{r.get('total_passed', 0)} | {r.get('overall_score', 0.0):.1f}% | " - f"{r.get('error_samples', 0)} |" + f"| **TOTAL** | {total_tests} | {total_passed} | " + f"**{100.0 * score:.1f}%** | {total_errored} |" ) - lines.append( - f"| **TOTAL** | {total_tests} | {total_passed} | " - f"**{100.0 * score:.1f}%** | {total_errored} |" - ) - write_github_step_summary("\n".join(lines) + "\n") + write_github_step_summary("\n".join(lines) + "\n") - # Guard against a silent empty run before comparing the score. - self.assertGreater( - total_tests, 0, f"olmOCR-bench scored 0 tests for {self.model}" - ) - self.assertGreaterEqual( - score, - self.accuracy, - f"olmOCR-bench aggregate for {self.model} is {100.0 * score:.1f}%, " - f"below the {100.0 * self.accuracy:.0f}% threshold " - f"({total_errored} samples errored)", - ) + # Guard against a silent empty run before comparing the score. + self.assertGreater( + total_tests, 0, f"olmOCR-bench scored 0 tests for {self.model}" + ) + self.assertGreaterEqual( + score, + self.accuracy, + f"olmOCR-bench aggregate for {self.model} is {100.0 * score:.1f}%, " + f"below the {100.0 * self.accuracy:.0f}% threshold " + f"({total_errored} samples errored)", + ) + except Exception as e: + model_metrics["error"] = str(e) + raise + finally: + write_results_to_github_step_summary({self.model: model_metrics}) if __name__ == "__main__":