minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[XPU][CI] Fix empty nightly dashboard (#37800)

Browse Source
This commit is contained in:
ashwini rathi
2026-09-07 09:29:51 +08:00
committed by GitHub
parent e0a83a2215
commit 0afba909e7
4 changed files with 164 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/nightly-test-intel.yml
+59 -1
View File
@@ -59,6 +59,17 @@ jobs:
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: Overlay in-tree sglang test helpers onto image site-packages
run: |
# Bridges the up-to-24h window between merge and nightly image rebuild.
dest=$(docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
"import sglang, os; print(os.path.dirname(sglang.__file__))")
docker exec ci_sglang_xpu cp \
/sglang-checkout/python/sglang/test/ci/ci_utils.py \
"$dest/test/ci/ci_utils.py"
docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
"import sglang.test.ci.ci_utils; print('overlay ok')"
- name: HF login + install run_suite extras
timeout-minutes: 10
run: |
@@ -101,6 +112,7 @@ jobs:
cd /sglang-checkout/test &&
OLMOCR_BENCH_DIR=/sglang-checkout/olmOCR-bench/bench_data \
GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \
SGLANG_IS_IN_CI=true \
SGLANG_TEST_METRICS_FILE=/sglang-checkout/nightly-xpu-1-gpu-metrics.jsonl \
python3 run_suite.py --hw xpu --suite nightly-xpu-1-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
" || TEST_EXIT_CODE=$?
@@ -154,6 +166,17 @@ jobs:
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: Overlay in-tree sglang test helpers onto image site-packages
run: |
# Bridges the up-to-24h window between merge and nightly image rebuild.
dest=$(docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
"import sglang, os; print(os.path.dirname(sglang.__file__))")
docker exec ci_sglang_xpu cp \
/sglang-checkout/python/sglang/test/ci/ci_utils.py \
"$dest/test/ci/ci_utils.py"
docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
"import sglang.test.ci.ci_utils; print('overlay ok')"
- name: HF login + install run_suite extras
timeout-minutes: 10
run: |
@@ -185,6 +208,7 @@ jobs:
source /opt/venv/bin/activate &&
cd /sglang-checkout/test &&
GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \
SGLANG_IS_IN_CI=true \
SGLANG_TEST_METRICS_FILE=/sglang-checkout/nightly-xpu-2-gpu-metrics.jsonl \
python3 run_suite.py --hw xpu --suite nightly-xpu-2-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
" || TEST_EXIT_CODE=$?
@@ -238,6 +262,17 @@ jobs:
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: Overlay in-tree sglang test helpers onto image site-packages
run: |
# Bridges the up-to-24h window between merge and nightly image rebuild.
dest=$(docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
"import sglang, os; print(os.path.dirname(sglang.__file__))")
docker exec ci_sglang_xpu cp \
/sglang-checkout/python/sglang/test/ci/ci_utils.py \
"$dest/test/ci/ci_utils.py"
docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
"import sglang.test.ci.ci_utils; print('overlay ok')"
- name: HF login + install run_suite extras
timeout-minutes: 10
run: |
@@ -269,6 +304,7 @@ jobs:
source /opt/venv/bin/activate &&
cd /sglang-checkout/test &&
GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \
SGLANG_IS_IN_CI=true \
SGLANG_TEST_METRICS_FILE=/sglang-checkout/nightly-xpu-4-gpu-metrics.jsonl \
python3 run_suite.py --hw xpu --suite nightly-xpu-4-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
" || TEST_EXIT_CODE=$?
@@ -322,6 +358,17 @@ jobs:
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: Overlay in-tree sglang test helpers onto image site-packages
run: |
# Bridges the up-to-24h window between merge and nightly image rebuild.
dest=$(docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
"import sglang, os; print(os.path.dirname(sglang.__file__))")
docker exec ci_sglang_xpu cp \
/sglang-checkout/python/sglang/test/ci/ci_utils.py \
"$dest/test/ci/ci_utils.py"
docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
"import sglang.test.ci.ci_utils; print('overlay ok')"
- name: HF login + install run_suite extras
timeout-minutes: 10
run: |
@@ -348,16 +395,27 @@ jobs:
- name: Nightly Test (8-GPU XPU)
timeout-minutes: 480
run: |
touch github_summary.md
touch github_summary.md nightly-xpu-8-gpu-metrics.jsonl
docker exec ci_sglang_xpu bash -c "
source /opt/venv/bin/activate &&
cd /sglang-checkout/test &&
GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \
SGLANG_IS_IN_CI=true \
SGLANG_TEST_METRICS_FILE=/sglang-checkout/nightly-xpu-8-gpu-metrics.jsonl \
python3 run_suite.py --hw xpu --suite nightly-xpu-8-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
" || TEST_EXIT_CODE=$?
echo "$(<github_summary.md)" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
- name: Upload per-model metrics jsonl
if: always()
uses: actions/upload-artifact@v4
with:
name: nightly-xpu-8-gpu-metrics
path: nightly-xpu-8-gpu-metrics.jsonl
if-no-files-found: warn
retention-days: 30
- name: Cleanup container
if: always()
run: |
.github/workflows/xpu-ci-job-monitor.yml
+2 -2
View File
@@ -288,10 +288,10 @@ jobs:
with open(snap_path) as f:
snapshot = json.load(f)
# Only look at nightly-test-intel jobs
# Snapshot writer emits `workflow` (filename), not `workflow_path`.
nightly = [
j for j in snapshot.get("jobs", [])
if j.get("workflow_path", "").endswith("nightly-test-intel.yml")
if (j.get("workflow") or "").endswith("nightly-test-intel.yml")
]
# status_summary style counts, per job name
python/sglang/test/ci/ci_utils.py
+22 -25
View File
@@ -417,8 +417,28 @@ def run_unittest_files(
if not file_passed:
success = False
if not continue_on_error:
break
# Flush per-file so a SIGKILL mid-suite still surfaces completed files.
metrics_path = os.environ.get("SGLANG_TEST_METRICS_FILE")
if metrics_path and filename in file_elapsed:
try:
incremental_record = {
"kind": "file",
"test_file": os.path.basename(filename),
"status": "pass" if file_passed else "fail",
"duration": round(file_elapsed[filename], 2),
}
if not file_passed:
reason = next((r for f, r in failed_tests if f == filename), None)
if reason:
incremental_record["error"] = reason
with open(metrics_path, "a") as f:
f.write(json.dumps(incremental_record) + "\n")
except OSError:
pass
if not file_passed and not continue_on_error:
break
if fork_worker is not None:
fork_worker.close()
@@ -484,27 +504,4 @@ def run_unittest_files(
summary += f"- ✗ Still failed: {', '.join(failed_after_retry)}\n"
write_github_step_summary(summary)
# Fully guarded auto-record for SGLANG_TEST_METRICS_FILE: unset (the default)
# means zero delta for every non-XPU-nightly suite. OSError is swallowed so
# a bad filesystem cannot turn a passing run red. Any new test file added
# to run_suite.py is picked up here without per-test wiring.
metrics_path = os.environ.get("SGLANG_TEST_METRICS_FILE")
if metrics_path:
passed_set = set(passed_tests)
failed_reasons = dict(failed_tests)
try:
with open(metrics_path, "a") as f:
for fname, elapsed in file_elapsed.items():
record = {
"kind": "file",
"test_file": os.path.basename(fname),
"status": "pass" if fname in passed_set else "fail",
"duration": round(elapsed, 2),
}
if fname in failed_reasons:
record["error"] = failed_reasons[fname]
f.write(json.dumps(record) + "\n")
except OSError:
pass
return 0 if success else -1
test/registered/xpu/test_deepseek_ocr_2_olmbench.py
+81 -64
View File
@@ -27,6 +27,7 @@ from sglang.test.test_utils import (
popen_launch_server,
write_github_step_summary,
)
from sglang.test.xpu.test_xpu_utils import write_results_to_github_step_summary
register_xpu_ci(est_time=7200, suite="nightly-xpu-1-gpu", nightly=True)
@@ -93,78 +94,94 @@ class TestDeepSeekOCR2OlmBenchXPU(CustomTestCase):
kill_process_tree(cls.process.pid)
def test_olmocr_bench(self):
if not self.bench_dir.exists():
self.fail(
f"olmOCR-bench data not found at {self.bench_dir}. Download it first:\n"
" hf download --repo-type dataset allenai/olmOCR-bench "
"--local-dir ./olmOCR-bench"
)
port = urlparse(self.base_url).port
cmd = [
sys.executable,
str(_REPO_ROOT / "benchmark" / "ocr" / "bench_sglang.py"),
"--port",
str(port),
"--split",
self.split,
"--concurrency",
str(self.concurrency),
"--model",
self.model,
*(["--max-samples", str(self.max_samples)] if self.max_samples > 0 else []),
"--bench-dir",
str(self.bench_dir),
"--output-dir",
str(self.output_dir),
]
model_metrics = {
"client": "olmOCR-bench",
"accuracy_threshold": self.accuracy,
}
try:
subprocess.run(cmd, check=True, cwd=str(_REPO_ROOT))
except subprocess.CalledProcessError as e:
self.fail(f"olmOCR-bench run failed for {self.model}: {e}")
if not self.bench_dir.exists():
self.fail(
f"olmOCR-bench data not found at {self.bench_dir}. Download it first:\n"
" hf download --repo-type dataset allenai/olmOCR-bench "
"--local-dir ./olmOCR-bench"
)
summary_path = self.output_dir / "summary.json"
if not summary_path.exists():
self.fail(f"Benchmark produced no summary at {summary_path}")
port = urlparse(self.base_url).port
cmd = [
sys.executable,
str(_REPO_ROOT / "benchmark" / "ocr" / "bench_sglang.py"),
"--port",
str(port),
"--split",
self.split,
"--concurrency",
str(self.concurrency),
"--model",
self.model,
*(
["--max-samples", str(self.max_samples)]
if self.max_samples > 0
else []
),
"--bench-dir",
str(self.bench_dir),
"--output-dir",
str(self.output_dir),
]
with open(summary_path, encoding="utf-8") as f:
results = json.load(f)
try:
subprocess.run(cmd, check=True, cwd=str(_REPO_ROOT))
except subprocess.CalledProcessError as e:
self.fail(f"olmOCR-bench run failed for {self.model}: {e}")
total_tests = sum(r.get("total_tests", 0) for r in results.values())
total_passed = sum(r.get("total_passed", 0) for r in results.values())
total_errored = sum(r.get("error_samples", 0) for r in results.values())
score = total_passed / total_tests if total_tests else 0.0
summary_path = self.output_dir / "summary.json"
if not summary_path.exists():
self.fail(f"Benchmark produced no summary at {summary_path}")
lines = [
f"## DeepSeek-OCR-2 olmOCR-bench (XPU, concurrency {self.concurrency})",
"",
"| Split | Tests | Passed | Score | Errored |",
"| --- | ---: | ---: | ---: | ---: |",
]
for split, r in results.items():
with open(summary_path, encoding="utf-8") as f:
results = json.load(f)
total_tests = sum(r.get("total_tests", 0) for r in results.values())
total_passed = sum(r.get("total_passed", 0) for r in results.values())
total_errored = sum(r.get("error_samples", 0) for r in results.values())
score = total_passed / total_tests if total_tests else 0.0
model_metrics["accuracy"] = score
model_metrics["num_prompts"] = total_tests
lines = [
f"## DeepSeek-OCR-2 olmOCR-bench (XPU, concurrency {self.concurrency})",
"",
"| Split | Tests | Passed | Score | Errored |",
"| --- | ---: | ---: | ---: | ---: |",
]
for split, r in results.items():
lines.append(
f"| {split} | {r.get('total_tests', 0)} | "
f"{r.get('total_passed', 0)} | {r.get('overall_score', 0.0):.1f}% | "
f"{r.get('error_samples', 0)} |"
)
lines.append(
f"| {split} | {r.get('total_tests', 0)} | "
f"{r.get('total_passed', 0)} | {r.get('overall_score', 0.0):.1f}% | "
f"{r.get('error_samples', 0)} |"
f"| **TOTAL** | {total_tests} | {total_passed} | "
f"**{100.0 * score:.1f}%** | {total_errored} |"
)
lines.append(
f"| **TOTAL** | {total_tests} | {total_passed} | "
f"**{100.0 * score:.1f}%** | {total_errored} |"
)
write_github_step_summary("\n".join(lines) + "\n")
write_github_step_summary("\n".join(lines) + "\n")
# Guard against a silent empty run before comparing the score.
self.assertGreater(
total_tests, 0, f"olmOCR-bench scored 0 tests for {self.model}"
)
self.assertGreaterEqual(
score,
self.accuracy,
f"olmOCR-bench aggregate for {self.model} is {100.0 * score:.1f}%, "
f"below the {100.0 * self.accuracy:.0f}% threshold "
f"({total_errored} samples errored)",
)
# Guard against a silent empty run before comparing the score.
self.assertGreater(
total_tests, 0, f"olmOCR-bench scored 0 tests for {self.model}"
)
self.assertGreaterEqual(
score,
self.accuracy,
f"olmOCR-bench aggregate for {self.model} is {100.0 * score:.1f}%, "
f"below the {100.0 * self.accuracy:.0f}% threshold "
f"({total_errored} samples errored)",
)
except Exception as e:
model_metrics["error"] = str(e)
raise
finally:
write_results_to_github_step_summary({self.model: model_metrics})
if __name__ == "__main__":