minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[XPU][CI] Fix empty nightly dashboard (#37800)

Browse Source
This commit is contained in:
ashwini rathi
2026-09-07 09:29:51 +08:00
committed by GitHub
parent e0a83a2215
commit 0afba909e7
4 changed files with 164 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/nightly-test-intel.yml
+59 -1
View File
@@ -59,6 +59,17 @@ jobs:
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: Overlay in-tree sglang test helpers onto image site-packages
run: |
# Bridges the up-to-24h window between merge and nightly image rebuild.
dest=$(docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
"import sglang, os; print(os.path.dirname(sglang.__file__))")
docker exec ci_sglang_xpu cp \
/sglang-checkout/python/sglang/test/ci/ci_utils.py \
"$dest/test/ci/ci_utils.py"
docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
"import sglang.test.ci.ci_utils; print('overlay ok')"
- name: HF login + install run_suite extras
timeout-minutes: 10
run: |
@@ -101,6 +112,7 @@ jobs:
cd /sglang-checkout/test &&
OLMOCR_BENCH_DIR=/sglang-checkout/olmOCR-bench/bench_data \
GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \
SGLANG_IS_IN_CI=true \
SGLANG_TEST_METRICS_FILE=/sglang-checkout/nightly-xpu-1-gpu-metrics.jsonl \
python3 run_suite.py --hw xpu --suite nightly-xpu-1-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
" || TEST_EXIT_CODE=$?
@@ -154,6 +166,17 @@ jobs:
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: Overlay in-tree sglang test helpers onto image site-packages
run: |
# Bridges the up-to-24h window between merge and nightly image rebuild.
dest=$(docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
"import sglang, os; print(os.path.dirname(sglang.__file__))")
docker exec ci_sglang_xpu cp \
/sglang-checkout/python/sglang/test/ci/ci_utils.py \
"$dest/test/ci/ci_utils.py"
docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
"import sglang.test.ci.ci_utils; print('overlay ok')"
- name: HF login + install run_suite extras
timeout-minutes: 10
run: |
@@ -185,6 +208,7 @@ jobs:
source /opt/venv/bin/activate &&
cd /sglang-checkout/test &&
GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \
SGLANG_IS_IN_CI=true \
SGLANG_TEST_METRICS_FILE=/sglang-checkout/nightly-xpu-2-gpu-metrics.jsonl \
python3 run_suite.py --hw xpu --suite nightly-xpu-2-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
" || TEST_EXIT_CODE=$?
@@ -238,6 +262,17 @@ jobs:
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: Overlay in-tree sglang test helpers onto image site-packages
run: |
# Bridges the up-to-24h window between merge and nightly image rebuild.
dest=$(docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
"import sglang, os; print(os.path.dirname(sglang.__file__))")
docker exec ci_sglang_xpu cp \
/sglang-checkout/python/sglang/test/ci/ci_utils.py \
"$dest/test/ci/ci_utils.py"
docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
"import sglang.test.ci.ci_utils; print('overlay ok')"
- name: HF login + install run_suite extras
timeout-minutes: 10
run: |
@@ -269,6 +304,7 @@ jobs:
source /opt/venv/bin/activate &&
cd /sglang-checkout/test &&
GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \
SGLANG_IS_IN_CI=true \
SGLANG_TEST_METRICS_FILE=/sglang-checkout/nightly-xpu-4-gpu-metrics.jsonl \
python3 run_suite.py --hw xpu --suite nightly-xpu-4-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
" || TEST_EXIT_CODE=$?
@@ -322,6 +358,17 @@ jobs:
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: Overlay in-tree sglang test helpers onto image site-packages
run: |
# Bridges the up-to-24h window between merge and nightly image rebuild.
dest=$(docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
"import sglang, os; print(os.path.dirname(sglang.__file__))")
docker exec ci_sglang_xpu cp \
/sglang-checkout/python/sglang/test/ci/ci_utils.py \
"$dest/test/ci/ci_utils.py"
docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
"import sglang.test.ci.ci_utils; print('overlay ok')"
- name: HF login + install run_suite extras
timeout-minutes: 10
run: |
@@ -348,16 +395,27 @@ jobs:
- name: Nightly Test (8-GPU XPU)
timeout-minutes: 480
run: |
touch github_summary.md
touch github_summary.md nightly-xpu-8-gpu-metrics.jsonl
docker exec ci_sglang_xpu bash -c "
source /opt/venv/bin/activate &&
cd /sglang-checkout/test &&
GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \
SGLANG_IS_IN_CI=true \
SGLANG_TEST_METRICS_FILE=/sglang-checkout/nightly-xpu-8-gpu-metrics.jsonl \
python3 run_suite.py --hw xpu --suite nightly-xpu-8-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
" || TEST_EXIT_CODE=$?
echo "$(<github_summary.md)" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
- name: Upload per-model metrics jsonl
if: always()
uses: actions/upload-artifact@v4
with:
name: nightly-xpu-8-gpu-metrics
path: nightly-xpu-8-gpu-metrics.jsonl
if-no-files-found: warn
retention-days: 30
- name: Cleanup container
if: always()
run: |
.github/workflows/xpu-ci-job-monitor.yml
+2 -2
View File
@@ -288,10 +288,10 @@ jobs:
with open(snap_path) as f:
snapshot = json.load(f)
# Only look at nightly-test-intel jobs
# Snapshot writer emits `workflow` (filename), not `workflow_path`.
nightly = [
j for j in snapshot.get("jobs", [])
if j.get("workflow_path", "").endswith("nightly-test-intel.yml")
if (j.get("workflow") or "").endswith("nightly-test-intel.yml")
]
# status_summary style counts, per job name
python/sglang/test/ci/ci_utils.py
+21 -24
View File
@@ -417,7 +417,27 @@ def run_unittest_files(
if not file_passed:
success = False
if not continue_on_error:
# Flush per-file so a SIGKILL mid-suite still surfaces completed files.
metrics_path = os.environ.get("SGLANG_TEST_METRICS_FILE")
if metrics_path and filename in file_elapsed:
try:
incremental_record = {
"kind": "file",
"test_file": os.path.basename(filename),
"status": "pass" if file_passed else "fail",
"duration": round(file_elapsed[filename], 2),
}
if not file_passed:
reason = next((r for f, r in failed_tests if f == filename), None)
if reason:
incremental_record["error"] = reason
with open(metrics_path, "a") as f:
f.write(json.dumps(incremental_record) + "\n")
except OSError:
pass
if not file_passed and not continue_on_error:
break
if fork_worker is not None:
@@ -484,27 +504,4 @@ def run_unittest_files(
summary += f"- ✗ Still failed: {', '.join(failed_after_retry)}\n"
write_github_step_summary(summary)
# Fully guarded auto-record for SGLANG_TEST_METRICS_FILE: unset (the default)
# means zero delta for every non-XPU-nightly suite. OSError is swallowed so
# a bad filesystem cannot turn a passing run red. Any new test file added
# to run_suite.py is picked up here without per-test wiring.
metrics_path = os.environ.get("SGLANG_TEST_METRICS_FILE")
if metrics_path:
passed_set = set(passed_tests)
failed_reasons = dict(failed_tests)
try:
with open(metrics_path, "a") as f:
for fname, elapsed in file_elapsed.items():
record = {
"kind": "file",
"test_file": os.path.basename(fname),
"status": "pass" if fname in passed_set else "fail",
"duration": round(elapsed, 2),
}
if fname in failed_reasons:
record["error"] = failed_reasons[fname]
f.write(json.dumps(record) + "\n")
except OSError:
pass
return 0 if success else -1
test/registered/xpu/test_deepseek_ocr_2_olmbench.py
+18 -1
View File
@@ -27,6 +27,7 @@ from sglang.test.test_utils import (
popen_launch_server,
write_github_step_summary,
)
from sglang.test.xpu.test_xpu_utils import write_results_to_github_step_summary
register_xpu_ci(est_time=7200, suite="nightly-xpu-1-gpu", nightly=True)
@@ -93,6 +94,11 @@ class TestDeepSeekOCR2OlmBenchXPU(CustomTestCase):
kill_process_tree(cls.process.pid)
def test_olmocr_bench(self):
model_metrics = {
"client": "olmOCR-bench",
"accuracy_threshold": self.accuracy,
}
try:
if not self.bench_dir.exists():
self.fail(
f"olmOCR-bench data not found at {self.bench_dir}. Download it first:\n"
@@ -112,7 +118,11 @@ class TestDeepSeekOCR2OlmBenchXPU(CustomTestCase):
str(self.concurrency),
"--model",
self.model,
*(["--max-samples", str(self.max_samples)] if self.max_samples > 0 else []),
*(
["--max-samples", str(self.max_samples)]
if self.max_samples > 0
else []
),
"--bench-dir",
str(self.bench_dir),
"--output-dir",
@@ -135,6 +145,8 @@ class TestDeepSeekOCR2OlmBenchXPU(CustomTestCase):
total_passed = sum(r.get("total_passed", 0) for r in results.values())
total_errored = sum(r.get("error_samples", 0) for r in results.values())
score = total_passed / total_tests if total_tests else 0.0
model_metrics["accuracy"] = score
model_metrics["num_prompts"] = total_tests
lines = [
f"## DeepSeek-OCR-2 olmOCR-bench (XPU, concurrency {self.concurrency})",
@@ -165,6 +177,11 @@ class TestDeepSeekOCR2OlmBenchXPU(CustomTestCase):
f"below the {100.0 * self.accuracy:.0f}% threshold "
f"({total_errored} samples errored)",
)
except Exception as e:
model_metrics["error"] = str(e)
raise
finally:
write_results_to_github_step_summary({self.model: model_metrics})
if __name__ == "__main__":