minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[AMD][CI] Name the ROCm Image That Actually Ran in AMD Job Names (#35686)

Browse Source 
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Bingxu Chen
2026-08-24 01:07:25 -07:00
committed by GitHub
co-authored by Cursor
parent 5c4622341f
commit 7de80e566c
9 changed files with 327 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/ci/utils/query_job_status.py
+3 -1
View File
@@ -6,7 +6,9 @@ Usage:
# Per-job reports (original mode)
python scripts/ci/utils/query_job_status.py --job "stage-c-test-large-8-gpu-amd-mi35x"
python scripts/ci/utils/query_job_status.py --job "stage-c-test-large-8-gpu-amd-mi35x" --hours 48
python scripts/ci/utils/query_job_status.py --job "stage-c-test-large-8-gpu-amd-mi35x-rocm720" --workflow "pr-test-amd-rocm720.yml" --input-data-file actions-job-snapshot.json --summary
# A stem matches every ROCm flavor; use the full display name to pin one
python scripts/ci/utils/query_job_status.py --job "stage-c-test-large-8-gpu-amd-mi35x" --workflow "pr-test-amd-rocm720.yml" --input-data-file actions-job-snapshot.json --summary
python scripts/ci/utils/query_job_status.py --job "nightly-test-1-gpu-unit (rocm724, linux-mi300-1gpu-sglang)" --workflow "nightly-test-amd-rocm720.yml" --input-data-file actions-job-snapshot.json --summary
# Runner fleet report (cross-workflow runner analytics)
python scripts/ci/utils/query_job_status.py --runner-report --workflow "pr-test-amd-rocm720.yml,nightly-test-amd-rocm720.yml" --hours 24
scripts/ci_monitor/ci_failures_analysis.py
+60
View File
@@ -27,6 +27,51 @@ from typing import Dict, List, Optional, Tuple
import requests
def _filter_legacy_amd_job_rows(job_data: Dict[str, Dict]) -> Dict[str, Dict]:
"""Drop pre-cutover AMD names without changing the shared analyzer."""
filtered = {}
for full_name, data in job_data.items():
# This caller was renamed by the AMD job-name cutover. Other outer
# callers, including AITER's *-rocm720 callers, are still current.
name_parts = full_name.split(" / ")
if "call-pr-test-amd-extra-rocm720" in name_parts[:-1]:
continue
leaf_name = name_parts[-1]
if leaf_name.startswith(("wait-for-stage-a-amd", "wait-for-stage-b-amd")):
continue
if leaf_name in {
"call-gate",
"call-pr-test-amd-extra",
"check-all-jobs",
"check-changes",
"pr-gate",
"pr-test-amd-extra-finish",
"pr-test-amd-finish",
"pr-test-amd-rocm720-finish",
}:
continue
# Old display-name stems ended in -rocm<digits>. Their version was not
# reliable enough to map forward: a -rocm720 PR job could run rocm724.
stem = leaf_name.split(" (", 1)[0]
_, separator, version = stem.rpartition("-rocm")
if separator and version.isdigit():
continue
# An intermediate nightly schema showed only the ROCm flavor. Current
# names always include both the flavor and runner inside parentheses.
if leaf_name.endswith(")") and " (" in leaf_name:
details = leaf_name.rsplit(" (", 1)[1][:-1]
if details.startswith("rocm") and details[4:].isdigit():
continue
filtered[full_name] = data
return filtered
class SGLangFailuresAnalyzer:
"""Analyzes consecutive failures in GitHub Actions workflows."""
@@ -2648,6 +2693,21 @@ def main():
else ({}, {})
)
# AMD renamed its display names to carry the actual ROCm flavor. Reset
# only AMD history at that boundary so legacy failures cannot remain
# "current" under names that no longer exist. Other platforms continue
# to use the unmodified shared analyzer results. General AMD reports
# intentionally omit old-schema rows from branches that have not moved
# to the new names; their ROCm flavor cannot be mapped reliably.
pr_test_amd_scheduled_data = _filter_legacy_amd_job_rows(
pr_test_amd_scheduled_data
)
nightly_amd_scheduled_data = _filter_legacy_amd_job_rows(
nightly_amd_scheduled_data
)
pr_test_amd_general_data = _filter_legacy_amd_job_rows(pr_test_amd_general_data)
nightly_amd_general_data = _filter_legacy_amd_job_rows(nightly_amd_general_data)
# Analyze runner health and consecutive failures on all runs
(
runner_stats,
scripts/ci_monitor/test_ci_failures_analysis.py
+57
View File
@@ -0,0 +1,57 @@
"""AMD job-name cutover tests.
Run with:
python -m unittest discover -s scripts/ci_monitor -p 'test_ci_failures_analysis.py'
"""
import os
import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from ci_failures_analysis import _filter_legacy_amd_job_rows # noqa: E402
class TestFilterLegacyAmdJobRows(unittest.TestCase):
def test_drops_legacy_names_and_nested_utilities(self):
rows = {
"stage-b-test-1-gpu-small-amd-rocm720 (linux-mi300-1gpu-sglang, 0)": {},
"nightly-accuracy-2-gpu-rocm720 (rocm724)": {},
"nightly-accuracy-2-gpu-rocm724": {},
"nightly-test-1-gpu-unit (rocm724)": {},
"call-pr-test-amd-rocm720 / call-pr-test-amd-extra-rocm720 / extra-a-test-1-gpu-small-amd (linux-mi300-1gpu-sglang)": {},
"wait-for-stage-a-amd": {},
"call-pr-test-amd-extra / pr-test-amd-extra-finish": {},
"call-pr-test-amd-extra / call-gate / pr-gate": {},
}
self.assertEqual(_filter_legacy_amd_job_rows(rows), {})
def test_keeps_current_flavors_and_nested_callers_separate(self):
new_success = {"current_streak": 0}
rows = {
"stage-b-test-1-gpu-small-amd (rocm724, linux-mi300-1gpu-sglang, 0)": new_success,
"nightly-accuracy-2-gpu (rocm720, linux-mi300-2gpu-sglang)": {
"current_streak": 1
},
"call-pr-test-amd-rocm720 / stage-c-test-4-gpu-amd (rocm724, linux-mi300-4gpu-sglang, 0)": {
"current_streak": 0
},
"call-pr-test-amd-rocm720 / call-pr-test-amd-extra / extra-a-test-1-gpu-small-amd (rocm724, linux-mi300-1gpu-sglang)": {
"current_streak": 0
},
}
filtered = _filter_legacy_amd_job_rows(rows)
self.assertEqual(set(filtered), set(rows))
self.assertIs(
filtered[
"stage-b-test-1-gpu-small-amd (rocm724, linux-mi300-1gpu-sglang, 0)"
],
new_success,
)
if __name__ == "__main__":
unittest.main()