minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 10 Packages Projects Releases Wiki Activity

[AMD][CI] Consolidate AMD workflows and retire ROCm 7.0 CI (#38632)

Browse Source
This commit is contained in:
Bingxu Chen
2026-09-15 23:45:22 +08:00
committed by GitHub
parent 47a157f257
commit 03ea13a545
17 changed files with 1190 additions and 4932 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/ci/amd/amd_ci_start_container.sh
+6 -12
View File
@@ -19,8 +19,9 @@ else
fi
# Default base tags (can be overridden by command line arguments)
ROCM_VERSION="rocm700"
# Default base tags (can be overridden by command line arguments).
# Keep this aligned with the default AMD PR and Nightly CI lane.
ROCM_VERSION="rocm10"
DEFAULT_MI30X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi30x"
DEFAULT_MI35X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi35x"
LOCAL_DOCKER_REGISTRY="10.44.14.109:5000"
@@ -52,8 +53,8 @@ while [[ $# -gt 0 ]]; do
echo " --mi35x-base-tag TAG Override MI35x base image tag"
echo " --custom-image IMAGE Use a specific Docker image directly (or set AMD_CI_IMAGE)"
echo " --build-from-dockerfile Build image from docker/rocm.Dockerfile"
echo " --gpu-arch ARCH GPU architecture for Dockerfile build (e.g., gfx950-rocm720)"
echo " --rocm-version VERSION Override ROCm version for image lookup (e.g., rocm720)"
echo " --gpu-arch ARCH GPU architecture for Dockerfile build (e.g., gfx950-rocm1000)"
echo " --rocm-version VERSION Override ROCm version for image lookup (e.g., rocm10)"
echo ""
echo "Environment:"
echo " ENABLE_CACHE_HOST=1|0"
@@ -211,13 +212,6 @@ find_latest_image() {
echo "rocm/sgl-dev:v0.5.8.post1-rocm720-mi30x-20260211-preview"
fi
;;
rocm700)
if [[ "${gpu_arch}" == "mi35x" ]]; then
echo "rocm/sgl-dev:v0.5.8.post1-rocm700-mi35x-20260211"
else
echo "rocm/sgl-dev:v0.5.8.post1-rocm700-mi30x-20260211"
fi
;;
*)
echo "Error: no hard-coded fallback available for ${ROCM_VERSION}" >&2
return 1
@@ -253,7 +247,7 @@ elif [[ -n "${BUILD_FROM_DOCKERFILE}" ]]; then
IMAGE="sglang-ci:${GPU_ARCH_BUILD}-$(date +%Y%m%d)"
echo "Building Docker image from ${DOCKERFILE} with GPU_ARCH=${GPU_ARCH_BUILD}..."
# Pass full GPU_ARCH (e.g., gfx950-rocm720) - Dockerfile handles stripping suffix
# Pass full GPU_ARCH (e.g., gfx950-rocm1000) - Dockerfile handles stripping suffix
docker build \
--build-arg GPU_ARCH="${GPU_ARCH_BUILD}" \
--build-arg SGL_BRANCH="main" \
scripts/ci/amd/amd_ci_start_container_disagg.sh
+3 -9
View File
@@ -19,8 +19,9 @@ else
fi
# Default base tags (can be overridden by command line arguments)
ROCM_VERSION="rocm700"
# Default base tags (can be overridden by command line arguments).
# Keep this aligned with the default AMD PR and Nightly CI lane.
ROCM_VERSION="rocm10"
DEFAULT_MI30X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi30x"
DEFAULT_MI35X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi35x"
LOCAL_DOCKER_REGISTRY="10.44.14.109:5000"
@@ -196,13 +197,6 @@ find_latest_image() {
echo "rocm/sgl-dev:v0.5.8.post1-rocm720-mi30x-20260211-preview"
fi
;;
rocm700)
if [[ "${gpu_arch}" == "mi35x" ]]; then
echo "rocm/sgl-dev:v0.5.8.post1-rocm700-mi35x-20260211"
else
echo "rocm/sgl-dev:v0.5.8.post1-rocm700-mi30x-20260211"
fi
;;
*)
echo "Error: no hard-coded fallback available for ${ROCM_VERSION}" >&2
return 1
scripts/ci_monitor/ci_failures_analysis.py
+21 -9
View File
@@ -30,12 +30,18 @@ import requests
def _filter_legacy_amd_job_rows(job_data: Dict[str, Dict]) -> Dict[str, Dict]:
"""Drop pre-cutover AMD names without changing the shared analyzer."""
legacy_callers = {
"call-nightly-amd-rocm720",
"call-pr-test-amd-extra-rocm720",
"call-pr-test-amd-rocm720",
}
filtered = {}
for full_name, data in job_data.items():
# This caller was renamed by the AMD job-name cutover. Other outer
# callers, including AITER's *-rocm720 callers, are still current.
# These callers were renamed by the AMD workflow canonicalization. Keep
# recognizing their historical spellings so old rows do not merge into
# current streaks.
name_parts = full_name.split(" / ")
if "call-pr-test-amd-extra-rocm720" in name_parts[:-1]:
if any(part in legacy_callers for part in name_parts[:-1]):
continue
leaf_name = name_parts[-1]
@@ -49,7 +55,7 @@ def _filter_legacy_amd_job_rows(job_data: Dict[str, Dict]) -> Dict[str, Dict]:
"pr-gate",
"pr-test-amd-extra-finish",
"pr-test-amd-finish",
"pr-test-amd-rocm720-finish",
"pr-test-amd-rocm720-finish", # Legacy pre-canonicalization teardown
}:
continue
@@ -67,6 +73,12 @@ def _filter_legacy_amd_job_rows(job_data: Dict[str, Dict]) -> Dict[str, Dict]:
if details.startswith("rocm") and details[4:].isdigit():
continue
# The canonical workflow paths reuse the history of the retired
# ROCm 7.0 workflows. Do not let those rows contribute to current
# multi-version streaks during the lookback window.
if details.split(",", 1)[0] == "rocm700":
continue
filtered[full_name] = data
return filtered
@@ -92,7 +104,7 @@ class SGLangFailuresAnalyzer:
"check-changes",
"pr-test-finish", # Nvidia workflow teardown
"pr-test-amd-finish", # AMD workflow teardown
"pr-test-amd-rocm720-finish", # Default AMD ROCm 7.2 teardown
"pr-test-amd-rocm720-finish", # Legacy AMD teardown
"call-gate",
"pr-gate",
"check-all-jobs",
@@ -2502,7 +2514,7 @@ def main():
# These 4 don't have scheduled events, so filter by main branch instead
pr_test_amd_scheduled_runs = analyzer.get_recent_runs(
limit=pr_test_scheduled_limit,
workflow_filter=["pr-test-amd-rocm720.yml"],
workflow_filter=["pr-test-amd.yml"],
filters={"branch": "main"},
)
pr_test_xeon_scheduled_runs = analyzer.get_recent_runs(
@@ -2529,7 +2541,7 @@ def main():
)
nightly_amd_scheduled_runs = analyzer.get_recent_runs(
limit=nightly_scheduled_limit,
workflow_filter=["nightly-test-amd-rocm720.yml"],
workflow_filter=["nightly-test-amd.yml"],
filters={"event": "schedule"},
)
nightly_intel_scheduled_runs = analyzer.get_recent_runs(
@@ -2551,7 +2563,7 @@ def main():
)
pr_test_amd_general_runs = analyzer.get_recent_runs(
limit=args.limit,
workflow_filter=["pr-test-amd-rocm720.yml"],
workflow_filter=["pr-test-amd.yml"],
)
pr_test_xeon_general_runs = analyzer.get_recent_runs(
limit=args.limit,
@@ -2573,7 +2585,7 @@ def main():
)
nightly_amd_general_runs = analyzer.get_recent_runs(
limit=args.limit,
workflow_filter=["nightly-test-amd-rocm720.yml"],
workflow_filter=["nightly-test-amd.yml"],
)
nightly_intel_general_runs = analyzer.get_recent_runs(
limit=args.limit,
scripts/ci_monitor/test_ci_failures_analysis.py
-57
View File
@@ -1,57 +0,0 @@
"""AMD job-name cutover tests.
Run with:
python -m unittest discover -s scripts/ci_monitor -p 'test_ci_failures_analysis.py'
"""
import os
import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from ci_failures_analysis import _filter_legacy_amd_job_rows # noqa: E402
class TestFilterLegacyAmdJobRows(unittest.TestCase):
def test_drops_legacy_names_and_nested_utilities(self):
rows = {
"stage-b-test-1-gpu-small-amd-rocm720 (linux-mi300-1gpu-sglang, 0)": {},
"nightly-accuracy-2-gpu-rocm720 (rocm724)": {},
"nightly-accuracy-2-gpu-rocm724": {},
"nightly-test-1-gpu-unit (rocm724)": {},
"call-pr-test-amd-rocm720 / call-pr-test-amd-extra-rocm720 / extra-a-test-1-gpu-small-amd (linux-mi300-1gpu-sglang)": {},
"wait-for-stage-a-amd": {},
"call-pr-test-amd-extra / pr-test-amd-extra-finish": {},
"call-pr-test-amd-extra / call-gate / pr-gate": {},
}
self.assertEqual(_filter_legacy_amd_job_rows(rows), {})
def test_keeps_current_flavors_and_nested_callers_separate(self):
new_success = {"current_streak": 0}
rows = {
"stage-b-test-1-gpu-small-amd (rocm724, linux-mi300-1gpu-sglang, 0)": new_success,
"nightly-accuracy-2-gpu (rocm720, linux-mi300-2gpu-sglang)": {
"current_streak": 1
},
"call-pr-test-amd-rocm720 / stage-c-test-4-gpu-amd (rocm724, linux-mi300-4gpu-sglang, 0)": {
"current_streak": 0
},
"call-pr-test-amd-rocm720 / call-pr-test-amd-extra / extra-a-test-1-gpu-small-amd (rocm724, linux-mi300-1gpu-sglang)": {
"current_streak": 0
},
}
filtered = _filter_legacy_amd_job_rows(rows)
self.assertEqual(set(filtered), set(rows))
self.assertIs(
filtered[
"stage-b-test-1-gpu-small-amd (rocm724, linux-mi300-1gpu-sglang, 0)"
],
new_success,
)
if __name__ == "__main__":
unittest.main()