minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[AMD][CI] Swap the AMD PR gate to ROCm 7.2 and demote ROCm 7.0 to a daily shadow (#34204)

Browse Source 
Co-authored-by: yctseng0211 <yctseng@smci355-ccs-aus-m12-17.cs-aus.dcgpu>
Co-authored-by: Chen <bingxche@amd.com>
This commit is contained in:
YC Yen-Ching Tseng
2026-08-12 03:31:03 -07:00
committed by GitHub
co-authored by yctseng0211 Chen
parent b3bffef70a
commit 00bdafe944
12 changed files with 515 additions and 432 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/ci/amd/amd_ci_install_dependency.sh
+25 -1
View File
@@ -238,6 +238,19 @@ DOCKERFILE="docker/rocm.Dockerfile"
GPU_ARCH="${GPU_ARCH:-mi30x}"
echo "[CI-AITER-CHECK] Runner GPU_ARCH=${GPU_ARCH}"
# ROCm 7.0 keeps the Triton its base image ships; later ROCm images run on the
# Triton AITER pins, so a rebuilt AITER has to bring its own along.
IMAGE_HIP_VERSION=$(docker exec ci_sglang python3 -c 'import torch; print(torch.version.hip or "")')
case "${IMAGE_HIP_VERSION}" in
7.0*) INSTALL_AITER_TRITON="false" ;;
7.*) INSTALL_AITER_TRITON="true" ;;
*)
echo "[CI-AITER-CHECK] ERROR: Unsupported or empty HIP version: '${IMAGE_HIP_VERSION}'"
exit 1
;;
esac
echo "[CI-AITER-CHECK] Container HIP=${IMAGE_HIP_VERSION}, install AITER's Triton on rebuild=${INSTALL_AITER_TRITON}"
#############################################
# 1. Extract AITER_COMMIT from correct Dockerfile block
#############################################
@@ -292,7 +305,6 @@ else
NEED_REBUILD="true"
fi
#############################################
# 4. Rebuild AITER if needed
#############################################
@@ -329,6 +341,18 @@ if [[ "${NEED_REBUILD}" == "true" ]]; then
fi
echo "[CI-AITER-CHECK] GPU_ARCH_LIST=${GPU_ARCH_LIST}"
# Run the installer here rather than letting setup.py do it: setup.py
# swallows its errors, and the AITER_USE_SYSTEM_TRITON=1 below then keeps
# whatever Triton is already installed. Doing it up front fails closed.
if [[ "${INSTALL_AITER_TRITON}" == "true" ]]; then
docker exec ci_sglang bash -c "
set -euo pipefail
cd /sgl-workspace/aiter
test -f .github/scripts/install_triton.sh
bash .github/scripts/install_triton.sh
"
fi
# build AITER
docker exec ci_sglang bash -c "
cd /sgl-workspace/aiter && \
scripts/ci/utils/query_job_status.py
+4 -4
View File
@@ -6,10 +6,10 @@ Usage:
# Per-job reports (original mode)
python scripts/ci/utils/query_job_status.py --job "stage-c-test-large-8-gpu-amd-mi35x"
python scripts/ci/utils/query_job_status.py --job "stage-c-test-large-8-gpu-amd-mi35x" --hours 48
python scripts/ci/utils/query_job_status.py --job "stage-c-test-large-8-gpu-amd-mi35x" --workflow "pr-test-amd.yml" --input-data-file actions-job-snapshot.json --summary
python scripts/ci/utils/query_job_status.py --job "stage-c-test-large-8-gpu-amd-mi35x-rocm720" --workflow "pr-test-amd-rocm720.yml" --input-data-file actions-job-snapshot.json --summary
# Runner fleet report (cross-workflow runner analytics)
python scripts/ci/utils/query_job_status.py --runner-report --workflow "pr-test-amd.yml,nightly-test-amd.yml" --hours 24
python scripts/ci/utils/query_job_status.py --runner-report --workflow "pr-test-amd-rocm720.yml,nightly-test-amd-rocm720.yml" --hours 24
python scripts/ci/utils/query_job_status.py --runner-report --workflow "pr-test-amd.yml,nightly-test-amd.yml,pr-test-amd-rocm720.yml,nightly-test-amd-rocm720.yml" --summary
python scripts/ci/utils/query_job_status.py --workflow "pr-test-amd.yml,nightly-test-amd.yml,pr-test-amd-rocm720.yml,nightly-test-amd-rocm720.yml" --dump-data-file actions-job-snapshot.json
@@ -1719,8 +1719,8 @@ def main():
)
parser.add_argument(
"--workflow",
default="pr-test-amd.yml",
help="Workflow file name, or comma-separated list for --runner-report (default: pr-test-amd.yml)",
default="pr-test-amd-rocm720.yml",
help="Workflow file name, or comma-separated list for --runner-report (default: pr-test-amd-rocm720.yml)",
)
parser.add_argument(
"--hours",
scripts/ci/utils/slash_command_handler.py
+1 -1
View File
@@ -1401,7 +1401,7 @@ def main():
"- **Full CI rerun** (with extra coverage): add the `run-ci` or "
"`run-ci-extra` label and push a new commit (or use `/tag-and-rerun-ci`).\n\n"
"**AMD CI**: stage-level dispatch is still available via "
"Actions UI → *PR Test (AMD)* / *PR Test ROCm 7.2 (AMD)* → "
"Actions UI → *PR Test ROCm 7.2 (AMD)* (default) / *PR Test ROCm 7.0 (AMD)* → "
"*Run workflow* → pick a stage from the dropdown."
)
scripts/ci_monitor/ci_failures_analysis.py
+5 -4
View File
@@ -47,6 +47,7 @@ class SGLangFailuresAnalyzer:
"check-changes",
"pr-test-finish", # Nvidia workflow teardown
"pr-test-amd-finish", # AMD workflow teardown
"pr-test-amd-rocm720-finish", # Default AMD ROCm 7.2 teardown
"call-gate",
"pr-gate",
"check-all-jobs",
@@ -2456,7 +2457,7 @@ def main():
# These 4 don't have scheduled events, so filter by main branch instead
pr_test_amd_scheduled_runs = analyzer.get_recent_runs(
limit=pr_test_scheduled_limit,
workflow_filter=["pr-test-amd.yml"],
workflow_filter=["pr-test-amd-rocm720.yml"],
filters={"branch": "main"},
)
pr_test_xeon_scheduled_runs = analyzer.get_recent_runs(
@@ -2483,7 +2484,7 @@ def main():
)
nightly_amd_scheduled_runs = analyzer.get_recent_runs(
limit=nightly_scheduled_limit,
workflow_filter=["nightly-test-amd.yml"],
workflow_filter=["nightly-test-amd-rocm720.yml"],
filters={"event": "schedule"},
)
nightly_intel_scheduled_runs = analyzer.get_recent_runs(
@@ -2505,7 +2506,7 @@ def main():
)
pr_test_amd_general_runs = analyzer.get_recent_runs(
limit=args.limit,
workflow_filter=["pr-test-amd.yml"],
workflow_filter=["pr-test-amd-rocm720.yml"],
)
pr_test_xeon_general_runs = analyzer.get_recent_runs(
limit=args.limit,
@@ -2527,7 +2528,7 @@ def main():
)
nightly_amd_general_runs = analyzer.get_recent_runs(
limit=args.limit,
workflow_filter=["nightly-test-amd.yml"],
workflow_filter=["nightly-test-amd-rocm720.yml"],
)
nightly_intel_general_runs = analyzer.get_recent_runs(
limit=args.limit,