minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[AMD][CI] Swap the AMD PR gate to ROCm 7.2 and demote ROCm 7.0 to a daily shadow (#34204)

Browse Source 
Co-authored-by: yctseng0211 <yctseng@smci355-ccs-aus-m12-17.cs-aus.dcgpu>
Co-authored-by: Chen <bingxche@amd.com>
This commit is contained in:
YC Yen-Ching Tseng
2026-08-12 03:31:03 -07:00
committed by GitHub
co-authored by yctseng0211 Chen
parent b3bffef70a
commit 00bdafe944
12 changed files with 515 additions and 432 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/amd-aiter-scout.yml
+3 -3
View File
@@ -12,7 +12,7 @@ on:
type: string
default: 'main'
job_filter:
description: 'Comma-separated workflows to run: nightly-amd, nightly-amd-rocm720, pr-test-amd, pr-test-amd-rocm720. Default: all'
description: 'Workflows: nightly-amd, nightly-amd-rocm720, pr-test-amd (ROCm 7.0 shadow), pr-test-amd-rocm720 (ROCm 7.2 gate). Default: all'
required: false
type: string
default: 'all'
@@ -154,8 +154,8 @@ jobs:
echo "|----------|--------|" >> $GITHUB_STEP_SUMMARY
echo "| Nightly AMD (AITER Latest) | \`${{ needs.call-nightly-amd.result }}\` |" >> $GITHUB_STEP_SUMMARY
echo "| Nightly AMD ROCm 7.2 | \`${{ needs.call-nightly-amd-rocm720.result }}\` |" >> $GITHUB_STEP_SUMMARY
echo "| PR Test AMD (AITER Latest) | \`${{ needs.call-pr-test-amd.result }}\` |" >> $GITHUB_STEP_SUMMARY
echo "| PR Test AMD ROCm 7.2 | \`${{ needs.call-pr-test-amd-rocm720.result }}\` |" >> $GITHUB_STEP_SUMMARY
echo "| PR Test AMD ROCm 7.0 Shadow | \`${{ needs.call-pr-test-amd.result }}\` |" >> $GITHUB_STEP_SUMMARY
echo "| PR Test AMD ROCm 7.2 Gate | \`${{ needs.call-pr-test-amd-rocm720.result }}\` |" >> $GITHUB_STEP_SUMMARY
- name: Check if any job failed
run: |
.github/workflows/amd-ci-job-monitor.yml
+98 -98
View File
@@ -53,7 +53,7 @@ jobs:
id: select-workflows
run: |
if [[ -n "${{ inputs.job_filter }}" ]]; then
echo "workflows=pr-test-amd.yml" >> "$GITHUB_OUTPUT"
echo "workflows=pr-test-amd-rocm720.yml" >> "$GITHUB_OUTPUT"
else
echo "workflows=pr-test-amd.yml,nightly-test-amd.yml,pr-test-amd-rocm720.yml,nightly-test-amd-rocm720.yml,nightly-amd-mi355x-disagg.yml" >> "$GITHUB_OUTPUT"
fi
@@ -106,7 +106,7 @@ jobs:
python scripts/ci/utils/query_job_status.py \
--repo ${{ github.repository }} \
--job "${{ inputs.job_filter }}" \
--workflow "pr-test-amd.yml" \
--workflow "pr-test-amd-rocm720.yml" \
--hours ${{ inputs.hours || '24' }} \
--input-data-file ci-data/actions-job-snapshot.json \
--summary
@@ -119,8 +119,8 @@ jobs:
outputs:
pr_jobs: ${{ steps.parse.outputs.pr_jobs }}
nightly_jobs: ${{ steps.parse.outputs.nightly_jobs }}
pr_rocm720_jobs: ${{ steps.parse.outputs.pr_rocm720_jobs }}
nightly_rocm720_jobs: ${{ steps.parse.outputs.nightly_rocm720_jobs }}
pr_rocm700_jobs: ${{ steps.parse.outputs.pr_rocm700_jobs }}
nightly_rocm700_jobs: ${{ steps.parse.outputs.nightly_rocm700_jobs }}
disagg_jobs: ${{ steps.parse.outputs.disagg_jobs }}
steps:
- name: Checkout code
@@ -129,37 +129,37 @@ jobs:
- name: Parse workflow files
id: parse
run: |
# Parse pr-test-amd.yml and extract job names (exclude utility jobs)
# Excluded: call-gate, check-changes, pr-test-amd-finish, cancel, check-all-jobs
pr_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/pr-test-amd.yml | \
grep -v -E '^(call-gate|check-changes|pr-test-amd-finish|cancel|check-all-jobs)$' | \
# Parse the default ROCm 7.2 PR gate (exclude utility jobs)
# Excluded: call-gate, check-changes, finish, cancel, check-all-jobs
pr_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/pr-test-amd-rocm720.yml | \
grep -v -E '^(call-gate|check-changes|pr-test-amd-rocm720-finish|cancel|check-all-jobs)$' | \
jq -R -s -c 'split("\n") | map(select(length > 0))')
echo "pr_jobs=$pr_jobs" >> $GITHUB_OUTPUT
echo "PR jobs: $pr_jobs"
# Parse nightly-test-amd.yml and extract job names (exclude utility jobs)
# Parse the default ROCm 7.2 nightly (exclude utility jobs)
# Excluded: check-all-jobs
nightly_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/nightly-test-amd.yml | \
nightly_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/nightly-test-amd-rocm720.yml | \
grep -v -E '^(check-all-jobs)$' | \
jq -R -s -c 'split("\n") | map(select(length > 0))')
echo "nightly_jobs=$nightly_jobs" >> $GITHUB_OUTPUT
echo "Nightly jobs: $nightly_jobs"
# Parse pr-test-amd-rocm720.yml (exclude utility jobs)
# Excluded: call-gate, check-changes, pr-test-amd-finish, cancel, check-all-jobs
pr_rocm720_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/pr-test-amd-rocm720.yml | \
# Parse the ROCm 7.0 rollback shadow (exclude utility jobs)
# Excluded: call-gate, check-changes, finish, cancel, check-all-jobs
pr_rocm700_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/pr-test-amd.yml | \
grep -v -E '^(call-gate|check-changes|pr-test-amd-finish|cancel|check-all-jobs)$' | \
jq -R -s -c 'split("\n") | map(select(length > 0))')
echo "pr_rocm720_jobs=$pr_rocm720_jobs" >> $GITHUB_OUTPUT
echo "PR ROCm 7.2 jobs: $pr_rocm720_jobs"
echo "pr_rocm700_jobs=$pr_rocm700_jobs" >> $GITHUB_OUTPUT
echo "PR ROCm 7.0 shadow jobs: $pr_rocm700_jobs"
# Parse nightly-test-amd-rocm720.yml (exclude utility jobs)
# Parse the ROCm 7.0 rollback shadow nightly (exclude utility jobs)
# Excluded: check-all-jobs
nightly_rocm720_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/nightly-test-amd-rocm720.yml | \
nightly_rocm700_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/nightly-test-amd.yml | \
grep -v -E '^(check-all-jobs)$' | \
jq -R -s -c 'split("\n") | map(select(length > 0))')
echo "nightly_rocm720_jobs=$nightly_rocm720_jobs" >> $GITHUB_OUTPUT
echo "Nightly ROCm 7.2 jobs: $nightly_rocm720_jobs"
echo "nightly_rocm700_jobs=$nightly_rocm700_jobs" >> $GITHUB_OUTPUT
echo "Nightly ROCm 7.0 shadow jobs: $nightly_rocm700_jobs"
# Parse nightly-amd-mi355x-disagg.yml (exclude utility jobs)
# Excluded: setup, collect-results -- both run on ubuntu-latest and the
@@ -204,7 +204,7 @@ jobs:
python scripts/ci/utils/query_job_status.py \
--repo ${{ github.repository }} \
--job "${{ matrix.job_name }}" \
--workflow "pr-test-amd.yml" \
--workflow "pr-test-amd-rocm720.yml" \
--hours ${{ inputs.hours || '24' }} \
--input-data-file ci-data/actions-job-snapshot.json \
--summary
@@ -238,84 +238,6 @@ jobs:
path: ci-data
- name: Generate Nightly Report
timeout-minutes: 15
run: |
python scripts/ci/utils/query_job_status.py \
--repo ${{ github.repository }} \
--job "${{ matrix.job_name }}" \
--workflow "nightly-test-amd.yml" \
--hours ${{ inputs.hours || '24' }} \
--input-data-file ci-data/actions-job-snapshot.json \
--summary
# PR ROCm 7.2 CI reports using dynamic matrix
pr-rocm720-ci-reports:
name: PR ROCm720 - ${{ matrix.job_name }}
needs: [parse-workflows, fetch-actions-data]
if: ${{ !inputs.job_filter }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
job_name: ${{ fromJson(needs.parse-workflows.outputs.pr_rocm720_jobs) }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.10'
- name: Install dependencies
run: pip install tabulate
- name: Download Actions data snapshot
uses: actions/download-artifact@v4
with:
name: actions-job-snapshot
path: ci-data
- name: Generate PR ROCm 7.2 Report
timeout-minutes: 15
run: |
python scripts/ci/utils/query_job_status.py \
--repo ${{ github.repository }} \
--job "${{ matrix.job_name }}" \
--workflow "pr-test-amd-rocm720.yml" \
--hours ${{ inputs.hours || '24' }} \
--input-data-file ci-data/actions-job-snapshot.json \
--summary
# Nightly ROCm 7.2 reports using dynamic matrix
nightly-rocm720-reports:
name: Nightly ROCm720 - ${{ matrix.job_name }}
needs: [parse-workflows, fetch-actions-data]
if: ${{ !inputs.job_filter }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
job_name: ${{ fromJson(needs.parse-workflows.outputs.nightly_rocm720_jobs) }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.10'
- name: Install dependencies
run: pip install tabulate
- name: Download Actions data snapshot
uses: actions/download-artifact@v4
with:
name: actions-job-snapshot
path: ci-data
- name: Generate Nightly ROCm 7.2 Report
timeout-minutes: 15
run: |
python scripts/ci/utils/query_job_status.py \
@@ -326,6 +248,84 @@ jobs:
--input-data-file ci-data/actions-job-snapshot.json \
--summary
# ROCm 7.0 rollback-shadow reports using dynamic matrix
pr-rocm700-ci-reports:
name: PR ROCm700 Shadow - ${{ matrix.job_name }}
needs: [parse-workflows, fetch-actions-data]
if: ${{ !inputs.job_filter }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
job_name: ${{ fromJson(needs.parse-workflows.outputs.pr_rocm700_jobs) }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.10'
- name: Install dependencies
run: pip install tabulate
- name: Download Actions data snapshot
uses: actions/download-artifact@v4
with:
name: actions-job-snapshot
path: ci-data
- name: Generate PR ROCm 7.0 Shadow Report
timeout-minutes: 15
run: |
python scripts/ci/utils/query_job_status.py \
--repo ${{ github.repository }} \
--job "${{ matrix.job_name }}" \
--workflow "pr-test-amd.yml" \
--hours ${{ inputs.hours || '24' }} \
--input-data-file ci-data/actions-job-snapshot.json \
--summary
# ROCm 7.0 rollback-shadow nightly reports using dynamic matrix
nightly-rocm700-reports:
name: Nightly ROCm700 Shadow - ${{ matrix.job_name }}
needs: [parse-workflows, fetch-actions-data]
if: ${{ !inputs.job_filter }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
job_name: ${{ fromJson(needs.parse-workflows.outputs.nightly_rocm700_jobs) }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.10'
- name: Install dependencies
run: pip install tabulate
- name: Download Actions data snapshot
uses: actions/download-artifact@v4
with:
name: actions-job-snapshot
path: ci-data
- name: Generate Nightly ROCm 7.0 Shadow Report
timeout-minutes: 15
run: |
python scripts/ci/utils/query_job_status.py \
--repo ${{ github.repository }} \
--job "${{ matrix.job_name }}" \
--workflow "nightly-test-amd.yml" \
--hours ${{ inputs.hours || '24' }} \
--input-data-file ci-data/actions-job-snapshot.json \
--summary
# MI355X 2N 1P1D disagg nightly reports using dynamic matrix
nightly-disagg-reports:
name: Nightly MI355X Disagg - ${{ matrix.job_name }}
.github/workflows/bot-bump-sglang-version.yml
+1 -1
View File
@@ -62,7 +62,7 @@ jobs:
run-nightly-tests-amd:
needs: bump-sglang-version
uses: ./.github/workflows/nightly-test-amd.yml
uses: ./.github/workflows/nightly-test-amd-rocm720.yml
with:
ref: ${{ needs.bump-sglang-version.outputs.branch_name }}
secrets: inherit
.github/workflows/nightly-test-amd.yml
+1 -1
View File
@@ -1,4 +1,4 @@
name: Nightly Test (AMD)
name: Nightly Test (AMD ROCm 7.0)
on:
schedule:
.github/workflows/pr-test-amd-extra.yml
+13 -12
View File
@@ -37,13 +37,13 @@ on:
- mi300
- mi325
rocm_version:
description: 'ROCm container variant (empty = Dockerfile default; rocm720 = ROCm 7.2.0)'
description: 'ROCm container variant (rocm720 = ROCm 7.2.0, the PR default; rocm700 = ROCm 7.0.0)'
required: false
type: choice
default: ''
default: rocm720
options:
- ''
- rocm720
- rocm700
aiter_ref:
description: 'Override AITER commit (optional, leave empty to use Dockerfile default)'
required: false
@@ -67,10 +67,10 @@ on:
type: string
default: mi300
rocm_version:
description: 'ROCm container variant (empty = Dockerfile default; rocm720 = ROCm 7.2.0)'
description: 'ROCm container variant (rocm720 = ROCm 7.2.0, the PR default; rocm700 = ROCm 7.0.0)'
required: false
type: string
default: ''
default: rocm720
aiter_ref:
description: 'Override AITER commit (optional, leave empty to use Dockerfile default)'
required: false
@@ -130,7 +130,7 @@ jobs:
# setup across scarce AMD GPUs to shave only a couple minutes of test time,
# so one GPU running the whole suite sequentially is the better trade.
extra-a-test-1-gpu-small-amd:
name: ${{ format('extra-a-test-1-gpu-small-amd{0} (linux-{1}-1gpu-sglang)', inputs.rocm_version && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi300') }}
name: ${{ format('extra-a-test-1-gpu-small-amd{0} (linux-{1}-1gpu-sglang)', inputs.rocm_version && inputs.rocm_version != 'rocm720' && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi300') }}
needs: [call-gate]
if: ${{ !cancelled() && needs.call-gate.result == 'success' }}
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
@@ -144,8 +144,9 @@ jobs:
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Start CI container
# `rocm_version` (e.g. rocm720) selects an alternate ROCm container; empty uses the Dockerfile default.
run: bash scripts/ci/amd/amd_ci_start_container.sh ${{ inputs.rocm_version && format('--rocm-version {0}', inputs.rocm_version) || '' }}
# `inputs` is empty on pull_request events, so fall back explicitly to
# the ROCm 7.2 PR default. Reusable callers can still request ROCm 7.0.
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ inputs.rocm_version || 'rocm720' }}
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
@@ -162,7 +163,7 @@ jobs:
# pool as small (AMD GPUs carry enough VRAM that "large" here is a CUDA
# memory-tier label, not a separate AMD runner pool).
extra-a-test-1-gpu-large-amd:
name: ${{ format('extra-a-test-1-gpu-large-amd{0} (linux-{1}-1gpu-sglang)', inputs.rocm_version && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi300') }}
name: ${{ format('extra-a-test-1-gpu-large-amd{0} (linux-{1}-1gpu-sglang)', inputs.rocm_version && inputs.rocm_version != 'rocm720' && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi300') }}
needs: [call-gate]
if: ${{ !cancelled() && needs.call-gate.result == 'success' }}
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
@@ -176,7 +177,7 @@ jobs:
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Start CI container
run: bash scripts/ci/amd/amd_ci_start_container.sh ${{ inputs.rocm_version && format('--rocm-version {0}', inputs.rocm_version) || '' }}
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ inputs.rocm_version || 'rocm720' }}
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
@@ -192,7 +193,7 @@ jobs:
# Multi-GPU TP / PP / PD mock-model + kv_canary e2e tests. Mirrors CUDA's
# extra-a 2-gpu-large stage; runs on the 2-GPU AMD pool.
extra-a-test-2-gpu-large-amd:
name: ${{ format('extra-a-test-2-gpu-large-amd{0} (linux-{1}-2gpu-sglang)', inputs.rocm_version && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi300') }}
name: ${{ format('extra-a-test-2-gpu-large-amd{0} (linux-{1}-2gpu-sglang)', inputs.rocm_version && inputs.rocm_version != 'rocm720' && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi300') }}
needs: [call-gate]
if: ${{ !cancelled() && needs.call-gate.result == 'success' }}
runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }}
@@ -206,7 +207,7 @@ jobs:
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Start CI container
run: bash scripts/ci/amd/amd_ci_start_container.sh ${{ inputs.rocm_version && format('--rocm-version {0}', inputs.rocm_version) || '' }}
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ inputs.rocm_version || 'rocm720' }}
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
.github/workflows/pr-test-amd-rocm720.yml
+354 -289
View File
File diff suppressed because it is too large Load Diff
.github/workflows/pr-test-amd.yml
+8 -16
View File
@@ -1,19 +1,16 @@
name: PR Test (AMD)
name: PR Test ROCm 7.0 (AMD)
# Dynamic run-name for /rerun-stage commands to enable URL lookup
# Format: "[stage-name] sha" for fork PRs, "[stage-name]" for non-fork, default for normal runs
run-name: ${{ (inputs.target_stage || inputs.target_stage_select) && (inputs.pr_head_sha && format('[{0}] {1}', inputs.target_stage || inputs.target_stage_select, inputs.pr_head_sha) || format('[{0}]', inputs.target_stage || inputs.target_stage_select)) || '' }}
on:
schedule:
- cron: '0 */12 * * *' # Run every 12 hours (UTC)
pull_request:
paths:
- "python/**"
- "scripts/ci/**"
- "test/**"
- "python/sglang/kernels/aot/**"
- ".github/workflows/pr-test-amd.yml"
- "docker/rocm.Dockerfile"
# ROCm 7.0 rollback shadow. pr-test-amd-rocm720.yml is the PR gate; this
# workflow now only proves ROCm 7.0 still works, so it fires once a day
# alongside nightly-test-amd.yml and chases coverage rather than latency:
# no matrix parallelism caps, no stage fast-fail (the `schedule` escape in
# each job's `if`), and continue-on-error via check-changes.
- cron: '30 17 * * *'
workflow_dispatch:
inputs:
target_stage_select:
@@ -223,6 +220,7 @@ jobs:
with:
ref: ${{ inputs.pr_head_sha || inputs.ref || '' }}
runner_arch: ${{ inputs.runner_arch || 'mi300' }}
rocm_version: rocm700
aiter_ref: ${{ inputs.aiter_ref }}
continue_on_error: true
secrets: inherit
@@ -479,7 +477,6 @@ jobs:
)
strategy:
fail-fast: false
max-parallel: 4
matrix:
part: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
@@ -595,7 +592,6 @@ jobs:
)
strategy:
fail-fast: false
max-parallel: 1
matrix:
part: [0, 1, 2]
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
@@ -636,7 +632,6 @@ jobs:
)
strategy:
fail-fast: false
max-parallel: 1
matrix:
part: [0, 1]
runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }}
@@ -677,7 +672,6 @@ jobs:
)
strategy:
fail-fast: false
max-parallel: 1
matrix:
part: [0, 1, 2, 3]
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
@@ -808,7 +802,6 @@ jobs:
)
strategy:
fail-fast: false
max-parallel: 1
matrix:
part: [0, 1, 2] # 3 partitions: 2 parametrized + 1 standalone (single_test_file/test_disagg_server.py)
runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }}
@@ -1082,7 +1075,6 @@ jobs:
RUNNER_LABELS: ${{ format('linux-{0}-8gpu-sglang', inputs.runner_arch || 'mi300') }}
strategy:
fail-fast: false
max-parallel: 2
matrix:
part: [0, 1, 2, 3]
runs-on: ${{ format('linux-{0}-8gpu-sglang', inputs.runner_arch || 'mi300') }}
.github/workflows/release-branch-cut.yml
+2 -2
View File
@@ -174,7 +174,7 @@ jobs:
run-pr-tests-amd:
needs: cut-release-branch
uses: ./.github/workflows/pr-test-amd.yml
uses: ./.github/workflows/pr-test-amd-rocm720.yml
with:
ref: ${{ needs.cut-release-branch.outputs.branch_name }}
run_all_tests: true
@@ -213,7 +213,7 @@ jobs:
run-nightly-tests-amd:
needs: cut-release-branch
uses: ./.github/workflows/nightly-test-amd.yml
uses: ./.github/workflows/nightly-test-amd-rocm720.yml
with:
ref: ${{ needs.cut-release-branch.outputs.branch_name }}
secrets: inherit