[AMD][CI] Make ROCm 10 the Default for AMD PR and Nightly Tests (#38659)

Co-authored-by: Chen Bingxu <bingxche@amd.com>
This commit is contained in:
YC Yen-Ching Tseng
2026-09-09 17:28:20 +08:00
committed by GitHub
co-authored by Chen Bingxu
parent 72d5c5bb73
commit 708f51e44b
8 changed files with 32 additions and 32 deletions
+5 -5
View File
@@ -12,7 +12,7 @@ on:
type: string
default: 'main'
job_filter:
description: 'Workflows: nightly-amd, nightly-amd-rocm720, pr-test-amd (ROCm 7.0 shadow), pr-test-amd-rocm720 (ROCm 7.2 gate). Default: all'
description: 'Workflows: nightly-amd, nightly-amd-rocm720, pr-test-amd (ROCm 7.0 shadow), pr-test-amd-rocm720 (ROCm 10 gate). Default: all'
required: false
type: string
default: 'all'
@@ -106,7 +106,7 @@ jobs:
secrets: inherit
with:
ref: amd/aiter-ci
rocm_version: rocm724
rocm_version: rocm10
aiter_ref: ${{ needs.resolve-aiter.outputs.aiter_sha }}
job_filter: 'all'
continue_on_error: ${{ inputs.continue_on_error == '' && true || inputs.continue_on_error }}
@@ -129,7 +129,7 @@ jobs:
secrets: inherit
with:
ref: amd/aiter-ci
rocm_version: rocm724
rocm_version: rocm10
run_all_tests: true
aiter_ref: ${{ needs.resolve-aiter.outputs.aiter_sha }}
continue_on_error: ${{ inputs.continue_on_error == '' && true || inputs.continue_on_error }}
@@ -155,9 +155,9 @@ jobs:
echo "| Workflow | Result |" >> $GITHUB_STEP_SUMMARY
echo "|----------|--------|" >> $GITHUB_STEP_SUMMARY
echo "| Nightly AMD (AITER Latest) | \`${{ needs.call-nightly-amd.result }}\` |" >> $GITHUB_STEP_SUMMARY
echo "| Nightly AMD ROCm 7.2 | \`${{ needs.call-nightly-amd-rocm720.result }}\` |" >> $GITHUB_STEP_SUMMARY
echo "| Nightly AMD ROCm 10 | \`${{ needs.call-nightly-amd-rocm720.result }}\` |" >> $GITHUB_STEP_SUMMARY
echo "| PR Test AMD ROCm 7.0 Shadow | \`${{ needs.call-pr-test-amd.result }}\` |" >> $GITHUB_STEP_SUMMARY
echo "| PR Test AMD ROCm 7.2 Gate | \`${{ needs.call-pr-test-amd-rocm720.result }}\` |" >> $GITHUB_STEP_SUMMARY
echo "| PR Test AMD ROCm 10 Gate | \`${{ needs.call-pr-test-amd-rocm720.result }}\` |" >> $GITHUB_STEP_SUMMARY
- name: Check if any job failed
run: |
@@ -65,7 +65,7 @@ jobs:
uses: ./.github/workflows/nightly-test-amd-rocm720.yml
with:
ref: ${{ needs.bump-sglang-version.outputs.branch_name }}
rocm_version: rocm724
rocm_version: rocm10
secrets: inherit
run-nightly-tests-npu:
@@ -116,7 +116,7 @@ on:
type: string
# A single flavor, unlike the schedule and the dispatch form: a caller
# that says nothing should not silently get multi-version GPU cost.
default: rocm724
default: rocm10
ref:
description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
required: false
+1 -1
View File
@@ -224,7 +224,7 @@ jobs:
'',
`Latest PR Test (Base): ${ptStart}${ptText}${ptEnd}`,
`Latest PR Test (Extra): ${peStart}${peText}${peEnd}`,
`Latest PR Test (AMD ROCm 7.2): ${amdStart}${amdText}${amdEnd}`,
`Latest PR Test (AMD ROCm 10): ${amdStart}${amdText}${amdEnd}`,
outerEnd,
].join('\n');
+12 -12
View File
@@ -37,10 +37,10 @@ on:
- mi300
- mi325
rocm_version:
description: 'ROCm container variant (ROCm 7.2.4 by default)'
description: 'ROCm container variant (ROCm 10 by default)'
required: false
type: choice
default: rocm724
default: rocm10
options:
- rocm10
- rocm724
@@ -74,10 +74,10 @@ on:
type: string
default: mi300
rocm_version:
description: 'ROCm container variant (ROCm 7.2.4 by default)'
description: 'ROCm container variant (ROCm 10 by default)'
required: false
type: string
default: rocm724
default: rocm10
aiter_ref:
description: 'Override AITER commit (optional, leave empty to use Dockerfile default)'
required: false
@@ -105,7 +105,7 @@ concurrency:
# scheduled reusable-workflow matrix calls cannot replace each other's
# pending runs. PR runs still share a per-branch group so pushes cancel stale
# runs. In a reusable workflow github.event_name is the originating event.
group: pr-test-amd-extra-${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && format('full-{0}-{1}', github.run_id, inputs.rocm_version || 'rocm724') || github.head_ref || github.ref_name || inputs.ref || 'default' }}
group: pr-test-amd-extra-${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && format('full-{0}-{1}', github.run_id, inputs.rocm_version || 'rocm10') || github.head_ref || github.ref_name || inputs.ref || 'default' }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
permissions:
@@ -143,7 +143,7 @@ jobs:
# setup across scarce AMD GPUs to shave only a couple minutes of test time,
# so one GPU running the whole suite sequentially is the better trade.
extra-a-test-1-gpu-small-amd:
name: ${{ format('extra-a-test-1-gpu-small-amd ({0}, linux-{1}-1gpu-sglang)', inputs.rocm_version || 'rocm724', inputs.runner_arch || 'mi300') }}
name: ${{ format('extra-a-test-1-gpu-small-amd ({0}, linux-{1}-1gpu-sglang)', inputs.rocm_version || 'rocm10', inputs.runner_arch || 'mi300') }}
needs: [call-gate]
if: ${{ !cancelled() && needs.call-gate.result == 'success' }}
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
@@ -158,8 +158,8 @@ jobs:
- name: Start CI container
# `inputs` is empty on pull_request events, so fall back explicitly to
# the ROCm 7.2 PR default. Reusable callers can request another variant.
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ inputs.rocm_version || 'rocm724' }}
# the ROCm 10 PR default. Reusable callers can request another variant.
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ inputs.rocm_version || 'rocm10' }}
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
@@ -176,7 +176,7 @@ jobs:
# pool as small (AMD GPUs carry enough VRAM that "large" here is a CUDA
# memory-tier label, not a separate AMD runner pool).
extra-a-test-1-gpu-large-amd:
name: ${{ format('extra-a-test-1-gpu-large-amd ({0}, linux-{1}-1gpu-sglang)', inputs.rocm_version || 'rocm724', inputs.runner_arch || 'mi300') }}
name: ${{ format('extra-a-test-1-gpu-large-amd ({0}, linux-{1}-1gpu-sglang)', inputs.rocm_version || 'rocm10', inputs.runner_arch || 'mi300') }}
needs: [call-gate]
if: ${{ !cancelled() && needs.call-gate.result == 'success' }}
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
@@ -190,7 +190,7 @@ jobs:
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Start CI container
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ inputs.rocm_version || 'rocm724' }}
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ inputs.rocm_version || 'rocm10' }}
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
@@ -206,7 +206,7 @@ jobs:
# Multi-GPU TP / PP / PD mock-model + kv_canary e2e tests. Mirrors CUDA's
# extra-a 2-gpu-large stage; runs on the 2-GPU AMD pool.
extra-a-test-2-gpu-large-amd:
name: ${{ format('extra-a-test-2-gpu-large-amd ({0}, linux-{1}-2gpu-sglang)', inputs.rocm_version || 'rocm724', inputs.runner_arch || 'mi300') }}
name: ${{ format('extra-a-test-2-gpu-large-amd ({0}, linux-{1}-2gpu-sglang)', inputs.rocm_version || 'rocm10', inputs.runner_arch || 'mi300') }}
needs: [call-gate]
if: ${{ !cancelled() && needs.call-gate.result == 'success' }}
runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }}
@@ -220,7 +220,7 @@ jobs:
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Start CI container
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ inputs.rocm_version || 'rocm724' }}
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ inputs.rocm_version || 'rocm10' }}
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
+2 -2
View File
@@ -53,7 +53,7 @@ jobs:
fi
- name: Start disaggregation container
run: bash scripts/ci/amd/amd_ci_start_container_disagg.sh --rocm-version rocm724
run: bash scripts/ci/amd/amd_ci_start_container_disagg.sh --rocm-version rocm10
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
@@ -84,7 +84,7 @@ jobs:
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Start CI container
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm724
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm10
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
ENABLE_CACHE_HOST: "1"
+8 -8
View File
@@ -5,8 +5,8 @@ run-name: ${{ (inputs.target_stage || inputs.target_stage_select) && (inputs.pr_
on:
schedule:
- cron: '0 */12 * * *' # rocm724: twice daily (UTC)
- cron: '30 17 * * *' # rocm10/rocm720: once daily, aligned with ROCm 7.0
- cron: '0 */12 * * *' # rocm10: twice daily (UTC)
- cron: '30 17 * * *' # rocm724/rocm720: once daily, aligned with ROCm 7.0
pull_request:
paths:
- "python/**"
@@ -21,7 +21,7 @@ on:
description: 'ROCm image version'
required: false
type: choice
default: rocm724
default: rocm10
options:
- rocm10
- rocm724
@@ -95,7 +95,7 @@ on:
description: 'ROCm image version'
required: false
type: string
default: rocm724
default: rocm10
ref:
description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
required: false
@@ -165,11 +165,11 @@ jobs:
jit_kernel: ${{ steps.filter.outputs.jit_kernel || steps.run-mode.outputs.run_all_tests }}
multimodal_gen: ${{ steps.filter.outputs.multimodal_gen || steps.run-mode.outputs.run_all_tests }}
continue_on_error: ${{ steps.set-continue-on-error.outputs.continue_on_error }}
# Keep rocm724 on its existing twice-daily cadence. The 17:30 UTC
# schedule runs rocm10 and rocm720 once daily alongside the ROCm 7.0
# Keep rocm10 on the twice-daily cadence. The 17:30 UTC schedule runs
# rocm724 and rocm720 once daily alongside the ROCm 7.0
# shadow workflow. Dispatch/call inputs select one version, while a
# pull_request falls through to the single rocm724 default.
rocm_versions: ${{ inputs.rocm_version && format('["{0}"]', inputs.rocm_version) || github.event.schedule == '30 17 * * *' && '["rocm10","rocm720"]' || '["rocm724"]' }}
# pull_request falls through to the single rocm10 default.
rocm_versions: ${{ inputs.rocm_version && format('["{0}"]', inputs.rocm_version) || github.event.schedule == '30 17 * * *' && '["rocm724","rocm720"]' || '["rocm10"]' }}
steps:
- name: Checkout code
uses: actions/checkout@v4
+2 -2
View File
@@ -177,7 +177,7 @@ jobs:
uses: ./.github/workflows/pr-test-amd-rocm720.yml
with:
ref: ${{ needs.cut-release-branch.outputs.branch_name }}
rocm_version: rocm724
rocm_version: rocm10
run_all_tests: true
secrets: inherit
@@ -217,7 +217,7 @@ jobs:
uses: ./.github/workflows/nightly-test-amd-rocm720.yml
with:
ref: ${{ needs.cut-release-branch.outputs.branch_name }}
rocm_version: rocm724
rocm_version: rocm10
secrets: inherit
run-nightly-tests-npu: