[AMD][CI] Make ROCm 10 the Default for AMD PR and Nightly Tests (#38659)
Co-authored-by: Chen Bingxu <bingxche@amd.com>
This commit is contained in:
co-authored by
Chen Bingxu
parent
72d5c5bb73
commit
708f51e44b
@@ -12,7 +12,7 @@ on:
|
||||
type: string
|
||||
default: 'main'
|
||||
job_filter:
|
||||
description: 'Workflows: nightly-amd, nightly-amd-rocm720, pr-test-amd (ROCm 7.0 shadow), pr-test-amd-rocm720 (ROCm 7.2 gate). Default: all'
|
||||
description: 'Workflows: nightly-amd, nightly-amd-rocm720, pr-test-amd (ROCm 7.0 shadow), pr-test-amd-rocm720 (ROCm 10 gate). Default: all'
|
||||
required: false
|
||||
type: string
|
||||
default: 'all'
|
||||
@@ -106,7 +106,7 @@ jobs:
|
||||
secrets: inherit
|
||||
with:
|
||||
ref: amd/aiter-ci
|
||||
rocm_version: rocm724
|
||||
rocm_version: rocm10
|
||||
aiter_ref: ${{ needs.resolve-aiter.outputs.aiter_sha }}
|
||||
job_filter: 'all'
|
||||
continue_on_error: ${{ inputs.continue_on_error == '' && true || inputs.continue_on_error }}
|
||||
@@ -129,7 +129,7 @@ jobs:
|
||||
secrets: inherit
|
||||
with:
|
||||
ref: amd/aiter-ci
|
||||
rocm_version: rocm724
|
||||
rocm_version: rocm10
|
||||
run_all_tests: true
|
||||
aiter_ref: ${{ needs.resolve-aiter.outputs.aiter_sha }}
|
||||
continue_on_error: ${{ inputs.continue_on_error == '' && true || inputs.continue_on_error }}
|
||||
@@ -155,9 +155,9 @@ jobs:
|
||||
echo "| Workflow | Result |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "|----------|--------|" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| Nightly AMD (AITER Latest) | \`${{ needs.call-nightly-amd.result }}\` |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| Nightly AMD ROCm 7.2 | \`${{ needs.call-nightly-amd-rocm720.result }}\` |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| Nightly AMD ROCm 10 | \`${{ needs.call-nightly-amd-rocm720.result }}\` |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| PR Test AMD ROCm 7.0 Shadow | \`${{ needs.call-pr-test-amd.result }}\` |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| PR Test AMD ROCm 7.2 Gate | \`${{ needs.call-pr-test-amd-rocm720.result }}\` |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| PR Test AMD ROCm 10 Gate | \`${{ needs.call-pr-test-amd-rocm720.result }}\` |" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
- name: Check if any job failed
|
||||
run: |
|
||||
|
||||
@@ -65,7 +65,7 @@ jobs:
|
||||
uses: ./.github/workflows/nightly-test-amd-rocm720.yml
|
||||
with:
|
||||
ref: ${{ needs.bump-sglang-version.outputs.branch_name }}
|
||||
rocm_version: rocm724
|
||||
rocm_version: rocm10
|
||||
secrets: inherit
|
||||
|
||||
run-nightly-tests-npu:
|
||||
|
||||
@@ -116,7 +116,7 @@ on:
|
||||
type: string
|
||||
# A single flavor, unlike the schedule and the dispatch form: a caller
|
||||
# that says nothing should not silently get multi-version GPU cost.
|
||||
default: rocm724
|
||||
default: rocm10
|
||||
ref:
|
||||
description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
|
||||
required: false
|
||||
|
||||
@@ -224,7 +224,7 @@ jobs:
|
||||
'',
|
||||
`Latest PR Test (Base): ${ptStart}${ptText}${ptEnd}`,
|
||||
`Latest PR Test (Extra): ${peStart}${peText}${peEnd}`,
|
||||
`Latest PR Test (AMD ROCm 7.2): ${amdStart}${amdText}${amdEnd}`,
|
||||
`Latest PR Test (AMD ROCm 10): ${amdStart}${amdText}${amdEnd}`,
|
||||
outerEnd,
|
||||
].join('\n');
|
||||
|
||||
|
||||
@@ -37,10 +37,10 @@ on:
|
||||
- mi300
|
||||
- mi325
|
||||
rocm_version:
|
||||
description: 'ROCm container variant (ROCm 7.2.4 by default)'
|
||||
description: 'ROCm container variant (ROCm 10 by default)'
|
||||
required: false
|
||||
type: choice
|
||||
default: rocm724
|
||||
default: rocm10
|
||||
options:
|
||||
- rocm10
|
||||
- rocm724
|
||||
@@ -74,10 +74,10 @@ on:
|
||||
type: string
|
||||
default: mi300
|
||||
rocm_version:
|
||||
description: 'ROCm container variant (ROCm 7.2.4 by default)'
|
||||
description: 'ROCm container variant (ROCm 10 by default)'
|
||||
required: false
|
||||
type: string
|
||||
default: rocm724
|
||||
default: rocm10
|
||||
aiter_ref:
|
||||
description: 'Override AITER commit (optional, leave empty to use Dockerfile default)'
|
||||
required: false
|
||||
@@ -105,7 +105,7 @@ concurrency:
|
||||
# scheduled reusable-workflow matrix calls cannot replace each other's
|
||||
# pending runs. PR runs still share a per-branch group so pushes cancel stale
|
||||
# runs. In a reusable workflow github.event_name is the originating event.
|
||||
group: pr-test-amd-extra-${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && format('full-{0}-{1}', github.run_id, inputs.rocm_version || 'rocm724') || github.head_ref || github.ref_name || inputs.ref || 'default' }}
|
||||
group: pr-test-amd-extra-${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && format('full-{0}-{1}', github.run_id, inputs.rocm_version || 'rocm10') || github.head_ref || github.ref_name || inputs.ref || 'default' }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
permissions:
|
||||
@@ -143,7 +143,7 @@ jobs:
|
||||
# setup across scarce AMD GPUs to shave only a couple minutes of test time,
|
||||
# so one GPU running the whole suite sequentially is the better trade.
|
||||
extra-a-test-1-gpu-small-amd:
|
||||
name: ${{ format('extra-a-test-1-gpu-small-amd ({0}, linux-{1}-1gpu-sglang)', inputs.rocm_version || 'rocm724', inputs.runner_arch || 'mi300') }}
|
||||
name: ${{ format('extra-a-test-1-gpu-small-amd ({0}, linux-{1}-1gpu-sglang)', inputs.rocm_version || 'rocm10', inputs.runner_arch || 'mi300') }}
|
||||
needs: [call-gate]
|
||||
if: ${{ !cancelled() && needs.call-gate.result == 'success' }}
|
||||
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
|
||||
@@ -158,8 +158,8 @@ jobs:
|
||||
|
||||
- name: Start CI container
|
||||
# `inputs` is empty on pull_request events, so fall back explicitly to
|
||||
# the ROCm 7.2 PR default. Reusable callers can request another variant.
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ inputs.rocm_version || 'rocm724' }}
|
||||
# the ROCm 10 PR default. Reusable callers can request another variant.
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ inputs.rocm_version || 'rocm10' }}
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
@@ -176,7 +176,7 @@ jobs:
|
||||
# pool as small (AMD GPUs carry enough VRAM that "large" here is a CUDA
|
||||
# memory-tier label, not a separate AMD runner pool).
|
||||
extra-a-test-1-gpu-large-amd:
|
||||
name: ${{ format('extra-a-test-1-gpu-large-amd ({0}, linux-{1}-1gpu-sglang)', inputs.rocm_version || 'rocm724', inputs.runner_arch || 'mi300') }}
|
||||
name: ${{ format('extra-a-test-1-gpu-large-amd ({0}, linux-{1}-1gpu-sglang)', inputs.rocm_version || 'rocm10', inputs.runner_arch || 'mi300') }}
|
||||
needs: [call-gate]
|
||||
if: ${{ !cancelled() && needs.call-gate.result == 'success' }}
|
||||
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
|
||||
@@ -190,7 +190,7 @@ jobs:
|
||||
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
|
||||
|
||||
- name: Start CI container
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ inputs.rocm_version || 'rocm724' }}
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ inputs.rocm_version || 'rocm10' }}
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
@@ -206,7 +206,7 @@ jobs:
|
||||
# Multi-GPU TP / PP / PD mock-model + kv_canary e2e tests. Mirrors CUDA's
|
||||
# extra-a 2-gpu-large stage; runs on the 2-GPU AMD pool.
|
||||
extra-a-test-2-gpu-large-amd:
|
||||
name: ${{ format('extra-a-test-2-gpu-large-amd ({0}, linux-{1}-2gpu-sglang)', inputs.rocm_version || 'rocm724', inputs.runner_arch || 'mi300') }}
|
||||
name: ${{ format('extra-a-test-2-gpu-large-amd ({0}, linux-{1}-2gpu-sglang)', inputs.rocm_version || 'rocm10', inputs.runner_arch || 'mi300') }}
|
||||
needs: [call-gate]
|
||||
if: ${{ !cancelled() && needs.call-gate.result == 'success' }}
|
||||
runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }}
|
||||
@@ -220,7 +220,7 @@ jobs:
|
||||
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
|
||||
|
||||
- name: Start CI container
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ inputs.rocm_version || 'rocm724' }}
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ inputs.rocm_version || 'rocm10' }}
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
|
||||
@@ -53,7 +53,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Start disaggregation container
|
||||
run: bash scripts/ci/amd/amd_ci_start_container_disagg.sh --rocm-version rocm724
|
||||
run: bash scripts/ci/amd/amd_ci_start_container_disagg.sh --rocm-version rocm10
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
@@ -84,7 +84,7 @@ jobs:
|
||||
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
|
||||
|
||||
- name: Start CI container
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm724
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm10
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
ENABLE_CACHE_HOST: "1"
|
||||
|
||||
@@ -5,8 +5,8 @@ run-name: ${{ (inputs.target_stage || inputs.target_stage_select) && (inputs.pr_
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 */12 * * *' # rocm724: twice daily (UTC)
|
||||
- cron: '30 17 * * *' # rocm10/rocm720: once daily, aligned with ROCm 7.0
|
||||
- cron: '0 */12 * * *' # rocm10: twice daily (UTC)
|
||||
- cron: '30 17 * * *' # rocm724/rocm720: once daily, aligned with ROCm 7.0
|
||||
pull_request:
|
||||
paths:
|
||||
- "python/**"
|
||||
@@ -21,7 +21,7 @@ on:
|
||||
description: 'ROCm image version'
|
||||
required: false
|
||||
type: choice
|
||||
default: rocm724
|
||||
default: rocm10
|
||||
options:
|
||||
- rocm10
|
||||
- rocm724
|
||||
@@ -95,7 +95,7 @@ on:
|
||||
description: 'ROCm image version'
|
||||
required: false
|
||||
type: string
|
||||
default: rocm724
|
||||
default: rocm10
|
||||
ref:
|
||||
description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
|
||||
required: false
|
||||
@@ -165,11 +165,11 @@ jobs:
|
||||
jit_kernel: ${{ steps.filter.outputs.jit_kernel || steps.run-mode.outputs.run_all_tests }}
|
||||
multimodal_gen: ${{ steps.filter.outputs.multimodal_gen || steps.run-mode.outputs.run_all_tests }}
|
||||
continue_on_error: ${{ steps.set-continue-on-error.outputs.continue_on_error }}
|
||||
# Keep rocm724 on its existing twice-daily cadence. The 17:30 UTC
|
||||
# schedule runs rocm10 and rocm720 once daily alongside the ROCm 7.0
|
||||
# Keep rocm10 on the twice-daily cadence. The 17:30 UTC schedule runs
|
||||
# rocm724 and rocm720 once daily alongside the ROCm 7.0
|
||||
# shadow workflow. Dispatch/call inputs select one version, while a
|
||||
# pull_request falls through to the single rocm724 default.
|
||||
rocm_versions: ${{ inputs.rocm_version && format('["{0}"]', inputs.rocm_version) || github.event.schedule == '30 17 * * *' && '["rocm10","rocm720"]' || '["rocm724"]' }}
|
||||
# pull_request falls through to the single rocm10 default.
|
||||
rocm_versions: ${{ inputs.rocm_version && format('["{0}"]', inputs.rocm_version) || github.event.schedule == '30 17 * * *' && '["rocm724","rocm720"]' || '["rocm10"]' }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
@@ -177,7 +177,7 @@ jobs:
|
||||
uses: ./.github/workflows/pr-test-amd-rocm720.yml
|
||||
with:
|
||||
ref: ${{ needs.cut-release-branch.outputs.branch_name }}
|
||||
rocm_version: rocm724
|
||||
rocm_version: rocm10
|
||||
run_all_tests: true
|
||||
secrets: inherit
|
||||
|
||||
@@ -217,7 +217,7 @@ jobs:
|
||||
uses: ./.github/workflows/nightly-test-amd-rocm720.yml
|
||||
with:
|
||||
ref: ${{ needs.cut-release-branch.outputs.branch_name }}
|
||||
rocm_version: rocm724
|
||||
rocm_version: rocm10
|
||||
secrets: inherit
|
||||
|
||||
run-nightly-tests-npu:
|
||||
|
||||
Reference in New Issue
Block a user