From 708f51e44bc64f546a60fa9631f0e7d99493d0a0 Mon Sep 17 00:00:00 2001 From: YC Yen-Ching Tseng Date: Wed, 9 Sep 2026 11:28:20 +0200 Subject: [PATCH] [AMD][CI] Make ROCm 10 the Default for AMD PR and Nightly Tests (#38659) Co-authored-by: Chen Bingxu --- .github/workflows/amd-aiter-scout.yml | 10 ++++---- .github/workflows/bot-bump-sglang-version.yml | 2 +- .../workflows/nightly-test-amd-rocm720.yml | 2 +- .github/workflows/pr-states.yml | 2 +- .github/workflows/pr-test-amd-extra.yml | 24 +++++++++---------- .github/workflows/pr-test-amd-mori.yml | 4 ++-- .github/workflows/pr-test-amd-rocm720.yml | 16 ++++++------- .github/workflows/release-branch-cut.yml | 4 ++-- 8 files changed, 32 insertions(+), 32 deletions(-) diff --git a/.github/workflows/amd-aiter-scout.yml b/.github/workflows/amd-aiter-scout.yml index 8365856ad..8c680213a 100644 --- a/.github/workflows/amd-aiter-scout.yml +++ b/.github/workflows/amd-aiter-scout.yml @@ -12,7 +12,7 @@ on: type: string default: 'main' job_filter: - description: 'Workflows: nightly-amd, nightly-amd-rocm720, pr-test-amd (ROCm 7.0 shadow), pr-test-amd-rocm720 (ROCm 7.2 gate). Default: all' + description: 'Workflows: nightly-amd, nightly-amd-rocm720, pr-test-amd (ROCm 7.0 shadow), pr-test-amd-rocm720 (ROCm 10 gate). Default: all' required: false type: string default: 'all' @@ -106,7 +106,7 @@ jobs: secrets: inherit with: ref: amd/aiter-ci - rocm_version: rocm724 + rocm_version: rocm10 aiter_ref: ${{ needs.resolve-aiter.outputs.aiter_sha }} job_filter: 'all' continue_on_error: ${{ inputs.continue_on_error == '' && true || inputs.continue_on_error }} @@ -129,7 +129,7 @@ jobs: secrets: inherit with: ref: amd/aiter-ci - rocm_version: rocm724 + rocm_version: rocm10 run_all_tests: true aiter_ref: ${{ needs.resolve-aiter.outputs.aiter_sha }} continue_on_error: ${{ inputs.continue_on_error == '' && true || inputs.continue_on_error }} @@ -155,9 +155,9 @@ jobs: echo "| Workflow | Result |" >> $GITHUB_STEP_SUMMARY echo "|----------|--------|" >> $GITHUB_STEP_SUMMARY echo "| Nightly AMD (AITER Latest) | \`${{ needs.call-nightly-amd.result }}\` |" >> $GITHUB_STEP_SUMMARY - echo "| Nightly AMD ROCm 7.2 | \`${{ needs.call-nightly-amd-rocm720.result }}\` |" >> $GITHUB_STEP_SUMMARY + echo "| Nightly AMD ROCm 10 | \`${{ needs.call-nightly-amd-rocm720.result }}\` |" >> $GITHUB_STEP_SUMMARY echo "| PR Test AMD ROCm 7.0 Shadow | \`${{ needs.call-pr-test-amd.result }}\` |" >> $GITHUB_STEP_SUMMARY - echo "| PR Test AMD ROCm 7.2 Gate | \`${{ needs.call-pr-test-amd-rocm720.result }}\` |" >> $GITHUB_STEP_SUMMARY + echo "| PR Test AMD ROCm 10 Gate | \`${{ needs.call-pr-test-amd-rocm720.result }}\` |" >> $GITHUB_STEP_SUMMARY - name: Check if any job failed run: | diff --git a/.github/workflows/bot-bump-sglang-version.yml b/.github/workflows/bot-bump-sglang-version.yml index b764d0b50..ef3dd5591 100644 --- a/.github/workflows/bot-bump-sglang-version.yml +++ b/.github/workflows/bot-bump-sglang-version.yml @@ -65,7 +65,7 @@ jobs: uses: ./.github/workflows/nightly-test-amd-rocm720.yml with: ref: ${{ needs.bump-sglang-version.outputs.branch_name }} - rocm_version: rocm724 + rocm_version: rocm10 secrets: inherit run-nightly-tests-npu: diff --git a/.github/workflows/nightly-test-amd-rocm720.yml b/.github/workflows/nightly-test-amd-rocm720.yml index 8503fba05..96361f658 100644 --- a/.github/workflows/nightly-test-amd-rocm720.yml +++ b/.github/workflows/nightly-test-amd-rocm720.yml @@ -116,7 +116,7 @@ on: type: string # A single flavor, unlike the schedule and the dispatch form: a caller # that says nothing should not silently get multi-version GPU cost. - default: rocm724 + default: rocm10 ref: description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.' required: false diff --git a/.github/workflows/pr-states.yml b/.github/workflows/pr-states.yml index 3a6da1eff..77c66da0e 100644 --- a/.github/workflows/pr-states.yml +++ b/.github/workflows/pr-states.yml @@ -224,7 +224,7 @@ jobs: '', `Latest PR Test (Base): ${ptStart}${ptText}${ptEnd}`, `Latest PR Test (Extra): ${peStart}${peText}${peEnd}`, - `Latest PR Test (AMD ROCm 7.2): ${amdStart}${amdText}${amdEnd}`, + `Latest PR Test (AMD ROCm 10): ${amdStart}${amdText}${amdEnd}`, outerEnd, ].join('\n'); diff --git a/.github/workflows/pr-test-amd-extra.yml b/.github/workflows/pr-test-amd-extra.yml index 991c1403b..335c64254 100644 --- a/.github/workflows/pr-test-amd-extra.yml +++ b/.github/workflows/pr-test-amd-extra.yml @@ -37,10 +37,10 @@ on: - mi300 - mi325 rocm_version: - description: 'ROCm container variant (ROCm 7.2.4 by default)' + description: 'ROCm container variant (ROCm 10 by default)' required: false type: choice - default: rocm724 + default: rocm10 options: - rocm10 - rocm724 @@ -74,10 +74,10 @@ on: type: string default: mi300 rocm_version: - description: 'ROCm container variant (ROCm 7.2.4 by default)' + description: 'ROCm container variant (ROCm 10 by default)' required: false type: string - default: rocm724 + default: rocm10 aiter_ref: description: 'Override AITER commit (optional, leave empty to use Dockerfile default)' required: false @@ -105,7 +105,7 @@ concurrency: # scheduled reusable-workflow matrix calls cannot replace each other's # pending runs. PR runs still share a per-branch group so pushes cancel stale # runs. In a reusable workflow github.event_name is the originating event. - group: pr-test-amd-extra-${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && format('full-{0}-{1}', github.run_id, inputs.rocm_version || 'rocm724') || github.head_ref || github.ref_name || inputs.ref || 'default' }} + group: pr-test-amd-extra-${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && format('full-{0}-{1}', github.run_id, inputs.rocm_version || 'rocm10') || github.head_ref || github.ref_name || inputs.ref || 'default' }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} permissions: @@ -143,7 +143,7 @@ jobs: # setup across scarce AMD GPUs to shave only a couple minutes of test time, # so one GPU running the whole suite sequentially is the better trade. extra-a-test-1-gpu-small-amd: - name: ${{ format('extra-a-test-1-gpu-small-amd ({0}, linux-{1}-1gpu-sglang)', inputs.rocm_version || 'rocm724', inputs.runner_arch || 'mi300') }} + name: ${{ format('extra-a-test-1-gpu-small-amd ({0}, linux-{1}-1gpu-sglang)', inputs.rocm_version || 'rocm10', inputs.runner_arch || 'mi300') }} needs: [call-gate] if: ${{ !cancelled() && needs.call-gate.result == 'success' }} runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} @@ -158,8 +158,8 @@ jobs: - name: Start CI container # `inputs` is empty on pull_request events, so fall back explicitly to - # the ROCm 7.2 PR default. Reusable callers can request another variant. - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ inputs.rocm_version || 'rocm724' }} + # the ROCm 10 PR default. Reusable callers can request another variant. + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ inputs.rocm_version || 'rocm10' }} env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -176,7 +176,7 @@ jobs: # pool as small (AMD GPUs carry enough VRAM that "large" here is a CUDA # memory-tier label, not a separate AMD runner pool). extra-a-test-1-gpu-large-amd: - name: ${{ format('extra-a-test-1-gpu-large-amd ({0}, linux-{1}-1gpu-sglang)', inputs.rocm_version || 'rocm724', inputs.runner_arch || 'mi300') }} + name: ${{ format('extra-a-test-1-gpu-large-amd ({0}, linux-{1}-1gpu-sglang)', inputs.rocm_version || 'rocm10', inputs.runner_arch || 'mi300') }} needs: [call-gate] if: ${{ !cancelled() && needs.call-gate.result == 'success' }} runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} @@ -190,7 +190,7 @@ jobs: run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ inputs.rocm_version || 'rocm724' }} + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ inputs.rocm_version || 'rocm10' }} env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -206,7 +206,7 @@ jobs: # Multi-GPU TP / PP / PD mock-model + kv_canary e2e tests. Mirrors CUDA's # extra-a 2-gpu-large stage; runs on the 2-GPU AMD pool. extra-a-test-2-gpu-large-amd: - name: ${{ format('extra-a-test-2-gpu-large-amd ({0}, linux-{1}-2gpu-sglang)', inputs.rocm_version || 'rocm724', inputs.runner_arch || 'mi300') }} + name: ${{ format('extra-a-test-2-gpu-large-amd ({0}, linux-{1}-2gpu-sglang)', inputs.rocm_version || 'rocm10', inputs.runner_arch || 'mi300') }} needs: [call-gate] if: ${{ !cancelled() && needs.call-gate.result == 'success' }} runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }} @@ -220,7 +220,7 @@ jobs: run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ inputs.rocm_version || 'rocm724' }} + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ inputs.rocm_version || 'rocm10' }} env: GITHUB_WORKSPACE: ${{ github.workspace }} diff --git a/.github/workflows/pr-test-amd-mori.yml b/.github/workflows/pr-test-amd-mori.yml index fde156b6e..e43d4ee9a 100644 --- a/.github/workflows/pr-test-amd-mori.yml +++ b/.github/workflows/pr-test-amd-mori.yml @@ -53,7 +53,7 @@ jobs: fi - name: Start disaggregation container - run: bash scripts/ci/amd/amd_ci_start_container_disagg.sh --rocm-version rocm724 + run: bash scripts/ci/amd/amd_ci_start_container_disagg.sh --rocm-version rocm10 env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -84,7 +84,7 @@ jobs: run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm724 + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm10 env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" diff --git a/.github/workflows/pr-test-amd-rocm720.yml b/.github/workflows/pr-test-amd-rocm720.yml index c84cfe2df..0a0f158ff 100644 --- a/.github/workflows/pr-test-amd-rocm720.yml +++ b/.github/workflows/pr-test-amd-rocm720.yml @@ -5,8 +5,8 @@ run-name: ${{ (inputs.target_stage || inputs.target_stage_select) && (inputs.pr_ on: schedule: - - cron: '0 */12 * * *' # rocm724: twice daily (UTC) - - cron: '30 17 * * *' # rocm10/rocm720: once daily, aligned with ROCm 7.0 + - cron: '0 */12 * * *' # rocm10: twice daily (UTC) + - cron: '30 17 * * *' # rocm724/rocm720: once daily, aligned with ROCm 7.0 pull_request: paths: - "python/**" @@ -21,7 +21,7 @@ on: description: 'ROCm image version' required: false type: choice - default: rocm724 + default: rocm10 options: - rocm10 - rocm724 @@ -95,7 +95,7 @@ on: description: 'ROCm image version' required: false type: string - default: rocm724 + default: rocm10 ref: description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.' required: false @@ -165,11 +165,11 @@ jobs: jit_kernel: ${{ steps.filter.outputs.jit_kernel || steps.run-mode.outputs.run_all_tests }} multimodal_gen: ${{ steps.filter.outputs.multimodal_gen || steps.run-mode.outputs.run_all_tests }} continue_on_error: ${{ steps.set-continue-on-error.outputs.continue_on_error }} - # Keep rocm724 on its existing twice-daily cadence. The 17:30 UTC - # schedule runs rocm10 and rocm720 once daily alongside the ROCm 7.0 + # Keep rocm10 on the twice-daily cadence. The 17:30 UTC schedule runs + # rocm724 and rocm720 once daily alongside the ROCm 7.0 # shadow workflow. Dispatch/call inputs select one version, while a - # pull_request falls through to the single rocm724 default. - rocm_versions: ${{ inputs.rocm_version && format('["{0}"]', inputs.rocm_version) || github.event.schedule == '30 17 * * *' && '["rocm10","rocm720"]' || '["rocm724"]' }} + # pull_request falls through to the single rocm10 default. + rocm_versions: ${{ inputs.rocm_version && format('["{0}"]', inputs.rocm_version) || github.event.schedule == '30 17 * * *' && '["rocm724","rocm720"]' || '["rocm10"]' }} steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/.github/workflows/release-branch-cut.yml b/.github/workflows/release-branch-cut.yml index 8c733c8dc..46d219cad 100644 --- a/.github/workflows/release-branch-cut.yml +++ b/.github/workflows/release-branch-cut.yml @@ -177,7 +177,7 @@ jobs: uses: ./.github/workflows/pr-test-amd-rocm720.yml with: ref: ${{ needs.cut-release-branch.outputs.branch_name }} - rocm_version: rocm724 + rocm_version: rocm10 run_all_tests: true secrets: inherit @@ -217,7 +217,7 @@ jobs: uses: ./.github/workflows/nightly-test-amd-rocm720.yml with: ref: ${{ needs.cut-release-branch.outputs.branch_name }} - rocm_version: rocm724 + rocm_version: rocm10 secrets: inherit run-nightly-tests-npu: