minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[AMD][CI] Consolidate AMD workflows and retire ROCm 7.0 CI (#38632)

Browse Source
This commit is contained in:
Bingxu Chen
2026-09-15 23:45:22 +08:00
committed by GitHub
parent 47a157f257
commit 03ea13a545
17 changed files with 1190 additions and 4932 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/pr-test-amd.yml
+246 -42
View File
@@ -1,18 +1,32 @@
name: PR Test ROCm 7.0 (AMD)
name: PR Test (AMD)
# Dynamic run-name for manual stage dispatches to enable URL lookup
# Format: "[stage-name] sha" for fork PRs, "[stage-name]" for non-fork, default for normal runs
run-name: ${{ (inputs.target_stage || inputs.target_stage_select) && (inputs.pr_head_sha && format('[{0}] {1}', inputs.target_stage || inputs.target_stage_select, inputs.pr_head_sha) || format('[{0}]', inputs.target_stage || inputs.target_stage_select)) || '' }}
on:
schedule:
# ROCm 7.0 rollback shadow. pr-test-amd-rocm720.yml is the PR gate; this
# workflow now only proves ROCm 7.0 still works, so it fires every two days
# alongside nightly-test-amd.yml and chases coverage rather than latency:
# no matrix parallelism caps, no stage fast-fail (the `schedule` escape in
# each job's `if`), and continue-on-error via check-changes.
- cron: '30 17 */2 * *'
- cron: '0 */12 * * *' # rocm10: twice daily (UTC)
- cron: '30 17 * * *' # rocm724/rocm720: once daily (UTC)
pull_request:
paths:
- "python/**"
- "scripts/ci/**"
- "test/**"
- "python/sglang/kernels/aot/**"
- ".github/workflows/pr-test-amd.yml"
- "docker/rocm.Dockerfile"
workflow_dispatch:
inputs:
rocm_version:
description: 'ROCm image version ("all" runs rocm10, rocm724, and rocm720)'
required: false
type: choice
default: rocm10
options:
- all
- rocm10
- rocm724
- rocm720
target_stage_select:
description: "Select a stage to run from dropdown (leave empty for auto-detect)"
required: false
@@ -33,6 +47,8 @@ on:
- multimodal-gen-test-1-gpu-amd
- multimodal-gen-test-2-gpu-amd
- stage-c-test-4-gpu-amd
- stage-c-dsv4-flash-fp4-fp8-amd-mi35x
- stage-c-dsv4-pro-fp4-amd-mi35x
- stage-c-test-large-8-gpu-amd
- stage-c-test-large-8-gpu-amd-mi35x
- stage-b-test-large-8-gpu-mi35x-disaggregation-amd
@@ -51,6 +67,11 @@ on:
required: false
type: string
default: ''
amd_ci_image:
description: 'Override AMD CI Docker image (optional, e.g. rocm/sgl-dev:tag)'
required: false
type: string
default: ''
continue_on_error:
description: 'Continue on error (do not fail the workflow on test failures)'
required: false
@@ -71,6 +92,11 @@ on:
default: false
workflow_call:
inputs:
rocm_version:
description: 'ROCm image version ("all" runs rocm10, rocm724, and rocm720)'
required: false
type: string
default: rocm10
ref:
description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
required: false
@@ -91,6 +117,11 @@ on:
required: false
type: string
default: ''
amd_ci_image:
description: 'Override AMD CI Docker image (optional, e.g. rocm/sgl-dev:tag)'
required: false
type: string
default: ''
continue_on_error:
description: 'Continue on error (do not fail the workflow on test failures)'
required: false
@@ -110,6 +141,7 @@ permissions:
env:
AITER_COMMIT_OVERRIDE: ${{ inputs.aiter_ref }}
AMD_CI_IMAGE: ${{ inputs.amd_ci_image }}
DOCKERHUB_AMD_USERNAME: ${{ secrets.DOCKERHUB_AMD_USERNAME }}
DOCKERHUB_AMD_TOKEN: ${{ secrets.DOCKERHUB_AMD_TOKEN }}
@@ -134,6 +166,10 @@ jobs:
jit_kernel: ${{ steps.filter.outputs.jit_kernel || steps.run-mode.outputs.run_all_tests }}
multimodal_gen: ${{ steps.filter.outputs.multimodal_gen || steps.run-mode.outputs.run_all_tests }}
continue_on_error: ${{ steps.set-continue-on-error.outputs.continue_on_error }}
# Keep rocm10 on the twice-daily cadence. The 17:30 UTC schedule runs
# rocm724 and rocm720 once daily. Dispatch/call inputs select one version
# or all three, while a pull_request falls through to the rocm10 default.
rocm_versions: ${{ inputs.rocm_version == 'all' && '["rocm10","rocm724","rocm720"]' || inputs.rocm_version && format('["{0}"]', inputs.rocm_version) || github.event.schedule == '30 17 * * *' && '["rocm724","rocm720"]' || '["rocm10"]' }}
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -212,21 +248,32 @@ jobs:
# Not added to `pr-test-amd-finish` so the base AMD gate never depends on
# the opt-in extra suite.
call-pr-test-amd-extra:
name: call-pr-test-amd-extra
needs: [check-changes]
if: |
(github.event_name == 'schedule' || inputs.run_all_tests == true) &&
!(inputs.target_stage || inputs.target_stage_select)
strategy:
fail-fast: false
matrix:
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
uses: ./.github/workflows/pr-test-amd-extra.yml
with:
ref: ${{ inputs.pr_head_sha || inputs.ref || '' }}
runner_arch: ${{ inputs.runner_arch || 'mi300' }}
rocm_version: rocm700
rocm_version: ${{ matrix.rocm_version }}
aiter_ref: ${{ inputs.aiter_ref }}
amd_ci_image: ${{ inputs.amd_ci_image }}
continue_on_error: true
secrets: inherit
# =============================================== sgl-kernel ====================================================
sgl-kernel-unit-test-amd:
name: ${{ format('sgl-kernel-unit-test-amd (rocm700, linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }}
name: ${{ format('sgl-kernel-unit-test-amd ({0}, linux-{1}-1gpu-sglang)', matrix.rocm_version, inputs.runner_arch || 'mi300') }}
strategy:
fail-fast: false
matrix:
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
needs: [check-changes, call-gate]
if: |
always() && !cancelled() &&
@@ -249,7 +296,7 @@ jobs:
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Start CI container
run: bash scripts/ci/amd/amd_ci_start_container.sh
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
@@ -284,7 +331,11 @@ jobs:
exit $failures
sgl-kernel-unit-test-2-gpu-amd:
name: ${{ format('sgl-kernel-unit-test-2-gpu-amd (rocm700, linux-{0}-2gpu-sglang)', inputs.runner_arch || 'mi300') }}
name: ${{ format('sgl-kernel-unit-test-2-gpu-amd ({0}, linux-{1}-2gpu-sglang)', matrix.rocm_version, inputs.runner_arch || 'mi300') }}
strategy:
fail-fast: false
matrix:
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
needs: [check-changes, call-gate]
if: |
always() && !cancelled() &&
@@ -307,7 +358,7 @@ jobs:
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Start CI container
run: bash scripts/ci/amd/amd_ci_start_container.sh
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
@@ -325,7 +376,11 @@ jobs:
# =============================================== primary ====================================================
stage-a-test-1-gpu-small-amd:
name: ${{ format('stage-a-test-1-gpu-small-amd (rocm700, linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }}
name: ${{ format('stage-a-test-1-gpu-small-amd ({0}, linux-{1}-1gpu-sglang)', matrix.rocm_version, inputs.runner_arch || 'mi300') }}
strategy:
fail-fast: false
matrix:
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
needs: [check-changes, call-gate]
if: |
always() && !cancelled() &&
@@ -348,7 +403,7 @@ jobs:
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Start CI container
run: bash scripts/ci/amd/amd_ci_start_container.sh
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
@@ -362,7 +417,11 @@ jobs:
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-a-test-1-gpu-small-amd ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
jit-kernel-unit-test-amd:
name: ${{ format('jit-kernel-unit-test-amd (rocm700, linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }}
name: ${{ format('jit-kernel-unit-test-amd ({0}, linux-{1}-1gpu-sglang)', matrix.rocm_version, inputs.runner_arch || 'mi300') }}
strategy:
fail-fast: false
matrix:
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
needs: [check-changes, call-gate]
if: |
always() && !cancelled() &&
@@ -385,7 +444,7 @@ jobs:
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Start CI container
run: bash scripts/ci/amd/amd_ci_start_container.sh
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
@@ -399,7 +458,11 @@ jobs:
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite jit-kernel-unit-test-amd ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
jit-kernel-benchmark-test-amd:
name: ${{ format('jit-kernel-benchmark-test-amd (rocm700, linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }}
name: ${{ format('jit-kernel-benchmark-test-amd ({0}, linux-{1}-1gpu-sglang)', matrix.rocm_version, inputs.runner_arch || 'mi300') }}
strategy:
fail-fast: false
matrix:
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
needs: [check-changes, call-gate]
if: |
always() && !cancelled() &&
@@ -422,7 +485,7 @@ jobs:
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Start CI container
run: bash scripts/ci/amd/amd_ci_start_container.sh
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
@@ -441,6 +504,7 @@ jobs:
# For scheduled runs: wait jobs are skipped, enabling parallel execution of all stages.
wait-for-stage-a-amd:
name: wait-for-stage-a-amd
needs: [check-changes, call-gate]
if: |
always() &&
@@ -462,7 +526,7 @@ jobs:
max-wait-minutes: '240'
stage-b-test-1-gpu-small-amd:
name: ${{ format('stage-b-test-1-gpu-small-amd (rocm700, linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }}
name: ${{ format('stage-b-test-1-gpu-small-amd ({0}, linux-{1}-1gpu-sglang, {2})', matrix.rocm_version, inputs.runner_arch || 'mi300', matrix.part) }}
needs: [check-changes, wait-for-stage-a-amd]
if: |
always() &&
@@ -476,7 +540,9 @@ jobs:
)
strategy:
fail-fast: false
max-parallel: 4
matrix:
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
part: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
steps:
@@ -489,7 +555,7 @@ jobs:
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Start CI container
run: bash scripts/ci/amd/amd_ci_start_container.sh
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
@@ -502,7 +568,11 @@ jobs:
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 14 --timeout-per-file 2400 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
stage-b-test-1-gpu-small-amd-nondeterministic:
name: ${{ format('stage-b-test-1-gpu-small-amd-nondeterministic (rocm700, linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }}
name: ${{ format('stage-b-test-1-gpu-small-amd-nondeterministic ({0}, linux-{1}-1gpu-sglang)', matrix.rocm_version, inputs.runner_arch || 'mi300') }}
strategy:
fail-fast: false
matrix:
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
needs: [check-changes, wait-for-stage-a-amd]
if: |
always() &&
@@ -525,7 +595,7 @@ jobs:
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Start CI container
run: bash scripts/ci/amd/amd_ci_start_container.sh
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
@@ -538,7 +608,7 @@ jobs:
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-nondeterministic --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
stage-b-test-1-gpu-small-amd-mi35x:
name: ${{ format('stage-b-test-1-gpu-small-amd-mi35x (rocm700, {0})', matrix.runner) }}
name: ${{ format('stage-b-test-1-gpu-small-amd-mi35x ({0}, {1})', matrix.rocm_version, matrix.runner) }}
needs: [check-changes, wait-for-stage-a-amd]
if: |
always() &&
@@ -553,6 +623,7 @@ jobs:
strategy:
fail-fast: false
matrix:
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
runner: [linux-mi35x-gpu-1]
runs-on: ${{matrix.runner}}
steps:
@@ -565,7 +636,7 @@ jobs:
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Start CI container
run: bash scripts/ci/amd/amd_ci_start_container.sh
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
@@ -578,7 +649,7 @@ jobs:
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-mi35x ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
stage-b-test-1-gpu-large-amd:
name: ${{ format('stage-b-test-1-gpu-large-amd (rocm700, linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }}
name: ${{ format('stage-b-test-1-gpu-large-amd ({0}, linux-{1}-1gpu-sglang, {2})', matrix.rocm_version, inputs.runner_arch || 'mi300', matrix.part) }}
needs: [check-changes, wait-for-stage-a-amd]
if: |
always() &&
@@ -592,7 +663,9 @@ jobs:
)
strategy:
fail-fast: false
max-parallel: 1
matrix:
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
part: [0, 1, 2]
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
steps:
@@ -605,7 +678,7 @@ jobs:
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Start CI container
run: bash scripts/ci/amd/amd_ci_start_container.sh
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
@@ -618,7 +691,7 @@ jobs:
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 2700 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
stage-b-test-2-gpu-large-amd:
name: ${{ format('stage-b-test-2-gpu-large-amd (rocm700, linux-{0}-2gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }}
name: ${{ format('stage-b-test-2-gpu-large-amd ({0}, linux-{1}-2gpu-sglang, {2})', matrix.rocm_version, inputs.runner_arch || 'mi300', matrix.part) }}
needs: [check-changes, wait-for-stage-a-amd]
if: |
always() &&
@@ -632,7 +705,9 @@ jobs:
)
strategy:
fail-fast: false
max-parallel: 1
matrix:
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
part: [0, 1]
runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }}
steps:
@@ -645,7 +720,7 @@ jobs:
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Start CI container
run: bash scripts/ci/amd/amd_ci_start_container.sh
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
@@ -658,7 +733,7 @@ jobs:
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-2-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file ${{ (inputs.runner_arch || 'mi300') == 'mi300' && 5400 || 2700 }} ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
multimodal-gen-test-1-gpu-amd:
name: ${{ format('multimodal-gen-test-1-gpu-amd (rocm700, linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }}
name: ${{ format('multimodal-gen-test-1-gpu-amd ({0}, linux-{1}-1gpu-sglang, {2})', matrix.rocm_version, inputs.runner_arch || 'mi300', matrix.part) }}
needs: [check-changes, call-gate]
if: |
always() && !cancelled() &&
@@ -672,7 +747,9 @@ jobs:
)
strategy:
fail-fast: false
max-parallel: 1
matrix:
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
part: [0, 1, 2, 3]
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
steps:
@@ -693,7 +770,7 @@ jobs:
pattern: wheel-python3.10-cuda12.9
- name: Start CI container
run: bash scripts/ci/amd/amd_ci_start_container.sh
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
@@ -800,13 +877,13 @@ jobs:
if: always()
uses: actions/upload-artifact@v4
with:
name: diffusion-failures-amd-1gpu-${{ matrix.part }}-${{ github.run_attempt }}
name: diffusion-failures-amd-1gpu-${{ matrix.rocm_version }}-${{ matrix.part }}-${{ github.run_attempt }}
path: diffusion-failures/
if-no-files-found: ignore
retention-days: 7
multimodal-gen-test-2-gpu-amd:
name: ${{ format('multimodal-gen-test-2-gpu-amd (rocm700, linux-{0}-2gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }}
name: ${{ format('multimodal-gen-test-2-gpu-amd ({0}, linux-{1}-2gpu-sglang, {2})', matrix.rocm_version, inputs.runner_arch || 'mi300', matrix.part) }}
needs: [check-changes, call-gate]
if: |
always() && !cancelled() &&
@@ -820,7 +897,9 @@ jobs:
)
strategy:
fail-fast: false
max-parallel: 1
matrix:
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
# run_suite.py load-balances the suite's parametrized cases and
# standalone files over these partitions, so the count is free to
# choose. Two shards measured 45-65 min of case work each, inside the
@@ -846,7 +925,7 @@ jobs:
pattern: wheel-python3.10-cuda12.9
- name: Start CI container
run: bash scripts/ci/amd/amd_ci_start_container.sh
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
@@ -934,12 +1013,13 @@ jobs:
if: always()
uses: actions/upload-artifact@v4
with:
name: diffusion-failures-amd-2gpu-${{ matrix.part }}-${{ github.run_attempt }}
name: diffusion-failures-amd-2gpu-${{ matrix.rocm_version }}-${{ matrix.part }}-${{ github.run_attempt }}
path: diffusion-failures/
if-no-files-found: ignore
retention-days: 7
wait-for-stage-b-amd:
name: wait-for-stage-b-amd
needs: [check-changes, call-gate, wait-for-stage-a-amd]
if: |
always() &&
@@ -967,7 +1047,7 @@ jobs:
max-wait-minutes: '480'
stage-c-test-4-gpu-amd:
name: ${{ format('stage-c-test-4-gpu-amd (rocm700, linux-{0}-4gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }}
name: ${{ format('stage-c-test-4-gpu-amd ({0}, linux-{1}-4gpu-sglang, {2})', matrix.rocm_version, inputs.runner_arch || 'mi300', matrix.part) }}
needs: [check-changes, call-gate, wait-for-stage-b-amd]
if: |
always() &&
@@ -982,6 +1062,7 @@ jobs:
strategy:
fail-fast: false
matrix:
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
part: [0]
runs-on: ${{ format('linux-{0}-4gpu-sglang', inputs.runner_arch || 'mi300') }}
steps:
@@ -994,7 +1075,7 @@ jobs:
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Start CI container
run: bash scripts/ci/amd/amd_ci_start_container.sh
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
@@ -1023,7 +1104,7 @@ jobs:
${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
stage-c-test-large-8-gpu-amd:
name: ${{ format('stage-c-test-large-8-gpu-amd (rocm700, linux-{0}-8gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }}
name: ${{ format('stage-c-test-large-8-gpu-amd ({0}, linux-{1}-8gpu-sglang, {2})', matrix.rocm_version, inputs.runner_arch || 'mi300', matrix.part) }}
needs: [check-changes, call-gate, wait-for-stage-b-amd]
if: |
always() &&
@@ -1039,7 +1120,9 @@ jobs:
RUNNER_LABELS: ${{ format('linux-{0}-8gpu-sglang', inputs.runner_arch || 'mi300') }}
strategy:
fail-fast: false
max-parallel: 2
matrix:
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
part: [0, 1, 2, 3]
runs-on: ${{ format('linux-{0}-8gpu-sglang', inputs.runner_arch || 'mi300') }}
steps:
@@ -1052,7 +1135,7 @@ jobs:
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Start CI container
run: bash scripts/ci/amd/amd_ci_start_container.sh
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
@@ -1071,7 +1154,7 @@ jobs:
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 4 --timeout-per-file 5400 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
stage-c-test-large-8-gpu-amd-mi35x:
name: ${{ format('stage-c-test-large-8-gpu-amd-mi35x (rocm700, {0}, {1})', matrix.runner, matrix.part) }}
name: ${{ format('stage-c-test-large-8-gpu-amd-mi35x ({0}, {1}, {2})', matrix.rocm_version, matrix.runner, matrix.part) }}
needs: [check-changes, call-gate, wait-for-stage-b-amd]
if: |
always() &&
@@ -1086,6 +1169,7 @@ jobs:
strategy:
fail-fast: false
matrix:
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
runner: [linux-mi35x-gpu-8]
part: [0, 1, 2]
runs-on: ${{matrix.runner}}
@@ -1099,7 +1183,7 @@ jobs:
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Start CI container
run: bash scripts/ci/amd/amd_ci_start_container.sh
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
@@ -1111,9 +1195,124 @@ jobs:
run: |
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
# =============================================== DeepSeek-V4 (MI35x, 8-GPU) ====================================================
# GSM8K accuracy on the nightly dsv4 suites, ~20min each. Gated as stage-C so
# every PR that touches the main package covers DeepSeek-V4.
stage-c-dsv4-flash-fp4-fp8-amd-mi35x:
name: ${{ format('stage-c-dsv4-flash-fp4-fp8-amd-mi35x ({0}, {1})', matrix.rocm_version, matrix.runner) }}
needs: [check-changes, call-gate, wait-for-stage-b-amd]
if: |
always() &&
(
(contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-c-dsv4-flash-fp4-fp8-amd-mi35x,')) ||
(
!(inputs.target_stage || inputs.target_stage_select) &&
((github.event_name == 'schedule') || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
)
)
strategy:
fail-fast: false
matrix:
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
runner: [linux-mi35x-gpu-8]
runs-on: ${{matrix.runner}}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }}
- name: Ensure VRAM is clear
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Setup docker (ROCm)
run: |
touch github_summary.md
bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
ENABLE_CACHE_HOST: "1"
- name: Install dependencies
run: |
# --skip-test-time-deps: GSM8K + bench_one_batch_server don't need lmms-eval / human-eval.
bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate
- name: Accuracy Test MI35x ROCm (8-GPU DeepSeek-V4-Flash FP4 + FP8)
timeout-minutes: 60
run: |
> github_summary.md # Clear summary file
# SGLANG_DSV4_ACCURACY_ONLY=1 makes the dsv4 test files skip their perf test
# (test_b_perf_8k_1k); only the GSM8K accuracy test runs in this workflow.
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
-e SGLANG_DSV4_ACCURACY_ONLY=1 \
-e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-flash --nightly --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
stage-c-dsv4-pro-fp4-amd-mi35x:
name: ${{ format('stage-c-dsv4-pro-fp4-amd-mi35x ({0}, {1})', matrix.rocm_version, matrix.runner) }}
needs: [check-changes, call-gate, wait-for-stage-b-amd]
if: |
always() &&
(
(contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-c-dsv4-pro-fp4-amd-mi35x,')) ||
(
!(inputs.target_stage || inputs.target_stage_select) &&
((github.event_name == 'schedule') || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
)
)
strategy:
fail-fast: false
matrix:
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
runner: [linux-mi35x-gpu-8]
runs-on: ${{matrix.runner}}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }}
- name: Ensure VRAM is clear
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Setup docker (ROCm)
run: |
touch github_summary.md
bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
ENABLE_CACHE_HOST: "1"
- name: Install dependencies
run: |
# --skip-test-time-deps: GSM8K + bench_one_batch_server don't need lmms-eval / human-eval.
bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate
- name: Accuracy Test MI35x ROCm (8-GPU DeepSeek-V4-Pro FP4)
timeout-minutes: 60
run: |
> github_summary.md # Clear summary file
# SGLANG_DSV4_ACCURACY_ONLY=1 makes the dsv4 test files skip their perf test
# (test_b_perf_8k_1k); only the GSM8K accuracy test runs in this workflow.
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
-e SGLANG_DSV4_ACCURACY_ONLY=1 \
-e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
python3 registered/amd/test_deepseek_v4_pro_fp4.py || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
# =============================================== Disaggregation ====================================================
stage-b-test-large-8-gpu-mi35x-disaggregation-amd:
name: ${{ format('stage-b-test-large-8-gpu-mi35x-disaggregation-amd (rocm700, {0})', matrix.runner) }}
name: ${{ format('stage-b-test-large-8-gpu-mi35x-disaggregation-amd ({0}, {1})', matrix.rocm_version, matrix.runner) }}
needs: [check-changes, wait-for-stage-a-amd]
if: |
always() &&
@@ -1128,6 +1327,7 @@ jobs:
strategy:
fail-fast: false
matrix:
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
runner: [linux-mi35x-gpu-8.fabric]
runs-on: ${{matrix.runner}}
@@ -1180,7 +1380,7 @@ jobs:
echo "=== Host RDMA Check Complete ==="
- name: Start Special Container
run: bash scripts/ci/amd/amd_ci_start_container_disagg.sh
run: bash scripts/ci/amd/amd_ci_start_container_disagg.sh --rocm-version "${{ matrix.rocm_version }}"
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
@@ -1247,6 +1447,7 @@ jobs:
-w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-8-gpu-mi35x-disaggregation-amd --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
pr-test-amd-finish:
name: pr-test-amd-finish
needs:
[
call-gate,
@@ -1271,6 +1472,9 @@ jobs:
stage-c-test-4-gpu-amd,
stage-c-test-large-8-gpu-amd,
stage-c-test-large-8-gpu-amd-mi35x,
stage-c-dsv4-flash-fp4-fp8-amd-mi35x,
stage-c-dsv4-pro-fp4-amd-mi35x,
]
if: always()
runs-on: ubuntu-latest