minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[AMD][CI] Swap the AMD PR gate to ROCm 7.2 and demote ROCm 7.0 to a daily shadow (#34204)

Browse Source 
Co-authored-by: yctseng0211 <yctseng@smci355-ccs-aus-m12-17.cs-aus.dcgpu>
Co-authored-by: Chen <bingxche@amd.com>
This commit is contained in:
YC Yen-Ching Tseng
2026-08-12 03:31:03 -07:00
committed by GitHub
co-authored by yctseng0211 Chen
parent b3bffef70a
commit 00bdafe944
12 changed files with 515 additions and 432 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/amd-aiter-scout.yml
+3 -3
View File
@@ -12,7 +12,7 @@ on:
type: string type: string
default: 'main' default: 'main'
job_filter: job_filter:
description: 'Comma-separated workflows to run: nightly-amd, nightly-amd-rocm720, pr-test-amd, pr-test-amd-rocm720. Default: all' description: 'Workflows: nightly-amd, nightly-amd-rocm720, pr-test-amd (ROCm 7.0 shadow), pr-test-amd-rocm720 (ROCm 7.2 gate). Default: all'
required: false required: false
type: string type: string
default: 'all' default: 'all'
@@ -154,8 +154,8 @@ jobs:
echo "|----------|--------|" >> $GITHUB_STEP_SUMMARY echo "|----------|--------|" >> $GITHUB_STEP_SUMMARY
echo "| Nightly AMD (AITER Latest) | \`${{ needs.call-nightly-amd.result }}\` |" >> $GITHUB_STEP_SUMMARY echo "| Nightly AMD (AITER Latest) | \`${{ needs.call-nightly-amd.result }}\` |" >> $GITHUB_STEP_SUMMARY
echo "| Nightly AMD ROCm 7.2 | \`${{ needs.call-nightly-amd-rocm720.result }}\` |" >> $GITHUB_STEP_SUMMARY echo "| Nightly AMD ROCm 7.2 | \`${{ needs.call-nightly-amd-rocm720.result }}\` |" >> $GITHUB_STEP_SUMMARY
echo "| PR Test AMD (AITER Latest) | \`${{ needs.call-pr-test-amd.result }}\` |" >> $GITHUB_STEP_SUMMARY echo "| PR Test AMD ROCm 7.0 Shadow | \`${{ needs.call-pr-test-amd.result }}\` |" >> $GITHUB_STEP_SUMMARY
echo "| PR Test AMD ROCm 7.2 | \`${{ needs.call-pr-test-amd-rocm720.result }}\` |" >> $GITHUB_STEP_SUMMARY echo "| PR Test AMD ROCm 7.2 Gate | \`${{ needs.call-pr-test-amd-rocm720.result }}\` |" >> $GITHUB_STEP_SUMMARY
- name: Check if any job failed - name: Check if any job failed
run: | run: |
.github/workflows/amd-ci-job-monitor.yml
+98 -98
View File
@@ -53,7 +53,7 @@ jobs:
id: select-workflows id: select-workflows
run: | run: |
if [[ -n "${{ inputs.job_filter }}" ]]; then if [[ -n "${{ inputs.job_filter }}" ]]; then
echo "workflows=pr-test-amd.yml" >> "$GITHUB_OUTPUT" echo "workflows=pr-test-amd-rocm720.yml" >> "$GITHUB_OUTPUT"
else else
echo "workflows=pr-test-amd.yml,nightly-test-amd.yml,pr-test-amd-rocm720.yml,nightly-test-amd-rocm720.yml,nightly-amd-mi355x-disagg.yml" >> "$GITHUB_OUTPUT" echo "workflows=pr-test-amd.yml,nightly-test-amd.yml,pr-test-amd-rocm720.yml,nightly-test-amd-rocm720.yml,nightly-amd-mi355x-disagg.yml" >> "$GITHUB_OUTPUT"
fi fi
@@ -106,7 +106,7 @@ jobs:
python scripts/ci/utils/query_job_status.py \ python scripts/ci/utils/query_job_status.py \
--repo ${{ github.repository }} \ --repo ${{ github.repository }} \
--job "${{ inputs.job_filter }}" \ --job "${{ inputs.job_filter }}" \
--workflow "pr-test-amd.yml" \ --workflow "pr-test-amd-rocm720.yml" \
--hours ${{ inputs.hours || '24' }} \ --hours ${{ inputs.hours || '24' }} \
--input-data-file ci-data/actions-job-snapshot.json \ --input-data-file ci-data/actions-job-snapshot.json \
--summary --summary
@@ -119,8 +119,8 @@ jobs:
outputs: outputs:
pr_jobs: ${{ steps.parse.outputs.pr_jobs }} pr_jobs: ${{ steps.parse.outputs.pr_jobs }}
nightly_jobs: ${{ steps.parse.outputs.nightly_jobs }} nightly_jobs: ${{ steps.parse.outputs.nightly_jobs }}
pr_rocm720_jobs: ${{ steps.parse.outputs.pr_rocm720_jobs }} pr_rocm700_jobs: ${{ steps.parse.outputs.pr_rocm700_jobs }}
nightly_rocm720_jobs: ${{ steps.parse.outputs.nightly_rocm720_jobs }} nightly_rocm700_jobs: ${{ steps.parse.outputs.nightly_rocm700_jobs }}
disagg_jobs: ${{ steps.parse.outputs.disagg_jobs }} disagg_jobs: ${{ steps.parse.outputs.disagg_jobs }}
steps: steps:
- name: Checkout code - name: Checkout code
@@ -129,37 +129,37 @@ jobs:
- name: Parse workflow files - name: Parse workflow files
id: parse id: parse
run: | run: |
# Parse pr-test-amd.yml and extract job names (exclude utility jobs) # Parse the default ROCm 7.2 PR gate (exclude utility jobs)
# Excluded: call-gate, check-changes, pr-test-amd-finish, cancel, check-all-jobs # Excluded: call-gate, check-changes, finish, cancel, check-all-jobs
pr_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/pr-test-amd.yml | \ pr_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/pr-test-amd-rocm720.yml | \
grep -v -E '^(call-gate|check-changes|pr-test-amd-finish|cancel|check-all-jobs)$' | \ grep -v -E '^(call-gate|check-changes|pr-test-amd-rocm720-finish|cancel|check-all-jobs)$' | \
jq -R -s -c 'split("\n") | map(select(length > 0))') jq -R -s -c 'split("\n") | map(select(length > 0))')
echo "pr_jobs=$pr_jobs" >> $GITHUB_OUTPUT echo "pr_jobs=$pr_jobs" >> $GITHUB_OUTPUT
echo "PR jobs: $pr_jobs" echo "PR jobs: $pr_jobs"
# Parse nightly-test-amd.yml and extract job names (exclude utility jobs) # Parse the default ROCm 7.2 nightly (exclude utility jobs)
# Excluded: check-all-jobs # Excluded: check-all-jobs
nightly_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/nightly-test-amd.yml | \ nightly_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/nightly-test-amd-rocm720.yml | \
grep -v -E '^(check-all-jobs)$' | \ grep -v -E '^(check-all-jobs)$' | \
jq -R -s -c 'split("\n") | map(select(length > 0))') jq -R -s -c 'split("\n") | map(select(length > 0))')
echo "nightly_jobs=$nightly_jobs" >> $GITHUB_OUTPUT echo "nightly_jobs=$nightly_jobs" >> $GITHUB_OUTPUT
echo "Nightly jobs: $nightly_jobs" echo "Nightly jobs: $nightly_jobs"
# Parse pr-test-amd-rocm720.yml (exclude utility jobs) # Parse the ROCm 7.0 rollback shadow (exclude utility jobs)
# Excluded: call-gate, check-changes, pr-test-amd-finish, cancel, check-all-jobs # Excluded: call-gate, check-changes, finish, cancel, check-all-jobs
pr_rocm720_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/pr-test-amd-rocm720.yml | \ pr_rocm700_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/pr-test-amd.yml | \
grep -v -E '^(call-gate|check-changes|pr-test-amd-finish|cancel|check-all-jobs)$' | \ grep -v -E '^(call-gate|check-changes|pr-test-amd-finish|cancel|check-all-jobs)$' | \
jq -R -s -c 'split("\n") | map(select(length > 0))') jq -R -s -c 'split("\n") | map(select(length > 0))')
echo "pr_rocm720_jobs=$pr_rocm720_jobs" >> $GITHUB_OUTPUT echo "pr_rocm700_jobs=$pr_rocm700_jobs" >> $GITHUB_OUTPUT
echo "PR ROCm 7.2 jobs: $pr_rocm720_jobs" echo "PR ROCm 7.0 shadow jobs: $pr_rocm700_jobs"
# Parse nightly-test-amd-rocm720.yml (exclude utility jobs) # Parse the ROCm 7.0 rollback shadow nightly (exclude utility jobs)
# Excluded: check-all-jobs # Excluded: check-all-jobs
nightly_rocm720_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/nightly-test-amd-rocm720.yml | \ nightly_rocm700_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/nightly-test-amd.yml | \
grep -v -E '^(check-all-jobs)$' | \ grep -v -E '^(check-all-jobs)$' | \
jq -R -s -c 'split("\n") | map(select(length > 0))') jq -R -s -c 'split("\n") | map(select(length > 0))')
echo "nightly_rocm720_jobs=$nightly_rocm720_jobs" >> $GITHUB_OUTPUT echo "nightly_rocm700_jobs=$nightly_rocm700_jobs" >> $GITHUB_OUTPUT
echo "Nightly ROCm 7.2 jobs: $nightly_rocm720_jobs" echo "Nightly ROCm 7.0 shadow jobs: $nightly_rocm700_jobs"
# Parse nightly-amd-mi355x-disagg.yml (exclude utility jobs) # Parse nightly-amd-mi355x-disagg.yml (exclude utility jobs)
# Excluded: setup, collect-results -- both run on ubuntu-latest and the # Excluded: setup, collect-results -- both run on ubuntu-latest and the
@@ -204,7 +204,7 @@ jobs:
python scripts/ci/utils/query_job_status.py \ python scripts/ci/utils/query_job_status.py \
--repo ${{ github.repository }} \ --repo ${{ github.repository }} \
--job "${{ matrix.job_name }}" \ --job "${{ matrix.job_name }}" \
--workflow "pr-test-amd.yml" \ --workflow "pr-test-amd-rocm720.yml" \
--hours ${{ inputs.hours || '24' }} \ --hours ${{ inputs.hours || '24' }} \
--input-data-file ci-data/actions-job-snapshot.json \ --input-data-file ci-data/actions-job-snapshot.json \
--summary --summary
@@ -238,84 +238,6 @@ jobs:
path: ci-data path: ci-data
- name: Generate Nightly Report - name: Generate Nightly Report
timeout-minutes: 15
run: |
python scripts/ci/utils/query_job_status.py \
--repo ${{ github.repository }} \
--job "${{ matrix.job_name }}" \
--workflow "nightly-test-amd.yml" \
--hours ${{ inputs.hours || '24' }} \
--input-data-file ci-data/actions-job-snapshot.json \
--summary
# PR ROCm 7.2 CI reports using dynamic matrix
pr-rocm720-ci-reports:
name: PR ROCm720 - ${{ matrix.job_name }}
needs: [parse-workflows, fetch-actions-data]
if: ${{ !inputs.job_filter }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
job_name: ${{ fromJson(needs.parse-workflows.outputs.pr_rocm720_jobs) }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.10'
- name: Install dependencies
run: pip install tabulate
- name: Download Actions data snapshot
uses: actions/download-artifact@v4
with:
name: actions-job-snapshot
path: ci-data
- name: Generate PR ROCm 7.2 Report
timeout-minutes: 15
run: |
python scripts/ci/utils/query_job_status.py \
--repo ${{ github.repository }} \
--job "${{ matrix.job_name }}" \
--workflow "pr-test-amd-rocm720.yml" \
--hours ${{ inputs.hours || '24' }} \
--input-data-file ci-data/actions-job-snapshot.json \
--summary
# Nightly ROCm 7.2 reports using dynamic matrix
nightly-rocm720-reports:
name: Nightly ROCm720 - ${{ matrix.job_name }}
needs: [parse-workflows, fetch-actions-data]
if: ${{ !inputs.job_filter }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
job_name: ${{ fromJson(needs.parse-workflows.outputs.nightly_rocm720_jobs) }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.10'
- name: Install dependencies
run: pip install tabulate
- name: Download Actions data snapshot
uses: actions/download-artifact@v4
with:
name: actions-job-snapshot
path: ci-data
- name: Generate Nightly ROCm 7.2 Report
timeout-minutes: 15 timeout-minutes: 15
run: | run: |
python scripts/ci/utils/query_job_status.py \ python scripts/ci/utils/query_job_status.py \
@@ -326,6 +248,84 @@ jobs:
--input-data-file ci-data/actions-job-snapshot.json \ --input-data-file ci-data/actions-job-snapshot.json \
--summary --summary
# ROCm 7.0 rollback-shadow reports using dynamic matrix
pr-rocm700-ci-reports:
name: PR ROCm700 Shadow - ${{ matrix.job_name }}
needs: [parse-workflows, fetch-actions-data]
if: ${{ !inputs.job_filter }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
job_name: ${{ fromJson(needs.parse-workflows.outputs.pr_rocm700_jobs) }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.10'
- name: Install dependencies
run: pip install tabulate
- name: Download Actions data snapshot
uses: actions/download-artifact@v4
with:
name: actions-job-snapshot
path: ci-data
- name: Generate PR ROCm 7.0 Shadow Report
timeout-minutes: 15
run: |
python scripts/ci/utils/query_job_status.py \
--repo ${{ github.repository }} \
--job "${{ matrix.job_name }}" \
--workflow "pr-test-amd.yml" \
--hours ${{ inputs.hours || '24' }} \
--input-data-file ci-data/actions-job-snapshot.json \
--summary
# ROCm 7.0 rollback-shadow nightly reports using dynamic matrix
nightly-rocm700-reports:
name: Nightly ROCm700 Shadow - ${{ matrix.job_name }}
needs: [parse-workflows, fetch-actions-data]
if: ${{ !inputs.job_filter }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
job_name: ${{ fromJson(needs.parse-workflows.outputs.nightly_rocm700_jobs) }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.10'
- name: Install dependencies
run: pip install tabulate
- name: Download Actions data snapshot
uses: actions/download-artifact@v4
with:
name: actions-job-snapshot
path: ci-data
- name: Generate Nightly ROCm 7.0 Shadow Report
timeout-minutes: 15
run: |
python scripts/ci/utils/query_job_status.py \
--repo ${{ github.repository }} \
--job "${{ matrix.job_name }}" \
--workflow "nightly-test-amd.yml" \
--hours ${{ inputs.hours || '24' }} \
--input-data-file ci-data/actions-job-snapshot.json \
--summary
# MI355X 2N 1P1D disagg nightly reports using dynamic matrix # MI355X 2N 1P1D disagg nightly reports using dynamic matrix
nightly-disagg-reports: nightly-disagg-reports:
name: Nightly MI355X Disagg - ${{ matrix.job_name }} name: Nightly MI355X Disagg - ${{ matrix.job_name }}
.github/workflows/bot-bump-sglang-version.yml
+1 -1
View File
@@ -62,7 +62,7 @@ jobs:
run-nightly-tests-amd: run-nightly-tests-amd:
needs: bump-sglang-version needs: bump-sglang-version
uses: ./.github/workflows/nightly-test-amd.yml uses: ./.github/workflows/nightly-test-amd-rocm720.yml
with: with:
ref: ${{ needs.bump-sglang-version.outputs.branch_name }} ref: ${{ needs.bump-sglang-version.outputs.branch_name }}
secrets: inherit secrets: inherit
.github/workflows/nightly-test-amd.yml
+1 -1
View File
@@ -1,4 +1,4 @@
name: Nightly Test (AMD) name: Nightly Test (AMD ROCm 7.0)
on: on:
schedule: schedule:
.github/workflows/pr-test-amd-extra.yml
+13 -12
View File
@@ -37,13 +37,13 @@ on:
- mi300 - mi300
- mi325 - mi325
rocm_version: rocm_version:
description: 'ROCm container variant (empty = Dockerfile default; rocm720 = ROCm 7.2.0)' description: 'ROCm container variant (rocm720 = ROCm 7.2.0, the PR default; rocm700 = ROCm 7.0.0)'
required: false required: false
type: choice type: choice
default: '' default: rocm720
options: options:
- ''
- rocm720 - rocm720
- rocm700
aiter_ref: aiter_ref:
description: 'Override AITER commit (optional, leave empty to use Dockerfile default)' description: 'Override AITER commit (optional, leave empty to use Dockerfile default)'
required: false required: false
@@ -67,10 +67,10 @@ on:
type: string type: string
default: mi300 default: mi300
rocm_version: rocm_version:
description: 'ROCm container variant (empty = Dockerfile default; rocm720 = ROCm 7.2.0)' description: 'ROCm container variant (rocm720 = ROCm 7.2.0, the PR default; rocm700 = ROCm 7.0.0)'
required: false required: false
type: string type: string
default: '' default: rocm720
aiter_ref: aiter_ref:
description: 'Override AITER commit (optional, leave empty to use Dockerfile default)' description: 'Override AITER commit (optional, leave empty to use Dockerfile default)'
required: false required: false
@@ -130,7 +130,7 @@ jobs:
# setup across scarce AMD GPUs to shave only a couple minutes of test time, # setup across scarce AMD GPUs to shave only a couple minutes of test time,
# so one GPU running the whole suite sequentially is the better trade. # so one GPU running the whole suite sequentially is the better trade.
extra-a-test-1-gpu-small-amd: extra-a-test-1-gpu-small-amd:
name: ${{ format('extra-a-test-1-gpu-small-amd{0} (linux-{1}-1gpu-sglang)', inputs.rocm_version && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi300') }} name: ${{ format('extra-a-test-1-gpu-small-amd{0} (linux-{1}-1gpu-sglang)', inputs.rocm_version && inputs.rocm_version != 'rocm720' && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi300') }}
needs: [call-gate] needs: [call-gate]
if: ${{ !cancelled() && needs.call-gate.result == 'success' }} if: ${{ !cancelled() && needs.call-gate.result == 'success' }}
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
@@ -144,8 +144,9 @@ jobs:
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Start CI container - name: Start CI container
# `rocm_version` (e.g. rocm720) selects an alternate ROCm container; empty uses the Dockerfile default. # `inputs` is empty on pull_request events, so fall back explicitly to
run: bash scripts/ci/amd/amd_ci_start_container.sh ${{ inputs.rocm_version && format('--rocm-version {0}', inputs.rocm_version) || '' }} # the ROCm 7.2 PR default. Reusable callers can still request ROCm 7.0.
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ inputs.rocm_version || 'rocm720' }}
env: env:
GITHUB_WORKSPACE: ${{ github.workspace }} GITHUB_WORKSPACE: ${{ github.workspace }}
@@ -162,7 +163,7 @@ jobs:
# pool as small (AMD GPUs carry enough VRAM that "large" here is a CUDA # pool as small (AMD GPUs carry enough VRAM that "large" here is a CUDA
# memory-tier label, not a separate AMD runner pool). # memory-tier label, not a separate AMD runner pool).
extra-a-test-1-gpu-large-amd: extra-a-test-1-gpu-large-amd:
name: ${{ format('extra-a-test-1-gpu-large-amd{0} (linux-{1}-1gpu-sglang)', inputs.rocm_version && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi300') }} name: ${{ format('extra-a-test-1-gpu-large-amd{0} (linux-{1}-1gpu-sglang)', inputs.rocm_version && inputs.rocm_version != 'rocm720' && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi300') }}
needs: [call-gate] needs: [call-gate]
if: ${{ !cancelled() && needs.call-gate.result == 'success' }} if: ${{ !cancelled() && needs.call-gate.result == 'success' }}
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
@@ -176,7 +177,7 @@ jobs:
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Start CI container - name: Start CI container
run: bash scripts/ci/amd/amd_ci_start_container.sh ${{ inputs.rocm_version && format('--rocm-version {0}', inputs.rocm_version) || '' }} run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ inputs.rocm_version || 'rocm720' }}
env: env:
GITHUB_WORKSPACE: ${{ github.workspace }} GITHUB_WORKSPACE: ${{ github.workspace }}
@@ -192,7 +193,7 @@ jobs:
# Multi-GPU TP / PP / PD mock-model + kv_canary e2e tests. Mirrors CUDA's # Multi-GPU TP / PP / PD mock-model + kv_canary e2e tests. Mirrors CUDA's
# extra-a 2-gpu-large stage; runs on the 2-GPU AMD pool. # extra-a 2-gpu-large stage; runs on the 2-GPU AMD pool.
extra-a-test-2-gpu-large-amd: extra-a-test-2-gpu-large-amd:
name: ${{ format('extra-a-test-2-gpu-large-amd{0} (linux-{1}-2gpu-sglang)', inputs.rocm_version && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi300') }} name: ${{ format('extra-a-test-2-gpu-large-amd{0} (linux-{1}-2gpu-sglang)', inputs.rocm_version && inputs.rocm_version != 'rocm720' && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi300') }}
needs: [call-gate] needs: [call-gate]
if: ${{ !cancelled() && needs.call-gate.result == 'success' }} if: ${{ !cancelled() && needs.call-gate.result == 'success' }}
runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }} runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }}
@@ -206,7 +207,7 @@ jobs:
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Start CI container - name: Start CI container
run: bash scripts/ci/amd/amd_ci_start_container.sh ${{ inputs.rocm_version && format('--rocm-version {0}', inputs.rocm_version) || '' }} run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ inputs.rocm_version || 'rocm720' }}
env: env:
GITHUB_WORKSPACE: ${{ github.workspace }} GITHUB_WORKSPACE: ${{ github.workspace }}
.github/workflows/pr-test-amd-rocm720.yml
+354 -289
View File
File diff suppressed because it is too large Load Diff
.github/workflows/pr-test-amd.yml
+8 -16
View File
@@ -1,19 +1,16 @@
name: PR Test (AMD) name: PR Test ROCm 7.0 (AMD)
# Dynamic run-name for /rerun-stage commands to enable URL lookup # Dynamic run-name for /rerun-stage commands to enable URL lookup
# Format: "[stage-name] sha" for fork PRs, "[stage-name]" for non-fork, default for normal runs # Format: "[stage-name] sha" for fork PRs, "[stage-name]" for non-fork, default for normal runs
run-name: ${{ (inputs.target_stage || inputs.target_stage_select) && (inputs.pr_head_sha && format('[{0}] {1}', inputs.target_stage || inputs.target_stage_select, inputs.pr_head_sha) || format('[{0}]', inputs.target_stage || inputs.target_stage_select)) || '' }} run-name: ${{ (inputs.target_stage || inputs.target_stage_select) && (inputs.pr_head_sha && format('[{0}] {1}', inputs.target_stage || inputs.target_stage_select, inputs.pr_head_sha) || format('[{0}]', inputs.target_stage || inputs.target_stage_select)) || '' }}
on: on:
schedule: schedule:
- cron: '0 */12 * * *' # Run every 12 hours (UTC) # ROCm 7.0 rollback shadow. pr-test-amd-rocm720.yml is the PR gate; this
pull_request: # workflow now only proves ROCm 7.0 still works, so it fires once a day
paths: # alongside nightly-test-amd.yml and chases coverage rather than latency:
- "python/**" # no matrix parallelism caps, no stage fast-fail (the `schedule` escape in
- "scripts/ci/**" # each job's `if`), and continue-on-error via check-changes.
- "test/**" - cron: '30 17 * * *'
- "python/sglang/kernels/aot/**"
- ".github/workflows/pr-test-amd.yml"
- "docker/rocm.Dockerfile"
workflow_dispatch: workflow_dispatch:
inputs: inputs:
target_stage_select: target_stage_select:
@@ -223,6 +220,7 @@ jobs:
with: with:
ref: ${{ inputs.pr_head_sha || inputs.ref || '' }} ref: ${{ inputs.pr_head_sha || inputs.ref || '' }}
runner_arch: ${{ inputs.runner_arch || 'mi300' }} runner_arch: ${{ inputs.runner_arch || 'mi300' }}
rocm_version: rocm700
aiter_ref: ${{ inputs.aiter_ref }} aiter_ref: ${{ inputs.aiter_ref }}
continue_on_error: true continue_on_error: true
secrets: inherit secrets: inherit
@@ -479,7 +477,6 @@ jobs:
) )
strategy: strategy:
fail-fast: false fail-fast: false
max-parallel: 4
matrix: matrix:
part: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13] part: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
@@ -595,7 +592,6 @@ jobs:
) )
strategy: strategy:
fail-fast: false fail-fast: false
max-parallel: 1
matrix: matrix:
part: [0, 1, 2] part: [0, 1, 2]
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
@@ -636,7 +632,6 @@ jobs:
) )
strategy: strategy:
fail-fast: false fail-fast: false
max-parallel: 1
matrix: matrix:
part: [0, 1] part: [0, 1]
runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }} runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }}
@@ -677,7 +672,6 @@ jobs:
) )
strategy: strategy:
fail-fast: false fail-fast: false
max-parallel: 1
matrix: matrix:
part: [0, 1, 2, 3] part: [0, 1, 2, 3]
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
@@ -808,7 +802,6 @@ jobs:
) )
strategy: strategy:
fail-fast: false fail-fast: false
max-parallel: 1
matrix: matrix:
part: [0, 1, 2] # 3 partitions: 2 parametrized + 1 standalone (single_test_file/test_disagg_server.py) part: [0, 1, 2] # 3 partitions: 2 parametrized + 1 standalone (single_test_file/test_disagg_server.py)
runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }} runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }}
@@ -1082,7 +1075,6 @@ jobs:
RUNNER_LABELS: ${{ format('linux-{0}-8gpu-sglang', inputs.runner_arch || 'mi300') }} RUNNER_LABELS: ${{ format('linux-{0}-8gpu-sglang', inputs.runner_arch || 'mi300') }}
strategy: strategy:
fail-fast: false fail-fast: false
max-parallel: 2
matrix: matrix:
part: [0, 1, 2, 3] part: [0, 1, 2, 3]
runs-on: ${{ format('linux-{0}-8gpu-sglang', inputs.runner_arch || 'mi300') }} runs-on: ${{ format('linux-{0}-8gpu-sglang', inputs.runner_arch || 'mi300') }}
.github/workflows/release-branch-cut.yml
+2 -2
View File
@@ -174,7 +174,7 @@ jobs:
run-pr-tests-amd: run-pr-tests-amd:
needs: cut-release-branch needs: cut-release-branch
uses: ./.github/workflows/pr-test-amd.yml uses: ./.github/workflows/pr-test-amd-rocm720.yml
with: with:
ref: ${{ needs.cut-release-branch.outputs.branch_name }} ref: ${{ needs.cut-release-branch.outputs.branch_name }}
run_all_tests: true run_all_tests: true
@@ -213,7 +213,7 @@ jobs:
run-nightly-tests-amd: run-nightly-tests-amd:
needs: cut-release-branch needs: cut-release-branch
uses: ./.github/workflows/nightly-test-amd.yml uses: ./.github/workflows/nightly-test-amd-rocm720.yml
with: with:
ref: ${{ needs.cut-release-branch.outputs.branch_name }} ref: ${{ needs.cut-release-branch.outputs.branch_name }}
secrets: inherit secrets: inherit
scripts/ci/amd/amd_ci_install_dependency.sh
+25 -1
View File
@@ -238,6 +238,19 @@ DOCKERFILE="docker/rocm.Dockerfile"
GPU_ARCH="${GPU_ARCH:-mi30x}" GPU_ARCH="${GPU_ARCH:-mi30x}"
echo "[CI-AITER-CHECK] Runner GPU_ARCH=${GPU_ARCH}" echo "[CI-AITER-CHECK] Runner GPU_ARCH=${GPU_ARCH}"
# ROCm 7.0 keeps the Triton its base image ships; later ROCm images run on the
# Triton AITER pins, so a rebuilt AITER has to bring its own along.
IMAGE_HIP_VERSION=$(docker exec ci_sglang python3 -c 'import torch; print(torch.version.hip or "")')
case "${IMAGE_HIP_VERSION}" in
7.0*) INSTALL_AITER_TRITON="false" ;;
7.*) INSTALL_AITER_TRITON="true" ;;
*)
echo "[CI-AITER-CHECK] ERROR: Unsupported or empty HIP version: '${IMAGE_HIP_VERSION}'"
exit 1
;;
esac
echo "[CI-AITER-CHECK] Container HIP=${IMAGE_HIP_VERSION}, install AITER's Triton on rebuild=${INSTALL_AITER_TRITON}"
############################################# #############################################
# 1. Extract AITER_COMMIT from correct Dockerfile block # 1. Extract AITER_COMMIT from correct Dockerfile block
############################################# #############################################
@@ -292,7 +305,6 @@ else
NEED_REBUILD="true" NEED_REBUILD="true"
fi fi
############################################# #############################################
# 4. Rebuild AITER if needed # 4. Rebuild AITER if needed
############################################# #############################################
@@ -329,6 +341,18 @@ if [[ "${NEED_REBUILD}" == "true" ]]; then
fi fi
echo "[CI-AITER-CHECK] GPU_ARCH_LIST=${GPU_ARCH_LIST}" echo "[CI-AITER-CHECK] GPU_ARCH_LIST=${GPU_ARCH_LIST}"
# Run the installer here rather than letting setup.py do it: setup.py
# swallows its errors, and the AITER_USE_SYSTEM_TRITON=1 below then keeps
# whatever Triton is already installed. Doing it up front fails closed.
if [[ "${INSTALL_AITER_TRITON}" == "true" ]]; then
docker exec ci_sglang bash -c "
set -euo pipefail
cd /sgl-workspace/aiter
test -f .github/scripts/install_triton.sh
bash .github/scripts/install_triton.sh
"
fi
# build AITER # build AITER
docker exec ci_sglang bash -c " docker exec ci_sglang bash -c "
cd /sgl-workspace/aiter && \ cd /sgl-workspace/aiter && \
scripts/ci/utils/query_job_status.py
+4 -4
View File
@@ -6,10 +6,10 @@ Usage:
# Per-job reports (original mode) # Per-job reports (original mode)
python scripts/ci/utils/query_job_status.py --job "stage-c-test-large-8-gpu-amd-mi35x" python scripts/ci/utils/query_job_status.py --job "stage-c-test-large-8-gpu-amd-mi35x"
python scripts/ci/utils/query_job_status.py --job "stage-c-test-large-8-gpu-amd-mi35x" --hours 48 python scripts/ci/utils/query_job_status.py --job "stage-c-test-large-8-gpu-amd-mi35x" --hours 48
python scripts/ci/utils/query_job_status.py --job "stage-c-test-large-8-gpu-amd-mi35x" --workflow "pr-test-amd.yml" --input-data-file actions-job-snapshot.json --summary python scripts/ci/utils/query_job_status.py --job "stage-c-test-large-8-gpu-amd-mi35x-rocm720" --workflow "pr-test-amd-rocm720.yml" --input-data-file actions-job-snapshot.json --summary
# Runner fleet report (cross-workflow runner analytics) # Runner fleet report (cross-workflow runner analytics)
python scripts/ci/utils/query_job_status.py --runner-report --workflow "pr-test-amd.yml,nightly-test-amd.yml" --hours 24 python scripts/ci/utils/query_job_status.py --runner-report --workflow "pr-test-amd-rocm720.yml,nightly-test-amd-rocm720.yml" --hours 24
python scripts/ci/utils/query_job_status.py --runner-report --workflow "pr-test-amd.yml,nightly-test-amd.yml,pr-test-amd-rocm720.yml,nightly-test-amd-rocm720.yml" --summary python scripts/ci/utils/query_job_status.py --runner-report --workflow "pr-test-amd.yml,nightly-test-amd.yml,pr-test-amd-rocm720.yml,nightly-test-amd-rocm720.yml" --summary
python scripts/ci/utils/query_job_status.py --workflow "pr-test-amd.yml,nightly-test-amd.yml,pr-test-amd-rocm720.yml,nightly-test-amd-rocm720.yml" --dump-data-file actions-job-snapshot.json python scripts/ci/utils/query_job_status.py --workflow "pr-test-amd.yml,nightly-test-amd.yml,pr-test-amd-rocm720.yml,nightly-test-amd-rocm720.yml" --dump-data-file actions-job-snapshot.json
@@ -1719,8 +1719,8 @@ def main():
) )
parser.add_argument( parser.add_argument(
"--workflow", "--workflow",
default="pr-test-amd.yml", default="pr-test-amd-rocm720.yml",
help="Workflow file name, or comma-separated list for --runner-report (default: pr-test-amd.yml)", help="Workflow file name, or comma-separated list for --runner-report (default: pr-test-amd-rocm720.yml)",
) )
parser.add_argument( parser.add_argument(
"--hours", "--hours",
scripts/ci/utils/slash_command_handler.py
+1 -1
View File
@@ -1401,7 +1401,7 @@ def main():
"- **Full CI rerun** (with extra coverage): add the `run-ci` or " "- **Full CI rerun** (with extra coverage): add the `run-ci` or "
"`run-ci-extra` label and push a new commit (or use `/tag-and-rerun-ci`).\n\n" "`run-ci-extra` label and push a new commit (or use `/tag-and-rerun-ci`).\n\n"
"**AMD CI**: stage-level dispatch is still available via " "**AMD CI**: stage-level dispatch is still available via "
"Actions UI → *PR Test (AMD)* / *PR Test ROCm 7.2 (AMD)* → " "Actions UI → *PR Test ROCm 7.2 (AMD)* (default) / *PR Test ROCm 7.0 (AMD)* → "
"*Run workflow* → pick a stage from the dropdown." "*Run workflow* → pick a stage from the dropdown."
) )
scripts/ci_monitor/ci_failures_analysis.py
+5 -4
View File
@@ -47,6 +47,7 @@ class SGLangFailuresAnalyzer:
"check-changes", "check-changes",
"pr-test-finish", # Nvidia workflow teardown "pr-test-finish", # Nvidia workflow teardown
"pr-test-amd-finish", # AMD workflow teardown "pr-test-amd-finish", # AMD workflow teardown
"pr-test-amd-rocm720-finish", # Default AMD ROCm 7.2 teardown
"call-gate", "call-gate",
"pr-gate", "pr-gate",
"check-all-jobs", "check-all-jobs",
@@ -2456,7 +2457,7 @@ def main():
# These 4 don't have scheduled events, so filter by main branch instead # These 4 don't have scheduled events, so filter by main branch instead
pr_test_amd_scheduled_runs = analyzer.get_recent_runs( pr_test_amd_scheduled_runs = analyzer.get_recent_runs(
limit=pr_test_scheduled_limit, limit=pr_test_scheduled_limit,
workflow_filter=["pr-test-amd.yml"], workflow_filter=["pr-test-amd-rocm720.yml"],
filters={"branch": "main"}, filters={"branch": "main"},
) )
pr_test_xeon_scheduled_runs = analyzer.get_recent_runs( pr_test_xeon_scheduled_runs = analyzer.get_recent_runs(
@@ -2483,7 +2484,7 @@ def main():
) )
nightly_amd_scheduled_runs = analyzer.get_recent_runs( nightly_amd_scheduled_runs = analyzer.get_recent_runs(
limit=nightly_scheduled_limit, limit=nightly_scheduled_limit,
workflow_filter=["nightly-test-amd.yml"], workflow_filter=["nightly-test-amd-rocm720.yml"],
filters={"event": "schedule"}, filters={"event": "schedule"},
) )
nightly_intel_scheduled_runs = analyzer.get_recent_runs( nightly_intel_scheduled_runs = analyzer.get_recent_runs(
@@ -2505,7 +2506,7 @@ def main():
) )
pr_test_amd_general_runs = analyzer.get_recent_runs( pr_test_amd_general_runs = analyzer.get_recent_runs(
limit=args.limit, limit=args.limit,
workflow_filter=["pr-test-amd.yml"], workflow_filter=["pr-test-amd-rocm720.yml"],
) )
pr_test_xeon_general_runs = analyzer.get_recent_runs( pr_test_xeon_general_runs = analyzer.get_recent_runs(
limit=args.limit, limit=args.limit,
@@ -2527,7 +2528,7 @@ def main():
) )
nightly_amd_general_runs = analyzer.get_recent_runs( nightly_amd_general_runs = analyzer.get_recent_runs(
limit=args.limit, limit=args.limit,
workflow_filter=["nightly-test-amd.yml"], workflow_filter=["nightly-test-amd-rocm720.yml"],
) )
nightly_intel_general_runs = analyzer.get_recent_runs( nightly_intel_general_runs = analyzer.get_recent_runs(
limit=args.limit, limit=args.limit,