[AMD][CI] Consolidate AMD workflows and retire ROCm 7.0 CI (#38632)
This commit is contained in:
Bingxu Chen
@@ -30,7 +30,7 @@ Examples:
|
||||
$babysit-pr-to-pass-ci
|
||||
$babysit-pr-to-pass-ci 12345
|
||||
$babysit-pr-to-pass-ci https://github.com/sgl-project/sglang/pull/12345 pr-test-extra.yml
|
||||
$babysit-pr-to-pass-ci 12345 --only pr-test-amd-rocm720.yml
|
||||
$babysit-pr-to-pass-ci 12345 --only pr-test-amd.yml
|
||||
```
|
||||
|
||||
## Start or continue the durable goal
|
||||
|
||||
@@ -12,7 +12,7 @@ on:
|
||||
type: string
|
||||
default: 'main'
|
||||
job_filter:
|
||||
description: 'Workflows: nightly-amd, nightly-amd-rocm720, pr-test-amd (ROCm 7.0 shadow), pr-test-amd-rocm720 (ROCm 10 gate). Default: all'
|
||||
description: 'Workflows: nightly-amd, pr-test-amd. Default: all'
|
||||
required: false
|
||||
type: string
|
||||
default: 'all'
|
||||
@@ -38,9 +38,7 @@ jobs:
|
||||
outputs:
|
||||
aiter_sha: ${{ steps.resolve.outputs.sha }}
|
||||
run_nightly_amd: ${{ steps.parse.outputs.run_nightly_amd }}
|
||||
run_nightly_amd_rocm720: ${{ steps.parse.outputs.run_nightly_amd_rocm720 }}
|
||||
run_pr_test_amd: ${{ steps.parse.outputs.run_pr_test_amd }}
|
||||
run_pr_test_amd_rocm720: ${{ steps.parse.outputs.run_pr_test_amd_rocm720 }}
|
||||
steps:
|
||||
- name: Resolve AITER commit
|
||||
id: resolve
|
||||
@@ -73,63 +71,39 @@ jobs:
|
||||
|
||||
if [[ "$FILTER" == "all" ]]; then
|
||||
echo "run_nightly_amd=true" >> $GITHUB_OUTPUT
|
||||
echo "run_nightly_amd_rocm720=true" >> $GITHUB_OUTPUT
|
||||
echo "run_pr_test_amd=true" >> $GITHUB_OUTPUT
|
||||
echo "run_pr_test_amd_rocm720=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
# Wrap with commas for exact substring matching (avoids "nightly-amd" matching "nightly-amd-rocm720")
|
||||
# Wrap with commas for exact substring matching.
|
||||
PADDED=",${FILTER// /},"
|
||||
echo "run_nightly_amd=$(echo "$PADDED" | grep -q ',nightly-amd,' && echo true || echo false)" >> $GITHUB_OUTPUT
|
||||
echo "run_nightly_amd_rocm720=$(echo "$PADDED" | grep -q ',nightly-amd-rocm720,' && echo true || echo false)" >> $GITHUB_OUTPUT
|
||||
echo "run_pr_test_amd=$(echo "$PADDED" | grep -q ',pr-test-amd,' && echo true || echo false)" >> $GITHUB_OUTPUT
|
||||
echo "run_pr_test_amd_rocm720=$(echo "$PADDED" | grep -q ',pr-test-amd-rocm720,' && echo true || echo false)" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
echo "### Job Filter" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Filter:** \`${FILTER}\`" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
call-nightly-amd:
|
||||
name: Nightly AMD (all ROCm versions)
|
||||
if: needs.resolve-aiter.outputs.run_nightly_amd == 'true'
|
||||
needs: resolve-aiter
|
||||
uses: ./.github/workflows/nightly-test-amd.yml
|
||||
secrets: inherit
|
||||
with:
|
||||
ref: amd/aiter-ci
|
||||
aiter_ref: ${{ needs.resolve-aiter.outputs.aiter_sha }}
|
||||
job_filter: 'all'
|
||||
continue_on_error: ${{ inputs.continue_on_error == '' && true || inputs.continue_on_error }}
|
||||
|
||||
call-nightly-amd-rocm720:
|
||||
if: needs.resolve-aiter.outputs.run_nightly_amd_rocm720 == 'true'
|
||||
needs: resolve-aiter
|
||||
uses: ./.github/workflows/nightly-test-amd-rocm720.yml
|
||||
secrets: inherit
|
||||
with:
|
||||
ref: amd/aiter-ci
|
||||
rocm_version: rocm10
|
||||
rocm_version: all
|
||||
aiter_ref: ${{ needs.resolve-aiter.outputs.aiter_sha }}
|
||||
job_filter: 'all'
|
||||
continue_on_error: ${{ inputs.continue_on_error == '' && true || inputs.continue_on_error }}
|
||||
|
||||
call-pr-test-amd:
|
||||
name: PR Test AMD (all ROCm versions)
|
||||
if: needs.resolve-aiter.outputs.run_pr_test_amd == 'true'
|
||||
needs: resolve-aiter
|
||||
uses: ./.github/workflows/pr-test-amd.yml
|
||||
secrets: inherit
|
||||
with:
|
||||
ref: amd/aiter-ci
|
||||
run_all_tests: true
|
||||
aiter_ref: ${{ needs.resolve-aiter.outputs.aiter_sha }}
|
||||
continue_on_error: ${{ inputs.continue_on_error == '' && true || inputs.continue_on_error }}
|
||||
|
||||
call-pr-test-amd-rocm720:
|
||||
if: needs.resolve-aiter.outputs.run_pr_test_amd_rocm720 == 'true'
|
||||
needs: resolve-aiter
|
||||
uses: ./.github/workflows/pr-test-amd-rocm720.yml
|
||||
secrets: inherit
|
||||
with:
|
||||
ref: amd/aiter-ci
|
||||
rocm_version: rocm10
|
||||
rocm_version: all
|
||||
run_all_tests: true
|
||||
aiter_ref: ${{ needs.resolve-aiter.outputs.aiter_sha }}
|
||||
continue_on_error: ${{ inputs.continue_on_error == '' && true || inputs.continue_on_error }}
|
||||
@@ -139,9 +113,7 @@ jobs:
|
||||
needs:
|
||||
- resolve-aiter
|
||||
- call-nightly-amd
|
||||
- call-nightly-amd-rocm720
|
||||
- call-pr-test-amd
|
||||
- call-pr-test-amd-rocm720
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Summary
|
||||
@@ -154,10 +126,8 @@ jobs:
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| Workflow | Result |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "|----------|--------|" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| Nightly AMD (AITER Latest) | \`${{ needs.call-nightly-amd.result }}\` |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| Nightly AMD ROCm 10 | \`${{ needs.call-nightly-amd-rocm720.result }}\` |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| PR Test AMD ROCm 7.0 Shadow | \`${{ needs.call-pr-test-amd.result }}\` |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| PR Test AMD ROCm 10 Gate | \`${{ needs.call-pr-test-amd-rocm720.result }}\` |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| Nightly AMD ROCm10 + ROCm7.2.4 + ROCm7.2.0 | \`${{ needs.call-nightly-amd.result }}\` |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| PR Test AMD ROCm10 + ROCm7.2.4 + ROCm7.2.0 Gate | \`${{ needs.call-pr-test-amd.result }}\` |" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
- name: Check if any job failed
|
||||
run: |
|
||||
|
||||
@@ -53,9 +53,9 @@ jobs:
|
||||
id: select-workflows
|
||||
run: |
|
||||
if [[ -n "${{ inputs.job_filter }}" ]]; then
|
||||
echo "workflows=pr-test-amd-rocm720.yml" >> "$GITHUB_OUTPUT"
|
||||
echo "workflows=pr-test-amd.yml" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "workflows=pr-test-amd.yml,nightly-test-amd.yml,pr-test-amd-rocm720.yml,nightly-test-amd-rocm720.yml,nightly-amd-mi355x-disagg.yml" >> "$GITHUB_OUTPUT"
|
||||
echo "workflows=pr-test-amd.yml,nightly-test-amd.yml,nightly-amd-mi355x-disagg.yml" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Fetch Actions data snapshot
|
||||
@@ -106,7 +106,7 @@ jobs:
|
||||
python scripts/ci/utils/query_job_status.py \
|
||||
--repo ${{ github.repository }} \
|
||||
--job "${{ inputs.job_filter }}" \
|
||||
--workflow "pr-test-amd-rocm720.yml" \
|
||||
--workflow "pr-test-amd.yml" \
|
||||
--hours ${{ inputs.hours || '24' }} \
|
||||
--input-data-file ci-data/actions-job-snapshot.json \
|
||||
--summary
|
||||
@@ -119,8 +119,6 @@ jobs:
|
||||
outputs:
|
||||
pr_jobs: ${{ steps.parse.outputs.pr_jobs }}
|
||||
nightly_jobs: ${{ steps.parse.outputs.nightly_jobs }}
|
||||
pr_rocm700_jobs: ${{ steps.parse.outputs.pr_rocm700_jobs }}
|
||||
nightly_rocm700_jobs: ${{ steps.parse.outputs.nightly_rocm700_jobs }}
|
||||
disagg_jobs: ${{ steps.parse.outputs.disagg_jobs }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -131,21 +129,18 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# Parse the default ROCm 7.2 PR gate (exclude utility jobs)
|
||||
# Parse the canonical multi-version AMD PR gate (exclude utility jobs)
|
||||
# Excluded: call-gate, check-changes, finish, cancel, check-all-jobs,
|
||||
# plus the stage waits -- they run on ubuntu-latest, so the
|
||||
# self-hosted-only snapshot has nothing for them. Keep the extra call:
|
||||
# its delegated GPU jobs use the explicit caller name as their prefix.
|
||||
# Job ids keep the -rocm720 lane suffix but display names no longer do,
|
||||
# so strip it to get the prefix query_job_status.py matches against.
|
||||
pr_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/pr-test-amd-rocm720.yml | \
|
||||
grep -v -E '^(call-gate|check-changes|pr-test-amd-rocm720-finish|cancel|check-all-jobs|wait-for-stage-[ab]-amd-rocm720)$' | \
|
||||
sed 's/-rocm720$//' | \
|
||||
pr_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/pr-test-amd.yml | \
|
||||
grep -v -E '^(call-gate|check-changes|pr-test-amd-finish|cancel|check-all-jobs|wait-for-stage-[ab]-amd)$' | \
|
||||
jq -R -s -c 'split("\n") | map(select(length > 0))')
|
||||
echo "pr_jobs=$pr_jobs" >> $GITHUB_OUTPUT
|
||||
echo "PR jobs: $pr_jobs"
|
||||
|
||||
# Parse the default ROCm 7.2 nightly (exclude utility jobs)
|
||||
# Parse the canonical multi-version AMD nightly (exclude utility jobs)
|
||||
# Excluded: check-all-jobs
|
||||
# One nightly run covers every flavor, so ask for each by its full name:
|
||||
# a stem-only prefix matches every flavor and merges them into a single
|
||||
@@ -153,33 +148,15 @@ jobs:
|
||||
# names carry the runner too, so pair each job id with its runs-on.
|
||||
# Keep in sync with the rocm_version matrix in the nightly workflow.
|
||||
rocm_flavors="rocm10 rocm724 rocm720"
|
||||
nightly_jobs=$(yq -r '.jobs | to_entries[] | .key + " " + .value."runs-on"' .github/workflows/nightly-test-amd-rocm720.yml | \
|
||||
nightly_jobs=$(yq -r '.jobs | to_entries[] | .key + " " + .value."runs-on"' .github/workflows/nightly-test-amd.yml | \
|
||||
grep -v -E '^check-all-jobs ' | \
|
||||
while read -r id runner; do
|
||||
stem=${id%-rocm720}
|
||||
for f in $rocm_flavors; do echo "$stem ($f, $runner)"; done
|
||||
for f in $rocm_flavors; do echo "$id ($f, $runner)"; done
|
||||
done | \
|
||||
jq -R -s -c 'split("\n") | map(select(length > 0))')
|
||||
echo "nightly_jobs=$nightly_jobs" >> $GITHUB_OUTPUT
|
||||
echo "Nightly jobs: $nightly_jobs"
|
||||
|
||||
# Parse the ROCm 7.0 rollback shadow (exclude utility jobs)
|
||||
# Excluded: call-gate, check-changes, finish, and the stage waits.
|
||||
# Keep the reusable extra caller so its nested GPU jobs remain covered.
|
||||
pr_rocm700_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/pr-test-amd.yml | \
|
||||
grep -v -E '^(call-gate|check-changes|pr-test-amd-finish|wait-for-stage-[ab]-amd)$' | \
|
||||
jq -R -s -c 'split("\n") | map(select(length > 0))')
|
||||
echo "pr_rocm700_jobs=$pr_rocm700_jobs" >> $GITHUB_OUTPUT
|
||||
echo "PR ROCm 7.0 shadow jobs: $pr_rocm700_jobs"
|
||||
|
||||
# Parse the ROCm 7.0 rollback shadow nightly (exclude utility jobs)
|
||||
# Excluded: check-all-jobs
|
||||
nightly_rocm700_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/nightly-test-amd.yml | \
|
||||
grep -v -E '^(check-all-jobs)$' | \
|
||||
jq -R -s -c 'split("\n") | map(select(length > 0))')
|
||||
echo "nightly_rocm700_jobs=$nightly_rocm700_jobs" >> $GITHUB_OUTPUT
|
||||
echo "Nightly ROCm 7.0 shadow jobs: $nightly_rocm700_jobs"
|
||||
|
||||
# Parse nightly-amd-mi355x-disagg.yml (exclude utility jobs)
|
||||
# Excluded: setup, collect-results -- both run on ubuntu-latest and the
|
||||
# snapshot only keeps self-hosted jobs, so they would report as empty.
|
||||
@@ -223,7 +200,7 @@ jobs:
|
||||
python scripts/ci/utils/query_job_status.py \
|
||||
--repo ${{ github.repository }} \
|
||||
--job "${{ matrix.job_name }}" \
|
||||
--workflow "pr-test-amd-rocm720.yml" \
|
||||
--workflow "pr-test-amd.yml" \
|
||||
--hours ${{ inputs.hours || '24' }} \
|
||||
--input-data-file ci-data/actions-job-snapshot.json \
|
||||
--summary
|
||||
@@ -257,84 +234,6 @@ jobs:
|
||||
path: ci-data
|
||||
|
||||
- name: Generate Nightly Report
|
||||
timeout-minutes: 15
|
||||
run: |
|
||||
python scripts/ci/utils/query_job_status.py \
|
||||
--repo ${{ github.repository }} \
|
||||
--job "${{ matrix.job_name }}" \
|
||||
--workflow "nightly-test-amd-rocm720.yml" \
|
||||
--hours ${{ inputs.hours || '24' }} \
|
||||
--input-data-file ci-data/actions-job-snapshot.json \
|
||||
--summary
|
||||
|
||||
# ROCm 7.0 rollback-shadow reports using dynamic matrix
|
||||
pr-rocm700-ci-reports:
|
||||
name: PR ROCm700 Shadow - ${{ matrix.job_name }}
|
||||
needs: [parse-workflows, fetch-actions-data]
|
||||
if: ${{ !inputs.job_filter }}
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
job_name: ${{ fromJson(needs.parse-workflows.outputs.pr_rocm700_jobs) }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.10'
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install tabulate
|
||||
|
||||
- name: Download Actions data snapshot
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: actions-job-snapshot
|
||||
path: ci-data
|
||||
|
||||
- name: Generate PR ROCm 7.0 Shadow Report
|
||||
timeout-minutes: 15
|
||||
run: |
|
||||
python scripts/ci/utils/query_job_status.py \
|
||||
--repo ${{ github.repository }} \
|
||||
--job "${{ matrix.job_name }}" \
|
||||
--workflow "pr-test-amd.yml" \
|
||||
--hours ${{ inputs.hours || '24' }} \
|
||||
--input-data-file ci-data/actions-job-snapshot.json \
|
||||
--summary
|
||||
|
||||
# ROCm 7.0 rollback-shadow nightly reports using dynamic matrix
|
||||
nightly-rocm700-reports:
|
||||
name: Nightly ROCm700 Shadow - ${{ matrix.job_name }}
|
||||
needs: [parse-workflows, fetch-actions-data]
|
||||
if: ${{ !inputs.job_filter }}
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
job_name: ${{ fromJson(needs.parse-workflows.outputs.nightly_rocm700_jobs) }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.10'
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install tabulate
|
||||
|
||||
- name: Download Actions data snapshot
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: actions-job-snapshot
|
||||
path: ci-data
|
||||
|
||||
- name: Generate Nightly ROCm 7.0 Shadow Report
|
||||
timeout-minutes: 15
|
||||
run: |
|
||||
python scripts/ci/utils/query_job_status.py \
|
||||
@@ -414,7 +313,7 @@ jobs:
|
||||
python scripts/ci/utils/query_job_status.py \
|
||||
--repo ${{ github.repository }} \
|
||||
--runner-report \
|
||||
--workflow "pr-test-amd.yml,nightly-test-amd.yml,pr-test-amd-rocm720.yml,nightly-test-amd-rocm720.yml,nightly-amd-mi355x-disagg.yml" \
|
||||
--workflow "pr-test-amd.yml,nightly-test-amd.yml,nightly-amd-mi355x-disagg.yml" \
|
||||
--hours ${{ inputs.hours || '24' }} \
|
||||
--input-data-file ci-data/actions-job-snapshot.json \
|
||||
--summary
|
||||
|
||||
@@ -62,7 +62,7 @@ jobs:
|
||||
|
||||
run-nightly-tests-amd:
|
||||
needs: bump-sglang-version
|
||||
uses: ./.github/workflows/nightly-test-amd-rocm720.yml
|
||||
uses: ./.github/workflows/nightly-test-amd.yml
|
||||
with:
|
||||
ref: ${{ needs.bump-sglang-version.outputs.branch_name }}
|
||||
rocm_version: rocm10
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -14,7 +14,7 @@ on:
|
||||
pull_request_target:
|
||||
types: [opened, synchronize, reopened, labeled, unlabeled]
|
||||
workflow_run:
|
||||
workflows: ["PR Test Base", "PR Test Extra", "PR Test ROCm 7.2 (AMD)"]
|
||||
workflows: ["PR Test Base", "PR Test Extra", "PR Test (AMD)"]
|
||||
types: [requested, completed]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
@@ -163,7 +163,7 @@ jobs:
|
||||
'pr-test-extra.yml',
|
||||
(hasCI && hasExtra) ? labelOnOpts : labelOffOpts,
|
||||
),
|
||||
findRun('pr-test-amd-rocm720.yml', labelOffOpts),
|
||||
findRun('pr-test-amd.yml', labelOffOpts),
|
||||
]);
|
||||
|
||||
// Skipped run = "no real run" -- happens when a label is added
|
||||
@@ -214,8 +214,8 @@ jobs:
|
||||
const ptEnd = '<!-- slot:pr-test:end -->';
|
||||
const peStart = '<!-- slot:pr-test-extra:start -->';
|
||||
const peEnd = '<!-- slot:pr-test-extra:end -->';
|
||||
const amdStart = '<!-- slot:pr-test-amd-rocm720:start -->';
|
||||
const amdEnd = '<!-- slot:pr-test-amd-rocm720:end -->';
|
||||
const amdStart = '<!-- slot:pr-test-amd:start -->';
|
||||
const amdEnd = '<!-- slot:pr-test-amd:end -->';
|
||||
|
||||
const newBlock = [
|
||||
outerStart,
|
||||
|
||||
@@ -45,7 +45,6 @@ on:
|
||||
- rocm10
|
||||
- rocm724
|
||||
- rocm720
|
||||
- rocm700
|
||||
aiter_ref:
|
||||
description: 'Override AITER commit (optional, leave empty to use Dockerfile default)'
|
||||
required: false
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,18 +1,32 @@
|
||||
name: PR Test ROCm 7.0 (AMD)
|
||||
name: PR Test (AMD)
|
||||
# Dynamic run-name for manual stage dispatches to enable URL lookup
|
||||
# Format: "[stage-name] sha" for fork PRs, "[stage-name]" for non-fork, default for normal runs
|
||||
run-name: ${{ (inputs.target_stage || inputs.target_stage_select) && (inputs.pr_head_sha && format('[{0}] {1}', inputs.target_stage || inputs.target_stage_select, inputs.pr_head_sha) || format('[{0}]', inputs.target_stage || inputs.target_stage_select)) || '' }}
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# ROCm 7.0 rollback shadow. pr-test-amd-rocm720.yml is the PR gate; this
|
||||
# workflow now only proves ROCm 7.0 still works, so it fires every two days
|
||||
# alongside nightly-test-amd.yml and chases coverage rather than latency:
|
||||
# no matrix parallelism caps, no stage fast-fail (the `schedule` escape in
|
||||
# each job's `if`), and continue-on-error via check-changes.
|
||||
- cron: '30 17 */2 * *'
|
||||
- cron: '0 */12 * * *' # rocm10: twice daily (UTC)
|
||||
- cron: '30 17 * * *' # rocm724/rocm720: once daily (UTC)
|
||||
pull_request:
|
||||
paths:
|
||||
- "python/**"
|
||||
- "scripts/ci/**"
|
||||
- "test/**"
|
||||
- "python/sglang/kernels/aot/**"
|
||||
- ".github/workflows/pr-test-amd.yml"
|
||||
- "docker/rocm.Dockerfile"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
rocm_version:
|
||||
description: 'ROCm image version ("all" runs rocm10, rocm724, and rocm720)'
|
||||
required: false
|
||||
type: choice
|
||||
default: rocm10
|
||||
options:
|
||||
- all
|
||||
- rocm10
|
||||
- rocm724
|
||||
- rocm720
|
||||
target_stage_select:
|
||||
description: "Select a stage to run from dropdown (leave empty for auto-detect)"
|
||||
required: false
|
||||
@@ -33,6 +47,8 @@ on:
|
||||
- multimodal-gen-test-1-gpu-amd
|
||||
- multimodal-gen-test-2-gpu-amd
|
||||
- stage-c-test-4-gpu-amd
|
||||
- stage-c-dsv4-flash-fp4-fp8-amd-mi35x
|
||||
- stage-c-dsv4-pro-fp4-amd-mi35x
|
||||
- stage-c-test-large-8-gpu-amd
|
||||
- stage-c-test-large-8-gpu-amd-mi35x
|
||||
- stage-b-test-large-8-gpu-mi35x-disaggregation-amd
|
||||
@@ -51,6 +67,11 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
default: ''
|
||||
amd_ci_image:
|
||||
description: 'Override AMD CI Docker image (optional, e.g. rocm/sgl-dev:tag)'
|
||||
required: false
|
||||
type: string
|
||||
default: ''
|
||||
continue_on_error:
|
||||
description: 'Continue on error (do not fail the workflow on test failures)'
|
||||
required: false
|
||||
@@ -71,6 +92,11 @@ on:
|
||||
default: false
|
||||
workflow_call:
|
||||
inputs:
|
||||
rocm_version:
|
||||
description: 'ROCm image version ("all" runs rocm10, rocm724, and rocm720)'
|
||||
required: false
|
||||
type: string
|
||||
default: rocm10
|
||||
ref:
|
||||
description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
|
||||
required: false
|
||||
@@ -91,6 +117,11 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
default: ''
|
||||
amd_ci_image:
|
||||
description: 'Override AMD CI Docker image (optional, e.g. rocm/sgl-dev:tag)'
|
||||
required: false
|
||||
type: string
|
||||
default: ''
|
||||
continue_on_error:
|
||||
description: 'Continue on error (do not fail the workflow on test failures)'
|
||||
required: false
|
||||
@@ -110,6 +141,7 @@ permissions:
|
||||
|
||||
env:
|
||||
AITER_COMMIT_OVERRIDE: ${{ inputs.aiter_ref }}
|
||||
AMD_CI_IMAGE: ${{ inputs.amd_ci_image }}
|
||||
DOCKERHUB_AMD_USERNAME: ${{ secrets.DOCKERHUB_AMD_USERNAME }}
|
||||
DOCKERHUB_AMD_TOKEN: ${{ secrets.DOCKERHUB_AMD_TOKEN }}
|
||||
|
||||
@@ -134,6 +166,10 @@ jobs:
|
||||
jit_kernel: ${{ steps.filter.outputs.jit_kernel || steps.run-mode.outputs.run_all_tests }}
|
||||
multimodal_gen: ${{ steps.filter.outputs.multimodal_gen || steps.run-mode.outputs.run_all_tests }}
|
||||
continue_on_error: ${{ steps.set-continue-on-error.outputs.continue_on_error }}
|
||||
# Keep rocm10 on the twice-daily cadence. The 17:30 UTC schedule runs
|
||||
# rocm724 and rocm720 once daily. Dispatch/call inputs select one version
|
||||
# or all three, while a pull_request falls through to the rocm10 default.
|
||||
rocm_versions: ${{ inputs.rocm_version == 'all' && '["rocm10","rocm724","rocm720"]' || inputs.rocm_version && format('["{0}"]', inputs.rocm_version) || github.event.schedule == '30 17 * * *' && '["rocm724","rocm720"]' || '["rocm10"]' }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
@@ -212,21 +248,32 @@ jobs:
|
||||
# Not added to `pr-test-amd-finish` so the base AMD gate never depends on
|
||||
# the opt-in extra suite.
|
||||
call-pr-test-amd-extra:
|
||||
name: call-pr-test-amd-extra
|
||||
needs: [check-changes]
|
||||
if: |
|
||||
(github.event_name == 'schedule' || inputs.run_all_tests == true) &&
|
||||
!(inputs.target_stage || inputs.target_stage_select)
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
|
||||
uses: ./.github/workflows/pr-test-amd-extra.yml
|
||||
with:
|
||||
ref: ${{ inputs.pr_head_sha || inputs.ref || '' }}
|
||||
runner_arch: ${{ inputs.runner_arch || 'mi300' }}
|
||||
rocm_version: rocm700
|
||||
rocm_version: ${{ matrix.rocm_version }}
|
||||
aiter_ref: ${{ inputs.aiter_ref }}
|
||||
amd_ci_image: ${{ inputs.amd_ci_image }}
|
||||
continue_on_error: true
|
||||
secrets: inherit
|
||||
|
||||
# =============================================== sgl-kernel ====================================================
|
||||
sgl-kernel-unit-test-amd:
|
||||
name: ${{ format('sgl-kernel-unit-test-amd (rocm700, linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }}
|
||||
name: ${{ format('sgl-kernel-unit-test-amd ({0}, linux-{1}-1gpu-sglang)', matrix.rocm_version, inputs.runner_arch || 'mi300') }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
|
||||
needs: [check-changes, call-gate]
|
||||
if: |
|
||||
always() && !cancelled() &&
|
||||
@@ -249,7 +296,7 @@ jobs:
|
||||
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
|
||||
|
||||
- name: Start CI container
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
@@ -284,7 +331,11 @@ jobs:
|
||||
exit $failures
|
||||
|
||||
sgl-kernel-unit-test-2-gpu-amd:
|
||||
name: ${{ format('sgl-kernel-unit-test-2-gpu-amd (rocm700, linux-{0}-2gpu-sglang)', inputs.runner_arch || 'mi300') }}
|
||||
name: ${{ format('sgl-kernel-unit-test-2-gpu-amd ({0}, linux-{1}-2gpu-sglang)', matrix.rocm_version, inputs.runner_arch || 'mi300') }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
|
||||
needs: [check-changes, call-gate]
|
||||
if: |
|
||||
always() && !cancelled() &&
|
||||
@@ -307,7 +358,7 @@ jobs:
|
||||
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
|
||||
|
||||
- name: Start CI container
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
@@ -325,7 +376,11 @@ jobs:
|
||||
# =============================================== primary ====================================================
|
||||
|
||||
stage-a-test-1-gpu-small-amd:
|
||||
name: ${{ format('stage-a-test-1-gpu-small-amd (rocm700, linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }}
|
||||
name: ${{ format('stage-a-test-1-gpu-small-amd ({0}, linux-{1}-1gpu-sglang)', matrix.rocm_version, inputs.runner_arch || 'mi300') }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
|
||||
needs: [check-changes, call-gate]
|
||||
if: |
|
||||
always() && !cancelled() &&
|
||||
@@ -348,7 +403,7 @@ jobs:
|
||||
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
|
||||
|
||||
- name: Start CI container
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
@@ -362,7 +417,11 @@ jobs:
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-a-test-1-gpu-small-amd ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
jit-kernel-unit-test-amd:
|
||||
name: ${{ format('jit-kernel-unit-test-amd (rocm700, linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }}
|
||||
name: ${{ format('jit-kernel-unit-test-amd ({0}, linux-{1}-1gpu-sglang)', matrix.rocm_version, inputs.runner_arch || 'mi300') }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
|
||||
needs: [check-changes, call-gate]
|
||||
if: |
|
||||
always() && !cancelled() &&
|
||||
@@ -385,7 +444,7 @@ jobs:
|
||||
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
|
||||
|
||||
- name: Start CI container
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
@@ -399,7 +458,11 @@ jobs:
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite jit-kernel-unit-test-amd ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
jit-kernel-benchmark-test-amd:
|
||||
name: ${{ format('jit-kernel-benchmark-test-amd (rocm700, linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }}
|
||||
name: ${{ format('jit-kernel-benchmark-test-amd ({0}, linux-{1}-1gpu-sglang)', matrix.rocm_version, inputs.runner_arch || 'mi300') }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
|
||||
needs: [check-changes, call-gate]
|
||||
if: |
|
||||
always() && !cancelled() &&
|
||||
@@ -422,7 +485,7 @@ jobs:
|
||||
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
|
||||
|
||||
- name: Start CI container
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
@@ -441,6 +504,7 @@ jobs:
|
||||
# For scheduled runs: wait jobs are skipped, enabling parallel execution of all stages.
|
||||
|
||||
wait-for-stage-a-amd:
|
||||
name: wait-for-stage-a-amd
|
||||
needs: [check-changes, call-gate]
|
||||
if: |
|
||||
always() &&
|
||||
@@ -462,7 +526,7 @@ jobs:
|
||||
max-wait-minutes: '240'
|
||||
|
||||
stage-b-test-1-gpu-small-amd:
|
||||
name: ${{ format('stage-b-test-1-gpu-small-amd (rocm700, linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }}
|
||||
name: ${{ format('stage-b-test-1-gpu-small-amd ({0}, linux-{1}-1gpu-sglang, {2})', matrix.rocm_version, inputs.runner_arch || 'mi300', matrix.part) }}
|
||||
needs: [check-changes, wait-for-stage-a-amd]
|
||||
if: |
|
||||
always() &&
|
||||
@@ -476,7 +540,9 @@ jobs:
|
||||
)
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 4
|
||||
matrix:
|
||||
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
|
||||
part: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
|
||||
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
|
||||
steps:
|
||||
@@ -489,7 +555,7 @@ jobs:
|
||||
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
|
||||
|
||||
- name: Start CI container
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
@@ -502,7 +568,11 @@ jobs:
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 14 --timeout-per-file 2400 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
stage-b-test-1-gpu-small-amd-nondeterministic:
|
||||
name: ${{ format('stage-b-test-1-gpu-small-amd-nondeterministic (rocm700, linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }}
|
||||
name: ${{ format('stage-b-test-1-gpu-small-amd-nondeterministic ({0}, linux-{1}-1gpu-sglang)', matrix.rocm_version, inputs.runner_arch || 'mi300') }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
|
||||
needs: [check-changes, wait-for-stage-a-amd]
|
||||
if: |
|
||||
always() &&
|
||||
@@ -525,7 +595,7 @@ jobs:
|
||||
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
|
||||
|
||||
- name: Start CI container
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
@@ -538,7 +608,7 @@ jobs:
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-nondeterministic --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
stage-b-test-1-gpu-small-amd-mi35x:
|
||||
name: ${{ format('stage-b-test-1-gpu-small-amd-mi35x (rocm700, {0})', matrix.runner) }}
|
||||
name: ${{ format('stage-b-test-1-gpu-small-amd-mi35x ({0}, {1})', matrix.rocm_version, matrix.runner) }}
|
||||
needs: [check-changes, wait-for-stage-a-amd]
|
||||
if: |
|
||||
always() &&
|
||||
@@ -553,6 +623,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
|
||||
runner: [linux-mi35x-gpu-1]
|
||||
runs-on: ${{matrix.runner}}
|
||||
steps:
|
||||
@@ -565,7 +636,7 @@ jobs:
|
||||
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
|
||||
|
||||
- name: Start CI container
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
@@ -578,7 +649,7 @@ jobs:
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-mi35x ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
stage-b-test-1-gpu-large-amd:
|
||||
name: ${{ format('stage-b-test-1-gpu-large-amd (rocm700, linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }}
|
||||
name: ${{ format('stage-b-test-1-gpu-large-amd ({0}, linux-{1}-1gpu-sglang, {2})', matrix.rocm_version, inputs.runner_arch || 'mi300', matrix.part) }}
|
||||
needs: [check-changes, wait-for-stage-a-amd]
|
||||
if: |
|
||||
always() &&
|
||||
@@ -592,7 +663,9 @@ jobs:
|
||||
)
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 1
|
||||
matrix:
|
||||
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
|
||||
part: [0, 1, 2]
|
||||
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
|
||||
steps:
|
||||
@@ -605,7 +678,7 @@ jobs:
|
||||
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
|
||||
|
||||
- name: Start CI container
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
@@ -618,7 +691,7 @@ jobs:
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 2700 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
stage-b-test-2-gpu-large-amd:
|
||||
name: ${{ format('stage-b-test-2-gpu-large-amd (rocm700, linux-{0}-2gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }}
|
||||
name: ${{ format('stage-b-test-2-gpu-large-amd ({0}, linux-{1}-2gpu-sglang, {2})', matrix.rocm_version, inputs.runner_arch || 'mi300', matrix.part) }}
|
||||
needs: [check-changes, wait-for-stage-a-amd]
|
||||
if: |
|
||||
always() &&
|
||||
@@ -632,7 +705,9 @@ jobs:
|
||||
)
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 1
|
||||
matrix:
|
||||
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
|
||||
part: [0, 1]
|
||||
runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }}
|
||||
steps:
|
||||
@@ -645,7 +720,7 @@ jobs:
|
||||
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
|
||||
|
||||
- name: Start CI container
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
@@ -658,7 +733,7 @@ jobs:
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-2-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file ${{ (inputs.runner_arch || 'mi300') == 'mi300' && 5400 || 2700 }} ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
multimodal-gen-test-1-gpu-amd:
|
||||
name: ${{ format('multimodal-gen-test-1-gpu-amd (rocm700, linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }}
|
||||
name: ${{ format('multimodal-gen-test-1-gpu-amd ({0}, linux-{1}-1gpu-sglang, {2})', matrix.rocm_version, inputs.runner_arch || 'mi300', matrix.part) }}
|
||||
needs: [check-changes, call-gate]
|
||||
if: |
|
||||
always() && !cancelled() &&
|
||||
@@ -672,7 +747,9 @@ jobs:
|
||||
)
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 1
|
||||
matrix:
|
||||
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
|
||||
part: [0, 1, 2, 3]
|
||||
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
|
||||
steps:
|
||||
@@ -693,7 +770,7 @@ jobs:
|
||||
pattern: wheel-python3.10-cuda12.9
|
||||
|
||||
- name: Start CI container
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
@@ -800,13 +877,13 @@ jobs:
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: diffusion-failures-amd-1gpu-${{ matrix.part }}-${{ github.run_attempt }}
|
||||
name: diffusion-failures-amd-1gpu-${{ matrix.rocm_version }}-${{ matrix.part }}-${{ github.run_attempt }}
|
||||
path: diffusion-failures/
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
multimodal-gen-test-2-gpu-amd:
|
||||
name: ${{ format('multimodal-gen-test-2-gpu-amd (rocm700, linux-{0}-2gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }}
|
||||
name: ${{ format('multimodal-gen-test-2-gpu-amd ({0}, linux-{1}-2gpu-sglang, {2})', matrix.rocm_version, inputs.runner_arch || 'mi300', matrix.part) }}
|
||||
needs: [check-changes, call-gate]
|
||||
if: |
|
||||
always() && !cancelled() &&
|
||||
@@ -820,7 +897,9 @@ jobs:
|
||||
)
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 1
|
||||
matrix:
|
||||
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
|
||||
# run_suite.py load-balances the suite's parametrized cases and
|
||||
# standalone files over these partitions, so the count is free to
|
||||
# choose. Two shards measured 45-65 min of case work each, inside the
|
||||
@@ -846,7 +925,7 @@ jobs:
|
||||
pattern: wheel-python3.10-cuda12.9
|
||||
|
||||
- name: Start CI container
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
@@ -934,12 +1013,13 @@ jobs:
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: diffusion-failures-amd-2gpu-${{ matrix.part }}-${{ github.run_attempt }}
|
||||
name: diffusion-failures-amd-2gpu-${{ matrix.rocm_version }}-${{ matrix.part }}-${{ github.run_attempt }}
|
||||
path: diffusion-failures/
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
wait-for-stage-b-amd:
|
||||
name: wait-for-stage-b-amd
|
||||
needs: [check-changes, call-gate, wait-for-stage-a-amd]
|
||||
if: |
|
||||
always() &&
|
||||
@@ -967,7 +1047,7 @@ jobs:
|
||||
max-wait-minutes: '480'
|
||||
|
||||
stage-c-test-4-gpu-amd:
|
||||
name: ${{ format('stage-c-test-4-gpu-amd (rocm700, linux-{0}-4gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }}
|
||||
name: ${{ format('stage-c-test-4-gpu-amd ({0}, linux-{1}-4gpu-sglang, {2})', matrix.rocm_version, inputs.runner_arch || 'mi300', matrix.part) }}
|
||||
needs: [check-changes, call-gate, wait-for-stage-b-amd]
|
||||
if: |
|
||||
always() &&
|
||||
@@ -982,6 +1062,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
|
||||
part: [0]
|
||||
runs-on: ${{ format('linux-{0}-4gpu-sglang', inputs.runner_arch || 'mi300') }}
|
||||
steps:
|
||||
@@ -994,7 +1075,7 @@ jobs:
|
||||
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
|
||||
|
||||
- name: Start CI container
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
@@ -1023,7 +1104,7 @@ jobs:
|
||||
${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
stage-c-test-large-8-gpu-amd:
|
||||
name: ${{ format('stage-c-test-large-8-gpu-amd (rocm700, linux-{0}-8gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }}
|
||||
name: ${{ format('stage-c-test-large-8-gpu-amd ({0}, linux-{1}-8gpu-sglang, {2})', matrix.rocm_version, inputs.runner_arch || 'mi300', matrix.part) }}
|
||||
needs: [check-changes, call-gate, wait-for-stage-b-amd]
|
||||
if: |
|
||||
always() &&
|
||||
@@ -1039,7 +1120,9 @@ jobs:
|
||||
RUNNER_LABELS: ${{ format('linux-{0}-8gpu-sglang', inputs.runner_arch || 'mi300') }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 2
|
||||
matrix:
|
||||
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
|
||||
part: [0, 1, 2, 3]
|
||||
runs-on: ${{ format('linux-{0}-8gpu-sglang', inputs.runner_arch || 'mi300') }}
|
||||
steps:
|
||||
@@ -1052,7 +1135,7 @@ jobs:
|
||||
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
|
||||
|
||||
- name: Start CI container
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
@@ -1071,7 +1154,7 @@ jobs:
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 4 --timeout-per-file 5400 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
stage-c-test-large-8-gpu-amd-mi35x:
|
||||
name: ${{ format('stage-c-test-large-8-gpu-amd-mi35x (rocm700, {0}, {1})', matrix.runner, matrix.part) }}
|
||||
name: ${{ format('stage-c-test-large-8-gpu-amd-mi35x ({0}, {1}, {2})', matrix.rocm_version, matrix.runner, matrix.part) }}
|
||||
needs: [check-changes, call-gate, wait-for-stage-b-amd]
|
||||
if: |
|
||||
always() &&
|
||||
@@ -1086,6 +1169,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
|
||||
runner: [linux-mi35x-gpu-8]
|
||||
part: [0, 1, 2]
|
||||
runs-on: ${{matrix.runner}}
|
||||
@@ -1099,7 +1183,7 @@ jobs:
|
||||
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
|
||||
|
||||
- name: Start CI container
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
@@ -1111,9 +1195,124 @@ jobs:
|
||||
run: |
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
# =============================================== DeepSeek-V4 (MI35x, 8-GPU) ====================================================
|
||||
# GSM8K accuracy on the nightly dsv4 suites, ~20min each. Gated as stage-C so
|
||||
# every PR that touches the main package covers DeepSeek-V4.
|
||||
stage-c-dsv4-flash-fp4-fp8-amd-mi35x:
|
||||
name: ${{ format('stage-c-dsv4-flash-fp4-fp8-amd-mi35x ({0}, {1})', matrix.rocm_version, matrix.runner) }}
|
||||
needs: [check-changes, call-gate, wait-for-stage-b-amd]
|
||||
if: |
|
||||
always() &&
|
||||
(
|
||||
(contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-c-dsv4-flash-fp4-fp8-amd-mi35x,')) ||
|
||||
(
|
||||
!(inputs.target_stage || inputs.target_stage_select) &&
|
||||
((github.event_name == 'schedule') || (!failure() && !cancelled())) &&
|
||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||
)
|
||||
)
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
|
||||
runner: [linux-mi35x-gpu-8]
|
||||
runs-on: ${{matrix.runner}}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }}
|
||||
|
||||
- name: Ensure VRAM is clear
|
||||
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
|
||||
|
||||
- name: Setup docker (ROCm)
|
||||
run: |
|
||||
touch github_summary.md
|
||||
bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
ENABLE_CACHE_HOST: "1"
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
# --skip-test-time-deps: GSM8K + bench_one_batch_server don't need lmms-eval / human-eval.
|
||||
bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
|
||||
bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate
|
||||
|
||||
- name: Accuracy Test MI35x ROCm (8-GPU DeepSeek-V4-Flash FP4 + FP8)
|
||||
timeout-minutes: 60
|
||||
run: |
|
||||
> github_summary.md # Clear summary file
|
||||
# SGLANG_DSV4_ACCURACY_ONLY=1 makes the dsv4 test files skip their perf test
|
||||
# (test_b_perf_8k_1k); only the GSM8K accuracy test runs in this workflow.
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
-e SGLANG_DSV4_ACCURACY_ONLY=1 \
|
||||
-e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-flash --nightly --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
stage-c-dsv4-pro-fp4-amd-mi35x:
|
||||
name: ${{ format('stage-c-dsv4-pro-fp4-amd-mi35x ({0}, {1})', matrix.rocm_version, matrix.runner) }}
|
||||
needs: [check-changes, call-gate, wait-for-stage-b-amd]
|
||||
if: |
|
||||
always() &&
|
||||
(
|
||||
(contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-c-dsv4-pro-fp4-amd-mi35x,')) ||
|
||||
(
|
||||
!(inputs.target_stage || inputs.target_stage_select) &&
|
||||
((github.event_name == 'schedule') || (!failure() && !cancelled())) &&
|
||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||
)
|
||||
)
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
|
||||
runner: [linux-mi35x-gpu-8]
|
||||
runs-on: ${{matrix.runner}}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }}
|
||||
|
||||
- name: Ensure VRAM is clear
|
||||
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
|
||||
|
||||
- name: Setup docker (ROCm)
|
||||
run: |
|
||||
touch github_summary.md
|
||||
bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}"
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
ENABLE_CACHE_HOST: "1"
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
# --skip-test-time-deps: GSM8K + bench_one_batch_server don't need lmms-eval / human-eval.
|
||||
bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
|
||||
bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate
|
||||
|
||||
- name: Accuracy Test MI35x ROCm (8-GPU DeepSeek-V4-Pro FP4)
|
||||
timeout-minutes: 60
|
||||
run: |
|
||||
> github_summary.md # Clear summary file
|
||||
# SGLANG_DSV4_ACCURACY_ONLY=1 makes the dsv4 test files skip their perf test
|
||||
# (test_b_perf_8k_1k); only the GSM8K accuracy test runs in this workflow.
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
-e SGLANG_DSV4_ACCURACY_ONLY=1 \
|
||||
-e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
|
||||
python3 registered/amd/test_deepseek_v4_pro_fp4.py || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# =============================================== Disaggregation ====================================================
|
||||
stage-b-test-large-8-gpu-mi35x-disaggregation-amd:
|
||||
name: ${{ format('stage-b-test-large-8-gpu-mi35x-disaggregation-amd (rocm700, {0})', matrix.runner) }}
|
||||
name: ${{ format('stage-b-test-large-8-gpu-mi35x-disaggregation-amd ({0}, {1})', matrix.rocm_version, matrix.runner) }}
|
||||
needs: [check-changes, wait-for-stage-a-amd]
|
||||
if: |
|
||||
always() &&
|
||||
@@ -1128,6 +1327,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }}
|
||||
runner: [linux-mi35x-gpu-8.fabric]
|
||||
|
||||
runs-on: ${{matrix.runner}}
|
||||
@@ -1180,7 +1380,7 @@ jobs:
|
||||
echo "=== Host RDMA Check Complete ==="
|
||||
|
||||
- name: Start Special Container
|
||||
run: bash scripts/ci/amd/amd_ci_start_container_disagg.sh
|
||||
run: bash scripts/ci/amd/amd_ci_start_container_disagg.sh --rocm-version "${{ matrix.rocm_version }}"
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
@@ -1247,6 +1447,7 @@ jobs:
|
||||
-w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-8-gpu-mi35x-disaggregation-amd --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
pr-test-amd-finish:
|
||||
name: pr-test-amd-finish
|
||||
needs:
|
||||
[
|
||||
call-gate,
|
||||
@@ -1271,6 +1472,9 @@ jobs:
|
||||
stage-c-test-4-gpu-amd,
|
||||
stage-c-test-large-8-gpu-amd,
|
||||
stage-c-test-large-8-gpu-amd-mi35x,
|
||||
|
||||
stage-c-dsv4-flash-fp4-fp8-amd-mi35x,
|
||||
stage-c-dsv4-pro-fp4-amd-mi35x,
|
||||
]
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -174,7 +174,7 @@ jobs:
|
||||
|
||||
run-pr-tests-amd:
|
||||
needs: cut-release-branch
|
||||
uses: ./.github/workflows/pr-test-amd-rocm720.yml
|
||||
uses: ./.github/workflows/pr-test-amd.yml
|
||||
with:
|
||||
ref: ${{ needs.cut-release-branch.outputs.branch_name }}
|
||||
rocm_version: rocm10
|
||||
@@ -214,7 +214,7 @@ jobs:
|
||||
|
||||
run-nightly-tests-amd:
|
||||
needs: cut-release-branch
|
||||
uses: ./.github/workflows/nightly-test-amd-rocm720.yml
|
||||
uses: ./.github/workflows/nightly-test-amd.yml
|
||||
with:
|
||||
ref: ${{ needs.cut-release-branch.outputs.branch_name }}
|
||||
rocm_version: rocm10
|
||||
|
||||
@@ -1,252 +0,0 @@
|
||||
name: Release Docker Images Nightly ROCm7.0 (AMD)
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: '0 12 * * *'
|
||||
|
||||
concurrency:
|
||||
# A PR number if a pull request and otherwise the commit hash. This cancels
|
||||
# queued and in-progress runs for the same PR (presubmit) or commit
|
||||
# (postsubmit). The workflow name is prepended to avoid conflicts between
|
||||
# different workflows.
|
||||
group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
if: github.repository == 'sgl-project/sglang'
|
||||
runs-on: amd-docker-scale
|
||||
environment: 'prod'
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
gpu_arch: ['gfx942', 'gfx950']
|
||||
build_type: ['all']
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required for git describe to find tags
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.10"
|
||||
|
||||
- name: "Set Date"
|
||||
run: |
|
||||
echo "DATE=$(date +%Y%m%d)" >> $GITHUB_ENV
|
||||
|
||||
- name: Get version from latest tag
|
||||
id: version
|
||||
run: |
|
||||
# Use the shared helper so stable/post releases sort above rc tags.
|
||||
VERSION=$(python3 scripts/release/get_version_tag.py --tag-only | sed 's/^v//')
|
||||
|
||||
if [ -z "$VERSION" ]; then
|
||||
echo "::error::Could not determine version from git tags"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Get short commit hash of current HEAD
|
||||
COMMIT_HASH=$(git rev-parse --short HEAD)
|
||||
|
||||
# Compose pretend version for setuptools_scm: e.g., 0.5.8.dev20260129+g1a2b3c4
|
||||
PRETEND_VERSION="${VERSION}.dev${{ env.DATE }}+g${COMMIT_HASH}"
|
||||
|
||||
echo "version=${VERSION}" >> $GITHUB_OUTPUT
|
||||
echo "pretend_version=${PRETEND_VERSION}" >> $GITHUB_OUTPUT
|
||||
echo "Detected version: ${VERSION}"
|
||||
echo "Pretend version for pip: ${PRETEND_VERSION}"
|
||||
|
||||
- name: Login to Docker Hub (AMD)
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_AMD_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_AMD_TOKEN }}
|
||||
|
||||
- name: Build and Push to rocm/sgl-dev
|
||||
run: |
|
||||
version=${{ steps.version.outputs.version }}
|
||||
pretend_version=${{ steps.version.outputs.pretend_version }}
|
||||
echo "Version: ${version}"
|
||||
echo "Pretend version: ${pretend_version}"
|
||||
|
||||
if [ "${{ matrix.gpu_arch }}" = "gfx942" ]; then
|
||||
rocm_tag="rocm700-mi30x"
|
||||
elif [ "${{ matrix.gpu_arch }}" = "gfx950" ]; then
|
||||
rocm_tag="rocm700-mi35x"
|
||||
else
|
||||
echo "Unsupported gfx arch"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
tag=v${version}-${rocm_tag}
|
||||
echo "IMAGE_TAG=${tag}-${{ env.DATE }}" >> $GITHUB_ENV
|
||||
|
||||
# remove --build-arg NIC_BACKEND=ainic for auto detection nic support in mori
|
||||
# UBUNTU_MIRROR forces apt over HTTPS to dodge port-80 reachability flakes
|
||||
# to Canonical's archive.ubuntu.com mirror IPs from the amd-docker-scale runner.
|
||||
docker build . -f docker/rocm.Dockerfile --build-arg SGL_BRANCH=${{ github.ref_name }} --build-arg BUILD_TYPE=${{ matrix.build_type }} --build-arg GPU_ARCH=${{ matrix.gpu_arch }} --build-arg ENABLE_MORI=1 --build-arg ENABLE_NIXL=1 --build-arg SETUPTOOLS_SCM_PRETEND_VERSION=${pretend_version} --build-arg UBUNTU_MIRROR=https://archive.ubuntu.com -t rocm/sgl-dev:${tag}-${{ env.DATE }} --no-cache
|
||||
docker push rocm/sgl-dev:${tag}-${{ env.DATE }}
|
||||
|
||||
# Persist the tag right after rocm/sgl-dev push succeeds so the local
|
||||
# registry mirror can run even if a later step in this job (lmsys push)
|
||||
# fails. By default this step only runs when the previous step succeeded,
|
||||
# so the artifact only exists when an image actually landed on Docker Hub.
|
||||
- name: Save published image tag
|
||||
run: |
|
||||
mkdir -p image-tag
|
||||
echo "${{ env.IMAGE_TAG }}" > "image-tag/${{ matrix.gpu_arch }}.txt"
|
||||
|
||||
- name: Upload image tag artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: image-tag-${{ matrix.gpu_arch }}
|
||||
path: image-tag/${{ matrix.gpu_arch }}.txt
|
||||
retention-days: 1
|
||||
|
||||
- name: Login to Docker Hub (lmsys)
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Push to lmsysorg/sglang-rocm
|
||||
run: |
|
||||
docker tag rocm/sgl-dev:${{ env.IMAGE_TAG }} lmsysorg/sglang-rocm:${{ env.IMAGE_TAG }}
|
||||
docker push lmsysorg/sglang-rocm:${{ env.IMAGE_TAG }}
|
||||
|
||||
# Mirror the freshly published rocm/sgl-dev image to the in-network Docker
|
||||
# registry so AMD CI runners can pull without hitting Docker Hub rate limits.
|
||||
# The tag is read verbatim from the publish job's artifact so this job uses
|
||||
# exactly the same tag that publish pushed (only the registry prefix differs).
|
||||
# `!cancelled()` lets us still mirror successful matrix legs when other legs
|
||||
# of publish failed; legs without an artifact will fail at download and be
|
||||
# the only ones marked red.
|
||||
push_local_registry:
|
||||
if: ${{ false }}
|
||||
runs-on: linux-mi300-1gpu-sglang
|
||||
environment: 'prod'
|
||||
needs: publish
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
gpu_arch: ['gfx942', 'gfx950']
|
||||
steps:
|
||||
- name: Download image tag artifact
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: image-tag-${{ matrix.gpu_arch }}
|
||||
|
||||
- name: Read image tag
|
||||
run: |
|
||||
image_tag=$(tr -d '[:space:]' < "${{ matrix.gpu_arch }}.txt")
|
||||
if [ -z "${image_tag}" ]; then
|
||||
echo "::error::Image tag artifact is empty"
|
||||
exit 1
|
||||
fi
|
||||
echo "IMAGE_TAG=${image_tag}" >> $GITHUB_ENV
|
||||
echo "Resolved IMAGE_TAG=${image_tag}"
|
||||
|
||||
- name: Login to Docker Hub (AMD)
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_AMD_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_AMD_TOKEN }}
|
||||
|
||||
- name: Mirror rocm/sgl-dev to local registry
|
||||
run: |
|
||||
src="rocm/sgl-dev:${{ env.IMAGE_TAG }}"
|
||||
dst="10.44.14.109:5000/rocm/sgl-dev:${{ env.IMAGE_TAG }}"
|
||||
docker pull "${src}"
|
||||
docker tag "${src}" "${dst}"
|
||||
docker push "${dst}"
|
||||
|
||||
# Temporarily disable docker cache seeding until performant storage is in place
|
||||
# cache:
|
||||
# if: false
|
||||
# # if: always() && github.repository == 'sgl-project/sglang'
|
||||
# runs-on: linux-mi300-gpu-1
|
||||
# environment: 'prod'
|
||||
# needs: publish
|
||||
# strategy:
|
||||
# fail-fast: false
|
||||
# matrix:
|
||||
# gpu_arch: ['gfx942']
|
||||
# build_type: ['all']
|
||||
# steps:
|
||||
# - name: Checkout repository
|
||||
# uses: actions/checkout@v4
|
||||
# with:
|
||||
# fetch-depth: 0 # Required for git describe to find tags
|
||||
|
||||
# - name: "Set Date"
|
||||
# run: |
|
||||
# echo "DATE=$(date +%Y%m%d)" >> $GITHUB_ENV
|
||||
|
||||
# - name: Get version from latest tag
|
||||
# id: version
|
||||
# run: |
|
||||
# # Use the shared helper so stable/post releases sort above rc tags.
|
||||
# VERSION=$(python3 scripts/release/get_version_tag.py --tag-only | sed 's/^v//')
|
||||
|
||||
# if [ -z "$VERSION" ]; then
|
||||
# echo "::error::Could not determine version from git tags"
|
||||
# exit 1
|
||||
# fi
|
||||
|
||||
# echo "version=${VERSION}" >> $GITHUB_OUTPUT
|
||||
# echo "Detected version: ${VERSION}"
|
||||
|
||||
# - name: Login to Docker Hub
|
||||
# uses: docker/login-action@v2
|
||||
# with:
|
||||
# username: ${{ secrets.DOCKERHUB_AMD_USERNAME }}
|
||||
# password: ${{ secrets.DOCKERHUB_AMD_TOKEN }}
|
||||
|
||||
# - name: Pull and Save Docker Image to Cache
|
||||
# run: |
|
||||
# set -euxo pipefail
|
||||
|
||||
# version=${{ steps.version.outputs.version }}
|
||||
# echo "Version: ${version}"
|
||||
|
||||
# if [ "${{ matrix.gpu_arch }}" = "gfx942" ]; then
|
||||
# rocm_tag="rocm700-mi30x"
|
||||
# else
|
||||
# echo "Unsupported gfx arch"
|
||||
# exit 1
|
||||
# fi
|
||||
|
||||
# tag=v${version}-${rocm_tag}
|
||||
|
||||
# if [ "${{ matrix.build_type }}" = "all" ]; then
|
||||
# tag_suffix=""
|
||||
# else
|
||||
# echo "Unsupported build type"
|
||||
# exit 1
|
||||
# fi
|
||||
|
||||
# image="rocm/sgl-dev:${tag}-${{ env.DATE }}${tag_suffix}"
|
||||
|
||||
# # Determine target cache file name based on ROCm variant
|
||||
# if [[ "${rocm_tag}" == rocm700* ]]; then
|
||||
# final_path="/home/runner/sgl-data/docker/image-700.tar"
|
||||
# else
|
||||
# echo "Unexpected ROCm tag: ${rocm_tag}"
|
||||
# exit 1
|
||||
# fi
|
||||
|
||||
# tmp_path="${final_path}.tmp"
|
||||
|
||||
# echo "Pulling image: ${image}"
|
||||
# docker pull "${image}"
|
||||
|
||||
# echo "Saving to temp file: ${tmp_path}"
|
||||
# docker save "${image}" -o "${tmp_path}"
|
||||
|
||||
# echo "Moving to final path: ${final_path}"
|
||||
# mv -f "${tmp_path}" "${final_path}"
|
||||
|
||||
# echo "Cache populated successfully at ${final_path}"
|
||||
@@ -18,7 +18,6 @@ on:
|
||||
- rocm10
|
||||
- rocm724
|
||||
- rocm720
|
||||
- rocm700
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
@@ -26,11 +25,11 @@ jobs:
|
||||
runs-on: amd-docker-scale
|
||||
environment: 'prod'
|
||||
strategy:
|
||||
# Eight flavors publish independently: a failure in one must not cancel
|
||||
# the others mid-push and leave the release tag with a partial image set.
|
||||
# Six flavors publish independently: a failure in one must not cancel the
|
||||
# others mid-push and leave the release tag with a partial image set.
|
||||
fail-fast: false
|
||||
matrix:
|
||||
rocm_version: ${{ fromJson((github.event_name == 'workflow_dispatch' && inputs.rocm_version != 'all' && inputs.rocm_version != '') && format('["{0}"]', inputs.rocm_version) || '["rocm700", "rocm720", "rocm724", "rocm10"]') }}
|
||||
rocm_version: ${{ fromJson((github.event_name == 'workflow_dispatch' && inputs.rocm_version != 'all' && inputs.rocm_version != '') && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm720", "rocm724"]') }}
|
||||
gpu_arch: ['gfx942', 'gfx950']
|
||||
build_type: ['all']
|
||||
steps:
|
||||
@@ -71,16 +70,7 @@ jobs:
|
||||
echo "Version: ${version}"
|
||||
|
||||
gpu_arch_suffix=""
|
||||
if [ "${{ matrix.rocm_version }}" = "rocm700" ]; then
|
||||
if [ "${{ matrix.gpu_arch }}" = "gfx942" ]; then
|
||||
rocm_tag="rocm700-mi30x"
|
||||
elif [ "${{ matrix.gpu_arch }}" = "gfx950" ]; then
|
||||
rocm_tag="rocm700-mi35x"
|
||||
else
|
||||
echo "Unsupported gfx arch"
|
||||
exit 1
|
||||
fi
|
||||
elif [ "${{ matrix.rocm_version }}" = "rocm720" ] || [ "${{ matrix.rocm_version }}" = "rocm724" ]; then
|
||||
if [ "${{ matrix.rocm_version }}" = "rocm720" ] || [ "${{ matrix.rocm_version }}" = "rocm724" ]; then
|
||||
gpu_arch_suffix="-${{ matrix.rocm_version }}"
|
||||
if [ "${{ matrix.gpu_arch }}" = "gfx942" ]; then
|
||||
rocm_tag="${{ matrix.rocm_version }}-mi30x"
|
||||
|
||||
@@ -19,8 +19,9 @@ else
|
||||
fi
|
||||
|
||||
|
||||
# Default base tags (can be overridden by command line arguments)
|
||||
ROCM_VERSION="rocm700"
|
||||
# Default base tags (can be overridden by command line arguments).
|
||||
# Keep this aligned with the default AMD PR and Nightly CI lane.
|
||||
ROCM_VERSION="rocm10"
|
||||
DEFAULT_MI30X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi30x"
|
||||
DEFAULT_MI35X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi35x"
|
||||
LOCAL_DOCKER_REGISTRY="10.44.14.109:5000"
|
||||
@@ -52,8 +53,8 @@ while [[ $# -gt 0 ]]; do
|
||||
echo " --mi35x-base-tag TAG Override MI35x base image tag"
|
||||
echo " --custom-image IMAGE Use a specific Docker image directly (or set AMD_CI_IMAGE)"
|
||||
echo " --build-from-dockerfile Build image from docker/rocm.Dockerfile"
|
||||
echo " --gpu-arch ARCH GPU architecture for Dockerfile build (e.g., gfx950-rocm720)"
|
||||
echo " --rocm-version VERSION Override ROCm version for image lookup (e.g., rocm720)"
|
||||
echo " --gpu-arch ARCH GPU architecture for Dockerfile build (e.g., gfx950-rocm1000)"
|
||||
echo " --rocm-version VERSION Override ROCm version for image lookup (e.g., rocm10)"
|
||||
echo ""
|
||||
echo "Environment:"
|
||||
echo " ENABLE_CACHE_HOST=1|0"
|
||||
@@ -211,13 +212,6 @@ find_latest_image() {
|
||||
echo "rocm/sgl-dev:v0.5.8.post1-rocm720-mi30x-20260211-preview"
|
||||
fi
|
||||
;;
|
||||
rocm700)
|
||||
if [[ "${gpu_arch}" == "mi35x" ]]; then
|
||||
echo "rocm/sgl-dev:v0.5.8.post1-rocm700-mi35x-20260211"
|
||||
else
|
||||
echo "rocm/sgl-dev:v0.5.8.post1-rocm700-mi30x-20260211"
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
echo "Error: no hard-coded fallback available for ${ROCM_VERSION}" >&2
|
||||
return 1
|
||||
@@ -253,7 +247,7 @@ elif [[ -n "${BUILD_FROM_DOCKERFILE}" ]]; then
|
||||
IMAGE="sglang-ci:${GPU_ARCH_BUILD}-$(date +%Y%m%d)"
|
||||
echo "Building Docker image from ${DOCKERFILE} with GPU_ARCH=${GPU_ARCH_BUILD}..."
|
||||
|
||||
# Pass full GPU_ARCH (e.g., gfx950-rocm720) - Dockerfile handles stripping suffix
|
||||
# Pass full GPU_ARCH (e.g., gfx950-rocm1000) - Dockerfile handles stripping suffix
|
||||
docker build \
|
||||
--build-arg GPU_ARCH="${GPU_ARCH_BUILD}" \
|
||||
--build-arg SGL_BRANCH="main" \
|
||||
|
||||
@@ -19,8 +19,9 @@ else
|
||||
fi
|
||||
|
||||
|
||||
# Default base tags (can be overridden by command line arguments)
|
||||
ROCM_VERSION="rocm700"
|
||||
# Default base tags (can be overridden by command line arguments).
|
||||
# Keep this aligned with the default AMD PR and Nightly CI lane.
|
||||
ROCM_VERSION="rocm10"
|
||||
DEFAULT_MI30X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi30x"
|
||||
DEFAULT_MI35X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi35x"
|
||||
LOCAL_DOCKER_REGISTRY="10.44.14.109:5000"
|
||||
@@ -196,13 +197,6 @@ find_latest_image() {
|
||||
echo "rocm/sgl-dev:v0.5.8.post1-rocm720-mi30x-20260211-preview"
|
||||
fi
|
||||
;;
|
||||
rocm700)
|
||||
if [[ "${gpu_arch}" == "mi35x" ]]; then
|
||||
echo "rocm/sgl-dev:v0.5.8.post1-rocm700-mi35x-20260211"
|
||||
else
|
||||
echo "rocm/sgl-dev:v0.5.8.post1-rocm700-mi30x-20260211"
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
echo "Error: no hard-coded fallback available for ${ROCM_VERSION}" >&2
|
||||
return 1
|
||||
|
||||
@@ -30,12 +30,18 @@ import requests
|
||||
def _filter_legacy_amd_job_rows(job_data: Dict[str, Dict]) -> Dict[str, Dict]:
|
||||
"""Drop pre-cutover AMD names without changing the shared analyzer."""
|
||||
|
||||
legacy_callers = {
|
||||
"call-nightly-amd-rocm720",
|
||||
"call-pr-test-amd-extra-rocm720",
|
||||
"call-pr-test-amd-rocm720",
|
||||
}
|
||||
filtered = {}
|
||||
for full_name, data in job_data.items():
|
||||
# This caller was renamed by the AMD job-name cutover. Other outer
|
||||
# callers, including AITER's *-rocm720 callers, are still current.
|
||||
# These callers were renamed by the AMD workflow canonicalization. Keep
|
||||
# recognizing their historical spellings so old rows do not merge into
|
||||
# current streaks.
|
||||
name_parts = full_name.split(" / ")
|
||||
if "call-pr-test-amd-extra-rocm720" in name_parts[:-1]:
|
||||
if any(part in legacy_callers for part in name_parts[:-1]):
|
||||
continue
|
||||
|
||||
leaf_name = name_parts[-1]
|
||||
@@ -49,7 +55,7 @@ def _filter_legacy_amd_job_rows(job_data: Dict[str, Dict]) -> Dict[str, Dict]:
|
||||
"pr-gate",
|
||||
"pr-test-amd-extra-finish",
|
||||
"pr-test-amd-finish",
|
||||
"pr-test-amd-rocm720-finish",
|
||||
"pr-test-amd-rocm720-finish", # Legacy pre-canonicalization teardown
|
||||
}:
|
||||
continue
|
||||
|
||||
@@ -67,6 +73,12 @@ def _filter_legacy_amd_job_rows(job_data: Dict[str, Dict]) -> Dict[str, Dict]:
|
||||
if details.startswith("rocm") and details[4:].isdigit():
|
||||
continue
|
||||
|
||||
# The canonical workflow paths reuse the history of the retired
|
||||
# ROCm 7.0 workflows. Do not let those rows contribute to current
|
||||
# multi-version streaks during the lookback window.
|
||||
if details.split(",", 1)[0] == "rocm700":
|
||||
continue
|
||||
|
||||
filtered[full_name] = data
|
||||
|
||||
return filtered
|
||||
@@ -92,7 +104,7 @@ class SGLangFailuresAnalyzer:
|
||||
"check-changes",
|
||||
"pr-test-finish", # Nvidia workflow teardown
|
||||
"pr-test-amd-finish", # AMD workflow teardown
|
||||
"pr-test-amd-rocm720-finish", # Default AMD ROCm 7.2 teardown
|
||||
"pr-test-amd-rocm720-finish", # Legacy AMD teardown
|
||||
"call-gate",
|
||||
"pr-gate",
|
||||
"check-all-jobs",
|
||||
@@ -2502,7 +2514,7 @@ def main():
|
||||
# These 4 don't have scheduled events, so filter by main branch instead
|
||||
pr_test_amd_scheduled_runs = analyzer.get_recent_runs(
|
||||
limit=pr_test_scheduled_limit,
|
||||
workflow_filter=["pr-test-amd-rocm720.yml"],
|
||||
workflow_filter=["pr-test-amd.yml"],
|
||||
filters={"branch": "main"},
|
||||
)
|
||||
pr_test_xeon_scheduled_runs = analyzer.get_recent_runs(
|
||||
@@ -2529,7 +2541,7 @@ def main():
|
||||
)
|
||||
nightly_amd_scheduled_runs = analyzer.get_recent_runs(
|
||||
limit=nightly_scheduled_limit,
|
||||
workflow_filter=["nightly-test-amd-rocm720.yml"],
|
||||
workflow_filter=["nightly-test-amd.yml"],
|
||||
filters={"event": "schedule"},
|
||||
)
|
||||
nightly_intel_scheduled_runs = analyzer.get_recent_runs(
|
||||
@@ -2551,7 +2563,7 @@ def main():
|
||||
)
|
||||
pr_test_amd_general_runs = analyzer.get_recent_runs(
|
||||
limit=args.limit,
|
||||
workflow_filter=["pr-test-amd-rocm720.yml"],
|
||||
workflow_filter=["pr-test-amd.yml"],
|
||||
)
|
||||
pr_test_xeon_general_runs = analyzer.get_recent_runs(
|
||||
limit=args.limit,
|
||||
@@ -2573,7 +2585,7 @@ def main():
|
||||
)
|
||||
nightly_amd_general_runs = analyzer.get_recent_runs(
|
||||
limit=args.limit,
|
||||
workflow_filter=["nightly-test-amd-rocm720.yml"],
|
||||
workflow_filter=["nightly-test-amd.yml"],
|
||||
)
|
||||
nightly_intel_general_runs = analyzer.get_recent_runs(
|
||||
limit=args.limit,
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
"""AMD job-name cutover tests.
|
||||
|
||||
Run with:
|
||||
python -m unittest discover -s scripts/ci_monitor -p 'test_ci_failures_analysis.py'
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from ci_failures_analysis import _filter_legacy_amd_job_rows # noqa: E402
|
||||
|
||||
|
||||
class TestFilterLegacyAmdJobRows(unittest.TestCase):
|
||||
def test_drops_legacy_names_and_nested_utilities(self):
|
||||
rows = {
|
||||
"stage-b-test-1-gpu-small-amd-rocm720 (linux-mi300-1gpu-sglang, 0)": {},
|
||||
"nightly-accuracy-2-gpu-rocm720 (rocm724)": {},
|
||||
"nightly-accuracy-2-gpu-rocm724": {},
|
||||
"nightly-test-1-gpu-unit (rocm724)": {},
|
||||
"call-pr-test-amd-rocm720 / call-pr-test-amd-extra-rocm720 / extra-a-test-1-gpu-small-amd (linux-mi300-1gpu-sglang)": {},
|
||||
"wait-for-stage-a-amd": {},
|
||||
"call-pr-test-amd-extra / pr-test-amd-extra-finish": {},
|
||||
"call-pr-test-amd-extra / call-gate / pr-gate": {},
|
||||
}
|
||||
|
||||
self.assertEqual(_filter_legacy_amd_job_rows(rows), {})
|
||||
|
||||
def test_keeps_current_flavors_and_nested_callers_separate(self):
|
||||
new_success = {"current_streak": 0}
|
||||
rows = {
|
||||
"stage-b-test-1-gpu-small-amd (rocm724, linux-mi300-1gpu-sglang, 0)": new_success,
|
||||
"nightly-accuracy-2-gpu (rocm720, linux-mi300-2gpu-sglang)": {
|
||||
"current_streak": 1
|
||||
},
|
||||
"call-pr-test-amd-rocm720 / stage-c-test-4-gpu-amd (rocm724, linux-mi300-4gpu-sglang, 0)": {
|
||||
"current_streak": 0
|
||||
},
|
||||
"call-pr-test-amd-rocm720 / call-pr-test-amd-extra / extra-a-test-1-gpu-small-amd (rocm724, linux-mi300-1gpu-sglang)": {
|
||||
"current_streak": 0
|
||||
},
|
||||
}
|
||||
|
||||
filtered = _filter_legacy_amd_job_rows(rows)
|
||||
|
||||
self.assertEqual(set(filtered), set(rows))
|
||||
self.assertIs(
|
||||
filtered[
|
||||
"stage-b-test-1-gpu-small-amd (rocm724, linux-mi300-1gpu-sglang, 0)"
|
||||
],
|
||||
new_success,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user