diff --git a/.claude/skills/babysit-pr-to-pass-ci/SKILL.md b/.claude/skills/babysit-pr-to-pass-ci/SKILL.md index 9afaa11a2..350b4ac86 100644 --- a/.claude/skills/babysit-pr-to-pass-ci/SKILL.md +++ b/.claude/skills/babysit-pr-to-pass-ci/SKILL.md @@ -30,7 +30,7 @@ Examples: $babysit-pr-to-pass-ci $babysit-pr-to-pass-ci 12345 $babysit-pr-to-pass-ci https://github.com/sgl-project/sglang/pull/12345 pr-test-extra.yml -$babysit-pr-to-pass-ci 12345 --only pr-test-amd-rocm720.yml +$babysit-pr-to-pass-ci 12345 --only pr-test-amd.yml ``` ## Start or continue the durable goal diff --git a/.github/workflows/amd-aiter-scout.yml b/.github/workflows/amd-aiter-scout.yml index 8c680213a..19e6cf06f 100644 --- a/.github/workflows/amd-aiter-scout.yml +++ b/.github/workflows/amd-aiter-scout.yml @@ -12,7 +12,7 @@ on: type: string default: 'main' job_filter: - description: 'Workflows: nightly-amd, nightly-amd-rocm720, pr-test-amd (ROCm 7.0 shadow), pr-test-amd-rocm720 (ROCm 10 gate). Default: all' + description: 'Workflows: nightly-amd, pr-test-amd. Default: all' required: false type: string default: 'all' @@ -38,9 +38,7 @@ jobs: outputs: aiter_sha: ${{ steps.resolve.outputs.sha }} run_nightly_amd: ${{ steps.parse.outputs.run_nightly_amd }} - run_nightly_amd_rocm720: ${{ steps.parse.outputs.run_nightly_amd_rocm720 }} run_pr_test_amd: ${{ steps.parse.outputs.run_pr_test_amd }} - run_pr_test_amd_rocm720: ${{ steps.parse.outputs.run_pr_test_amd_rocm720 }} steps: - name: Resolve AITER commit id: resolve @@ -73,63 +71,39 @@ jobs: if [[ "$FILTER" == "all" ]]; then echo "run_nightly_amd=true" >> $GITHUB_OUTPUT - echo "run_nightly_amd_rocm720=true" >> $GITHUB_OUTPUT echo "run_pr_test_amd=true" >> $GITHUB_OUTPUT - echo "run_pr_test_amd_rocm720=true" >> $GITHUB_OUTPUT else - # Wrap with commas for exact substring matching (avoids "nightly-amd" matching "nightly-amd-rocm720") + # Wrap with commas for exact substring matching. PADDED=",${FILTER// /}," echo "run_nightly_amd=$(echo "$PADDED" | grep -q ',nightly-amd,' && echo true || echo false)" >> $GITHUB_OUTPUT - echo "run_nightly_amd_rocm720=$(echo "$PADDED" | grep -q ',nightly-amd-rocm720,' && echo true || echo false)" >> $GITHUB_OUTPUT echo "run_pr_test_amd=$(echo "$PADDED" | grep -q ',pr-test-amd,' && echo true || echo false)" >> $GITHUB_OUTPUT - echo "run_pr_test_amd_rocm720=$(echo "$PADDED" | grep -q ',pr-test-amd-rocm720,' && echo true || echo false)" >> $GITHUB_OUTPUT fi echo "### Job Filter" >> $GITHUB_STEP_SUMMARY echo "- **Filter:** \`${FILTER}\`" >> $GITHUB_STEP_SUMMARY call-nightly-amd: + name: Nightly AMD (all ROCm versions) if: needs.resolve-aiter.outputs.run_nightly_amd == 'true' needs: resolve-aiter uses: ./.github/workflows/nightly-test-amd.yml secrets: inherit with: ref: amd/aiter-ci - aiter_ref: ${{ needs.resolve-aiter.outputs.aiter_sha }} - job_filter: 'all' - continue_on_error: ${{ inputs.continue_on_error == '' && true || inputs.continue_on_error }} - - call-nightly-amd-rocm720: - if: needs.resolve-aiter.outputs.run_nightly_amd_rocm720 == 'true' - needs: resolve-aiter - uses: ./.github/workflows/nightly-test-amd-rocm720.yml - secrets: inherit - with: - ref: amd/aiter-ci - rocm_version: rocm10 + rocm_version: all aiter_ref: ${{ needs.resolve-aiter.outputs.aiter_sha }} job_filter: 'all' continue_on_error: ${{ inputs.continue_on_error == '' && true || inputs.continue_on_error }} call-pr-test-amd: + name: PR Test AMD (all ROCm versions) if: needs.resolve-aiter.outputs.run_pr_test_amd == 'true' needs: resolve-aiter uses: ./.github/workflows/pr-test-amd.yml secrets: inherit with: ref: amd/aiter-ci - run_all_tests: true - aiter_ref: ${{ needs.resolve-aiter.outputs.aiter_sha }} - continue_on_error: ${{ inputs.continue_on_error == '' && true || inputs.continue_on_error }} - - call-pr-test-amd-rocm720: - if: needs.resolve-aiter.outputs.run_pr_test_amd_rocm720 == 'true' - needs: resolve-aiter - uses: ./.github/workflows/pr-test-amd-rocm720.yml - secrets: inherit - with: - ref: amd/aiter-ci - rocm_version: rocm10 + rocm_version: all run_all_tests: true aiter_ref: ${{ needs.resolve-aiter.outputs.aiter_sha }} continue_on_error: ${{ inputs.continue_on_error == '' && true || inputs.continue_on_error }} @@ -139,9 +113,7 @@ jobs: needs: - resolve-aiter - call-nightly-amd - - call-nightly-amd-rocm720 - call-pr-test-amd - - call-pr-test-amd-rocm720 runs-on: ubuntu-latest steps: - name: Summary @@ -154,10 +126,8 @@ jobs: echo "" >> $GITHUB_STEP_SUMMARY echo "| Workflow | Result |" >> $GITHUB_STEP_SUMMARY echo "|----------|--------|" >> $GITHUB_STEP_SUMMARY - echo "| Nightly AMD (AITER Latest) | \`${{ needs.call-nightly-amd.result }}\` |" >> $GITHUB_STEP_SUMMARY - echo "| Nightly AMD ROCm 10 | \`${{ needs.call-nightly-amd-rocm720.result }}\` |" >> $GITHUB_STEP_SUMMARY - echo "| PR Test AMD ROCm 7.0 Shadow | \`${{ needs.call-pr-test-amd.result }}\` |" >> $GITHUB_STEP_SUMMARY - echo "| PR Test AMD ROCm 10 Gate | \`${{ needs.call-pr-test-amd-rocm720.result }}\` |" >> $GITHUB_STEP_SUMMARY + echo "| Nightly AMD ROCm10 + ROCm7.2.4 + ROCm7.2.0 | \`${{ needs.call-nightly-amd.result }}\` |" >> $GITHUB_STEP_SUMMARY + echo "| PR Test AMD ROCm10 + ROCm7.2.4 + ROCm7.2.0 Gate | \`${{ needs.call-pr-test-amd.result }}\` |" >> $GITHUB_STEP_SUMMARY - name: Check if any job failed run: | diff --git a/.github/workflows/amd-ci-job-monitor.yml b/.github/workflows/amd-ci-job-monitor.yml index fe6c58a5f..e091fc04e 100644 --- a/.github/workflows/amd-ci-job-monitor.yml +++ b/.github/workflows/amd-ci-job-monitor.yml @@ -53,9 +53,9 @@ jobs: id: select-workflows run: | if [[ -n "${{ inputs.job_filter }}" ]]; then - echo "workflows=pr-test-amd-rocm720.yml" >> "$GITHUB_OUTPUT" + echo "workflows=pr-test-amd.yml" >> "$GITHUB_OUTPUT" else - echo "workflows=pr-test-amd.yml,nightly-test-amd.yml,pr-test-amd-rocm720.yml,nightly-test-amd-rocm720.yml,nightly-amd-mi355x-disagg.yml" >> "$GITHUB_OUTPUT" + echo "workflows=pr-test-amd.yml,nightly-test-amd.yml,nightly-amd-mi355x-disagg.yml" >> "$GITHUB_OUTPUT" fi - name: Fetch Actions data snapshot @@ -106,7 +106,7 @@ jobs: python scripts/ci/utils/query_job_status.py \ --repo ${{ github.repository }} \ --job "${{ inputs.job_filter }}" \ - --workflow "pr-test-amd-rocm720.yml" \ + --workflow "pr-test-amd.yml" \ --hours ${{ inputs.hours || '24' }} \ --input-data-file ci-data/actions-job-snapshot.json \ --summary @@ -119,8 +119,6 @@ jobs: outputs: pr_jobs: ${{ steps.parse.outputs.pr_jobs }} nightly_jobs: ${{ steps.parse.outputs.nightly_jobs }} - pr_rocm700_jobs: ${{ steps.parse.outputs.pr_rocm700_jobs }} - nightly_rocm700_jobs: ${{ steps.parse.outputs.nightly_rocm700_jobs }} disagg_jobs: ${{ steps.parse.outputs.disagg_jobs }} steps: - name: Checkout code @@ -131,21 +129,18 @@ jobs: run: | set -euo pipefail - # Parse the default ROCm 7.2 PR gate (exclude utility jobs) + # Parse the canonical multi-version AMD PR gate (exclude utility jobs) # Excluded: call-gate, check-changes, finish, cancel, check-all-jobs, # plus the stage waits -- they run on ubuntu-latest, so the # self-hosted-only snapshot has nothing for them. Keep the extra call: # its delegated GPU jobs use the explicit caller name as their prefix. - # Job ids keep the -rocm720 lane suffix but display names no longer do, - # so strip it to get the prefix query_job_status.py matches against. - pr_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/pr-test-amd-rocm720.yml | \ - grep -v -E '^(call-gate|check-changes|pr-test-amd-rocm720-finish|cancel|check-all-jobs|wait-for-stage-[ab]-amd-rocm720)$' | \ - sed 's/-rocm720$//' | \ + pr_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/pr-test-amd.yml | \ + grep -v -E '^(call-gate|check-changes|pr-test-amd-finish|cancel|check-all-jobs|wait-for-stage-[ab]-amd)$' | \ jq -R -s -c 'split("\n") | map(select(length > 0))') echo "pr_jobs=$pr_jobs" >> $GITHUB_OUTPUT echo "PR jobs: $pr_jobs" - # Parse the default ROCm 7.2 nightly (exclude utility jobs) + # Parse the canonical multi-version AMD nightly (exclude utility jobs) # Excluded: check-all-jobs # One nightly run covers every flavor, so ask for each by its full name: # a stem-only prefix matches every flavor and merges them into a single @@ -153,33 +148,15 @@ jobs: # names carry the runner too, so pair each job id with its runs-on. # Keep in sync with the rocm_version matrix in the nightly workflow. rocm_flavors="rocm10 rocm724 rocm720" - nightly_jobs=$(yq -r '.jobs | to_entries[] | .key + " " + .value."runs-on"' .github/workflows/nightly-test-amd-rocm720.yml | \ + nightly_jobs=$(yq -r '.jobs | to_entries[] | .key + " " + .value."runs-on"' .github/workflows/nightly-test-amd.yml | \ grep -v -E '^check-all-jobs ' | \ while read -r id runner; do - stem=${id%-rocm720} - for f in $rocm_flavors; do echo "$stem ($f, $runner)"; done + for f in $rocm_flavors; do echo "$id ($f, $runner)"; done done | \ jq -R -s -c 'split("\n") | map(select(length > 0))') echo "nightly_jobs=$nightly_jobs" >> $GITHUB_OUTPUT echo "Nightly jobs: $nightly_jobs" - # Parse the ROCm 7.0 rollback shadow (exclude utility jobs) - # Excluded: call-gate, check-changes, finish, and the stage waits. - # Keep the reusable extra caller so its nested GPU jobs remain covered. - pr_rocm700_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/pr-test-amd.yml | \ - grep -v -E '^(call-gate|check-changes|pr-test-amd-finish|wait-for-stage-[ab]-amd)$' | \ - jq -R -s -c 'split("\n") | map(select(length > 0))') - echo "pr_rocm700_jobs=$pr_rocm700_jobs" >> $GITHUB_OUTPUT - echo "PR ROCm 7.0 shadow jobs: $pr_rocm700_jobs" - - # Parse the ROCm 7.0 rollback shadow nightly (exclude utility jobs) - # Excluded: check-all-jobs - nightly_rocm700_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/nightly-test-amd.yml | \ - grep -v -E '^(check-all-jobs)$' | \ - jq -R -s -c 'split("\n") | map(select(length > 0))') - echo "nightly_rocm700_jobs=$nightly_rocm700_jobs" >> $GITHUB_OUTPUT - echo "Nightly ROCm 7.0 shadow jobs: $nightly_rocm700_jobs" - # Parse nightly-amd-mi355x-disagg.yml (exclude utility jobs) # Excluded: setup, collect-results -- both run on ubuntu-latest and the # snapshot only keeps self-hosted jobs, so they would report as empty. @@ -223,7 +200,7 @@ jobs: python scripts/ci/utils/query_job_status.py \ --repo ${{ github.repository }} \ --job "${{ matrix.job_name }}" \ - --workflow "pr-test-amd-rocm720.yml" \ + --workflow "pr-test-amd.yml" \ --hours ${{ inputs.hours || '24' }} \ --input-data-file ci-data/actions-job-snapshot.json \ --summary @@ -257,84 +234,6 @@ jobs: path: ci-data - name: Generate Nightly Report - timeout-minutes: 15 - run: | - python scripts/ci/utils/query_job_status.py \ - --repo ${{ github.repository }} \ - --job "${{ matrix.job_name }}" \ - --workflow "nightly-test-amd-rocm720.yml" \ - --hours ${{ inputs.hours || '24' }} \ - --input-data-file ci-data/actions-job-snapshot.json \ - --summary - - # ROCm 7.0 rollback-shadow reports using dynamic matrix - pr-rocm700-ci-reports: - name: PR ROCm700 Shadow - ${{ matrix.job_name }} - needs: [parse-workflows, fetch-actions-data] - if: ${{ !inputs.job_filter }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - job_name: ${{ fromJson(needs.parse-workflows.outputs.pr_rocm700_jobs) }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - - name: Install dependencies - run: pip install tabulate - - - name: Download Actions data snapshot - uses: actions/download-artifact@v4 - with: - name: actions-job-snapshot - path: ci-data - - - name: Generate PR ROCm 7.0 Shadow Report - timeout-minutes: 15 - run: | - python scripts/ci/utils/query_job_status.py \ - --repo ${{ github.repository }} \ - --job "${{ matrix.job_name }}" \ - --workflow "pr-test-amd.yml" \ - --hours ${{ inputs.hours || '24' }} \ - --input-data-file ci-data/actions-job-snapshot.json \ - --summary - - # ROCm 7.0 rollback-shadow nightly reports using dynamic matrix - nightly-rocm700-reports: - name: Nightly ROCm700 Shadow - ${{ matrix.job_name }} - needs: [parse-workflows, fetch-actions-data] - if: ${{ !inputs.job_filter }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - job_name: ${{ fromJson(needs.parse-workflows.outputs.nightly_rocm700_jobs) }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - - name: Install dependencies - run: pip install tabulate - - - name: Download Actions data snapshot - uses: actions/download-artifact@v4 - with: - name: actions-job-snapshot - path: ci-data - - - name: Generate Nightly ROCm 7.0 Shadow Report timeout-minutes: 15 run: | python scripts/ci/utils/query_job_status.py \ @@ -414,7 +313,7 @@ jobs: python scripts/ci/utils/query_job_status.py \ --repo ${{ github.repository }} \ --runner-report \ - --workflow "pr-test-amd.yml,nightly-test-amd.yml,pr-test-amd-rocm720.yml,nightly-test-amd-rocm720.yml,nightly-amd-mi355x-disagg.yml" \ + --workflow "pr-test-amd.yml,nightly-test-amd.yml,nightly-amd-mi355x-disagg.yml" \ --hours ${{ inputs.hours || '24' }} \ --input-data-file ci-data/actions-job-snapshot.json \ --summary diff --git a/.github/workflows/bot-bump-sglang-version.yml b/.github/workflows/bot-bump-sglang-version.yml index ef3dd5591..125229b30 100644 --- a/.github/workflows/bot-bump-sglang-version.yml +++ b/.github/workflows/bot-bump-sglang-version.yml @@ -62,7 +62,7 @@ jobs: run-nightly-tests-amd: needs: bump-sglang-version - uses: ./.github/workflows/nightly-test-amd-rocm720.yml + uses: ./.github/workflows/nightly-test-amd.yml with: ref: ${{ needs.bump-sglang-version.outputs.branch_name }} rocm_version: rocm10 diff --git a/.github/workflows/nightly-test-amd-rocm720.yml b/.github/workflows/nightly-test-amd-rocm720.yml deleted file mode 100644 index 96361f658..000000000 --- a/.github/workflows/nightly-test-amd-rocm720.yml +++ /dev/null @@ -1,2392 +0,0 @@ -name: Nightly Test (AMD ROCm 7.2) - -on: - schedule: - - cron: '30 17 * * *' - push: - branches: - - main - paths: - - "python/sglang/version.py" - workflow_dispatch: - inputs: - rocm_version: - description: 'ROCm image version ("all" runs rocm10, rocm724, and rocm720)' - required: false - type: choice - default: 'all' - options: - - 'all' - - rocm10 - - rocm724 - - rocm720 - aiter_ref: - description: 'Override AITER commit (optional, leave empty to use Dockerfile default)' - required: false - type: string - default: '' - continue_on_error: - description: 'Continue on error (do not fail the workflow on test failures)' - required: false - type: boolean - default: true - job_select: - description: 'Select a job to run from dropdown (choose "all" to run all jobs)' - required: false - type: choice - default: 'all' - options: - - 'all' - # 1-GPU Unit Tests (MI30x + MI35x) - - nightly-test-1-gpu-unit-rocm720 - - nightly-test-1-gpu-kernel-rocm720 - - nightly-test-1-gpu-mi35x-rocm720 - # 2-GPU and 4-GPU Tests (MI30x + MI35x) - - nightly-accuracy-2-gpu-rocm720 - - nightly-accuracy-2-gpu-vlm-rocm720 - - nightly-perf-2-gpu-text-rocm720 - - nightly-perf-2-gpu-vlm-rocm720 - - nightly-4-gpu-rocm720 - # 2-GPU GLM-5.1-MXFP4 (MI35x only) - - nightly-2-gpu-mi35x-glm51-mxfp4-rocm720 - # 2-GPU DeepSeek-R1-MXFP4 TP2 (MI35x only) - - nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2-rocm720 - # 8-GPU GPT-OSS (MI35x mixes Qwen3-Coder-Next) - - nightly-accuracy-8-gpu-rocm720 - - nightly-accuracy-8-gpu-mi35x-rocm720 - # 8-GPU Grok1-INT4 (MI30x + MI35x) - - nightly-8-gpu-grok1-int4-rocm720 - - nightly-8-gpu-mi35x-grok1-int4-rocm720 - # 8-GPU Grok2 (MI30x + MI35x) - - nightly-8-gpu-grok2-rocm720 - - nightly-8-gpu-mi35x-grok2-rocm720 - # 8-GPU DeepSeek-V3.2 (MI30x) - - nightly-8-gpu-deepseek-v32-rocm720 - - nightly-8-gpu-deepseek-v32-mtp-rocm720 - # 8-GPU DeepSeek-V3.2 (MI35x) - - nightly-8-gpu-mi35x-deepseek-v32-rocm720 - - nightly-8-gpu-mi35x-deepseek-v32-mtp-rocm720 - # 8-GPU DeepSeek-R1 (MI35x only) - - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-rocm720 - - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8-rocm720 - - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion-rocm720 - - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4-rocm720 - - nightly-8-gpu-mi35x-deepseek-r1-hicache-rocm720 - # 8-GPU DeepSeek-V4-Flash (MI30x) - - nightly-8-gpu-deepseek-v4-flash-rocm720 - # 8-GPU DeepSeek-V4 (MI35x) - - nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720 - - nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720 - - nightly-8-gpu-mi35x-deepseek-v4-pro-mtp-rocm720 - - nightly-8-gpu-mi35x-deepseek-v4-pro-dspark-rocm720 - # 8-GPU Kimi-K2.6 (MI30x) - - nightly-8-gpu-kimi-k26-rocm720 - # 8-GPU Kimi-K3 (MI35x only - native MXFP4 needs gfx95x) - - nightly-8-gpu-mi35x-kimi-k3-rocm720 - # 8-GPU Qwen3.8-MXFP4 (MI35x only - MXFP4 needs gfx95x) - - nightly-8-gpu-mi35x-qwen38-mxfp4-rocm720 - # 8-GPU Qwen 3.5 (MI30x + MI35x) - - nightly-8-gpu-qwen35-rocm720 - - nightly-8-gpu-mi35x-qwen35-rocm720 - - nightly-8-gpu-mi35x-qwen35-triton-dcp-rocm720 - # 8-GPU GLM-5.1 (MI30x) - - nightly-8-gpu-glm51-rocm720 - # 8-GPU GLM-5.2-FP8 (MI35x accuracy + performance) - - nightly-8-gpu-mi35x-glm52-fp8-rocm720 - # 8-GPU GLM-5-MXFP4 (MI35x only) - - nightly-8-gpu-mi35x-glm5-mxfp4-rocm720 - # 4-GPU MiniMax-M2.5 (MI35x) - - nightly-4-gpu-mi35x-minimax-m25-rocm720 - # 4-GPU MiniMax-M3 MXFP8 (MI35x accuracy + performance) - - nightly-4-gpu-mi35x-minimax-m3-rocm720 - # 8-GPU MiniMax-M2.7 (MI30x only) - - nightly-8-gpu-minimax-m27-rocm720 - # Diffusion (MI30x) - - nightly-1-gpu-zimage-turbo-rocm720 - job_filter: - description: 'Or type comma-separated job names (overrides dropdown if non-empty)' - required: false - type: string - default: '' - workflow_call: - inputs: - rocm_version: - description: 'ROCm image version ("all" runs rocm10, rocm724, and rocm720)' - required: false - type: string - # A single flavor, unlike the schedule and the dispatch form: a caller - # that says nothing should not silently get multi-version GPU cost. - default: rocm10 - ref: - description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.' - required: false - type: string - default: '' - aiter_ref: - description: 'Override AITER commit (optional, leave empty to use Dockerfile default)' - required: false - type: string - default: '' - job_filter: - description: 'Select which job to run (leave empty or "all" to run all jobs)' - required: false - type: string - default: 'all' - continue_on_error: - description: 'Continue on error (do not fail the workflow on test failures)' - required: false - type: boolean - default: true - -env: - AITER_COMMIT_OVERRIDE: ${{ inputs.aiter_ref }} - DOCKERHUB_AMD_USERNAME: ${{ secrets.DOCKERHUB_AMD_USERNAME }} - DOCKERHUB_AMD_TOKEN: ${{ secrets.DOCKERHUB_AMD_TOKEN }} - -concurrency: - # When called via workflow_call with ref set, use a unique group per caller run to avoid - # collisions with direct schedule/push triggers. We use inputs.ref (not github.event_name) - # to detect this, because github.event_name inherits from the caller in workflow_call. - # Manual dispatch and scheduled runs also get unique groups so a version bump - # or the next daily run cannot cancel an in-progress three-version nightly. - group: nightly-test-amd-rocm720-${{ github.event_name == 'workflow_dispatch' && format('manual-{0}', github.run_id) || github.event_name == 'schedule' && format('scheduled-{0}', github.run_id) || inputs.ref && format('caller-{0}', github.run_id) || github.ref }} - cancel-in-progress: ${{ !inputs.ref && github.event_name != 'workflow_call' && github.event_name != 'workflow_dispatch' && github.event_name != 'schedule' }} - -jobs: - # ============================================== MI30x ROCm 7.2 Unit Tests ============================================== - # 1-GPU Unit Tests - LoRA, debug utils, scheduler, etc. (MI30x ROCm 7.2) - - # ============================================================================== - # 1-GPU Unit Tests (MI30x + MI35x) - # ============================================================================== - - nightly-test-1-gpu-unit-rocm720: - name: ${{ format('nightly-test-1-gpu-unit ({0}, linux-mi300-1gpu-sglang)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-unit-rocm720,')) - runs-on: linux-mi300-1gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - name: Nightly Unit Test ROCm 7.2 (1-GPU) - timeout-minutes: 90 - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-1-gpu --nightly --timeout-per-file 900 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - nightly-test-1-gpu-kernel-rocm720: - name: ${{ format('nightly-test-1-gpu-kernel ({0}, linux-mi300-1gpu-sglang)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-kernel-rocm720,')) - runs-on: linux-mi300-1gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - name: Nightly Kernel Benchmark ROCm 7.2 (1-GPU) - timeout-minutes: 60 - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-kernel-1-gpu --nightly --timeout-per-file 900 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - nightly-test-1-gpu-mi35x-rocm720: - name: ${{ format('nightly-test-1-gpu-mi35x ({0}, linux-mi35x-gpu-1)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-mi35x-rocm720,')) - runs-on: linux-mi35x-gpu-1 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh - - name: Nightly Test MI35x ROCm 7.2 (1-GPU) - timeout-minutes: 90 - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-1-gpu-mi35x --nightly --timeout-per-file 900 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # ============================================================================== - # 2-GPU and 4-GPU Tests (MI30x + MI35x) - # ============================================================================== - - nightly-accuracy-2-gpu-rocm720: - name: ${{ format('nightly-accuracy-2-gpu ({0}, linux-mi300-2gpu-sglang)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-2-gpu-rocm720,')) - runs-on: linux-mi300-2gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - name: Nightly Test ROCm 7.2 (2-GPU) - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - nightly-2-gpu-mi35x-glm51-mxfp4-rocm720: - name: ${{ format('nightly-2-gpu-mi35x-glm51-mxfp4 ({0}, linux-mi35x-gpu-2)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-2-gpu-mi35x-glm51-mxfp4-rocm720,')) - runs-on: linux-mi35x-gpu-2 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Accuracy Test MI35x ROCm 7.2 (2-GPU GLM-5.1-MXFP4 GSM8K) - timeout-minutes: 120 - run: | - > github_summary.md - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-2-gpu-mi35x-glm51-mxfp4 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2-rocm720: - name: ${{ format('nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2 ({0}, linux-mi35x-gpu-2)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2-rocm720,')) - runs-on: linux-mi35x-gpu-2 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Accuracy Test MI35x ROCm 7.2 (2-GPU DeepSeek-R1-MXFP4 TP2) - timeout-minutes: 180 - run: | - > github_summary.md - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-2-gpu-mi35x-deepseek-r1-mxfp4-tp2 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - nightly-accuracy-2-gpu-vlm-rocm720: - name: ${{ format('nightly-accuracy-2-gpu-vlm ({0}, linux-mi300-2gpu-sglang)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-2-gpu-vlm-rocm720,')) - runs-on: linux-mi300-2gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - name: Nightly Accuracy Test ROCm 7.2 (2-GPU VLM MMMU) - timeout-minutes: 180 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-2-gpu-vlm --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - nightly-perf-2-gpu-text-rocm720: - name: ${{ format('nightly-perf-2-gpu-text ({0}, linux-mi300-2gpu-sglang)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-2-gpu-text-rocm720,')) - runs-on: linux-mi300-2gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - name: Performance Test (2-GPU Text Models) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e SGLANG_USE_AITER=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-perf-text-2-gpu --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - nightly-perf-2-gpu-vlm-rocm720: - name: ${{ format('nightly-perf-2-gpu-vlm ({0}, linux-mi300-2gpu-sglang)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-2-gpu-vlm-rocm720,')) - runs-on: linux-mi300-2gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - name: Performance Test ROCm 7.2 (2-GPU VLM Models) - timeout-minutes: 180 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e SGLANG_USE_AITER=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-perf-vlm-2-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - nightly-4-gpu-rocm720: - name: ${{ format('nightly-4-gpu ({0}, linux-mi300-4gpu-sglang)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu-rocm720,')) - runs-on: linux-mi300-4gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Nightly Test ROCm 7.2 (4-GPU) - timeout-minutes: 120 - run: | - > github_summary.md - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-4-gpu --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # ============================================================================== - # 8-GPU GPT-OSS (MI35x mixes Qwen3-Coder-Next) - # ============================================================================== - - nightly-accuracy-8-gpu-rocm720: - name: ${{ format('nightly-accuracy-8-gpu ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-rocm720,')) - runs-on: linux-mi300-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - - name: Accuracy Test ROCm 7.2 (8-GPU GPT-OSS) - timeout-minutes: 180 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-gpt-oss --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # Shares this job with the accuracy step above: it already serves the same - # two models, and an MI30x job spends ~49 min pulling the image and - # installing dependencies before it runs anything. - - name: Performance Test ROCm 7.2 (8-GPU GPT-OSS) - if: ${{ !cancelled() }} - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-gpt-oss --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - nightly-accuracy-8-gpu-mi35x-rocm720: - name: ${{ format('nightly-accuracy-8-gpu-mi35x ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Accuracy Test MI35x ROCm 7.2 (8-GPU GPT-OSS) - timeout-minutes: 180 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # Shares this job with the accuracy step above rather than taking its own, - # so the container setup and the GPT-OSS weight cache are paid for once. - - name: Performance Test MI35x ROCm 7.2 (8-GPU GPT-OSS) - if: ${{ !cancelled() }} - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-gpt-oss --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # ============================================================================== - # 8-GPU Grok1-INT4 (MI30x + MI35x) - # ============================================================================== - - nightly-8-gpu-grok1-int4-rocm720: - name: ${{ format('nightly-8-gpu-grok1-int4 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok1-int4-rocm720,')) - runs-on: linux-mi300-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - - name: Accuracy Test ROCm 7.2 (8-GPU Grok1-INT4) - timeout-minutes: 60 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e RCCL_MSCCL_ENABLE=0 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-grok1-int4 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test ROCm 7.2 (8-GPU Grok1-INT4) - timeout-minutes: 60 - continue-on-error: true - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e RCCL_MSCCL_ENABLE=0 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-grok1-int4 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - nightly-8-gpu-mi35x-grok1-int4-rocm720: - name: ${{ format('nightly-8-gpu-mi35x-grok1-int4 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-grok1-int4-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Accuracy Test MI35x ROCm 7.2 (8-GPU Grok1-INT4) - timeout-minutes: 60 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e RCCL_MSCCL_ENABLE=0 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-grok1-int4 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test MI35x ROCm 7.2 (8-GPU Grok1-INT4) - timeout-minutes: 60 - continue-on-error: true - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e RCCL_MSCCL_ENABLE=0 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-grok1-int4 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # ============================================================================== - # 8-GPU Grok2 (MI30x + MI35x) - # ============================================================================== - - nightly-8-gpu-grok2-rocm720: - name: ${{ format('nightly-8-gpu-grok2 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok2-rocm720,')) - runs-on: linux-mi300-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - - name: Accuracy Test ROCm 7.2 (8-GPU Grok2) - timeout-minutes: 60 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e RCCL_MSCCL_ENABLE=0 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-grok2 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test ROCm 7.2 (8-GPU Grok2) - timeout-minutes: 60 - continue-on-error: true - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e RCCL_MSCCL_ENABLE=0 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-grok2 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - nightly-8-gpu-mi35x-grok2-rocm720: - name: ${{ format('nightly-8-gpu-mi35x-grok2 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-grok2-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Accuracy Test MI35x ROCm 7.2 (8-GPU Grok2) - timeout-minutes: 60 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e RCCL_MSCCL_ENABLE=0 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-grok2 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test MI35x ROCm 7.2 (8-GPU Grok2) - timeout-minutes: 60 - continue-on-error: true - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e RCCL_MSCCL_ENABLE=0 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-grok2 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # ============================================================================== - # 8-GPU DeepSeek-V3.2 (MI30x) - # - # V3-0324 and V3.1 are no longer scheduled here: V3.2 covers the same MI30x - # paths (aiter MLA, EAGLE MTP, multithread weight load) on a current - # checkpoint, and DeepSeek-R1/V4 carry the MI35x side. - # ============================================================================== - - nightly-8-gpu-deepseek-v32-rocm720: - name: ${{ format('nightly-8-gpu-deepseek-v32 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v32-rocm720,')) - runs-on: linux-mi300-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - - name: Accuracy Test ROCm 7.2 (8-GPU DeepSeek-V3.2 Basic) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v32 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test ROCm 7.2 (8-GPU DeepSeek-V3.2 Basic) - timeout-minutes: 150 - continue-on-error: true - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-deepseek-v32-basic --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - nightly-8-gpu-deepseek-v32-mtp-rocm720: - name: ${{ format('nightly-8-gpu-deepseek-v32-mtp ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v32-mtp-rocm720,')) - runs-on: linux-mi300-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - - name: Accuracy Test ROCm 7.2 (8-GPU DeepSeek-V3.2 MTP) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v32-mtp --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test ROCm 7.2 (8-GPU DeepSeek-V3.2 MTP) - timeout-minutes: 180 - continue-on-error: true - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-deepseek-v32-mtp --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # ============================================================================== - # 8-GPU DeepSeek-V3.2 (MI35x) - # - # Accuracy and performance share one job per config, as the MI30x V3.2 jobs - # above already do: both steps serve the same weights, so splitting them cost - # a second container setup and a second cold weight load for nothing. - # ============================================================================== - - nightly-8-gpu-mi35x-deepseek-v32-rocm720: - name: ${{ format('nightly-8-gpu-mi35x-deepseek-v32 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v32-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Accuracy Test MI35x ROCm 7.2 (8-GPU DeepSeek-V3.2) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v32 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test MI35x (8-GPU DeepSeek-V3.2 Basic) - timeout-minutes: 150 - continue-on-error: true - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-deepseek-v32-basic --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - nightly-8-gpu-mi35x-deepseek-v32-mtp-rocm720: - name: ${{ format('nightly-8-gpu-mi35x-deepseek-v32-mtp ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v32-mtp-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Accuracy Test MI35x ROCm 7.2 (8-GPU DeepSeek-V3.2 TP+MTP) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-deepseek-v32-mtp --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test MI35x ROCm 7.2 (8-GPU DeepSeek-V3.2 MTP) - timeout-minutes: 180 - continue-on-error: true - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-deepseek-v32-mtp --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # ============================================================================== - # 8-GPU DeepSeek-R1 (MI35x only) - # ============================================================================== - - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-rocm720: - name: ${{ format('nightly-8-gpu-mi35x-deepseek-r1-mxfp4 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Accuracy Test MI35x ROCm 7.2 (8-GPU DeepSeek-R1-MXFP4) - timeout-minutes: 180 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-r1-mxfp4 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test MI35x ROCm 7.2 (8-GPU DeepSeek-R1-MXFP4) - timeout-minutes: 300 - continue-on-error: true - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 registered/amd/perf/mi35x/test_deepseek_r1_mxfp4_perf_mi35x.py || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4-rocm720: - name: ${{ format('nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Accuracy Test MI35x ROCm 7.2 (DeepSeek-R1-MXFP4 TP4) - timeout-minutes: 180 - run: | - > github_summary.md - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 registered/amd/accuracy/mi35x/test_deepseek_r1_mxfp4_tp4_mi35x.py || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Accuracy Test MI35x ROCm 7.2 (DeepSeek-R1-MXFP4 TP4 MTP) - if: ${{ !cancelled() }} - timeout-minutes: 180 - run: | - > github_summary.md - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 registered/amd/accuracy/mi35x/test_deepseek_r1_mxfp4_tp4_mtp_mi35x.py || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8-rocm720: - name: ${{ format('nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Accuracy Test MI35x ROCm 7.2 (8-GPU DeepSeek-R1-MXFP4 KV FP8) - timeout-minutes: 180 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test MI35x ROCm 7.2 (8-GPU DeepSeek-R1-MXFP4 KV FP8) - timeout-minutes: 300 - continue-on-error: true - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 registered/amd/perf/mi35x/test_deepseek_r1_mxfp4_kv_fp8_perf_mi35x.py || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion-rocm720: - name: ${{ format('nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Accuracy Test MI35x ROCm 7.2 (8-GPU DeepSeek-R1-MXFP4 AllReduce Fusion) - timeout-minutes: 180 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test MI35x ROCm 7.2 (8-GPU DeepSeek-R1-MXFP4 AllReduce Fusion) - timeout-minutes: 300 - continue-on-error: true - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 registered/amd/perf/mi35x/test_deepseek_r1_mxfp4_ar_fusion_perf_mi35x.py || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - nightly-8-gpu-mi35x-deepseek-r1-hicache-rocm720: - name: ${{ format('nightly-8-gpu-mi35x-deepseek-r1-hicache ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-hicache-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Accuracy Test MI35x ROCm 7.2 (8-GPU DeepSeek-R1 HiCache) - timeout-minutes: 180 - run: | - > github_summary.md - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-r1-hicache --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # ============================================================================== - # 8-GPU DeepSeek-V4-Flash (MI30x) - # - # Accuracy only: the MI35x jobs below carry the DSV4 perf numbers, and gfx942 - # has no DSV4 perf baseline to regress against yet. Runs the cookbook's - # MI300X Flash FP8 low-latency cell, so it also covers MLA + KV-FP8 on gfx942. - # ============================================================================== - - nightly-8-gpu-deepseek-v4-flash-rocm720: - name: ${{ format('nightly-8-gpu-deepseek-v4-flash ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v4-flash-rocm720,')) - runs-on: linux-mi300-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - - name: Accuracy Test ROCm 7.2 (8-GPU DeepSeek-V4-Flash FP8) - timeout-minutes: 180 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v4-flash --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # ============================================================================== - # 8-GPU DeepSeek-V4 (MI35x) - # ============================================================================== - - nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720: - name: ${{ format('nightly-8-gpu-mi35x-deepseek-v4-flash ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: | - # --skip-test-time-deps: GSM8K + bench_one_batch_server don't need lmms-eval / human-eval. - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Accuracy + Performance Test MI35x ROCm 7.2 (8-GPU DeepSeek-V4-Flash FP8 + FP4, unified_kv_triton) - timeout-minutes: 300 - run: | - > github_summary.md # Clear summary file - echo "## SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton" >> github_summary.md - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-flash --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} - echo "$(> $GITHUB_STEP_SUMMARY || true - - - name: Accuracy + Performance Test MI35x ROCm 7.2 (8-GPU DeepSeek-V4-Flash FP8 + FP4, triton) - if: ${{ !cancelled() }} - timeout-minutes: 300 - run: | - > github_summary.md # Clear summary file - echo "## SGLANG_HACK_FLASHMLA_BACKEND=triton" >> github_summary.md - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_HACK_FLASHMLA_BACKEND=triton \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-flash --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} - echo "$(> $GITHUB_STEP_SUMMARY || true - - nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720: - name: ${{ format('nightly-8-gpu-mi35x-deepseek-v4-pro ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: | - # --skip-test-time-deps: GSM8K + bench_one_batch_server don't need lmms-eval / human-eval. - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Accuracy + Performance Test MI35x ROCm 7.2 (8-GPU DeepSeek-V4-Pro FP8 + FP4, unified_kv_triton) - timeout-minutes: 480 - run: | - > github_summary.md # Clear summary file - echo "## SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton" >> github_summary.md - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-pro --nightly --timeout-per-file 14400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} - echo "$(> $GITHUB_STEP_SUMMARY || true - - - name: Accuracy + Performance Test MI35x ROCm 7.2 (8-GPU DeepSeek-V4-Pro FP8 + FP4, triton) - if: ${{ !cancelled() }} - timeout-minutes: 480 - run: | - > github_summary.md # Clear summary file - echo "## SGLANG_HACK_FLASHMLA_BACKEND=triton" >> github_summary.md - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_HACK_FLASHMLA_BACKEND=triton \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-pro --nightly --timeout-per-file 14400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} - echo "$(> $GITHUB_STEP_SUMMARY || true - - nightly-8-gpu-mi35x-deepseek-v4-pro-mtp-rocm720: - name: ${{ format('nightly-8-gpu-mi35x-deepseek-v4-pro-mtp ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v4-pro-mtp-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: | - # --skip-test-time-deps: GSM8K + send_one don't need lmms-eval / human-eval. - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Accuracy + MTP Accept Length Test MI35x ROCm 7.2 (8-GPU DeepSeek-V4-Pro FP8 MTP, unified_kv_triton) - timeout-minutes: 240 - run: | - > github_summary.md # Clear summary file - echo "## SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton" >> github_summary.md - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-pro-mtp --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} - echo "$(> $GITHUB_STEP_SUMMARY || true - - - name: Accuracy + MTP Accept Length Test MI35x ROCm 7.2 (8-GPU DeepSeek-V4-Pro FP8 MTP, triton) - if: ${{ !cancelled() }} - timeout-minutes: 240 - run: | - > github_summary.md # Clear summary file - echo "## SGLANG_HACK_FLASHMLA_BACKEND=triton" >> github_summary.md - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_HACK_FLASHMLA_BACKEND=triton \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-pro-mtp --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} - echo "$(> $GITHUB_STEP_SUMMARY || true - - nightly-8-gpu-mi35x-deepseek-v4-pro-dspark-rocm720: - name: ${{ format('nightly-8-gpu-mi35x-deepseek-v4-pro-dspark ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v4-pro-dspark-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: | - # --skip-test-time-deps: GSM8K doesn't need lmms-eval / human-eval. - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Accuracy + DSpark Accept Length Test MI35x ROCm 7.2 (8-GPU DeepSeek-V4-Pro-DSpark FP4, unified_kv_triton) - timeout-minutes: 300 - run: | - > github_summary.md # Clear summary file - echo "## SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton" >> github_summary.md - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-pro-dspark --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} - echo "$(> $GITHUB_STEP_SUMMARY || true - - # ============================================================================== - # 8-GPU Kimi-K2.6 (MI30x) - # - # Kept on MI30x: Kimi-K3 needs gfx95x for its native MXFP4 weights, so it - # cannot take over here the way it does on MI35x. - # ============================================================================== - - nightly-8-gpu-kimi-k26-rocm720: - name: ${{ format('nightly-8-gpu-kimi-k26 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-kimi-k26-rocm720,')) - runs-on: linux-mi300-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - - name: Accuracy Test ROCm 7.2 (8-GPU Kimi-K2.6) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-kimi-k26 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # ============================================================================== - # 8-GPU Kimi-K3 (MI35x only - native MXFP4 needs gfx95x) - # ============================================================================== - - nightly-8-gpu-mi35x-kimi-k3-rocm720: - name: ${{ format('nightly-8-gpu-mi35x-kimi-k3 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-kimi-k3-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - # The 2.8T MXFP4 checkpoint is ~1.56 TB, so weight download and load - # dominate wall time well before the eval itself starts. - - name: Accuracy Test MI35x ROCm 7.2 (8-GPU Kimi-K3) - timeout-minutes: 300 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-kimi-k3 --nightly --timeout-per-file 14400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # Shares the job with the eval above so the checkpoint is already cached - # and only one 8-GPU MI35x slot is consumed. Step ordering is also the - # accuracy gate: a failed eval fails the job before this runs, so - # throughput is never measured on a build that got the tokens wrong. - # continue-on-error matches every other combined accuracy + perf job here, - # so a perf hiccup cannot turn the accuracy result red. - - name: Performance Test MI35x ROCm 7.2 (8-GPU Kimi-K3) - timeout-minutes: 300 - continue-on-error: true - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 registered/amd/perf/mi35x/test_kimi_k3_perf_mi35x.py || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # ============================================================================== - # 8-GPU Qwen 3.5 (MI30x + MI35x) - # ============================================================================== - - nightly-8-gpu-qwen35-rocm720: - name: ${{ format('nightly-8-gpu-qwen35 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-qwen35-rocm720,')) - runs-on: linux-mi300-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-aiter-build --skip-test-time-deps - bash scripts/ci/amd/amd_ci_exec.sh pip install mistral-common "lm-eval[api]" - - - name: Accuracy Test ROCm 7.2 (8-GPU Qwen 3.5) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-qwen35 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test ROCm 7.2 (8-GPU Qwen 3.5 FP8) - timeout-minutes: 120 - continue-on-error: true - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e SGLANG_USE_AITER=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-qwen35-fp8 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - nightly-8-gpu-mi35x-qwen35-rocm720: - name: ${{ format('nightly-8-gpu-mi35x-qwen35 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen35-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-aiter-build --skip-test-time-deps - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - bash scripts/ci/amd/amd_ci_exec.sh pip install mistral-common "lm-eval[api]" - - - name: Accuracy Test MI35x ROCm 7.2 (8-GPU Qwen 3.5) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-qwen35 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test MI35x ROCm 7.2 (8-GPU Qwen 3.5 FP8) - timeout-minutes: 120 - continue-on-error: true - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_USE_AITER=1 \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-qwen35-fp8 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - nightly-8-gpu-mi35x-qwen35-triton-dcp-rocm720: - name: ${{ format('nightly-8-gpu-mi35x-qwen35-triton-dcp ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen35-triton-dcp-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-aiter-build --skip-test-time-deps - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Accuracy Test MI35x ROCm 7.2 (8-GPU Qwen 3.5 Triton DCP) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-qwen35-triton-dcp --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # ============================================================================== - # 8-GPU Qwen3.8-MXFP4 (MI35x only - MXFP4 needs gfx95x) - # ============================================================================== - - nightly-8-gpu-mi35x-qwen38-mxfp4-rocm720: - name: ${{ format('nightly-8-gpu-mi35x-qwen38-mxfp4 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen38-mxfp4-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - # ~1.2 TB of weights against a fleet-shared cache that runs at 100% used. - # Run 32196787596 died 40 minutes into the download on ENOSPC with no - # free-space figure anywhere in its log; record one up front instead. - - name: Check model cache space - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout \ - bash scripts/ci/amd/check_hf_cache_space.sh \ - amd/Qwen3.8-2.4T-A95B-Quark-MXFP4 1250 - - # One step, not the usual accuracy/perf pair: both phases live in the - # same suite so they launch from one server config. The 2.4T MXFP4 - # checkpoint is ~1.2 TB and is loaded once per phase, which dominates - # wall time well before either the eval or the benchmark starts. - - name: Accuracy Test + Performance Test MI35x ROCm 7.2 (8-GPU Qwen3.8-MXFP4) - timeout-minutes: 420 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-qwen38-mxfp4 --nightly --timeout-per-file 21600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # ============================================================================== - # 8-GPU GLM-5.1 (MI30x + MI35x) - # ============================================================================== - - nightly-8-gpu-glm51-rocm720: - name: ${{ format('nightly-8-gpu-glm51 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-glm51-rocm720,')) - runs-on: linux-mi300-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - - name: Accuracy Test ROCm 7.2 (8-GPU GLM-5.1 DSA) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-glm51 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test ROCm 7.2 (8-GPU GLM-5.1) - timeout-minutes: 120 - continue-on-error: true - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e SGLANG_USE_AITER=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-glm51 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # ============================================================================== - # 8-GPU GLM-5.2-FP8 (MI35x accuracy + performance) - # ============================================================================== - - nightly-8-gpu-mi35x-glm52-fp8-rocm720: - name: ${{ format('nightly-8-gpu-mi35x-glm52-fp8 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm52-fp8-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - # The ~700 GB FP8 checkpoint means weight download and load dominate wall - # time well before the eval itself starts. - - name: Accuracy Test MI35x ROCm 7.2 (8-GPU GLM-5.2-FP8 DSA) - timeout-minutes: 180 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-glm52-fp8 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # Run perf only after correctness passes. Keeping both steps in this job - # reuses the cached ~700 GB checkpoint and one scarce 8-GPU MI35x slot. - - name: Performance Test MI35x ROCm 7.2 (8-GPU GLM-5.2-FP8) - timeout-minutes: 120 - continue-on-error: true - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-glm52-fp8 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # ============================================================================== - # 8-GPU GLM-5-MXFP4 (MI35x only) - # ============================================================================== - - nightly-8-gpu-mi35x-glm5-mxfp4-rocm720: - name: ${{ format('nightly-8-gpu-mi35x-glm5-mxfp4 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm5-mxfp4-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Accuracy Test MI35x ROCm 7.2 (8-GPU GLM-5-MXFP4) - timeout-minutes: 180 - run: | - > github_summary.md - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_USE_AITER=1 \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-glm5-mxfp4 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test MI35x ROCm 7.2 (8-GPU GLM-5-MXFP4) - timeout-minutes: 300 - continue-on-error: true - run: | - > github_summary.md - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_USE_AITER=1 \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 registered/amd/perf/mi35x/test_glm5_mxfp4_perf_mi35x.py || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # ============================================================================== - # 4-GPU MiniMax-M2.5 (MI35x) - # ============================================================================== - - nightly-4-gpu-mi35x-minimax-m25-rocm720: - name: ${{ format('nightly-4-gpu-mi35x-minimax-m25 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu-mi35x-minimax-m25-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - - name: Accuracy Test MI35x ROCm 7.2 (4-GPU MiniMax-M2.5) - timeout-minutes: 120 - run: | - > github_summary.md - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_USE_AITER=1 \ - -e SGLANG_USE_AITER_UNIFIED_ATTN=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-4-gpu-mi35x-minimax-m25-tp4 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # ============================================================================== - # 4-GPU MiniMax-M3 MXFP8 (MI35x accuracy + performance) - # ============================================================================== - - nightly-4-gpu-mi35x-minimax-m3-rocm720: - name: ${{ format('nightly-4-gpu-mi35x-minimax-m3 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu-mi35x-minimax-m3-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - - name: Accuracy Test MI35x ROCm 7.2 (4-GPU MiniMax-M3 MXFP8) - timeout-minutes: 120 - run: | - > github_summary.md - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_USE_AITER=1 \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-4-gpu-mi35x-minimax-m3-tp4 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # Run perf only after correctness passes. Keeping both steps in this job - # reuses the cached checkpoint and one scarce MI35x slot. - - name: Performance Test MI35x ROCm 7.2 (4-GPU MiniMax-M3 MXFP8) - timeout-minutes: 120 - continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed - run: | - > github_summary.md - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_USE_AITER=1 \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-4-gpu-mi35x-minimax-m3 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # ============================================================================== - # 8-GPU MiniMax-M2.7 (MI30x only) - # ============================================================================== - - nightly-8-gpu-minimax-m27-rocm720: - name: ${{ format('nightly-8-gpu-minimax-m27 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-minimax-m27-rocm720,')) - runs-on: linux-mi300-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - - name: Accuracy Test ROCm 7.2 (8-GPU MiniMax-M2.7) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e SGLANG_USE_AITER=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-minimax-m27 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test ROCm 7.2 (8-GPU MiniMax-M2.7) - timeout-minutes: 120 - continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - -e SGLANG_USE_AITER=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-minimax-m27 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # ============================================================================== - # Diffusion (MI30x) - # ============================================================================== - - nightly-1-gpu-zimage-turbo-rocm720: - name: ${{ format('nightly-1-gpu-zimage-turbo ({0}, linux-mi300-1gpu-sglang)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-1-gpu-zimage-turbo-rocm720,')) - runs-on: linux-mi300-1gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Z-Image-Turbo Diffusion Test ROCm 7.2 (1-GPU) - timeout-minutes: 45 - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - -e SGLANG_DIFFUSION_ARTIFACT_DIR="/sglang-checkout/diffusion-artifacts" \ - pytest test/registered/amd/test_zimage_turbo.py -v -s ${{ inputs.continue_on_error && '|| true' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Upload generated images - if: always() - uses: actions/upload-artifact@v4 - with: - name: zimage-turbo-outputs-${{ matrix.rocm_version }} - path: diffusion-artifacts/ - if-no-files-found: ignore - retention-days: 30 - - check-all-jobs: - if: always() && (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch') - needs: - # 1-GPU Unit Tests (MI30x + MI35x) - - nightly-test-1-gpu-unit-rocm720 - - nightly-test-1-gpu-mi35x-rocm720 - # 2-GPU and 4-GPU Tests (MI30x + MI35x) - - nightly-accuracy-2-gpu-rocm720 - - nightly-accuracy-2-gpu-vlm-rocm720 - - nightly-perf-2-gpu-text-rocm720 - - nightly-perf-2-gpu-vlm-rocm720 - - nightly-4-gpu-rocm720 - - nightly-2-gpu-mi35x-glm51-mxfp4-rocm720 - - nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2-rocm720 - # 8-GPU GPT-OSS (MI35x mixes Qwen3-Coder-Next) - - nightly-accuracy-8-gpu-rocm720 - - nightly-accuracy-8-gpu-mi35x-rocm720 - # 8-GPU Grok1-INT4 (MI30x + MI35x) - - nightly-8-gpu-grok1-int4-rocm720 - - nightly-8-gpu-mi35x-grok1-int4-rocm720 - # 8-GPU Grok2 (MI30x + MI35x) - - nightly-8-gpu-grok2-rocm720 - - nightly-8-gpu-mi35x-grok2-rocm720 - # 8-GPU DeepSeek-V3.2 (MI30x) - - nightly-8-gpu-deepseek-v32-rocm720 - - nightly-8-gpu-deepseek-v32-mtp-rocm720 - # 8-GPU DeepSeek-V3.2 (MI35x) - - nightly-8-gpu-mi35x-deepseek-v32-rocm720 - - nightly-8-gpu-mi35x-deepseek-v32-mtp-rocm720 - # 8-GPU DeepSeek-R1 (MI35x only) - - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-rocm720 - - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8-rocm720 - - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion-rocm720 - - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4-rocm720 - - nightly-8-gpu-mi35x-deepseek-r1-hicache-rocm720 - # 8-GPU DeepSeek-V4-Flash (MI30x) - - nightly-8-gpu-deepseek-v4-flash-rocm720 - # 8-GPU DeepSeek-V4 (MI35x) - - nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720 - - nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720 - - nightly-8-gpu-mi35x-deepseek-v4-pro-mtp-rocm720 - - nightly-8-gpu-mi35x-deepseek-v4-pro-dspark-rocm720 - # 8-GPU Kimi-K2.6 (MI30x) - - nightly-8-gpu-kimi-k26-rocm720 - # 8-GPU Kimi-K3 (MI35x only - native MXFP4 needs gfx95x) - - nightly-8-gpu-mi35x-kimi-k3-rocm720 - # 8-GPU Qwen3.8-MXFP4 (MI35x only - MXFP4 needs gfx95x) - - nightly-8-gpu-mi35x-qwen38-mxfp4-rocm720 - # 8-GPU Qwen 3.5 (MI30x + MI35x) - - nightly-8-gpu-qwen35-rocm720 - - nightly-8-gpu-mi35x-qwen35-rocm720 - - nightly-8-gpu-mi35x-qwen35-triton-dcp-rocm720 - # 8-GPU GLM-5.1 (MI30x) - - nightly-8-gpu-glm51-rocm720 - # 8-GPU GLM-5.2-FP8 (MI35x accuracy + performance) - - nightly-8-gpu-mi35x-glm52-fp8-rocm720 - # 8-GPU GLM-5-MXFP4 (MI35x only) - - nightly-8-gpu-mi35x-glm5-mxfp4-rocm720 - # 4-GPU MiniMax-M2.5 (MI35x) - - nightly-4-gpu-mi35x-minimax-m25-rocm720 - # 4-GPU MiniMax-M3 MXFP8 (MI35x accuracy + performance) - - nightly-4-gpu-mi35x-minimax-m3-rocm720 - # 8-GPU MiniMax-M2.7 (MI30x only) - - nightly-8-gpu-minimax-m27-rocm720 - # Diffusion (MI30x) - - nightly-1-gpu-zimage-turbo-rocm720 - runs-on: ubuntu-latest - steps: - - name: Check if any job failed - run: | - if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then - echo "One or more ROCm 7.2 nightly test jobs failed" - exit 1 - fi - if [[ "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then - echo "One or more ROCm 7.2 nightly test jobs were cancelled" - exit 1 - fi - echo "All ROCm 7.2 nightly test jobs passed" diff --git a/.github/workflows/nightly-test-amd.yml b/.github/workflows/nightly-test-amd.yml index c748e4098..c2a931bec 100644 --- a/.github/workflows/nightly-test-amd.yml +++ b/.github/workflows/nightly-test-amd.yml @@ -1,8 +1,8 @@ -name: Nightly Test (AMD ROCm 7.0) +name: Nightly Test (AMD) on: schedule: - - cron: '30 17 */2 * *' + - cron: '30 17 * * *' push: branches: - main @@ -10,6 +10,16 @@ on: - "python/sglang/version.py" workflow_dispatch: inputs: + rocm_version: + description: 'ROCm image version ("all" runs rocm10, rocm724, and rocm720)' + required: false + type: choice + default: rocm10 + options: + - 'all' + - rocm10 + - rocm724 + - rocm720 aiter_ref: description: 'Override AITER commit (optional, leave empty to use Dockerfile default)' required: false @@ -50,40 +60,44 @@ on: # 8-GPU Grok2 (MI30x + MI35x) - nightly-8-gpu-grok2 - nightly-8-gpu-mi35x-grok2 - # 8-GPU DeepSeek-V3.x (MI30x) - - nightly-8-gpu-deepseek-v31 + # 8-GPU DeepSeek-V3.2 (MI30x) - nightly-8-gpu-deepseek-v32 - nightly-8-gpu-deepseek-v32-mtp - - nightly-8-gpu-deepseek-v3-kv-fp8 # 8-GPU DeepSeek-V3.2 (MI35x) - - nightly-accuracy-8-gpu-mi35x-deepseek-v32 - - nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp - - nightly-perf-8-gpu-mi35x-deepseek-v32-basic - - nightly-perf-8-gpu-mi35x-deepseek-v32-mtp + - nightly-8-gpu-mi35x-deepseek-v32 + - nightly-8-gpu-mi35x-deepseek-v32-mtp # 8-GPU DeepSeek-R1 (MI35x only) - nightly-8-gpu-mi35x-deepseek-r1-mxfp4 - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8 - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4 - nightly-8-gpu-mi35x-deepseek-r1-hicache - # 8-GPU Kimi-K2.6 (MI30x + MI35x) + # 8-GPU DeepSeek-V4-Flash (MI30x) + - nightly-8-gpu-deepseek-v4-flash + # 8-GPU DeepSeek-V4 (MI35x) + - nightly-8-gpu-mi35x-deepseek-v4-flash + - nightly-8-gpu-mi35x-deepseek-v4-pro + - nightly-8-gpu-mi35x-deepseek-v4-pro-mtp + - nightly-8-gpu-mi35x-deepseek-v4-pro-dspark + # 8-GPU Kimi-K2.6 (MI30x) - nightly-8-gpu-kimi-k26 - - nightly-8-gpu-mi35x-kimi-k26 - # 8-GPU Qwen3-235B (MI30x + MI35x MXFP4) - - nightly-8-gpu-qwen3-235b - - nightly-8-gpu-mi35x-qwen3-235b-mxfp4 + # 8-GPU Kimi-K3 (MI35x only - native MXFP4 needs gfx95x) + - nightly-8-gpu-mi35x-kimi-k3 + # 8-GPU Qwen3.8-MXFP4 (MI35x only - MXFP4 needs gfx95x) + - nightly-8-gpu-mi35x-qwen38-mxfp4 # 8-GPU Qwen 3.5 (MI30x + MI35x) - nightly-8-gpu-qwen35 - nightly-8-gpu-mi35x-qwen35 - nightly-8-gpu-mi35x-qwen35-triton-dcp - # 8-GPU GLM-5.1 (MI30x + MI35x) + # 8-GPU GLM-5.1 (MI30x) - nightly-8-gpu-glm51 - - nightly-8-gpu-mi35x-glm51 + # 8-GPU GLM-5.2-FP8 (MI35x accuracy + performance) + - nightly-8-gpu-mi35x-glm52-fp8 # 8-GPU GLM-5-MXFP4 (MI35x only) - nightly-8-gpu-mi35x-glm5-mxfp4 # 4-GPU MiniMax-M2.5 (MI35x) - nightly-4-gpu-mi35x-minimax-m25 - # 4-GPU MiniMax-M3 MXFP8 (MI35x) + # 4-GPU MiniMax-M3 MXFP8 (MI35x accuracy + performance) - nightly-4-gpu-mi35x-minimax-m3 # 8-GPU MiniMax-M2.7 (MI30x only) - nightly-8-gpu-minimax-m27 @@ -96,6 +110,13 @@ on: default: '' workflow_call: inputs: + rocm_version: + description: 'ROCm image version ("all" runs rocm10, rocm724, and rocm720)' + required: false + type: string + # A caller that says nothing gets the same single-flavor default as a + # manual dispatch; pass "all" explicitly for multi-version GPU coverage. + default: rocm10 ref: description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.' required: false @@ -126,20 +147,25 @@ concurrency: # When called via workflow_call with ref set, use a unique group per caller run to avoid # collisions with direct schedule/push triggers. We use inputs.ref (not github.event_name) # to detect this, because github.event_name inherits from the caller in workflow_call. - # Manual dispatch runs also get unique groups so they never cancel each other. - group: nightly-test-amd-${{ github.event_name == 'workflow_dispatch' && format('manual-{0}', github.run_id) || inputs.ref && format('caller-{0}', github.run_id) || github.ref }} - cancel-in-progress: ${{ !inputs.ref && github.event_name != 'workflow_call' && github.event_name != 'workflow_dispatch' }} + # Manual dispatch and scheduled runs also get unique groups so a version bump + # or the next daily run cannot cancel an in-progress three-version nightly. + group: nightly-test-amd-${{ github.event_name == 'workflow_dispatch' && format('manual-{0}', github.run_id) || github.event_name == 'schedule' && format('scheduled-{0}', github.run_id) || inputs.ref && format('caller-{0}', github.run_id) || github.ref }} + cancel-in-progress: ${{ !inputs.ref && github.event_name != 'workflow_call' && github.event_name != 'workflow_dispatch' && github.event_name != 'schedule' }} jobs: - # ============================================== MI30x Unit Tests ============================================== - # 1-GPU Unit Tests - LoRA, debug utils, scheduler, etc. (MI30x only) + # ============================================== MI30x ROCm Unit Tests ============================================== + # 1-GPU Unit Tests - LoRA, debug utils, scheduler, etc. (MI30x ROCm) # ============================================================================== # 1-GPU Unit Tests (MI30x + MI35x) # ============================================================================== nightly-test-1-gpu-unit: - name: nightly-test-1-gpu-unit (rocm700, linux-mi300-1gpu-sglang) + name: ${{ format('nightly-test-1-gpu-unit ({0}, linux-mi300-1gpu-sglang)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-unit,')) runs-on: linux-mi300-1gpu-sglang steps: @@ -151,17 +177,16 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Nightly Unit Test (1-GPU) + - name: Nightly Unit Test ROCm (1-GPU) timeout-minutes: 90 run: | bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ @@ -171,7 +196,11 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-test-1-gpu-kernel: - name: nightly-test-1-gpu-kernel (rocm700, linux-mi300-1gpu-sglang) + name: ${{ format('nightly-test-1-gpu-kernel ({0}, linux-mi300-1gpu-sglang)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-kernel,')) runs-on: linux-mi300-1gpu-sglang steps: @@ -183,17 +212,16 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Nightly Kernel Benchmark (1-GPU) + - name: Nightly Kernel Benchmark ROCm (1-GPU) timeout-minutes: 60 run: | bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ @@ -203,7 +231,11 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-test-1-gpu-mi35x: - name: nightly-test-1-gpu-mi35x (rocm700, linux-mi35x-gpu-1) + name: ${{ format('nightly-test-1-gpu-mi35x ({0}, linux-mi35x-gpu-1)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-mi35x,')) runs-on: linux-mi35x-gpu-1 steps: @@ -215,20 +247,17 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: | bash scripts/ci/amd/amd_ci_install_dependency.sh - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Nightly Test MI35x (1-GPU) + - name: Nightly Test MI35x ROCm (1-GPU) timeout-minutes: 90 run: | bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ @@ -242,7 +271,11 @@ jobs: # ============================================================================== nightly-accuracy-2-gpu: - name: nightly-accuracy-2-gpu (rocm700, linux-mi300-2gpu-sglang) + name: ${{ format('nightly-accuracy-2-gpu ({0}, linux-mi300-2gpu-sglang)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-2-gpu,')) runs-on: linux-mi300-2gpu-sglang steps: @@ -254,27 +287,31 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Nightly Test (2-GPU) + - name: Nightly Test ROCm (2-GPU) run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} nightly-2-gpu-mi35x-glm51-mxfp4: - name: nightly-2-gpu-mi35x-glm51-mxfp4 (rocm700, linux-mi35x-gpu-2) + name: ${{ format('nightly-2-gpu-mi35x-glm51-mxfp4 ({0}, linux-mi35x-gpu-2)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-2-gpu-mi35x-glm51-mxfp4,')) runs-on: linux-mi35x-gpu-2 steps: @@ -286,30 +323,35 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - name: Accuracy Test MI35x (2-GPU GLM-5.1-MXFP4 GSM8K) + - name: Accuracy Test MI35x ROCm (2-GPU GLM-5.1-MXFP4 GSM8K) timeout-minutes: 120 run: | > github_summary.md bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-2-gpu-mi35x-glm51-mxfp4 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2: - name: nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2 (rocm700, linux-mi35x-gpu-2) + name: ${{ format('nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2 ({0}, linux-mi35x-gpu-2)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2,')) runs-on: linux-mi35x-gpu-2 steps: @@ -321,31 +363,36 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - name: Accuracy Test MI35x (2-GPU DeepSeek-R1-MXFP4 TP2) + - name: Accuracy Test MI35x ROCm (2-GPU DeepSeek-R1-MXFP4 TP2) timeout-minutes: 180 run: | > github_summary.md bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-2-gpu-mi35x-deepseek-r1-mxfp4-tp2 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} nightly-accuracy-2-gpu-vlm: - name: nightly-accuracy-2-gpu-vlm (rocm700, linux-mi300-2gpu-sglang) + name: ${{ format('nightly-accuracy-2-gpu-vlm ({0}, linux-mi300-2gpu-sglang)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-2-gpu-vlm,')) runs-on: linux-mi300-2gpu-sglang steps: @@ -357,28 +404,32 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Nightly Accuracy Test (2-GPU VLM MMMU) + - name: Nightly Accuracy Test ROCm (2-GPU VLM MMMU) timeout-minutes: 180 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-accuracy-2-gpu-vlm --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} nightly-perf-2-gpu-text: - name: nightly-perf-2-gpu-text (rocm700, linux-mi300-2gpu-sglang) + name: ${{ format('nightly-perf-2-gpu-text ({0}, linux-mi300-2gpu-sglang)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-2-gpu-text,')) runs-on: linux-mi300-2gpu-sglang steps: @@ -390,21 +441,21 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - name: Performance Test (2-GPU Text Models) timeout-minutes: 120 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e SGLANG_USE_AITER=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-perf-text-2-gpu --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? @@ -412,7 +463,11 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-perf-2-gpu-vlm: - name: nightly-perf-2-gpu-vlm (rocm700, linux-mi300-2gpu-sglang) + name: ${{ format('nightly-perf-2-gpu-vlm ({0}, linux-mi300-2gpu-sglang)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-2-gpu-vlm,')) runs-on: linux-mi300-2gpu-sglang steps: @@ -424,21 +479,21 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Performance Test (2-GPU VLM Models) + - name: Performance Test ROCm (2-GPU VLM Models) timeout-minutes: 180 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e SGLANG_USE_AITER=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-perf-vlm-2-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? @@ -446,7 +501,11 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-4-gpu: - name: nightly-4-gpu (rocm700, linux-mi300-4gpu-sglang) + name: ${{ format('nightly-4-gpu ({0}, linux-mi300-4gpu-sglang)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu,')) runs-on: linux-mi300-4gpu-sglang steps: @@ -458,17 +517,17 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - name: Nightly Test (4-GPU) + - name: Nightly Test ROCm (4-GPU) timeout-minutes: 120 run: | > github_summary.md @@ -483,7 +542,11 @@ jobs: # ============================================================================== nightly-accuracy-8-gpu: - name: nightly-accuracy-8-gpu (rocm700, linux-mi300-8gpu-sglang) + name: ${{ format('nightly-accuracy-8-gpu ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -495,27 +558,48 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh + run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - name: Accuracy Test (8-GPU GPT-OSS) + - name: Accuracy Test ROCm (8-GPU GPT-OSS) timeout-minutes: 180 run: | + > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-gpt-oss --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} + # Shares this job with the accuracy step above: it already serves the same + # two models, and an MI30x job spends ~49 min pulling the image and + # installing dependencies before it runs anything. + - name: Performance Test ROCm (8-GPU GPT-OSS) + if: ${{ !cancelled() }} + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-gpt-oss --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + nightly-accuracy-8-gpu-mi35x: - name: nightly-accuracy-8-gpu-mi35x (rocm700, linux-mi35x-gpu-8) + name: ${{ format('nightly-accuracy-8-gpu-mi35x ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x,')) runs-on: linux-mi35x-gpu-8 steps: @@ -527,34 +611,54 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps # Install tabulate for run_suite.py (missing in MI35x container) bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - name: Accuracy Test MI35x (8-GPU GPT-OSS) + - name: Accuracy Test MI35x ROCm (8-GPU GPT-OSS) timeout-minutes: 180 run: | + > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} + # Shares this job with the accuracy step above rather than taking its own, + # so the container setup and the GPT-OSS weight cache are paid for once. + - name: Performance Test MI35x ROCm (8-GPU GPT-OSS) + if: ${{ !cancelled() }} + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-gpt-oss --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + # ============================================================================== # 8-GPU Grok1-INT4 (MI30x + MI35x) # ============================================================================== nightly-8-gpu-grok1-int4: - name: nightly-8-gpu-grok1-int4 (rocm700, linux-mi300-8gpu-sglang) + name: ${{ format('nightly-8-gpu-grok1-int4 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok1-int4,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -566,17 +670,17 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh + run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - name: Accuracy Test (8-GPU Grok1-INT4) + - name: Accuracy Test ROCm (8-GPU Grok1-INT4) timeout-minutes: 60 run: | > github_summary.md # Clear summary file @@ -587,9 +691,9 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - - name: Performance Test (8-GPU Grok1-INT4) + - name: Performance Test ROCm (8-GPU Grok1-INT4) timeout-minutes: 60 - continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed + continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ @@ -600,7 +704,11 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-grok1-int4: - name: nightly-8-gpu-mi35x-grok1-int4 (rocm700, linux-mi35x-gpu-8) + name: ${{ format('nightly-8-gpu-mi35x-grok1-int4 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-grok1-int4,')) runs-on: linux-mi35x-gpu-8 steps: @@ -612,33 +720,33 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps # Install tabulate for run_suite.py (missing in MI35x container) bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - name: Accuracy Test MI35x (8-GPU Grok1-INT4) - timeout-minutes: 90 + - name: Accuracy Test MI35x ROCm (8-GPU Grok1-INT4) + timeout-minutes: 60 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e RCCL_MSCCL_ENABLE=0 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-grok1-int4 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-grok1-int4 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - - name: Performance Test MI35x (8-GPU Grok1-INT4) + - name: Performance Test MI35x ROCm (8-GPU Grok1-INT4) timeout-minutes: 60 - continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed + continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ @@ -653,7 +761,11 @@ jobs: # ============================================================================== nightly-8-gpu-grok2: - name: nightly-8-gpu-grok2 (rocm700, linux-mi300-8gpu-sglang) + name: ${{ format('nightly-8-gpu-grok2 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok2,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -665,17 +777,17 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh + run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - name: Accuracy Test (8-GPU Grok2) + - name: Accuracy Test ROCm (8-GPU Grok2) timeout-minutes: 60 run: | > github_summary.md # Clear summary file @@ -686,9 +798,9 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - - name: Performance Test (8-GPU Grok2) + - name: Performance Test ROCm (8-GPU Grok2) timeout-minutes: 60 - continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed + continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ @@ -699,7 +811,11 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-grok2: - name: nightly-8-gpu-mi35x-grok2 (rocm700, linux-mi35x-gpu-8) + name: ${{ format('nightly-8-gpu-mi35x-grok2 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-grok2,')) runs-on: linux-mi35x-gpu-8 steps: @@ -711,20 +827,20 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps # Install tabulate for run_suite.py (missing in MI35x container) bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - name: Accuracy Test MI35x (8-GPU Grok2) + - name: Accuracy Test MI35x ROCm (8-GPU Grok2) timeout-minutes: 60 run: | > github_summary.md # Clear summary file @@ -735,9 +851,9 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - - name: Performance Test MI35x (8-GPU Grok2) + - name: Performance Test MI35x ROCm (8-GPU Grok2) timeout-minutes: 60 - continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed + continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ @@ -748,58 +864,19 @@ jobs: exit ${TEST_EXIT_CODE:-0} # ============================================================================== - # 8-GPU DeepSeek-V3.x (MI30x) + # 8-GPU DeepSeek-V3.2 (MI30x) + # + # V3-0324 and V3.1 are no longer scheduled here: V3.2 covers the same MI30x + # paths (aiter MLA, EAGLE MTP, multithread weight load) on a current + # checkpoint, and DeepSeek-R1/V4 carry the MI35x side. # ============================================================================== - nightly-8-gpu-deepseek-v31: - name: nightly-8-gpu-deepseek-v31 (rocm700, linux-mi300-8gpu-sglang) - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v31,')) - runs-on: linux-mi300-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Accuracy Test (8-GPU DeepSeek-V3.1) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_USE_AITER=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v31 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test (8-GPU DeepSeek-V3.1) - timeout-minutes: 300 - continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_USE_ROCM700A=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-deepseek-v31 --nightly --timeout-per-file 18000 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - nightly-8-gpu-deepseek-v32: - name: nightly-8-gpu-deepseek-v32 (rocm700, linux-mi300-8gpu-sglang) + name: ${{ format('nightly-8-gpu-deepseek-v32 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v32,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -811,18 +888,18 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh + run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - name: Accuracy Test (8-GPU DeepSeek-V3.2 Basic) + - name: Accuracy Test ROCm (8-GPU DeepSeek-V3.2 Basic) timeout-minutes: 120 run: | > github_summary.md # Clear summary file @@ -832,9 +909,9 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - - name: Performance Test (8-GPU DeepSeek-V3.2 Basic) + - name: Performance Test ROCm (8-GPU DeepSeek-V3.2 Basic) timeout-minutes: 150 - continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed + continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ @@ -844,7 +921,11 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-deepseek-v32-mtp: - name: nightly-8-gpu-deepseek-v32-mtp (rocm700, linux-mi300-8gpu-sglang) + name: ${{ format('nightly-8-gpu-deepseek-v32-mtp ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v32-mtp,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -856,18 +937,18 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh + run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - name: Accuracy Test (8-GPU DeepSeek-V3.2 MTP) + - name: Accuracy Test ROCm (8-GPU DeepSeek-V3.2 MTP) timeout-minutes: 120 run: | > github_summary.md # Clear summary file @@ -877,9 +958,9 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - - name: Performance Test (8-GPU DeepSeek-V3.2 MTP) + - name: Performance Test ROCm (8-GPU DeepSeek-V3.2 MTP) timeout-minutes: 180 - continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed + continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ @@ -888,47 +969,21 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - nightly-8-gpu-deepseek-v3-kv-fp8: - name: nightly-8-gpu-deepseek-v3-kv-fp8 (rocm700, linux-mi300-8gpu-sglang) - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v3-kv-fp8,')) - runs-on: linux-mi300-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: DeepSeek-V3 KV FP8 Test (8-GPU Basic + MTP) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-deepseek-v3-kv-fp8 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - # ============================================================================== # 8-GPU DeepSeek-V3.2 (MI35x) + # + # Accuracy and performance share one job per config, as the MI30x V3.2 jobs + # above already do: both steps serve the same weights, so splitting them cost + # a second container setup and a second cold weight load for nothing. # ============================================================================== - nightly-accuracy-8-gpu-mi35x-deepseek-v32: - name: nightly-accuracy-8-gpu-mi35x-deepseek-v32 (rocm700, linux-mi35x-gpu-8) - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x-deepseek-v32,')) + nightly-8-gpu-mi35x-deepseek-v32: + name: ${{ format('nightly-8-gpu-mi35x-deepseek-v32 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v32,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code @@ -939,33 +994,50 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps # Install tabulate for run_suite.py (missing in MI35x container) bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - name: Accuracy Test MI35x (8-GPU DeepSeek-V3.2) + - name: Accuracy Test MI35x ROCm (8-GPU DeepSeek-V3.2) timeout-minutes: 120 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v32 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp: - name: nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp (rocm700, linux-mi35x-gpu-8) - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp,')) + - name: Performance Test MI35x (8-GPU DeepSeek-V3.2 Basic) + timeout-minutes: 150 + continue-on-error: true + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-deepseek-v32-basic --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + nightly-8-gpu-mi35x-deepseek-v32-mtp: + name: ${{ format('nightly-8-gpu-mi35x-deepseek-v32-mtp ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v32-mtp,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code @@ -976,21 +1048,21 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps # Install tabulate for run_suite.py (missing in MI35x container) bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - name: Accuracy Test MI35x (8-GPU DeepSeek-V3.2 TP+MTP) + - name: Accuracy Test MI35x ROCm (8-GPU DeepSeek-V3.2 TP+MTP) timeout-minutes: 120 run: | > github_summary.md # Clear summary file @@ -1000,72 +1072,9 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - nightly-perf-8-gpu-mi35x-deepseek-v32-basic: - name: nightly-perf-8-gpu-mi35x-deepseek-v32-basic (rocm700, linux-mi35x-gpu-8) - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-8-gpu-mi35x-deepseek-v32-basic,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Performance Test MI35x (8-GPU DeepSeek-V3.2 Basic) - timeout-minutes: 150 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-deepseek-v32-basic --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - nightly-perf-8-gpu-mi35x-deepseek-v32-mtp: - name: nightly-perf-8-gpu-mi35x-deepseek-v32-mtp (rocm700, linux-mi35x-gpu-8) - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-8-gpu-mi35x-deepseek-v32-mtp,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Performance Test MI35x (8-GPU DeepSeek-V3.2 MTP) + - name: Performance Test MI35x ROCm (8-GPU DeepSeek-V3.2 MTP) timeout-minutes: 180 + continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ @@ -1079,7 +1088,11 @@ jobs: # ============================================================================== nightly-8-gpu-mi35x-deepseek-r1-mxfp4: - name: nightly-8-gpu-mi35x-deepseek-r1-mxfp4 (rocm700, linux-mi35x-gpu-8) + name: ${{ format('nightly-8-gpu-mi35x-deepseek-r1-mxfp4 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1091,43 +1104,49 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps # Install tabulate for run_suite.py (missing in MI35x container) bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - name: Accuracy Test MI35x (8-GPU DeepSeek-R1-MXFP4) + - name: Accuracy Test MI35x ROCm (8-GPU DeepSeek-R1-MXFP4) timeout-minutes: 180 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-r1-mxfp4 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - - name: Performance Test MI35x (8-GPU DeepSeek-R1-MXFP4) + - name: Performance Test MI35x ROCm (8-GPU DeepSeek-R1-MXFP4) timeout-minutes: 300 - continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed + continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 registered/amd/perf/mi35x/test_deepseek_r1_mxfp4_perf_mi35x.py || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4: - name: nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4 (rocm700, linux-mi35x-gpu-8) + name: ${{ format('nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1139,42 +1158,48 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - name: Accuracy Test MI35x (DeepSeek-R1-MXFP4 TP4) + - name: Accuracy Test MI35x ROCm (DeepSeek-R1-MXFP4 TP4) timeout-minutes: 180 run: | > github_summary.md bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 registered/amd/accuracy/mi35x/test_deepseek_r1_mxfp4_tp4_mi35x.py || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - - name: Accuracy Test MI35x (DeepSeek-R1-MXFP4 TP4 MTP) + - name: Accuracy Test MI35x ROCm (DeepSeek-R1-MXFP4 TP4 MTP) if: ${{ !cancelled() }} timeout-minutes: 180 run: | > github_summary.md bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 registered/amd/accuracy/mi35x/test_deepseek_r1_mxfp4_tp4_mtp_mi35x.py || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8: - name: nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8 (rocm700, linux-mi35x-gpu-8) + name: ${{ format('nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1186,43 +1211,49 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps # Install tabulate for run_suite.py (missing in MI35x container) bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - name: Accuracy Test MI35x (8-GPU DeepSeek-R1-MXFP4 KV FP8) + - name: Accuracy Test MI35x ROCm (8-GPU DeepSeek-R1-MXFP4 KV FP8) timeout-minutes: 180 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - - name: Performance Test MI35x (8-GPU DeepSeek-R1-MXFP4 KV FP8) + - name: Performance Test MI35x ROCm (8-GPU DeepSeek-R1-MXFP4 KV FP8) timeout-minutes: 300 - continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed + continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 registered/amd/perf/mi35x/test_deepseek_r1_mxfp4_kv_fp8_perf_mi35x.py || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion: - name: nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion (rocm700, linux-mi35x-gpu-8) + name: ${{ format('nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1234,43 +1265,49 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps # Install tabulate for run_suite.py (missing in MI35x container) bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - name: Accuracy Test MI35x (8-GPU DeepSeek-R1-MXFP4 AllReduce Fusion) + - name: Accuracy Test MI35x ROCm (8-GPU DeepSeek-R1-MXFP4 AllReduce Fusion) timeout-minutes: 180 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - - name: Performance Test MI35x (8-GPU DeepSeek-R1-MXFP4 AllReduce Fusion) + - name: Performance Test MI35x ROCm (8-GPU DeepSeek-R1-MXFP4 AllReduce Fusion) timeout-minutes: 300 - continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed + continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 registered/amd/perf/mi35x/test_deepseek_r1_mxfp4_ar_fusion_perf_mi35x.py || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-deepseek-r1-hicache: - name: nightly-8-gpu-mi35x-deepseek-r1-hicache (rocm700, linux-mi35x-gpu-8) + name: ${{ format('nightly-8-gpu-mi35x-deepseek-r1-hicache ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-hicache,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1282,19 +1319,19 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - name: Accuracy Test MI35x (8-GPU DeepSeek-R1 HiCache) + - name: Accuracy Test MI35x ROCm (8-GPU DeepSeek-R1 HiCache) timeout-minutes: 180 run: | > github_summary.md @@ -1305,11 +1342,280 @@ jobs: exit ${TEST_EXIT_CODE:-0} # ============================================================================== - # 8-GPU Kimi-K2.6 (MI30x + MI35x) + # 8-GPU DeepSeek-V4-Flash (MI30x) + # + # Accuracy only: the MI35x jobs below carry the DSV4 perf numbers, and gfx942 + # has no DSV4 perf baseline to regress against yet. Runs the cookbook's + # MI300X Flash FP8 low-latency cell, so it also covers MLA + KV-FP8 on gfx942. + # ============================================================================== + + nightly-8-gpu-deepseek-v4-flash: + name: ${{ format('nightly-8-gpu-deepseek-v4-flash ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v4-flash,')) + runs-on: linux-mi300-8gpu-sglang + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (${{ matrix.rocm_version }}) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + ENABLE_CACHE_HOST: "1" + + - name: Install dependencies + run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + + - name: Accuracy Test ROCm (8-GPU DeepSeek-V4-Flash FP8) + timeout-minutes: 180 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v4-flash --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + # ============================================================================== + # 8-GPU DeepSeek-V4 (MI35x) + # ============================================================================== + + nightly-8-gpu-mi35x-deepseek-v4-flash: + name: ${{ format('nightly-8-gpu-mi35x-deepseek-v4-flash ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v4-flash,')) + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (${{ matrix.rocm_version }}) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + ENABLE_CACHE_HOST: "1" + + - name: Install dependencies + run: | + # --skip-test-time-deps: GSM8K + bench_one_batch_server don't need lmms-eval / human-eval. + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + + - name: Accuracy + Performance Test MI35x ROCm (8-GPU DeepSeek-V4-Flash FP8 + FP4, unified_kv_triton) + timeout-minutes: 300 + run: | + > github_summary.md # Clear summary file + echo "## SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton" >> github_summary.md + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-flash --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} + echo "$(> $GITHUB_STEP_SUMMARY || true + + - name: Accuracy + Performance Test MI35x ROCm (8-GPU DeepSeek-V4-Flash FP8 + FP4, triton) + if: ${{ !cancelled() }} + timeout-minutes: 300 + run: | + > github_summary.md # Clear summary file + echo "## SGLANG_HACK_FLASHMLA_BACKEND=triton" >> github_summary.md + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_HACK_FLASHMLA_BACKEND=triton \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-flash --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} + echo "$(> $GITHUB_STEP_SUMMARY || true + + nightly-8-gpu-mi35x-deepseek-v4-pro: + name: ${{ format('nightly-8-gpu-mi35x-deepseek-v4-pro ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v4-pro,')) + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (${{ matrix.rocm_version }}) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + ENABLE_CACHE_HOST: "1" + + - name: Install dependencies + run: | + # --skip-test-time-deps: GSM8K + bench_one_batch_server don't need lmms-eval / human-eval. + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + + - name: Accuracy + Performance Test MI35x ROCm (8-GPU DeepSeek-V4-Pro FP8 + FP4, unified_kv_triton) + timeout-minutes: 480 + run: | + > github_summary.md # Clear summary file + echo "## SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton" >> github_summary.md + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-pro --nightly --timeout-per-file 14400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} + echo "$(> $GITHUB_STEP_SUMMARY || true + + - name: Accuracy + Performance Test MI35x ROCm (8-GPU DeepSeek-V4-Pro FP8 + FP4, triton) + if: ${{ !cancelled() }} + timeout-minutes: 480 + run: | + > github_summary.md # Clear summary file + echo "## SGLANG_HACK_FLASHMLA_BACKEND=triton" >> github_summary.md + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_HACK_FLASHMLA_BACKEND=triton \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-pro --nightly --timeout-per-file 14400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} + echo "$(> $GITHUB_STEP_SUMMARY || true + + nightly-8-gpu-mi35x-deepseek-v4-pro-mtp: + name: ${{ format('nightly-8-gpu-mi35x-deepseek-v4-pro-mtp ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v4-pro-mtp,')) + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (${{ matrix.rocm_version }}) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + ENABLE_CACHE_HOST: "1" + + - name: Install dependencies + run: | + # --skip-test-time-deps: GSM8K + send_one don't need lmms-eval / human-eval. + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + + - name: Accuracy + MTP Accept Length Test MI35x ROCm (8-GPU DeepSeek-V4-Pro FP8 MTP, unified_kv_triton) + timeout-minutes: 240 + run: | + > github_summary.md # Clear summary file + echo "## SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton" >> github_summary.md + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-pro-mtp --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} + echo "$(> $GITHUB_STEP_SUMMARY || true + + - name: Accuracy + MTP Accept Length Test MI35x ROCm (8-GPU DeepSeek-V4-Pro FP8 MTP, triton) + if: ${{ !cancelled() }} + timeout-minutes: 240 + run: | + > github_summary.md # Clear summary file + echo "## SGLANG_HACK_FLASHMLA_BACKEND=triton" >> github_summary.md + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_HACK_FLASHMLA_BACKEND=triton \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-pro-mtp --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} + echo "$(> $GITHUB_STEP_SUMMARY || true + + nightly-8-gpu-mi35x-deepseek-v4-pro-dspark: + name: ${{ format('nightly-8-gpu-mi35x-deepseek-v4-pro-dspark ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v4-pro-dspark,')) + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (${{ matrix.rocm_version }}) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + ENABLE_CACHE_HOST: "1" + + - name: Install dependencies + run: | + # --skip-test-time-deps: GSM8K doesn't need lmms-eval / human-eval. + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + + - name: Accuracy + DSpark Accept Length Test MI35x ROCm (8-GPU DeepSeek-V4-Pro-DSpark FP4, unified_kv_triton) + timeout-minutes: 300 + run: | + > github_summary.md # Clear summary file + echo "## SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton" >> github_summary.md + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-pro-dspark --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} + echo "$(> $GITHUB_STEP_SUMMARY || true + + # ============================================================================== + # 8-GPU Kimi-K2.6 (MI30x) + # + # Kept on MI30x: Kimi-K3 needs gfx95x for its native MXFP4 weights, so it + # cannot take over here the way it does on MI35x. # ============================================================================== nightly-8-gpu-kimi-k26: - name: nightly-8-gpu-kimi-k26 (rocm700, linux-mi300-8gpu-sglang) + name: ${{ format('nightly-8-gpu-kimi-k26 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-kimi-k26,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -1321,30 +1627,39 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh + run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - name: Accuracy Test (8-GPU Kimi-K2.6) + - name: Accuracy Test ROCm (8-GPU Kimi-K2.6) timeout-minutes: 120 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-kimi-k26 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - nightly-8-gpu-mi35x-kimi-k26: - name: nightly-8-gpu-mi35x-kimi-k26 (rocm700, linux-mi35x-gpu-8) - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-kimi-k26,')) + # ============================================================================== + # 8-GPU Kimi-K3 (MI35x only - native MXFP4 needs gfx95x) + # ============================================================================== + + nightly-8-gpu-mi35x-kimi-k3: + name: ${{ format('nightly-8-gpu-mi35x-kimi-k3 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-kimi-k3,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code @@ -1355,101 +1670,48 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps # Install tabulate for run_suite.py (missing in MI35x container) bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - name: Accuracy Test MI35x (8-GPU Kimi-K2.6) - timeout-minutes: 180 + # The 2.8T MXFP4 checkpoint is ~1.56 TB, so weight download and load + # dominate wall time well before the eval itself starts. + - name: Accuracy Test MI35x ROCm (8-GPU Kimi-K3) + timeout-minutes: 300 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-kimi-k26 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-kimi-k3 --nightly --timeout-per-file 14400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # ============================================================================== - # 8-GPU Qwen3-235B (MI30x + MI35x MXFP4) - # ============================================================================== - - nightly-8-gpu-qwen3-235b: - name: nightly-8-gpu-qwen3-235b (rocm700, linux-mi300-8gpu-sglang) - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-qwen3-235b,')) - runs-on: linux-mi300-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Accuracy Test + Performance Test (8-GPU Qwen3) - timeout-minutes: 120 + # Shares the job with the eval above so the checkpoint is already cached + # and only one 8-GPU MI35x slot is consumed. Step ordering is also the + # accuracy gate: a failed eval fails the job before this runs, so + # throughput is never measured on a build that got the tokens wrong. + # continue-on-error matches every other combined accuracy + perf job here, + # so a perf hiccup cannot turn the accuracy result red. + - name: Performance Test MI35x ROCm (8-GPU Kimi-K3) + timeout-minutes: 300 + continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-8-gpu-qwen3-235b --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - nightly-8-gpu-mi35x-qwen3-235b-mxfp4: - name: nightly-8-gpu-mi35x-qwen3-235b-mxfp4 (rocm700, linux-mi35x-gpu-8) - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen3-235b-mxfp4,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Accuracy Test + Performance Test MI35x (8-GPU Qwen3-235B-MXFP4) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-8-gpu-mi35x-qwen3-235b-mxfp4 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + python3 registered/amd/perf/mi35x/test_kimi_k3_perf_mi35x.py || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} @@ -1458,7 +1720,11 @@ jobs: # ============================================================================== nightly-8-gpu-qwen35: - name: nightly-8-gpu-qwen35 (rocm700, linux-mi300-8gpu-sglang) + name: ${{ format('nightly-8-gpu-qwen35 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-qwen35,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -1470,20 +1736,20 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-aiter-build --skip-test-time-deps bash scripts/ci/amd/amd_ci_exec.sh pip install mistral-common "lm-eval[api]" - - name: Accuracy Test (8-GPU Qwen 3.5) + - name: Accuracy Test ROCm (8-GPU Qwen 3.5) timeout-minutes: 120 run: | > github_summary.md # Clear summary file @@ -1494,12 +1760,13 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - - name: Performance Test (8-GPU Qwen 3.5 FP8) + - name: Performance Test ROCm (8-GPU Qwen 3.5 FP8) timeout-minutes: 120 continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e SGLANG_USE_AITER=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-qwen35-fp8 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? @@ -1507,7 +1774,11 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-qwen35: - name: nightly-8-gpu-mi35x-qwen35 (rocm700, linux-mi35x-gpu-8) + name: ${{ format('nightly-8-gpu-mi35x-qwen35 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen35,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1519,43 +1790,49 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-aiter-build --skip-test-time-deps bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate bash scripts/ci/amd/amd_ci_exec.sh pip install mistral-common "lm-eval[api]" - - name: Accuracy Test MI35x (8-GPU Qwen 3.5) + - name: Accuracy Test MI35x ROCm (8-GPU Qwen 3.5) timeout-minutes: 120 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-qwen35 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - - name: Performance Test MI35x (8-GPU Qwen 3.5 FP8) + - name: Performance Test MI35x ROCm (8-GPU Qwen 3.5 FP8) timeout-minutes: 120 continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_USE_AITER=1 \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-qwen35-fp8 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-qwen35-triton-dcp: - name: nightly-8-gpu-mi35x-qwen35-triton-dcp (rocm700, linux-mi35x-gpu-8) + name: ${{ format('nightly-8-gpu-mi35x-qwen35-triton-dcp ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen35-triton-dcp,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1567,34 +1844,98 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-aiter-build --skip-test-time-deps bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - name: Accuracy Test MI35x (8-GPU Qwen 3.5 Triton DCP) + - name: Accuracy Test MI35x ROCm (8-GPU Qwen 3.5 Triton DCP) timeout-minutes: 120 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-qwen35-triton-dcp --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} + # ============================================================================== + # 8-GPU Qwen3.8-MXFP4 (MI35x only - MXFP4 needs gfx95x) + # ============================================================================== + + nightly-8-gpu-mi35x-qwen38-mxfp4: + name: ${{ format('nightly-8-gpu-mi35x-qwen38-mxfp4 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen38-mxfp4,')) + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (${{ matrix.rocm_version }}) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + ENABLE_CACHE_HOST: "1" + + - name: Install dependencies + run: | + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + # Install tabulate for run_suite.py (missing in MI35x container) + bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + + # ~1.2 TB of weights against a fleet-shared cache that runs at 100% used. + # Run 32196787596 died 40 minutes into the download on ENOSPC with no + # free-space figure anywhere in its log; record one up front instead. + - name: Check model cache space + run: | + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout \ + bash scripts/ci/amd/check_hf_cache_space.sh \ + amd/Qwen3.8-2.4T-A95B-Quark-MXFP4 1250 + + # One step, not the usual accuracy/perf pair: both phases live in the + # same suite so they launch from one server config. The 2.4T MXFP4 + # checkpoint is ~1.2 TB and is loaded once per phase, which dominates + # wall time well before either the eval or the benchmark starts. + - name: Accuracy Test + Performance Test MI35x ROCm (8-GPU Qwen3.8-MXFP4) + timeout-minutes: 420 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-qwen38-mxfp4 --nightly --timeout-per-file 21600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + # ============================================================================== # 8-GPU GLM-5.1 (MI30x + MI35x) # ============================================================================== nightly-8-gpu-glm51: - name: nightly-8-gpu-glm51 (rocm700, linux-mi300-8gpu-sglang) + name: ${{ format('nightly-8-gpu-glm51 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-glm51,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -1606,42 +1947,52 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - name: Accuracy Test (8-GPU GLM-5.1 DSA) + - name: Accuracy Test ROCm (8-GPU GLM-5.1 DSA) timeout-minutes: 120 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-glm51 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - - name: Performance Test (8-GPU GLM-5.1) + - name: Performance Test ROCm (8-GPU GLM-5.1) timeout-minutes: 120 continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e SGLANG_USE_AITER=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-glm51 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - nightly-8-gpu-mi35x-glm51: - name: nightly-8-gpu-mi35x-glm51 (rocm700, linux-mi35x-gpu-8) - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm51,')) + # ============================================================================== + # 8-GPU GLM-5.2-FP8 (MI35x accuracy + performance) + # ============================================================================== + + nightly-8-gpu-mi35x-glm52-fp8: + name: ${{ format('nightly-8-gpu-mi35x-glm52-fp8 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm52-fp8,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code @@ -1652,36 +2003,43 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} + ENABLE_CACHE_HOST: "1" - name: Install dependencies run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - name: Accuracy Test MI35x (8-GPU GLM-5.1 DSA) + # The ~700 GB FP8 checkpoint means weight download and load dominate wall + # time well before the eval itself starts. + - name: Accuracy Test MI35x ROCm (8-GPU GLM-5.2-FP8 DSA) timeout-minutes: 180 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-glm51 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-glm52-fp8 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - - name: Performance Test MI35x (8-GPU GLM-5.1) + # Run perf only after correctness passes. Keeping both steps in this job + # reuses the cached ~700 GB checkpoint and one scarce 8-GPU MI35x slot. + - name: Performance Test MI35x ROCm (8-GPU GLM-5.2-FP8) timeout-minutes: 120 continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-glm51 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-glm52-fp8 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} @@ -1690,7 +2048,11 @@ jobs: # ============================================================================== nightly-8-gpu-mi35x-glm5-mxfp4: - name: nightly-8-gpu-mi35x-glm5-mxfp4 (rocm700, linux-mi35x-gpu-8) + name: ${{ format('nightly-8-gpu-mi35x-glm5-mxfp4 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm5-mxfp4,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1702,36 +2064,38 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - name: Accuracy Test MI35x (8-GPU GLM-5-MXFP4) + - name: Accuracy Test MI35x ROCm (8-GPU GLM-5-MXFP4) timeout-minutes: 180 run: | > github_summary.md bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_USE_AITER=1 \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-glm5-mxfp4 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - - name: Performance Test MI35x (8-GPU GLM-5-MXFP4) + - name: Performance Test MI35x ROCm (8-GPU GLM-5-MXFP4) timeout-minutes: 300 continue-on-error: true run: | > github_summary.md bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_USE_AITER=1 \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 registered/amd/perf/mi35x/test_glm5_mxfp4_perf_mi35x.py || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true @@ -1742,7 +2106,11 @@ jobs: # ============================================================================== nightly-4-gpu-mi35x-minimax-m25: - name: nightly-4-gpu-mi35x-minimax-m25 (rocm700, linux-mi35x-gpu-8) + name: ${{ format('nightly-4-gpu-mi35x-minimax-m25 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu-mi35x-minimax-m25,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1754,10 +2122,10 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.0) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" @@ -1765,7 +2133,7 @@ jobs: - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - name: Accuracy Test MI35x (4-GPU MiniMax-M2.5) + - name: Accuracy Test MI35x ROCm (4-GPU MiniMax-M2.5) timeout-minutes: 120 run: | > github_summary.md @@ -1778,11 +2146,15 @@ jobs: exit ${TEST_EXIT_CODE:-0} # ============================================================================== - # 4-GPU MiniMax-M3 MXFP8 (MI35x) + # 4-GPU MiniMax-M3 MXFP8 (MI35x accuracy + performance) # ============================================================================== nightly-4-gpu-mi35x-minimax-m3: - name: nightly-4-gpu-mi35x-minimax-m3 (rocm700, linux-mi35x-gpu-8) + name: ${{ format('nightly-4-gpu-mi35x-minimax-m3 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu-mi35x-minimax-m3,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1794,10 +2166,10 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.0) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" @@ -1805,23 +2177,43 @@ jobs: - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - name: Accuracy Test MI35x (4-GPU MiniMax-M3 MXFP8) + - name: Accuracy Test MI35x ROCm (4-GPU MiniMax-M3 MXFP8) timeout-minutes: 120 run: | > github_summary.md bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_USE_AITER=1 \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-4-gpu-mi35x-minimax-m3-tp4 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} + # Run perf only after correctness passes. Keeping both steps in this job + # reuses the cached checkpoint and one scarce MI35x slot. + - name: Performance Test MI35x ROCm (4-GPU MiniMax-M3 MXFP8) + timeout-minutes: 120 + continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed + run: | + > github_summary.md + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_USE_AITER=1 \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-4-gpu-mi35x-minimax-m3 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + # ============================================================================== # 8-GPU MiniMax-M2.7 (MI30x only) # ============================================================================== nightly-8-gpu-minimax-m27: - name: nightly-8-gpu-minimax-m27 (rocm700, linux-mi300-8gpu-sglang) + name: ${{ format('nightly-8-gpu-minimax-m27 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-minimax-m27,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -1833,34 +2225,36 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh + run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - name: Accuracy Test (8-GPU MiniMax-M2.7) + - name: Accuracy Test ROCm (8-GPU MiniMax-M2.7) timeout-minutes: 120 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e SGLANG_USE_AITER=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-minimax-m27 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - - name: Performance Test (8-GPU MiniMax-M2.7) + - name: Performance Test ROCm (8-GPU MiniMax-M2.7) timeout-minutes: 120 continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e SGLANG_USE_AITER=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-minimax-m27 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? @@ -1872,7 +2266,11 @@ jobs: # ============================================================================== nightly-1-gpu-zimage-turbo: - name: nightly-1-gpu-zimage-turbo (rocm700, linux-mi300-1gpu-sglang) + name: ${{ format('nightly-1-gpu-zimage-turbo ({0}, linux-mi300-1gpu-sglang)', matrix.rocm_version) }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-1-gpu-zimage-turbo,')) runs-on: linux-mi300-1gpu-sglang steps: @@ -1884,17 +2282,17 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - name: Z-Image-Turbo Diffusion Test (1-GPU) + - name: Z-Image-Turbo Diffusion Test ROCm (1-GPU) timeout-minutes: 45 run: | bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout \ @@ -1908,7 +2306,7 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: zimage-turbo-outputs + name: zimage-turbo-outputs-${{ matrix.rocm_version }} path: diffusion-artifacts/ if-no-files-found: ignore retention-days: 30 @@ -1918,12 +2316,13 @@ jobs: needs: # 1-GPU Unit Tests (MI30x + MI35x) - nightly-test-1-gpu-unit + - nightly-test-1-gpu-kernel - nightly-test-1-gpu-mi35x # 2-GPU and 4-GPU Tests (MI30x + MI35x) - nightly-accuracy-2-gpu - nightly-accuracy-2-gpu-vlm - # - nightly-perf-2-gpu-text # excluded: perf failures don't block CI - # - nightly-perf-2-gpu-vlm # excluded: perf failures don't block CI + - nightly-perf-2-gpu-text + - nightly-perf-2-gpu-vlm - nightly-4-gpu - nightly-2-gpu-mi35x-glm51-mxfp4 - nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2 @@ -1936,40 +2335,44 @@ jobs: # 8-GPU Grok2 (MI30x + MI35x) - nightly-8-gpu-grok2 - nightly-8-gpu-mi35x-grok2 - # 8-GPU DeepSeek-V3.x (MI30x) - - nightly-8-gpu-deepseek-v31 + # 8-GPU DeepSeek-V3.2 (MI30x) - nightly-8-gpu-deepseek-v32 - nightly-8-gpu-deepseek-v32-mtp - - nightly-8-gpu-deepseek-v3-kv-fp8 # 8-GPU DeepSeek-V3.2 (MI35x) - - nightly-accuracy-8-gpu-mi35x-deepseek-v32 - - nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp - # - nightly-perf-8-gpu-mi35x-deepseek-v32-basic # excluded: perf failures don't block CI - # - nightly-perf-8-gpu-mi35x-deepseek-v32-mtp # excluded: perf failures don't block CI + - nightly-8-gpu-mi35x-deepseek-v32 + - nightly-8-gpu-mi35x-deepseek-v32-mtp # 8-GPU DeepSeek-R1 (MI35x only) - nightly-8-gpu-mi35x-deepseek-r1-mxfp4 - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8 - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4 - nightly-8-gpu-mi35x-deepseek-r1-hicache - # 8-GPU Kimi-K2.6 (MI30x + MI35x) + # 8-GPU DeepSeek-V4-Flash (MI30x) + - nightly-8-gpu-deepseek-v4-flash + # 8-GPU DeepSeek-V4 (MI35x) + - nightly-8-gpu-mi35x-deepseek-v4-flash + - nightly-8-gpu-mi35x-deepseek-v4-pro + - nightly-8-gpu-mi35x-deepseek-v4-pro-mtp + - nightly-8-gpu-mi35x-deepseek-v4-pro-dspark + # 8-GPU Kimi-K2.6 (MI30x) - nightly-8-gpu-kimi-k26 - - nightly-8-gpu-mi35x-kimi-k26 - # 8-GPU Qwen3-235B (MI30x + MI35x MXFP4) - - nightly-8-gpu-qwen3-235b - - nightly-8-gpu-mi35x-qwen3-235b-mxfp4 + # 8-GPU Kimi-K3 (MI35x only - native MXFP4 needs gfx95x) + - nightly-8-gpu-mi35x-kimi-k3 + # 8-GPU Qwen3.8-MXFP4 (MI35x only - MXFP4 needs gfx95x) + - nightly-8-gpu-mi35x-qwen38-mxfp4 # 8-GPU Qwen 3.5 (MI30x + MI35x) - nightly-8-gpu-qwen35 - nightly-8-gpu-mi35x-qwen35 - nightly-8-gpu-mi35x-qwen35-triton-dcp - # 8-GPU GLM-5.1 (MI30x + MI35x) + # 8-GPU GLM-5.1 (MI30x) - nightly-8-gpu-glm51 - - nightly-8-gpu-mi35x-glm51 + # 8-GPU GLM-5.2-FP8 (MI35x accuracy + performance) + - nightly-8-gpu-mi35x-glm52-fp8 # 8-GPU GLM-5-MXFP4 (MI35x only) - nightly-8-gpu-mi35x-glm5-mxfp4 # 4-GPU MiniMax-M2.5 (MI35x) - nightly-4-gpu-mi35x-minimax-m25 - # 4-GPU MiniMax-M3 MXFP8 (MI35x) + # 4-GPU MiniMax-M3 MXFP8 (MI35x accuracy + performance) - nightly-4-gpu-mi35x-minimax-m3 # 8-GPU MiniMax-M2.7 (MI30x only) - nightly-8-gpu-minimax-m27 @@ -1980,11 +2383,11 @@ jobs: - name: Check if any job failed run: | if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then - echo "One or more nightly test jobs failed" + echo "One or more ROCm nightly test jobs failed" exit 1 fi if [[ "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then - echo "One or more nightly test jobs were cancelled" + echo "One or more ROCm nightly test jobs were cancelled" exit 1 fi - echo "All nightly test jobs passed" + echo "All ROCm nightly test jobs passed" diff --git a/.github/workflows/pr-states.yml b/.github/workflows/pr-states.yml index 77c66da0e..9a053d000 100644 --- a/.github/workflows/pr-states.yml +++ b/.github/workflows/pr-states.yml @@ -14,7 +14,7 @@ on: pull_request_target: types: [opened, synchronize, reopened, labeled, unlabeled] workflow_run: - workflows: ["PR Test Base", "PR Test Extra", "PR Test ROCm 7.2 (AMD)"] + workflows: ["PR Test Base", "PR Test Extra", "PR Test (AMD)"] types: [requested, completed] workflow_dispatch: inputs: @@ -163,7 +163,7 @@ jobs: 'pr-test-extra.yml', (hasCI && hasExtra) ? labelOnOpts : labelOffOpts, ), - findRun('pr-test-amd-rocm720.yml', labelOffOpts), + findRun('pr-test-amd.yml', labelOffOpts), ]); // Skipped run = "no real run" -- happens when a label is added @@ -214,8 +214,8 @@ jobs: const ptEnd = ''; const peStart = ''; const peEnd = ''; - const amdStart = ''; - const amdEnd = ''; + const amdStart = ''; + const amdEnd = ''; const newBlock = [ outerStart, diff --git a/.github/workflows/pr-test-amd-extra.yml b/.github/workflows/pr-test-amd-extra.yml index 335c64254..f06236601 100644 --- a/.github/workflows/pr-test-amd-extra.yml +++ b/.github/workflows/pr-test-amd-extra.yml @@ -45,7 +45,6 @@ on: - rocm10 - rocm724 - rocm720 - - rocm700 aiter_ref: description: 'Override AITER commit (optional, leave empty to use Dockerfile default)' required: false diff --git a/.github/workflows/pr-test-amd-rocm720.yml b/.github/workflows/pr-test-amd-rocm720.yml deleted file mode 100644 index 0a0f158ff..000000000 --- a/.github/workflows/pr-test-amd-rocm720.yml +++ /dev/null @@ -1,1506 +0,0 @@ -name: PR Test ROCm 7.2 (AMD) -# Dynamic run-name for manual stage dispatches to enable URL lookup -# Format: "[stage-name] sha" for fork PRs, "[stage-name]" for non-fork, default for normal runs -run-name: ${{ (inputs.target_stage || inputs.target_stage_select) && (inputs.pr_head_sha && format('[{0}] {1}', inputs.target_stage || inputs.target_stage_select, inputs.pr_head_sha) || format('[{0}]', inputs.target_stage || inputs.target_stage_select)) || '' }} - -on: - schedule: - - cron: '0 */12 * * *' # rocm10: twice daily (UTC) - - cron: '30 17 * * *' # rocm724/rocm720: once daily, aligned with ROCm 7.0 - pull_request: - paths: - - "python/**" - - "scripts/ci/**" - - "test/**" - - "python/sglang/kernels/aot/**" - - ".github/workflows/pr-test-amd-rocm720.yml" - - "docker/rocm.Dockerfile" - workflow_dispatch: - inputs: - rocm_version: - description: 'ROCm image version' - required: false - type: choice - default: rocm10 - options: - - rocm10 - - rocm724 - - rocm720 - target_stage_select: - description: "Select a stage to run from dropdown (leave empty for auto-detect)" - required: false - type: choice - default: '' - options: - - '' - - sgl-kernel-unit-test-amd-rocm720 - - sgl-kernel-unit-test-2-gpu-amd-rocm720 - - stage-a-test-1-gpu-small-amd-rocm720 - - jit-kernel-unit-test-amd-rocm720 - - jit-kernel-benchmark-test-amd-rocm720 - - stage-b-test-1-gpu-small-amd-rocm720 - - stage-b-test-1-gpu-small-amd-nondeterministic-rocm720 - - stage-b-test-1-gpu-small-amd-mi35x-rocm720 - - stage-b-test-1-gpu-large-amd-rocm720 - - stage-b-test-2-gpu-large-amd-rocm720 - - multimodal-gen-test-1-gpu-amd-rocm720 - - multimodal-gen-test-2-gpu-amd-rocm720 - - stage-c-test-4-gpu-amd-rocm720 - - stage-c-dsv4-flash-fp4-fp8-amd-mi35x-rocm720 - - stage-c-dsv4-pro-fp4-amd-mi35x-rocm720 - - stage-c-test-large-8-gpu-amd-rocm720 - - stage-c-test-large-8-gpu-amd-mi35x-rocm720 - - stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720 - target_stage: - description: "Or type comma-separated stage names (overrides dropdown if non-empty)" - required: false - type: string - default: "" - pr_head_sha: - description: "PR head SHA to checkout (for stage dispatches on fork PRs)" - required: false - type: string - default: "" - aiter_ref: - description: 'Override AITER commit (optional, leave empty to use Dockerfile default)' - required: false - type: string - default: '' - amd_ci_image: - description: 'Override AMD CI Docker image (optional, e.g. rocm/sgl-dev:tag)' - required: false - type: string - default: '' - continue_on_error: - description: 'Continue on error (do not fail the workflow on test failures)' - required: false - type: boolean - default: false - runner_arch: - description: 'AMD runner pool to dispatch GPU jobs to' - required: false - type: choice - default: mi300 - options: - - mi300 - - mi325 - run_all_tests: - description: 'Run all tests (skip change detection). Ignored when target_stage / target_stage_select is set.' - required: false - type: boolean - default: false - workflow_call: - inputs: - rocm_version: - description: 'ROCm image version' - required: false - type: string - default: rocm10 - ref: - description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.' - required: false - type: string - default: '' - runner_arch: - description: 'AMD runner pool to dispatch GPU jobs to' - required: false - type: string - default: mi300 - run_all_tests: - description: "Run all tests (for releasing or testing purpose)" - required: false - type: boolean - default: false - aiter_ref: - description: 'Override AITER commit (optional, leave empty to use Dockerfile default)' - required: false - type: string - default: '' - amd_ci_image: - description: 'Override AMD CI Docker image (optional, e.g. rocm/sgl-dev:tag)' - required: false - type: string - default: '' - continue_on_error: - description: 'Continue on error (do not fail the workflow on test failures)' - required: false - type: boolean - default: false - -# Mirror pr-test.yml: the chained extra suite (call-pr-test-amd-extra-rocm720 -> -# pr-test-amd-extra.yml) declares actions: write / issues: read / -# pull-requests: read. A called reusable workflow can only use scopes the -# caller already holds, so without this block the call fails workflow -# validation ("requesting actions: write... but only allowed ...none"). -permissions: - actions: write - contents: read - issues: read - pull-requests: read - -env: - AITER_COMMIT_OVERRIDE: ${{ inputs.aiter_ref }} - AMD_CI_IMAGE: ${{ inputs.amd_ci_image }} - DOCKERHUB_AMD_USERNAME: ${{ secrets.DOCKERHUB_AMD_USERNAME }} - DOCKERHUB_AMD_TOKEN: ${{ secrets.DOCKERHUB_AMD_TOKEN }} - -concurrency: - # Scheduled, run_all_tests, and manual dispatch runs get unique groups (never cancel each other). - # PR runs share a group per branch so new pushes cancel stale runs. - group: pr-test-amd-rocm720-${{ (inputs.run_all_tests || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && format('full-{0}', github.run_id) || inputs.pr_head_sha || inputs.ref || github.ref }} - cancel-in-progress: ${{ !inputs.run_all_tests && github.event_name != 'workflow_call' && github.event_name != 'schedule' && github.event_name != 'workflow_dispatch' }} - -jobs: - call-gate: - if: github.event_name != 'schedule' - uses: ./.github/workflows/pr-gate.yml - secrets: inherit - check-changes: - needs: [call-gate] - if: always() - runs-on: ubuntu-latest - outputs: - main_package: ${{ steps.filter.outputs.main_package || steps.run-mode.outputs.run_all_tests }} - sgl_kernel: ${{ steps.filter.outputs.sgl_kernel || steps.run-mode.outputs.run_all_tests }} - jit_kernel: ${{ steps.filter.outputs.jit_kernel || steps.run-mode.outputs.run_all_tests }} - multimodal_gen: ${{ steps.filter.outputs.multimodal_gen || steps.run-mode.outputs.run_all_tests }} - continue_on_error: ${{ steps.set-continue-on-error.outputs.continue_on_error }} - # Keep rocm10 on the twice-daily cadence. The 17:30 UTC schedule runs - # rocm724 and rocm720 once daily alongside the ROCm 7.0 - # shadow workflow. Dispatch/call inputs select one version, while a - # pull_request falls through to the single rocm10 default. - rocm_versions: ${{ inputs.rocm_version && format('["{0}"]', inputs.rocm_version) || github.event.schedule == '30 17 * * *' && '["rocm724","rocm720"]' || '["rocm10"]' }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Determine run mode - id: run-mode - run: | - if [[ "${{ inputs.run_all_tests }}" == "true" || "${{ github.event_name }}" == "schedule" ]]; then - echo "run_all_tests=true" >> $GITHUB_OUTPUT - echo "Run mode: ALL TESTS (run_all_tests=${{ inputs.run_all_tests }}, event=${{ github.event_name }})" - else - echo "run_all_tests=false" >> $GITHUB_OUTPUT - echo "Run mode: FILTERED (triggered by ${{ github.event_name }})" - fi - - - name: Set continue-on-error for schedule/full runs - id: set-continue-on-error - env: - # `bypass-fastfail` PR label: also disable within-suite fast-fail - # here. The shared actions/wait-for-jobs already honors the same - # label to skip cross-stage waits. - BYPASS_FASTFAIL_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'bypass-fastfail') }} - run: | - if [[ "${{ steps.run-mode.outputs.run_all_tests }}" == "true" \ - || "${{ inputs.continue_on_error }}" == "true" \ - || "$BYPASS_FASTFAIL_LABEL" == "true" ]]; then - echo "continue_on_error=true" >> $GITHUB_OUTPUT - echo "Continue-on-error: ENABLED (run_all_tests=${{ steps.run-mode.outputs.run_all_tests }}, input=${{ inputs.continue_on_error }}, bypass-fastfail=$BYPASS_FASTFAIL_LABEL)" - else - echo "continue_on_error=false" >> $GITHUB_OUTPUT - echo "Continue-on-error: DISABLED" - fi - - - name: Detect file changes - id: filter - uses: dorny/paths-filter@v3 - if: steps.run-mode.outputs.run_all_tests != 'true' - with: - filters: | - main_package: - - "python/sglang/!(multimodal_gen|kernels)/**/!(*.md)" - - "python/sglang/kernels/!(*.md)" - - "python/sglang/kernels/!(aot)/**/!(*.md)" - - "python/pyproject_rocm.toml" - - "python/pyproject_other.toml" - - "scripts/ci/amd/*" - - "scripts/ci/utils/*" - - "test/**/!(*.md)" - - ".github/workflows/pr-test-amd-rocm720.yml" - sgl_kernel: - - "python/sglang/kernels/aot/**/!(*.md|THIRDPARTYNOTICES.txt|LICENSE)" - - ".github/workflows/pr-test-amd-rocm720.yml" - jit_kernel: - - "python/sglang/kernels/!(*.md)" - - "python/sglang/kernels/!(aot)/**" - - "test/registered/kernels/**" - - ".github/workflows/pr-test-amd-rocm720.yml" - multimodal_gen: - - "python/sglang/multimodal_gen/**/!(*.md|*.ipynb)" - - "python/sglang/cli/**" - - "python/sglang/srt/observability/**" - - "python/sglang/kernels/ops/diffusion/**" - - "test/registered/kernels/ops/diffusion/**" - - "test/registered/kernels/benchmark/diffusion/**" - - "python/pyproject_rocm.toml" - - "python/pyproject_other.toml" - - # =============================================== extra (scheduled) ==================================================== - # Chain the label-gated AMD extra tier into the scheduled run, mirroring - # pr-test.yml's `call-pr-test-extra`. On `schedule` (and run_all_tests - # dispatch) the extra suite runs on `main` without needing the - # `run-ci-extra` label (pr-gate.yml only enforces labels on pull_request - # events). Targeted stage dispatches (target_stage set) are excluded. - # Not added to `pr-test-amd-rocm720-finish` so the base AMD gate never depends on - # the opt-in extra suite. - call-pr-test-amd-extra-rocm720: - name: call-pr-test-amd-extra - needs: [check-changes] - if: | - (github.event_name == 'schedule' || inputs.run_all_tests == true) && - !(inputs.target_stage || inputs.target_stage_select) - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} - uses: ./.github/workflows/pr-test-amd-extra.yml - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || '' }} - runner_arch: ${{ inputs.runner_arch || 'mi300' }} - rocm_version: ${{ matrix.rocm_version }} - aiter_ref: ${{ inputs.aiter_ref }} - amd_ci_image: ${{ inputs.amd_ci_image }} - continue_on_error: true - secrets: inherit - - # =============================================== sgl-kernel ==================================================== - sgl-kernel-unit-test-amd-rocm720: - name: ${{ format('sgl-kernel-unit-test-amd ({0}, linux-{1}-1gpu-sglang)', matrix.rocm_version, inputs.runner_arch || 'mi300') }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} - needs: [check-changes, call-gate] - if: | - always() && !cancelled() && - ( - (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',sgl-kernel-unit-test-amd-rocm720,')) || - ( - !(inputs.target_stage || inputs.target_stage_select) && - (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') && - needs.check-changes.outputs.sgl_kernel == 'true' - ) - ) - runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Run test - timeout-minutes: 30 - env: - CONTINUE_ON_ERROR: ${{ needs.check-changes.outputs.continue_on_error }} - run: | - # In continue-on-error mode (schedule/full runs), keep running all pytest - # files and aggregate the exit code. In PR mode, preserve fail-fast. - failures=0 - run_pytest() { - if [[ "$CONTINUE_ON_ERROR" == "true" ]]; then - "$@" || failures=$((failures + 1)) - else - "$@" - fi - } - run_pytest docker exec -w /sglang-checkout/python/sglang/kernels/aot/tests ci_sglang python3 -m pytest test_moe_align.py - run_pytest docker exec -w /sglang-checkout/python/sglang/kernels/aot/tests ci_sglang python3 -m pytest test_moe_topk_softmax.py - run_pytest docker exec -w /sglang-checkout/python/sglang/kernels/aot/tests/speculative ci_sglang python3 -m pytest test_eagle_utils.py - run_pytest docker exec -w /sglang-checkout/python/sglang/kernels/aot/tests ci_sglang python3 -m pytest test_apply_token_bitmask_inplace.py - run_pytest docker exec -w /sglang-checkout/python/sglang/kernels/aot/tests ci_sglang python3 -m pytest test_activation.py - run_pytest docker exec -w /sglang-checkout/python/sglang/kernels/aot/tests ci_sglang python3 -m pytest test_topk.py - run_pytest docker exec -w /sglang-checkout/python/sglang/kernels/aot/tests ci_sglang python3 -m pytest test_kvcacheio.py - run_pytest docker exec -w /sglang-checkout/python/sglang/kernels/aot/tests ci_sglang python3 -m pytest test_moe_topk_sigmoid.py - run_pytest docker exec -w /sglang-checkout/python/sglang/kernels/aot/tests ci_sglang python3 -m pytest test_torch_defaults_reset.py - exit $failures - - sgl-kernel-unit-test-2-gpu-amd-rocm720: - name: ${{ format('sgl-kernel-unit-test-2-gpu-amd ({0}, linux-{1}-2gpu-sglang)', matrix.rocm_version, inputs.runner_arch || 'mi300') }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} - needs: [check-changes, call-gate] - if: | - always() && !cancelled() && - ( - (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',sgl-kernel-unit-test-2-gpu-amd-rocm720,')) || - ( - !(inputs.target_stage || inputs.target_stage_select) && - (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') && - needs.check-changes.outputs.sgl_kernel == 'true' - ) - ) - runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Run test - timeout-minutes: 30 - env: - CONTINUE_ON_ERROR: ${{ needs.check-changes.outputs.continue_on_error }} - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite sgl-kernel-unit-test-2-gpu-amd ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - - # =============================================== primary ==================================================== - - stage-a-test-1-gpu-small-amd-rocm720: - name: ${{ format('stage-a-test-1-gpu-small-amd ({0}, linux-{1}-1gpu-sglang)', matrix.rocm_version, inputs.runner_arch || 'mi300') }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} - needs: [check-changes, call-gate] - if: | - always() && !cancelled() && - ( - (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-a-test-1-gpu-small-amd-rocm720,')) || - ( - !(inputs.target_stage || inputs.target_stage_select) && - (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Run test - timeout-minutes: 30 - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-a-test-1-gpu-small-amd ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - - jit-kernel-unit-test-amd-rocm720: - name: ${{ format('jit-kernel-unit-test-amd ({0}, linux-{1}-1gpu-sglang)', matrix.rocm_version, inputs.runner_arch || 'mi300') }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} - needs: [check-changes, call-gate] - if: | - always() && !cancelled() && - ( - (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',jit-kernel-unit-test-amd-rocm720,')) || - ( - !(inputs.target_stage || inputs.target_stage_select) && - (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') && - needs.check-changes.outputs.jit_kernel == 'true' - ) - ) - runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Run JIT kernel unit tests - timeout-minutes: 30 - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite jit-kernel-unit-test-amd ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - - jit-kernel-benchmark-test-amd-rocm720: - name: ${{ format('jit-kernel-benchmark-test-amd ({0}, linux-{1}-1gpu-sglang)', matrix.rocm_version, inputs.runner_arch || 'mi300') }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} - needs: [check-changes, call-gate] - if: | - always() && !cancelled() && - ( - (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',jit-kernel-benchmark-test-amd-rocm720,')) || - ( - !(inputs.target_stage || inputs.target_stage_select) && - (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') && - needs.check-changes.outputs.jit_kernel == 'true' - ) - ) - runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Run JIT kernel benchmarks - timeout-minutes: 30 - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite jit-kernel-benchmark-test-amd ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - - # =============================================== Wait Jobs for Sequential PR Execution ==================================================== - # These jobs poll GitHub API to wait for previous stages to complete. - # For PR runs: wait jobs run and enforce sequential execution via polling. - # For scheduled runs: wait jobs are skipped, enabling parallel execution of all stages. - - wait-for-stage-a-amd-rocm720: - name: wait-for-stage-a-amd - needs: [check-changes, call-gate] - if: | - always() && - !cancelled() && - github.event_name == 'pull_request' && - !(inputs.target_stage || inputs.target_stage_select) && - (needs.check-changes.outputs.main_package == 'true' || needs.check-changes.outputs.sgl_kernel == 'true') && - (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') - runs-on: ubuntu-latest - outputs: - stage_a_result: ${{ steps.wait.outputs.result }} - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/wait-for-jobs - id: wait - with: - stage-name: stage-a-amd - jobs: '[{"prefix": "stage-a-test-1-gpu-small-amd", "expected_count": 1}]' - max-wait-minutes: '240' - - stage-b-test-1-gpu-small-amd-rocm720: - name: ${{ format('stage-b-test-1-gpu-small-amd ({0}, linux-{1}-1gpu-sglang, {2})', matrix.rocm_version, inputs.runner_arch || 'mi300', matrix.part) }} - needs: [check-changes, wait-for-stage-a-amd-rocm720] - if: | - always() && - ( - (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-b-test-1-gpu-small-amd-rocm720,')) || - ( - !(inputs.target_stage || inputs.target_stage_select) && - ((github.event_name == 'schedule') || (!failure() && !cancelled())) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - strategy: - fail-fast: false - max-parallel: 4 - matrix: - rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} - part: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13] - runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Run test - timeout-minutes: ${{ (inputs.runner_arch || 'mi300') == 'mi300' && 75 || 60 }} - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 14 --timeout-per-file 2400 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - - stage-b-test-1-gpu-small-amd-nondeterministic-rocm720: - name: ${{ format('stage-b-test-1-gpu-small-amd-nondeterministic ({0}, linux-{1}-1gpu-sglang)', matrix.rocm_version, inputs.runner_arch || 'mi300') }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} - needs: [check-changes, wait-for-stage-a-amd-rocm720] - if: | - always() && - ( - (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-b-test-1-gpu-small-amd-nondeterministic-rocm720,')) || - ( - !(inputs.target_stage || inputs.target_stage_select) && - ((github.event_name == 'schedule') || (!failure() && !cancelled())) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Run test - timeout-minutes: 75 - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-nondeterministic --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - - stage-b-test-1-gpu-small-amd-mi35x-rocm720: - name: ${{ format('stage-b-test-1-gpu-small-amd-mi35x ({0}, {1})', matrix.rocm_version, matrix.runner) }} - needs: [check-changes, wait-for-stage-a-amd-rocm720] - if: | - always() && - ( - (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-b-test-1-gpu-small-amd-mi35x-rocm720,')) || - ( - !(inputs.target_stage || inputs.target_stage_select) && - ((github.event_name == 'schedule') || (!failure() && !cancelled())) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} - runner: [linux-mi35x-gpu-1] - runs-on: ${{matrix.runner}} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Run test - timeout-minutes: 30 - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-mi35x ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - - stage-b-test-1-gpu-large-amd-rocm720: - name: ${{ format('stage-b-test-1-gpu-large-amd ({0}, linux-{1}-1gpu-sglang, {2})', matrix.rocm_version, inputs.runner_arch || 'mi300', matrix.part) }} - needs: [check-changes, wait-for-stage-a-amd-rocm720] - if: | - always() && - ( - (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-b-test-1-gpu-large-amd-rocm720,')) || - ( - !(inputs.target_stage || inputs.target_stage_select) && - ((github.event_name == 'schedule') || (!failure() && !cancelled())) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - strategy: - fail-fast: false - max-parallel: 1 - matrix: - rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} - part: [0, 1, 2] - runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Run test - timeout-minutes: ${{ (inputs.runner_arch || 'mi300') == 'mi300' && 60 || 45 }} - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 2700 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - - stage-b-test-2-gpu-large-amd-rocm720: - name: ${{ format('stage-b-test-2-gpu-large-amd ({0}, linux-{1}-2gpu-sglang, {2})', matrix.rocm_version, inputs.runner_arch || 'mi300', matrix.part) }} - needs: [check-changes, wait-for-stage-a-amd-rocm720] - if: | - always() && - ( - (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-b-test-2-gpu-large-amd-rocm720,')) || - ( - !(inputs.target_stage || inputs.target_stage_select) && - ((github.event_name == 'schedule') || (!failure() && !cancelled())) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - strategy: - fail-fast: false - max-parallel: 1 - matrix: - rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} - part: [0, 1] - runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Run test - timeout-minutes: ${{ (inputs.runner_arch || 'mi300') == 'mi300' && 120 || 45 }} - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-2-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file ${{ (inputs.runner_arch || 'mi300') == 'mi300' && 5400 || 2700 }} ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - - multimodal-gen-test-1-gpu-amd-rocm720: - name: ${{ format('multimodal-gen-test-1-gpu-amd ({0}, linux-{1}-1gpu-sglang, {2})', matrix.rocm_version, inputs.runner_arch || 'mi300', matrix.part) }} - needs: [check-changes, call-gate] - if: | - always() && !cancelled() && - ( - (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',multimodal-gen-test-1-gpu-amd-rocm720,')) || - ( - !(inputs.target_stage || inputs.target_stage_select) && - (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') && - needs.check-changes.outputs.multimodal_gen == 'true' - ) - ) - strategy: - fail-fast: false - max-parallel: 1 - matrix: - rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} - part: [0, 1, 2, 3] - runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Download artifacts - if: needs.check-changes.outputs.sgl_kernel == 'true' - uses: actions/download-artifact@v4 - with: - path: python/sglang/kernels/aot/dist/ - merge-multiple: true - pattern: wheel-python3.10-cuda12.9 - - - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh diffusion - - - name: Setup kernel caches - run: | - docker exec ci_sglang mkdir -p /sgl-data/miopen-cache /sgl-data/hf-cache/hub - - # Create persistent cache marker if /sgl-data is a real mount (not ephemeral) - # This tells the test cleanup code to NOT delete downloaded models - if docker exec ci_sglang test -d /sgl-data && docker exec ci_sglang mountpoint -q /sgl-data 2>/dev/null; then - docker exec ci_sglang touch /sgl-data/hf-cache/.persistent_cache - echo "Created .persistent_cache marker - HF cache will persist" - else - echo "WARNING: /sgl-data is not a mount point - models will be cleaned up after each test" - fi - - # Check MIOpen cache (VAE convolution kernels) - miopen_files=$(docker exec ci_sglang find /sgl-data/miopen-cache -name "*.udb" 2>/dev/null | wc -l || echo "0") - echo "Found ${miopen_files} MIOpen cache files" - - - name: Diagnose HF cache and system resources - run: | - echo "=== System Memory Status ===" - free -h - echo "" - echo "=== Disk Space ===" - df -h /home/runner/sglang-data 2>/dev/null || df -h - echo "" - echo "=== HF Cache Directory Structure ===" - docker exec ci_sglang ls -la /sgl-data/hf-cache/ 2>/dev/null || echo "HF cache dir not found" - docker exec ci_sglang ls -la /sgl-data/hf-cache/hub/ 2>/dev/null || echo "HF hub cache not found" - echo "" - echo "=== Checking for cached diffusion models (1-GPU tests) ===" - # Models used in 1-GPU tests: Wan2.1-T2V-1.3B, HunyuanVideo, Qwen-Image, FLUX.1, FLUX.2 - for model in "Wan-AI--Wan2.1-T2V-1.3B-Diffusers" "tencent--HunyuanVideo" "Qwen--Qwen-Image" "black-forest-labs--FLUX.1-dev" "black-forest-labs--FLUX.2-dev"; do - cache_path="/sgl-data/hf-cache/hub/models--${model}" - if docker exec ci_sglang test -d "$cache_path"; then - size=$(docker exec ci_sglang du -sh "$cache_path" 2>/dev/null | cut -f1) - echo "✓ CACHED: $model ($size)" - else - echo "✗ NOT CACHED: $model" - fi - done - echo "" - echo "=== GPU Memory Status ===" - docker exec ci_sglang rocm-smi --showmeminfo vram 2>/dev/null || echo "rocm-smi not available" - - - name: Run diffusion server tests (1-GPU) - timeout-minutes: 90 - run: | - # AMD CI: All 1-GPU tests except FLUX.2 (FLUX.1 covers same code path) - # Tests: T2V, T2I, I2V, LoRA - # - # HF download env vars: - # - HF_HUB_ENABLE_HF_TRANSFER=1: Use faster hf_transfer for downloads (if available) - # - HF_HUB_DISABLE_SYMLINKS_WARNING=1: Suppress symlink warnings - docker exec \ - -e SGLANG_E2E_TOLERANCE=0.3 \ - -e SGLANG_STAGE_TIME_TOLERANCE=0.2 \ - -e SGLANG_NON_DENOISE_STAGE_TIME_TOLERANCE=0.6 \ - -e SGLANG_DENOISE_STEP_TOLERANCE=0.6 \ - -e SGLANG_DENOISE_AGG_TOLERANCE=0.3 \ - -e SGLANG_SKIP_CONSISTENCY=1 \ - -e SGLANG_TEST_NUM_INFERENCE_STEPS=5 \ - -e SGLANG_DIFFUSION_ARTIFACT_DIR=/sglang-checkout/diffusion-failures \ - -e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \ - -e HF_HUB_ENABLE_HF_TRANSFER=1 \ - -e HF_HUB_DISABLE_SYMLINKS_WARNING=1 \ - -w /sglang-checkout/python \ - ci_sglang python3 sglang/multimodal_gen/test/run_suite.py \ - --suite 1-gpu \ - --partition-id ${{ matrix.part }} \ - --total-partitions 4 \ - -k "not flux_2" \ - ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - - # Post-test diagnostics - echo "=== Post-test System Memory Status ===" - free -h - - # The mm_gen `unit` suite is portable CPU-style tests that finish in a - # few minutes, which does not justify a job of its own: on mi300 the - # container pull plus dependency install costs ~2h before any test runs. - # Ride along on the shard that is already warm. Runs after the diffusion - # tests, and on failure too, so a diffusion regression cannot hide it. - # Skips the CUDA-only ltx2_vae_channels_last memory-format assertion. - - name: Run diffusion unit tests - if: matrix.part == 0 && (success() || failure()) - timeout-minutes: 30 - run: | - docker exec \ - -e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \ - -w /sglang-checkout/python \ - ci_sglang python3 sglang/multimodal_gen/test/run_suite.py \ - --suite unit \ - -k "not ltx2_vae_channels_last" \ - ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - - - name: Upload diffusion failure artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: diffusion-failures-amd-1gpu-${{ matrix.rocm_version }}-${{ matrix.part }}-${{ github.run_attempt }} - path: diffusion-failures/ - if-no-files-found: ignore - retention-days: 7 - - multimodal-gen-test-2-gpu-amd-rocm720: - name: ${{ format('multimodal-gen-test-2-gpu-amd ({0}, linux-{1}-2gpu-sglang, {2})', matrix.rocm_version, inputs.runner_arch || 'mi300', matrix.part) }} - needs: [check-changes, call-gate] - if: | - always() && !cancelled() && - ( - (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',multimodal-gen-test-2-gpu-amd-rocm720,')) || - ( - !(inputs.target_stage || inputs.target_stage_select) && - (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') && - needs.check-changes.outputs.multimodal_gen == 'true' - ) - ) - strategy: - fail-fast: false - max-parallel: 1 - matrix: - rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} - # run_suite.py load-balances the suite's parametrized cases and - # standalone files over these partitions, so the count is free to - # choose. Two shards measured 45-65 min of case work each, inside the - # 180 min step timeout, and halve the ~2h per-job container pull plus - # dependency install. - part: [0, 1] - runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Download artifacts - if: needs.check-changes.outputs.sgl_kernel == 'true' - uses: actions/download-artifact@v4 - with: - path: python/sglang/kernels/aot/dist/ - merge-multiple: true - pattern: wheel-python3.10-cuda12.9 - - - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh diffusion - - - name: Setup kernel caches - run: | - docker exec ci_sglang mkdir -p /sgl-data/miopen-cache /sgl-data/hf-cache/hub - - # Create persistent cache marker if /sgl-data is a real mount (not ephemeral) - # This tells the test cleanup code to NOT delete downloaded models - if docker exec ci_sglang test -d /sgl-data && docker exec ci_sglang mountpoint -q /sgl-data 2>/dev/null; then - docker exec ci_sglang touch /sgl-data/hf-cache/.persistent_cache - echo "Created .persistent_cache marker - HF cache will persist" - else - echo "WARNING: /sgl-data is not a mount point - models will be cleaned up after each test" - fi - - # Check MIOpen cache (VAE convolution kernels) - miopen_files=$(docker exec ci_sglang find /sgl-data/miopen-cache -name "*.udb" 2>/dev/null | wc -l || echo "0") - echo "Found ${miopen_files} MIOpen cache files" - - - name: Diagnose HF cache and system resources - run: | - echo "=== System Memory Status ===" - free -h - echo "" - echo "=== Disk Space ===" - df -h /home/runner/sglang-data 2>/dev/null || df -h - echo "" - echo "=== HF Cache Directory Structure ===" - docker exec ci_sglang ls -la /sgl-data/hf-cache/ 2>/dev/null || echo "HF cache dir not found" - docker exec ci_sglang ls -la /sgl-data/hf-cache/hub/ 2>/dev/null || echo "HF hub cache not found" - echo "" - echo "=== Checking for cached diffusion models (2-GPU tests) ===" - # Models used in 2-GPU tests: Wan2.2-T2V-A14B, Wan2.1-T2V-14B, Qwen-Image, FLUX.1 - for model in "Wan-AI--Wan2.2-T2V-A14B-Diffusers" "Wan-AI--Wan2.1-T2V-14B-Diffusers" "Qwen--Qwen-Image" "black-forest-labs--FLUX.1-dev"; do - cache_path="/sgl-data/hf-cache/hub/models--${model}" - if docker exec ci_sglang test -d "$cache_path"; then - size=$(docker exec ci_sglang du -sh "$cache_path" 2>/dev/null | cut -f1) - echo "✓ CACHED: $model ($size)" - else - echo "✗ NOT CACHED: $model" - fi - done - echo "" - echo "=== GPU Memory Status ===" - docker exec ci_sglang rocm-smi --showmeminfo vram 2>/dev/null || echo "rocm-smi not available" - - - name: Run diffusion server tests (2-GPU) - timeout-minutes: ${{ (inputs.runner_arch || 'mi300') == 'mi300' && 180 || 150 }} - run: | - # AMD CI: All 2-GPU tests including LoRA - # Tests: T2V, T2I, I2V, LoRA - # - # HF download env vars: - # - HF_HUB_ENABLE_HF_TRANSFER=1: Use faster hf_transfer for downloads (if available) - # - HF_HUB_DISABLE_SYMLINKS_WARNING=1: Suppress symlink warnings - docker exec \ - -e SGLANG_E2E_TOLERANCE=0.3 \ - -e SGLANG_STAGE_TIME_TOLERANCE=0.2 \ - -e SGLANG_NON_DENOISE_STAGE_TIME_TOLERANCE=0.6 \ - -e SGLANG_DENOISE_STEP_TOLERANCE=0.6 \ - -e SGLANG_DENOISE_AGG_TOLERANCE=0.3 \ - -e SGLANG_SKIP_CONSISTENCY=1 \ - -e SGLANG_TEST_NUM_INFERENCE_STEPS=5 \ - -e SGLANG_DIFFUSION_ARTIFACT_DIR=/sglang-checkout/diffusion-failures \ - -e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \ - -e HF_HUB_ENABLE_HF_TRANSFER=1 \ - -e HF_HUB_DISABLE_SYMLINKS_WARNING=1 \ - -w /sglang-checkout/python \ - ci_sglang python3 sglang/multimodal_gen/test/run_suite.py \ - --suite 2-gpu \ - --partition-id ${{ matrix.part }} \ - --total-partitions 2 \ - ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - - # Post-test diagnostics - echo "=== Post-test System Memory Status ===" - free -h - - - name: Upload diffusion failure artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: diffusion-failures-amd-2gpu-${{ matrix.rocm_version }}-${{ matrix.part }}-${{ github.run_attempt }} - path: diffusion-failures/ - if-no-files-found: ignore - retention-days: 7 - - wait-for-stage-b-amd-rocm720: - name: wait-for-stage-b-amd - needs: [check-changes, call-gate, wait-for-stage-a-amd-rocm720] - if: | - always() && - !cancelled() && - github.event_name == 'pull_request' && - !(inputs.target_stage || inputs.target_stage_select) && - (needs.check-changes.outputs.main_package == 'true' || needs.check-changes.outputs.sgl_kernel == 'true') && - (needs.wait-for-stage-a-amd-rocm720.result == 'success' || needs.wait-for-stage-a-amd-rocm720.result == 'skipped') && - (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') - runs-on: ubuntu-latest - outputs: - stage_b_result: ${{ steps.wait.outputs.result }} - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/wait-for-jobs - id: wait - with: - stage-name: stage-b-amd - jobs: | - [ - {"prefix": "stage-b-test-1-gpu-small-amd", "expected_count": 14}, - {"prefix": "stage-b-test-1-gpu-large-amd", "expected_count": 3}, - {"prefix": "stage-b-test-2-gpu-large-amd", "expected_count": 2} - ] - max-wait-minutes: '480' - - stage-c-test-4-gpu-amd-rocm720: - name: ${{ format('stage-c-test-4-gpu-amd ({0}, linux-{1}-4gpu-sglang, {2})', matrix.rocm_version, inputs.runner_arch || 'mi300', matrix.part) }} - needs: [check-changes, call-gate, wait-for-stage-b-amd-rocm720] - if: | - always() && - ( - (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-c-test-4-gpu-amd-rocm720,')) || - ( - !(inputs.target_stage || inputs.target_stage_select) && - ((github.event_name == 'schedule') || (!failure() && !cancelled())) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} - part: [0] - runs-on: ${{ format('linux-{0}-4gpu-sglang', inputs.runner_arch || 'mi300') }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Run test - timeout-minutes: 90 - run: | - bash scripts/ci/amd/amd_ci_exec.sh \ - -e NCCL_CUMEM_ENABLE=0 \ - -e NCCL_NVLS_ENABLE=0 \ - -e RCCL_MSCCL_ENABLE=0 \ - -e SGLANG_USE_ROCM700A=1 \ - -w "/sglang-checkout/test" \ - python3 run_suite.py \ - --hw amd \ - --suite stage-c-test-4-gpu-amd \ - --auto-partition-id ${{ matrix.part }} \ - --auto-partition-size 1 \ - --timeout-per-file 5400 \ - --enable-retry \ - --max-attempts 2 \ - --retry-wait-seconds 120 \ - --retry-timeout-increase 0 \ - ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - - stage-c-test-large-8-gpu-amd-rocm720: - name: ${{ format('stage-c-test-large-8-gpu-amd ({0}, linux-{1}-8gpu-sglang, {2})', matrix.rocm_version, inputs.runner_arch || 'mi300', matrix.part) }} - needs: [check-changes, call-gate, wait-for-stage-b-amd-rocm720] - if: | - always() && - ( - (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-c-test-large-8-gpu-amd-rocm720,')) || - ( - !(inputs.target_stage || inputs.target_stage_select) && - ((github.event_name == 'schedule') || (!failure() && !cancelled())) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - env: - RUNNER_LABELS: ${{ format('linux-{0}-8gpu-sglang', inputs.runner_arch || 'mi300') }} - strategy: - fail-fast: false - max-parallel: 2 - matrix: - rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} - part: [0, 1, 2, 3] - runs-on: ${{ format('linux-{0}-8gpu-sglang', inputs.runner_arch || 'mi300') }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Test RCCL multi-GPU communication - timeout-minutes: 5 - run: | - echo "Testing RCCL multi-GPU communication with debug info..." - docker exec ci_sglang bash -c "cd /sglang-checkout && NCCL_DEBUG=INFO RCCL_DEBUG=INFO torchrun --nproc_per_node=8 scripts/ci/amd/test_rccl_multi_gpu.py" - - - name: Run test - timeout-minutes: 120 - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 4 --timeout-per-file 5400 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - - stage-c-test-large-8-gpu-amd-mi35x-rocm720: - name: ${{ format('stage-c-test-large-8-gpu-amd-mi35x ({0}, {1}, {2})', matrix.rocm_version, matrix.runner, matrix.part) }} - needs: [check-changes, call-gate, wait-for-stage-b-amd-rocm720] - if: | - always() && - ( - (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-c-test-large-8-gpu-amd-mi35x-rocm720,')) || - ( - !(inputs.target_stage || inputs.target_stage_select) && - ((github.event_name == 'schedule') || (!failure() && !cancelled())) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} - runner: [linux-mi35x-gpu-8] - part: [0, 1, 2] - runs-on: ${{matrix.runner}} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Run test - timeout-minutes: 60 - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - - # =============================================== DeepSeek-V4 (MI35x, 8-GPU) ==================================================== - # GSM8K accuracy on the nightly dsv4 suites, ~20min each. Gated as stage-C so - # every PR that touches the main package covers DeepSeek-V4. - stage-c-dsv4-flash-fp4-fp8-amd-mi35x-rocm720: - name: ${{ format('stage-c-dsv4-flash-fp4-fp8-amd-mi35x ({0}, {1})', matrix.rocm_version, matrix.runner) }} - needs: [check-changes, call-gate, wait-for-stage-b-amd-rocm720] - if: | - always() && - ( - (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-c-dsv4-flash-fp4-fp8-amd-mi35x-rocm720,')) || - ( - !(inputs.target_stage || inputs.target_stage_select) && - ((github.event_name == 'schedule') || (!failure() && !cancelled())) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} - runner: [linux-mi35x-gpu-8] - runs-on: ${{matrix.runner}} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (ROCm 7.2) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: | - # --skip-test-time-deps: GSM8K + bench_one_batch_server don't need lmms-eval / human-eval. - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Accuracy Test MI35x ROCm 7.2 (8-GPU DeepSeek-V4-Flash FP4 + FP8) - timeout-minutes: 60 - run: | - > github_summary.md # Clear summary file - # SGLANG_DSV4_ACCURACY_ONLY=1 makes the dsv4 test files skip their perf test - # (test_b_perf_8k_1k); only the GSM8K accuracy test runs in this workflow. - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - -e SGLANG_DSV4_ACCURACY_ONLY=1 \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-flash --nightly --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - stage-c-dsv4-pro-fp4-amd-mi35x-rocm720: - name: ${{ format('stage-c-dsv4-pro-fp4-amd-mi35x ({0}, {1})', matrix.rocm_version, matrix.runner) }} - needs: [check-changes, call-gate, wait-for-stage-b-amd-rocm720] - if: | - always() && - ( - (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-c-dsv4-pro-fp4-amd-mi35x-rocm720,')) || - ( - !(inputs.target_stage || inputs.target_stage_select) && - ((github.event_name == 'schedule') || (!failure() && !cancelled())) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} - runner: [linux-mi35x-gpu-8] - runs-on: ${{matrix.runner}} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (ROCm 7.2) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: | - # --skip-test-time-deps: GSM8K + bench_one_batch_server don't need lmms-eval / human-eval. - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Accuracy Test MI35x ROCm 7.2 (8-GPU DeepSeek-V4-Pro FP4) - timeout-minutes: 60 - run: | - > github_summary.md # Clear summary file - # SGLANG_DSV4_ACCURACY_ONLY=1 makes the dsv4 test files skip their perf test - # (test_b_perf_8k_1k); only the GSM8K accuracy test runs in this workflow. - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - -e SGLANG_DSV4_ACCURACY_ONLY=1 \ - -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ - python3 registered/amd/test_deepseek_v4_pro_fp4.py || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # =============================================== Disaggregation ==================================================== - stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720: - name: ${{ format('stage-b-test-large-8-gpu-mi35x-disaggregation-amd ({0}, {1})', matrix.rocm_version, matrix.runner) }} - needs: [check-changes, wait-for-stage-a-amd-rocm720] - if: | - always() && - ( - (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720,')) || - ( - !(inputs.target_stage || inputs.target_stage_select) && - ((github.event_name == 'schedule') || (!failure() && !cancelled())) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} - runner: [linux-mi35x-gpu-8.fabric] - - runs-on: ${{matrix.runner}} - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Check Host RDMA Environment - id: rdma_detect - run: | - set +e - echo "=== Checking Host RDMA Environment ===" - - echo "" - echo "=== 1. Ionic driver library check ===" - ls -l /usr/lib/x86_64-linux-gnu/libibverbs/libionic* 2>/dev/null || echo "libionic not found in standard path" - - echo "" - echo "=== 2. Infiniband devices ===" - ls -la /dev/infiniband/ 2>/dev/null || echo "/dev/infiniband not found" - ls -la /sys/class/infiniband/ 2>/dev/null || echo "/sys/class/infiniband not found" - - echo "" - echo "=== 3. ibv_devinfo ===" - which ibv_devinfo 2>/dev/null && ibv_devinfo 2>&1 || echo "ibv_devinfo not available" - - echo "" - echo "=== 4. Kernel modules ===" - lsmod 2>/dev/null | grep -E "ib_|rdma|ionic" || echo "No RDMA kernel modules loaded" - - echo "" - echo "=== 5. Detect RDMA Devices for test environment ===" - if [ -d "/sys/class/infiniband" ]; then - # RDMA_DEVS=$(ls /sys/class/infiniband | paste -sd "," -) - RDMA_DEVS=rdma0 - echo "Detected RDMA Devices: $RDMA_DEVS" - echo "SGLANG_TEST_RDMA_DEVICE=$RDMA_DEVS" >> $GITHUB_ENV - else - echo "No RDMA devices found in /sys/class/infiniband" - echo "SGLANG_TEST_RDMA_DEVICE=" >> $GITHUB_ENV - fi - - echo "" - echo "=== Host RDMA Check Complete ===" - - - name: Start Special Container - run: bash scripts/ci/amd/amd_ci_start_container_disagg.sh --rocm-version "${{ matrix.rocm_version }}" - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Verify NIXL in Container - run: | - docker exec ci_sglang bash -lc ' - set -eux - export LD_LIBRARY_PATH=/opt/ucx/lib:/opt/rocm/lib:${LD_LIBRARY_PATH:-} - python -c "import nixl; print(\"nixl OK:\", nixl.__file__)" - test -d /opt/ucx/lib - UCX_INFO=$(command -v ucx_info || true) - if [ -z "$UCX_INFO" ]; then - UCX_INFO=/opt/ucx/bin/ucx_info - fi - test -x "$UCX_INFO" - "$UCX_INFO" -v - "$UCX_INFO" -d | grep -iq rocm - python -c "from nixl._api import nixl_agent, nixl_agent_config; nixl_agent(\"ci-smoke\", nixl_agent_config(backends=[\"UCX\"])); print(\"nixl UCX backend init OK\")" - ' - - - name: Verify RDMA in Container - run: | - docker exec -u root ci_sglang bash -c ' - echo "=== Container RDMA Verification ===" - echo "Device nodes:" - ls -la /dev/infiniband/ - echo "" - echo "Provider libraries:" - ls /usr/lib/x86_64-linux-gnu/libibverbs/ | grep -E "ionic|mlx" || echo "No Ionic/Mellanox providers" - echo "" - echo "HCA devices:" - HCA_COUNT=$(ibv_devinfo -list 2>&1 | grep -oE "^[0-9]+ HCAs? found" | grep -oE "^[0-9]+" || echo "0") - ibv_devinfo -list - if [ "$HCA_COUNT" -gt 0 ]; then - echo "" - echo "=== SUCCESS: RDMA setup complete. Found $HCA_COUNT HCA(s) ===" - else - echo "" - echo "=== WARNING: No HCAs detected. RDMA tests may fail ===" - fi - ' - - - name: Run Aiter Op Test (RMSNorm) - timeout-minutes: 10 - run: | - echo "Running pre-check: test_rmsnorm2d.py" - docker exec \ - -e MAX_JOBS=192 \ - ci_sglang \ - python /sgl-workspace/aiter/op_tests/test_rmsnorm2d.py - - - name: Run test_disaggregation - timeout-minutes: 60 - run: | - docker exec ci_sglang rocm-smi --showtopotype || true - docker exec ci_sglang bash -c 'strings $(python3 -c "import mooncake,os;print(os.path.dirname(mooncake.__file__))")/engine*.so | grep -iE "rdma,hip|Unsupported multi-protocol"' || true - bash scripts/ci/amd/amd_ci_exec.sh \ - -e SGLANG_TEST_RDMA_DEVICE="${{ env.SGLANG_TEST_RDMA_DEVICE }}" \ - -e MC_LOG_LEVEL=TRACE \ - -e GLOG_logtostderr=1 \ - -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-8-gpu-mi35x-disaggregation-amd --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - - pr-test-amd-rocm720-finish: - name: pr-test-amd-finish - needs: - [ - call-gate, - check-changes, - - sgl-kernel-unit-test-amd-rocm720, - sgl-kernel-unit-test-2-gpu-amd-rocm720, - multimodal-gen-test-1-gpu-amd-rocm720, - multimodal-gen-test-2-gpu-amd-rocm720, - - wait-for-stage-a-amd-rocm720, - stage-a-test-1-gpu-small-amd-rocm720, - jit-kernel-unit-test-amd-rocm720, - jit-kernel-benchmark-test-amd-rocm720, - wait-for-stage-b-amd-rocm720, - stage-b-test-1-gpu-small-amd-rocm720, - stage-b-test-1-gpu-small-amd-nondeterministic-rocm720, - stage-b-test-1-gpu-small-amd-mi35x-rocm720, - stage-b-test-1-gpu-large-amd-rocm720, - stage-b-test-2-gpu-large-amd-rocm720, - stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720, - stage-c-test-4-gpu-amd-rocm720, - stage-c-test-large-8-gpu-amd-rocm720, - stage-c-test-large-8-gpu-amd-mi35x-rocm720, - - stage-c-dsv4-flash-fp4-fp8-amd-mi35x-rocm720, - stage-c-dsv4-pro-fp4-amd-mi35x-rocm720, - ] - if: always() - runs-on: ubuntu-latest - steps: - - name: Check all dependent job statuses - run: | - # Convert the 'needs' context to a JSON string - json_needs='${{ toJson(needs) }}' - - # Get a list of all job names from the JSON keys - job_names=$(echo "$json_needs" | jq -r 'keys_unsorted[]') - - for job in $job_names; do - # For each job, extract its result - result=$(echo "$json_needs" | jq -r --arg j "$job" '.[$j].result') - - # Print the job name and its result - echo "$job: $result" - - # Check for failure or cancellation and exit if found - if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then - echo "The above jobs failed." - exit 1 - fi - done - - # If the loop completes, all jobs were successful - echo "All jobs completed successfully" - exit 0 diff --git a/.github/workflows/pr-test-amd.yml b/.github/workflows/pr-test-amd.yml index e848ccb27..713dbbf98 100644 --- a/.github/workflows/pr-test-amd.yml +++ b/.github/workflows/pr-test-amd.yml @@ -1,18 +1,32 @@ -name: PR Test ROCm 7.0 (AMD) +name: PR Test (AMD) # Dynamic run-name for manual stage dispatches to enable URL lookup # Format: "[stage-name] sha" for fork PRs, "[stage-name]" for non-fork, default for normal runs run-name: ${{ (inputs.target_stage || inputs.target_stage_select) && (inputs.pr_head_sha && format('[{0}] {1}', inputs.target_stage || inputs.target_stage_select, inputs.pr_head_sha) || format('[{0}]', inputs.target_stage || inputs.target_stage_select)) || '' }} on: schedule: - # ROCm 7.0 rollback shadow. pr-test-amd-rocm720.yml is the PR gate; this - # workflow now only proves ROCm 7.0 still works, so it fires every two days - # alongside nightly-test-amd.yml and chases coverage rather than latency: - # no matrix parallelism caps, no stage fast-fail (the `schedule` escape in - # each job's `if`), and continue-on-error via check-changes. - - cron: '30 17 */2 * *' + - cron: '0 */12 * * *' # rocm10: twice daily (UTC) + - cron: '30 17 * * *' # rocm724/rocm720: once daily (UTC) + pull_request: + paths: + - "python/**" + - "scripts/ci/**" + - "test/**" + - "python/sglang/kernels/aot/**" + - ".github/workflows/pr-test-amd.yml" + - "docker/rocm.Dockerfile" workflow_dispatch: inputs: + rocm_version: + description: 'ROCm image version ("all" runs rocm10, rocm724, and rocm720)' + required: false + type: choice + default: rocm10 + options: + - all + - rocm10 + - rocm724 + - rocm720 target_stage_select: description: "Select a stage to run from dropdown (leave empty for auto-detect)" required: false @@ -33,6 +47,8 @@ on: - multimodal-gen-test-1-gpu-amd - multimodal-gen-test-2-gpu-amd - stage-c-test-4-gpu-amd + - stage-c-dsv4-flash-fp4-fp8-amd-mi35x + - stage-c-dsv4-pro-fp4-amd-mi35x - stage-c-test-large-8-gpu-amd - stage-c-test-large-8-gpu-amd-mi35x - stage-b-test-large-8-gpu-mi35x-disaggregation-amd @@ -51,6 +67,11 @@ on: required: false type: string default: '' + amd_ci_image: + description: 'Override AMD CI Docker image (optional, e.g. rocm/sgl-dev:tag)' + required: false + type: string + default: '' continue_on_error: description: 'Continue on error (do not fail the workflow on test failures)' required: false @@ -71,6 +92,11 @@ on: default: false workflow_call: inputs: + rocm_version: + description: 'ROCm image version ("all" runs rocm10, rocm724, and rocm720)' + required: false + type: string + default: rocm10 ref: description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.' required: false @@ -91,6 +117,11 @@ on: required: false type: string default: '' + amd_ci_image: + description: 'Override AMD CI Docker image (optional, e.g. rocm/sgl-dev:tag)' + required: false + type: string + default: '' continue_on_error: description: 'Continue on error (do not fail the workflow on test failures)' required: false @@ -110,6 +141,7 @@ permissions: env: AITER_COMMIT_OVERRIDE: ${{ inputs.aiter_ref }} + AMD_CI_IMAGE: ${{ inputs.amd_ci_image }} DOCKERHUB_AMD_USERNAME: ${{ secrets.DOCKERHUB_AMD_USERNAME }} DOCKERHUB_AMD_TOKEN: ${{ secrets.DOCKERHUB_AMD_TOKEN }} @@ -134,6 +166,10 @@ jobs: jit_kernel: ${{ steps.filter.outputs.jit_kernel || steps.run-mode.outputs.run_all_tests }} multimodal_gen: ${{ steps.filter.outputs.multimodal_gen || steps.run-mode.outputs.run_all_tests }} continue_on_error: ${{ steps.set-continue-on-error.outputs.continue_on_error }} + # Keep rocm10 on the twice-daily cadence. The 17:30 UTC schedule runs + # rocm724 and rocm720 once daily. Dispatch/call inputs select one version + # or all three, while a pull_request falls through to the rocm10 default. + rocm_versions: ${{ inputs.rocm_version == 'all' && '["rocm10","rocm724","rocm720"]' || inputs.rocm_version && format('["{0}"]', inputs.rocm_version) || github.event.schedule == '30 17 * * *' && '["rocm724","rocm720"]' || '["rocm10"]' }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -212,21 +248,32 @@ jobs: # Not added to `pr-test-amd-finish` so the base AMD gate never depends on # the opt-in extra suite. call-pr-test-amd-extra: + name: call-pr-test-amd-extra + needs: [check-changes] if: | (github.event_name == 'schedule' || inputs.run_all_tests == true) && !(inputs.target_stage || inputs.target_stage_select) + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} uses: ./.github/workflows/pr-test-amd-extra.yml with: ref: ${{ inputs.pr_head_sha || inputs.ref || '' }} runner_arch: ${{ inputs.runner_arch || 'mi300' }} - rocm_version: rocm700 + rocm_version: ${{ matrix.rocm_version }} aiter_ref: ${{ inputs.aiter_ref }} + amd_ci_image: ${{ inputs.amd_ci_image }} continue_on_error: true secrets: inherit # =============================================== sgl-kernel ==================================================== sgl-kernel-unit-test-amd: - name: ${{ format('sgl-kernel-unit-test-amd (rocm700, linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }} + name: ${{ format('sgl-kernel-unit-test-amd ({0}, linux-{1}-1gpu-sglang)', matrix.rocm_version, inputs.runner_arch || 'mi300') }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} needs: [check-changes, call-gate] if: | always() && !cancelled() && @@ -249,7 +296,7 @@ jobs: run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -284,7 +331,11 @@ jobs: exit $failures sgl-kernel-unit-test-2-gpu-amd: - name: ${{ format('sgl-kernel-unit-test-2-gpu-amd (rocm700, linux-{0}-2gpu-sglang)', inputs.runner_arch || 'mi300') }} + name: ${{ format('sgl-kernel-unit-test-2-gpu-amd ({0}, linux-{1}-2gpu-sglang)', matrix.rocm_version, inputs.runner_arch || 'mi300') }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} needs: [check-changes, call-gate] if: | always() && !cancelled() && @@ -307,7 +358,7 @@ jobs: run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -325,7 +376,11 @@ jobs: # =============================================== primary ==================================================== stage-a-test-1-gpu-small-amd: - name: ${{ format('stage-a-test-1-gpu-small-amd (rocm700, linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }} + name: ${{ format('stage-a-test-1-gpu-small-amd ({0}, linux-{1}-1gpu-sglang)', matrix.rocm_version, inputs.runner_arch || 'mi300') }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} needs: [check-changes, call-gate] if: | always() && !cancelled() && @@ -348,7 +403,7 @@ jobs: run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -362,7 +417,11 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-a-test-1-gpu-small-amd ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} jit-kernel-unit-test-amd: - name: ${{ format('jit-kernel-unit-test-amd (rocm700, linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }} + name: ${{ format('jit-kernel-unit-test-amd ({0}, linux-{1}-1gpu-sglang)', matrix.rocm_version, inputs.runner_arch || 'mi300') }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} needs: [check-changes, call-gate] if: | always() && !cancelled() && @@ -385,7 +444,7 @@ jobs: run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -399,7 +458,11 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite jit-kernel-unit-test-amd ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} jit-kernel-benchmark-test-amd: - name: ${{ format('jit-kernel-benchmark-test-amd (rocm700, linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }} + name: ${{ format('jit-kernel-benchmark-test-amd ({0}, linux-{1}-1gpu-sglang)', matrix.rocm_version, inputs.runner_arch || 'mi300') }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} needs: [check-changes, call-gate] if: | always() && !cancelled() && @@ -422,7 +485,7 @@ jobs: run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -441,6 +504,7 @@ jobs: # For scheduled runs: wait jobs are skipped, enabling parallel execution of all stages. wait-for-stage-a-amd: + name: wait-for-stage-a-amd needs: [check-changes, call-gate] if: | always() && @@ -462,7 +526,7 @@ jobs: max-wait-minutes: '240' stage-b-test-1-gpu-small-amd: - name: ${{ format('stage-b-test-1-gpu-small-amd (rocm700, linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} + name: ${{ format('stage-b-test-1-gpu-small-amd ({0}, linux-{1}-1gpu-sglang, {2})', matrix.rocm_version, inputs.runner_arch || 'mi300', matrix.part) }} needs: [check-changes, wait-for-stage-a-amd] if: | always() && @@ -476,7 +540,9 @@ jobs: ) strategy: fail-fast: false + max-parallel: 4 matrix: + rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} part: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13] runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} steps: @@ -489,7 +555,7 @@ jobs: run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -502,7 +568,11 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 14 --timeout-per-file 2400 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-b-test-1-gpu-small-amd-nondeterministic: - name: ${{ format('stage-b-test-1-gpu-small-amd-nondeterministic (rocm700, linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }} + name: ${{ format('stage-b-test-1-gpu-small-amd-nondeterministic ({0}, linux-{1}-1gpu-sglang)', matrix.rocm_version, inputs.runner_arch || 'mi300') }} + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} needs: [check-changes, wait-for-stage-a-amd] if: | always() && @@ -525,7 +595,7 @@ jobs: run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -538,7 +608,7 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-nondeterministic --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-b-test-1-gpu-small-amd-mi35x: - name: ${{ format('stage-b-test-1-gpu-small-amd-mi35x (rocm700, {0})', matrix.runner) }} + name: ${{ format('stage-b-test-1-gpu-small-amd-mi35x ({0}, {1})', matrix.rocm_version, matrix.runner) }} needs: [check-changes, wait-for-stage-a-amd] if: | always() && @@ -553,6 +623,7 @@ jobs: strategy: fail-fast: false matrix: + rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} runner: [linux-mi35x-gpu-1] runs-on: ${{matrix.runner}} steps: @@ -565,7 +636,7 @@ jobs: run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -578,7 +649,7 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-mi35x ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-b-test-1-gpu-large-amd: - name: ${{ format('stage-b-test-1-gpu-large-amd (rocm700, linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} + name: ${{ format('stage-b-test-1-gpu-large-amd ({0}, linux-{1}-1gpu-sglang, {2})', matrix.rocm_version, inputs.runner_arch || 'mi300', matrix.part) }} needs: [check-changes, wait-for-stage-a-amd] if: | always() && @@ -592,7 +663,9 @@ jobs: ) strategy: fail-fast: false + max-parallel: 1 matrix: + rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} part: [0, 1, 2] runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} steps: @@ -605,7 +678,7 @@ jobs: run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -618,7 +691,7 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 2700 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-b-test-2-gpu-large-amd: - name: ${{ format('stage-b-test-2-gpu-large-amd (rocm700, linux-{0}-2gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} + name: ${{ format('stage-b-test-2-gpu-large-amd ({0}, linux-{1}-2gpu-sglang, {2})', matrix.rocm_version, inputs.runner_arch || 'mi300', matrix.part) }} needs: [check-changes, wait-for-stage-a-amd] if: | always() && @@ -632,7 +705,9 @@ jobs: ) strategy: fail-fast: false + max-parallel: 1 matrix: + rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} part: [0, 1] runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }} steps: @@ -645,7 +720,7 @@ jobs: run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -658,7 +733,7 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-2-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file ${{ (inputs.runner_arch || 'mi300') == 'mi300' && 5400 || 2700 }} ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} multimodal-gen-test-1-gpu-amd: - name: ${{ format('multimodal-gen-test-1-gpu-amd (rocm700, linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} + name: ${{ format('multimodal-gen-test-1-gpu-amd ({0}, linux-{1}-1gpu-sglang, {2})', matrix.rocm_version, inputs.runner_arch || 'mi300', matrix.part) }} needs: [check-changes, call-gate] if: | always() && !cancelled() && @@ -672,7 +747,9 @@ jobs: ) strategy: fail-fast: false + max-parallel: 1 matrix: + rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} part: [0, 1, 2, 3] runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} steps: @@ -693,7 +770,7 @@ jobs: pattern: wheel-python3.10-cuda12.9 - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -800,13 +877,13 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: diffusion-failures-amd-1gpu-${{ matrix.part }}-${{ github.run_attempt }} + name: diffusion-failures-amd-1gpu-${{ matrix.rocm_version }}-${{ matrix.part }}-${{ github.run_attempt }} path: diffusion-failures/ if-no-files-found: ignore retention-days: 7 multimodal-gen-test-2-gpu-amd: - name: ${{ format('multimodal-gen-test-2-gpu-amd (rocm700, linux-{0}-2gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} + name: ${{ format('multimodal-gen-test-2-gpu-amd ({0}, linux-{1}-2gpu-sglang, {2})', matrix.rocm_version, inputs.runner_arch || 'mi300', matrix.part) }} needs: [check-changes, call-gate] if: | always() && !cancelled() && @@ -820,7 +897,9 @@ jobs: ) strategy: fail-fast: false + max-parallel: 1 matrix: + rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} # run_suite.py load-balances the suite's parametrized cases and # standalone files over these partitions, so the count is free to # choose. Two shards measured 45-65 min of case work each, inside the @@ -846,7 +925,7 @@ jobs: pattern: wheel-python3.10-cuda12.9 - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -934,12 +1013,13 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: diffusion-failures-amd-2gpu-${{ matrix.part }}-${{ github.run_attempt }} + name: diffusion-failures-amd-2gpu-${{ matrix.rocm_version }}-${{ matrix.part }}-${{ github.run_attempt }} path: diffusion-failures/ if-no-files-found: ignore retention-days: 7 wait-for-stage-b-amd: + name: wait-for-stage-b-amd needs: [check-changes, call-gate, wait-for-stage-a-amd] if: | always() && @@ -967,7 +1047,7 @@ jobs: max-wait-minutes: '480' stage-c-test-4-gpu-amd: - name: ${{ format('stage-c-test-4-gpu-amd (rocm700, linux-{0}-4gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} + name: ${{ format('stage-c-test-4-gpu-amd ({0}, linux-{1}-4gpu-sglang, {2})', matrix.rocm_version, inputs.runner_arch || 'mi300', matrix.part) }} needs: [check-changes, call-gate, wait-for-stage-b-amd] if: | always() && @@ -982,6 +1062,7 @@ jobs: strategy: fail-fast: false matrix: + rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} part: [0] runs-on: ${{ format('linux-{0}-4gpu-sglang', inputs.runner_arch || 'mi300') }} steps: @@ -994,7 +1075,7 @@ jobs: run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -1023,7 +1104,7 @@ jobs: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-c-test-large-8-gpu-amd: - name: ${{ format('stage-c-test-large-8-gpu-amd (rocm700, linux-{0}-8gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} + name: ${{ format('stage-c-test-large-8-gpu-amd ({0}, linux-{1}-8gpu-sglang, {2})', matrix.rocm_version, inputs.runner_arch || 'mi300', matrix.part) }} needs: [check-changes, call-gate, wait-for-stage-b-amd] if: | always() && @@ -1039,7 +1120,9 @@ jobs: RUNNER_LABELS: ${{ format('linux-{0}-8gpu-sglang', inputs.runner_arch || 'mi300') }} strategy: fail-fast: false + max-parallel: 2 matrix: + rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} part: [0, 1, 2, 3] runs-on: ${{ format('linux-{0}-8gpu-sglang', inputs.runner_arch || 'mi300') }} steps: @@ -1052,7 +1135,7 @@ jobs: run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -1071,7 +1154,7 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 4 --timeout-per-file 5400 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-c-test-large-8-gpu-amd-mi35x: - name: ${{ format('stage-c-test-large-8-gpu-amd-mi35x (rocm700, {0}, {1})', matrix.runner, matrix.part) }} + name: ${{ format('stage-c-test-large-8-gpu-amd-mi35x ({0}, {1}, {2})', matrix.rocm_version, matrix.runner, matrix.part) }} needs: [check-changes, call-gate, wait-for-stage-b-amd] if: | always() && @@ -1086,6 +1169,7 @@ jobs: strategy: fail-fast: false matrix: + rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} runner: [linux-mi35x-gpu-8] part: [0, 1, 2] runs-on: ${{matrix.runner}} @@ -1099,7 +1183,7 @@ jobs: run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -1111,9 +1195,124 @@ jobs: run: | bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + # =============================================== DeepSeek-V4 (MI35x, 8-GPU) ==================================================== + # GSM8K accuracy on the nightly dsv4 suites, ~20min each. Gated as stage-C so + # every PR that touches the main package covers DeepSeek-V4. + stage-c-dsv4-flash-fp4-fp8-amd-mi35x: + name: ${{ format('stage-c-dsv4-flash-fp4-fp8-amd-mi35x ({0}, {1})', matrix.rocm_version, matrix.runner) }} + needs: [check-changes, call-gate, wait-for-stage-b-amd] + if: | + always() && + ( + (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-c-dsv4-flash-fp4-fp8-amd-mi35x,')) || + ( + !(inputs.target_stage || inputs.target_stage_select) && + ((github.event_name == 'schedule') || (!failure() && !cancelled())) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + ) + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} + runner: [linux-mi35x-gpu-8] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (ROCm) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + ENABLE_CACHE_HOST: "1" + + - name: Install dependencies + run: | + # --skip-test-time-deps: GSM8K + bench_one_batch_server don't need lmms-eval / human-eval. + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + + - name: Accuracy Test MI35x ROCm (8-GPU DeepSeek-V4-Flash FP4 + FP8) + timeout-minutes: 60 + run: | + > github_summary.md # Clear summary file + # SGLANG_DSV4_ACCURACY_ONLY=1 makes the dsv4 test files skip their perf test + # (test_b_perf_8k_1k); only the GSM8K accuracy test runs in this workflow. + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + -e SGLANG_DSV4_ACCURACY_ONLY=1 \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ + python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-flash --nightly --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + stage-c-dsv4-pro-fp4-amd-mi35x: + name: ${{ format('stage-c-dsv4-pro-fp4-amd-mi35x ({0}, {1})', matrix.rocm_version, matrix.runner) }} + needs: [check-changes, call-gate, wait-for-stage-b-amd] + if: | + always() && + ( + (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-c-dsv4-pro-fp4-amd-mi35x,')) || + ( + !(inputs.target_stage || inputs.target_stage_select) && + ((github.event_name == 'schedule') || (!failure() && !cancelled())) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + ) + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} + runner: [linux-mi35x-gpu-8] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (ROCm) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "${{ matrix.rocm_version }}" + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + ENABLE_CACHE_HOST: "1" + + - name: Install dependencies + run: | + # --skip-test-time-deps: GSM8K + bench_one_batch_server don't need lmms-eval / human-eval. + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + + - name: Accuracy Test MI35x ROCm (8-GPU DeepSeek-V4-Pro FP4) + timeout-minutes: 60 + run: | + > github_summary.md # Clear summary file + # SGLANG_DSV4_ACCURACY_ONLY=1 makes the dsv4 test files skip their perf test + # (test_b_perf_8k_1k); only the GSM8K accuracy test runs in this workflow. + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + -e SGLANG_DSV4_ACCURACY_ONLY=1 \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ + python3 registered/amd/test_deepseek_v4_pro_fp4.py || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + # =============================================== Disaggregation ==================================================== stage-b-test-large-8-gpu-mi35x-disaggregation-amd: - name: ${{ format('stage-b-test-large-8-gpu-mi35x-disaggregation-amd (rocm700, {0})', matrix.runner) }} + name: ${{ format('stage-b-test-large-8-gpu-mi35x-disaggregation-amd ({0}, {1})', matrix.rocm_version, matrix.runner) }} needs: [check-changes, wait-for-stage-a-amd] if: | always() && @@ -1128,6 +1327,7 @@ jobs: strategy: fail-fast: false matrix: + rocm_version: ${{ fromJson(needs.check-changes.outputs.rocm_versions) }} runner: [linux-mi35x-gpu-8.fabric] runs-on: ${{matrix.runner}} @@ -1180,7 +1380,7 @@ jobs: echo "=== Host RDMA Check Complete ===" - name: Start Special Container - run: bash scripts/ci/amd/amd_ci_start_container_disagg.sh + run: bash scripts/ci/amd/amd_ci_start_container_disagg.sh --rocm-version "${{ matrix.rocm_version }}" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -1247,6 +1447,7 @@ jobs: -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-8-gpu-mi35x-disaggregation-amd --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} pr-test-amd-finish: + name: pr-test-amd-finish needs: [ call-gate, @@ -1271,6 +1472,9 @@ jobs: stage-c-test-4-gpu-amd, stage-c-test-large-8-gpu-amd, stage-c-test-large-8-gpu-amd-mi35x, + + stage-c-dsv4-flash-fp4-fp8-amd-mi35x, + stage-c-dsv4-pro-fp4-amd-mi35x, ] if: always() runs-on: ubuntu-latest diff --git a/.github/workflows/release-branch-cut.yml b/.github/workflows/release-branch-cut.yml index 46d219cad..1edeec727 100644 --- a/.github/workflows/release-branch-cut.yml +++ b/.github/workflows/release-branch-cut.yml @@ -174,7 +174,7 @@ jobs: run-pr-tests-amd: needs: cut-release-branch - uses: ./.github/workflows/pr-test-amd-rocm720.yml + uses: ./.github/workflows/pr-test-amd.yml with: ref: ${{ needs.cut-release-branch.outputs.branch_name }} rocm_version: rocm10 @@ -214,7 +214,7 @@ jobs: run-nightly-tests-amd: needs: cut-release-branch - uses: ./.github/workflows/nightly-test-amd-rocm720.yml + uses: ./.github/workflows/nightly-test-amd.yml with: ref: ${{ needs.cut-release-branch.outputs.branch_name }} rocm_version: rocm10 diff --git a/.github/workflows/release-docker-amd-nightly.yml b/.github/workflows/release-docker-amd-nightly.yml deleted file mode 100644 index 98b958d9f..000000000 --- a/.github/workflows/release-docker-amd-nightly.yml +++ /dev/null @@ -1,252 +0,0 @@ -name: Release Docker Images Nightly ROCm7.0 (AMD) -on: - workflow_dispatch: - schedule: - - cron: '0 12 * * *' - -concurrency: - # A PR number if a pull request and otherwise the commit hash. This cancels - # queued and in-progress runs for the same PR (presubmit) or commit - # (postsubmit). The workflow name is prepended to avoid conflicts between - # different workflows. - group: ${{ github.workflow }}-${{ github.event.number || github.sha }} - cancel-in-progress: true - -jobs: - publish: - if: github.repository == 'sgl-project/sglang' - runs-on: amd-docker-scale - environment: 'prod' - strategy: - fail-fast: false - matrix: - gpu_arch: ['gfx942', 'gfx950'] - build_type: ['all'] - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Required for git describe to find tags - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: "Set Date" - run: | - echo "DATE=$(date +%Y%m%d)" >> $GITHUB_ENV - - - name: Get version from latest tag - id: version - run: | - # Use the shared helper so stable/post releases sort above rc tags. - VERSION=$(python3 scripts/release/get_version_tag.py --tag-only | sed 's/^v//') - - if [ -z "$VERSION" ]; then - echo "::error::Could not determine version from git tags" - exit 1 - fi - - # Get short commit hash of current HEAD - COMMIT_HASH=$(git rev-parse --short HEAD) - - # Compose pretend version for setuptools_scm: e.g., 0.5.8.dev20260129+g1a2b3c4 - PRETEND_VERSION="${VERSION}.dev${{ env.DATE }}+g${COMMIT_HASH}" - - echo "version=${VERSION}" >> $GITHUB_OUTPUT - echo "pretend_version=${PRETEND_VERSION}" >> $GITHUB_OUTPUT - echo "Detected version: ${VERSION}" - echo "Pretend version for pip: ${PRETEND_VERSION}" - - - name: Login to Docker Hub (AMD) - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_AMD_USERNAME }} - password: ${{ secrets.DOCKERHUB_AMD_TOKEN }} - - - name: Build and Push to rocm/sgl-dev - run: | - version=${{ steps.version.outputs.version }} - pretend_version=${{ steps.version.outputs.pretend_version }} - echo "Version: ${version}" - echo "Pretend version: ${pretend_version}" - - if [ "${{ matrix.gpu_arch }}" = "gfx942" ]; then - rocm_tag="rocm700-mi30x" - elif [ "${{ matrix.gpu_arch }}" = "gfx950" ]; then - rocm_tag="rocm700-mi35x" - else - echo "Unsupported gfx arch" - exit 1 - fi - - tag=v${version}-${rocm_tag} - echo "IMAGE_TAG=${tag}-${{ env.DATE }}" >> $GITHUB_ENV - - # remove --build-arg NIC_BACKEND=ainic for auto detection nic support in mori - # UBUNTU_MIRROR forces apt over HTTPS to dodge port-80 reachability flakes - # to Canonical's archive.ubuntu.com mirror IPs from the amd-docker-scale runner. - docker build . -f docker/rocm.Dockerfile --build-arg SGL_BRANCH=${{ github.ref_name }} --build-arg BUILD_TYPE=${{ matrix.build_type }} --build-arg GPU_ARCH=${{ matrix.gpu_arch }} --build-arg ENABLE_MORI=1 --build-arg ENABLE_NIXL=1 --build-arg SETUPTOOLS_SCM_PRETEND_VERSION=${pretend_version} --build-arg UBUNTU_MIRROR=https://archive.ubuntu.com -t rocm/sgl-dev:${tag}-${{ env.DATE }} --no-cache - docker push rocm/sgl-dev:${tag}-${{ env.DATE }} - - # Persist the tag right after rocm/sgl-dev push succeeds so the local - # registry mirror can run even if a later step in this job (lmsys push) - # fails. By default this step only runs when the previous step succeeded, - # so the artifact only exists when an image actually landed on Docker Hub. - - name: Save published image tag - run: | - mkdir -p image-tag - echo "${{ env.IMAGE_TAG }}" > "image-tag/${{ matrix.gpu_arch }}.txt" - - - name: Upload image tag artifact - uses: actions/upload-artifact@v4 - with: - name: image-tag-${{ matrix.gpu_arch }} - path: image-tag/${{ matrix.gpu_arch }}.txt - retention-days: 1 - - - name: Login to Docker Hub (lmsys) - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Push to lmsysorg/sglang-rocm - run: | - docker tag rocm/sgl-dev:${{ env.IMAGE_TAG }} lmsysorg/sglang-rocm:${{ env.IMAGE_TAG }} - docker push lmsysorg/sglang-rocm:${{ env.IMAGE_TAG }} - - # Mirror the freshly published rocm/sgl-dev image to the in-network Docker - # registry so AMD CI runners can pull without hitting Docker Hub rate limits. - # The tag is read verbatim from the publish job's artifact so this job uses - # exactly the same tag that publish pushed (only the registry prefix differs). - # `!cancelled()` lets us still mirror successful matrix legs when other legs - # of publish failed; legs without an artifact will fail at download and be - # the only ones marked red. - push_local_registry: - if: ${{ false }} - runs-on: linux-mi300-1gpu-sglang - environment: 'prod' - needs: publish - strategy: - fail-fast: false - matrix: - gpu_arch: ['gfx942', 'gfx950'] - steps: - - name: Download image tag artifact - uses: actions/download-artifact@v4 - with: - name: image-tag-${{ matrix.gpu_arch }} - - - name: Read image tag - run: | - image_tag=$(tr -d '[:space:]' < "${{ matrix.gpu_arch }}.txt") - if [ -z "${image_tag}" ]; then - echo "::error::Image tag artifact is empty" - exit 1 - fi - echo "IMAGE_TAG=${image_tag}" >> $GITHUB_ENV - echo "Resolved IMAGE_TAG=${image_tag}" - - - name: Login to Docker Hub (AMD) - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_AMD_USERNAME }} - password: ${{ secrets.DOCKERHUB_AMD_TOKEN }} - - - name: Mirror rocm/sgl-dev to local registry - run: | - src="rocm/sgl-dev:${{ env.IMAGE_TAG }}" - dst="10.44.14.109:5000/rocm/sgl-dev:${{ env.IMAGE_TAG }}" - docker pull "${src}" - docker tag "${src}" "${dst}" - docker push "${dst}" - - # Temporarily disable docker cache seeding until performant storage is in place - # cache: - # if: false - # # if: always() && github.repository == 'sgl-project/sglang' - # runs-on: linux-mi300-gpu-1 - # environment: 'prod' - # needs: publish - # strategy: - # fail-fast: false - # matrix: - # gpu_arch: ['gfx942'] - # build_type: ['all'] - # steps: - # - name: Checkout repository - # uses: actions/checkout@v4 - # with: - # fetch-depth: 0 # Required for git describe to find tags - - # - name: "Set Date" - # run: | - # echo "DATE=$(date +%Y%m%d)" >> $GITHUB_ENV - - # - name: Get version from latest tag - # id: version - # run: | - # # Use the shared helper so stable/post releases sort above rc tags. - # VERSION=$(python3 scripts/release/get_version_tag.py --tag-only | sed 's/^v//') - - # if [ -z "$VERSION" ]; then - # echo "::error::Could not determine version from git tags" - # exit 1 - # fi - - # echo "version=${VERSION}" >> $GITHUB_OUTPUT - # echo "Detected version: ${VERSION}" - - # - name: Login to Docker Hub - # uses: docker/login-action@v2 - # with: - # username: ${{ secrets.DOCKERHUB_AMD_USERNAME }} - # password: ${{ secrets.DOCKERHUB_AMD_TOKEN }} - - # - name: Pull and Save Docker Image to Cache - # run: | - # set -euxo pipefail - - # version=${{ steps.version.outputs.version }} - # echo "Version: ${version}" - - # if [ "${{ matrix.gpu_arch }}" = "gfx942" ]; then - # rocm_tag="rocm700-mi30x" - # else - # echo "Unsupported gfx arch" - # exit 1 - # fi - - # tag=v${version}-${rocm_tag} - - # if [ "${{ matrix.build_type }}" = "all" ]; then - # tag_suffix="" - # else - # echo "Unsupported build type" - # exit 1 - # fi - - # image="rocm/sgl-dev:${tag}-${{ env.DATE }}${tag_suffix}" - - # # Determine target cache file name based on ROCm variant - # if [[ "${rocm_tag}" == rocm700* ]]; then - # final_path="/home/runner/sgl-data/docker/image-700.tar" - # else - # echo "Unexpected ROCm tag: ${rocm_tag}" - # exit 1 - # fi - - # tmp_path="${final_path}.tmp" - - # echo "Pulling image: ${image}" - # docker pull "${image}" - - # echo "Saving to temp file: ${tmp_path}" - # docker save "${image}" -o "${tmp_path}" - - # echo "Moving to final path: ${final_path}" - # mv -f "${tmp_path}" "${final_path}" - - # echo "Cache populated successfully at ${final_path}" diff --git a/.github/workflows/release-docker-amd.yml b/.github/workflows/release-docker-amd.yml index ebe4b2261..7973232a0 100644 --- a/.github/workflows/release-docker-amd.yml +++ b/.github/workflows/release-docker-amd.yml @@ -18,7 +18,6 @@ on: - rocm10 - rocm724 - rocm720 - - rocm700 jobs: publish: @@ -26,11 +25,11 @@ jobs: runs-on: amd-docker-scale environment: 'prod' strategy: - # Eight flavors publish independently: a failure in one must not cancel - # the others mid-push and leave the release tag with a partial image set. + # Six flavors publish independently: a failure in one must not cancel the + # others mid-push and leave the release tag with a partial image set. fail-fast: false matrix: - rocm_version: ${{ fromJson((github.event_name == 'workflow_dispatch' && inputs.rocm_version != 'all' && inputs.rocm_version != '') && format('["{0}"]', inputs.rocm_version) || '["rocm700", "rocm720", "rocm724", "rocm10"]') }} + rocm_version: ${{ fromJson((github.event_name == 'workflow_dispatch' && inputs.rocm_version != 'all' && inputs.rocm_version != '') && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm720", "rocm724"]') }} gpu_arch: ['gfx942', 'gfx950'] build_type: ['all'] steps: @@ -71,16 +70,7 @@ jobs: echo "Version: ${version}" gpu_arch_suffix="" - if [ "${{ matrix.rocm_version }}" = "rocm700" ]; then - if [ "${{ matrix.gpu_arch }}" = "gfx942" ]; then - rocm_tag="rocm700-mi30x" - elif [ "${{ matrix.gpu_arch }}" = "gfx950" ]; then - rocm_tag="rocm700-mi35x" - else - echo "Unsupported gfx arch" - exit 1 - fi - elif [ "${{ matrix.rocm_version }}" = "rocm720" ] || [ "${{ matrix.rocm_version }}" = "rocm724" ]; then + if [ "${{ matrix.rocm_version }}" = "rocm720" ] || [ "${{ matrix.rocm_version }}" = "rocm724" ]; then gpu_arch_suffix="-${{ matrix.rocm_version }}" if [ "${{ matrix.gpu_arch }}" = "gfx942" ]; then rocm_tag="${{ matrix.rocm_version }}-mi30x" diff --git a/scripts/ci/amd/amd_ci_start_container.sh b/scripts/ci/amd/amd_ci_start_container.sh index e11a18d30..0bd926393 100755 --- a/scripts/ci/amd/amd_ci_start_container.sh +++ b/scripts/ci/amd/amd_ci_start_container.sh @@ -19,8 +19,9 @@ else fi -# Default base tags (can be overridden by command line arguments) -ROCM_VERSION="rocm700" +# Default base tags (can be overridden by command line arguments). +# Keep this aligned with the default AMD PR and Nightly CI lane. +ROCM_VERSION="rocm10" DEFAULT_MI30X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi30x" DEFAULT_MI35X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi35x" LOCAL_DOCKER_REGISTRY="10.44.14.109:5000" @@ -52,8 +53,8 @@ while [[ $# -gt 0 ]]; do echo " --mi35x-base-tag TAG Override MI35x base image tag" echo " --custom-image IMAGE Use a specific Docker image directly (or set AMD_CI_IMAGE)" echo " --build-from-dockerfile Build image from docker/rocm.Dockerfile" - echo " --gpu-arch ARCH GPU architecture for Dockerfile build (e.g., gfx950-rocm720)" - echo " --rocm-version VERSION Override ROCm version for image lookup (e.g., rocm720)" + echo " --gpu-arch ARCH GPU architecture for Dockerfile build (e.g., gfx950-rocm1000)" + echo " --rocm-version VERSION Override ROCm version for image lookup (e.g., rocm10)" echo "" echo "Environment:" echo " ENABLE_CACHE_HOST=1|0" @@ -211,13 +212,6 @@ find_latest_image() { echo "rocm/sgl-dev:v0.5.8.post1-rocm720-mi30x-20260211-preview" fi ;; - rocm700) - if [[ "${gpu_arch}" == "mi35x" ]]; then - echo "rocm/sgl-dev:v0.5.8.post1-rocm700-mi35x-20260211" - else - echo "rocm/sgl-dev:v0.5.8.post1-rocm700-mi30x-20260211" - fi - ;; *) echo "Error: no hard-coded fallback available for ${ROCM_VERSION}" >&2 return 1 @@ -253,7 +247,7 @@ elif [[ -n "${BUILD_FROM_DOCKERFILE}" ]]; then IMAGE="sglang-ci:${GPU_ARCH_BUILD}-$(date +%Y%m%d)" echo "Building Docker image from ${DOCKERFILE} with GPU_ARCH=${GPU_ARCH_BUILD}..." - # Pass full GPU_ARCH (e.g., gfx950-rocm720) - Dockerfile handles stripping suffix + # Pass full GPU_ARCH (e.g., gfx950-rocm1000) - Dockerfile handles stripping suffix docker build \ --build-arg GPU_ARCH="${GPU_ARCH_BUILD}" \ --build-arg SGL_BRANCH="main" \ diff --git a/scripts/ci/amd/amd_ci_start_container_disagg.sh b/scripts/ci/amd/amd_ci_start_container_disagg.sh index 1dcfe71d9..0ac689300 100755 --- a/scripts/ci/amd/amd_ci_start_container_disagg.sh +++ b/scripts/ci/amd/amd_ci_start_container_disagg.sh @@ -19,8 +19,9 @@ else fi -# Default base tags (can be overridden by command line arguments) -ROCM_VERSION="rocm700" +# Default base tags (can be overridden by command line arguments). +# Keep this aligned with the default AMD PR and Nightly CI lane. +ROCM_VERSION="rocm10" DEFAULT_MI30X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi30x" DEFAULT_MI35X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi35x" LOCAL_DOCKER_REGISTRY="10.44.14.109:5000" @@ -196,13 +197,6 @@ find_latest_image() { echo "rocm/sgl-dev:v0.5.8.post1-rocm720-mi30x-20260211-preview" fi ;; - rocm700) - if [[ "${gpu_arch}" == "mi35x" ]]; then - echo "rocm/sgl-dev:v0.5.8.post1-rocm700-mi35x-20260211" - else - echo "rocm/sgl-dev:v0.5.8.post1-rocm700-mi30x-20260211" - fi - ;; *) echo "Error: no hard-coded fallback available for ${ROCM_VERSION}" >&2 return 1 diff --git a/scripts/ci_monitor/ci_failures_analysis.py b/scripts/ci_monitor/ci_failures_analysis.py index 9f40103f6..f1477dcf3 100644 --- a/scripts/ci_monitor/ci_failures_analysis.py +++ b/scripts/ci_monitor/ci_failures_analysis.py @@ -30,12 +30,18 @@ import requests def _filter_legacy_amd_job_rows(job_data: Dict[str, Dict]) -> Dict[str, Dict]: """Drop pre-cutover AMD names without changing the shared analyzer.""" + legacy_callers = { + "call-nightly-amd-rocm720", + "call-pr-test-amd-extra-rocm720", + "call-pr-test-amd-rocm720", + } filtered = {} for full_name, data in job_data.items(): - # This caller was renamed by the AMD job-name cutover. Other outer - # callers, including AITER's *-rocm720 callers, are still current. + # These callers were renamed by the AMD workflow canonicalization. Keep + # recognizing their historical spellings so old rows do not merge into + # current streaks. name_parts = full_name.split(" / ") - if "call-pr-test-amd-extra-rocm720" in name_parts[:-1]: + if any(part in legacy_callers for part in name_parts[:-1]): continue leaf_name = name_parts[-1] @@ -49,7 +55,7 @@ def _filter_legacy_amd_job_rows(job_data: Dict[str, Dict]) -> Dict[str, Dict]: "pr-gate", "pr-test-amd-extra-finish", "pr-test-amd-finish", - "pr-test-amd-rocm720-finish", + "pr-test-amd-rocm720-finish", # Legacy pre-canonicalization teardown }: continue @@ -67,6 +73,12 @@ def _filter_legacy_amd_job_rows(job_data: Dict[str, Dict]) -> Dict[str, Dict]: if details.startswith("rocm") and details[4:].isdigit(): continue + # The canonical workflow paths reuse the history of the retired + # ROCm 7.0 workflows. Do not let those rows contribute to current + # multi-version streaks during the lookback window. + if details.split(",", 1)[0] == "rocm700": + continue + filtered[full_name] = data return filtered @@ -92,7 +104,7 @@ class SGLangFailuresAnalyzer: "check-changes", "pr-test-finish", # Nvidia workflow teardown "pr-test-amd-finish", # AMD workflow teardown - "pr-test-amd-rocm720-finish", # Default AMD ROCm 7.2 teardown + "pr-test-amd-rocm720-finish", # Legacy AMD teardown "call-gate", "pr-gate", "check-all-jobs", @@ -2502,7 +2514,7 @@ def main(): # These 4 don't have scheduled events, so filter by main branch instead pr_test_amd_scheduled_runs = analyzer.get_recent_runs( limit=pr_test_scheduled_limit, - workflow_filter=["pr-test-amd-rocm720.yml"], + workflow_filter=["pr-test-amd.yml"], filters={"branch": "main"}, ) pr_test_xeon_scheduled_runs = analyzer.get_recent_runs( @@ -2529,7 +2541,7 @@ def main(): ) nightly_amd_scheduled_runs = analyzer.get_recent_runs( limit=nightly_scheduled_limit, - workflow_filter=["nightly-test-amd-rocm720.yml"], + workflow_filter=["nightly-test-amd.yml"], filters={"event": "schedule"}, ) nightly_intel_scheduled_runs = analyzer.get_recent_runs( @@ -2551,7 +2563,7 @@ def main(): ) pr_test_amd_general_runs = analyzer.get_recent_runs( limit=args.limit, - workflow_filter=["pr-test-amd-rocm720.yml"], + workflow_filter=["pr-test-amd.yml"], ) pr_test_xeon_general_runs = analyzer.get_recent_runs( limit=args.limit, @@ -2573,7 +2585,7 @@ def main(): ) nightly_amd_general_runs = analyzer.get_recent_runs( limit=args.limit, - workflow_filter=["nightly-test-amd-rocm720.yml"], + workflow_filter=["nightly-test-amd.yml"], ) nightly_intel_general_runs = analyzer.get_recent_runs( limit=args.limit, diff --git a/scripts/ci_monitor/test_ci_failures_analysis.py b/scripts/ci_monitor/test_ci_failures_analysis.py deleted file mode 100644 index 3434f5ad0..000000000 --- a/scripts/ci_monitor/test_ci_failures_analysis.py +++ /dev/null @@ -1,57 +0,0 @@ -"""AMD job-name cutover tests. - -Run with: - python -m unittest discover -s scripts/ci_monitor -p 'test_ci_failures_analysis.py' -""" - -import os -import sys -import unittest - -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from ci_failures_analysis import _filter_legacy_amd_job_rows # noqa: E402 - - -class TestFilterLegacyAmdJobRows(unittest.TestCase): - def test_drops_legacy_names_and_nested_utilities(self): - rows = { - "stage-b-test-1-gpu-small-amd-rocm720 (linux-mi300-1gpu-sglang, 0)": {}, - "nightly-accuracy-2-gpu-rocm720 (rocm724)": {}, - "nightly-accuracy-2-gpu-rocm724": {}, - "nightly-test-1-gpu-unit (rocm724)": {}, - "call-pr-test-amd-rocm720 / call-pr-test-amd-extra-rocm720 / extra-a-test-1-gpu-small-amd (linux-mi300-1gpu-sglang)": {}, - "wait-for-stage-a-amd": {}, - "call-pr-test-amd-extra / pr-test-amd-extra-finish": {}, - "call-pr-test-amd-extra / call-gate / pr-gate": {}, - } - - self.assertEqual(_filter_legacy_amd_job_rows(rows), {}) - - def test_keeps_current_flavors_and_nested_callers_separate(self): - new_success = {"current_streak": 0} - rows = { - "stage-b-test-1-gpu-small-amd (rocm724, linux-mi300-1gpu-sglang, 0)": new_success, - "nightly-accuracy-2-gpu (rocm720, linux-mi300-2gpu-sglang)": { - "current_streak": 1 - }, - "call-pr-test-amd-rocm720 / stage-c-test-4-gpu-amd (rocm724, linux-mi300-4gpu-sglang, 0)": { - "current_streak": 0 - }, - "call-pr-test-amd-rocm720 / call-pr-test-amd-extra / extra-a-test-1-gpu-small-amd (rocm724, linux-mi300-1gpu-sglang)": { - "current_streak": 0 - }, - } - - filtered = _filter_legacy_amd_job_rows(rows) - - self.assertEqual(set(filtered), set(rows)) - self.assertIs( - filtered[ - "stage-b-test-1-gpu-small-amd (rocm724, linux-mi300-1gpu-sglang, 0)" - ], - new_success, - ) - - -if __name__ == "__main__": - unittest.main()