name: Nightly Test (AMD) on: schedule: - cron: '30 17 * * *' push: branches: - main paths: - "python/sglang/version.py" workflow_dispatch: inputs: rocm_version: description: 'ROCm image version ("all" runs rocm10, rocm724, and rocm720)' required: false type: choice default: rocm10 options: - 'all' - rocm10 - rocm724 - rocm720 aiter_ref: description: 'Override AITER commit (optional, leave empty to use Dockerfile default)' required: false type: string default: '' continue_on_error: description: 'Continue on error (do not fail the workflow on test failures)' required: false type: boolean default: true job_select: description: 'Select a job to run from dropdown (choose "all" to run all jobs)' required: false type: choice default: 'all' options: - 'all' # 1-GPU Unit Tests (MI30x + MI35x) - nightly-test-1-gpu-unit - nightly-test-1-gpu-kernel - nightly-test-1-gpu-mi35x # 2-GPU and 4-GPU Tests (MI30x + MI35x) - nightly-accuracy-2-gpu - nightly-accuracy-2-gpu-vlm - nightly-perf-2-gpu-text - nightly-perf-2-gpu-vlm - nightly-4-gpu # 2-GPU GLM-5.1-MXFP4 (MI35x only) - nightly-2-gpu-mi35x-glm51-mxfp4 # 2-GPU DeepSeek-R1-MXFP4 TP2 (MI35x only) - nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2 # 8-GPU GPT-OSS (MI35x mixes Qwen3-Coder-Next) - nightly-accuracy-8-gpu - nightly-accuracy-8-gpu-mi35x # 8-GPU Grok1-INT4 (MI30x + MI35x) - nightly-8-gpu-grok1-int4 - nightly-8-gpu-mi35x-grok1-int4 # 8-GPU Grok2 (MI30x + MI35x) - nightly-8-gpu-grok2 - nightly-8-gpu-mi35x-grok2 # 8-GPU DeepSeek-V3.2 (MI30x) - nightly-8-gpu-deepseek-v32 - nightly-8-gpu-deepseek-v32-mtp # 8-GPU DeepSeek-V3.2 (MI35x) - nightly-8-gpu-mi35x-deepseek-v32 - nightly-8-gpu-mi35x-deepseek-v32-mtp # 8-GPU DeepSeek-R1 (MI35x only) - nightly-8-gpu-mi35x-deepseek-r1-mxfp4 - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8 - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4 - nightly-8-gpu-mi35x-deepseek-r1-hicache # 8-GPU DeepSeek-V4-Flash (MI30x) - nightly-8-gpu-deepseek-v4-flash # 8-GPU DeepSeek-V4 (MI35x) - nightly-8-gpu-mi35x-deepseek-v4-flash - nightly-8-gpu-mi35x-deepseek-v4-pro - nightly-8-gpu-mi35x-deepseek-v4-pro-mtp - nightly-8-gpu-mi35x-deepseek-v4-pro-dspark # 8-GPU Kimi-K2.6 (MI30x) - nightly-8-gpu-kimi-k26 # 8-GPU Kimi-K3 (MI35x only - native MXFP4 needs gfx95x) - nightly-8-gpu-mi35x-kimi-k3 # 8-GPU Qwen3.8-MXFP4 (MI35x only - MXFP4 needs gfx95x) - nightly-8-gpu-mi35x-qwen38-mxfp4 # 8-GPU Qwen 3.5 (MI30x + MI35x) - nightly-8-gpu-qwen35 - nightly-8-gpu-mi35x-qwen35 - nightly-8-gpu-mi35x-qwen35-triton-dcp # 8-GPU GLM-5.1 (MI30x) - nightly-8-gpu-glm51 # 8-GPU GLM-5.2-FP8 (MI35x accuracy + performance) - nightly-8-gpu-mi35x-glm52-fp8 # 8-GPU GLM-5-MXFP4 (MI35x only) - nightly-8-gpu-mi35x-glm5-mxfp4 # 4-GPU MiniMax-M2.5 (MI35x) - nightly-4-gpu-mi35x-minimax-m25 # 4-GPU MiniMax-M3 MXFP8 (MI35x accuracy + performance) - nightly-4-gpu-mi35x-minimax-m3 # 8-GPU MiniMax-M2.7 (MI30x only) - nightly-8-gpu-minimax-m27 # Diffusion (MI30x) - nightly-1-gpu-zimage-turbo job_filter: description: 'Or type comma-separated job names (overrides dropdown if non-empty)' required: false type: string default: '' workflow_call: inputs: rocm_version: description: 'ROCm image version ("all" runs rocm10, rocm724, and rocm720)' required: false type: string # A caller that says nothing gets the same single-flavor default as a # manual dispatch; pass "all" explicitly for multi-version GPU coverage. default: rocm10 ref: description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.' required: false type: string default: '' aiter_ref: description: 'Override AITER commit (optional, leave empty to use Dockerfile default)' required: false type: string default: '' job_filter: description: 'Select which job to run (leave empty or "all" to run all jobs)' required: false type: string default: 'all' continue_on_error: description: 'Continue on error (do not fail the workflow on test failures)' required: false type: boolean default: true env: AITER_COMMIT_OVERRIDE: ${{ inputs.aiter_ref }} DOCKERHUB_AMD_USERNAME: ${{ secrets.DOCKERHUB_AMD_USERNAME }} DOCKERHUB_AMD_TOKEN: ${{ secrets.DOCKERHUB_AMD_TOKEN }} concurrency: # When called via workflow_call with ref set, use a unique group per caller run to avoid # collisions with direct schedule/push triggers. We use inputs.ref (not github.event_name) # to detect this, because github.event_name inherits from the caller in workflow_call. # Manual dispatch and scheduled runs also get unique groups so a version bump # or the next daily run cannot cancel an in-progress three-version nightly. group: nightly-test-amd-${{ github.event_name == 'workflow_dispatch' && format('manual-{0}', github.run_id) || github.event_name == 'schedule' && format('scheduled-{0}', github.run_id) || inputs.ref && format('caller-{0}', github.run_id) || github.ref }} cancel-in-progress: ${{ !inputs.ref && github.event_name != 'workflow_call' && github.event_name != 'workflow_dispatch' && github.event_name != 'schedule' }} jobs: # ============================================== MI30x ROCm Unit Tests ============================================== # 1-GPU Unit Tests - LoRA, debug utils, scheduler, etc. (MI30x ROCm) # ============================================================================== # 1-GPU Unit Tests (MI30x + MI35x) # ============================================================================== nightly-test-1-gpu-unit: name: ${{ format('nightly-test-1-gpu-unit ({0}, linux-mi300-1gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-unit,')) runs-on: linux-mi300-1gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh - name: Nightly Unit Test ROCm (1-GPU) timeout-minutes: 90 run: | bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-1-gpu --nightly --timeout-per-file 900 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} nightly-test-1-gpu-kernel: name: ${{ format('nightly-test-1-gpu-kernel ({0}, linux-mi300-1gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-kernel,')) runs-on: linux-mi300-1gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh - name: Nightly Kernel Benchmark ROCm (1-GPU) timeout-minutes: 60 run: | bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-kernel-1-gpu --nightly --timeout-per-file 900 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} nightly-test-1-gpu-mi35x: name: ${{ format('nightly-test-1-gpu-mi35x ({0}, linux-mi35x-gpu-1)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-mi35x,')) runs-on: linux-mi35x-gpu-1 steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: | bash scripts/ci/amd/amd_ci_install_dependency.sh - name: Nightly Test MI35x ROCm (1-GPU) timeout-minutes: 90 run: | bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-1-gpu-mi35x --nightly --timeout-per-file 900 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} # ============================================================================== # 2-GPU and 4-GPU Tests (MI30x + MI35x) # ============================================================================== nightly-accuracy-2-gpu: name: ${{ format('nightly-accuracy-2-gpu ({0}, linux-mi300-2gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-2-gpu,')) runs-on: linux-mi300-2gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh - name: Nightly Test ROCm (2-GPU) run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} nightly-2-gpu-mi35x-glm51-mxfp4: name: ${{ format('nightly-2-gpu-mi35x-glm51-mxfp4 ({0}, linux-mi35x-gpu-2)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-2-gpu-mi35x-glm51-mxfp4,')) runs-on: linux-mi35x-gpu-2 steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: | bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - name: Accuracy Test MI35x ROCm (2-GPU GLM-5.1-MXFP4 GSM8K) timeout-minutes: 120 run: | > github_summary.md bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-2-gpu-mi35x-glm51-mxfp4 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2: name: ${{ format('nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2 ({0}, linux-mi35x-gpu-2)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2,')) runs-on: linux-mi35x-gpu-2 steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: | bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - name: Accuracy Test MI35x ROCm (2-GPU DeepSeek-R1-MXFP4 TP2) timeout-minutes: 180 run: | > github_summary.md bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-2-gpu-mi35x-deepseek-r1-mxfp4-tp2 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} nightly-accuracy-2-gpu-vlm: name: ${{ format('nightly-accuracy-2-gpu-vlm ({0}, linux-mi300-2gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-2-gpu-vlm,')) runs-on: linux-mi300-2gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh - name: Nightly Accuracy Test ROCm (2-GPU VLM MMMU) timeout-minutes: 180 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-accuracy-2-gpu-vlm --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} nightly-perf-2-gpu-text: name: ${{ format('nightly-perf-2-gpu-text ({0}, linux-mi300-2gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-2-gpu-text,')) runs-on: linux-mi300-2gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh - name: Performance Test (2-GPU Text Models) timeout-minutes: 120 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e SGLANG_USE_AITER=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-perf-text-2-gpu --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} nightly-perf-2-gpu-vlm: name: ${{ format('nightly-perf-2-gpu-vlm ({0}, linux-mi300-2gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-2-gpu-vlm,')) runs-on: linux-mi300-2gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh - name: Performance Test ROCm (2-GPU VLM Models) timeout-minutes: 180 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e SGLANG_USE_AITER=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-perf-vlm-2-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} nightly-4-gpu: name: ${{ format('nightly-4-gpu ({0}, linux-mi300-4gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu,')) runs-on: linux-mi300-4gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh - name: Nightly Test ROCm (4-GPU) timeout-minutes: 120 run: | > github_summary.md bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-4-gpu --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} # ============================================================================== # 8-GPU GPT-OSS (MI35x mixes Qwen3-Coder-Next) # ============================================================================== nightly-accuracy-8-gpu: name: ${{ format('nightly-accuracy-8-gpu ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu,')) runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - name: Accuracy Test ROCm (8-GPU GPT-OSS) timeout-minutes: 180 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-gpt-oss --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} # Shares this job with the accuracy step above: it already serves the same # two models, and an MI30x job spends ~49 min pulling the image and # installing dependencies before it runs anything. - name: Performance Test ROCm (8-GPU GPT-OSS) if: ${{ !cancelled() }} timeout-minutes: 120 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-gpt-oss --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} nightly-accuracy-8-gpu-mi35x: name: ${{ format('nightly-accuracy-8-gpu-mi35x ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: | bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps # Install tabulate for run_suite.py (missing in MI35x container) bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - name: Accuracy Test MI35x ROCm (8-GPU GPT-OSS) timeout-minutes: 180 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} # Shares this job with the accuracy step above rather than taking its own, # so the container setup and the GPT-OSS weight cache are paid for once. - name: Performance Test MI35x ROCm (8-GPU GPT-OSS) if: ${{ !cancelled() }} timeout-minutes: 120 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-gpt-oss --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} # ============================================================================== # 8-GPU Grok1-INT4 (MI30x + MI35x) # ============================================================================== nightly-8-gpu-grok1-int4: name: ${{ format('nightly-8-gpu-grok1-int4 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok1-int4,')) runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - name: Accuracy Test ROCm (8-GPU Grok1-INT4) timeout-minutes: 60 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e RCCL_MSCCL_ENABLE=0 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-grok1-int4 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - name: Performance Test ROCm (8-GPU Grok1-INT4) timeout-minutes: 60 continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e RCCL_MSCCL_ENABLE=0 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-grok1-int4 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-grok1-int4: name: ${{ format('nightly-8-gpu-mi35x-grok1-int4 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-grok1-int4,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: | bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps # Install tabulate for run_suite.py (missing in MI35x container) bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - name: Accuracy Test MI35x ROCm (8-GPU Grok1-INT4) timeout-minutes: 60 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e RCCL_MSCCL_ENABLE=0 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-grok1-int4 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - name: Performance Test MI35x ROCm (8-GPU Grok1-INT4) timeout-minutes: 60 continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e RCCL_MSCCL_ENABLE=0 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-grok1-int4 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} # ============================================================================== # 8-GPU Grok2 (MI30x + MI35x) # ============================================================================== nightly-8-gpu-grok2: name: ${{ format('nightly-8-gpu-grok2 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok2,')) runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - name: Accuracy Test ROCm (8-GPU Grok2) timeout-minutes: 60 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e RCCL_MSCCL_ENABLE=0 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-grok2 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - name: Performance Test ROCm (8-GPU Grok2) timeout-minutes: 60 continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e RCCL_MSCCL_ENABLE=0 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-grok2 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-grok2: name: ${{ format('nightly-8-gpu-mi35x-grok2 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-grok2,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: | bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps # Install tabulate for run_suite.py (missing in MI35x container) bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - name: Accuracy Test MI35x ROCm (8-GPU Grok2) timeout-minutes: 60 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e RCCL_MSCCL_ENABLE=0 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-grok2 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - name: Performance Test MI35x ROCm (8-GPU Grok2) timeout-minutes: 60 continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e RCCL_MSCCL_ENABLE=0 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-grok2 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} # ============================================================================== # 8-GPU DeepSeek-V3.2 (MI30x) # # V3-0324 and V3.1 are no longer scheduled here: V3.2 covers the same MI30x # paths (aiter MLA, EAGLE MTP, multithread weight load) on a current # checkpoint, and DeepSeek-R1/V4 carry the MI35x side. # ============================================================================== nightly-8-gpu-deepseek-v32: name: ${{ format('nightly-8-gpu-deepseek-v32 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v32,')) runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - name: Accuracy Test ROCm (8-GPU DeepSeek-V3.2 Basic) timeout-minutes: 120 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v32 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - name: Performance Test ROCm (8-GPU DeepSeek-V3.2 Basic) timeout-minutes: 150 continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-deepseek-v32-basic --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-deepseek-v32-mtp: name: ${{ format('nightly-8-gpu-deepseek-v32-mtp ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v32-mtp,')) runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - name: Accuracy Test ROCm (8-GPU DeepSeek-V3.2 MTP) timeout-minutes: 120 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v32-mtp --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - name: Performance Test ROCm (8-GPU DeepSeek-V3.2 MTP) timeout-minutes: 180 continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-deepseek-v32-mtp --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} # ============================================================================== # 8-GPU DeepSeek-V3.2 (MI35x) # # Accuracy and performance share one job per config, as the MI30x V3.2 jobs # above already do: both steps serve the same weights, so splitting them cost # a second container setup and a second cold weight load for nothing. # ============================================================================== nightly-8-gpu-mi35x-deepseek-v32: name: ${{ format('nightly-8-gpu-mi35x-deepseek-v32 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v32,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: | bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps # Install tabulate for run_suite.py (missing in MI35x container) bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - name: Accuracy Test MI35x ROCm (8-GPU DeepSeek-V3.2) timeout-minutes: 120 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v32 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - name: Performance Test MI35x (8-GPU DeepSeek-V3.2 Basic) timeout-minutes: 150 continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-deepseek-v32-basic --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-deepseek-v32-mtp: name: ${{ format('nightly-8-gpu-mi35x-deepseek-v32-mtp ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v32-mtp,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: | bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps # Install tabulate for run_suite.py (missing in MI35x container) bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - name: Accuracy Test MI35x ROCm (8-GPU DeepSeek-V3.2 TP+MTP) timeout-minutes: 120 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-deepseek-v32-mtp --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - name: Performance Test MI35x ROCm (8-GPU DeepSeek-V3.2 MTP) timeout-minutes: 180 continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-deepseek-v32-mtp --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} # ============================================================================== # 8-GPU DeepSeek-R1 (MI35x only) # ============================================================================== nightly-8-gpu-mi35x-deepseek-r1-mxfp4: name: ${{ format('nightly-8-gpu-mi35x-deepseek-r1-mxfp4 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: | bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps # Install tabulate for run_suite.py (missing in MI35x container) bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - name: Accuracy Test MI35x ROCm (8-GPU DeepSeek-R1-MXFP4) timeout-minutes: 180 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-r1-mxfp4 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - name: Performance Test MI35x ROCm (8-GPU DeepSeek-R1-MXFP4) timeout-minutes: 300 continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 registered/amd/perf/mi35x/test_deepseek_r1_mxfp4_perf_mi35x.py || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4: name: ${{ format('nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: | bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - name: Accuracy Test MI35x ROCm (DeepSeek-R1-MXFP4 TP4) timeout-minutes: 180 run: | > github_summary.md bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 registered/amd/accuracy/mi35x/test_deepseek_r1_mxfp4_tp4_mi35x.py || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - name: Accuracy Test MI35x ROCm (DeepSeek-R1-MXFP4 TP4 MTP) if: ${{ !cancelled() }} timeout-minutes: 180 run: | > github_summary.md bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 registered/amd/accuracy/mi35x/test_deepseek_r1_mxfp4_tp4_mtp_mi35x.py || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8: name: ${{ format('nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: | bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps # Install tabulate for run_suite.py (missing in MI35x container) bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - name: Accuracy Test MI35x ROCm (8-GPU DeepSeek-R1-MXFP4 KV FP8) timeout-minutes: 180 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - name: Performance Test MI35x ROCm (8-GPU DeepSeek-R1-MXFP4 KV FP8) timeout-minutes: 300 continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 registered/amd/perf/mi35x/test_deepseek_r1_mxfp4_kv_fp8_perf_mi35x.py || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion: name: ${{ format('nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: | bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps # Install tabulate for run_suite.py (missing in MI35x container) bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - name: Accuracy Test MI35x ROCm (8-GPU DeepSeek-R1-MXFP4 AllReduce Fusion) timeout-minutes: 180 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - name: Performance Test MI35x ROCm (8-GPU DeepSeek-R1-MXFP4 AllReduce Fusion) timeout-minutes: 300 continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 registered/amd/perf/mi35x/test_deepseek_r1_mxfp4_ar_fusion_perf_mi35x.py || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-deepseek-r1-hicache: name: ${{ format('nightly-8-gpu-mi35x-deepseek-r1-hicache ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-hicache,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: | bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - name: Accuracy Test MI35x ROCm (8-GPU DeepSeek-R1 HiCache) timeout-minutes: 180 run: | > github_summary.md bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-r1-hicache --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} # ============================================================================== # 8-GPU DeepSeek-V4-Flash (MI30x) # # Accuracy only: the MI35x jobs below carry the DSV4 perf numbers, and gfx942 # has no DSV4 perf baseline to regress against yet. Runs the cookbook's # MI300X Flash FP8 low-latency cell, so it also covers MLA + KV-FP8 on gfx942. # ============================================================================== nightly-8-gpu-deepseek-v4-flash: name: ${{ format('nightly-8-gpu-deepseek-v4-flash ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v4-flash,')) runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - name: Accuracy Test ROCm (8-GPU DeepSeek-V4-Flash FP8) timeout-minutes: 180 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v4-flash --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} # ============================================================================== # 8-GPU DeepSeek-V4 (MI35x) # ============================================================================== nightly-8-gpu-mi35x-deepseek-v4-flash: name: ${{ format('nightly-8-gpu-mi35x-deepseek-v4-flash ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v4-flash,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: | # --skip-test-time-deps: GSM8K + bench_one_batch_server don't need lmms-eval / human-eval. bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - name: Accuracy + Performance Test MI35x ROCm (8-GPU DeepSeek-V4-Flash FP8 + FP4, unified_kv_triton) timeout-minutes: 300 run: | > github_summary.md # Clear summary file echo "## SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton" >> github_summary.md bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-flash --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} echo "$(> $GITHUB_STEP_SUMMARY || true - name: Accuracy + Performance Test MI35x ROCm (8-GPU DeepSeek-V4-Flash FP8 + FP4, triton) if: ${{ !cancelled() }} timeout-minutes: 300 run: | > github_summary.md # Clear summary file echo "## SGLANG_HACK_FLASHMLA_BACKEND=triton" >> github_summary.md bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_HACK_FLASHMLA_BACKEND=triton \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-flash --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} echo "$(> $GITHUB_STEP_SUMMARY || true nightly-8-gpu-mi35x-deepseek-v4-pro: name: ${{ format('nightly-8-gpu-mi35x-deepseek-v4-pro ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v4-pro,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: | # --skip-test-time-deps: GSM8K + bench_one_batch_server don't need lmms-eval / human-eval. bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - name: Accuracy + Performance Test MI35x ROCm (8-GPU DeepSeek-V4-Pro FP8 + FP4, unified_kv_triton) timeout-minutes: 480 run: | > github_summary.md # Clear summary file echo "## SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton" >> github_summary.md bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-pro --nightly --timeout-per-file 14400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} echo "$(> $GITHUB_STEP_SUMMARY || true - name: Accuracy + Performance Test MI35x ROCm (8-GPU DeepSeek-V4-Pro FP8 + FP4, triton) if: ${{ !cancelled() }} timeout-minutes: 480 run: | > github_summary.md # Clear summary file echo "## SGLANG_HACK_FLASHMLA_BACKEND=triton" >> github_summary.md bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_HACK_FLASHMLA_BACKEND=triton \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-pro --nightly --timeout-per-file 14400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} echo "$(> $GITHUB_STEP_SUMMARY || true nightly-8-gpu-mi35x-deepseek-v4-pro-mtp: name: ${{ format('nightly-8-gpu-mi35x-deepseek-v4-pro-mtp ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v4-pro-mtp,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: | # --skip-test-time-deps: GSM8K + send_one don't need lmms-eval / human-eval. bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - name: Accuracy + MTP Accept Length Test MI35x ROCm (8-GPU DeepSeek-V4-Pro FP8 MTP, unified_kv_triton) timeout-minutes: 240 run: | > github_summary.md # Clear summary file echo "## SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton" >> github_summary.md bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-pro-mtp --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} echo "$(> $GITHUB_STEP_SUMMARY || true - name: Accuracy + MTP Accept Length Test MI35x ROCm (8-GPU DeepSeek-V4-Pro FP8 MTP, triton) if: ${{ !cancelled() }} timeout-minutes: 240 run: | > github_summary.md # Clear summary file echo "## SGLANG_HACK_FLASHMLA_BACKEND=triton" >> github_summary.md bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_HACK_FLASHMLA_BACKEND=triton \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-pro-mtp --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} echo "$(> $GITHUB_STEP_SUMMARY || true nightly-8-gpu-mi35x-deepseek-v4-pro-dspark: name: ${{ format('nightly-8-gpu-mi35x-deepseek-v4-pro-dspark ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v4-pro-dspark,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: | # --skip-test-time-deps: GSM8K doesn't need lmms-eval / human-eval. bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - name: Accuracy + DSpark Accept Length Test MI35x ROCm (8-GPU DeepSeek-V4-Pro-DSpark FP4, unified_kv_triton) timeout-minutes: 300 run: | > github_summary.md # Clear summary file echo "## SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton" >> github_summary.md bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-pro-dspark --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} echo "$(> $GITHUB_STEP_SUMMARY || true # ============================================================================== # 8-GPU Kimi-K2.6 (MI30x) # # Kept on MI30x: Kimi-K3 needs gfx95x for its native MXFP4 weights, so it # cannot take over here the way it does on MI35x. # ============================================================================== nightly-8-gpu-kimi-k26: name: ${{ format('nightly-8-gpu-kimi-k26 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-kimi-k26,')) runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - name: Accuracy Test ROCm (8-GPU Kimi-K2.6) timeout-minutes: 120 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-kimi-k26 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} # ============================================================================== # 8-GPU Kimi-K3 (MI35x only - native MXFP4 needs gfx95x) # ============================================================================== nightly-8-gpu-mi35x-kimi-k3: name: ${{ format('nightly-8-gpu-mi35x-kimi-k3 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-kimi-k3,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: | bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps # Install tabulate for run_suite.py (missing in MI35x container) bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate # The 2.8T MXFP4 checkpoint is ~1.56 TB, so weight download and load # dominate wall time well before the eval itself starts. - name: Accuracy Test MI35x ROCm (8-GPU Kimi-K3) timeout-minutes: 300 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-kimi-k3 --nightly --timeout-per-file 14400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} # Shares the job with the eval above so the checkpoint is already cached # and only one 8-GPU MI35x slot is consumed. Step ordering is also the # accuracy gate: a failed eval fails the job before this runs, so # throughput is never measured on a build that got the tokens wrong. # continue-on-error matches every other combined accuracy + perf job here, # so a perf hiccup cannot turn the accuracy result red. - name: Performance Test MI35x ROCm (8-GPU Kimi-K3) timeout-minutes: 300 continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 registered/amd/perf/mi35x/test_kimi_k3_perf_mi35x.py || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} # ============================================================================== # 8-GPU Qwen 3.5 (MI30x + MI35x) # ============================================================================== nightly-8-gpu-qwen35: name: ${{ format('nightly-8-gpu-qwen35 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-qwen35,')) runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: | bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-aiter-build --skip-test-time-deps bash scripts/ci/amd/amd_ci_exec.sh pip install mistral-common "lm-eval[api]" - name: Accuracy Test ROCm (8-GPU Qwen 3.5) timeout-minutes: 120 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-qwen35 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - name: Performance Test ROCm (8-GPU Qwen 3.5 FP8) timeout-minutes: 120 continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e SGLANG_USE_AITER=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-qwen35-fp8 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-qwen35: name: ${{ format('nightly-8-gpu-mi35x-qwen35 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen35,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: | bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-aiter-build --skip-test-time-deps bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate bash scripts/ci/amd/amd_ci_exec.sh pip install mistral-common "lm-eval[api]" - name: Accuracy Test MI35x ROCm (8-GPU Qwen 3.5) timeout-minutes: 120 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-qwen35 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - name: Performance Test MI35x ROCm (8-GPU Qwen 3.5 FP8) timeout-minutes: 120 continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_USE_AITER=1 \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-qwen35-fp8 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-qwen35-triton-dcp: name: ${{ format('nightly-8-gpu-mi35x-qwen35-triton-dcp ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen35-triton-dcp,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: | bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-aiter-build --skip-test-time-deps bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - name: Accuracy Test MI35x ROCm (8-GPU Qwen 3.5 Triton DCP) timeout-minutes: 120 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-qwen35-triton-dcp --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} # ============================================================================== # 8-GPU Qwen3.8-MXFP4 (MI35x only - MXFP4 needs gfx95x) # ============================================================================== nightly-8-gpu-mi35x-qwen38-mxfp4: name: ${{ format('nightly-8-gpu-mi35x-qwen38-mxfp4 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen38-mxfp4,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: | bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps # Install tabulate for run_suite.py (missing in MI35x container) bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate # ~1.2 TB of weights against a fleet-shared cache that runs at 100% used. # Run 32196787596 died 40 minutes into the download on ENOSPC with no # free-space figure anywhere in its log; record one up front instead. - name: Check model cache space run: | bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout \ bash scripts/ci/amd/check_hf_cache_space.sh \ amd/Qwen3.8-2.4T-A95B-Quark-MXFP4 1250 # One step, not the usual accuracy/perf pair: both phases live in the # same suite so they launch from one server config. The 2.4T MXFP4 # checkpoint is ~1.2 TB and is loaded once per phase, which dominates # wall time well before either the eval or the benchmark starts. - name: Accuracy Test + Performance Test MI35x ROCm (8-GPU Qwen3.8-MXFP4) timeout-minutes: 420 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-qwen38-mxfp4 --nightly --timeout-per-file 21600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} # ============================================================================== # 8-GPU GLM-5.1 (MI30x + MI35x) # ============================================================================== nightly-8-gpu-glm51: name: ${{ format('nightly-8-gpu-glm51 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-glm51,')) runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: | bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - name: Accuracy Test ROCm (8-GPU GLM-5.1 DSA) timeout-minutes: 120 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-glm51 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - name: Performance Test ROCm (8-GPU GLM-5.1) timeout-minutes: 120 continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e SGLANG_USE_AITER=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-glm51 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} # ============================================================================== # 8-GPU GLM-5.2-FP8 (MI35x accuracy + performance) # ============================================================================== nightly-8-gpu-mi35x-glm52-fp8: name: ${{ format('nightly-8-gpu-mi35x-glm52-fp8 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm52-fp8,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: | bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate # The ~700 GB FP8 checkpoint means weight download and load dominate wall # time well before the eval itself starts. - name: Accuracy Test MI35x ROCm (8-GPU GLM-5.2-FP8 DSA) timeout-minutes: 180 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-glm52-fp8 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} # Run perf only after correctness passes. Keeping both steps in this job # reuses the cached ~700 GB checkpoint and one scarce 8-GPU MI35x slot. - name: Performance Test MI35x ROCm (8-GPU GLM-5.2-FP8) timeout-minutes: 120 continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-glm52-fp8 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} # ============================================================================== # 8-GPU GLM-5-MXFP4 (MI35x only) # ============================================================================== nightly-8-gpu-mi35x-glm5-mxfp4: name: ${{ format('nightly-8-gpu-mi35x-glm5-mxfp4 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm5-mxfp4,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: | bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - name: Accuracy Test MI35x ROCm (8-GPU GLM-5-MXFP4) timeout-minutes: 180 run: | > github_summary.md bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_USE_AITER=1 \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-glm5-mxfp4 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - name: Performance Test MI35x ROCm (8-GPU GLM-5-MXFP4) timeout-minutes: 300 continue-on-error: true run: | > github_summary.md bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_USE_AITER=1 \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 registered/amd/perf/mi35x/test_glm5_mxfp4_perf_mi35x.py || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} # ============================================================================== # 4-GPU MiniMax-M2.5 (MI35x) # ============================================================================== nightly-4-gpu-mi35x-minimax-m25: name: ${{ format('nightly-4-gpu-mi35x-minimax-m25 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu-mi35x-minimax-m25,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - name: Accuracy Test MI35x ROCm (4-GPU MiniMax-M2.5) timeout-minutes: 120 run: | > github_summary.md bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_USE_AITER=1 \ -e SGLANG_USE_AITER_UNIFIED_ATTN=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-4-gpu-mi35x-minimax-m25-tp4 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} # ============================================================================== # 4-GPU MiniMax-M3 MXFP8 (MI35x accuracy + performance) # ============================================================================== nightly-4-gpu-mi35x-minimax-m3: name: ${{ format('nightly-4-gpu-mi35x-minimax-m3 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu-mi35x-minimax-m3,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - name: Accuracy Test MI35x ROCm (4-GPU MiniMax-M3 MXFP8) timeout-minutes: 120 run: | > github_summary.md bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_USE_AITER=1 \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-4-gpu-mi35x-minimax-m3-tp4 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} # Run perf only after correctness passes. Keeping both steps in this job # reuses the cached checkpoint and one scarce MI35x slot. - name: Performance Test MI35x ROCm (4-GPU MiniMax-M3 MXFP8) timeout-minutes: 120 continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed run: | > github_summary.md bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_USE_AITER=1 \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-perf-4-gpu-mi35x-minimax-m3 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} # ============================================================================== # 8-GPU MiniMax-M2.7 (MI30x only) # ============================================================================== nightly-8-gpu-minimax-m27: name: ${{ format('nightly-8-gpu-minimax-m27 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-minimax-m27,')) runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - name: Accuracy Test ROCm (8-GPU MiniMax-M2.7) timeout-minutes: 120 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e SGLANG_USE_AITER=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-minimax-m27 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - name: Performance Test ROCm (8-GPU MiniMax-M2.7) timeout-minutes: 120 continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e SGLANG_USE_AITER=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-minimax-m27 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} # ============================================================================== # Diffusion (MI30x) # ============================================================================== nightly-1-gpu-zimage-turbo: name: ${{ format('nightly-1-gpu-zimage-turbo ({0}, linux-mi300-1gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-1-gpu-zimage-turbo,')) runs-on: linux-mi300-1gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.sha }} - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh - name: Z-Image-Turbo Diffusion Test ROCm (1-GPU) timeout-minutes: 45 run: | bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ -e SGLANG_DIFFUSION_ARTIFACT_DIR="/sglang-checkout/diffusion-artifacts" \ pytest test/registered/amd/test_zimage_turbo.py -v -s ${{ inputs.continue_on_error && '|| true' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - name: Upload generated images if: always() uses: actions/upload-artifact@v4 with: name: zimage-turbo-outputs-${{ matrix.rocm_version }} path: diffusion-artifacts/ if-no-files-found: ignore retention-days: 30 check-all-jobs: if: always() && (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch') needs: # 1-GPU Unit Tests (MI30x + MI35x) - nightly-test-1-gpu-unit - nightly-test-1-gpu-kernel - nightly-test-1-gpu-mi35x # 2-GPU and 4-GPU Tests (MI30x + MI35x) - nightly-accuracy-2-gpu - nightly-accuracy-2-gpu-vlm - nightly-perf-2-gpu-text - nightly-perf-2-gpu-vlm - nightly-4-gpu - nightly-2-gpu-mi35x-glm51-mxfp4 - nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2 # 8-GPU GPT-OSS (MI35x mixes Qwen3-Coder-Next) - nightly-accuracy-8-gpu - nightly-accuracy-8-gpu-mi35x # 8-GPU Grok1-INT4 (MI30x + MI35x) - nightly-8-gpu-grok1-int4 - nightly-8-gpu-mi35x-grok1-int4 # 8-GPU Grok2 (MI30x + MI35x) - nightly-8-gpu-grok2 - nightly-8-gpu-mi35x-grok2 # 8-GPU DeepSeek-V3.2 (MI30x) - nightly-8-gpu-deepseek-v32 - nightly-8-gpu-deepseek-v32-mtp # 8-GPU DeepSeek-V3.2 (MI35x) - nightly-8-gpu-mi35x-deepseek-v32 - nightly-8-gpu-mi35x-deepseek-v32-mtp # 8-GPU DeepSeek-R1 (MI35x only) - nightly-8-gpu-mi35x-deepseek-r1-mxfp4 - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8 - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4 - nightly-8-gpu-mi35x-deepseek-r1-hicache # 8-GPU DeepSeek-V4-Flash (MI30x) - nightly-8-gpu-deepseek-v4-flash # 8-GPU DeepSeek-V4 (MI35x) - nightly-8-gpu-mi35x-deepseek-v4-flash - nightly-8-gpu-mi35x-deepseek-v4-pro - nightly-8-gpu-mi35x-deepseek-v4-pro-mtp - nightly-8-gpu-mi35x-deepseek-v4-pro-dspark # 8-GPU Kimi-K2.6 (MI30x) - nightly-8-gpu-kimi-k26 # 8-GPU Kimi-K3 (MI35x only - native MXFP4 needs gfx95x) - nightly-8-gpu-mi35x-kimi-k3 # 8-GPU Qwen3.8-MXFP4 (MI35x only - MXFP4 needs gfx95x) - nightly-8-gpu-mi35x-qwen38-mxfp4 # 8-GPU Qwen 3.5 (MI30x + MI35x) - nightly-8-gpu-qwen35 - nightly-8-gpu-mi35x-qwen35 - nightly-8-gpu-mi35x-qwen35-triton-dcp # 8-GPU GLM-5.1 (MI30x) - nightly-8-gpu-glm51 # 8-GPU GLM-5.2-FP8 (MI35x accuracy + performance) - nightly-8-gpu-mi35x-glm52-fp8 # 8-GPU GLM-5-MXFP4 (MI35x only) - nightly-8-gpu-mi35x-glm5-mxfp4 # 4-GPU MiniMax-M2.5 (MI35x) - nightly-4-gpu-mi35x-minimax-m25 # 4-GPU MiniMax-M3 MXFP8 (MI35x accuracy + performance) - nightly-4-gpu-mi35x-minimax-m3 # 8-GPU MiniMax-M2.7 (MI30x only) - nightly-8-gpu-minimax-m27 # Diffusion (MI30x) - nightly-1-gpu-zimage-turbo runs-on: ubuntu-latest steps: - name: Check if any job failed run: | if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then echo "One or more ROCm nightly test jobs failed" exit 1 fi if [[ "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then echo "One or more ROCm nightly test jobs were cancelled" exit 1 fi echo "All ROCm nightly test jobs passed"