sglang/.github/workflows/nightly-test-amd.yml

name: Nightly Test (AMD)

on:
  schedule:
    - cron: '30 17 * * *'
  push:
    branches:
      - main
    paths:
      - "python/sglang/version.py"
  workflow_dispatch:
    inputs:
      rocm_version:
        description: 'ROCm image version ("all" runs rocm10, rocm724, and rocm720)'
        required: false
        type: choice
        default: rocm10
        options:
          - 'all'
          - rocm10
          - rocm724
          - rocm720
      aiter_ref:
        description: 'Override AITER commit (optional, leave empty to use Dockerfile default)'
        required: false
        type: string
        default: ''
      continue_on_error:
        description: 'Continue on error (do not fail the workflow on test failures)'
        required: false
        type: boolean
        default: true
      job_select:
        description: 'Select a job to run from dropdown (choose "all" to run all jobs)'
        required: false
        type: choice
        default: 'all'
        options:
          - 'all'
          # 1-GPU Unit Tests (MI30x + MI35x)
          - nightly-test-1-gpu-unit
          - nightly-test-1-gpu-kernel
          - nightly-test-1-gpu-mi35x
          # 2-GPU and 4-GPU Tests (MI30x + MI35x)
          - nightly-accuracy-2-gpu
          - nightly-accuracy-2-gpu-vlm
          - nightly-perf-2-gpu-text
          - nightly-perf-2-gpu-vlm
          - nightly-4-gpu
          # 2-GPU GLM-5.1-MXFP4 (MI35x only)
          - nightly-2-gpu-mi35x-glm51-mxfp4
          # 2-GPU DeepSeek-R1-MXFP4 TP2 (MI35x only)
          - nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2
          # 8-GPU GPT-OSS (MI35x mixes Qwen3-Coder-Next)
          - nightly-accuracy-8-gpu
          - nightly-accuracy-8-gpu-mi35x
          # 8-GPU Grok1-INT4 (MI30x + MI35x)
          - nightly-8-gpu-grok1-int4
          - nightly-8-gpu-mi35x-grok1-int4
          # 8-GPU Grok2 (MI30x + MI35x)
          - nightly-8-gpu-grok2
          - nightly-8-gpu-mi35x-grok2
          # 8-GPU DeepSeek-V3.2 (MI30x)
          - nightly-8-gpu-deepseek-v32
          - nightly-8-gpu-deepseek-v32-mtp
          # 8-GPU DeepSeek-V3.2 (MI35x)
          - nightly-8-gpu-mi35x-deepseek-v32
          - nightly-8-gpu-mi35x-deepseek-v32-mtp
          # 8-GPU DeepSeek-R1 (MI35x only)
          - nightly-8-gpu-mi35x-deepseek-r1-mxfp4
          - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8
          - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion
          - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4
          - nightly-8-gpu-mi35x-deepseek-r1-hicache
          # 8-GPU DeepSeek-V4-Flash (MI30x)
          - nightly-8-gpu-deepseek-v4-flash
          # 8-GPU DeepSeek-V4 (MI35x)
          - nightly-8-gpu-mi35x-deepseek-v4-flash
          - nightly-8-gpu-mi35x-deepseek-v4-pro
          - nightly-8-gpu-mi35x-deepseek-v4-pro-mtp
          - nightly-8-gpu-mi35x-deepseek-v4-pro-dspark
          # 8-GPU Kimi-K2.6 (MI30x)
          - nightly-8-gpu-kimi-k26
          # 8-GPU Kimi-K3 (MI35x only - native MXFP4 needs gfx95x)
          - nightly-8-gpu-mi35x-kimi-k3
          # 8-GPU Qwen3.8-MXFP4 (MI35x only - MXFP4 needs gfx95x)
          - nightly-8-gpu-mi35x-qwen38-mxfp4
          # 8-GPU Qwen 3.5 (MI30x + MI35x)
          - nightly-8-gpu-qwen35
          - nightly-8-gpu-mi35x-qwen35
          - nightly-8-gpu-mi35x-qwen35-triton-dcp
          # 8-GPU GLM-5.1 (MI30x)
          - nightly-8-gpu-glm51
          # 8-GPU GLM-5.2-FP8 (MI35x accuracy + performance)
          - nightly-8-gpu-mi35x-glm52-fp8
          # 8-GPU GLM-5-MXFP4 (MI35x only)
          - nightly-8-gpu-mi35x-glm5-mxfp4
          # 4-GPU MiniMax-M2.5 (MI35x)
          - nightly-4-gpu-mi35x-minimax-m25
          # 4-GPU MiniMax-M3 MXFP8 (MI35x accuracy + performance)
          - nightly-4-gpu-mi35x-minimax-m3
          # 8-GPU MiniMax-M2.7 (MI30x only)
          - nightly-8-gpu-minimax-m27
          # Diffusion (MI30x)
          - nightly-1-gpu-zimage-turbo
      job_filter:
        description: 'Or type comma-separated job names (overrides dropdown if non-empty)'
        required: false
        type: string
        default: ''
  workflow_call:
    inputs:
      rocm_version:
        description: 'ROCm image version ("all" runs rocm10, rocm724, and rocm720)'
        required: false
        type: string
        # A caller that says nothing gets the same single-flavor default as a
        # manual dispatch; pass "all" explicitly for multi-version GPU coverage.
        default: rocm10
      ref:
        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
        required: false
        type: string
        default: ''
      aiter_ref:
        description: 'Override AITER commit (optional, leave empty to use Dockerfile default)'
        required: false
        type: string
        default: ''
      job_filter:
        description: 'Select which job to run (leave empty or "all" to run all jobs)'
        required: false
        type: string
        default: 'all'
      continue_on_error:
        description: 'Continue on error (do not fail the workflow on test failures)'
        required: false
        type: boolean
        default: true

env:
  AITER_COMMIT_OVERRIDE: ${{ inputs.aiter_ref }}
  DOCKERHUB_AMD_USERNAME: ${{ secrets.DOCKERHUB_AMD_USERNAME }}
  DOCKERHUB_AMD_TOKEN: ${{ secrets.DOCKERHUB_AMD_TOKEN }}

concurrency:
  # When called via workflow_call with ref set, use a unique group per caller run to avoid
  # collisions with direct schedule/push triggers. We use inputs.ref (not github.event_name)
  # to detect this, because github.event_name inherits from the caller in workflow_call.
  # Manual dispatch and scheduled runs also get unique groups so a version bump
  # or the next daily run cannot cancel an in-progress three-version nightly.
  group: nightly-test-amd-${{ github.event_name == 'workflow_dispatch' && format('manual-{0}', github.run_id) || github.event_name == 'schedule' && format('scheduled-{0}', github.run_id) || inputs.ref && format('caller-{0}', github.run_id) || github.ref }}
  cancel-in-progress: ${{ !inputs.ref && github.event_name != 'workflow_call' && github.event_name != 'workflow_dispatch' && github.event_name != 'schedule' }}

jobs:
  # ============================================== MI30x ROCm Unit Tests ==============================================
  # 1-GPU Unit Tests - LoRA, debug utils, scheduler, etc. (MI30x ROCm)

  # ==============================================================================
  # 1-GPU Unit Tests (MI30x + MI35x)
  # ==============================================================================

  nightly-test-1-gpu-unit:
    name: ${{ format('nightly-test-1-gpu-unit ({0}, linux-mi300-1gpu-sglang)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-unit,'))
    runs-on: linux-mi300-1gpu-sglang
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: bash scripts/ci/amd/amd_ci_install_dependency.sh
      - name: Nightly Unit Test ROCm (1-GPU)
        timeout-minutes: 90
        run: |
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-1-gpu --nightly --timeout-per-file 900 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  nightly-test-1-gpu-kernel:
    name: ${{ format('nightly-test-1-gpu-kernel ({0}, linux-mi300-1gpu-sglang)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-kernel,'))
    runs-on: linux-mi300-1gpu-sglang
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: bash scripts/ci/amd/amd_ci_install_dependency.sh
      - name: Nightly Kernel Benchmark ROCm (1-GPU)
        timeout-minutes: 60
        run: |
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-kernel-1-gpu --nightly --timeout-per-file 900 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  nightly-test-1-gpu-mi35x:
    name: ${{ format('nightly-test-1-gpu-mi35x ({0}, linux-mi35x-gpu-1)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-mi35x,'))
    runs-on: linux-mi35x-gpu-1
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: |
          bash scripts/ci/amd/amd_ci_install_dependency.sh
      - name: Nightly Test MI35x ROCm (1-GPU)
        timeout-minutes: 90
        run: |
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-1-gpu-mi35x --nightly --timeout-per-file 900 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  # ==============================================================================
  # 2-GPU and 4-GPU Tests (MI30x + MI35x)
  # ==============================================================================

  nightly-accuracy-2-gpu:
    name: ${{ format('nightly-accuracy-2-gpu ({0}, linux-mi300-2gpu-sglang)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-2-gpu,'))
    runs-on: linux-mi300-2gpu-sglang
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: bash scripts/ci/amd/amd_ci_install_dependency.sh
      - name: Nightly Test ROCm (2-GPU)
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  nightly-2-gpu-mi35x-glm51-mxfp4:
    name: ${{ format('nightly-2-gpu-mi35x-glm51-mxfp4 ({0}, linux-mi35x-gpu-2)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-2-gpu-mi35x-glm51-mxfp4,'))
    runs-on: linux-mi35x-gpu-2
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: |
          bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
          bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate

      - name: Accuracy Test MI35x ROCm (2-GPU GLM-5.1-MXFP4 GSM8K)
        timeout-minutes: 120
        run: |
          > github_summary.md
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-2-gpu-mi35x-glm51-mxfp4 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2:
    name: ${{ format('nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2 ({0}, linux-mi35x-gpu-2)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2,'))
    runs-on: linux-mi35x-gpu-2
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}
          ENABLE_CACHE_HOST: "1"

      - name: Install dependencies
        run: |
          bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
          bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate

      - name: Accuracy Test MI35x ROCm (2-GPU DeepSeek-R1-MXFP4 TP2)
        timeout-minutes: 180
        run: |
          > github_summary.md
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-2-gpu-mi35x-deepseek-r1-mxfp4-tp2 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  nightly-accuracy-2-gpu-vlm:
    name: ${{ format('nightly-accuracy-2-gpu-vlm ({0}, linux-mi300-2gpu-sglang)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-2-gpu-vlm,'))
    runs-on: linux-mi300-2gpu-sglang
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: bash scripts/ci/amd/amd_ci_install_dependency.sh
      - name: Nightly Accuracy Test ROCm (2-GPU VLM MMMU)
        timeout-minutes: 180
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-accuracy-2-gpu-vlm --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  nightly-perf-2-gpu-text:
    name: ${{ format('nightly-perf-2-gpu-text ({0}, linux-mi300-2gpu-sglang)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-2-gpu-text,'))
    runs-on: linux-mi300-2gpu-sglang
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: bash scripts/ci/amd/amd_ci_install_dependency.sh
      - name: Performance Test (2-GPU Text Models)
        timeout-minutes: 120
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e SGLANG_USE_AITER=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-perf-text-2-gpu --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  nightly-perf-2-gpu-vlm:
    name: ${{ format('nightly-perf-2-gpu-vlm ({0}, linux-mi300-2gpu-sglang)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-2-gpu-vlm,'))
    runs-on: linux-mi300-2gpu-sglang
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: bash scripts/ci/amd/amd_ci_install_dependency.sh
      - name: Performance Test ROCm (2-GPU VLM Models)
        timeout-minutes: 180
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e SGLANG_USE_AITER=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-perf-vlm-2-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  nightly-4-gpu:
    name: ${{ format('nightly-4-gpu ({0}, linux-mi300-4gpu-sglang)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu,'))
    runs-on: linux-mi300-4gpu-sglang
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: bash scripts/ci/amd/amd_ci_install_dependency.sh

      - name: Nightly Test ROCm (4-GPU)
        timeout-minutes: 120
        run: |
          > github_summary.md
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-4-gpu --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  # ==============================================================================
  # 8-GPU GPT-OSS (MI35x mixes Qwen3-Coder-Next)
  # ==============================================================================

  nightly-accuracy-8-gpu:
    name: ${{ format('nightly-accuracy-8-gpu ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu,'))
    runs-on: linux-mi300-8gpu-sglang
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps

      - name: Accuracy Test ROCm (8-GPU GPT-OSS)
        timeout-minutes: 180
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-gpt-oss --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

      # Shares this job with the accuracy step above: it already serves the same
      # two models, and an MI30x job spends ~49 min pulling the image and
      # installing dependencies before it runs anything.
      - name: Performance Test ROCm (8-GPU GPT-OSS)
        if: ${{ !cancelled() }}
        timeout-minutes: 120
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-gpt-oss --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  nightly-accuracy-8-gpu-mi35x:
    name: ${{ format('nightly-accuracy-8-gpu-mi35x ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x,'))
    runs-on: linux-mi35x-gpu-8
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: |
          bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
          # Install tabulate for run_suite.py (missing in MI35x container)
          bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate

      - name: Accuracy Test MI35x ROCm (8-GPU GPT-OSS)
        timeout-minutes: 180
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

      # Shares this job with the accuracy step above rather than taking its own,
      # so the container setup and the GPT-OSS weight cache are paid for once.
      - name: Performance Test MI35x ROCm (8-GPU GPT-OSS)
        if: ${{ !cancelled() }}
        timeout-minutes: 120
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-gpt-oss --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  # ==============================================================================
  # 8-GPU Grok1-INT4 (MI30x + MI35x)
  # ==============================================================================

  nightly-8-gpu-grok1-int4:
    name: ${{ format('nightly-8-gpu-grok1-int4 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok1-int4,'))
    runs-on: linux-mi300-8gpu-sglang
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps

      - name: Accuracy Test ROCm (8-GPU Grok1-INT4)
        timeout-minutes: 60
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e RCCL_MSCCL_ENABLE=0 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-grok1-int4 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

      - name: Performance Test ROCm (8-GPU Grok1-INT4)
        timeout-minutes: 60
        continue-on-error: true
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e RCCL_MSCCL_ENABLE=0 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-grok1-int4 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  nightly-8-gpu-mi35x-grok1-int4:
    name: ${{ format('nightly-8-gpu-mi35x-grok1-int4 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-grok1-int4,'))
    runs-on: linux-mi35x-gpu-8
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: |
          bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
          # Install tabulate for run_suite.py (missing in MI35x container)
          bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate

      - name: Accuracy Test MI35x ROCm (8-GPU Grok1-INT4)
        timeout-minutes: 60
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e RCCL_MSCCL_ENABLE=0 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-grok1-int4 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

      - name: Performance Test MI35x ROCm (8-GPU Grok1-INT4)
        timeout-minutes: 60
        continue-on-error: true
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e RCCL_MSCCL_ENABLE=0 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-grok1-int4 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  # ==============================================================================
  # 8-GPU Grok2 (MI30x + MI35x)
  # ==============================================================================

  nightly-8-gpu-grok2:
    name: ${{ format('nightly-8-gpu-grok2 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok2,'))
    runs-on: linux-mi300-8gpu-sglang
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps

      - name: Accuracy Test ROCm (8-GPU Grok2)
        timeout-minutes: 60
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e RCCL_MSCCL_ENABLE=0 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-grok2 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

      - name: Performance Test ROCm (8-GPU Grok2)
        timeout-minutes: 60
        continue-on-error: true
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e RCCL_MSCCL_ENABLE=0 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-grok2 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  nightly-8-gpu-mi35x-grok2:
    name: ${{ format('nightly-8-gpu-mi35x-grok2 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-grok2,'))
    runs-on: linux-mi35x-gpu-8
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: |
          bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
          # Install tabulate for run_suite.py (missing in MI35x container)
          bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate

      - name: Accuracy Test MI35x ROCm (8-GPU Grok2)
        timeout-minutes: 60
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e RCCL_MSCCL_ENABLE=0 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-grok2 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

      - name: Performance Test MI35x ROCm (8-GPU Grok2)
        timeout-minutes: 60
        continue-on-error: true
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e RCCL_MSCCL_ENABLE=0 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-grok2 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  # ==============================================================================
  # 8-GPU DeepSeek-V3.2 (MI30x)
  #
  # V3-0324 and V3.1 are no longer scheduled here: V3.2 covers the same MI30x
  # paths (aiter MLA, EAGLE MTP, multithread weight load) on a current
  # checkpoint, and DeepSeek-R1/V4 carry the MI35x side.
  # ==============================================================================

  nightly-8-gpu-deepseek-v32:
    name: ${{ format('nightly-8-gpu-deepseek-v32 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v32,'))
    runs-on: linux-mi300-8gpu-sglang
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}
          ENABLE_CACHE_HOST: "1"

      - name: Install dependencies
        run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps

      - name: Accuracy Test ROCm (8-GPU DeepSeek-V3.2 Basic)
        timeout-minutes: 120
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v32 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

      - name: Performance Test ROCm (8-GPU DeepSeek-V3.2 Basic)
        timeout-minutes: 150
        continue-on-error: true
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-deepseek-v32-basic --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  nightly-8-gpu-deepseek-v32-mtp:
    name: ${{ format('nightly-8-gpu-deepseek-v32-mtp ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v32-mtp,'))
    runs-on: linux-mi300-8gpu-sglang
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}
          ENABLE_CACHE_HOST: "1"

      - name: Install dependencies
        run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps

      - name: Accuracy Test ROCm (8-GPU DeepSeek-V3.2 MTP)
        timeout-minutes: 120
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v32-mtp --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

      - name: Performance Test ROCm (8-GPU DeepSeek-V3.2 MTP)
        timeout-minutes: 180
        continue-on-error: true
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-deepseek-v32-mtp --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  # ==============================================================================
  # 8-GPU DeepSeek-V3.2 (MI35x)
  #
  # Accuracy and performance share one job per config, as the MI30x V3.2 jobs
  # above already do: both steps serve the same weights, so splitting them cost
  # a second container setup and a second cold weight load for nothing.
  # ==============================================================================

  nightly-8-gpu-mi35x-deepseek-v32:
    name: ${{ format('nightly-8-gpu-mi35x-deepseek-v32 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v32,'))
    runs-on: linux-mi35x-gpu-8
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}
          ENABLE_CACHE_HOST: "1"

      - name: Install dependencies
        run: |
          bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
          # Install tabulate for run_suite.py (missing in MI35x container)
          bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate

      - name: Accuracy Test MI35x ROCm (8-GPU DeepSeek-V3.2)
        timeout-minutes: 120
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v32 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

      - name: Performance Test MI35x (8-GPU DeepSeek-V3.2 Basic)
        timeout-minutes: 150
        continue-on-error: true
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-deepseek-v32-basic --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  nightly-8-gpu-mi35x-deepseek-v32-mtp:
    name: ${{ format('nightly-8-gpu-mi35x-deepseek-v32-mtp ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v32-mtp,'))
    runs-on: linux-mi35x-gpu-8
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}
          ENABLE_CACHE_HOST: "1"

      - name: Install dependencies
        run: |
          bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
          # Install tabulate for run_suite.py (missing in MI35x container)
          bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate

      - name: Accuracy Test MI35x ROCm (8-GPU DeepSeek-V3.2 TP+MTP)
        timeout-minutes: 120
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-deepseek-v32-mtp --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

      - name: Performance Test MI35x ROCm (8-GPU DeepSeek-V3.2 MTP)
        timeout-minutes: 180
        continue-on-error: true
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-deepseek-v32-mtp --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  # ==============================================================================
  # 8-GPU DeepSeek-R1 (MI35x only)
  # ==============================================================================

  nightly-8-gpu-mi35x-deepseek-r1-mxfp4:
    name: ${{ format('nightly-8-gpu-mi35x-deepseek-r1-mxfp4 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4,'))
    runs-on: linux-mi35x-gpu-8
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}
          ENABLE_CACHE_HOST: "1"

      - name: Install dependencies
        run: |
          bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
          # Install tabulate for run_suite.py (missing in MI35x container)
          bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate

      - name: Accuracy Test MI35x ROCm (8-GPU DeepSeek-R1-MXFP4)
        timeout-minutes: 180
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-r1-mxfp4 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

      - name: Performance Test MI35x ROCm (8-GPU DeepSeek-R1-MXFP4)
        timeout-minutes: 300
        continue-on-error: true
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 registered/amd/perf/mi35x/test_deepseek_r1_mxfp4_perf_mi35x.py || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4:
    name: ${{ format('nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4,'))
    runs-on: linux-mi35x-gpu-8
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}
          ENABLE_CACHE_HOST: "1"

      - name: Install dependencies
        run: |
          bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
          bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate

      - name: Accuracy Test MI35x ROCm (DeepSeek-R1-MXFP4 TP4)
        timeout-minutes: 180
        run: |
          > github_summary.md
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 registered/amd/accuracy/mi35x/test_deepseek_r1_mxfp4_tp4_mi35x.py || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

      - name: Accuracy Test MI35x ROCm (DeepSeek-R1-MXFP4 TP4 MTP)
        if: ${{ !cancelled() }}
        timeout-minutes: 180
        run: |
          > github_summary.md
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 registered/amd/accuracy/mi35x/test_deepseek_r1_mxfp4_tp4_mtp_mi35x.py || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8:
    name: ${{ format('nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8,'))
    runs-on: linux-mi35x-gpu-8
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}
          ENABLE_CACHE_HOST: "1"

      - name: Install dependencies
        run: |
          bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
          # Install tabulate for run_suite.py (missing in MI35x container)
          bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate

      - name: Accuracy Test MI35x ROCm (8-GPU DeepSeek-R1-MXFP4 KV FP8)
        timeout-minutes: 180
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

      - name: Performance Test MI35x ROCm (8-GPU DeepSeek-R1-MXFP4 KV FP8)
        timeout-minutes: 300
        continue-on-error: true
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 registered/amd/perf/mi35x/test_deepseek_r1_mxfp4_kv_fp8_perf_mi35x.py || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion:
    name: ${{ format('nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion,'))
    runs-on: linux-mi35x-gpu-8
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}
          ENABLE_CACHE_HOST: "1"

      - name: Install dependencies
        run: |
          bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
          # Install tabulate for run_suite.py (missing in MI35x container)
          bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate

      - name: Accuracy Test MI35x ROCm (8-GPU DeepSeek-R1-MXFP4 AllReduce Fusion)
        timeout-minutes: 180
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

      - name: Performance Test MI35x ROCm (8-GPU DeepSeek-R1-MXFP4 AllReduce Fusion)
        timeout-minutes: 300
        continue-on-error: true
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 registered/amd/perf/mi35x/test_deepseek_r1_mxfp4_ar_fusion_perf_mi35x.py || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  nightly-8-gpu-mi35x-deepseek-r1-hicache:
    name: ${{ format('nightly-8-gpu-mi35x-deepseek-r1-hicache ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-hicache,'))
    runs-on: linux-mi35x-gpu-8
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: |
          bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
          bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate

      - name: Accuracy Test MI35x ROCm (8-GPU DeepSeek-R1 HiCache)
        timeout-minutes: 180
        run: |
          > github_summary.md
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-r1-hicache --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  # ==============================================================================
  # 8-GPU DeepSeek-V4-Flash (MI30x)
  #
  # Accuracy only: the MI35x jobs below carry the DSV4 perf numbers, and gfx942
  # has no DSV4 perf baseline to regress against yet. Runs the cookbook's
  # MI300X Flash FP8 low-latency cell, so it also covers MLA + KV-FP8 on gfx942.
  # ==============================================================================

  nightly-8-gpu-deepseek-v4-flash:
    name: ${{ format('nightly-8-gpu-deepseek-v4-flash ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v4-flash,'))
    runs-on: linux-mi300-8gpu-sglang
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}
          ENABLE_CACHE_HOST: "1"

      - name: Install dependencies
        run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps

      - name: Accuracy Test ROCm (8-GPU DeepSeek-V4-Flash FP8)
        timeout-minutes: 180
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v4-flash --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  # ==============================================================================
  # 8-GPU DeepSeek-V4 (MI35x)
  # ==============================================================================

  nightly-8-gpu-mi35x-deepseek-v4-flash:
    name: ${{ format('nightly-8-gpu-mi35x-deepseek-v4-flash ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v4-flash,'))
    runs-on: linux-mi35x-gpu-8
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}
          ENABLE_CACHE_HOST: "1"

      - name: Install dependencies
        run: |
          # --skip-test-time-deps: GSM8K + bench_one_batch_server don't need lmms-eval / human-eval.
          bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
          bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate

      - name: Accuracy + Performance Test MI35x ROCm (8-GPU DeepSeek-V4-Flash FP8 + FP4, unified_kv_triton)
        timeout-minutes: 300
        run: |
          > github_summary.md  # Clear summary file
          echo "## SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton" >> github_summary.md
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-flash --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true

      - name: Accuracy + Performance Test MI35x ROCm (8-GPU DeepSeek-V4-Flash FP8 + FP4, triton)
        if: ${{ !cancelled() }}
        timeout-minutes: 300
        run: |
          > github_summary.md  # Clear summary file
          echo "## SGLANG_HACK_FLASHMLA_BACKEND=triton" >> github_summary.md
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_HACK_FLASHMLA_BACKEND=triton \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-flash --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true

  nightly-8-gpu-mi35x-deepseek-v4-pro:
    name: ${{ format('nightly-8-gpu-mi35x-deepseek-v4-pro ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v4-pro,'))
    runs-on: linux-mi35x-gpu-8
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}
          ENABLE_CACHE_HOST: "1"

      - name: Install dependencies
        run: |
          # --skip-test-time-deps: GSM8K + bench_one_batch_server don't need lmms-eval / human-eval.
          bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
          bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate

      - name: Accuracy + Performance Test MI35x ROCm (8-GPU DeepSeek-V4-Pro FP8 + FP4, unified_kv_triton)
        timeout-minutes: 480
        run: |
          > github_summary.md  # Clear summary file
          echo "## SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton" >> github_summary.md
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-pro --nightly --timeout-per-file 14400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true

      - name: Accuracy + Performance Test MI35x ROCm (8-GPU DeepSeek-V4-Pro FP8 + FP4, triton)
        if: ${{ !cancelled() }}
        timeout-minutes: 480
        run: |
          > github_summary.md  # Clear summary file
          echo "## SGLANG_HACK_FLASHMLA_BACKEND=triton" >> github_summary.md
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_HACK_FLASHMLA_BACKEND=triton \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-pro --nightly --timeout-per-file 14400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true

  nightly-8-gpu-mi35x-deepseek-v4-pro-mtp:
    name: ${{ format('nightly-8-gpu-mi35x-deepseek-v4-pro-mtp ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v4-pro-mtp,'))
    runs-on: linux-mi35x-gpu-8
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}
          ENABLE_CACHE_HOST: "1"

      - name: Install dependencies
        run: |
          # --skip-test-time-deps: GSM8K + send_one don't need lmms-eval / human-eval.
          bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
          bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate

      - name: Accuracy + MTP Accept Length Test MI35x ROCm (8-GPU DeepSeek-V4-Pro FP8 MTP, unified_kv_triton)
        timeout-minutes: 240
        run: |
          > github_summary.md  # Clear summary file
          echo "## SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton" >> github_summary.md
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-pro-mtp --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true

      - name: Accuracy + MTP Accept Length Test MI35x ROCm (8-GPU DeepSeek-V4-Pro FP8 MTP, triton)
        if: ${{ !cancelled() }}
        timeout-minutes: 240
        run: |
          > github_summary.md  # Clear summary file
          echo "## SGLANG_HACK_FLASHMLA_BACKEND=triton" >> github_summary.md
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_HACK_FLASHMLA_BACKEND=triton \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-pro-mtp --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true

  nightly-8-gpu-mi35x-deepseek-v4-pro-dspark:
    name: ${{ format('nightly-8-gpu-mi35x-deepseek-v4-pro-dspark ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v4-pro-dspark,'))
    runs-on: linux-mi35x-gpu-8
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}
          ENABLE_CACHE_HOST: "1"

      - name: Install dependencies
        run: |
          # --skip-test-time-deps: GSM8K doesn't need lmms-eval / human-eval.
          bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
          bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate

      - name: Accuracy + DSpark Accept Length Test MI35x ROCm (8-GPU DeepSeek-V4-Pro-DSpark FP4, unified_kv_triton)
        timeout-minutes: 300
        run: |
          > github_summary.md  # Clear summary file
          echo "## SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton" >> github_summary.md
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-pro-dspark --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true

  # ==============================================================================
  # 8-GPU Kimi-K2.6 (MI30x)
  #
  # Kept on MI30x: Kimi-K3 needs gfx95x for its native MXFP4 weights, so it
  # cannot take over here the way it does on MI35x.
  # ==============================================================================

  nightly-8-gpu-kimi-k26:
    name: ${{ format('nightly-8-gpu-kimi-k26 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-kimi-k26,'))
    runs-on: linux-mi300-8gpu-sglang
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}
          ENABLE_CACHE_HOST: "1"

      - name: Install dependencies
        run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps

      - name: Accuracy Test ROCm (8-GPU Kimi-K2.6)
        timeout-minutes: 120
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-kimi-k26 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  # ==============================================================================
  # 8-GPU Kimi-K3 (MI35x only - native MXFP4 needs gfx95x)
  # ==============================================================================

  nightly-8-gpu-mi35x-kimi-k3:
    name: ${{ format('nightly-8-gpu-mi35x-kimi-k3 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-kimi-k3,'))
    runs-on: linux-mi35x-gpu-8
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}
          ENABLE_CACHE_HOST: "1"

      - name: Install dependencies
        run: |
          bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
          # Install tabulate for run_suite.py (missing in MI35x container)
          bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate

      # The 2.8T MXFP4 checkpoint is ~1.56 TB, so weight download and load
      # dominate wall time well before the eval itself starts.
      - name: Accuracy Test MI35x ROCm (8-GPU Kimi-K3)
        timeout-minutes: 300
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-kimi-k3 --nightly --timeout-per-file 14400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

      # Shares the job with the eval above so the checkpoint is already cached
      # and only one 8-GPU MI35x slot is consumed. Step ordering is also the
      # accuracy gate: a failed eval fails the job before this runs, so
      # throughput is never measured on a build that got the tokens wrong.
      # continue-on-error matches every other combined accuracy + perf job here,
      # so a perf hiccup cannot turn the accuracy result red.
      - name: Performance Test MI35x ROCm (8-GPU Kimi-K3)
        timeout-minutes: 300
        continue-on-error: true
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 registered/amd/perf/mi35x/test_kimi_k3_perf_mi35x.py || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  # ==============================================================================
  # 8-GPU Qwen 3.5 (MI30x + MI35x)
  # ==============================================================================

  nightly-8-gpu-qwen35:
    name: ${{ format('nightly-8-gpu-qwen35 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-qwen35,'))
    runs-on: linux-mi300-8gpu-sglang
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}
          ENABLE_CACHE_HOST: "1"

      - name: Install dependencies
        run: |
          bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-aiter-build --skip-test-time-deps
          bash scripts/ci/amd/amd_ci_exec.sh pip install mistral-common "lm-eval[api]"

      - name: Accuracy Test ROCm (8-GPU Qwen 3.5)
        timeout-minutes: 120
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-qwen35 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

      - name: Performance Test ROCm (8-GPU Qwen 3.5 FP8)
        timeout-minutes: 120
        continue-on-error: true
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e SGLANG_USE_AITER=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-qwen35-fp8 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  nightly-8-gpu-mi35x-qwen35:
    name: ${{ format('nightly-8-gpu-mi35x-qwen35 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen35,'))
    runs-on: linux-mi35x-gpu-8
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: |
          bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-aiter-build --skip-test-time-deps
          bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate
          bash scripts/ci/amd/amd_ci_exec.sh pip install mistral-common "lm-eval[api]"

      - name: Accuracy Test MI35x ROCm (8-GPU Qwen 3.5)
        timeout-minutes: 120
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-qwen35 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

      - name: Performance Test MI35x ROCm (8-GPU Qwen 3.5 FP8)
        timeout-minutes: 120
        continue-on-error: true
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_USE_AITER=1 \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-qwen35-fp8 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  nightly-8-gpu-mi35x-qwen35-triton-dcp:
    name: ${{ format('nightly-8-gpu-mi35x-qwen35-triton-dcp ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen35-triton-dcp,'))
    runs-on: linux-mi35x-gpu-8
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: |
          bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-aiter-build --skip-test-time-deps
          bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate

      - name: Accuracy Test MI35x ROCm (8-GPU Qwen 3.5 Triton DCP)
        timeout-minutes: 120
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-qwen35-triton-dcp --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  # ==============================================================================
  # 8-GPU Qwen3.8-MXFP4 (MI35x only - MXFP4 needs gfx95x)
  # ==============================================================================

  nightly-8-gpu-mi35x-qwen38-mxfp4:
    name: ${{ format('nightly-8-gpu-mi35x-qwen38-mxfp4 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen38-mxfp4,'))
    runs-on: linux-mi35x-gpu-8
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}
          ENABLE_CACHE_HOST: "1"

      - name: Install dependencies
        run: |
          bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
          # Install tabulate for run_suite.py (missing in MI35x container)
          bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate

      # ~1.2 TB of weights against a fleet-shared cache that runs at 100% used.
      # Run 32196787596 died 40 minutes into the download on ENOSPC with no
      # free-space figure anywhere in its log; record one up front instead.
      - name: Check model cache space
        run: |
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout \
            bash scripts/ci/amd/check_hf_cache_space.sh \
            amd/Qwen3.8-2.4T-A95B-Quark-MXFP4 1250

      # One step, not the usual accuracy/perf pair: both phases live in the
      # same suite so they launch from one server config. The 2.4T MXFP4
      # checkpoint is ~1.2 TB and is loaded once per phase, which dominates
      # wall time well before either the eval or the benchmark starts.
      - name: Accuracy Test + Performance Test MI35x ROCm (8-GPU Qwen3.8-MXFP4)
        timeout-minutes: 420
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-qwen38-mxfp4 --nightly --timeout-per-file 21600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  # ==============================================================================
  # 8-GPU GLM-5.1 (MI30x + MI35x)
  # ==============================================================================

  nightly-8-gpu-glm51:
    name: ${{ format('nightly-8-gpu-glm51 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-glm51,'))
    runs-on: linux-mi300-8gpu-sglang
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: |
          bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps

      - name: Accuracy Test ROCm (8-GPU GLM-5.1 DSA)
        timeout-minutes: 120
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-glm51 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

      - name: Performance Test ROCm (8-GPU GLM-5.1)
        timeout-minutes: 120
        continue-on-error: true
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e SGLANG_USE_AITER=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-glm51 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  # ==============================================================================
  # 8-GPU GLM-5.2-FP8 (MI35x accuracy + performance)
  # ==============================================================================

  nightly-8-gpu-mi35x-glm52-fp8:
    name: ${{ format('nightly-8-gpu-mi35x-glm52-fp8 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm52-fp8,'))
    runs-on: linux-mi35x-gpu-8
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}
          ENABLE_CACHE_HOST: "1"

      - name: Install dependencies
        run: |
          bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
          bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate

      # The ~700 GB FP8 checkpoint means weight download and load dominate wall
      # time well before the eval itself starts.
      - name: Accuracy Test MI35x ROCm (8-GPU GLM-5.2-FP8 DSA)
        timeout-minutes: 180
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-glm52-fp8 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

      # Run perf only after correctness passes. Keeping both steps in this job
      # reuses the cached ~700 GB checkpoint and one scarce 8-GPU MI35x slot.
      - name: Performance Test MI35x ROCm (8-GPU GLM-5.2-FP8)
        timeout-minutes: 120
        continue-on-error: true
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-glm52-fp8 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  # ==============================================================================
  # 8-GPU GLM-5-MXFP4 (MI35x only)
  # ==============================================================================

  nightly-8-gpu-mi35x-glm5-mxfp4:
    name: ${{ format('nightly-8-gpu-mi35x-glm5-mxfp4 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm5-mxfp4,'))
    runs-on: linux-mi35x-gpu-8
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: |
          bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
          bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate

      - name: Accuracy Test MI35x ROCm (8-GPU GLM-5-MXFP4)
        timeout-minutes: 180
        run: |
          > github_summary.md
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_USE_AITER=1 \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-glm5-mxfp4 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

      - name: Performance Test MI35x ROCm (8-GPU GLM-5-MXFP4)
        timeout-minutes: 300
        continue-on-error: true
        run: |
          > github_summary.md
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_USE_AITER=1 \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 registered/amd/perf/mi35x/test_glm5_mxfp4_perf_mi35x.py || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  # ==============================================================================
  # 4-GPU MiniMax-M2.5 (MI35x)
  # ==============================================================================

  nightly-4-gpu-mi35x-minimax-m25:
    name: ${{ format('nightly-4-gpu-mi35x-minimax-m25 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu-mi35x-minimax-m25,'))
    runs-on: linux-mi35x-gpu-8
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}
          ENABLE_CACHE_HOST: "1"

      - name: Install dependencies
        run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps

      - name: Accuracy Test MI35x ROCm (4-GPU MiniMax-M2.5)
        timeout-minutes: 120
        run: |
          > github_summary.md
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_USE_AITER=1 \
            -e SGLANG_USE_AITER_UNIFIED_ATTN=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-4-gpu-mi35x-minimax-m25-tp4 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  # ==============================================================================
  # 4-GPU MiniMax-M3 MXFP8 (MI35x accuracy + performance)
  # ==============================================================================

  nightly-4-gpu-mi35x-minimax-m3:
    name: ${{ format('nightly-4-gpu-mi35x-minimax-m3 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu-mi35x-minimax-m3,'))
    runs-on: linux-mi35x-gpu-8
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}
          ENABLE_CACHE_HOST: "1"

      - name: Install dependencies
        run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps

      - name: Accuracy Test MI35x ROCm (4-GPU MiniMax-M3 MXFP8)
        timeout-minutes: 120
        run: |
          > github_summary.md
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_USE_AITER=1 \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-4-gpu-mi35x-minimax-m3-tp4 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

      # Run perf only after correctness passes. Keeping both steps in this job
      # reuses the cached checkpoint and one scarce MI35x slot.
      - name: Performance Test MI35x ROCm (4-GPU MiniMax-M3 MXFP8)
        timeout-minutes: 120
        continue-on-error: true  # Perf test failure doesn't fail the job if accuracy passed
        run: |
          > github_summary.md
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_USE_AITER=1 \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-perf-4-gpu-mi35x-minimax-m3 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  # ==============================================================================
  # 8-GPU MiniMax-M2.7 (MI30x only)
  # ==============================================================================

  nightly-8-gpu-minimax-m27:
    name: ${{ format('nightly-8-gpu-minimax-m27 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-minimax-m27,'))
    runs-on: linux-mi300-8gpu-sglang
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}
          ENABLE_CACHE_HOST: "1"

      - name: Install dependencies
        run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps

      - name: Accuracy Test ROCm (8-GPU MiniMax-M2.7)
        timeout-minutes: 120
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e SGLANG_USE_AITER=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-minimax-m27 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

      - name: Performance Test ROCm (8-GPU MiniMax-M2.7)
        timeout-minutes: 120
        continue-on-error: true  # Perf test failure doesn't fail the job if accuracy passed
        run: |
          > github_summary.md  # Clear summary file
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
            -e SGLANG_USE_AITER=1 \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-minimax-m27 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

  # ==============================================================================
  # Diffusion (MI30x)
  # ==============================================================================

  nightly-1-gpu-zimage-turbo:
    name: ${{ format('nightly-1-gpu-zimage-turbo ({0}, linux-mi300-1gpu-sglang)', matrix.rocm_version) }}
    strategy:
      fail-fast: false
      matrix:
        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-1-gpu-zimage-turbo,'))
    runs-on: linux-mi300-1gpu-sglang
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Ensure VRAM is clear
        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm

      - name: Setup docker (${{ matrix.rocm_version }})
        run: |
          touch github_summary.md
          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: bash scripts/ci/amd/amd_ci_install_dependency.sh

      - name: Z-Image-Turbo Diffusion Test ROCm (1-GPU)
        timeout-minutes: 45
        run: |
          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout \
            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
            -e SGLANG_DIFFUSION_ARTIFACT_DIR="/sglang-checkout/diffusion-artifacts" \
            pytest test/registered/amd/test_zimage_turbo.py -v -s ${{ inputs.continue_on_error && '|| true' || '' }} || TEST_EXIT_CODE=$?
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

      - name: Upload generated images
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: zimage-turbo-outputs-${{ matrix.rocm_version }}
          path: diffusion-artifacts/
          if-no-files-found: ignore
          retention-days: 30

  check-all-jobs:
    if: always() && (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch')
    needs:
      # 1-GPU Unit Tests (MI30x + MI35x)
      - nightly-test-1-gpu-unit
      - nightly-test-1-gpu-kernel
      - nightly-test-1-gpu-mi35x
      # 2-GPU and 4-GPU Tests (MI30x + MI35x)
      - nightly-accuracy-2-gpu
      - nightly-accuracy-2-gpu-vlm
      - nightly-perf-2-gpu-text
      - nightly-perf-2-gpu-vlm
      - nightly-4-gpu
      - nightly-2-gpu-mi35x-glm51-mxfp4
      - nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2
      # 8-GPU GPT-OSS (MI35x mixes Qwen3-Coder-Next)
      - nightly-accuracy-8-gpu
      - nightly-accuracy-8-gpu-mi35x
      # 8-GPU Grok1-INT4 (MI30x + MI35x)
      - nightly-8-gpu-grok1-int4
      - nightly-8-gpu-mi35x-grok1-int4
      # 8-GPU Grok2 (MI30x + MI35x)
      - nightly-8-gpu-grok2
      - nightly-8-gpu-mi35x-grok2
      # 8-GPU DeepSeek-V3.2 (MI30x)
      - nightly-8-gpu-deepseek-v32
      - nightly-8-gpu-deepseek-v32-mtp
      # 8-GPU DeepSeek-V3.2 (MI35x)
      - nightly-8-gpu-mi35x-deepseek-v32
      - nightly-8-gpu-mi35x-deepseek-v32-mtp
      # 8-GPU DeepSeek-R1 (MI35x only)
      - nightly-8-gpu-mi35x-deepseek-r1-mxfp4
      - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8
      - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion
      - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4
      - nightly-8-gpu-mi35x-deepseek-r1-hicache
      # 8-GPU DeepSeek-V4-Flash (MI30x)
      - nightly-8-gpu-deepseek-v4-flash
      # 8-GPU DeepSeek-V4 (MI35x)
      - nightly-8-gpu-mi35x-deepseek-v4-flash
      - nightly-8-gpu-mi35x-deepseek-v4-pro
      - nightly-8-gpu-mi35x-deepseek-v4-pro-mtp
      - nightly-8-gpu-mi35x-deepseek-v4-pro-dspark
      # 8-GPU Kimi-K2.6 (MI30x)
      - nightly-8-gpu-kimi-k26
      # 8-GPU Kimi-K3 (MI35x only - native MXFP4 needs gfx95x)
      - nightly-8-gpu-mi35x-kimi-k3
      # 8-GPU Qwen3.8-MXFP4 (MI35x only - MXFP4 needs gfx95x)
      - nightly-8-gpu-mi35x-qwen38-mxfp4
      # 8-GPU Qwen 3.5 (MI30x + MI35x)
      - nightly-8-gpu-qwen35
      - nightly-8-gpu-mi35x-qwen35
      - nightly-8-gpu-mi35x-qwen35-triton-dcp
      # 8-GPU GLM-5.1 (MI30x)
      - nightly-8-gpu-glm51
      # 8-GPU GLM-5.2-FP8 (MI35x accuracy + performance)
      - nightly-8-gpu-mi35x-glm52-fp8
      # 8-GPU GLM-5-MXFP4 (MI35x only)
      - nightly-8-gpu-mi35x-glm5-mxfp4
      # 4-GPU MiniMax-M2.5 (MI35x)
      - nightly-4-gpu-mi35x-minimax-m25
      # 4-GPU MiniMax-M3 MXFP8 (MI35x accuracy + performance)
      - nightly-4-gpu-mi35x-minimax-m3
      # 8-GPU MiniMax-M2.7 (MI30x only)
      - nightly-8-gpu-minimax-m27
      # Diffusion (MI30x)
      - nightly-1-gpu-zimage-turbo
    runs-on: ubuntu-latest
    steps:
      - name: Check if any job failed
        run: |
          if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then
            echo "One or more ROCm nightly test jobs failed"
            exit 1
          fi
          if [[ "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then
            echo "One or more ROCm nightly test jobs were cancelled"
            exit 1
          fi
          echo "All ROCm nightly test jobs passed"