diff --git a/.github/workflows/amd-aiter-scout.yml b/.github/workflows/amd-aiter-scout.yml index 098240776..8365856ad 100644 --- a/.github/workflows/amd-aiter-scout.yml +++ b/.github/workflows/amd-aiter-scout.yml @@ -106,6 +106,7 @@ jobs: secrets: inherit with: ref: amd/aiter-ci + rocm_version: rocm724 aiter_ref: ${{ needs.resolve-aiter.outputs.aiter_sha }} job_filter: 'all' continue_on_error: ${{ inputs.continue_on_error == '' && true || inputs.continue_on_error }} diff --git a/.github/workflows/bot-bump-sglang-version.yml b/.github/workflows/bot-bump-sglang-version.yml index 21b795eac..b764d0b50 100644 --- a/.github/workflows/bot-bump-sglang-version.yml +++ b/.github/workflows/bot-bump-sglang-version.yml @@ -65,6 +65,7 @@ jobs: uses: ./.github/workflows/nightly-test-amd-rocm720.yml with: ref: ${{ needs.bump-sglang-version.outputs.branch_name }} + rocm_version: rocm724 secrets: inherit run-nightly-tests-npu: diff --git a/.github/workflows/nightly-test-amd-rocm720.yml b/.github/workflows/nightly-test-amd-rocm720.yml index d6e6e86ec..1db903d70 100644 --- a/.github/workflows/nightly-test-amd-rocm720.yml +++ b/.github/workflows/nightly-test-amd-rocm720.yml @@ -10,6 +10,15 @@ on: - "python/sglang/version.py" workflow_dispatch: inputs: + rocm_version: + description: 'ROCm image version ("all" runs every flavor, as the nightly schedule does)' + required: false + type: choice + default: 'all' + options: + - 'all' + - rocm724 + - rocm720 aiter_ref: description: 'Override AITER commit (optional, leave empty to use Dockerfile default)' required: false @@ -103,6 +112,13 @@ on: default: '' workflow_call: inputs: + rocm_version: + description: 'ROCm image version ("all" runs every flavor)' + required: false + type: string + # A single flavor, unlike the schedule and the dispatch form: a caller + # that says nothing should not silently get double the GPU cost. + default: rocm724 ref: description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.' required: false @@ -146,6 +162,10 @@ jobs: # ============================================================================== nightly-test-1-gpu-unit-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-unit-rocm720,')) runs-on: linux-mi300-1gpu-sglang steps: @@ -160,7 +180,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -176,6 +196,10 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-test-1-gpu-kernel-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-kernel-rocm720,')) runs-on: linux-mi300-1gpu-sglang steps: @@ -190,7 +214,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -206,6 +230,10 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-test-1-gpu-mi35x-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-mi35x-rocm720,')) runs-on: linux-mi35x-gpu-1 steps: @@ -220,7 +248,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -241,6 +269,10 @@ jobs: # ============================================================================== nightly-accuracy-2-gpu-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-2-gpu-rocm720,')) runs-on: linux-mi300-2gpu-sglang steps: @@ -255,7 +287,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -271,6 +303,10 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-2-gpu-mi35x-glm51-mxfp4-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-2-gpu-mi35x-glm51-mxfp4-rocm720,')) runs-on: linux-mi35x-gpu-2 steps: @@ -285,7 +321,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -305,6 +341,10 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2-rocm720,')) runs-on: linux-mi35x-gpu-2 steps: @@ -319,7 +359,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" @@ -340,6 +380,10 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-accuracy-2-gpu-vlm-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-2-gpu-vlm-rocm720,')) runs-on: linux-mi300-2gpu-sglang steps: @@ -354,7 +398,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -371,6 +415,10 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-perf-2-gpu-text-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-2-gpu-text-rocm720,')) runs-on: linux-mi300-2gpu-sglang steps: @@ -385,7 +433,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -403,6 +451,10 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-perf-2-gpu-vlm-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-2-gpu-vlm-rocm720,')) runs-on: linux-mi300-2gpu-sglang steps: @@ -417,7 +469,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -435,6 +487,10 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-4-gpu-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu-rocm720,')) runs-on: linux-mi300-4gpu-sglang steps: @@ -449,7 +505,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -471,6 +527,10 @@ jobs: # ============================================================================== nightly-accuracy-8-gpu-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-rocm720,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -485,7 +545,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -519,6 +579,10 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-accuracy-8-gpu-mi35x-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x-rocm720,')) runs-on: linux-mi35x-gpu-8 steps: @@ -533,7 +597,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -571,6 +635,10 @@ jobs: # ============================================================================== nightly-8-gpu-grok1-int4-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok1-int4-rocm720,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -585,7 +653,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -616,6 +684,10 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-grok1-int4-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-grok1-int4-rocm720,')) runs-on: linux-mi35x-gpu-8 steps: @@ -630,7 +702,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -668,6 +740,10 @@ jobs: # ============================================================================== nightly-8-gpu-grok2-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok2-rocm720,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -682,7 +758,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -713,6 +789,10 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-grok2-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-grok2-rocm720,')) runs-on: linux-mi35x-gpu-8 steps: @@ -727,7 +807,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -765,6 +845,10 @@ jobs: # ============================================================================== nightly-8-gpu-deepseek-v31-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v31-rocm720,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -779,7 +863,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" @@ -811,6 +895,10 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-deepseek-v32-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v32-rocm720,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -825,7 +913,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" @@ -855,6 +943,10 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-deepseek-v32-mtp-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v32-mtp-rocm720,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -869,7 +961,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" @@ -899,6 +991,10 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-deepseek-v3-kv-fp8-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v3-kv-fp8-rocm720,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -913,7 +1009,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" @@ -936,6 +1032,10 @@ jobs: # ============================================================================== nightly-accuracy-8-gpu-mi35x-deepseek-v32-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x-deepseek-v32-rocm720,')) runs-on: linux-mi35x-gpu-8 steps: @@ -950,7 +1050,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" @@ -972,6 +1072,10 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp-rocm720,')) runs-on: linux-mi35x-gpu-8 steps: @@ -986,7 +1090,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" @@ -1008,6 +1112,10 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-perf-8-gpu-mi35x-deepseek-v32-basic-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-8-gpu-mi35x-deepseek-v32-basic-rocm720,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1022,7 +1130,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" @@ -1044,6 +1152,10 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-perf-8-gpu-mi35x-deepseek-v32-mtp-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-8-gpu-mi35x-deepseek-v32-mtp-rocm720,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1058,7 +1170,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" @@ -1084,6 +1196,10 @@ jobs: # ============================================================================== nightly-8-gpu-mi35x-deepseek-r1-mxfp4-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-rocm720,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1098,7 +1214,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" @@ -1131,6 +1247,10 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4-rocm720,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1145,7 +1265,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" @@ -1177,6 +1297,10 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8-rocm720,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1191,7 +1315,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" @@ -1224,6 +1348,10 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion-rocm720,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1238,7 +1366,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" @@ -1271,6 +1399,10 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-deepseek-r1-hicache-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-hicache-rocm720,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1285,7 +1417,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -1309,6 +1441,10 @@ jobs: # ============================================================================== nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1323,7 +1459,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" @@ -1360,6 +1496,10 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1374,7 +1514,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" @@ -1411,6 +1551,10 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true nightly-8-gpu-mi35x-deepseek-v4-pro-mtp-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v4-pro-mtp-rocm720,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1425,7 +1569,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" @@ -1462,6 +1606,10 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true nightly-8-gpu-mi35x-deepseek-v4-pro-dspark-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v4-pro-dspark-rocm720,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1476,7 +1624,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" @@ -1507,6 +1655,10 @@ jobs: # ============================================================================== nightly-8-gpu-kimi-k26-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-kimi-k26-rocm720,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -1521,7 +1673,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" @@ -1544,6 +1696,10 @@ jobs: # ============================================================================== nightly-8-gpu-mi35x-kimi-k3-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-kimi-k3-rocm720,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1558,7 +1714,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" @@ -1603,6 +1759,10 @@ jobs: # ============================================================================== nightly-8-gpu-qwen3-235b-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-qwen3-235b-rocm720,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -1617,7 +1777,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" @@ -1637,6 +1797,10 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1651,7 +1815,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -1676,6 +1840,10 @@ jobs: # ============================================================================== nightly-8-gpu-qwen35-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-qwen35-rocm720,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -1690,7 +1858,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" @@ -1724,6 +1892,10 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-qwen35-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen35-rocm720,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1738,7 +1910,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -1771,6 +1943,10 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-qwen35-triton-dcp-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen35-triton-dcp-rocm720,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1785,7 +1961,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -1809,6 +1985,10 @@ jobs: # ============================================================================== nightly-8-gpu-glm51-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-glm51-rocm720,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -1823,7 +2003,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -1858,6 +2038,10 @@ jobs: # ============================================================================== nightly-8-gpu-mi35x-glm52-fp8-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm52-fp8-rocm720,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1872,7 +2056,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" @@ -1912,6 +2096,10 @@ jobs: # ============================================================================== nightly-8-gpu-mi35x-glm5-mxfp4-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm5-mxfp4-rocm720,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1926,7 +2114,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -1963,6 +2151,10 @@ jobs: # ============================================================================== nightly-4-gpu-mi35x-minimax-m25-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu-mi35x-minimax-m25-rocm720,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1977,7 +2169,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" @@ -2002,6 +2194,10 @@ jobs: # ============================================================================== nightly-4-gpu-mi35x-minimax-m3-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu-mi35x-minimax-m3-rocm720,')) runs-on: linux-mi35x-gpu-8 steps: @@ -2016,7 +2212,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" @@ -2040,6 +2236,10 @@ jobs: # ============================================================================== nightly-8-gpu-minimax-m27-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-minimax-m27-rocm720,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -2054,7 +2254,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" @@ -2090,6 +2290,10 @@ jobs: # ============================================================================== nightly-1-gpu-zimage-turbo-rocm720: + strategy: + fail-fast: false + matrix: + rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-1-gpu-zimage-turbo-rocm720,')) runs-on: linux-mi300-1gpu-sglang steps: @@ -2104,7 +2308,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} env: GITHUB_WORKSPACE: ${{ github.workspace }} diff --git a/.github/workflows/release-branch-cut.yml b/.github/workflows/release-branch-cut.yml index f726644ff..8c733c8dc 100644 --- a/.github/workflows/release-branch-cut.yml +++ b/.github/workflows/release-branch-cut.yml @@ -217,6 +217,7 @@ jobs: uses: ./.github/workflows/nightly-test-amd-rocm720.yml with: ref: ${{ needs.cut-release-branch.outputs.branch_name }} + rocm_version: rocm724 secrets: inherit run-nightly-tests-npu: