diff --git a/.github/workflows/nightly-test-amd-rocm720.yml b/.github/workflows/nightly-test-amd-rocm720.yml index 129d0d0fd..fe0b20b6a 100644 --- a/.github/workflows/nightly-test-amd-rocm720.yml +++ b/.github/workflows/nightly-test-amd-rocm720.yml @@ -63,10 +63,8 @@ on: - nightly-8-gpu-deepseek-v32-rocm720 - nightly-8-gpu-deepseek-v32-mtp-rocm720 # 8-GPU DeepSeek-V3.2 (MI35x) - - nightly-accuracy-8-gpu-mi35x-deepseek-v32-rocm720 - - nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp-rocm720 - - nightly-perf-8-gpu-mi35x-deepseek-v32-basic-rocm720 - - nightly-perf-8-gpu-mi35x-deepseek-v32-mtp-rocm720 + - nightly-8-gpu-mi35x-deepseek-v32-rocm720 + - nightly-8-gpu-mi35x-deepseek-v32-mtp-rocm720 # 8-GPU DeepSeek-R1 (MI35x only) - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-rocm720 - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8-rocm720 @@ -963,15 +961,19 @@ jobs: # ============================================================================== # 8-GPU DeepSeek-V3.2 (MI35x) + # + # Accuracy and performance share one job per config, as the MI30x V3.2 jobs + # above already do: both steps serve the same weights, so splitting them cost + # a second container setup and a second cold weight load for nothing. # ============================================================================== - nightly-accuracy-8-gpu-mi35x-deepseek-v32-rocm720: - name: ${{ format('nightly-accuracy-8-gpu-mi35x-deepseek-v32 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} + nightly-8-gpu-mi35x-deepseek-v32-rocm720: + name: ${{ format('nightly-8-gpu-mi35x-deepseek-v32 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x-deepseek-v32-rocm720,')) + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v32-rocm720,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code @@ -1006,13 +1008,24 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp-rocm720: - name: ${{ format('nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} + - name: Performance Test MI35x ROCm 7.2 (8-GPU DeepSeek-V3.2 Basic) + timeout-minutes: 150 + continue-on-error: true + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-deepseek-v32-basic --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + nightly-8-gpu-mi35x-deepseek-v32-mtp-rocm720: + name: ${{ format('nightly-8-gpu-mi35x-deepseek-v32-mtp ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp-rocm720,')) + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v32-mtp-rocm720,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code @@ -1047,80 +1060,9 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - nightly-perf-8-gpu-mi35x-deepseek-v32-basic-rocm720: - name: ${{ format('nightly-perf-8-gpu-mi35x-deepseek-v32-basic ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-8-gpu-mi35x-deepseek-v32-basic-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Performance Test MI35x ROCm 7.2 (8-GPU DeepSeek-V3.2 Basic) - timeout-minutes: 150 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-deepseek-v32-basic --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - nightly-perf-8-gpu-mi35x-deepseek-v32-mtp-rocm720: - name: ${{ format('nightly-perf-8-gpu-mi35x-deepseek-v32-mtp ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-8-gpu-mi35x-deepseek-v32-mtp-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - name: Performance Test MI35x ROCm 7.2 (8-GPU DeepSeek-V3.2 MTP) timeout-minutes: 180 + continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ @@ -2358,10 +2300,8 @@ jobs: - nightly-8-gpu-deepseek-v32-rocm720 - nightly-8-gpu-deepseek-v32-mtp-rocm720 # 8-GPU DeepSeek-V3.2 (MI35x) - - nightly-accuracy-8-gpu-mi35x-deepseek-v32-rocm720 - - nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp-rocm720 - - nightly-perf-8-gpu-mi35x-deepseek-v32-basic-rocm720 - - nightly-perf-8-gpu-mi35x-deepseek-v32-mtp-rocm720 + - nightly-8-gpu-mi35x-deepseek-v32-rocm720 + - nightly-8-gpu-mi35x-deepseek-v32-mtp-rocm720 # 8-GPU DeepSeek-R1 (MI35x only) - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-rocm720 - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8-rocm720