diff --git a/.github/workflows/nightly-test-amd-rocm720.yml b/.github/workflows/nightly-test-amd-rocm720.yml index b01544daf..a4845fb00 100644 --- a/.github/workflows/nightly-test-amd-rocm720.yml +++ b/.github/workflows/nightly-test-amd-rocm720.yml @@ -1064,8 +1064,7 @@ jobs: nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4-rocm720,')) - # TP=4 workload: runs on the dedicated 4-GPU mi35x slice, not a whole 8-GPU node. - runs-on: linux-mi35x-gpu-4 + runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code uses: actions/checkout@v4 @@ -1761,8 +1760,7 @@ jobs: nightly-4-gpu-mi35x-minimax-m25-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu-mi35x-minimax-m25-rocm720,')) - # TP=4 workload: runs on the dedicated 4-GPU mi35x slice, not a whole 8-GPU node. - runs-on: linux-mi35x-gpu-4 + runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/.github/workflows/nightly-test-amd.yml b/.github/workflows/nightly-test-amd.yml index 1d09a53b3..f34c27257 100644 --- a/.github/workflows/nightly-test-amd.yml +++ b/.github/workflows/nightly-test-amd.yml @@ -1068,8 +1068,7 @@ jobs: nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4,')) - # TP=4 workload: runs on the dedicated 4-GPU mi35x slice, not a whole 8-GPU node. - runs-on: linux-mi35x-gpu-4 + runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code uses: actions/checkout@v4 @@ -1617,8 +1616,7 @@ jobs: nightly-4-gpu-mi35x-minimax-m25: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu-mi35x-minimax-m25,')) - # TP=4 workload: runs on the dedicated 4-GPU mi35x slice, not a whole 8-GPU node. - runs-on: linux-mi35x-gpu-4 + runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/.github/workflows/pr-test-amd-rocm720.yml b/.github/workflows/pr-test-amd-rocm720.yml index 0a193bbe8..dd1c7f12e 100644 --- a/.github/workflows/pr-test-amd-rocm720.yml +++ b/.github/workflows/pr-test-amd-rocm720.yml @@ -48,7 +48,6 @@ on: - stage-c-test-large-8-gpu-amd-mi35x-rocm720 - stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720 - stage-c-test-4-gpu-amd-rocm720 - - stage-c-test-4-gpu-amd-mi35x-rocm720 - dsv4-flash-fp4-fp8-amd-rocm720 - dsv4-pro-fp4-amd-rocm720 target_stage: @@ -989,48 +988,7 @@ jobs: fail-fast: false matrix: runner: [linux-mi35x-gpu-8] - part: [0] - runs-on: ${{matrix.runner}} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - name: Run test - timeout-minutes: 120 - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 1 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - - # MI35x stage-c tests that only need 4 GPUs (e.g. DeepSeek-R1-MXFP4 TP=4), - # split off the 8-GPU mi35x suite onto the dedicated 4-GPU mi35x slice. - stage-c-test-4-gpu-amd-mi35x-rocm720: - needs: [check-changes] - if: | - always() && - ( - (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-c-test-4-gpu-amd-mi35x-rocm720,')) || - ( - !(inputs.target_stage || inputs.target_stage_select) && - (!failure() && !cancelled()) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - strategy: - fail-fast: false - matrix: - runner: [linux-mi35x-gpu-4] - part: [0] + part: [0, 1] runs-on: ${{matrix.runner}} steps: - name: Checkout code @@ -1051,7 +1009,7 @@ jobs: - name: Run test timeout-minutes: 60 run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-4-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 1 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} # =============================================== Disaggregation ==================================================== stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720: @@ -1297,7 +1255,6 @@ jobs: stage-b-test-2-gpu-large-amd-rocm720, stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720, stage-c-test-4-gpu-amd-rocm720, - stage-c-test-4-gpu-amd-mi35x-rocm720, stage-c-test-large-8-gpu-amd-rocm720, stage-c-test-large-8-gpu-amd-mi35x-rocm720, diff --git a/.github/workflows/pr-test-amd.yml b/.github/workflows/pr-test-amd.yml index 427731703..a08d00967 100644 --- a/.github/workflows/pr-test-amd.yml +++ b/.github/workflows/pr-test-amd.yml @@ -35,7 +35,6 @@ on: - multimodal-gen-test-1-gpu-amd - multimodal-gen-test-2-gpu-amd - stage-c-test-4-gpu-amd - - stage-c-test-4-gpu-amd-mi35x - stage-c-test-large-8-gpu-amd - stage-c-test-large-8-gpu-amd-mi35x - stage-b-test-large-8-gpu-mi35x-disaggregation-amd @@ -1037,50 +1036,7 @@ jobs: fail-fast: false matrix: runner: [linux-mi35x-gpu-8] - part: [0] - runs-on: ${{matrix.runner}} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Run test - timeout-minutes: 120 - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 1 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - - # MI35x stage-c tests that only need 4 GPUs (e.g. DeepSeek-R1-MXFP4 TP=4). - # Split off the 8-GPU mi35x suite so they no longer hold a whole 8-GPU node - # and instead run on the dedicated 4-GPU mi35x slice. - stage-c-test-4-gpu-amd-mi35x: - needs: [check-changes, call-gate, wait-for-stage-b-amd] - if: | - always() && - ( - (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-c-test-4-gpu-amd-mi35x,')) || - ( - !(inputs.target_stage || inputs.target_stage_select) && - ((github.event_name == 'schedule') || (!failure() && !cancelled())) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - strategy: - fail-fast: false - matrix: - runner: [linux-mi35x-gpu-4] - part: [0] + part: [0, 1] runs-on: ${{matrix.runner}} steps: - name: Checkout code @@ -1102,7 +1058,7 @@ jobs: - name: Run test timeout-minutes: 60 run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-4-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 1 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} # =============================================== Disaggregation ==================================================== stage-b-test-large-8-gpu-mi35x-disaggregation-amd: @@ -1238,7 +1194,6 @@ jobs: stage-b-test-2-gpu-large-amd, stage-b-test-large-8-gpu-mi35x-disaggregation-amd, stage-c-test-4-gpu-amd, - stage-c-test-4-gpu-amd-mi35x, stage-c-test-large-8-gpu-amd, stage-c-test-large-8-gpu-amd-mi35x, ] diff --git a/test/registered/amd/test_deepseek_r1_mxfp4_4gpu.py b/test/registered/amd/test_deepseek_r1_mxfp4_8gpu.py similarity index 91% rename from test/registered/amd/test_deepseek_r1_mxfp4_4gpu.py rename to test/registered/amd/test_deepseek_r1_mxfp4_8gpu.py index 23857582d..e426ec694 100644 --- a/test/registered/amd/test_deepseek_r1_mxfp4_4gpu.py +++ b/test/registered/amd/test_deepseek_r1_mxfp4_8gpu.py @@ -16,7 +16,7 @@ from sglang.test.test_utils import ( write_github_step_summary, ) -register_amd_ci(est_time=3600, suite="stage-c-test-4-gpu-amd-mi35x") +register_amd_ci(est_time=3600, suite="stage-c-test-large-8-gpu-amd-mi35x") DEEPSEEK_R1_MODEL_PATH = "amd/DeepSeek-R1-MXFP4-Preview" SERVER_LAUNCH_TIMEOUT = 1800 @@ -30,12 +30,12 @@ class TestDeepseekR1MXFP4(CustomTestCase): # Workaround: AITER custom all-gather corrupts CUDA-graph IPC buffer # registration and triggers a decode-time "Memory access fault" on - # MI35x. Disable until the AITER-side fix lands (see PR body). + # MI35x TP=8. Disable until the AITER-side fix lands (see PR body). envs.SGLANG_USE_AITER_AG.set(False) other_args = [ "--tp", - "4", + "8", "--chunked-prefill-size", "131072", "--model-loader-extra-config", @@ -90,8 +90,7 @@ class TestDeepseekR1MXFP4(CustomTestCase): write_github_step_summary( f"### test_bs_1_speed (deepseek-r1-mxfp4)\n" f"{speed=:.2f} token/s\n" ) - # Report-only: the previous >75 tok/s gate was calibrated for TP=8. - # Decode throughput at TP=4 differs; re-calibrate before re-enabling. + self.assertGreater(speed, 75) class TestDeepseekR1MXFP4MTP(CustomTestCase): @@ -106,7 +105,7 @@ class TestDeepseekR1MXFP4MTP(CustomTestCase): other_args = [ "--tp", - "4", + "8", "--chunked-prefill-size", "131072", "--speculative-algorithm", @@ -176,8 +175,7 @@ class TestDeepseekR1MXFP4MTP(CustomTestCase): f"{speed=:.2f} token/s\n" ) self.assertGreater(acc_length, 2.04) - # Report-only: the previous >150 tok/s gate was calibrated for TP=8. - # Decode throughput at TP=4 differs; re-calibrate before re-enabling. + self.assertGreater(speed, 150) if __name__ == "__main__": diff --git a/test/run_suite.py b/test/run_suite.py index 7d955630f..de7d3247f 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -41,7 +41,6 @@ PER_COMMIT_SUITES = { "jit-kernel-unit-test-amd", "sgl-kernel-unit-test-2-gpu-amd", "stage-c-test-4-gpu-amd", - "stage-c-test-4-gpu-amd-mi35x", "stage-c-test-large-8-gpu-amd", "stage-c-test-large-8-gpu-amd-mi35x", # extra-a: label-gated PR opt-in suites in pr-test-amd-extra.yml