diff --git a/.github/workflows/nightly-test-amd-rocm720.yml b/.github/workflows/nightly-test-amd-rocm720.yml index a4845fb00..b01544daf 100644 --- a/.github/workflows/nightly-test-amd-rocm720.yml +++ b/.github/workflows/nightly-test-amd-rocm720.yml @@ -1064,7 +1064,8 @@ jobs: nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4-rocm720,')) - runs-on: linux-mi35x-gpu-8 + # TP=4 workload: runs on the dedicated 4-GPU mi35x slice, not a whole 8-GPU node. + runs-on: linux-mi35x-gpu-4 steps: - name: Checkout code uses: actions/checkout@v4 @@ -1760,7 +1761,8 @@ jobs: nightly-4-gpu-mi35x-minimax-m25-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu-mi35x-minimax-m25-rocm720,')) - runs-on: linux-mi35x-gpu-8 + # TP=4 workload: runs on the dedicated 4-GPU mi35x slice, not a whole 8-GPU node. + runs-on: linux-mi35x-gpu-4 steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/.github/workflows/nightly-test-amd.yml b/.github/workflows/nightly-test-amd.yml index f34c27257..1d09a53b3 100644 --- a/.github/workflows/nightly-test-amd.yml +++ b/.github/workflows/nightly-test-amd.yml @@ -1068,7 +1068,8 @@ jobs: nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4,')) - runs-on: linux-mi35x-gpu-8 + # TP=4 workload: runs on the dedicated 4-GPU mi35x slice, not a whole 8-GPU node. + runs-on: linux-mi35x-gpu-4 steps: - name: Checkout code uses: actions/checkout@v4 @@ -1616,7 +1617,8 @@ jobs: nightly-4-gpu-mi35x-minimax-m25: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu-mi35x-minimax-m25,')) - runs-on: linux-mi35x-gpu-8 + # TP=4 workload: runs on the dedicated 4-GPU mi35x slice, not a whole 8-GPU node. + runs-on: linux-mi35x-gpu-4 steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/.github/workflows/pr-test-amd-rocm720.yml b/.github/workflows/pr-test-amd-rocm720.yml index dd1c7f12e..0a193bbe8 100644 --- a/.github/workflows/pr-test-amd-rocm720.yml +++ b/.github/workflows/pr-test-amd-rocm720.yml @@ -48,6 +48,7 @@ on: - stage-c-test-large-8-gpu-amd-mi35x-rocm720 - stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720 - stage-c-test-4-gpu-amd-rocm720 + - stage-c-test-4-gpu-amd-mi35x-rocm720 - dsv4-flash-fp4-fp8-amd-rocm720 - dsv4-pro-fp4-amd-rocm720 target_stage: @@ -988,7 +989,48 @@ jobs: fail-fast: false matrix: runner: [linux-mi35x-gpu-8] - part: [0, 1] + part: [0] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Start CI container + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd/amd_ci_install_dependency.sh + - name: Run test + timeout-minutes: 120 + run: | + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 1 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + + # MI35x stage-c tests that only need 4 GPUs (e.g. DeepSeek-R1-MXFP4 TP=4), + # split off the 8-GPU mi35x suite onto the dedicated 4-GPU mi35x slice. + stage-c-test-4-gpu-amd-mi35x-rocm720: + needs: [check-changes] + if: | + always() && + ( + (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-c-test-4-gpu-amd-mi35x-rocm720,')) || + ( + !(inputs.target_stage || inputs.target_stage_select) && + (!failure() && !cancelled()) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + ) + strategy: + fail-fast: false + matrix: + runner: [linux-mi35x-gpu-4] + part: [0] runs-on: ${{matrix.runner}} steps: - name: Checkout code @@ -1009,7 +1051,7 @@ jobs: - name: Run test timeout-minutes: 60 run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-4-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 1 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} # =============================================== Disaggregation ==================================================== stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720: @@ -1255,6 +1297,7 @@ jobs: stage-b-test-2-gpu-large-amd-rocm720, stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720, stage-c-test-4-gpu-amd-rocm720, + stage-c-test-4-gpu-amd-mi35x-rocm720, stage-c-test-large-8-gpu-amd-rocm720, stage-c-test-large-8-gpu-amd-mi35x-rocm720, diff --git a/.github/workflows/pr-test-amd.yml b/.github/workflows/pr-test-amd.yml index a08d00967..427731703 100644 --- a/.github/workflows/pr-test-amd.yml +++ b/.github/workflows/pr-test-amd.yml @@ -35,6 +35,7 @@ on: - multimodal-gen-test-1-gpu-amd - multimodal-gen-test-2-gpu-amd - stage-c-test-4-gpu-amd + - stage-c-test-4-gpu-amd-mi35x - stage-c-test-large-8-gpu-amd - stage-c-test-large-8-gpu-amd-mi35x - stage-b-test-large-8-gpu-mi35x-disaggregation-amd @@ -1036,7 +1037,50 @@ jobs: fail-fast: false matrix: runner: [linux-mi35x-gpu-8] - part: [0, 1] + part: [0] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Start CI container + run: bash scripts/ci/amd/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd/amd_ci_install_dependency.sh + + - name: Run test + timeout-minutes: 120 + run: | + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 1 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + + # MI35x stage-c tests that only need 4 GPUs (e.g. DeepSeek-R1-MXFP4 TP=4). + # Split off the 8-GPU mi35x suite so they no longer hold a whole 8-GPU node + # and instead run on the dedicated 4-GPU mi35x slice. + stage-c-test-4-gpu-amd-mi35x: + needs: [check-changes, call-gate, wait-for-stage-b-amd] + if: | + always() && + ( + (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-c-test-4-gpu-amd-mi35x,')) || + ( + !(inputs.target_stage || inputs.target_stage_select) && + ((github.event_name == 'schedule') || (!failure() && !cancelled())) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + ) + strategy: + fail-fast: false + matrix: + runner: [linux-mi35x-gpu-4] + part: [0] runs-on: ${{matrix.runner}} steps: - name: Checkout code @@ -1058,7 +1102,7 @@ jobs: - name: Run test timeout-minutes: 60 run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-4-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 1 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} # =============================================== Disaggregation ==================================================== stage-b-test-large-8-gpu-mi35x-disaggregation-amd: @@ -1194,6 +1238,7 @@ jobs: stage-b-test-2-gpu-large-amd, stage-b-test-large-8-gpu-mi35x-disaggregation-amd, stage-c-test-4-gpu-amd, + stage-c-test-4-gpu-amd-mi35x, stage-c-test-large-8-gpu-amd, stage-c-test-large-8-gpu-amd-mi35x, ] diff --git a/test/registered/amd/test_deepseek_r1_mxfp4_8gpu.py b/test/registered/amd/test_deepseek_r1_mxfp4_4gpu.py similarity index 91% rename from test/registered/amd/test_deepseek_r1_mxfp4_8gpu.py rename to test/registered/amd/test_deepseek_r1_mxfp4_4gpu.py index e426ec694..23857582d 100644 --- a/test/registered/amd/test_deepseek_r1_mxfp4_8gpu.py +++ b/test/registered/amd/test_deepseek_r1_mxfp4_4gpu.py @@ -16,7 +16,7 @@ from sglang.test.test_utils import ( write_github_step_summary, ) -register_amd_ci(est_time=3600, suite="stage-c-test-large-8-gpu-amd-mi35x") +register_amd_ci(est_time=3600, suite="stage-c-test-4-gpu-amd-mi35x") DEEPSEEK_R1_MODEL_PATH = "amd/DeepSeek-R1-MXFP4-Preview" SERVER_LAUNCH_TIMEOUT = 1800 @@ -30,12 +30,12 @@ class TestDeepseekR1MXFP4(CustomTestCase): # Workaround: AITER custom all-gather corrupts CUDA-graph IPC buffer # registration and triggers a decode-time "Memory access fault" on - # MI35x TP=8. Disable until the AITER-side fix lands (see PR body). + # MI35x. Disable until the AITER-side fix lands (see PR body). envs.SGLANG_USE_AITER_AG.set(False) other_args = [ "--tp", - "8", + "4", "--chunked-prefill-size", "131072", "--model-loader-extra-config", @@ -90,7 +90,8 @@ class TestDeepseekR1MXFP4(CustomTestCase): write_github_step_summary( f"### test_bs_1_speed (deepseek-r1-mxfp4)\n" f"{speed=:.2f} token/s\n" ) - self.assertGreater(speed, 75) + # Report-only: the previous >75 tok/s gate was calibrated for TP=8. + # Decode throughput at TP=4 differs; re-calibrate before re-enabling. class TestDeepseekR1MXFP4MTP(CustomTestCase): @@ -105,7 +106,7 @@ class TestDeepseekR1MXFP4MTP(CustomTestCase): other_args = [ "--tp", - "8", + "4", "--chunked-prefill-size", "131072", "--speculative-algorithm", @@ -175,7 +176,8 @@ class TestDeepseekR1MXFP4MTP(CustomTestCase): f"{speed=:.2f} token/s\n" ) self.assertGreater(acc_length, 2.04) - self.assertGreater(speed, 150) + # Report-only: the previous >150 tok/s gate was calibrated for TP=8. + # Decode throughput at TP=4 differs; re-calibrate before re-enabling. if __name__ == "__main__": diff --git a/test/run_suite.py b/test/run_suite.py index de7d3247f..7d955630f 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -41,6 +41,7 @@ PER_COMMIT_SUITES = { "jit-kernel-unit-test-amd", "sgl-kernel-unit-test-2-gpu-amd", "stage-c-test-4-gpu-amd", + "stage-c-test-4-gpu-amd-mi35x", "stage-c-test-large-8-gpu-amd", "stage-c-test-large-8-gpu-amd-mi35x", # extra-a: label-gated PR opt-in suites in pr-test-amd-extra.yml