ci: add 4-GPU mi35x runner and rebalance off the saturated 8-GPU pool (#28745)
Co-authored-by: michaelzhang-ai <michaelzhang-ai@users.noreply.github.com>
This commit is contained in:
Michael
co-authored by
michaelzhang-ai
michaelzhang-ai
parent
3a574846ff
commit
13aab2fc06
@@ -1064,7 +1064,8 @@ jobs:
|
||||
|
||||
nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4-rocm720:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4-rocm720,'))
|
||||
runs-on: linux-mi35x-gpu-8
|
||||
# TP=4 workload: runs on the dedicated 4-GPU mi35x slice, not a whole 8-GPU node.
|
||||
runs-on: linux-mi35x-gpu-4
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
@@ -1760,7 +1761,8 @@ jobs:
|
||||
|
||||
nightly-4-gpu-mi35x-minimax-m25-rocm720:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu-mi35x-minimax-m25-rocm720,'))
|
||||
runs-on: linux-mi35x-gpu-8
|
||||
# TP=4 workload: runs on the dedicated 4-GPU mi35x slice, not a whole 8-GPU node.
|
||||
runs-on: linux-mi35x-gpu-4
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
@@ -1068,7 +1068,8 @@ jobs:
|
||||
|
||||
nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4,'))
|
||||
runs-on: linux-mi35x-gpu-8
|
||||
# TP=4 workload: runs on the dedicated 4-GPU mi35x slice, not a whole 8-GPU node.
|
||||
runs-on: linux-mi35x-gpu-4
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
@@ -1616,7 +1617,8 @@ jobs:
|
||||
|
||||
nightly-4-gpu-mi35x-minimax-m25:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu-mi35x-minimax-m25,'))
|
||||
runs-on: linux-mi35x-gpu-8
|
||||
# TP=4 workload: runs on the dedicated 4-GPU mi35x slice, not a whole 8-GPU node.
|
||||
runs-on: linux-mi35x-gpu-4
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
@@ -48,6 +48,7 @@ on:
|
||||
- stage-c-test-large-8-gpu-amd-mi35x-rocm720
|
||||
- stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720
|
||||
- stage-c-test-4-gpu-amd-rocm720
|
||||
- stage-c-test-4-gpu-amd-mi35x-rocm720
|
||||
- dsv4-flash-fp4-fp8-amd-rocm720
|
||||
- dsv4-pro-fp4-amd-rocm720
|
||||
target_stage:
|
||||
@@ -988,7 +989,48 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
runner: [linux-mi35x-gpu-8]
|
||||
part: [0, 1]
|
||||
part: [0]
|
||||
runs-on: ${{matrix.runner}}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }}
|
||||
|
||||
- name: Ensure VRAM is clear
|
||||
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
|
||||
|
||||
- name: Start CI container
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: bash scripts/ci/amd/amd_ci_install_dependency.sh
|
||||
- name: Run test
|
||||
timeout-minutes: 120
|
||||
run: |
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 1 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
# MI35x stage-c tests that only need 4 GPUs (e.g. DeepSeek-R1-MXFP4 TP=4),
|
||||
# split off the 8-GPU mi35x suite onto the dedicated 4-GPU mi35x slice.
|
||||
stage-c-test-4-gpu-amd-mi35x-rocm720:
|
||||
needs: [check-changes]
|
||||
if: |
|
||||
always() &&
|
||||
(
|
||||
(contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-c-test-4-gpu-amd-mi35x-rocm720,')) ||
|
||||
(
|
||||
!(inputs.target_stage || inputs.target_stage_select) &&
|
||||
(!failure() && !cancelled()) &&
|
||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||
)
|
||||
)
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
runner: [linux-mi35x-gpu-4]
|
||||
part: [0]
|
||||
runs-on: ${{matrix.runner}}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -1009,7 +1051,7 @@ jobs:
|
||||
- name: Run test
|
||||
timeout-minutes: 60
|
||||
run: |
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-4-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 1 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
# =============================================== Disaggregation ====================================================
|
||||
stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720:
|
||||
@@ -1255,6 +1297,7 @@ jobs:
|
||||
stage-b-test-2-gpu-large-amd-rocm720,
|
||||
stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720,
|
||||
stage-c-test-4-gpu-amd-rocm720,
|
||||
stage-c-test-4-gpu-amd-mi35x-rocm720,
|
||||
stage-c-test-large-8-gpu-amd-rocm720,
|
||||
stage-c-test-large-8-gpu-amd-mi35x-rocm720,
|
||||
|
||||
|
||||
@@ -35,6 +35,7 @@ on:
|
||||
- multimodal-gen-test-1-gpu-amd
|
||||
- multimodal-gen-test-2-gpu-amd
|
||||
- stage-c-test-4-gpu-amd
|
||||
- stage-c-test-4-gpu-amd-mi35x
|
||||
- stage-c-test-large-8-gpu-amd
|
||||
- stage-c-test-large-8-gpu-amd-mi35x
|
||||
- stage-b-test-large-8-gpu-mi35x-disaggregation-amd
|
||||
@@ -1036,7 +1037,50 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
runner: [linux-mi35x-gpu-8]
|
||||
part: [0, 1]
|
||||
part: [0]
|
||||
runs-on: ${{matrix.runner}}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }}
|
||||
|
||||
- name: Ensure VRAM is clear
|
||||
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
|
||||
|
||||
- name: Start CI container
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: bash scripts/ci/amd/amd_ci_install_dependency.sh
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 120
|
||||
run: |
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 1 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
# MI35x stage-c tests that only need 4 GPUs (e.g. DeepSeek-R1-MXFP4 TP=4).
|
||||
# Split off the 8-GPU mi35x suite so they no longer hold a whole 8-GPU node
|
||||
# and instead run on the dedicated 4-GPU mi35x slice.
|
||||
stage-c-test-4-gpu-amd-mi35x:
|
||||
needs: [check-changes, call-gate, wait-for-stage-b-amd]
|
||||
if: |
|
||||
always() &&
|
||||
(
|
||||
(contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-c-test-4-gpu-amd-mi35x,')) ||
|
||||
(
|
||||
!(inputs.target_stage || inputs.target_stage_select) &&
|
||||
((github.event_name == 'schedule') || (!failure() && !cancelled())) &&
|
||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||
)
|
||||
)
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
runner: [linux-mi35x-gpu-4]
|
||||
part: [0]
|
||||
runs-on: ${{matrix.runner}}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -1058,7 +1102,7 @@ jobs:
|
||||
- name: Run test
|
||||
timeout-minutes: 60
|
||||
run: |
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-4-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 1 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
# =============================================== Disaggregation ====================================================
|
||||
stage-b-test-large-8-gpu-mi35x-disaggregation-amd:
|
||||
@@ -1194,6 +1238,7 @@ jobs:
|
||||
stage-b-test-2-gpu-large-amd,
|
||||
stage-b-test-large-8-gpu-mi35x-disaggregation-amd,
|
||||
stage-c-test-4-gpu-amd,
|
||||
stage-c-test-4-gpu-amd-mi35x,
|
||||
stage-c-test-large-8-gpu-amd,
|
||||
stage-c-test-large-8-gpu-amd-mi35x,
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user