[AMD][CI] Merge the four MI35x DeepSeek-V3.2 nightly jobs into two to save runtime (#36736)
Co-authored-by: Cursor Agent <cursoragent@cursor.com> Co-authored-by: quitenode <quitenode@users.noreply.github.com>
This commit is contained in:
Michael
co-authored by
Cursor Agent
quitenode
Cursor Agent
quitenode
parent
0132848349
commit
e283c9f8ea
@@ -63,10 +63,8 @@ on:
|
||||
- nightly-8-gpu-deepseek-v32-rocm720
|
||||
- nightly-8-gpu-deepseek-v32-mtp-rocm720
|
||||
# 8-GPU DeepSeek-V3.2 (MI35x)
|
||||
- nightly-accuracy-8-gpu-mi35x-deepseek-v32-rocm720
|
||||
- nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp-rocm720
|
||||
- nightly-perf-8-gpu-mi35x-deepseek-v32-basic-rocm720
|
||||
- nightly-perf-8-gpu-mi35x-deepseek-v32-mtp-rocm720
|
||||
- nightly-8-gpu-mi35x-deepseek-v32-rocm720
|
||||
- nightly-8-gpu-mi35x-deepseek-v32-mtp-rocm720
|
||||
# 8-GPU DeepSeek-R1 (MI35x only)
|
||||
- nightly-8-gpu-mi35x-deepseek-r1-mxfp4-rocm720
|
||||
- nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8-rocm720
|
||||
@@ -963,15 +961,19 @@ jobs:
|
||||
|
||||
# ==============================================================================
|
||||
# 8-GPU DeepSeek-V3.2 (MI35x)
|
||||
#
|
||||
# Accuracy and performance share one job per config, as the MI30x V3.2 jobs
|
||||
# above already do: both steps serve the same weights, so splitting them cost
|
||||
# a second container setup and a second cold weight load for nothing.
|
||||
# ==============================================================================
|
||||
|
||||
nightly-accuracy-8-gpu-mi35x-deepseek-v32-rocm720:
|
||||
name: ${{ format('nightly-accuracy-8-gpu-mi35x-deepseek-v32 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
|
||||
nightly-8-gpu-mi35x-deepseek-v32-rocm720:
|
||||
name: ${{ format('nightly-8-gpu-mi35x-deepseek-v32 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }}
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x-deepseek-v32-rocm720,'))
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v32-rocm720,'))
|
||||
runs-on: linux-mi35x-gpu-8
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -1006,13 +1008,24 @@ jobs:
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp-rocm720:
|
||||
name: ${{ format('nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
|
||||
- name: Performance Test MI35x ROCm 7.2 (8-GPU DeepSeek-V3.2 Basic)
|
||||
timeout-minutes: 150
|
||||
continue-on-error: true
|
||||
run: |
|
||||
> github_summary.md # Clear summary file
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-deepseek-v32-basic --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
nightly-8-gpu-mi35x-deepseek-v32-mtp-rocm720:
|
||||
name: ${{ format('nightly-8-gpu-mi35x-deepseek-v32-mtp ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }}
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp-rocm720,'))
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v32-mtp-rocm720,'))
|
||||
runs-on: linux-mi35x-gpu-8
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -1047,80 +1060,9 @@ jobs:
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
nightly-perf-8-gpu-mi35x-deepseek-v32-basic-rocm720:
|
||||
name: ${{ format('nightly-perf-8-gpu-mi35x-deepseek-v32-basic ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }}
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-8-gpu-mi35x-deepseek-v32-basic-rocm720,'))
|
||||
runs-on: linux-mi35x-gpu-8
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.sha }}
|
||||
|
||||
- name: Ensure VRAM is clear
|
||||
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
|
||||
|
||||
- name: Setup docker (${{ matrix.rocm_version }})
|
||||
run: |
|
||||
touch github_summary.md
|
||||
bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
ENABLE_CACHE_HOST: "1"
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
|
||||
# Install tabulate for run_suite.py (missing in MI35x container)
|
||||
bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate
|
||||
|
||||
- name: Performance Test MI35x ROCm 7.2 (8-GPU DeepSeek-V3.2 Basic)
|
||||
timeout-minutes: 150
|
||||
run: |
|
||||
> github_summary.md # Clear summary file
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-deepseek-v32-basic --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
nightly-perf-8-gpu-mi35x-deepseek-v32-mtp-rocm720:
|
||||
name: ${{ format('nightly-perf-8-gpu-mi35x-deepseek-v32-mtp ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }}
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-8-gpu-mi35x-deepseek-v32-mtp-rocm720,'))
|
||||
runs-on: linux-mi35x-gpu-8
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.sha }}
|
||||
|
||||
- name: Ensure VRAM is clear
|
||||
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
|
||||
|
||||
- name: Setup docker (${{ matrix.rocm_version }})
|
||||
run: |
|
||||
touch github_summary.md
|
||||
bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
ENABLE_CACHE_HOST: "1"
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
|
||||
# Install tabulate for run_suite.py (missing in MI35x container)
|
||||
bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate
|
||||
|
||||
- name: Performance Test MI35x ROCm 7.2 (8-GPU DeepSeek-V3.2 MTP)
|
||||
timeout-minutes: 180
|
||||
continue-on-error: true
|
||||
run: |
|
||||
> github_summary.md # Clear summary file
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
@@ -2358,10 +2300,8 @@ jobs:
|
||||
- nightly-8-gpu-deepseek-v32-rocm720
|
||||
- nightly-8-gpu-deepseek-v32-mtp-rocm720
|
||||
# 8-GPU DeepSeek-V3.2 (MI35x)
|
||||
- nightly-accuracy-8-gpu-mi35x-deepseek-v32-rocm720
|
||||
- nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp-rocm720
|
||||
- nightly-perf-8-gpu-mi35x-deepseek-v32-basic-rocm720
|
||||
- nightly-perf-8-gpu-mi35x-deepseek-v32-mtp-rocm720
|
||||
- nightly-8-gpu-mi35x-deepseek-v32-rocm720
|
||||
- nightly-8-gpu-mi35x-deepseek-v32-mtp-rocm720
|
||||
# 8-GPU DeepSeek-R1 (MI35x only)
|
||||
- nightly-8-gpu-mi35x-deepseek-r1-mxfp4-rocm720
|
||||
- nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8-rocm720
|
||||
|
||||
Reference in New Issue
Block a user