[AMD][CI] Add the Qwen3.8 MXFP4 MI35x nightly (#35383)
This commit is contained in:
Michael
@@ -84,9 +84,8 @@ on:
|
||||
- nightly-8-gpu-kimi-k26-rocm720
|
||||
# 8-GPU Kimi-K3 (MI35x only - native MXFP4 needs gfx95x)
|
||||
- nightly-8-gpu-mi35x-kimi-k3-rocm720
|
||||
# 8-GPU Qwen3-235B (MI30x + MI35x MXFP4)
|
||||
- nightly-8-gpu-qwen3-235b-rocm720
|
||||
- nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720
|
||||
# 8-GPU Qwen3.8-MXFP4 (MI35x only - MXFP4 needs gfx95x)
|
||||
- nightly-8-gpu-mi35x-qwen38-mxfp4-rocm720
|
||||
# 8-GPU Qwen 3.5 (MI30x + MI35x)
|
||||
- nightly-8-gpu-qwen35-rocm720
|
||||
- nightly-8-gpu-mi35x-qwen35-rocm720
|
||||
@@ -1754,87 +1753,6 @@ jobs:
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# ==============================================================================
|
||||
# 8-GPU Qwen3-235B (MI30x + MI35x MXFP4)
|
||||
# ==============================================================================
|
||||
|
||||
nightly-8-gpu-qwen3-235b-rocm720:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }}
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-qwen3-235b-rocm720,'))
|
||||
runs-on: linux-mi300-8gpu-sglang
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.sha }}
|
||||
|
||||
- name: Ensure VRAM is clear
|
||||
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
|
||||
|
||||
- name: Setup docker (ROCm 7.2)
|
||||
run: |
|
||||
touch github_summary.md
|
||||
bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
ENABLE_CACHE_HOST: "1"
|
||||
|
||||
- name: Install dependencies
|
||||
run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
|
||||
|
||||
- name: Accuracy Test + Performance Test ROCm 7.2 (8-GPU Qwen3)
|
||||
timeout-minutes: 120
|
||||
run: |
|
||||
> github_summary.md # Clear summary file
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-8-gpu-qwen3-235b --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }}
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720,'))
|
||||
runs-on: linux-mi35x-gpu-8
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.sha }}
|
||||
|
||||
- name: Ensure VRAM is clear
|
||||
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
|
||||
|
||||
- name: Setup docker (ROCm 7.2)
|
||||
run: |
|
||||
touch github_summary.md
|
||||
bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
|
||||
# Install tabulate for run_suite.py (missing in MI35x container)
|
||||
bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate
|
||||
|
||||
- name: Accuracy Test + Performance Test MI35x ROCm 7.2 (8-GPU Qwen3-235B-MXFP4)
|
||||
timeout-minutes: 120
|
||||
run: |
|
||||
> github_summary.md # Clear summary file
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-8-gpu-mi35x-qwen3-235b-mxfp4 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# ==============================================================================
|
||||
# 8-GPU Qwen 3.5 (MI30x + MI35x)
|
||||
# ==============================================================================
|
||||
@@ -1980,6 +1898,63 @@ jobs:
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# ==============================================================================
|
||||
# 8-GPU Qwen3.8-MXFP4 (MI35x only - MXFP4 needs gfx95x)
|
||||
# ==============================================================================
|
||||
|
||||
nightly-8-gpu-mi35x-qwen38-mxfp4-rocm720:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }}
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen38-mxfp4-rocm720,'))
|
||||
runs-on: linux-mi35x-gpu-8
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.sha }}
|
||||
|
||||
- name: Ensure VRAM is clear
|
||||
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
|
||||
|
||||
- name: Setup docker (ROCm 7.2)
|
||||
run: |
|
||||
touch github_summary.md
|
||||
bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
ENABLE_CACHE_HOST: "1"
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
|
||||
# Install tabulate for run_suite.py (missing in MI35x container)
|
||||
bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate
|
||||
|
||||
# ~1.2 TB of weights against a fleet-shared cache that runs at 100% used.
|
||||
# Run 32196787596 died 40 minutes into the download on ENOSPC with no
|
||||
# free-space figure anywhere in its log; record one up front instead.
|
||||
- name: Check model cache space
|
||||
run: |
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout \
|
||||
bash scripts/ci/amd/check_hf_cache_space.sh \
|
||||
amd/Qwen3.8-2.4T-A95B-Quark-MXFP4 1250
|
||||
|
||||
# One step, not the usual accuracy/perf pair: both phases live in the
|
||||
# same suite so they launch from one server config. The 2.4T MXFP4
|
||||
# checkpoint is ~1.2 TB and is loaded once per phase, which dominates
|
||||
# wall time well before either the eval or the benchmark starts.
|
||||
- name: Accuracy Test + Performance Test MI35x ROCm 7.2 (8-GPU Qwen3.8-MXFP4)
|
||||
timeout-minutes: 420
|
||||
run: |
|
||||
> github_summary.md # Clear summary file
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-qwen38-mxfp4 --nightly --timeout-per-file 21600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# ==============================================================================
|
||||
# 8-GPU GLM-5.1 (MI30x + MI35x)
|
||||
# ==============================================================================
|
||||
@@ -2382,9 +2357,8 @@ jobs:
|
||||
- nightly-8-gpu-kimi-k26-rocm720
|
||||
# 8-GPU Kimi-K3 (MI35x only - native MXFP4 needs gfx95x)
|
||||
- nightly-8-gpu-mi35x-kimi-k3-rocm720
|
||||
# 8-GPU Qwen3-235B (MI30x + MI35x MXFP4)
|
||||
- nightly-8-gpu-qwen3-235b-rocm720
|
||||
- nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720
|
||||
# 8-GPU Qwen3.8-MXFP4 (MI35x only - MXFP4 needs gfx95x)
|
||||
- nightly-8-gpu-mi35x-qwen38-mxfp4-rocm720
|
||||
# 8-GPU Qwen 3.5 (MI30x + MI35x)
|
||||
- nightly-8-gpu-qwen35-rocm720
|
||||
- nightly-8-gpu-mi35x-qwen35-rocm720
|
||||
|
||||
Reference in New Issue
Block a user