minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[AMD] [CI] Register MI35x GSM8K nightly tests (#26478)

Browse Source 
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Bingxu Chen
2026-05-28 10:31:43 +08:00
committed by GitHub
co-authored by Cursor
parent 14c1bb2721
commit 81663cb5f1
4 changed files with 214 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/nightly-test-amd-rocm720.yml
+80 -6
View File
@@ -30,12 +30,14 @@ on:
# 1-GPU Unit Tests (MI30x + MI35x) # 1-GPU Unit Tests (MI30x + MI35x)
- nightly-test-1-gpu-unit-rocm720 - nightly-test-1-gpu-unit-rocm720
- nightly-test-1-gpu-mi35x-rocm720 - nightly-test-1-gpu-mi35x-rocm720
# 2-GPU and 4-GPU Tests (MI30x) # 2-GPU and 4-GPU Tests (MI30x + MI35x)
- nightly-accuracy-2-gpu-rocm720 - nightly-accuracy-2-gpu-rocm720
- nightly-accuracy-2-gpu-vlm-rocm720 - nightly-accuracy-2-gpu-vlm-rocm720
- nightly-perf-2-gpu-text-rocm720 - nightly-perf-2-gpu-text-rocm720
- nightly-perf-2-gpu-vlm-rocm720 - nightly-perf-2-gpu-vlm-rocm720
- nightly-4-gpu-rocm720 - nightly-4-gpu-rocm720
# 2-GPU GLM-5.1-MXFP4 (MI35x only)
- nightly-2-gpu-mi35x-glm51-mxfp4-rocm720
# 8-GPU GPT-OSS (MI30x mixes Grok1-FP8; MI35x mixes Qwen3-Coder-Next) # 8-GPU GPT-OSS (MI30x mixes Grok1-FP8; MI35x mixes Qwen3-Coder-Next)
- nightly-accuracy-8-gpu-rocm720 - nightly-accuracy-8-gpu-rocm720
- nightly-accuracy-8-gpu-mi35x-rocm720 - nightly-accuracy-8-gpu-mi35x-rocm720
@@ -55,10 +57,11 @@ on:
- nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp-rocm720 - nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp-rocm720
- nightly-perf-8-gpu-mi35x-deepseek-v32-basic-rocm720 - nightly-perf-8-gpu-mi35x-deepseek-v32-basic-rocm720
- nightly-perf-8-gpu-mi35x-deepseek-v32-mtp-rocm720 - nightly-perf-8-gpu-mi35x-deepseek-v32-mtp-rocm720
# 8-GPU DeepSeek-R1-MXFP4 (MI35x only) # 8-GPU DeepSeek-R1 (MI35x only)
- nightly-8-gpu-mi35x-deepseek-r1-mxfp4-rocm720 - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-rocm720
- nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8-rocm720 - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8-rocm720
- nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion-rocm720 - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion-rocm720
- nightly-8-gpu-mi35x-deepseek-r1-hicache-rocm720
# 8-GPU DeepSeek-V4 (MI35x only) # 8-GPU DeepSeek-V4 (MI35x only)
- nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720 - nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720
- nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720 - nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720
@@ -191,7 +194,7 @@ jobs:
exit ${TEST_EXIT_CODE:-0} exit ${TEST_EXIT_CODE:-0}
# ============================================================================== # ==============================================================================
# 2-GPU and 4-GPU Tests (MI30x) # 2-GPU and 4-GPU Tests (MI30x + MI35x)
# ============================================================================== # ==============================================================================
nightly-accuracy-2-gpu-rocm720: nightly-accuracy-2-gpu-rocm720:
@@ -224,6 +227,41 @@ jobs:
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0} exit ${TEST_EXIT_CODE:-0}
nightly-2-gpu-mi35x-glm51-mxfp4-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-2-gpu-mi35x-glm51-mxfp4-rocm720,'))
runs-on: linux-mi35x-gpu-2
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.sha }}
- name: Ensure VRAM is clear
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Setup docker (ROCm 7.2)
run: |
touch github_summary.md
bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: Install dependencies
run: |
bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate
bash scripts/ci/amd/amd_ci_exec.sh pip install git+https://github.com/huggingface/transformers.git@96f807a33b75
- name: Accuracy Test MI35x ROCm 7.2 (2-GPU GLM-5.1-MXFP4 GSM8K)
timeout-minutes: 120
run: |
> github_summary.md
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-amd-2-gpu-mi35x-glm51-mxfp4 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
nightly-accuracy-2-gpu-vlm-rocm720: nightly-accuracy-2-gpu-vlm-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-2-gpu-vlm-rocm720,')) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-2-gpu-vlm-rocm720,'))
runs-on: linux-mi325-2gpu-sglang runs-on: linux-mi325-2gpu-sglang
@@ -936,7 +974,7 @@ jobs:
exit ${TEST_EXIT_CODE:-0} exit ${TEST_EXIT_CODE:-0}
# ============================================================================== # ==============================================================================
# 8-GPU DeepSeek-R1-MXFP4 (MI35x only) # 8-GPU DeepSeek-R1 (MI35x only)
# ============================================================================== # ==============================================================================
nightly-8-gpu-mi35x-deepseek-r1-mxfp4-rocm720: nightly-8-gpu-mi35x-deepseek-r1-mxfp4-rocm720:
@@ -1077,6 +1115,40 @@ jobs:
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0} exit ${TEST_EXIT_CODE:-0}
nightly-8-gpu-mi35x-deepseek-r1-hicache-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-hicache-rocm720,'))
runs-on: linux-mi35x-gpu-8
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.sha }}
- name: Ensure VRAM is clear
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Setup docker (ROCm 7.2)
run: |
touch github_summary.md
bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: Install dependencies
run: |
bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate
- name: Accuracy Test MI35x ROCm 7.2 (8-GPU DeepSeek-R1 HiCache)
timeout-minutes: 180
run: |
> github_summary.md
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-r1-hicache --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
# ============================================================================== # ==============================================================================
# 8-GPU DeepSeek-V4 (MI35x only) # 8-GPU DeepSeek-V4 (MI35x only)
# ============================================================================== # ==============================================================================
@@ -1693,12 +1765,13 @@ jobs:
# 1-GPU Unit Tests (MI30x + MI35x) # 1-GPU Unit Tests (MI30x + MI35x)
- nightly-test-1-gpu-unit-rocm720 - nightly-test-1-gpu-unit-rocm720
- nightly-test-1-gpu-mi35x-rocm720 - nightly-test-1-gpu-mi35x-rocm720
# 2-GPU and 4-GPU Tests (MI30x) # 2-GPU and 4-GPU Tests (MI30x + MI35x)
- nightly-accuracy-2-gpu-rocm720 - nightly-accuracy-2-gpu-rocm720
- nightly-accuracy-2-gpu-vlm-rocm720 - nightly-accuracy-2-gpu-vlm-rocm720
- nightly-perf-2-gpu-text-rocm720 - nightly-perf-2-gpu-text-rocm720
- nightly-perf-2-gpu-vlm-rocm720 - nightly-perf-2-gpu-vlm-rocm720
- nightly-4-gpu-rocm720 - nightly-4-gpu-rocm720
- nightly-2-gpu-mi35x-glm51-mxfp4-rocm720
# 8-GPU GPT-OSS (MI30x mixes Grok1-FP8; MI35x mixes Qwen3-Coder-Next) # 8-GPU GPT-OSS (MI30x mixes Grok1-FP8; MI35x mixes Qwen3-Coder-Next)
- nightly-accuracy-8-gpu-rocm720 - nightly-accuracy-8-gpu-rocm720
- nightly-accuracy-8-gpu-mi35x-rocm720 - nightly-accuracy-8-gpu-mi35x-rocm720
@@ -1718,10 +1791,11 @@ jobs:
- nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp-rocm720 - nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp-rocm720
- nightly-perf-8-gpu-mi35x-deepseek-v32-basic-rocm720 - nightly-perf-8-gpu-mi35x-deepseek-v32-basic-rocm720
- nightly-perf-8-gpu-mi35x-deepseek-v32-mtp-rocm720 - nightly-perf-8-gpu-mi35x-deepseek-v32-mtp-rocm720
# 8-GPU DeepSeek-R1-MXFP4 (MI35x only) # 8-GPU DeepSeek-R1 (MI35x only)
- nightly-8-gpu-mi35x-deepseek-r1-mxfp4-rocm720 - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-rocm720
- nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8-rocm720 - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8-rocm720
- nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion-rocm720 - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion-rocm720
- nightly-8-gpu-mi35x-deepseek-r1-hicache-rocm720
# 8-GPU DeepSeek-V4 (MI35x only) # 8-GPU DeepSeek-V4 (MI35x only)
- nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720 - nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720
- nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720 - nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720
.github/workflows/nightly-test-amd.yml
+80 -6
View File
@@ -30,12 +30,14 @@ on:
# 1-GPU Unit Tests (MI30x + MI35x) # 1-GPU Unit Tests (MI30x + MI35x)
- nightly-test-1-gpu-unit - nightly-test-1-gpu-unit
- nightly-test-1-gpu-mi35x - nightly-test-1-gpu-mi35x
# 2-GPU and 4-GPU Tests (MI30x) # 2-GPU and 4-GPU Tests (MI30x + MI35x)
- nightly-accuracy-2-gpu - nightly-accuracy-2-gpu
- nightly-accuracy-2-gpu-vlm - nightly-accuracy-2-gpu-vlm
- nightly-perf-2-gpu-text - nightly-perf-2-gpu-text
- nightly-perf-2-gpu-vlm - nightly-perf-2-gpu-vlm
- nightly-4-gpu - nightly-4-gpu
# 2-GPU GLM-5.1-MXFP4 (MI35x only)
- nightly-2-gpu-mi35x-glm51-mxfp4
# 8-GPU GPT-OSS (MI30x mixes Grok1-FP8; MI35x mixes Qwen3-Coder-Next) # 8-GPU GPT-OSS (MI30x mixes Grok1-FP8; MI35x mixes Qwen3-Coder-Next)
- nightly-accuracy-8-gpu - nightly-accuracy-8-gpu
- nightly-accuracy-8-gpu-mi35x - nightly-accuracy-8-gpu-mi35x
@@ -55,10 +57,11 @@ on:
- nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp - nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp
- nightly-perf-8-gpu-mi35x-deepseek-v32-basic - nightly-perf-8-gpu-mi35x-deepseek-v32-basic
- nightly-perf-8-gpu-mi35x-deepseek-v32-mtp - nightly-perf-8-gpu-mi35x-deepseek-v32-mtp
# 8-GPU DeepSeek-R1-MXFP4 (MI35x only) # 8-GPU DeepSeek-R1 (MI35x only)
- nightly-8-gpu-mi35x-deepseek-r1-mxfp4 - nightly-8-gpu-mi35x-deepseek-r1-mxfp4
- nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8 - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8
- nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion
- nightly-8-gpu-mi35x-deepseek-r1-hicache
# 8-GPU Kimi-K2.6 (MI30x + MI35x) # 8-GPU Kimi-K2.6 (MI30x + MI35x)
- nightly-8-gpu-kimi-k26 - nightly-8-gpu-kimi-k26
- nightly-8-gpu-mi35x-kimi-k26 - nightly-8-gpu-mi35x-kimi-k26
@@ -192,7 +195,7 @@ jobs:
exit ${TEST_EXIT_CODE:-0} exit ${TEST_EXIT_CODE:-0}
# ============================================================================== # ==============================================================================
# 2-GPU and 4-GPU Tests (MI30x) # 2-GPU and 4-GPU Tests (MI30x + MI35x)
# ============================================================================== # ==============================================================================
nightly-accuracy-2-gpu: nightly-accuracy-2-gpu:
@@ -226,6 +229,41 @@ jobs:
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0} exit ${TEST_EXIT_CODE:-0}
nightly-2-gpu-mi35x-glm51-mxfp4:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-2-gpu-mi35x-glm51-mxfp4,'))
runs-on: linux-mi35x-gpu-2
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.sha }}
- name: Ensure VRAM is clear
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Setup docker
run: |
touch github_summary.md
bash scripts/ci/amd/amd_ci_start_container.sh
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: Install dependencies
run: |
bash scripts/ci/amd/amd_ci_install_dependency.sh
bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate
bash scripts/ci/amd/amd_ci_exec.sh pip install git+https://github.com/huggingface/transformers.git@96f807a33b75
- name: Accuracy Test MI35x (2-GPU GLM-5.1-MXFP4 GSM8K)
timeout-minutes: 120
run: |
> github_summary.md
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-amd-2-gpu-mi35x-glm51-mxfp4 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
nightly-accuracy-2-gpu-vlm: nightly-accuracy-2-gpu-vlm:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-2-gpu-vlm,')) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-2-gpu-vlm,'))
runs-on: linux-mi325-2gpu-sglang runs-on: linux-mi325-2gpu-sglang
@@ -941,7 +979,7 @@ jobs:
exit ${TEST_EXIT_CODE:-0} exit ${TEST_EXIT_CODE:-0}
# ============================================================================== # ==============================================================================
# 8-GPU DeepSeek-R1-MXFP4 (MI35x only) # 8-GPU DeepSeek-R1 (MI35x only)
# ============================================================================== # ==============================================================================
nightly-8-gpu-mi35x-deepseek-r1-mxfp4: nightly-8-gpu-mi35x-deepseek-r1-mxfp4:
@@ -1082,6 +1120,40 @@ jobs:
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0} exit ${TEST_EXIT_CODE:-0}
nightly-8-gpu-mi35x-deepseek-r1-hicache:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-hicache,'))
runs-on: linux-mi35x-gpu-8
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.sha }}
- name: Ensure VRAM is clear
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Setup docker
run: |
touch github_summary.md
bash scripts/ci/amd/amd_ci_start_container.sh
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: Install dependencies
run: |
bash scripts/ci/amd/amd_ci_install_dependency.sh
bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate
- name: Accuracy Test MI35x (8-GPU DeepSeek-R1 HiCache)
timeout-minutes: 180
run: |
> github_summary.md
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-r1-hicache --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
# ============================================================================== # ==============================================================================
# 8-GPU Kimi-K2.6 (MI30x + MI35x) # 8-GPU Kimi-K2.6 (MI30x + MI35x)
# ============================================================================== # ==============================================================================
@@ -1569,12 +1641,13 @@ jobs:
# 1-GPU Unit Tests (MI30x + MI35x) # 1-GPU Unit Tests (MI30x + MI35x)
- nightly-test-1-gpu-unit - nightly-test-1-gpu-unit
- nightly-test-1-gpu-mi35x - nightly-test-1-gpu-mi35x
# 2-GPU and 4-GPU Tests (MI30x) # 2-GPU and 4-GPU Tests (MI30x + MI35x)
- nightly-accuracy-2-gpu - nightly-accuracy-2-gpu
- nightly-accuracy-2-gpu-vlm - nightly-accuracy-2-gpu-vlm
# - nightly-perf-2-gpu-text # excluded: perf failures don't block CI # - nightly-perf-2-gpu-text # excluded: perf failures don't block CI
# - nightly-perf-2-gpu-vlm # excluded: perf failures don't block CI # - nightly-perf-2-gpu-vlm # excluded: perf failures don't block CI
- nightly-4-gpu - nightly-4-gpu
- nightly-2-gpu-mi35x-glm51-mxfp4
# 8-GPU GPT-OSS (MI30x mixes Grok1-FP8; MI35x mixes Qwen3-Coder-Next) # 8-GPU GPT-OSS (MI30x mixes Grok1-FP8; MI35x mixes Qwen3-Coder-Next)
- nightly-accuracy-8-gpu - nightly-accuracy-8-gpu
- nightly-accuracy-8-gpu-mi35x - nightly-accuracy-8-gpu-mi35x
@@ -1594,10 +1667,11 @@ jobs:
- nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp - nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp
# - nightly-perf-8-gpu-mi35x-deepseek-v32-basic # excluded: perf failures don't block CI # - nightly-perf-8-gpu-mi35x-deepseek-v32-basic # excluded: perf failures don't block CI
# - nightly-perf-8-gpu-mi35x-deepseek-v32-mtp # excluded: perf failures don't block CI # - nightly-perf-8-gpu-mi35x-deepseek-v32-mtp # excluded: perf failures don't block CI
# 8-GPU DeepSeek-R1-MXFP4 (MI35x only) # 8-GPU DeepSeek-R1 (MI35x only)
- nightly-8-gpu-mi35x-deepseek-r1-mxfp4 - nightly-8-gpu-mi35x-deepseek-r1-mxfp4
- nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8 - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8
- nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion
- nightly-8-gpu-mi35x-deepseek-r1-hicache
# 8-GPU Kimi-K2.6 (MI30x + MI35x) # 8-GPU Kimi-K2.6 (MI30x + MI35x)
- nightly-8-gpu-kimi-k26 - nightly-8-gpu-kimi-k26
- nightly-8-gpu-mi35x-kimi-k26 - nightly-8-gpu-mi35x-kimi-k26
test/registered/amd/accuracy/mi35x/test_glm51_mxfp4_tp2_gsm8k_mi35x.py
+34 -62
View File
@@ -1,25 +1,21 @@
"""MI355X GLM-5.1-MXFP4 TP=2 GSM8K accuracy gate. """MI355X GLM-5.1-MXFP4 TP=2 GSM8K accuracy gate.
This is a PR Test (AMD) regression test for the GLM-5.1-MXFP4 TP=2 This is a nightly AMD regression test for the GLM-5.1-MXFP4 TP=2
accuracy drop seen on MI355X/gfx950 when aiter selected a bad BF16 GEMM path. accuracy drop seen on MI355X/gfx950 when aiter selected a bad BF16 GEMM path.
Registry: stage-c-test-large-8-gpu-amd-mi35x suite Registry: nightly-amd-2-gpu-mi35x-glm51-mxfp4 suite
""" """
import os
import resource import resource
import unittest import unittest
from types import SimpleNamespace from types import SimpleNamespace
from urllib.parse import urlparse
os.environ.setdefault("HF_HOME", "/data2/models/huggingface")
os.environ.setdefault("HF_HUB_CACHE", "/data2/models/huggingface/hub")
from sglang.srt.utils import kill_process_tree from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k_eval from sglang.test.run_eval import run_eval
from sglang.test.test_utils import ( from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST, DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci, is_in_ci,
popen_launch_server, popen_launch_server,
write_github_step_summary, write_github_step_summary,
@@ -27,19 +23,16 @@ from sglang.test.test_utils import (
register_amd_ci( register_amd_ci(
est_time=3600, est_time=3600,
suite="stage-c-test-large-8-gpu-amd-mi35x", suite="nightly-amd-2-gpu-mi35x-glm51-mxfp4",
nightly=True,
) )
GLM51_MXFP4_MODEL_ID = "amd/GLM-5.1-MXFP4" GLM51_MXFP4_MODEL_ID = "amd/GLM-5.1-MXFP4"
GLM51_MXFP4_LOCAL_PATHS = ( SERVER_LAUNCH_TIMEOUT = 5400
"/data2/models/amd-GLM-5.1-MXFP4",
"/data/huggingface/hub/amd/GLM-5.1-MXFP4",
)
GSM8K_ACCURACY_THRESHOLD = 0.92 GSM8K_ACCURACY_THRESHOLD = 0.92
GSM8K_INVALID_THRESHOLD = 0.02 GSM8K_NUM_EXAMPLES = None
DEFAULT_NUM_QUESTIONS = 1200 GSM8K_NUM_THREADS = 512
DEFAULT_PARALLEL = 1200
def _raise_nofile_limit() -> None: def _raise_nofile_limit() -> None:
@@ -50,26 +43,16 @@ def _raise_nofile_limit() -> None:
resource.setrlimit(resource.RLIMIT_NOFILE, (target, hard)) resource.setrlimit(resource.RLIMIT_NOFILE, (target, hard))
def _get_model_path() -> str: class TestGLM51MXFP4TP2GSM8KMI35x(CustomTestCase):
env_path = os.environ.get("GLM51_MXFP4_MODEL_PATH")
if env_path:
return env_path
for path in GLM51_MXFP4_LOCAL_PATHS:
if os.path.exists(path):
return path
return GLM51_MXFP4_MODEL_ID
class TestGLM51MXFP4TP2GSM8KMI35x(unittest.TestCase):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
_raise_nofile_limit() _raise_nofile_limit()
cls.model = _get_model_path() cls.model = GLM51_MXFP4_MODEL_ID
cls.base_url = DEFAULT_URL_FOR_TEST cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server( cls.process = popen_launch_server(
model=cls.model, model=cls.model,
base_url=cls.base_url, base_url=cls.base_url,
timeout=5400, timeout=SERVER_LAUNCH_TIMEOUT,
other_args=[ other_args=[
"--tp", "--tp",
"2", "2",
@@ -100,44 +83,33 @@ class TestGLM51MXFP4TP2GSM8KMI35x(unittest.TestCase):
def tearDownClass(cls): def tearDownClass(cls):
kill_process_tree(cls.process.pid) kill_process_tree(cls.process.pid)
def test_gsm8k_accuracy(self): def test_gsm8k(self):
num_questions = int(
os.environ.get("GLM51_MXFP4_GSM8K_NUM_QUESTIONS", DEFAULT_NUM_QUESTIONS)
)
parallel = int(os.environ.get("GLM51_MXFP4_GSM8K_PARALLEL", DEFAULT_PARALLEL))
url = urlparse(self.base_url)
args = SimpleNamespace( args = SimpleNamespace(
num_shots=5, base_url=self.base_url,
data_path=None, model=self.model,
num_questions=num_questions, eval_name="gsm8k",
max_new_tokens=512, api="completion",
parallel=parallel, num_examples=GSM8K_NUM_EXAMPLES,
host=url.hostname or "127.0.0.1", num_threads=GSM8K_NUM_THREADS,
port=url.port or 30000, max_tokens=512,
temperature=0.0, temperature=0.0,
) )
metrics = run_gsm8k_eval(args) metrics = run_eval(args)
accuracy = metrics["accuracy"] print(f"{metrics=}", flush=True)
invalid = metrics["invalid"] score = metrics["score"]
summary = (
"### GLM-5.1-MXFP4 TP=2 GSM8K (MI355X)\n\n"
"| Model | TP | Questions | Accuracy | Invalid | Threshold | Status |\n"
"| ----- | -- | --------- | -------- | ------- | --------- | ------ |\n"
)
passed = (
accuracy >= GSM8K_ACCURACY_THRESHOLD and invalid <= GSM8K_INVALID_THRESHOLD
)
status = "PASS" if passed else "FAIL"
summary += (
f"| {self.model} | 2 | {num_questions} | {accuracy:.3f} | "
f"{invalid:.3f} | accuracy >= {GSM8K_ACCURACY_THRESHOLD:.2f} | {status} |\n"
)
if is_in_ci():
write_github_step_summary(summary)
self.assertGreaterEqual(accuracy, GSM8K_ACCURACY_THRESHOLD) if is_in_ci():
self.assertLessEqual(invalid, GSM8K_INVALID_THRESHOLD) write_github_step_summary(
"### GLM-5.1-MXFP4 TP=2 GSM8K (MI355X)\n\n"
"| Model | Examples | Max Parallel | Score | Threshold | Latency |\n"
"| ----- | --------- | ------------ | ----- | --------- | ------- |\n"
f"| {self.model} | full | default ({GSM8K_NUM_THREADS}) | "
f"{score:.3f} | {GSM8K_ACCURACY_THRESHOLD:.2f} | "
f"{metrics.get('latency', 0):.1f}s |\n"
)
self.assertGreaterEqual(score, GSM8K_ACCURACY_THRESHOLD)
if __name__ == "__main__": if __name__ == "__main__":
test/registered/amd/test_deepseek_r1_hicache_mi35x.py
+20 -25
View File
@@ -1,4 +1,4 @@
"""MI35x DeepSeek-R1-0528 FP8 HiCache PR Test (8-GPU) """MI35x DeepSeek-R1-0528 FP8 HiCache Nightly Test (8-GPU)
Regression guard: launches DeepSeek-R1-0528 (native FP8, MLA, aiter attention Regression guard: launches DeepSeek-R1-0528 (native FP8, MLA, aiter attention
backend) on MI35x with the full L1+L2+L3 HiCache hierarchy wired up backend) on MI35x with the full L1+L2+L3 HiCache hierarchy wired up
@@ -6,13 +6,13 @@ backend) on MI35x with the full L1+L2+L3 HiCache hierarchy wired up
GSM8K few-shot completion and asserts the accuracy still matches the GSM8K few-shot completion and asserts the accuracy still matches the
established threshold. The goal is to catch regressions where HiCache established threshold. The goal is to catch regressions where HiCache
breaks DSR1-0528 generation correctness, not to stress-test the cascade breaks DSR1-0528 generation correctness, not to stress-test the cascade
overflow path (that lives in the nightly suite). overflow path.
Acceptance: GSM8K (200 questions, 5-shot, completion API) score >= 0.93, Acceptance: GSM8K (1319 questions, 5-shot, completion API) score >= 0.93,
matching ``test_deepseek_r1_eval_mi35x.py`` / matching ``test_deepseek_r1_eval_mi35x.py`` /
``test_deepseek_r1_eval_amd.py``. ``test_deepseek_r1_eval_amd.py``.
Registry: stage-c-test-large-8-gpu-amd-mi35x (per-commit PR suite). Registry: nightly-amd-8-gpu-mi35x-deepseek-r1-hicache suite.
""" """
import os import os
@@ -21,12 +21,6 @@ import tempfile
import unittest import unittest
from types import SimpleNamespace from types import SimpleNamespace
# MI35x CI runner caches HF models on a fast local volume. We only fall
# back to this if HF_HOME isn't already set by the runner (e.g. repro_ci.sh
# points HF_HOME at /sgl-data/hf-cache); never force HF_HUB_CACHE so
# huggingface_hub keeps deriving it as $HF_HOME/hub.
os.environ.setdefault("HF_HOME", "/data2/models/huggingface")
from sglang.srt.utils import kill_process_tree from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.run_eval import run_eval from sglang.test.run_eval import run_eval
@@ -38,18 +32,21 @@ from sglang.test.test_utils import (
write_github_step_summary, write_github_step_summary,
) )
# ~15 min: 5-8 min weight load + ~2-3 min GSM8K + slack. # DSR1-0528 can spend 20+ min in weight loading on MI35x before warmup.
register_amd_ci(est_time=900, suite="stage-c-test-large-8-gpu-amd-mi35x") register_amd_ci(
est_time=5400,
suite="nightly-amd-8-gpu-mi35x-deepseek-r1-hicache",
nightly=True,
)
DEEPSEEK_R1_MODEL_PATH = "deepseek-ai/DeepSeek-R1-0528" DEEPSEEK_R1_MODEL_PATH = "deepseek-ai/DeepSeek-R1-0528"
SERVER_LAUNCH_TIMEOUT = 1500 SERVER_LAUNCH_TIMEOUT = 3600
# Threshold matches the existing nightly AMD DSR1-0528 accuracy tests: # Threshold matches the existing nightly AMD DSR1-0528 accuracy tests:
# test/registered/amd/accuracy/mi35x/test_deepseek_r1_eval_mi35x.py # test/registered/amd/accuracy/mi35x/test_deepseek_r1_eval_mi35x.py
# test/registered/amd/accuracy/mi30x/test_deepseek_r1_eval_amd.py # test/registered/amd/accuracy/mi30x/test_deepseek_r1_eval_amd.py
GSM8K_ACCURACY_THRESHOLD = 0.93 GSM8K_ACCURACY_THRESHOLD = 0.93
GSM8K_NUM_EXAMPLES = 200 GSM8K_NUM_EXAMPLES = None
GSM8K_NUM_SHOTS = 5
GSM8K_NUM_THREADS = 64 GSM8K_NUM_THREADS = 64
@@ -140,31 +137,29 @@ class TestDeepSeekR1HiCacheMI35x(CustomTestCase):
"""GSM8K few-shot completion against the HiCache-enabled DSR1-0528.""" """GSM8K few-shot completion against the HiCache-enabled DSR1-0528."""
args = SimpleNamespace( args = SimpleNamespace(
base_url=self.base_url, base_url=self.base_url,
model=self.model,
eval_name="gsm8k", eval_name="gsm8k",
api="completion", api="completion",
num_examples=GSM8K_NUM_EXAMPLES, num_examples=GSM8K_NUM_EXAMPLES,
num_shots=GSM8K_NUM_SHOTS,
num_threads=GSM8K_NUM_THREADS, num_threads=GSM8K_NUM_THREADS,
max_tokens=512, max_tokens=512,
temperature=0.0, temperature=0.0,
) )
metrics = run_eval(args) metrics = run_eval(args)
print(f"{metrics=}", flush=True)
score = metrics["score"] score = metrics["score"]
print(f"GSM8K {metrics=}", flush=True)
if is_in_ci(): if is_in_ci():
write_github_step_summary( write_github_step_summary(
"### DeepSeek-R1-0528 FP8 HiCache GSM8K (MI35x)\n\n" "### DeepSeek-R1-0528 FP8 HiCache GSM8K (MI35x)\n\n"
f"- score: `{score:.3f}` (threshold `{GSM8K_ACCURACY_THRESHOLD}`)\n" "| Model | Examples | Max Parallel | Score | Threshold | Latency |\n"
f"- latency: `{metrics.get('latency', 0):.1f}s`\n" "| ----- | --------- | ------------ | ----- | --------- | ------- |\n"
f"| {self.model} | full | {GSM8K_NUM_THREADS} | "
f"{score:.3f} | {GSM8K_ACCURACY_THRESHOLD:.2f} | "
f"{metrics.get('latency', 0):.1f}s |\n"
) )
self.assertGreater( self.assertGreaterEqual(score, GSM8K_ACCURACY_THRESHOLD)
score,
GSM8K_ACCURACY_THRESHOLD,
f"DSR1-0528 FP8 + HiCache GSM8K accuracy {score:.3f} "
f"below threshold {GSM8K_ACCURACY_THRESHOLD}",
)
if __name__ == "__main__": if __name__ == "__main__":