minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

[AMD][CI] Add DeepSeek-V4-Flash FP8 accuracy coverage on MI30x (#36396)

Browse Source
This commit is contained in:
Michael
2026-08-26 19:30:07 -07:00
committed by GitHub
parent 07570202f0
commit 1c8f2b38cb
3 changed files with 201 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/nightly-test-amd-rocm720.yml
+61 -99
View File
@@ -59,11 +59,9 @@ on:
# 8-GPU Grok2 (MI30x + MI35x)
- nightly-8-gpu-grok2-rocm720
- nightly-8-gpu-mi35x-grok2-rocm720
# 8-GPU DeepSeek-V3.x (MI30x)
- nightly-8-gpu-deepseek-v31-rocm720
# 8-GPU DeepSeek-V3.2 (MI30x)
- nightly-8-gpu-deepseek-v32-rocm720
- nightly-8-gpu-deepseek-v32-mtp-rocm720
- nightly-8-gpu-deepseek-v3-kv-fp8-rocm720
# 8-GPU DeepSeek-V3.2 (MI35x)
- nightly-accuracy-8-gpu-mi35x-deepseek-v32-rocm720
- nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp-rocm720
@@ -75,7 +73,9 @@ on:
- nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion-rocm720
- nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4-rocm720
- nightly-8-gpu-mi35x-deepseek-r1-hicache-rocm720
# 8-GPU DeepSeek-V4 (MI35x only)
# 8-GPU DeepSeek-V4-Flash (MI30x)
- nightly-8-gpu-deepseek-v4-flash-rocm720
# 8-GPU DeepSeek-V4 (MI35x)
- nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720
- nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720
- nightly-8-gpu-mi35x-deepseek-v4-pro-mtp-rocm720
@@ -856,60 +856,13 @@ jobs:
exit ${TEST_EXIT_CODE:-0}
# ==============================================================================
# 8-GPU DeepSeek-V3.x (MI30x)
# 8-GPU DeepSeek-V3.2 (MI30x)
#
# V3-0324 and V3.1 are no longer scheduled here: V3.2 covers the same MI30x
# paths (aiter MLA, EAGLE MTP, multithread weight load) on a current
# checkpoint, and DeepSeek-R1/V4 carry the MI35x side.
# ==============================================================================
nightly-8-gpu-deepseek-v31-rocm720:
name: ${{ format('nightly-8-gpu-deepseek-v31 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }}
strategy:
fail-fast: false
matrix:
rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }}
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v31-rocm720,'))
runs-on: linux-mi300-8gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.sha }}
- name: Ensure VRAM is clear
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Setup docker (${{ matrix.rocm_version }})
run: |
touch github_summary.md
bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
ENABLE_CACHE_HOST: "1"
- name: Install dependencies
run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
- name: Accuracy Test ROCm 7.2 (8-GPU DeepSeek-V3.1)
timeout-minutes: 120
run: |
> github_summary.md # Clear summary file
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e SGLANG_USE_AITER=1 \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v31 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
- name: Performance Test ROCm 7.2 (8-GPU DeepSeek-V3.1)
timeout-minutes: 300
continue-on-error: true
run: |
> github_summary.md # Clear summary file
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e SGLANG_USE_ROCM700A=1 \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-deepseek-v31 --nightly --timeout-per-file 18000 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
nightly-8-gpu-deepseek-v32-rocm720:
name: ${{ format('nightly-8-gpu-deepseek-v32 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }}
strategy:
@@ -1008,44 +961,6 @@ jobs:
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
nightly-8-gpu-deepseek-v3-kv-fp8-rocm720:
name: ${{ format('nightly-8-gpu-deepseek-v3-kv-fp8 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }}
strategy:
fail-fast: false
matrix:
rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }}
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v3-kv-fp8-rocm720,'))
runs-on: linux-mi300-8gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.sha }}
- name: Ensure VRAM is clear
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Setup docker (${{ matrix.rocm_version }})
run: |
touch github_summary.md
bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
ENABLE_CACHE_HOST: "1"
- name: Install dependencies
run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
- name: DeepSeek-V3 KV FP8 Test ROCm 7.2 (8-GPU Basic + MTP)
timeout-minutes: 120
run: |
> github_summary.md # Clear summary file
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-deepseek-v3-kv-fp8 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
# ==============================================================================
# 8-GPU DeepSeek-V3.2 (MI35x)
# ==============================================================================
@@ -1465,7 +1380,54 @@ jobs:
exit ${TEST_EXIT_CODE:-0}
# ==============================================================================
# 8-GPU DeepSeek-V4 (MI35x only)
# 8-GPU DeepSeek-V4-Flash (MI30x)
#
# Accuracy only: the MI35x jobs below carry the DSV4 perf numbers, and gfx942
# has no DSV4 perf baseline to regress against yet. Runs the cookbook's
# MI300X Flash FP8 low-latency cell, so it also covers MLA + KV-FP8 on gfx942.
# ==============================================================================
nightly-8-gpu-deepseek-v4-flash-rocm720:
name: ${{ format('nightly-8-gpu-deepseek-v4-flash ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }}
strategy:
fail-fast: false
matrix:
rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }}
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v4-flash-rocm720,'))
runs-on: linux-mi300-8gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.sha }}
- name: Ensure VRAM is clear
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Setup docker (${{ matrix.rocm_version }})
run: |
touch github_summary.md
bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
ENABLE_CACHE_HOST: "1"
- name: Install dependencies
run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
- name: Accuracy Test ROCm 7.2 (8-GPU DeepSeek-V4-Flash FP8)
timeout-minutes: 180
run: |
> github_summary.md # Clear summary file
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v4-flash --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
# ==============================================================================
# 8-GPU DeepSeek-V4 (MI35x)
# ==============================================================================
nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720:
@@ -2392,11 +2354,9 @@ jobs:
# 8-GPU Grok2 (MI30x + MI35x)
- nightly-8-gpu-grok2-rocm720
- nightly-8-gpu-mi35x-grok2-rocm720
# 8-GPU DeepSeek-V3.x (MI30x)
- nightly-8-gpu-deepseek-v31-rocm720
# 8-GPU DeepSeek-V3.2 (MI30x)
- nightly-8-gpu-deepseek-v32-rocm720
- nightly-8-gpu-deepseek-v32-mtp-rocm720
- nightly-8-gpu-deepseek-v3-kv-fp8-rocm720
# 8-GPU DeepSeek-V3.2 (MI35x)
- nightly-accuracy-8-gpu-mi35x-deepseek-v32-rocm720
- nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp-rocm720
@@ -2408,7 +2368,9 @@ jobs:
- nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion-rocm720
- nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4-rocm720
- nightly-8-gpu-mi35x-deepseek-r1-hicache-rocm720
# 8-GPU DeepSeek-V4 (MI35x only)
# 8-GPU DeepSeek-V4-Flash (MI30x)
- nightly-8-gpu-deepseek-v4-flash-rocm720
# 8-GPU DeepSeek-V4 (MI35x)
- nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720
- nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720
- nightly-8-gpu-mi35x-deepseek-v4-pro-mtp-rocm720
test/registered/amd/test_deepseek_v4_flash_fp8_mi30x.py
+139
View File
@@ -0,0 +1,139 @@
"""MI30x DeepSeek-V4-Flash FP8 Accuracy Test (8-GPU)
GSM8K few-shot accuracy for DeepSeek-V4-Flash FP8 on MI30x (gfx942) ROCm 7.2.
The launch config is the cookbook's MI300X Flash FP8 low-latency single-node
recipe, verbatim -- the one the docs tell users to run, marked `verified: true`
in docs/src/snippets/configs/deepseek-ai/deepseek-v4.jsx. Of the three published
MI300X strategies it is the only TP-only one (balanced and high-throughput add
DP attention and the prefill delayer), so it is the narrowest cell that still
covers the gfx942 serving path. Keep this in sync with that cell.
The cell as published could not load this checkpoint; the fix is in the
cookbook PR and is the SGLANG_DSV4_FP4_EXPERTS entry below.
Accuracy only: gfx942 has no DSV4 perf baseline to regress against yet, and the
MI35x suite already carries the 8k/1k throughput numbers.
Unlike the MI35x FP8 test this recipe also runs `--kv-cache-dtype fp8_e4m3` and
EAGLE, so it is the gfx942 MLA + KV-FP8 signal as well. That extra quantization
is also why the threshold below is 0.90 rather than the 0.91 the MI35x FP8 test
uses: measured 0.9174 and 0.9257 here, a spread wide enough that 0.91 would sit
only ~2 sd off the mean and fail a good build roughly one night in forty. 0.90
keeps ~2 points of headroom and still catches every regression worth gating on
-- the failure modes this guards against (see #36390) collapse output entirely
rather than shaving a point.
Registry: nightly-amd-accuracy-8-gpu-deepseek-v4-flash suite
"""
import os
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
register_amd_ci(
est_time=5400,
suite="nightly-amd-accuracy-8-gpu-deepseek-v4-flash",
nightly=True,
)
DEEPSEEK_V4_FP8_MODEL_PATH = os.environ.get(
"DEEPSEEK_V4_FP8_MODEL_PATH", "sgl-project/DeepSeek-V4-Flash-FP8"
)
SERVER_LAUNCH_TIMEOUT = 3600
ENV_VARS = {
# The routed experts of this checkpoint are FP8, not mxfp4-packed. The env
# default is mxfp4 and the auto-detect fallback reads the safetensors header
# from the local HF cache, so it returns None on a runner that has not pulled
# the weights yet and the wrong default wins -- weight load then dies on a
# factor-of-2 shape mismatch. Set it rather than depend on cache state.
"SGLANG_DSV4_FP4_EXPERTS": "0",
"SGLANG_USE_ROCM700A": "0",
"SGLANG_HACK_FLASHMLA_BACKEND": "unified_kv_triton",
"AITER_BF16_FP8_MOE_BOUND": "0",
}
class TestDeepseekV4FlashFp8Mi30x(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V4_FP8_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
env = os.environ.copy()
env.update(ENV_VARS)
other_args = [
"--trust-remote-code",
"--tp",
"8",
"--attention-backend",
"dsv4",
"--page-size",
"256",
"--mem-fraction-static",
"0.90",
"--swa-full-tokens-ratio",
"0.1",
"--disable-shared-experts-fusion",
"--kv-cache-dtype",
"fp8_e4m3",
"--chunked-prefill-size",
"8192",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=SERVER_LAUNCH_TIMEOUT,
other_args=other_args,
env=env,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
args = SimpleNamespace(
num_shots=8,
data_path=None,
num_questions=1319,
parallel=1319,
max_new_tokens=512,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval_few_shot_gsm8k(args)
print(f"{metrics=}")
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v4-flash-fp8, mi30x)\n"
f'{metrics["accuracy"]=:.3f}\n'
)
self.assertGreater(metrics["accuracy"], 0.90)
if __name__ == "__main__":
unittest.main()
test/run_suite.py
+1
View File
@@ -147,6 +147,7 @@ NIGHTLY_SUITES = {
"nightly-amd-4-gpu",
"nightly-amd-8-gpu",
"nightly-amd-vlm",
"nightly-amd-accuracy-8-gpu-deepseek-v4-flash",
"nightly-amd-8-gpu-mi35x-deepseek-v4-flash",
# MI35x 8-GPU suite (different model configs)
"nightly-amd-8-gpu-mi35x",