From 5eff3c489a71b49abe3fea558ebc6e5806df3a4d Mon Sep 17 00:00:00 2001 From: Bingxu Chen Date: Mon, 4 May 2026 15:02:33 +0800 Subject: [PATCH] [AMD] Deepseek v4 Flash / Pro nightly tests for MI35x ROCm 7.2 (#24203) Co-authored-by: YC Yen-Ching Tseng --- .../workflows/nightly-test-amd-rocm720.yml | 140 ++++++++++++ test/registered/amd/test_deepseek_v4_fp4.py | 207 +++++++++++++++++ test/registered/amd/test_deepseek_v4_fp8.py | 207 +++++++++++++++++ .../amd/test_deepseek_v4_pro_fp4.py | 209 ++++++++++++++++++ .../amd/test_deepseek_v4_pro_fp8.py | 209 ++++++++++++++++++ 5 files changed, 972 insertions(+) create mode 100644 test/registered/amd/test_deepseek_v4_fp4.py create mode 100644 test/registered/amd/test_deepseek_v4_fp8.py create mode 100644 test/registered/amd/test_deepseek_v4_pro_fp4.py create mode 100644 test/registered/amd/test_deepseek_v4_pro_fp8.py diff --git a/.github/workflows/nightly-test-amd-rocm720.yml b/.github/workflows/nightly-test-amd-rocm720.yml index ea17d82ef..e3eda3732 100644 --- a/.github/workflows/nightly-test-amd-rocm720.yml +++ b/.github/workflows/nightly-test-amd-rocm720.yml @@ -57,6 +57,8 @@ on: - nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp-rocm720 - nightly-perf-8-gpu-mi35x-deepseek-v32-basic-rocm720 - nightly-perf-8-gpu-mi35x-deepseek-v32-mtp-rocm720 + - nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720 + - nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720 - nightly-8-gpu-mi35x-kimi-k25-rocm720 - nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720 - nightly-8-gpu-mi35x-qwen35-rocm720 @@ -1413,6 +1415,142 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} + # MI35x 8-GPU DeepSeek-V4-Flash FP8 + FP4 (Accuracy + Performance combined) ROCm 7.2 + # NOTE on runtime sourcing: the DSv4 docker image (tag suffix `-DSv4`) bakes + # in sglang built from a specific commit of the amd/deepseek_v4 branch (the + # 7-char sha in the image tag is that commit). To keep the runtime as exactly + # that image-frozen sglang/aiter, we pass `--skip-sglang-build` and + # `--skip-aiter-build` so install_dependency.sh does NOT `pip install -e + # /sglang-checkout/python` (which would override the image's sglang with + # whatever this checkout happens to be) and does NOT rebuild aiter from this + # checkout's docker/rocm.Dockerfile. The /sglang-checkout mount is still used + # for shell scripts and for run_suite.py discovering test files; it does not + # poison Python imports because the image's site-packages .pth points at + # /sgl-workspace/sglang/python (a different path). + nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720,')) + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Resolve DSv4 image tag + id: dsv4_image + run: | + # Pick the latest Docker Hub tag matching rocm720-mi35x---DSv4. + # Docker Hub returns results sorted by last_updated DESC by default, so the + # first regex match is the most recent daily build. + AUTH_HEADER=() + if [[ -n "${DOCKERHUB_AMD_USERNAME:-}" && -n "${DOCKERHUB_AMD_TOKEN:-}" ]]; then + TOKEN=$(curl -s -H "Content-Type: application/json" \ + -X POST -d "{\"username\":\"${DOCKERHUB_AMD_USERNAME}\",\"password\":\"${DOCKERHUB_AMD_TOKEN}\"}" \ + https://hub.docker.com/v2/users/login/ | python3 -c "import json,sys; print(json.load(sys.stdin).get('token',''))") + if [[ -n "$TOKEN" ]]; then + AUTH_HEADER=(-H "Authorization: JWT $TOKEN") + fi + fi + TAG=$(curl -s "${AUTH_HEADER[@]}" \ + "https://hub.docker.com/v2/repositories/rocm/sgl-dev/tags?page_size=100&name=DSv4" \ + | grep -oE '"name":"rocm720-mi35x-[a-f0-9]{7}-[0-9]{8}-DSv4"' \ + | head -n 1 | cut -d'"' -f4) + if [ -z "$TAG" ]; then + echo "::error::No DSv4 image found matching rocm720-mi35x---DSv4 on Docker Hub" + exit 1 + fi + echo "image=rocm/sgl-dev:$TAG" >> "$GITHUB_OUTPUT" + echo "Resolved DSv4 image: rocm/sgl-dev:$TAG" + + - name: Setup docker (ROCm 7.2 DSv4) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --custom-image ${{ steps.dsv4_image.outputs.image }} + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies (preserve DSv4 sglang/aiter from image) + run: | + # --skip-sglang-build: keep the image's pre-installed DSv4 sglang + # (default would `pip install -e /sglang-checkout/python` and clobber it with main's source). + # --skip-aiter-build: keep the image's DSv4-tuned aiter + # (default reads /sglang-checkout/docker/rocm.Dockerfile from main and rebuilds aiter to that commit). + # --skip-test-time-deps: GSM8K + bench_one_batch_server don't need lmms-eval / human-eval. + bash scripts/ci/amd/amd_ci_install_dependency.sh \ + --skip-sglang-build --skip-aiter-build --skip-test-time-deps + # tabulate is the only thing run_suite.py imports that may not be in the DSv4 image. + bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + + - name: Accuracy + Performance Test MI35x ROCm 7.2 (8-GPU DeepSeek-V4-Flash FP8 + FP4) + timeout-minutes: 300 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-flash --nightly --timeout-per-file 7200 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + # MI35x 8-GPU DeepSeek-V4-Pro FP8 + FP4 (Accuracy + Performance combined) ROCm 7.2 + # Pro is 1.6T (vs Flash 285B); load + warmup is much longer, so timeout-per-file + # and the job timeout are both larger than the Flash job. + # Same image / branch / install strategy as the Flash job above — see the comment + # block on `nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720` for the rationale. + nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720,')) + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Resolve DSv4 image tag + id: dsv4_image + run: | + AUTH_HEADER=() + if [[ -n "${DOCKERHUB_AMD_USERNAME:-}" && -n "${DOCKERHUB_AMD_TOKEN:-}" ]]; then + TOKEN=$(curl -s -H "Content-Type: application/json" \ + -X POST -d "{\"username\":\"${DOCKERHUB_AMD_USERNAME}\",\"password\":\"${DOCKERHUB_AMD_TOKEN}\"}" \ + https://hub.docker.com/v2/users/login/ | python3 -c "import json,sys; print(json.load(sys.stdin).get('token',''))") + if [[ -n "$TOKEN" ]]; then + AUTH_HEADER=(-H "Authorization: JWT $TOKEN") + fi + fi + TAG=$(curl -s "${AUTH_HEADER[@]}" \ + "https://hub.docker.com/v2/repositories/rocm/sgl-dev/tags?page_size=100&name=DSv4" \ + | grep -oE '"name":"rocm720-mi35x-[a-f0-9]{7}-[0-9]{8}-DSv4"' \ + | head -n 1 | cut -d'"' -f4) + if [ -z "$TAG" ]; then + echo "::error::No DSv4 image found matching rocm720-mi35x---DSv4 on Docker Hub" + exit 1 + fi + echo "image=rocm/sgl-dev:$TAG" >> "$GITHUB_OUTPUT" + echo "Resolved DSv4 image: rocm/sgl-dev:$TAG" + + - name: Setup docker (ROCm 7.2 DSv4) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --custom-image ${{ steps.dsv4_image.outputs.image }} + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies (preserve DSv4 sglang/aiter from image) + run: | + bash scripts/ci/amd/amd_ci_install_dependency.sh \ + --skip-sglang-build --skip-aiter-build --skip-test-time-deps + bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + + - name: Accuracy + Performance Test MI35x ROCm 7.2 (8-GPU DeepSeek-V4-Pro FP8 + FP4) + timeout-minutes: 480 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-pro --nightly --timeout-per-file 14400 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + check-all-jobs: if: always() && (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch') needs: @@ -1453,6 +1591,8 @@ jobs: - nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp-rocm720 - nightly-perf-8-gpu-mi35x-deepseek-v32-basic-rocm720 - nightly-perf-8-gpu-mi35x-deepseek-v32-mtp-rocm720 + - nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720 + - nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720 - nightly-8-gpu-mi35x-kimi-k25-rocm720 - nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720 - nightly-8-gpu-mi35x-qwen35-rocm720 diff --git a/test/registered/amd/test_deepseek_v4_fp4.py b/test/registered/amd/test_deepseek_v4_fp4.py new file mode 100644 index 000000000..0cf9cbaf5 --- /dev/null +++ b/test/registered/amd/test_deepseek_v4_fp4.py @@ -0,0 +1,207 @@ +"""MI35x DeepSeek-V4-Flash FP4 Test (8-GPU) + +Combined accuracy + performance test for DeepSeek-V4-Flash FP4 on MI35x ROCm 7.2. +- Accuracy: GSM8K few-shot eval +- Performance: bench_one_batch_server with input_len=8192, output_len=1024 (bs=1) + +Both tests share a single launched server. + +Registry: nightly-amd-8-gpu-mi35x-deepseek-v4-flash suite +""" + +import json +import os +import subprocess +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +register_amd_ci( + est_time=7200, suite="nightly-amd-8-gpu-mi35x-deepseek-v4-flash", nightly=True +) + +DEEPSEEK_V4_FP4_MODEL_PATH = os.environ.get( + "DEEPSEEK_V4_FP4_MODEL_PATH", "deepseek-ai/DeepSeek-V4-Flash" +) +SERVER_LAUNCH_TIMEOUT = 3600 + +# Common DeepSeek-V4 env vars (AMD ROCm 7.2 path: tilelang + AITER + ROCm700A). +# Source of truth: python/run_dsv4.sh. +COMMON_ENV_VARS = { + "SGLANG_OPT_USE_FUSED_COMPRESS": "false", + "SGLANG_OPT_USE_OLD_COMPRESSOR": "true", + "SGLANG_OPT_USE_TILELANG_SWA_PREPARE": "false", + "SGLANG_OPT_USE_JIT_KERNEL_FUSED_TOPK": "false", + "SGLANG_OPT_USE_FUSED_HASH_TOPK": "false", + "SGLANG_OPT_DEEPGEMM_HC_PRENORM": "false", + "SGLANG_OPT_USE_TILELANG_MHC_PRE": "false", + "SGLANG_OPT_USE_TILELANG_MHC_POST": "false", + "SGLANG_ENABLE_THINKING": "1", + "SGLANG_USE_AITER": "1", + "SGLANG_USE_ROCM700A": "1", + "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH": "1", + "SGLANG_OPT_DPSK_V4_RADIX": "0", + "SGLANG_OPT_USE_OVERLAP_STORE_CACHE": "false", + "SGLANG_OPT_USE_FUSED_STORE_CACHE": "false", + "SGLANG_TOPK_TRANSFORM_512_TORCH": "1", + "SGLANG_OPT_USE_TILELANG_INDEXER": "true", + "SGLANG_HACK_FLASHMLA_BACKEND": "tilelang", + "SGLANG_REASONING_EFFORT": "max", +} + +# FP4 variant: FP4 mixed-precision experts. +FP4_ENV_VARS = { + "SGLANG_DSV4_FP4_EXPERTS": "true", + "SGLANG_FORCE_TRITON_MOE_FP8": "0", +} + + +class TestDeepseekV4Fp4(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEEPSEEK_V4_FP4_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + + env = os.environ.copy() + env.update(COMMON_ENV_VARS) + env.update(FP4_ENV_VARS) + + other_args = [ + "--trust-remote-code", + "--tp", + "8", + "--disable-radix-cache", + "--attention-backend", + "compressed", + "--max-running-requests", + "256", + "--page-size", + "256", + "--chunked-prefill-size", + "8192", + "--disable-shared-experts-fusion", + "--tool-call-parser", + "deepseekv4", + "--reasoning-parser", + "deepseek-v4", + ] + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=SERVER_LAUNCH_TIMEOUT, + other_args=other_args, + env=env, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_a_gsm8k(self): + # `a` prefix to run first (alphabetical) and warm up the server. + args = SimpleNamespace( + num_shots=8, + data_path=None, + num_questions=1319, + parallel=1319, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_gsm8k (deepseek-v4-flash-fp4)\n" + f'{metrics["accuracy"]=:.3f}\n' + ) + self.assertGreater(metrics["accuracy"], 0.91) + + def test_b_perf_8k_1k(self): + json_output = "/tmp/deepseek_v4_flash_fp4_perf.json" + if os.path.exists(json_output): + os.remove(json_output) + + # First "1" is a warmup; the markdown report below skips it. + batch_sizes = ["1", "1", "2", "4", "8", "16", "32"] + cmd = [ + "python3", + "-m", + "sglang.bench_one_batch_server", + "--model", + "None", + "--base-url", + self.base_url, + "--batch-size", + *batch_sizes, + "--input-len", + "8192", + "--output-len", + "1024", + "--show-report", + f"--pydantic-result-filename={json_output}", + "--no-append-to-github-summary", + "--trust-remote-code", + ] + print(f"Running benchmark: {' '.join(cmd)}") + result = subprocess.run(cmd, capture_output=True, text=True) + print(result.stdout) + if result.returncode != 0: + print(f"STDERR: {result.stderr}") + self.fail(f"bench_one_batch_server failed (rc={result.returncode})") + + self.assertTrue( + os.path.exists(json_output), + f"Benchmark JSON output {json_output} not found", + ) + with open(json_output) as f: + results_data = json.load(f) + self.assertTrue(results_data, "No benchmark results returned") + + if ( + len(results_data) > 1 + and results_data[0]["batch_size"] == results_data[1]["batch_size"] + ): + report_results = results_data[1:] + else: + report_results = results_data + + summary_lines = [ + "### test_perf_8k_1k (deepseek-v4-flash-fp4)", + "input_len=8192 output_len=1024", + "", + "| batch size | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |", + "| ---------- | ----------- | ------------------------ | ------------------------- | -------- |", + ] + for r in report_results: + bs = r["batch_size"] + latency = r.get("latency", 0.0) + in_tp = r.get("input_throughput", 0.0) + out_tp = r.get("output_throughput", 0.0) + itl = 1 / (out_tp / bs) * 1000 if out_tp > 0 else float("inf") + summary_lines.append( + f"| {bs} | {latency:.2f} | {in_tp:.2f} | {out_tp:.2f} | {itl:.2f} |" + ) + print( + f"bs={bs} latency={latency:.2f}s " + f"in_tp={in_tp:.2f} tok/s out_tp={out_tp:.2f} tok/s ITL={itl:.2f}ms" + ) + + if is_in_ci(): + write_github_step_summary("\n".join(summary_lines) + "\n") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/test_deepseek_v4_fp8.py b/test/registered/amd/test_deepseek_v4_fp8.py new file mode 100644 index 000000000..12d7cd034 --- /dev/null +++ b/test/registered/amd/test_deepseek_v4_fp8.py @@ -0,0 +1,207 @@ +"""MI35x DeepSeek-V4-Flash FP8 Test (8-GPU) + +Combined accuracy + performance test for DeepSeek-V4-Flash FP8 on MI35x ROCm 7.2. +- Accuracy: GSM8K few-shot eval +- Performance: bench_one_batch_server with input_len=8192, output_len=1024 (bs=1) + +Both tests share a single launched server. + +Registry: nightly-amd-8-gpu-mi35x-deepseek-v4-flash suite +""" + +import json +import os +import subprocess +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +register_amd_ci( + est_time=7200, suite="nightly-amd-8-gpu-mi35x-deepseek-v4-flash", nightly=True +) + +DEEPSEEK_V4_FP8_MODEL_PATH = os.environ.get( + "DEEPSEEK_V4_FP8_MODEL_PATH", "sgl-project/DeepSeek-V4-Flash-FP8" +) +SERVER_LAUNCH_TIMEOUT = 3600 + +# Common DeepSeek-V4 env vars (AMD ROCm 7.2 path: tilelang + AITER + ROCm700A). +# Source of truth: python/run_dsv4.sh. +COMMON_ENV_VARS = { + "SGLANG_OPT_USE_FUSED_COMPRESS": "false", + "SGLANG_OPT_USE_OLD_COMPRESSOR": "true", + "SGLANG_OPT_USE_TILELANG_SWA_PREPARE": "false", + "SGLANG_OPT_USE_JIT_KERNEL_FUSED_TOPK": "false", + "SGLANG_OPT_USE_FUSED_HASH_TOPK": "false", + "SGLANG_OPT_DEEPGEMM_HC_PRENORM": "false", + "SGLANG_OPT_USE_TILELANG_MHC_PRE": "false", + "SGLANG_OPT_USE_TILELANG_MHC_POST": "false", + "SGLANG_ENABLE_THINKING": "1", + "SGLANG_USE_AITER": "1", + "SGLANG_USE_ROCM700A": "1", + "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH": "1", + "SGLANG_OPT_DPSK_V4_RADIX": "0", + "SGLANG_OPT_USE_OVERLAP_STORE_CACHE": "false", + "SGLANG_OPT_USE_FUSED_STORE_CACHE": "false", + "SGLANG_TOPK_TRANSFORM_512_TORCH": "1", + "SGLANG_OPT_USE_TILELANG_INDEXER": "true", + "SGLANG_HACK_FLASHMLA_BACKEND": "tilelang", + "SGLANG_REASONING_EFFORT": "max", +} + +# FP8 variant: dense-FP8 experts via the Triton MoE FP8 path. +FP8_ENV_VARS = { + "SGLANG_DSV4_FP4_EXPERTS": "false", + "SGLANG_FORCE_TRITON_MOE_FP8": "1", +} + + +class TestDeepseekV4Fp8(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEEPSEEK_V4_FP8_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + + env = os.environ.copy() + env.update(COMMON_ENV_VARS) + env.update(FP8_ENV_VARS) + + other_args = [ + "--trust-remote-code", + "--tp", + "8", + "--disable-radix-cache", + "--attention-backend", + "compressed", + "--max-running-requests", + "256", + "--page-size", + "256", + "--chunked-prefill-size", + "8192", + "--disable-shared-experts-fusion", + "--tool-call-parser", + "deepseekv4", + "--reasoning-parser", + "deepseek-v4", + ] + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=SERVER_LAUNCH_TIMEOUT, + other_args=other_args, + env=env, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_a_gsm8k(self): + # `a` prefix to run first (alphabetical) and warm up the server. + args = SimpleNamespace( + num_shots=8, + data_path=None, + num_questions=1319, + parallel=1319, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_gsm8k (deepseek-v4-flash-fp8)\n" + f'{metrics["accuracy"]=:.3f}\n' + ) + self.assertGreater(metrics["accuracy"], 0.91) + + def test_b_perf_8k_1k(self): + json_output = "/tmp/deepseek_v4_flash_fp8_perf.json" + if os.path.exists(json_output): + os.remove(json_output) + + # First "1" is a warmup; the markdown report below skips it. + batch_sizes = ["1", "1", "2", "4", "8", "16", "32"] + cmd = [ + "python3", + "-m", + "sglang.bench_one_batch_server", + "--model", + "None", + "--base-url", + self.base_url, + "--batch-size", + *batch_sizes, + "--input-len", + "8192", + "--output-len", + "1024", + "--show-report", + f"--pydantic-result-filename={json_output}", + "--no-append-to-github-summary", + "--trust-remote-code", + ] + print(f"Running benchmark: {' '.join(cmd)}") + result = subprocess.run(cmd, capture_output=True, text=True) + print(result.stdout) + if result.returncode != 0: + print(f"STDERR: {result.stderr}") + self.fail(f"bench_one_batch_server failed (rc={result.returncode})") + + self.assertTrue( + os.path.exists(json_output), + f"Benchmark JSON output {json_output} not found", + ) + with open(json_output) as f: + results_data = json.load(f) + self.assertTrue(results_data, "No benchmark results returned") + + if ( + len(results_data) > 1 + and results_data[0]["batch_size"] == results_data[1]["batch_size"] + ): + report_results = results_data[1:] + else: + report_results = results_data + + summary_lines = [ + "### test_perf_8k_1k (deepseek-v4-flash-fp8)", + "input_len=8192 output_len=1024", + "", + "| batch size | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |", + "| ---------- | ----------- | ------------------------ | ------------------------- | -------- |", + ] + for r in report_results: + bs = r["batch_size"] + latency = r.get("latency", 0.0) + in_tp = r.get("input_throughput", 0.0) + out_tp = r.get("output_throughput", 0.0) + itl = 1 / (out_tp / bs) * 1000 if out_tp > 0 else float("inf") + summary_lines.append( + f"| {bs} | {latency:.2f} | {in_tp:.2f} | {out_tp:.2f} | {itl:.2f} |" + ) + print( + f"bs={bs} latency={latency:.2f}s " + f"in_tp={in_tp:.2f} tok/s out_tp={out_tp:.2f} tok/s ITL={itl:.2f}ms" + ) + + if is_in_ci(): + write_github_step_summary("\n".join(summary_lines) + "\n") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/test_deepseek_v4_pro_fp4.py b/test/registered/amd/test_deepseek_v4_pro_fp4.py new file mode 100644 index 000000000..7ee91300a --- /dev/null +++ b/test/registered/amd/test_deepseek_v4_pro_fp4.py @@ -0,0 +1,209 @@ +"""MI35x DeepSeek-V4-Pro FP4 Test (8-GPU) + +Combined accuracy + performance test for DeepSeek-V4-Pro (1.6T) FP4 on +MI35x ROCm 7.2. +- Accuracy: GSM8K few-shot eval +- Performance: bench_one_batch_server with input_len=8192, output_len=1024 (bs=1) + +Both tests share a single launched server. + +Registry: nightly-amd-8-gpu-mi35x-deepseek-v4-pro suite +""" + +import json +import os +import subprocess +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +register_amd_ci( + est_time=14400, suite="nightly-amd-8-gpu-mi35x-deepseek-v4-pro", nightly=True +) + +DEEPSEEK_V4_PRO_FP4_MODEL_PATH = os.environ.get( + "DEEPSEEK_V4_PRO_MODEL_PATH_FP4", "deepseek-ai/DeepSeek-V4-Pro" +) +# Pro is 1.6T; weight load + warmup is much longer than Flash 285B. +SERVER_LAUNCH_TIMEOUT = 5400 + +# Common DeepSeek-V4 env vars (AMD ROCm 7.2 path: tilelang + AITER + ROCm700A). +# Source of truth: python/run_dsv4.sh. +COMMON_ENV_VARS = { + "SGLANG_OPT_USE_FUSED_COMPRESS": "false", + "SGLANG_OPT_USE_OLD_COMPRESSOR": "true", + "SGLANG_OPT_USE_TILELANG_SWA_PREPARE": "false", + "SGLANG_OPT_USE_JIT_KERNEL_FUSED_TOPK": "false", + "SGLANG_OPT_USE_FUSED_HASH_TOPK": "false", + "SGLANG_OPT_DEEPGEMM_HC_PRENORM": "false", + "SGLANG_OPT_USE_TILELANG_MHC_PRE": "false", + "SGLANG_OPT_USE_TILELANG_MHC_POST": "false", + "SGLANG_ENABLE_THINKING": "1", + "SGLANG_USE_AITER": "1", + "SGLANG_USE_ROCM700A": "1", + "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH": "1", + "SGLANG_OPT_DPSK_V4_RADIX": "0", + "SGLANG_OPT_USE_OVERLAP_STORE_CACHE": "false", + "SGLANG_OPT_USE_FUSED_STORE_CACHE": "false", + "SGLANG_TOPK_TRANSFORM_512_TORCH": "1", + "SGLANG_OPT_USE_TILELANG_INDEXER": "true", + "SGLANG_HACK_FLASHMLA_BACKEND": "tilelang", + "SGLANG_REASONING_EFFORT": "max", +} + +# FP4 variant: FP4 mixed-precision experts. +FP4_ENV_VARS = { + "SGLANG_DSV4_FP4_EXPERTS": "true", + "SGLANG_FORCE_TRITON_MOE_FP8": "0", +} + + +class TestDeepseekV4ProFp4(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEEPSEEK_V4_PRO_FP4_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + + env = os.environ.copy() + env.update(COMMON_ENV_VARS) + env.update(FP4_ENV_VARS) + + other_args = [ + "--trust-remote-code", + "--tp", + "8", + "--disable-radix-cache", + "--attention-backend", + "compressed", + "--max-running-requests", + "256", + "--page-size", + "256", + "--chunked-prefill-size", + "8192", + "--disable-shared-experts-fusion", + "--tool-call-parser", + "deepseekv4", + "--reasoning-parser", + "deepseek-v4", + ] + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=SERVER_LAUNCH_TIMEOUT, + other_args=other_args, + env=env, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_a_gsm8k(self): + # `a` prefix to run first (alphabetical) and warm up the server. + args = SimpleNamespace( + num_shots=8, + data_path=None, + num_questions=1319, + parallel=1319, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_gsm8k (deepseek-v4-pro-fp4)\n" + f'{metrics["accuracy"]=:.3f}\n' + ) + self.assertGreater(metrics["accuracy"], 0.92) + + def test_b_perf_8k_1k(self): + json_output = "/tmp/deepseek_v4_pro_fp4_perf.json" + if os.path.exists(json_output): + os.remove(json_output) + + # First "1" is a warmup; the markdown report below skips it. + batch_sizes = ["1", "1", "2", "4", "8", "16", "32"] + cmd = [ + "python3", + "-m", + "sglang.bench_one_batch_server", + "--model", + "None", + "--base-url", + self.base_url, + "--batch-size", + *batch_sizes, + "--input-len", + "8192", + "--output-len", + "1024", + "--show-report", + f"--pydantic-result-filename={json_output}", + "--no-append-to-github-summary", + "--trust-remote-code", + ] + print(f"Running benchmark: {' '.join(cmd)}") + result = subprocess.run(cmd, capture_output=True, text=True) + print(result.stdout) + if result.returncode != 0: + print(f"STDERR: {result.stderr}") + self.fail(f"bench_one_batch_server failed (rc={result.returncode})") + + self.assertTrue( + os.path.exists(json_output), + f"Benchmark JSON output {json_output} not found", + ) + with open(json_output) as f: + results_data = json.load(f) + self.assertTrue(results_data, "No benchmark results returned") + + if ( + len(results_data) > 1 + and results_data[0]["batch_size"] == results_data[1]["batch_size"] + ): + report_results = results_data[1:] + else: + report_results = results_data + + summary_lines = [ + "### test_perf_8k_1k (deepseek-v4-pro-fp4)", + "input_len=8192 output_len=1024", + "", + "| batch size | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |", + "| ---------- | ----------- | ------------------------ | ------------------------- | -------- |", + ] + for r in report_results: + bs = r["batch_size"] + latency = r.get("latency", 0.0) + in_tp = r.get("input_throughput", 0.0) + out_tp = r.get("output_throughput", 0.0) + itl = 1 / (out_tp / bs) * 1000 if out_tp > 0 else float("inf") + summary_lines.append( + f"| {bs} | {latency:.2f} | {in_tp:.2f} | {out_tp:.2f} | {itl:.2f} |" + ) + print( + f"bs={bs} latency={latency:.2f}s " + f"in_tp={in_tp:.2f} tok/s out_tp={out_tp:.2f} tok/s ITL={itl:.2f}ms" + ) + + if is_in_ci(): + write_github_step_summary("\n".join(summary_lines) + "\n") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/test_deepseek_v4_pro_fp8.py b/test/registered/amd/test_deepseek_v4_pro_fp8.py new file mode 100644 index 000000000..c4595aa3d --- /dev/null +++ b/test/registered/amd/test_deepseek_v4_pro_fp8.py @@ -0,0 +1,209 @@ +"""MI35x DeepSeek-V4-Pro FP8 Test (8-GPU) + +Combined accuracy + performance test for DeepSeek-V4-Pro (1.6T) FP8 on +MI35x ROCm 7.2. +- Accuracy: GSM8K few-shot eval +- Performance: bench_one_batch_server with input_len=8192, output_len=1024 (bs=1) + +Both tests share a single launched server. + +Registry: nightly-amd-8-gpu-mi35x-deepseek-v4-pro suite +""" + +import json +import os +import subprocess +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +register_amd_ci( + est_time=14400, suite="nightly-amd-8-gpu-mi35x-deepseek-v4-pro", nightly=True +) + +DEEPSEEK_V4_PRO_FP8_MODEL_PATH = os.environ.get( + "DEEPSEEK_V4_PRO_MODEL_PATH_FP8", "sgl-project/DeepSeek-V4-Pro-FP8" +) +# Pro is 1.6T; weight load + warmup is much longer than Flash 285B. +SERVER_LAUNCH_TIMEOUT = 5400 + +# Common DeepSeek-V4 env vars (AMD ROCm 7.2 path: tilelang + AITER + ROCm700A). +# Source of truth: python/run_dsv4.sh. +COMMON_ENV_VARS = { + "SGLANG_OPT_USE_FUSED_COMPRESS": "false", + "SGLANG_OPT_USE_OLD_COMPRESSOR": "true", + "SGLANG_OPT_USE_TILELANG_SWA_PREPARE": "false", + "SGLANG_OPT_USE_JIT_KERNEL_FUSED_TOPK": "false", + "SGLANG_OPT_USE_FUSED_HASH_TOPK": "false", + "SGLANG_OPT_DEEPGEMM_HC_PRENORM": "false", + "SGLANG_OPT_USE_TILELANG_MHC_PRE": "false", + "SGLANG_OPT_USE_TILELANG_MHC_POST": "false", + "SGLANG_ENABLE_THINKING": "1", + "SGLANG_USE_AITER": "1", + "SGLANG_USE_ROCM700A": "1", + "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH": "1", + "SGLANG_OPT_DPSK_V4_RADIX": "0", + "SGLANG_OPT_USE_OVERLAP_STORE_CACHE": "false", + "SGLANG_OPT_USE_FUSED_STORE_CACHE": "false", + "SGLANG_TOPK_TRANSFORM_512_TORCH": "1", + "SGLANG_OPT_USE_TILELANG_INDEXER": "true", + "SGLANG_HACK_FLASHMLA_BACKEND": "tilelang", + "SGLANG_REASONING_EFFORT": "max", +} + +# FP8 variant: dense-FP8 experts via the Triton MoE FP8 path. +FP8_ENV_VARS = { + "SGLANG_DSV4_FP4_EXPERTS": "false", + "SGLANG_FORCE_TRITON_MOE_FP8": "1", +} + + +class TestDeepseekV4ProFp8(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEEPSEEK_V4_PRO_FP8_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + + env = os.environ.copy() + env.update(COMMON_ENV_VARS) + env.update(FP8_ENV_VARS) + + other_args = [ + "--trust-remote-code", + "--tp", + "8", + "--disable-radix-cache", + "--attention-backend", + "compressed", + "--max-running-requests", + "256", + "--page-size", + "256", + "--chunked-prefill-size", + "8192", + "--disable-shared-experts-fusion", + "--tool-call-parser", + "deepseekv4", + "--reasoning-parser", + "deepseek-v4", + ] + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=SERVER_LAUNCH_TIMEOUT, + other_args=other_args, + env=env, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_a_gsm8k(self): + # `a` prefix to run first (alphabetical) and warm up the server. + args = SimpleNamespace( + num_shots=8, + data_path=None, + num_questions=1319, + parallel=1319, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_gsm8k (deepseek-v4-pro-fp8)\n" + f'{metrics["accuracy"]=:.3f}\n' + ) + self.assertGreater(metrics["accuracy"], 0.91) + + def test_b_perf_8k_1k(self): + json_output = "/tmp/deepseek_v4_pro_fp8_perf.json" + if os.path.exists(json_output): + os.remove(json_output) + + # First "1" is a warmup; the markdown report below skips it. + batch_sizes = ["1", "1", "2", "4", "8", "16", "32"] + cmd = [ + "python3", + "-m", + "sglang.bench_one_batch_server", + "--model", + "None", + "--base-url", + self.base_url, + "--batch-size", + *batch_sizes, + "--input-len", + "8192", + "--output-len", + "1024", + "--show-report", + f"--pydantic-result-filename={json_output}", + "--no-append-to-github-summary", + "--trust-remote-code", + ] + print(f"Running benchmark: {' '.join(cmd)}") + result = subprocess.run(cmd, capture_output=True, text=True) + print(result.stdout) + if result.returncode != 0: + print(f"STDERR: {result.stderr}") + self.fail(f"bench_one_batch_server failed (rc={result.returncode})") + + self.assertTrue( + os.path.exists(json_output), + f"Benchmark JSON output {json_output} not found", + ) + with open(json_output) as f: + results_data = json.load(f) + self.assertTrue(results_data, "No benchmark results returned") + + if ( + len(results_data) > 1 + and results_data[0]["batch_size"] == results_data[1]["batch_size"] + ): + report_results = results_data[1:] + else: + report_results = results_data + + summary_lines = [ + "### test_perf_8k_1k (deepseek-v4-pro-fp8)", + "input_len=8192 output_len=1024", + "", + "| batch size | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |", + "| ---------- | ----------- | ------------------------ | ------------------------- | -------- |", + ] + for r in report_results: + bs = r["batch_size"] + latency = r.get("latency", 0.0) + in_tp = r.get("input_throughput", 0.0) + out_tp = r.get("output_throughput", 0.0) + itl = 1 / (out_tp / bs) * 1000 if out_tp > 0 else float("inf") + summary_lines.append( + f"| {bs} | {latency:.2f} | {in_tp:.2f} | {out_tp:.2f} | {itl:.2f} |" + ) + print( + f"bs={bs} latency={latency:.2f}s " + f"in_tp={in_tp:.2f} tok/s out_tp={out_tp:.2f} tok/s ITL={itl:.2f}ms" + ) + + if is_in_ci(): + write_github_step_summary("\n".join(summary_lines) + "\n") + + +if __name__ == "__main__": + unittest.main()