[AMD] Deepseek v4 Flash / Pro nightly tests for MI35x ROCm 7.2 (#24203)
Co-authored-by: YC Yen-Ching Tseng <yctseng@amd.com>
This commit is contained in:
Bingxu Chen
co-authored by
YC Yen-Ching Tseng
YC Yen-Ching Tseng
parent
aea527afdc
commit
5eff3c489a
@@ -57,6 +57,8 @@ on:
|
||||
- nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp-rocm720
|
||||
- nightly-perf-8-gpu-mi35x-deepseek-v32-basic-rocm720
|
||||
- nightly-perf-8-gpu-mi35x-deepseek-v32-mtp-rocm720
|
||||
- nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720
|
||||
- nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720
|
||||
- nightly-8-gpu-mi35x-kimi-k25-rocm720
|
||||
- nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720
|
||||
- nightly-8-gpu-mi35x-qwen35-rocm720
|
||||
@@ -1413,6 +1415,142 @@ jobs:
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# MI35x 8-GPU DeepSeek-V4-Flash FP8 + FP4 (Accuracy + Performance combined) ROCm 7.2
|
||||
# NOTE on runtime sourcing: the DSv4 docker image (tag suffix `-DSv4`) bakes
|
||||
# in sglang built from a specific commit of the amd/deepseek_v4 branch (the
|
||||
# 7-char sha in the image tag is that commit). To keep the runtime as exactly
|
||||
# that image-frozen sglang/aiter, we pass `--skip-sglang-build` and
|
||||
# `--skip-aiter-build` so install_dependency.sh does NOT `pip install -e
|
||||
# /sglang-checkout/python` (which would override the image's sglang with
|
||||
# whatever this checkout happens to be) and does NOT rebuild aiter from this
|
||||
# checkout's docker/rocm.Dockerfile. The /sglang-checkout mount is still used
|
||||
# for shell scripts and for run_suite.py discovering test files; it does not
|
||||
# poison Python imports because the image's site-packages .pth points at
|
||||
# /sgl-workspace/sglang/python (a different path).
|
||||
nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720,'))
|
||||
runs-on: linux-mi35x-gpu-8
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.sha }}
|
||||
|
||||
- name: Resolve DSv4 image tag
|
||||
id: dsv4_image
|
||||
run: |
|
||||
# Pick the latest Docker Hub tag matching rocm720-mi35x-<sha7>-<YYYYMMDD>-DSv4.
|
||||
# Docker Hub returns results sorted by last_updated DESC by default, so the
|
||||
# first regex match is the most recent daily build.
|
||||
AUTH_HEADER=()
|
||||
if [[ -n "${DOCKERHUB_AMD_USERNAME:-}" && -n "${DOCKERHUB_AMD_TOKEN:-}" ]]; then
|
||||
TOKEN=$(curl -s -H "Content-Type: application/json" \
|
||||
-X POST -d "{\"username\":\"${DOCKERHUB_AMD_USERNAME}\",\"password\":\"${DOCKERHUB_AMD_TOKEN}\"}" \
|
||||
https://hub.docker.com/v2/users/login/ | python3 -c "import json,sys; print(json.load(sys.stdin).get('token',''))")
|
||||
if [[ -n "$TOKEN" ]]; then
|
||||
AUTH_HEADER=(-H "Authorization: JWT $TOKEN")
|
||||
fi
|
||||
fi
|
||||
TAG=$(curl -s "${AUTH_HEADER[@]}" \
|
||||
"https://hub.docker.com/v2/repositories/rocm/sgl-dev/tags?page_size=100&name=DSv4" \
|
||||
| grep -oE '"name":"rocm720-mi35x-[a-f0-9]{7}-[0-9]{8}-DSv4"' \
|
||||
| head -n 1 | cut -d'"' -f4)
|
||||
if [ -z "$TAG" ]; then
|
||||
echo "::error::No DSv4 image found matching rocm720-mi35x-<sha7>-<YYYYMMDD>-DSv4 on Docker Hub"
|
||||
exit 1
|
||||
fi
|
||||
echo "image=rocm/sgl-dev:$TAG" >> "$GITHUB_OUTPUT"
|
||||
echo "Resolved DSv4 image: rocm/sgl-dev:$TAG"
|
||||
|
||||
- name: Setup docker (ROCm 7.2 DSv4)
|
||||
run: |
|
||||
touch github_summary.md
|
||||
bash scripts/ci/amd/amd_ci_start_container.sh --custom-image ${{ steps.dsv4_image.outputs.image }}
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
- name: Install dependencies (preserve DSv4 sglang/aiter from image)
|
||||
run: |
|
||||
# --skip-sglang-build: keep the image's pre-installed DSv4 sglang
|
||||
# (default would `pip install -e /sglang-checkout/python` and clobber it with main's source).
|
||||
# --skip-aiter-build: keep the image's DSv4-tuned aiter
|
||||
# (default reads /sglang-checkout/docker/rocm.Dockerfile from main and rebuilds aiter to that commit).
|
||||
# --skip-test-time-deps: GSM8K + bench_one_batch_server don't need lmms-eval / human-eval.
|
||||
bash scripts/ci/amd/amd_ci_install_dependency.sh \
|
||||
--skip-sglang-build --skip-aiter-build --skip-test-time-deps
|
||||
# tabulate is the only thing run_suite.py imports that may not be in the DSv4 image.
|
||||
bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate
|
||||
|
||||
- name: Accuracy + Performance Test MI35x ROCm 7.2 (8-GPU DeepSeek-V4-Flash FP8 + FP4)
|
||||
timeout-minutes: 300
|
||||
run: |
|
||||
> github_summary.md # Clear summary file
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-flash --nightly --timeout-per-file 7200 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# MI35x 8-GPU DeepSeek-V4-Pro FP8 + FP4 (Accuracy + Performance combined) ROCm 7.2
|
||||
# Pro is 1.6T (vs Flash 285B); load + warmup is much longer, so timeout-per-file
|
||||
# and the job timeout are both larger than the Flash job.
|
||||
# Same image / branch / install strategy as the Flash job above — see the comment
|
||||
# block on `nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720` for the rationale.
|
||||
nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720,'))
|
||||
runs-on: linux-mi35x-gpu-8
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.sha }}
|
||||
|
||||
- name: Resolve DSv4 image tag
|
||||
id: dsv4_image
|
||||
run: |
|
||||
AUTH_HEADER=()
|
||||
if [[ -n "${DOCKERHUB_AMD_USERNAME:-}" && -n "${DOCKERHUB_AMD_TOKEN:-}" ]]; then
|
||||
TOKEN=$(curl -s -H "Content-Type: application/json" \
|
||||
-X POST -d "{\"username\":\"${DOCKERHUB_AMD_USERNAME}\",\"password\":\"${DOCKERHUB_AMD_TOKEN}\"}" \
|
||||
https://hub.docker.com/v2/users/login/ | python3 -c "import json,sys; print(json.load(sys.stdin).get('token',''))")
|
||||
if [[ -n "$TOKEN" ]]; then
|
||||
AUTH_HEADER=(-H "Authorization: JWT $TOKEN")
|
||||
fi
|
||||
fi
|
||||
TAG=$(curl -s "${AUTH_HEADER[@]}" \
|
||||
"https://hub.docker.com/v2/repositories/rocm/sgl-dev/tags?page_size=100&name=DSv4" \
|
||||
| grep -oE '"name":"rocm720-mi35x-[a-f0-9]{7}-[0-9]{8}-DSv4"' \
|
||||
| head -n 1 | cut -d'"' -f4)
|
||||
if [ -z "$TAG" ]; then
|
||||
echo "::error::No DSv4 image found matching rocm720-mi35x-<sha7>-<YYYYMMDD>-DSv4 on Docker Hub"
|
||||
exit 1
|
||||
fi
|
||||
echo "image=rocm/sgl-dev:$TAG" >> "$GITHUB_OUTPUT"
|
||||
echo "Resolved DSv4 image: rocm/sgl-dev:$TAG"
|
||||
|
||||
- name: Setup docker (ROCm 7.2 DSv4)
|
||||
run: |
|
||||
touch github_summary.md
|
||||
bash scripts/ci/amd/amd_ci_start_container.sh --custom-image ${{ steps.dsv4_image.outputs.image }}
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
- name: Install dependencies (preserve DSv4 sglang/aiter from image)
|
||||
run: |
|
||||
bash scripts/ci/amd/amd_ci_install_dependency.sh \
|
||||
--skip-sglang-build --skip-aiter-build --skip-test-time-deps
|
||||
bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate
|
||||
|
||||
- name: Accuracy + Performance Test MI35x ROCm 7.2 (8-GPU DeepSeek-V4-Pro FP8 + FP4)
|
||||
timeout-minutes: 480
|
||||
run: |
|
||||
> github_summary.md # Clear summary file
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-pro --nightly --timeout-per-file 14400 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
check-all-jobs:
|
||||
if: always() && (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch')
|
||||
needs:
|
||||
@@ -1453,6 +1591,8 @@ jobs:
|
||||
- nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp-rocm720
|
||||
- nightly-perf-8-gpu-mi35x-deepseek-v32-basic-rocm720
|
||||
- nightly-perf-8-gpu-mi35x-deepseek-v32-mtp-rocm720
|
||||
- nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720
|
||||
- nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720
|
||||
- nightly-8-gpu-mi35x-kimi-k25-rocm720
|
||||
- nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720
|
||||
- nightly-8-gpu-mi35x-qwen35-rocm720
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
"""MI35x DeepSeek-V4-Flash FP4 Test (8-GPU)
|
||||
|
||||
Combined accuracy + performance test for DeepSeek-V4-Flash FP4 on MI35x ROCm 7.2.
|
||||
- Accuracy: GSM8K few-shot eval
|
||||
- Performance: bench_one_batch_server with input_len=8192, output_len=1024 (bs=1)
|
||||
|
||||
Both tests share a single launched server.
|
||||
|
||||
Registry: nightly-amd-8-gpu-mi35x-deepseek-v4-flash suite
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_amd_ci
|
||||
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
register_amd_ci(
|
||||
est_time=7200, suite="nightly-amd-8-gpu-mi35x-deepseek-v4-flash", nightly=True
|
||||
)
|
||||
|
||||
DEEPSEEK_V4_FP4_MODEL_PATH = os.environ.get(
|
||||
"DEEPSEEK_V4_FP4_MODEL_PATH", "deepseek-ai/DeepSeek-V4-Flash"
|
||||
)
|
||||
SERVER_LAUNCH_TIMEOUT = 3600
|
||||
|
||||
# Common DeepSeek-V4 env vars (AMD ROCm 7.2 path: tilelang + AITER + ROCm700A).
|
||||
# Source of truth: python/run_dsv4.sh.
|
||||
COMMON_ENV_VARS = {
|
||||
"SGLANG_OPT_USE_FUSED_COMPRESS": "false",
|
||||
"SGLANG_OPT_USE_OLD_COMPRESSOR": "true",
|
||||
"SGLANG_OPT_USE_TILELANG_SWA_PREPARE": "false",
|
||||
"SGLANG_OPT_USE_JIT_KERNEL_FUSED_TOPK": "false",
|
||||
"SGLANG_OPT_USE_FUSED_HASH_TOPK": "false",
|
||||
"SGLANG_OPT_DEEPGEMM_HC_PRENORM": "false",
|
||||
"SGLANG_OPT_USE_TILELANG_MHC_PRE": "false",
|
||||
"SGLANG_OPT_USE_TILELANG_MHC_POST": "false",
|
||||
"SGLANG_ENABLE_THINKING": "1",
|
||||
"SGLANG_USE_AITER": "1",
|
||||
"SGLANG_USE_ROCM700A": "1",
|
||||
"SGLANG_FP8_PAGED_MQA_LOGITS_TORCH": "1",
|
||||
"SGLANG_OPT_DPSK_V4_RADIX": "0",
|
||||
"SGLANG_OPT_USE_OVERLAP_STORE_CACHE": "false",
|
||||
"SGLANG_OPT_USE_FUSED_STORE_CACHE": "false",
|
||||
"SGLANG_TOPK_TRANSFORM_512_TORCH": "1",
|
||||
"SGLANG_OPT_USE_TILELANG_INDEXER": "true",
|
||||
"SGLANG_HACK_FLASHMLA_BACKEND": "tilelang",
|
||||
"SGLANG_REASONING_EFFORT": "max",
|
||||
}
|
||||
|
||||
# FP4 variant: FP4 mixed-precision experts.
|
||||
FP4_ENV_VARS = {
|
||||
"SGLANG_DSV4_FP4_EXPERTS": "true",
|
||||
"SGLANG_FORCE_TRITON_MOE_FP8": "0",
|
||||
}
|
||||
|
||||
|
||||
class TestDeepseekV4Fp4(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = DEEPSEEK_V4_FP4_MODEL_PATH
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update(COMMON_ENV_VARS)
|
||||
env.update(FP4_ENV_VARS)
|
||||
|
||||
other_args = [
|
||||
"--trust-remote-code",
|
||||
"--tp",
|
||||
"8",
|
||||
"--disable-radix-cache",
|
||||
"--attention-backend",
|
||||
"compressed",
|
||||
"--max-running-requests",
|
||||
"256",
|
||||
"--page-size",
|
||||
"256",
|
||||
"--chunked-prefill-size",
|
||||
"8192",
|
||||
"--disable-shared-experts-fusion",
|
||||
"--tool-call-parser",
|
||||
"deepseekv4",
|
||||
"--reasoning-parser",
|
||||
"deepseek-v4",
|
||||
]
|
||||
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=SERVER_LAUNCH_TIMEOUT,
|
||||
other_args=other_args,
|
||||
env=env,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
kill_process_tree(cls.process.pid)
|
||||
|
||||
def test_a_gsm8k(self):
|
||||
# `a` prefix to run first (alphabetical) and warm up the server.
|
||||
args = SimpleNamespace(
|
||||
num_shots=8,
|
||||
data_path=None,
|
||||
num_questions=1319,
|
||||
parallel=1319,
|
||||
max_new_tokens=512,
|
||||
host="http://127.0.0.1",
|
||||
port=int(self.base_url.split(":")[-1]),
|
||||
)
|
||||
metrics = run_eval_few_shot_gsm8k(args)
|
||||
print(f"{metrics=}")
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_gsm8k (deepseek-v4-flash-fp4)\n"
|
||||
f'{metrics["accuracy"]=:.3f}\n'
|
||||
)
|
||||
self.assertGreater(metrics["accuracy"], 0.91)
|
||||
|
||||
def test_b_perf_8k_1k(self):
|
||||
json_output = "/tmp/deepseek_v4_flash_fp4_perf.json"
|
||||
if os.path.exists(json_output):
|
||||
os.remove(json_output)
|
||||
|
||||
# First "1" is a warmup; the markdown report below skips it.
|
||||
batch_sizes = ["1", "1", "2", "4", "8", "16", "32"]
|
||||
cmd = [
|
||||
"python3",
|
||||
"-m",
|
||||
"sglang.bench_one_batch_server",
|
||||
"--model",
|
||||
"None",
|
||||
"--base-url",
|
||||
self.base_url,
|
||||
"--batch-size",
|
||||
*batch_sizes,
|
||||
"--input-len",
|
||||
"8192",
|
||||
"--output-len",
|
||||
"1024",
|
||||
"--show-report",
|
||||
f"--pydantic-result-filename={json_output}",
|
||||
"--no-append-to-github-summary",
|
||||
"--trust-remote-code",
|
||||
]
|
||||
print(f"Running benchmark: {' '.join(cmd)}")
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
print(result.stdout)
|
||||
if result.returncode != 0:
|
||||
print(f"STDERR: {result.stderr}")
|
||||
self.fail(f"bench_one_batch_server failed (rc={result.returncode})")
|
||||
|
||||
self.assertTrue(
|
||||
os.path.exists(json_output),
|
||||
f"Benchmark JSON output {json_output} not found",
|
||||
)
|
||||
with open(json_output) as f:
|
||||
results_data = json.load(f)
|
||||
self.assertTrue(results_data, "No benchmark results returned")
|
||||
|
||||
if (
|
||||
len(results_data) > 1
|
||||
and results_data[0]["batch_size"] == results_data[1]["batch_size"]
|
||||
):
|
||||
report_results = results_data[1:]
|
||||
else:
|
||||
report_results = results_data
|
||||
|
||||
summary_lines = [
|
||||
"### test_perf_8k_1k (deepseek-v4-flash-fp4)",
|
||||
"input_len=8192 output_len=1024",
|
||||
"",
|
||||
"| batch size | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |",
|
||||
"| ---------- | ----------- | ------------------------ | ------------------------- | -------- |",
|
||||
]
|
||||
for r in report_results:
|
||||
bs = r["batch_size"]
|
||||
latency = r.get("latency", 0.0)
|
||||
in_tp = r.get("input_throughput", 0.0)
|
||||
out_tp = r.get("output_throughput", 0.0)
|
||||
itl = 1 / (out_tp / bs) * 1000 if out_tp > 0 else float("inf")
|
||||
summary_lines.append(
|
||||
f"| {bs} | {latency:.2f} | {in_tp:.2f} | {out_tp:.2f} | {itl:.2f} |"
|
||||
)
|
||||
print(
|
||||
f"bs={bs} latency={latency:.2f}s "
|
||||
f"in_tp={in_tp:.2f} tok/s out_tp={out_tp:.2f} tok/s ITL={itl:.2f}ms"
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary("\n".join(summary_lines) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,207 @@
|
||||
"""MI35x DeepSeek-V4-Flash FP8 Test (8-GPU)
|
||||
|
||||
Combined accuracy + performance test for DeepSeek-V4-Flash FP8 on MI35x ROCm 7.2.
|
||||
- Accuracy: GSM8K few-shot eval
|
||||
- Performance: bench_one_batch_server with input_len=8192, output_len=1024 (bs=1)
|
||||
|
||||
Both tests share a single launched server.
|
||||
|
||||
Registry: nightly-amd-8-gpu-mi35x-deepseek-v4-flash suite
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_amd_ci
|
||||
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
register_amd_ci(
|
||||
est_time=7200, suite="nightly-amd-8-gpu-mi35x-deepseek-v4-flash", nightly=True
|
||||
)
|
||||
|
||||
DEEPSEEK_V4_FP8_MODEL_PATH = os.environ.get(
|
||||
"DEEPSEEK_V4_FP8_MODEL_PATH", "sgl-project/DeepSeek-V4-Flash-FP8"
|
||||
)
|
||||
SERVER_LAUNCH_TIMEOUT = 3600
|
||||
|
||||
# Common DeepSeek-V4 env vars (AMD ROCm 7.2 path: tilelang + AITER + ROCm700A).
|
||||
# Source of truth: python/run_dsv4.sh.
|
||||
COMMON_ENV_VARS = {
|
||||
"SGLANG_OPT_USE_FUSED_COMPRESS": "false",
|
||||
"SGLANG_OPT_USE_OLD_COMPRESSOR": "true",
|
||||
"SGLANG_OPT_USE_TILELANG_SWA_PREPARE": "false",
|
||||
"SGLANG_OPT_USE_JIT_KERNEL_FUSED_TOPK": "false",
|
||||
"SGLANG_OPT_USE_FUSED_HASH_TOPK": "false",
|
||||
"SGLANG_OPT_DEEPGEMM_HC_PRENORM": "false",
|
||||
"SGLANG_OPT_USE_TILELANG_MHC_PRE": "false",
|
||||
"SGLANG_OPT_USE_TILELANG_MHC_POST": "false",
|
||||
"SGLANG_ENABLE_THINKING": "1",
|
||||
"SGLANG_USE_AITER": "1",
|
||||
"SGLANG_USE_ROCM700A": "1",
|
||||
"SGLANG_FP8_PAGED_MQA_LOGITS_TORCH": "1",
|
||||
"SGLANG_OPT_DPSK_V4_RADIX": "0",
|
||||
"SGLANG_OPT_USE_OVERLAP_STORE_CACHE": "false",
|
||||
"SGLANG_OPT_USE_FUSED_STORE_CACHE": "false",
|
||||
"SGLANG_TOPK_TRANSFORM_512_TORCH": "1",
|
||||
"SGLANG_OPT_USE_TILELANG_INDEXER": "true",
|
||||
"SGLANG_HACK_FLASHMLA_BACKEND": "tilelang",
|
||||
"SGLANG_REASONING_EFFORT": "max",
|
||||
}
|
||||
|
||||
# FP8 variant: dense-FP8 experts via the Triton MoE FP8 path.
|
||||
FP8_ENV_VARS = {
|
||||
"SGLANG_DSV4_FP4_EXPERTS": "false",
|
||||
"SGLANG_FORCE_TRITON_MOE_FP8": "1",
|
||||
}
|
||||
|
||||
|
||||
class TestDeepseekV4Fp8(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = DEEPSEEK_V4_FP8_MODEL_PATH
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update(COMMON_ENV_VARS)
|
||||
env.update(FP8_ENV_VARS)
|
||||
|
||||
other_args = [
|
||||
"--trust-remote-code",
|
||||
"--tp",
|
||||
"8",
|
||||
"--disable-radix-cache",
|
||||
"--attention-backend",
|
||||
"compressed",
|
||||
"--max-running-requests",
|
||||
"256",
|
||||
"--page-size",
|
||||
"256",
|
||||
"--chunked-prefill-size",
|
||||
"8192",
|
||||
"--disable-shared-experts-fusion",
|
||||
"--tool-call-parser",
|
||||
"deepseekv4",
|
||||
"--reasoning-parser",
|
||||
"deepseek-v4",
|
||||
]
|
||||
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=SERVER_LAUNCH_TIMEOUT,
|
||||
other_args=other_args,
|
||||
env=env,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
kill_process_tree(cls.process.pid)
|
||||
|
||||
def test_a_gsm8k(self):
|
||||
# `a` prefix to run first (alphabetical) and warm up the server.
|
||||
args = SimpleNamespace(
|
||||
num_shots=8,
|
||||
data_path=None,
|
||||
num_questions=1319,
|
||||
parallel=1319,
|
||||
max_new_tokens=512,
|
||||
host="http://127.0.0.1",
|
||||
port=int(self.base_url.split(":")[-1]),
|
||||
)
|
||||
metrics = run_eval_few_shot_gsm8k(args)
|
||||
print(f"{metrics=}")
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_gsm8k (deepseek-v4-flash-fp8)\n"
|
||||
f'{metrics["accuracy"]=:.3f}\n'
|
||||
)
|
||||
self.assertGreater(metrics["accuracy"], 0.91)
|
||||
|
||||
def test_b_perf_8k_1k(self):
|
||||
json_output = "/tmp/deepseek_v4_flash_fp8_perf.json"
|
||||
if os.path.exists(json_output):
|
||||
os.remove(json_output)
|
||||
|
||||
# First "1" is a warmup; the markdown report below skips it.
|
||||
batch_sizes = ["1", "1", "2", "4", "8", "16", "32"]
|
||||
cmd = [
|
||||
"python3",
|
||||
"-m",
|
||||
"sglang.bench_one_batch_server",
|
||||
"--model",
|
||||
"None",
|
||||
"--base-url",
|
||||
self.base_url,
|
||||
"--batch-size",
|
||||
*batch_sizes,
|
||||
"--input-len",
|
||||
"8192",
|
||||
"--output-len",
|
||||
"1024",
|
||||
"--show-report",
|
||||
f"--pydantic-result-filename={json_output}",
|
||||
"--no-append-to-github-summary",
|
||||
"--trust-remote-code",
|
||||
]
|
||||
print(f"Running benchmark: {' '.join(cmd)}")
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
print(result.stdout)
|
||||
if result.returncode != 0:
|
||||
print(f"STDERR: {result.stderr}")
|
||||
self.fail(f"bench_one_batch_server failed (rc={result.returncode})")
|
||||
|
||||
self.assertTrue(
|
||||
os.path.exists(json_output),
|
||||
f"Benchmark JSON output {json_output} not found",
|
||||
)
|
||||
with open(json_output) as f:
|
||||
results_data = json.load(f)
|
||||
self.assertTrue(results_data, "No benchmark results returned")
|
||||
|
||||
if (
|
||||
len(results_data) > 1
|
||||
and results_data[0]["batch_size"] == results_data[1]["batch_size"]
|
||||
):
|
||||
report_results = results_data[1:]
|
||||
else:
|
||||
report_results = results_data
|
||||
|
||||
summary_lines = [
|
||||
"### test_perf_8k_1k (deepseek-v4-flash-fp8)",
|
||||
"input_len=8192 output_len=1024",
|
||||
"",
|
||||
"| batch size | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |",
|
||||
"| ---------- | ----------- | ------------------------ | ------------------------- | -------- |",
|
||||
]
|
||||
for r in report_results:
|
||||
bs = r["batch_size"]
|
||||
latency = r.get("latency", 0.0)
|
||||
in_tp = r.get("input_throughput", 0.0)
|
||||
out_tp = r.get("output_throughput", 0.0)
|
||||
itl = 1 / (out_tp / bs) * 1000 if out_tp > 0 else float("inf")
|
||||
summary_lines.append(
|
||||
f"| {bs} | {latency:.2f} | {in_tp:.2f} | {out_tp:.2f} | {itl:.2f} |"
|
||||
)
|
||||
print(
|
||||
f"bs={bs} latency={latency:.2f}s "
|
||||
f"in_tp={in_tp:.2f} tok/s out_tp={out_tp:.2f} tok/s ITL={itl:.2f}ms"
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary("\n".join(summary_lines) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,209 @@
|
||||
"""MI35x DeepSeek-V4-Pro FP4 Test (8-GPU)
|
||||
|
||||
Combined accuracy + performance test for DeepSeek-V4-Pro (1.6T) FP4 on
|
||||
MI35x ROCm 7.2.
|
||||
- Accuracy: GSM8K few-shot eval
|
||||
- Performance: bench_one_batch_server with input_len=8192, output_len=1024 (bs=1)
|
||||
|
||||
Both tests share a single launched server.
|
||||
|
||||
Registry: nightly-amd-8-gpu-mi35x-deepseek-v4-pro suite
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_amd_ci
|
||||
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
register_amd_ci(
|
||||
est_time=14400, suite="nightly-amd-8-gpu-mi35x-deepseek-v4-pro", nightly=True
|
||||
)
|
||||
|
||||
DEEPSEEK_V4_PRO_FP4_MODEL_PATH = os.environ.get(
|
||||
"DEEPSEEK_V4_PRO_MODEL_PATH_FP4", "deepseek-ai/DeepSeek-V4-Pro"
|
||||
)
|
||||
# Pro is 1.6T; weight load + warmup is much longer than Flash 285B.
|
||||
SERVER_LAUNCH_TIMEOUT = 5400
|
||||
|
||||
# Common DeepSeek-V4 env vars (AMD ROCm 7.2 path: tilelang + AITER + ROCm700A).
|
||||
# Source of truth: python/run_dsv4.sh.
|
||||
COMMON_ENV_VARS = {
|
||||
"SGLANG_OPT_USE_FUSED_COMPRESS": "false",
|
||||
"SGLANG_OPT_USE_OLD_COMPRESSOR": "true",
|
||||
"SGLANG_OPT_USE_TILELANG_SWA_PREPARE": "false",
|
||||
"SGLANG_OPT_USE_JIT_KERNEL_FUSED_TOPK": "false",
|
||||
"SGLANG_OPT_USE_FUSED_HASH_TOPK": "false",
|
||||
"SGLANG_OPT_DEEPGEMM_HC_PRENORM": "false",
|
||||
"SGLANG_OPT_USE_TILELANG_MHC_PRE": "false",
|
||||
"SGLANG_OPT_USE_TILELANG_MHC_POST": "false",
|
||||
"SGLANG_ENABLE_THINKING": "1",
|
||||
"SGLANG_USE_AITER": "1",
|
||||
"SGLANG_USE_ROCM700A": "1",
|
||||
"SGLANG_FP8_PAGED_MQA_LOGITS_TORCH": "1",
|
||||
"SGLANG_OPT_DPSK_V4_RADIX": "0",
|
||||
"SGLANG_OPT_USE_OVERLAP_STORE_CACHE": "false",
|
||||
"SGLANG_OPT_USE_FUSED_STORE_CACHE": "false",
|
||||
"SGLANG_TOPK_TRANSFORM_512_TORCH": "1",
|
||||
"SGLANG_OPT_USE_TILELANG_INDEXER": "true",
|
||||
"SGLANG_HACK_FLASHMLA_BACKEND": "tilelang",
|
||||
"SGLANG_REASONING_EFFORT": "max",
|
||||
}
|
||||
|
||||
# FP4 variant: FP4 mixed-precision experts.
|
||||
FP4_ENV_VARS = {
|
||||
"SGLANG_DSV4_FP4_EXPERTS": "true",
|
||||
"SGLANG_FORCE_TRITON_MOE_FP8": "0",
|
||||
}
|
||||
|
||||
|
||||
class TestDeepseekV4ProFp4(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = DEEPSEEK_V4_PRO_FP4_MODEL_PATH
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update(COMMON_ENV_VARS)
|
||||
env.update(FP4_ENV_VARS)
|
||||
|
||||
other_args = [
|
||||
"--trust-remote-code",
|
||||
"--tp",
|
||||
"8",
|
||||
"--disable-radix-cache",
|
||||
"--attention-backend",
|
||||
"compressed",
|
||||
"--max-running-requests",
|
||||
"256",
|
||||
"--page-size",
|
||||
"256",
|
||||
"--chunked-prefill-size",
|
||||
"8192",
|
||||
"--disable-shared-experts-fusion",
|
||||
"--tool-call-parser",
|
||||
"deepseekv4",
|
||||
"--reasoning-parser",
|
||||
"deepseek-v4",
|
||||
]
|
||||
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=SERVER_LAUNCH_TIMEOUT,
|
||||
other_args=other_args,
|
||||
env=env,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
kill_process_tree(cls.process.pid)
|
||||
|
||||
def test_a_gsm8k(self):
|
||||
# `a` prefix to run first (alphabetical) and warm up the server.
|
||||
args = SimpleNamespace(
|
||||
num_shots=8,
|
||||
data_path=None,
|
||||
num_questions=1319,
|
||||
parallel=1319,
|
||||
max_new_tokens=512,
|
||||
host="http://127.0.0.1",
|
||||
port=int(self.base_url.split(":")[-1]),
|
||||
)
|
||||
metrics = run_eval_few_shot_gsm8k(args)
|
||||
print(f"{metrics=}")
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_gsm8k (deepseek-v4-pro-fp4)\n"
|
||||
f'{metrics["accuracy"]=:.3f}\n'
|
||||
)
|
||||
self.assertGreater(metrics["accuracy"], 0.92)
|
||||
|
||||
def test_b_perf_8k_1k(self):
|
||||
json_output = "/tmp/deepseek_v4_pro_fp4_perf.json"
|
||||
if os.path.exists(json_output):
|
||||
os.remove(json_output)
|
||||
|
||||
# First "1" is a warmup; the markdown report below skips it.
|
||||
batch_sizes = ["1", "1", "2", "4", "8", "16", "32"]
|
||||
cmd = [
|
||||
"python3",
|
||||
"-m",
|
||||
"sglang.bench_one_batch_server",
|
||||
"--model",
|
||||
"None",
|
||||
"--base-url",
|
||||
self.base_url,
|
||||
"--batch-size",
|
||||
*batch_sizes,
|
||||
"--input-len",
|
||||
"8192",
|
||||
"--output-len",
|
||||
"1024",
|
||||
"--show-report",
|
||||
f"--pydantic-result-filename={json_output}",
|
||||
"--no-append-to-github-summary",
|
||||
"--trust-remote-code",
|
||||
]
|
||||
print(f"Running benchmark: {' '.join(cmd)}")
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
print(result.stdout)
|
||||
if result.returncode != 0:
|
||||
print(f"STDERR: {result.stderr}")
|
||||
self.fail(f"bench_one_batch_server failed (rc={result.returncode})")
|
||||
|
||||
self.assertTrue(
|
||||
os.path.exists(json_output),
|
||||
f"Benchmark JSON output {json_output} not found",
|
||||
)
|
||||
with open(json_output) as f:
|
||||
results_data = json.load(f)
|
||||
self.assertTrue(results_data, "No benchmark results returned")
|
||||
|
||||
if (
|
||||
len(results_data) > 1
|
||||
and results_data[0]["batch_size"] == results_data[1]["batch_size"]
|
||||
):
|
||||
report_results = results_data[1:]
|
||||
else:
|
||||
report_results = results_data
|
||||
|
||||
summary_lines = [
|
||||
"### test_perf_8k_1k (deepseek-v4-pro-fp4)",
|
||||
"input_len=8192 output_len=1024",
|
||||
"",
|
||||
"| batch size | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |",
|
||||
"| ---------- | ----------- | ------------------------ | ------------------------- | -------- |",
|
||||
]
|
||||
for r in report_results:
|
||||
bs = r["batch_size"]
|
||||
latency = r.get("latency", 0.0)
|
||||
in_tp = r.get("input_throughput", 0.0)
|
||||
out_tp = r.get("output_throughput", 0.0)
|
||||
itl = 1 / (out_tp / bs) * 1000 if out_tp > 0 else float("inf")
|
||||
summary_lines.append(
|
||||
f"| {bs} | {latency:.2f} | {in_tp:.2f} | {out_tp:.2f} | {itl:.2f} |"
|
||||
)
|
||||
print(
|
||||
f"bs={bs} latency={latency:.2f}s "
|
||||
f"in_tp={in_tp:.2f} tok/s out_tp={out_tp:.2f} tok/s ITL={itl:.2f}ms"
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary("\n".join(summary_lines) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,209 @@
|
||||
"""MI35x DeepSeek-V4-Pro FP8 Test (8-GPU)
|
||||
|
||||
Combined accuracy + performance test for DeepSeek-V4-Pro (1.6T) FP8 on
|
||||
MI35x ROCm 7.2.
|
||||
- Accuracy: GSM8K few-shot eval
|
||||
- Performance: bench_one_batch_server with input_len=8192, output_len=1024 (bs=1)
|
||||
|
||||
Both tests share a single launched server.
|
||||
|
||||
Registry: nightly-amd-8-gpu-mi35x-deepseek-v4-pro suite
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_amd_ci
|
||||
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
register_amd_ci(
|
||||
est_time=14400, suite="nightly-amd-8-gpu-mi35x-deepseek-v4-pro", nightly=True
|
||||
)
|
||||
|
||||
DEEPSEEK_V4_PRO_FP8_MODEL_PATH = os.environ.get(
|
||||
"DEEPSEEK_V4_PRO_MODEL_PATH_FP8", "sgl-project/DeepSeek-V4-Pro-FP8"
|
||||
)
|
||||
# Pro is 1.6T; weight load + warmup is much longer than Flash 285B.
|
||||
SERVER_LAUNCH_TIMEOUT = 5400
|
||||
|
||||
# Common DeepSeek-V4 env vars (AMD ROCm 7.2 path: tilelang + AITER + ROCm700A).
|
||||
# Source of truth: python/run_dsv4.sh.
|
||||
COMMON_ENV_VARS = {
|
||||
"SGLANG_OPT_USE_FUSED_COMPRESS": "false",
|
||||
"SGLANG_OPT_USE_OLD_COMPRESSOR": "true",
|
||||
"SGLANG_OPT_USE_TILELANG_SWA_PREPARE": "false",
|
||||
"SGLANG_OPT_USE_JIT_KERNEL_FUSED_TOPK": "false",
|
||||
"SGLANG_OPT_USE_FUSED_HASH_TOPK": "false",
|
||||
"SGLANG_OPT_DEEPGEMM_HC_PRENORM": "false",
|
||||
"SGLANG_OPT_USE_TILELANG_MHC_PRE": "false",
|
||||
"SGLANG_OPT_USE_TILELANG_MHC_POST": "false",
|
||||
"SGLANG_ENABLE_THINKING": "1",
|
||||
"SGLANG_USE_AITER": "1",
|
||||
"SGLANG_USE_ROCM700A": "1",
|
||||
"SGLANG_FP8_PAGED_MQA_LOGITS_TORCH": "1",
|
||||
"SGLANG_OPT_DPSK_V4_RADIX": "0",
|
||||
"SGLANG_OPT_USE_OVERLAP_STORE_CACHE": "false",
|
||||
"SGLANG_OPT_USE_FUSED_STORE_CACHE": "false",
|
||||
"SGLANG_TOPK_TRANSFORM_512_TORCH": "1",
|
||||
"SGLANG_OPT_USE_TILELANG_INDEXER": "true",
|
||||
"SGLANG_HACK_FLASHMLA_BACKEND": "tilelang",
|
||||
"SGLANG_REASONING_EFFORT": "max",
|
||||
}
|
||||
|
||||
# FP8 variant: dense-FP8 experts via the Triton MoE FP8 path.
|
||||
FP8_ENV_VARS = {
|
||||
"SGLANG_DSV4_FP4_EXPERTS": "false",
|
||||
"SGLANG_FORCE_TRITON_MOE_FP8": "1",
|
||||
}
|
||||
|
||||
|
||||
class TestDeepseekV4ProFp8(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = DEEPSEEK_V4_PRO_FP8_MODEL_PATH
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update(COMMON_ENV_VARS)
|
||||
env.update(FP8_ENV_VARS)
|
||||
|
||||
other_args = [
|
||||
"--trust-remote-code",
|
||||
"--tp",
|
||||
"8",
|
||||
"--disable-radix-cache",
|
||||
"--attention-backend",
|
||||
"compressed",
|
||||
"--max-running-requests",
|
||||
"256",
|
||||
"--page-size",
|
||||
"256",
|
||||
"--chunked-prefill-size",
|
||||
"8192",
|
||||
"--disable-shared-experts-fusion",
|
||||
"--tool-call-parser",
|
||||
"deepseekv4",
|
||||
"--reasoning-parser",
|
||||
"deepseek-v4",
|
||||
]
|
||||
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=SERVER_LAUNCH_TIMEOUT,
|
||||
other_args=other_args,
|
||||
env=env,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
kill_process_tree(cls.process.pid)
|
||||
|
||||
def test_a_gsm8k(self):
|
||||
# `a` prefix to run first (alphabetical) and warm up the server.
|
||||
args = SimpleNamespace(
|
||||
num_shots=8,
|
||||
data_path=None,
|
||||
num_questions=1319,
|
||||
parallel=1319,
|
||||
max_new_tokens=512,
|
||||
host="http://127.0.0.1",
|
||||
port=int(self.base_url.split(":")[-1]),
|
||||
)
|
||||
metrics = run_eval_few_shot_gsm8k(args)
|
||||
print(f"{metrics=}")
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_gsm8k (deepseek-v4-pro-fp8)\n"
|
||||
f'{metrics["accuracy"]=:.3f}\n'
|
||||
)
|
||||
self.assertGreater(metrics["accuracy"], 0.91)
|
||||
|
||||
def test_b_perf_8k_1k(self):
|
||||
json_output = "/tmp/deepseek_v4_pro_fp8_perf.json"
|
||||
if os.path.exists(json_output):
|
||||
os.remove(json_output)
|
||||
|
||||
# First "1" is a warmup; the markdown report below skips it.
|
||||
batch_sizes = ["1", "1", "2", "4", "8", "16", "32"]
|
||||
cmd = [
|
||||
"python3",
|
||||
"-m",
|
||||
"sglang.bench_one_batch_server",
|
||||
"--model",
|
||||
"None",
|
||||
"--base-url",
|
||||
self.base_url,
|
||||
"--batch-size",
|
||||
*batch_sizes,
|
||||
"--input-len",
|
||||
"8192",
|
||||
"--output-len",
|
||||
"1024",
|
||||
"--show-report",
|
||||
f"--pydantic-result-filename={json_output}",
|
||||
"--no-append-to-github-summary",
|
||||
"--trust-remote-code",
|
||||
]
|
||||
print(f"Running benchmark: {' '.join(cmd)}")
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
print(result.stdout)
|
||||
if result.returncode != 0:
|
||||
print(f"STDERR: {result.stderr}")
|
||||
self.fail(f"bench_one_batch_server failed (rc={result.returncode})")
|
||||
|
||||
self.assertTrue(
|
||||
os.path.exists(json_output),
|
||||
f"Benchmark JSON output {json_output} not found",
|
||||
)
|
||||
with open(json_output) as f:
|
||||
results_data = json.load(f)
|
||||
self.assertTrue(results_data, "No benchmark results returned")
|
||||
|
||||
if (
|
||||
len(results_data) > 1
|
||||
and results_data[0]["batch_size"] == results_data[1]["batch_size"]
|
||||
):
|
||||
report_results = results_data[1:]
|
||||
else:
|
||||
report_results = results_data
|
||||
|
||||
summary_lines = [
|
||||
"### test_perf_8k_1k (deepseek-v4-pro-fp8)",
|
||||
"input_len=8192 output_len=1024",
|
||||
"",
|
||||
"| batch size | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |",
|
||||
"| ---------- | ----------- | ------------------------ | ------------------------- | -------- |",
|
||||
]
|
||||
for r in report_results:
|
||||
bs = r["batch_size"]
|
||||
latency = r.get("latency", 0.0)
|
||||
in_tp = r.get("input_throughput", 0.0)
|
||||
out_tp = r.get("output_throughput", 0.0)
|
||||
itl = 1 / (out_tp / bs) * 1000 if out_tp > 0 else float("inf")
|
||||
summary_lines.append(
|
||||
f"| {bs} | {latency:.2f} | {in_tp:.2f} | {out_tp:.2f} | {itl:.2f} |"
|
||||
)
|
||||
print(
|
||||
f"bs={bs} latency={latency:.2f}s "
|
||||
f"in_tp={in_tp:.2f} tok/s out_tp={out_tp:.2f} tok/s ITL={itl:.2f}ms"
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary("\n".join(summary_lines) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user