minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[AMD] Add GLM-5.2 MI35x nightly accuracy and perf benchmark (#32570)

Browse Source
This commit is contained in:
Michael
2026-08-19 22:49:43 -07:00
committed by GitHub
parent 58e327480a
commit 02b93e7e01
4 changed files with 231 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/nightly-test-amd-rocm720.yml
+21 -10
View File
@@ -82,9 +82,10 @@ on:
- nightly-8-gpu-qwen35-rocm720
- nightly-8-gpu-mi35x-qwen35-rocm720
- nightly-8-gpu-mi35x-qwen35-triton-dcp-rocm720
# 8-GPU GLM-5.1 (MI30x + MI35x)
# 8-GPU GLM-5.1 (MI30x)
- nightly-8-gpu-glm51-rocm720
- nightly-8-gpu-mi35x-glm51-rocm720
# 8-GPU GLM-5.2-FP8 (MI35x accuracy + performance)
- nightly-8-gpu-mi35x-glm52-fp8-rocm720
# 8-GPU GLM-5-MXFP4 (MI35x only)
- nightly-8-gpu-mi35x-glm5-mxfp4-rocm720
# 4-GPU MiniMax-M2.5 (MI35x)
@@ -1852,8 +1853,12 @@ jobs:
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
nightly-8-gpu-mi35x-glm51-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm51-rocm720,'))
# ==============================================================================
# 8-GPU GLM-5.2-FP8 (MI35x accuracy + performance)
# ==============================================================================
nightly-8-gpu-mi35x-glm52-fp8-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm52-fp8-rocm720,'))
runs-on: linux-mi35x-gpu-8
steps:
- name: Checkout code
@@ -1870,30 +1875,35 @@ jobs:
bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
ENABLE_CACHE_HOST: "1"
- name: Install dependencies
run: |
bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate
- name: Accuracy Test MI35x ROCm 7.2 (8-GPU GLM-5.1 DSA)
# The ~700 GB FP8 checkpoint means weight download and load dominate wall
# time well before the eval itself starts.
- name: Accuracy Test MI35x ROCm 7.2 (8-GPU GLM-5.2-FP8 DSA)
timeout-minutes: 180
run: |
> github_summary.md # Clear summary file
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-glm51 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-glm52-fp8 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
- name: Performance Test MI35x ROCm 7.2 (8-GPU GLM-5.1)
# Run perf only after correctness passes. Keeping both steps in this job
# reuses the cached ~700 GB checkpoint and one scarce 8-GPU MI35x slot.
- name: Performance Test MI35x ROCm 7.2 (8-GPU GLM-5.2-FP8)
timeout-minutes: 120
continue-on-error: true
run: |
> github_summary.md # Clear summary file
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-glm51 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-glm52-fp8 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
@@ -2175,9 +2185,10 @@ jobs:
- nightly-8-gpu-qwen35-rocm720
- nightly-8-gpu-mi35x-qwen35-rocm720
- nightly-8-gpu-mi35x-qwen35-triton-dcp-rocm720
# 8-GPU GLM-5.1 (MI30x + MI35x)
# 8-GPU GLM-5.1 (MI30x)
- nightly-8-gpu-glm51-rocm720
- nightly-8-gpu-mi35x-glm51-rocm720
# 8-GPU GLM-5.2-FP8 (MI35x accuracy + performance)
- nightly-8-gpu-mi35x-glm52-fp8-rocm720
# 8-GPU GLM-5-MXFP4 (MI35x only)
- nightly-8-gpu-mi35x-glm5-mxfp4-rocm720
# 4-GPU MiniMax-M2.5 (MI35x)
test/registered/amd/accuracy/mi35x/test_glm52_fp8_eval_mi35x.py
+103
View File
@@ -0,0 +1,103 @@
"""MI35x GLM-5.2-FP8 GSM8K Accuracy Evaluation Test (8-GPU)
Tests zai-org/GLM-5.2-FP8 with the DSA tilelang backend on MI35x (gfx950).
Server arguments track the GLM-5.2 cookbook's MI355X / FP8 / low-latency /
single-node cell: TP8, DSA tilelang prefill+decode, 131072 chunked prefill,
0.80 static memory fraction, and a 20-minute watchdog for weight loading.
That cell ships as `verified: true` with published benchmarks, but nothing in
CI re-checks it, so this nightly is what keeps it honest.
gfx950 is the arch that needs the guard. An earlier ROCm 7.2 miscompile of
aiter's block-FP8 `gemm_a8w8_blockscale_bpreshuffle` GEMM was small per layer
but compounded across all 78 layers: GSM8K collapsed to ~0 while short factual
prompts still looked fine. Only a multi-step reasoning eval catches that class
of regression, and MI300X/MI325X (gfx942) were never affected -- hence MI35x
only.
ROCm 7.2 only, and that is a measured decision rather than a cautious one. Both
lanes were run on current same-day images (`v0.5.16-rocm7{00,20}-mi35x-20260727`)
and 7.2 scored 0.931 while 7.0 scored 0.270. The 7.0 image ships Triton 3.4.0,
which is below the >=3.5.0 the aiter gluon DSA kernels need, so it logs
"falling back to legacy page_size=1 / KVBlockSize=1 path" and that fallback is
what loses the accuracy. Re-add a 7.0 job once its image ships Triton >=3.5.0,
or once the legacy DSA fallback is fixed on gfx950.
The eval matches the CUDA GLM-5.2-FP8 nightly (`test/registered/8-gpu-models/
test_glm52_fp8.py`): same dataset and same 0.92 baseline, so a red run here
means AMD diverged from CUDA rather than the harness diverging.
Headroom is thinner than the cookbook's ~0.96 suggests: this harness measured
0.931 against a 0.92 baseline. The cookbook number comes from a different GSM8K
setup, so the two are not directly comparable, but it does mean a ~1 point
regression trips this test. Treat a marginal failure as worth reading rather
than as noise to retry away.
Accuracy runs before the companion performance benchmark in the same workflow
job. A correctness failure therefore skips performance, while a green result
lets the benchmark reuse the cached checkpoint and the same scarce runner slot.
Registry: nightly-amd-8-gpu-mi35x-glm52-fp8 suite
"""
import unittest
from sglang.test.accuracy_test_runner import AccuracyTestParams
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.run_combined_tests import run_combined_tests
from sglang.test.test_utils import ModelLaunchSettings
# Register for AMD CI - MI35x GLM-5.2-FP8 accuracy test (~90 min: the ~700 GB
# FP8 checkpoint dominates startup, then GSM8K 5-shot at TP8)
register_amd_ci(
est_time=5400,
suite="nightly-amd-8-gpu-mi35x-glm52-fp8",
nightly=True,
)
GLM_52_FP8_MODEL_PATH = "zai-org/GLM-5.2-FP8"
# Fetching and loading a ~700 GB checkpoint against a cold cache is what this
# budget has to cover; the default launch timeout is nowhere near enough.
SERVER_LAUNCH_TIMEOUT = 5400
class TestGLM52FP8EvalMI35x(unittest.TestCase):
"""GLM-5.2-FP8 GSM8K Accuracy Evaluation Test for MI35x."""
def test_glm_52_fp8(self):
"""Run accuracy test for GLM-5.2-FP8."""
cookbook_args = [
"--trust-remote-code",
"--reasoning-parser=glm45",
"--tool-call-parser=glm47",
"--dsa-prefill-backend=tilelang",
"--dsa-decode-backend=tilelang",
"--chunked-prefill-size=131072",
"--mem-fraction-static=0.80",
"--watchdog-timeout=1200",
# Not part of the cookbook cell; purely a load-time win on a
# checkpoint this large, with no effect on numerics.
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
]
variants = [
ModelLaunchSettings(
GLM_52_FP8_MODEL_PATH,
tp_size=8,
extra_args=cookbook_args,
variant="TP8",
launch_timeout=SERVER_LAUNCH_TIMEOUT,
),
]
run_combined_tests(
models=variants,
test_name="GLM-5.2-FP8 (MI35x)",
accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.92),
)
if __name__ == "__main__":
unittest.main()
test/registered/amd/perf/mi35x/test_glm52_fp8_perf_mi35x.py
+106
View File
@@ -0,0 +1,106 @@
"""MI35x nightly performance benchmark for GLM-5.2-FP8 (8-GPU).
Benchmarks zai-org/GLM-5.2-FP8 at TP8 with the cookbook's MI355X / FP8 /
low-latency / single-node server configuration. At 8K-input / 1K-output,
batch 1 and 16 are the concurrency points the cookbook publishes, so drift from
the recipe shows up directly; 8 and 64 extend the sweep to keep the batch range
the retired GLM-5.1 MI35x benchmark used to cover.
This runs after the GLM-5.2 accuracy test in the same ROCm 7.2 job. The
checkpoint is therefore already cached, accuracy gates performance, and the
two tests consume one scarce 8-GPU MI35x slot rather than two.
Registry: nightly-perf-8-gpu-mi35x-glm52-fp8 suite
"""
import os
import unittest
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.nightly_bench_utils import generate_simple_markdown_report
from sglang.test.nightly_utils import NightlyBenchmarkRunner
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
register_amd_ci(
est_time=5400,
suite="nightly-perf-8-gpu-mi35x-glm52-fp8",
nightly=True,
)
GLM_52_FP8_MODEL_PATH = os.environ.get("GLM52_FP8_MODEL_PATH", "zai-org/GLM-5.2-FP8")
RESULT_DIR = "performance_results_glm52_fp8_mi35x"
class TestGLM52FP8PerfMI35x(unittest.TestCase):
"""GLM-5.2-FP8 low-latency throughput on AMD MI35x."""
@classmethod
def setUpClass(cls):
cls.base_url = DEFAULT_URL_FOR_TEST
# The leading 1 is repeated so the report helper drops it as a warmup
# run: this step launches its own server, so the first request pays for
# warmup and batch 1 is the row that distorts most.
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "8192"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "1024"))
cls.model_config = {
"name": "low-latency-tp8",
"model_path": GLM_52_FP8_MODEL_PATH,
"other_args": [
"--trust-remote-code",
"--reasoning-parser",
"glm45",
"--tool-call-parser",
"glm47",
"--tp",
"8",
"--dsa-prefill-backend",
"tilelang",
"--dsa-decode-backend",
"tilelang",
"--chunked-prefill-size",
"131072",
"--mem-fraction-static",
"0.80",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
"--watchdog-timeout",
"1200",
],
}
os.environ.setdefault("SGLANG_BENCH_TIMEOUT", "3600")
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
cls.runner.full_report = f"## {cls.__name__}\n"
def test_glm52_fp8_perf(self):
"""Run the GLM-5.2-FP8 batch-size sweep."""
try:
result_tuple = self.runner.run_benchmark_for_model(
model_path=self.model_config["model_path"],
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=self.model_config["other_args"],
variant=self.model_config["name"],
extra_bench_args=["--trust-remote-code"],
timeout=5400,
)
results, success = result_tuple[0], result_tuple[1]
if results:
self.runner.full_report += (
generate_simple_markdown_report(results, default_gpu_config="MI35x")
+ "\n"
)
self.assertTrue(
success, f"Benchmark failed for {GLM_52_FP8_MODEL_PATH} on MI35x"
)
finally:
self.runner.write_final_report()
if __name__ == "__main__":
unittest.main()
test/run_suite.py
+1
View File
@@ -141,6 +141,7 @@ NIGHTLY_SUITES = {
"nightly-amd-2-gpu-mi35x-deepseek-r1-mxfp4-tp2",
"nightly-amd-8-gpu-mi35x-deepseek-r1-mxfp4-tp4",
"nightly-amd-accuracy-8-gpu-mi35x-kimi-k3",
"nightly-amd-8-gpu-mi35x-glm52-fp8",
"nightly-amd-4-gpu",
"nightly-amd-8-gpu",
"nightly-amd-vlm",