From 02b93e7e01cd0fba348c093d3971a5ba04fedf29 Mon Sep 17 00:00:00 2001 From: Michael <13900043+michaelzhang-ai@users.noreply.github.com> Date: Wed, 19 Aug 2026 22:49:43 -0700 Subject: [PATCH] [AMD] Add GLM-5.2 MI35x nightly accuracy and perf benchmark (#32570) --- .../workflows/nightly-test-amd-rocm720.yml | 31 +++-- .../mi35x/test_glm52_fp8_eval_mi35x.py | 103 +++++++++++++++++ .../perf/mi35x/test_glm52_fp8_perf_mi35x.py | 106 ++++++++++++++++++ test/run_suite.py | 1 + 4 files changed, 231 insertions(+), 10 deletions(-) create mode 100644 test/registered/amd/accuracy/mi35x/test_glm52_fp8_eval_mi35x.py create mode 100644 test/registered/amd/perf/mi35x/test_glm52_fp8_perf_mi35x.py diff --git a/.github/workflows/nightly-test-amd-rocm720.yml b/.github/workflows/nightly-test-amd-rocm720.yml index aefa1ef0c..d6e6e86ec 100644 --- a/.github/workflows/nightly-test-amd-rocm720.yml +++ b/.github/workflows/nightly-test-amd-rocm720.yml @@ -82,9 +82,10 @@ on: - nightly-8-gpu-qwen35-rocm720 - nightly-8-gpu-mi35x-qwen35-rocm720 - nightly-8-gpu-mi35x-qwen35-triton-dcp-rocm720 - # 8-GPU GLM-5.1 (MI30x + MI35x) + # 8-GPU GLM-5.1 (MI30x) - nightly-8-gpu-glm51-rocm720 - - nightly-8-gpu-mi35x-glm51-rocm720 + # 8-GPU GLM-5.2-FP8 (MI35x accuracy + performance) + - nightly-8-gpu-mi35x-glm52-fp8-rocm720 # 8-GPU GLM-5-MXFP4 (MI35x only) - nightly-8-gpu-mi35x-glm5-mxfp4-rocm720 # 4-GPU MiniMax-M2.5 (MI35x) @@ -1852,8 +1853,12 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - nightly-8-gpu-mi35x-glm51-rocm720: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm51-rocm720,')) + # ============================================================================== + # 8-GPU GLM-5.2-FP8 (MI35x accuracy + performance) + # ============================================================================== + + nightly-8-gpu-mi35x-glm52-fp8-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm52-fp8-rocm720,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code @@ -1870,30 +1875,35 @@ jobs: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 env: GITHUB_WORKSPACE: ${{ github.workspace }} + ENABLE_CACHE_HOST: "1" - name: Install dependencies run: | bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - name: Accuracy Test MI35x ROCm 7.2 (8-GPU GLM-5.1 DSA) + # The ~700 GB FP8 checkpoint means weight download and load dominate wall + # time well before the eval itself starts. + - name: Accuracy Test MI35x ROCm 7.2 (8-GPU GLM-5.2-FP8 DSA) timeout-minutes: 180 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-glm51 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-glm52-fp8 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - - name: Performance Test MI35x ROCm 7.2 (8-GPU GLM-5.1) + # Run perf only after correctness passes. Keeping both steps in this job + # reuses the cached ~700 GB checkpoint and one scarce 8-GPU MI35x slot. + - name: Performance Test MI35x ROCm 7.2 (8-GPU GLM-5.2-FP8) timeout-minutes: 120 continue-on-error: true run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-glm51 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-glm52-fp8 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} @@ -2175,9 +2185,10 @@ jobs: - nightly-8-gpu-qwen35-rocm720 - nightly-8-gpu-mi35x-qwen35-rocm720 - nightly-8-gpu-mi35x-qwen35-triton-dcp-rocm720 - # 8-GPU GLM-5.1 (MI30x + MI35x) + # 8-GPU GLM-5.1 (MI30x) - nightly-8-gpu-glm51-rocm720 - - nightly-8-gpu-mi35x-glm51-rocm720 + # 8-GPU GLM-5.2-FP8 (MI35x accuracy + performance) + - nightly-8-gpu-mi35x-glm52-fp8-rocm720 # 8-GPU GLM-5-MXFP4 (MI35x only) - nightly-8-gpu-mi35x-glm5-mxfp4-rocm720 # 4-GPU MiniMax-M2.5 (MI35x) diff --git a/test/registered/amd/accuracy/mi35x/test_glm52_fp8_eval_mi35x.py b/test/registered/amd/accuracy/mi35x/test_glm52_fp8_eval_mi35x.py new file mode 100644 index 000000000..4a0ccd0b5 --- /dev/null +++ b/test/registered/amd/accuracy/mi35x/test_glm52_fp8_eval_mi35x.py @@ -0,0 +1,103 @@ +"""MI35x GLM-5.2-FP8 GSM8K Accuracy Evaluation Test (8-GPU) + +Tests zai-org/GLM-5.2-FP8 with the DSA tilelang backend on MI35x (gfx950). + +Server arguments track the GLM-5.2 cookbook's MI355X / FP8 / low-latency / +single-node cell: TP8, DSA tilelang prefill+decode, 131072 chunked prefill, +0.80 static memory fraction, and a 20-minute watchdog for weight loading. +That cell ships as `verified: true` with published benchmarks, but nothing in +CI re-checks it, so this nightly is what keeps it honest. + +gfx950 is the arch that needs the guard. An earlier ROCm 7.2 miscompile of +aiter's block-FP8 `gemm_a8w8_blockscale_bpreshuffle` GEMM was small per layer +but compounded across all 78 layers: GSM8K collapsed to ~0 while short factual +prompts still looked fine. Only a multi-step reasoning eval catches that class +of regression, and MI300X/MI325X (gfx942) were never affected -- hence MI35x +only. + +ROCm 7.2 only, and that is a measured decision rather than a cautious one. Both +lanes were run on current same-day images (`v0.5.16-rocm7{00,20}-mi35x-20260727`) +and 7.2 scored 0.931 while 7.0 scored 0.270. The 7.0 image ships Triton 3.4.0, +which is below the >=3.5.0 the aiter gluon DSA kernels need, so it logs +"falling back to legacy page_size=1 / KVBlockSize=1 path" and that fallback is +what loses the accuracy. Re-add a 7.0 job once its image ships Triton >=3.5.0, +or once the legacy DSA fallback is fixed on gfx950. + +The eval matches the CUDA GLM-5.2-FP8 nightly (`test/registered/8-gpu-models/ +test_glm52_fp8.py`): same dataset and same 0.92 baseline, so a red run here +means AMD diverged from CUDA rather than the harness diverging. + +Headroom is thinner than the cookbook's ~0.96 suggests: this harness measured +0.931 against a 0.92 baseline. The cookbook number comes from a different GSM8K +setup, so the two are not directly comparable, but it does mean a ~1 point +regression trips this test. Treat a marginal failure as worth reading rather +than as noise to retry away. + +Accuracy runs before the companion performance benchmark in the same workflow +job. A correctness failure therefore skips performance, while a green result +lets the benchmark reuse the cached checkpoint and the same scarce runner slot. + +Registry: nightly-amd-8-gpu-mi35x-glm52-fp8 suite +""" + +import unittest + +from sglang.test.accuracy_test_runner import AccuracyTestParams +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.run_combined_tests import run_combined_tests +from sglang.test.test_utils import ModelLaunchSettings + +# Register for AMD CI - MI35x GLM-5.2-FP8 accuracy test (~90 min: the ~700 GB +# FP8 checkpoint dominates startup, then GSM8K 5-shot at TP8) +register_amd_ci( + est_time=5400, + suite="nightly-amd-8-gpu-mi35x-glm52-fp8", + nightly=True, +) + +GLM_52_FP8_MODEL_PATH = "zai-org/GLM-5.2-FP8" + +# Fetching and loading a ~700 GB checkpoint against a cold cache is what this +# budget has to cover; the default launch timeout is nowhere near enough. +SERVER_LAUNCH_TIMEOUT = 5400 + + +class TestGLM52FP8EvalMI35x(unittest.TestCase): + """GLM-5.2-FP8 GSM8K Accuracy Evaluation Test for MI35x.""" + + def test_glm_52_fp8(self): + """Run accuracy test for GLM-5.2-FP8.""" + cookbook_args = [ + "--trust-remote-code", + "--reasoning-parser=glm45", + "--tool-call-parser=glm47", + "--dsa-prefill-backend=tilelang", + "--dsa-decode-backend=tilelang", + "--chunked-prefill-size=131072", + "--mem-fraction-static=0.80", + "--watchdog-timeout=1200", + # Not part of the cookbook cell; purely a load-time win on a + # checkpoint this large, with no effect on numerics. + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + ] + + variants = [ + ModelLaunchSettings( + GLM_52_FP8_MODEL_PATH, + tp_size=8, + extra_args=cookbook_args, + variant="TP8", + launch_timeout=SERVER_LAUNCH_TIMEOUT, + ), + ] + + run_combined_tests( + models=variants, + test_name="GLM-5.2-FP8 (MI35x)", + accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.92), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/perf/mi35x/test_glm52_fp8_perf_mi35x.py b/test/registered/amd/perf/mi35x/test_glm52_fp8_perf_mi35x.py new file mode 100644 index 000000000..4fe5f6fbf --- /dev/null +++ b/test/registered/amd/perf/mi35x/test_glm52_fp8_perf_mi35x.py @@ -0,0 +1,106 @@ +"""MI35x nightly performance benchmark for GLM-5.2-FP8 (8-GPU). + +Benchmarks zai-org/GLM-5.2-FP8 at TP8 with the cookbook's MI355X / FP8 / +low-latency / single-node server configuration. At 8K-input / 1K-output, +batch 1 and 16 are the concurrency points the cookbook publishes, so drift from +the recipe shows up directly; 8 and 64 extend the sweep to keep the batch range +the retired GLM-5.1 MI35x benchmark used to cover. + +This runs after the GLM-5.2 accuracy test in the same ROCm 7.2 job. The +checkpoint is therefore already cached, accuracy gates performance, and the +two tests consume one scarce 8-GPU MI35x slot rather than two. + +Registry: nightly-perf-8-gpu-mi35x-glm52-fp8 suite +""" + +import os +import unittest + +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.nightly_bench_utils import generate_simple_markdown_report +from sglang.test.nightly_utils import NightlyBenchmarkRunner +from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env + +register_amd_ci( + est_time=5400, + suite="nightly-perf-8-gpu-mi35x-glm52-fp8", + nightly=True, +) + +GLM_52_FP8_MODEL_PATH = os.environ.get("GLM52_FP8_MODEL_PATH", "zai-org/GLM-5.2-FP8") +RESULT_DIR = "performance_results_glm52_fp8_mi35x" + + +class TestGLM52FP8PerfMI35x(unittest.TestCase): + """GLM-5.2-FP8 low-latency throughput on AMD MI35x.""" + + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + # The leading 1 is repeated so the report helper drops it as a warmup + # run: this step launches its own server, so the first request pays for + # warmup and batch 1 is the row that distorts most. + cls.batch_sizes = [1, 1, 8, 16, 64] + cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "8192")) + cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "1024")) + cls.model_config = { + "name": "low-latency-tp8", + "model_path": GLM_52_FP8_MODEL_PATH, + "other_args": [ + "--trust-remote-code", + "--reasoning-parser", + "glm45", + "--tool-call-parser", + "glm47", + "--tp", + "8", + "--dsa-prefill-backend", + "tilelang", + "--dsa-decode-backend", + "tilelang", + "--chunked-prefill-size", + "131072", + "--mem-fraction-static", + "0.80", + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + "--watchdog-timeout", + "1200", + ], + } + + os.environ.setdefault("SGLANG_BENCH_TIMEOUT", "3600") + cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url) + cls.runner.setup_result_directory() + cls.runner.full_report = f"## {cls.__name__}\n" + + def test_glm52_fp8_perf(self): + """Run the GLM-5.2-FP8 batch-size sweep.""" + try: + result_tuple = self.runner.run_benchmark_for_model( + model_path=self.model_config["model_path"], + batch_sizes=self.batch_sizes, + input_lens=self.input_lens, + output_lens=self.output_lens, + other_args=self.model_config["other_args"], + variant=self.model_config["name"], + extra_bench_args=["--trust-remote-code"], + timeout=5400, + ) + results, success = result_tuple[0], result_tuple[1] + + if results: + self.runner.full_report += ( + generate_simple_markdown_report(results, default_gpu_config="MI35x") + + "\n" + ) + + self.assertTrue( + success, f"Benchmark failed for {GLM_52_FP8_MODEL_PATH} on MI35x" + ) + finally: + self.runner.write_final_report() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/run_suite.py b/test/run_suite.py index ef753276e..ebbcb869b 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -141,6 +141,7 @@ NIGHTLY_SUITES = { "nightly-amd-2-gpu-mi35x-deepseek-r1-mxfp4-tp2", "nightly-amd-8-gpu-mi35x-deepseek-r1-mxfp4-tp4", "nightly-amd-accuracy-8-gpu-mi35x-kimi-k3", + "nightly-amd-8-gpu-mi35x-glm52-fp8", "nightly-amd-4-gpu", "nightly-amd-8-gpu", "nightly-amd-vlm",