diff --git a/.github/workflows/nightly-test-amd-rocm720.yml b/.github/workflows/nightly-test-amd-rocm720.yml index 253d50b8f..fbea53065 100644 --- a/.github/workflows/nightly-test-amd-rocm720.yml +++ b/.github/workflows/nightly-test-amd-rocm720.yml @@ -96,9 +96,7 @@ on: - nightly-8-gpu-mi35x-glm52-fp8-rocm720 # 8-GPU GLM-5-MXFP4 (MI35x only) - nightly-8-gpu-mi35x-glm5-mxfp4-rocm720 - # 4-GPU MiniMax-M2.5 (MI35x) - - nightly-4-gpu-mi35x-minimax-m25-rocm720 - # 4-GPU MiniMax-M3 MXFP8 (MI35x) + # 4-GPU MiniMax-M3 MXFP8 (MI35x accuracy + performance) - nightly-4-gpu-mi35x-minimax-m3-rocm720 # 8-GPU MiniMax-M2.7 (MI30x only) - nightly-8-gpu-minimax-m27-rocm720 @@ -2164,51 +2162,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} # ============================================================================== - # 4-GPU MiniMax-M2.5 (MI35x) - # ============================================================================== - - nightly-4-gpu-mi35x-minimax-m25-rocm720: - name: ${{ format('nightly-4-gpu-mi35x-minimax-m25 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} - strategy: - fail-fast: false - matrix: - rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }} - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu-mi35x-minimax-m25-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (${{ matrix.rocm_version }}) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - ENABLE_CACHE_HOST: "1" - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - - name: Accuracy Test MI35x ROCm 7.2 (4-GPU MiniMax-M2.5) - timeout-minutes: 120 - run: | - > github_summary.md - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_USE_AITER=1 \ - -e SGLANG_USE_AITER_UNIFIED_ATTN=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-4-gpu-mi35x-minimax-m25-tp4 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # ============================================================================== - # 4-GPU MiniMax-M3 MXFP8 (MI35x) + # 4-GPU MiniMax-M3 MXFP8 (MI35x accuracy + performance) # ============================================================================== nightly-4-gpu-mi35x-minimax-m3-rocm720: @@ -2250,6 +2204,20 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} + # Run perf only after correctness passes. Keeping both steps in this job + # reuses the cached checkpoint and one scarce MI35x slot. + - name: Performance Test MI35x ROCm 7.2 (4-GPU MiniMax-M3 MXFP8) + timeout-minutes: 120 + continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed + run: | + > github_summary.md + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_USE_AITER=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-4-gpu-mi35x-minimax-m3 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + # ============================================================================== # 8-GPU MiniMax-M2.7 (MI30x only) # ============================================================================== @@ -2415,9 +2383,7 @@ jobs: - nightly-8-gpu-mi35x-glm52-fp8-rocm720 # 8-GPU GLM-5-MXFP4 (MI35x only) - nightly-8-gpu-mi35x-glm5-mxfp4-rocm720 - # 4-GPU MiniMax-M2.5 (MI35x) - - nightly-4-gpu-mi35x-minimax-m25-rocm720 - # 4-GPU MiniMax-M3 MXFP8 (MI35x) + # 4-GPU MiniMax-M3 MXFP8 (MI35x accuracy + performance) - nightly-4-gpu-mi35x-minimax-m3-rocm720 # 8-GPU MiniMax-M2.7 (MI30x only) - nightly-8-gpu-minimax-m27-rocm720 diff --git a/test/registered/amd/perf/mi35x/test_minimax_m3_perf_mi35x.py b/test/registered/amd/perf/mi35x/test_minimax_m3_perf_mi35x.py new file mode 100644 index 000000000..5ff0318a7 --- /dev/null +++ b/test/registered/amd/perf/mi35x/test_minimax_m3_perf_mi35x.py @@ -0,0 +1,151 @@ +"""MI35x nightly performance benchmark for MiniMax-M3-MXFP8 (4-GPU, TP=4). + +Benchmarks MiniMaxAI/MiniMax-M3-MXFP8 with the same TP=4 recipe the MI35x +accuracy test validated (aiter attention, fp8 e4m3 KV, block-fp8 linear path, +quick INT4 all-reduce), so a throughput regression cannot be confused with a +configuration difference. MI35x (gfx950 / CDNA4) has hardware MX-scaled matmul, +so the MXFP8 MoE weights are served natively. + +This runs as the step after the eval inside nightly-4-gpu-mi35x-minimax-m3-rocm720 +rather than as a job of its own, which is how the other combined accuracy plus +performance jobs in that workflow are arranged. Sharing the job reuses the +already-cached checkpoint and one MI35x runner slot, and step ordering supplies +the gate for free: a failed eval fails the job, so this never measures +throughput on a build that got the tokens wrong. + +1K-input / 1K-output is the workload shape the cookbook publishes for MI355X, and +it weights decode heavily, which is what matters for a reasoning model that +spends most of its tokens inside ``. That published row is a tp8 +bench_serving run (~1678 output tok/s at concurrency 64) while this sweep is tp4 +under bench_one_batch_server, so treat it as a shape reference, not a target. + +Registry: nightly-perf-4-gpu-mi35x-minimax-m3 suite +""" + +import os +import unittest + +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.nightly_bench_utils import generate_simple_markdown_report +from sglang.test.nightly_utils import NightlyBenchmarkRunner +from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env + +register_amd_ci( + est_time=5400, + suite="nightly-perf-4-gpu-mi35x-minimax-m3", + nightly=True, +) + +MINIMAX_M3_MODEL_PATH = os.environ.get( + "MINIMAX_M3_MODEL_PATH", "MiniMaxAI/MiniMax-M3-MXFP8" +) +RESULT_DIR = "performance_results_minimax_m3_mi35x" +MAX_BATCH_SIZE = 64 + + +class TestNightlyMiniMaxM3PerformanceMI35x(unittest.TestCase): + """MiniMax-M3-MXFP8 TP=4 throughput on AMD MI35x.""" + + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + # bench_one_batch_server already warms every unique batch size before it + # times anything, but only at output_len 16. The repeated leading 1 buys + # one throwaway full-length decode -- the report helper drops it -- so + # whatever is first touched over 1024 output tokens lands there instead + # of in a published row. + cls.batch_sizes = [1, 1, 8, 16, MAX_BATCH_SIZE] + cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "1024")) + cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "1024")) + + cls.model_config = { + "name": "TP4+MXFP8+aiterAttn+fp8KV+blockFP8+quickAR", + "model_path": MINIMAX_M3_MODEL_PATH, + # Mirrors test_minimax_m3_tp4_eval_mi35x.py, plus multithread weight + # load and a decode graph capped at the largest timed batch (capture + # is not spent on batches this sweep never sends). + "other_args": [ + "--quantization", + "mxfp8", + "--dtype", + "bfloat16", + "--trust-remote-code", + "--tp", + "4", + "--attention-backend", + "aiter", + "--kv-cache-dtype", + "fp8_e4m3", + "--disable-radix-cache", + "--chunked-prefill-size", + "8192", + "--mem-fraction-static", + "0.80", + "--cuda-graph-max-bs-decode", + str(MAX_BATCH_SIZE), + "--max-running-requests", + str(MAX_BATCH_SIZE), + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + "--watchdog-timeout", + "1200", + ], + # See the accuracy test for what each of these does: the block-fp8 + # linear path (#32036) and custom/quick INT4 all-reduce (#32230) are + # both opt-in on gfx950, and the fp32 router GEMM is what ROCm 7.0's + # rocBLAS accepts. + "env_vars": { + "SGLANG_USE_AITER": "1", + "SGLANG_OPT_USE_BF16_ROUTER_GEMM": "0", + "SGLANG_FORCE_MXFP8_BLOCK_CONVERT": "1", + "SGLANG_M3_ALLOW_CUSTOM_AR": "1", + "ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", + "ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16": "1", + }, + } + + cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url) + cls.runner.setup_result_directory() + cls.runner.full_report = f"## {cls.__name__}\n" + + def test_bench_minimax_m3(self): + """Run the MiniMax-M3-MXFP8 batch-size sweep.""" + old_env = {} + for key, value in self.model_config["env_vars"].items(): + old_env[key] = os.environ.get(key) + os.environ[key] = value + print(f"Setting env: {key}={value}") + + try: + result_tuple = self.runner.run_benchmark_for_model( + model_path=self.model_config["model_path"], + batch_sizes=self.batch_sizes, + input_lens=self.input_lens, + output_lens=self.output_lens, + other_args=self.model_config["other_args"], + variant=self.model_config["name"], + extra_bench_args=["--trust-remote-code"], + timeout=5400, + ) + results, success = result_tuple[0], result_tuple[1] + + if results: + self.runner.full_report += ( + generate_simple_markdown_report(results, default_gpu_config="MI35x") + + "\n" + ) + + self.assertTrue( + success, f"Benchmark failed for {MINIMAX_M3_MODEL_PATH} on MI35x" + ) + finally: + for key, value in old_env.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + self.runner.write_final_report() + + +if __name__ == "__main__": + unittest.main()