diff --git a/.github/workflows/nightly-test-amd-rocm720.yml b/.github/workflows/nightly-test-amd-rocm720.yml index 2245c81ea..3a81f23bd 100644 --- a/.github/workflows/nightly-test-amd-rocm720.yml +++ b/.github/workflows/nightly-test-amd-rocm720.yml @@ -493,6 +493,7 @@ jobs: - name: Accuracy Test ROCm 7.2 (8-GPU GPT-OSS) timeout-minutes: 180 run: | + > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ @@ -500,6 +501,21 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} + # Shares this job with the accuracy step above: it already serves the same + # two models, and an MI30x job spends ~49 min pulling the image and + # installing dependencies before it runs anything. + - name: Performance Test ROCm 7.2 (8-GPU GPT-OSS) + if: ${{ !cancelled() }} + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-gpt-oss --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + nightly-accuracy-8-gpu-mi35x-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x-rocm720,')) runs-on: linux-mi35x-gpu-8 @@ -528,12 +544,26 @@ jobs: - name: Accuracy Test MI35x ROCm 7.2 (8-GPU GPT-OSS) timeout-minutes: 180 run: | + > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} + # Shares this job with the accuracy step above rather than taking its own, + # so the container setup and the GPT-OSS weight cache are paid for once. + - name: Performance Test MI35x ROCm 7.2 (8-GPU GPT-OSS) + if: ${{ !cancelled() }} + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-gpt-oss --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + # ============================================================================== # 8-GPU Grok1-INT4 (MI30x + MI35x) # ============================================================================== diff --git a/python/sglang/test/nightly_bench_utils.py b/python/sglang/test/nightly_bench_utils.py index b6cfcd79c..d298d0a29 100644 --- a/python/sglang/test/nightly_bench_utils.py +++ b/python/sglang/test/nightly_bench_utils.py @@ -73,6 +73,46 @@ def generate_markdown_report( return summary +def generate_simple_markdown_report( + results: List[BenchmarkResult], default_gpu_config: str = "" +) -> str: + """Generate a markdown report without the H100-priced cost columns. + + Drops the leading result when it is a warmup run, which the caller requests + by repeating the first batch size. + """ + model_header = results[0].model_path + if results[0].run_name and results[0].run_name != "default": + model_header += f" ({results[0].run_name})" + + gpu_config = os.getenv("GPU_CONFIG", default_gpu_config) + if gpu_config: + model_header += f" [{gpu_config}]" + + summary = f"### {model_header}\n" + summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n" + summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n" + + report_results = ( + results[1:] + if len(results) > 1 and results[0].batch_size == results[1].batch_size + else results + ) + + for result in report_results: + itl = ( + 1 / (result.output_throughput / result.batch_size) * 1000 + if result.output_throughput > 0 + else 0 + ) + summary += ( + f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | " + f"{result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n" + ) + + return summary + + def save_results_as_pydantic_models( results: List, pydantic_result_filename: str, diff --git a/test/registered/amd/perf/mi30x/test_gpt_oss_perf_amd.py b/test/registered/amd/perf/mi30x/test_gpt_oss_perf_amd.py new file mode 100644 index 000000000..26ff9c452 --- /dev/null +++ b/test/registered/amd/perf/mi30x/test_gpt_oss_perf_amd.py @@ -0,0 +1,106 @@ +"""MI30x nightly performance benchmark for GPT-OSS (8-GPU). + +Benchmarks the bf16 GPT-OSS conversions (lmsys/gpt-oss-20b-bf16, +lmsys/gpt-oss-120b-bf16) with the same TP8 AITER server configuration the +MI30x GPT-OSS accuracy test uses, so a throughput change here points at the +serving stack rather than at a different recipe. + +MI30x cannot serve the MXFP4 checkpoints natively, which is why the model +paths differ from the MI35x benchmark. + +Registry: nightly-perf-8-gpu-gpt-oss suite + +Example usage: + python3 test_gpt_oss_perf_amd.py +""" + +import os +import unittest + +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.nightly_bench_utils import generate_simple_markdown_report +from sglang.test.nightly_utils import NightlyBenchmarkRunner +from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env + +# Register for AMD CI - MI30x GPT-OSS perf benchmark (~60 min for both sizes) +register_amd_ci(est_time=3600, suite="nightly-perf-8-gpu-gpt-oss", nightly=True) + +RESULT_DIR = "performance_results_gpt_oss_mi30x" + +GPT_OSS_20B_MODEL_PATH = os.environ.get( + "GPT_OSS_20B_BF16_MODEL_PATH", "lmsys/gpt-oss-20b-bf16" +) +GPT_OSS_120B_MODEL_PATH = os.environ.get( + "GPT_OSS_120B_BF16_MODEL_PATH", "lmsys/gpt-oss-120b-bf16" +) + +# Matches test/registered/amd/accuracy/mi30x/test_gpt_oss_eval_amd.py. +SERVER_ARGS = [ + "--trust-remote-code", + "--tp", + "8", + "--attention-backend", + "triton", + "--chunked-prefill-size", + "130172", + "--max-running-requests", + "128", + "--mem-fraction-static", + "0.85", +] + +ENV_VARS = {"SGLANG_USE_AITER": "1"} + + +class TestNightlyGptOssPerformanceAMD(unittest.TestCase): + """MI30x nightly performance benchmark for the bf16 GPT-OSS models.""" + + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + # The leading duplicate is a warmup: this benchmark launches its own + # server, so nothing else has paid for JIT and autotuning first. + cls.batch_sizes = [1, 1, 8, 16, 64] + cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) + cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) + cls.models = [GPT_OSS_20B_MODEL_PATH, GPT_OSS_120B_MODEL_PATH] + + cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url) + cls.runner.setup_result_directory() + cls.runner.full_report = f"## {cls.__name__}\n" + + def test_bench_one_batch(self): + """Benchmark every GPT-OSS size.""" + failures = [] + env = os.environ.copy() + env.update(ENV_VARS) + + try: + for model_path in self.models: + with self.subTest(model=model_path): + results, success, _ = self.runner.run_benchmark_for_model( + model_path=model_path, + batch_sizes=self.batch_sizes, + input_lens=self.input_lens, + output_lens=self.output_lens, + other_args=SERVER_ARGS, + extra_bench_args=["--trust-remote-code"], + env=env, + ) + + if results: + self.runner.full_report += ( + generate_simple_markdown_report(results, "MI30x") + "\n" + ) + + if not success: + failures.append(f"benchmark failed for {model_path}") + finally: + self.runner.write_final_report() + + if failures: + self.fail("\n".join(failures)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/perf/mi35x/test_gpt_oss_perf_mi35x.py b/test/registered/amd/perf/mi35x/test_gpt_oss_perf_mi35x.py new file mode 100644 index 000000000..8a3504174 --- /dev/null +++ b/test/registered/amd/perf/mi35x/test_gpt_oss_perf_mi35x.py @@ -0,0 +1,107 @@ +"""MI35x nightly performance benchmark for GPT-OSS (8-GPU). + +Benchmarks the MXFP4 GPT-OSS checkpoints (openai/gpt-oss-20b, +openai/gpt-oss-120b) with the same TP8 AITER server configuration the MI35x +GPT-OSS accuracy test uses, so a throughput change here points at the +serving stack rather than at a different recipe. + +Registry: nightly-perf-8-gpu-mi35x-gpt-oss suite + +Example usage: + python3 test_gpt_oss_perf_mi35x.py +""" + +import os +import unittest + +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.nightly_bench_utils import generate_simple_markdown_report +from sglang.test.nightly_utils import NightlyBenchmarkRunner +from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env + +# Register for AMD CI - MI35x GPT-OSS perf benchmark (~60 min for both sizes) +register_amd_ci(est_time=3600, suite="nightly-perf-8-gpu-mi35x-gpt-oss", nightly=True) + +RESULT_DIR = "performance_results_gpt_oss_mi35x" + +# MI35x serves the MXFP4 checkpoints directly; MI30x uses the bf16 conversions. +GPT_OSS_20B_MODEL_PATH = os.environ.get("GPT_OSS_20B_MODEL_PATH", "openai/gpt-oss-20b") +GPT_OSS_120B_MODEL_PATH = os.environ.get( + "GPT_OSS_120B_MODEL_PATH", "openai/gpt-oss-120b" +) + +# Matches test/registered/amd/accuracy/mi35x/test_gpt_oss_eval_mi35x.py. +SERVER_ARGS = [ + "--trust-remote-code", + "--tp", + "8", + "--attention-backend", + "triton", + "--chunked-prefill-size", + "130172", + "--max-running-requests", + "128", + "--mem-fraction-static", + "0.85", +] + +# AITER's MXFP4 fused MoE for gpt-oss uses the separated gate/up tile layout; +# other AITER MXFP4 callers default to interleave, so opt out explicitly. +ENV_VARS = { + "SGLANG_USE_AITER": "1", + "SGLANG_USE_AITER_MOE_GU_ITLV": "1", +} + + +class TestNightlyGptOssPerformanceMI35x(unittest.TestCase): + """MI35x nightly performance benchmark for the MXFP4 GPT-OSS models.""" + + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + # The leading duplicate is a warmup: this benchmark launches its own + # server, so nothing else has paid for JIT and autotuning first. + cls.batch_sizes = [1, 1, 8, 16, 64] + cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) + cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) + cls.models = [GPT_OSS_20B_MODEL_PATH, GPT_OSS_120B_MODEL_PATH] + + cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url) + cls.runner.setup_result_directory() + cls.runner.full_report = f"## {cls.__name__}\n" + + def test_bench_one_batch(self): + """Benchmark every GPT-OSS size.""" + failures = [] + env = os.environ.copy() + env.update(ENV_VARS) + + try: + for model_path in self.models: + with self.subTest(model=model_path): + results, success, _ = self.runner.run_benchmark_for_model( + model_path=model_path, + batch_sizes=self.batch_sizes, + input_lens=self.input_lens, + output_lens=self.output_lens, + other_args=SERVER_ARGS, + extra_bench_args=["--trust-remote-code"], + env=env, + ) + + if results: + self.runner.full_report += ( + generate_simple_markdown_report(results, "MI35x") + "\n" + ) + + if not success: + failures.append(f"benchmark failed for {model_path}") + finally: + self.runner.write_final_report() + + if failures: + self.fail("\n".join(failures)) + + +if __name__ == "__main__": + unittest.main()