[AMD][CI] Add GPT-OSS perf benchmarks to the ROCm 7.2 nightly (#34645)

Co-authored-by: Cursor Agent <cursoragent@cursor.com>
Co-authored-by: Michael <michaelzhang-ai@users.noreply.github.com>

.github/workflows/nightly-test-amd-rocm720.yml

@@ -493,6 +493,7 @@ jobs:
       - name: Accuracy Test ROCm 7.2 (8-GPU GPT-OSS)
         timeout-minutes: 180
         run: |
+          > github_summary.md  # Clear summary file
           bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
             -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
             -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
@@ -500,6 +501,21 @@ jobs:
           echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
           exit ${TEST_EXIT_CODE:-0}
 
+      # Shares this job with the accuracy step above: it already serves the same
+      # two models, and an MI30x job spends ~49 min pulling the image and
+      # installing dependencies before it runs anything.
+      - name: Performance Test ROCm 7.2 (8-GPU GPT-OSS)
+        if: ${{ !cancelled() }}
+        timeout-minutes: 120
+        run: |
+          > github_summary.md  # Clear summary file
+          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
+            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
+            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
+            python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-gpt-oss --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
+          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
+          exit ${TEST_EXIT_CODE:-0}
+
   nightly-accuracy-8-gpu-mi35x-rocm720:
     if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x-rocm720,'))
     runs-on: linux-mi35x-gpu-8
@@ -528,12 +544,26 @@ jobs:
       - name: Accuracy Test MI35x ROCm 7.2 (8-GPU GPT-OSS)
         timeout-minutes: 180
         run: |
+          > github_summary.md  # Clear summary file
           bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
             -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
             python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
           echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
           exit ${TEST_EXIT_CODE:-0}
 
+      # Shares this job with the accuracy step above rather than taking its own,
+      # so the container setup and the GPT-OSS weight cache are paid for once.
+      - name: Performance Test MI35x ROCm 7.2 (8-GPU GPT-OSS)
+        if: ${{ !cancelled() }}
+        timeout-minutes: 120
+        run: |
+          > github_summary.md  # Clear summary file
+          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
+            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
+            python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-gpt-oss --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
+          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
+          exit ${TEST_EXIT_CODE:-0}
+
   # ==============================================================================
   # 8-GPU Grok1-INT4 (MI30x + MI35x)
   # ==============================================================================

python/sglang/test/nightly_bench_utils.py

@@ -73,6 +73,46 @@ def generate_markdown_report(
     return summary
 
 
+def generate_simple_markdown_report(
+    results: List[BenchmarkResult], default_gpu_config: str = ""
+) -> str:
+    """Generate a markdown report without the H100-priced cost columns.
+
+    Drops the leading result when it is a warmup run, which the caller requests
+    by repeating the first batch size.
+    """
+    model_header = results[0].model_path
+    if results[0].run_name and results[0].run_name != "default":
+        model_header += f" ({results[0].run_name})"
+
+    gpu_config = os.getenv("GPU_CONFIG", default_gpu_config)
+    if gpu_config:
+        model_header += f" [{gpu_config}]"
+
+    summary = f"### {model_header}\n"
+    summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n"
+    summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n"
+
+    report_results = (
+        results[1:]
+        if len(results) > 1 and results[0].batch_size == results[1].batch_size
+        else results
+    )
+
+    for result in report_results:
+        itl = (
+            1 / (result.output_throughput / result.batch_size) * 1000
+            if result.output_throughput > 0
+            else 0
+        )
+        summary += (
+            f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | "
+            f"{result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n"
+        )
+
+    return summary
+
+
 def save_results_as_pydantic_models(
     results: List,
     pydantic_result_filename: str,

test/registered/amd/perf/mi30x/test_gpt_oss_perf_amd.py

@@ -0,0 +1,106 @@
+"""MI30x nightly performance benchmark for GPT-OSS (8-GPU).
+
+Benchmarks the bf16 GPT-OSS conversions (lmsys/gpt-oss-20b-bf16,
+lmsys/gpt-oss-120b-bf16) with the same TP8 AITER server configuration the
+MI30x GPT-OSS accuracy test uses, so a throughput change here points at the
+serving stack rather than at a different recipe.
+
+MI30x cannot serve the MXFP4 checkpoints natively, which is why the model
+paths differ from the MI35x benchmark.
+
+Registry: nightly-perf-8-gpu-gpt-oss suite
+
+Example usage:
+    python3 test_gpt_oss_perf_amd.py
+"""
+
+import os
+import unittest
+
+from sglang.test.ci.ci_register import register_amd_ci
+from sglang.test.nightly_bench_utils import generate_simple_markdown_report
+from sglang.test.nightly_utils import NightlyBenchmarkRunner
+from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
+
+# Register for AMD CI - MI30x GPT-OSS perf benchmark (~60 min for both sizes)
+register_amd_ci(est_time=3600, suite="nightly-perf-8-gpu-gpt-oss", nightly=True)
+
+RESULT_DIR = "performance_results_gpt_oss_mi30x"
+
+GPT_OSS_20B_MODEL_PATH = os.environ.get(
+    "GPT_OSS_20B_BF16_MODEL_PATH", "lmsys/gpt-oss-20b-bf16"
+)
+GPT_OSS_120B_MODEL_PATH = os.environ.get(
+    "GPT_OSS_120B_BF16_MODEL_PATH", "lmsys/gpt-oss-120b-bf16"
+)
+
+# Matches test/registered/amd/accuracy/mi30x/test_gpt_oss_eval_amd.py.
+SERVER_ARGS = [
+    "--trust-remote-code",
+    "--tp",
+    "8",
+    "--attention-backend",
+    "triton",
+    "--chunked-prefill-size",
+    "130172",
+    "--max-running-requests",
+    "128",
+    "--mem-fraction-static",
+    "0.85",
+]
+
+ENV_VARS = {"SGLANG_USE_AITER": "1"}
+
+
+class TestNightlyGptOssPerformanceAMD(unittest.TestCase):
+    """MI30x nightly performance benchmark for the bf16 GPT-OSS models."""
+
+    @classmethod
+    def setUpClass(cls):
+        cls.base_url = DEFAULT_URL_FOR_TEST
+        # The leading duplicate is a warmup: this benchmark launches its own
+        # server, so nothing else has paid for JIT and autotuning first.
+        cls.batch_sizes = [1, 1, 8, 16, 64]
+        cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
+        cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
+        cls.models = [GPT_OSS_20B_MODEL_PATH, GPT_OSS_120B_MODEL_PATH]
+
+        cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
+        cls.runner.setup_result_directory()
+        cls.runner.full_report = f"## {cls.__name__}\n"
+
+    def test_bench_one_batch(self):
+        """Benchmark every GPT-OSS size."""
+        failures = []
+        env = os.environ.copy()
+        env.update(ENV_VARS)
+
+        try:
+            for model_path in self.models:
+                with self.subTest(model=model_path):
+                    results, success, _ = self.runner.run_benchmark_for_model(
+                        model_path=model_path,
+                        batch_sizes=self.batch_sizes,
+                        input_lens=self.input_lens,
+                        output_lens=self.output_lens,
+                        other_args=SERVER_ARGS,
+                        extra_bench_args=["--trust-remote-code"],
+                        env=env,
+                    )
+
+                    if results:
+                        self.runner.full_report += (
+                            generate_simple_markdown_report(results, "MI30x") + "\n"
+                        )
+
+                    if not success:
+                        failures.append(f"benchmark failed for {model_path}")
+        finally:
+            self.runner.write_final_report()
+
+        if failures:
+            self.fail("\n".join(failures))
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/amd/perf/mi35x/test_gpt_oss_perf_mi35x.py

@@ -0,0 +1,107 @@
+"""MI35x nightly performance benchmark for GPT-OSS (8-GPU).
+
+Benchmarks the MXFP4 GPT-OSS checkpoints (openai/gpt-oss-20b,
+openai/gpt-oss-120b) with the same TP8 AITER server configuration the MI35x
+GPT-OSS accuracy test uses, so a throughput change here points at the
+serving stack rather than at a different recipe.
+
+Registry: nightly-perf-8-gpu-mi35x-gpt-oss suite
+
+Example usage:
+    python3 test_gpt_oss_perf_mi35x.py
+"""
+
+import os
+import unittest
+
+from sglang.test.ci.ci_register import register_amd_ci
+from sglang.test.nightly_bench_utils import generate_simple_markdown_report
+from sglang.test.nightly_utils import NightlyBenchmarkRunner
+from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
+
+# Register for AMD CI - MI35x GPT-OSS perf benchmark (~60 min for both sizes)
+register_amd_ci(est_time=3600, suite="nightly-perf-8-gpu-mi35x-gpt-oss", nightly=True)
+
+RESULT_DIR = "performance_results_gpt_oss_mi35x"
+
+# MI35x serves the MXFP4 checkpoints directly; MI30x uses the bf16 conversions.
+GPT_OSS_20B_MODEL_PATH = os.environ.get("GPT_OSS_20B_MODEL_PATH", "openai/gpt-oss-20b")
+GPT_OSS_120B_MODEL_PATH = os.environ.get(
+    "GPT_OSS_120B_MODEL_PATH", "openai/gpt-oss-120b"
+)
+
+# Matches test/registered/amd/accuracy/mi35x/test_gpt_oss_eval_mi35x.py.
+SERVER_ARGS = [
+    "--trust-remote-code",
+    "--tp",
+    "8",
+    "--attention-backend",
+    "triton",
+    "--chunked-prefill-size",
+    "130172",
+    "--max-running-requests",
+    "128",
+    "--mem-fraction-static",
+    "0.85",
+]
+
+# AITER's MXFP4 fused MoE for gpt-oss uses the separated gate/up tile layout;
+# other AITER MXFP4 callers default to interleave, so opt out explicitly.
+ENV_VARS = {
+    "SGLANG_USE_AITER": "1",
+    "SGLANG_USE_AITER_MOE_GU_ITLV": "1",
+}
+
+
+class TestNightlyGptOssPerformanceMI35x(unittest.TestCase):
+    """MI35x nightly performance benchmark for the MXFP4 GPT-OSS models."""
+
+    @classmethod
+    def setUpClass(cls):
+        cls.base_url = DEFAULT_URL_FOR_TEST
+        # The leading duplicate is a warmup: this benchmark launches its own
+        # server, so nothing else has paid for JIT and autotuning first.
+        cls.batch_sizes = [1, 1, 8, 16, 64]
+        cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
+        cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
+        cls.models = [GPT_OSS_20B_MODEL_PATH, GPT_OSS_120B_MODEL_PATH]
+
+        cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
+        cls.runner.setup_result_directory()
+        cls.runner.full_report = f"## {cls.__name__}\n"
+
+    def test_bench_one_batch(self):
+        """Benchmark every GPT-OSS size."""
+        failures = []
+        env = os.environ.copy()
+        env.update(ENV_VARS)
+
+        try:
+            for model_path in self.models:
+                with self.subTest(model=model_path):
+                    results, success, _ = self.runner.run_benchmark_for_model(
+                        model_path=model_path,
+                        batch_sizes=self.batch_sizes,
+                        input_lens=self.input_lens,
+                        output_lens=self.output_lens,
+                        other_args=SERVER_ARGS,
+                        extra_bench_args=["--trust-remote-code"],
+                        env=env,
+                    )
+
+                    if results:
+                        self.runner.full_report += (
+                            generate_simple_markdown_report(results, "MI35x") + "\n"
+                        )
+
+                    if not success:
+                        failures.append(f"benchmark failed for {model_path}")
+        finally:
+            self.runner.write_final_report()
+
+        if failures:
+            self.fail("\n".join(failures))
+
+
+if __name__ == "__main__":
+    unittest.main()