minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 10 Packages Projects Releases Wiki Activity

[AMD][CI] Add GPT-OSS perf benchmarks to the ROCm 7.2 nightly (#34645)

Browse Source 
Co-authored-by: Cursor Agent <cursoragent@cursor.com>
Co-authored-by: Michael <michaelzhang-ai@users.noreply.github.com>
This commit is contained in:
Michael
2026-08-16 16:35:09 -07:00
committed by GitHub
co-authored by Cursor Agent Michael
parent f7cb328eb7
commit d91c3682b0
4 changed files with 283 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/nightly-test-amd-rocm720.yml
+30
View File
@@ -493,6 +493,7 @@ jobs:
- name: Accuracy Test ROCm 7.2 (8-GPU GPT-OSS)
timeout-minutes: 180
run: |
> github_summary.md # Clear summary file
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
@@ -500,6 +501,21 @@ jobs:
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
# Shares this job with the accuracy step above: it already serves the same
# two models, and an MI30x job spends ~49 min pulling the image and
# installing dependencies before it runs anything.
- name: Performance Test ROCm 7.2 (8-GPU GPT-OSS)
if: ${{ !cancelled() }}
timeout-minutes: 120
run: |
> github_summary.md # Clear summary file
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-gpt-oss --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
nightly-accuracy-8-gpu-mi35x-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x-rocm720,'))
runs-on: linux-mi35x-gpu-8
@@ -528,12 +544,26 @@ jobs:
- name: Accuracy Test MI35x ROCm 7.2 (8-GPU GPT-OSS)
timeout-minutes: 180
run: |
> github_summary.md # Clear summary file
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
# Shares this job with the accuracy step above rather than taking its own,
# so the container setup and the GPT-OSS weight cache are paid for once.
- name: Performance Test MI35x ROCm 7.2 (8-GPU GPT-OSS)
if: ${{ !cancelled() }}
timeout-minutes: 120
run: |
> github_summary.md # Clear summary file
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-gpt-oss --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
# ==============================================================================
# 8-GPU Grok1-INT4 (MI30x + MI35x)
# ==============================================================================
python/sglang/test/nightly_bench_utils.py
+40
View File
@@ -73,6 +73,46 @@ def generate_markdown_report(
return summary
def generate_simple_markdown_report(
results: List[BenchmarkResult], default_gpu_config: str = ""
) -> str:
"""Generate a markdown report without the H100-priced cost columns.
Drops the leading result when it is a warmup run, which the caller requests
by repeating the first batch size.
"""
model_header = results[0].model_path
if results[0].run_name and results[0].run_name != "default":
model_header += f" ({results[0].run_name})"
gpu_config = os.getenv("GPU_CONFIG", default_gpu_config)
if gpu_config:
model_header += f" [{gpu_config}]"
summary = f"### {model_header}\n"
summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n"
summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n"
report_results = (
results[1:]
if len(results) > 1 and results[0].batch_size == results[1].batch_size
else results
)
for result in report_results:
itl = (
1 / (result.output_throughput / result.batch_size) * 1000
if result.output_throughput > 0
else 0
)
summary += (
f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | "
f"{result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n"
)
return summary
def save_results_as_pydantic_models(
results: List,
pydantic_result_filename: str,
test/registered/amd/perf/mi30x/test_gpt_oss_perf_amd.py
+106
View File
@@ -0,0 +1,106 @@
"""MI30x nightly performance benchmark for GPT-OSS (8-GPU).
Benchmarks the bf16 GPT-OSS conversions (lmsys/gpt-oss-20b-bf16,
lmsys/gpt-oss-120b-bf16) with the same TP8 AITER server configuration the
MI30x GPT-OSS accuracy test uses, so a throughput change here points at the
serving stack rather than at a different recipe.
MI30x cannot serve the MXFP4 checkpoints natively, which is why the model
paths differ from the MI35x benchmark.
Registry: nightly-perf-8-gpu-gpt-oss suite
Example usage:
python3 test_gpt_oss_perf_amd.py
"""
import os
import unittest
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.nightly_bench_utils import generate_simple_markdown_report
from sglang.test.nightly_utils import NightlyBenchmarkRunner
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
# Register for AMD CI - MI30x GPT-OSS perf benchmark (~60 min for both sizes)
register_amd_ci(est_time=3600, suite="nightly-perf-8-gpu-gpt-oss", nightly=True)
RESULT_DIR = "performance_results_gpt_oss_mi30x"
GPT_OSS_20B_MODEL_PATH = os.environ.get(
"GPT_OSS_20B_BF16_MODEL_PATH", "lmsys/gpt-oss-20b-bf16"
)
GPT_OSS_120B_MODEL_PATH = os.environ.get(
"GPT_OSS_120B_BF16_MODEL_PATH", "lmsys/gpt-oss-120b-bf16"
)
# Matches test/registered/amd/accuracy/mi30x/test_gpt_oss_eval_amd.py.
SERVER_ARGS = [
"--trust-remote-code",
"--tp",
"8",
"--attention-backend",
"triton",
"--chunked-prefill-size",
"130172",
"--max-running-requests",
"128",
"--mem-fraction-static",
"0.85",
]
ENV_VARS = {"SGLANG_USE_AITER": "1"}
class TestNightlyGptOssPerformanceAMD(unittest.TestCase):
"""MI30x nightly performance benchmark for the bf16 GPT-OSS models."""
@classmethod
def setUpClass(cls):
cls.base_url = DEFAULT_URL_FOR_TEST
# The leading duplicate is a warmup: this benchmark launches its own
# server, so nothing else has paid for JIT and autotuning first.
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
cls.models = [GPT_OSS_20B_MODEL_PATH, GPT_OSS_120B_MODEL_PATH]
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
cls.runner.full_report = f"## {cls.__name__}\n"
def test_bench_one_batch(self):
"""Benchmark every GPT-OSS size."""
failures = []
env = os.environ.copy()
env.update(ENV_VARS)
try:
for model_path in self.models:
with self.subTest(model=model_path):
results, success, _ = self.runner.run_benchmark_for_model(
model_path=model_path,
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=SERVER_ARGS,
extra_bench_args=["--trust-remote-code"],
env=env,
)
if results:
self.runner.full_report += (
generate_simple_markdown_report(results, "MI30x") + "\n"
)
if not success:
failures.append(f"benchmark failed for {model_path}")
finally:
self.runner.write_final_report()
if failures:
self.fail("\n".join(failures))
if __name__ == "__main__":
unittest.main()
test/registered/amd/perf/mi35x/test_gpt_oss_perf_mi35x.py
+107
View File
@@ -0,0 +1,107 @@
"""MI35x nightly performance benchmark for GPT-OSS (8-GPU).
Benchmarks the MXFP4 GPT-OSS checkpoints (openai/gpt-oss-20b,
openai/gpt-oss-120b) with the same TP8 AITER server configuration the MI35x
GPT-OSS accuracy test uses, so a throughput change here points at the
serving stack rather than at a different recipe.
Registry: nightly-perf-8-gpu-mi35x-gpt-oss suite
Example usage:
python3 test_gpt_oss_perf_mi35x.py
"""
import os
import unittest
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.nightly_bench_utils import generate_simple_markdown_report
from sglang.test.nightly_utils import NightlyBenchmarkRunner
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
# Register for AMD CI - MI35x GPT-OSS perf benchmark (~60 min for both sizes)
register_amd_ci(est_time=3600, suite="nightly-perf-8-gpu-mi35x-gpt-oss", nightly=True)
RESULT_DIR = "performance_results_gpt_oss_mi35x"
# MI35x serves the MXFP4 checkpoints directly; MI30x uses the bf16 conversions.
GPT_OSS_20B_MODEL_PATH = os.environ.get("GPT_OSS_20B_MODEL_PATH", "openai/gpt-oss-20b")
GPT_OSS_120B_MODEL_PATH = os.environ.get(
"GPT_OSS_120B_MODEL_PATH", "openai/gpt-oss-120b"
)
# Matches test/registered/amd/accuracy/mi35x/test_gpt_oss_eval_mi35x.py.
SERVER_ARGS = [
"--trust-remote-code",
"--tp",
"8",
"--attention-backend",
"triton",
"--chunked-prefill-size",
"130172",
"--max-running-requests",
"128",
"--mem-fraction-static",
"0.85",
]
# AITER's MXFP4 fused MoE for gpt-oss uses the separated gate/up tile layout;
# other AITER MXFP4 callers default to interleave, so opt out explicitly.
ENV_VARS = {
"SGLANG_USE_AITER": "1",
"SGLANG_USE_AITER_MOE_GU_ITLV": "1",
}
class TestNightlyGptOssPerformanceMI35x(unittest.TestCase):
"""MI35x nightly performance benchmark for the MXFP4 GPT-OSS models."""
@classmethod
def setUpClass(cls):
cls.base_url = DEFAULT_URL_FOR_TEST
# The leading duplicate is a warmup: this benchmark launches its own
# server, so nothing else has paid for JIT and autotuning first.
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
cls.models = [GPT_OSS_20B_MODEL_PATH, GPT_OSS_120B_MODEL_PATH]
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
cls.runner.full_report = f"## {cls.__name__}\n"
def test_bench_one_batch(self):
"""Benchmark every GPT-OSS size."""
failures = []
env = os.environ.copy()
env.update(ENV_VARS)
try:
for model_path in self.models:
with self.subTest(model=model_path):
results, success, _ = self.runner.run_benchmark_for_model(
model_path=model_path,
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=SERVER_ARGS,
extra_bench_args=["--trust-remote-code"],
env=env,
)
if results:
self.runner.full_report += (
generate_simple_markdown_report(results, "MI35x") + "\n"
)
if not success:
failures.append(f"benchmark failed for {model_path}")
finally:
self.runner.write_final_report()
if failures:
self.fail("\n".join(failures))
if __name__ == "__main__":
unittest.main()