minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[CI] Remove profiling from nightly tests (#33832)

Browse Source
This commit is contained in:
Baizhou Zhang
2026-08-06 01:16:08 -07:00
committed by GitHub
parent c11ce7c514
commit 0e584529f5
65 changed files with 214 additions and 429 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/manual/nightly/test_deepseek_v31_perf.py
+3 -3
View File
@@ -4,7 +4,7 @@ from sglang.test.nightly_utils import NightlyBenchmarkRunner
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
DEEPSEEK_V31_MODEL_PATH = "deepseek-ai/DeepSeek-V3.1"
PROFILE_DIR = "performance_profiles_deepseek_v31"
RESULT_DIR = "performance_results_deepseek_v31"
class TestNightlyDeepseekV31Performance(unittest.TestCase):
@@ -50,8 +50,8 @@ class TestNightlyDeepseekV31Performance(unittest.TestCase):
},
]
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
def test_bench_one_batch(self):
failed_variants = []
test/manual/nightly/test_deepseek_v32_perf.py
+3 -3
View File
@@ -4,7 +4,7 @@ from sglang.test.nightly_utils import NightlyBenchmarkRunner
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2"
PROFILE_DIR = "performance_profiles_deepseek_v32"
RESULT_DIR = "performance_results_deepseek_v32"
class TestNightlyDeepseekV32Performance(unittest.TestCase):
@@ -91,8 +91,8 @@ class TestNightlyDeepseekV32Performance(unittest.TestCase):
},
]
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
def test_bench_one_batch(self):
failed_variants = []
test/manual/nightly/test_text_models_perf.py
+3 -3
View File
@@ -8,7 +8,7 @@ from sglang.test.test_utils import (
parse_models,
)
PROFILE_DIR = "performance_profiles_text_models"
RESULT_DIR = "performance_results_text_models"
class TestNightlyTextModelsPerformance(unittest.TestCase):
@@ -28,8 +28,8 @@ class TestNightlyTextModelsPerformance(unittest.TestCase):
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
def test_bench_one_batch(self):
all_model_succeed = True
test/manual/nightly/test_vlms_perf.py
+3 -3
View File
@@ -10,7 +10,7 @@ from sglang.test.test_utils import (
parse_models,
)
PROFILE_DIR = "performance_profiles_vlms"
RESULT_DIR = "performance_results_vlms"
MODEL_DEFAULTS = [
# Keep conservative defaults. Can be overridden by env NIGHTLY_VLM_MODELS
@@ -49,8 +49,8 @@ class TestNightlyVLMModelsPerformance(unittest.TestCase):
cls.batch_sizes = _parse_int_list_env("NIGHTLY_VLM_BATCH_SIZES", "1,1,2,8,16")
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_VLM_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_VLM_OUTPUT_LENS", "512"))
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
def test_bench_one_batch(self):
all_model_succeed = True
test/manual/test_deepseek_v31.py
+1 -1
View File
@@ -61,7 +61,7 @@ class TestDeepseekV31(unittest.TestCase):
dataset="gsm8k", baseline_accuracy=0.935
),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_deepseek_v31",
result_dir="performance_results_deepseek_v31",
),
)
test/manual/test_glm_46_fp8.py
+1 -1
View File
@@ -50,7 +50,7 @@ class TestGLM46FP8(unittest.TestCase):
test_name="GLM-4.6-FP8",
accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.80),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_glm_4_6_fp8",
result_dir="performance_results_glm_4_6_fp8",
),
)
test/manual/test_qwen3_235b.py
+1 -1
View File
@@ -61,7 +61,7 @@ class TestQwen3235BFP8(unittest.TestCase):
test_name="Qwen3-235B-FP8",
accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.88),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_qwen3_235b_fp8",
result_dir="performance_results_qwen3_235b_fp8",
),
)
test/registered/8-gpu-models/test_glm52_fp8.py
+1 -1
View File
@@ -59,7 +59,7 @@ class TestGlm52Fp8(unittest.TestCase):
test_name="GLM-5.2-FP8",
accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.92),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_glm_52_fp8",
result_dir="performance_results_glm_52_fp8",
),
)
test/registered/8-gpu-models/test_glm_46.py
+1 -1
View File
@@ -43,7 +43,7 @@ class TestGLM46(unittest.TestCase):
test_name="GLM-4.6",
accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.80),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_glm_4_6",
result_dir="performance_results_glm_4_6",
),
)
test/registered/8-gpu-models/test_gpt_oss_120b.py
+1 -1
View File
@@ -74,7 +74,7 @@ class TestGptOss120B(unittest.TestCase):
test_name="GPT-OSS-120B",
accuracy_params=None,
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_gpt_oss_120b",
result_dir="performance_results_gpt_oss_120b",
),
)
test/registered/8-gpu-models/test_inkling_nvfp4_nightly.py
+1 -1
View File
@@ -70,7 +70,7 @@ class TestInklingNVFP4Nightly(unittest.TestCase):
repeat=1,
),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_inkling_nvfp4",
result_dir="performance_results_inkling_nvfp4",
),
)
test/registered/8-gpu-models/test_kimi_k25.py
+1 -1
View File
@@ -54,7 +54,7 @@ class TestKimiK25(unittest.TestCase):
test_name="Kimi-K2.5",
accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.92),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_kimi_k25",
result_dir="performance_results_kimi_k25",
),
)
test/registered/8-gpu-models/test_llama4.py
+1 -1
View File
@@ -47,7 +47,7 @@ class TestLlama4(unittest.TestCase):
test_name="Llama-4-Scout",
accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.9),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_llama4",
result_dir="performance_results_llama4",
),
)
test/registered/8-gpu-models/test_longcat_flash_lite_fp8.py
+1 -1
View File
@@ -58,7 +58,7 @@ class TestLongCatFlashLiteFp8(unittest.TestCase):
num_examples=200,
),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_longcat_flash_lite_fp8",
result_dir="performance_results_longcat_flash_lite_fp8",
),
)
test/registered/8-gpu-models/test_minimax_m25.py
+1 -1
View File
@@ -54,7 +54,7 @@ class TestMiniMaxM25(unittest.TestCase):
test_name="MiniMax-M2.5",
accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.80),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_minimax_m25",
result_dir="performance_results_minimax_m25",
),
)
test/registered/8-gpu-models/test_mistral_large3.py
+1 -1
View File
@@ -89,7 +89,7 @@ class TestMistralLarge3(unittest.TestCase):
test_name="Mistral-Large-3",
accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.85),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_mistral_large3",
result_dir="performance_results_mistral_large3",
),
)
test/registered/8-gpu-models/test_nvidia_nemotron_3_super_nightly.py
+2 -2
View File
@@ -92,7 +92,7 @@ class TestNvidiaNemotron3SuperNightly(unittest.TestCase):
repeat=1,
),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_nemotron_3_super_bf16",
result_dir="performance_results_nemotron_3_super_bf16",
),
)
@@ -129,7 +129,7 @@ class TestNvidiaNemotron3SuperNightly(unittest.TestCase):
repeat=1,
),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_nemotron_3_super_nvfp4",
result_dir="performance_results_nemotron_3_super_nvfp4",
),
)
test/registered/8-gpu-models/test_qwen35.py
+1 -1
View File
@@ -71,7 +71,7 @@ class TestQwen35(unittest.TestCase):
num_examples=200,
),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_qwen35",
result_dir="performance_results_qwen35",
),
)
test/registered/amd/perf/mi30x/test_deepseek_v31_perf.py
+6 -7
View File
@@ -22,7 +22,7 @@ register_amd_ci(est_time=18000, suite="nightly-perf-8-gpu-deepseek-v31", nightly
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
"""Generate a simplified markdown report without traces and cost columns.
"""Generate a simplified markdown report without cost columns.
Skips the first result if it's a warmup run (duplicate batch_size).
"""
@@ -56,7 +56,7 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
DEEPSEEK_V31_MODEL_PATH = os.environ.get(
"DEEPSEEK_V31_MODEL_PATH", "deepseek-ai/DeepSeek-V3.1"
)
PROFILE_DIR = "performance_profiles_deepseek_v31"
RESULT_DIR = "performance_results_deepseek_v31"
class TestNightlyDeepseekV31Performance(unittest.TestCase):
@@ -109,9 +109,9 @@ class TestNightlyDeepseekV31Performance(unittest.TestCase):
},
]
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
# Override full_report to remove traces help text
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
# Set the report header for this test
cls.runner.full_report = f"## {cls.__name__}\n"
def test_bench_one_batch(self):
@@ -129,7 +129,6 @@ class TestNightlyDeepseekV31Performance(unittest.TestCase):
other_args=variant_config["other_args"],
variant=variant_config["name"],
extra_bench_args=["--trust-remote-code"],
enable_profile=False, # Disable profiling for AMD tests
)
results = result_tuple[0]
success = result_tuple[1]
@@ -137,7 +136,7 @@ class TestNightlyDeepseekV31Performance(unittest.TestCase):
if not success:
failed_variants.append(variant_config["name"])
# Use simplified report format without traces
# Use the simplified report format
if results:
self.runner.full_report += (
generate_simple_markdown_report(results) + "\n"
test/registered/amd/perf/mi30x/test_deepseek_v32_basic_perf_amd.py
+6 -7
View File
@@ -26,7 +26,7 @@ register_amd_ci(
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
"""Generate a simplified markdown report without traces and cost columns.
"""Generate a simplified markdown report without cost columns.
Skips the first result if it's a warmup run (duplicate batch_size).
"""
@@ -60,7 +60,7 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
DEEPSEEK_V32_MODEL_PATH = os.environ.get(
"DEEPSEEK_V32_MODEL_PATH", "deepseek-ai/DeepSeek-V3.2"
)
PROFILE_DIR = "performance_profiles_deepseek_v32_basic_mi325"
RESULT_DIR = "performance_results_deepseek_v32_basic_mi325"
class TestNightlyDeepseekV32BasicPerformance(unittest.TestCase):
@@ -99,9 +99,9 @@ class TestNightlyDeepseekV32BasicPerformance(unittest.TestCase):
"env_vars": {"SGLANG_USE_AITER": "1"},
}
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
# Override full_report to remove traces help text
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
# Set the report header for this test
cls.runner.full_report = f"## {cls.__name__}\n"
def test_bench_one_batch(self):
@@ -115,7 +115,6 @@ class TestNightlyDeepseekV32BasicPerformance(unittest.TestCase):
other_args=self.variant_config["other_args"],
variant=self.variant_config["name"],
extra_bench_args=["--trust-remote-code"],
enable_profile=False, # Disable profiling for AMD tests
)
results = result_tuple[0]
success = result_tuple[1]
@@ -125,7 +124,7 @@ class TestNightlyDeepseekV32BasicPerformance(unittest.TestCase):
if avg_spec_accept_length is not None:
print(f" avg_spec_accept_length={avg_spec_accept_length:.2f}")
# Use simplified report format without traces
# Use the simplified report format
if results:
self.runner.full_report += (
generate_simple_markdown_report(results) + "\n"
test/registered/amd/perf/mi30x/test_deepseek_v32_mtp_perf_amd.py
+6 -7
View File
@@ -27,7 +27,7 @@ register_amd_ci(
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
"""Generate a simplified markdown report without traces and cost columns.
"""Generate a simplified markdown report without cost columns.
Skips the first result if it's a warmup run (duplicate batch_size).
"""
@@ -61,7 +61,7 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
DEEPSEEK_V32_MODEL_PATH = os.environ.get(
"DEEPSEEK_V32_MODEL_PATH", "deepseek-ai/DeepSeek-V3.2"
)
PROFILE_DIR = "performance_profiles_deepseek_v32_mtp_mi325"
RESULT_DIR = "performance_results_deepseek_v32_mtp_mi325"
class TestNightlyDeepseekV32MTPPerformance(unittest.TestCase):
@@ -108,9 +108,9 @@ class TestNightlyDeepseekV32MTPPerformance(unittest.TestCase):
"env_vars": {"SGLANG_USE_AITER": "1"},
}
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
# Override full_report to remove traces help text
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
# Set the report header for this test
cls.runner.full_report = f"## {cls.__name__}\n"
def test_bench_one_batch(self):
@@ -124,7 +124,6 @@ class TestNightlyDeepseekV32MTPPerformance(unittest.TestCase):
other_args=self.variant_config["other_args"],
variant=self.variant_config["name"],
extra_bench_args=["--trust-remote-code"],
enable_profile=False, # Disable profiling for AMD tests
)
results = result_tuple[0]
success = result_tuple[1]
@@ -134,7 +133,7 @@ class TestNightlyDeepseekV32MTPPerformance(unittest.TestCase):
if avg_spec_accept_length is not None:
print(f" avg_spec_accept_length={avg_spec_accept_length:.2f}")
# Use simplified report format without traces
# Use the simplified report format
if results:
self.runner.full_report += (
generate_simple_markdown_report(results) + "\n"
test/registered/amd/perf/mi30x/test_deepseek_v3_perf.py
+6 -7
View File
@@ -22,7 +22,7 @@ register_amd_ci(est_time=18000, suite="nightly-perf-8-gpu-deepseek-v3", nightly=
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
"""Generate a simplified markdown report without traces and cost columns."""
"""Generate a simplified markdown report without cost columns."""
model_header = results[0].model_path
if results[0].run_name and results[0].run_name != "default":
model_header += f" ({results[0].run_name})"
@@ -46,7 +46,7 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
DEEPSEEK_V3_MODEL_PATH = os.environ.get(
"DEEPSEEK_V3_MODEL_PATH", "deepseek-ai/DeepSeek-V3-0324"
)
PROFILE_DIR = "performance_profiles_deepseek_v3"
RESULT_DIR = "performance_results_deepseek_v3"
class TestNightlyDeepseekV3Performance(unittest.TestCase):
@@ -99,9 +99,9 @@ class TestNightlyDeepseekV3Performance(unittest.TestCase):
},
]
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
# Override full_report to remove traces help text
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
# Set the report header for this test
cls.runner.full_report = f"## {cls.__name__}\n"
def test_bench_one_batch(self):
@@ -119,7 +119,6 @@ class TestNightlyDeepseekV3Performance(unittest.TestCase):
other_args=variant_config["other_args"],
variant=variant_config["name"],
extra_bench_args=["--trust-remote-code"],
enable_profile=False, # Disable profiling for AMD tests
)
results = result_tuple[0]
success = result_tuple[1]
@@ -127,7 +126,7 @@ class TestNightlyDeepseekV3Performance(unittest.TestCase):
if not success:
failed_variants.append(variant_config["name"])
# Use simplified report format without traces
# Use the simplified report format
if results:
self.runner.full_report += (
generate_simple_markdown_report(results) + "\n"
test/registered/amd/perf/mi30x/test_glm51_perf_amd.py
+3 -4
View File
@@ -47,7 +47,7 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
GLM51_MODEL_PATH = os.environ.get("GLM51_MODEL_PATH", "zai-org/GLM-5.1-FP8")
PROFILE_DIR = "performance_profiles_glm51"
RESULT_DIR = "performance_results_glm51"
class TestNightlyGLM51Performance(unittest.TestCase):
@@ -94,8 +94,8 @@ class TestNightlyGLM51Performance(unittest.TestCase):
},
}
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
cls.runner.full_report = f"## {cls.__name__}\n"
def test_bench_glm51(self):
@@ -113,7 +113,6 @@ class TestNightlyGLM51Performance(unittest.TestCase):
other_args=self.model_config["other_args"],
variant=self.model_config["name"],
extra_bench_args=["--trust-remote-code"],
enable_profile=False,
timeout=5400,
)
results = result_tuple[0]
test/registered/amd/perf/mi30x/test_glm5_perf_amd.py
+3 -4
View File
@@ -48,7 +48,7 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
GLM5_MODEL_PATH = os.environ.get("GLM5_MODEL_PATH", "zai-org/GLM-5-FP8")
PROFILE_DIR = "performance_profiles_glm5"
RESULT_DIR = "performance_results_glm5"
class TestNightlyGLM5Performance(unittest.TestCase):
@@ -95,8 +95,8 @@ class TestNightlyGLM5Performance(unittest.TestCase):
},
}
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
cls.runner.full_report = f"## {cls.__name__}\n"
def test_bench_glm5(self):
@@ -115,7 +115,6 @@ class TestNightlyGLM5Performance(unittest.TestCase):
other_args=self.model_config["other_args"],
variant=self.model_config["name"],
extra_bench_args=["--trust-remote-code"],
enable_profile=False,
timeout=5400,
)
results = result_tuple[0]
test/registered/amd/perf/mi30x/test_grok1_fp8_perf.py
+4 -5
View File
@@ -24,7 +24,7 @@ register_amd_ci(est_time=1500, suite="nightly-perf-8-gpu-grok1-fp8", nightly=Tru
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
"""Generate a simplified markdown report without traces and cost columns."""
"""Generate a simplified markdown report without cost columns."""
model_header = results[0].model_path
if results[0].run_name and results[0].run_name != "default":
model_header += f" ({results[0].run_name})"
@@ -47,7 +47,7 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
# Model and tokenizer paths can be overridden via environment variables
GROK1_MODEL_PATH = os.environ.get("GROK1_MODEL_PATH", "lmzheng/grok-1")
GROK1_TOKENIZER_PATH = os.environ.get("GROK1_TOKENIZER_PATH", "Xenova/grok-1-tokenizer")
PROFILE_DIR = "performance_profiles_grok1_fp8"
RESULT_DIR = "performance_results_grok1_fp8"
class TestNightlyGrok1FP8Performance(unittest.TestCase):
@@ -87,8 +87,8 @@ class TestNightlyGrok1FP8Performance(unittest.TestCase):
},
}
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
cls.runner.full_report = f"## {cls.__name__}\n"
def test_bench_grok1_fp8(self):
@@ -109,7 +109,6 @@ class TestNightlyGrok1FP8Performance(unittest.TestCase):
other_args=self.model_config["other_args"],
variant=self.model_config["name"],
extra_bench_args=["--trust-remote-code"],
enable_profile=False, # Disable profiling for AMD tests
)
results = result_tuple[0]
success = result_tuple[1]
test/registered/amd/perf/mi30x/test_grok1_int4_perf.py
+4 -5
View File
@@ -24,7 +24,7 @@ register_amd_ci(est_time=1500, suite="nightly-perf-8-gpu-grok1-int4", nightly=Tr
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
"""Generate a simplified markdown report without traces and cost columns.
"""Generate a simplified markdown report without cost columns.
Skips the first result if it's a warmup run (duplicate batch_size).
"""
@@ -57,7 +57,7 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
# Model and tokenizer paths can be overridden via environment variables
GROK1_MODEL_PATH = os.environ.get("GROK1_MODEL_PATH", "amd/grok-1-W4A8KV8")
GROK1_TOKENIZER_PATH = os.environ.get("GROK1_TOKENIZER_PATH", "Xenova/grok-1-tokenizer")
PROFILE_DIR = "performance_profiles_grok1_int4"
RESULT_DIR = "performance_results_grok1_int4"
class TestNightlyGrok1INT4Performance(unittest.TestCase):
@@ -97,8 +97,8 @@ class TestNightlyGrok1INT4Performance(unittest.TestCase):
},
}
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
cls.runner.full_report = f"## {cls.__name__}\n"
def test_bench_grok1_int4(self):
@@ -119,7 +119,6 @@ class TestNightlyGrok1INT4Performance(unittest.TestCase):
other_args=self.model_config["other_args"],
variant=self.model_config["name"],
extra_bench_args=["--trust-remote-code"],
enable_profile=False, # Disable profiling for AMD tests
)
results = result_tuple[0]
success = result_tuple[1]
test/registered/amd/perf/mi30x/test_grok2_perf.py
+4 -5
View File
@@ -24,7 +24,7 @@ register_amd_ci(est_time=1500, suite="nightly-perf-8-gpu-grok2", nightly=True)
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
"""Generate a simplified markdown report without traces and cost columns.
"""Generate a simplified markdown report without cost columns.
Skips the first result if it's a warmup run (duplicate batch_size).
"""
@@ -59,7 +59,7 @@ GROK2_MODEL_PATH = os.environ.get("GROK2_MODEL_PATH", "xai-org/grok-2")
GROK2_TOKENIZER_PATH = os.environ.get(
"GROK2_TOKENIZER_PATH", "alvarobartt/grok-2-tokenizer"
)
PROFILE_DIR = "performance_profiles_grok2"
RESULT_DIR = "performance_results_grok2"
class TestNightlyGrok2Performance(unittest.TestCase):
@@ -99,8 +99,8 @@ class TestNightlyGrok2Performance(unittest.TestCase):
},
}
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
cls.runner.full_report = f"## {cls.__name__}\n"
def test_bench_grok2(self):
@@ -121,7 +121,6 @@ class TestNightlyGrok2Performance(unittest.TestCase):
other_args=self.model_config["other_args"],
variant=self.model_config["name"],
extra_bench_args=["--trust-remote-code"],
enable_profile=False, # Disable profiling for AMD tests
)
results = result_tuple[0]
success = result_tuple[1]
test/registered/amd/perf/mi30x/test_kimi_k26_perf_amd.py
+4 -5
View File
@@ -28,7 +28,7 @@ register_amd_ci(est_time=5400, suite="nightly-perf-8-gpu-kimi-k26", nightly=True
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
"""Generate a simplified markdown report without traces and cost columns.
"""Generate a simplified markdown report without cost columns.
Skips the first result if it's a warmup run (duplicate batch_size).
"""
@@ -58,7 +58,7 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
KIMI_K26_MODEL_PATH = os.environ.get("KIMI_K26_MODEL_PATH", "moonshotai/Kimi-K2.6")
PROFILE_DIR = "performance_profiles_kimi_k26"
RESULT_DIR = "performance_results_kimi_k26"
class TestNightlyKimiK26Performance(unittest.TestCase):
@@ -101,8 +101,8 @@ class TestNightlyKimiK26Performance(unittest.TestCase):
},
}
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
cls.runner.full_report = f"## {cls.__name__}\n"
def test_bench_kimi_k26(self):
@@ -121,7 +121,6 @@ class TestNightlyKimiK26Performance(unittest.TestCase):
other_args=self.model_config["other_args"],
variant=self.model_config["name"],
extra_bench_args=["--trust-remote-code"],
enable_profile=False,
timeout=5400,
)
results = result_tuple[0]
test/registered/amd/perf/mi30x/test_minimax_m25_perf_amd.py
+4 -5
View File
@@ -23,7 +23,7 @@ register_amd_ci(est_time=5400, suite="nightly-perf-8-gpu-minimax-m25", nightly=T
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
"""Generate a simplified markdown report without traces and cost columns.
"""Generate a simplified markdown report without cost columns.
Skips the first result if it's a warmup run (duplicate batch_size).
"""
@@ -55,7 +55,7 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
MINIMAX_M25_MODEL_PATH = os.environ.get(
"MINIMAX_M25_MODEL_PATH", "MiniMaxAI/MiniMax-M2.5"
)
PROFILE_DIR = "performance_profiles_minimax_m25"
RESULT_DIR = "performance_results_minimax_m25"
class TestNightlyMiniMaxM25Performance(unittest.TestCase):
@@ -94,8 +94,8 @@ class TestNightlyMiniMaxM25Performance(unittest.TestCase):
},
}
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
cls.runner.full_report = f"## {cls.__name__}\n"
def test_bench_minimax_m25(self):
@@ -115,7 +115,6 @@ class TestNightlyMiniMaxM25Performance(unittest.TestCase):
other_args=self.model_config["other_args"],
variant=self.model_config["name"],
extra_bench_args=["--trust-remote-code"],
enable_profile=False,
timeout=5400,
)
results = result_tuple[0]
test/registered/amd/perf/mi30x/test_minimax_m27_perf_amd.py
+4 -5
View File
@@ -23,7 +23,7 @@ register_amd_ci(est_time=5400, suite="nightly-perf-8-gpu-minimax-m27", nightly=T
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
"""Generate a simplified markdown report without traces and cost columns.
"""Generate a simplified markdown report without cost columns.
Skips the first result if it's a warmup run (duplicate batch_size).
"""
@@ -55,7 +55,7 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
MINIMAX_M27_MODEL_PATH = os.environ.get(
"MINIMAX_M27_MODEL_PATH", "MiniMaxAI/MiniMax-M2.7"
)
PROFILE_DIR = "performance_profiles_minimax_m27"
RESULT_DIR = "performance_results_minimax_m27"
class TestNightlyMiniMaxM27Performance(unittest.TestCase):
@@ -94,8 +94,8 @@ class TestNightlyMiniMaxM27Performance(unittest.TestCase):
},
}
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
cls.runner.full_report = f"## {cls.__name__}\n"
def test_bench_minimax_m27(self):
@@ -115,7 +115,6 @@ class TestNightlyMiniMaxM27Performance(unittest.TestCase):
other_args=self.model_config["other_args"],
variant=self.model_config["name"],
extra_bench_args=["--trust-remote-code"],
enable_profile=False,
timeout=5400,
)
results = result_tuple[0]
test/registered/amd/perf/mi30x/test_qwen35_fp8_perf_amd.py
+4 -5
View File
@@ -24,7 +24,7 @@ register_amd_ci(est_time=5400, suite="nightly-perf-8-gpu-qwen35-fp8", nightly=Tr
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
"""Generate a simplified markdown report without traces and cost columns.
"""Generate a simplified markdown report without cost columns.
Skips the first result if it's a warmup run (duplicate batch_size).
"""
@@ -56,7 +56,7 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
QWEN35_FP8_MODEL_PATH = os.environ.get(
"QWEN35_FP8_MODEL_PATH", "Qwen/Qwen3.5-397B-A17B-FP8"
)
PROFILE_DIR = "performance_profiles_qwen35_fp8"
RESULT_DIR = "performance_results_qwen35_fp8"
class TestNightlyQwen35Fp8Performance(unittest.TestCase):
@@ -94,8 +94,8 @@ class TestNightlyQwen35Fp8Performance(unittest.TestCase):
},
}
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
cls.runner.full_report = f"## {cls.__name__}\n"
def test_bench_qwen35_fp8(self):
@@ -114,7 +114,6 @@ class TestNightlyQwen35Fp8Performance(unittest.TestCase):
other_args=self.model_config["other_args"],
variant=self.model_config["name"],
extra_bench_args=["--trust-remote-code"],
enable_profile=False,
timeout=5400,
)
results = result_tuple[0]
test/registered/amd/perf/mi30x/test_text_models_perf_amd.py
+4 -5
View File
@@ -25,11 +25,11 @@ from sglang.test.test_utils import (
# Register for AMD CI - Text models benchmark (~60 min)
register_amd_ci(est_time=3600, suite="nightly-amd-perf-text-2-gpu", nightly=True)
PROFILE_DIR = "performance_profiles_text_models_amd"
RESULT_DIR = "performance_results_text_models_amd"
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
"""Generate a simplified markdown report without traces and cost columns.
"""Generate a simplified markdown report without cost columns.
Skips the first result if it's a warmup run (duplicate batch_size).
"""
@@ -89,8 +89,8 @@ class TestNightlyTextModelsPerfAMD(unittest.TestCase):
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
cls.runner.full_report = f"## {cls.__name__}\n"
def test_bench_one_batch(self):
@@ -110,7 +110,6 @@ class TestNightlyTextModelsPerfAMD(unittest.TestCase):
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=other_args,
enable_profile=False, # Disable profiling for AMD tests
)
results = result_tuple[0]
success = result_tuple[1]
test/registered/amd/perf/mi30x/test_vlms_perf_amd.py
+4 -5
View File
@@ -26,7 +26,7 @@ from sglang.test.test_utils import (
# Register for AMD CI - VLM models benchmark (~120 min)
register_amd_ci(est_time=7200, suite="nightly-amd-perf-vlm-2-gpu", nightly=True)
PROFILE_DIR = "performance_profiles_vlms_amd"
RESULT_DIR = "performance_results_vlms_amd"
# VLM models suitable for AMD
MODEL_DEFAULTS = [
@@ -42,7 +42,7 @@ MODEL_DEFAULTS = [
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
"""Generate a simplified markdown report without traces and cost columns.
"""Generate a simplified markdown report without cost columns.
Skips the first result if it's a warmup run (duplicate batch_size).
"""
@@ -95,8 +95,8 @@ class TestNightlyVLMsPerfAMD(unittest.TestCase):
cls.batch_sizes = _parse_int_list_env("NIGHTLY_VLM_BATCH_SIZES", "1,1,2,8,16")
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_VLM_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_VLM_OUTPUT_LENS", "512"))
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
cls.runner.full_report = f"## {cls.__name__}\n"
def test_bench_one_batch(self):
@@ -123,7 +123,6 @@ class TestNightlyVLMsPerfAMD(unittest.TestCase):
output_lens=self.output_lens,
other_args=other_args,
extra_bench_args=extra_bench_args,
enable_profile=False, # Disable profiling for AMD tests
)
results = result_tuple[0]
success = result_tuple[1]
test/registered/amd/perf/mi35x/test_deepseek_r1_mxfp4_ar_fusion_perf_mi35x.py
+4 -5
View File
@@ -24,7 +24,7 @@ register_amd_ci(
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
"""Generate a simplified markdown report without traces and cost columns.
"""Generate a simplified markdown report without cost columns.
Skips the first result if it's a warmup run (duplicate batch_size).
"""
@@ -54,7 +54,7 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
return summary
PROFILE_DIR = "performance_profiles_deepseek_r1_mxfp4_ar_fusion_mi35x"
RESULT_DIR = "performance_results_deepseek_r1_mxfp4_ar_fusion_mi35x"
class TestDeepseekR1MXFP4ArFusionPerfMI35x(unittest.TestCase):
@@ -90,8 +90,8 @@ class TestDeepseekR1MXFP4ArFusionPerfMI35x(unittest.TestCase):
},
]
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
cls.runner.full_report = f"## {cls.__name__}\n"
def test_bench_one_batch(self):
@@ -109,7 +109,6 @@ class TestDeepseekR1MXFP4ArFusionPerfMI35x(unittest.TestCase):
other_args=variant_config["other_args"],
variant=variant_config["name"],
extra_bench_args=["--trust-remote-code"],
enable_profile=False,
)
results = result_tuple[0]
success = result_tuple[1]
test/registered/amd/perf/mi35x/test_deepseek_r1_mxfp4_kv_fp8_perf_mi35x.py
+4 -5
View File
@@ -24,7 +24,7 @@ register_amd_ci(
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
"""Generate a simplified markdown report without traces and cost columns.
"""Generate a simplified markdown report without cost columns.
Skips the first result if it's a warmup run (duplicate batch_size).
"""
@@ -54,7 +54,7 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
return summary
PROFILE_DIR = "performance_profiles_deepseek_r1_mxfp4_kv_fp8_mi35x"
RESULT_DIR = "performance_results_deepseek_r1_mxfp4_kv_fp8_mi35x"
class TestDeepseekR1MXFP4KvFp8PerfMI35x(unittest.TestCase):
@@ -91,8 +91,8 @@ class TestDeepseekR1MXFP4KvFp8PerfMI35x(unittest.TestCase):
},
]
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
cls.runner.full_report = f"## {cls.__name__}\n"
def test_bench_one_batch(self):
@@ -110,7 +110,6 @@ class TestDeepseekR1MXFP4KvFp8PerfMI35x(unittest.TestCase):
other_args=variant_config["other_args"],
variant=variant_config["name"],
extra_bench_args=["--trust-remote-code"],
enable_profile=False,
)
results = result_tuple[0]
success = result_tuple[1]
test/registered/amd/perf/mi35x/test_deepseek_r1_mxfp4_perf_mi35x.py
+6 -7
View File
@@ -21,7 +21,7 @@ register_amd_ci(
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
"""Generate a simplified markdown report without traces and cost columns.
"""Generate a simplified markdown report without cost columns.
Skips the first result if it's a warmup run (duplicate batch_size).
"""
@@ -51,7 +51,7 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
return summary
PROFILE_DIR = "performance_profiles_deepseek_r1_mxfp4_mi35x"
RESULT_DIR = "performance_results_deepseek_r1_mxfp4_mi35x"
class TestDeepseekR1MXFP4PerfMI35x(unittest.TestCase):
@@ -88,9 +88,9 @@ class TestDeepseekR1MXFP4PerfMI35x(unittest.TestCase):
},
]
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
# Override full_report to remove traces help text
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
# Set the report header for this test
cls.runner.full_report = f"## {cls.__name__}\n"
def test_bench_one_batch(self):
@@ -108,7 +108,6 @@ class TestDeepseekR1MXFP4PerfMI35x(unittest.TestCase):
other_args=variant_config["other_args"],
variant=variant_config["name"],
extra_bench_args=["--trust-remote-code"],
enable_profile=False, # Disable profiling for AMD tests
)
results = result_tuple[0]
success = result_tuple[1]
@@ -116,7 +115,7 @@ class TestDeepseekR1MXFP4PerfMI35x(unittest.TestCase):
if not success:
failed_variants.append(variant_config["name"])
# Use simplified report format without traces
# Use the simplified report format
if results:
self.runner.full_report += (
generate_simple_markdown_report(results) + "\n"
test/registered/amd/perf/mi35x/test_deepseek_v32_basic_perf_mi35x.py
+6 -7
View File
@@ -26,7 +26,7 @@ register_amd_ci(
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
"""Generate a simplified markdown report without traces and cost columns.
"""Generate a simplified markdown report without cost columns.
Skips the first result if it's a warmup run (duplicate batch_size).
"""
@@ -60,7 +60,7 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
DEEPSEEK_V32_MODEL_PATH = os.environ.get(
"DEEPSEEK_V32_MODEL_PATH", "deepseek-ai/DeepSeek-V3.2"
)
PROFILE_DIR = "performance_profiles_deepseek_v32_basic"
RESULT_DIR = "performance_results_deepseek_v32_basic"
class TestNightlyDeepseekV32BasicPerformance(unittest.TestCase):
@@ -98,9 +98,9 @@ class TestNightlyDeepseekV32BasicPerformance(unittest.TestCase):
],
}
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
# Override full_report to remove traces help text
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
# Set the report header for this test
cls.runner.full_report = f"## {cls.__name__}\n"
def test_bench_one_batch(self):
@@ -114,13 +114,12 @@ class TestNightlyDeepseekV32BasicPerformance(unittest.TestCase):
other_args=self.variant_config["other_args"],
variant=self.variant_config["name"],
extra_bench_args=["--trust-remote-code"],
enable_profile=False, # Disable profiling for AMD tests
timeout=5400, # Extended timeout for large model loading
)
results = result_tuple[0]
success = result_tuple[1]
# Use simplified report format without traces
# Use the simplified report format
if results:
self.runner.full_report += (
generate_simple_markdown_report(results) + "\n"
test/registered/amd/perf/mi35x/test_deepseek_v32_mtp_perf_mi35x.py
+7 -11
View File
@@ -32,7 +32,7 @@ register_amd_ci(
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
"""Generate a simplified markdown report without traces and cost columns.
"""Generate a simplified markdown report without cost columns.
Skips the first result if it's a warmup run (duplicate batch_size).
"""
@@ -82,9 +82,7 @@ def _run_benchmark_with_timeout(
timeout=timeout,
)
try:
profile_path_prefix, json_output_file = runner.generate_profile_filename(
model_path, variant
)
json_output_file = runner.generate_result_filename(model_path, variant)
bench_args = list(extra_bench_args) if extra_bench_args else []
if variant:
bench_args.extend(["--run-name", variant])
@@ -93,10 +91,8 @@ def _run_benchmark_with_timeout(
batch_sizes,
input_lens,
output_lens,
profile_path_prefix,
json_output_file,
extra_args=bench_args,
enable_profile=False, # Disable profiling for AMD tests
)
_, cmd_success = runner.run_benchmark_command(command, model_description)
if not cmd_success:
@@ -113,7 +109,7 @@ def _run_benchmark_with_timeout(
DEEPSEEK_V32_MODEL_PATH = os.environ.get(
"DEEPSEEK_V32_MODEL_PATH", "deepseek-ai/DeepSeek-V3.2"
)
PROFILE_DIR = "performance_profiles_deepseek_v32_mtp"
RESULT_DIR = "performance_results_deepseek_v32_mtp"
SERVER_LAUNCH_TIMEOUT = 5400
@@ -160,9 +156,9 @@ class TestNightlyDeepseekV32MTPPerformance(unittest.TestCase):
],
}
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
# Override full_report to remove traces help text
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
# Set the report header for this test
cls.runner.full_report = f"## {cls.__name__}\n"
def test_bench_one_batch(self):
@@ -187,7 +183,7 @@ class TestNightlyDeepseekV32MTPPerformance(unittest.TestCase):
if avg_spec_accept_length is not None:
print(f" avg_spec_accept_length={avg_spec_accept_length:.2f}")
# Use simplified report format without traces
# Use the simplified report format
if results:
self.runner.full_report += (
generate_simple_markdown_report(results) + "\n"
test/registered/amd/perf/mi35x/test_glm51_perf_mi35x.py
+3 -4
View File
@@ -45,7 +45,7 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
GLM51_MODEL_PATH = os.environ.get("GLM51_MODEL_PATH", "zai-org/GLM-5.1-FP8")
PROFILE_DIR = "performance_profiles_glm51_mi35x"
RESULT_DIR = "performance_results_glm51_mi35x"
class TestGLM51PerfMI35x(unittest.TestCase):
@@ -96,8 +96,8 @@ class TestGLM51PerfMI35x(unittest.TestCase):
}
os.environ.setdefault("SGLANG_BENCH_TIMEOUT", "3600")
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
cls.runner.full_report = f"## {cls.__name__}\n"
def test_glm51_perf(self):
@@ -115,7 +115,6 @@ class TestGLM51PerfMI35x(unittest.TestCase):
other_args=self.model_config["other_args"],
variant=self.model_config["name"],
extra_bench_args=["--trust-remote-code"],
enable_profile=False,
timeout=5400,
)
results = result_tuple[0]
test/registered/amd/perf/mi35x/test_glm5_mxfp4_perf_mi35x.py
+4 -5
View File
@@ -25,7 +25,7 @@ register_amd_ci(
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
"""Generate a simplified markdown report without traces and cost columns.
"""Generate a simplified markdown report without cost columns.
Skips the first result if it's a warmup run (duplicate batch_size).
"""
@@ -58,7 +58,7 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
return summary
PROFILE_DIR = "performance_profiles_glm5_mxfp4_mi35x"
RESULT_DIR = "performance_results_glm5_mxfp4_mi35x"
class TestGLM5MXFP4PerfMI35x(unittest.TestCase):
@@ -99,8 +99,8 @@ class TestGLM5MXFP4PerfMI35x(unittest.TestCase):
},
]
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
cls.runner.full_report = f"## {cls.__name__}\n"
def test_bench_one_batch(self):
@@ -124,7 +124,6 @@ class TestGLM5MXFP4PerfMI35x(unittest.TestCase):
other_args=variant_config["other_args"],
variant=variant_config["name"],
extra_bench_args=["--trust-remote-code"],
enable_profile=False,
)
results = result_tuple[0]
success = result_tuple[1]
test/registered/amd/perf/mi35x/test_glm5_perf_mi35x.py
+3 -4
View File
@@ -44,7 +44,7 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
GLM5_MODEL_PATH = os.environ.get("GLM5_MODEL_PATH", "zai-org/GLM-5-FP8")
PROFILE_DIR = "performance_profiles_glm5_mi35x"
RESULT_DIR = "performance_results_glm5_mi35x"
class TestGLM5PerfMI35x(unittest.TestCase):
@@ -94,8 +94,8 @@ class TestGLM5PerfMI35x(unittest.TestCase):
}
os.environ.setdefault("SGLANG_BENCH_TIMEOUT", "3600")
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
cls.runner.full_report = f"## {cls.__name__}\n"
def test_glm5_perf(self):
@@ -114,7 +114,6 @@ class TestGLM5PerfMI35x(unittest.TestCase):
other_args=self.model_config["other_args"],
variant=self.model_config["name"],
extra_bench_args=["--trust-remote-code"],
enable_profile=False,
timeout=5400,
)
results = result_tuple[0]
test/registered/amd/perf/mi35x/test_grok1_int4_perf_mi35x.py
+4 -5
View File
@@ -21,7 +21,7 @@ register_amd_ci(
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
"""Generate a simplified markdown report without traces and cost columns.
"""Generate a simplified markdown report without cost columns.
Skips the first result if it's a warmup run (duplicate batch_size).
"""
@@ -54,7 +54,7 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
# Model and tokenizer paths can be overridden via environment variables
GROK1_MODEL_PATH = os.environ.get("GROK1_MODEL_PATH", "amd/grok-1-W4A8KV8")
GROK1_TOKENIZER_PATH = os.environ.get("GROK1_TOKENIZER_PATH", "Xenova/grok-1-tokenizer")
PROFILE_DIR = "performance_profiles_grok1_int4_mi35x"
RESULT_DIR = "performance_results_grok1_int4_mi35x"
class TestGrok1INT4PerfMI35x(unittest.TestCase):
@@ -90,8 +90,8 @@ class TestGrok1INT4PerfMI35x(unittest.TestCase):
},
}
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
cls.runner.full_report = f"## {cls.__name__}\n"
def test_grok1_int4_perf(self):
@@ -112,7 +112,6 @@ class TestGrok1INT4PerfMI35x(unittest.TestCase):
other_args=self.model_config["other_args"],
variant=self.model_config["name"],
extra_bench_args=["--trust-remote-code"],
enable_profile=False, # Disable profiling for AMD tests
)
results = result_tuple[0]
success = result_tuple[1]
test/registered/amd/perf/mi35x/test_grok2_perf_mi35x.py
+4 -5
View File
@@ -19,7 +19,7 @@ register_amd_ci(est_time=1500, suite="nightly-perf-8-gpu-mi35x-grok2", nightly=T
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
"""Generate a simplified markdown report without traces and cost columns.
"""Generate a simplified markdown report without cost columns.
Skips the first result if it's a warmup run (duplicate batch_size).
"""
@@ -54,7 +54,7 @@ GROK2_MODEL_PATH = os.environ.get("GROK2_MODEL_PATH", "xai-org/grok-2")
GROK2_TOKENIZER_PATH = os.environ.get(
"GROK2_TOKENIZER_PATH", "alvarobartt/grok-2-tokenizer"
)
PROFILE_DIR = "performance_profiles_grok2_mi35x"
RESULT_DIR = "performance_results_grok2_mi35x"
class TestGrok2PerfMI35x(unittest.TestCase):
@@ -90,8 +90,8 @@ class TestGrok2PerfMI35x(unittest.TestCase):
},
}
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
cls.runner.full_report = f"## {cls.__name__}\n"
def test_grok2_perf(self):
@@ -112,7 +112,6 @@ class TestGrok2PerfMI35x(unittest.TestCase):
other_args=self.model_config["other_args"],
variant=self.model_config["name"],
extra_bench_args=["--trust-remote-code"],
enable_profile=False, # Disable profiling for AMD tests
)
results = result_tuple[0]
success = result_tuple[1]
test/registered/amd/perf/mi35x/test_kimi_k26_perf_mi35x.py
+4 -5
View File
@@ -28,7 +28,7 @@ register_amd_ci(est_time=5400, suite="nightly-perf-8-gpu-mi35x-kimi-k26", nightl
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
"""Generate a simplified markdown report without traces and cost columns.
"""Generate a simplified markdown report without cost columns.
Skips the first result if it's a warmup run (duplicate batch_size).
"""
@@ -58,7 +58,7 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
KIMI_K26_MODEL_PATH = os.environ.get("KIMI_K26_MODEL_PATH", "moonshotai/Kimi-K2.6")
PROFILE_DIR = "performance_profiles_kimi_k26_mi35x"
RESULT_DIR = "performance_results_kimi_k26_mi35x"
class TestNightlyKimiK26PerformanceMI35x(unittest.TestCase):
@@ -101,8 +101,8 @@ class TestNightlyKimiK26PerformanceMI35x(unittest.TestCase):
},
}
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
cls.runner.full_report = f"## {cls.__name__}\n"
def test_bench_kimi_k26(self):
@@ -121,7 +121,6 @@ class TestNightlyKimiK26PerformanceMI35x(unittest.TestCase):
other_args=self.model_config["other_args"],
variant=self.model_config["name"],
extra_bench_args=["--trust-remote-code"],
enable_profile=False,
timeout=5400,
)
results = result_tuple[0]
test/registered/amd/perf/mi35x/test_minimax_m25_perf_mi35x.py
+4 -5
View File
@@ -25,7 +25,7 @@ register_amd_ci(
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
"""Generate a simplified markdown report without traces and cost columns.
"""Generate a simplified markdown report without cost columns.
Skips the first result if it's a warmup run (duplicate batch_size).
"""
@@ -57,7 +57,7 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
MINIMAX_M25_MODEL_PATH = os.environ.get(
"MINIMAX_M25_MODEL_PATH", "MiniMaxAI/MiniMax-M2.5"
)
PROFILE_DIR = "performance_profiles_minimax_m25_mi35x"
RESULT_DIR = "performance_results_minimax_m25_mi35x"
class TestNightlyMiniMaxM25PerformanceMI35x(unittest.TestCase):
@@ -96,8 +96,8 @@ class TestNightlyMiniMaxM25PerformanceMI35x(unittest.TestCase):
},
}
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
cls.runner.full_report = f"## {cls.__name__}\n"
def test_bench_minimax_m25(self):
@@ -117,7 +117,6 @@ class TestNightlyMiniMaxM25PerformanceMI35x(unittest.TestCase):
other_args=self.model_config["other_args"],
variant=self.model_config["name"],
extra_bench_args=["--trust-remote-code"],
enable_profile=False,
timeout=5400,
)
results = result_tuple[0]
test/registered/amd/perf/mi35x/test_minimax_m27_perf_mi35x.py
+4 -5
View File
@@ -25,7 +25,7 @@ register_amd_ci(
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
"""Generate a simplified markdown report without traces and cost columns.
"""Generate a simplified markdown report without cost columns.
Skips the first result if it's a warmup run (duplicate batch_size).
"""
@@ -57,7 +57,7 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
MINIMAX_M27_MODEL_PATH = os.environ.get(
"MINIMAX_M27_MODEL_PATH", "MiniMaxAI/MiniMax-M2.7"
)
PROFILE_DIR = "performance_profiles_minimax_m27_mi35x"
RESULT_DIR = "performance_results_minimax_m27_mi35x"
class TestNightlyMiniMaxM27PerformanceMI35x(unittest.TestCase):
@@ -96,8 +96,8 @@ class TestNightlyMiniMaxM27PerformanceMI35x(unittest.TestCase):
},
}
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
cls.runner.full_report = f"## {cls.__name__}\n"
def test_bench_minimax_m27(self):
@@ -117,7 +117,6 @@ class TestNightlyMiniMaxM27PerformanceMI35x(unittest.TestCase):
other_args=self.model_config["other_args"],
variant=self.model_config["name"],
extra_bench_args=["--trust-remote-code"],
enable_profile=False,
timeout=5400,
)
results = result_tuple[0]
test/registered/amd/perf/mi35x/test_qwen35_fp8_perf_mi35x.py
+4 -5
View File
@@ -21,7 +21,7 @@ register_amd_ci(
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
"""Generate a simplified markdown report without traces and cost columns.
"""Generate a simplified markdown report without cost columns.
Skips the first result if it's a warmup run (duplicate batch_size).
"""
@@ -53,7 +53,7 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
QWEN35_FP8_MODEL_PATH = os.environ.get(
"QWEN35_FP8_MODEL_PATH", "Qwen/Qwen3.5-397B-A17B-FP8"
)
PROFILE_DIR = "performance_profiles_qwen35_fp8_mi35x"
RESULT_DIR = "performance_results_qwen35_fp8_mi35x"
class TestQwen35Fp8PerfMI35x(unittest.TestCase):
@@ -87,8 +87,8 @@ class TestQwen35Fp8PerfMI35x(unittest.TestCase):
},
}
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
cls.runner.full_report = f"## {cls.__name__}\n"
def test_qwen35_fp8_perf(self):
@@ -107,7 +107,6 @@ class TestQwen35Fp8PerfMI35x(unittest.TestCase):
other_args=self.model_config["other_args"],
variant=self.model_config["name"],
extra_bench_args=["--trust-remote-code"],
enable_profile=False,
timeout=5400,
)
results = result_tuple[0]
test/registered/gb300/test_deepseek_v4_pro_fp4.py
+1 -1
View File
@@ -136,7 +136,7 @@ class TestDeepSeekV4ProFp4(unittest.TestCase):
accuracy_params=accuracy_params,
performance_params=PerformanceTestParams(
batch_sizes=PERFORMANCE_BATCH_SIZES[variant.variant],
profile_dir="performance_profiles_gb300",
result_dir="performance_results_gb300",
),
)
except AssertionError as e:
test/registered/gb300/test_glm52_nvfp4.py
+1 -1
View File
@@ -61,7 +61,7 @@ class TestGlm52Nvfp4(unittest.TestCase):
test_name="GLM-5.2-NVFP4",
accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.92),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_gb300",
result_dir="performance_results_gb300",
),
)
test/registered/gb300/test_kimi_k25.py
+1 -1
View File
@@ -54,7 +54,7 @@ class TestKimiK25(unittest.TestCase):
dataset="mmmu-pro", baseline_accuracy=0.69, repeat=1, max_tokens=32768
),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_gb300",
result_dir="performance_results_gb300",
),
)
test/registered/gb300/test_kimi_k25_nvfp4.py
+1 -1
View File
@@ -69,7 +69,7 @@ class TestKimiK25Nvfp4(unittest.TestCase):
dataset="mmmu-pro", baseline_accuracy=0.69, repeat=1, max_tokens=32768
),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_gb300",
result_dir="performance_results_gb300",
),
)
test/registered/gb300/test_qwen35_fp8.py
+1 -1
View File
@@ -65,7 +65,7 @@ class TestQwen35Fp8(unittest.TestCase):
dataset="mmmu-pro", baseline_accuracy=0.76, repeat=1, max_tokens=32768
),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_gb300",
result_dir="performance_results_gb300",
),
)
test/registered/gb300/test_qwen35_nvfp4.py
+1 -1
View File
@@ -74,7 +74,7 @@ class TestQwen35Nvfp4(unittest.TestCase):
dataset="mmmu-pro", baseline_accuracy=0.76, repeat=1, max_tokens=32768
),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_gb300",
result_dir="performance_results_gb300",
),
)
test/registered/perf/test_dpsk_v3_fp4_4gpu_perf.py
+1 -1
View File
@@ -68,7 +68,7 @@ class TestDeepseekR1FP4Unified(unittest.TestCase):
api="completion",
),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_deepseek_v3_fp4",
result_dir="performance_results_deepseek_v3_fp4",
),
)
test/registered/perf/test_gpt_oss_4gpu_perf.py
+3 -3
View File
@@ -6,7 +6,7 @@ from sglang.test.test_utils import DEFAULT_URL_FOR_TEST
register_cuda_ci(est_time=600, suite="nightly-4-gpu-b200", nightly=True)
PROFILE_DIR = "performance_profiles_gpt_oss_4gpu"
RESULT_DIR = "performance_results_gpt_oss_4gpu"
class TestNightlyGptOss4GpuPerformance(unittest.TestCase):
@@ -29,8 +29,8 @@ class TestNightlyGptOss4GpuPerformance(unittest.TestCase):
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = (4096,)
cls.output_lens = (512,)
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
def test_bench_one_batch(self):
all_model_succeed = True
test/registered/perf/test_text_models_perf.py
+3 -3
View File
@@ -11,7 +11,7 @@ from sglang.test.test_utils import (
register_cuda_ci(est_time=3600, suite="nightly-perf-text-2-gpu", nightly=True)
PROFILE_DIR = "performance_profiles_text_models"
RESULT_DIR = "performance_results_text_models"
class TestNightlyTextModelsPerformance(unittest.TestCase):
@@ -31,8 +31,8 @@ class TestNightlyTextModelsPerformance(unittest.TestCase):
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
def test_bench_one_batch(self):
all_model_succeed = True
test/registered/perf/test_vlms_perf.py
+3 -3
View File
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
register_cuda_ci(est_time=7200, suite="nightly-perf-vlm-2-gpu", nightly=True)
PROFILE_DIR = "performance_profiles_vlms"
RESULT_DIR = "performance_results_vlms"
MODEL_DEFAULTS = [
# Keep conservative defaults. Can be overridden by env NIGHTLY_VLM_MODELS
@@ -52,8 +52,8 @@ class TestNightlyVLMModelsPerformance(unittest.TestCase):
cls.batch_sizes = _parse_int_list_env("NIGHTLY_VLM_BATCH_SIZES", "1,1,2,8,16")
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_VLM_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_VLM_OUTPUT_LENS", "512"))
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner = NightlyBenchmarkRunner(RESULT_DIR, cls.__name__, cls.base_url)
cls.runner.setup_result_directory()
def test_bench_one_batch(self):
all_model_succeed = True
test/registered/quant/test_kimi_k25_nvfp4_eagle.py
+1 -1
View File
@@ -59,7 +59,7 @@ class TestKimiK25Nvfp4Eagle(unittest.TestCase):
performance_params=PerformanceTestParams(
batch_sizes=[1, 8, 16],
spec_accept_length_threshold=2.8,
profile_dir="performance_profiles_kimi_k25_nvfp4_eagle",
result_dir="performance_results_kimi_k25_nvfp4_eagle",
),
)
test/registered/quant/test_kimi_k26_nvfp4_dflash.py
+1 -1
View File
@@ -61,7 +61,7 @@ class TestKimiK26Nvfp4Dflash(unittest.TestCase):
performance_params=PerformanceTestParams(
batch_sizes=[1, 8, 16],
spec_accept_length_threshold=2.0,
profile_dir="performance_profiles_kimi_k26_nvfp4_dflash",
result_dir="performance_results_kimi_k26_nvfp4_dflash",
),
)