minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[CI] Remove profiling from nightly tests (#33832)

Browse Source
This commit is contained in:
Baizhou Zhang
2026-08-06 01:16:08 -07:00
committed by GitHub
parent c11ce7c514
commit 0e584529f5
65 changed files with 214 additions and 429 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/test/nightly_bench_utils.py
+6 -55
View File
@@ -1,12 +1,9 @@
import json
import logging
import os
from typing import List, Optional
from pydantic import BaseModel
logger = logging.getLogger(__name__)
# TODO:
# There is huge redundancy between BenchmarkResult and BenchOneCaseResult, and redundancy between to_markdown_row, generate_markdown_report, get_report_summary.
# We should refactor them to reduce the code duplication.
@@ -33,17 +30,7 @@ class BenchmarkResult(BaseModel):
profile_link_decode: Optional[str] = None
server_args: Optional[List[str]] = None
@staticmethod
def help_str() -> str:
return f"""
Note: To view the traces through perfetto-ui, please:
1. open with Google Chrome
2. allow popup
"""
def to_markdown_row(
self, trace_dir, base_url: str = "", relay_base: str = ""
) -> str:
def to_markdown_row(self) -> str:
"""Convert this benchmark result to a markdown table row."""
hourly_cost_per_gpu = 2 # $2/hour for one H100
@@ -54,42 +41,11 @@ Note: To view the traces through perfetto-ui, please:
input_cost = 1e6 / (self.input_throughput * input_util) / 3600 * hourly_cost
output_cost = 1e6 / self.output_throughput / 3600 * hourly_cost
def get_perfetto_relay_link_from_trace_file(trace_file: str):
from urllib.parse import quote
rel_path = os.path.relpath(trace_file, trace_dir)
raw_file_link = f"{base_url}/{rel_path}"
relay_link = (
f"{relay_base}?src={quote(raw_file_link, safe='')}"
if relay_base
else raw_file_link
)
return relay_link
# Handle profile links
profile_link = "NA | NA"
if self.profile_link_extend or self.profile_link_decode:
# Create a combined link or use the first available one
trace_files = [self.profile_link_extend, self.profile_link_decode]
if any(trace_file is None for trace_file in trace_files):
logger.error("Some trace files are None", f"{trace_files=}")
trace_files_relay_links = [
(
f"[trace]({get_perfetto_relay_link_from_trace_file(trace_file)})"
if trace_file
else "N/A"
)
for trace_file in trace_files
]
profile_link = " | ".join(trace_files_relay_links)
# Build the row
return f"| {self.batch_size} | {self.input_len} | {self.latency:.2f} | {self.input_throughput:.2f} | {self.output_throughput:.2f} | {accept_length} | {itl:.2f} | {input_cost:.2f} | {output_cost:.2f} | {profile_link} |\n"
return f"| {self.batch_size} | {self.input_len} | {self.latency:.2f} | {self.input_throughput:.2f} | {self.output_throughput:.2f} | {accept_length} | {itl:.2f} | {input_cost:.2f} | {output_cost:.2f} |\n"
def generate_markdown_report(
trace_dir, results: List[BenchmarkResult], variant: Optional[str] = None
results: List[BenchmarkResult], variant: Optional[str] = None
) -> str:
"""Generate a markdown report from a list of BenchmarkResult object from a single run."""
# Build model header with run_name if it's not "default"
@@ -107,17 +63,12 @@ def generate_markdown_report(
summary = f"### {model_header}\n"
summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | acc length | ITL (ms) | input cost ($/1M) | output cost ($/1M) | profile (extend) | profile (decode)|\n"
summary += "| ---------- | --------- | ----------- | ------------------------- | ------------------------- | ---------- | -------- | ----------------- | ------------------ | ---------------- | --------------- |\n"
summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | acc length | ITL (ms) | input cost ($/1M) | output cost ($/1M) |\n"
summary += "| ---------- | --------- | ----------- | ------------------------- | ------------------------- | ---------- | -------- | ----------------- | ------------------ |\n"
# all results should share the same isl & osl
for result in results:
base_url = os.getenv("TRACE_BASE_URL", "").rstrip("/")
relay_base = os.getenv(
"PERFETTO_RELAY_URL",
"",
).rstrip("/")
summary += result.to_markdown_row(trace_dir, base_url, relay_base)
summary += result.to_markdown_row()
return summary
python/sglang/test/nightly_utils.py
+17 -44
View File
@@ -1,4 +1,4 @@
"""Utilities for running nightly performance benchmarks with profiling."""
"""Utilities for running nightly performance benchmarks."""
import json
import os
@@ -19,16 +19,16 @@ from sglang.test.test_utils import (
class NightlyBenchmarkRunner:
"""Helper class for running nightly performance benchmarks with profiling.
"""Helper class for running nightly performance benchmarks.
This class encapsulates common patterns used across nightly performance tests,
including profile directory management, benchmark command construction,
including result directory management, benchmark command construction,
result parsing, and report generation.
"""
def __init__(
self,
profile_dir: str,
result_dir: str,
test_name: str,
base_url: str,
gpu_config: str = None,
@@ -36,12 +36,12 @@ class NightlyBenchmarkRunner:
"""Initialize the benchmark runner.
Args:
profile_dir: Directory to store performance profiles
result_dir: Directory to store benchmark results
test_name: Name of the test (used for reporting)
base_url: Base URL for the server
gpu_config: Optional GPU configuration string (e.g., "2-gpu-h100", "8-gpu-b200")
"""
self.profile_dir = profile_dir
self.result_dir = result_dir
self.test_name = test_name
self.base_url = base_url
self.gpu_config = gpu_config or os.environ.get("GPU_CONFIG", "")
@@ -51,38 +51,32 @@ class NightlyBenchmarkRunner:
if self.gpu_config:
header += f" ({self.gpu_config})"
header += "\n"
self.full_report = header + BenchmarkResult.help_str()
self.full_report = header
def setup_profile_directory(self) -> None:
"""Create the profile directory if it doesn't exist."""
os.makedirs(self.profile_dir, exist_ok=True)
def setup_result_directory(self) -> None:
"""Create the result directory if it doesn't exist."""
os.makedirs(self.result_dir, exist_ok=True)
def generate_profile_filename(
self, model_path: str, variant: str = ""
) -> Tuple[str, str]:
"""Generate unique profile filename and path for the model.
def generate_result_filename(self, model_path: str, variant: str = "") -> str:
"""Generate a unique result filename for the model.
Args:
model_path: Path to the model (e.g., "deepseek-ai/DeepSeek-V3.1")
variant: Optional variant suffix (e.g., "basic", "mtp", "dsa")
Returns:
Tuple of (profile_path_prefix, json_output_file)
Path to the JSON result file
"""
timestamp = int(time.time())
model_safe_name = model_path.replace("/", "_")
# Build filename with optional variant
if variant:
profile_filename = f"{model_safe_name}_{variant}_{timestamp}"
json_filename = f"results_{model_safe_name}_{variant}_{timestamp}.json"
else:
profile_filename = f"{model_safe_name}_{timestamp}"
json_filename = f"results_{model_safe_name}_{timestamp}.json"
profile_path_prefix = os.path.join(self.profile_dir, profile_filename)
return profile_path_prefix, json_filename
return os.path.join(self.result_dir, json_filename)
def build_benchmark_command(
self,
@@ -90,11 +84,9 @@ class NightlyBenchmarkRunner:
batch_sizes: List[int],
input_lens: Tuple[int, ...],
output_lens: Tuple[int, ...],
profile_path_prefix: str,
json_output_file: str,
extra_args: Optional[List[str]] = None,
server_args: Optional[List[str]] = None,
enable_profile: bool = True,
) -> List[str]:
"""Build the benchmark command with all required arguments.
@@ -103,11 +95,9 @@ class NightlyBenchmarkRunner:
batch_sizes: List of batch sizes to test
input_lens: Tuple of input lengths to test
output_lens: Tuple of output lengths to test
profile_path_prefix: Prefix for profile output files
json_output_file: Path to JSON output file
extra_args: Optional extra arguments to append to command
server_args: Optional server launch arguments to record in metrics
enable_profile: Whether to enable profiling (default True for NVIDIA)
Returns:
List of command arguments ready for subprocess.run()
@@ -132,17 +122,6 @@ class NightlyBenchmarkRunner:
"--trust-remote-code",
]
# Add profiling flags only if enabled (disabled for AMD tests)
if enable_profile and profile_path_prefix:
command.extend(
[
"--profile",
"--profile-by-stage",
"--profile-output-dir",
profile_path_prefix,
]
)
if extra_args:
command.extend(extra_args)
@@ -227,7 +206,6 @@ class NightlyBenchmarkRunner:
other_args: Optional[List[str]] = None,
variant: str = "",
extra_bench_args: Optional[List[str]] = None,
enable_profile: bool = True,
timeout: Optional[int] = None,
env: Optional[dict] = None,
) -> Tuple[List[BenchmarkResult], bool, Optional[float]]:
@@ -235,7 +213,7 @@ class NightlyBenchmarkRunner:
This method handles:
- Server launch and cleanup
- Profile filename generation
- Result filename generation
- Benchmark command construction and execution
- Result loading and parsing
- Fetching speculative decoding accept length (for MTP/EAGLE)
@@ -248,7 +226,6 @@ class NightlyBenchmarkRunner:
other_args: Arguments to pass to server launch
variant: Optional variant suffix (e.g., "basic", "mtp")
extra_bench_args: Extra arguments for the benchmark command
enable_profile: Whether to enable profiling (default True for NVIDIA)
timeout: Optional timeout for server launch (defaults to DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH)
env: Environment dict for subprocess
@@ -275,9 +252,7 @@ class NightlyBenchmarkRunner:
)
# Generate filenames
profile_path_prefix, json_output_file = self.generate_profile_filename(
model_path, variant
)
json_output_file = self.generate_result_filename(model_path, variant)
# Build and run benchmark command
# Prepare extra args with run_name if variant is specified
@@ -290,11 +265,9 @@ class NightlyBenchmarkRunner:
batch_sizes,
input_lens,
output_lens,
profile_path_prefix,
json_output_file,
extra_args=bench_args,
server_args=other_args,
enable_profile=enable_profile,
)
result, cmd_success = self.run_benchmark_command(command, model_description)
@@ -346,7 +319,7 @@ class NightlyBenchmarkRunner:
results: List of BenchmarkResult objects to add to report
"""
if results:
report_part = generate_markdown_report(self.profile_dir, results, variant)
report_part = generate_markdown_report(results, variant)
self.full_report += report_part + "\n"
def write_final_report(self) -> None:
python/sglang/test/performance_test_runner.py
+7 -8
View File
@@ -13,7 +13,7 @@ class PerformanceTestParams:
batch_sizes: List[int] = field(default_factory=lambda: [1, 8, 16])
input_lens: Tuple[int, ...] = (8192,)
output_lens: Tuple[int, ...] = (512,)
profile_dir: Optional[str] = None # None = auto-generate based on is_vlm
result_dir: Optional[str] = None # None = auto-generate based on is_vlm
dataset_name: str = "mmmu" # For VLM perf test
# MTP/EAGLE speculative decoding: minimum accept length threshold (None = no validation)
spec_accept_length_threshold: Optional[float] = None
@@ -86,9 +86,8 @@ def run_performance_test(
perf_runner.add_report(results, variant=model.variant)
print(f"✓ Performance test succeeded for {model.model_path}")
# The cumulative /server_info accept length is reset by the cache
# flush before the profiling phase, so it can be missing here. Fall
# back to the per-run accept lengths captured during benchmarking.
# Fall back to the per-run accept lengths captured during benchmarking
# when the cumulative /server_info metric is unavailable.
if avg_spec_accept_length is None:
run_accept_lengths = [
r.acc_length
@@ -155,7 +154,7 @@ def run_performance_test(
def run_performance_for_models(
models: List[ModelLaunchSettings],
profile_dir: str,
result_dir: str,
test_name: str,
base_url: Optional[str] = None,
batch_sizes: List[int] = None,
@@ -168,7 +167,7 @@ def run_performance_for_models(
Args:
models: List of ModelLaunchSettings to test
profile_dir: Directory for performance profiles
result_dir: Directory for performance results
test_name: Name for the test (used in reports)
base_url: Server base URL (default: DEFAULT_URL_FOR_TEST)
batch_sizes: Batch sizes for perf test
@@ -188,11 +187,11 @@ def run_performance_for_models(
# Setup performance runner
perf_runner = NightlyBenchmarkRunner(
profile_dir=profile_dir,
result_dir=result_dir,
test_name=test_name,
base_url=base_url,
)
perf_runner.setup_profile_directory()
perf_runner.setup_result_directory()
all_results = []
all_passed = True
python/sglang/test/run_combined_tests.py
+4 -6
View File
@@ -76,18 +76,16 @@ def run_combined_tests(
# Set up performance parameters
if run_perf:
perf = performance_params
profile_dir = perf.profile_dir or (
"performance_profiles_vlms"
if is_vlm
else "performance_profiles_text_models"
result_dir = perf.result_dir or (
"performance_results_vlms" if is_vlm else "performance_results_text_models"
)
perf_runner = NightlyBenchmarkRunner(
profile_dir=profile_dir,
result_dir=result_dir,
test_name=test_name,
base_url=base_url,
)
perf_runner.setup_profile_directory()
perf_runner.setup_result_directory()
else:
perf_runner = None