minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Add P90/99 e2e latency in bench_serving script (#16245)

Browse Source
This commit is contained in:
Kangyan-Zhou
2025-12-31 15:40:33 -08:00
committed by GitHub
parent 851845579c
commit 12b89e51d8
1 changed files with 9 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/bench_serving.py
+9
View File
@@ -932,6 +932,7 @@ class BenchmarkMetrics:
mean_e2e_latency_ms: float mean_e2e_latency_ms: float
median_e2e_latency_ms: float median_e2e_latency_ms: float
std_e2e_latency_ms: float std_e2e_latency_ms: float
p90_e2e_latency_ms: float
p99_e2e_latency_ms: float p99_e2e_latency_ms: float
concurrency: float concurrency: float
max_output_tokens_per_s: float = 0.0 max_output_tokens_per_s: float = 0.0
@@ -2081,6 +2082,7 @@ def calculate_metrics(
mean_e2e_latency_ms=np.mean(e2e_latencies) * 1000, mean_e2e_latency_ms=np.mean(e2e_latencies) * 1000,
median_e2e_latency_ms=np.median(e2e_latencies) * 1000, median_e2e_latency_ms=np.median(e2e_latencies) * 1000,
std_e2e_latency_ms=np.std(e2e_latencies) * 1000, std_e2e_latency_ms=np.std(e2e_latencies) * 1000,
p90_e2e_latency_ms=np.percentile(e2e_latencies, 90) * 1000,
p99_e2e_latency_ms=np.percentile(e2e_latencies, 99) * 1000, p99_e2e_latency_ms=np.percentile(e2e_latencies, 99) * 1000,
concurrency=np.sum(e2e_latencies) / dur_s, concurrency=np.sum(e2e_latencies) / dur_s,
max_output_tokens_per_s=max_output_tokens_per_s, max_output_tokens_per_s=max_output_tokens_per_s,
@@ -2409,6 +2411,12 @@ async def benchmark(
"Median E2E Latency (ms):", metrics.median_e2e_latency_ms "Median E2E Latency (ms):", metrics.median_e2e_latency_ms
) )
) )
print(
"{:<40} {:<10.2f}".format("P90 E2E Latency (ms):", metrics.p90_e2e_latency_ms)
)
print(
"{:<40} {:<10.2f}".format("P99 E2E Latency (ms):", metrics.p99_e2e_latency_ms)
)
print("{s:{c}^{n}}".format(s="Time to First Token", n=50, c="-")) print("{s:{c}^{n}}".format(s="Time to First Token", n=50, c="-"))
print("{:<40} {:<10.2f}".format("Mean TTFT (ms):", metrics.mean_ttft_ms)) print("{:<40} {:<10.2f}".format("Mean TTFT (ms):", metrics.mean_ttft_ms))
print("{:<40} {:<10.2f}".format("Median TTFT (ms):", metrics.median_ttft_ms)) print("{:<40} {:<10.2f}".format("Median TTFT (ms):", metrics.median_ttft_ms))
@@ -2463,6 +2471,7 @@ async def benchmark(
"mean_e2e_latency_ms": metrics.mean_e2e_latency_ms, "mean_e2e_latency_ms": metrics.mean_e2e_latency_ms,
"median_e2e_latency_ms": metrics.median_e2e_latency_ms, "median_e2e_latency_ms": metrics.median_e2e_latency_ms,
"std_e2e_latency_ms": metrics.std_e2e_latency_ms, "std_e2e_latency_ms": metrics.std_e2e_latency_ms,
"p90_e2e_latency_ms": metrics.p90_e2e_latency_ms,
"p99_e2e_latency_ms": metrics.p99_e2e_latency_ms, "p99_e2e_latency_ms": metrics.p99_e2e_latency_ms,
"mean_ttft_ms": metrics.mean_ttft_ms, "mean_ttft_ms": metrics.mean_ttft_ms,
"median_ttft_ms": metrics.median_ttft_ms, "median_ttft_ms": metrics.median_ttft_ms,