[Feat] bench: report goodput and SLO attainment in bench_serving (#43967)

This commit is contained in:
Bowen Wang
2026-10-18 13:31:48 +08:00
committed by GitHub
parent 62d4261cf4
commit 3671503e44
4 changed files with 190 additions and 0 deletions
+68
View File
@@ -0,0 +1,68 @@
"""Goodput for the serving benchmark: the requests that met every given latency SLO."""
from __future__ import annotations
from typing import TYPE_CHECKING
import msgspec
if TYPE_CHECKING:
from sglang.benchmark.serving import RequestFuncOutput
GOODPUT_SLO_METRICS = ("ttft", "tpot", "e2el")
class GoodputMetrics(msgspec.Struct, frozen=True):
"""Field names are the keys written to the benchmark result file."""
goodput_slos_ms: dict[str, float]
request_goodput: float # good requests per second
slo_attainment: float # good requests / requests sent
slo_attainment_by_metric: dict[str, float]
def calculate_goodput(
outputs: list[RequestFuncOutput], goodput_slos: dict[str, float], dur_s: float
) -> GoodputMetrics:
num_good = 0
num_met = dict.fromkeys(goodput_slos, 0)
for output in outputs:
if not output.success:
continue
# A single-token response has no inter-token interval; its TPOT counts as 0.
tpot = 0.0
if output.output_len > 1:
tpot = (output.latency - output.ttft) / (output.output_len - 1)
observed_ms = {
"ttft": output.ttft * 1000,
"tpot": tpot * 1000,
"e2el": output.latency * 1000,
}
met = [name for name, slo in goodput_slos.items() if observed_ms[name] <= slo]
for name in met:
num_met[name] += 1
if len(met) == len(goodput_slos):
num_good += 1
return GoodputMetrics(
goodput_slos_ms=goodput_slos,
request_goodput=num_good / dur_s,
slo_attainment=num_good / len(outputs),
slo_attainment_by_metric={
name: count / len(outputs) for name, count in num_met.items()
},
)
def print_goodput(goodput: GoodputMetrics) -> None:
slos = " ".join(f"{name}:{slo:g}" for name, slo in goodput.goodput_slos_ms.items())
print("{s:{c}^{n}}".format(s="Goodput", n=50, c="-"))
print("{:<40} {:<10}".format("SLOs (ms):", slos))
print(
"{:<40} {:<10.2f}".format("Request goodput (req/s):", goodput.request_goodput)
)
print(
"{:<40} {:<10.2f}".format("SLO attainment (%):", goodput.slo_attainment * 100)
)
for name, attainment in goodput.slo_attainment_by_metric.items():
label = f"{name.upper()} SLO attainment (%):"
print("{:<40} {:<10.2f}".format(label, attainment * 100))
+40
View File
@@ -34,6 +34,7 @@ from pathlib import Path
from typing import Any, AsyncGenerator, Callable, Dict, List, Optional, Tuple, Union
import aiohttp
import msgspec
import numpy as np
import orjson
import requests
@@ -42,6 +43,11 @@ from transformers import AutoTokenizer, PreTrainedTokenizerBase
from sglang.benchmark.datasets import DatasetRow, get_dataset
from sglang.benchmark.datasets.mooncake import get_mooncake_request_over_time
from sglang.benchmark.goodput import (
GOODPUT_SLO_METRICS,
calculate_goodput,
print_goodput,
)
from sglang.benchmark.utils import (
get_tokenizer,
parse_custom_headers,
@@ -1666,6 +1672,11 @@ async def benchmark(
accept_length=accept_length,
plot_throughput=args.plot_throughput,
)
goodput = None
if args.goodput:
goodput = calculate_goodput(
outputs=outputs, goodput_slos=dict(args.goodput), dur_s=benchmark_duration
)
print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="="))
print("{:<40} {:<10}".format("Backend:", backend))
@@ -1735,6 +1746,8 @@ async def benchmark(
print("{:<40} {:<10.2f}".format("Concurrency:", metrics.concurrency))
if accept_length:
print("{:<40} {:<10.2f}".format("Accept length:", accept_length))
if goodput:
print_goodput(goodput)
print("{s:{c}^{n}}".format(s="End-to-End Latency", n=50, c="-"))
print(
"{:<40} {:<10.2f}".format("Mean E2E Latency (ms):", metrics.mean_e2e_latency_ms)
@@ -1894,6 +1907,9 @@ async def benchmark(
"max_concurrent_requests": metrics.max_concurrent_requests,
}
if goodput:
result.update(msgspec.structs.asdict(goodput))
if args.cache_report:
result["cache_report"] = {
"total_prompt_tokens": total_prompt_tokens,
@@ -2026,6 +2042,9 @@ def run_benchmark(args_: argparse.Namespace):
if not hasattr(args, "cache_report"):
args.cache_report = False
if not hasattr(args, "goodput"):
args.goodput = None
if getattr(args, "print_requests", False):
assert args.backend == "sglang-oai-chat" # only support this now
@@ -2226,6 +2245,17 @@ def _finite_positive_float(value) -> float:
return parsed
def _goodput_slo(pair: str) -> Tuple[str, float]:
"""argparse type for one --goodput KEY:VALUE entry; VALUE is in milliseconds."""
name, _, value = pair.partition(":")
if name not in GOODPUT_SLO_METRICS:
raise argparse.ArgumentTypeError(
f"expected KEY:VALUE with KEY in {', '.join(GOODPUT_SLO_METRICS)}, "
f"got {pair!r}"
)
return name, _finite_positive_float(value)
def _validate_parsed_gsp_args(
parser: argparse.ArgumentParser, args: argparse.Namespace
) -> None:
@@ -2454,6 +2484,16 @@ def cli_main():
parser.add_argument(
"--output-details", action="store_true", help="Output details of benchmarking."
)
parser.add_argument(
"--goodput",
nargs="+",
type=_goodput_slo,
default=None,
metavar="KEY:VALUE",
help="SLOs for goodput as KEY:VALUE pairs in milliseconds, with KEY in "
f"{', '.join(GOODPUT_SLO_METRICS)}, e.g. --goodput ttft:500 tpot:50. A request "
"counts toward goodput only if it succeeds and meets every given SLO.",
)
parser.add_argument(
"--print-requests",
action="store_true",