[Feat] bench: report goodput and SLO attainment in bench_serving (#43967)

This commit is contained in:
Bowen Wang
2026-10-18 13:31:48 +08:00
committed by GitHub
parent 62d4261cf4
commit 3671503e44
4 changed files with 190 additions and 0 deletions
@@ -245,6 +245,25 @@ Printed after each run:
The script also retokenizes generated text with the configured tokenizer and reports "retokenized" counts.
### Goodput
Throughput counts every completed request, including the ones that came back too late to be useful. Pass service level objectives with `--goodput` to count only the requests that met them:
```bash Command
python3 -m sglang.bench_serving \
--backend sglang --host 127.0.0.1 --port 30000 \
--num-prompts 1000 --request-rate 16 \
--goodput ttft:500 tpot:50 e2el:20000
```
Each entry is `KEY:VALUE` in milliseconds, with `KEY` one of `ttft`, `tpot`, `e2el` (end-to-end latency). A request is good if it succeeded and met every listed SLO; a single-token response has a TPOT of 0. The run then also prints:
- Request goodput (req/s): good requests per second
- SLO attainment (%): good requests over all requests sent, so failed requests count as misses
- One attainment line per listed SLO, which shows which SLO is the binding one
Sweeping `--request-rate` and reading goodput gives the highest rate the deployment sustains within its SLOs: throughput keeps rising or flattens under overload, goodput peaks and then falls.
### JSONL output format
When `--output-file` is set, one JSON object is appended per run. Base fields:
@@ -253,6 +272,7 @@ When `--output-file` is set, one JSON object is appended per run. Base fields:
- Duration and totals: completed, total_input_tokens, total_output_tokens, retokenized totals
- Throughputs and latency statistics as printed in the console
- `accept_length` when available (sglang)
- `goodput_slos_ms`, `request_goodput`, `slo_attainment`, `slo_attainment_by_metric` when `--goodput` is given
With `--output-details`, an extended object also includes arrays:
+68
View File
@@ -0,0 +1,68 @@
"""Goodput for the serving benchmark: the requests that met every given latency SLO."""
from __future__ import annotations
from typing import TYPE_CHECKING
import msgspec
if TYPE_CHECKING:
from sglang.benchmark.serving import RequestFuncOutput
GOODPUT_SLO_METRICS = ("ttft", "tpot", "e2el")
class GoodputMetrics(msgspec.Struct, frozen=True):
"""Field names are the keys written to the benchmark result file."""
goodput_slos_ms: dict[str, float]
request_goodput: float # good requests per second
slo_attainment: float # good requests / requests sent
slo_attainment_by_metric: dict[str, float]
def calculate_goodput(
outputs: list[RequestFuncOutput], goodput_slos: dict[str, float], dur_s: float
) -> GoodputMetrics:
num_good = 0
num_met = dict.fromkeys(goodput_slos, 0)
for output in outputs:
if not output.success:
continue
# A single-token response has no inter-token interval; its TPOT counts as 0.
tpot = 0.0
if output.output_len > 1:
tpot = (output.latency - output.ttft) / (output.output_len - 1)
observed_ms = {
"ttft": output.ttft * 1000,
"tpot": tpot * 1000,
"e2el": output.latency * 1000,
}
met = [name for name, slo in goodput_slos.items() if observed_ms[name] <= slo]
for name in met:
num_met[name] += 1
if len(met) == len(goodput_slos):
num_good += 1
return GoodputMetrics(
goodput_slos_ms=goodput_slos,
request_goodput=num_good / dur_s,
slo_attainment=num_good / len(outputs),
slo_attainment_by_metric={
name: count / len(outputs) for name, count in num_met.items()
},
)
def print_goodput(goodput: GoodputMetrics) -> None:
slos = " ".join(f"{name}:{slo:g}" for name, slo in goodput.goodput_slos_ms.items())
print("{s:{c}^{n}}".format(s="Goodput", n=50, c="-"))
print("{:<40} {:<10}".format("SLOs (ms):", slos))
print(
"{:<40} {:<10.2f}".format("Request goodput (req/s):", goodput.request_goodput)
)
print(
"{:<40} {:<10.2f}".format("SLO attainment (%):", goodput.slo_attainment * 100)
)
for name, attainment in goodput.slo_attainment_by_metric.items():
label = f"{name.upper()} SLO attainment (%):"
print("{:<40} {:<10.2f}".format(label, attainment * 100))
+40
View File
@@ -34,6 +34,7 @@ from pathlib import Path
from typing import Any, AsyncGenerator, Callable, Dict, List, Optional, Tuple, Union
import aiohttp
import msgspec
import numpy as np
import orjson
import requests
@@ -42,6 +43,11 @@ from transformers import AutoTokenizer, PreTrainedTokenizerBase
from sglang.benchmark.datasets import DatasetRow, get_dataset
from sglang.benchmark.datasets.mooncake import get_mooncake_request_over_time
from sglang.benchmark.goodput import (
GOODPUT_SLO_METRICS,
calculate_goodput,
print_goodput,
)
from sglang.benchmark.utils import (
get_tokenizer,
parse_custom_headers,
@@ -1666,6 +1672,11 @@ async def benchmark(
accept_length=accept_length,
plot_throughput=args.plot_throughput,
)
goodput = None
if args.goodput:
goodput = calculate_goodput(
outputs=outputs, goodput_slos=dict(args.goodput), dur_s=benchmark_duration
)
print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="="))
print("{:<40} {:<10}".format("Backend:", backend))
@@ -1735,6 +1746,8 @@ async def benchmark(
print("{:<40} {:<10.2f}".format("Concurrency:", metrics.concurrency))
if accept_length:
print("{:<40} {:<10.2f}".format("Accept length:", accept_length))
if goodput:
print_goodput(goodput)
print("{s:{c}^{n}}".format(s="End-to-End Latency", n=50, c="-"))
print(
"{:<40} {:<10.2f}".format("Mean E2E Latency (ms):", metrics.mean_e2e_latency_ms)
@@ -1894,6 +1907,9 @@ async def benchmark(
"max_concurrent_requests": metrics.max_concurrent_requests,
}
if goodput:
result.update(msgspec.structs.asdict(goodput))
if args.cache_report:
result["cache_report"] = {
"total_prompt_tokens": total_prompt_tokens,
@@ -2026,6 +2042,9 @@ def run_benchmark(args_: argparse.Namespace):
if not hasattr(args, "cache_report"):
args.cache_report = False
if not hasattr(args, "goodput"):
args.goodput = None
if getattr(args, "print_requests", False):
assert args.backend == "sglang-oai-chat" # only support this now
@@ -2226,6 +2245,17 @@ def _finite_positive_float(value) -> float:
return parsed
def _goodput_slo(pair: str) -> Tuple[str, float]:
"""argparse type for one --goodput KEY:VALUE entry; VALUE is in milliseconds."""
name, _, value = pair.partition(":")
if name not in GOODPUT_SLO_METRICS:
raise argparse.ArgumentTypeError(
f"expected KEY:VALUE with KEY in {', '.join(GOODPUT_SLO_METRICS)}, "
f"got {pair!r}"
)
return name, _finite_positive_float(value)
def _validate_parsed_gsp_args(
parser: argparse.ArgumentParser, args: argparse.Namespace
) -> None:
@@ -2454,6 +2484,16 @@ def cli_main():
parser.add_argument(
"--output-details", action="store_true", help="Output details of benchmarking."
)
parser.add_argument(
"--goodput",
nargs="+",
type=_goodput_slo,
default=None,
metavar="KEY:VALUE",
help="SLOs for goodput as KEY:VALUE pairs in milliseconds, with KEY in "
f"{', '.join(GOODPUT_SLO_METRICS)}, e.g. --goodput ttft:500 tpot:50. A request "
"counts toward goodput only if it succeeds and meets every given SLO.",
)
parser.add_argument(
"--print-requests",
action="store_true",
@@ -0,0 +1,62 @@
"""Unit tests for the goodput bench_serving reports under --goodput."""
import argparse
import unittest
from sglang.benchmark.goodput import calculate_goodput
from sglang.benchmark.serving import RequestFuncOutput, _goodput_slo
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
DURATION_S = 10.0
def _output(ttft, latency, output_len, success=True):
return RequestFuncOutput(
success=success, ttft=ttft, latency=latency, output_len=output_len
)
# ttft / tpot / e2el in ms: 100 / 100 / 900, 500 / 100 / 900, 200 / 200 / 2200,
# 200 / 0 / 200 (a single token), then one failed request.
OUTPUTS = [
_output(ttft=0.1, latency=0.9, output_len=9),
_output(ttft=0.5, latency=0.9, output_len=5),
_output(ttft=0.2, latency=2.2, output_len=11),
_output(ttft=0.2, latency=0.2, output_len=1),
_output(ttft=0.0, latency=0.0, output_len=0, success=False),
]
class TestBenchServingGoodput(CustomTestCase):
def test_request_is_good_only_if_it_meets_every_listed_slo(self):
all_three = {"ttft": 3 / 5, "tpot": 3 / 5, "e2el": 3 / 5}
# (--goodput entries, good requests, attainment of each listed SLO alone)
cases = [
(["ttft:300", "tpot:150", "e2el:1000"], 2, all_three),
(["ttft:300"], 3, {"ttft": 3 / 5}),
# Dividing by output_len instead would give 88.9 and 80 ms, and pass.
(["tpot:95"], 1, {"tpot": 1 / 5}),
]
for entries, num_good, by_metric in cases:
with self.subTest(entries=entries):
goodput = calculate_goodput(
outputs=OUTPUTS,
goodput_slos=dict(map(_goodput_slo, entries)),
dur_s=DURATION_S,
)
self.assertEqual(goodput.request_goodput, num_good / DURATION_S)
self.assertEqual(goodput.slo_attainment, num_good / len(OUTPUTS))
self.assertEqual(goodput.slo_attainment_by_metric, by_metric)
def test_invalid_slo_entries_are_rejected(self):
for entry in ["itl:10", "ttft:0"]:
with self.subTest(entry=entry):
with self.assertRaises(argparse.ArgumentTypeError):
_goodput_slo(entry)
if __name__ == "__main__":
unittest.main()