[Feat] bench: report goodput and SLO attainment in bench_serving (#43967)
This commit is contained in:
@@ -0,0 +1,68 @@
|
||||
"""Goodput for the serving benchmark: the requests that met every given latency SLO."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import msgspec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.benchmark.serving import RequestFuncOutput
|
||||
|
||||
GOODPUT_SLO_METRICS = ("ttft", "tpot", "e2el")
|
||||
|
||||
|
||||
class GoodputMetrics(msgspec.Struct, frozen=True):
|
||||
"""Field names are the keys written to the benchmark result file."""
|
||||
|
||||
goodput_slos_ms: dict[str, float]
|
||||
request_goodput: float # good requests per second
|
||||
slo_attainment: float # good requests / requests sent
|
||||
slo_attainment_by_metric: dict[str, float]
|
||||
|
||||
|
||||
def calculate_goodput(
|
||||
outputs: list[RequestFuncOutput], goodput_slos: dict[str, float], dur_s: float
|
||||
) -> GoodputMetrics:
|
||||
num_good = 0
|
||||
num_met = dict.fromkeys(goodput_slos, 0)
|
||||
for output in outputs:
|
||||
if not output.success:
|
||||
continue
|
||||
# A single-token response has no inter-token interval; its TPOT counts as 0.
|
||||
tpot = 0.0
|
||||
if output.output_len > 1:
|
||||
tpot = (output.latency - output.ttft) / (output.output_len - 1)
|
||||
observed_ms = {
|
||||
"ttft": output.ttft * 1000,
|
||||
"tpot": tpot * 1000,
|
||||
"e2el": output.latency * 1000,
|
||||
}
|
||||
met = [name for name, slo in goodput_slos.items() if observed_ms[name] <= slo]
|
||||
for name in met:
|
||||
num_met[name] += 1
|
||||
if len(met) == len(goodput_slos):
|
||||
num_good += 1
|
||||
return GoodputMetrics(
|
||||
goodput_slos_ms=goodput_slos,
|
||||
request_goodput=num_good / dur_s,
|
||||
slo_attainment=num_good / len(outputs),
|
||||
slo_attainment_by_metric={
|
||||
name: count / len(outputs) for name, count in num_met.items()
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def print_goodput(goodput: GoodputMetrics) -> None:
|
||||
slos = " ".join(f"{name}:{slo:g}" for name, slo in goodput.goodput_slos_ms.items())
|
||||
print("{s:{c}^{n}}".format(s="Goodput", n=50, c="-"))
|
||||
print("{:<40} {:<10}".format("SLOs (ms):", slos))
|
||||
print(
|
||||
"{:<40} {:<10.2f}".format("Request goodput (req/s):", goodput.request_goodput)
|
||||
)
|
||||
print(
|
||||
"{:<40} {:<10.2f}".format("SLO attainment (%):", goodput.slo_attainment * 100)
|
||||
)
|
||||
for name, attainment in goodput.slo_attainment_by_metric.items():
|
||||
label = f"{name.upper()} SLO attainment (%):"
|
||||
print("{:<40} {:<10.2f}".format(label, attainment * 100))
|
||||
@@ -34,6 +34,7 @@ from pathlib import Path
|
||||
from typing import Any, AsyncGenerator, Callable, Dict, List, Optional, Tuple, Union
|
||||
|
||||
import aiohttp
|
||||
import msgspec
|
||||
import numpy as np
|
||||
import orjson
|
||||
import requests
|
||||
@@ -42,6 +43,11 @@ from transformers import AutoTokenizer, PreTrainedTokenizerBase
|
||||
|
||||
from sglang.benchmark.datasets import DatasetRow, get_dataset
|
||||
from sglang.benchmark.datasets.mooncake import get_mooncake_request_over_time
|
||||
from sglang.benchmark.goodput import (
|
||||
GOODPUT_SLO_METRICS,
|
||||
calculate_goodput,
|
||||
print_goodput,
|
||||
)
|
||||
from sglang.benchmark.utils import (
|
||||
get_tokenizer,
|
||||
parse_custom_headers,
|
||||
@@ -1666,6 +1672,11 @@ async def benchmark(
|
||||
accept_length=accept_length,
|
||||
plot_throughput=args.plot_throughput,
|
||||
)
|
||||
goodput = None
|
||||
if args.goodput:
|
||||
goodput = calculate_goodput(
|
||||
outputs=outputs, goodput_slos=dict(args.goodput), dur_s=benchmark_duration
|
||||
)
|
||||
|
||||
print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="="))
|
||||
print("{:<40} {:<10}".format("Backend:", backend))
|
||||
@@ -1735,6 +1746,8 @@ async def benchmark(
|
||||
print("{:<40} {:<10.2f}".format("Concurrency:", metrics.concurrency))
|
||||
if accept_length:
|
||||
print("{:<40} {:<10.2f}".format("Accept length:", accept_length))
|
||||
if goodput:
|
||||
print_goodput(goodput)
|
||||
print("{s:{c}^{n}}".format(s="End-to-End Latency", n=50, c="-"))
|
||||
print(
|
||||
"{:<40} {:<10.2f}".format("Mean E2E Latency (ms):", metrics.mean_e2e_latency_ms)
|
||||
@@ -1894,6 +1907,9 @@ async def benchmark(
|
||||
"max_concurrent_requests": metrics.max_concurrent_requests,
|
||||
}
|
||||
|
||||
if goodput:
|
||||
result.update(msgspec.structs.asdict(goodput))
|
||||
|
||||
if args.cache_report:
|
||||
result["cache_report"] = {
|
||||
"total_prompt_tokens": total_prompt_tokens,
|
||||
@@ -2026,6 +2042,9 @@ def run_benchmark(args_: argparse.Namespace):
|
||||
if not hasattr(args, "cache_report"):
|
||||
args.cache_report = False
|
||||
|
||||
if not hasattr(args, "goodput"):
|
||||
args.goodput = None
|
||||
|
||||
if getattr(args, "print_requests", False):
|
||||
assert args.backend == "sglang-oai-chat" # only support this now
|
||||
|
||||
@@ -2226,6 +2245,17 @@ def _finite_positive_float(value) -> float:
|
||||
return parsed
|
||||
|
||||
|
||||
def _goodput_slo(pair: str) -> Tuple[str, float]:
|
||||
"""argparse type for one --goodput KEY:VALUE entry; VALUE is in milliseconds."""
|
||||
name, _, value = pair.partition(":")
|
||||
if name not in GOODPUT_SLO_METRICS:
|
||||
raise argparse.ArgumentTypeError(
|
||||
f"expected KEY:VALUE with KEY in {', '.join(GOODPUT_SLO_METRICS)}, "
|
||||
f"got {pair!r}"
|
||||
)
|
||||
return name, _finite_positive_float(value)
|
||||
|
||||
|
||||
def _validate_parsed_gsp_args(
|
||||
parser: argparse.ArgumentParser, args: argparse.Namespace
|
||||
) -> None:
|
||||
@@ -2454,6 +2484,16 @@ def cli_main():
|
||||
parser.add_argument(
|
||||
"--output-details", action="store_true", help="Output details of benchmarking."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--goodput",
|
||||
nargs="+",
|
||||
type=_goodput_slo,
|
||||
default=None,
|
||||
metavar="KEY:VALUE",
|
||||
help="SLOs for goodput as KEY:VALUE pairs in milliseconds, with KEY in "
|
||||
f"{', '.join(GOODPUT_SLO_METRICS)}, e.g. --goodput ttft:500 tpot:50. A request "
|
||||
"counts toward goodput only if it succeeds and meets every given SLO.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--print-requests",
|
||||
action="store_true",
|
||||
|
||||
Reference in New Issue
Block a user