From e98c6d7b251ac79627efe30c41ff56c59106b706 Mon Sep 17 00:00:00 2001 From: Rain Jiang <96632942+rainj-me@users.noreply.github.com> Date: Tue, 13 Oct 2026 11:07:42 -0700 Subject: [PATCH] Rainj me/rust bench (#43950) --- python/sglang/benchmark/rust_client.py | 224 ++++++ python/sglang/benchmark/serving.py | 39 ++ rust/Cargo.lock | 22 + rust/Cargo.toml | 1 + rust/sglang-bench/Cargo.toml | 57 ++ rust/sglang-bench/README.md | 176 +++++ rust/sglang-bench/src/args.rs | 601 ++++++++++++++++ rust/sglang-bench/src/dataset.rs | 368 ++++++++++ rust/sglang-bench/src/hf.rs | 230 ++++++ rust/sglang-bench/src/lib.rs | 88 +++ rust/sglang-bench/src/metrics.rs | 670 ++++++++++++++++++ rust/sglang-bench/src/request.rs | 925 +++++++++++++++++++++++++ rust/sglang-bench/src/run.rs | 729 +++++++++++++++++++ rust/sglang-bench/tests/grpc.rs | 184 +++++ 14 files changed, 4314 insertions(+) create mode 100644 python/sglang/benchmark/rust_client.py create mode 100644 rust/sglang-bench/Cargo.toml create mode 100644 rust/sglang-bench/README.md create mode 100644 rust/sglang-bench/src/args.rs create mode 100644 rust/sglang-bench/src/dataset.rs create mode 100644 rust/sglang-bench/src/hf.rs create mode 100644 rust/sglang-bench/src/lib.rs create mode 100644 rust/sglang-bench/src/metrics.rs create mode 100644 rust/sglang-bench/src/request.rs create mode 100644 rust/sglang-bench/src/run.rs create mode 100644 rust/sglang-bench/tests/grpc.rs diff --git a/python/sglang/benchmark/rust_client.py b/python/sglang/benchmark/rust_client.py new file mode 100644 index 0000000000..c737834f8c --- /dev/null +++ b/python/sglang/benchmark/rust_client.py @@ -0,0 +1,224 @@ +"""Run the benchmark through the Rust client (``sglang-bench``). + +The Python client in :mod:`sglang.benchmark.serving` drives every request and +parses every response stream on one asyncio thread, so past a few thousand +concurrent streams it, not the server, caps the measured throughput. The Rust +client parses each stream on a Tokio worker thread instead. It reports the same +metrics, prints the same table, and appends the same result keys, so a run from +either side stays comparable. + +Two ways in: + +* ``python -m sglang.benchmark.rust_client --help`` forwards the command line + to the Rust client, which owns the flag definitions. This path imports no + model tooling, so it starts in well under a second. +* ``python -m sglang.benchmark.serving --rust-client ...`` routes an + already-parsed argument namespace here, for callers on the existing entry + point. + +The Rust client covers fewer backends and datasets than the Python one; an +unsupported configuration raises here rather than quietly measuring something +else. See ``rust/sglang-bench/README.md``. +""" + +from __future__ import annotations + +import json +import math +import sys +from typing import Any, Dict, Iterable, Optional + +EXTENSION_MODULE = "sglang.srt.rust_extensions._bench" + +#: Backends the Rust client implements. The Python client's remaining +#: backends (``trt``, ``gserver``, ``truss``, and the embedding ones) have no +#: Rust counterpart. +SUPPORTED_BACKENDS = frozenset( + { + "sglang", + "sglang-native", + "sglang-oai", + "sglang-oai-chat", + "vllm", + "vllm-chat", + "lmdeploy", + "lmdeploy-chat", + } +) + +#: Datasets the Rust client implements. +SUPPORTED_DATASETS = frozenset({"random", "random-ids", "sharegpt"}) + +#: Namespace attributes that map onto a Rust flag of the same name. Anything +#: needing conversion is handled in :func:`build_config`. +_DIRECT_FIELDS = ( + "backend", + # Rust-only: ``http`` (default) or ``grpc``. `serving.py` has no such + # flag, so this arrives only from a programmatic caller. + "protocol", + "base_url", + "host", + "port", + "ready_check_timeout_sec", + "dataset_name", + "dataset_path", + "model", + "served_model_name", + "tokenizer", + "num_prompts", + "sharegpt_output_len", + "sharegpt_context_len", + "random_input_len", + "random_output_len", + "random_range_ratio", + "max_concurrency", + "warmup_requests", + "output_file", + "output_details", + "disable_tqdm", + "disable_stream", + "disable_ignore_eos", + "seed", + "temperature", + "top_p", + "return_logprob", + "top_logprobs_num", + "logprob_start_len", + "extra_request_body", + "tokenize_prompt", + "flush_cache", + "flush_cache_timeout", + "profile", + "tag", +) + +#: Options the Python client accepts that the Rust one does not implement. +#: Each maps to the value that means "not requested", so only a caller who +#: actually asked for one is turned away. +_UNSUPPORTED_OPTIONS = { + "lora_name": None, + "use_trace_timestamps": False, + "apply_chat_template": False, + "plot_throughput": False, + "cache_report": False, + "pd_separated": False, + "return_routed_experts": False, + "token_ids_logprob": None, + "print_requests": False, + "prompt_suffix": "", + "fake_prefill": False, + "profile_steps": None, + "profile_num_steps": None, + "profile_by_stage": False, +} + + +class UnsupportedByRustClient(ValueError): + """The requested configuration has no Rust implementation.""" + + +def check_supported(args: Any) -> None: + """Raise unless the Rust client implements everything ``args`` asks for.""" + backend = getattr(args, "backend", None) + if backend not in SUPPORTED_BACKENDS: + raise UnsupportedByRustClient( + f"--rust-client does not implement --backend {backend}; " + f"supported: {', '.join(sorted(SUPPORTED_BACKENDS))}" + ) + dataset = getattr(args, "dataset_name", None) + if dataset not in SUPPORTED_DATASETS: + raise UnsupportedByRustClient( + f"--rust-client does not implement --dataset-name {dataset}; " + f"supported: {', '.join(sorted(SUPPORTED_DATASETS))}" + ) + requested = [ + name + for name, unset in _UNSUPPORTED_OPTIONS.items() + if getattr(args, name, unset) != unset + ] + if requested: + raise UnsupportedByRustClient( + "--rust-client does not implement: " + f"{', '.join(sorted(requested))}. Drop --rust-client to run these " + "on the Python client." + ) + + +def build_config(args: Any) -> Dict[str, Any]: + """The Rust client's config object for an argument namespace. + + Only keys the caller actually set are sent; every omitted key takes the + Rust flag's own default, which is the same value the Python flag has. + """ + config: Dict[str, Any] = {} + for field in _DIRECT_FIELDS: + value = getattr(args, field, None) + if value is not None: + config[field] = value + + # The Rust flag takes a string so it can carry `inf`, which JSON cannot. + rate = getattr(args, "request_rate", float("inf")) + config["request_rate"] = "inf" if rate is None or math.isinf(rate) else str(rate) + + # The Python flag spells a header `Key=Value`; the Rust one `Key: Value`. + config["headers"] = list(_as_headers(getattr(args, "header", None))) + + worker_threads = getattr(args, "rust_worker_threads", None) + if worker_threads is not None: + config["worker_threads"] = worker_threads + return config + + +def _as_headers(header: Any) -> Iterable[str]: + """`--header Key=Value ...` in the Rust client's `Key: Value` spelling.""" + if not header: + return () + raw = (header,) if isinstance(header, str) else tuple(header) + rewritten = [] + for entry in raw: + name, separator, value = entry.partition("=") + if not separator or not name or not value: + raise UnsupportedByRustClient( + f"--header must be `Key=Value`, got {entry!r}" + ) + rewritten.append(f"{name}: {value}") + return tuple(rewritten) + + +def _extension(extension: Any = None) -> Any: + if extension is not None: + return extension + from sglang.srt.rust_extensions import load_rust_extension + + return load_rust_extension(EXTENSION_MODULE) + + +def run_rust_benchmark(args: Any, extension: Any = None) -> Dict[str, Any]: + """Run the benchmark described by ``args`` and return its result document. + + The Rust client prints the report and appends the result line itself, so + the returned mapping is for a caller that wants the numbers in process. + """ + check_supported(args) + config = build_config(args) + result = _extension(extension).run_config(json.dumps(config)) + return json.loads(result) + + +def main(argv: Optional[list] = None, extension: Any = None) -> int: + """Forward a command line to the Rust client, which owns the flags.""" + argv = list(sys.argv if argv is None else argv) + if not argv: + argv = ["sglang-bench"] + # The Rust side expects argv[0] to be the program name, so clap's usage + # and error messages name the command the user actually typed. + argv[0] = "python -m sglang.benchmark.rust_client" + try: + _extension(extension).run_argv(argv) + except SystemExit as exit_request: + return int(exit_request.code or 0) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python/sglang/benchmark/serving.py b/python/sglang/benchmark/serving.py index ecfec3609e..ecca2def42 100644 --- a/python/sglang/benchmark/serving.py +++ b/python/sglang/benchmark/serving.py @@ -1971,6 +1971,13 @@ def run_benchmark(args_: argparse.Namespace): global args args = args_ + # The Rust client owns the whole run, from dataset to result file, so hand + # it the arguments before this module sets up anything of its own. + if getattr(args, "rust_client", False): + from sglang.benchmark.rust_client import run_rust_benchmark + + return run_rust_benchmark(args) + # Set default value for max_concurrency if not present if not hasattr(args, "max_concurrency"): args.max_concurrency = None @@ -2798,7 +2805,39 @@ def cli_main(): default=None, help="Custom HTTP headers in Key=Value format. Example: --header MyHeader=MY_VALUE MyAnotherHeader=myanothervalue", ) + parser.add_argument( + "--rust-client", + action="store_true", + help="Run through the Rust client (sglang.srt.rust_extensions._bench), " + "which parses each response stream on its own thread instead of one " + "shared asyncio thread. Same metrics and result keys; fewer backends " + "and datasets (see rust/sglang-bench/README.md). For a command line " + "that skips this module's model-tooling imports entirely, run " + "`python -m sglang.benchmark.rust_client` instead.", + ) + parser.add_argument( + "--rust-worker-threads", + type=int, + default=None, + help="Response-stream parsing threads for --rust-client. " + "Defaults to the available parallelism.", + ) args = parser.parse_args() + if args.rust_worker_threads is not None and not args.rust_client: + parser.error("--rust-worker-threads requires --rust-client") + if args.rust_client: + # Checked here so an unsupported combination is a usage error rather + # than a traceback; `run_benchmark` still raises for a programmatic + # caller, which has no parser to report through. + from sglang.benchmark.rust_client import ( + UnsupportedByRustClient, + check_supported, + ) + + try: + check_supported(args) + except UnsupportedByRustClient as error: + parser.error(str(error)) _validate_parsed_gsp_args(parser, args) run_benchmark(args) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index aa2ee255a9..68b797a82d 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -3707,6 +3707,28 @@ dependencies = [ "tonic-prost", ] +[[package]] +name = "sglang-bench" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "clap", + "futures", + "hf-hub", + "indicatif", + "pyo3", + "rand 0.8.7", + "reqwest", + "serde", + "serde_json", + "sglang-api-types", + "tokenizers", + "tokio", + "tokio-stream", + "tonic", +] + [[package]] name = "sglang-grpc" version = "0.1.0" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index b1a92e4eb1..ed0ac007cd 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -3,6 +3,7 @@ resolver = "3" members = [ "sglang-api-codegen", "sglang-api-types", + "sglang-bench", "sglang-grpc", "sglang-mm", "sglang-processor", diff --git a/rust/sglang-bench/Cargo.toml b/rust/sglang-bench/Cargo.toml new file mode 100644 index 0000000000..4070e4f1b6 --- /dev/null +++ b/rust/sglang-bench/Cargo.toml @@ -0,0 +1,57 @@ +[package] +name = "sglang-bench" +description = "Serving benchmark client: the Rust port of python/sglang/benchmark/serving.py" +version.workspace = true +edition.workspace = true +license.workspace = true + +# Consumed by python/setup.py: registers this crate as a PyO3 extension module +# of the main sglang wheel at the given import path. debug = false keeps +# editable installs on release builds, since a debug client cannot keep up with +# the server it is measuring. +[package.metadata.sglang] +python-module = "sglang.srt.rust_extensions._bench" +debug = false + +[lib] +# Unique per-crate artifact name: the shared workspace target/ dir cannot hold +# two cdylibs of the same name. The importable Python module is `_bench`, named +# by the `#[pymodule]` entry point, and setuptools-rust renames the artifact. +name = "sglang_bench" +# cdylib: the PyO3 module (`sglang.srt.rust_extensions._bench`). +# rlib: so integration tests and any future standalone bin can link the core. +crate-type = ["cdylib", "rlib"] + +[dependencies] +bytes = { workspace = true } +futures = { workspace = true } +pyo3 = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +# `--protocol grpc` speaks the server's api.v1 service with the same +# generated request type the HTTP route decodes. +sglang_api_types = { package = "sglang-api-types", path = "../sglang-api-types" } +tokio = { workspace = true } +tonic = { workspace = true } + +# Declared here rather than in [workspace.dependencies]: no other member uses +# them, and a benchmark client's dependency choices should not constrain the +# server's graph. The versions match what sibling crates already resolve to, +# so this adds no package to Cargo.lock. +anyhow = "1" +clap = { version = "4", features = ["derive"] } +# Cache lookups only (same shape as sglang-server); the Hub download itself +# goes through this crate's own reqwest client. +hf-hub = { version = "0.4", default-features = false } +indicatif = "0.17" +rand = "0.8" +reqwest = { version = "0.12", default-features = false, features = [ + "json", + "rustls-tls", + "stream", +] } +tokenizers = { version = "0.21", default-features = false, features = ["onig"] } + +[dev-dependencies] +# The gRPC end-to-end test hosts a tonic mock of the api.v1 service. +tokio-stream = { workspace = true } diff --git a/rust/sglang-bench/README.md b/rust/sglang-bench/README.md new file mode 100644 index 0000000000..1225d112c8 --- /dev/null +++ b/rust/sglang-bench/README.md @@ -0,0 +1,176 @@ +# sglang-bench + +The serving benchmark client, ported from `python/sglang/benchmark/serving.py`. + +## Why + +The Python client drives every request and parses every response stream on one +asyncio thread, and it builds a fresh aiohttp session, and so a fresh +connection, per request. Both bite as the request count grows: past some +point the client, not the server, sets the measured throughput, and the +benchmark reports the client's limit. Here each request is a Tokio task on a +shared pooled HTTP client, so stream parsing spreads across +`--worker-threads` cores and connections are reused. + +What that is worth was measured on one host against a mock server, 96 output +tokens per request at concurrency 256, two repeats each: + +| Requests | Python | Rust | +|---|---|---| +| 500 | 1.8 s, 27.1k tok/s | 1.7 s, 28.9k tok/s | +| 2500 | 106.8 s, 2.2k tok/s | 13.2 s, 18.2k tok/s | + +So: no difference at 500 requests, and 8x at 2500, with the cliff somewhere +between. Mean TTFT follows the same shape, 196 ms against 153 ms at 500 +requests and 446 ms against 144 ms at 2500. + +**Treat those figures as a direction, not a specification.** The mock is a +Python `ThreadingHTTPServer` sharing the host's CPUs with the client, so it +caps both clients and punishes the Python one's connection churn harder than +a real server might. An earlier run of a different workload on the same setup +showed 2x rather than 8x. The client-side CPU drawn per request is comparable +between the two, which says the Python client here is blocked rather than +compute-bound, so where your own cliff falls depends on your server. Measure +it on the server you care about before quoting a number. + +## Usage + +Python loads it as the `sglang.srt.rust_extensions._bench` extension module, +the same way the Rust server is loaded. Two entry points: + +```bash +# Its own command line. Imports no model tooling, so it starts fast. +python -m sglang.benchmark.rust_client \ + --backend sglang --dataset-name random \ + --num-prompts 20000 --random-input-len 1024 --random-output-len 512 \ + --max-concurrency 512 + +# Or from the existing entry point, for a caller already on that command line. +python -m sglang.benchmark.serving --rust-client \ + --backend sglang --dataset-name random --num-prompts 20000 +``` + +The Rust client owns the flag definitions, so `--help` on the first form is +authoritative. Flag names, defaults, the printed report, and the result +`.jsonl` keys are the Python client's, so an existing command line runs +unchanged and anything reading the result file keeps working. +`--worker-threads` and `--disable-retokenize` are the additions; through +`serving.py` the first is spelled `--rust-worker-threads`. + +In process: + +```python +from sglang.benchmark.rust_client import run_rust_benchmark + +result = run_rust_benchmark(args) # args as `serving.py` parses them +``` + +`--rust-client` refuses a configuration the Rust client does not implement +rather than measuring something else; `serving.py` reports it as a usage +error. + +### Building + +A wheel build picks the crate up from `[package.metadata.sglang]`, as it does +for the other extensions. In a source tree the loader compiles and caches it +on first use, which costs about a minute once and roughly 8 seconds of +workspace fingerprinting per run after that; a wheel's bundled module skips +both. To stage it by hand: + +```bash +cd rust && cargo build --release -p sglang-bench +cp target/release/libsglang_bench.so \ + ../python/sglang/srt/rust_extensions/_bench.cpython-312-aarch64-linux-gnu.so +``` + + +## Covered + +- **Backends**: `sglang` / `sglang-native` (`/generate`), `sglang-oai`, + `vllm`, `lmdeploy` (`/v1/completions`), `sglang-oai-chat`, `vllm-chat`, + `lmdeploy-chat` (`/v1/chat/completions`), streaming and not. +- **Protocols**: `--protocol http` (default) and `--protocol grpc`, the Rust + server's `sglang.api.v1` service (below). +- **Datasets**: `random`, `random-ids`, `sharegpt`. +- Poisson request pacing (`--request-rate`), a concurrency cap, warmup, cache + flush, profiler start/stop, the readiness probe, `--extra-request-body`, + custom headers, and API-key auth. +- The full metric set, including the per-second peak-throughput and + peak-concurrency windows, with NumPy's percentile interpolation and + population standard deviation so the numbers line up with the Python + script's. + +### Token ids on both ends + +`--tokenize-prompt` sends `input_ids` instead of `text`. On the response side +tokens are counted from `meta_info.completion_tokens`, with the `output_ids` +a frame carries as the fallback, so a server launched with +`--skip-tokenizer-init`, which returns ids and an empty `text`, measures the +same way a decoding server does. The Python client cannot benchmark such a +server: it counts a token only where it sees text. + +With `--dataset-name random-ids --tokenize-prompt --vocab-size N` the client +loads no tokenizer at all and needs no model files; the retokenized totals +then report 0, since there is no text to re-encode. That combination takes +the tokenizer and detokenizer out of the measurement on both ends, which is +the datapath `sglang-server` is built around. + +### gRPC + +`--protocol grpc` sends the same backend's requests over the Rust server's +`sglang.api.v1` service (`Generate`, `GetModelInfo`, `GetServerInfo`) instead +of HTTP. It is a transport choice, not a backend: only the native API has a +gRPC service, so it pairs with `--backend sglang`, and the default port is the +server's HTTP port plus 10000, matching `--grpc-port`. The request is the +same schema the `/generate` route decodes, built once and deserialized into +the generated type, and each streamed frame is fed to the same accounting as +an SSE frame through its JSON form, so an HTTP run and a gRPC run differ only +in transport and are directly comparable. The result line carries a +`protocol` key, the one key the Python client does not write, so the two +kinds of run stay apart in a shared file. Two differences to know about: all +request streams share one HTTP/2 connection, and cache flush and the profiler +are HTTP-only controls, so a gRPC run reports and skips them. + +## Not covered + +Flags for these are absent rather than accepted and ignored, because a +silently dropped flag would report a benchmark nobody ran. + +- **Backends**: `trt`, `gserver`, `truss`, and the embedding backends. +- **Datasets**: `image`, `mmmu`, `mooncake`, `agentic-trace`, + `generated-shared-prefix`, `custom`, `openai`, `longbench-v2`, + `speed-bench`, and trace-timestamp replay. +- Multi-turn replay, LoRA request distributions, `--cache-report`, + `--plot-throughput`, PD-separated profiling, and the per-stage profiler + options. + +Reach for the Python script for any of those. Both write the same result +shape, so a mixed set of runs stays comparable. + +## Parity + +Counted quantities match exactly. Against one mock server with +`--random-range-ratio 1.0` (which fixes every length, making the two clients' +different random draws irrelevant), both clients report identical +`completed`, `total_input_tokens`, `total_output_tokens`, +`total_output_tokens_retokenized` and `accept_length`, and their result files +have identical key sets. + +Sampling does not match: the two use different random number generators, so +the same `--seed` draws different prompts. A run reproduces itself, and +distributions match, but individual prompts do not. + +Latency figures are not expected to match either, and that is the point: the +Python client's TTFT on the run above is 2.4x the Rust client's on the same +server, because its single parse thread is queueing. + +## Tests + +`cargo test -p sglang-bench` covers the wire parsers against recorded frame +sequences, the statistics against NumPy's definitions, dataset sampling, and +argument and config validation. CI runs them through the workspace +`cargo test` in `test/registered/rust/test_run_rust_tests.py`. + +`test/registered/unit/benchmark/test_rust_client.py` covers the Python +boundary (config translation and the refusals) with the extension mocked, so +it needs no compiled artifact. diff --git a/rust/sglang-bench/src/args.rs b/rust/sglang-bench/src/args.rs new file mode 100644 index 0000000000..b4781f6f4a --- /dev/null +++ b/rust/sglang-bench/src/args.rs @@ -0,0 +1,601 @@ +//! The command line, mirroring `python/sglang/benchmark/serving.py`'s flags. +//! +//! Every flag that this port implements keeps its Python spelling and default, +//! so an existing command line runs here unchanged. Flags the Python script +//! has for surfaces this port does not implement (the TRT / gserver / truss +//! backends, the image, mooncake, agentic-trace and generated-shared-prefix +//! datasets, LoRA, profiling by stage) are absent rather than accepted and +//! ignored: a silently dropped flag would report a benchmark nobody ran. + +use clap::{Parser, ValueEnum}; +use serde::Deserialize; + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, ValueEnum)] +#[serde(rename_all = "kebab-case")] +pub enum Backend { + /// `/generate`, the native SGLang route. + Sglang, + /// Alias of `sglang`, as in the Python script. + SglangNative, + /// `/v1/completions` on an SGLang server. + SglangOai, + /// `/v1/chat/completions` on an SGLang server. + SglangOaiChat, + Vllm, + VllmChat, + Lmdeploy, + LmdeployChat, +} + +/// The wire the request travels on. Orthogonal to `--backend`, which picks +/// the API shape: `grpc` carries the same `/generate` contract as HTTP, on +/// the Rust server's `sglang.api.v1` service. +#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq, ValueEnum)] +#[serde(rename_all = "kebab-case")] +pub enum Protocol { + /// JSON over HTTP, streamed as SSE. What the Python client speaks. + #[default] + Http, + /// The Rust server's gRPC service; native-API backends only. + Grpc, +} + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, ValueEnum)] +#[serde(rename_all = "kebab-case")] +pub enum DatasetName { + /// Prompts sampled from ShareGPT and repeated or truncated to the + /// requested input length. + Random, + /// Prompts built from arithmetic token-id sequences, no corpus needed. + RandomIds, + /// The first turn of each ShareGPT conversation, with its reply's length + /// as the output length. + Sharegpt, +} + +#[derive(Debug, Deserialize, Parser)] +// Every field defaults, so a JSON config sets only what it means to change. +// `deny_unknown_fields` makes a misspelled key an error rather than a setting +// that silently did not apply. +#[serde(default, deny_unknown_fields)] +#[command( + name = "sglang-bench", + about = "Serving benchmark client (the Rust port of sglang.benchmark.serving)", + after_help = "Examples:\n \ + sglang-bench --dataset-name random --num-prompts 20000 --random-input-len 1024 --random-output-len 512\n \ + sglang-bench --backend sglang-oai-chat --dataset-name sharegpt --num-prompts 5000 --max-concurrency 512" +)] +pub struct Args { + #[arg(long, value_enum, default_value = "sglang")] + pub backend: Backend, + + /// Wire protocol. `grpc` needs a native-API backend (`sglang`) and talks + /// to the Rust server's gRPC listener, whose default port is the HTTP + /// port plus 10000. + #[arg(long, value_enum, default_value = "http")] + pub protocol: Protocol, + + /// Full base URL. Overrides `--host` / `--port`. + #[arg(long)] + pub base_url: Option, + + #[arg(long, default_value = "0.0.0.0")] + pub host: String, + + /// Defaults per backend, as in the Python script (30000 for SGLang, + /// 8000 for vLLM, 23333 for LMDeploy). + #[arg(long)] + pub port: Option, + + #[arg(long, default_value_t = 60)] + pub ready_check_timeout_sec: u64, + + #[arg(long, value_enum, default_value = "sharegpt")] + pub dataset_name: DatasetName, + + /// Path to a local ShareGPT JSON. Empty downloads and caches it. + #[arg(long, default_value = "")] + pub dataset_path: String, + + /// Name or path of the model. Defaults to the server's `/v1/models`. + #[arg(long)] + pub model: Option, + + /// The model name to send, when it differs from `--model`. + #[arg(long)] + pub served_model_name: Option, + + /// Tokenizer path or HF repo id. Defaults to `--model`. + #[arg(long)] + pub tokenizer: Option, + + #[arg(long, default_value_t = 1000)] + pub num_prompts: usize, + + #[arg(long)] + pub sharegpt_output_len: Option, + + #[arg(long)] + pub sharegpt_context_len: Option, + + #[arg(long, default_value_t = 1024)] + pub random_input_len: usize, + + #[arg(long, default_value_t = 1024)] + pub random_output_len: usize, + + #[arg(long, default_value_t = 0.0)] + pub random_range_ratio: f64, + + /// Requests per second, Poisson-paced. `inf` sends everything at once. + #[arg(long, default_value = "inf")] + pub request_rate: String, + + /// Cap on requests in flight. Unset means unbounded. + #[arg(long)] + pub max_concurrency: Option, + + #[arg(long, default_value_t = 1)] + pub warmup_requests: usize, + + /// Tokio worker threads that parse the response streams. Defaults to the + /// available parallelism; this is the knob the Python script lacks, where + /// one asyncio thread parses every stream. + #[arg(long)] + pub worker_threads: Option, + + #[arg(long)] + pub output_file: Option, + + /// Also write the per-request columns (input/output lens, TTFTs, ITLs, + /// generated texts, errors) into the result line. + #[arg(long)] + pub output_details: bool, + + #[arg(long)] + pub disable_tqdm: bool, + + #[arg(long)] + pub disable_stream: bool, + + #[arg(long)] + pub disable_ignore_eos: bool, + + #[arg(long, default_value_t = 42)] + pub seed: u64, + + #[arg(long, default_value_t = 0.0)] + pub temperature: f64, + + #[arg(long, default_value_t = 1.0)] + pub top_p: f64, + + #[arg(long)] + pub return_logprob: bool, + + #[arg(long, default_value_t = 0)] + pub top_logprobs_num: u32, + + #[arg(long, default_value_t = -1)] + pub logprob_start_len: i64, + + /// JSON object merged into every request body, overriding the defaults. + #[arg(long)] + pub extra_request_body: Option, + + /// `key: value` header, repeatable. + #[arg(long = "header", value_name = "KEY: VALUE")] + pub headers: Vec, + + /// Send `input_ids` instead of `text`. Only for the random datasets. + /// The response side follows automatically: a server launched with + /// `--skip-tokenizer-init` returns `output_ids` and no text, and tokens + /// are then counted from the ids. + #[arg(long)] + pub tokenize_prompt: bool, + + /// Vocabulary size for `random-ids` with `--tokenize-prompt`. Given, no + /// tokenizer is loaded at all, so the client needs no model files; the + /// retokenized totals are then reported as 0. + #[arg(long)] + pub vocab_size: Option, + + /// Skip re-tokenizing the generated text. The retokenized totals are then + /// reported as 0; use it when only throughput and latency matter. + #[arg(long)] + pub disable_retokenize: bool, + + /// POST `/flush_cache` after the warmup. + #[arg(long)] + pub flush_cache: bool, + + #[arg(long, default_value_t = 60.0)] + pub flush_cache_timeout: f64, + + /// Start and stop the server profiler around the measured run. + #[arg(long)] + pub profile: bool, + + /// Free-form label recorded in the result line. + #[arg(long)] + pub tag: Option, +} + +/// The defaults are clap's, read back from an empty command line, so the two +/// entry points cannot drift: there is one place a default is written. +impl Default for Args { + fn default() -> Self { + Self::parse_from(["sglang-bench"]) + } +} + +impl Backend { + /// The Python `_BACKEND_API_PATHS` entry. + pub fn api_path(self) -> &'static str { + match self { + Self::Sglang | Self::SglangNative => "/generate", + Self::SglangOai | Self::Vllm | Self::Lmdeploy => "/v1/completions", + Self::SglangOaiChat | Self::VllmChat | Self::LmdeployChat => "/v1/chat/completions", + } + } + + /// The Python per-backend default port. The Rust server's gRPC listener + /// defaults to its HTTP port plus 10000 (`--grpc-port`). + pub fn default_port(self, protocol: Protocol) -> u16 { + let http = match self { + Self::Sglang | Self::SglangNative | Self::SglangOai | Self::SglangOaiChat => 30000, + Self::Vllm | Self::VllmChat => 8000, + Self::Lmdeploy | Self::LmdeployChat => 23333, + }; + match protocol { + Protocol::Http => http, + Protocol::Grpc => http + 10000, + } + } + + /// The API shape this backend speaks, which decides how a request is + /// built and how a streamed chunk is read. + pub fn api(self) -> Api { + match self { + Self::Sglang | Self::SglangNative => Api::SglangGenerate, + Self::SglangOai | Self::Vllm | Self::Lmdeploy => Api::OpenaiCompletions, + Self::SglangOaiChat | Self::VllmChat | Self::LmdeployChat => Api::OpenaiChat, + } + } + + /// The name this backend reports as, matching the Python `--backend` value. + pub fn as_str(self) -> &'static str { + match self { + Self::Sglang => "sglang", + Self::SglangNative => "sglang-native", + Self::SglangOai => "sglang-oai", + Self::SglangOaiChat => "sglang-oai-chat", + Self::Vllm => "vllm", + Self::VllmChat => "vllm-chat", + Self::Lmdeploy => "lmdeploy", + Self::LmdeployChat => "lmdeploy-chat", + } + } + + pub fn is_sglang(self) -> bool { + self.as_str().starts_with("sglang") + } +} + +/// The request and frame shape, independent of the wire it travels on. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Api { + /// SGLang's native `/generate`, over HTTP or the gRPC service. + SglangGenerate, + OpenaiCompletions, + OpenaiChat, +} + +impl Protocol { + pub fn as_str(self) -> &'static str { + match self { + Self::Http => "http", + Self::Grpc => "grpc", + } + } +} + +impl DatasetName { + pub fn as_str(self) -> &'static str { + match self { + Self::Random => "random", + Self::RandomIds => "random-ids", + Self::Sharegpt => "sharegpt", + } + } + + pub fn is_random(self) -> bool { + matches!(self, Self::Random | Self::RandomIds) + } +} + +impl Args { + /// `--request-rate`, with `inf` as the unpaced default. + pub fn parse_request_rate(&self) -> anyhow::Result { + let raw = self.request_rate.trim(); + if raw.eq_ignore_ascii_case("inf") || raw.eq_ignore_ascii_case("infinity") { + return Ok(f64::INFINITY); + } + let rate: f64 = raw + .parse() + .map_err(|_| anyhow::anyhow!("--request-rate must be a number or `inf`, got {raw}"))?; + anyhow::ensure!(rate > 0.0, "--request-rate must be positive, got {rate}"); + Ok(rate) + } + + /// The base URL the client posts to, `--base-url` or `http://host:port`. + /// A bare IPv6 host is bracketed, as `resolve_base_url` does in Python. + pub fn base_url(&self) -> String { + if let Some(url) = &self.base_url { + return url.trim_end_matches('/').to_owned(); + } + let port = self + .port + .unwrap_or_else(|| self.backend.default_port(self.protocol)); + let host = if self.host.contains(':') && !self.host.starts_with('[') { + format!("[{}]", self.host) + } else { + self.host.clone() + }; + format!("http://{host}:{port}") + } + + /// Where requests go: the route under the base URL over HTTP, the bare + /// endpoint over gRPC, whose service lives at the connection's root. + pub fn api_url(&self) -> String { + match self.protocol { + Protocol::Http => format!("{}{}", self.base_url(), self.backend.api_path()), + Protocol::Grpc => self.base_url(), + } + } + + /// `--extra-request-body`, which must be a JSON object. + pub fn parse_extra_body(&self) -> anyhow::Result> { + let Some(raw) = &self.extra_request_body else { + return Ok(serde_json::Map::new()); + }; + match serde_json::from_str(raw) { + Ok(serde_json::Value::Object(map)) => Ok(map), + Ok(_) => anyhow::bail!("--extra-request-body must be a JSON object"), + Err(error) => anyhow::bail!("--extra-request-body is not valid JSON: {error}"), + } + } + + /// Validate the flag combinations the Python script asserts on. + pub fn validate(&self) -> anyhow::Result<()> { + anyhow::ensure!(self.num_prompts > 0, "--num-prompts must be at least 1"); + anyhow::ensure!( + (0.0..=1.0).contains(&self.random_range_ratio), + "--random-range-ratio must be in [0, 1], got {}", + self.random_range_ratio + ); + anyhow::ensure!( + !self.tokenize_prompt || self.dataset_name.is_random(), + "--tokenize-prompt is only supported by the random datasets" + ); + anyhow::ensure!( + !self.tokenize_prompt || self.backend.api() != Api::OpenaiChat, + "--tokenize-prompt cannot be used with a chat backend, which sends messages" + ); + anyhow::ensure!( + self.protocol == Protocol::Http || self.backend.api() == Api::SglangGenerate, + "--protocol grpc carries the native /generate contract only; use --backend sglang \ + (the OpenAI routes have no gRPC service)" + ); + if let Some(len) = self.sharegpt_output_len { + anyhow::ensure!(len >= 4, "--sharegpt-output-len is too small, got {len}"); + } + if let Some(limit) = self.max_concurrency { + anyhow::ensure!(limit > 0, "--max-concurrency must be at least 1"); + } + if let Some(size) = self.vocab_size { + anyhow::ensure!(size > 0, "--vocab-size must be at least 1"); + anyhow::ensure!( + self.tokenize_prompt && self.dataset_name == DatasetName::RandomIds, + "--vocab-size is only meaningful with --tokenize-prompt --dataset-name random-ids; \ + every other dataset needs the tokenizer" + ); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn args(extra: &[&str]) -> Args { + let mut argv = vec!["sglang-bench"]; + argv.extend_from_slice(extra); + Args::parse_from(argv) + } + + /// The URL is assembled as the Python script's `resolve_base_url` does, + /// including the per-backend default port and IPv6 bracketing. + #[test] + fn urls_follow_the_backend_defaults() { + assert_eq!(args(&[]).api_url(), "http://0.0.0.0:30000/generate"); + assert_eq!( + args(&["--backend", "vllm"]).api_url(), + "http://0.0.0.0:8000/v1/completions" + ); + assert_eq!( + args(&["--backend", "lmdeploy-chat", "--port", "1"]).api_url(), + "http://0.0.0.0:1/v1/chat/completions" + ); + assert_eq!( + args(&["--host", "::1"]).api_url(), + "http://[::1]:30000/generate" + ); + assert_eq!( + args(&["--base-url", "https://example.com/"]).api_url(), + "https://example.com/generate" + ); + // gRPC has no route path, and its default port is the server's + // HTTP port plus 10000. + assert_eq!( + args(&["--protocol", "grpc"]).api_url(), + "http://0.0.0.0:40000" + ); + assert_eq!( + args(&["--protocol", "grpc", "--port", "7"]).api_url(), + "http://0.0.0.0:7" + ); + } + + /// Only the native API has a gRPC service, so the protocol is rejected + /// for the OpenAI-shaped backends rather than quietly sent over HTTP. + #[test] + fn grpc_requires_a_native_backend() { + assert!(args(&["--protocol", "grpc"]).validate().is_ok()); + assert!( + args(&["--protocol", "grpc", "--backend", "sglang-native"]) + .validate() + .is_ok() + ); + for backend in ["sglang-oai", "sglang-oai-chat", "vllm", "lmdeploy-chat"] { + assert!( + args(&["--protocol", "grpc", "--backend", backend]) + .validate() + .is_err(), + "{backend}" + ); + } + } + + #[test] + fn request_rate_accepts_inf_and_rejects_nonsense() { + assert!(args(&[]).parse_request_rate().unwrap().is_infinite()); + assert_eq!( + args(&["--request-rate", "7.5"]) + .parse_request_rate() + .unwrap(), + 7.5 + ); + assert!(args(&["--request-rate", "0"]).parse_request_rate().is_err()); + assert!( + args(&["--request-rate", "fast"]) + .parse_request_rate() + .is_err() + ); + } + + #[test] + fn extra_request_body_must_be_an_object() { + let parsed = args(&["--extra-request-body", r#"{"top_k": 5}"#]) + .parse_extra_body() + .unwrap(); + assert_eq!(parsed["top_k"], 5); + assert!( + args(&["--extra-request-body", "[1]"]) + .parse_extra_body() + .is_err() + ); + assert!( + args(&["--extra-request-body", "{"]) + .parse_extra_body() + .is_err() + ); + assert!(args(&[]).parse_extra_body().unwrap().is_empty()); + } + + /// Flag combinations the Python script asserts on must fail here too, + /// rather than producing a benchmark whose shape nobody asked for. + #[test] + fn validation_rejects_unsupported_combinations() { + assert!(args(&[]).validate().is_ok()); + assert!(args(&["--num-prompts", "0"]).validate().is_err()); + assert!(args(&["--random-range-ratio", "1.5"]).validate().is_err()); + assert!(args(&["--tokenize-prompt"]).validate().is_err()); + assert!( + args(&["--tokenize-prompt", "--dataset-name", "random"]) + .validate() + .is_ok() + ); + assert!( + args(&[ + "--tokenize-prompt", + "--dataset-name", + "random", + "--backend", + "sglang-oai-chat" + ]) + .validate() + .is_err() + ); + assert!(args(&["--sharegpt-output-len", "2"]).validate().is_err()); + assert!(args(&["--max-concurrency", "0"]).validate().is_err()); + } + + /// `--vocab-size` replaces the tokenizer, so it is only coherent where the + /// tokenizer's sole job was to supply that number. + #[test] + fn vocab_size_requires_the_tokenizer_free_dataset() { + let ok = [ + "--vocab-size", + "1000", + "--tokenize-prompt", + "--dataset-name", + "random-ids", + ]; + assert!(args(&ok).validate().is_ok()); + assert!(args(&["--vocab-size", "1000"]).validate().is_err()); + assert!( + args(&[ + "--vocab-size", + "1000", + "--tokenize-prompt", + "--dataset-name", + "random" + ]) + .validate() + .is_err() + ); + assert!( + args(&[ + "--vocab-size", + "0", + "--tokenize-prompt", + "--dataset-name", + "random-ids" + ]) + .validate() + .is_err() + ); + } + + /// A JSON config sets only the keys it names; the rest take the same + /// defaults the command line would apply. + #[test] + fn a_config_fills_missing_keys_from_the_cli_defaults() { + let args: Args = serde_json::from_str( + r#"{"backend": "sglang-oai-chat", "dataset_name": "random-ids", "num_prompts": 7}"#, + ) + .unwrap(); + assert_eq!(args.backend, Backend::SglangOaiChat); + assert_eq!(args.dataset_name, DatasetName::RandomIds); + assert_eq!(args.num_prompts, 7); + // Untouched keys match the flag defaults. + assert_eq!(args.random_input_len, 1024); + assert_eq!(args.seed, 42); + assert_eq!(args.host, "0.0.0.0"); + assert!(args.parse_request_rate().unwrap().is_infinite()); + assert!(!args.disable_stream); + } + + /// An empty config is the full default set, and a misspelled key is an + /// error: silently ignoring it would report a benchmark that did not run + /// the way the caller asked. + #[test] + fn a_config_rejects_unknown_keys() { + let defaults: Args = serde_json::from_str("{}").unwrap(); + assert_eq!(defaults.num_prompts, Args::default().num_prompts); + let typo = serde_json::from_str::(r#"{"num_promts": 3}"#); + assert!(typo.is_err()); + } +} diff --git a/rust/sglang-bench/src/dataset.rs b/rust/sglang-bench/src/dataset.rs new file mode 100644 index 0000000000..fab806d97e --- /dev/null +++ b/rust/sglang-bench/src/dataset.rs @@ -0,0 +1,368 @@ +//! Prompt generation, ported from `python/sglang/benchmark/datasets/`. +//! +//! The `random`, `random-ids` and `sharegpt` samplers are here; they are the +//! ones a large-batch run uses. Sampling is seeded, so a run reproduces +//! itself, but it does not reproduce the Python script's selection: the two +//! use different random number generators, so the same `--seed` draws +//! different prompts. Totals and distributions match; individual prompts do +//! not. + +use std::path::Path; + +use anyhow::{Context, Result}; +use rand::prelude::*; +use rand::rngs::StdRng; +use serde::Deserialize; +use tokenizers::Tokenizer; + +use crate::args::{Args, DatasetName}; + +/// What goes on the wire as the prompt. +#[derive(Clone, Debug)] +pub enum Prompt { + Text(String), + /// `--tokenize-prompt`: skip the server's tokenizer and send ids. + TokenIds(Vec), +} + +#[derive(Clone, Debug)] +pub struct DatasetRow { + pub prompt: Prompt, + pub prompt_len: usize, + pub output_len: usize, + pub text_prompt_len: usize, + /// Always 0 here: this port has no image datasets. The field exists so the + /// reported totals keep the Python script's shape. + pub vision_prompt_len: usize, +} + +impl DatasetRow { + fn new(prompt: Prompt, prompt_len: usize, output_len: usize) -> Self { + Self { + prompt, + prompt_len, + output_len, + text_prompt_len: prompt_len, + vision_prompt_len: 0, + } + } +} + +/// Whether `--dataset-name` needs the ShareGPT corpus, which the caller +/// resolves (and may download) before sampling. +pub fn needs_corpus(name: DatasetName) -> bool { + matches!(name, DatasetName::Random | DatasetName::Sharegpt) +} + +/// Build the request set for `--dataset-name`. `corpus` is the ShareGPT path +/// when [`needs_corpus`] asked for one. +/// `tokenizer` may be absent only for `random-ids` with `--tokenize-prompt +/// --vocab-size`, the one combination that reads nothing from it. +pub fn load( + args: &Args, + tokenizer: Option<&Tokenizer>, + corpus: Option<&Path>, +) -> Result> { + let mut rng = StdRng::seed_from_u64(args.seed); + let rows = match args.dataset_name { + DatasetName::Random | DatasetName::RandomIds => { + sample_random(args, tokenizer, corpus, &mut rng)? + } + DatasetName::Sharegpt => { + let corpus = corpus.context("the sharegpt dataset needs a corpus path")?; + sample_sharegpt(args, required(tokenizer)?, corpus, &mut rng)? + } + }; + anyhow::ensure!( + !rows.is_empty(), + "the dataset produced no requests; check --num-prompts and the dataset filters" + ); + let input_tokens: usize = rows.iter().map(|row| row.prompt_len).sum(); + let output_tokens: usize = rows.iter().map(|row| row.output_len).sum(); + println!("#Input tokens: {input_tokens}"); + println!("#Output tokens: {output_tokens}"); + Ok(rows) +} + +/// Per-request lengths drawn uniformly from `[full * ratio, full]`, as the +/// Python `compute_random_lens` does. A ratio of 0 still has a floor of 1, so +/// no request asks for an empty prompt. +fn random_lens(full_len: usize, range_ratio: f64, count: usize, rng: &mut StdRng) -> Vec { + if full_len == 0 { + return vec![0; count]; + } + let low = ((full_len as f64 * range_ratio) as usize).max(1); + (0..count).map(|_| rng.gen_range(low..=full_len)).collect() +} + +/// The `random` and `random-ids` datasets. +/// +/// `random` repeats or truncates real ShareGPT prompts to the requested +/// length, which keeps the token distribution realistic. `random-ids` builds +/// arithmetic id sequences instead, needing no corpus, at the cost of prompts +/// that no tokenizer would produce. +fn sample_random( + args: &Args, + tokenizer: Option<&Tokenizer>, + corpus: Option<&Path>, + rng: &mut StdRng, +) -> Result> { + let count = args.num_prompts; + let mut input_lens = random_lens(args.random_input_len, args.random_range_ratio, count, rng); + let output_lens = random_lens(args.random_output_len, args.random_range_ratio, count, rng); + + let return_text = !args.tokenize_prompt; + if return_text { + // The server adds its own special tokens, so leave room for them or + // every prompt arrives longer than requested. + let special = num_special_tokens(required(tokenizer)?); + for len in &mut input_lens { + *len = len.saturating_sub(special).max(1); + } + } + + if args.dataset_name == DatasetName::RandomIds { + let vocab_size = match args.vocab_size { + Some(size) => size, + None => required(tokenizer)?.get_vocab_size(false) as u32, + }; + anyhow::ensure!(vocab_size > 0, "the tokenizer reports an empty vocabulary"); + let mut rows = Vec::with_capacity(count); + for (index, (&input_len, &output_len)) in + input_lens.iter().zip(output_lens.iter()).enumerate() + { + let offset: u32 = rng.gen_range(0..vocab_size); + let ids: Vec = (0..input_len) + .map(|position| { + let raw = offset as u64 + index as u64 + position as u64; + (raw % u64::from(vocab_size)) as u32 + }) + .collect(); + rows.push(DatasetRow::new( + to_prompt(ids, return_text, tokenizer)?, + input_len, + output_len, + )); + } + return Ok(rows); + } + + let corpus = corpus.context("the random dataset samples prompts from a corpus")?; + let tokenizer = required(tokenizer)?; + let corpus = load_sharegpt_pairs(corpus, rng)?; + let mut rows = Vec::with_capacity(count); + for (prompt, _) in &corpus { + let index = rows.len(); + if index == count { + break; + } + let token_ids = encode(tokenizer, prompt, true)?; + if token_ids.is_empty() { + continue; + } + let target = input_lens[index]; + let ids: Vec = token_ids.iter().copied().cycle().take(target).collect(); + rows.push(DatasetRow::new( + to_prompt(ids, return_text, Some(tokenizer))?, + target, + output_lens[index], + )); + } + anyhow::ensure!( + rows.len() == count, + "the corpus held {} usable prompts, fewer than --num-prompts {count}", + rows.len() + ); + Ok(rows) +} + +/// The `sharegpt` dataset: each conversation's first turn is the prompt and +/// its reply's length is the output length, with the too-short and too-long +/// rows pruned. +fn sample_sharegpt( + args: &Args, + tokenizer: &Tokenizer, + corpus: &Path, + rng: &mut StdRng, +) -> Result> { + let corpus = load_sharegpt_pairs(corpus, rng)?; + let mut rows = Vec::with_capacity(args.num_prompts); + for (prompt, completion) in &corpus { + if rows.len() == args.num_prompts { + break; + } + let prompt_len = encode(tokenizer, prompt, true)?.len(); + let output_len = match args.sharegpt_output_len { + Some(len) => len, + None => encode(tokenizer, completion, true)?.len(), + }; + // Prune sequences too short to measure a decode interval on. + if prompt_len < 2 || output_len < 2 { + continue; + } + if let Some(context_len) = args.sharegpt_context_len + && prompt_len + output_len > context_len + { + continue; + } + rows.push(DatasetRow::new( + Prompt::Text(prompt.clone()), + prompt_len, + output_len, + )); + } + Ok(rows) +} + +/// A ShareGPT conversation, keeping only what the samplers read. +#[derive(Deserialize)] +struct Conversation { + #[serde(alias = "conversation", default)] + conversations: Vec, +} + +#[derive(Deserialize)] +struct Turn { + #[serde(default)] + value: String, +} + +/// The first two turns of every conversation that has at least two, shuffled. +fn load_sharegpt_pairs(path: &Path, rng: &mut StdRng) -> Result> { + let bytes = std::fs::read(path).with_context(|| format!("cannot read {}", path.display()))?; + let conversations: Vec = serde_json::from_slice(&bytes) + .with_context(|| format!("{} is not the ShareGPT JSON", path.display()))?; + let mut pairs: Vec<(String, String)> = conversations + .into_iter() + .filter(|conversation| conversation.conversations.len() >= 2) + .map(|mut conversation| { + let mut turns = conversation.conversations.drain(..2); + let prompt = turns.next().expect("two turns").value; + let completion = turns.next().expect("two turns").value; + (prompt, completion) + }) + .collect(); + anyhow::ensure!( + !pairs.is_empty(), + "{} held no conversation with two turns", + path.display() + ); + pairs.shuffle(rng); + Ok(pairs) +} + +fn required(tokenizer: Option<&Tokenizer>) -> Result<&Tokenizer> { + tokenizer.context( + "this dataset needs a tokenizer; --vocab-size only covers random-ids with --tokenize-prompt", + ) +} + +fn to_prompt(ids: Vec, return_text: bool, tokenizer: Option<&Tokenizer>) -> Result { + if !return_text { + return Ok(Prompt::TokenIds(ids)); + } + let text = required(tokenizer)? + .decode(&ids, false) + .map_err(|error| anyhow::anyhow!("decode failed: {error}"))?; + Ok(Prompt::Text(text)) +} + +fn encode(tokenizer: &Tokenizer, text: &str, add_special_tokens: bool) -> Result> { + let encoding = tokenizer + .encode(text, add_special_tokens) + .map_err(|error| anyhow::anyhow!("encode failed: {error}"))?; + Ok(encoding.get_ids().to_vec()) +} + +/// How many special tokens the tokenizer adds around a prompt, measured the +/// only way its API allows: encode nothing and count what appears. +fn num_special_tokens(tokenizer: &Tokenizer) -> usize { + tokenizer + .encode("", true) + .map(|encoding| encoding.len()) + .unwrap_or(0) +} + +/// Re-tokenize the generated texts to report the server's token count against +/// an independent one. Batched, so the tokenizer's own parallelism applies; +/// the Python script encodes them one at a time, which dominates its +/// post-processing on a large run. +pub fn retokenized_lens(tokenizer: &Tokenizer, texts: &[&str]) -> Result> { + if texts.is_empty() { + return Ok(Vec::new()); + } + let encodings = tokenizer + .encode_batch(texts.to_vec(), false) + .map_err(|error| anyhow::anyhow!("batch encode failed: {error}"))?; + Ok(encodings.iter().map(tokenizers::Encoding::len).collect()) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Lengths stay inside `[full * ratio, full]`, and a ratio of 0 keeps the + /// floor at 1 so no prompt is empty. + #[test] + fn random_lens_respect_the_range_ratio() { + let mut rng = StdRng::seed_from_u64(1); + let lens = random_lens(100, 0.5, 64, &mut rng); + assert_eq!(lens.len(), 64); + assert!(lens.iter().all(|&len| (50..=100).contains(&len))); + + let full = random_lens(10, 1.0, 8, &mut rng); + assert!(full.iter().all(|&len| len == 10)); + + let floored = random_lens(4, 0.0, 32, &mut rng); + assert!(floored.iter().all(|&len| (1..=4).contains(&len))); + + // An output length of 0 is valid and must not panic on the floor. + assert_eq!(random_lens(0, 0.5, 3, &mut rng), vec![0, 0, 0]); + } + + /// The same seed draws the same lengths, so two runs of this binary are + /// comparable even though they do not match the Python script's draw. + #[test] + fn sampling_is_reproducible_under_a_seed() { + let first = random_lens(64, 0.3, 16, &mut StdRng::seed_from_u64(7)); + let second = random_lens(64, 0.3, 16, &mut StdRng::seed_from_u64(7)); + assert_eq!(first, second); + let other = random_lens(64, 0.3, 16, &mut StdRng::seed_from_u64(8)); + assert_ne!(first, other); + } + + /// Both ShareGPT spellings parse, conversations with one turn are dropped, + /// and only the first two turns are kept. + #[test] + fn sharegpt_parsing_accepts_both_key_spellings() { + let json = serde_json::json!([ + {"conversations": [{"value": "q1"}, {"value": "a1"}, {"value": "ignored"}]}, + {"conversation": [{"value": "q2"}, {"value": "a2"}]}, + {"conversations": [{"value": "orphan"}]}, + {"id": "no conversation key at all"}, + ]); + let path = + std::env::temp_dir().join(format!("sgl-bench-sharegpt-{}.json", std::process::id())); + std::fs::write(&path, serde_json::to_vec(&json).unwrap()).unwrap(); + + let mut pairs = load_sharegpt_pairs(&path, &mut StdRng::seed_from_u64(3)).unwrap(); + pairs.sort(); + assert_eq!( + pairs, + vec![ + ("q1".to_owned(), "a1".to_owned()), + ("q2".to_owned(), "a2".to_owned()) + ] + ); + std::fs::remove_file(&path).unwrap(); + } + + #[test] + fn a_corpus_without_usable_pairs_is_an_error() { + let path = + std::env::temp_dir().join(format!("sgl-bench-empty-{}.json", std::process::id())); + std::fs::write(&path, b"[]").unwrap(); + assert!(load_sharegpt_pairs(&path, &mut StdRng::seed_from_u64(1)).is_err()); + std::fs::remove_file(&path).unwrap(); + } +} diff --git a/rust/sglang-bench/src/hf.rs b/rust/sglang-bench/src/hf.rs new file mode 100644 index 0000000000..040323c042 --- /dev/null +++ b/rust/sglang-bench/src/hf.rs @@ -0,0 +1,230 @@ +//! Tokenizer and corpus files: the local HF cache first, then the Hub. +//! +//! The cache lookup mirrors `rust/sglang-server`'s `resolve_model_file`, so a +//! model the Python scheduler already downloaded is found without network +//! access. Only a cache miss reaches the Hub, and the download is cached for +//! the next run. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use tokenizers::Tokenizer; + +pub const SHAREGPT_REPO: &str = "anon8231489123/ShareGPT_Vicuna_unfiltered"; +pub const SHAREGPT_FILE: &str = "ShareGPT_V3_unfiltered_cleaned_split.json"; + +/// Locate a `tokenizer.json`, given a file, a directory holding one, or an HF +/// repo id. +pub async fn resolve_tokenizer(http: &reqwest::Client, source: &str) -> Result { + resolve_file(http, source, "tokenizer.json", RepoKind::Model) + .await + .with_context(|| format!("cannot find a tokenizer for {source}")) +} + +/// Load a tokenizer from an already-resolved path. +pub fn load_tokenizer(path: &Path) -> Result { + Tokenizer::from_file(path) + .map_err(|error| anyhow::anyhow!("{}: {error}", path.display())) + .context("tokenizer.json failed to load") +} + +/// The ShareGPT conversations file, from `--dataset-path` or the Hub. +pub async fn resolve_sharegpt(http: &reqwest::Client, dataset_path: &str) -> Result { + if !dataset_path.is_empty() { + let path = PathBuf::from(dataset_path); + anyhow::ensure!( + path.is_file(), + "--dataset-path {dataset_path} is not a file" + ); + return Ok(path); + } + resolve_file(http, SHAREGPT_REPO, SHAREGPT_FILE, RepoKind::Dataset) + .await + .context("cannot obtain the ShareGPT dataset; pass --dataset-path") +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum RepoKind { + Model, + Dataset, +} + +impl RepoKind { + /// The Hub URL segment; a dataset repo is namespaced, a model repo is not. + fn url_prefix(self) -> &'static str { + match self { + Self::Model => "", + Self::Dataset => "datasets/", + } + } + + fn hub_type(self) -> hf_hub::RepoType { + match self { + Self::Model => hf_hub::RepoType::Model, + Self::Dataset => hf_hub::RepoType::Dataset, + } + } +} + +/// A local path as given, a file inside a given directory, or `filename` from +/// the repo `source` names. +async fn resolve_file( + http: &reqwest::Client, + source: &str, + filename: &str, + kind: RepoKind, +) -> Result { + let path = Path::new(source); + if path.is_file() { + return Ok(path.to_path_buf()); + } + if path.is_dir() { + let candidate = path.join(filename); + anyhow::ensure!(candidate.is_file(), "{} has no {filename}", path.display()); + return Ok(candidate); + } + if let Some(cached) = from_hub_cache(source, filename, kind) { + return Ok(cached); + } + download(http, source, filename, kind).await +} + +/// Look `filename` up in the local HF hub cache. Python resolves the cache +/// directory as `HF_HUB_CACHE` > `HUGGINGFACE_HUB_CACHE` > `HF_HOME/hub`, +/// while the hf-hub crate only knows `HF_HOME`; honour the overrides first or +/// a model the scheduler downloaded is missed. +fn from_hub_cache(repo_id: &str, filename: &str, kind: RepoKind) -> Option { + use hf_hub::{Cache, Repo}; + + let cache = ["HF_HUB_CACHE", "HUGGINGFACE_HUB_CACHE"] + .iter() + .find_map(|var| std::env::var(var).ok()) + .map(|dir| Cache::new(dir.into())) + .unwrap_or_else(Cache::from_env); + cache + .repo(Repo::with_revision( + repo_id.to_owned(), + kind.hub_type(), + "main".to_owned(), + )) + .get(filename) +} + +/// Fetch `filename` from the Hub into this crate's own cache directory. +/// Downloads to a temporary file and renames, so an interrupted run does not +/// leave a truncated file that the next one would read as complete. +async fn download( + http: &reqwest::Client, + repo_id: &str, + filename: &str, + kind: RepoKind, +) -> Result { + let destination = download_cache_dir()?.join(format!( + "{}--{filename}", + repo_id.replace(['/', '\\'], "--") + )); + if destination.is_file() { + return Ok(destination); + } + + let url = format!( + "https://huggingface.co/{}{repo_id}/resolve/main/{filename}", + kind.url_prefix() + ); + eprintln!("downloading {url}"); + let mut request = http.get(&url); + if let Some(token) = hf_token() { + request = request.bearer_auth(token); + } + let response = request.send().await.with_context(|| format!("GET {url}"))?; + anyhow::ensure!( + response.status().is_success(), + "GET {url} returned {}", + response.status() + ); + let body = response.bytes().await?; + + let partial = destination.with_extension("partial"); + std::fs::write(&partial, &body) + .with_context(|| format!("cannot write {}", partial.display()))?; + std::fs::rename(&partial, &destination)?; + eprintln!("cached at {}", destination.display()); + Ok(destination) +} + +fn hf_token() -> Option { + ["HF_TOKEN", "HUGGING_FACE_HUB_TOKEN"] + .iter() + .find_map(|var| std::env::var(var).ok()) + .filter(|token| !token.is_empty()) +} + +/// Where downloads land: under the HF home when one is set, else a per-user +/// directory in the temp dir. +fn download_cache_dir() -> Result { + let root = std::env::var("HF_HOME") + .map(PathBuf::from) + .unwrap_or_else(|_| std::env::temp_dir()) + .join("sglang-bench"); + std::fs::create_dir_all(&root).with_context(|| format!("cannot create {}", root.display()))?; + Ok(root) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A path that exists is used as given; a directory is searched for the + /// file; a missing file inside a real directory is an error rather than a + /// silent fall through to a Hub download of the directory's name. + #[tokio::test] + async fn local_paths_resolve_before_the_hub() { + let dir = std::env::temp_dir().join(format!("sglang-bench-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let file = dir.join("tokenizer.json"); + std::fs::write(&file, b"{}").unwrap(); + + let http = reqwest::Client::new(); + let from_file = resolve_file( + &http, + file.to_str().unwrap(), + "tokenizer.json", + RepoKind::Model, + ) + .await; + assert_eq!(from_file.unwrap(), file); + + let from_dir = resolve_file( + &http, + dir.to_str().unwrap(), + "tokenizer.json", + RepoKind::Model, + ) + .await; + assert_eq!(from_dir.unwrap(), file); + + let missing = + resolve_file(&http, dir.to_str().unwrap(), "absent.json", RepoKind::Model).await; + assert!(missing.is_err()); + + std::fs::remove_dir_all(&dir).unwrap(); + } + + /// A dataset repo lives under a different Hub path than a model repo; + /// swapping them yields a 404 on every download. + #[test] + fn dataset_and_model_repos_use_different_url_prefixes() { + assert_eq!(RepoKind::Model.url_prefix(), ""); + assert_eq!(RepoKind::Dataset.url_prefix(), "datasets/"); + } + + #[tokio::test] + async fn an_explicit_dataset_path_must_exist() { + let http = reqwest::Client::new(); + assert!( + resolve_sharegpt(&http, "/nonexistent/sharegpt.json") + .await + .is_err() + ); + } +} diff --git a/rust/sglang-bench/src/lib.rs b/rust/sglang-bench/src/lib.rs new file mode 100644 index 0000000000..cf4b09f5e3 --- /dev/null +++ b/rust/sglang-bench/src/lib.rs @@ -0,0 +1,88 @@ +//! The serving benchmark client, ported from +//! `python/sglang/benchmark/serving.py`. +//! +//! Why a port: the Python client drives every request and parses every +//! response stream on one asyncio thread. Past a few thousand concurrent +//! streams that thread, not the server, sets the measured throughput, and the +//! benchmark reports the client's limit. Here each request is a Tokio task, so +//! stream parsing spreads across `--worker-threads` cores, and the one +//! remaining serial step (re-tokenizing the outputs) is batched. +//! +//! The report, the result JSON keys, and the flag names are kept identical, so +//! an existing command line and anything reading the `.jsonl` keep working. +//! See README.md for what this port does not cover. +//! +//! Python loads this as `sglang.srt.rust_extensions._bench`; see +//! `python/sglang/benchmark/rust_client.py`. + +mod args; +use clap::Parser; +mod dataset; +mod hf; +mod metrics; +mod request; +mod run; +use pyo3::prelude::*; + +pub use args::Args; +pub use run::run_blocking; + +/// Run a benchmark described by a command line, as the CLI would. +/// +/// `argv` starts with the program name, so clap's errors and `--help` name +/// the caller correctly. A usage error raises `SystemExit` with clap's own +/// exit code after printing clap's message, which keeps +/// `python -m sglang.benchmark.rust_client --help` behaving like any other +/// command line tool. +#[pyfunction] +fn run_argv(py: Python<'_>, argv: Vec) -> PyResult { + let args = match Args::try_parse_from(&argv) { + Ok(args) => args, + Err(error) => { + // clap renders both help and usage errors; print it the way it + // asks to (help on stdout, errors on stderr). + let _ = error.print(); + return Err(PyErr::new::( + error.exit_code(), + )); + } + }; + run(py, args) +} + +/// Run a benchmark described by a JSON object whose keys are the long flag +/// names without their dashes. Missing keys take the flag's default, so a +/// caller sends only what it means to set. +/// +/// This is the entry point for a Python caller that already parsed its own +/// arguments; `run_argv` is the one for a bare command line. +#[pyfunction] +fn run_config(py: Python<'_>, config: &str) -> PyResult { + let args: Args = serde_json::from_str(config).map_err(|error| { + PyErr::new::(format!( + "invalid benchmark config: {error}" + )) + })?; + run(py, args) +} + +/// The GIL is released for the whole run: it owns a Tokio runtime with its +/// own threads and never touches Python, so holding the GIL would only +/// block the caller's other threads for the length of the benchmark. +fn run(py: Python<'_>, args: Args) -> PyResult { + let result = py + .detach(|| run_blocking(args)) + .map_err(|error| PyErr::new::(format!("{error:#}")))?; + serde_json::to_string(&result).map_err(|error| { + PyErr::new::(format!( + "cannot serialize the benchmark result: {error}" + )) + }) +} + +#[pymodule] +fn _bench(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_argv, module)?)?; + module.add_function(wrap_pyfunction!(run_config, module)?)?; + Ok(()) +} diff --git a/rust/sglang-bench/src/metrics.rs b/rust/sglang-bench/src/metrics.rs new file mode 100644 index 0000000000..2786bb4468 --- /dev/null +++ b/rust/sglang-bench/src/metrics.rs @@ -0,0 +1,670 @@ +//! Metric aggregation and the result report. +//! +//! The statistics follow NumPy, because the Python script's numbers are +//! NumPy's: `percentile` interpolates linearly between the two neighbouring +//! order statistics, and `std` is the population standard deviation (ddof=0). +//! An empty sample reports 0 where the Python script writes `x or 0`, and NaN +//! for the end-to-end latencies, which it does not guard. + +use crate::args::{Args, Backend}; +use crate::request::RequestOutput; + +/// NumPy's `percentile` with the default linear interpolation. `q` is in +/// `[0, 100]`; `values` need not be sorted. +pub fn percentile(values: &[f64], q: f64) -> f64 { + if values.is_empty() { + return 0.0; + } + let mut sorted = values.to_vec(); + sorted.sort_by(f64::total_cmp); + let rank = q / 100.0 * (sorted.len() - 1) as f64; + let lower = rank.floor() as usize; + let upper = rank.ceil() as usize; + if lower == upper { + return sorted[lower]; + } + sorted[lower] + (sorted[upper] - sorted[lower]) * (rank - lower as f64) +} + +pub fn mean(values: &[f64]) -> f64 { + if values.is_empty() { + return 0.0; + } + values.iter().sum::() / values.len() as f64 +} + +/// Population standard deviation, NumPy's default. +pub fn std_dev(values: &[f64]) -> f64 { + if values.is_empty() { + return 0.0; + } + let mean = mean(values); + let variance = values.iter().map(|v| (v - mean).powi(2)).sum::() / values.len() as f64; + variance.sqrt() +} + +pub fn max(values: &[f64]) -> f64 { + values.iter().copied().fold(0.0, f64::max) +} + +/// `np.mean` on an unguarded empty sample: NaN, as the Python script reports +/// for the end-to-end latencies when every request failed. +fn mean_or_nan(values: &[f64]) -> f64 { + if values.is_empty() { + f64::NAN + } else { + mean(values) + } +} + +fn std_or_nan(values: &[f64]) -> f64 { + if values.is_empty() { + f64::NAN + } else { + std_dev(values) + } +} + +fn percentile_or_nan(values: &[f64], q: f64) -> f64 { + if values.is_empty() { + f64::NAN + } else { + percentile(values, q) + } +} + +/// One statistic family, reported as mean / median / std / p90 / p95 / p99. +#[derive(Clone, Copy, Debug, Default)] +pub struct Summary { + pub mean: f64, + pub median: f64, + pub std: f64, + pub p90: f64, + pub p95: f64, + pub p99: f64, +} + +impl Summary { + /// Milliseconds from a sample of seconds, with 0 for an empty sample. + fn from_seconds(values: &[f64]) -> Self { + Self { + mean: mean(values) * 1e3, + median: percentile(values, 50.0) * 1e3, + std: std_dev(values) * 1e3, + p90: percentile(values, 90.0) * 1e3, + p95: percentile(values, 95.0) * 1e3, + p99: percentile(values, 99.0) * 1e3, + } + } + + /// The same, for the unguarded end-to-end family. + fn from_seconds_or_nan(values: &[f64]) -> Self { + Self { + mean: mean_or_nan(values) * 1e3, + median: percentile_or_nan(values, 50.0) * 1e3, + std: std_or_nan(values) * 1e3, + p90: percentile_or_nan(values, 90.0) * 1e3, + p95: percentile_or_nan(values, 95.0) * 1e3, + p99: percentile_or_nan(values, 99.0) * 1e3, + } + } +} + +#[derive(Debug)] +pub struct BenchmarkMetrics { + pub completed: usize, + pub total_input: usize, + pub total_input_text: usize, + pub total_input_vision: usize, + pub total_output: usize, + pub total_output_retokenized: usize, + + pub request_throughput: f64, + pub input_throughput: f64, + pub output_throughput: f64, + pub total_throughput: f64, + + pub ttft: Summary, + pub tpot: Summary, + pub itl: Summary, + pub max_itl_ms: f64, + pub e2e: Summary, + + pub concurrency: f64, + pub max_output_tokens_per_s: f64, + pub max_concurrent_requests: usize, + /// Per-request output lengths in request order, 0 for a failure. + pub output_lens: Vec, +} + +/// Fold the per-request outputs into the reported metrics. `retokenized_lens` +/// is parallel to `outputs` and already 0 for failures; pass all zeros when +/// re-tokenization is disabled. +pub fn calculate( + outputs: &[RequestOutput], + retokenized_lens: &[usize], + duration_s: f64, +) -> BenchmarkMetrics { + let mut output_lens = Vec::with_capacity(outputs.len()); + let (mut total_input, mut total_input_text, mut total_input_vision) = (0, 0, 0); + let mut completed = 0usize; + let mut itls: Vec = Vec::new(); + let mut tpots: Vec = Vec::new(); + let mut ttfts: Vec = Vec::new(); + let mut e2e_latencies: Vec = Vec::new(); + + for output in outputs { + if !output.success { + output_lens.push(0); + continue; + } + output_lens.push(output.output_len); + total_input += output.dataset_prompt_len; + total_input_text += output.dataset_text_prompt_len; + total_input_vision += output.dataset_vision_prompt_len; + if output.output_len > 1 { + tpots.push((output.latency - output.ttft) / (output.output_len - 1) as f64); + } + itls.extend_from_slice(&output.itl); + ttfts.push(output.ttft); + e2e_latencies.push(output.latency); + completed += 1; + } + + let total_output: usize = output_lens.iter().sum(); + let total_output_retokenized: usize = retokenized_lens.iter().sum(); + let (max_output_tokens_per_s, max_concurrent_requests) = peak_windows(outputs); + + BenchmarkMetrics { + completed, + total_input, + total_input_text, + total_input_vision, + total_output, + total_output_retokenized, + request_throughput: completed as f64 / duration_s, + input_throughput: total_input as f64 / duration_s, + output_throughput: total_output as f64 / duration_s, + total_throughput: (total_input + total_output) as f64 / duration_s, + ttft: Summary::from_seconds(&ttfts), + tpot: Summary::from_seconds(&tpots), + itl: Summary::from_seconds(&itls), + max_itl_ms: max(&itls) * 1e3, + e2e: Summary::from_seconds_or_nan(&e2e_latencies), + concurrency: e2e_latencies.iter().sum::() / duration_s, + max_output_tokens_per_s, + max_concurrent_requests, + output_lens, + } +} + +/// Peak output tokens per second and peak concurrent requests, bucketed into +/// one-second windows from the first request's start. Token times are +/// reconstructed from each request's TTFT plus its inter-token latencies, +/// which is how the Python script derives them. +fn peak_windows(outputs: &[RequestOutput]) -> (f64, usize) { + let successful = || outputs.iter().filter(|o| o.success); + let Some(min_start) = successful().map(|o| o.start_time).reduce(f64::min) else { + return (0.0, 0); + }; + let max_end = successful() + .map(|o| o.start_time + o.latency) + .fold(f64::NEG_INFINITY, f64::max); + + let buckets = (max_end - min_start).ceil() as usize + 1; + let mut tokens_per_second = vec![0u64; buckets]; + let mut concurrent_per_second = vec![0u64; buckets]; + + for output in successful() { + let mut token_time = output.start_time + output.ttft; + let mut record = |time: f64| { + let bucket = (time - min_start) as isize; + if bucket >= 0 && (bucket as usize) < buckets { + tokens_per_second[bucket as usize] += 1; + } + }; + record(token_time); + for itl in &output.itl { + token_time += itl; + record(token_time); + } + + let start_second = (output.start_time - min_start) as usize; + let end_second = ((output.start_time + output.latency) - min_start) as usize; + let last = (end_second + 1).min(buckets); + for bucket in &mut concurrent_per_second[start_second.min(last)..last] { + *bucket += 1; + } + } + + ( + tokens_per_second.iter().copied().max().unwrap_or(0) as f64, + concurrent_per_second.iter().copied().max().unwrap_or(0) as usize, + ) +} + +/// The `=== Serving Benchmark Result ===` block, line for line as the Python +/// script prints it: a 40-column label, then the value in 10 columns. +pub fn print_report( + metrics: &BenchmarkMetrics, + args: &Args, + request_rate: f64, + duration_s: f64, + accept_length: Option, +) { + println!("\n{:=^50}", " Serving Benchmark Result "); + println!("{:<40} {:<10}", "Backend:", args.backend.as_str()); + println!("{:<40} {:<10}", "Protocol:", args.protocol.as_str()); + let rate = if request_rate.is_infinite() { + "inf".to_owned() + } else { + format!("{request_rate}") + }; + println!("{:<40} {:<10}", "Traffic request rate:", rate); + println!( + "{:<40} {:<10}", + "Max request concurrency:", + args.max_concurrency + .map_or_else(|| "not set".to_owned(), |limit| limit.to_string()) + ); + println!("{:<40} {:<10}", "Successful requests:", metrics.completed); + println!("{:<40} {:<10.2}", "Benchmark duration (s):", duration_s); + println!("{:<40} {:<10}", "Total input tokens:", metrics.total_input); + println!( + "{:<40} {:<10}", + "Total input text tokens:", metrics.total_input_text + ); + println!( + "{:<40} {:<10}", + "Total generated tokens:", metrics.total_output + ); + println!( + "{:<40} {:<10}", + "Total generated tokens (retokenized):", metrics.total_output_retokenized + ); + println!( + "{:<40} {:<10.2}", + "Request throughput (req/s):", metrics.request_throughput + ); + println!( + "{:<40} {:<10.2}", + "Input token throughput (tok/s):", metrics.input_throughput + ); + println!( + "{:<40} {:<10.2}", + "Output token throughput (tok/s):", metrics.output_throughput + ); + println!( + "{:<40} {:<10.2}", + "Peak output token throughput (tok/s):", metrics.max_output_tokens_per_s + ); + println!( + "{:<40} {:<10}", + "Peak concurrent requests:", metrics.max_concurrent_requests + ); + println!( + "{:<40} {:<10.2}", + "Total token throughput (tok/s):", metrics.total_throughput + ); + println!("{:<40} {:<10.2}", "Concurrency:", metrics.concurrency); + if let Some(accept_length) = accept_length { + println!("{:<40} {:<10.2}", "Accept length:", accept_length); + } + + print_family("End-to-End Latency", "E2E Latency", &metrics.e2e); + print_family("Time to First Token", "TTFT", &metrics.ttft); + print_family( + "Time per Output Token (excl. 1st token)", + "TPOT", + &metrics.tpot, + ); + println!("{:-^50}", "Inter-Token Latency"); + println!("{:<40} {:<10.2}", "Mean ITL (ms):", metrics.itl.mean); + println!("{:<40} {:<10.2}", "Median ITL (ms):", metrics.itl.median); + println!("{:<40} {:<10.2}", "P90 ITL (ms):", metrics.itl.p90); + println!("{:<40} {:<10.2}", "P95 ITL (ms):", metrics.itl.p95); + println!("{:<40} {:<10.2}", "P99 ITL (ms):", metrics.itl.p99); + println!("{:<40} {:<10.2}", "Max ITL (ms):", metrics.max_itl_ms); + println!("{}", "=".repeat(50)); +} + +fn print_family(heading: &str, label: &str, summary: &Summary) { + println!("{heading:-^50}"); + println!( + "{:<40} {:<10.2}", + format!("Mean {label} (ms):"), + summary.mean + ); + println!( + "{:<40} {:<10.2}", + format!("Median {label} (ms):"), + summary.median + ); + println!("{:<40} {:<10.2}", format!("P90 {label} (ms):"), summary.p90); + println!("{:<40} {:<10.2}", format!("P95 {label} (ms):"), summary.p95); + println!("{:<40} {:<10.2}", format!("P99 {label} (ms):"), summary.p99); +} + +/// The default result filename, as the Python script derives it: +/// `____.jsonl` for a random dataset, +/// `___.jsonl` otherwise. +pub fn default_output_file(args: &Args, now: &str) -> String { + let backend = args.backend.as_str(); + if args.dataset_name.is_random() { + format!( + "{backend}_{now}_{}_{}_{}.jsonl", + args.num_prompts, args.random_input_len, args.random_output_len + ) + } else { + format!( + "{backend}_{now}_{}_{}.jsonl", + args.num_prompts, + args.dataset_name.as_str() + ) + } +} + +/// The JSON result line, with the Python script's keys so existing readers of +/// the `.jsonl` keep working. +pub fn result_json( + metrics: &BenchmarkMetrics, + args: &Args, + request_rate: f64, + duration_s: f64, + accept_length: Option, + server_info: Option, + outputs: &[RequestOutput], +) -> serde_json::Value { + let rate = if request_rate.is_infinite() { + serde_json::Value::String("inf".into()) + } else { + serde_json::json!(request_rate) + }; + // Built by insertion rather than one `json!` literal: fifty keys in one + // macro invocation exceeds the expander's recursion limit. + let mut object = serde_json::Map::new(); + // Scoped so the borrow ends before the map is moved into the result. + { + let mut put = |key: &str, value: serde_json::Value| { + object.insert(key.to_owned(), value); + }; + // Arguments. + put("tag", serde_json::json!(args.tag)); + put("backend", serde_json::json!(args.backend.as_str())); + // The one key the Python client does not write: an HTTP and a gRPC run of + // the same backend land in the same default file and must stay apart. + put("protocol", serde_json::json!(args.protocol.as_str())); + put( + "dataset_name", + serde_json::json!(args.dataset_name.as_str()), + ); + put("request_rate", rate); + put("max_concurrency", serde_json::json!(args.max_concurrency)); + put( + "sharegpt_output_len", + serde_json::json!(args.sharegpt_output_len), + ); + put("random_input_len", serde_json::json!(args.random_input_len)); + put( + "random_output_len", + serde_json::json!(args.random_output_len), + ); + put( + "random_range_ratio", + serde_json::json!(args.random_range_ratio), + ); + // Information. + put("server_info", serde_json::json!(server_info)); + // Results. + put("duration", serde_json::json!(duration_s)); + put("completed", serde_json::json!(metrics.completed)); + put("total_input_tokens", serde_json::json!(metrics.total_input)); + put( + "total_input_text_tokens", + serde_json::json!(metrics.total_input_text), + ); + put( + "total_input_vision_tokens", + serde_json::json!(metrics.total_input_vision), + ); + put( + "total_output_tokens", + serde_json::json!(metrics.total_output), + ); + put( + "total_output_tokens_retokenized", + serde_json::json!(metrics.total_output_retokenized), + ); + put( + "request_throughput", + serde_json::json!(metrics.request_throughput), + ); + put( + "input_throughput", + serde_json::json!(metrics.input_throughput), + ); + put( + "output_throughput", + serde_json::json!(metrics.output_throughput), + ); + put( + "total_throughput", + serde_json::json!(metrics.total_throughput), + ); + for (prefix, summary) in [ + ("e2e_latency", &metrics.e2e), + ("ttft", &metrics.ttft), + ("tpot", &metrics.tpot), + ("itl", &metrics.itl), + ] { + for (stat, value) in [ + ("mean", summary.mean), + ("median", summary.median), + ("std", summary.std), + ("p90", summary.p90), + ("p95", summary.p95), + ("p99", summary.p99), + ] { + // NaN has no JSON form; the Python script writes it as a bare + // `NaN` token, which strict readers reject, so it becomes null. + let value = if value.is_finite() { + serde_json::json!(value) + } else { + serde_json::Value::Null + }; + put(&format!("{stat}_{prefix}_ms"), value); + } + } + put("concurrency", serde_json::json!(metrics.concurrency)); + put("accept_length", serde_json::json!(accept_length)); + put( + "max_output_tokens_per_s", + serde_json::json!(metrics.max_output_tokens_per_s), + ); + put( + "max_concurrent_requests", + serde_json::json!(metrics.max_concurrent_requests), + ); + } + let mut result = serde_json::Value::Object(object); + + if args.output_details { + let details = serde_json::json!({ + "input_lens": outputs.iter().map(|o| o.prompt_len).collect::>(), + "output_lens": metrics.output_lens, + "ttfts": outputs.iter().map(|o| o.ttft).collect::>(), + "itls": outputs.iter().map(|o| o.itl.clone()).collect::>(), + "generated_texts": outputs.iter().map(|o| o.generated_text.clone()).collect::>(), + "errors": outputs.iter().map(|o| o.error.clone()).collect::>(), + }); + let (Some(result), Some(details)) = (result.as_object_mut(), details.as_object()) else { + unreachable!("both are JSON objects"); + }; + for (key, value) in details { + result.insert(key.clone(), value.clone()); + } + } + result +} + +/// `avg_spec_accept_length` out of a `/server_info` body, following the PD +/// shape where the decode node's info is nested under `decode[0]`. +pub fn accept_length_from_server_info(info: &serde_json::Value) -> Option { + let node = info + .get("decode") + .and_then(|decode| decode.get(0)) + .unwrap_or(info); + node.get("internal_states")? + .get(0)? + .get("avg_spec_accept_length")? + .as_f64() +} + +/// Whether this backend reports spec-decoding and cache stats on +/// `/server_info` at all. +pub fn reports_server_info(backend: Backend) -> bool { + backend.is_sglang() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn output(ttft: f64, itl: Vec, latency: f64, output_len: usize) -> RequestOutput { + RequestOutput { + success: true, + ttft, + itl, + latency, + output_len, + prompt_len: 10, + dataset_prompt_len: 10, + dataset_text_prompt_len: 10, + start_time: 0.0, + ..Default::default() + } + } + + /// NumPy's linear interpolation, pinned against values computed with + /// `np.percentile`: the Python script's reported percentiles are these. + #[test] + fn percentile_matches_numpy_linear_interpolation() { + let values = [1.0, 2.0, 3.0, 4.0]; + assert_eq!(percentile(&values, 0.0), 1.0); + assert_eq!(percentile(&values, 50.0), 2.5); + assert_eq!(percentile(&values, 100.0), 4.0); + // rank = 0.9 * 3 = 2.7 -> 3 + (4-3)*0.7 + assert!((percentile(&values, 90.0) - 3.7).abs() < 1e-12); + // Unsorted input must give the same answer. + assert_eq!(percentile(&[4.0, 1.0, 3.0, 2.0], 50.0), 2.5); + // A single sample is every percentile. + assert_eq!(percentile(&[7.0], 99.0), 7.0); + // `np.percentile(x or 0, q)` on an empty sample is 0. + assert_eq!(percentile(&[], 99.0), 0.0); + } + + /// `np.std` defaults to the population deviation (ddof=0), not the sample + /// one; reporting the sample deviation would inflate every std by + /// `sqrt(n/(n-1))`. + #[test] + fn std_dev_is_the_population_deviation() { + assert!((std_dev(&[1.0, 2.0, 3.0, 4.0]) - 1.118_033_988_749_895).abs() < 1e-12); + assert_eq!(std_dev(&[5.0]), 0.0); + assert_eq!(std_dev(&[]), 0.0); + } + + /// A request contributes one TPOT sample only when it produced more than + /// one token, and ITLs pool across requests. + #[test] + fn metrics_fold_per_request_samples() { + let outputs = vec![ + output(0.1, vec![0.01, 0.01], 0.12, 3), + output(0.2, vec![0.02], 0.22, 2), + // One token: no TPOT sample, since there is no decode interval. + output(0.3, vec![], 0.3, 1), + RequestOutput { + success: false, + output_len: 99, + ..Default::default() + }, + ]; + let metrics = calculate(&outputs, &[3, 2, 1, 0], 2.0); + + assert_eq!(metrics.completed, 3); + // A failed request contributes 0, not its requested length. + assert_eq!(metrics.output_lens, vec![3, 2, 1, 0]); + assert_eq!(metrics.total_output, 6); + assert_eq!(metrics.total_input, 30); + assert_eq!(metrics.request_throughput, 1.5); + assert_eq!(metrics.output_throughput, 3.0); + // TPOT from the two multi-token requests only. + assert!((metrics.tpot.mean - 15.0).abs() < 1e-9); + // ITLs pooled: 10, 10, 20 ms. + assert!((metrics.itl.mean - 40.0 / 3.0).abs() < 1e-9); + assert!((metrics.max_itl_ms - 20.0).abs() < 1e-9); + assert!((metrics.concurrency - 0.32) < 1e-9); + } + + /// Every request failing leaves the end-to-end family NaN (as NumPy's + /// unguarded `mean([])` does) while the guarded families report 0, so the + /// report distinguishes "no data" from "zero latency". + #[test] + fn all_failures_report_nan_e2e_and_zero_elsewhere() { + let outputs = vec![RequestOutput { + success: false, + ..Default::default() + }]; + let metrics = calculate(&outputs, &[0], 1.0); + assert_eq!(metrics.completed, 0); + assert!(metrics.e2e.mean.is_nan()); + assert!(metrics.e2e.p99.is_nan()); + assert_eq!(metrics.ttft.mean, 0.0); + assert_eq!(metrics.max_itl_ms, 0.0); + assert_eq!(metrics.max_concurrent_requests, 0); + } + + /// The peak windows bucket token arrivals by whole second from the first + /// start, so two requests overlapping in second 0 peak at 2. + #[test] + fn peak_windows_bucket_by_second() { + let outputs = vec![ + RequestOutput { + success: true, + start_time: 10.0, + ttft: 0.1, + itl: vec![0.1, 0.1], + latency: 0.3, + output_len: 3, + ..Default::default() + }, + RequestOutput { + success: true, + start_time: 10.5, + ttft: 0.1, + itl: vec![1.0], + latency: 1.1, + output_len: 2, + ..Default::default() + }, + ]; + let (peak_tokens, peak_requests) = peak_windows(&outputs); + // Second 0 holds 3 tokens from the first request plus 1 from the second. + assert_eq!(peak_tokens, 4.0); + assert_eq!(peak_requests, 2); + } + + #[test] + fn accept_length_reads_plain_and_pd_shapes() { + let plain = serde_json::json!({"internal_states": [{"avg_spec_accept_length": 2.5}]}); + assert_eq!(accept_length_from_server_info(&plain), Some(2.5)); + let pd = serde_json::json!({ + "decode": [{"internal_states": [{"avg_spec_accept_length": 3.5}]}] + }); + assert_eq!(accept_length_from_server_info(&pd), Some(3.5)); + assert_eq!( + accept_length_from_server_info(&serde_json::json!({"internal_states": []})), + None + ); + assert_eq!(accept_length_from_server_info(&serde_json::json!({})), None); + } +} diff --git a/rust/sglang-bench/src/request.rs b/rust/sglang-bench/src/request.rs new file mode 100644 index 0000000000..05adea9cc7 --- /dev/null +++ b/rust/sglang-bench/src/request.rs @@ -0,0 +1,925 @@ +//! One request's lifetime: build the body, post it, and read the streamed +//! response into per-token timings. +//! +//! Each request is an independent task, so the response streams are parsed on +//! whichever Tokio worker thread polls them. That is the one structural +//! difference from the Python script, where a single asyncio thread parses +//! every stream and becomes the bottleneck well before the server does. +//! +//! The three wire protocols are read exactly as their Python counterparts in +//! `serving.py` do, including where each one takes its timestamps and how it +//! attributes inter-token latency. + +use std::time::Instant; + +use bytes::BytesMut; +use futures::StreamExt; +use reqwest::header::HeaderMap; +use serde_json::{Map, Value}; +use sglang_api_types::api::v1 as api; +use sglang_api_types::api::v1::sglang_service_client::SglangServiceClient; +use tonic::transport::Channel; + +use crate::args::Api; +use crate::dataset::{DatasetRow, Prompt}; + +/// The api.v1 client over one shared channel. Cloning shares the connection. +pub type GrpcClient = SglangServiceClient; + +/// Read buffer growth hint. The Python client raises aiohttp's read buffer to +/// 10 MB because cumulative `/generate` frames outgrow the 64 KB default under +/// load; reqwest hands us whatever the socket yields, so the only sizing +/// decision left is this crate's line buffer. +const LINE_BUFFER_HINT: usize = 64 * 1024; + +#[derive(Clone, Debug, Default)] +pub struct RequestOutput { + pub generated_text: String, + pub success: bool, + /// Seconds from the request's first byte out to its last chunk in. + pub latency: f64, + pub ttft: f64, + /// One entry per generated token after the first. + pub itl: Vec, + /// Prompt length as the server reported it, else the dataset row's. + pub prompt_len: usize, + pub error: String, + pub output_len: usize, + /// Request start, in seconds from this process's time base. The peak + /// windows in `metrics` place every request on this one timeline. + pub start_time: f64, + /// The dataset row's lengths. The input totals sum these, so one row + /// counts once even where the server reports its own prompt length. + pub dataset_prompt_len: usize, + pub dataset_text_prompt_len: usize, + pub dataset_vision_prompt_len: usize, +} + +/// Everything about a request body that does not vary per prompt. +#[derive(Clone, Debug)] +pub struct RequestTemplate { + pub api: Api, + pub api_url: String, + pub model: String, + pub stream: bool, + pub temperature: f64, + pub top_p: f64, + pub ignore_eos: bool, + pub return_logprob: bool, + pub top_logprobs_num: u32, + pub logprob_start_len: i64, + pub extra_body: Map, + pub headers: HeaderMap, + /// Set for the gRPC backend; the HTTP backends leave it `None`. + pub grpc: Option, +} + +impl RequestTemplate { + /// The JSON body for one row, with `--extra-request-body` merged last so a + /// caller can override any default this builds. + fn body(&self, row: &DatasetRow, output_len: usize) -> Value { + // gRPC carries the same schema the native route decodes, so the body + // is built once and the gRPC path deserializes it into the generated + // request type. + let mut body = match self.api { + Api::SglangGenerate => { + let mut sampling = serde_json::json!({ + "temperature": self.temperature, + "max_new_tokens": output_len, + "ignore_eos": self.ignore_eos, + }); + if self.top_p < 1.0 { + sampling["top_p"] = serde_json::json!(self.top_p); + } + let mut body = serde_json::json!({ + "sampling_params": sampling, + "stream": self.stream, + "return_logprob": self.return_logprob, + "logprob_start_len": self.logprob_start_len, + }); + match &row.prompt { + Prompt::Text(text) => body["text"] = serde_json::json!(text), + Prompt::TokenIds(ids) => body["input_ids"] = serde_json::json!(ids), + } + if self.top_logprobs_num > 0 { + body["top_logprobs_num"] = serde_json::json!(self.top_logprobs_num); + } + body + } + Api::OpenaiCompletions => { + let prompt = match &row.prompt { + Prompt::Text(text) => serde_json::json!(text), + Prompt::TokenIds(ids) => serde_json::json!(ids), + }; + let mut body = serde_json::json!({ + "model": self.model, + "prompt": prompt, + "best_of": 1, + "max_tokens": output_len, + "stream": self.stream, + }); + if !self.extra_body.contains_key("temperature") { + body["temperature"] = serde_json::json!(0.0); + } + if !self.extra_body.contains_key("ignore_eos") { + body["ignore_eos"] = serde_json::json!(self.ignore_eos); + } + if self.return_logprob && self.top_logprobs_num > 0 { + body["logprobs"] = serde_json::json!(self.top_logprobs_num); + } + body + } + Api::OpenaiChat => { + let Prompt::Text(text) = &row.prompt else { + unreachable!("--tokenize-prompt is rejected for chat backends"); + }; + let mut body = serde_json::json!({ + "model": self.model, + "messages": [{"role": "user", "content": text}], + "max_completion_tokens": output_len, + "stream": self.stream, + }); + if !self.extra_body.contains_key("temperature") { + body["temperature"] = serde_json::json!(0.0); + } + if !self.extra_body.contains_key("ignore_eos") { + body["ignore_eos"] = serde_json::json!(self.ignore_eos); + } + body + } + }; + + let Some(object) = body.as_object_mut() else { + unreachable!("every branch builds a JSON object"); + }; + for (key, value) in &self.extra_body { + object.insert(key.clone(), value.clone()); + } + body + } +} + +/// Send one request and read its response to completion. +/// +/// Never returns `Err`: a transport failure, a non-200 status, or a malformed +/// stream is recorded on the output as the Python script records it, so one +/// bad request does not end the run. +pub async fn send( + http: &reqwest::Client, + template: &RequestTemplate, + row: &DatasetRow, + base: Instant, +) -> RequestOutput { + let mut output = RequestOutput { + prompt_len: row.prompt_len, + output_len: row.output_len, + dataset_prompt_len: row.prompt_len, + dataset_text_prompt_len: row.text_prompt_len, + dataset_vision_prompt_len: row.vision_prompt_len, + ..Default::default() + }; + + let body = template.body(row, row.output_len); + if let Some(client) = &template.grpc { + send_grpc(client.clone(), body, base, &mut output).await; + return output; + } + let start = Instant::now(); + output.start_time = start.duration_since(base).as_secs_f64(); + + let response = match http + .post(&template.api_url) + .headers(template.headers.clone()) + .json(&body) + .send() + .await + { + Ok(response) => response, + Err(error) => { + output.error = error.to_string(); + return output; + } + }; + + let status = response.status(); + if !status.is_success() { + let reason = status.canonical_reason().unwrap_or(""); + let text = response.text().await.unwrap_or_default(); + output.error = format!("{reason}: {text}"); + return output; + } + + // A non-streaming chat reply is one JSON object, not a stream of deltas, + // so it is read whole; the other protocols' non-streaming replies are a + // single frame that the streaming parser already handles. + if !template.stream && template.api == Api::OpenaiChat { + match response.json::().await { + Ok(data) => { + finish_non_streaming_chat(&data, start.elapsed().as_secs_f64(), &mut output) + } + Err(error) => output.error = format!("response was not JSON: {error}"), + } + return output; + } + + match read_stream(response, template.api, start, &mut output).await { + Ok(()) => output.success = true, + Err(error) => { + output.success = false; + output.error = error; + } + } + output +} + +/// One request over the api.v1 gRPC service. Each streamed frame is the typed +/// twin of a `/generate` SSE frame, so it is handed to the same accounting +/// through its JSON form; the generated types serialize to exactly the HTTP +/// shape, which is what keeps the two backends' metrics comparable. +async fn send_grpc(mut client: GrpcClient, body: Value, base: Instant, output: &mut RequestOutput) { + let request: api::GenerateRequest = match serde_json::from_value(body) { + Ok(request) => request, + Err(error) => { + output.error = format!("request does not fit the api.v1 schema: {error}"); + return; + } + }; + let start = Instant::now(); + output.start_time = start.duration_since(base).as_secs_f64(); + + let mut stream = match client.generate(request).await { + Ok(response) => response.into_inner(), + Err(status) => { + output.error = format!("{:?}: {}", status.code(), status.message()); + return; + } + }; + let mut state = StreamState::new(Api::SglangGenerate, output.output_len); + loop { + match stream.message().await { + Ok(Some(api::GenerateStreamItem { + item: Some(api::generate_stream_item::Item::Frame(frame)), + })) => match serde_json::to_value(&frame) { + Ok(data) => state.feed_value(&data, start, output), + Err(error) => { + output.error = format!("frame does not serialize: {error}"); + return; + } + }, + Ok(Some(api::GenerateStreamItem { + item: Some(api::generate_stream_item::Item::Error(error)), + })) => { + let body = error.error.unwrap_or_default(); + output.error = format!("{}: {}", body.code, body.message); + return; + } + // An item with neither arm is a protocol violation; skip it as + // the SSE parser skips a frame it cannot read. + Ok(Some(_)) => {} + Ok(None) => break, + Err(status) => { + output.error = format!("stream failed: {:?}: {}", status.code(), status.message()); + return; + } + } + } + match state.finish(output) { + Ok(()) => output.success = true, + Err(error) => output.error = error, + } +} + +/// Consume the response body line by line, feeding each line to the +/// protocol's parser. aiohttp's `async for chunk in response.content` yields +/// newline-delimited pieces, which is what the Python parsers assume, so the +/// framing is reproduced here rather than parsing raw socket chunks. +async fn read_stream( + response: reqwest::Response, + api: Api, + start: Instant, + output: &mut RequestOutput, +) -> Result<(), String> { + let mut state = StreamState::new(api, output.output_len); + let mut body = response.bytes_stream(); + let mut buffer = BytesMut::with_capacity(LINE_BUFFER_HINT); + // Where the newline search left off. Without it a long cumulative frame + // reassembled from many socket reads would be rescanned from the start on + // every read, which is quadratic in the frame size. + let mut scanned = 0usize; + + while let Some(chunk) = body.next().await { + let chunk = chunk.map_err(|error| format!("stream read failed: {error}"))?; + buffer.extend_from_slice(&chunk); + while let Some(offset) = buffer[scanned..].iter().position(|&byte| byte == b'\n') { + let end = scanned + offset; + let line = buffer.split_to(end + 1); + scanned = 0; + state.feed(&line[..end], start, output); + } + scanned = buffer.len(); + } + if !buffer.is_empty() { + state.feed(&buffer, start, output); + } + + state.finish(output) +} + +/// The per-request parse state, one arm per wire protocol. +struct StreamState { + api: Api, + /// Latency of the most recent chunk, which becomes the request's latency. + /// `None` means nothing arrived, which the Python script surfaces as a + /// failure (its `latency` local stays unbound). + latency: Option, + /// Instant of the last counted token, for the next inter-token latency. + most_recent: Instant, + /// Tokens counted so far, for the `/generate` cumulative frames. + last_output_len: usize, + /// Falls back to the requested length until the server reports one. + output_len: usize, + generated_text: String, + ttft: Option, +} + +impl StreamState { + fn new(api: Api, requested_output_len: usize) -> Self { + Self { + api, + latency: None, + most_recent: Instant::now(), + last_output_len: 0, + output_len: requested_output_len, + generated_text: String::new(), + ttft: None, + } + } + + fn feed(&mut self, line: &[u8], start: Instant, output: &mut RequestOutput) { + let line = trim(line); + if line.is_empty() { + return; + } + let payload = line.strip_prefix(b"data: ").unwrap_or(line); + if payload == b"[DONE]" { + self.latency = Some(start.elapsed().as_secs_f64()); + return; + } + let Ok(data) = serde_json::from_slice::(payload) else { + // A frame this client cannot read is not the server's failure to + // report; skip it as the Python parsers' `json.loads` would raise + // and end the request. + self.latency = Some(start.elapsed().as_secs_f64()); + return; + }; + self.feed_value(&data, start, output); + } + + /// One decoded frame, from either transport. + fn feed_value(&mut self, data: &Value, start: Instant, output: &mut RequestOutput) { + self.latency = Some(start.elapsed().as_secs_f64()); + record_server_prompt_len(data, output); + match self.api { + Api::SglangGenerate => self.feed_generate(data, start, output), + Api::OpenaiCompletions => self.feed_completions(data, start, output), + Api::OpenaiChat => self.feed_chat(data, start, output), + } + } + + /// `/generate`: every frame carries the running token count, so a frame's + /// inter-token latency is spread evenly over the tokens it added. + /// + /// Progress is read from the count, not the text: a server launched with + /// `--skip-tokenizer-init` returns `output_ids` and an empty `text`, and a + /// token is a token whether or not it was decoded. The count comes from + /// `meta_info.completion_tokens` (cumulative on every server and mode), + /// else from the `output_ids` the frame carried, else, for a server that + /// reports neither, the first non-empty text marks the first token and + /// no later frame can be attributed. + fn feed_generate(&mut self, data: &Value, start: Instant, output: &mut RequestOutput) { + let text = data.get("text").and_then(Value::as_str).unwrap_or_default(); + let completion_tokens = data + .get("meta_info") + .and_then(|meta| meta.get("completion_tokens")) + .and_then(Value::as_u64) + .map(|tokens| tokens as usize); + let ids_carried = data + .get("output_ids") + .and_then(Value::as_array) + .map(Vec::len) + .unwrap_or(0); + let reached = match completion_tokens { + Some(count) => Some(count), + None if ids_carried > 0 => Some(self.last_output_len + ids_carried), + None => None, + }; + let progressed = match reached { + Some(count) => count > self.last_output_len, + None => !text.is_empty() && self.ttft.is_none(), + }; + + if !text.is_empty() { + // Cumulative text: the frame holds everything generated so far. + self.generated_text.clear(); + self.generated_text.push_str(text); + } + if !progressed { + return; + } + let timestamp = Instant::now(); + if let Some(count) = reached { + self.output_len = count; + } + if self.ttft.is_none() { + self.ttft = Some(timestamp.duration_since(start).as_secs_f64()); + } else if let Some(count) = reached { + let new_tokens = count - self.last_output_len; + let gap = timestamp.duration_since(self.most_recent).as_secs_f64(); + output + .itl + .extend(std::iter::repeat_n(gap / new_tokens as f64, new_tokens)); + } + self.most_recent = timestamp; + if let Some(count) = reached { + self.last_output_len = count; + } + } + + /// `/v1/completions`: each frame carries one delta, so each frame after + /// the first is one inter-token latency. + fn feed_completions(&mut self, data: &Value, start: Instant, output: &mut RequestOutput) { + let text = data + .get("choices") + .and_then(|choices| choices.get(0)) + .and_then(|choice| choice.get("text")) + .and_then(Value::as_str) + .unwrap_or_default(); + if text.is_empty() { + return; + } + self.count_delta(text, start, output); + if let Some(tokens) = completion_tokens(data) { + self.output_len = tokens; + } + } + + /// `/v1/chat/completions`: as completions, except the usage block may + /// arrive on a frame with no choices, and reasoning models stream their + /// thoughts in a sibling field that counts as content. + fn feed_chat(&mut self, data: &Value, start: Instant, output: &mut RequestOutput) { + if let Some(tokens) = completion_tokens(data) { + self.output_len = tokens; + } + let Some(delta) = data + .get("choices") + .and_then(|choices| choices.get(0)) + .and_then(|choice| choice.get("delta")) + else { + return; + }; + let content = chat_content(delta); + if content.is_empty() { + return; + } + self.count_delta(&content, start, output); + } + + /// The shared delta accounting for the two OpenAI protocols. + fn count_delta(&mut self, text: &str, start: Instant, output: &mut RequestOutput) { + let timestamp = Instant::now(); + if self.ttft.is_none() { + self.ttft = Some(timestamp.duration_since(start).as_secs_f64()); + } else { + output + .itl + .push(timestamp.duration_since(self.most_recent).as_secs_f64()); + } + self.most_recent = timestamp; + self.generated_text.push_str(text); + } + + /// Commit the parse to the output, or report why the response was unusable. + fn finish(self, output: &mut RequestOutput) -> Result<(), String> { + let Some(latency) = self.latency else { + return Err("the response body was empty".to_owned()); + }; + output.generated_text = self.generated_text; + output.latency = latency; + output.output_len = self.output_len; + // A 200 that produced no token is still a failed request: with no TTFT + // there is nothing to report, and counting it would dilute every + // latency percentile with a zero. + match self.ttft { + Some(ttft) => { + output.ttft = ttft; + Ok(()) + } + None => Err("the response contained no generated tokens".to_owned()), + } + } +} + +/// A non-streaming chat reply, whose content sits under `message` rather than +/// arriving as `delta` frames. TTFT is the whole latency, as in Python. +fn finish_non_streaming_chat(data: &Value, latency: f64, output: &mut RequestOutput) { + record_server_prompt_len(data, output); + let message = data + .get("choices") + .and_then(|choices| choices.get(0)) + .and_then(|choice| choice.get("message")); + output.generated_text = message.map(chat_content).unwrap_or_default(); + output.latency = latency; + output.ttft = latency; + if let Some(tokens) = completion_tokens(data) { + output.output_len = tokens; + } + output.success = true; +} + +/// Reasoning models stream their thoughts in `reasoning_content`, and vLLM's +/// Kimi parser uses `reasoning`. Prefer the standard spelling so a server +/// exposing both aliases does not count the same tokens twice. +fn chat_content(message: &Value) -> String { + let reasoning = message + .get("reasoning_content") + .and_then(Value::as_str) + .or_else(|| message.get("reasoning").and_then(Value::as_str)) + .unwrap_or_default(); + let content = message + .get("content") + .and_then(Value::as_str) + .unwrap_or_default(); + let mut combined = String::with_capacity(reasoning.len() + content.len()); + combined.push_str(reasoning); + combined.push_str(content); + combined +} + +fn completion_tokens(data: &Value) -> Option { + data.get("usage")? + .get("completion_tokens")? + .as_u64() + .map(|tokens| tokens as usize) +} + +/// Take the prompt length from the server, the only side that knows it: +/// `usage.prompt_tokens` on the OpenAI routes, `meta_info.prompt_tokens` on +/// the native one. A server that reports neither leaves the row's value. +fn record_server_prompt_len(data: &Value, output: &mut RequestOutput) { + let reported = data.get("usage").or_else(|| data.get("meta_info")); + let Some(tokens) = reported + .and_then(|reported| reported.get("prompt_tokens")) + .and_then(Value::as_u64) + else { + return; + }; + if tokens > 0 { + output.prompt_len = tokens as usize; + } +} + +fn trim(line: &[u8]) -> &[u8] { + let start = line + .iter() + .position(|byte| !byte.is_ascii_whitespace()) + .unwrap_or(line.len()); + let end = line + .iter() + .rposition(|byte| !byte.is_ascii_whitespace()) + .map_or(start, |index| index + 1); + &line[start..end] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::dataset::DatasetRow; + + fn template(api: Api) -> RequestTemplate { + RequestTemplate { + api, + api_url: "http://localhost/x".into(), + model: "m".into(), + stream: true, + temperature: 0.0, + top_p: 1.0, + ignore_eos: true, + return_logprob: false, + top_logprobs_num: 0, + logprob_start_len: -1, + extra_body: Map::new(), + headers: HeaderMap::new(), + grpc: None, + } + } + + fn row(prompt: Prompt) -> DatasetRow { + DatasetRow { + prompt, + prompt_len: 7, + output_len: 16, + text_prompt_len: 7, + vision_prompt_len: 0, + } + } + + fn feed_all(api: Api, lines: &[&str]) -> RequestOutput { + let mut output = RequestOutput { + output_len: 16, + ..Default::default() + }; + let start = Instant::now(); + let mut state = StreamState::new(api, 16); + for line in lines { + state.feed(line.as_bytes(), start, &mut output); + } + let result = state.finish(&mut output); + output.success = result.is_ok(); + if let Err(error) = result { + output.error = error; + } + output + } + + /// `/generate` sends `text` and `sampling_params`, and the token-id form + /// switches the key rather than the value's type. + #[test] + fn generate_body_matches_the_native_route() { + let template = template(Api::SglangGenerate); + let body = template.body(&row(Prompt::Text("hi".into())), 8); + assert_eq!(body["text"], "hi"); + assert_eq!(body["sampling_params"]["max_new_tokens"], 8); + assert_eq!(body["sampling_params"]["ignore_eos"], true); + assert_eq!(body["stream"], true); + // top_p is only sent when it narrows the distribution. + assert!(body["sampling_params"].get("top_p").is_none()); + + let ids = template.body(&row(Prompt::TokenIds(vec![1, 2])), 8); + assert_eq!(ids["input_ids"], serde_json::json!([1, 2])); + assert!(ids.get("text").is_none()); + } + + /// `--extra-request-body` is merged last, so it overrides a default this + /// crate chose, and suppresses the OpenAI temperature/ignore_eos defaults. + #[test] + fn extra_body_overrides_every_default() { + let mut template = template(Api::OpenaiCompletions); + template.extra_body = + serde_json::from_str(r#"{"temperature": 0.9, "max_tokens": 3, "ignore_eos": false}"#) + .unwrap(); + let body = template.body(&row(Prompt::Text("hi".into())), 8); + assert_eq!(body["temperature"], 0.9); + assert_eq!(body["max_tokens"], 3); + assert_eq!(body["ignore_eos"], false); + } + + #[test] + fn chat_body_wraps_the_prompt_in_one_user_message() { + let body = template(Api::OpenaiChat).body(&row(Prompt::Text("hi".into())), 8); + assert_eq!(body["messages"][0]["role"], "user"); + assert_eq!(body["messages"][0]["content"], "hi"); + assert_eq!(body["max_completion_tokens"], 8); + } + + /// The native route's frames are cumulative, so a frame that advances the + /// token count by k contributes k inter-token latencies, each the frame's + /// gap divided by k. Reporting one latency per frame instead would + /// understate ITL on a server that batches tokens into a frame. + #[test] + fn generate_spreads_a_frame_gap_over_the_tokens_it_added() { + let output = feed_all( + Api::SglangGenerate, + &[ + r#"data: {"text":"a","meta_info":{"completion_tokens":1,"prompt_tokens":5}}"#, + r#"data: {"text":"abcd","meta_info":{"completion_tokens":4}}"#, + "data: [DONE]", + ], + ); + assert!(output.success, "{}", output.error); + assert_eq!(output.generated_text, "abcd"); + assert_eq!(output.output_len, 4); + // Three tokens arrived on the second frame, so three equal ITLs. + assert_eq!(output.itl.len(), 3); + assert!((output.itl[0] - output.itl[2]).abs() < 1e-12); + // The server's prompt length wins over the row's. + assert_eq!(output.prompt_len, 5); + assert!(output.ttft > 0.0); + } + + /// A frame that repeats the token count adds no latency sample. + #[test] + fn generate_ignores_a_frame_that_added_no_token() { + let output = feed_all( + Api::SglangGenerate, + &[ + r#"data: {"text":"a","meta_info":{"completion_tokens":1}}"#, + r#"data: {"text":"a","meta_info":{"completion_tokens":1}}"#, + r#"data: {"text":"ab","meta_info":{"completion_tokens":2}}"#, + ], + ); + assert_eq!(output.itl.len(), 1); + assert_eq!(output.output_len, 2); + } + + /// The OpenAI routes send deltas, so the text concatenates and each frame + /// after the first is exactly one inter-token latency. + #[test] + fn completions_concatenates_deltas() { + let output = feed_all( + Api::OpenaiCompletions, + &[ + r#"data: {"choices":[{"text":"He"}],"usage":{"prompt_tokens":4}}"#, + r#"data: {"choices":[{"text":"llo"}]}"#, + r#"data: {"choices":[{"text":""}],"usage":{"completion_tokens":9}}"#, + "data: [DONE]", + ], + ); + assert!(output.success, "{}", output.error); + assert_eq!(output.generated_text, "Hello"); + assert_eq!(output.itl.len(), 1); + assert_eq!(output.prompt_len, 4); + // An empty-text usage frame does not update the length, matching the + // Python parser, which reads usage only beside a non-empty delta. + assert_eq!(output.output_len, 16); + } + + /// A chat stream may close with a usage-only frame that carries no + /// choices; its token count must still be taken. + #[test] + fn chat_reads_a_usage_only_final_frame() { + let output = feed_all( + Api::OpenaiChat, + &[ + r#"data: {"choices":[{"delta":{"content":"a"}}]}"#, + r#"data: {"choices":[{"delta":{"content":"b"}}]}"#, + r#"data: {"choices":[],"usage":{"completion_tokens":2,"prompt_tokens":3}}"#, + "data: [DONE]", + ], + ); + assert!(output.success, "{}", output.error); + assert_eq!(output.generated_text, "ab"); + assert_eq!(output.output_len, 2); + assert_eq!(output.prompt_len, 3); + assert_eq!(output.itl.len(), 1); + } + + /// Reasoning tokens count as generated content, and the standard spelling + /// wins when a server exposes both aliases. + #[test] + fn chat_counts_reasoning_content_once() { + let output = feed_all( + Api::OpenaiChat, + &[ + r#"data: {"choices":[{"delta":{"reasoning_content":"think","reasoning":"dup","content":"!"}}]}"#, + ], + ); + assert_eq!(output.generated_text, "think!"); + } + + /// An empty body and a 200 that generated nothing are both failures: the + /// first has no latency to report, the second no TTFT, and counting either + /// as a success would pull every percentile toward zero. + #[test] + fn a_response_without_tokens_is_a_failure() { + let empty = feed_all(Api::SglangGenerate, &[]); + assert!(!empty.success); + assert!(empty.error.contains("empty")); + + let no_tokens = feed_all(Api::OpenaiChat, &["data: [DONE]"]); + assert!(!no_tokens.success); + assert!(no_tokens.error.contains("no generated tokens")); + } + + /// A server launched with `--skip-tokenizer-init` sends `output_ids` and + /// an empty `text`. Tokens must be counted from the ids, or every such + /// request reports zero tokens and is recorded as a failure. + #[test] + fn generate_counts_tokens_from_ids_when_there_is_no_text() { + let output = feed_all( + Api::SglangGenerate, + &[ + r#"data: {"text":"","output_ids":[11],"meta_info":{"completion_tokens":1}}"#, + r#"data: {"text":"","output_ids":[11,12,13],"meta_info":{"completion_tokens":3}}"#, + "data: [DONE]", + ], + ); + assert!(output.success, "{}", output.error); + assert_eq!(output.output_len, 3); + assert_eq!(output.itl.len(), 2); + assert!(output.ttft > 0.0); + assert!(output.generated_text.is_empty()); + } + + /// Without a count, the ids a frame carries stand in for it, read as + /// deltas (the Python server's streaming shape). + #[test] + fn generate_falls_back_to_carried_ids_as_deltas() { + let output = feed_all( + Api::SglangGenerate, + &[ + r#"data: {"output_ids":[11]}"#, + r#"data: {"output_ids":[12,13]}"#, + ], + ); + assert!(output.success, "{}", output.error); + assert_eq!(output.output_len, 3); + assert_eq!(output.itl.len(), 2); + } + + /// A server reporting neither a count nor ids still gets a TTFT from its + /// first text, and nothing more, since no later frame can be attributed. + #[test] + fn generate_without_counts_records_only_the_first_token() { + let output = feed_all( + Api::SglangGenerate, + &[r#"data: {"text":"a"}"#, r#"data: {"text":"ab"}"#], + ); + assert!(output.success, "{}", output.error); + assert!(output.ttft > 0.0); + assert!(output.itl.is_empty()); + assert_eq!(output.generated_text, "ab"); + // No count arrived, so the requested length stands. + assert_eq!(output.output_len, 16); + } + + /// The gRPC backend sends the body the native route would receive, so it + /// must decode into the generated request type for both prompt forms. + #[test] + fn grpc_body_fits_the_api_v1_schema() { + let template = template(Api::SglangGenerate); + for prompt in [Prompt::Text("hi".into()), Prompt::TokenIds(vec![1, 2])] { + let body = template.body(&row(prompt), 8); + let request: api::GenerateRequest = + serde_json::from_value(body).expect("native body is the api.v1 schema"); + assert_eq!(request.stream, Some(true)); + } + } + + /// A typed gRPC frame serializes to the native JSON frame, which is what + /// lets both transports share one accounting path. + #[test] + fn grpc_frame_serializes_to_the_native_frame() { + let frame = api::GenerateResponse { + text: "ab".into(), + meta_info: Some(api::GenerateMetaInfo { + id: "r".into(), + prompt_tokens: 3, + completion_tokens: 2, + ..Default::default() + }), + output_ids: Some(api::TokenIds { ids: vec![7, 8] }), + index: None, + }; + let data = serde_json::to_value(&frame).unwrap(); + assert_eq!(data["text"], "ab"); + assert_eq!(data["meta_info"]["completion_tokens"], 2); + assert_eq!(data["output_ids"], serde_json::json!([7, 8])); + + let mut output = RequestOutput::default(); + let start = Instant::now(); + let mut state = StreamState::new(Api::SglangGenerate, 16); + state.feed_value(&data, start, &mut output); + assert!(state.finish(&mut output).is_ok()); + assert_eq!(output.output_len, 2); + assert_eq!(output.prompt_len, 3); + } + + /// SSE framing details: blank keep-alive lines are skipped, a frame with + /// no `data: ` prefix is still parsed, and a frame this client cannot read + /// does not abort the request. + #[test] + fn framing_tolerates_blank_and_bare_lines() { + let output = feed_all( + Api::OpenaiCompletions, + &[ + "", + " ", + r#"{"choices":[{"text":"x"}]}"#, + "data: not json", + r#"data: {"choices":[{"text":"y"}]}"#, + ], + ); + assert!(output.success, "{}", output.error); + assert_eq!(output.generated_text, "xy"); + } + + #[test] + fn non_streaming_chat_reports_latency_as_ttft() { + let mut output = RequestOutput { + output_len: 16, + ..Default::default() + }; + let data = serde_json::json!({ + "choices": [{"message": {"content": "hello"}}], + "usage": {"completion_tokens": 2, "prompt_tokens": 6}, + }); + finish_non_streaming_chat(&data, 1.5, &mut output); + assert!(output.success); + assert_eq!(output.generated_text, "hello"); + assert_eq!(output.ttft, 1.5); + assert_eq!(output.latency, 1.5); + assert_eq!(output.output_len, 2); + assert_eq!(output.prompt_len, 6); + assert!(output.itl.is_empty()); + } +} diff --git a/rust/sglang-bench/src/run.rs b/rust/sglang-bench/src/run.rs new file mode 100644 index 0000000000..6447b2aaff --- /dev/null +++ b/rust/sglang-bench/src/run.rs @@ -0,0 +1,729 @@ +//! The benchmark run: set up, dispatch, measure, report. + +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result}; +use indicatif::{ProgressBar, ProgressStyle}; +use rand::prelude::*; +use rand::rngs::StdRng; +use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; +use sglang_api_types::api::v1 as api; +use tokio::sync::Semaphore; + +use crate::args::{Args, Protocol}; +use crate::dataset::DatasetRow; +use crate::request::{GrpcClient, RequestOutput, RequestTemplate}; +use crate::{dataset, hf, metrics, request}; + +/// aiohttp's benchmark session allows 6 hours for one request; a long +/// generation under heavy load legitimately takes minutes, and a shorter +/// timeout would report the client's impatience as a server failure. +const REQUEST_TIMEOUT: Duration = Duration::from_secs(6 * 60 * 60); + +/// The Rust server's gRPC listener accepts 64 MiB messages; a cumulative +/// `/generate` frame on a long request outgrows tonic's 4 MiB default. +const GRPC_MAX_MESSAGE_SIZE: usize = 64 * 1024 * 1024; + +/// Run one benchmark to completion and return its result document, the same +/// object the Python client writes to its `.jsonl`. +/// +/// Blocking: it owns the Tokio runtime for the run. Callers holding the GIL +/// must release it first, since this neither needs nor touches Python. +pub fn run_blocking(args: Args) -> Result { + args.validate()?; + let worker_threads = args.worker_threads.unwrap_or_else(|| { + std::thread::available_parallelism() + .map(std::num::NonZeroUsize::get) + .unwrap_or(4) + }); + anyhow::ensure!(worker_threads > 0, "--worker-threads must be at least 1"); + + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(worker_threads) + .enable_all() + .build() + .context("cannot start the Tokio runtime")?; + runtime.block_on(run(args, worker_threads)) +} + +async fn run(args: Args, worker_threads: usize) -> Result { + let base_url = args.base_url(); + let request_rate = args.parse_request_rate()?; + let headers = build_headers(&args)?; + let http = reqwest::Client::builder() + .timeout(REQUEST_TIMEOUT) + // One connection per in-flight request, so a request never waits on + // the pool instead of the server. Unbounded concurrency falls back to + // a generous ceiling. + .pool_max_idle_per_host(args.max_concurrency.unwrap_or(4096)) + .build() + .context("cannot build the HTTP client")?; + + // gRPC speaks to the server through one multiplexed HTTP/2 channel; the + // HTTP backends keep using `http`. + let grpc = if args.protocol == Protocol::Grpc { + Some(connect_grpc(&base_url)?) + } else { + None + }; + let server = Server { + http: &http, + base_url: &base_url, + headers: &headers, + grpc: grpc.clone(), + }; + + if args.ready_check_timeout_sec > 0 { + server + .wait_until_ready(args.ready_check_timeout_sec) + .await?; + } + + let model_info = server.model_info().await; + let model = match &args.model { + Some(model) => model.clone(), + None => model_info + .as_ref() + .and_then(|info| info.served_model_name.clone()) + .with_context(|| { + format!("no model found at {base_url}; pass --model, or check --host and --port") + })?, + }; + let model_id = args + .served_model_name + .clone() + .unwrap_or_else(|| model.clone()); + let tokenizer_source = match &args.tokenizer { + Some(source) => source.clone(), + None => model_info + .as_ref() + .and_then(|info| info.tokenizer_path.clone()) + .unwrap_or_else(|| model.clone()), + }; + + // `--vocab-size` is the one configuration that reads nothing from the + // tokenizer, so none is loaded: the client then needs no model files. + let tokenizer = if args.vocab_size.is_some() { + println!("Running without a tokenizer (--vocab-size given); retokenized totals report 0."); + None + } else { + println!("Loading tokenizer from {tokenizer_source}..."); + let tokenizer_path = hf::resolve_tokenizer(&http, &tokenizer_source).await?; + Some(hf::load_tokenizer(&tokenizer_path)?) + }; + let corpus = if dataset::needs_corpus(args.dataset_name) { + Some(hf::resolve_sharegpt(&http, &args.dataset_path).await?) + } else { + None + }; + let rows = dataset::load(&args, tokenizer.as_ref(), corpus.as_deref())?; + + let template = RequestTemplate { + api: args.backend.api(), + api_url: args.api_url(), + model: model_id, + stream: !args.disable_stream, + temperature: args.temperature, + top_p: args.top_p, + ignore_eos: !args.disable_ignore_eos, + return_logprob: args.return_logprob, + top_logprobs_num: args.top_logprobs_num, + logprob_start_len: args.logprob_start_len, + extra_body: args.parse_extra_body()?, + headers: headers.clone(), + grpc, + }; + + // One time base for every request, so the peak-window buckets in `metrics` + // place all of them on a single timeline. + let base = Instant::now(); + warmup(&http, &template, &rows, &args, base).await?; + // Cache flush is an HTTP-only control; a gRPC run skips it loudly rather + // than measuring a run that silently kept its cache. + if should_flush_cache(&args) { + if server.grpc.is_some() { + eprintln!("flush_cache is not exposed over gRPC; skipping"); + } else { + flush_cache( + &http, + &base_url, + &template.headers, + args.flush_cache_timeout, + ) + .await; + } + } + tokio::time::sleep(Duration::from_secs(1)).await; + + // The profiler is an HTTP-only control; a gRPC run skips it loudly. + if args.profile { + if server.grpc.is_some() { + eprintln!("the profiler is not exposed over gRPC; skipping"); + } else { + profile(&http, &base_url, &template.headers, "start").await; + } + } + + println!( + "Running {} requests on {worker_threads} worker threads (concurrency: {})...", + rows.len(), + args.max_concurrency + .map_or_else(|| "unbounded".to_owned(), |limit| limit.to_string()) + ); + let started = Instant::now(); + let outputs = dispatch(&http, &template, &rows, &args, request_rate, base).await; + let duration_s = started.elapsed().as_secs_f64(); + + if args.profile && server.grpc.is_none() { + profile(&http, &base_url, &template.headers, "stop").await; + } + + let server_info = if metrics::reports_server_info(args.backend) { + server.server_info().await + } else { + None + }; + let accept_length = server_info + .as_ref() + .and_then(metrics::accept_length_from_server_info); + + let retokenized = retokenize(tokenizer.as_ref(), &outputs, &args)?; + let metrics = metrics::calculate(&outputs, &retokenized, duration_s); + metrics::print_report(&metrics, &args, request_rate, duration_s, accept_length); + report_failures(&outputs); + + let result = metrics::result_json( + &metrics, + &args, + request_rate, + duration_s, + accept_length, + server_info, + &outputs, + ); + write_result(&args, &result)?; + Ok(result) +} + +/// Submit every request, pacing arrivals and capping concurrency. +/// +/// Arrivals are paced on this task while the requests themselves run as +/// independent tasks, so a slow response never delays the next arrival. The +/// outputs come back in request order, which the per-request result columns +/// rely on. +async fn dispatch( + http: &reqwest::Client, + template: &RequestTemplate, + rows: &[DatasetRow], + args: &Args, + request_rate: f64, + base: Instant, +) -> Vec { + let permits = args + .max_concurrency + .map(|limit| Arc::new(Semaphore::new(limit))); + let completed = Arc::new(AtomicU64::new(0)); + let progress = progress_bar(args, rows.len(), completed.clone()); + + let mut rng = StdRng::seed_from_u64(args.seed ^ 0x5eed); + let mut handles = Vec::with_capacity(rows.len()); + for row in rows { + let (http, template, row) = (http.clone(), template.clone(), row.clone()); + let permits = permits.clone(); + let completed = completed.clone(); + handles.push(tokio::spawn(async move { + // Held for the request's lifetime, so the cap counts requests in + // flight rather than requests started. + let _permit = match &permits { + Some(permits) => Some(permits.acquire().await.expect("semaphore is never closed")), + None => None, + }; + let output = request::send(&http, &template, &row, base).await; + completed.fetch_add(1, Ordering::Relaxed); + output + })); + + if request_rate.is_finite() { + // Poisson arrivals: exponential gaps with mean 1/rate. Sampled + // from the uniform inverse CDF, since the gap distribution is the + // only random draw this loop makes. + let uniform: f64 = rng.gen_range(f64::MIN_POSITIVE..1.0); + let interval = -uniform.ln() / request_rate; + tokio::time::sleep(Duration::from_secs_f64(interval)).await; + } + } + + let mut outputs = Vec::with_capacity(handles.len()); + for handle in handles { + match handle.await { + Ok(output) => outputs.push(output), + // A panicked task is this client's bug, not a server failure, but + // dropping the run would discard every other measurement. + Err(error) => outputs.push(RequestOutput { + error: format!("benchmark task failed: {error}"), + ..Default::default() + }), + } + } + if let Some(progress) = progress { + progress.finish_and_clear(); + } + outputs +} + +/// Send a few requests before measuring, so the first measured request does +/// not pay for the server's lazy initialization. Run at the measured +/// concurrency: an unbounded warmup burst is its own, different load. +async fn warmup( + http: &reqwest::Client, + template: &RequestTemplate, + rows: &[DatasetRow], + args: &Args, + base: Instant, +) -> Result<()> { + if args.warmup_requests == 0 { + return Ok(()); + } + println!("Starting warmup with {} sequences...", args.warmup_requests); + let mut row = rows[0].clone(); + row.output_len = row.output_len.min(32); + + let permits = args + .max_concurrency + .map(|limit| Arc::new(Semaphore::new(limit))); + let mut handles = Vec::with_capacity(args.warmup_requests); + for _ in 0..args.warmup_requests { + let (http, template, row) = (http.clone(), template.clone(), row.clone()); + let permits = permits.clone(); + handles.push(tokio::spawn(async move { + let _permit = match &permits { + Some(permits) => Some(permits.acquire().await.expect("semaphore is never closed")), + None => None, + }; + request::send(&http, &template, &row, base).await + })); + } + + let mut first_error = String::new(); + let mut any_succeeded = false; + for handle in handles { + match handle.await { + Ok(output) if output.success => any_succeeded = true, + Ok(output) if first_error.is_empty() => first_error = output.error, + Ok(_) => {} + Err(error) if first_error.is_empty() => first_error = error.to_string(), + Err(_) => {} + } + } + anyhow::ensure!( + any_succeeded, + "warmup failed; check the benchmark arguments. Error: {first_error}" + ); + println!( + "Warmup completed with {} sequences. Starting main benchmark run...", + args.warmup_requests + ); + Ok(()) +} + +/// Flush the server's prefix cache so the measured run does not reuse the +/// warmup's prefixes. On by default in CI, as in the Python script. +fn should_flush_cache(args: &Args) -> bool { + if args.flush_cache { + return true; + } + args.backend.is_sglang() + && std::env::var("SGLANG_IS_IN_CI") + .map(|value| matches!(value.to_lowercase().as_str(), "true" | "1")) + .unwrap_or(false) +} + +async fn flush_cache(http: &reqwest::Client, base_url: &str, headers: &HeaderMap, timeout_s: f64) { + let url = format!("{base_url}/flush_cache"); + let result = http + .post(&url) + .headers(headers.clone()) + .timeout(Duration::from_secs_f64(timeout_s)) + .send() + .await; + match result { + Ok(response) if response.status().is_success() => println!("Cache flushed."), + Ok(response) => eprintln!("flush_cache returned {}", response.status()), + Err(error) => eprintln!("flush_cache failed: {error}"), + } +} + +async fn profile(http: &reqwest::Client, base_url: &str, headers: &HeaderMap, action: &str) { + println!( + "{}ing profiler...", + &action[..action.len() - 1].to_uppercase() + ); + let url = format!("{base_url}/{action}_profile"); + match http.post(&url).headers(headers.clone()).send().await { + Ok(response) if response.status().is_success() => println!("Profiler {action}ed"), + Ok(response) => eprintln!("{url} returned {}", response.status()), + Err(error) => eprintln!("{url} failed: {error}"), + } +} + +/// Re-tokenize the generated texts, batched so the tokenizer's own +/// parallelism applies. `--disable-retokenize` reports zeros instead. +fn retokenize( + tokenizer: Option<&tokenizers::Tokenizer>, + outputs: &[RequestOutput], + args: &Args, +) -> Result> { + let Some(tokenizer) = tokenizer.filter(|_| !args.disable_retokenize) else { + return Ok(vec![0; outputs.len()]); + }; + let texts: Vec<&str> = outputs + .iter() + .map(|output| { + if output.success { + output.generated_text.as_str() + } else { + "" + } + }) + .collect(); + dataset::retokenized_lens(tokenizer, &texts) +} + +fn progress_bar(args: &Args, total: usize, completed: Arc) -> Option { + if args.disable_tqdm { + return None; + } + let bar = ProgressBar::new(total as u64); + bar.set_style( + ProgressStyle::with_template("{wide_bar} {pos}/{len} [{elapsed_precise}<{eta_precise}]") + .expect("the template is valid"), + ); + // The bar is driven from one place rather than from every request: at tens + // of thousands of requests, each task taking the bar's lock to report its + // own completion is itself measurable. + let ticker = bar.clone(); + tokio::spawn(async move { + loop { + let done = completed.load(Ordering::Relaxed); + ticker.set_position(done); + if done >= total as u64 { + return; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + }); + Some(bar) +} + +/// Summarize the failures, so a run that measured far fewer requests than it +/// sent says why on the terminal rather than only in the result file. +fn report_failures(outputs: &[RequestOutput]) { + let failed: Vec<&RequestOutput> = outputs.iter().filter(|output| !output.success).collect(); + if failed.is_empty() { + return; + } + eprintln!("\n{} of {} requests failed.", failed.len(), outputs.len()); + let mut counts: std::collections::BTreeMap<&str, usize> = std::collections::BTreeMap::new(); + for output in &failed { + *counts.entry(output.error.as_str()).or_default() += 1; + } + for (error, count) in counts.iter().take(5) { + let truncated: String = error.chars().take(200).collect(); + eprintln!(" {count} x {truncated}"); + } +} + +fn write_result(args: &Args, result: &serde_json::Value) -> Result<()> { + use std::io::Write; + + let path = match &args.output_file { + Some(path) => path.clone(), + None => metrics::default_output_file(args, &month_day()), + }; + let mut file = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&path) + .with_context(|| format!("cannot open {path}"))?; + writeln!(file, "{}", serde_json::to_string(result)?) + .with_context(|| format!("cannot write {path}"))?; + println!("Result appended to {path}"); + Ok(()) +} + +/// `MMDD` in local time, the stamp the Python script puts in the default +/// result filename. Derived from the Unix epoch so the crate needs no date +/// library; the civil-date arithmetic is Howard Hinnant's algorithm. +fn month_day() -> String { + let secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or(0); + let days = secs.div_euclid(86_400); + let (_, month, day) = civil_from_days(days); + format!("{month:02}{day:02}") +} + +/// Days since 1970-01-01 to a `(year, month, day)` civil date. +fn civil_from_days(days: i64) -> (i64, u32, u32) { + let z = days + 719_468; + let era = z.div_euclid(146_097); + let doe = z.rem_euclid(146_097); + let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365; + let year = yoe + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let day = (doy - (153 * mp + 2) / 5 + 1) as u32; + let month = if mp < 10 { mp + 3 } else { mp - 9 } as u32; + (year + i64::from(month <= 2), month, day) +} + +/// `OPENAI_API_KEY` as a bearer token, else `API_KEY` verbatim, plus any +/// `--header` values. Same precedence as the Python `get_auth_headers`. +fn build_headers(args: &Args) -> Result { + let mut headers = HeaderMap::new(); + match (std::env::var("OPENAI_API_KEY"), std::env::var("API_KEY")) { + (Ok(key), _) if !key.is_empty() => { + headers.insert( + reqwest::header::AUTHORIZATION, + HeaderValue::from_str(&format!("Bearer {key}"))?, + ); + } + (_, Ok(key)) if !key.is_empty() => { + headers.insert(reqwest::header::AUTHORIZATION, HeaderValue::from_str(&key)?); + } + _ => {} + } + for raw in &args.headers { + let (name, value) = raw + .split_once(':') + .with_context(|| format!("--header must be `Key: Value`, got {raw}"))?; + headers.insert( + HeaderName::from_bytes(name.trim().as_bytes()) + .with_context(|| format!("invalid header name in {raw}"))?, + HeaderValue::from_str(value.trim()) + .with_context(|| format!("invalid header value in {raw}"))?, + ); + } + Ok(headers) +} + +/// Open the api.v1 channel. One connection carries every request stream, +/// multiplexed by HTTP/2; it connects on first use so a server that is still +/// starting is handled by the readiness poll rather than here. +fn connect_grpc(base_url: &str) -> Result { + let channel = tonic::transport::Endpoint::from_shared(base_url.to_owned()) + .with_context(|| format!("{base_url} is not a valid gRPC endpoint"))? + .connect_timeout(Duration::from_secs(5)) + .connect_lazy(); + Ok(GrpcClient::new(channel).max_decoding_message_size(GRPC_MAX_MESSAGE_SIZE)) +} + +/// What the server says about itself, in the shape both transports share. +struct ServerModelInfo { + served_model_name: Option, + tokenizer_path: Option, +} + +/// The control-plane calls, each over whichever transport the run uses. +struct Server<'a> { + http: &'a reqwest::Client, + base_url: &'a str, + headers: &'a HeaderMap, + grpc: Option, +} + +impl Server<'_> { + /// Poll until the server answers, so a benchmark started alongside a + /// server waits for it instead of reporting a wall of connection errors. + async fn wait_until_ready(&self, timeout_sec: u64) -> Result<()> { + let target = match &self.grpc { + Some(_) => format!("{} (gRPC GetModelInfo)", self.base_url), + None => format!("{}/v1/models", self.base_url), + }; + println!("Waiting up to {timeout_sec}s for {target} to become ready..."); + let started = Instant::now(); + loop { + if self.is_ready().await { + println!("Server ready in {:.1}s.", started.elapsed().as_secs_f64()); + return Ok(()); + } + anyhow::ensure!( + started.elapsed() < Duration::from_secs(timeout_sec), + "server at {target} did not become ready within {timeout_sec}s" + ); + tokio::time::sleep(Duration::from_secs(1)).await; + } + } + + async fn is_ready(&self) -> bool { + match &self.grpc { + Some(client) => client + .clone() + .get_model_info(api::GetModelInfoRequest {}) + .await + .is_ok(), + None => self + .http + .get(format!("{}/v1/models", self.base_url)) + .headers(self.headers.clone()) + .timeout(Duration::from_secs(5)) + .send() + .await + .is_ok_and(|response| response.status().is_success()), + } + } + + /// The served model name and the tokenizer the server itself loaded. + /// HTTP reads `/v1/models` for the name and `/model_info` for the path; + /// gRPC gets both from `GetModelInfo`. + async fn model_info(&self) -> Option { + if let Some(client) = &self.grpc { + let info = client + .clone() + .get_model_info(api::GetModelInfoRequest {}) + .await + .ok()? + .into_inner(); + return Some(ServerModelInfo { + served_model_name: non_empty(info.served_model_name) + .or_else(|| non_empty(info.model_path.clone())), + tokenizer_path: non_empty(info.tokenizer_path) + .or_else(|| non_empty(info.model_path)), + }); + } + let served_model_name = fetch_json( + self.http, + &format!("{}/v1/models", self.base_url), + self.headers, + ) + .await + .and_then(|body| { + body.get("data")? + .get(0)? + .get("id")? + .as_str() + .map(str::to_owned) + }); + let tokenizer_path = fetch_json( + self.http, + &format!("{}/model_info", self.base_url), + self.headers, + ) + .await + .and_then(|info| { + ["tokenizer_path", "model_path"].iter().find_map(|key| { + info.get(key) + .and_then(serde_json::Value::as_str) + .filter(|path| !path.is_empty()) + .map(str::to_owned) + }) + }); + Some(ServerModelInfo { + served_model_name, + tokenizer_path, + }) + } + + /// `/server_info`, or its gRPC twin rendered to the same JSON shape so + /// the accept-length extractor reads both. + async fn server_info(&self) -> Option { + match &self.grpc { + Some(client) => { + let info = client + .clone() + .get_server_info(api::GetServerInfoRequest {}) + .await + .ok()? + .into_inner(); + serde_json::to_value(info).ok() + } + None => { + fetch_json( + self.http, + &format!("{}/server_info", self.base_url), + self.headers, + ) + .await + } + } + } +} + +fn non_empty(value: String) -> Option { + (!value.is_empty()).then_some(value) +} + +async fn fetch_json( + http: &reqwest::Client, + url: &str, + headers: &HeaderMap, +) -> Option { + let response = http + .get(url) + .headers(headers.clone()) + .timeout(Duration::from_secs(10)) + .send() + .await + .ok()?; + if !response.status().is_success() { + return None; + } + response.json().await.ok() +} + +#[cfg(test)] +mod tests { + use clap::Parser; + + use super::*; + + /// The civil-date arithmetic behind the default result filename, checked + /// on epoch day 0, a leap day, and a year boundary. + #[test] + fn civil_dates_round_trip_known_days() { + assert_eq!(civil_from_days(0), (1970, 1, 1)); + // 2020-02-29, a leap day. + assert_eq!(civil_from_days(18_321), (2020, 2, 29)); + // 2024-12-31 and the next day. + assert_eq!(civil_from_days(20_088), (2024, 12, 31)); + assert_eq!(civil_from_days(20_089), (2025, 1, 1)); + } + + #[test] + fn headers_combine_auth_and_custom_values() { + let args = Args::parse_from([ + "sglang-bench", + "--header", + "X-Route: a", + "--header", + "X-Other: b", + ]); + let headers = build_headers(&args).unwrap(); + assert_eq!(headers["x-route"], "a"); + assert_eq!(headers["x-other"], "b"); + } + + /// A malformed `--header` is rejected rather than dropped, since a header + /// the server never sees would change what the run measured. + #[test] + fn a_header_without_a_colon_is_rejected() { + let args = Args::parse_from(["sglang-bench", "--header", "no-colon"]); + assert!(build_headers(&args).is_err()); + } + + /// CI flushes the cache for SGLang backends so the measured run does not + /// inherit the warmup's prefixes; `--flush-cache` forces it everywhere. + #[test] + fn cache_flush_follows_the_backend_and_ci() { + let plain = Args::parse_from(["sglang-bench"]); + let forced = Args::parse_from(["sglang-bench", "--flush-cache"]); + let vllm = Args::parse_from(["sglang-bench", "--backend", "vllm"]); + assert!(should_flush_cache(&forced)); + // SGLANG_IS_IN_CI is not set in this test process. + assert!(!should_flush_cache(&plain)); + assert!(!should_flush_cache(&vllm)); + } +} diff --git a/rust/sglang-bench/tests/grpc.rs b/rust/sglang-bench/tests/grpc.rs new file mode 100644 index 0000000000..66c41dc62e --- /dev/null +++ b/rust/sglang-bench/tests/grpc.rs @@ -0,0 +1,184 @@ +//! End to end over the api.v1 gRPC transport: a tonic mock of the server's +//! service answers model info, server info and a token-id generate stream, +//! and the client is driven through its public entry point with ids on both +//! ends and no tokenizer. This is the one path the unit tests cannot reach, +//! since they stop at the frame parser. + +use std::net::TcpListener; +use std::pin::Pin; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use futures::Stream; +use sglang_api_types::api::v1 as api; +use sglang_api_types::api::v1::sglang_service_server::{SglangService, SglangServiceServer}; +use tokio_stream::wrappers::TcpListenerStream; +use tonic::{Request, Response, Status}; + +/// Serves `tokens_per_request` ids per generate call, two per frame, with an +/// empty `text` as a `--skip-tokenizer-init` server would. +struct Mock { + tokens_per_request: usize, + /// Requests that arrived with `input_ids` rather than `text`. + id_requests: Arc, +} + +type ItemStream = Pin> + Send>>; + +#[tonic::async_trait] +impl SglangService for Mock { + type GenerateStream = ItemStream; + + async fn generate( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + if request.input_ids.is_some() && request.text.is_none() { + self.id_requests.fetch_add(1, Ordering::Relaxed); + } + let want = self.tokens_per_request; + let mut frames = Vec::new(); + let mut ids: Vec = Vec::new(); + while ids.len() < want { + ids.push(ids.len() as i64 + 100); + if ids.len() < want { + ids.push(ids.len() as i64 + 100); + } + let done = ids.len() == want; + frames.push(Ok(api::GenerateStreamItem { + item: Some(api::generate_stream_item::Item::Frame( + api::GenerateResponse { + text: String::new(), + meta_info: Some(api::GenerateMetaInfo { + id: "mock".into(), + prompt_tokens: 9, + completion_tokens: ids.len() as u64, + finish_reason: done.then_some(api::FinishReason { + kind: Some(api::finish_reason::Kind::Length(api::FinishLength { + length: Some(want as u64), + })), + }), + ..Default::default() + }), + output_ids: Some(api::TokenIds { ids: ids.clone() }), + index: None, + }, + )), + })); + } + Ok(Response::new(Box::pin(futures::stream::iter(frames)))) + } + + async fn health_check( + &self, + _: Request, + ) -> Result, Status> { + Ok(Response::new(api::HealthCheckResponse { healthy: true })) + } + + async fn get_model_info( + &self, + _: Request, + ) -> Result, Status> { + Ok(Response::new(api::GetModelInfoResponse { + model_path: "/models/mock".into(), + served_model_name: "mock-model".into(), + tokenizer_path: "/models/mock".into(), + is_generation: true, + ..Default::default() + })) + } + + async fn get_server_info( + &self, + _: Request, + ) -> Result, Status> { + Ok(Response::new(api::GetServerInfoResponse { + model_path: "/models/mock".into(), + served_model_name: "mock-model".into(), + tokenizer_path: "/models/mock".into(), + max_context_length: 4096, + max_total_num_tokens: Some(8192), + version: "test".into(), + internal_states: vec![r#"{"avg_spec_accept_length": 2.5}"#.into()], + })) + } +} + +/// Bind on a free port and serve the mock on its own runtime thread, so the +/// client's own Tokio runtime is what the test exercises. +fn serve_mock(mock: Mock) -> u16 { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + listener.set_nonblocking(true).unwrap(); + std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async { + let listener = tokio::net::TcpListener::from_std(listener).unwrap(); + tonic::transport::Server::builder() + .add_service(SglangServiceServer::new(mock)) + .serve_with_incoming(TcpListenerStream::new(listener)) + .await + .unwrap(); + }); + }); + port +} + +/// The whole client, ids in and ids out, with no tokenizer loaded: every +/// request completes, tokens are counted from the ids the frames carry, the +/// accept length is read from the gRPC server info, and the per-run result +/// lands in the file with the Python client's keys. +#[test] +fn grpc_backend_runs_end_to_end_on_token_ids() { + let id_requests = Arc::new(AtomicUsize::new(0)); + let port = serve_mock(Mock { + tokens_per_request: 6, + id_requests: id_requests.clone(), + }); + let output_file = + std::env::temp_dir().join(format!("sglang-bench-grpc-{}.jsonl", std::process::id())); + + let args: sglang_bench::Args = serde_json::from_value(serde_json::json!({ + "backend": "sglang", + "protocol": "grpc", + "base_url": format!("http://127.0.0.1:{port}"), + "dataset_name": "random-ids", + "tokenize_prompt": true, + "vocab_size": 1000, + "num_prompts": 12, + "random_input_len": 8, + "random_output_len": 6, + "random_range_ratio": 1.0, + "max_concurrency": 4, + "warmup_requests": 1, + "ready_check_timeout_sec": 10, + "disable_tqdm": true, + "output_file": output_file.to_str().unwrap(), + })) + .unwrap(); + + let result = sglang_bench::run_blocking(args).expect("benchmark runs"); + + assert_eq!(result["completed"], 12); + assert_eq!(result["backend"], "sglang"); + assert_eq!(result["protocol"], "grpc"); + // Every prompt is 8 ids and every reply 6 ids; no text crossed the wire, + // so the retokenized total is 0 by design. + assert_eq!(result["total_input_tokens"], 12 * 8); + assert_eq!(result["total_output_tokens"], 12 * 6); + assert_eq!(result["total_output_tokens_retokenized"], 0); + assert_eq!(result["accept_length"], 2.5); + assert!(result["mean_ttft_ms"].as_f64().unwrap() > 0.0); + // 12 measured requests plus the warmup, all as input_ids. + assert_eq!(id_requests.load(Ordering::Relaxed), 13); + + let written = std::fs::read_to_string(&output_file).unwrap(); + let line: serde_json::Value = serde_json::from_str(written.trim()).unwrap(); + assert_eq!(line["completed"], 12); + std::fs::remove_file(&output_file).unwrap(); +}