Rainj me/rust bench (#43950)

This commit is contained in:
Rain Jiang
2026-10-13 11:07:42 -07:00
committed by GitHub
parent c0584cf9af
commit e98c6d7b25
14 changed files with 4314 additions and 0 deletions
+224
View File
@@ -0,0 +1,224 @@
"""Run the benchmark through the Rust client (``sglang-bench``).
The Python client in :mod:`sglang.benchmark.serving` drives every request and
parses every response stream on one asyncio thread, so past a few thousand
concurrent streams it, not the server, caps the measured throughput. The Rust
client parses each stream on a Tokio worker thread instead. It reports the same
metrics, prints the same table, and appends the same result keys, so a run from
either side stays comparable.
Two ways in:
* ``python -m sglang.benchmark.rust_client --help`` forwards the command line
to the Rust client, which owns the flag definitions. This path imports no
model tooling, so it starts in well under a second.
* ``python -m sglang.benchmark.serving --rust-client ...`` routes an
already-parsed argument namespace here, for callers on the existing entry
point.
The Rust client covers fewer backends and datasets than the Python one; an
unsupported configuration raises here rather than quietly measuring something
else. See ``rust/sglang-bench/README.md``.
"""
from __future__ import annotations
import json
import math
import sys
from typing import Any, Dict, Iterable, Optional
EXTENSION_MODULE = "sglang.srt.rust_extensions._bench"
#: Backends the Rust client implements. The Python client's remaining
#: backends (``trt``, ``gserver``, ``truss``, and the embedding ones) have no
#: Rust counterpart.
SUPPORTED_BACKENDS = frozenset(
{
"sglang",
"sglang-native",
"sglang-oai",
"sglang-oai-chat",
"vllm",
"vllm-chat",
"lmdeploy",
"lmdeploy-chat",
}
)
#: Datasets the Rust client implements.
SUPPORTED_DATASETS = frozenset({"random", "random-ids", "sharegpt"})
#: Namespace attributes that map onto a Rust flag of the same name. Anything
#: needing conversion is handled in :func:`build_config`.
_DIRECT_FIELDS = (
"backend",
# Rust-only: ``http`` (default) or ``grpc``. `serving.py` has no such
# flag, so this arrives only from a programmatic caller.
"protocol",
"base_url",
"host",
"port",
"ready_check_timeout_sec",
"dataset_name",
"dataset_path",
"model",
"served_model_name",
"tokenizer",
"num_prompts",
"sharegpt_output_len",
"sharegpt_context_len",
"random_input_len",
"random_output_len",
"random_range_ratio",
"max_concurrency",
"warmup_requests",
"output_file",
"output_details",
"disable_tqdm",
"disable_stream",
"disable_ignore_eos",
"seed",
"temperature",
"top_p",
"return_logprob",
"top_logprobs_num",
"logprob_start_len",
"extra_request_body",
"tokenize_prompt",
"flush_cache",
"flush_cache_timeout",
"profile",
"tag",
)
#: Options the Python client accepts that the Rust one does not implement.
#: Each maps to the value that means "not requested", so only a caller who
#: actually asked for one is turned away.
_UNSUPPORTED_OPTIONS = {
"lora_name": None,
"use_trace_timestamps": False,
"apply_chat_template": False,
"plot_throughput": False,
"cache_report": False,
"pd_separated": False,
"return_routed_experts": False,
"token_ids_logprob": None,
"print_requests": False,
"prompt_suffix": "",
"fake_prefill": False,
"profile_steps": None,
"profile_num_steps": None,
"profile_by_stage": False,
}
class UnsupportedByRustClient(ValueError):
"""The requested configuration has no Rust implementation."""
def check_supported(args: Any) -> None:
"""Raise unless the Rust client implements everything ``args`` asks for."""
backend = getattr(args, "backend", None)
if backend not in SUPPORTED_BACKENDS:
raise UnsupportedByRustClient(
f"--rust-client does not implement --backend {backend}; "
f"supported: {', '.join(sorted(SUPPORTED_BACKENDS))}"
)
dataset = getattr(args, "dataset_name", None)
if dataset not in SUPPORTED_DATASETS:
raise UnsupportedByRustClient(
f"--rust-client does not implement --dataset-name {dataset}; "
f"supported: {', '.join(sorted(SUPPORTED_DATASETS))}"
)
requested = [
name
for name, unset in _UNSUPPORTED_OPTIONS.items()
if getattr(args, name, unset) != unset
]
if requested:
raise UnsupportedByRustClient(
"--rust-client does not implement: "
f"{', '.join(sorted(requested))}. Drop --rust-client to run these "
"on the Python client."
)
def build_config(args: Any) -> Dict[str, Any]:
"""The Rust client's config object for an argument namespace.
Only keys the caller actually set are sent; every omitted key takes the
Rust flag's own default, which is the same value the Python flag has.
"""
config: Dict[str, Any] = {}
for field in _DIRECT_FIELDS:
value = getattr(args, field, None)
if value is not None:
config[field] = value
# The Rust flag takes a string so it can carry `inf`, which JSON cannot.
rate = getattr(args, "request_rate", float("inf"))
config["request_rate"] = "inf" if rate is None or math.isinf(rate) else str(rate)
# The Python flag spells a header `Key=Value`; the Rust one `Key: Value`.
config["headers"] = list(_as_headers(getattr(args, "header", None)))
worker_threads = getattr(args, "rust_worker_threads", None)
if worker_threads is not None:
config["worker_threads"] = worker_threads
return config
def _as_headers(header: Any) -> Iterable[str]:
"""`--header Key=Value ...` in the Rust client's `Key: Value` spelling."""
if not header:
return ()
raw = (header,) if isinstance(header, str) else tuple(header)
rewritten = []
for entry in raw:
name, separator, value = entry.partition("=")
if not separator or not name or not value:
raise UnsupportedByRustClient(
f"--header must be `Key=Value`, got {entry!r}"
)
rewritten.append(f"{name}: {value}")
return tuple(rewritten)
def _extension(extension: Any = None) -> Any:
if extension is not None:
return extension
from sglang.srt.rust_extensions import load_rust_extension
return load_rust_extension(EXTENSION_MODULE)
def run_rust_benchmark(args: Any, extension: Any = None) -> Dict[str, Any]:
"""Run the benchmark described by ``args`` and return its result document.
The Rust client prints the report and appends the result line itself, so
the returned mapping is for a caller that wants the numbers in process.
"""
check_supported(args)
config = build_config(args)
result = _extension(extension).run_config(json.dumps(config))
return json.loads(result)
def main(argv: Optional[list] = None, extension: Any = None) -> int:
"""Forward a command line to the Rust client, which owns the flags."""
argv = list(sys.argv if argv is None else argv)
if not argv:
argv = ["sglang-bench"]
# The Rust side expects argv[0] to be the program name, so clap's usage
# and error messages name the command the user actually typed.
argv[0] = "python -m sglang.benchmark.rust_client"
try:
_extension(extension).run_argv(argv)
except SystemExit as exit_request:
return int(exit_request.code or 0)
return 0
if __name__ == "__main__":
sys.exit(main())
+39
View File
@@ -1971,6 +1971,13 @@ def run_benchmark(args_: argparse.Namespace):
global args
args = args_
# The Rust client owns the whole run, from dataset to result file, so hand
# it the arguments before this module sets up anything of its own.
if getattr(args, "rust_client", False):
from sglang.benchmark.rust_client import run_rust_benchmark
return run_rust_benchmark(args)
# Set default value for max_concurrency if not present
if not hasattr(args, "max_concurrency"):
args.max_concurrency = None
@@ -2798,7 +2805,39 @@ def cli_main():
default=None,
help="Custom HTTP headers in Key=Value format. Example: --header MyHeader=MY_VALUE MyAnotherHeader=myanothervalue",
)
parser.add_argument(
"--rust-client",
action="store_true",
help="Run through the Rust client (sglang.srt.rust_extensions._bench), "
"which parses each response stream on its own thread instead of one "
"shared asyncio thread. Same metrics and result keys; fewer backends "
"and datasets (see rust/sglang-bench/README.md). For a command line "
"that skips this module's model-tooling imports entirely, run "
"`python -m sglang.benchmark.rust_client` instead.",
)
parser.add_argument(
"--rust-worker-threads",
type=int,
default=None,
help="Response-stream parsing threads for --rust-client. "
"Defaults to the available parallelism.",
)
args = parser.parse_args()
if args.rust_worker_threads is not None and not args.rust_client:
parser.error("--rust-worker-threads requires --rust-client")
if args.rust_client:
# Checked here so an unsupported combination is a usage error rather
# than a traceback; `run_benchmark` still raises for a programmatic
# caller, which has no parser to report through.
from sglang.benchmark.rust_client import (
UnsupportedByRustClient,
check_supported,
)
try:
check_supported(args)
except UnsupportedByRustClient as error:
parser.error(str(error))
_validate_parsed_gsp_args(parser, args)
run_benchmark(args)