diff --git a/python/sglang/bench_one_batch_server.py b/python/sglang/bench_one_batch_server.py index 654589902..3bac6b038 100644 --- a/python/sglang/bench_one_batch_server.py +++ b/python/sglang/bench_one_batch_server.py @@ -1,13 +1,15 @@ -"""Back-compat shim. The implementation now lives in -``sglang.benchmark.one_batch_server``; this module preserves the -``python -m sglang.bench_one_batch_server`` entry point and the -``from sglang.bench_one_batch_server import ...`` imports. +"""Deprecated import path for ``sglang.benchmark.one_batch_server``. + +``python -m sglang.bench_one_batch_server`` and +``from sglang.bench_one_batch_server import ...`` still work, but the +implementation now lives in ``sglang.benchmark.one_batch_server``. +Update references to the new path. """ import warnings from sglang.benchmark.one_batch_server import * # noqa: F401,F403 -from sglang.benchmark.one_batch_server import main +from sglang.benchmark.one_batch_server import cli_main warnings.warn( "`sglang.bench_one_batch_server` is deprecated and will be removed in a " @@ -18,4 +20,4 @@ warnings.warn( ) if __name__ == "__main__": - main() + cli_main() diff --git a/python/sglang/bench_serving.py b/python/sglang/bench_serving.py index ddf0874db..6b25d1e85 100644 --- a/python/sglang/bench_serving.py +++ b/python/sglang/bench_serving.py @@ -1,20 +1,22 @@ -# SPDX-License-Identifier: Apache-2.0 -"""Compatibility shim for the relocated serving benchmark entrypoint.""" +"""Deprecated import path for ``sglang.benchmark.serving``. + +``python -m sglang.bench_serving`` and ``from sglang.bench_serving import ...`` +still work, but the implementation now lives in ``sglang.benchmark.serving``. +Update references to the new path. +""" import warnings -from sglang.benchmark.serving import * # noqa: F403 -from sglang.benchmark.serving import ( # noqa: F401 - _create_bench_client_session, - cli_main, -) +from sglang.benchmark.serving import * # noqa: F401,F403 +from sglang.benchmark.serving import cli_main warnings.warn( - "sglang.bench_serving is deprecated; use sglang.benchmark.serving instead.", + "`sglang.bench_serving` is deprecated and will be removed in a future " + "release; use `sglang.benchmark.serving` instead " + "(e.g. `python -m sglang.benchmark.serving`).", FutureWarning, stacklevel=1, ) - if __name__ == "__main__": cli_main() diff --git a/python/sglang/benchmark/datasets/autobench.py b/python/sglang/benchmark/datasets/autobench.py index cc5598133..c02bce1c3 100644 --- a/python/sglang/benchmark/datasets/autobench.py +++ b/python/sglang/benchmark/datasets/autobench.py @@ -173,7 +173,7 @@ def _estimate_prompt_lens( prompt_len = len(prompt) return prompt_len, prompt_len, 0 - # Multi-turn prompt lists are handled specially by bench_serving and do not + # Multi-turn prompt lists are handled specially by the serving benchmark and do not # contribute reliable static prompt lengths. return 0, 0, 0 diff --git a/python/sglang/benchmark/datasets/generated_shared_prefix.py b/python/sglang/benchmark/datasets/generated_shared_prefix.py index 6b5ac3426..f238990ed 100644 --- a/python/sglang/benchmark/datasets/generated_shared_prefix.py +++ b/python/sglang/benchmark/datasets/generated_shared_prefix.py @@ -60,7 +60,7 @@ class GeneratedSharedPrefixDataset(BaseDataset): # Defensive validation for in-process callers that construct a # Namespace by hand and bypass the argparse boundary in - # bench_serving.py. The CLI hook enforces the same rules first. + # serving.py. The CLI hook enforces the same rules first. if group_distribution not in ("uniform", "zipf"): raise ValueError( f"--gsp-group-distribution must be 'uniform' or 'zipf', " diff --git a/python/sglang/benchmark/datasets/openai_dataset.py b/python/sglang/benchmark/datasets/openai_dataset.py index 3ae807056..7c3bb9dfd 100644 --- a/python/sglang/benchmark/datasets/openai_dataset.py +++ b/python/sglang/benchmark/datasets/openai_dataset.py @@ -97,7 +97,7 @@ def sample_openai_requests( tools_tokens = len(tokenizer.encode(tools_str)) prompt_len += tools_tokens - # Pass messages list directly - bench_serving handles List[Dict] prompts + # Pass messages list directly - the serving benchmark handles List[Dict] prompts filtered_dataset.append( DatasetRow( prompt=messages, diff --git a/python/sglang/benchmark/datasets/speed_bench.py b/python/sglang/benchmark/datasets/speed_bench.py index df4c9e343..2ebabb21d 100644 --- a/python/sglang/benchmark/datasets/speed_bench.py +++ b/python/sglang/benchmark/datasets/speed_bench.py @@ -1,4 +1,4 @@ -"""SPEED-Bench (nvidia/SPEED-Bench) dataset for SGLang bench_serving. +"""SPEED-Bench (nvidia/SPEED-Bench) dataset for the SGLang serving benchmark. Reads the pre-downloaded throughput_1k JSONL produced by prepare_speed_bench.sh (or equivalent), optionally filtering by category (low_entropy / mixed / diff --git a/python/sglang/benchmark/offline_throughput.py b/python/sglang/benchmark/offline_throughput.py index 2002db6b4..945fb5a5a 100644 --- a/python/sglang/benchmark/offline_throughput.py +++ b/python/sglang/benchmark/offline_throughput.py @@ -1,13 +1,13 @@ """ Benchmark the throughput in the offline mode. -It accepts server arguments (the same as launch_server.py) and benchmark arguments (the same as bench_serving.py). +It accepts server arguments (the same as launch_server.py) and benchmark arguments (the same as serving.py). # Usage ## Sharegpt dataset with default args -python -m sglang.bench_offline_throughput --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --num-prompts 10 +python -m sglang.benchmark.offline_throughput --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --num-prompts 10 ## Random dataset with default args -python -m sglang.bench_offline_throughput --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --dataset-name random --random-input 1024 --random-output 1024 +python -m sglang.benchmark.offline_throughput --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --dataset-name random --random-input 1024 --random-output 1024 """ import argparse diff --git a/python/sglang/benchmark/one_batch.py b/python/sglang/benchmark/one_batch.py index b96d4d7ca..834e1518f 100644 --- a/python/sglang/benchmark/one_batch.py +++ b/python/sglang/benchmark/one_batch.py @@ -6,18 +6,18 @@ It accepts server arguments (the same as launch_server.py) and benchmark argumen # Usage (latency test) ## with dummy weights: -python -m sglang.bench_one_batch --model-path meta-llama/Meta-Llama-3-8B-Instruct --load-format dummy +python -m sglang.benchmark.one_batch --model-path meta-llama/Meta-Llama-3-8B-Instruct --load-format dummy ## sweep through multiple data points and store (append) the results in a jsonl file: -python -m sglang.bench_one_batch --model-path meta-llama/Meta-Llama-3-8B-Instruct --batch-size 1 12 14 --input-len 256 512 --output-len 32 256 --run-name test_run +python -m sglang.benchmark.one_batch --model-path meta-llama/Meta-Llama-3-8B-Instruct --batch-size 1 12 14 --input-len 256 512 --output-len 32 256 --run-name test_run ## run with profiling: -python -m sglang.bench_one_batch --model-path meta-llama/Meta-Llama-3-8B-Instruct --batch-size 1 12 14 --input-len 256 512 --profile +python -m sglang.benchmark.one_batch --model-path meta-llama/Meta-Llama-3-8B-Instruct --batch-size 1 12 14 --input-len 256 512 --profile ## run with profiling to custom directory: export SGLANG_TORCH_PROFILER_DIR=/root/sglang/profile_log -python -m sglang.bench_one_batch --model-path meta-llama/Meta-Llama-3-8B-Instruct --batch-size 1 --input-len 256 --profile +python -m sglang.benchmark.one_batch --model-path meta-llama/Meta-Llama-3-8B-Instruct --batch-size 1 --input-len 256 --profile ## run with CUDA profiler (nsys): -nsys profile --force-overwrite=true -o bench_one_batch python -m sglang.bench_one_batch --model-path meta-llama/Meta-Llama-3-8B-Instruct --batch-size 1 --input-len 256 --profile --profile-activities CUDA_PROFILER +nsys profile --force-overwrite=true -o bench_one_batch python -m sglang.benchmark.one_batch --model-path meta-llama/Meta-Llama-3-8B-Instruct --batch-size 1 --input-len 256 --profile --profile-activities CUDA_PROFILER # Usage (correctness test): -python -m sglang.bench_one_batch --model-path TinyLlama/TinyLlama-1.1B-Chat-v0.4 --correct +python -m sglang.benchmark.one_batch --model-path TinyLlama/TinyLlama-1.1B-Chat-v0.4 --correct ## Reference output (of the correctness test above, can be gpu dependent): input_ids=[[1, 450, 7483, 310, 3444, 338], [1, 450, 7483, 310, 278, 3303, 13187, 290, 338], [1, 20628, 338, 263, 6575, 1460, 2462, 322, 306, 763]] diff --git a/python/sglang/benchmark/one_batch_server.py b/python/sglang/benchmark/one_batch_server.py index e41fdb249..df896106d 100644 --- a/python/sglang/benchmark/one_batch_server.py +++ b/python/sglang/benchmark/one_batch_server.py @@ -5,11 +5,11 @@ This script launches a server and uses the HTTP interface. It accepts server arguments (the same as launch_server.py) and benchmark arguments (e.g., batch size, input lengths). Usage: -python3 -m sglang.bench_one_batch_server --model meta-llama/Meta-Llama-3.1-8B --batch-size 1 16 64 --input-len 1024 --output-len 8 +python3 -m sglang.benchmark.one_batch_server --model meta-llama/Meta-Llama-3.1-8B --batch-size 1 16 64 --input-len 1024 --output-len 8 -python3 -m sglang.bench_one_batch_server --model None --base-url http://localhost:30000 --batch-size 16 --input-len 1024 --output-len 8 -python3 -m sglang.bench_one_batch_server --model None --base-url http://localhost:30000 --batch-size 16 --input-len 1024 --output-len 8 --show-report --profile --profile-by-stage -python3 -m sglang.bench_one_batch_server --model None --base-url http://localhost:30000 --batch-size 16 --input-len 1024 --output-len 8 --result-filename results.jsonl --profile +python3 -m sglang.benchmark.one_batch_server --model None --base-url http://localhost:30000 --batch-size 16 --input-len 1024 --output-len 8 +python3 -m sglang.benchmark.one_batch_server --model None --base-url http://localhost:30000 --batch-size 16 --input-len 1024 --output-len 8 --show-report --profile --profile-by-stage +python3 -m sglang.benchmark.one_batch_server --model None --base-url http://localhost:30000 --batch-size 16 --input-len 1024 --output-len 8 --result-filename results.jsonl --profile """ import argparse @@ -309,7 +309,7 @@ class BenchArgs: default=BenchArgs.lora_request_distribution, choices=["uniform", "distinct", "skewed"], help="How to sample a LoRA adapter per prompt when more than one " - "is listed in --lora-name. Mirrors bench_serving.py. " + "is listed in --lora-name. Mirrors serving.py. " "'uniform' picks uniformly at random, 'distinct' round-robins so " "consecutive prompts get different adapters, 'skewed' samples " "from a Zipf distribution over --lora-name (alpha controls the " @@ -507,7 +507,7 @@ def run_one_case( else: _flush_cache_with_retry(url, "/flush_cache") - # Load input token ids via bench_serving.get_dataset + # Load input token ids via benchmark.datasets.get_dataset supported_datasets = ("random", "random-ids", "mmmu", "generated-shared-prefix") if dataset_name not in supported_datasets: raise ValueError( @@ -968,7 +968,7 @@ def run_benchmark_internal( f"to actually exercise multi-batch." ) - # LoRA distribution args: mirror bench_serving.py semantics so multi-LoRA + # LoRA distribution args: mirror serving.py semantics so multi-LoRA # benchmarks behave consistently across harnesses. if bench_args.lora_request_distribution in ("distinct", "skewed"): assert bench_args.lora_name is not None and len(bench_args.lora_name) > 1, ( @@ -1152,7 +1152,7 @@ def run_benchmark(server_args: ServerArgs, bench_args: BenchArgs): return results, server_info -def main(): +def cli_main(): parser = argparse.ArgumentParser() ServerArgs.add_cli_args(parser) BenchArgs.add_cli_args(parser) @@ -1165,4 +1165,4 @@ def main(): if __name__ == "__main__": - main() + cli_main()