[Benchmark] Add an opt-out for the token-capacity check (#39284)
This commit is contained in:
@@ -124,6 +124,7 @@ class BenchArgs:
|
||||
base_url: str = ""
|
||||
local_tokenizer_path: str = ""
|
||||
skip_warmup: bool = False
|
||||
skip_token_capacity_check: bool = False
|
||||
show_report: bool = False
|
||||
profile: bool = False
|
||||
profile_activities: Tuple[str] = ("CPU", "GPU")
|
||||
@@ -207,6 +208,11 @@ class BenchArgs:
|
||||
),
|
||||
)
|
||||
parser.add_argument("--skip-warmup", action="store_true")
|
||||
parser.add_argument(
|
||||
"--skip-token-capacity-check",
|
||||
action="store_true",
|
||||
help="Skip the raw-token capacity check; keep max-running-requests checks.",
|
||||
)
|
||||
parser.add_argument("--show-report", action="store_true")
|
||||
parser.add_argument("--profile", action="store_true")
|
||||
parser.add_argument(
|
||||
@@ -1230,6 +1236,9 @@ def run_benchmark_internal(
|
||||
"token_capacity", 1000000000
|
||||
)
|
||||
|
||||
if bench_args.skip_token_capacity_check:
|
||||
skip_token_capacity_threshold = float("inf")
|
||||
|
||||
# Router /get_server_info responses carry "router_manager"; worker
|
||||
# responses never do, so its presence confirms a router by design.
|
||||
if not internal_states and server_info.get("router_manager"):
|
||||
|
||||
Reference in New Issue
Block a user