[Benchmark] Limit warmup concurrency in serving benchmark (#39398)
This commit is contained in:
@@ -1441,9 +1441,16 @@ async def benchmark(
|
|||||||
|
|
||||||
# Run warmup requests
|
# Run warmup requests
|
||||||
warmup_tasks = []
|
warmup_tasks = []
|
||||||
|
# Keep warmup at the same concurrency as the measured run. Otherwise,
|
||||||
|
# --warmup-requests also becomes an unbounded warmup burst.
|
||||||
for _ in range(warmup_requests):
|
for _ in range(warmup_requests):
|
||||||
warmup_tasks.append(
|
warmup_tasks.append(
|
||||||
asyncio.create_task(request_func(request_func_input=test_input))
|
asyncio.create_task(
|
||||||
|
limited_request_func(
|
||||||
|
request_func_input=test_input,
|
||||||
|
pbar=None,
|
||||||
|
)
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
warmup_outputs = await asyncio.gather(*warmup_tasks)
|
warmup_outputs = await asyncio.gather(*warmup_tasks)
|
||||||
@@ -1470,7 +1477,7 @@ async def benchmark(
|
|||||||
if should_flush_cache:
|
if should_flush_cache:
|
||||||
flush_server_cache(base_url, backend, flush_cache_timeout)
|
flush_server_cache(base_url, backend, flush_cache_timeout)
|
||||||
|
|
||||||
time.sleep(1.0)
|
await asyncio.sleep(1.0)
|
||||||
|
|
||||||
# Build profile URLs for PD separated mode (do this once at the beginning)
|
# Build profile URLs for PD separated mode (do this once at the beginning)
|
||||||
pd_profile_urls = []
|
pd_profile_urls = []
|
||||||
|
|||||||
Reference in New Issue
Block a user