minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[HiCache] fix: graceful shutdown of pending async tasks in bench_mix.py (#20276)

Browse Source
This commit is contained in:
shuwenn
2026-03-29 00:46:32 -07:00
committed by GitHub
parent d2fa8d67ba
commit c34593f951
1 changed files with 8 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmark/hicache/bench_mix.py
+8 -1
View File
@@ -426,11 +426,13 @@ class WorkloadGenerator:
def request_sender(self): def request_sender(self):
async def request_loop(): async def request_loop():
tasks = []
while True: while True:
if self.sent_requests - self.completed_requests < self.max_parallel: if self.sent_requests - self.completed_requests < self.max_parallel:
new_request = self.user_generator.pop() new_request = self.user_generator.pop()
if new_request: if new_request:
asyncio.create_task(self.handle_request(new_request)) task = asyncio.create_task(self.handle_request(new_request))
tasks.append(task)
self.sent_requests += 1 self.sent_requests += 1
else: else:
await asyncio.sleep(0.05) await asyncio.sleep(0.05)
@@ -440,6 +442,11 @@ class WorkloadGenerator:
self.done = True self.done = True
break break
# Cancel all pending tasks and wait for them to finish
for task in tasks:
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
loop = asyncio.new_event_loop() loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop) asyncio.set_event_loop(loop)
loop.run_until_complete(request_loop()) loop.run_until_complete(request_loop())