diff --git a/sgl-model-gateway/bindings/python/src/sglang_router/mini_lb.py b/sgl-model-gateway/bindings/python/src/sglang_router/mini_lb.py index abdbd9523..2156a2dfe 100644 --- a/sgl-model-gateway/bindings/python/src/sglang_router/mini_lb.py +++ b/sgl-model-gateway/bindings/python/src/sglang_router/mini_lb.py @@ -258,12 +258,18 @@ async def health_generate(): @app.post("/flush_cache") -async def flush_cache(): +async def flush_cache(timeout: Optional[float] = None): + # `timeout` must reach the workers. The scheduler treats a missing or + # non-positive timeout as "flush now, skip the idle check", so dropping it + # here frees KV buffers while a PD KV transfer is still reading them: the + # transfer then fails for real and the peer session gets blacklisted. + # Forwarding it keeps the scheduler on its deferred, drain-first path. + params = None if timeout is None else {"timeout": timeout} async with aiohttp.ClientSession() as session: # Create the tasks tasks = [] for server in chain(lb.prefill_urls, lb.decode_urls): - tasks.append(session.post(f"{server}/flush_cache")) + tasks.append(session.post(f"{server}/flush_cache", params=params)) for i, response in enumerate(asyncio.as_completed(tasks)): await response return Response(status_code=200)