From f464e77d17a3908ad0ea32547b1e8b039bcbd354 Mon Sep 17 00:00:00 2001 From: Shangming Cai Date: Mon, 24 Aug 2026 15:32:57 +0800 Subject: [PATCH] fix(mini-lb): forward the flush_cache timeout param to workers (#36150) --- .../bindings/python/src/sglang_router/mini_lb.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/sgl-model-gateway/bindings/python/src/sglang_router/mini_lb.py b/sgl-model-gateway/bindings/python/src/sglang_router/mini_lb.py index abdbd9523..2156a2dfe 100644 --- a/sgl-model-gateway/bindings/python/src/sglang_router/mini_lb.py +++ b/sgl-model-gateway/bindings/python/src/sglang_router/mini_lb.py @@ -258,12 +258,18 @@ async def health_generate(): @app.post("/flush_cache") -async def flush_cache(): +async def flush_cache(timeout: Optional[float] = None): + # `timeout` must reach the workers. The scheduler treats a missing or + # non-positive timeout as "flush now, skip the idle check", so dropping it + # here frees KV buffers while a PD KV transfer is still reading them: the + # transfer then fails for real and the peer session gets blacklisted. + # Forwarding it keeps the scheduler on its deferred, drain-first path. + params = None if timeout is None else {"timeout": timeout} async with aiohttp.ClientSession() as session: # Create the tasks tasks = [] for server in chain(lb.prefill_urls, lb.decode_urls): - tasks.append(session.post(f"{server}/flush_cache")) + tasks.append(session.post(f"{server}/flush_cache", params=params)) for i, response in enumerate(asyncio.as_completed(tasks)): await response return Response(status_code=200)