From 71a72b8fea18fa104d371f128fee420b2aa9b9cd Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Mon, 12 Oct 2026 08:58:03 +0800 Subject: [PATCH] gateway g9i: keep large prefills off fast-P even without least-wait MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit regular_plan returned Default when regular_pool_least_wait=false, letting the router policy pick across ALL prefill workers including the fast-P node — ~1/3 of large prefills landed on fast-P under random policy (observed 2026-10-12), polluting its small-prefill queue and L2. With a fast-P configured, 'regular' now explicitly restricts to the regular pool via the least-wait aggregate pick. Default is kept only when the fast pool is empty (then it equals the regular pool anyway). Co-authored-by: Kimi --- sgl-model-gateway/src/routers/http/g9i.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/sgl-model-gateway/src/routers/http/g9i.rs b/sgl-model-gateway/src/routers/http/g9i.rs index 9e78a0027e..cbe4314a83 100644 --- a/sgl-model-gateway/src/routers/http/g9i.rs +++ b/sgl-model-gateway/src/routers/http/g9i.rs @@ -744,7 +744,14 @@ impl G9iState { }); } if !self.config.regular_pool_least_wait { - return PrefillPlan::Default; + // Default lets the router policy pick across ALL prefill workers, + // including fast-P — large prefills would land on the fast-P node + // and defeat its isolation (~1/3 of big prefills under random + // policy, observed 2026-10-12). With a fast-P configured, restrict + // "regular" explicitly to the regular pool (least-wait aggregate). + if self.fast_pool(prefill_workers).is_empty() { + return PrefillPlan::Default; + } } match picked { Some(w) => {