diff --git a/sgl-model-gateway/src/routers/http/g9i.rs b/sgl-model-gateway/src/routers/http/g9i.rs index ab256eede3..223986474a 100644 --- a/sgl-model-gateway/src/routers/http/g9i.rs +++ b/sgl-model-gateway/src/routers/http/g9i.rs @@ -559,13 +559,18 @@ impl G9iState { .collect(); let regular_sorted = self.sorted_by_wait(®ular); - let mut probe: Option<(&Arc, u64)> = None; + let mut probes: Vec<(&Arc, ProbeHit)> = Vec::new(); let mut last_err: Option = None; for cand in fast_sorted.iter().chain(regular_sorted.iter()) { - match self.query_new_tokens(client, cand.as_ref(), body).await { - Ok(Some(n)) => { - probe = Some((*cand, n)); - break; + match self.query_hit(client, cand.as_ref(), body).await { + Ok(Some(h)) => { + let fully_local = h.local_new() == 0; + probes.push((*cand, h)); + if fully_local { + // Cannot do better than a node that holds the whole + // prompt locally; save the remaining probes. + break; + } } Ok(None) => { // Unclassifiable (multimodal 400): same answer on every @@ -580,7 +585,22 @@ impl G9iState { } } } - let (hit_worker, new_tokens) = match probe { + // Best local hit wins; ties keep the first probed (fast pool first), + // so a fresh short prompt with no hits anywhere still lands on fast-P. + // L3 hits are deliberately NOT counted here: L3 is shared, so an L3 + // hit on node X does not mean X can serve the prefix — mid-put pages + // are metadata-visible before fetchable, which previously sent + // multi-turn requests onto fast-P with a local miss while their home + // regular-P sat on an immediate L2 hit. + let mut best: Option<(&Arc, u64)> = None; + for (w, h) in &probes { + let n = h.local_new(); + match best { + Some((_, bn)) if bn <= n => {} + _ => best = Some((*w, n)), + } + } + let (hit_worker, new_tokens) = match best { Some(x) => x, None => { warn!( @@ -597,13 +617,14 @@ impl G9iState { return (self.regular_plan(&prefill_workers), "regular"); } - // Read-your-own-node: the prefix lives on whichever node answered the - // probe. If that node is a regular-P, route there directly — an - // immediate L2 hit beats fast-P, which would miss locally and either - // recompute the whole prefix or wait for the cross-node L3 backup to - // become visible (seconds for long prefixes). Multi-turn sessions - // therefore stay on their home P; fast-P only serves sessions whose - // prefix it already holds (its L2) or brand-new short prefills. + // Read-your-own-node: route to the node that actually holds the + // prefix locally. If that node is a regular-P, an immediate L2 hit + // beats fast-P, which would miss locally and either recompute the + // whole prefix or wait for the cross-node L3 backup to become + // visible (seconds for long prefixes; observed 10-20s at 76k). + // Multi-turn sessions therefore stay on their home P; fast-P serves + // sessions whose prefix it already holds (its own L2) and brand-new + // short prefills. let hit_is_fast = self .config .fast_prefill_urls @@ -634,7 +655,7 @@ impl G9iState { } metrics::counter!(METRIC_DECISIONS, "path" => "routed_home_hit").increment(1); debug!( - "G9i: small prefill (new={} < {}) with prefix on regular node {} — routing home (L2 hit)", + "G9i: small prefill (local_new={} < {}) with prefix on regular node {} — routing home (L2 hit)", new_tokens, self.config.max_new_tokens, hit_worker.url(), @@ -645,7 +666,7 @@ impl G9iState { ); } - // Small prefill whose prefix fast-P holds -> fast-P pool, least-wait pick. + // Small prefill whose prefix fast-P holds locally -> fast-P pool, least-wait pick. let (picked, est_wait, fresh) = self.least_wait(&fast_pool); if fresh == 0 { // Lost our snapshots between probe and pick; fail-open to regular. @@ -762,13 +783,33 @@ impl G9iState { Ok(()) } - /// Probe a fast-P's /query_cache_hit; Ok(None) = "not classifiable". - async fn query_new_tokens( + /// Breakdown of one node's /query_cache_hit answer. L3 is reported + /// separately: it is shared storage, so an L3 hit on a node says nothing + /// about whether THIS node can serve the prefix quickly (mid-put pages + /// are metadata-visible before they are fetchable — observed 2026-10-12). + /// Locality decisions must use l1 + l2 only. + #[derive(Clone, Copy, Debug)] + struct ProbeHit { + total: u64, + l1: u64, + l2: u64, + #[allow(dead_code)] + l3: u64, + } + + impl ProbeHit { + fn local_new(&self) -> u64 { + self.total.saturating_sub(self.l1.saturating_add(self.l2)) + } + } + + /// Probe a node's /query_cache_hit; Ok(None) = "not classifiable". + async fn query_hit( &self, client: &Client, worker: &dyn Worker, body: &Value, - ) -> Result, String> { + ) -> Result, String> { let url = format!("{}/query_cache_hit", normalize_url(&worker.url().to_string())); let mut req = client .post(&url) @@ -790,9 +831,13 @@ impl G9iState { return Ok(None); } let json: Value = resp.json().await.map_err(|e| e.to_string())?; - Ok(json - .get("new_tokens") - .and_then(|v| v.as_u64())) + let get_u64 = |k: &str| json.get(k).and_then(|v| v.as_u64()).unwrap_or(0); + Ok(Some(ProbeHit { + total: get_u64("total_tokens"), + l1: get_u64("l1_hit_tokens"), + l2: get_u64("l2_hit_tokens"), + l3: get_u64("l3_hit_tokens"), + })) } }