gateway g9i: classify on local (L1+L2) hits only, probe the whole pool
Follow-up to the home-hit routing fix. Two flaws remained: 1. The engine's /query_cache_hit new_tokens counts shared-L3 hits. Mooncake pages are metadata-visible (Exist) before the put finishes, so fast-P claimed long prefixes it could not actually serve: the gateway classified 'small prefill, fast-P has it' while the engine prefetch found nothing and recomputed 76k cold — and the probe loop broke at fast-P before ever asking the home regular-P, which held an immediate L2 copy. 2. The probe loop stopped at the first answering node, so 'which node holds the prefix' was only ever 'fast-P or whoever got asked first'. Now every fresh candidate is probed (fast pool first, early-out only on a fully-local prompt), and the routing decision uses local_new = total - l1 - l2. Ties keep probe order so fresh short prompts with no hits still land on fast-P. Co-authored-by: Kimi
This commit is contained in:
@@ -559,13 +559,18 @@ impl G9iState {
|
||||
.collect();
|
||||
let regular_sorted = self.sorted_by_wait(®ular);
|
||||
|
||||
let mut probe: Option<(&Arc<dyn Worker>, u64)> = None;
|
||||
let mut probes: Vec<(&Arc<dyn Worker>, ProbeHit)> = Vec::new();
|
||||
let mut last_err: Option<String> = None;
|
||||
for cand in fast_sorted.iter().chain(regular_sorted.iter()) {
|
||||
match self.query_new_tokens(client, cand.as_ref(), body).await {
|
||||
Ok(Some(n)) => {
|
||||
probe = Some((*cand, n));
|
||||
break;
|
||||
match self.query_hit(client, cand.as_ref(), body).await {
|
||||
Ok(Some(h)) => {
|
||||
let fully_local = h.local_new() == 0;
|
||||
probes.push((*cand, h));
|
||||
if fully_local {
|
||||
// Cannot do better than a node that holds the whole
|
||||
// prompt locally; save the remaining probes.
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(None) => {
|
||||
// Unclassifiable (multimodal 400): same answer on every
|
||||
@@ -580,7 +585,22 @@ impl G9iState {
|
||||
}
|
||||
}
|
||||
}
|
||||
let (hit_worker, new_tokens) = match probe {
|
||||
// Best local hit wins; ties keep the first probed (fast pool first),
|
||||
// so a fresh short prompt with no hits anywhere still lands on fast-P.
|
||||
// L3 hits are deliberately NOT counted here: L3 is shared, so an L3
|
||||
// hit on node X does not mean X can serve the prefix — mid-put pages
|
||||
// are metadata-visible before fetchable, which previously sent
|
||||
// multi-turn requests onto fast-P with a local miss while their home
|
||||
// regular-P sat on an immediate L2 hit.
|
||||
let mut best: Option<(&Arc<dyn Worker>, u64)> = None;
|
||||
for (w, h) in &probes {
|
||||
let n = h.local_new();
|
||||
match best {
|
||||
Some((_, bn)) if bn <= n => {}
|
||||
_ => best = Some((*w, n)),
|
||||
}
|
||||
}
|
||||
let (hit_worker, new_tokens) = match best {
|
||||
Some(x) => x,
|
||||
None => {
|
||||
warn!(
|
||||
@@ -597,13 +617,14 @@ impl G9iState {
|
||||
return (self.regular_plan(&prefill_workers), "regular");
|
||||
}
|
||||
|
||||
// Read-your-own-node: the prefix lives on whichever node answered the
|
||||
// probe. If that node is a regular-P, route there directly — an
|
||||
// immediate L2 hit beats fast-P, which would miss locally and either
|
||||
// recompute the whole prefix or wait for the cross-node L3 backup to
|
||||
// become visible (seconds for long prefixes). Multi-turn sessions
|
||||
// therefore stay on their home P; fast-P only serves sessions whose
|
||||
// prefix it already holds (its L2) or brand-new short prefills.
|
||||
// Read-your-own-node: route to the node that actually holds the
|
||||
// prefix locally. If that node is a regular-P, an immediate L2 hit
|
||||
// beats fast-P, which would miss locally and either recompute the
|
||||
// whole prefix or wait for the cross-node L3 backup to become
|
||||
// visible (seconds for long prefixes; observed 10-20s at 76k).
|
||||
// Multi-turn sessions therefore stay on their home P; fast-P serves
|
||||
// sessions whose prefix it already holds (its own L2) and brand-new
|
||||
// short prefills.
|
||||
let hit_is_fast = self
|
||||
.config
|
||||
.fast_prefill_urls
|
||||
@@ -634,7 +655,7 @@ impl G9iState {
|
||||
}
|
||||
metrics::counter!(METRIC_DECISIONS, "path" => "routed_home_hit").increment(1);
|
||||
debug!(
|
||||
"G9i: small prefill (new={} < {}) with prefix on regular node {} — routing home (L2 hit)",
|
||||
"G9i: small prefill (local_new={} < {}) with prefix on regular node {} — routing home (L2 hit)",
|
||||
new_tokens,
|
||||
self.config.max_new_tokens,
|
||||
hit_worker.url(),
|
||||
@@ -645,7 +666,7 @@ impl G9iState {
|
||||
);
|
||||
}
|
||||
|
||||
// Small prefill whose prefix fast-P holds -> fast-P pool, least-wait pick.
|
||||
// Small prefill whose prefix fast-P holds locally -> fast-P pool, least-wait pick.
|
||||
let (picked, est_wait, fresh) = self.least_wait(&fast_pool);
|
||||
if fresh == 0 {
|
||||
// Lost our snapshots between probe and pick; fail-open to regular.
|
||||
@@ -762,13 +783,33 @@ impl G9iState {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Probe a fast-P's /query_cache_hit; Ok(None) = "not classifiable".
|
||||
async fn query_new_tokens(
|
||||
/// Breakdown of one node's /query_cache_hit answer. L3 is reported
|
||||
/// separately: it is shared storage, so an L3 hit on a node says nothing
|
||||
/// about whether THIS node can serve the prefix quickly (mid-put pages
|
||||
/// are metadata-visible before they are fetchable — observed 2026-10-12).
|
||||
/// Locality decisions must use l1 + l2 only.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
struct ProbeHit {
|
||||
total: u64,
|
||||
l1: u64,
|
||||
l2: u64,
|
||||
#[allow(dead_code)]
|
||||
l3: u64,
|
||||
}
|
||||
|
||||
impl ProbeHit {
|
||||
fn local_new(&self) -> u64 {
|
||||
self.total.saturating_sub(self.l1.saturating_add(self.l2))
|
||||
}
|
||||
}
|
||||
|
||||
/// Probe a node's /query_cache_hit; Ok(None) = "not classifiable".
|
||||
async fn query_hit(
|
||||
&self,
|
||||
client: &Client,
|
||||
worker: &dyn Worker,
|
||||
body: &Value,
|
||||
) -> Result<Option<u64>, String> {
|
||||
) -> Result<Option<ProbeHit>, String> {
|
||||
let url = format!("{}/query_cache_hit", normalize_url(&worker.url().to_string()));
|
||||
let mut req = client
|
||||
.post(&url)
|
||||
@@ -790,9 +831,13 @@ impl G9iState {
|
||||
return Ok(None);
|
||||
}
|
||||
let json: Value = resp.json().await.map_err(|e| e.to_string())?;
|
||||
Ok(json
|
||||
.get("new_tokens")
|
||||
.and_then(|v| v.as_u64()))
|
||||
let get_u64 = |k: &str| json.get(k).and_then(|v| v.as_u64()).unwrap_or(0);
|
||||
Ok(Some(ProbeHit {
|
||||
total: get_u64("total_tokens"),
|
||||
l1: get_u64("l1_hit_tokens"),
|
||||
l2: get_u64("l2_hit_tokens"),
|
||||
l3: get_u64("l3_hit_tokens"),
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user