diff --git a/sgl-model-gateway/src/routers/http/g9i.rs b/sgl-model-gateway/src/routers/http/g9i.rs index 2e40f2c880..640e49e079 100644 --- a/sgl-model-gateway/src/routers/http/g9i.rs +++ b/sgl-model-gateway/src/routers/http/g9i.rs @@ -2031,6 +2031,7 @@ impl G9iState { ctx: &ShadowResponseContext, resp_json: &Value, actual_ttft_secs: Option, + pre_strip_breakdown: Option, ) { if let Some(lock) = &self.shadow { if let Ok(mut table) = lock.write() { @@ -2039,9 +2040,25 @@ impl G9iState { &ctx.chain, &ctx.node, resp_json, + pre_strip_breakdown, ); } } + // T22: shadow hit/miss prediction vs the engine's actual cached + // tokens (2x2 agreement). Fires only on responses with real usage; + // error/abort paths go through shadow_release instead. This is the + // always-on divergence signal between the shadow ledger and the + // engine's real cache state. + if let Some(total) = super::g9i_shadow::extract_cached_total(resp_json, pre_strip_breakdown) + { + let (shadow, engine) = super::g9i_shadow::agreement_labels(ctx.shadow_hit, total); + metrics::counter!( + super::g9i_shadow::METRIC_SHADOW_ENGINE_AGREEMENT, + "shadow" => shadow, + "engine" => engine + ) + .increment(1); + } if let Some(lock) = &self.shadow_queue { if let Ok(mut q) = lock.write() { if let Some(class) = ctx.class { diff --git a/sgl-model-gateway/src/routers/http/g9i_shadow.rs b/sgl-model-gateway/src/routers/http/g9i_shadow.rs index c80b824d65..45b96c2d7e 100644 --- a/sgl-model-gateway/src/routers/http/g9i_shadow.rs +++ b/sgl-model-gateway/src/routers/http/g9i_shadow.rs @@ -70,6 +70,13 @@ const METRIC_EVICTED_AGE: &str = "smg_g9i_shadow_prefix_table_evicted_age_second const METRIC_CACHE_HIT_TOKENS: &str = "smg_g9i_shadow_cache_hit_tokens_total"; const METRIC_SHADOW_DECISION: &str = "smg_g9i_shadow_decision_total"; const METRIC_AGREEMENT: &str = "smg_g9i_shadow_agreement_total"; +/// T22: shadow hit/miss prediction vs the engine's actual cached tokens +/// (2x2, labels shadow=hit|miss x engine=hit|miss). shadow=hit iff the +/// classifier anchored on a real prefix-table hit; engine=hit iff the +/// response reports any cached tokens (standard +/// `usage.prompt_tokens_details.cached_tokens` — the only channel PD +/// streaming fills — or the sglext tier breakdown pre-strip). +pub const METRIC_SHADOW_ENGINE_AGREEMENT: &str = "smg_g9i_shadow_engine_agreement_total"; const METRIC_LOOKUP_DEPTH: &str = "smg_g9i_shadow_prefix_lookup_depth"; const METRIC_LOOKUP_RESULT: &str = "smg_g9i_shadow_prefix_table_lookup_total"; const METRIC_DIVERGENCE: &str = "smg_g9i_shadow_prefix_divergence_total"; @@ -1534,6 +1541,11 @@ pub struct ShadowResponseContext { pub decode_node: String, pub strip_usage: bool, pub strip_cached_details: bool, + /// T22: the request-time chain lookup anchored on a real prefix-table + /// hit (decision.matched_node.is_some() — placeholders don't count). + /// Paired with the response's actual cached tokens for the + /// shadow×engine hit/miss agreement counter. + pub shadow_hit: bool, /// Shadow classification (None only in defensive edge cases — a /// classifiable request always gets classified at request time). /// Queue admit/complete are skipped without it, keeping the queue @@ -1561,6 +1573,10 @@ pub struct ShadowResponseContext { impl ShadowResponseContext { pub fn from_request(ctx: ShadowRequestContext, node: String, decode_node: String) -> Self { + let shadow_hit = ctx + .decision + .as_ref() + .is_some_and(|d| d.matched_node.is_some()); let (class, est_new_tokens, est_ttft_secs, est_tp_secs, est_td_secs) = match &ctx.decision { Some(d) => ( Some(d.class), @@ -1577,6 +1593,7 @@ impl ShadowResponseContext { decode_node, strip_usage: ctx.strip_usage, strip_cached_details: ctx.strip_cached_details, + shadow_hit, class, est_new_tokens, est_ttft_secs, @@ -1604,17 +1621,26 @@ pub fn extract_prompt_tokens(resp: &Value) -> Option { /// per-tier hit tokens are counted (`tier=l1|l2|l3`); without it the table /// write still happens (the anchor token count is the table's payload — /// the tier split is §7.1 observability, not table state). +/// +/// `pre_strip_breakdown`: non-stream callers strip the injected +/// `cached_tokens_details` from the body BEFORE this function sees the JSON +/// (the response rewrite), so they pass the breakdown extracted pre-strip; +/// streaming callers harvest the usage event pre-strip already and pass +/// None (the in-function extraction then works). Passing the pre-strip +/// value fixes the 2026-10-17 order bug that kept +/// `smg_g9i_shadow_cache_hit_tokens_total` permanently absent. pub fn record_response_json( table: &mut PrefixTable, chain: &[u64], node: &str, resp: &Value, + pre_strip_breakdown: Option, ) -> bool { let Some(prompt_tokens) = extract_prompt_tokens(resp) else { return false; }; table.record(chain, node, prompt_tokens); - if let Some(bd) = extract_cached_breakdown(resp) { + if let Some(bd) = pre_strip_breakdown.or_else(|| extract_cached_breakdown(resp)) { metrics::counter!(METRIC_CACHE_HIT_TOKENS, "tier" => "l1").increment(bd.device); metrics::counter!(METRIC_CACHE_HIT_TOKENS, "tier" => "l2").increment(bd.host); metrics::counter!(METRIC_CACHE_HIT_TOKENS, "tier" => "l3").increment(bd.storage); @@ -1643,22 +1669,62 @@ pub fn extract_cached_breakdown(resp: &Value) -> Option { }) } +/// Total cached tokens reported by the response, whichever channel carries +/// it: the standard OpenAI `usage.prompt_tokens_details.cached_tokens` +/// (the only channel PD streaming fills) or the sglext tier breakdown +/// (non-streaming; callers pass the pre-strip value because the JSON here +/// may already be stripped). `None` when the response carries no real +/// usage at all; a present-but-detail-less usage means "the engine +/// reported a completion with no cache hit" and yields `Some(0)` — the +/// hit/miss agreement counter wants that miss signal, not silence. +pub fn extract_cached_total(resp: &Value, pre_strip_breakdown: Option) -> Option { + let usage = resp.get("usage")?; + extract_prompt_tokens(resp)?; + if let Some(v) = usage + .get("prompt_tokens_details") + .and_then(|d| d.get("cached_tokens")) + .and_then(Value::as_u64) + { + return Some(v); + } + if let Some(bd) = pre_strip_breakdown.or_else(|| extract_cached_breakdown(resp)) { + return Some(bd.device + bd.host + bd.storage); + } + Some(0) +} + +/// T22 agreement labels: shadow=hit iff the request-time lookup anchored on +/// a real prefix-table entry; engine=hit iff the response reported any +/// cached tokens. Pure so the mapping is unit-tested once and never drifts +/// between the streaming / non-streaming call sites. +pub fn agreement_labels(shadow_hit: bool, cached_total: u64) -> (&'static str, &'static str) { + let shadow = if shadow_hit { "hit" } else { "miss" }; + let engine = if cached_total > 0 { "hit" } else { "miss" }; + (shadow, engine) +} + /// Non-streaming bookkeeping helper: parse the response body and strip /// `cached_tokens_details` when we injected the request flag. -/// Returns `(parsed_json_for_record, rewritten_body)`. Both are `None` on -/// unparseable bodies (fail-open: forward the original bytes untouched); -/// `rewritten_body` is `None` when nothing was stripped, so callers can -/// forward the original bytes byte-identically. -pub fn strip_and_extract(body: &[u8], strip_cached_details: bool) -> (Option, Option>) { +/// Returns `(parsed_json_for_record, rewritten_body, pre_strip_breakdown)`. +/// All are `None` on unparseable bodies (fail-open: forward the original +/// bytes untouched); `rewritten_body` is `None` when nothing was stripped, +/// so callers can forward the original bytes byte-identically. The +/// breakdown is extracted BEFORE stripping — stripping first blinded the +/// tier bookkeeping (the 2026-10-17 order bug). +pub fn strip_and_extract( + body: &[u8], + strip_cached_details: bool, +) -> (Option, Option>, Option) { let Ok(mut json) = serde_json::from_slice::(body) else { - return (None, None); + return (None, None, None); }; + let breakdown = extract_cached_breakdown(&json); let rewritten = if strip_cached_details && strip_cached_details_from(&mut json) { serde_json::to_vec(&json).ok() } else { None }; - (Some(json), rewritten) + (Some(json), rewritten, breakdown) } /// Process one upstream SSE chunk: harvest the usage event (if any) for @@ -1927,14 +1993,15 @@ mod tests { fn record_response_json_requires_prompt_tokens() { let mut t = table(); // No usage -> no write (429 / aborted stream discipline). - assert!(!record_response_json(&mut t, &[1], "n", &json!({"id": "x"}))); + assert!(!record_response_json(&mut t, &[1], "n", &json!({"id": "x"}), None)); assert!(t.is_empty()); // usage without prompt_tokens -> no write. assert!(!record_response_json( &mut t, &[1], "n", - &json!({"usage": {"completion_tokens": 3}}) + &json!({"usage": {"completion_tokens": 3}}), + None )); assert!(t.is_empty()); // Real usage -> written, breakdown optional. @@ -1944,7 +2011,7 @@ mod tests { "cached_tokens_details": {"device": 30, "host": 10, "storage": 2} } }); - assert!(record_response_json(&mut t, &[1], "n", &resp)); + assert!(record_response_json(&mut t, &[1], "n", &resp, None)); let (_, e) = t.lookup(&[1]).unwrap(); assert_eq!(e.prompt_tokens, 42); assert_eq!( @@ -2058,21 +2125,39 @@ mod tests { #[test] fn strip_and_extract_non_streaming() { let body = br#"{"id":"x","usage":{"prompt_tokens":42,"cached_tokens_details":{"device":40,"host":2,"storage":0}}}"#; - let (json, rewritten) = strip_and_extract(body, true); + let (json, rewritten, breakdown) = strip_and_extract(body, true); assert!(json.is_some()); + // Order-bug regression pin: the breakdown is extracted BEFORE the + // strip, so the tier counters see it even though the returned JSON + // no longer carries the field. + assert_eq!( + breakdown, + Some(CachedBreakdown { + device: 40, + host: 2, + storage: 0 + }) + ); + assert!( + extract_cached_breakdown(&json.unwrap()).is_none(), + "returned json is post-strip" + ); let rewritten = rewritten.expect("details stripped"); let text = String::from_utf8_lossy(&rewritten); assert!(text.contains("\"prompt_tokens\":42")); assert!(!text.contains("cached_tokens_details")); - // Nothing to strip -> no rewrite (caller forwards original bytes). - let (_, rewritten) = strip_and_extract(body, false); + // Nothing to strip -> no rewrite (caller forwards original bytes), + // but the breakdown is still extracted. + let (_, rewritten, breakdown) = strip_and_extract(body, false); assert!(rewritten.is_none()); + assert!(breakdown.is_some()); - // Unparseable -> fail-open, no json, no rewrite. - let (json, rewritten) = strip_and_extract(b"not json", true); + // Unparseable -> fail-open, no json, no rewrite, no breakdown. + let (json, rewritten, breakdown) = strip_and_extract(b"not json", true); assert!(json.is_none()); assert!(rewritten.is_none()); + assert!(breakdown.is_none()); } // ==================== sglext placement ==================== @@ -2105,7 +2190,7 @@ mod tests { // sglext carries other client-requested fields -> only the injected // details go away, the rest of the container survives. let body = br#"{"id":"x","usage":{"prompt_tokens":42},"sglext":{"routed_experts":"e0","cached_tokens_details":{"device":40,"host":2,"storage":0}}}"#; - let (json, rewritten) = strip_and_extract(body, true); + let (json, rewritten, _) = strip_and_extract(body, true); assert!(json.is_some()); let text = { let bytes = rewritten.expect("details stripped"); @@ -2118,7 +2203,7 @@ mod tests { // sglext carried ONLY the injected details -> the whole key goes // (the engine omits sglext when all fields are None). let body = br#"{"id":"x","usage":{"prompt_tokens":42},"sglext":{"cached_tokens_details":{"device":40,"host":0}}}"#; - let (_, rewritten) = strip_and_extract(body, true); + let (_, rewritten, _) = strip_and_extract(body, true); let text = { let bytes = rewritten.expect("sglext emptied"); String::from_utf8_lossy(&bytes).into_owned() @@ -2127,6 +2212,93 @@ mod tests { assert!(text.contains("\"prompt_tokens\":42")); } + // ==================== T22: cached total + shadow×engine agreement ==================== + + #[test] + fn extract_cached_total_channels() { + let bd = Some(CachedBreakdown { + device: 30, + host: 10, + storage: 2, + }); + // Standard OpenAI field wins (the only channel PD streaming fills). + let resp = + json!({"usage": {"prompt_tokens": 100, "prompt_tokens_details": {"cached_tokens": 96}}}); + assert_eq!(extract_cached_total(&resp, None), Some(96)); + // Pre-strip breakdown is the fallback (non-streaming post-strip json). + let resp = json!({"usage": {"prompt_tokens": 100}}); + assert_eq!(extract_cached_total(&resp, bd), Some(42)); + // In-json breakdown also works (streaming harvest is pre-strip). + let resp = + json!({"usage": {"prompt_tokens": 100, "cached_tokens_details": {"device": 5, "host": 1}}}); + assert_eq!(extract_cached_total(&resp, None), Some(6)); + // Usage present but no details anywhere = engine reported a + // completion with zero cache hit — a real miss signal, not None. + let resp = json!({"usage": {"prompt_tokens": 100, "prompt_tokens_details": null}}); + assert_eq!(extract_cached_total(&resp, None), Some(0)); + // No usage = no signal at all (error envelopes, aborted streams). + assert_eq!(extract_cached_total(&json!({"id": "x"}), None), None); + assert_eq!( + extract_cached_total(&json!({"usage": {"completion_tokens": 3}}), None), + None + ); + } + + #[test] + fn agreement_labels_truth_table() { + assert_eq!(agreement_labels(true, 1024), ("hit", "hit")); + assert_eq!(agreement_labels(true, 0), ("hit", "miss")); + assert_eq!(agreement_labels(false, 1024), ("miss", "hit")); + assert_eq!(agreement_labels(false, 0), ("miss", "miss")); + } + + #[test] + fn response_context_carries_shadow_hit() { + let body = json!({ + "model": "m", + "messages": [ + {"role": "system", "content": "s"}, + {"role": "user", "content": "u1"}, + {"role": "user", "content": "u2"} + ] + }); + // Real-anchor hit: decision has matched_node -> shadow_hit true. + let mut ctx = ShadowRequestContext::from_body(&body, false).unwrap(); + let mut t = table(); + t.record(&ctx.chain, "http://p1:30000", 100); + let decision = classify(&mut t, &ctx, 1_000_000); + assert!(decision.matched_node.is_some()); + ctx.decision = Some(decision); + let rctx = ShadowResponseContext::from_request( + ctx, + "http://p1:30000".into(), + "http://d:30300".into(), + ); + assert!(rctx.shadow_hit); + + // Miss on a fresh table: no anchor -> shadow_hit false. + let mut ctx = ShadowRequestContext::from_body(&body, false).unwrap(); + let mut t = table(); + let decision = classify(&mut t, &ctx, 1_000_000); + assert!(decision.matched_node.is_none()); + ctx.decision = Some(decision); + let rctx = ShadowResponseContext::from_request( + ctx, + "http://p1:30000".into(), + "http://d:30300".into(), + ); + assert!(!rctx.shadow_hit); + + // No decision at all (defensive edge) -> false. + let ctx = ShadowRequestContext::from_body(&body, false).unwrap(); + let rctx = ShadowResponseContext::from_request( + ctx, + "http://p1:30000".into(), + "http://d:30300".into(), + ); + assert!(!rctx.shadow_hit); + } + #[test] fn sse_sglext_standalone_chunk_dropped() { // ids-framed path: the engine emits a standalone `choices=[]` chunk diff --git a/sgl-model-gateway/src/routers/http/pd_router.rs b/sgl-model-gateway/src/routers/http/pd_router.rs index dd72b54b33..af85d0c6d0 100644 --- a/sgl-model-gateway/src/routers/http/pd_router.rs +++ b/sgl-model-gateway/src/routers/http/pd_router.rs @@ -1157,9 +1157,11 @@ impl PDRouter { Ok(decode_body) => { let decode_body = self.shadow_record_non_stream(decode_body, status, &shadow); - let mut response = Response::new(Body::from(decode_body)); - *response.status_mut() = status; - *response.headers_mut() = response_headers; + let response = Self::non_stream_passthrough_response( + status, + decode_body, + response_headers, + ); // T17: non-streaming e2e (observability only). if let Some(g9i) = &self.g9i { g9i.record_nonstream_e2e(start_time.elapsed().as_secs_f64()); @@ -1469,7 +1471,11 @@ impl PDRouter { // (malformed) repeat usage event from // double-completing the queue slot. if !shadow_done { - g9i.shadow_record(sctx, &json, shadow_ttft_secs); + // Streaming harvests the usage + // event pre-strip, so the + // breakdown extraction inside + // works unaided (None). + g9i.shadow_record(sctx, &json, shadow_ttft_secs, None); shadow_done = true; } } @@ -1662,12 +1668,15 @@ impl PDRouter { g9i.shadow_release(ctx); return body; } - let (parsed, rewritten) = + let (parsed, rewritten, breakdown) = super::g9i_shadow::strip_and_extract(&body, ctx.strip_cached_details); match &parsed { // TTFT calibration is streaming-only (non-streaming has no - // observable first-token moment) -> None. - Some(json) => g9i.shadow_record(ctx, json, None), + // observable first-token moment) -> None. The breakdown was + // extracted pre-strip and is passed through — the parsed JSON + // here is post-strip, so re-extracting from it would always + // miss (the 2026-10-17 order bug). + Some(json) => g9i.shadow_record(ctx, json, None, breakdown), // Unparseable success body: nothing to record, but the dispatch // is over — release rather than leak the outstanding count. None => g9i.shadow_release(ctx), @@ -1678,6 +1687,27 @@ impl PDRouter { } } + /// Assemble the non-streaming passthrough response. The shadow strip + /// may have REWRITTEN (shortened) the body, so the upstream + /// Content-Length in `headers` is stale; forwarding it verbatim + /// truncated cache-hit responses at exactly the stripped length (the + /// 2026-10-17 IncompleteRead incident). Pin Content-Length to the + /// actual body length — correct for rewritten and untouched bodies + /// alike. + fn non_stream_passthrough_response( + status: StatusCode, + body: bytes::Bytes, + mut headers: HeaderMap, + ) -> Response { + if let Ok(v) = HeaderValue::from_str(body.len().to_string().as_str()) { + headers.insert(CONTENT_LENGTH, v); + } + let mut response = Response::new(Body::from(body)); + *response.status_mut() = status; + *response.headers_mut() = headers; + response + } + // Helper to process prefill response and extract body if needed for logprobs async fn process_prefill_response( &self, @@ -2205,6 +2235,39 @@ mod tests { Box::new(worker) } + #[test] + fn non_stream_passthrough_pins_content_length_to_body() { + // 2026-10-17 IncompleteRead regression: the shadow strip shortens + // the body on engine cache hits; the upstream Content-Length must + // never survive onto the rewritten body. + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_LENGTH, HeaderValue::from_static("637")); + let body = bytes::Bytes::from_static(b"{}"); + let resp = + PDRouter::non_stream_passthrough_response(StatusCode::OK, body.clone(), headers); + assert_eq!( + resp.headers().get(CONTENT_LENGTH).unwrap(), + &HeaderValue::from_static("2"), + "stale upstream length is replaced by the actual body length" + ); + + // Untouched body: the pinned length matches the original. + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_LENGTH, HeaderValue::from_static("2")); + headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); + let resp = PDRouter::non_stream_passthrough_response(StatusCode::OK, body, headers); + assert_eq!( + resp.headers().get(CONTENT_LENGTH).unwrap(), + &HeaderValue::from_static("2") + ); + assert_eq!( + resp.headers().get(CONTENT_TYPE).unwrap(), + &HeaderValue::from_static("application/json"), + "other preserved headers survive" + ); + assert_eq!(resp.status(), StatusCode::OK); + } + #[test] fn test_chat_request_text_uses_full_conversation() { // Regression test for https://github.com/sgl-project/sglang/issues/26263