gateway g9i: fix non-stream CL truncation on shadow strip + pre-strip breakdown extraction + shadowxengine agreement metric (T22)
- Bug A (user-visible): non-stream passthrough forwarded the upstream
Content-Length after the shadow strip had shortened the body ->
cache-hit non-stream responses truncated (~60B, IncompleteRead).
non_stream_passthrough_response now pins CL to the actual body length.
- Bug B (order): strip_and_extract returned the post-strip JSON to
shadow_record, so extract_cached_breakdown always missed and
smg_g9i_shadow_cache_hit_tokens_total never fired. The breakdown is now
extracted pre-strip and threaded through record_response_json.
- T22 observability: extract_cached_total (standard
usage.prompt_tokens_details.cached_tokens first, tier breakdown as
fallback) + smg_g9i_shadow_engine_agreement_total{shadow,engine} 2x2
counter pairing the request-time lookup hit with the engine's actual
cached tokens; ShadowResponseContext carries shadow_hit.
This commit is contained in:
@@ -2031,6 +2031,7 @@ impl G9iState {
|
||||
ctx: &ShadowResponseContext,
|
||||
resp_json: &Value,
|
||||
actual_ttft_secs: Option<f64>,
|
||||
pre_strip_breakdown: Option<super::g9i_shadow::CachedBreakdown>,
|
||||
) {
|
||||
if let Some(lock) = &self.shadow {
|
||||
if let Ok(mut table) = lock.write() {
|
||||
@@ -2039,9 +2040,25 @@ impl G9iState {
|
||||
&ctx.chain,
|
||||
&ctx.node,
|
||||
resp_json,
|
||||
pre_strip_breakdown,
|
||||
);
|
||||
}
|
||||
}
|
||||
// T22: shadow hit/miss prediction vs the engine's actual cached
|
||||
// tokens (2x2 agreement). Fires only on responses with real usage;
|
||||
// error/abort paths go through shadow_release instead. This is the
|
||||
// always-on divergence signal between the shadow ledger and the
|
||||
// engine's real cache state.
|
||||
if let Some(total) = super::g9i_shadow::extract_cached_total(resp_json, pre_strip_breakdown)
|
||||
{
|
||||
let (shadow, engine) = super::g9i_shadow::agreement_labels(ctx.shadow_hit, total);
|
||||
metrics::counter!(
|
||||
super::g9i_shadow::METRIC_SHADOW_ENGINE_AGREEMENT,
|
||||
"shadow" => shadow,
|
||||
"engine" => engine
|
||||
)
|
||||
.increment(1);
|
||||
}
|
||||
if let Some(lock) = &self.shadow_queue {
|
||||
if let Ok(mut q) = lock.write() {
|
||||
if let Some(class) = ctx.class {
|
||||
|
||||
@@ -70,6 +70,13 @@ const METRIC_EVICTED_AGE: &str = "smg_g9i_shadow_prefix_table_evicted_age_second
|
||||
const METRIC_CACHE_HIT_TOKENS: &str = "smg_g9i_shadow_cache_hit_tokens_total";
|
||||
const METRIC_SHADOW_DECISION: &str = "smg_g9i_shadow_decision_total";
|
||||
const METRIC_AGREEMENT: &str = "smg_g9i_shadow_agreement_total";
|
||||
/// T22: shadow hit/miss prediction vs the engine's actual cached tokens
|
||||
/// (2x2, labels shadow=hit|miss x engine=hit|miss). shadow=hit iff the
|
||||
/// classifier anchored on a real prefix-table hit; engine=hit iff the
|
||||
/// response reports any cached tokens (standard
|
||||
/// `usage.prompt_tokens_details.cached_tokens` — the only channel PD
|
||||
/// streaming fills — or the sglext tier breakdown pre-strip).
|
||||
pub const METRIC_SHADOW_ENGINE_AGREEMENT: &str = "smg_g9i_shadow_engine_agreement_total";
|
||||
const METRIC_LOOKUP_DEPTH: &str = "smg_g9i_shadow_prefix_lookup_depth";
|
||||
const METRIC_LOOKUP_RESULT: &str = "smg_g9i_shadow_prefix_table_lookup_total";
|
||||
const METRIC_DIVERGENCE: &str = "smg_g9i_shadow_prefix_divergence_total";
|
||||
@@ -1534,6 +1541,11 @@ pub struct ShadowResponseContext {
|
||||
pub decode_node: String,
|
||||
pub strip_usage: bool,
|
||||
pub strip_cached_details: bool,
|
||||
/// T22: the request-time chain lookup anchored on a real prefix-table
|
||||
/// hit (decision.matched_node.is_some() — placeholders don't count).
|
||||
/// Paired with the response's actual cached tokens for the
|
||||
/// shadow×engine hit/miss agreement counter.
|
||||
pub shadow_hit: bool,
|
||||
/// Shadow classification (None only in defensive edge cases — a
|
||||
/// classifiable request always gets classified at request time).
|
||||
/// Queue admit/complete are skipped without it, keeping the queue
|
||||
@@ -1561,6 +1573,10 @@ pub struct ShadowResponseContext {
|
||||
|
||||
impl ShadowResponseContext {
|
||||
pub fn from_request(ctx: ShadowRequestContext, node: String, decode_node: String) -> Self {
|
||||
let shadow_hit = ctx
|
||||
.decision
|
||||
.as_ref()
|
||||
.is_some_and(|d| d.matched_node.is_some());
|
||||
let (class, est_new_tokens, est_ttft_secs, est_tp_secs, est_td_secs) = match &ctx.decision {
|
||||
Some(d) => (
|
||||
Some(d.class),
|
||||
@@ -1577,6 +1593,7 @@ impl ShadowResponseContext {
|
||||
decode_node,
|
||||
strip_usage: ctx.strip_usage,
|
||||
strip_cached_details: ctx.strip_cached_details,
|
||||
shadow_hit,
|
||||
class,
|
||||
est_new_tokens,
|
||||
est_ttft_secs,
|
||||
@@ -1604,17 +1621,26 @@ pub fn extract_prompt_tokens(resp: &Value) -> Option<u64> {
|
||||
/// per-tier hit tokens are counted (`tier=l1|l2|l3`); without it the table
|
||||
/// write still happens (the anchor token count is the table's payload —
|
||||
/// the tier split is §7.1 observability, not table state).
|
||||
///
|
||||
/// `pre_strip_breakdown`: non-stream callers strip the injected
|
||||
/// `cached_tokens_details` from the body BEFORE this function sees the JSON
|
||||
/// (the response rewrite), so they pass the breakdown extracted pre-strip;
|
||||
/// streaming callers harvest the usage event pre-strip already and pass
|
||||
/// None (the in-function extraction then works). Passing the pre-strip
|
||||
/// value fixes the 2026-10-17 order bug that kept
|
||||
/// `smg_g9i_shadow_cache_hit_tokens_total` permanently absent.
|
||||
pub fn record_response_json(
|
||||
table: &mut PrefixTable,
|
||||
chain: &[u64],
|
||||
node: &str,
|
||||
resp: &Value,
|
||||
pre_strip_breakdown: Option<CachedBreakdown>,
|
||||
) -> bool {
|
||||
let Some(prompt_tokens) = extract_prompt_tokens(resp) else {
|
||||
return false;
|
||||
};
|
||||
table.record(chain, node, prompt_tokens);
|
||||
if let Some(bd) = extract_cached_breakdown(resp) {
|
||||
if let Some(bd) = pre_strip_breakdown.or_else(|| extract_cached_breakdown(resp)) {
|
||||
metrics::counter!(METRIC_CACHE_HIT_TOKENS, "tier" => "l1").increment(bd.device);
|
||||
metrics::counter!(METRIC_CACHE_HIT_TOKENS, "tier" => "l2").increment(bd.host);
|
||||
metrics::counter!(METRIC_CACHE_HIT_TOKENS, "tier" => "l3").increment(bd.storage);
|
||||
@@ -1643,22 +1669,62 @@ pub fn extract_cached_breakdown(resp: &Value) -> Option<CachedBreakdown> {
|
||||
})
|
||||
}
|
||||
|
||||
/// Total cached tokens reported by the response, whichever channel carries
|
||||
/// it: the standard OpenAI `usage.prompt_tokens_details.cached_tokens`
|
||||
/// (the only channel PD streaming fills) or the sglext tier breakdown
|
||||
/// (non-streaming; callers pass the pre-strip value because the JSON here
|
||||
/// may already be stripped). `None` when the response carries no real
|
||||
/// usage at all; a present-but-detail-less usage means "the engine
|
||||
/// reported a completion with no cache hit" and yields `Some(0)` — the
|
||||
/// hit/miss agreement counter wants that miss signal, not silence.
|
||||
pub fn extract_cached_total(resp: &Value, pre_strip_breakdown: Option<CachedBreakdown>) -> Option<u64> {
|
||||
let usage = resp.get("usage")?;
|
||||
extract_prompt_tokens(resp)?;
|
||||
if let Some(v) = usage
|
||||
.get("prompt_tokens_details")
|
||||
.and_then(|d| d.get("cached_tokens"))
|
||||
.and_then(Value::as_u64)
|
||||
{
|
||||
return Some(v);
|
||||
}
|
||||
if let Some(bd) = pre_strip_breakdown.or_else(|| extract_cached_breakdown(resp)) {
|
||||
return Some(bd.device + bd.host + bd.storage);
|
||||
}
|
||||
Some(0)
|
||||
}
|
||||
|
||||
/// T22 agreement labels: shadow=hit iff the request-time lookup anchored on
|
||||
/// a real prefix-table entry; engine=hit iff the response reported any
|
||||
/// cached tokens. Pure so the mapping is unit-tested once and never drifts
|
||||
/// between the streaming / non-streaming call sites.
|
||||
pub fn agreement_labels(shadow_hit: bool, cached_total: u64) -> (&'static str, &'static str) {
|
||||
let shadow = if shadow_hit { "hit" } else { "miss" };
|
||||
let engine = if cached_total > 0 { "hit" } else { "miss" };
|
||||
(shadow, engine)
|
||||
}
|
||||
|
||||
/// Non-streaming bookkeeping helper: parse the response body and strip
|
||||
/// `cached_tokens_details` when we injected the request flag.
|
||||
/// Returns `(parsed_json_for_record, rewritten_body)`. Both are `None` on
|
||||
/// unparseable bodies (fail-open: forward the original bytes untouched);
|
||||
/// `rewritten_body` is `None` when nothing was stripped, so callers can
|
||||
/// forward the original bytes byte-identically.
|
||||
pub fn strip_and_extract(body: &[u8], strip_cached_details: bool) -> (Option<Value>, Option<Vec<u8>>) {
|
||||
/// Returns `(parsed_json_for_record, rewritten_body, pre_strip_breakdown)`.
|
||||
/// All are `None` on unparseable bodies (fail-open: forward the original
|
||||
/// bytes untouched); `rewritten_body` is `None` when nothing was stripped,
|
||||
/// so callers can forward the original bytes byte-identically. The
|
||||
/// breakdown is extracted BEFORE stripping — stripping first blinded the
|
||||
/// tier bookkeeping (the 2026-10-17 order bug).
|
||||
pub fn strip_and_extract(
|
||||
body: &[u8],
|
||||
strip_cached_details: bool,
|
||||
) -> (Option<Value>, Option<Vec<u8>>, Option<CachedBreakdown>) {
|
||||
let Ok(mut json) = serde_json::from_slice::<Value>(body) else {
|
||||
return (None, None);
|
||||
return (None, None, None);
|
||||
};
|
||||
let breakdown = extract_cached_breakdown(&json);
|
||||
let rewritten = if strip_cached_details && strip_cached_details_from(&mut json) {
|
||||
serde_json::to_vec(&json).ok()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
(Some(json), rewritten)
|
||||
(Some(json), rewritten, breakdown)
|
||||
}
|
||||
|
||||
/// Process one upstream SSE chunk: harvest the usage event (if any) for
|
||||
@@ -1927,14 +1993,15 @@ mod tests {
|
||||
fn record_response_json_requires_prompt_tokens() {
|
||||
let mut t = table();
|
||||
// No usage -> no write (429 / aborted stream discipline).
|
||||
assert!(!record_response_json(&mut t, &[1], "n", &json!({"id": "x"})));
|
||||
assert!(!record_response_json(&mut t, &[1], "n", &json!({"id": "x"}), None));
|
||||
assert!(t.is_empty());
|
||||
// usage without prompt_tokens -> no write.
|
||||
assert!(!record_response_json(
|
||||
&mut t,
|
||||
&[1],
|
||||
"n",
|
||||
&json!({"usage": {"completion_tokens": 3}})
|
||||
&json!({"usage": {"completion_tokens": 3}}),
|
||||
None
|
||||
));
|
||||
assert!(t.is_empty());
|
||||
// Real usage -> written, breakdown optional.
|
||||
@@ -1944,7 +2011,7 @@ mod tests {
|
||||
"cached_tokens_details": {"device": 30, "host": 10, "storage": 2}
|
||||
}
|
||||
});
|
||||
assert!(record_response_json(&mut t, &[1], "n", &resp));
|
||||
assert!(record_response_json(&mut t, &[1], "n", &resp, None));
|
||||
let (_, e) = t.lookup(&[1]).unwrap();
|
||||
assert_eq!(e.prompt_tokens, 42);
|
||||
assert_eq!(
|
||||
@@ -2058,21 +2125,39 @@ mod tests {
|
||||
#[test]
|
||||
fn strip_and_extract_non_streaming() {
|
||||
let body = br#"{"id":"x","usage":{"prompt_tokens":42,"cached_tokens_details":{"device":40,"host":2,"storage":0}}}"#;
|
||||
let (json, rewritten) = strip_and_extract(body, true);
|
||||
let (json, rewritten, breakdown) = strip_and_extract(body, true);
|
||||
assert!(json.is_some());
|
||||
// Order-bug regression pin: the breakdown is extracted BEFORE the
|
||||
// strip, so the tier counters see it even though the returned JSON
|
||||
// no longer carries the field.
|
||||
assert_eq!(
|
||||
breakdown,
|
||||
Some(CachedBreakdown {
|
||||
device: 40,
|
||||
host: 2,
|
||||
storage: 0
|
||||
})
|
||||
);
|
||||
assert!(
|
||||
extract_cached_breakdown(&json.unwrap()).is_none(),
|
||||
"returned json is post-strip"
|
||||
);
|
||||
let rewritten = rewritten.expect("details stripped");
|
||||
let text = String::from_utf8_lossy(&rewritten);
|
||||
assert!(text.contains("\"prompt_tokens\":42"));
|
||||
assert!(!text.contains("cached_tokens_details"));
|
||||
|
||||
// Nothing to strip -> no rewrite (caller forwards original bytes).
|
||||
let (_, rewritten) = strip_and_extract(body, false);
|
||||
// Nothing to strip -> no rewrite (caller forwards original bytes),
|
||||
// but the breakdown is still extracted.
|
||||
let (_, rewritten, breakdown) = strip_and_extract(body, false);
|
||||
assert!(rewritten.is_none());
|
||||
assert!(breakdown.is_some());
|
||||
|
||||
// Unparseable -> fail-open, no json, no rewrite.
|
||||
let (json, rewritten) = strip_and_extract(b"not json", true);
|
||||
// Unparseable -> fail-open, no json, no rewrite, no breakdown.
|
||||
let (json, rewritten, breakdown) = strip_and_extract(b"not json", true);
|
||||
assert!(json.is_none());
|
||||
assert!(rewritten.is_none());
|
||||
assert!(breakdown.is_none());
|
||||
}
|
||||
|
||||
// ==================== sglext placement ====================
|
||||
@@ -2105,7 +2190,7 @@ mod tests {
|
||||
// sglext carries other client-requested fields -> only the injected
|
||||
// details go away, the rest of the container survives.
|
||||
let body = br#"{"id":"x","usage":{"prompt_tokens":42},"sglext":{"routed_experts":"e0","cached_tokens_details":{"device":40,"host":2,"storage":0}}}"#;
|
||||
let (json, rewritten) = strip_and_extract(body, true);
|
||||
let (json, rewritten, _) = strip_and_extract(body, true);
|
||||
assert!(json.is_some());
|
||||
let text = {
|
||||
let bytes = rewritten.expect("details stripped");
|
||||
@@ -2118,7 +2203,7 @@ mod tests {
|
||||
// sglext carried ONLY the injected details -> the whole key goes
|
||||
// (the engine omits sglext when all fields are None).
|
||||
let body = br#"{"id":"x","usage":{"prompt_tokens":42},"sglext":{"cached_tokens_details":{"device":40,"host":0}}}"#;
|
||||
let (_, rewritten) = strip_and_extract(body, true);
|
||||
let (_, rewritten, _) = strip_and_extract(body, true);
|
||||
let text = {
|
||||
let bytes = rewritten.expect("sglext emptied");
|
||||
String::from_utf8_lossy(&bytes).into_owned()
|
||||
@@ -2127,6 +2212,93 @@ mod tests {
|
||||
assert!(text.contains("\"prompt_tokens\":42"));
|
||||
}
|
||||
|
||||
// ==================== T22: cached total + shadow×engine agreement ====================
|
||||
|
||||
#[test]
|
||||
fn extract_cached_total_channels() {
|
||||
let bd = Some(CachedBreakdown {
|
||||
device: 30,
|
||||
host: 10,
|
||||
storage: 2,
|
||||
});
|
||||
// Standard OpenAI field wins (the only channel PD streaming fills).
|
||||
let resp =
|
||||
json!({"usage": {"prompt_tokens": 100, "prompt_tokens_details": {"cached_tokens": 96}}});
|
||||
assert_eq!(extract_cached_total(&resp, None), Some(96));
|
||||
// Pre-strip breakdown is the fallback (non-streaming post-strip json).
|
||||
let resp = json!({"usage": {"prompt_tokens": 100}});
|
||||
assert_eq!(extract_cached_total(&resp, bd), Some(42));
|
||||
// In-json breakdown also works (streaming harvest is pre-strip).
|
||||
let resp =
|
||||
json!({"usage": {"prompt_tokens": 100, "cached_tokens_details": {"device": 5, "host": 1}}});
|
||||
assert_eq!(extract_cached_total(&resp, None), Some(6));
|
||||
// Usage present but no details anywhere = engine reported a
|
||||
// completion with zero cache hit — a real miss signal, not None.
|
||||
let resp = json!({"usage": {"prompt_tokens": 100, "prompt_tokens_details": null}});
|
||||
assert_eq!(extract_cached_total(&resp, None), Some(0));
|
||||
// No usage = no signal at all (error envelopes, aborted streams).
|
||||
assert_eq!(extract_cached_total(&json!({"id": "x"}), None), None);
|
||||
assert_eq!(
|
||||
extract_cached_total(&json!({"usage": {"completion_tokens": 3}}), None),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn agreement_labels_truth_table() {
|
||||
assert_eq!(agreement_labels(true, 1024), ("hit", "hit"));
|
||||
assert_eq!(agreement_labels(true, 0), ("hit", "miss"));
|
||||
assert_eq!(agreement_labels(false, 1024), ("miss", "hit"));
|
||||
assert_eq!(agreement_labels(false, 0), ("miss", "miss"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_context_carries_shadow_hit() {
|
||||
let body = json!({
|
||||
"model": "m",
|
||||
"messages": [
|
||||
{"role": "system", "content": "s"},
|
||||
{"role": "user", "content": "u1"},
|
||||
{"role": "user", "content": "u2"}
|
||||
]
|
||||
});
|
||||
// Real-anchor hit: decision has matched_node -> shadow_hit true.
|
||||
let mut ctx = ShadowRequestContext::from_body(&body, false).unwrap();
|
||||
let mut t = table();
|
||||
t.record(&ctx.chain, "http://p1:30000", 100);
|
||||
let decision = classify(&mut t, &ctx, 1_000_000);
|
||||
assert!(decision.matched_node.is_some());
|
||||
ctx.decision = Some(decision);
|
||||
let rctx = ShadowResponseContext::from_request(
|
||||
ctx,
|
||||
"http://p1:30000".into(),
|
||||
"http://d:30300".into(),
|
||||
);
|
||||
assert!(rctx.shadow_hit);
|
||||
|
||||
// Miss on a fresh table: no anchor -> shadow_hit false.
|
||||
let mut ctx = ShadowRequestContext::from_body(&body, false).unwrap();
|
||||
let mut t = table();
|
||||
let decision = classify(&mut t, &ctx, 1_000_000);
|
||||
assert!(decision.matched_node.is_none());
|
||||
ctx.decision = Some(decision);
|
||||
let rctx = ShadowResponseContext::from_request(
|
||||
ctx,
|
||||
"http://p1:30000".into(),
|
||||
"http://d:30300".into(),
|
||||
);
|
||||
assert!(!rctx.shadow_hit);
|
||||
|
||||
// No decision at all (defensive edge) -> false.
|
||||
let ctx = ShadowRequestContext::from_body(&body, false).unwrap();
|
||||
let rctx = ShadowResponseContext::from_request(
|
||||
ctx,
|
||||
"http://p1:30000".into(),
|
||||
"http://d:30300".into(),
|
||||
);
|
||||
assert!(!rctx.shadow_hit);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sse_sglext_standalone_chunk_dropped() {
|
||||
// ids-framed path: the engine emits a standalone `choices=[]` chunk
|
||||
|
||||
@@ -1157,9 +1157,11 @@ impl PDRouter {
|
||||
Ok(decode_body) => {
|
||||
let decode_body =
|
||||
self.shadow_record_non_stream(decode_body, status, &shadow);
|
||||
let mut response = Response::new(Body::from(decode_body));
|
||||
*response.status_mut() = status;
|
||||
*response.headers_mut() = response_headers;
|
||||
let response = Self::non_stream_passthrough_response(
|
||||
status,
|
||||
decode_body,
|
||||
response_headers,
|
||||
);
|
||||
// T17: non-streaming e2e (observability only).
|
||||
if let Some(g9i) = &self.g9i {
|
||||
g9i.record_nonstream_e2e(start_time.elapsed().as_secs_f64());
|
||||
@@ -1469,7 +1471,11 @@ impl PDRouter {
|
||||
// (malformed) repeat usage event from
|
||||
// double-completing the queue slot.
|
||||
if !shadow_done {
|
||||
g9i.shadow_record(sctx, &json, shadow_ttft_secs);
|
||||
// Streaming harvests the usage
|
||||
// event pre-strip, so the
|
||||
// breakdown extraction inside
|
||||
// works unaided (None).
|
||||
g9i.shadow_record(sctx, &json, shadow_ttft_secs, None);
|
||||
shadow_done = true;
|
||||
}
|
||||
}
|
||||
@@ -1662,12 +1668,15 @@ impl PDRouter {
|
||||
g9i.shadow_release(ctx);
|
||||
return body;
|
||||
}
|
||||
let (parsed, rewritten) =
|
||||
let (parsed, rewritten, breakdown) =
|
||||
super::g9i_shadow::strip_and_extract(&body, ctx.strip_cached_details);
|
||||
match &parsed {
|
||||
// TTFT calibration is streaming-only (non-streaming has no
|
||||
// observable first-token moment) -> None.
|
||||
Some(json) => g9i.shadow_record(ctx, json, None),
|
||||
// observable first-token moment) -> None. The breakdown was
|
||||
// extracted pre-strip and is passed through — the parsed JSON
|
||||
// here is post-strip, so re-extracting from it would always
|
||||
// miss (the 2026-10-17 order bug).
|
||||
Some(json) => g9i.shadow_record(ctx, json, None, breakdown),
|
||||
// Unparseable success body: nothing to record, but the dispatch
|
||||
// is over — release rather than leak the outstanding count.
|
||||
None => g9i.shadow_release(ctx),
|
||||
@@ -1678,6 +1687,27 @@ impl PDRouter {
|
||||
}
|
||||
}
|
||||
|
||||
/// Assemble the non-streaming passthrough response. The shadow strip
|
||||
/// may have REWRITTEN (shortened) the body, so the upstream
|
||||
/// Content-Length in `headers` is stale; forwarding it verbatim
|
||||
/// truncated cache-hit responses at exactly the stripped length (the
|
||||
/// 2026-10-17 IncompleteRead incident). Pin Content-Length to the
|
||||
/// actual body length — correct for rewritten and untouched bodies
|
||||
/// alike.
|
||||
fn non_stream_passthrough_response(
|
||||
status: StatusCode,
|
||||
body: bytes::Bytes,
|
||||
mut headers: HeaderMap,
|
||||
) -> Response {
|
||||
if let Ok(v) = HeaderValue::from_str(body.len().to_string().as_str()) {
|
||||
headers.insert(CONTENT_LENGTH, v);
|
||||
}
|
||||
let mut response = Response::new(Body::from(body));
|
||||
*response.status_mut() = status;
|
||||
*response.headers_mut() = headers;
|
||||
response
|
||||
}
|
||||
|
||||
// Helper to process prefill response and extract body if needed for logprobs
|
||||
async fn process_prefill_response(
|
||||
&self,
|
||||
@@ -2205,6 +2235,39 @@ mod tests {
|
||||
Box::new(worker)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_stream_passthrough_pins_content_length_to_body() {
|
||||
// 2026-10-17 IncompleteRead regression: the shadow strip shortens
|
||||
// the body on engine cache hits; the upstream Content-Length must
|
||||
// never survive onto the rewritten body.
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(CONTENT_LENGTH, HeaderValue::from_static("637"));
|
||||
let body = bytes::Bytes::from_static(b"{}");
|
||||
let resp =
|
||||
PDRouter::non_stream_passthrough_response(StatusCode::OK, body.clone(), headers);
|
||||
assert_eq!(
|
||||
resp.headers().get(CONTENT_LENGTH).unwrap(),
|
||||
&HeaderValue::from_static("2"),
|
||||
"stale upstream length is replaced by the actual body length"
|
||||
);
|
||||
|
||||
// Untouched body: the pinned length matches the original.
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(CONTENT_LENGTH, HeaderValue::from_static("2"));
|
||||
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
let resp = PDRouter::non_stream_passthrough_response(StatusCode::OK, body, headers);
|
||||
assert_eq!(
|
||||
resp.headers().get(CONTENT_LENGTH).unwrap(),
|
||||
&HeaderValue::from_static("2")
|
||||
);
|
||||
assert_eq!(
|
||||
resp.headers().get(CONTENT_TYPE).unwrap(),
|
||||
&HeaderValue::from_static("application/json"),
|
||||
"other preserved headers survive"
|
||||
);
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_chat_request_text_uses_full_conversation() {
|
||||
// Regression test for https://github.com/sgl-project/sglang/issues/26263
|
||||
|
||||
Reference in New Issue
Block a user