diff --git a/proto/sglang/api/v1/generate.proto b/proto/sglang/api/v1/generate.proto index 549ae1460c..0c71613c20 100644 --- a/proto/sglang/api/v1/generate.proto +++ b/proto/sglang/api/v1/generate.proto @@ -129,6 +129,10 @@ message GenerateMetaInfo { // Seconds from normalized-request acceptance to the terminal output; only // on the terminal frame. optional double e2e_latency = 12; + // Prompt tokens served from the prefix cache, and output tokens spent + // reasoning so far; reported on every frame. + uint32 cached_tokens = 13; + uint64 reasoning_tokens = 14; } message GenerateResponse { diff --git a/python/sglang/api/v1/api_types.py b/python/sglang/api/v1/api_types.py index fb5611159c..afc90096c5 100644 --- a/python/sglang/api/v1/api_types.py +++ b/python/sglang/api/v1/api_types.py @@ -302,6 +302,8 @@ class GenerateMetaInfo(msgspec.Struct, kw_only=True, frozen=True): input_token_ids_logprobs: List[NullableTopLogprobs] = [] hidden_states: List[HiddenStateRow] = [] e2e_latency: Optional[float] = None + cached_tokens: int = 0 + reasoning_tokens: int = 0 @classmethod def from_json_value(cls, value: Any) -> "GenerateMetaInfo": @@ -1257,6 +1259,10 @@ def decode_GenerateMetaInfo(value: Any) -> GenerateMetaInfo: kw["hidden_states"] = [] if x is None else [decode_HiddenStateRow(e) for e in _expect_seq(x, expected="a sequence")] elif key == "e2e_latency": kw["e2e_latency"] = None if x is None else _expect_float(x, expected="f64") + elif key == "cached_tokens": + kw["cached_tokens"] = _expect_uint(x, expected="u32") + elif key == "reasoning_tokens": + kw["reasoning_tokens"] = _expect_uint(x, expected="u64") return GenerateMetaInfo(**kw) @@ -1282,6 +1288,8 @@ def encode_GenerateMetaInfo(v: GenerateMetaInfo) -> Dict[str, Any]: d["hidden_states"] = [encode_HiddenStateRow(e) for e in v.hidden_states] if v.e2e_latency is not None: d["e2e_latency"] = float(v.e2e_latency) + d["cached_tokens"] = v.cached_tokens + d["reasoning_tokens"] = v.reasoning_tokens return d diff --git a/python/sglang/srt/managers/scheduler_components/output_streamer.py b/python/sglang/srt/managers/scheduler_components/output_streamer.py index fedd34247d..9fb430fb5b 100644 --- a/python/sglang/srt/managers/scheduler_components/output_streamer.py +++ b/python/sglang/srt/managers/scheduler_components/output_streamer.py @@ -472,6 +472,8 @@ class _GenerationStreamAccumulator: self.output_ids.append(output_ids_[send_token_offset:]) req.send_token_offset = len(output_ids_) self.prompt_tokens.append(len(req.origin_input_ids)) + self.reasoning_tokens.append(req.reasoning_tokens) + self.cached_tokens.append(req.cached_tokens) # Index-aligned with the batch items so mixed batches resolve per-item # on the tokenizer side; None for non-beam items and aborted groups. beam_output = ( @@ -498,13 +500,11 @@ class _GenerationStreamAccumulator: req.sampling_params.spaces_between_special_tokens ) self.no_stop_trim.append(req.sampling_params.no_stop_trim) - self.reasoning_tokens.append(req.reasoning_tokens) self.completion_tokens.append( beam_completion_tokens(beam_output) if beam_output is not None else len(output_ids_) ) - self.cached_tokens.append(req.cached_tokens) # Collect detailed cache breakdown if available self.cached_tokens_details.append(self.get_cached_tokens_details(req)) diff --git a/python/sglang/srt/rust_server/server.py b/python/sglang/srt/rust_server/server.py index 6d72b0c78b..f605cb4829 100644 --- a/python/sglang/srt/rust_server/server.py +++ b/python/sglang/srt/rust_server/server.py @@ -364,7 +364,8 @@ class RustServer: bulk numeric columns never go through msgpack: - ``header``: msgpack ``BatchHeader`` positional array — the per-request - scalar columns (``rids, finish_reasons, prompt_tokens, tok_lens``) plus + scalar columns (``rids, finish_reasons, prompt_tokens, tok_lens, + cached_tokens, reasoning_tokens``) plus the shape metadata for the optional families (``*_lens`` element counts for the flat logprob columns, ``*_reqlens``/``*_poslens`` for the ragged and hidden ones). @@ -412,7 +413,14 @@ class RustServer: # Column order here MUST match BatchHeader (header_cols) and # for_each_chunk's read order (data_cols); the extras contribution # is ordered by the `extras` tuple below. - header_cols = [rids, finish_reasons, prompt_tokens, tok_lens] + header_cols = [ + rids, + finish_reasons, + prompt_tokens, + tok_lens, + payload.cached_tokens, + payload.reasoning_tokens, + ] data_cols = [flat_ids] if has_extra: diff --git a/rust/sglang-api-types/src/generated/sglang.api.v1.rs b/rust/sglang-api-types/src/generated/sglang.api.v1.rs index f1293a80ba..9670bb0609 100644 --- a/rust/sglang-api-types/src/generated/sglang.api.v1.rs +++ b/rust/sglang-api-types/src/generated/sglang.api.v1.rs @@ -473,6 +473,12 @@ pub struct GenerateMetaInfo { /// on the terminal frame. #[prost(double, optional, tag = "12")] pub e2e_latency: ::core::option::Option, + /// Prompt tokens served from the prefix cache, and output tokens spent + /// reasoning so far; reported on every frame. + #[prost(uint32, tag = "13")] + pub cached_tokens: u32, + #[prost(uint64, tag = "14")] + pub reasoning_tokens: u64, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct GenerateResponse { diff --git a/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs b/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs index 7579d355ee..39faf1fb5c 100644 --- a/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs +++ b/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs @@ -2084,6 +2084,16 @@ impl ::serde::Serialize for GenerateMetaInfo { if let Some(v) = &self.e2e_latency { ::serde::ser::SerializeMap::serialize_entry(&mut map, "e2e_latency", v)?; } + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "cached_tokens", + &self.cached_tokens, + )?; + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "reasoning_tokens", + &self.reasoning_tokens, + )?; ::serde::ser::SerializeMap::end(map) } } @@ -2112,6 +2122,8 @@ impl<'de> ::serde::Deserialize<'de> for GenerateMetaInfo { let mut f_input_token_ids_logprobs = None; let mut f_hidden_states = None; let mut f_e2e_latency = None; + let mut f_cached_tokens = None; + let mut f_reasoning_tokens = None; while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { match key.as_ref() { "id" => { @@ -2217,6 +2229,20 @@ impl<'de> ::serde::Deserialize<'de> for GenerateMetaInfo { } f_e2e_latency = Some(map.next_value::>()?); } + "cached_tokens" => { + if f_cached_tokens.is_some() { + return Err(::serde::de::Error::duplicate_field("cached_tokens")); + } + f_cached_tokens = Some(map.next_value::()?); + } + "reasoning_tokens" => { + if f_reasoning_tokens.is_some() { + return Err(::serde::de::Error::duplicate_field( + "reasoning_tokens", + )); + } + f_reasoning_tokens = Some(map.next_value::()?); + } _ => { map.next_value::<::serde::de::IgnoredAny>()?; } @@ -2235,6 +2261,8 @@ impl<'de> ::serde::Deserialize<'de> for GenerateMetaInfo { input_token_ids_logprobs: f_input_token_ids_logprobs.unwrap_or_default(), hidden_states: f_hidden_states.unwrap_or_default(), e2e_latency: f_e2e_latency.flatten(), + cached_tokens: f_cached_tokens.unwrap_or_default(), + reasoning_tokens: f_reasoning_tokens.unwrap_or_default(), }) } } diff --git a/rust/sglang-api-types/testdata/json_parity.json b/rust/sglang-api-types/testdata/json_parity.json index 67ba4017ef..9de2711db9 100644 --- a/rust/sglang-api-types/testdata/json_parity.json +++ b/rust/sglang-api-types/testdata/json_parity.json @@ -143,11 +143,11 @@ 8 ] }, - "canonical": "{\"text\":\"ok\",\"meta_info\":{\"id\":\"client-rid\",\"prompt_tokens\":5,\"completion_tokens\":2,\"finish_reason\":null},\"output_ids\":[7,8]}" + "canonical": "{\"text\":\"ok\",\"meta_info\":{\"id\":\"client-rid\",\"prompt_tokens\":5,\"completion_tokens\":2,\"finish_reason\":null,\"cached_tokens\":0,\"reasoning_tokens\":0},\"output_ids\":[7,8]}" }, { "type": "GenerateResponse", - "name": "terminal batch frame with every logprob family, null rows, hidden states, e2e_latency, index", + "name": "terminal batch frame with every logprob family, null rows, hidden states, e2e_latency, cache and reasoning counts, index", "input": { "text": "ok", "meta_info": { @@ -212,14 +212,16 @@ ], [] ], - "e2e_latency": 0.01 + "e2e_latency": 0.01, + "cached_tokens": 4, + "reasoning_tokens": 2 }, "output_ids": [ 5 ], "index": 0 }, - "canonical": "{\"text\":\"ok\",\"meta_info\":{\"id\":\"r\",\"prompt_tokens\":1,\"completion_tokens\":1,\"finish_reason\":{\"type\":\"stop\",\"matched\":[1,2]},\"output_token_logprobs\":[[-0.5,5,\"x\"],[null,6,null]],\"input_token_logprobs\":[],\"output_top_logprobs\":[null,[[-0.25,6,null]]],\"input_top_logprobs\":[[[-0.125,1,\"\"]]],\"output_token_ids_logprobs\":[[[-0.5,9,null]]],\"input_token_ids_logprobs\":[null],\"hidden_states\":[[0.5,0.25],[]],\"e2e_latency\":0.01},\"output_ids\":[5],\"index\":0}" + "canonical": "{\"text\":\"ok\",\"meta_info\":{\"id\":\"r\",\"prompt_tokens\":1,\"completion_tokens\":1,\"finish_reason\":{\"type\":\"stop\",\"matched\":[1,2]},\"output_token_logprobs\":[[-0.5,5,\"x\"],[null,6,null]],\"input_token_logprobs\":[],\"output_top_logprobs\":[null,[[-0.25,6,null]]],\"input_top_logprobs\":[[[-0.125,1,\"\"]]],\"output_token_ids_logprobs\":[[[-0.5,9,null]]],\"input_token_ids_logprobs\":[null],\"hidden_states\":[[0.5,0.25],[]],\"e2e_latency\":0.01,\"cached_tokens\":4,\"reasoning_tokens\":2},\"output_ids\":[5],\"index\":0}" }, { "type": "GenerateResponse", @@ -236,7 +238,7 @@ } } }, - "canonical": "{\"text\":\"\",\"meta_info\":{\"id\":\"r\",\"prompt_tokens\":1,\"completion_tokens\":9,\"finish_reason\":{\"type\":\"length\",\"length\":9}}}" + "canonical": "{\"text\":\"\",\"meta_info\":{\"id\":\"r\",\"prompt_tokens\":1,\"completion_tokens\":9,\"finish_reason\":{\"type\":\"length\",\"length\":9},\"cached_tokens\":0,\"reasoning_tokens\":0}}" }, { "type": "GenerateResponse", @@ -255,7 +257,7 @@ } } }, - "canonical": "{\"text\":\"\",\"meta_info\":{\"id\":\"r\",\"prompt_tokens\":1,\"completion_tokens\":0,\"finish_reason\":{\"type\":\"abort\",\"message\":\"over the limit\",\"status_code\":400,\"err_type\":null}}}" + "canonical": "{\"text\":\"\",\"meta_info\":{\"id\":\"r\",\"prompt_tokens\":1,\"completion_tokens\":0,\"finish_reason\":{\"type\":\"abort\",\"message\":\"over the limit\",\"status_code\":400,\"err_type\":null},\"cached_tokens\":0,\"reasoning_tokens\":0}}" }, { "type": "GenerateResponse", @@ -272,7 +274,7 @@ } } }, - "canonical": "{\"text\":\"\",\"meta_info\":{\"id\":\"r\",\"prompt_tokens\":1,\"completion_tokens\":0,\"finish_reason\":{\"detail\":3,\"type\":\"novel\"}}}" + "canonical": "{\"text\":\"\",\"meta_info\":{\"id\":\"r\",\"prompt_tokens\":1,\"completion_tokens\":0,\"finish_reason\":{\"detail\":3,\"type\":\"novel\"},\"cached_tokens\":0,\"reasoning_tokens\":0}}" }, { "type": "GenerateStreamError", @@ -366,7 +368,7 @@ } } }, - "canonical": "{\"text\":\"\",\"meta_info\":{\"id\":\"r\",\"prompt_tokens\":1,\"completion_tokens\":0,\"finish_reason\":{\"type\":\"novel\",\"detail\":3,\"again\":[1,{\"z\":1,\"a\":2}]}}}" + "canonical": "{\"text\":\"\",\"meta_info\":{\"id\":\"r\",\"prompt_tokens\":1,\"completion_tokens\":0,\"finish_reason\":{\"type\":\"novel\",\"detail\":3,\"again\":[1,{\"z\":1,\"a\":2}]},\"cached_tokens\":0,\"reasoning_tokens\":0}}" } ] } diff --git a/rust/sglang-api-types/tests/json_contract.rs b/rust/sglang-api-types/tests/json_contract.rs index ac822f4469..8d0c283e8c 100644 --- a/rust/sglang-api-types/tests/json_contract.rs +++ b/rust/sglang-api-types/tests/json_contract.rs @@ -247,13 +247,14 @@ fn generate_response_matches_frame_value_shape() { prompt_tokens: 5, completion_tokens: 2, finish_reason: None, + cached_tokens: 3, ..Default::default() }), index: None, }; assert_eq!( serde_json::to_string(&frame).unwrap(), - r#"{"text":"ok","meta_info":{"id":"client-rid","prompt_tokens":5,"completion_tokens":2,"finish_reason":null},"output_ids":[7,8]}"# + r#"{"text":"ok","meta_info":{"id":"client-rid","prompt_tokens":5,"completion_tokens":2,"finish_reason":null,"cached_tokens":3,"reasoning_tokens":0},"output_ids":[7,8]}"# ); } diff --git a/rust/sglang-server/src/api_server/core/frame.rs b/rust/sglang-server/src/api_server/core/frame.rs index 90a82c9bb4..b8c359a143 100644 --- a/rust/sglang-server/src/api_server/core/frame.rs +++ b/rust/sglang-server/src/api_server/core/frame.rs @@ -24,6 +24,8 @@ pub(crate) struct CoreOutput { pub(crate) token_ids: TokenIds, pub(crate) finish_reason: Option, pub(crate) prompt_tokens: u32, + pub(crate) cached_tokens: u32, + pub(crate) reasoning_tokens: u64, pub(crate) text: String, pub(crate) completion_tokens: u64, pub(crate) extras: Option>, @@ -36,6 +38,8 @@ impl From for CoreOutput { token_ids, finish_reason, prompt_tokens, + cached_tokens, + reasoning_tokens, text, completion_tokens, extras, @@ -44,6 +48,8 @@ impl From for CoreOutput { token_ids, finish_reason, prompt_tokens, + cached_tokens, + reasoning_tokens, text, completion_tokens, extras, @@ -63,6 +69,8 @@ impl CoreOutput { self.token_ids.extend_from_slice(&delta.token_ids); self.completion_tokens += delta.completion_tokens; self.prompt_tokens = delta.prompt_tokens; + self.cached_tokens = delta.cached_tokens; + self.reasoning_tokens = delta.reasoning_tokens; if delta.finish_reason.is_some() { self.finish_reason = delta.finish_reason.clone(); } @@ -196,6 +204,8 @@ impl CoreOutput { completion_tokens: self.completion_tokens, finish_reason: self.finish_reason.as_ref().map(finish_reason), e2e_latency, + cached_tokens: self.cached_tokens, + reasoning_tokens: self.reasoning_tokens, ..Default::default() }; let Some(extras) = self.extras.as_deref() else { diff --git a/rust/sglang-server/src/api_server/grpc/service.rs b/rust/sglang-server/src/api_server/grpc/service.rs index 46f36d8c35..a73204b88c 100644 --- a/rust/sglang-server/src/api_server/grpc/service.rs +++ b/rust/sglang-server/src/api_server/grpc/service.rs @@ -254,6 +254,8 @@ mod tests { token_ids: vec![token_id], finish_reason: finished.then(stop_reason), prompt_tokens: 3, + cached_tokens: 2, + reasoning_tokens: 0, text: text.into(), completion_tokens: 1, extras: None, @@ -361,6 +363,7 @@ mod tests { let first_meta = meta(&first); assert_eq!(first_meta.id, "request-1"); assert_eq!(first_meta.prompt_tokens, 3); + assert_eq!(first_meta.cached_tokens, 2); assert_eq!(first_meta.completion_tokens, 1); assert!(first_meta.finish_reason.is_none()); assert!(first_meta.e2e_latency.is_none()); diff --git a/rust/sglang-server/src/message/response.rs b/rust/sglang-server/src/message/response.rs index c2cc222bb0..eda46cd817 100644 --- a/rust/sglang-server/src/message/response.rs +++ b/rust/sglang-server/src/message/response.rs @@ -133,9 +133,9 @@ pub fn frame_decode_batch_cols(header: &[u8], data_cols: &[&[u8]]) -> Bytes { Bytes::from(buf) } -/// Columnar scalar header for a whole decode batch. The first four fields are -/// required; every field after `tok_lens` defaults empty, so the hot path emits -/// a four-element header. Field order is the wire ABI and must match +/// Columnar scalar header for a whole decode batch. The first six fields are +/// required; every field after `reasoning_tokens` defaults empty, so the hot +/// path emits a six-element header. Field order is the wire ABI and must match /// `RustTokenizerManager.push_generation`'s `header_cols` in /// `python/sglang/srt/rust_server/server.py`. /// @@ -155,6 +155,9 @@ pub struct BatchHeader { pub finish_reasons: Vec>, pub prompt_tokens: Vec, pub tok_lens: Vec, + /// Python's `cached_tokens` / `reasoning_tokens`: running totals, not deltas. + pub cached_tokens: Vec, + pub reasoning_tokens: Vec, #[serde(default)] pub out_lp_lens: Vec, #[serde(default)] @@ -276,8 +279,13 @@ pub fn for_each_chunk(body: &[u8], mut route: impl FnMut(ChunkEvent)) -> Decoded // `tok_lens` delivered a 200 with an empty completion, and a short // `finish_reasons` dropped the terminal marker so the request never completed // and its unary drain pended forever. The producer already asserts this for the - // extras columns; the four core ones were unchecked. - if h.finish_reasons.len() != n || h.prompt_tokens.len() != n || h.tok_lens.len() != n { + // extras columns; the core ones were unchecked. + if h.finish_reasons.len() != n + || h.prompt_tokens.len() != n + || h.tok_lens.len() != n + || h.cached_tokens.len() != n + || h.reasoning_tokens.len() != n + { reject!() } // The per-request extras columns are either absent (no request asked) or one @@ -465,6 +473,8 @@ pub fn for_each_chunk(body: &[u8], mut route: impl FnMut(ChunkEvent)) -> Decoded token_ids, finish_reason: h.finish_reasons.get(i).cloned().flatten(), prompt_tokens: h.prompt_tokens.get(i).copied().unwrap_or(0), + cached_tokens: h.cached_tokens.get(i).copied().unwrap_or(0), + reasoning_tokens: h.reasoning_tokens.get(i).copied().unwrap_or(0), extras, // Listed explicitly, NOT `..Default::default()`: a new column added to // `ChunkEvent` and wired into the response must fail to compile here @@ -568,6 +578,10 @@ pub struct ChunkEvent { pub finish_reason: Option, /// Prompt token count for this request (constant across its chunks). pub prompt_tokens: u32, + /// Prefix-cache hits and reasoning tokens so far: running totals, so the + /// latest chunk's value is the request's. + pub cached_tokens: u32, + pub reasoning_tokens: u64, /// Decoded text **delta** for this chunk (empty in skip mode / on partial UTF-8), /// filled by the detok shard. `token_ids` doubles as `output_ids`; /// `completion_tokens` is this chunk's count. @@ -665,7 +679,8 @@ mod tests { fn decodes_batch_frame() { use rmpv::Value; // 3 requests: rids "1","2","3"; finish [nil, {type:stop,matched:5}, nil]; - // prompt_tokens [4,5,6]; tok_lens [2,0,1] -> ids [10,11 | (none) | 12]. + // prompt_tokens [4,5,6]; tok_lens [2,0,1] -> ids [10,11 | (none) | 12]; + // cached_tokens [3,0,1]; reasoning_tokens [0,0,7]. let stop = Value::Map(vec![ (Value::from("type"), Value::from("stop")), (Value::from("matched"), Value::from(5)), @@ -683,6 +698,16 @@ mod tests { Value::from(0u32), Value::from(1u32), ]), + Value::Array(vec![ + Value::from(3u32), + Value::from(0u32), + Value::from(1u32), + ]), + Value::Array(vec![ + Value::from(0u64), + Value::from(0u64), + Value::from(7u64), + ]), ]); let mut header = Vec::new(); rmpv::encode::write_value(&mut header, &header_arr).unwrap(); @@ -699,6 +724,8 @@ mod tests { assert_eq!(events[0].rid, Rid::from("1")); assert_eq!(events[0].token_ids, vec![10, 11]); assert_eq!(events[0].prompt_tokens, 4); + assert_eq!(events[0].cached_tokens, 3); + assert_eq!(events[2].reasoning_tokens, 7); assert!(events[0].finish_reason.is_none()); assert_eq!(events[1].rid, Rid::from("2")); assert!(events[1].token_ids.is_empty()); @@ -738,6 +765,8 @@ mod tests { Value::Array(vec![Value::Nil]), // finish_reasons Value::Array(vec![Value::from(0u32)]), // prompt_tokens Value::Array(vec![Value::from(10u32)]), // tok_lens (claims 80 bytes) + Value::Array(vec![Value::from(0u32)]), // cached_tokens + Value::Array(vec![Value::from(0u32)]), // reasoning_tokens Value::Array(vec![Value::from(1u32)]), // out_lp_lens (base now past data) ]); let mut header = Vec::new(); @@ -776,6 +805,8 @@ mod tests { finish.clone(), arr_u(&[0, 0]), // prompt arr_u(&[1, 1]), // tok_lens + arr_u(&[0, 0]), // cached_tokens + arr_u(&[0, 0]), // reasoning_tokens arr_u(&[0, 0]), // out_lp_lens arr_u(&[0, 0]), // in_lp_lens arr_u(&[2, 1]), // out_top_reqlens — 3 positions claimed @@ -800,6 +831,8 @@ mod tests { finish, arr_u(&[0, 0]), // prompt arr_u(&[1, 1]), // tok_lens + arr_u(&[0, 0]), // cached_tokens + arr_u(&[0, 0]), // reasoning_tokens arr_u(&[0, 0]), // out_lp_lens arr_u(&[0, 0]), // in_lp_lens arr_u(&[0, 0]), // out_top_reqlens @@ -837,6 +870,8 @@ mod tests { Value::Array(vec![Value::Nil, Value::Nil]), Value::Array(vec![Value::from(0u32), Value::from(0u32)]), Value::Array(vec![Value::from(9u32), Value::from(9u32)]), // tok_lens past data + Value::Array(vec![Value::from(0u32), Value::from(0u32)]), // cached_tokens + Value::Array(vec![Value::from(0u32), Value::from(0u32)]), // reasoning_tokens ]); let mut header = Vec::new(); rmpv::encode::write_value(&mut header, &header_arr).unwrap(); @@ -863,6 +898,8 @@ mod tests { Value::Array(vec![Value::Nil]), Value::Array(vec![Value::from(0u32)]), Value::Array(vec![Value::from(1u32)]), // tok_lens: 1 id = 8 bytes + Value::Array(vec![Value::from(0u32)]), // cached_tokens + Value::Array(vec![Value::from(0u32)]), // reasoning_tokens ]); let mut header = Vec::new(); rmpv::encode::write_value(&mut header, &header_arr).unwrap(); @@ -885,6 +922,8 @@ mod tests { Value::Array(vec![Value::Nil]), // finish_reasons Value::Array(vec![Value::from(1u32)]), // prompt_tokens Value::Array(vec![Value::from(1u32)]), // tok_lens + Value::Array(vec![Value::from(0u32)]), // cached_tokens + Value::Array(vec![Value::from(0u32)]), // reasoning_tokens ]); let mut header = Vec::new(); rmpv::encode::write_value(&mut header, &header_arr).unwrap(); @@ -929,7 +968,8 @@ mod tests { let i = |xs: &[i32]| -> Vec { xs.iter().flat_map(|x| x.to_le_bytes()).collect() }; let q = |xs: &[i64]| -> Vec { xs.iter().flat_map(|x| x.to_le_bytes()).collect() }; let arr_u = |xs: &[u32]| Value::Array(xs.iter().map(|&x| Value::from(x)).collect()); - // header: rids, finish, prompt, tok_lens, out_lp_lens, in_lp_lens, + // header: rids, finish, prompt, tok_lens, cached_tokens, reasoning_tokens, + // out_lp_lens, in_lp_lens, // out_top_reqlens, out_top_poslens, in_top_*, out_tokids_lp_*, // in_tokids_lp_*, // hidden_reqlens, hidden_poslens @@ -938,6 +978,8 @@ mod tests { Value::Array(vec![Value::Nil, Value::Nil]), // finish arr_u(&[3, 4]), // prompt arr_u(&[1, 1]), // tok_lens + arr_u(&[0, 0]), // cached_tokens + arr_u(&[0, 0]), // reasoning_tokens arr_u(&[2, 0]), // out_lp_lens arr_u(&[0, 0]), // in_lp_lens arr_u(&[1, 0]), // out_top_reqlens (req0: 1 pos) @@ -1011,6 +1053,8 @@ mod tests { Value::Array(vec![Value::Nil, Value::Nil]), // finish arr_u(&[3, 4]), // prompt arr_u(&[1, 1]), // tok_lens + arr_u(&[0, 0]), // cached_tokens + arr_u(&[0, 0]), // reasoning_tokens arr_u(&[2, 0]), // out_lp_lens (ACTIVE) reqlens.clone(), // in_lp_lens reqlens.clone(), // out_top_reqlens @@ -1092,6 +1136,8 @@ mod tests { Value::Array(vec![Value::Nil]), // finish arr_u(&[9]), // prompt arr_u(&[1]), // tok_lens + arr_u(&[0]), // cached_tokens + arr_u(&[0]), // reasoning_tokens arr_u(&[2]), // out_lp_lens (2 flat) arr_u(&[1]), // in_lp_lens (1 flat) arr_u(&[1]), // out_top_reqlens (1 position… @@ -1165,6 +1211,8 @@ mod tests { Value::Array(vec![Value::Nil, Value::Nil]), Value::Array(vec![Value::from(0u32), Value::from(0u32)]), Value::Array(vec![Value::from(1u32), Value::from(1u32)]), + Value::Array(vec![Value::from(0u32), Value::from(0u32)]), + Value::Array(vec![Value::from(0u32), Value::from(0u32)]), ]); let mut header = Vec::new(); rmpv::encode::write_value(&mut header, &header_arr).unwrap(); diff --git a/rust/sglang-server/src/utils/runtime.rs b/rust/sglang-server/src/utils/runtime.rs index 2047e07c04..181c9f47cb 100644 --- a/rust/sglang-server/src/utils/runtime.rs +++ b/rust/sglang-server/src/utils/runtime.rs @@ -453,6 +453,8 @@ mod tests { )], prompt_tokens: vec![3], tok_lens: vec![output_ids.len() as u32], + cached_tokens: vec![0], + reasoning_tokens: vec![0], ..Default::default() }; let header = rmp_serde::to_vec(&header).unwrap(); diff --git a/test/registered/unit/api/v1/test_json_contract.py b/test/registered/unit/api/v1/test_json_contract.py index 1f8c8c2496..bde00d2757 100644 --- a/test/registered/unit/api/v1/test_json_contract.py +++ b/test/registered/unit/api/v1/test_json_contract.py @@ -295,7 +295,7 @@ class TestFrames(CustomTestCase): ) self.assertEqual( compact(frame.to_json_value()), - '{"text":"ok","meta_info":{"id":"client-rid","prompt_tokens":5,"completion_tokens":2,"finish_reason":null},"output_ids":[7,8]}', + '{"text":"ok","meta_info":{"id":"client-rid","prompt_tokens":5,"completion_tokens":2,"finish_reason":null,"cached_tokens":0,"reasoning_tokens":0},"output_ids":[7,8]}', ) def test_frame_round_trip_with_logprobs(self): @@ -310,6 +310,8 @@ class TestFrames(CustomTestCase): "output_top_logprobs": [None, [[-0.2, 6, None]]], "hidden_states": [[0.5, 0.25]], "e2e_latency": 0.01, + "cached_tokens": 1, + "reasoning_tokens": 0, }, "output_ids": [5], "index": 0, diff --git a/test/registered/unit/api/v1/test_parity.py b/test/registered/unit/api/v1/test_parity.py index 68605119d8..4058fb4bb2 100644 --- a/test/registered/unit/api/v1/test_parity.py +++ b/test/registered/unit/api/v1/test_parity.py @@ -224,6 +224,8 @@ class TestResponseParity(CustomTestCase): "completion_tokens", "finish_reason", "e2e_latency", + "cached_tokens", + "reasoning_tokens", ], ) self.assertEqual( @@ -243,7 +245,10 @@ class TestResponseParity(CustomTestCase): list(GenerateMetaInfo.__struct_fields__)[:4], ["id", "prompt_tokens", "completion_tokens", "finish_reason"], ) - self.assertEqual(list(GenerateMetaInfo.__struct_fields__)[-1], "e2e_latency") + self.assertEqual( + list(GenerateMetaInfo.__struct_fields__)[-3:], + ["e2e_latency", "cached_tokens", "reasoning_tokens"], + ) if __name__ == "__main__":