[Feat] rust-server: Report cached_tokens and reasoning_tokens in meta_info (#44531)

Co-authored-by: fei-xx <fei-xx@users.noreply.github.com>
Co-authored-by: shodoco <shodoco@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-15 17:50:27 -07:00
committed by GitHub
co-authored by fei-xx shodoco
parent d393298736
commit f110b2f801
14 changed files with 149 additions and 22 deletions
+8
View File
@@ -302,6 +302,8 @@ class GenerateMetaInfo(msgspec.Struct, kw_only=True, frozen=True):
input_token_ids_logprobs: List[NullableTopLogprobs] = []
hidden_states: List[HiddenStateRow] = []
e2e_latency: Optional[float] = None
cached_tokens: int = 0
reasoning_tokens: int = 0
@classmethod
def from_json_value(cls, value: Any) -> "GenerateMetaInfo":
@@ -1257,6 +1259,10 @@ def decode_GenerateMetaInfo(value: Any) -> GenerateMetaInfo:
kw["hidden_states"] = [] if x is None else [decode_HiddenStateRow(e) for e in _expect_seq(x, expected="a sequence")]
elif key == "e2e_latency":
kw["e2e_latency"] = None if x is None else _expect_float(x, expected="f64")
elif key == "cached_tokens":
kw["cached_tokens"] = _expect_uint(x, expected="u32")
elif key == "reasoning_tokens":
kw["reasoning_tokens"] = _expect_uint(x, expected="u64")
return GenerateMetaInfo(**kw)
@@ -1282,6 +1288,8 @@ def encode_GenerateMetaInfo(v: GenerateMetaInfo) -> Dict[str, Any]:
d["hidden_states"] = [encode_HiddenStateRow(e) for e in v.hidden_states]
if v.e2e_latency is not None:
d["e2e_latency"] = float(v.e2e_latency)
d["cached_tokens"] = v.cached_tokens
d["reasoning_tokens"] = v.reasoning_tokens
return d
@@ -472,6 +472,8 @@ class _GenerationStreamAccumulator:
self.output_ids.append(output_ids_[send_token_offset:])
req.send_token_offset = len(output_ids_)
self.prompt_tokens.append(len(req.origin_input_ids))
self.reasoning_tokens.append(req.reasoning_tokens)
self.cached_tokens.append(req.cached_tokens)
# Index-aligned with the batch items so mixed batches resolve per-item
# on the tokenizer side; None for non-beam items and aborted groups.
beam_output = (
@@ -498,13 +500,11 @@ class _GenerationStreamAccumulator:
req.sampling_params.spaces_between_special_tokens
)
self.no_stop_trim.append(req.sampling_params.no_stop_trim)
self.reasoning_tokens.append(req.reasoning_tokens)
self.completion_tokens.append(
beam_completion_tokens(beam_output)
if beam_output is not None
else len(output_ids_)
)
self.cached_tokens.append(req.cached_tokens)
# Collect detailed cache breakdown if available
self.cached_tokens_details.append(self.get_cached_tokens_details(req))
+10 -2
View File
@@ -364,7 +364,8 @@ class RustServer:
bulk numeric columns never go through msgpack:
- ``header``: msgpack ``BatchHeader`` positional array — the per-request
scalar columns (``rids, finish_reasons, prompt_tokens, tok_lens``) plus
scalar columns (``rids, finish_reasons, prompt_tokens, tok_lens,
cached_tokens, reasoning_tokens``) plus
the shape metadata for the optional families (``*_lens`` element counts
for the flat logprob columns, ``*_reqlens``/``*_poslens`` for the ragged
and hidden ones).
@@ -412,7 +413,14 @@ class RustServer:
# Column order here MUST match BatchHeader (header_cols) and
# for_each_chunk's read order (data_cols); the extras contribution
# is ordered by the `extras` tuple below.
header_cols = [rids, finish_reasons, prompt_tokens, tok_lens]
header_cols = [
rids,
finish_reasons,
prompt_tokens,
tok_lens,
payload.cached_tokens,
payload.reasoning_tokens,
]
data_cols = [flat_ids]
if has_extra: