[Fix] tokenizer_manager: re-report only emitted customized_info keys on cumulative streams (#44537)
Co-authored-by: fei-xx <fei-xx@users.noreply.github.com> Co-authored-by: shodoco <shodoco@users.noreply.github.com>
This commit is contained in:
co-authored by
fei-xx
shodoco
parent
bda6ef7e01
commit
d393298736
@@ -42,6 +42,7 @@ from typing import (
|
||||
Iterable,
|
||||
List,
|
||||
Optional,
|
||||
Set,
|
||||
Tuple,
|
||||
Union,
|
||||
)
|
||||
@@ -350,6 +351,9 @@ class ReqState:
|
||||
customized_info_accumulated: Dict[str, List[Any]] = dataclasses.field(
|
||||
default_factory=dict
|
||||
)
|
||||
# Accumulated keys written to meta_info. Cumulative streams re-report only
|
||||
# these, so a subclass can accumulate values it keeps out of responses.
|
||||
customized_info_reported_keys: Set[str] = dataclasses.field(default_factory=set)
|
||||
|
||||
# For return_prompt_token_ids: stores prompt token IDs captured after tokenization
|
||||
prompt_token_ids: Optional[List[int]] = None
|
||||
@@ -2005,12 +2009,13 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
if (
|
||||
is_stream
|
||||
and not incremental_stream
|
||||
and state.customized_info_accumulated
|
||||
and state.customized_info_reported_keys
|
||||
):
|
||||
meta_info = out.get("meta_info")
|
||||
if meta_info is not None:
|
||||
for key, values in state.customized_info_accumulated.items():
|
||||
meta_info.setdefault(key, list(values))
|
||||
if key in state.customized_info_reported_keys:
|
||||
meta_info.setdefault(key, list(values))
|
||||
|
||||
# Resolve deferred text for non-incremental streaming.
|
||||
# _handle_batch_output sets "text": None on intermediate chunks
|
||||
@@ -2827,6 +2832,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
accumulated = state.customized_info_accumulated.setdefault(key, [])
|
||||
accumulated.extend(values)
|
||||
meta_info[key] = accumulated
|
||||
state.customized_info_reported_keys.add(key)
|
||||
|
||||
def update_request_meta_info(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user