From d3932987363a82258ffd18eb2449e14e9908d7ae Mon Sep 17 00:00:00 2001 From: metamergebot Date: Thu, 15 Oct 2026 17:49:49 -0700 Subject: [PATCH] [Fix] tokenizer_manager: re-report only emitted customized_info keys on cumulative streams (#44537) Co-authored-by: fei-xx Co-authored-by: shodoco --- python/sglang/srt/managers/tokenizer_manager.py | 10 ++++++++-- .../managers/test_tokenizer_manager_rid_cleanup.py | 14 +++++++++----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index ae7b0015f0..e8e4b450f9 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -42,6 +42,7 @@ from typing import ( Iterable, List, Optional, + Set, Tuple, Union, ) @@ -350,6 +351,9 @@ class ReqState: customized_info_accumulated: Dict[str, List[Any]] = dataclasses.field( default_factory=dict ) + # Accumulated keys written to meta_info. Cumulative streams re-report only + # these, so a subclass can accumulate values it keeps out of responses. + customized_info_reported_keys: Set[str] = dataclasses.field(default_factory=set) # For return_prompt_token_ids: stores prompt token IDs captured after tokenization prompt_token_ids: Optional[List[int]] = None @@ -2005,12 +2009,13 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): if ( is_stream and not incremental_stream - and state.customized_info_accumulated + and state.customized_info_reported_keys ): meta_info = out.get("meta_info") if meta_info is not None: for key, values in state.customized_info_accumulated.items(): - meta_info.setdefault(key, list(values)) + if key in state.customized_info_reported_keys: + meta_info.setdefault(key, list(values)) # Resolve deferred text for non-incremental streaming. # _handle_batch_output sets "text": None on intermediate chunks @@ -2827,6 +2832,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): accumulated = state.customized_info_accumulated.setdefault(key, []) accumulated.extend(values) meta_info[key] = accumulated + state.customized_info_reported_keys.add(key) def update_request_meta_info( self, diff --git a/test/registered/unit/managers/test_tokenizer_manager_rid_cleanup.py b/test/registered/unit/managers/test_tokenizer_manager_rid_cleanup.py index 6d42f3b7d3..2ffc3488f3 100644 --- a/test/registered/unit/managers/test_tokenizer_manager_rid_cleanup.py +++ b/test/registered/unit/managers/test_tokenizer_manager_rid_cleanup.py @@ -269,14 +269,17 @@ class TestEngineResponseWait(unittest.IsolatedAsyncioTestCase): """ self.tm.incremental_streaming_output = False key = "request_score" + hidden_key = "hidden_metric" stream = self.tm._wait_one_response(self.state.obj) - self.state.customized_info_accumulated[key] = [386.25] + first_meta_info = {"finish_reason": None} + self.tm.update_request_meta_info( + first_meta_info, self.state, {key: [[386.25]]}, 0, None + ) + # Accumulated without being written to meta_info, as a subclass may do. + self.state.customized_info_accumulated[hidden_key] = [1.0, 2.0] self.state.out_list.extend( [ - { - "output_ids": [1], - "meta_info": {"finish_reason": None, key: [386.25]}, - }, + {"output_ids": [1], "meta_info": first_meta_info}, { "output_ids": [1, 2], "meta_info": {"finish_reason": {"type": "length"}}, @@ -290,6 +293,7 @@ class TestEngineResponseWait(unittest.IsolatedAsyncioTestCase): self.assertEqual(out["output_ids"], [1, 2]) self.assertEqual(out["meta_info"][key], [386.25]) + self.assertNotIn(hidden_key, out["meta_info"]) with self.assertRaises(StopAsyncIteration): await anext(stream)