From c6fd9a00c7ccaecb0211676139ca98f43798410b Mon Sep 17 00:00:00 2001 From: Alex Nails Date: Sat, 11 Apr 2026 23:05:36 -0700 Subject: [PATCH] =?UTF-8?q?[tokenizer]=20eliminate=20O(n=C2=B2)=20copy=20i?= =?UTF-8?q?n=20non-incremental=20streaming=20(#22567)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Claude Opus 4.6 (1M context) --- .../sglang/srt/managers/tokenizer_manager.py | 56 ++++++++++++++----- 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index e3eb3d4ab..30e4ab9f9 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -1268,6 +1268,17 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerScoreMixin): else: out = out_list[-1] + # Resolve deferred text for non-incremental streaming. + # _handle_batch_output sets "text": None on intermediate chunks + # to avoid O(n) string rebuild per step (O(n^2) total). + if ( + is_stream + and not incremental_stream + and "text" in out + and out["text"] is None + ): + out["text"] = state.get_text() + if finished: # For non-streaming cases, response has not been sent yet (`response_sent_to_client_time` has not been set yet). # Record response sent time right before we log finished results and metrics. @@ -1708,15 +1719,26 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerScoreMixin): output_token_ids = delta_output_ids _slice_streaming_output_meta_info(meta_info, output_offset) state.last_output_offset = len(state.output_ids) - output_text = delta_text + out_dict = { + "text": delta_text, + "output_ids": output_token_ids, + "meta_info": meta_info, + } + elif state.finished: + out_dict = { + "text": state.get_text(), + "output_ids": state.output_ids.copy(), + "meta_info": meta_info, + } else: - output_token_ids = state.output_ids.copy() - output_text = state.get_text() - out_dict = { - "text": output_text, - "output_ids": output_token_ids, - "meta_info": meta_info, - } + # Non-incremental intermediate: pass reference (no + # copy) and defer text to _wait_one_response to avoid + # O(n) per-step cost that compounds to O(n^2). + out_dict = { + "text": None, + "output_ids": state.output_ids, + "meta_info": meta_info, + } elif state.finished: out_dict = { "text": state.get_text(), @@ -1739,12 +1761,20 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerScoreMixin): output_token_ids = delta_output_ids _slice_streaming_output_meta_info(meta_info, output_offset) state.last_output_offset = len(state.output_ids) + out_dict = { + "output_ids": output_token_ids, + "meta_info": meta_info, + } + elif state.finished: + out_dict = { + "output_ids": state.output_ids.copy(), + "meta_info": meta_info, + } else: - output_token_ids = state.output_ids.copy() - out_dict = { - "output_ids": output_token_ids, - "meta_info": meta_info, - } + out_dict = { + "output_ids": state.output_ids, + "meta_info": meta_info, + } elif state.finished: out_dict = { "output_ids": state.output_ids.copy(),