From 468e26f253b2f1d42bf3545742b7c363d0c6592e Mon Sep 17 00:00:00 2001 From: metamergebot Date: Tue, 20 Oct 2026 13:24:14 -0700 Subject: [PATCH] [Perf] detokenizer: Add SGLANG_BATCH_TOKENIZER_OUTPUTS to opt out of grouped return sends (#45280) Co-authored-by: joshuadeng --- python/sglang/srt/environ.py | 2 + .../srt/managers/multi_tokenizer_mixin.py | 7 ++- .../managers/test_multi_tokenizer_mixin.py | 45 ++++++++++++------- 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 283a7d9470..d6b54ecf70 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1575,6 +1575,8 @@ class Envs: # Think tokens budget: negative means unlimited, >= 0 caps thinking tokens SGLANG_MAX_THINK_TOKENS = EnvInt(-1) SGLANG_PATCH_TOKENIZER = EnvBool(True) + # Group replies by tokenizer worker; disable for per-request delivery. + SGLANG_BATCH_TOKENIZER_OUTPUTS = EnvBool(True) # Encode long rendered chat prompts as chunks on the tokenizers thread pool. SGLANG_PARALLEL_PROMPT_ENCODE = EnvBool(True) # Shorter prompts use the single-call encode; below this the gain is eaten diff --git a/python/sglang/srt/managers/multi_tokenizer_mixin.py b/python/sglang/srt/managers/multi_tokenizer_mixin.py index 1cfe6e346a..d0768ee365 100644 --- a/python/sglang/srt/managers/multi_tokenizer_mixin.py +++ b/python/sglang/srt/managers/multi_tokenizer_mixin.py @@ -39,6 +39,7 @@ import zmq import zmq.asyncio from sglang.srt.disaggregation.utils import TransferBackend +from sglang.srt.environ import envs from sglang.srt.managers.disagg_service import start_disagg_service from sglang.srt.managers.io_struct import ( BaseBatchReq, @@ -542,7 +543,11 @@ def _split_batch_by_key( def _send_output_to_tokenizer_workers( output: Any, ipcs: List[Optional[str]], socket_mapping: SocketMapping ) -> None: - grouped = _split_batch_by_key(output, ipcs) + grouped = ( + _split_batch_by_key(output, ipcs) + if envs.SGLANG_BATCH_TOKENIZER_OUTPUTS.get() + else None + ) if grouped is None: for index, ipc_name in enumerate(ipcs): single_output = _handle_output_by_index(output, index) diff --git a/test/registered/unit/managers/test_multi_tokenizer_mixin.py b/test/registered/unit/managers/test_multi_tokenizer_mixin.py index 23f1d99ba1..708bef7146 100644 --- a/test/registered/unit/managers/test_multi_tokenizer_mixin.py +++ b/test/registered/unit/managers/test_multi_tokenizer_mixin.py @@ -16,6 +16,7 @@ maybe_stub_sgl_kernel() from sglang.srt.beam_search.types import BeamSearchSequence from sglang.srt.entrypoints import http_server +from sglang.srt.environ import envs from sglang.srt.managers import multi_tokenizer_mixin from sglang.srt.managers.io_struct import ( AbortReq, @@ -501,7 +502,7 @@ class TestMultiTokenizerMixin(CustomTestCase): ], ) - def test_return_path_groups_noncontiguous_owners(self): + def test_return_path_batching_preserves_owner_order(self): output = _handle_output_by_indices(_make_batch_str_output(), [0, 1, 0]) output.rids = ["rid-0", "rid-1", "rid-2"] output.output_strs = ["first", "second", "third"] @@ -515,22 +516,32 @@ class TestMultiTokenizerMixin(CustomTestCase): ) ) - _send_output_to_tokenizer_workers(output, ipcs, mapping) - - self.assertEqual( - sent, - [ - ( - "worker-b", - ["rid-0", "rid-2"], - ["first", "third"], - ["worker-b", "worker-b"], - True, - ), - ("worker-a", ["rid-1"], ["second"], ["worker-a"], True), - ], - ) - self.assertEqual(pickle.dumps(output), original) + expected_grouped = [ + ( + "worker-b", + ["rid-0", "rid-2"], + ["first", "third"], + ["worker-b", "worker-b"], + True, + ), + ("worker-a", ["rid-1"], ["second"], ["worker-a"], True), + ] + expected_individual = [ + ("worker-b", ["rid-0"], ["first"], None, True), + ("worker-a", ["rid-1"], ["second"], None, True), + ("worker-b", ["rid-2"], ["third"], None, True), + ] + for batching in (True, False): + sent.clear() + with ( + self.subTest(batching=batching), + envs.SGLANG_BATCH_TOKENIZER_OUTPUTS.override(batching), + ): + _send_output_to_tokenizer_workers(output, ipcs, mapping) + self.assertEqual( + sent, expected_grouped if batching else expected_individual + ) + self.assertEqual(pickle.dumps(output), original) def test_return_path_falls_back_before_sending_any_group(self): output = _make_batch_embedding_output()