[Perf] detokenizer: Add SGLANG_BATCH_TOKENIZER_OUTPUTS to opt out of grouped return sends (#45280)

Co-authored-by: joshuadeng <joshuadeng@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-20 13:24:14 -07:00
committed by GitHub
co-authored by joshuadeng
parent bab4a808d8
commit 468e26f253
3 changed files with 36 additions and 18 deletions
+2
View File
@@ -1575,6 +1575,8 @@ class Envs:
# Think tokens budget: negative means unlimited, >= 0 caps thinking tokens
SGLANG_MAX_THINK_TOKENS = EnvInt(-1)
SGLANG_PATCH_TOKENIZER = EnvBool(True)
# Group replies by tokenizer worker; disable for per-request delivery.
SGLANG_BATCH_TOKENIZER_OUTPUTS = EnvBool(True)
# Encode long rendered chat prompts as chunks on the tokenizers thread pool.
SGLANG_PARALLEL_PROMPT_ENCODE = EnvBool(True)
# Shorter prompts use the single-call encode; below this the gain is eaten
@@ -39,6 +39,7 @@ import zmq
import zmq.asyncio
from sglang.srt.disaggregation.utils import TransferBackend
from sglang.srt.environ import envs
from sglang.srt.managers.disagg_service import start_disagg_service
from sglang.srt.managers.io_struct import (
BaseBatchReq,
@@ -542,7 +543,11 @@ def _split_batch_by_key(
def _send_output_to_tokenizer_workers(
output: Any, ipcs: List[Optional[str]], socket_mapping: SocketMapping
) -> None:
grouped = _split_batch_by_key(output, ipcs)
grouped = (
_split_batch_by_key(output, ipcs)
if envs.SGLANG_BATCH_TOKENIZER_OUTPUTS.get()
else None
)
if grouped is None:
for index, ipc_name in enumerate(ipcs):
single_output = _handle_output_by_index(output, index)