[Perf] detokenizer: Add SGLANG_BATCH_TOKENIZER_OUTPUTS to opt out of grouped return sends (#45280)
Co-authored-by: joshuadeng <joshuadeng@users.noreply.github.com>
This commit is contained in:
co-authored by
joshuadeng
parent
bab4a808d8
commit
468e26f253
@@ -1575,6 +1575,8 @@ class Envs:
|
||||
# Think tokens budget: negative means unlimited, >= 0 caps thinking tokens
|
||||
SGLANG_MAX_THINK_TOKENS = EnvInt(-1)
|
||||
SGLANG_PATCH_TOKENIZER = EnvBool(True)
|
||||
# Group replies by tokenizer worker; disable for per-request delivery.
|
||||
SGLANG_BATCH_TOKENIZER_OUTPUTS = EnvBool(True)
|
||||
# Encode long rendered chat prompts as chunks on the tokenizers thread pool.
|
||||
SGLANG_PARALLEL_PROMPT_ENCODE = EnvBool(True)
|
||||
# Shorter prompts use the single-call encode; below this the gain is eaten
|
||||
|
||||
@@ -39,6 +39,7 @@ import zmq
|
||||
import zmq.asyncio
|
||||
|
||||
from sglang.srt.disaggregation.utils import TransferBackend
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.managers.disagg_service import start_disagg_service
|
||||
from sglang.srt.managers.io_struct import (
|
||||
BaseBatchReq,
|
||||
@@ -542,7 +543,11 @@ def _split_batch_by_key(
|
||||
def _send_output_to_tokenizer_workers(
|
||||
output: Any, ipcs: List[Optional[str]], socket_mapping: SocketMapping
|
||||
) -> None:
|
||||
grouped = _split_batch_by_key(output, ipcs)
|
||||
grouped = (
|
||||
_split_batch_by_key(output, ipcs)
|
||||
if envs.SGLANG_BATCH_TOKENIZER_OUTPUTS.get()
|
||||
else None
|
||||
)
|
||||
if grouped is None:
|
||||
for index, ipc_name in enumerate(ipcs):
|
||||
single_output = _handle_output_by_index(output, index)
|
||||
|
||||
Reference in New Issue
Block a user