[Perf] detokenizer: Group return IPC sends by tokenizer worker (#44288)

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
Co-authored-by: hnyls2002 <lsyincs@gmail.com>
Co-authored-by: Liangsheng Yin <hnyls2002@gmail.com>
This commit is contained in:
Liu Ke
2026-10-14 14:26:51 -07:00
committed by GitHub
co-authored by Claude Opus 4.6 hnyls2002 Liangsheng Yin
parent d3dc14eb38
commit 4273814771
2 changed files with 132 additions and 31 deletions
@@ -29,7 +29,7 @@ import sys
import threading
import zlib
from multiprocessing import shared_memory
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Type
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Type
import msgspec
import psutil
@@ -495,6 +495,41 @@ def _handle_output_by_indices(output: Any, indices: List[int]) -> Any:
)
def _split_batch_by_key(
output: Any, ipcs: List[Optional[str]], key_fn: Callable[[Optional[str]], Any]
) -> Optional[List[Tuple[Any, Any]]]:
indices_by_key: Dict[Any, List[int]] = {}
for index, ipc_key in enumerate(ipcs):
indices_by_key.setdefault(key_fn(ipc_key), []).append(index)
# Build every sub-batch before any send, so a fallback never duplicates one.
try:
grouped = [
(key, _handle_output_by_indices(output, indices), indices)
for key, indices in indices_by_key.items()
]
except _CannotSplitBatch:
return None
for _, sub_output, indices in grouped:
sub_output.http_worker_ipcs = [ipcs[index] for index in indices]
return [(key, sub_output) for key, sub_output, _ in grouped]
def _send_output_to_tokenizer_workers(
output: Any, ipcs: List[Optional[str]], socket_mapping: SocketMapping
) -> None:
grouped = _split_batch_by_key(output, ipcs, lambda ipc_name: ipc_name)
if grouped is None:
for index, ipc_name in enumerate(ipcs):
single_output = _handle_output_by_index(output, index)
socket_mapping.send_output(ipc_name, single_output, is_tokenizer=True)
return
for ipc_name, grouped_output in grouped:
socket_mapping.send_output(ipc_name, grouped_output, is_tokenizer=True)
class MultiHttpWorkerDetokenizerMixin:
"""Mixin class for DetokenizerManager"""
@@ -516,11 +551,9 @@ class MultiHttpWorkerDetokenizerMixin:
# In multi-detokenizer mode the upstream MultiDetokenizerRouter may
# forward either batched or single requests, so handle both shapes.
if isinstance(recv_obj, BaseBatchReq):
for i, ipc_name in enumerate(recv_obj.http_worker_ipcs):
new_output = _handle_output_by_index(output, i)
self.socket_mapping.send_output(
ipc_name, new_output, is_tokenizer=True
)
_send_output_to_tokenizer_workers(
output, recv_obj.http_worker_ipcs, self.socket_mapping
)
elif isinstance(recv_obj, BaseReq):
self.socket_mapping.send_output(
recv_obj.http_worker_ipc, output, is_tokenizer=True
@@ -681,7 +714,7 @@ class MultiDetokenizerRouter:
def _send(self, ipc_name: str, obj: Any) -> None:
self.socket_mapping.send_output(ipc_name, obj, is_tokenizer=False)
def _split_batch_by_target(self, recv_obj: BaseBatchReq):
def _send_batch(self, recv_obj: BaseBatchReq) -> None:
ipcs = recv_obj.http_worker_ipcs
assert (
ipcs is not None
@@ -689,29 +722,7 @@ class MultiDetokenizerRouter:
and all(x is not None for x in ipcs)
), f"Batch req {recv_obj.rids=} has invalid http_worker_ipcs"
indices_by_target: Dict[str, List[int]] = {}
for index, ipc_key in enumerate(ipcs):
target = self._pick(ipc_key)
indices_by_target.setdefault(target, []).append(index)
try:
grouped = [
(
target,
_handle_output_by_indices(recv_obj, indices),
indices,
)
for target, indices in indices_by_target.items()
]
except _CannotSplitBatch:
return None
for _, output, indices in grouped:
output.http_worker_ipcs = [ipcs[index] for index in indices]
return grouped
def _send_batch(self, recv_obj: BaseBatchReq) -> None:
grouped = self._split_batch_by_target(recv_obj)
grouped = _split_batch_by_key(recv_obj, ipcs, self._pick)
if grouped is None:
for index, ipc_key in enumerate(recv_obj.http_worker_ipcs):
one = _handle_output_by_index(recv_obj, index)
@@ -721,7 +732,7 @@ class MultiDetokenizerRouter:
self._send(self._pick(ipc_key), one)
return
for target, output, _ in grouped:
for target, output in grouped:
self._send(target, output)
def event_loop(self):