[Perf] detokenizer: Group return IPC sends by tokenizer worker (#44288)
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> Co-authored-by: hnyls2002 <lsyincs@gmail.com> Co-authored-by: Liangsheng Yin <hnyls2002@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
hnyls2002
Liangsheng Yin
parent
d3dc14eb38
commit
4273814771
@@ -29,7 +29,7 @@ import sys
|
||||
import threading
|
||||
import zlib
|
||||
from multiprocessing import shared_memory
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Type
|
||||
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Type
|
||||
|
||||
import msgspec
|
||||
import psutil
|
||||
@@ -495,6 +495,41 @@ def _handle_output_by_indices(output: Any, indices: List[int]) -> Any:
|
||||
)
|
||||
|
||||
|
||||
def _split_batch_by_key(
|
||||
output: Any, ipcs: List[Optional[str]], key_fn: Callable[[Optional[str]], Any]
|
||||
) -> Optional[List[Tuple[Any, Any]]]:
|
||||
indices_by_key: Dict[Any, List[int]] = {}
|
||||
for index, ipc_key in enumerate(ipcs):
|
||||
indices_by_key.setdefault(key_fn(ipc_key), []).append(index)
|
||||
|
||||
# Build every sub-batch before any send, so a fallback never duplicates one.
|
||||
try:
|
||||
grouped = [
|
||||
(key, _handle_output_by_indices(output, indices), indices)
|
||||
for key, indices in indices_by_key.items()
|
||||
]
|
||||
except _CannotSplitBatch:
|
||||
return None
|
||||
|
||||
for _, sub_output, indices in grouped:
|
||||
sub_output.http_worker_ipcs = [ipcs[index] for index in indices]
|
||||
return [(key, sub_output) for key, sub_output, _ in grouped]
|
||||
|
||||
|
||||
def _send_output_to_tokenizer_workers(
|
||||
output: Any, ipcs: List[Optional[str]], socket_mapping: SocketMapping
|
||||
) -> None:
|
||||
grouped = _split_batch_by_key(output, ipcs, lambda ipc_name: ipc_name)
|
||||
if grouped is None:
|
||||
for index, ipc_name in enumerate(ipcs):
|
||||
single_output = _handle_output_by_index(output, index)
|
||||
socket_mapping.send_output(ipc_name, single_output, is_tokenizer=True)
|
||||
return
|
||||
|
||||
for ipc_name, grouped_output in grouped:
|
||||
socket_mapping.send_output(ipc_name, grouped_output, is_tokenizer=True)
|
||||
|
||||
|
||||
class MultiHttpWorkerDetokenizerMixin:
|
||||
"""Mixin class for DetokenizerManager"""
|
||||
|
||||
@@ -516,11 +551,9 @@ class MultiHttpWorkerDetokenizerMixin:
|
||||
# In multi-detokenizer mode the upstream MultiDetokenizerRouter may
|
||||
# forward either batched or single requests, so handle both shapes.
|
||||
if isinstance(recv_obj, BaseBatchReq):
|
||||
for i, ipc_name in enumerate(recv_obj.http_worker_ipcs):
|
||||
new_output = _handle_output_by_index(output, i)
|
||||
self.socket_mapping.send_output(
|
||||
ipc_name, new_output, is_tokenizer=True
|
||||
)
|
||||
_send_output_to_tokenizer_workers(
|
||||
output, recv_obj.http_worker_ipcs, self.socket_mapping
|
||||
)
|
||||
elif isinstance(recv_obj, BaseReq):
|
||||
self.socket_mapping.send_output(
|
||||
recv_obj.http_worker_ipc, output, is_tokenizer=True
|
||||
@@ -681,7 +714,7 @@ class MultiDetokenizerRouter:
|
||||
def _send(self, ipc_name: str, obj: Any) -> None:
|
||||
self.socket_mapping.send_output(ipc_name, obj, is_tokenizer=False)
|
||||
|
||||
def _split_batch_by_target(self, recv_obj: BaseBatchReq):
|
||||
def _send_batch(self, recv_obj: BaseBatchReq) -> None:
|
||||
ipcs = recv_obj.http_worker_ipcs
|
||||
assert (
|
||||
ipcs is not None
|
||||
@@ -689,29 +722,7 @@ class MultiDetokenizerRouter:
|
||||
and all(x is not None for x in ipcs)
|
||||
), f"Batch req {recv_obj.rids=} has invalid http_worker_ipcs"
|
||||
|
||||
indices_by_target: Dict[str, List[int]] = {}
|
||||
for index, ipc_key in enumerate(ipcs):
|
||||
target = self._pick(ipc_key)
|
||||
indices_by_target.setdefault(target, []).append(index)
|
||||
|
||||
try:
|
||||
grouped = [
|
||||
(
|
||||
target,
|
||||
_handle_output_by_indices(recv_obj, indices),
|
||||
indices,
|
||||
)
|
||||
for target, indices in indices_by_target.items()
|
||||
]
|
||||
except _CannotSplitBatch:
|
||||
return None
|
||||
|
||||
for _, output, indices in grouped:
|
||||
output.http_worker_ipcs = [ipcs[index] for index in indices]
|
||||
return grouped
|
||||
|
||||
def _send_batch(self, recv_obj: BaseBatchReq) -> None:
|
||||
grouped = self._split_batch_by_target(recv_obj)
|
||||
grouped = _split_batch_by_key(recv_obj, ipcs, self._pick)
|
||||
if grouped is None:
|
||||
for index, ipc_key in enumerate(recv_obj.http_worker_ipcs):
|
||||
one = _handle_output_by_index(recv_obj, index)
|
||||
@@ -721,7 +732,7 @@ class MultiDetokenizerRouter:
|
||||
self._send(self._pick(ipc_key), one)
|
||||
return
|
||||
|
||||
for target, output, _ in grouped:
|
||||
for target, output in grouped:
|
||||
self._send(target, output)
|
||||
|
||||
def event_loop(self):
|
||||
|
||||
Reference in New Issue
Block a user