From 427381477133c7da86c69fa0e180a7013a71ed1d Mon Sep 17 00:00:00 2001 From: Liu Ke <48812234+hope1262946533@users.noreply.github.com> Date: Thu, 15 Oct 2026 05:26:51 +0800 Subject: [PATCH] [Perf] detokenizer: Group return IPC sends by tokenizer worker (#44288) Co-authored-by: Claude Opus 4.6 Co-authored-by: hnyls2002 Co-authored-by: Liangsheng Yin --- .../srt/managers/multi_tokenizer_mixin.py | 73 ++++++++------- .../managers/test_multi_tokenizer_mixin.py | 90 +++++++++++++++++++ 2 files changed, 132 insertions(+), 31 deletions(-) diff --git a/python/sglang/srt/managers/multi_tokenizer_mixin.py b/python/sglang/srt/managers/multi_tokenizer_mixin.py index 09be583aa1..5570a394bf 100644 --- a/python/sglang/srt/managers/multi_tokenizer_mixin.py +++ b/python/sglang/srt/managers/multi_tokenizer_mixin.py @@ -29,7 +29,7 @@ import sys import threading import zlib from multiprocessing import shared_memory -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Type +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Type import msgspec import psutil @@ -495,6 +495,41 @@ def _handle_output_by_indices(output: Any, indices: List[int]) -> Any: ) +def _split_batch_by_key( + output: Any, ipcs: List[Optional[str]], key_fn: Callable[[Optional[str]], Any] +) -> Optional[List[Tuple[Any, Any]]]: + indices_by_key: Dict[Any, List[int]] = {} + for index, ipc_key in enumerate(ipcs): + indices_by_key.setdefault(key_fn(ipc_key), []).append(index) + + # Build every sub-batch before any send, so a fallback never duplicates one. + try: + grouped = [ + (key, _handle_output_by_indices(output, indices), indices) + for key, indices in indices_by_key.items() + ] + except _CannotSplitBatch: + return None + + for _, sub_output, indices in grouped: + sub_output.http_worker_ipcs = [ipcs[index] for index in indices] + return [(key, sub_output) for key, sub_output, _ in grouped] + + +def _send_output_to_tokenizer_workers( + output: Any, ipcs: List[Optional[str]], socket_mapping: SocketMapping +) -> None: + grouped = _split_batch_by_key(output, ipcs, lambda ipc_name: ipc_name) + if grouped is None: + for index, ipc_name in enumerate(ipcs): + single_output = _handle_output_by_index(output, index) + socket_mapping.send_output(ipc_name, single_output, is_tokenizer=True) + return + + for ipc_name, grouped_output in grouped: + socket_mapping.send_output(ipc_name, grouped_output, is_tokenizer=True) + + class MultiHttpWorkerDetokenizerMixin: """Mixin class for DetokenizerManager""" @@ -516,11 +551,9 @@ class MultiHttpWorkerDetokenizerMixin: # In multi-detokenizer mode the upstream MultiDetokenizerRouter may # forward either batched or single requests, so handle both shapes. if isinstance(recv_obj, BaseBatchReq): - for i, ipc_name in enumerate(recv_obj.http_worker_ipcs): - new_output = _handle_output_by_index(output, i) - self.socket_mapping.send_output( - ipc_name, new_output, is_tokenizer=True - ) + _send_output_to_tokenizer_workers( + output, recv_obj.http_worker_ipcs, self.socket_mapping + ) elif isinstance(recv_obj, BaseReq): self.socket_mapping.send_output( recv_obj.http_worker_ipc, output, is_tokenizer=True @@ -681,7 +714,7 @@ class MultiDetokenizerRouter: def _send(self, ipc_name: str, obj: Any) -> None: self.socket_mapping.send_output(ipc_name, obj, is_tokenizer=False) - def _split_batch_by_target(self, recv_obj: BaseBatchReq): + def _send_batch(self, recv_obj: BaseBatchReq) -> None: ipcs = recv_obj.http_worker_ipcs assert ( ipcs is not None @@ -689,29 +722,7 @@ class MultiDetokenizerRouter: and all(x is not None for x in ipcs) ), f"Batch req {recv_obj.rids=} has invalid http_worker_ipcs" - indices_by_target: Dict[str, List[int]] = {} - for index, ipc_key in enumerate(ipcs): - target = self._pick(ipc_key) - indices_by_target.setdefault(target, []).append(index) - - try: - grouped = [ - ( - target, - _handle_output_by_indices(recv_obj, indices), - indices, - ) - for target, indices in indices_by_target.items() - ] - except _CannotSplitBatch: - return None - - for _, output, indices in grouped: - output.http_worker_ipcs = [ipcs[index] for index in indices] - return grouped - - def _send_batch(self, recv_obj: BaseBatchReq) -> None: - grouped = self._split_batch_by_target(recv_obj) + grouped = _split_batch_by_key(recv_obj, ipcs, self._pick) if grouped is None: for index, ipc_key in enumerate(recv_obj.http_worker_ipcs): one = _handle_output_by_index(recv_obj, index) @@ -721,7 +732,7 @@ class MultiDetokenizerRouter: self._send(self._pick(ipc_key), one) return - for target, output, _ in grouped: + for target, output in grouped: self._send(target, output) def event_loop(self): diff --git a/test/registered/unit/managers/test_multi_tokenizer_mixin.py b/test/registered/unit/managers/test_multi_tokenizer_mixin.py index 08445b3e2c..dcde88928f 100644 --- a/test/registered/unit/managers/test_multi_tokenizer_mixin.py +++ b/test/registered/unit/managers/test_multi_tokenizer_mixin.py @@ -1,4 +1,7 @@ +import pickle import unittest +from types import SimpleNamespace +from unittest.mock import MagicMock, patch import numpy as np @@ -18,9 +21,11 @@ from sglang.srt.managers.io_struct import ( ) from sglang.srt.managers.multi_tokenizer_mixin import ( MultiDetokenizerRouter, + MultiHttpWorkerDetokenizerMixin, TokenizerWorker, _handle_output_by_index, _handle_output_by_indices, + _send_output_to_tokenizer_workers, get_tokenizer_worker_class, ) @@ -328,6 +333,91 @@ class TestMultiTokenizerMixin(CustomTestCase): [("detok-a", ["embedding-rid-0"]), ("detok-a", ["embedding-rid-1"])], ) + def test_return_path_groups_noncontiguous_owners(self): + output = _handle_output_by_indices(_make_batch_str_output(), [0, 1, 0]) + output.rids = ["rid-0", "rid-1", "rid-2"] + output.output_strs = ["first", "second", "third"] + ipcs = ["worker-b", "worker-a", "worker-b"] + output.http_worker_ipcs = list(ipcs) + original = pickle.dumps(output) + sent = [] + mapping = SimpleNamespace( + send_output=lambda target, obj, *, is_tokenizer: sent.append( + (target, obj.rids, obj.output_strs, obj.http_worker_ipcs, is_tokenizer) + ) + ) + + _send_output_to_tokenizer_workers(output, ipcs, mapping) + + self.assertEqual( + sent, + [ + ( + "worker-b", + ["rid-0", "rid-2"], + ["first", "third"], + ["worker-b", "worker-b"], + True, + ), + ("worker-a", ["rid-1"], ["second"], ["worker-a"], True), + ], + ) + self.assertEqual(pickle.dumps(output), original) + + def test_return_path_falls_back_before_sending_any_group(self): + output = _make_batch_embedding_output() + output.rids.append("embedding-rid-2") + output.embeddings.append([0.5, 0.6]) + output.prompt_tokens.append(30) + output.cached_tokens.append(5) + output.finished_reasons.append(None) + sent = [] + mapping = SimpleNamespace( + send_output=lambda target, obj, *, is_tokenizer: sent.append((target, obj)) + ) + + _send_output_to_tokenizer_workers( + output, ["worker-b", "worker-a", "worker-a"], mapping + ) + + self.assertEqual( + [(target, obj.rids) for target, obj in sent], + [ + ("worker-b", ["embedding-rid-0"]), + ("worker-a", ["embedding-rid-1"]), + ("worker-a", ["embedding-rid-2"]), + ], + ) + + def test_event_loop_sends_one_message_per_owner(self): + output = _make_batch_str_output() + output.http_worker_ipcs = ["worker-a", "worker-a"] + worker = MultiHttpWorkerDetokenizerMixin() + worker.recv_from_scheduler = object() + worker.soft_watchdog = MagicMock() + worker._request_dispatcher = lambda obj: obj + sent = [] + mapping = SimpleNamespace( + send_output=lambda target, obj, *, is_tokenizer: sent.append( + (target, obj.rids) + ) + ) + + with ( + patch( + "sglang.srt.managers.multi_tokenizer_mixin.SocketMapping", + return_value=mapping, + ), + patch( + "sglang.srt.managers.multi_tokenizer_mixin.sock_recv", + side_effect=[output, EOFError("end of input")], + ), + self.assertRaisesRegex(EOFError, "end of input"), + ): + worker.multi_http_worker_event_loop() + + self.assertEqual(sent, [("worker-a", ["rid-0", "rid-1"])]) + def test_get_tokenizer_worker_class_uses_default(self): self.assertIs(get_tokenizer_worker_class(DefaultServerArgs()), TokenizerWorker)