From cb9bd1d4d4b14828efe8dc7758fcd94020ab501c Mon Sep 17 00:00:00 2001 From: Liu Ke <48812234+hope1262946533@users.noreply.github.com> Date: Sat, 10 Oct 2026 06:32:14 +0800 Subject: [PATCH] [Perf] Reduce multi-detokenizer router IPC sends (#42238) Co-authored-by: hnyls2002 Co-authored-by: Liangsheng Yin --- .../srt/managers/multi_tokenizer_mixin.py | 159 +++++++++++++-- .../managers/test_multi_tokenizer_mixin.py | 186 +++++++++++++++++- 2 files changed, 328 insertions(+), 17 deletions(-) diff --git a/python/sglang/srt/managers/multi_tokenizer_mixin.py b/python/sglang/srt/managers/multi_tokenizer_mixin.py index 48cb012c13..09be583aa1 100644 --- a/python/sglang/srt/managers/multi_tokenizer_mixin.py +++ b/python/sglang/srt/managers/multi_tokenizer_mixin.py @@ -31,6 +31,7 @@ import zlib from multiprocessing import shared_memory from typing import TYPE_CHECKING, Any, Dict, List, Optional, Type +import msgspec import psutil import setproctitle import zmq @@ -394,6 +395,106 @@ def _handle_output_by_index(output, i): return new_output +class _CannotSplitBatch(Exception): + pass + + +def _extract_field_by_indices( + output: Any, field_name: str, indices: List[int], check_length: bool = True +) -> Any: + field = object.__getattribute__(output, field_name) + if field is None: + return None + + should_wrap_result = field_name in ("customized_info", "time_stats") + if should_wrap_result: + field = unwrap_from_pickle(field) + if field is None: + return None + + max_index = max(indices) + if isinstance(field, dict): + new_field = {} + for key, values in field.items(): + if len(values) <= max_index: + raise _CannotSplitBatch( + f"{field_name}[{key!r}] cannot provide index {max_index}" + ) + selected = [values[index] for index in indices] + new_field[key] = selected + if should_wrap_result: + return wrap_as_pickle(new_field) if new_field else None + return new_field + + # Same as _extract_field_by_index: a field too short for every index (e.g. + # spec_* lists stay empty when speculative decoding is off) becomes None. + if check_length and len(field) <= min(indices): + return None + if len(field) <= max_index: + raise _CannotSplitBatch(f"{field_name} cannot provide index {max_index}") + + selected = [field[index] for index in indices] + return wrap_as_pickle(selected) if should_wrap_result else selected + + +# Per-request fields that may be shorter than rids; the rest must match rids or +# be empty. +_UNCHECKED_LENGTH_FIELDS = frozenset( + { + "input_token_logprobs_val", + "input_token_logprobs_idx", + "output_token_logprobs_val", + "output_token_logprobs_idx", + "input_top_logprobs_val", + "input_top_logprobs_idx", + "input_top_logprobs_val_flat", + "input_top_logprobs_idx_flat", + "input_top_logprobs_flat_null_prefix", + "output_top_logprobs_val", + "output_top_logprobs_idx", + "input_token_ids_logprobs_val", + "input_token_ids_logprobs_idx", + "output_token_ids_logprobs_val", + "output_token_ids_logprobs_idx", + "output_token_entropy_val", + "output_token_sampling_mask", + "output_hidden_states", + "routed_experts", + "indexer_topk", + "token_steps", + "customized_info", + "dp_ranks", + } +) +_NOT_SPLIT_FIELDS = frozenset( + {"rids", "http_worker_ipcs", "placeholder_tokens_idx", "placeholder_tokens_val"} +) + + +def _handle_output_by_indices(output: Any, indices: List[int]) -> Any: + if len(indices) == 1: + return _handle_output_by_index(output, indices[0]) + if not isinstance(output, (BatchTokenIDOutput, BatchStrOutput)): + raise _CannotSplitBatch(f"Cannot split {type(output)}") + + split_fields = { + field.name: _extract_field_by_indices( + output, + field.name, + indices, + check_length=field.name not in _UNCHECKED_LENGTH_FIELDS, + ) + for field in msgspec.structs.fields(output) + if field.name not in _NOT_SPLIT_FIELDS + } + return type(output)( + rids=[output.rids[index] for index in indices], + placeholder_tokens_idx=None, + placeholder_tokens_val=None, + **split_fields, + ) + + class MultiHttpWorkerDetokenizerMixin: """Mixin class for DetokenizerManager""" @@ -580,6 +681,49 @@ class MultiDetokenizerRouter: def _send(self, ipc_name: str, obj: Any) -> None: self.socket_mapping.send_output(ipc_name, obj, is_tokenizer=False) + def _split_batch_by_target(self, recv_obj: BaseBatchReq): + ipcs = recv_obj.http_worker_ipcs + assert ( + ipcs is not None + and len(ipcs) == len(recv_obj.rids) + and all(x is not None for x in ipcs) + ), f"Batch req {recv_obj.rids=} has invalid http_worker_ipcs" + + indices_by_target: Dict[str, List[int]] = {} + for index, ipc_key in enumerate(ipcs): + target = self._pick(ipc_key) + indices_by_target.setdefault(target, []).append(index) + + try: + grouped = [ + ( + target, + _handle_output_by_indices(recv_obj, indices), + indices, + ) + for target, indices in indices_by_target.items() + ] + except _CannotSplitBatch: + return None + + for _, output, indices in grouped: + output.http_worker_ipcs = [ipcs[index] for index in indices] + return grouped + + def _send_batch(self, recv_obj: BaseBatchReq) -> None: + grouped = self._split_batch_by_target(recv_obj) + if grouped is None: + for index, ipc_key in enumerate(recv_obj.http_worker_ipcs): + one = _handle_output_by_index(recv_obj, index) + if one is recv_obj: + raise TypeError(f"Cannot split {type(recv_obj)}") + one.http_worker_ipcs = [ipc_key] + self._send(self._pick(ipc_key), one) + return + + for target, output, _ in grouped: + self._send(target, output) + def event_loop(self): while True: recv_obj = sock_recv(self.recv_from_scheduler) @@ -606,20 +750,7 @@ class MultiDetokenizerRouter: self._send(ipc, recv_obj) continue - ipcs = recv_obj.http_worker_ipcs - assert ( - ipcs is not None - and len(ipcs) == len(recv_obj.rids) - and all(x is not None for x in ipcs) - ), f"Batch req {recv_obj.rids=} has invalid http_worker_ipcs" - - # Split per-item and route each by its own ipc. - for i, ipc_key in enumerate(ipcs): - one = _handle_output_by_index(recv_obj, i) - if one is recv_obj: - raise TypeError(f"Cannot split {type(recv_obj)}") - one.http_worker_ipcs = [ipc_key] - self._send(self._pick(ipc_key), one) + self._send_batch(recv_obj) continue raise ValueError( diff --git a/test/registered/unit/managers/test_multi_tokenizer_mixin.py b/test/registered/unit/managers/test_multi_tokenizer_mixin.py index 02a05ca209..08445b3e2c 100644 --- a/test/registered/unit/managers/test_multi_tokenizer_mixin.py +++ b/test/registered/unit/managers/test_multi_tokenizer_mixin.py @@ -5,14 +5,22 @@ import numpy as np from sglang.srt.sampling.sampling_mask import SamplingMaskChunk from sglang.srt.utils.weight_versions import WeightVersionSpan from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.test_utils import maybe_stub_sgl_kernel +from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel maybe_stub_sgl_kernel() -from sglang.srt.managers.io_struct import BatchStrOutput +from sglang.srt.beam_search.types import BeamSearchSequence +from sglang.srt.managers.io_struct import ( + BatchEmbeddingOutput, + BatchStrOutput, + BatchTokenIDOutput, + BeamSearchOutput, +) from sglang.srt.managers.multi_tokenizer_mixin import ( + MultiDetokenizerRouter, TokenizerWorker, _handle_output_by_index, + _handle_output_by_indices, get_tokenizer_worker_class, ) @@ -97,10 +105,89 @@ def _make_batch_str_output() -> BatchStrOutput: ], [WeightVersionSpan(version="v2", start=0, end=2)], ], + input_top_logprobs_val_flat=[np.array([[1.0]], np.float32), None], + input_top_logprobs_idx_flat=[np.array([[1]], np.int32), None], + input_top_logprobs_flat_null_prefix=[0, None], + beam_search_output=[ + BeamSearchOutput( + sequences=[ + BeamSearchSequence(tokens=[11], cum_logprob=-1.0), + ] + ), + None, + ], + token_steps=[[1], [2]], ) -class TestMultiTokenizerMixin(unittest.TestCase): +def _make_batch_token_id_output() -> BatchTokenIDOutput: + return BatchTokenIDOutput( + rids=["token-rid-0", "token-rid-1"], + # OutputStreamer leaves the spec_* lists empty without speculative decoding. + spec_verify_ct=[], + spec_num_correct_drafts=[], + spec_correct_drafts_histogram=[], + spec_num_block_accept_tokens=[], + spec_num_cap_tokens=[], + spec_cap_lens_histogram=[], + finished_reasons=[None, None], + decoded_texts=["first", "second"], + decode_ids=[[1], [2]], + read_offsets=[0, 1], + output_ids=[[1], [2]], + skip_special_tokens=[True, True], + spaces_between_special_tokens=[True, True], + no_stop_trim=[False, False], + prompt_tokens=[10, 20], + reasoning_tokens=[0, 0], + completion_tokens=[1, 2], + cached_tokens=[3, 4], + input_token_logprobs_val=[[], []], + input_token_logprobs_idx=[[], []], + output_token_logprobs_val=[[], []], + output_token_logprobs_idx=[[], []], + input_top_logprobs_val=[[], []], + input_top_logprobs_idx=[[], []], + output_top_logprobs_val=[[], []], + output_top_logprobs_idx=[[], []], + input_token_ids_logprobs_val=[[], []], + input_token_ids_logprobs_idx=[[], []], + output_token_ids_logprobs_val=[[], []], + output_token_ids_logprobs_idx=[[], []], + output_token_entropy_val=[None, None], + output_token_sampling_mask=[None, None], + output_hidden_states=[None, None], + routed_experts=[None, None], + indexer_topk=[None, None], + placeholder_tokens_idx=[None, None], + placeholder_tokens_val=[None, None], + ) + + +def _make_batch_embedding_output() -> BatchEmbeddingOutput: + return BatchEmbeddingOutput( + rids=["embedding-rid-0", "embedding-rid-1"], + finished_reasons=[None, None], + embeddings=[[0.1, 0.2], [0.3, 0.4]], + prompt_tokens=[10, 20], + cached_tokens=[3, 4], + placeholder_tokens_idx=[None, None], + placeholder_tokens_val=[None, None], + ) + + +def _make_router() -> MultiDetokenizerRouter: + router = MultiDetokenizerRouter.__new__(MultiDetokenizerRouter) + router.ipc_name_list = ["detok-a", "detok-b"] + router.num_workers = 2 + router._pick = lambda key: { + "worker-a": "detok-a", + "worker-b": "detok-b", + }[key] + return router + + +class TestMultiTokenizerMixin(CustomTestCase): def test_batch_str_output_preserves_cached_tokens_details(self): output = _make_batch_str_output() @@ -148,6 +235,99 @@ class TestMultiTokenizerMixin(unittest.TestCase): self.assertIsNone(_handle_output_by_index(output, 0).weight_versions) + def test_grouped_split_preserves_latest_fields_and_order(self): + output = _make_batch_str_output() + + grouped = _handle_output_by_indices(output, [1, 0]) + + self.assertEqual(grouped.rids, ["rid-1", "rid-0"]) + self.assertEqual(grouped.output_strs, ["second", "first"]) + self.assertEqual( + grouped.cached_tokens_details, + [{"device": 1, "host": 3}, {"device": 3, "host": 0}], + ) + self.assertEqual( + grouped.weight_versions, + [ + [WeightVersionSpan(version="v2", start=0, end=2)], + [ + WeightVersionSpan(version="v1", start=0, end=3), + WeightVersionSpan(version="v2", start=3, end=5), + ], + ], + ) + self.assertEqual(grouped.input_top_logprobs_flat_null_prefix, [None, 0]) + self.assertIsNone(grouped.beam_search_output[0]) + self.assertEqual( + grouped.beam_search_output[1].sequences[0].tokens, + [11], + ) + np.testing.assert_array_equal( + grouped.input_top_logprobs_val_flat[1], np.array([[1.0]], np.float32) + ) + + def test_router_groups_by_target_and_preserves_owner_ipcs(self): + output = _make_batch_str_output() + output.http_worker_ipcs = ["worker-a", "worker-a"] + router = _make_router() + sent = [] + router._send = lambda target, obj: sent.append((target, obj)) + + router._send_batch(output) + + self.assertEqual(len(sent), 1) + self.assertEqual(sent[0][0], "detok-a") + self.assertEqual(sent[0][1].rids, ["rid-0", "rid-1"]) + self.assertEqual(sent[0][1].http_worker_ipcs, ["worker-a", "worker-a"]) + self.assertEqual(output.rids, ["rid-0", "rid-1"]) + self.assertEqual(output.http_worker_ipcs, ["worker-a", "worker-a"]) + + def test_router_keeps_first_target_order_and_does_not_cross_messages(self): + output = _make_batch_str_output() + output.http_worker_ipcs = ["worker-b", "worker-a"] + router = _make_router() + sent = [] + router._send = lambda target, obj: sent.append((target, obj)) + + router._send_batch(output) + + self.assertEqual( + [(target, obj.rids, obj.http_worker_ipcs) for target, obj in sent], + [ + ("detok-b", ["rid-0"], ["worker-b"]), + ("detok-a", ["rid-1"], ["worker-a"]), + ], + ) + + def test_router_groups_token_output_without_speculative_decoding(self): + output = _make_batch_token_id_output() + output.http_worker_ipcs = ["worker-a", "worker-a"] + router = _make_router() + sent = [] + router._send = lambda target, obj: sent.append((target, obj)) + + router._send_batch(output) + + self.assertEqual(len(sent), 1) + self.assertEqual(sent[0][0], "detok-a") + self.assertEqual(sent[0][1].rids, ["token-rid-0", "token-rid-1"]) + self.assertEqual(sent[0][1].decoded_texts, ["first", "second"]) + self.assertIsNone(sent[0][1].spec_verify_ct) + + def test_router_sends_embedding_output_per_request(self): + output = _make_batch_embedding_output() + output.http_worker_ipcs = ["worker-a", "worker-a"] + router = _make_router() + sent = [] + router._send = lambda target, obj: sent.append((target, obj)) + + router._send_batch(output) + + self.assertEqual( + [(target, obj.rids) for target, obj in sent], + [("detok-a", ["embedding-rid-0"]), ("detok-a", ["embedding-rid-1"])], + ) + def test_get_tokenizer_worker_class_uses_default(self): self.assertIs(get_tokenizer_worker_class(DefaultServerArgs()), TokenizerWorker)