[Perf] Reduce multi-detokenizer router IPC sends (#42238)

Co-authored-by: hnyls2002 <lsyincs@gmail.com>
Co-authored-by: Liangsheng Yin <hnyls2002@gmail.com>
This commit is contained in:
Liu Ke
2026-10-09 15:32:14 -07:00
committed by GitHub
co-authored by hnyls2002 Liangsheng Yin
parent 66d840a8ef
commit cb9bd1d4d4
2 changed files with 328 additions and 17 deletions
@@ -31,6 +31,7 @@ import zlib
from multiprocessing import shared_memory
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Type
import msgspec
import psutil
import setproctitle
import zmq
@@ -394,6 +395,106 @@ def _handle_output_by_index(output, i):
return new_output
class _CannotSplitBatch(Exception):
pass
def _extract_field_by_indices(
output: Any, field_name: str, indices: List[int], check_length: bool = True
) -> Any:
field = object.__getattribute__(output, field_name)
if field is None:
return None
should_wrap_result = field_name in ("customized_info", "time_stats")
if should_wrap_result:
field = unwrap_from_pickle(field)
if field is None:
return None
max_index = max(indices)
if isinstance(field, dict):
new_field = {}
for key, values in field.items():
if len(values) <= max_index:
raise _CannotSplitBatch(
f"{field_name}[{key!r}] cannot provide index {max_index}"
)
selected = [values[index] for index in indices]
new_field[key] = selected
if should_wrap_result:
return wrap_as_pickle(new_field) if new_field else None
return new_field
# Same as _extract_field_by_index: a field too short for every index (e.g.
# spec_* lists stay empty when speculative decoding is off) becomes None.
if check_length and len(field) <= min(indices):
return None
if len(field) <= max_index:
raise _CannotSplitBatch(f"{field_name} cannot provide index {max_index}")
selected = [field[index] for index in indices]
return wrap_as_pickle(selected) if should_wrap_result else selected
# Per-request fields that may be shorter than rids; the rest must match rids or
# be empty.
_UNCHECKED_LENGTH_FIELDS = frozenset(
{
"input_token_logprobs_val",
"input_token_logprobs_idx",
"output_token_logprobs_val",
"output_token_logprobs_idx",
"input_top_logprobs_val",
"input_top_logprobs_idx",
"input_top_logprobs_val_flat",
"input_top_logprobs_idx_flat",
"input_top_logprobs_flat_null_prefix",
"output_top_logprobs_val",
"output_top_logprobs_idx",
"input_token_ids_logprobs_val",
"input_token_ids_logprobs_idx",
"output_token_ids_logprobs_val",
"output_token_ids_logprobs_idx",
"output_token_entropy_val",
"output_token_sampling_mask",
"output_hidden_states",
"routed_experts",
"indexer_topk",
"token_steps",
"customized_info",
"dp_ranks",
}
)
_NOT_SPLIT_FIELDS = frozenset(
{"rids", "http_worker_ipcs", "placeholder_tokens_idx", "placeholder_tokens_val"}
)
def _handle_output_by_indices(output: Any, indices: List[int]) -> Any:
if len(indices) == 1:
return _handle_output_by_index(output, indices[0])
if not isinstance(output, (BatchTokenIDOutput, BatchStrOutput)):
raise _CannotSplitBatch(f"Cannot split {type(output)}")
split_fields = {
field.name: _extract_field_by_indices(
output,
field.name,
indices,
check_length=field.name not in _UNCHECKED_LENGTH_FIELDS,
)
for field in msgspec.structs.fields(output)
if field.name not in _NOT_SPLIT_FIELDS
}
return type(output)(
rids=[output.rids[index] for index in indices],
placeholder_tokens_idx=None,
placeholder_tokens_val=None,
**split_fields,
)
class MultiHttpWorkerDetokenizerMixin:
"""Mixin class for DetokenizerManager"""
@@ -580,6 +681,49 @@ class MultiDetokenizerRouter:
def _send(self, ipc_name: str, obj: Any) -> None:
self.socket_mapping.send_output(ipc_name, obj, is_tokenizer=False)
def _split_batch_by_target(self, recv_obj: BaseBatchReq):
ipcs = recv_obj.http_worker_ipcs
assert (
ipcs is not None
and len(ipcs) == len(recv_obj.rids)
and all(x is not None for x in ipcs)
), f"Batch req {recv_obj.rids=} has invalid http_worker_ipcs"
indices_by_target: Dict[str, List[int]] = {}
for index, ipc_key in enumerate(ipcs):
target = self._pick(ipc_key)
indices_by_target.setdefault(target, []).append(index)
try:
grouped = [
(
target,
_handle_output_by_indices(recv_obj, indices),
indices,
)
for target, indices in indices_by_target.items()
]
except _CannotSplitBatch:
return None
for _, output, indices in grouped:
output.http_worker_ipcs = [ipcs[index] for index in indices]
return grouped
def _send_batch(self, recv_obj: BaseBatchReq) -> None:
grouped = self._split_batch_by_target(recv_obj)
if grouped is None:
for index, ipc_key in enumerate(recv_obj.http_worker_ipcs):
one = _handle_output_by_index(recv_obj, index)
if one is recv_obj:
raise TypeError(f"Cannot split {type(recv_obj)}")
one.http_worker_ipcs = [ipc_key]
self._send(self._pick(ipc_key), one)
return
for target, output, _ in grouped:
self._send(target, output)
def event_loop(self):
while True:
recv_obj = sock_recv(self.recv_from_scheduler)
@@ -606,20 +750,7 @@ class MultiDetokenizerRouter:
self._send(ipc, recv_obj)
continue
ipcs = recv_obj.http_worker_ipcs
assert (
ipcs is not None
and len(ipcs) == len(recv_obj.rids)
and all(x is not None for x in ipcs)
), f"Batch req {recv_obj.rids=} has invalid http_worker_ipcs"
# Split per-item and route each by its own ipc.
for i, ipc_key in enumerate(ipcs):
one = _handle_output_by_index(recv_obj, i)
if one is recv_obj:
raise TypeError(f"Cannot split {type(recv_obj)}")
one.http_worker_ipcs = [ipc_key]
self._send(self._pick(ipc_key), one)
self._send_batch(recv_obj)
continue
raise ValueError(
@@ -5,14 +5,22 @@ import numpy as np
from sglang.srt.sampling.sampling_mask import SamplingMaskChunk
from sglang.srt.utils.weight_versions import WeightVersionSpan
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import maybe_stub_sgl_kernel
from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel
maybe_stub_sgl_kernel()
from sglang.srt.managers.io_struct import BatchStrOutput
from sglang.srt.beam_search.types import BeamSearchSequence
from sglang.srt.managers.io_struct import (
BatchEmbeddingOutput,
BatchStrOutput,
BatchTokenIDOutput,
BeamSearchOutput,
)
from sglang.srt.managers.multi_tokenizer_mixin import (
MultiDetokenizerRouter,
TokenizerWorker,
_handle_output_by_index,
_handle_output_by_indices,
get_tokenizer_worker_class,
)
@@ -97,10 +105,89 @@ def _make_batch_str_output() -> BatchStrOutput:
],
[WeightVersionSpan(version="v2", start=0, end=2)],
],
input_top_logprobs_val_flat=[np.array([[1.0]], np.float32), None],
input_top_logprobs_idx_flat=[np.array([[1]], np.int32), None],
input_top_logprobs_flat_null_prefix=[0, None],
beam_search_output=[
BeamSearchOutput(
sequences=[
BeamSearchSequence(tokens=[11], cum_logprob=-1.0),
]
),
None,
],
token_steps=[[1], [2]],
)
class TestMultiTokenizerMixin(unittest.TestCase):
def _make_batch_token_id_output() -> BatchTokenIDOutput:
return BatchTokenIDOutput(
rids=["token-rid-0", "token-rid-1"],
# OutputStreamer leaves the spec_* lists empty without speculative decoding.
spec_verify_ct=[],
spec_num_correct_drafts=[],
spec_correct_drafts_histogram=[],
spec_num_block_accept_tokens=[],
spec_num_cap_tokens=[],
spec_cap_lens_histogram=[],
finished_reasons=[None, None],
decoded_texts=["first", "second"],
decode_ids=[[1], [2]],
read_offsets=[0, 1],
output_ids=[[1], [2]],
skip_special_tokens=[True, True],
spaces_between_special_tokens=[True, True],
no_stop_trim=[False, False],
prompt_tokens=[10, 20],
reasoning_tokens=[0, 0],
completion_tokens=[1, 2],
cached_tokens=[3, 4],
input_token_logprobs_val=[[], []],
input_token_logprobs_idx=[[], []],
output_token_logprobs_val=[[], []],
output_token_logprobs_idx=[[], []],
input_top_logprobs_val=[[], []],
input_top_logprobs_idx=[[], []],
output_top_logprobs_val=[[], []],
output_top_logprobs_idx=[[], []],
input_token_ids_logprobs_val=[[], []],
input_token_ids_logprobs_idx=[[], []],
output_token_ids_logprobs_val=[[], []],
output_token_ids_logprobs_idx=[[], []],
output_token_entropy_val=[None, None],
output_token_sampling_mask=[None, None],
output_hidden_states=[None, None],
routed_experts=[None, None],
indexer_topk=[None, None],
placeholder_tokens_idx=[None, None],
placeholder_tokens_val=[None, None],
)
def _make_batch_embedding_output() -> BatchEmbeddingOutput:
return BatchEmbeddingOutput(
rids=["embedding-rid-0", "embedding-rid-1"],
finished_reasons=[None, None],
embeddings=[[0.1, 0.2], [0.3, 0.4]],
prompt_tokens=[10, 20],
cached_tokens=[3, 4],
placeholder_tokens_idx=[None, None],
placeholder_tokens_val=[None, None],
)
def _make_router() -> MultiDetokenizerRouter:
router = MultiDetokenizerRouter.__new__(MultiDetokenizerRouter)
router.ipc_name_list = ["detok-a", "detok-b"]
router.num_workers = 2
router._pick = lambda key: {
"worker-a": "detok-a",
"worker-b": "detok-b",
}[key]
return router
class TestMultiTokenizerMixin(CustomTestCase):
def test_batch_str_output_preserves_cached_tokens_details(self):
output = _make_batch_str_output()
@@ -148,6 +235,99 @@ class TestMultiTokenizerMixin(unittest.TestCase):
self.assertIsNone(_handle_output_by_index(output, 0).weight_versions)
def test_grouped_split_preserves_latest_fields_and_order(self):
output = _make_batch_str_output()
grouped = _handle_output_by_indices(output, [1, 0])
self.assertEqual(grouped.rids, ["rid-1", "rid-0"])
self.assertEqual(grouped.output_strs, ["second", "first"])
self.assertEqual(
grouped.cached_tokens_details,
[{"device": 1, "host": 3}, {"device": 3, "host": 0}],
)
self.assertEqual(
grouped.weight_versions,
[
[WeightVersionSpan(version="v2", start=0, end=2)],
[
WeightVersionSpan(version="v1", start=0, end=3),
WeightVersionSpan(version="v2", start=3, end=5),
],
],
)
self.assertEqual(grouped.input_top_logprobs_flat_null_prefix, [None, 0])
self.assertIsNone(grouped.beam_search_output[0])
self.assertEqual(
grouped.beam_search_output[1].sequences[0].tokens,
[11],
)
np.testing.assert_array_equal(
grouped.input_top_logprobs_val_flat[1], np.array([[1.0]], np.float32)
)
def test_router_groups_by_target_and_preserves_owner_ipcs(self):
output = _make_batch_str_output()
output.http_worker_ipcs = ["worker-a", "worker-a"]
router = _make_router()
sent = []
router._send = lambda target, obj: sent.append((target, obj))
router._send_batch(output)
self.assertEqual(len(sent), 1)
self.assertEqual(sent[0][0], "detok-a")
self.assertEqual(sent[0][1].rids, ["rid-0", "rid-1"])
self.assertEqual(sent[0][1].http_worker_ipcs, ["worker-a", "worker-a"])
self.assertEqual(output.rids, ["rid-0", "rid-1"])
self.assertEqual(output.http_worker_ipcs, ["worker-a", "worker-a"])
def test_router_keeps_first_target_order_and_does_not_cross_messages(self):
output = _make_batch_str_output()
output.http_worker_ipcs = ["worker-b", "worker-a"]
router = _make_router()
sent = []
router._send = lambda target, obj: sent.append((target, obj))
router._send_batch(output)
self.assertEqual(
[(target, obj.rids, obj.http_worker_ipcs) for target, obj in sent],
[
("detok-b", ["rid-0"], ["worker-b"]),
("detok-a", ["rid-1"], ["worker-a"]),
],
)
def test_router_groups_token_output_without_speculative_decoding(self):
output = _make_batch_token_id_output()
output.http_worker_ipcs = ["worker-a", "worker-a"]
router = _make_router()
sent = []
router._send = lambda target, obj: sent.append((target, obj))
router._send_batch(output)
self.assertEqual(len(sent), 1)
self.assertEqual(sent[0][0], "detok-a")
self.assertEqual(sent[0][1].rids, ["token-rid-0", "token-rid-1"])
self.assertEqual(sent[0][1].decoded_texts, ["first", "second"])
self.assertIsNone(sent[0][1].spec_verify_ct)
def test_router_sends_embedding_output_per_request(self):
output = _make_batch_embedding_output()
output.http_worker_ipcs = ["worker-a", "worker-a"]
router = _make_router()
sent = []
router._send = lambda target, obj: sent.append((target, obj))
router._send_batch(output)
self.assertEqual(
[(target, obj.rids) for target, obj in sent],
[("detok-a", ["embedding-rid-0"]), ("detok-a", ["embedding-rid-1"])],
)
def test_get_tokenizer_worker_class_uses_default(self):
self.assertIs(get_tokenizer_worker_class(DefaultServerArgs()), TokenizerWorker)