[Perf] Reduce multi-detokenizer router IPC sends (#42238)

Co-authored-by: hnyls2002 <lsyincs@gmail.com>
Co-authored-by: Liangsheng Yin <hnyls2002@gmail.com>
This commit is contained in:
Liu Ke
2026-10-09 15:32:14 -07:00
committed by GitHub
co-authored by hnyls2002 Liangsheng Yin
parent 66d840a8ef
commit cb9bd1d4d4
2 changed files with 328 additions and 17 deletions
@@ -31,6 +31,7 @@ import zlib
from multiprocessing import shared_memory
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Type
import msgspec
import psutil
import setproctitle
import zmq
@@ -394,6 +395,106 @@ def _handle_output_by_index(output, i):
return new_output
class _CannotSplitBatch(Exception):
pass
def _extract_field_by_indices(
output: Any, field_name: str, indices: List[int], check_length: bool = True
) -> Any:
field = object.__getattribute__(output, field_name)
if field is None:
return None
should_wrap_result = field_name in ("customized_info", "time_stats")
if should_wrap_result:
field = unwrap_from_pickle(field)
if field is None:
return None
max_index = max(indices)
if isinstance(field, dict):
new_field = {}
for key, values in field.items():
if len(values) <= max_index:
raise _CannotSplitBatch(
f"{field_name}[{key!r}] cannot provide index {max_index}"
)
selected = [values[index] for index in indices]
new_field[key] = selected
if should_wrap_result:
return wrap_as_pickle(new_field) if new_field else None
return new_field
# Same as _extract_field_by_index: a field too short for every index (e.g.
# spec_* lists stay empty when speculative decoding is off) becomes None.
if check_length and len(field) <= min(indices):
return None
if len(field) <= max_index:
raise _CannotSplitBatch(f"{field_name} cannot provide index {max_index}")
selected = [field[index] for index in indices]
return wrap_as_pickle(selected) if should_wrap_result else selected
# Per-request fields that may be shorter than rids; the rest must match rids or
# be empty.
_UNCHECKED_LENGTH_FIELDS = frozenset(
{
"input_token_logprobs_val",
"input_token_logprobs_idx",
"output_token_logprobs_val",
"output_token_logprobs_idx",
"input_top_logprobs_val",
"input_top_logprobs_idx",
"input_top_logprobs_val_flat",
"input_top_logprobs_idx_flat",
"input_top_logprobs_flat_null_prefix",
"output_top_logprobs_val",
"output_top_logprobs_idx",
"input_token_ids_logprobs_val",
"input_token_ids_logprobs_idx",
"output_token_ids_logprobs_val",
"output_token_ids_logprobs_idx",
"output_token_entropy_val",
"output_token_sampling_mask",
"output_hidden_states",
"routed_experts",
"indexer_topk",
"token_steps",
"customized_info",
"dp_ranks",
}
)
_NOT_SPLIT_FIELDS = frozenset(
{"rids", "http_worker_ipcs", "placeholder_tokens_idx", "placeholder_tokens_val"}
)
def _handle_output_by_indices(output: Any, indices: List[int]) -> Any:
if len(indices) == 1:
return _handle_output_by_index(output, indices[0])
if not isinstance(output, (BatchTokenIDOutput, BatchStrOutput)):
raise _CannotSplitBatch(f"Cannot split {type(output)}")
split_fields = {
field.name: _extract_field_by_indices(
output,
field.name,
indices,
check_length=field.name not in _UNCHECKED_LENGTH_FIELDS,
)
for field in msgspec.structs.fields(output)
if field.name not in _NOT_SPLIT_FIELDS
}
return type(output)(
rids=[output.rids[index] for index in indices],
placeholder_tokens_idx=None,
placeholder_tokens_val=None,
**split_fields,
)
class MultiHttpWorkerDetokenizerMixin:
"""Mixin class for DetokenizerManager"""
@@ -580,6 +681,49 @@ class MultiDetokenizerRouter:
def _send(self, ipc_name: str, obj: Any) -> None:
self.socket_mapping.send_output(ipc_name, obj, is_tokenizer=False)
def _split_batch_by_target(self, recv_obj: BaseBatchReq):
ipcs = recv_obj.http_worker_ipcs
assert (
ipcs is not None
and len(ipcs) == len(recv_obj.rids)
and all(x is not None for x in ipcs)
), f"Batch req {recv_obj.rids=} has invalid http_worker_ipcs"
indices_by_target: Dict[str, List[int]] = {}
for index, ipc_key in enumerate(ipcs):
target = self._pick(ipc_key)
indices_by_target.setdefault(target, []).append(index)
try:
grouped = [
(
target,
_handle_output_by_indices(recv_obj, indices),
indices,
)
for target, indices in indices_by_target.items()
]
except _CannotSplitBatch:
return None
for _, output, indices in grouped:
output.http_worker_ipcs = [ipcs[index] for index in indices]
return grouped
def _send_batch(self, recv_obj: BaseBatchReq) -> None:
grouped = self._split_batch_by_target(recv_obj)
if grouped is None:
for index, ipc_key in enumerate(recv_obj.http_worker_ipcs):
one = _handle_output_by_index(recv_obj, index)
if one is recv_obj:
raise TypeError(f"Cannot split {type(recv_obj)}")
one.http_worker_ipcs = [ipc_key]
self._send(self._pick(ipc_key), one)
return
for target, output, _ in grouped:
self._send(target, output)
def event_loop(self):
while True:
recv_obj = sock_recv(self.recv_from_scheduler)
@@ -606,20 +750,7 @@ class MultiDetokenizerRouter:
self._send(ipc, recv_obj)
continue
ipcs = recv_obj.http_worker_ipcs
assert (
ipcs is not None
and len(ipcs) == len(recv_obj.rids)
and all(x is not None for x in ipcs)
), f"Batch req {recv_obj.rids=} has invalid http_worker_ipcs"
# Split per-item and route each by its own ipc.
for i, ipc_key in enumerate(ipcs):
one = _handle_output_by_index(recv_obj, i)
if one is recv_obj:
raise TypeError(f"Cannot split {type(recv_obj)}")
one.http_worker_ipcs = [ipc_key]
self._send(self._pick(ipc_key), one)
self._send_batch(recv_obj)
continue
raise ValueError(