[Fix] detokenizer router: Route by rid without tokenizer worker ipcs and broadcast logging config (#33715)

Co-authored-by: hnyls2002 <lsyincs@gmail.com>
This commit is contained in:
Aditya Singh
2026-10-14 17:20:21 -07:00
committed by GitHub
co-authored by hnyls2002
parent 0536cec73d
commit a6a8c56015
2 changed files with 82 additions and 21 deletions
@@ -30,7 +30,7 @@ import threading
import zlib
from functools import lru_cache
from multiprocessing import shared_memory
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Type
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type
import msgspec
import psutil
@@ -46,6 +46,7 @@ from sglang.srt.managers.io_struct import (
BatchEmbeddingOutput,
BatchStrOutput,
BatchTokenIDOutput,
ConfigureLoggingReq,
ContinueGenerationReqInput,
FreezeGCReq,
PauseContinueBroadcastReq,
@@ -516,11 +517,11 @@ def _handle_output_by_indices(output: Any, indices: List[int]) -> Any:
def _split_batch_by_key(
output: Any, ipcs: List[Optional[str]], key_fn: Callable[[Optional[str]], Any]
output: Any, keys: List[Any]
) -> Optional[List[Tuple[Any, Any]]]:
indices_by_key: Dict[Any, List[int]] = {}
for index, ipc_key in enumerate(ipcs):
indices_by_key.setdefault(key_fn(ipc_key), []).append(index)
for index, key in enumerate(keys):
indices_by_key.setdefault(key, []).append(index)
# Build every sub-batch before any send, so a fallback never duplicates one.
try:
@@ -531,15 +532,18 @@ def _split_batch_by_key(
except _CannotSplitBatch:
return None
ipcs = output.http_worker_ipcs
for _, sub_output, indices in grouped:
sub_output.http_worker_ipcs = [ipcs[index] for index in indices]
sub_output.http_worker_ipcs = (
None if ipcs is None else [ipcs[index] for index in indices]
)
return [(key, sub_output) for key, sub_output, _ in grouped]
def _send_output_to_tokenizer_workers(
output: Any, ipcs: List[Optional[str]], socket_mapping: SocketMapping
) -> None:
grouped = _split_batch_by_key(output, ipcs, lambda ipc_name: ipc_name)
grouped = _split_batch_by_key(output, ipcs)
if grouped is None:
for index, ipc_name in enumerate(ipcs):
single_output = _handle_output_by_index(output, index)
@@ -717,6 +721,8 @@ class MultiDetokenizerRouter:
Each request is pinned to a worker by hashing its ``http_worker_ipc`` with
``zlib.crc32`` (deterministic across runs), so all outputs of the same rid
always land on the same detokenizer and ``decode_status`` stays consistent.
With a single tokenizer worker no ``http_worker_ipc`` is stamped, and the
``rid`` is hashed instead.
"""
def __init__(self, ipc_name_list: List[str], port_args: PortArgs):
@@ -734,22 +740,24 @@ class MultiDetokenizerRouter:
def _send(self, ipc_name: str, obj: Any) -> None:
self.socket_mapping.send_output(ipc_name, obj, is_tokenizer=False)
def _pick_for(self, http_worker_ipc: Optional[str], rid: str) -> str:
return self._pick(rid if http_worker_ipc is None else http_worker_ipc)
def _send_batch(self, recv_obj: BaseBatchReq) -> None:
ipcs = recv_obj.http_worker_ipcs
assert (
ipcs is not None
and len(ipcs) == len(recv_obj.rids)
and all(x is not None for x in ipcs)
), f"Batch req {recv_obj.rids=} has invalid http_worker_ipcs"
assert ipcs is not None and len(ipcs) == len(recv_obj.rids), (
f"Batch req {recv_obj.rids=} has invalid http_worker_ipcs"
)
targets = [self._pick_for(ipc, rid) for ipc, rid in zip(ipcs, recv_obj.rids)]
grouped = _split_batch_by_key(recv_obj, ipcs, self._pick)
grouped = _split_batch_by_key(recv_obj, targets)
if grouped is None:
for index, ipc_key in enumerate(recv_obj.http_worker_ipcs):
for index, target in enumerate(targets):
one = _handle_output_by_index(recv_obj, index)
if one is recv_obj:
raise TypeError(f"Cannot split {type(recv_obj)}")
one.http_worker_ipcs = [ipc_key]
self._send(self._pick(ipc_key), one)
one.http_worker_ipcs = [ipcs[index]]
self._send(target, one)
return
for target, output in grouped:
@@ -759,18 +767,20 @@ class MultiDetokenizerRouter:
while True:
recv_obj = sock_recv(self.recv_from_scheduler)
# FreezeGCReq must freeze every detokenizer process.
if isinstance(recv_obj, FreezeGCReq):
# Control messages must reach every detokenizer process.
if isinstance(recv_obj, (FreezeGCReq, ConfigureLoggingReq)):
for ipc in self.ipc_name_list:
self._send(ipc, recv_obj)
continue
# Single request: route by its own http_worker_ipc.
# Single request: route like the batch items of the same rid.
if isinstance(recv_obj, BaseReq):
assert recv_obj.http_worker_ipc is not None, (
f"Single req {recv_obj.rid=} missing http_worker_ipc"
assert (
recv_obj.http_worker_ipc is not None or recv_obj.rid is not None
), f"Single req {type(recv_obj).__name__} has no routing key"
self._send(
self._pick_for(recv_obj.http_worker_ipc, recv_obj.rid), recv_obj
)
self._send(self._pick(recv_obj.http_worker_ipc), recv_obj)
continue
# Batch request.