diff --git a/python/sglang/srt/managers/multi_tokenizer_mixin.py b/python/sglang/srt/managers/multi_tokenizer_mixin.py index 973674a447..5a2ec5c6c8 100644 --- a/python/sglang/srt/managers/multi_tokenizer_mixin.py +++ b/python/sglang/srt/managers/multi_tokenizer_mixin.py @@ -30,7 +30,7 @@ import threading import zlib from functools import lru_cache from multiprocessing import shared_memory -from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Type +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type import msgspec import psutil @@ -46,6 +46,7 @@ from sglang.srt.managers.io_struct import ( BatchEmbeddingOutput, BatchStrOutput, BatchTokenIDOutput, + ConfigureLoggingReq, ContinueGenerationReqInput, FreezeGCReq, PauseContinueBroadcastReq, @@ -516,11 +517,11 @@ def _handle_output_by_indices(output: Any, indices: List[int]) -> Any: def _split_batch_by_key( - output: Any, ipcs: List[Optional[str]], key_fn: Callable[[Optional[str]], Any] + output: Any, keys: List[Any] ) -> Optional[List[Tuple[Any, Any]]]: indices_by_key: Dict[Any, List[int]] = {} - for index, ipc_key in enumerate(ipcs): - indices_by_key.setdefault(key_fn(ipc_key), []).append(index) + for index, key in enumerate(keys): + indices_by_key.setdefault(key, []).append(index) # Build every sub-batch before any send, so a fallback never duplicates one. try: @@ -531,15 +532,18 @@ def _split_batch_by_key( except _CannotSplitBatch: return None + ipcs = output.http_worker_ipcs for _, sub_output, indices in grouped: - sub_output.http_worker_ipcs = [ipcs[index] for index in indices] + sub_output.http_worker_ipcs = ( + None if ipcs is None else [ipcs[index] for index in indices] + ) return [(key, sub_output) for key, sub_output, _ in grouped] def _send_output_to_tokenizer_workers( output: Any, ipcs: List[Optional[str]], socket_mapping: SocketMapping ) -> None: - grouped = _split_batch_by_key(output, ipcs, lambda ipc_name: ipc_name) + grouped = _split_batch_by_key(output, ipcs) if grouped is None: for index, ipc_name in enumerate(ipcs): single_output = _handle_output_by_index(output, index) @@ -717,6 +721,8 @@ class MultiDetokenizerRouter: Each request is pinned to a worker by hashing its ``http_worker_ipc`` with ``zlib.crc32`` (deterministic across runs), so all outputs of the same rid always land on the same detokenizer and ``decode_status`` stays consistent. + With a single tokenizer worker no ``http_worker_ipc`` is stamped, and the + ``rid`` is hashed instead. """ def __init__(self, ipc_name_list: List[str], port_args: PortArgs): @@ -734,22 +740,24 @@ class MultiDetokenizerRouter: def _send(self, ipc_name: str, obj: Any) -> None: self.socket_mapping.send_output(ipc_name, obj, is_tokenizer=False) + def _pick_for(self, http_worker_ipc: Optional[str], rid: str) -> str: + return self._pick(rid if http_worker_ipc is None else http_worker_ipc) + def _send_batch(self, recv_obj: BaseBatchReq) -> None: ipcs = recv_obj.http_worker_ipcs - assert ( - ipcs is not None - and len(ipcs) == len(recv_obj.rids) - and all(x is not None for x in ipcs) - ), f"Batch req {recv_obj.rids=} has invalid http_worker_ipcs" + assert ipcs is not None and len(ipcs) == len(recv_obj.rids), ( + f"Batch req {recv_obj.rids=} has invalid http_worker_ipcs" + ) + targets = [self._pick_for(ipc, rid) for ipc, rid in zip(ipcs, recv_obj.rids)] - grouped = _split_batch_by_key(recv_obj, ipcs, self._pick) + grouped = _split_batch_by_key(recv_obj, targets) if grouped is None: - for index, ipc_key in enumerate(recv_obj.http_worker_ipcs): + for index, target in enumerate(targets): one = _handle_output_by_index(recv_obj, index) if one is recv_obj: raise TypeError(f"Cannot split {type(recv_obj)}") - one.http_worker_ipcs = [ipc_key] - self._send(self._pick(ipc_key), one) + one.http_worker_ipcs = [ipcs[index]] + self._send(target, one) return for target, output in grouped: @@ -759,18 +767,20 @@ class MultiDetokenizerRouter: while True: recv_obj = sock_recv(self.recv_from_scheduler) - # FreezeGCReq must freeze every detokenizer process. - if isinstance(recv_obj, FreezeGCReq): + # Control messages must reach every detokenizer process. + if isinstance(recv_obj, (FreezeGCReq, ConfigureLoggingReq)): for ipc in self.ipc_name_list: self._send(ipc, recv_obj) continue - # Single request: route by its own http_worker_ipc. + # Single request: route like the batch items of the same rid. if isinstance(recv_obj, BaseReq): - assert recv_obj.http_worker_ipc is not None, ( - f"Single req {recv_obj.rid=} missing http_worker_ipc" + assert ( + recv_obj.http_worker_ipc is not None or recv_obj.rid is not None + ), f"Single req {type(recv_obj).__name__} has no routing key" + self._send( + self._pick_for(recv_obj.http_worker_ipc, recv_obj.rid), recv_obj ) - self._send(self._pick(recv_obj.http_worker_ipc), recv_obj) continue # Batch request. diff --git a/test/registered/unit/managers/test_multi_tokenizer_mixin.py b/test/registered/unit/managers/test_multi_tokenizer_mixin.py index 89c4fa42f3..b881cca211 100644 --- a/test/registered/unit/managers/test_multi_tokenizer_mixin.py +++ b/test/registered/unit/managers/test_multi_tokenizer_mixin.py @@ -16,10 +16,12 @@ maybe_stub_sgl_kernel() from sglang.srt.beam_search.types import BeamSearchSequence from sglang.srt.managers import multi_tokenizer_mixin from sglang.srt.managers.io_struct import ( + AbortReq, BatchEmbeddingOutput, BatchStrOutput, BatchTokenIDOutput, BeamSearchOutput, + ConfigureLoggingReq, unwrap_from_pickle, wrap_as_pickle, ) @@ -409,6 +411,55 @@ class TestMultiTokenizerMixin(CustomTestCase): [("detok-a", ["embedding-rid-0"]), ("detok-a", ["embedding-rid-1"])], ) + def test_router_keys_on_rid_without_tokenizer_worker_ipcs(self): + output = _make_batch_str_output() + output.http_worker_ipcs = [None, None] + router = _make_router() + router._pick = lambda key: f"detok-for-{key}" + sent = [] + router._send = lambda target, obj: sent.append( + (target, obj.rids, obj.http_worker_ipcs) + ) + + router._send_batch(output) + + self.assertEqual( + sent, + [ + ("detok-for-rid-0", ["rid-0"], [None]), + ("detok-for-rid-1", ["rid-1"], [None]), + ], + ) + + def test_router_broadcasts_logging_config_and_routes_single_req_by_rid(self): + router = _make_router() + router.recv_from_scheduler = object() + router._pick = lambda key: f"detok-for-{key}" + sent = [] + router._send = lambda target, obj: sent.append((target, type(obj).__name__)) + + with ( + patch( + "sglang.srt.managers.multi_tokenizer_mixin.sock_recv", + side_effect=[ + ConfigureLoggingReq(log_level="debug"), + AbortReq(rid="rid-0"), + EOFError("end of input"), + ], + ), + self.assertRaisesRegex(EOFError, "end of input"), + ): + router.event_loop() + + self.assertEqual( + sent, + [ + ("detok-a", "ConfigureLoggingReq"), + ("detok-b", "ConfigureLoggingReq"), + ("detok-for-rid-0", "AbortReq"), + ], + ) + def test_return_path_groups_noncontiguous_owners(self): output = _handle_output_by_indices(_make_batch_str_output(), [0, 1, 0]) output.rids = ["rid-0", "rid-1", "rid-2"]