[Perf] Reduce multi-detokenizer router IPC sends (#42238)
Co-authored-by: hnyls2002 <lsyincs@gmail.com> Co-authored-by: Liangsheng Yin <hnyls2002@gmail.com>
This commit is contained in:
co-authored by
hnyls2002
Liangsheng Yin
parent
66d840a8ef
commit
cb9bd1d4d4
@@ -31,6 +31,7 @@ import zlib
|
||||
from multiprocessing import shared_memory
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Type
|
||||
|
||||
import msgspec
|
||||
import psutil
|
||||
import setproctitle
|
||||
import zmq
|
||||
@@ -394,6 +395,106 @@ def _handle_output_by_index(output, i):
|
||||
return new_output
|
||||
|
||||
|
||||
class _CannotSplitBatch(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def _extract_field_by_indices(
|
||||
output: Any, field_name: str, indices: List[int], check_length: bool = True
|
||||
) -> Any:
|
||||
field = object.__getattribute__(output, field_name)
|
||||
if field is None:
|
||||
return None
|
||||
|
||||
should_wrap_result = field_name in ("customized_info", "time_stats")
|
||||
if should_wrap_result:
|
||||
field = unwrap_from_pickle(field)
|
||||
if field is None:
|
||||
return None
|
||||
|
||||
max_index = max(indices)
|
||||
if isinstance(field, dict):
|
||||
new_field = {}
|
||||
for key, values in field.items():
|
||||
if len(values) <= max_index:
|
||||
raise _CannotSplitBatch(
|
||||
f"{field_name}[{key!r}] cannot provide index {max_index}"
|
||||
)
|
||||
selected = [values[index] for index in indices]
|
||||
new_field[key] = selected
|
||||
if should_wrap_result:
|
||||
return wrap_as_pickle(new_field) if new_field else None
|
||||
return new_field
|
||||
|
||||
# Same as _extract_field_by_index: a field too short for every index (e.g.
|
||||
# spec_* lists stay empty when speculative decoding is off) becomes None.
|
||||
if check_length and len(field) <= min(indices):
|
||||
return None
|
||||
if len(field) <= max_index:
|
||||
raise _CannotSplitBatch(f"{field_name} cannot provide index {max_index}")
|
||||
|
||||
selected = [field[index] for index in indices]
|
||||
return wrap_as_pickle(selected) if should_wrap_result else selected
|
||||
|
||||
|
||||
# Per-request fields that may be shorter than rids; the rest must match rids or
|
||||
# be empty.
|
||||
_UNCHECKED_LENGTH_FIELDS = frozenset(
|
||||
{
|
||||
"input_token_logprobs_val",
|
||||
"input_token_logprobs_idx",
|
||||
"output_token_logprobs_val",
|
||||
"output_token_logprobs_idx",
|
||||
"input_top_logprobs_val",
|
||||
"input_top_logprobs_idx",
|
||||
"input_top_logprobs_val_flat",
|
||||
"input_top_logprobs_idx_flat",
|
||||
"input_top_logprobs_flat_null_prefix",
|
||||
"output_top_logprobs_val",
|
||||
"output_top_logprobs_idx",
|
||||
"input_token_ids_logprobs_val",
|
||||
"input_token_ids_logprobs_idx",
|
||||
"output_token_ids_logprobs_val",
|
||||
"output_token_ids_logprobs_idx",
|
||||
"output_token_entropy_val",
|
||||
"output_token_sampling_mask",
|
||||
"output_hidden_states",
|
||||
"routed_experts",
|
||||
"indexer_topk",
|
||||
"token_steps",
|
||||
"customized_info",
|
||||
"dp_ranks",
|
||||
}
|
||||
)
|
||||
_NOT_SPLIT_FIELDS = frozenset(
|
||||
{"rids", "http_worker_ipcs", "placeholder_tokens_idx", "placeholder_tokens_val"}
|
||||
)
|
||||
|
||||
|
||||
def _handle_output_by_indices(output: Any, indices: List[int]) -> Any:
|
||||
if len(indices) == 1:
|
||||
return _handle_output_by_index(output, indices[0])
|
||||
if not isinstance(output, (BatchTokenIDOutput, BatchStrOutput)):
|
||||
raise _CannotSplitBatch(f"Cannot split {type(output)}")
|
||||
|
||||
split_fields = {
|
||||
field.name: _extract_field_by_indices(
|
||||
output,
|
||||
field.name,
|
||||
indices,
|
||||
check_length=field.name not in _UNCHECKED_LENGTH_FIELDS,
|
||||
)
|
||||
for field in msgspec.structs.fields(output)
|
||||
if field.name not in _NOT_SPLIT_FIELDS
|
||||
}
|
||||
return type(output)(
|
||||
rids=[output.rids[index] for index in indices],
|
||||
placeholder_tokens_idx=None,
|
||||
placeholder_tokens_val=None,
|
||||
**split_fields,
|
||||
)
|
||||
|
||||
|
||||
class MultiHttpWorkerDetokenizerMixin:
|
||||
"""Mixin class for DetokenizerManager"""
|
||||
|
||||
@@ -580,6 +681,49 @@ class MultiDetokenizerRouter:
|
||||
def _send(self, ipc_name: str, obj: Any) -> None:
|
||||
self.socket_mapping.send_output(ipc_name, obj, is_tokenizer=False)
|
||||
|
||||
def _split_batch_by_target(self, recv_obj: BaseBatchReq):
|
||||
ipcs = recv_obj.http_worker_ipcs
|
||||
assert (
|
||||
ipcs is not None
|
||||
and len(ipcs) == len(recv_obj.rids)
|
||||
and all(x is not None for x in ipcs)
|
||||
), f"Batch req {recv_obj.rids=} has invalid http_worker_ipcs"
|
||||
|
||||
indices_by_target: Dict[str, List[int]] = {}
|
||||
for index, ipc_key in enumerate(ipcs):
|
||||
target = self._pick(ipc_key)
|
||||
indices_by_target.setdefault(target, []).append(index)
|
||||
|
||||
try:
|
||||
grouped = [
|
||||
(
|
||||
target,
|
||||
_handle_output_by_indices(recv_obj, indices),
|
||||
indices,
|
||||
)
|
||||
for target, indices in indices_by_target.items()
|
||||
]
|
||||
except _CannotSplitBatch:
|
||||
return None
|
||||
|
||||
for _, output, indices in grouped:
|
||||
output.http_worker_ipcs = [ipcs[index] for index in indices]
|
||||
return grouped
|
||||
|
||||
def _send_batch(self, recv_obj: BaseBatchReq) -> None:
|
||||
grouped = self._split_batch_by_target(recv_obj)
|
||||
if grouped is None:
|
||||
for index, ipc_key in enumerate(recv_obj.http_worker_ipcs):
|
||||
one = _handle_output_by_index(recv_obj, index)
|
||||
if one is recv_obj:
|
||||
raise TypeError(f"Cannot split {type(recv_obj)}")
|
||||
one.http_worker_ipcs = [ipc_key]
|
||||
self._send(self._pick(ipc_key), one)
|
||||
return
|
||||
|
||||
for target, output, _ in grouped:
|
||||
self._send(target, output)
|
||||
|
||||
def event_loop(self):
|
||||
while True:
|
||||
recv_obj = sock_recv(self.recv_from_scheduler)
|
||||
@@ -606,20 +750,7 @@ class MultiDetokenizerRouter:
|
||||
self._send(ipc, recv_obj)
|
||||
continue
|
||||
|
||||
ipcs = recv_obj.http_worker_ipcs
|
||||
assert (
|
||||
ipcs is not None
|
||||
and len(ipcs) == len(recv_obj.rids)
|
||||
and all(x is not None for x in ipcs)
|
||||
), f"Batch req {recv_obj.rids=} has invalid http_worker_ipcs"
|
||||
|
||||
# Split per-item and route each by its own ipc.
|
||||
for i, ipc_key in enumerate(ipcs):
|
||||
one = _handle_output_by_index(recv_obj, i)
|
||||
if one is recv_obj:
|
||||
raise TypeError(f"Cannot split {type(recv_obj)}")
|
||||
one.http_worker_ipcs = [ipc_key]
|
||||
self._send(self._pick(ipc_key), one)
|
||||
self._send_batch(recv_obj)
|
||||
continue
|
||||
|
||||
raise ValueError(
|
||||
|
||||
@@ -5,14 +5,22 @@ import numpy as np
|
||||
from sglang.srt.sampling.sampling_mask import SamplingMaskChunk
|
||||
from sglang.srt.utils.weight_versions import WeightVersionSpan
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import maybe_stub_sgl_kernel
|
||||
from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel
|
||||
|
||||
maybe_stub_sgl_kernel()
|
||||
|
||||
from sglang.srt.managers.io_struct import BatchStrOutput
|
||||
from sglang.srt.beam_search.types import BeamSearchSequence
|
||||
from sglang.srt.managers.io_struct import (
|
||||
BatchEmbeddingOutput,
|
||||
BatchStrOutput,
|
||||
BatchTokenIDOutput,
|
||||
BeamSearchOutput,
|
||||
)
|
||||
from sglang.srt.managers.multi_tokenizer_mixin import (
|
||||
MultiDetokenizerRouter,
|
||||
TokenizerWorker,
|
||||
_handle_output_by_index,
|
||||
_handle_output_by_indices,
|
||||
get_tokenizer_worker_class,
|
||||
)
|
||||
|
||||
@@ -97,10 +105,89 @@ def _make_batch_str_output() -> BatchStrOutput:
|
||||
],
|
||||
[WeightVersionSpan(version="v2", start=0, end=2)],
|
||||
],
|
||||
input_top_logprobs_val_flat=[np.array([[1.0]], np.float32), None],
|
||||
input_top_logprobs_idx_flat=[np.array([[1]], np.int32), None],
|
||||
input_top_logprobs_flat_null_prefix=[0, None],
|
||||
beam_search_output=[
|
||||
BeamSearchOutput(
|
||||
sequences=[
|
||||
BeamSearchSequence(tokens=[11], cum_logprob=-1.0),
|
||||
]
|
||||
),
|
||||
None,
|
||||
],
|
||||
token_steps=[[1], [2]],
|
||||
)
|
||||
|
||||
|
||||
class TestMultiTokenizerMixin(unittest.TestCase):
|
||||
def _make_batch_token_id_output() -> BatchTokenIDOutput:
|
||||
return BatchTokenIDOutput(
|
||||
rids=["token-rid-0", "token-rid-1"],
|
||||
# OutputStreamer leaves the spec_* lists empty without speculative decoding.
|
||||
spec_verify_ct=[],
|
||||
spec_num_correct_drafts=[],
|
||||
spec_correct_drafts_histogram=[],
|
||||
spec_num_block_accept_tokens=[],
|
||||
spec_num_cap_tokens=[],
|
||||
spec_cap_lens_histogram=[],
|
||||
finished_reasons=[None, None],
|
||||
decoded_texts=["first", "second"],
|
||||
decode_ids=[[1], [2]],
|
||||
read_offsets=[0, 1],
|
||||
output_ids=[[1], [2]],
|
||||
skip_special_tokens=[True, True],
|
||||
spaces_between_special_tokens=[True, True],
|
||||
no_stop_trim=[False, False],
|
||||
prompt_tokens=[10, 20],
|
||||
reasoning_tokens=[0, 0],
|
||||
completion_tokens=[1, 2],
|
||||
cached_tokens=[3, 4],
|
||||
input_token_logprobs_val=[[], []],
|
||||
input_token_logprobs_idx=[[], []],
|
||||
output_token_logprobs_val=[[], []],
|
||||
output_token_logprobs_idx=[[], []],
|
||||
input_top_logprobs_val=[[], []],
|
||||
input_top_logprobs_idx=[[], []],
|
||||
output_top_logprobs_val=[[], []],
|
||||
output_top_logprobs_idx=[[], []],
|
||||
input_token_ids_logprobs_val=[[], []],
|
||||
input_token_ids_logprobs_idx=[[], []],
|
||||
output_token_ids_logprobs_val=[[], []],
|
||||
output_token_ids_logprobs_idx=[[], []],
|
||||
output_token_entropy_val=[None, None],
|
||||
output_token_sampling_mask=[None, None],
|
||||
output_hidden_states=[None, None],
|
||||
routed_experts=[None, None],
|
||||
indexer_topk=[None, None],
|
||||
placeholder_tokens_idx=[None, None],
|
||||
placeholder_tokens_val=[None, None],
|
||||
)
|
||||
|
||||
|
||||
def _make_batch_embedding_output() -> BatchEmbeddingOutput:
|
||||
return BatchEmbeddingOutput(
|
||||
rids=["embedding-rid-0", "embedding-rid-1"],
|
||||
finished_reasons=[None, None],
|
||||
embeddings=[[0.1, 0.2], [0.3, 0.4]],
|
||||
prompt_tokens=[10, 20],
|
||||
cached_tokens=[3, 4],
|
||||
placeholder_tokens_idx=[None, None],
|
||||
placeholder_tokens_val=[None, None],
|
||||
)
|
||||
|
||||
|
||||
def _make_router() -> MultiDetokenizerRouter:
|
||||
router = MultiDetokenizerRouter.__new__(MultiDetokenizerRouter)
|
||||
router.ipc_name_list = ["detok-a", "detok-b"]
|
||||
router.num_workers = 2
|
||||
router._pick = lambda key: {
|
||||
"worker-a": "detok-a",
|
||||
"worker-b": "detok-b",
|
||||
}[key]
|
||||
return router
|
||||
|
||||
|
||||
class TestMultiTokenizerMixin(CustomTestCase):
|
||||
def test_batch_str_output_preserves_cached_tokens_details(self):
|
||||
output = _make_batch_str_output()
|
||||
|
||||
@@ -148,6 +235,99 @@ class TestMultiTokenizerMixin(unittest.TestCase):
|
||||
|
||||
self.assertIsNone(_handle_output_by_index(output, 0).weight_versions)
|
||||
|
||||
def test_grouped_split_preserves_latest_fields_and_order(self):
|
||||
output = _make_batch_str_output()
|
||||
|
||||
grouped = _handle_output_by_indices(output, [1, 0])
|
||||
|
||||
self.assertEqual(grouped.rids, ["rid-1", "rid-0"])
|
||||
self.assertEqual(grouped.output_strs, ["second", "first"])
|
||||
self.assertEqual(
|
||||
grouped.cached_tokens_details,
|
||||
[{"device": 1, "host": 3}, {"device": 3, "host": 0}],
|
||||
)
|
||||
self.assertEqual(
|
||||
grouped.weight_versions,
|
||||
[
|
||||
[WeightVersionSpan(version="v2", start=0, end=2)],
|
||||
[
|
||||
WeightVersionSpan(version="v1", start=0, end=3),
|
||||
WeightVersionSpan(version="v2", start=3, end=5),
|
||||
],
|
||||
],
|
||||
)
|
||||
self.assertEqual(grouped.input_top_logprobs_flat_null_prefix, [None, 0])
|
||||
self.assertIsNone(grouped.beam_search_output[0])
|
||||
self.assertEqual(
|
||||
grouped.beam_search_output[1].sequences[0].tokens,
|
||||
[11],
|
||||
)
|
||||
np.testing.assert_array_equal(
|
||||
grouped.input_top_logprobs_val_flat[1], np.array([[1.0]], np.float32)
|
||||
)
|
||||
|
||||
def test_router_groups_by_target_and_preserves_owner_ipcs(self):
|
||||
output = _make_batch_str_output()
|
||||
output.http_worker_ipcs = ["worker-a", "worker-a"]
|
||||
router = _make_router()
|
||||
sent = []
|
||||
router._send = lambda target, obj: sent.append((target, obj))
|
||||
|
||||
router._send_batch(output)
|
||||
|
||||
self.assertEqual(len(sent), 1)
|
||||
self.assertEqual(sent[0][0], "detok-a")
|
||||
self.assertEqual(sent[0][1].rids, ["rid-0", "rid-1"])
|
||||
self.assertEqual(sent[0][1].http_worker_ipcs, ["worker-a", "worker-a"])
|
||||
self.assertEqual(output.rids, ["rid-0", "rid-1"])
|
||||
self.assertEqual(output.http_worker_ipcs, ["worker-a", "worker-a"])
|
||||
|
||||
def test_router_keeps_first_target_order_and_does_not_cross_messages(self):
|
||||
output = _make_batch_str_output()
|
||||
output.http_worker_ipcs = ["worker-b", "worker-a"]
|
||||
router = _make_router()
|
||||
sent = []
|
||||
router._send = lambda target, obj: sent.append((target, obj))
|
||||
|
||||
router._send_batch(output)
|
||||
|
||||
self.assertEqual(
|
||||
[(target, obj.rids, obj.http_worker_ipcs) for target, obj in sent],
|
||||
[
|
||||
("detok-b", ["rid-0"], ["worker-b"]),
|
||||
("detok-a", ["rid-1"], ["worker-a"]),
|
||||
],
|
||||
)
|
||||
|
||||
def test_router_groups_token_output_without_speculative_decoding(self):
|
||||
output = _make_batch_token_id_output()
|
||||
output.http_worker_ipcs = ["worker-a", "worker-a"]
|
||||
router = _make_router()
|
||||
sent = []
|
||||
router._send = lambda target, obj: sent.append((target, obj))
|
||||
|
||||
router._send_batch(output)
|
||||
|
||||
self.assertEqual(len(sent), 1)
|
||||
self.assertEqual(sent[0][0], "detok-a")
|
||||
self.assertEqual(sent[0][1].rids, ["token-rid-0", "token-rid-1"])
|
||||
self.assertEqual(sent[0][1].decoded_texts, ["first", "second"])
|
||||
self.assertIsNone(sent[0][1].spec_verify_ct)
|
||||
|
||||
def test_router_sends_embedding_output_per_request(self):
|
||||
output = _make_batch_embedding_output()
|
||||
output.http_worker_ipcs = ["worker-a", "worker-a"]
|
||||
router = _make_router()
|
||||
sent = []
|
||||
router._send = lambda target, obj: sent.append((target, obj))
|
||||
|
||||
router._send_batch(output)
|
||||
|
||||
self.assertEqual(
|
||||
[(target, obj.rids) for target, obj in sent],
|
||||
[("detok-a", ["embedding-rid-0"]), ("detok-a", ["embedding-rid-1"])],
|
||||
)
|
||||
|
||||
def test_get_tokenizer_worker_class_uses_default(self):
|
||||
self.assertIs(get_tokenizer_worker_class(DefaultServerArgs()), TokenizerWorker)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user