[BugFix] Preserve tokenizer worker fanout when skip_tokenizer_init is enabled (#30682)
Co-authored-by: hjzhang <zhanghjzzz@qq.com> Co-authored-by: hnyls2002 <lsyincs@gmail.com>
This commit is contained in:
co-authored by
hjzhang
hnyls2002
parent
4d60c4540c
commit
db7e6807de
@@ -53,7 +53,8 @@ class SchedulerIpcChannels:
|
||||
context, zmq.PUSH, port_args.tokenizer_ipc_name, False
|
||||
)
|
||||
if skip_tokenizer_init:
|
||||
# Directly send to the TokenizerManager
|
||||
# No decode work: send outputs straight to the tokenizer side
|
||||
# (MultiTokenizerRouter fans out when tokenizer_worker_num > 1).
|
||||
send_to_detokenizer_raw = get_zmq_socket(
|
||||
context, zmq.PUSH, port_args.tokenizer_ipc_name, False
|
||||
)
|
||||
|
||||
@@ -601,6 +601,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
(FreezeGCReq, lambda x: None),
|
||||
# For handling case when scheduler skips detokenizer and forwards back to the tokenizer manager, we ignore it.
|
||||
(HealthCheckOutput, lambda x: None),
|
||||
# Same skip-detokenizer forwarding case as above.
|
||||
(ConfigureLoggingReq, lambda x: None),
|
||||
(ActiveRanksOutput, self.update_active_ranks),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -6089,16 +6089,12 @@ class ServerArgs:
|
||||
)
|
||||
|
||||
if self.skip_tokenizer_init:
|
||||
if self.tokenizer_worker_num != 1:
|
||||
logger.warning(
|
||||
"skip_tokenizer_init=True disables tokenizer workers; forcing tokenizer_worker_num=1 "
|
||||
f"(requested {self.tokenizer_worker_num})."
|
||||
)
|
||||
self.tokenizer_worker_num = 1
|
||||
# Tokenizer workers still serve HTTP / state / output work, so
|
||||
# their fanout is preserved; detokenizer workers only decode.
|
||||
if self.detokenizer_worker_num != 1:
|
||||
logger.warning(
|
||||
"skip_tokenizer_init=True disables detokenizer workers; forcing detokenizer_worker_num=1 "
|
||||
f"(requested {self.detokenizer_worker_num})."
|
||||
"skip_tokenizer_init=True leaves no decode work for detokenizer workers; "
|
||||
f"forcing detokenizer_worker_num=1 (requested {self.detokenizer_worker_num})."
|
||||
)
|
||||
self.detokenizer_worker_num = 1
|
||||
|
||||
|
||||
Reference in New Issue
Block a user