[PD] Resolve missing bootstrap_room problem about fake-decode in load-balance method (#18399)
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: hnyls2002 <lsyincs@gmail.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot]
hnyls2002
parent
4c1d07fbdd
commit
5273f11fd8
@@ -559,16 +559,6 @@ class DataParallelController:
|
|||||||
if self.maybe_external_dp_rank_routing(req):
|
if self.maybe_external_dp_rank_routing(req):
|
||||||
return
|
return
|
||||||
|
|
||||||
# Set default bootstrap_room if in FAKE auto mode and room is None
|
|
||||||
if (
|
|
||||||
req.bootstrap_room is None
|
|
||||||
and self.server_args.disaggregation_transfer_backend == "fake"
|
|
||||||
):
|
|
||||||
req.bootstrap_room = self.round_robin_counter
|
|
||||||
self.round_robin_counter = (self.round_robin_counter + 1) % len(
|
|
||||||
self.workers
|
|
||||||
)
|
|
||||||
|
|
||||||
assert req.bootstrap_room is not None, (
|
assert req.bootstrap_room is not None, (
|
||||||
"req.bootstrap_room should not be None. Do not send requests directly to "
|
"req.bootstrap_room should not be None. Do not send requests directly to "
|
||||||
"prefill or decode instances; send to the router instead."
|
"prefill or decode instances; send to the router instead."
|
||||||
|
|||||||
@@ -440,6 +440,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
|||||||
self.server_args.disaggregation_mode
|
self.server_args.disaggregation_mode
|
||||||
)
|
)
|
||||||
self.bootstrap_server = start_disagg_service(self.server_args)
|
self.bootstrap_server = start_disagg_service(self.server_args)
|
||||||
|
# Single-source counter for auto-assigning fake bootstrap_room.
|
||||||
|
self.fake_bootstrap_room_counter = 0
|
||||||
|
|
||||||
# Encoder Disaggregation
|
# Encoder Disaggregation
|
||||||
if self.server_args.language_only:
|
if self.server_args.language_only:
|
||||||
@@ -974,6 +976,14 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
|||||||
SessionParams(**obj.session_params) if obj.session_params else None
|
SessionParams(**obj.session_params) if obj.session_params else None
|
||||||
)
|
)
|
||||||
|
|
||||||
|
bootstrap_room = obj.bootstrap_room
|
||||||
|
if (
|
||||||
|
bootstrap_room is None
|
||||||
|
and self.server_args.disaggregation_transfer_backend == "fake"
|
||||||
|
):
|
||||||
|
bootstrap_room = self.fake_bootstrap_room_counter
|
||||||
|
self.fake_bootstrap_room_counter += 1
|
||||||
|
|
||||||
tokenized_obj = TokenizedGenerateReqInput(
|
tokenized_obj = TokenizedGenerateReqInput(
|
||||||
input_text,
|
input_text,
|
||||||
input_ids,
|
input_ids,
|
||||||
@@ -988,7 +998,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
|||||||
http_worker_ipc=obj.http_worker_ipc,
|
http_worker_ipc=obj.http_worker_ipc,
|
||||||
bootstrap_host=obj.bootstrap_host,
|
bootstrap_host=obj.bootstrap_host,
|
||||||
bootstrap_port=obj.bootstrap_port,
|
bootstrap_port=obj.bootstrap_port,
|
||||||
bootstrap_room=obj.bootstrap_room,
|
bootstrap_room=bootstrap_room,
|
||||||
lora_id=obj.lora_id,
|
lora_id=obj.lora_id,
|
||||||
input_embeds=input_embeds,
|
input_embeds=input_embeds,
|
||||||
positional_embed_overrides=obj.positional_embed_overrides,
|
positional_embed_overrides=obj.positional_embed_overrides,
|
||||||
|
|||||||
Reference in New Issue
Block a user