[Fix] pd: Allow runtime role switching with DP and DP attention (#44215)
Co-authored-by: Shangming Cai <csmthu@gmail.com>
This commit is contained in:
co-authored by
Shangming Cai
parent
7163e97462
commit
af49ada16d
@@ -165,8 +165,6 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None:
|
||||
if cfg.enable_pd_role_switch:
|
||||
view = resolved_view(server_args)
|
||||
unsupported = []
|
||||
if attn_dp_enabled_of(view):
|
||||
unsupported.append(f"attention DP (--attn-dp-size {view.attn_dp_size})")
|
||||
if view.ep_size > 1:
|
||||
unsupported.append(f"expert parallelism (--ep-size {view.ep_size})")
|
||||
if view.moe_a2a_backend != "none":
|
||||
@@ -175,8 +173,11 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None:
|
||||
)
|
||||
if view.pp_size > 1:
|
||||
unsupported.append(f"pipeline parallelism (--pp-size {view.pp_size})")
|
||||
if view.dp_size > 1:
|
||||
unsupported.append(f"data parallelism (--dp-size {view.dp_size})")
|
||||
if (
|
||||
attn_dp_enabled_of(view)
|
||||
and view.enable_dp_attention_local_control_broadcast
|
||||
):
|
||||
unsupported.append("DP attention local control broadcast")
|
||||
if view.dcp_size > 1:
|
||||
unsupported.append(
|
||||
f"decode context parallelism (--dcp-size {view.dcp_size})"
|
||||
|
||||
@@ -110,6 +110,16 @@ def handle_pd_role_switch(
|
||||
safe_to_restore=True,
|
||||
)
|
||||
|
||||
if recv_req.prepare_only:
|
||||
# The tokenizer checks every DP worker before allowing any teardown.
|
||||
return PdRoleSwitchReqOutput(
|
||||
success=True,
|
||||
message="ready",
|
||||
old_role=old_role,
|
||||
new_role=new_role,
|
||||
safe_to_restore=True,
|
||||
)
|
||||
|
||||
scheduler._pd_role_switch_in_progress = True
|
||||
try:
|
||||
# Teardown + role flip + rebuild are one logical atomic step. If any of
|
||||
|
||||
@@ -2270,6 +2270,8 @@ class PdRoleSwitchReqInput(BaseReq, kw_only=True):
|
||||
decode_cuda_graph_bs: Optional[List[int]] = None
|
||||
# Measured graph footprint from a matching decode peer.
|
||||
decode_cuda_graph_memory_gb: Optional[float] = None
|
||||
# Internal preflight across DP workers; does not change roles.
|
||||
prepare_only: bool = False
|
||||
|
||||
|
||||
class PdRoleSwitchReqOutput(BaseReq, kw_only=True):
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import hashlib
|
||||
import logging
|
||||
import time
|
||||
@@ -927,10 +928,19 @@ class TokenizerControlMixin:
|
||||
new_role=obj.new_role,
|
||||
safe_to_restore=True,
|
||||
)
|
||||
results = await self.pd_role_switch_communicator(obj)
|
||||
all_success = all(r.success for r in results)
|
||||
# The caller must drain ingress and serialize role-switch requests.
|
||||
# Check every DP worker before allowing any of them to tear down.
|
||||
if get_parallel().num_dp_ranks > 1:
|
||||
prepare_req = copy.copy(obj)
|
||||
prepare_req.prepare_only = True
|
||||
results = await self.pd_role_switch_communicator(prepare_req)
|
||||
if results and all(r.success for r in results) and not obj.prepare_only:
|
||||
results = await self.pd_role_switch_communicator(obj)
|
||||
else:
|
||||
results = await self.pd_role_switch_communicator(obj)
|
||||
all_success = bool(results) and all(r.success for r in results)
|
||||
safe_to_restore = bool(results) and all(r.safe_to_restore for r in results)
|
||||
if all_success:
|
||||
if all_success and not obj.prepare_only:
|
||||
# Keep the tokenizer-manager's view of the role in sync so future
|
||||
# control ops and bootstrap routing behave consistently.
|
||||
self.record_config_updates(
|
||||
@@ -938,6 +948,8 @@ class TokenizerControlMixin:
|
||||
)
|
||||
self.disaggregation_mode = DisaggregationMode(obj.new_role)
|
||||
msg = "ok"
|
||||
elif all_success:
|
||||
msg = "ready"
|
||||
else:
|
||||
# Surface only the failing workers' messages.
|
||||
msg = "; ".join(r.message for r in results if not r.success)
|
||||
|
||||
Reference in New Issue
Block a user