[Fix] pd: Allow runtime role switching with DP and DP attention (#44215)

Co-authored-by: Shangming Cai <csmthu@gmail.com>
This commit is contained in:
Huang Long
2026-10-21 22:58:29 +08:00
committed by GitHub
co-authored by Shangming Cai
parent 7163e97462
commit af49ada16d
5 changed files with 202 additions and 17 deletions
@@ -165,8 +165,6 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None:
if cfg.enable_pd_role_switch:
view = resolved_view(server_args)
unsupported = []
if attn_dp_enabled_of(view):
unsupported.append(f"attention DP (--attn-dp-size {view.attn_dp_size})")
if view.ep_size > 1:
unsupported.append(f"expert parallelism (--ep-size {view.ep_size})")
if view.moe_a2a_backend != "none":
@@ -175,8 +173,11 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None:
)
if view.pp_size > 1:
unsupported.append(f"pipeline parallelism (--pp-size {view.pp_size})")
if view.dp_size > 1:
unsupported.append(f"data parallelism (--dp-size {view.dp_size})")
if (
attn_dp_enabled_of(view)
and view.enable_dp_attention_local_control_broadcast
):
unsupported.append("DP attention local control broadcast")
if view.dcp_size > 1:
unsupported.append(
f"decode context parallelism (--dcp-size {view.dcp_size})"
@@ -110,6 +110,16 @@ def handle_pd_role_switch(
safe_to_restore=True,
)
if recv_req.prepare_only:
# The tokenizer checks every DP worker before allowing any teardown.
return PdRoleSwitchReqOutput(
success=True,
message="ready",
old_role=old_role,
new_role=new_role,
safe_to_restore=True,
)
scheduler._pd_role_switch_in_progress = True
try:
# Teardown + role flip + rebuild are one logical atomic step. If any of
+2
View File
@@ -2270,6 +2270,8 @@ class PdRoleSwitchReqInput(BaseReq, kw_only=True):
decode_cuda_graph_bs: Optional[List[int]] = None
# Measured graph footprint from a matching decode peer.
decode_cuda_graph_memory_gb: Optional[float] = None
# Internal preflight across DP workers; does not change roles.
prepare_only: bool = False
class PdRoleSwitchReqOutput(BaseReq, kw_only=True):
@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
import copy
import hashlib
import logging
import time
@@ -927,10 +928,19 @@ class TokenizerControlMixin:
new_role=obj.new_role,
safe_to_restore=True,
)
results = await self.pd_role_switch_communicator(obj)
all_success = all(r.success for r in results)
# The caller must drain ingress and serialize role-switch requests.
# Check every DP worker before allowing any of them to tear down.
if get_parallel().num_dp_ranks > 1:
prepare_req = copy.copy(obj)
prepare_req.prepare_only = True
results = await self.pd_role_switch_communicator(prepare_req)
if results and all(r.success for r in results) and not obj.prepare_only:
results = await self.pd_role_switch_communicator(obj)
else:
results = await self.pd_role_switch_communicator(obj)
all_success = bool(results) and all(r.success for r in results)
safe_to_restore = bool(results) and all(r.safe_to_restore for r in results)
if all_success:
if all_success and not obj.prepare_only:
# Keep the tokenizer-manager's view of the role in sync so future
# control ops and bootstrap routing behave consistently.
self.record_config_updates(
@@ -938,6 +948,8 @@ class TokenizerControlMixin:
)
self.disaggregation_mode = DisaggregationMode(obj.new_role)
msg = "ok"
elif all_success:
msg = "ready"
else:
# Surface only the failing workers' messages.
msg = "; ".join(r.message for r in results if not r.success)