diff --git a/python/sglang/srt/arg_groups/pd_disaggregation_hook.py b/python/sglang/srt/arg_groups/pd_disaggregation_hook.py index 21784c9fc6..a9f297cd08 100644 --- a/python/sglang/srt/arg_groups/pd_disaggregation_hook.py +++ b/python/sglang/srt/arg_groups/pd_disaggregation_hook.py @@ -165,8 +165,6 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None: if cfg.enable_pd_role_switch: view = resolved_view(server_args) unsupported = [] - if attn_dp_enabled_of(view): - unsupported.append(f"attention DP (--attn-dp-size {view.attn_dp_size})") if view.ep_size > 1: unsupported.append(f"expert parallelism (--ep-size {view.ep_size})") if view.moe_a2a_backend != "none": @@ -175,8 +173,11 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None: ) if view.pp_size > 1: unsupported.append(f"pipeline parallelism (--pp-size {view.pp_size})") - if view.dp_size > 1: - unsupported.append(f"data parallelism (--dp-size {view.dp_size})") + if ( + attn_dp_enabled_of(view) + and view.enable_dp_attention_local_control_broadcast + ): + unsupported.append("DP attention local control broadcast") if view.dcp_size > 1: unsupported.append( f"decode context parallelism (--dcp-size {view.dcp_size})" diff --git a/python/sglang/srt/disaggregation/role_switch.py b/python/sglang/srt/disaggregation/role_switch.py index cc820a4e58..5c7ab1d9e1 100644 --- a/python/sglang/srt/disaggregation/role_switch.py +++ b/python/sglang/srt/disaggregation/role_switch.py @@ -110,6 +110,16 @@ def handle_pd_role_switch( safe_to_restore=True, ) + if recv_req.prepare_only: + # The tokenizer checks every DP worker before allowing any teardown. + return PdRoleSwitchReqOutput( + success=True, + message="ready", + old_role=old_role, + new_role=new_role, + safe_to_restore=True, + ) + scheduler._pd_role_switch_in_progress = True try: # Teardown + role flip + rebuild are one logical atomic step. If any of diff --git a/python/sglang/srt/managers/io_struct.py b/python/sglang/srt/managers/io_struct.py index 3148deb920..03241e90b9 100644 --- a/python/sglang/srt/managers/io_struct.py +++ b/python/sglang/srt/managers/io_struct.py @@ -2270,6 +2270,8 @@ class PdRoleSwitchReqInput(BaseReq, kw_only=True): decode_cuda_graph_bs: Optional[List[int]] = None # Measured graph footprint from a matching decode peer. decode_cuda_graph_memory_gb: Optional[float] = None + # Internal preflight across DP workers; does not change roles. + prepare_only: bool = False class PdRoleSwitchReqOutput(BaseReq, kw_only=True): diff --git a/python/sglang/srt/managers/tokenizer_control_mixin.py b/python/sglang/srt/managers/tokenizer_control_mixin.py index f62afbae8e..9d2dcd1227 100644 --- a/python/sglang/srt/managers/tokenizer_control_mixin.py +++ b/python/sglang/srt/managers/tokenizer_control_mixin.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import copy import hashlib import logging import time @@ -927,10 +928,19 @@ class TokenizerControlMixin: new_role=obj.new_role, safe_to_restore=True, ) - results = await self.pd_role_switch_communicator(obj) - all_success = all(r.success for r in results) + # The caller must drain ingress and serialize role-switch requests. + # Check every DP worker before allowing any of them to tear down. + if get_parallel().num_dp_ranks > 1: + prepare_req = copy.copy(obj) + prepare_req.prepare_only = True + results = await self.pd_role_switch_communicator(prepare_req) + if results and all(r.success for r in results) and not obj.prepare_only: + results = await self.pd_role_switch_communicator(obj) + else: + results = await self.pd_role_switch_communicator(obj) + all_success = bool(results) and all(r.success for r in results) safe_to_restore = bool(results) and all(r.safe_to_restore for r in results) - if all_success: + if all_success and not obj.prepare_only: # Keep the tokenizer-manager's view of the role in sync so future # control ops and bootstrap routing behave consistently. self.record_config_updates( @@ -938,6 +948,8 @@ class TokenizerControlMixin: ) self.disaggregation_mode = DisaggregationMode(obj.new_role) msg = "ok" + elif all_success: + msg = "ready" else: # Surface only the failing workers' messages. msg = "; ".join(r.message for r in results if not r.success) diff --git a/test/registered/unit/disaggregation/test_pd_role_switch.py b/test/registered/unit/disaggregation/test_pd_role_switch.py index b5b700e939..5bea2def91 100644 --- a/test/registered/unit/disaggregation/test_pd_role_switch.py +++ b/test/registered/unit/disaggregation/test_pd_role_switch.py @@ -1,7 +1,7 @@ import concurrent.futures import unittest from types import SimpleNamespace -from unittest.mock import MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, patch from sglang.srt import runtime_context as rc # noqa: E402 from sglang.srt.disaggregation import role_switch # noqa: E402 @@ -53,6 +53,55 @@ class TestHandlePdRoleSwitch(unittest.TestCase): s.tp_worker = MagicMock() return s + def test_preflight_checks_graph_headroom_without_releasing_resources(self): + s = self._scheduler(DisaggregationMode.PREFILL) + s.device = "cuda" + cases = ( + ("missing", None, [], 2.0, False, "is required"), + ("negative", -1.0, [], 2.0, False, "non-negative"), + ("insufficient", 1.0, [], 0.5, False, "insufficient"), + ( + "probe failure", + 1.0, + [], + RuntimeError("probe failed"), + False, + "failed to check", + ), + ("sufficient", 1.0, [], 1.0, True, "ready"), + ("already captured", None, [1], 0.0, True, "ready"), + ) + for name, budget, captured_bs, memory, success, message in cases: + with self.subTest(name=name): + s.tp_worker.get_decode_cuda_graph_bs.return_value = captured_bs + with patch.object( + role_switch, "get_available_gpu_memory" + ) as memory_probe: + if isinstance(memory, Exception): + memory_probe.side_effect = memory + else: + memory_probe.return_value = memory + out = Scheduler.handle_pd_role_switch( + s, + PdRoleSwitchReqInput( + new_role="decode", + prepare_only=True, + decode_cuda_graph_memory_gb=budget, + ), + ) + self.assertEqual(out.success, success) + self.assertTrue(out.safe_to_restore) + self.assertIn(message, out.message) + if not captured_bs and budget is not None and budget >= 0: + memory_probe.assert_called_once() + else: + memory_probe.assert_not_called() + s.tp_worker.ensure_decode_cuda_graphs.assert_not_called() + s.teardown_disaggregation.assert_not_called() + s.init_disaggregation.assert_not_called() + self.assertEqual(rc.get_disagg().disaggregation_mode, "prefill") + self.assertFalse(s._event_loop_should_restart) + def test_rejected_when_flag_disabled(self): s = self._scheduler(DisaggregationMode.PREFILL, enable=False) out = Scheduler.handle_pd_role_switch( @@ -296,7 +345,7 @@ class TestPdRoleSwitchReqSerialization(unittest.TestCase): class TestPdRoleSwitchStartupValidation(unittest.TestCase): """--enable-pd-role-switch only rebuilds the small role-specific disagg - structures on a flip; the per-role buffers of DP attention / EP / MoE + structures on a flip; the per-role buffers of EP / MoE all-to-all / pipeline parallelism are sized at startup and not rebuilt, so a flip with those on would silently deadlock. The PD arg hook must reject the combination up-front instead of failing at flip time.""" @@ -331,10 +380,18 @@ class TestPdRoleSwitchStartupValidation(unittest.TestCase): # No raise for the validated pure-TP configuration. self._run(self._sa()) - def test_reject_dp_attention(self): - with self.assertRaises(ValueError) as ctx: - self._run(self._sa(attn_dp_size=2)) - self.assertIn("attention DP", str(ctx.exception)) + def test_dp_attention_role_switch_accepted(self): + self._run(self._sa(tp_size=2, attn_dp_size=2)) + + def test_reject_dp_attention_local_control_broadcast(self): + with self.assertRaisesRegex(ValueError, "local control broadcast"): + self._run( + self._sa( + tp_size=2, + attn_dp_size=2, + enable_dp_attention_local_control_broadcast=True, + ) + ) def test_reject_expert_parallelism(self): with self.assertRaises(ValueError) as ctx: @@ -351,10 +408,8 @@ class TestPdRoleSwitchStartupValidation(unittest.TestCase): self._run(self._sa(pp_size=2)) self.assertIn("pipeline parallelism", str(ctx.exception)) - def test_reject_data_parallelism(self): - with self.assertRaises(ValueError) as ctx: - self._run(self._sa(dp_size=2)) - self.assertIn("data parallelism", str(ctx.exception)) + def test_data_parallel_role_switch_accepted(self): + self._run(self._sa(dp_size=2)) def test_reject_decode_context_parallelism(self): with self.assertRaises(ValueError) as ctx: @@ -597,6 +652,111 @@ class TestReleasePrefixCacheOnRoleSwitch(unittest.TestCase): s.tree_cache.reset.assert_called_once_with() +class TestDPRoleSwitch(unittest.IsolatedAsyncioTestCase): + async def test_all_workers_must_be_ready_before_switching(self): + for dp_size, attn_dp_size in ((2, 1), (1, 2)): + with self.subTest(dp_size=dp_size, attn_dp_size=attn_dp_size): + await self._check_preflight(dp_size, attn_dp_size) + + async def test_insufficient_headroom_prevents_switching_all_workers(self): + for dp_size, attn_dp_size in ((2, 1), (1, 2)): + with self.subTest(dp_size=dp_size, attn_dp_size=attn_dp_size): + await self._check_preflight( + dp_size, attn_dp_size, insufficient_memory=True + ) + + async def _check_preflight(self, dp_size, attn_dp_size, insufficient_memory=False): + from sglang.srt.managers import tokenizer_control_mixin as control + + fixture = TestHandlePdRoleSwitch() + fixture.setUp() + self.addCleanup(fixture.tearDown) + self.addCleanup(fixture.doCleanups) + schedulers = [fixture._scheduler(DisaggregationMode.PREFILL) for _ in range(2)] + available_memory = [2.0, 2.0] + for scheduler in schedulers: + scheduler.device = "cuda" + scheduler.tp_worker.get_decode_cuda_graph_bs.return_value = ( + [] if insufficient_memory else [1] + ) + scheduler.init_disaggregation.side_effect = lambda s=scheduler: setattr( + s, + "disaggregation_mode", + DisaggregationMode(rc.get_disagg().disaggregation_mode), + ) + + async def fanout(req): + results = [] + for index, scheduler in enumerate(schedulers): + rc.get_context().set_server_args(scheduler.server_args) + rc.get_context().override( + "test.role", disaggregation_mode=scheduler.disaggregation_mode.value + ) + with patch.object( + role_switch, + "get_available_gpu_memory", + return_value=available_memory[index], + ): + results.append(role_switch.handle_pd_role_switch(scheduler, req)) + if req.prepare_only: + for scheduler in schedulers: + scheduler.init_disaggregation.assert_not_called() + return results + + manager = SimpleNamespace( + server_args=SimpleNamespace(enable_pd_role_switch=True), + auto_create_handle_loop=MagicMock(), + pd_role_switch_communicator=AsyncMock(side_effect=fanout), + record_config_updates=MagicMock(), + disaggregation_mode=DisaggregationMode.PREFILL, + ) + req = PdRoleSwitchReqInput( + new_role="decode", + decode_cuda_graph_memory_gb=1.0 if insufficient_memory else None, + ) + with patch.object( + control, + "get_parallel", + return_value=SimpleNamespace( + dp_size=dp_size, + attn_dp_size=attn_dp_size, + num_dp_ranks=dp_size * attn_dp_size, + ), + ): + if insufficient_memory: + available_memory[1] = 0.5 + else: + schedulers[1].is_fully_idle.return_value = False + out = await control.TokenizerControlMixin.pd_role_switch(manager, req) + self.assertFalse(out.success) + self.assertTrue(out.safe_to_restore) + self.assertIn( + "insufficient" if insufficient_memory else "not idle", out.message + ) + manager.pd_role_switch_communicator.assert_awaited_once() + self.assertTrue( + manager.pd_role_switch_communicator.call_args.args[0].prepare_only + ) + manager.record_config_updates.assert_not_called() + for scheduler in schedulers: + scheduler.init_disaggregation.assert_not_called() + scheduler.teardown_disaggregation.assert_not_called() + self.assertEqual( + scheduler.disaggregation_mode, DisaggregationMode.PREFILL + ) + + available_memory[1] = 2.0 + schedulers[1].is_fully_idle.return_value = True + out = await control.TokenizerControlMixin.pd_role_switch(manager, req) + self.assertTrue(out.success) + self.assertEqual(manager.disaggregation_mode, DisaggregationMode.DECODE) + for scheduler in schedulers: + scheduler.init_disaggregation.assert_called_once() + self.assertEqual( + scheduler.disaggregation_mode, DisaggregationMode.DECODE + ) + + class TestStartupDecodeGraphReuse(unittest.TestCase): def test_switch_reuses_startup_graph_or_captures_when_missing(self): from sglang.srt.managers.tp_worker import TpModelWorker