From cf88ed4352c74e34d960171859ef2242157e734d Mon Sep 17 00:00:00 2001 From: FeiHong Guo <1187883824@qq.com> Date: Sun, 11 Oct 2026 21:08:55 +0800 Subject: [PATCH] [BugFix][DP] Prevent health checks from perturbing user request routing (#35721) Signed-off-by: Phoenix3334 --- .../srt/managers/data_parallel_controller.py | 39 +++++++- .../managers/test_data_parallel_controller.py | 96 ++++++++++++++++++- 2 files changed, 130 insertions(+), 5 deletions(-) diff --git a/python/sglang/srt/managers/data_parallel_controller.py b/python/sglang/srt/managers/data_parallel_controller.py index 09046957b1..dc948a16e8 100644 --- a/python/sglang/srt/managers/data_parallel_controller.py +++ b/python/sglang/srt/managers/data_parallel_controller.py @@ -46,6 +46,7 @@ from sglang.srt.managers.io_struct import ( from sglang.srt.managers.load_snapshot import create_load_snapshot_reader from sglang.srt.managers.schedule_batch import Req from sglang.srt.managers.scheduler import run_scheduler_process +from sglang.srt.managers.utils import is_health_check_generate_req from sglang.srt.observability.cpu_monitor import start_cpu_monitor_thread from sglang.srt.observability.req_time_stats import DPControllerReqTimeStats from sglang.srt.observability.startup_time import aggregate_scheduler_startup_times @@ -169,6 +170,7 @@ class DataParallelController: # Dispatch method self.round_robin_counter = 0 + self.health_round_robin_counter = 0 dispatch_lookup = { LoadBalanceMethod.ROUND_ROBIN: self.round_robin_scheduler, LoadBalanceMethod.FOLLOW_BOOTSTRAP_ROOM: self.follow_bootstrap_room_scheduler, @@ -332,7 +334,12 @@ class DataParallelController: self.dp_budget.update_budget(self.load_snapshot_reader.read_all()) def dispatching_with_trace(self, req: Req, refresh_load_budget: bool = True): - if refresh_load_budget and self.refresh_load_budget_on_dispatch: + is_health_check = is_health_check_generate_req(req) + if ( + not is_health_check + and refresh_load_budget + and self.refresh_load_budget_on_dispatch + ): self.refresh_load_budget() time_stats = DPControllerReqTimeStats.new_from_obj( @@ -341,12 +348,18 @@ class DataParallelController: time_stats.set_dp_dispatch_time() req.time_stats = wrap_as_pickle(time_stats) - self.dispatching(req) + if is_health_check: + self.dispatch_health_check(req) + else: + self.dispatching(req) req.time_stats = time_stats req.time_stats.set_dp_dispatch_finish_time() def dispatch_batch_generate(self, batch_req: BatchTokenizedGenerateReqInput): - if self.refresh_load_budget_on_dispatch: + has_user_request = any( + not is_health_check_generate_req(req) for req in batch_req + ) + if has_user_request and self.refresh_load_budget_on_dispatch: self.refresh_load_budget() for req in batch_req: self.dispatching_with_trace(req, refresh_load_budget=False) @@ -751,6 +764,26 @@ class DataParallelController: info.get("startup_time") for info in scheduler_info ) + def dispatch_health_check(self, req: Req) -> None: + # This policy routes by request metadata, not shared load-balancing state. + if self.load_balance_method == LoadBalanceMethod.FOLLOW_BOOTSTRAP_ROOM: + self.follow_bootstrap_room_scheduler(req) + return + + if self.maybe_external_dp_rank_routing(req): + return + + active = self._active_workers + if not active: + raise RuntimeError("No active DP workers are available for health checks.") + + slot = active[self.health_round_robin_counter % len(active)] + self.health_round_robin_counter = (self.health_round_robin_counter + 1) % len( + active + ) + logger.debug(f"Choose worker {slot} for health check") + sock_send(self.workers[slot], req) + def maybe_external_dp_rank_routing(self, req: Req): if req.routed_dp_rank is not None: rank = req.routed_dp_rank diff --git a/test/registered/unit/managers/test_data_parallel_controller.py b/test/registered/unit/managers/test_data_parallel_controller.py index 35e1fc13c4..a513cf872e 100644 --- a/test/registered/unit/managers/test_data_parallel_controller.py +++ b/test/registered/unit/managers/test_data_parallel_controller.py @@ -16,7 +16,7 @@ import time import unittest from contextlib import nullcontext from types import SimpleNamespace -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, call, patch import msgspec.structs @@ -53,14 +53,24 @@ def _make_controller(num_dp_ranks: int) -> DataParallelController: ctl.workers = [MagicMock(name=f"worker_{i}") for i in range(num_dp_ranks)] ctl.status = [True] * num_dp_ranks ctl._active_workers = list(range(num_dp_ranks)) + ctl.load_balance_method = LoadBalanceMethod.ROUND_ROBIN ctl.round_robin_counter = 0 + ctl.health_round_robin_counter = 0 ctl.dp_budget = DPBudget(num_dp_ranks=num_dp_ranks) return ctl -def _req(routed_dp_rank=None, bootstrap_room=None, input_ids=None): +def _req( + routed_dp_rank=None, + bootstrap_room=None, + input_ids=None, + rid="USER", + time_stats=None, +): """Req stand-in; SimpleNamespace avoids pinning to the Req dataclass schema.""" return SimpleNamespace( + time_stats=time_stats, + rid=rid, routed_dp_rank=routed_dp_rank, bootstrap_room=bootstrap_room, input_ids=input_ids or [], @@ -305,6 +315,88 @@ class TestRoundRobinScheduler(CustomTestCase): ctl.workers[0].send_pyobj.assert_called_once() +class TestDispatchingWithTrace(CustomTestCase): + def _make_traced_controller(self) -> DataParallelController: + ctl = _make_controller(num_dp_ranks=4) + ctl.refresh_load_budget_on_dispatch = True + ctl.refresh_load_budget = MagicMock() + ctl.dispatching = MagicMock() + ctl.dispatch_health_check = MagicMock() + return ctl + + def test_health_and_user_dispatch_use_separate_state(self): + for rid, is_health in [("HEALTH_CHECK_dispatch", True), ("user", False)]: + with self.subTest(rid=rid): + ctl = self._make_traced_controller() + req = _req(rid=rid) + ctl.dispatching_with_trace(req) + self.assertEqual(ctl.refresh_load_budget.call_count, int(not is_health)) + self.assertEqual( + ctl.dispatching.call_args_list, [] if is_health else [call(req)] + ) + self.assertEqual( + ctl.dispatch_health_check.call_args_list, + [call(req)] if is_health else [], + ) + + def test_batch_refreshes_budget_only_for_user_requests(self): + for rids, refreshes in [ + (["HEALTH_CHECK_batch"], 0), + (["HEALTH_CHECK_batch", "user"], 1), + ]: + with self.subTest(rids=rids): + ctl = self._make_traced_controller() + ctl.dispatching_with_trace = MagicMock() + reqs = [_req(rid=rid) for rid in rids] + ctl.dispatch_batch_generate(reqs) + self.assertEqual(ctl.refresh_load_budget.call_count, refreshes) + self.assertEqual( + ctl.dispatching_with_trace.call_args_list, + [call(req, refresh_load_budget=False) for req in reqs], + ) + + +class TestHealthCheckScheduler(CustomTestCase): + def test_health_and_user_round_robin_state_are_independent(self): + ctl = _make_controller(num_dp_ranks=4) + ctl.round_robin_scheduler(_req()) + self.assertEqual(ctl.health_round_robin_counter, 0) + ctl.dispatch_health_check(_req(rid="HEALTH_CHECK")) + self.assertEqual(ctl.round_robin_counter, 1) + self.assertEqual(ctl.health_round_robin_counter, 1) + ctl.round_robin_scheduler(_req()) + self.assertEqual(ctl.round_robin_counter, 2) + self.assertEqual(ctl.health_round_robin_counter, 1) + + def test_health_round_robin_uses_only_active_workers(self): + ctl = _make_controller(num_dp_ranks=4) + ctl._active_workers = [1, 3] + for _ in range(4): + ctl.dispatch_health_check(_req(rid="HEALTH_CHECK_active")) + ctl.workers[0].send_pyobj.assert_not_called() + self.assertEqual(ctl.workers[1].send_pyobj.call_count, 2) + ctl.workers[2].send_pyobj.assert_not_called() + self.assertEqual(ctl.workers[3].send_pyobj.call_count, 2) + + def test_external_routing_does_not_advance_health_counter(self): + ctl = _make_controller(num_dp_ranks=4) + req = _req(routed_dp_rank=2, rid="HEALTH_CHECK_routed") + ctl.dispatch_health_check(req) + ctl.workers[2].send_pyobj.assert_called_once() + self.assertEqual(ctl.health_round_robin_counter, 0) + + def test_health_preserves_follow_bootstrap_room_routing(self): + ctl = _make_controller(num_dp_ranks=4) + ctl.load_balance_method = LoadBalanceMethod.FOLLOW_BOOTSTRAP_ROOM + + req = _req(rid="HEALTH_CHECK_pd", bootstrap_room=3) + + ctl.dispatch_health_check(req) + + ctl.workers[3].send_pyobj.assert_called_once() + self.assertEqual(ctl.health_round_robin_counter, 0) + + class TestFollowBootstrapRoomScheduler(CustomTestCase): def test_dispatches_by_bootstrap_room_modulo(self): ctl = _make_controller(num_dp_ranks=4)