Files
sglang/test/registered/unit/managers/test_data_parallel_controller.py
T

567 lines
22 KiB
Python

"""DPBudget + DataParallelController dispatch tests.
The e2e counterpart, over the real scheduler load-report path, is
test/registered/disaggregation/test_disaggregation_dp_attention.py.
Fragility: scheduler tests bypass `DataParallelController.__init__` via
`__new__` and inject only the attrs the schedulers read (`workers`, `status`,
`_active_workers`, `round_robin_counter`, `dp_budget`). Update `_make_controller`
if a scheduler starts reading another attr. `maybe_external_dp_rank_routing`
is exercised as the real method, no mock. The refresh-throttle tests inject
`load_snapshot_reader` and `_last_refresh_time` on top of those.
"""
import threading
import time
import unittest
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import MagicMock, call, patch
import msgspec.structs
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel
maybe_stub_sgl_kernel()
from sglang.srt.managers.data_parallel_controller import (
DataParallelController,
DPBudget,
LoadBalanceMethod,
WorkerPortHandshake,
)
from sglang.srt.managers.load_snapshot import LoadSnapshot
from sglang.srt.runtime_context import get_context
register_cpu_ci(est_time=12, suite="base-a-test-cpu")
_BASE_LOAD = msgspec.structs.replace(
LoadSnapshot(dp_rank=0),
max_total_num_tokens=4096,
max_running_requests=128,
)
def _load(**overrides) -> LoadSnapshot:
return msgspec.structs.replace(_BASE_LOAD, **overrides)
def _make_controller(num_dp_ranks: int) -> DataParallelController:
"""Bypass __init__; inject only the attrs dispatch methods read."""
ctl = DataParallelController.__new__(DataParallelController)
ctl.workers = [MagicMock(name=f"worker_{i}") for i in range(num_dp_ranks)]
ctl.status = [True] * num_dp_ranks
ctl._active_workers = list(range(num_dp_ranks))
ctl.load_balance_method = LoadBalanceMethod.ROUND_ROBIN
ctl.round_robin_counter = 0
ctl.health_round_robin_counter = 0
ctl.dp_budget = DPBudget(num_dp_ranks=num_dp_ranks)
return ctl
def _req(
routed_dp_rank=None,
bootstrap_room=None,
input_ids=None,
rid="USER",
time_stats=None,
):
"""Req stand-in; SimpleNamespace avoids pinning to the Req dataclass schema."""
return SimpleNamespace(
time_stats=time_stats,
rid=rid,
routed_dp_rank=routed_dp_rank,
bootstrap_room=bootstrap_room,
input_ids=input_ids or [],
)
class TestJoinerWorkerPortAdmission(CustomTestCase):
def _launch_joiner(self, parallel, handshake, launch):
controller = DataParallelController.__new__(DataParallelController)
controller._receive_ports_as_client = MagicMock(return_value=handshake)
controller.launch_tensor_parallel_group = launch
execution = SimpleNamespace(moe=SimpleNamespace(is_ep_scale_joiner=True))
with (
patch(
"sglang.srt.managers.data_parallel_controller.get_parallel",
return_value=parallel,
),
patch(
"sglang.srt.managers.data_parallel_controller.get_exec",
return_value=execution,
),
):
controller.launch_dp_attention_schedulers(
SimpleNamespace(), SimpleNamespace()
)
def test_topology_mismatch_is_rejected_before_scheduler_launch(self):
mismatches = [
# A wider joiner would map physical offset 4 onto active primary slot 2.
((1, 1), (2, 1), 1),
# A narrower joiner would select beyond the primary's reserved slots.
((2, 1), (1, 1), 2),
]
for primary, joining, num_dp_ranks in mismatches:
with self.subTest(primary=primary, joining=joining):
parallel = SimpleNamespace(
dist_init_addr="127.0.0.1:20000",
ep_join_rank_offset=4,
num_dp_ranks=num_dp_ranks,
attn_tp_size=joining[0],
attn_cp_size=joining[1],
)
handshake = WorkerPortHandshake(
worker_ports=[1000, 1001, 1002, 1003],
attn_tp_size=primary[0],
attn_cp_size=primary[1],
)
launch = MagicMock()
with self.assertRaisesRegex(ValueError, "same attention topology"):
self._launch_joiner(parallel, handshake, launch)
launch.assert_not_called()
def test_matching_topology_selects_reserved_logical_dp_slot(self):
parallel = SimpleNamespace(
dist_init_addr="127.0.0.1:20000",
ep_join_rank_offset=4,
num_dp_ranks=1,
attn_tp_size=2,
attn_cp_size=1,
)
handshake = WorkerPortHandshake(
worker_ports=[1000, 1001, 1002, 1003],
attn_tp_size=2,
attn_cp_size=1,
)
launch = MagicMock()
self._launch_joiner(parallel, handshake, launch)
self.assertEqual(launch.call_args.args[-1], [1002])
class TestLocalKvEventSources(CustomTestCase):
@staticmethod
def _source(rank, block_size=64):
return {
"dp_rank": rank,
"endpoint": f"tcp://127.0.0.1:{5557 + rank}",
"topic": "kv",
"block_size": block_size,
}
def test_scheduler_reports_logical_block_size_or_no_source(self):
from sglang.srt.managers.scheduler import Scheduler
from sglang.srt.managers.scheduler_components.kv_events_publisher import (
SchedulerKvEventsPublisher,
)
component = SchedulerKvEventsPublisher.__new__(SchedulerKvEventsPublisher)
scheduler = Scheduler.__new__(Scheduler)
scheduler.max_total_num_tokens = 64
scheduler.max_req_input_len = 32
scheduler.startup_time = {}
scheduler.page_size = 64
scheduler.kv_events_publisher = component
publisher = MagicMock()
publisher.describe_local_source.return_value = self._source(4, 256)
for owned_publisher, expected in (
(publisher, [self._source(4, 256)]),
(None, []),
):
with (
self.subTest(has_publisher=owned_publisher is not None),
get_context().override_server_args(grpc_port=50051, dcp_size=4),
):
component.kv_event_publisher = owned_publisher
self.assertEqual(
scheduler.get_init_info()["kv_event_sources"], expected
)
publisher.describe_local_source.assert_called_once_with(256)
def test_controller_collects_all_scheduler_sources(self):
from sglang.srt.managers import data_parallel_controller as module
controller = DataParallelController.__new__(DataParallelController)
controller.env_lock = threading.Lock()
controller.scheduler_procs = []
controller.local_kv_event_sources = []
controller.run_scheduler_process_func = MagicMock()
controller.attn_replica_size = 1
controller.prelaunched_schedulers = None
# TP8/DP4 over two nodes: this follower's four schedulers own ranks 2,3.
readers = [
MagicMock(
recv=MagicMock(
return_value={
"max_total_num_tokens": 64,
"max_req_input_len": 32,
"kv_event_sources": sources,
}
)
)
for sources in ([self._source(2)], [], [self._source(3)], [])
]
with (
get_context().override_server_args(
tp_size=8,
dp_size=4,
enable_dp_attention=True,
nnodes=2,
node_rank=1,
pp_size=1,
attn_cp_size=1,
),
patch.object(module.mp, "Process"),
patch.object(
module.mp,
"Pipe",
side_effect=[(reader, MagicMock()) for reader in readers],
),
patch.object(module.PortArgs, "init_new", return_value=MagicMock()),
patch.object(module.TorchMemorySaverAdapter, "create"),
patch.object(module.numa_utils, "configure_subprocess"),
patch.object(
module, "maybe_reindex_device_id", return_value=nullcontext(0)
),
):
controller.launch_tensor_parallel_group(
MagicMock(), MagicMock(), 0, None, [7000, 7001, 7002, 7003]
)
self.assertEqual(
controller.local_kv_event_sources, [self._source(2), self._source(3)]
)
class TestDPBudgetUpdateBudget(CustomTestCase):
def test_maps_running_plus_waiting_to_total_requests(self):
budget = DPBudget(num_dp_ranks=2)
budget.update_budget(
[
_load(dp_rank=0, timestamp=1.0, num_running_reqs=3, num_waiting_reqs=2),
_load(dp_rank=1, timestamp=1.0, num_running_reqs=5, num_waiting_reqs=1),
]
)
self.assertEqual(budget.total_requests, [5, 6])
def test_maps_num_total_tokens_not_num_used_tokens(self):
budget = DPBudget(num_dp_ranks=2)
budget.update_budget(
[
_load(
dp_rank=0, timestamp=1.0, num_used_tokens=100, num_total_tokens=150
),
_load(
dp_rank=1, timestamp=1.0, num_used_tokens=80, num_total_tokens=80
),
]
)
self.assertEqual(budget.total_tokens, [150, 80])
def test_partial_update_only_affects_reported_rank(self):
budget = DPBudget(num_dp_ranks=3)
budget.update_budget(
[
_load(
dp_rank=0, timestamp=1.0, num_running_reqs=10, num_total_tokens=100
),
_load(
dp_rank=1, timestamp=1.0, num_running_reqs=20, num_total_tokens=200
),
_load(
dp_rank=2, timestamp=1.0, num_running_reqs=30, num_total_tokens=300
),
]
)
budget.update_budget(
[
_load(
dp_rank=1,
timestamp=2.0,
num_running_reqs=1,
num_waiting_reqs=1,
num_total_tokens=50,
)
]
)
self.assertEqual(budget.total_requests, [10, 2, 30])
self.assertEqual(budget.total_tokens, [100, 50, 300])
class TestRoundRobinScheduler(CustomTestCase):
def test_cycles_through_active_workers_in_order(self):
ctl = _make_controller(num_dp_ranks=4)
for _ in range(8):
ctl.round_robin_scheduler(_req())
# 8 reqs across 4 active workers — 2 each, in round-robin order
for i, worker in enumerate(ctl.workers):
self.assertEqual(worker.send_pyobj.call_count, 2, f"worker {i} call count")
def test_first_dispatch_picks_worker_zero(self):
ctl = _make_controller(num_dp_ranks=4)
ctl.round_robin_scheduler(_req())
ctl.workers[0].send_pyobj.assert_called_once()
for i in (1, 2, 3):
ctl.workers[i].send_pyobj.assert_not_called()
def test_skips_inactive_workers(self):
ctl = _make_controller(num_dp_ranks=4)
ctl.status[1] = False
ctl.status[3] = False
for _ in range(6):
ctl.round_robin_scheduler(_req())
# Only workers 0 and 2 are active — should split 6 reqs evenly
self.assertEqual(ctl.workers[0].send_pyobj.call_count, 3)
ctl.workers[1].send_pyobj.assert_not_called()
self.assertEqual(ctl.workers[2].send_pyobj.call_count, 3)
ctl.workers[3].send_pyobj.assert_not_called()
def test_routed_dp_rank_bypasses_counter(self):
"""External dp-rank routing must not advance the counter."""
ctl = _make_controller(num_dp_ranks=4)
ctl.round_robin_scheduler(_req(routed_dp_rank=2))
ctl.workers[2].send_pyobj.assert_called_once()
self.assertEqual(
ctl.round_robin_counter,
0,
"external routing must not advance the round-robin counter",
)
# Subsequent round-robin req still lands on worker 0
ctl.round_robin_scheduler(_req())
ctl.workers[0].send_pyobj.assert_called_once()
class TestDispatchingWithTrace(CustomTestCase):
def _make_traced_controller(self) -> DataParallelController:
ctl = _make_controller(num_dp_ranks=4)
ctl.refresh_load_budget_on_dispatch = True
ctl.refresh_load_budget = MagicMock()
ctl.dispatching = MagicMock()
ctl.dispatch_health_check = MagicMock()
return ctl
def test_health_and_user_dispatch_use_separate_state(self):
for rid, is_health in [("HEALTH_CHECK_dispatch", True), ("user", False)]:
with self.subTest(rid=rid):
ctl = self._make_traced_controller()
req = _req(rid=rid)
ctl.dispatching_with_trace(req)
self.assertEqual(ctl.refresh_load_budget.call_count, int(not is_health))
self.assertEqual(
ctl.dispatching.call_args_list, [] if is_health else [call(req)]
)
self.assertEqual(
ctl.dispatch_health_check.call_args_list,
[call(req)] if is_health else [],
)
def test_batch_refreshes_budget_only_for_user_requests(self):
for rids, refreshes in [
(["HEALTH_CHECK_batch"], 0),
(["HEALTH_CHECK_batch", "user"], 1),
]:
with self.subTest(rids=rids):
ctl = self._make_traced_controller()
ctl.dispatching_with_trace = MagicMock()
reqs = [_req(rid=rid) for rid in rids]
ctl.dispatch_batch_generate(reqs)
self.assertEqual(ctl.refresh_load_budget.call_count, refreshes)
self.assertEqual(
ctl.dispatching_with_trace.call_args_list,
[call(req, refresh_load_budget=False) for req in reqs],
)
class TestHealthCheckScheduler(CustomTestCase):
def test_health_and_user_round_robin_state_are_independent(self):
ctl = _make_controller(num_dp_ranks=4)
ctl.round_robin_scheduler(_req())
self.assertEqual(ctl.health_round_robin_counter, 0)
ctl.dispatch_health_check(_req(rid="HEALTH_CHECK"))
self.assertEqual(ctl.round_robin_counter, 1)
self.assertEqual(ctl.health_round_robin_counter, 1)
ctl.round_robin_scheduler(_req())
self.assertEqual(ctl.round_robin_counter, 2)
self.assertEqual(ctl.health_round_robin_counter, 1)
def test_health_round_robin_uses_only_active_workers(self):
ctl = _make_controller(num_dp_ranks=4)
ctl._active_workers = [1, 3]
for _ in range(4):
ctl.dispatch_health_check(_req(rid="HEALTH_CHECK_active"))
ctl.workers[0].send_pyobj.assert_not_called()
self.assertEqual(ctl.workers[1].send_pyobj.call_count, 2)
ctl.workers[2].send_pyobj.assert_not_called()
self.assertEqual(ctl.workers[3].send_pyobj.call_count, 2)
def test_external_routing_does_not_advance_health_counter(self):
ctl = _make_controller(num_dp_ranks=4)
req = _req(routed_dp_rank=2, rid="HEALTH_CHECK_routed")
ctl.dispatch_health_check(req)
ctl.workers[2].send_pyobj.assert_called_once()
self.assertEqual(ctl.health_round_robin_counter, 0)
def test_health_preserves_follow_bootstrap_room_routing(self):
ctl = _make_controller(num_dp_ranks=4)
ctl.load_balance_method = LoadBalanceMethod.FOLLOW_BOOTSTRAP_ROOM
req = _req(rid="HEALTH_CHECK_pd", bootstrap_room=3)
ctl.dispatch_health_check(req)
ctl.workers[3].send_pyobj.assert_called_once()
self.assertEqual(ctl.health_round_robin_counter, 0)
class TestFollowBootstrapRoomScheduler(CustomTestCase):
def test_dispatches_by_bootstrap_room_modulo(self):
ctl = _make_controller(num_dp_ranks=4)
for room, expected_rank in [
(0, 0),
(1, 1),
(4, 0),
(5, 1),
(100, 0),
(101, 1),
]:
ctl.follow_bootstrap_room_scheduler(_req(bootstrap_room=room))
ctl.workers[expected_rank].send_pyobj.assert_called()
def test_requires_bootstrap_room(self):
ctl = _make_controller(num_dp_ranks=4)
with self.assertRaises(AssertionError):
ctl.follow_bootstrap_room_scheduler(_req(bootstrap_room=None))
def test_routed_dp_rank_bypasses_bootstrap_room(self):
ctl = _make_controller(num_dp_ranks=4)
ctl.follow_bootstrap_room_scheduler(_req(routed_dp_rank=3, bootstrap_room=1))
ctl.workers[3].send_pyobj.assert_called_once()
ctl.workers[1].send_pyobj.assert_not_called()
class TestTotalRequestsScheduler(CustomTestCase):
def test_dispatches_to_min_request_worker(self):
ctl = _make_controller(num_dp_ranks=4)
ctl.dp_budget.total_requests = [5, 3, 1, 4]
ctl.total_requests_scheduler(_req())
ctl.workers[2].send_pyobj.assert_called_once()
for i in (0, 1, 3):
ctl.workers[i].send_pyobj.assert_not_called()
self.assertEqual(
ctl.dp_budget.total_requests[2],
2,
"DPBudget must record the dispatch by incrementing the counter",
)
def test_routed_dp_rank_bypasses_budget(self):
ctl = _make_controller(num_dp_ranks=4)
ctl.dp_budget.total_requests = [5, 3, 1, 4]
ctl.total_requests_scheduler(_req(routed_dp_rank=0))
ctl.workers[0].send_pyobj.assert_called_once()
# DPBudget must not be touched when bypassed
self.assertEqual(
ctl.dp_budget.total_requests,
[5, 3, 1, 4],
"external routing must not mutate DPBudget state",
)
class TestStatusAwarenessInconsistency(CustomTestCase):
"""Document a divergence: ``round_robin_scheduler`` skips workers whose
``status`` is False, but ``total_requests_scheduler`` /
``total_tokens_scheduler`` route purely by DPBudget — they do NOT
consult ``self.status``. If a future change unifies this behaviour,
this test will fail and force a reviewer to confirm intent."""
def test_total_requests_ignores_status(self):
ctl = _make_controller(num_dp_ranks=4)
# Worker 2 is the global minimum AND marked inactive.
ctl.dp_budget.total_requests = [5, 3, 1, 4]
ctl.status[2] = False
ctl.total_requests_scheduler(_req())
# Current behaviour: still dispatches to the inactive worker.
ctl.workers[2].send_pyobj.assert_called_once()
class TestRefreshLoadBudgetThrottle(CustomTestCase):
@staticmethod
def _controller_with_reader(num_dp_ranks, snapshots):
ctl = _make_controller(num_dp_ranks)
ctl.load_snapshot_reader = MagicMock()
ctl.load_snapshot_reader.read_all.return_value = snapshots
return ctl
def test_throttled_refresh_spreads_a_burst_across_ranks(self):
idle = [_load(dp_rank=i, timestamp=1.0, num_total_tokens=0) for i in range(4)]
ctl = self._controller_with_reader(num_dp_ranks=4, snapshots=idle)
# A refresh stamp in the future keeps every call inside the window, so
# the burst runs entirely on speculative counters.
ctl._last_refresh_time = time.perf_counter() + 3600.0
for _ in range(8):
ctl.refresh_load_budget()
ctl.total_tokens_scheduler(_req(input_ids=[0] * 100))
ctl.load_snapshot_reader.read_all.assert_not_called()
self.assertEqual(
ctl.dp_budget.total_tokens,
[200, 200, 200, 200],
"speculative increments should spread the burst evenly",
)
for i, worker in enumerate(ctl.workers):
self.assertEqual(
worker.send_pyobj.call_count, 2, f"worker {i} should get 2 of 8 reqs"
)
def test_refresh_outside_window_overwrites_speculative_increments(self):
reported = [
_load(dp_rank=0, timestamp=2.0, num_total_tokens=10),
_load(dp_rank=1, timestamp=2.0, num_total_tokens=20),
]
ctl = self._controller_with_reader(num_dp_ranks=2, snapshots=reported)
ctl._last_refresh_time = 0.0 # window has long passed
ctl.dp_budget.total_tokens = [999, 999]
ctl.refresh_load_budget()
ctl.load_snapshot_reader.read_all.assert_called_once()
self.assertEqual(
ctl.dp_budget.total_tokens,
[10, 20],
"a fresh snapshot must replace the speculative state",
)
self.assertGreater(ctl._last_refresh_time, 0.0)
def test_unchanged_snapshot_does_not_reset_the_burst(self):
frozen = [_load(dp_rank=i, timestamp=1.0, num_total_tokens=0) for i in range(2)]
ctl = self._controller_with_reader(num_dp_ranks=2, snapshots=frozen)
ctl._last_refresh_time = 0.0
ctl.refresh_load_budget() # adopts timestamp 1.0
for _ in range(4):
ctl.total_tokens_scheduler(_req(input_ids=[0] * 50))
after_burst = list(ctl.dp_budget.total_tokens)
ctl._last_refresh_time = 0.0 # let the next refresh through the throttle
ctl.refresh_load_budget() # same timestamp -> update_budget skips it
self.assertEqual(
after_burst, [100, 100], "burst should have spread over both ranks"
)
self.assertEqual(
ctl.dp_budget.total_tokens,
after_burst,
"a stale-timestamp snapshot must not wipe the speculative state",
)
if __name__ == "__main__":
unittest.main()