diff --git a/python/sglang/srt/entrypoints/engine.py b/python/sglang/srt/entrypoints/engine.py index fb6db42920..4147570728 100644 --- a/python/sglang/srt/entrypoints/engine.py +++ b/python/sglang/srt/entrypoints/engine.py @@ -66,6 +66,7 @@ from sglang.srt.environ import envs from sglang.srt.managers.data_parallel_controller import ( SCHEDULER_PIDS_ARG, run_data_parallel_controller_process, + run_scheduler_process_with_init_pipe, ) from sglang.srt.managers.detokenizer_manager import run_detokenizer_process from sglang.srt.managers.io_struct import ( @@ -895,8 +896,15 @@ class Engine(EngineScoreMixin, EngineBase): use_dp_controller = ( get_parallel().num_dp_ranks > 1 or get_exec().moe.ep_join_mode == "scale" ) + prelaunch_dp_schedulers = ( + use_dp_controller + and get_parallel().attn_dp_enabled + and get_exec().moe.elastic_ep_backend is None + and not get_exec().moe.is_ep_joiner + ) + prelaunched_schedulers = {} - if not use_dp_controller: + if not use_dp_controller or prelaunch_dp_schedulers: # Launch tensor parallel scheduler processes memory_saver_adapter = TorchMemorySaverAdapter.create( enable=get_exec().features.enable_memory_saver @@ -909,7 +917,7 @@ class Engine(EngineScoreMixin, EngineBase): for pp_rank in pp_rank_range: for tp_rank in tp_rank_range: - reader, writer = mp.Pipe(duplex=False) + reader, writer = mp.Pipe(duplex=prelaunch_dp_schedulers) gpu_id = ( get_device().base_gpu_id + ((pp_rank % pp_size_per_node) * tp_size_per_node) @@ -917,9 +925,12 @@ class Engine(EngineScoreMixin, EngineBase): ) with maybe_reindex_device_id(gpu_id) as gpu_id: - proc = mp.Process( - target=run_scheduler_process_func, - args=( + if prelaunch_dp_schedulers: + target = run_scheduler_process_with_init_pipe + args = (run_scheduler_process_func, writer) + else: + target = run_scheduler_process_func + args = ( server_args, port_args, gpu_id, @@ -927,8 +938,8 @@ class Engine(EngineScoreMixin, EngineBase): pp_rank, None, writer, - ), - ) + ) + proc = mp.Process(target=target, args=args) with ( memory_saver_adapter.configure_subprocess(), numa_utils.configure_subprocess(server_args, gpu_id), @@ -936,8 +947,13 @@ class Engine(EngineScoreMixin, EngineBase): proc.start() scheduler_procs.append(proc) - scheduler_pipe_readers.append(reader) - else: + if prelaunch_dp_schedulers: + prelaunched_schedulers[pp_rank, tp_rank] = (proc.pid, reader) + writer.close() + else: + scheduler_pipe_readers.append(reader) + + if use_dp_controller: # Launch the data parallel controller reader, writer = mp.Pipe(duplex=False) scheduler_pipe_readers = [reader] @@ -948,10 +964,15 @@ class Engine(EngineScoreMixin, EngineBase): port_args=port_args, pipe_writer=writer, run_scheduler_process_func=run_scheduler_process_func, + prelaunched_schedulers=prelaunched_schedulers or None, ), ) - proc.start() - scheduler_procs.append(proc) + try: + proc.start() + finally: + for _, init_pipe in prelaunched_schedulers.values(): + init_pipe.close() + scheduler_procs.insert(0, proc) all_child_pids = [proc.pid for proc in scheduler_procs] scheduler_infos = [] @@ -970,18 +991,26 @@ class Engine(EngineScoreMixin, EngineBase): key=lambda source: source["dp_rank"], ) scheduler_infos.extend(infos) - if use_dp_controller: + if use_dp_controller and not prelaunch_dp_schedulers: for info in infos: if SCHEDULER_PIDS_ARG in info: all_child_pids.extend(info[SCHEDULER_PIDS_ARG]) def block_until_scheduler_exits(): - for proc in scheduler_procs: - proc.join() - logger.error( - f"Scheduler or DataParallelController {proc.pid} " - f"terminated with {proc.exitcode}" - ) + watchdog = None + if prelaunch_dp_schedulers and get_parallel().node_rank > 0: + watchdog = SubprocessWatchdog(scheduler_procs) + watchdog.start() + try: + for proc in scheduler_procs: + proc.join() + logger.error( + f"Scheduler or DataParallelController {proc.pid} " + f"terminated with {proc.exitcode}" + ) + finally: + if watchdog is not None: + watchdog.stop() return ( SchedulerInitResult( diff --git a/python/sglang/srt/managers/data_parallel_controller.py b/python/sglang/srt/managers/data_parallel_controller.py index 33c4fb0f22..09046957b1 100644 --- a/python/sglang/srt/managers/data_parallel_controller.py +++ b/python/sglang/srt/managers/data_parallel_controller.py @@ -21,6 +21,7 @@ import threading import time from collections.abc import Callable from enum import Enum, auto +from multiprocessing.connection import Connection, wait import psutil import setproctitle @@ -146,6 +147,9 @@ class DataParallelController: server_args: ServerArgs, port_args: PortArgs, run_scheduler_process_func: Callable, + prelaunched_schedulers: ( + dict[tuple[int, int], tuple[int, Connection]] | None + ) = None, ) -> None: # Parse args self.server_args = server_args @@ -154,6 +158,7 @@ class DataParallelController: get_parallel().load_balance_method ) self.run_scheduler_process_func = run_scheduler_process_func + self.prelaunched_schedulers = prelaunched_schedulers # Init inter-process communication self.context = zmq.Context(1 + get_parallel().num_dp_ranks) @@ -672,7 +677,6 @@ class DataParallelController: rank_port_args.nccl_port = port_args.nccl_port rank_port_args.instance_id = port_args.instance_id - reader, writer = mp.Pipe(duplex=False) gpu_id = ( get_device().base_gpu_id + base_gpu_id @@ -697,27 +701,34 @@ class DataParallelController: display_dp_rank = dp_rank + offset if dp_rank is not None else None with self.env_lock, maybe_reindex_device_id(gpu_id) as gpu_id: - proc = mp.Process( - target=self.run_scheduler_process_func, - args=( - server_args, - rank_port_args, - gpu_id, - tp_rank, - pp_rank, - dp_rank, - writer, - display_tp_rank, - display_dp_rank, - display_moe_ep_rank, - ), + scheduler_args = ( + server_args, + rank_port_args, + gpu_id, + tp_rank, + pp_rank, + dp_rank, ) - with ( - memory_saver_adapter.configure_subprocess(), - numa_utils.configure_subprocess(server_args, gpu_id), - ): - proc.start() - self.scheduler_procs.append(proc) + display_ranks = ( + display_tp_rank, + display_dp_rank, + display_moe_ep_rank, + ) + if self.prelaunched_schedulers is not None: + _, reader = self.prelaunched_schedulers[pp_rank, tp_rank] + reader.send((scheduler_args, display_ranks)) + else: + reader, writer = mp.Pipe(duplex=False) + proc = mp.Process( + target=self.run_scheduler_process_func, + args=(*scheduler_args, writer, *display_ranks), + ) + with ( + memory_saver_adapter.configure_subprocess(), + numa_utils.configure_subprocess(server_args, gpu_id), + ): + proc.start() + self.scheduler_procs.append(proc) scheduler_pipe_readers.append(reader) # Wait for model to finish loading @@ -813,11 +824,22 @@ class DataParallelController: self._request_dispatcher(recv_req) +def run_scheduler_process_with_init_pipe(run_scheduler_process_func: Callable, pipe): + # Unpickling the callable imports the scheduler before waiting for DP setup. + kill_itself_when_parent_died() + try: + scheduler_args, display_ranks = pipe.recv() + run_scheduler_process_func(*scheduler_args, pipe, *display_ranks) + finally: + pipe.close() + + def run_data_parallel_controller_process( server_args: ServerArgs, port_args: PortArgs, pipe_writer, run_scheduler_process_func: Callable = run_scheduler_process, + prelaunched_schedulers: dict[tuple[int, int], tuple[int, Connection]] | None = None, ): setproctitle.setproctitle("sglang::data_parallel_controller") faulthandler.enable() @@ -843,11 +865,13 @@ def run_data_parallel_controller_process( try: controller = DataParallelController( - server_args, port_args, run_scheduler_process_func + server_args, port_args, run_scheduler_process_func, prelaunched_schedulers + ) + scheduler_pids = ( + [pid for pid, _ in prelaunched_schedulers.values()] + if prelaunched_schedulers is not None + else [proc.pid for proc in controller.scheduler_procs if proc is not None] ) - scheduler_pids = [ - proc.pid for proc in controller.scheduler_procs if proc is not None - ] init_info = { "status": "ready", "max_total_num_tokens": controller.max_total_num_tokens, @@ -865,6 +889,9 @@ def run_data_parallel_controller_process( # The primary owns routing for the expanded scheduler set. if get_parallel().node_rank == 0 and not get_exec().moe.is_ep_scale_joiner: controller.event_loop() + if prelaunched_schedulers is not None: + # The launcher owns these processes. Their pipes close when they exit. + wait([pipe for _, pipe in prelaunched_schedulers.values()]) for proc in controller.scheduler_procs: proc.join() logger.error( diff --git a/test/registered/unit/managers/test_data_parallel_controller.py b/test/registered/unit/managers/test_data_parallel_controller.py index 04a0dc069c..35e1fc13c4 100644 --- a/test/registered/unit/managers/test_data_parallel_controller.py +++ b/test/registered/unit/managers/test_data_parallel_controller.py @@ -114,6 +114,7 @@ class TestLocalKvEventSources(CustomTestCase): controller.scheduler_procs = [] controller.local_kv_event_sources = [] controller.run_scheduler_process_func = MagicMock() + controller.prelaunched_schedulers = None # TP8/DP4 over two nodes: this follower's four schedulers own ranks 2,3. readers = [ MagicMock(