Overlap scheduler startup with the data parallel controller (#43177)

Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com>
Co-authored-by: Yongji Wu <30348494+libertyeagle@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-09 10:54:22 -07:00
committed by GitHub
co-authored by Hanming Lu Yongji Wu
parent 5cbf949839
commit 3c612132fd
3 changed files with 100 additions and 43 deletions
+47 -18
View File
@@ -66,6 +66,7 @@ from sglang.srt.environ import envs
from sglang.srt.managers.data_parallel_controller import (
SCHEDULER_PIDS_ARG,
run_data_parallel_controller_process,
run_scheduler_process_with_init_pipe,
)
from sglang.srt.managers.detokenizer_manager import run_detokenizer_process
from sglang.srt.managers.io_struct import (
@@ -895,8 +896,15 @@ class Engine(EngineScoreMixin, EngineBase):
use_dp_controller = (
get_parallel().num_dp_ranks > 1 or get_exec().moe.ep_join_mode == "scale"
)
prelaunch_dp_schedulers = (
use_dp_controller
and get_parallel().attn_dp_enabled
and get_exec().moe.elastic_ep_backend is None
and not get_exec().moe.is_ep_joiner
)
prelaunched_schedulers = {}
if not use_dp_controller:
if not use_dp_controller or prelaunch_dp_schedulers:
# Launch tensor parallel scheduler processes
memory_saver_adapter = TorchMemorySaverAdapter.create(
enable=get_exec().features.enable_memory_saver
@@ -909,7 +917,7 @@ class Engine(EngineScoreMixin, EngineBase):
for pp_rank in pp_rank_range:
for tp_rank in tp_rank_range:
reader, writer = mp.Pipe(duplex=False)
reader, writer = mp.Pipe(duplex=prelaunch_dp_schedulers)
gpu_id = (
get_device().base_gpu_id
+ ((pp_rank % pp_size_per_node) * tp_size_per_node)
@@ -917,9 +925,12 @@ class Engine(EngineScoreMixin, EngineBase):
)
with maybe_reindex_device_id(gpu_id) as gpu_id:
proc = mp.Process(
target=run_scheduler_process_func,
args=(
if prelaunch_dp_schedulers:
target = run_scheduler_process_with_init_pipe
args = (run_scheduler_process_func, writer)
else:
target = run_scheduler_process_func
args = (
server_args,
port_args,
gpu_id,
@@ -927,8 +938,8 @@ class Engine(EngineScoreMixin, EngineBase):
pp_rank,
None,
writer,
),
)
)
proc = mp.Process(target=target, args=args)
with (
memory_saver_adapter.configure_subprocess(),
numa_utils.configure_subprocess(server_args, gpu_id),
@@ -936,8 +947,13 @@ class Engine(EngineScoreMixin, EngineBase):
proc.start()
scheduler_procs.append(proc)
scheduler_pipe_readers.append(reader)
else:
if prelaunch_dp_schedulers:
prelaunched_schedulers[pp_rank, tp_rank] = (proc.pid, reader)
writer.close()
else:
scheduler_pipe_readers.append(reader)
if use_dp_controller:
# Launch the data parallel controller
reader, writer = mp.Pipe(duplex=False)
scheduler_pipe_readers = [reader]
@@ -948,10 +964,15 @@ class Engine(EngineScoreMixin, EngineBase):
port_args=port_args,
pipe_writer=writer,
run_scheduler_process_func=run_scheduler_process_func,
prelaunched_schedulers=prelaunched_schedulers or None,
),
)
proc.start()
scheduler_procs.append(proc)
try:
proc.start()
finally:
for _, init_pipe in prelaunched_schedulers.values():
init_pipe.close()
scheduler_procs.insert(0, proc)
all_child_pids = [proc.pid for proc in scheduler_procs]
scheduler_infos = []
@@ -970,18 +991,26 @@ class Engine(EngineScoreMixin, EngineBase):
key=lambda source: source["dp_rank"],
)
scheduler_infos.extend(infos)
if use_dp_controller:
if use_dp_controller and not prelaunch_dp_schedulers:
for info in infos:
if SCHEDULER_PIDS_ARG in info:
all_child_pids.extend(info[SCHEDULER_PIDS_ARG])
def block_until_scheduler_exits():
for proc in scheduler_procs:
proc.join()
logger.error(
f"Scheduler or DataParallelController {proc.pid} "
f"terminated with {proc.exitcode}"
)
watchdog = None
if prelaunch_dp_schedulers and get_parallel().node_rank > 0:
watchdog = SubprocessWatchdog(scheduler_procs)
watchdog.start()
try:
for proc in scheduler_procs:
proc.join()
logger.error(
f"Scheduler or DataParallelController {proc.pid} "
f"terminated with {proc.exitcode}"
)
finally:
if watchdog is not None:
watchdog.stop()
return (
SchedulerInitResult(
@@ -21,6 +21,7 @@ import threading
import time
from collections.abc import Callable
from enum import Enum, auto
from multiprocessing.connection import Connection, wait
import psutil
import setproctitle
@@ -146,6 +147,9 @@ class DataParallelController:
server_args: ServerArgs,
port_args: PortArgs,
run_scheduler_process_func: Callable,
prelaunched_schedulers: (
dict[tuple[int, int], tuple[int, Connection]] | None
) = None,
) -> None:
# Parse args
self.server_args = server_args
@@ -154,6 +158,7 @@ class DataParallelController:
get_parallel().load_balance_method
)
self.run_scheduler_process_func = run_scheduler_process_func
self.prelaunched_schedulers = prelaunched_schedulers
# Init inter-process communication
self.context = zmq.Context(1 + get_parallel().num_dp_ranks)
@@ -672,7 +677,6 @@ class DataParallelController:
rank_port_args.nccl_port = port_args.nccl_port
rank_port_args.instance_id = port_args.instance_id
reader, writer = mp.Pipe(duplex=False)
gpu_id = (
get_device().base_gpu_id
+ base_gpu_id
@@ -697,27 +701,34 @@ class DataParallelController:
display_dp_rank = dp_rank + offset if dp_rank is not None else None
with self.env_lock, maybe_reindex_device_id(gpu_id) as gpu_id:
proc = mp.Process(
target=self.run_scheduler_process_func,
args=(
server_args,
rank_port_args,
gpu_id,
tp_rank,
pp_rank,
dp_rank,
writer,
display_tp_rank,
display_dp_rank,
display_moe_ep_rank,
),
scheduler_args = (
server_args,
rank_port_args,
gpu_id,
tp_rank,
pp_rank,
dp_rank,
)
with (
memory_saver_adapter.configure_subprocess(),
numa_utils.configure_subprocess(server_args, gpu_id),
):
proc.start()
self.scheduler_procs.append(proc)
display_ranks = (
display_tp_rank,
display_dp_rank,
display_moe_ep_rank,
)
if self.prelaunched_schedulers is not None:
_, reader = self.prelaunched_schedulers[pp_rank, tp_rank]
reader.send((scheduler_args, display_ranks))
else:
reader, writer = mp.Pipe(duplex=False)
proc = mp.Process(
target=self.run_scheduler_process_func,
args=(*scheduler_args, writer, *display_ranks),
)
with (
memory_saver_adapter.configure_subprocess(),
numa_utils.configure_subprocess(server_args, gpu_id),
):
proc.start()
self.scheduler_procs.append(proc)
scheduler_pipe_readers.append(reader)
# Wait for model to finish loading
@@ -813,11 +824,22 @@ class DataParallelController:
self._request_dispatcher(recv_req)
def run_scheduler_process_with_init_pipe(run_scheduler_process_func: Callable, pipe):
# Unpickling the callable imports the scheduler before waiting for DP setup.
kill_itself_when_parent_died()
try:
scheduler_args, display_ranks = pipe.recv()
run_scheduler_process_func(*scheduler_args, pipe, *display_ranks)
finally:
pipe.close()
def run_data_parallel_controller_process(
server_args: ServerArgs,
port_args: PortArgs,
pipe_writer,
run_scheduler_process_func: Callable = run_scheduler_process,
prelaunched_schedulers: dict[tuple[int, int], tuple[int, Connection]] | None = None,
):
setproctitle.setproctitle("sglang::data_parallel_controller")
faulthandler.enable()
@@ -843,11 +865,13 @@ def run_data_parallel_controller_process(
try:
controller = DataParallelController(
server_args, port_args, run_scheduler_process_func
server_args, port_args, run_scheduler_process_func, prelaunched_schedulers
)
scheduler_pids = (
[pid for pid, _ in prelaunched_schedulers.values()]
if prelaunched_schedulers is not None
else [proc.pid for proc in controller.scheduler_procs if proc is not None]
)
scheduler_pids = [
proc.pid for proc in controller.scheduler_procs if proc is not None
]
init_info = {
"status": "ready",
"max_total_num_tokens": controller.max_total_num_tokens,
@@ -865,6 +889,9 @@ def run_data_parallel_controller_process(
# The primary owns routing for the expanded scheduler set.
if get_parallel().node_rank == 0 and not get_exec().moe.is_ep_scale_joiner:
controller.event_loop()
if prelaunched_schedulers is not None:
# The launcher owns these processes. Their pipes close when they exit.
wait([pipe for _, pipe in prelaunched_schedulers.values()])
for proc in controller.scheduler_procs:
proc.join()
logger.error(