diff --git a/python/sglang/srt/arg_groups/fields/parallel.py b/python/sglang/srt/arg_groups/fields/parallel.py index 9bd0b1c1b1..80179b1e1b 100644 --- a/python/sglang/srt/arg_groups/fields/parallel.py +++ b/python/sglang/srt/arg_groups/fields/parallel.py @@ -98,6 +98,7 @@ class Parallel(msgspec.Struct): "follow_bootstrap_room", "total_requests", "total_tokens", + "uncached_tokens", ], ), ] = "auto" diff --git a/python/sglang/srt/managers/data_parallel_controller.py b/python/sglang/srt/managers/data_parallel_controller.py index 03f5002270..2f9eff34d3 100644 --- a/python/sglang/srt/managers/data_parallel_controller.py +++ b/python/sglang/srt/managers/data_parallel_controller.py @@ -1,988 +1,1040 @@ -# Copyright 2023-2024 SGLang Team -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A controller that dispatches requests to multiple data parallel workers.""" - -import faulthandler -import logging -import multiprocessing as mp -import signal -import threading -import time -from collections.abc import Callable -from dataclasses import dataclass -from enum import Enum, auto -from multiprocessing.connection import Connection, wait - -import psutil -import setproctitle -import zmq - -from sglang.srt.elastic_ep.topology import ( - attn_replica_size, - physical_ep_rank_to_dp_rank, - physical_ep_size_to_dp_size, - validate_matching_attention_topology, -) -from sglang.srt.environ import envs -from sglang.srt.layers.dp_attention import compute_dp_attention_world_info -from sglang.srt.managers.io_struct import ( - ActiveRanksOutput, - BatchTokenizedEmbeddingReqInput, - BatchTokenizedGenerateReqInput, - BlockReqInput, - ElasticScaleUpdateReq, - ProfileReq, - TokenizedEmbeddingReqInput, - TokenizedGenerateReqInput, - sock_recv, - sock_send, - unwrap_from_pickle, - wrap_as_pickle, -) -from sglang.srt.managers.load_snapshot import create_load_snapshot_reader -from sglang.srt.managers.schedule_batch import Req -from sglang.srt.managers.scheduler import run_scheduler_process -from sglang.srt.managers.utils import is_health_check_generate_req -from sglang.srt.observability.cpu_monitor import start_cpu_monitor_thread -from sglang.srt.observability.req_time_stats import DPControllerReqTimeStats -from sglang.srt.observability.startup_time import aggregate_scheduler_startup_times -from sglang.srt.observability.trace import process_tracing_init, trace_set_thread_info -from sglang.srt.plugins import load_plugins -from sglang.srt.runtime_context import ( - get_device, - get_disagg, - get_exec, - get_observability, - get_parallel, - get_serving, - publish, -) -from sglang.srt.server_args import ( - DP_ATTENTION_HANDSHAKE_PORT_DELTA, - PortArgs, - ServerArgs, -) -from sglang.srt.utils import numa_utils -from sglang.srt.utils.common import ( - configure_logger, - kill_itself_when_parent_died, - maybe_reindex_device_id, -) -from sglang.srt.utils.network import ( - NetworkAddress, - bind_port, - get_zmq_socket, - get_zmq_socket_on_host, -) -from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter -from sglang.srt.utils.watchdog import Watchdog -from sglang.utils import TypeBasedDispatcher, get_exception_traceback - -logger = logging.getLogger(__name__) - -SCHEDULER_PIDS_ARG = "scheduler_pids" - - -@dataclass(frozen=True) -class WorkerPortHandshake: - worker_ports: list[int] - attn_tp_size: int - attn_cp_size: int - - -class LoadBalanceMethod(Enum): - """Load balance method.""" - - ROUND_ROBIN = auto() - FOLLOW_BOOTSTRAP_ROOM = auto() - TOTAL_REQUESTS = auto() - TOTAL_TOKENS = auto() - - @classmethod - def from_str(cls, method: str): - method = method.upper() - try: - return cls[method] - except KeyError as exc: - raise ValueError(f"Invalid load balance method: {method}") from exc - - -class DPBudget: - def __init__(self, num_dp_ranks: int): - self.num_dp_ranks = num_dp_ranks - self.total_requests = [0] * num_dp_ranks - self.total_tokens = [0] * num_dp_ranks - self.last_timestamp = [0.0] * num_dp_ranks - - def update_budget(self, loads): - """Update budget from shm snapshots, skipping stale reads.""" - for load in loads: - if load.timestamp == self.last_timestamp[load.dp_rank]: - continue - self.last_timestamp[load.dp_rank] = load.timestamp - self.total_requests[load.dp_rank] = ( - load.num_running_reqs + load.num_waiting_reqs - ) - self.total_tokens[load.dp_rank] = load.num_total_tokens - - def dispatch(self, method: LoadBalanceMethod, estimated_tokens: int = 0): - if method == LoadBalanceMethod.TOTAL_REQUESTS: - target_rank = self.total_requests.index(min(self.total_requests)) - elif method == LoadBalanceMethod.TOTAL_TOKENS: - # Use total_requests as a tie-breaker when total_tokens are equal - target_rank = min( - range(self.num_dp_ranks), - key=lambda i: (self.total_tokens[i], self.total_requests[i]), - ) - else: - return None - - # Increment the load of that worker by one as a heuristic - self.total_requests[target_rank] += 1 - self.total_tokens[target_rank] += estimated_tokens - return target_rank - - -class DataParallelController: - """A controller that dispatches requests to multiple data parallel workers.""" - - def __init__( - self, - server_args: ServerArgs, - port_args: PortArgs, - run_scheduler_process_func: Callable, - prelaunched_schedulers: ( - dict[tuple[int, int], tuple[int, Connection]] | None - ) = None, - ) -> None: - # Parse args - self.server_args = server_args - self.port_args = port_args - self.load_balance_method = LoadBalanceMethod.from_str( - get_parallel().load_balance_method - ) - self.run_scheduler_process_func = run_scheduler_process_func - self.prelaunched_schedulers = prelaunched_schedulers - - # Init inter-process communication - self.context = zmq.Context(1 + get_parallel().num_dp_ranks) - if get_parallel().node_rank == 0: - self.recv_from_tokenizer = get_zmq_socket( - self.context, zmq.PULL, port_args.scheduler_input_ipc_name, False - ) - - # Dispatch method - self.round_robin_counter = 0 - self.health_round_robin_counter = 0 - dispatch_lookup = { - LoadBalanceMethod.ROUND_ROBIN: self.round_robin_scheduler, - LoadBalanceMethod.FOLLOW_BOOTSTRAP_ROOM: self.follow_bootstrap_room_scheduler, - LoadBalanceMethod.TOTAL_REQUESTS: self.total_requests_scheduler, - LoadBalanceMethod.TOTAL_TOKENS: self.total_tokens_scheduler, - } - self.dispatching = dispatch_lookup[self.load_balance_method] - self.refresh_load_budget_on_dispatch = self.load_balance_method in ( - LoadBalanceMethod.TOTAL_REQUESTS, - LoadBalanceMethod.TOTAL_TOKENS, - ) - - parallel = get_parallel() - self.launch_dp_size: int = parallel.num_dp_ranks - self.attn_replica_size = attn_replica_size() - if ( - get_exec().moe.elastic_ep_backend is not None - and parallel.max_ep_size is not None - and parallel.max_ep_size > parallel.tp_size - ): - self.max_dp_size = physical_ep_size_to_dp_size( - parallel.max_ep_size, - self.attn_replica_size, - ) - else: - self.max_dp_size = parallel.num_dp_ranks - assert self.max_dp_size >= self.launch_dp_size, ( - f"maximum DP size ({self.max_dp_size}) must be >= " - f"--dp ({self.launch_dp_size})." - ) - - self.dp_active: list[bool] = [True] * self.launch_dp_size + [False] * ( - self.max_dp_size - self.launch_dp_size - ) - - self.dp_budget = DPBudget(get_parallel().num_dp_ranks) - self.load_snapshot_reader = create_load_snapshot_reader( - port_args, - caller="DataParallelController", - ) - self._last_refresh_time = 0.0 - - # To protect changing env vars to set CUDA_VISIBLE_DEVICES. - self.env_lock = threading.Lock() - - # Launch data parallel workers - self.scheduler_procs = [] - self.local_kv_event_sources = [] - self.workers: list[zmq.Socket | None] = [None] * self.max_dp_size - self.status: list[bool] = list(self.dp_active) - self._active_workers: list[int] = list(range(self.launch_dp_size)) - self._active_count_cache: int = self.launch_dp_size - - if get_parallel().attn_dp_enabled: - self.launch_dp_attention_schedulers(server_args, port_args) - # When local control broadcast is enabled, send control messages to - # every DP group leader (attn_tp_rank=0) so each leader broadcasts - # within its own attn_tp_group instead of the full tp_group. - # Otherwise fall back to the original behaviour: send to only the - # first leader, which then broadcasts over the full tp_group. - local_ctrl = get_parallel().enable_dp_attention_local_control_broadcast - self.control_message_step = 1 if local_ctrl else get_parallel().tp_size - else: - self.launch_dp_schedulers(server_args, port_args) - self.control_message_step = 1 - - self.init_dispatcher() - - self.soft_watchdog = Watchdog.create( - debug_name="DataParallelController", - watchdog_timeout=get_device().soft_watchdog_timeout, - soft=True, - test_stuck_time=envs.SGLANG_TEST_STUCK_DP_CONTROLLER.get(), - ) - - if get_observability().enable_metrics: - start_cpu_monitor_thread("data_parallel_controller") - - def send_to_all_workers(self, obj): - for i, worker in enumerate(self.workers): - if worker is not None and self.status[i]: - sock_send(worker, obj) - - def send_control_message(self, obj): - for i in self._active_workers[:: self.control_message_step]: - worker = self.workers[i] - if worker is not None: - sock_send(worker, obj) - - def update_active_ranks(self, ranks: ActiveRanksOutput): - if get_exec().moe.elastic_ep_backend is not None: - if len(ranks.status) != self.max_dp_size: - logger.warning( - "[Elastic EP][DPC] active rank status len=%d != max_dp_size=%d; " - "ignoring update", - len(ranks.status), - self.max_dp_size, - ) - return - self.status = [ - self.dp_active[i] and bool(ranks.status[i]) - for i in range(self.max_dp_size) - ] - self._refresh_active_workers() - return - if len(ranks.status) != self.max_dp_size: - logger.warning( - "[DPC] update_active_ranks: status len=%d != max_dp_size=%d; " - "ignoring update", - len(ranks.status), - self.max_dp_size, - ) - return - self.status = list(ranks.status) - - def add_elastic_workers(self, slot_offset: int, slot_count: int): - """Activate a range of pre-bound worker slots.""" - end = slot_offset + slot_count - if end > self.max_dp_size: - raise ValueError( - f"[Elastic EP] add_elastic_workers: slot_offset={slot_offset} + " - f"slot_count={slot_count} exceeds max_dp_size={self.max_dp_size}. " - f"Restart with a larger --max-ep-size." - ) - - for slot in range(slot_offset, end): - if self.dp_active[slot]: - logger.debug( - "[Elastic EP] add_elastic_workers: slot %d already active; " - "skipping", - slot, - ) - continue - assert self.workers[slot] is not None, ( - f"[Elastic EP] add_elastic_workers: slot {slot} was not " - f"pre-bound at launch; expected a primary-bound PUSH socket." - ) - self.dp_active[slot] = True - self.status[slot] = True - - self._refresh_active_workers() - logger.debug( - "[Elastic EP] DataParallelController activated slots %s " - "(active=%d / max=%d)", - list(range(slot_offset, end)), - self._active_count_cache, - self.max_dp_size, - ) - - def _refresh_active_workers(self) -> None: - self._active_workers = [ - i for i, active in enumerate(self.dp_active) if active and self.status[i] - ] - self._active_count_cache = len(self._active_workers) - - def refresh_load_budget(self): - # Throttle to at most once per 20ms. When a burst of requests - # arrives, dispatching_with_trace() calls this before every - # dispatch. Each call reads the latest scheduler snapshot and - # overwrites the speculative +1 increments that DPBudget.dispatch() - # added for previously dispatched requests in this burst. Without - # throttling, the budget resets to the (stale) scheduler-reported - # value on every request, causing the entire burst to land on a - # single DP rank. The 20ms interval lets the burst complete - # using speculative counters, then refreshes from the real - # scheduler load for the next batch. - now = time.perf_counter() - if now - self._last_refresh_time < 0.02: - return - self._last_refresh_time = now - self.dp_budget.update_budget(self.load_snapshot_reader.read_all()) - - def dispatching_with_trace(self, req: Req, refresh_load_budget: bool = True): - is_health_check = is_health_check_generate_req(req) - if ( - not is_health_check - and refresh_load_budget - and self.refresh_load_budget_on_dispatch - ): - self.refresh_load_budget() - - time_stats = DPControllerReqTimeStats.new_from_obj( - unwrap_from_pickle(req.time_stats) - ) - - time_stats.set_dp_dispatch_time() - req.time_stats = wrap_as_pickle(time_stats) - if is_health_check: - self.dispatch_health_check(req) - else: - self.dispatching(req) - req.time_stats = time_stats - req.time_stats.set_dp_dispatch_finish_time() - - def dispatch_batch_generate(self, batch_req: BatchTokenizedGenerateReqInput): - has_user_request = any( - not is_health_check_generate_req(req) for req in batch_req - ) - if has_user_request and self.refresh_load_budget_on_dispatch: - self.refresh_load_budget() - for req in batch_req: - self.dispatching_with_trace(req, refresh_load_budget=False) - - def dispatch_batch_embedding(self, batch_req: BatchTokenizedEmbeddingReqInput): - if self.refresh_load_budget_on_dispatch: - self.refresh_load_budget() - for req in batch_req: - self.dispatching_with_trace(req, refresh_load_budget=False) - - def init_dispatcher(self): - self._request_dispatcher = TypeBasedDispatcher( - [ - (TokenizedGenerateReqInput, self.dispatching_with_trace), - (TokenizedEmbeddingReqInput, self.dispatching_with_trace), - (BatchTokenizedGenerateReqInput, self.dispatch_batch_generate), - (BatchTokenizedEmbeddingReqInput, self.dispatch_batch_embedding), - (BlockReqInput, self.send_to_all_workers), - (ProfileReq, self.send_to_all_workers), - (ActiveRanksOutput, self.update_active_ranks), - ( - ElasticScaleUpdateReq, - lambda msg: self.add_elastic_workers( - msg.slot_offset, msg.slot_count - ), - ), - ] - ) - self._request_dispatcher.add_fallback_fn(self.send_control_message) - - def launch_dp_schedulers(self, server_args, port_args): - base_gpu_id = 0 - - threads = [] - sockets = [] - ready_events = [] - for dp_rank in range(get_parallel().num_dp_ranks): - tmp_port_args = PortArgs.init_new(server_args) - tmp_port_args.tokenizer_ipc_name = port_args.tokenizer_ipc_name - tmp_port_args.detokenizer_ipc_name = port_args.detokenizer_ipc_name - tmp_port_args.instance_id = port_args.instance_id - - # This port is checked free in PortArgs.init_new. - # We hold it first so that the next dp worker gets a different port - sockets.append(bind_port(tmp_port_args.nccl_port)) - - ready_event = threading.Event() - ready_events.append(ready_event) - - # Create a thread for each worker - thread = threading.Thread( - target=self.launch_tensor_parallel_group_thread, - args=(server_args, tmp_port_args, base_gpu_id, dp_rank, ready_event), - ) - threads.append(thread) - base_gpu_id += ( - get_parallel().tp_size - * get_parallel().pp_size - * get_device().gpu_id_step - ) - - if get_parallel().node_rank == 0: - self.workers[dp_rank] = get_zmq_socket( - self.context, - zmq.PUSH, - tmp_port_args.scheduler_input_ipc_name, - True, - ) - - # Free all sockets before starting the threads to launch TP workers - for sock in sockets: - sock.close() - - # Start all threads - for thread in threads: - thread.start() - for event in ready_events: - event.wait() - - def launch_tensor_parallel_group_thread( - self, - server_args: ServerArgs, - port_args: PortArgs, - base_gpu_id: int, - dp_rank: int, - ready_event: threading.Event, - ): - self.launch_tensor_parallel_group(server_args, port_args, base_gpu_id, dp_rank) - ready_event.set() - - # This thread cannot be closed because otherwise the `kill_itself_when_parent_died` - # function in scheduler.py will kill the scheduler. - while True: - time.sleep(30 * 24 * 3600) - - def _broadcast_worker_ports( - self, server_args: ServerArgs, worker_ports: list[int] | None = None - ) -> list[int]: - """Broadcast worker ports from node 0 to all other nodes. - - Node 0 acts as the server, waiting for all other nodes to connect and - sending them the pre-allocated worker ports. Other nodes act as clients, - connecting to node 0 to receive their copy of the worker ports. - - Args: - server_args: Server arguments containing node configuration. - worker_ports: Pre-allocated worker ports to broadcast. - - Returns: - List of worker ports (same on all nodes after broadcast). - """ - is_joiner = get_exec().moe.is_ep_scale_joiner - if get_parallel().dist_init_addr is None or is_joiner: - na = NetworkAddress( - get_serving().host or "127.0.0.1", - get_serving().port + DP_ATTENTION_HANDSHAKE_PORT_DELTA, - ) - else: - na = NetworkAddress.parse(get_parallel().dist_init_addr) - na = NetworkAddress(na.host, na.port + DP_ATTENTION_HANDSHAKE_PORT_DELTA) - endpoint = na.to_tcp() - - if get_parallel().node_rank == 0: - # Node 0: Broadcast worker ports to all other nodes - return self._broadcast_ports_as_server( - endpoint, get_parallel().nnodes - 1, worker_ports - ) - else: - # Other nodes: Receive worker ports from node 0 - return self._receive_ports_as_client(endpoint).worker_ports - - @staticmethod - def _make_worker_port_handshake(worker_ports: list[int]) -> WorkerPortHandshake: - parallel = get_parallel() - return WorkerPortHandshake( - worker_ports=worker_ports, - attn_tp_size=parallel.attn_tp_size, - attn_cp_size=parallel.attn_cp_size, - ) - - def _broadcast_ports_as_server( - self, endpoint: str, expected_clients: int, worker_ports: list[int] - ) -> list[int]: - """Broadcast worker ports to all client nodes.""" - logger.debug(f"Broadcasting worker ports to {expected_clients} client nodes") - logger.debug(f"Worker ports: {worker_ports}") - handshake = self._make_worker_port_handshake(worker_ports) - - rep_socket = get_zmq_socket(self.context, zmq.REP, endpoint, True) - - try: - connected_clients = 0 - while connected_clients < expected_clients: - # Wait for client handshake - client_rank = sock_recv(rep_socket) - logger.debug(f"Received handshake from node {client_rank}") - - # Send worker ports to client - sock_send(rep_socket, wrap_as_pickle(handshake)) - connected_clients += 1 - logger.debug( - f"Sent worker ports to {connected_clients}/{expected_clients} nodes" - ) - - logger.debug("Worker port broadcast completed") - return worker_ports - finally: - if get_exec().moe.elastic_ep_backend is None: - rep_socket.close() - else: - threading.Thread( - target=self._reply_ports_as_server, - args=(rep_socket, handshake), - daemon=True, - ).start() - - def _reply_ports_as_server( - self, rep_socket: zmq.Socket, handshake: WorkerPortHandshake - ): - """Serve pre-bound worker ports and topology to elastic joiners.""" - while True: - try: - client_rank = sock_recv(rep_socket) - except Exception: - logger.exception( - "Failed to recv/decode handshake in reply thread; continue" - ) - continue - logger.debug(f"Received handshake from node {client_rank}") - - # Send worker ports to client - sock_send(rep_socket, wrap_as_pickle(handshake)) - logger.debug(f"Sent worker ports to node {client_rank}") - - def _receive_ports_as_client(self, endpoint: str) -> WorkerPortHandshake: - """Receive worker ports from the server node.""" - logger.debug("Connecting to node 0 to receive worker ports") - node_rank = get_parallel().node_rank - - req_socket = get_zmq_socket(self.context, zmq.REQ, endpoint, False) - req_socket.setsockopt(zmq.RCVTIMEO, 600 * 1000) # 10 minute timeout - req_socket.setsockopt(zmq.SNDTIMEO, 600 * 1000) - - try: - # Send handshake with our node rank - sock_send(req_socket, wrap_as_pickle(str(node_rank))) - - # Receive worker ports - handshake = sock_recv(req_socket) - if not isinstance(handshake, WorkerPortHandshake): - raise RuntimeError("Invalid worker-port handshake from node 0") - logger.debug( - f"Received {len(handshake.worker_ports)} worker ports from node 0" - ) - return handshake - except zmq.Again: - logger.error("Timeout waiting for worker ports from node 0") - raise RuntimeError( - "Failed to receive worker ports from node 0 within timeout" - ) - finally: - req_socket.close() - - def launch_dp_attention_schedulers( - self, server_args: ServerArgs, port_args: PortArgs - ): - if get_parallel().dist_init_addr is None: - bind_host = "127.0.0.1" - else: - bind_host = NetworkAddress.parse(get_parallel().dist_init_addr).host - - worker_ports = [] - if get_exec().moe.is_ep_scale_joiner: - # Scale joiners connect to their pre-bound primary worker sockets. - primary = NetworkAddress.parse(get_parallel().dist_init_addr) - primary_endpoint = NetworkAddress( - primary.host, primary.port + DP_ATTENTION_HANDSHAKE_PORT_DELTA - ).to_tcp() - handshake = self._receive_ports_as_client(primary_endpoint) - parallel = get_parallel() - validate_matching_attention_topology( - primary_attn_tp_size=handshake.attn_tp_size, - primary_attn_cp_size=handshake.attn_cp_size, - joining_attn_tp_size=parallel.attn_tp_size, - joining_attn_cp_size=parallel.attn_cp_size, - ) - primary_replica_size = handshake.attn_tp_size * handshake.attn_cp_size - offset = physical_ep_rank_to_dp_rank( - parallel.ep_join_rank_offset, - primary_replica_size, - ) - broadcasted_ports = handshake.worker_ports[ - offset : offset + parallel.num_dp_ranks - ] - if len(broadcasted_ports) != parallel.num_dp_ranks: - raise ValueError( - "Primary has insufficient reserved attention-DP worker ports " - f"for join offset {parallel.ep_join_rank_offset} " - f"(requested {parallel.num_dp_ranks}, got " - f"{len(broadcasted_ports)})." - ) - elif get_parallel().node_rank == 0: - # Elastic primaries reserve sockets for the maximum DP size. - bind_count = ( - self.max_dp_size - if get_exec().moe.elastic_ep_backend is not None - else get_parallel().num_dp_ranks - ) - for slot in range(bind_count): - worker_port, worker_socket = get_zmq_socket_on_host( - self.context, zmq.PUSH, host=bind_host, restrict_peers=True - ) - worker_ports.append(worker_port) - self.workers[slot] = worker_socket - logger.debug( - "Assigned port %s to worker slot %s on host %s", - worker_port, - slot, - bind_host, - ) - broadcasted_ports = self._broadcast_worker_ports(server_args, worker_ports) - else: - broadcasted_ports = self._broadcast_worker_ports(server_args, None) - - self.launch_tensor_parallel_group( - server_args, port_args, 0, None, broadcasted_ports - ) - - def launch_tensor_parallel_group( - self, - server_args: ServerArgs, - port_args: PortArgs, - base_gpu_id: int, - dp_rank: int | None, - worker_ports: list[int] | None = None, - ): - if not get_parallel().attn_dp_enabled: - logger.info(f"Launch DP{dp_rank} starting at GPU #{base_gpu_id}.") - - memory_saver_adapter = TorchMemorySaverAdapter.create( - enable=get_exec().features.enable_memory_saver - ) - - scheduler_pipe_readers = [] - - pp_size_per_node = max(get_parallel().pp_size // get_parallel().nnodes, 1) - nnodes_per_pp_rank = max(get_parallel().nnodes // get_parallel().pp_size, 1) - pp_rank_range = range( - pp_size_per_node * (get_parallel().node_rank // nnodes_per_pp_rank), - pp_size_per_node * (get_parallel().node_rank // nnodes_per_pp_rank + 1), - ) - - nnodes_per_tp_group = nnodes_per_pp_rank - tp_size_per_node = get_parallel().tp_size // nnodes_per_tp_group - if get_exec().moe.is_ep_scale_joiner: - # Scale joiners enumerate their full local TP span. - tp_rank_range = range(get_parallel().tp_size) - tp_size_per_node = get_parallel().tp_size - else: - tp_rank_range = range( - tp_size_per_node * (get_parallel().node_rank % nnodes_per_tp_group), - tp_size_per_node * (get_parallel().node_rank % nnodes_per_tp_group + 1), - ) - - for pp_rank in pp_rank_range: - for tp_rank in tp_rank_range: - rank_port_args = port_args - - if get_parallel().attn_dp_enabled: - # dp attention has different sharding logic - _, _, dp_rank, _ = compute_dp_attention_world_info( - tp_rank, - get_parallel().tp_size, - get_parallel().attn_dp_size, - get_parallel().attn_cp_size, - ) - # compute zmq ports for this dp rank - rank_port_args = PortArgs.init_new( - server_args, dp_rank, worker_ports - ) - if get_exec().moe.is_ep_scale_joiner: - # Scale-joiner outputs return through the primary tokenizer. - primary_addr = NetworkAddress.parse( - get_parallel().dist_init_addr - ) - primary_port_base = primary_addr.port + 1 - rank_port_args.tokenizer_ipc_name = NetworkAddress( - primary_addr.host, primary_port_base - ).to_tcp() - rank_port_args.detokenizer_ipc_name = NetworkAddress( - primary_addr.host, primary_port_base + 1 - ).to_tcp() - # Data parallelism reuses the tensor parallelism group, - # so all dp ranks should use the same nccl port. - rank_port_args.nccl_port = port_args.nccl_port - rank_port_args.instance_id = port_args.instance_id - - gpu_id = ( - get_device().base_gpu_id - + base_gpu_id - + ((pp_rank % pp_size_per_node) * tp_size_per_node) - + (tp_rank % tp_size_per_node) * get_device().gpu_id_step - ) - # Derive the child's EP rank for its display label. - moe_ep_rank = ( - tp_rank - % (get_parallel().tp_size // get_parallel().moe_dp_size) - // ( - get_parallel().tp_size - // get_parallel().moe_dp_size - // get_parallel().ep_size - ) - ) - - # Scheduler internals use local ranks; logs use global ranks. - offset = get_parallel().ep_join_rank_offset - dp_offset = physical_ep_rank_to_dp_rank(offset, self.attn_replica_size) - display_tp_rank = tp_rank + offset - display_moe_ep_rank = moe_ep_rank + offset - display_dp_rank = dp_rank + dp_offset if dp_rank is not None else None - - with self.env_lock, maybe_reindex_device_id(gpu_id) as gpu_id: - scheduler_args = ( - server_args, - rank_port_args, - gpu_id, - tp_rank, - pp_rank, - dp_rank, - ) - display_ranks = ( - display_tp_rank, - display_dp_rank, - display_moe_ep_rank, - ) - if self.prelaunched_schedulers is not None: - _, reader = self.prelaunched_schedulers[pp_rank, tp_rank] - reader.send((scheduler_args, display_ranks)) - else: - reader, writer = mp.Pipe(duplex=False) - proc = mp.Process( - target=self.run_scheduler_process_func, - args=(*scheduler_args, writer, *display_ranks), - ) - with ( - memory_saver_adapter.configure_subprocess(), - numa_utils.configure_subprocess(server_args, gpu_id), - ): - proc.start() - self.scheduler_procs.append(proc) - scheduler_pipe_readers.append(reader) - - # Wait for model to finish loading - scheduler_info = [] - for i in range(len(scheduler_pipe_readers)): - scheduler_info.append(scheduler_pipe_readers[i].recv()) - - # Pure-DP TP groups launch concurrently. Their ready replies contain - # only publishers owned by schedulers on this node. - with self.env_lock: - self.local_kv_event_sources.extend( - source - for info in scheduler_info - for source in info.get("kv_event_sources", []) - ) - - self.max_total_num_tokens = scheduler_info[0]["max_total_num_tokens"] - self.max_req_input_len = scheduler_info[0]["max_req_input_len"] - self.startup_time = aggregate_scheduler_startup_times( - info.get("startup_time") for info in scheduler_info - ) - - def dispatch_health_check(self, req: Req) -> None: - # This policy routes by request metadata, not shared load-balancing state. - if self.load_balance_method == LoadBalanceMethod.FOLLOW_BOOTSTRAP_ROOM: - self.follow_bootstrap_room_scheduler(req) - return - - if self.maybe_external_dp_rank_routing(req): - return - - active = self._active_workers - if not active: - raise RuntimeError("No active DP workers are available for health checks.") - - slot = active[self.health_round_robin_counter % len(active)] - self.health_round_robin_counter = (self.health_round_robin_counter + 1) % len( - active - ) - logger.debug(f"Choose worker {slot} for health check") - sock_send(self.workers[slot], req) - - def maybe_external_dp_rank_routing(self, req: Req): - if req.routed_dp_rank is not None: - rank = req.routed_dp_rank - if ( - rank < 0 - or rank >= len(self.workers) - or rank not in self._active_workers - or self.workers[rank] is None - ): - raise ValueError(f"DP rank {rank} is not active.") - logger.debug(f"Direct routing to DP rank {rank}") - sock_send(self.workers[rank], req) - return True - return False - - def round_robin_scheduler(self, req: Req): - if self.maybe_external_dp_rank_routing(req): - return - - active = self._active_workers - if not active: - raise RuntimeError("No active DP workers are available for routing.") - attempts = 0 - while attempts < len(active): - slot = active[self.round_robin_counter % len(active)] - self.round_robin_counter = (self.round_robin_counter + 1) % len(active) - if self.status[slot]: - logger.debug(f"Choose worker {slot}") - sock_send(self.workers[slot], req) - return - attempts += 1 - raise RuntimeError( - f"Cannot route request: all {len(active)} active DP workers " - "are unavailable." - ) - - def follow_bootstrap_room_scheduler(self, req: Req): - if self.maybe_external_dp_rank_routing(req): - return - - assert req.bootstrap_room is not None, ( - "req.bootstrap_room should not be None. Do not send requests directly to " - "prefill or decode instances; send to the router instead." - ) - target_rank = req.bootstrap_room % len(self.workers) - sock_send(self.workers[target_rank], req) - - def total_requests_scheduler(self, req: Req): - if self.maybe_external_dp_rank_routing(req): - return - target_worker = self.dp_budget.dispatch(LoadBalanceMethod.TOTAL_REQUESTS) - sock_send(self.workers[target_worker], req) - - def total_tokens_scheduler(self, req: Req): - if self.maybe_external_dp_rank_routing(req): - return - estimated_tokens = len(req.input_ids) - target_worker = self.dp_budget.dispatch( - LoadBalanceMethod.TOTAL_TOKENS, estimated_tokens=estimated_tokens - ) - sock_send(self.workers[target_worker], req) - - def event_loop(self): - while True: - while True: - self.soft_watchdog.feed() - try: - recv_req = sock_recv(self.recv_from_tokenizer, flags=zmq.NOBLOCK) - except zmq.ZMQError: - break - self._request_dispatcher(recv_req) - - -def run_scheduler_process_with_init_pipe(run_scheduler_process_func: Callable, pipe): - # Unpickling the callable imports the scheduler before waiting for DP setup. - kill_itself_when_parent_died() - try: - scheduler_args, display_ranks = pipe.recv() - run_scheduler_process_func(*scheduler_args, pipe, *display_ranks) - finally: - pipe.close() - - -def run_data_parallel_controller_process( - server_args: ServerArgs, - port_args: PortArgs, - pipe_writer, - run_scheduler_process_func: Callable = run_scheduler_process, - prelaunched_schedulers: dict[tuple[int, int], tuple[int, Connection]] | None = None, -): - setproctitle.setproctitle("sglang::data_parallel_controller") - faulthandler.enable() - kill_itself_when_parent_died() - parent_process = psutil.Process().parent() - - load_plugins() - # This process reads the config namespaces before spawning schedulers. - publish(server_args, role="dp_controller") - configure_logger(server_args) - if get_observability().enable_trace: - process_tracing_init( - get_observability().otlp_traces_endpoint, - get_observability().otlp_service_name, - trace_modules=get_observability().trace_modules, - ) - thread_label = "DP Controller" - if get_disagg().disaggregation_mode == "prefill": - thread_label = "Prefill DP Controller" - elif get_disagg().disaggregation_mode == "decode": - thread_label = "Decode DP Controller" - trace_set_thread_info(thread_label) - - try: - controller = DataParallelController( - server_args, port_args, run_scheduler_process_func, prelaunched_schedulers - ) - scheduler_pids = ( - [pid for pid, _ in prelaunched_schedulers.values()] - if prelaunched_schedulers is not None - else [proc.pid for proc in controller.scheduler_procs if proc is not None] - ) - init_info = { - "status": "ready", - "max_total_num_tokens": controller.max_total_num_tokens, - "max_req_input_len": controller.max_req_input_len, - "startup_time": controller.startup_time, - SCHEDULER_PIDS_ARG: scheduler_pids, - } - if get_serving().grpc_port is not None and not ( - get_serving().smg_grpc_mode or get_serving().grpc_mode - ): - init_info["kv_event_sources"] = sorted( - controller.local_kv_event_sources, key=lambda source: source["dp_rank"] - ) - pipe_writer.send(init_info) - # The primary owns routing for the expanded scheduler set. - if get_parallel().node_rank == 0 and not get_exec().moe.is_ep_scale_joiner: - controller.event_loop() - if prelaunched_schedulers is not None: - # The launcher owns these processes. Their pipes close when they exit. - wait([pipe for _, pipe in prelaunched_schedulers.values()]) - for proc in controller.scheduler_procs: - proc.join() - logger.error( - f"Scheduler or DataParallelController {proc.pid} terminated with {proc.exitcode}" - ) - except Exception: - traceback = get_exception_traceback() - logger.error(f"DataParallelController hit an exception: {traceback}") - parent_process.send_signal(signal.SIGQUIT) +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""A controller that dispatches requests to multiple data parallel workers.""" + +import faulthandler +import logging +import multiprocessing as mp +import signal +import threading +import time +from collections.abc import Callable +from dataclasses import dataclass +from enum import Enum, auto +from multiprocessing.connection import Connection, wait + +import psutil +import setproctitle +import zmq + +from sglang.srt.elastic_ep.topology import ( + attn_replica_size, + physical_ep_rank_to_dp_rank, + physical_ep_size_to_dp_size, + validate_matching_attention_topology, +) +from sglang.srt.environ import envs +from sglang.srt.layers.dp_attention import compute_dp_attention_world_info +from sglang.srt.managers.io_struct import ( + ActiveRanksOutput, + BatchTokenizedEmbeddingReqInput, + BatchTokenizedGenerateReqInput, + BlockReqInput, + ElasticScaleUpdateReq, + ProfileReq, + TokenizedEmbeddingReqInput, + TokenizedGenerateReqInput, + sock_recv, + sock_send, + unwrap_from_pickle, + wrap_as_pickle, +) +from sglang.srt.managers.load_snapshot import create_load_snapshot_reader +from sglang.srt.managers.schedule_batch import Req +from sglang.srt.managers.scheduler import run_scheduler_process +from sglang.srt.managers.utils import is_health_check_generate_req +from sglang.srt.observability.cpu_monitor import start_cpu_monitor_thread +from sglang.srt.observability.req_time_stats import DPControllerReqTimeStats +from sglang.srt.observability.startup_time import aggregate_scheduler_startup_times +from sglang.srt.observability.trace import process_tracing_init, trace_set_thread_info +from sglang.srt.plugins import load_plugins +from sglang.srt.runtime_context import ( + get_device, + get_disagg, + get_exec, + get_observability, + get_parallel, + get_serving, + publish, +) +from sglang.srt.server_args import ( + DP_ATTENTION_HANDSHAKE_PORT_DELTA, + PortArgs, + ServerArgs, +) +from sglang.srt.utils import numa_utils +from sglang.srt.utils.common import ( + configure_logger, + kill_itself_when_parent_died, + maybe_reindex_device_id, +) +from sglang.srt.utils.network import ( + NetworkAddress, + bind_port, + get_zmq_socket, + get_zmq_socket_on_host, +) +from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter +from sglang.srt.utils.watchdog import Watchdog +from sglang.utils import TypeBasedDispatcher, get_exception_traceback + +logger = logging.getLogger(__name__) + +SCHEDULER_PIDS_ARG = "scheduler_pids" + + +@dataclass(frozen=True) +class WorkerPortHandshake: + worker_ports: list[int] + attn_tp_size: int + attn_cp_size: int + + +class LoadBalanceMethod(Enum): + """Load balance method.""" + + ROUND_ROBIN = auto() + FOLLOW_BOOTSTRAP_ROOM = auto() + TOTAL_REQUESTS = auto() + TOTAL_TOKENS = auto() + UNCACHED_TOKENS = auto() + + @classmethod + def from_str(cls, method: str): + method = method.upper() + try: + return cls[method] + except KeyError as exc: + raise ValueError(f"Invalid load balance method: {method}") from exc + + +class DPBudget: + def __init__(self, num_dp_ranks: int): + self.num_dp_ranks = num_dp_ranks + self.total_requests = [0] * num_dp_ranks + self.total_tokens = [0] * num_dp_ranks + self.total_uncached_tokens = [0] * num_dp_ranks + self.cache_hit_rate = [0.0] * num_dp_ranks + self.last_timestamp = [0.0] * num_dp_ranks + + def update_budget(self, loads): + """Update budget from shm snapshots, skipping stale reads.""" + for load in loads: + if load.timestamp == self.last_timestamp[load.dp_rank]: + continue + self.last_timestamp[load.dp_rank] = load.timestamp + self.total_requests[load.dp_rank] = ( + load.num_running_reqs + load.num_waiting_reqs + ) + self.total_tokens[load.dp_rank] = load.num_total_tokens + self.total_uncached_tokens[load.dp_rank] = ( + load.num_waiting_uncached_tokens + ) + self.cache_hit_rate[load.dp_rank] = load.cache_hit_rate + + def dispatch( + self, + method: LoadBalanceMethod, + estimated_tokens: int = 0, + num_input_tokens: int = 0, + ): + if method == LoadBalanceMethod.TOTAL_REQUESTS: + target_rank = self.total_requests.index(min(self.total_requests)) + elif method == LoadBalanceMethod.TOTAL_TOKENS: + # Use total_requests as a tie-breaker when total_tokens are equal + target_rank = min( + range(self.num_dp_ranks), + key=lambda i: (self.total_tokens[i], self.total_requests[i]), + ) + elif method == LoadBalanceMethod.UNCACHED_TOKENS: + # Charge every candidate its own estimated uncached share: the same + # prompt costs less on a rank whose cache already holds more of it. + # The exact prefix match is only known inside the scheduler, so the + # rank's recent cache hit rate stands in for it. + estimated_uncached = [ + int(num_input_tokens * (1.0 - self.cache_hit_rate[i])) + for i in range(self.num_dp_ranks) + ] + target_rank = min( + range(self.num_dp_ranks), + key=lambda i: ( + self.total_uncached_tokens[i] + estimated_uncached[i], + self.total_requests[i], + ), + ) + self.total_requests[target_rank] += 1 + self.total_uncached_tokens[target_rank] += estimated_uncached[target_rank] + return target_rank + else: + return None + + # Increment the load of that worker by one as a heuristic + self.total_requests[target_rank] += 1 + self.total_tokens[target_rank] += estimated_tokens + return target_rank + + +class DataParallelController: + """A controller that dispatches requests to multiple data parallel workers.""" + + def __init__( + self, + server_args: ServerArgs, + port_args: PortArgs, + run_scheduler_process_func: Callable, + prelaunched_schedulers: ( + dict[tuple[int, int], tuple[int, Connection]] | None + ) = None, + ) -> None: + # Parse args + self.server_args = server_args + self.port_args = port_args + self.load_balance_method = LoadBalanceMethod.from_str( + get_parallel().load_balance_method + ) + self.run_scheduler_process_func = run_scheduler_process_func + self.prelaunched_schedulers = prelaunched_schedulers + + # Init inter-process communication + self.context = zmq.Context(1 + get_parallel().num_dp_ranks) + if get_parallel().node_rank == 0: + self.recv_from_tokenizer = get_zmq_socket( + self.context, zmq.PULL, port_args.scheduler_input_ipc_name, False + ) + + # Dispatch method + self.round_robin_counter = 0 + self.health_round_robin_counter = 0 + dispatch_lookup = { + LoadBalanceMethod.ROUND_ROBIN: self.round_robin_scheduler, + LoadBalanceMethod.FOLLOW_BOOTSTRAP_ROOM: self.follow_bootstrap_room_scheduler, + LoadBalanceMethod.TOTAL_REQUESTS: self.total_requests_scheduler, + LoadBalanceMethod.TOTAL_TOKENS: self.total_tokens_scheduler, + LoadBalanceMethod.UNCACHED_TOKENS: ( + self.total_uncached_tokens_scheduler + ), + } + self.dispatching = dispatch_lookup[self.load_balance_method] + self.refresh_load_budget_on_dispatch = self.load_balance_method in ( + LoadBalanceMethod.TOTAL_REQUESTS, + LoadBalanceMethod.TOTAL_TOKENS, + LoadBalanceMethod.UNCACHED_TOKENS, + ) + + parallel = get_parallel() + self.launch_dp_size: int = parallel.num_dp_ranks + self.attn_replica_size = attn_replica_size() + if ( + get_exec().moe.elastic_ep_backend is not None + and parallel.max_ep_size is not None + and parallel.max_ep_size > parallel.tp_size + ): + self.max_dp_size = physical_ep_size_to_dp_size( + parallel.max_ep_size, + self.attn_replica_size, + ) + else: + self.max_dp_size = parallel.num_dp_ranks + assert self.max_dp_size >= self.launch_dp_size, ( + f"maximum DP size ({self.max_dp_size}) must be >= " + f"--dp ({self.launch_dp_size})." + ) + + self.dp_active: list[bool] = [True] * self.launch_dp_size + [False] * ( + self.max_dp_size - self.launch_dp_size + ) + + self.dp_budget = DPBudget(get_parallel().num_dp_ranks) + self.load_snapshot_reader = create_load_snapshot_reader( + port_args, + caller="DataParallelController", + ) + self._last_refresh_time = 0.0 + + # To protect changing env vars to set CUDA_VISIBLE_DEVICES. + self.env_lock = threading.Lock() + + # Launch data parallel workers + self.scheduler_procs = [] + self.local_kv_event_sources = [] + self.workers: list[zmq.Socket | None] = [None] * self.max_dp_size + self.status: list[bool] = list(self.dp_active) + self._active_workers: list[int] = list(range(self.launch_dp_size)) + self._active_count_cache: int = self.launch_dp_size + + if get_parallel().attn_dp_enabled: + self.launch_dp_attention_schedulers(server_args, port_args) + # When local control broadcast is enabled, send control messages to + # every DP group leader (attn_tp_rank=0) so each leader broadcasts + # within its own attn_tp_group instead of the full tp_group. + # Otherwise fall back to the original behaviour: send to only the + # first leader, which then broadcasts over the full tp_group. + local_ctrl = get_parallel().enable_dp_attention_local_control_broadcast + self.control_message_step = 1 if local_ctrl else get_parallel().tp_size + else: + self.launch_dp_schedulers(server_args, port_args) + self.control_message_step = 1 + + self.init_dispatcher() + + self.soft_watchdog = Watchdog.create( + debug_name="DataParallelController", + watchdog_timeout=get_device().soft_watchdog_timeout, + soft=True, + test_stuck_time=envs.SGLANG_TEST_STUCK_DP_CONTROLLER.get(), + ) + + if get_observability().enable_metrics: + start_cpu_monitor_thread("data_parallel_controller") + + def send_to_all_workers(self, obj): + for i, worker in enumerate(self.workers): + if worker is not None and self.status[i]: + sock_send(worker, obj) + + def send_control_message(self, obj): + for i in self._active_workers[:: self.control_message_step]: + worker = self.workers[i] + if worker is not None: + sock_send(worker, obj) + + def update_active_ranks(self, ranks: ActiveRanksOutput): + if get_exec().moe.elastic_ep_backend is not None: + if len(ranks.status) != self.max_dp_size: + logger.warning( + "[Elastic EP][DPC] active rank status len=%d != max_dp_size=%d; " + "ignoring update", + len(ranks.status), + self.max_dp_size, + ) + return + self.status = [ + self.dp_active[i] and bool(ranks.status[i]) + for i in range(self.max_dp_size) + ] + self._refresh_active_workers() + return + if len(ranks.status) != self.max_dp_size: + logger.warning( + "[DPC] update_active_ranks: status len=%d != max_dp_size=%d; " + "ignoring update", + len(ranks.status), + self.max_dp_size, + ) + return + self.status = list(ranks.status) + + def add_elastic_workers(self, slot_offset: int, slot_count: int): + """Activate a range of pre-bound worker slots.""" + end = slot_offset + slot_count + if end > self.max_dp_size: + raise ValueError( + f"[Elastic EP] add_elastic_workers: slot_offset={slot_offset} + " + f"slot_count={slot_count} exceeds max_dp_size={self.max_dp_size}. " + f"Restart with a larger --max-ep-size." + ) + + for slot in range(slot_offset, end): + if self.dp_active[slot]: + logger.debug( + "[Elastic EP] add_elastic_workers: slot %d already active; " + "skipping", + slot, + ) + continue + assert self.workers[slot] is not None, ( + f"[Elastic EP] add_elastic_workers: slot {slot} was not " + f"pre-bound at launch; expected a primary-bound PUSH socket." + ) + self.dp_active[slot] = True + self.status[slot] = True + + self._refresh_active_workers() + logger.debug( + "[Elastic EP] DataParallelController activated slots %s " + "(active=%d / max=%d)", + list(range(slot_offset, end)), + self._active_count_cache, + self.max_dp_size, + ) + + def _refresh_active_workers(self) -> None: + self._active_workers = [ + i for i, active in enumerate(self.dp_active) if active and self.status[i] + ] + self._active_count_cache = len(self._active_workers) + + def refresh_load_budget(self): + # Throttle to at most once per 20ms. When a burst of requests + # arrives, dispatching_with_trace() calls this before every + # dispatch. Each call reads the latest scheduler snapshot and + # overwrites the speculative +1 increments that DPBudget.dispatch() + # added for previously dispatched requests in this burst. Without + # throttling, the budget resets to the (stale) scheduler-reported + # value on every request, causing the entire burst to land on a + # single DP rank. The 20ms interval lets the burst complete + # using speculative counters, then refreshes from the real + # scheduler load for the next batch. + now = time.perf_counter() + if now - self._last_refresh_time < 0.02: + return + self._last_refresh_time = now + self.dp_budget.update_budget(self.load_snapshot_reader.read_all()) + + def dispatching_with_trace(self, req: Req, refresh_load_budget: bool = True): + is_health_check = is_health_check_generate_req(req) + if ( + not is_health_check + and refresh_load_budget + and self.refresh_load_budget_on_dispatch + ): + self.refresh_load_budget() + + time_stats = DPControllerReqTimeStats.new_from_obj( + unwrap_from_pickle(req.time_stats) + ) + + time_stats.set_dp_dispatch_time() + req.time_stats = wrap_as_pickle(time_stats) + if is_health_check: + self.dispatch_health_check(req) + else: + self.dispatching(req) + req.time_stats = time_stats + req.time_stats.set_dp_dispatch_finish_time() + + def dispatch_batch_generate(self, batch_req: BatchTokenizedGenerateReqInput): + has_user_request = any( + not is_health_check_generate_req(req) for req in batch_req + ) + if has_user_request and self.refresh_load_budget_on_dispatch: + self.refresh_load_budget() + for req in batch_req: + self.dispatching_with_trace(req, refresh_load_budget=False) + + def dispatch_batch_embedding(self, batch_req: BatchTokenizedEmbeddingReqInput): + if self.refresh_load_budget_on_dispatch: + self.refresh_load_budget() + for req in batch_req: + self.dispatching_with_trace(req, refresh_load_budget=False) + + def init_dispatcher(self): + self._request_dispatcher = TypeBasedDispatcher( + [ + (TokenizedGenerateReqInput, self.dispatching_with_trace), + (TokenizedEmbeddingReqInput, self.dispatching_with_trace), + (BatchTokenizedGenerateReqInput, self.dispatch_batch_generate), + (BatchTokenizedEmbeddingReqInput, self.dispatch_batch_embedding), + (BlockReqInput, self.send_to_all_workers), + (ProfileReq, self.send_to_all_workers), + (ActiveRanksOutput, self.update_active_ranks), + ( + ElasticScaleUpdateReq, + lambda msg: self.add_elastic_workers( + msg.slot_offset, msg.slot_count + ), + ), + ] + ) + self._request_dispatcher.add_fallback_fn(self.send_control_message) + + def launch_dp_schedulers(self, server_args, port_args): + base_gpu_id = 0 + + threads = [] + sockets = [] + ready_events = [] + for dp_rank in range(get_parallel().num_dp_ranks): + tmp_port_args = PortArgs.init_new(server_args) + tmp_port_args.tokenizer_ipc_name = port_args.tokenizer_ipc_name + tmp_port_args.detokenizer_ipc_name = port_args.detokenizer_ipc_name + tmp_port_args.instance_id = port_args.instance_id + + # This port is checked free in PortArgs.init_new. + # We hold it first so that the next dp worker gets a different port + sockets.append(bind_port(tmp_port_args.nccl_port)) + + ready_event = threading.Event() + ready_events.append(ready_event) + + # Create a thread for each worker + thread = threading.Thread( + target=self.launch_tensor_parallel_group_thread, + args=(server_args, tmp_port_args, base_gpu_id, dp_rank, ready_event), + ) + threads.append(thread) + base_gpu_id += ( + get_parallel().tp_size + * get_parallel().pp_size + * get_device().gpu_id_step + ) + + if get_parallel().node_rank == 0: + self.workers[dp_rank] = get_zmq_socket( + self.context, + zmq.PUSH, + tmp_port_args.scheduler_input_ipc_name, + True, + ) + + # Free all sockets before starting the threads to launch TP workers + for sock in sockets: + sock.close() + + # Start all threads + for thread in threads: + thread.start() + for event in ready_events: + event.wait() + + def launch_tensor_parallel_group_thread( + self, + server_args: ServerArgs, + port_args: PortArgs, + base_gpu_id: int, + dp_rank: int, + ready_event: threading.Event, + ): + self.launch_tensor_parallel_group(server_args, port_args, base_gpu_id, dp_rank) + ready_event.set() + + # This thread cannot be closed because otherwise the `kill_itself_when_parent_died` + # function in scheduler.py will kill the scheduler. + while True: + time.sleep(30 * 24 * 3600) + + def _broadcast_worker_ports( + self, server_args: ServerArgs, worker_ports: list[int] | None = None + ) -> list[int]: + """Broadcast worker ports from node 0 to all other nodes. + + Node 0 acts as the server, waiting for all other nodes to connect and + sending them the pre-allocated worker ports. Other nodes act as clients, + connecting to node 0 to receive their copy of the worker ports. + + Args: + server_args: Server arguments containing node configuration. + worker_ports: Pre-allocated worker ports to broadcast. + + Returns: + List of worker ports (same on all nodes after broadcast). + """ + is_joiner = get_exec().moe.is_ep_scale_joiner + if get_parallel().dist_init_addr is None or is_joiner: + na = NetworkAddress( + get_serving().host or "127.0.0.1", + get_serving().port + DP_ATTENTION_HANDSHAKE_PORT_DELTA, + ) + else: + na = NetworkAddress.parse(get_parallel().dist_init_addr) + na = NetworkAddress(na.host, na.port + DP_ATTENTION_HANDSHAKE_PORT_DELTA) + endpoint = na.to_tcp() + + if get_parallel().node_rank == 0: + # Node 0: Broadcast worker ports to all other nodes + return self._broadcast_ports_as_server( + endpoint, get_parallel().nnodes - 1, worker_ports + ) + else: + # Other nodes: Receive worker ports from node 0 + return self._receive_ports_as_client(endpoint).worker_ports + + @staticmethod + def _make_worker_port_handshake(worker_ports: list[int]) -> WorkerPortHandshake: + parallel = get_parallel() + return WorkerPortHandshake( + worker_ports=worker_ports, + attn_tp_size=parallel.attn_tp_size, + attn_cp_size=parallel.attn_cp_size, + ) + + def _broadcast_ports_as_server( + self, endpoint: str, expected_clients: int, worker_ports: list[int] + ) -> list[int]: + """Broadcast worker ports to all client nodes.""" + logger.debug(f"Broadcasting worker ports to {expected_clients} client nodes") + logger.debug(f"Worker ports: {worker_ports}") + handshake = self._make_worker_port_handshake(worker_ports) + + rep_socket = get_zmq_socket(self.context, zmq.REP, endpoint, True) + + try: + connected_clients = 0 + while connected_clients < expected_clients: + # Wait for client handshake + client_rank = sock_recv(rep_socket) + logger.debug(f"Received handshake from node {client_rank}") + + # Send worker ports to client + sock_send(rep_socket, wrap_as_pickle(handshake)) + connected_clients += 1 + logger.debug( + f"Sent worker ports to {connected_clients}/{expected_clients} nodes" + ) + + logger.debug("Worker port broadcast completed") + return worker_ports + finally: + if get_exec().moe.elastic_ep_backend is None: + rep_socket.close() + else: + threading.Thread( + target=self._reply_ports_as_server, + args=(rep_socket, handshake), + daemon=True, + ).start() + + def _reply_ports_as_server( + self, rep_socket: zmq.Socket, handshake: WorkerPortHandshake + ): + """Serve pre-bound worker ports and topology to elastic joiners.""" + while True: + try: + client_rank = sock_recv(rep_socket) + except Exception: + logger.exception( + "Failed to recv/decode handshake in reply thread; continue" + ) + continue + logger.debug(f"Received handshake from node {client_rank}") + + # Send worker ports to client + sock_send(rep_socket, wrap_as_pickle(handshake)) + logger.debug(f"Sent worker ports to node {client_rank}") + + def _receive_ports_as_client(self, endpoint: str) -> WorkerPortHandshake: + """Receive worker ports from the server node.""" + logger.debug("Connecting to node 0 to receive worker ports") + node_rank = get_parallel().node_rank + + req_socket = get_zmq_socket(self.context, zmq.REQ, endpoint, False) + req_socket.setsockopt(zmq.RCVTIMEO, 600 * 1000) # 10 minute timeout + req_socket.setsockopt(zmq.SNDTIMEO, 600 * 1000) + + try: + # Send handshake with our node rank + sock_send(req_socket, wrap_as_pickle(str(node_rank))) + + # Receive worker ports + handshake = sock_recv(req_socket) + if not isinstance(handshake, WorkerPortHandshake): + raise RuntimeError("Invalid worker-port handshake from node 0") + logger.debug( + f"Received {len(handshake.worker_ports)} worker ports from node 0" + ) + return handshake + except zmq.Again: + logger.error("Timeout waiting for worker ports from node 0") + raise RuntimeError( + "Failed to receive worker ports from node 0 within timeout" + ) + finally: + req_socket.close() + + def launch_dp_attention_schedulers( + self, server_args: ServerArgs, port_args: PortArgs + ): + if get_parallel().dist_init_addr is None: + bind_host = "127.0.0.1" + else: + bind_host = NetworkAddress.parse(get_parallel().dist_init_addr).host + + worker_ports = [] + if get_exec().moe.is_ep_scale_joiner: + # Scale joiners connect to their pre-bound primary worker sockets. + primary = NetworkAddress.parse(get_parallel().dist_init_addr) + primary_endpoint = NetworkAddress( + primary.host, primary.port + DP_ATTENTION_HANDSHAKE_PORT_DELTA + ).to_tcp() + handshake = self._receive_ports_as_client(primary_endpoint) + parallel = get_parallel() + validate_matching_attention_topology( + primary_attn_tp_size=handshake.attn_tp_size, + primary_attn_cp_size=handshake.attn_cp_size, + joining_attn_tp_size=parallel.attn_tp_size, + joining_attn_cp_size=parallel.attn_cp_size, + ) + primary_replica_size = handshake.attn_tp_size * handshake.attn_cp_size + offset = physical_ep_rank_to_dp_rank( + parallel.ep_join_rank_offset, + primary_replica_size, + ) + broadcasted_ports = handshake.worker_ports[ + offset : offset + parallel.num_dp_ranks + ] + if len(broadcasted_ports) != parallel.num_dp_ranks: + raise ValueError( + "Primary has insufficient reserved attention-DP worker ports " + f"for join offset {parallel.ep_join_rank_offset} " + f"(requested {parallel.num_dp_ranks}, got " + f"{len(broadcasted_ports)})." + ) + elif get_parallel().node_rank == 0: + # Elastic primaries reserve sockets for the maximum DP size. + bind_count = ( + self.max_dp_size + if get_exec().moe.elastic_ep_backend is not None + else get_parallel().num_dp_ranks + ) + for slot in range(bind_count): + worker_port, worker_socket = get_zmq_socket_on_host( + self.context, zmq.PUSH, host=bind_host, restrict_peers=True + ) + worker_ports.append(worker_port) + self.workers[slot] = worker_socket + logger.debug( + "Assigned port %s to worker slot %s on host %s", + worker_port, + slot, + bind_host, + ) + broadcasted_ports = self._broadcast_worker_ports(server_args, worker_ports) + else: + broadcasted_ports = self._broadcast_worker_ports(server_args, None) + + self.launch_tensor_parallel_group( + server_args, port_args, 0, None, broadcasted_ports + ) + + def launch_tensor_parallel_group( + self, + server_args: ServerArgs, + port_args: PortArgs, + base_gpu_id: int, + dp_rank: int | None, + worker_ports: list[int] | None = None, + ): + if not get_parallel().attn_dp_enabled: + logger.info(f"Launch DP{dp_rank} starting at GPU #{base_gpu_id}.") + + memory_saver_adapter = TorchMemorySaverAdapter.create( + enable=get_exec().features.enable_memory_saver + ) + + scheduler_pipe_readers = [] + + pp_size_per_node = max(get_parallel().pp_size // get_parallel().nnodes, 1) + nnodes_per_pp_rank = max(get_parallel().nnodes // get_parallel().pp_size, 1) + pp_rank_range = range( + pp_size_per_node * (get_parallel().node_rank // nnodes_per_pp_rank), + pp_size_per_node * (get_parallel().node_rank // nnodes_per_pp_rank + 1), + ) + + nnodes_per_tp_group = nnodes_per_pp_rank + tp_size_per_node = get_parallel().tp_size // nnodes_per_tp_group + if get_exec().moe.is_ep_scale_joiner: + # Scale joiners enumerate their full local TP span. + tp_rank_range = range(get_parallel().tp_size) + tp_size_per_node = get_parallel().tp_size + else: + tp_rank_range = range( + tp_size_per_node * (get_parallel().node_rank % nnodes_per_tp_group), + tp_size_per_node * (get_parallel().node_rank % nnodes_per_tp_group + 1), + ) + + for pp_rank in pp_rank_range: + for tp_rank in tp_rank_range: + rank_port_args = port_args + + if get_parallel().attn_dp_enabled: + # dp attention has different sharding logic + _, _, dp_rank, _ = compute_dp_attention_world_info( + tp_rank, + get_parallel().tp_size, + get_parallel().attn_dp_size, + get_parallel().attn_cp_size, + ) + # compute zmq ports for this dp rank + rank_port_args = PortArgs.init_new( + server_args, dp_rank, worker_ports + ) + if get_exec().moe.is_ep_scale_joiner: + # Scale-joiner outputs return through the primary tokenizer. + primary_addr = NetworkAddress.parse( + get_parallel().dist_init_addr + ) + primary_port_base = primary_addr.port + 1 + rank_port_args.tokenizer_ipc_name = NetworkAddress( + primary_addr.host, primary_port_base + ).to_tcp() + rank_port_args.detokenizer_ipc_name = NetworkAddress( + primary_addr.host, primary_port_base + 1 + ).to_tcp() + # Data parallelism reuses the tensor parallelism group, + # so all dp ranks should use the same nccl port. + rank_port_args.nccl_port = port_args.nccl_port + rank_port_args.instance_id = port_args.instance_id + + gpu_id = ( + get_device().base_gpu_id + + base_gpu_id + + ((pp_rank % pp_size_per_node) * tp_size_per_node) + + (tp_rank % tp_size_per_node) * get_device().gpu_id_step + ) + # Derive the child's EP rank for its display label. + moe_ep_rank = ( + tp_rank + % (get_parallel().tp_size // get_parallel().moe_dp_size) + // ( + get_parallel().tp_size + // get_parallel().moe_dp_size + // get_parallel().ep_size + ) + ) + + # Scheduler internals use local ranks; logs use global ranks. + offset = get_parallel().ep_join_rank_offset + dp_offset = physical_ep_rank_to_dp_rank(offset, self.attn_replica_size) + display_tp_rank = tp_rank + offset + display_moe_ep_rank = moe_ep_rank + offset + display_dp_rank = dp_rank + dp_offset if dp_rank is not None else None + + with self.env_lock, maybe_reindex_device_id(gpu_id) as gpu_id: + scheduler_args = ( + server_args, + rank_port_args, + gpu_id, + tp_rank, + pp_rank, + dp_rank, + ) + display_ranks = ( + display_tp_rank, + display_dp_rank, + display_moe_ep_rank, + ) + if self.prelaunched_schedulers is not None: + _, reader = self.prelaunched_schedulers[pp_rank, tp_rank] + reader.send((scheduler_args, display_ranks)) + else: + reader, writer = mp.Pipe(duplex=False) + proc = mp.Process( + target=self.run_scheduler_process_func, + args=(*scheduler_args, writer, *display_ranks), + ) + with ( + memory_saver_adapter.configure_subprocess(), + numa_utils.configure_subprocess(server_args, gpu_id), + ): + proc.start() + self.scheduler_procs.append(proc) + scheduler_pipe_readers.append(reader) + + # Wait for model to finish loading + scheduler_info = [] + for i in range(len(scheduler_pipe_readers)): + scheduler_info.append(scheduler_pipe_readers[i].recv()) + + # Pure-DP TP groups launch concurrently. Their ready replies contain + # only publishers owned by schedulers on this node. + with self.env_lock: + self.local_kv_event_sources.extend( + source + for info in scheduler_info + for source in info.get("kv_event_sources", []) + ) + + self.max_total_num_tokens = scheduler_info[0]["max_total_num_tokens"] + self.max_req_input_len = scheduler_info[0]["max_req_input_len"] + self.startup_time = aggregate_scheduler_startup_times( + info.get("startup_time") for info in scheduler_info + ) + + def dispatch_health_check(self, req: Req) -> None: + # This policy routes by request metadata, not shared load-balancing state. + if self.load_balance_method == LoadBalanceMethod.FOLLOW_BOOTSTRAP_ROOM: + self.follow_bootstrap_room_scheduler(req) + return + + if self.maybe_external_dp_rank_routing(req): + return + + active = self._active_workers + if not active: + raise RuntimeError("No active DP workers are available for health checks.") + + slot = active[self.health_round_robin_counter % len(active)] + self.health_round_robin_counter = (self.health_round_robin_counter + 1) % len( + active + ) + logger.debug(f"Choose worker {slot} for health check") + sock_send(self.workers[slot], req) + + def maybe_external_dp_rank_routing(self, req: Req): + if req.routed_dp_rank is not None: + rank = req.routed_dp_rank + if ( + rank < 0 + or rank >= len(self.workers) + or rank not in self._active_workers + or self.workers[rank] is None + ): + raise ValueError(f"DP rank {rank} is not active.") + logger.debug(f"Direct routing to DP rank {rank}") + sock_send(self.workers[rank], req) + return True + return False + + def round_robin_scheduler(self, req: Req): + if self.maybe_external_dp_rank_routing(req): + return + + active = self._active_workers + if not active: + raise RuntimeError("No active DP workers are available for routing.") + attempts = 0 + while attempts < len(active): + slot = active[self.round_robin_counter % len(active)] + self.round_robin_counter = (self.round_robin_counter + 1) % len(active) + if self.status[slot]: + logger.debug(f"Choose worker {slot}") + sock_send(self.workers[slot], req) + return + attempts += 1 + raise RuntimeError( + f"Cannot route request: all {len(active)} active DP workers " + "are unavailable." + ) + + def follow_bootstrap_room_scheduler(self, req: Req): + if self.maybe_external_dp_rank_routing(req): + return + + assert req.bootstrap_room is not None, ( + "req.bootstrap_room should not be None. Do not send requests directly to " + "prefill or decode instances; send to the router instead." + ) + target_rank = req.bootstrap_room % len(self.workers) + sock_send(self.workers[target_rank], req) + + def total_requests_scheduler(self, req: Req): + if self.maybe_external_dp_rank_routing(req): + return + target_worker = self.dp_budget.dispatch(LoadBalanceMethod.TOTAL_REQUESTS) + sock_send(self.workers[target_worker], req) + + def total_tokens_scheduler(self, req: Req): + if self.maybe_external_dp_rank_routing(req): + return + estimated_tokens = len(req.input_ids) + target_worker = self.dp_budget.dispatch( + LoadBalanceMethod.TOTAL_TOKENS, estimated_tokens=estimated_tokens + ) + sock_send(self.workers[target_worker], req) + + def total_uncached_tokens_scheduler(self, req: Req): + """Balance on prefill work that will actually be computed. + + ``total_tokens`` charges a request's whole prompt, although with prefix + caching only the unmatched part costs prefill compute. On workloads with + a high cache hit rate that makes the signal mostly noise: per request the + uncached part is a few percent of the prompt but varies by more than an + order of magnitude, so one cold request outweighs many hot ones. + """ + if self.maybe_external_dp_rank_routing(req): + return + target_worker = self.dp_budget.dispatch( + LoadBalanceMethod.UNCACHED_TOKENS, + num_input_tokens=len(req.input_ids), + ) + sock_send(self.workers[target_worker], req) + + def event_loop(self): + while True: + while True: + self.soft_watchdog.feed() + try: + recv_req = sock_recv(self.recv_from_tokenizer, flags=zmq.NOBLOCK) + except zmq.ZMQError: + break + self._request_dispatcher(recv_req) + + +def run_scheduler_process_with_init_pipe(run_scheduler_process_func: Callable, pipe): + # Unpickling the callable imports the scheduler before waiting for DP setup. + kill_itself_when_parent_died() + try: + scheduler_args, display_ranks = pipe.recv() + run_scheduler_process_func(*scheduler_args, pipe, *display_ranks) + finally: + pipe.close() + + +def run_data_parallel_controller_process( + server_args: ServerArgs, + port_args: PortArgs, + pipe_writer, + run_scheduler_process_func: Callable = run_scheduler_process, + prelaunched_schedulers: dict[tuple[int, int], tuple[int, Connection]] | None = None, +): + setproctitle.setproctitle("sglang::data_parallel_controller") + faulthandler.enable() + kill_itself_when_parent_died() + parent_process = psutil.Process().parent() + + load_plugins() + # This process reads the config namespaces before spawning schedulers. + publish(server_args, role="dp_controller") + configure_logger(server_args) + if get_observability().enable_trace: + process_tracing_init( + get_observability().otlp_traces_endpoint, + get_observability().otlp_service_name, + trace_modules=get_observability().trace_modules, + ) + thread_label = "DP Controller" + if get_disagg().disaggregation_mode == "prefill": + thread_label = "Prefill DP Controller" + elif get_disagg().disaggregation_mode == "decode": + thread_label = "Decode DP Controller" + trace_set_thread_info(thread_label) + + try: + controller = DataParallelController( + server_args, port_args, run_scheduler_process_func, prelaunched_schedulers + ) + scheduler_pids = ( + [pid for pid, _ in prelaunched_schedulers.values()] + if prelaunched_schedulers is not None + else [proc.pid for proc in controller.scheduler_procs if proc is not None] + ) + init_info = { + "status": "ready", + "max_total_num_tokens": controller.max_total_num_tokens, + "max_req_input_len": controller.max_req_input_len, + "startup_time": controller.startup_time, + SCHEDULER_PIDS_ARG: scheduler_pids, + } + if get_serving().grpc_port is not None and not ( + get_serving().smg_grpc_mode or get_serving().grpc_mode + ): + init_info["kv_event_sources"] = sorted( + controller.local_kv_event_sources, key=lambda source: source["dp_rank"] + ) + pipe_writer.send(init_info) + # The primary owns routing for the expanded scheduler set. + if get_parallel().node_rank == 0 and not get_exec().moe.is_ep_scale_joiner: + controller.event_loop() + if prelaunched_schedulers is not None: + # The launcher owns these processes. Their pipes close when they exit. + wait([pipe for _, pipe in prelaunched_schedulers.values()]) + for proc in controller.scheduler_procs: + proc.join() + logger.error( + f"Scheduler or DataParallelController {proc.pid} terminated with {proc.exitcode}" + ) + except Exception: + traceback = get_exception_traceback() + logger.error(f"DataParallelController hit an exception: {traceback}") + parent_process.send_signal(signal.SIGQUIT) diff --git a/test/registered/unit/managers/test_data_parallel_controller.py b/test/registered/unit/managers/test_data_parallel_controller.py index b800686774..5dffb9ba29 100644 --- a/test/registered/unit/managers/test_data_parallel_controller.py +++ b/test/registered/unit/managers/test_data_parallel_controller.py @@ -265,6 +265,27 @@ class TestDPBudgetUpdateBudget(CustomTestCase): ) self.assertEqual(budget.total_tokens, [150, 80]) + def test_maps_waiting_uncached_tokens_and_hit_rate(self): + budget = DPBudget(num_dp_ranks=2) + budget.update_budget( + [ + _load( + dp_rank=0, + timestamp=1.0, + num_waiting_uncached_tokens=900, + cache_hit_rate=0.95, + ), + _load( + dp_rank=1, + timestamp=1.0, + num_waiting_uncached_tokens=100, + cache_hit_rate=0.5, + ), + ] + ) + self.assertEqual(budget.total_uncached_tokens, [900, 100]) + self.assertEqual(budget.cache_hit_rate, [0.95, 0.5]) + def test_partial_update_only_affects_reported_rank(self): budget = DPBudget(num_dp_ranks=3) budget.update_budget( @@ -473,6 +494,43 @@ class TestTotalRequestsScheduler(CustomTestCase): ) +class TestUncachedTokensScheduler(CustomTestCase): + def test_dispatches_to_rank_with_least_uncached_work(self): + ctl = _make_controller(num_dp_ranks=4) + ctl.dp_budget.total_uncached_tokens = [100, 50, 900, 10] + ctl.total_uncached_tokens_scheduler(_req(input_ids=[0] * 1000)) + ctl.workers[3].send_pyobj.assert_called_once() + for i in (0, 1, 2): + ctl.workers[i].send_pyobj.assert_not_called() + # No hit rate reported -> the whole prompt is charged as uncached. + self.assertEqual(ctl.dp_budget.total_uncached_tokens, [100, 50, 900, 1010]) + self.assertEqual(ctl.dp_budget.total_requests, [0, 0, 0, 1]) + + def test_charge_scales_with_target_hit_rate(self): + ctl = _make_controller(num_dp_ranks=2) + ctl.dp_budget.total_uncached_tokens = [0, 0] + ctl.dp_budget.cache_hit_rate = [0.5, 0.0] + ctl.total_uncached_tokens_scheduler(_req(input_ids=[0] * 1000)) + # rank 0 estimates 500 uncached tokens, rank 1 estimates 1000. + ctl.workers[0].send_pyobj.assert_called_once() + self.assertEqual(ctl.dp_budget.total_uncached_tokens, [500, 0]) + + def test_pending_backlog_beats_a_cheaper_candidate(self): + ctl = _make_controller(num_dp_ranks=2) + ctl.dp_budget.total_uncached_tokens = [500, 0] + ctl.dp_budget.cache_hit_rate = [0.0, 0.0] + ctl.total_uncached_tokens_scheduler(_req(input_ids=[0] * 100)) + ctl.workers[1].send_pyobj.assert_called_once() + self.assertEqual(ctl.dp_budget.total_uncached_tokens, [500, 100]) + + def test_routed_dp_rank_bypasses_budget(self): + ctl = _make_controller(num_dp_ranks=4) + ctl.dp_budget.total_uncached_tokens = [5, 3, 1, 4] + ctl.total_uncached_tokens_scheduler(_req(routed_dp_rank=0)) + ctl.workers[0].send_pyobj.assert_called_once() + self.assertEqual(ctl.dp_budget.total_uncached_tokens, [5, 3, 1, 4]) + + class TestStatusAwarenessInconsistency(CustomTestCase): """Document a divergence: ``round_robin_scheduler`` skips workers whose ``status`` is False, but ``total_requests_scheduler`` / diff --git a/test/registered/unit/managers/test_load_inquirer.py b/test/registered/unit/managers/test_load_inquirer.py index 87f7699f58..127d10173f 100644 --- a/test/registered/unit/managers/test_load_inquirer.py +++ b/test/registered/unit/managers/test_load_inquirer.py @@ -1,202 +1,203 @@ -import unittest -from dataclasses import replace -from types import SimpleNamespace -from unittest.mock import Mock - -from sglang.srt.disaggregation.utils import DisaggregationMode -from sglang.srt.managers.schedule_policy import ( - CacheAwarePolicy, - SchedulePolicy, -) -from sglang.srt.managers.scheduler_components.load_inquirer import ( - SchedulerLoadInquirer, -) -from sglang.srt.mem_cache.base_prefix_cache import CacheRequestHandle -from sglang.test.ci.ci_register import register_cpu_ci - -register_cpu_ci(est_time=10, suite="base-a-test-cpu") - - -class TestSchedulePolicyWaitingQueueMatching(unittest.TestCase): - def make_policy(self, policy, supports_fast_match_prefix): - schedule_policy = object.__new__(SchedulePolicy) - schedule_policy.policy = policy - schedule_policy.tree_cache = SimpleNamespace( - supports_fast_match_prefix=lambda: supports_fast_match_prefix - ) - return schedule_policy - - def test_lpm_queue_limit_respects_fast_matching_capability(self): - policy = self.make_policy(CacheAwarePolicy.LPM, False) - self.assertTrue(policy.waiting_queue_prefix_matched([None] * 128)) - self.assertFalse(policy.waiting_queue_prefix_matched([None] * 129)) - - policy.tree_cache = SimpleNamespace(supports_fast_match_prefix=lambda: True) - self.assertTrue(policy.waiting_queue_prefix_matched([None] * 129)) - - -class TestSchedulerLoadInquirer(unittest.TestCase): - def make_inquirer( - self, - waiting_queue_prefix_matched=True, - storage_span=None, - disaggregation_mode=DisaggregationMode.NULL, - ): - waiting_req = SimpleNamespace( - seqlen=100, - num_matched_prefix_tokens=20, - cache_request_handle=CacheRequestHandle("waiting", 0), - ) - chunked_req = SimpleNamespace(seqlen=50, prefix_len=10) - return SchedulerLoadInquirer( - disaggregation_mode=disaggregation_mode, - server_args=None, - max_total_num_tokens=1000, - max_running_requests=10, - pool_stats_observer=None, - tp_worker=None, - token_to_kv_pool_allocator=None, - spec_algorithm=None, - get_running_batch=None, - get_waiting_queue=lambda: [waiting_req], - waiting_queue_prefix_matched=lambda: waiting_queue_prefix_matched, - get_chunked_req=lambda: chunked_req, - get_recent_cache_hit_rate=lambda: 0.75, - get_stats=None, - get_disagg_prefill_bootstrap_queue=None, - get_disagg_prefill_inflight_queue=None, - get_disagg_decode_prealloc_queue=None, - get_disagg_decode_transfer_queue=None, - get_spec_total_num_accept_tokens=None, - get_spec_total_num_forward_ct=None, - get_total_prefill_uncached_tokens=None, - get_total_prefill_busy_us=None, - get_decode_moment_totals=None, - get_waiting_storage_span=Mock(return_value=storage_span), - ) - - def test_waiting_tokens_are_estimated_when_prefix_matching_is_skipped(self): - inquirer = self.make_inquirer(waiting_queue_prefix_matched=False) - - self.assertEqual( - SchedulerLoadInquirer.get_num_waiting_uncached_tokens(inquirer), - 65, - ) - - def test_waiting_tokens_use_exact_match_when_prefix_matching_is_done(self): - inquirer = self.make_inquirer(waiting_queue_prefix_matched=True) - - self.assertEqual( - SchedulerLoadInquirer.get_num_waiting_uncached_tokens(inquirer), - 120, - ) - - def test_storage_exclusion_is_clipped_to_uncached_prompt(self): - for span, expected in ( - (None, 120), - ((0, 10), 120), - ((0, 40), 100), - ((20, 80), 60), - ((60, 120), 80), - ((20, 120), 40), - ((20, 20), 120), - ((100, 120), 120), - ): - with self.subTest(span=span): - inquirer = self.make_inquirer(storage_span=span) - self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), expected) - inquirer.get_waiting_storage_span.assert_called_once_with( - CacheRequestHandle("waiting", 0) - ) - - def test_storage_exclusion_requires_per_request_matching(self): - for prefix_matched, expected in ((True, 60), (False, 65)): - with self.subTest(prefix_matched=prefix_matched): - inquirer = self.make_inquirer( - waiting_queue_prefix_matched=prefix_matched, - storage_span=(20, 80), - ) - self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), expected) - if not prefix_matched: - inquirer.get_waiting_storage_span.assert_not_called() - - def test_storage_exclusion_does_not_change_decode(self): - for prefix_matched in (False, True): - with self.subTest(prefix_matched=prefix_matched): - inquirer = self.make_inquirer( - waiting_queue_prefix_matched=prefix_matched, - storage_span=(20, 100), - disaggregation_mode=DisaggregationMode.DECODE, - ) - self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), 0) - inquirer.get_waiting_storage_span.assert_not_called() - - def test_storage_exclusion_preserves_the_historical_hit_rate_estimate(self): - for span in (None, (20, 50), (20, 80), (80, 100), (0, 120)): - with self.subTest(span=span): - inquirer = self.make_inquirer( - waiting_queue_prefix_matched=False, - storage_span=span, - ) - self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), 65) - inquirer.get_waiting_storage_span.assert_not_called() - - def test_historical_estimate_already_includes_storage_hits(self): - requests = [ - SimpleNamespace(seqlen=100, cache_request_handle=CacheRequestHandle(rid, 0)) - for rid in ("storage_hit", "cold") - ] - get_storage_span = Mock( - side_effect=lambda handle: (0, 80) if handle.rid == "storage_hit" else None - ) - inquirer = replace( - self.make_inquirer(waiting_queue_prefix_matched=False), - get_waiting_queue=lambda: requests, - get_chunked_req=lambda: None, - get_recent_cache_hit_rate=lambda: 0.4, - get_waiting_storage_span=get_storage_span, - ) - - # One 80-token storage hit and one cold prompt need 20 + 100 tokens. - self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), 120) - get_storage_span.assert_not_called() - - def test_storage_exclusion_without_a_provider_keeps_the_original_estimate(self): - inquirer = replace( - self.make_inquirer(), - get_waiting_storage_span=None, - ) - self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), 120) - - def test_scheduler_wires_storage_provider(self): - from sglang.srt.managers.scheduler import Scheduler - - req = self.make_inquirer().get_waiting_queue()[0] - get_storage_span = Mock(return_value=(20, 80)) - scheduler = SimpleNamespace( - disaggregation_mode=DisaggregationMode.PREFILL, - server_args=None, - max_total_num_tokens=1000, - kv_shard_widening=1, - max_running_requests=10, - pool_stats_observer=None, - tp_worker=None, - token_to_kv_pool_allocator=None, - spec_algorithm=None, - tree_cache=SimpleNamespace(waiting_storage_span=get_storage_span), - policy=SimpleNamespace(waiting_queue_prefix_matched=lambda queue: True), - metrics_reporter=SimpleNamespace(recent_cache_hit_rate=0.75), - waiting_queue=[req], - chunked_req=None, - ) - - Scheduler.init_load_inquirer(scheduler) - - inquirer = scheduler.load_inquirer - self.assertIs(inquirer.get_waiting_storage_span, get_storage_span) - self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), 20) - get_storage_span.assert_called_once_with(req.cache_request_handle) - - -if __name__ == "__main__": - unittest.main() +import unittest +from dataclasses import replace +from types import SimpleNamespace +from unittest.mock import Mock + +from sglang.srt.disaggregation.utils import DisaggregationMode +from sglang.srt.managers.schedule_policy import ( + CacheAwarePolicy, + SchedulePolicy, +) +from sglang.srt.managers.scheduler_components.load_inquirer import ( + SchedulerLoadInquirer, +) +from sglang.srt.mem_cache.base_prefix_cache import CacheRequestHandle +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + + +class TestSchedulePolicyWaitingQueueMatching(unittest.TestCase): + def make_policy(self, policy, supports_fast_match_prefix): + schedule_policy = object.__new__(SchedulePolicy) + schedule_policy.policy = policy + schedule_policy.tree_cache = SimpleNamespace( + supports_fast_match_prefix=lambda: supports_fast_match_prefix + ) + return schedule_policy + + def test_lpm_queue_limit_respects_fast_matching_capability(self): + policy = self.make_policy(CacheAwarePolicy.LPM, False) + self.assertTrue(policy.waiting_queue_prefix_matched([None] * 128)) + self.assertFalse(policy.waiting_queue_prefix_matched([None] * 129)) + + policy.tree_cache = SimpleNamespace(supports_fast_match_prefix=lambda: True) + self.assertTrue(policy.waiting_queue_prefix_matched([None] * 129)) + + +class TestSchedulerLoadInquirer(unittest.TestCase): + def make_inquirer( + self, + waiting_queue_prefix_matched=True, + storage_span=None, + disaggregation_mode=DisaggregationMode.NULL, + ): + waiting_req = SimpleNamespace( + seqlen=100, + num_matched_prefix_tokens=20, + cache_request_handle=CacheRequestHandle("waiting", 0), + ) + chunked_req = SimpleNamespace(seqlen=50, prefix_len=10) + return SchedulerLoadInquirer( + disaggregation_mode=disaggregation_mode, + server_args=None, + max_total_num_tokens=1000, + max_running_requests=10, + pool_stats_observer=None, + tp_worker=None, + token_to_kv_pool_allocator=None, + spec_algorithm=None, + get_running_batch=None, + get_waiting_queue=lambda: [waiting_req], + waiting_queue_prefix_matched=lambda: waiting_queue_prefix_matched, + get_chunked_req=lambda: chunked_req, + get_grammar_queue=lambda: [], + get_recent_cache_hit_rate=lambda: 0.75, + get_stats=None, + get_disagg_prefill_bootstrap_queue=None, + get_disagg_prefill_inflight_queue=None, + get_disagg_decode_prealloc_queue=None, + get_disagg_decode_transfer_queue=None, + get_spec_total_num_accept_tokens=None, + get_spec_total_num_forward_ct=None, + get_total_prefill_uncached_tokens=None, + get_total_prefill_busy_us=None, + get_decode_moment_totals=None, + get_waiting_storage_span=Mock(return_value=storage_span), + ) + + def test_waiting_tokens_are_estimated_when_prefix_matching_is_skipped(self): + inquirer = self.make_inquirer(waiting_queue_prefix_matched=False) + + self.assertEqual( + SchedulerLoadInquirer.get_num_waiting_uncached_tokens(inquirer), + 65, + ) + + def test_waiting_tokens_use_exact_match_when_prefix_matching_is_done(self): + inquirer = self.make_inquirer(waiting_queue_prefix_matched=True) + + self.assertEqual( + SchedulerLoadInquirer.get_num_waiting_uncached_tokens(inquirer), + 120, + ) + + def test_storage_exclusion_is_clipped_to_uncached_prompt(self): + for span, expected in ( + (None, 120), + ((0, 10), 120), + ((0, 40), 100), + ((20, 80), 60), + ((60, 120), 80), + ((20, 120), 40), + ((20, 20), 120), + ((100, 120), 120), + ): + with self.subTest(span=span): + inquirer = self.make_inquirer(storage_span=span) + self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), expected) + inquirer.get_waiting_storage_span.assert_called_once_with( + CacheRequestHandle("waiting", 0) + ) + + def test_storage_exclusion_requires_per_request_matching(self): + for prefix_matched, expected in ((True, 60), (False, 65)): + with self.subTest(prefix_matched=prefix_matched): + inquirer = self.make_inquirer( + waiting_queue_prefix_matched=prefix_matched, + storage_span=(20, 80), + ) + self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), expected) + if not prefix_matched: + inquirer.get_waiting_storage_span.assert_not_called() + + def test_storage_exclusion_does_not_change_decode(self): + for prefix_matched in (False, True): + with self.subTest(prefix_matched=prefix_matched): + inquirer = self.make_inquirer( + waiting_queue_prefix_matched=prefix_matched, + storage_span=(20, 100), + disaggregation_mode=DisaggregationMode.DECODE, + ) + self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), 0) + inquirer.get_waiting_storage_span.assert_not_called() + + def test_storage_exclusion_preserves_the_historical_hit_rate_estimate(self): + for span in (None, (20, 50), (20, 80), (80, 100), (0, 120)): + with self.subTest(span=span): + inquirer = self.make_inquirer( + waiting_queue_prefix_matched=False, + storage_span=span, + ) + self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), 65) + inquirer.get_waiting_storage_span.assert_not_called() + + def test_historical_estimate_already_includes_storage_hits(self): + requests = [ + SimpleNamespace(seqlen=100, cache_request_handle=CacheRequestHandle(rid, 0)) + for rid in ("storage_hit", "cold") + ] + get_storage_span = Mock( + side_effect=lambda handle: (0, 80) if handle.rid == "storage_hit" else None + ) + inquirer = replace( + self.make_inquirer(waiting_queue_prefix_matched=False), + get_waiting_queue=lambda: requests, + get_chunked_req=lambda: None, + get_recent_cache_hit_rate=lambda: 0.4, + get_waiting_storage_span=get_storage_span, + ) + + # One 80-token storage hit and one cold prompt need 20 + 100 tokens. + self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), 120) + get_storage_span.assert_not_called() + + def test_storage_exclusion_without_a_provider_keeps_the_original_estimate(self): + inquirer = replace( + self.make_inquirer(), + get_waiting_storage_span=None, + ) + self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), 120) + + def test_scheduler_wires_storage_provider(self): + from sglang.srt.managers.scheduler import Scheduler + + req = self.make_inquirer().get_waiting_queue()[0] + get_storage_span = Mock(return_value=(20, 80)) + scheduler = SimpleNamespace( + disaggregation_mode=DisaggregationMode.PREFILL, + server_args=None, + max_total_num_tokens=1000, + kv_shard_widening=1, + max_running_requests=10, + pool_stats_observer=None, + tp_worker=None, + token_to_kv_pool_allocator=None, + spec_algorithm=None, + tree_cache=SimpleNamespace(waiting_storage_span=get_storage_span), + policy=SimpleNamespace(waiting_queue_prefix_matched=lambda queue: True), + metrics_reporter=SimpleNamespace(recent_cache_hit_rate=0.75), + waiting_queue=[req], + chunked_req=None, + ) + + Scheduler.init_load_inquirer(scheduler) + + inquirer = scheduler.load_inquirer + self.assertIs(inquirer.get_waiting_storage_span, get_storage_span) + self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), 20) + get_storage_span.assert_called_once_with(req.cache_request_handle) + + +if __name__ == "__main__": + unittest.main()