From 87ed1b95266c4ccff9a9e5b0e2fb6143f6ecb7f8 Mon Sep 17 00:00:00 2001 From: Yoray Zack Date: Wed, 14 Oct 2026 20:17:01 +0300 Subject: [PATCH] [4/N] elastic-ep: Scale at DP-attention replica granularity (#33728) --- python/sglang/srt/arg_groups/parallel_hook.py | 50 +++++- python/sglang/srt/elastic_ep/elastic_ep.py | 38 +++++ python/sglang/srt/elastic_ep/topology.py | 59 +++++++ python/sglang/srt/layers/dcp/comm.py | 8 +- python/sglang/srt/layers/dp_attention.py | 18 ++- .../srt/layers/vocab_parallel_embedding.py | 8 +- .../srt/managers/data_parallel_controller.py | 108 +++++++++---- python/sglang/srt/managers/io_struct.py | 4 +- python/sglang/srt/managers/scheduler.py | 25 ++- .../managers/scheduler_components/dp_attn.py | 13 +- .../sglang/srt/managers/tokenizer_manager.py | 18 ++- .../sglang/srt/model_executor/model_runner.py | 46 ++++-- test/manual/ep/test_elastic_scale.py | 95 ++++++++++- .../test_vocab_parallel_embedding.py | 32 ++++ .../unit/elastic_ep/test_topology.py | 147 ++++++++++++++++++ .../unit/layers/test_dp_attention.py | 31 +++- .../managers/test_data_parallel_controller.py | 71 +++++++++ test/registered/unit/test_runtime_context.py | 11 +- 18 files changed, 695 insertions(+), 87 deletions(-) create mode 100644 python/sglang/srt/elastic_ep/topology.py create mode 100644 test/registered/unit/elastic_ep/test_topology.py diff --git a/python/sglang/srt/arg_groups/parallel_hook.py b/python/sglang/srt/arg_groups/parallel_hook.py index fdbc3e9add..bf52160671 100644 --- a/python/sglang/srt/arg_groups/parallel_hook.py +++ b/python/sglang/srt/arg_groups/parallel_hook.py @@ -558,6 +558,43 @@ def handle_elastic_ep(server_args: Any): ) if scaling_active: resolved = resolved_view(server_args) + from sglang.srt.elastic_ep.topology import physical_ep_size_to_dp_size + + try: + divisor = cfg.attn_dp_size * resolved.attn_cp_size + if ( + cfg.tp_size <= 0 + or cfg.attn_dp_size <= 0 + or resolved.attn_cp_size <= 0 + or cfg.tp_size % divisor + ): + raise ValueError( + "Invalid attention topology: " + f"tp_size={cfg.tp_size}, attn_dp_size={cfg.attn_dp_size}, " + f"attn_cp_size={resolved.attn_cp_size}." + ) + attn_tp_size = derive_attn_tp_size( + tp_size=cfg.tp_size, + attn_cp_size=resolved.attn_cp_size, + attn_dp_size=cfg.attn_dp_size, + ) + attn_replica_size = attn_tp_size * resolved.attn_cp_size + physical_ep_size_to_dp_size(cfg.max_ep_size, attn_replica_size) + physical_ep_size_to_dp_size( + cfg.elastic_ep_initial_size or cfg.tp_size, attn_replica_size + ) + if cfg.ep_join_mode == "scale": + physical_ep_size_to_dp_size(cfg.ep_join_rank_offset, attn_replica_size) + except ValueError as exc: + raise AssertionError(str(exc)) from exc + + if attn_tp_size > 1: + assert cfg.moe_dense_tp_size == 1, ( + "Elastic EP scale-up with attention TP greater than one requires " + "--moe-dense-tp-size 1 " + f"(got moe_dense_tp_size={cfg.moe_dense_tp_size})." + ) + assert cfg.elastic_ep_scale_timeout > 0, ( "--elastic-ep-scale-timeout must be greater than zero." ) @@ -622,6 +659,11 @@ def handle_elastic_ep(server_args: Any): "Elastic EP scale-up requires --pp-size 1 " f"(got pp_size={cfg.pp_size}); WORLD must not span PP stages." ) + assert attn_tp_size % cfg.dcp_size == 0, ( + "Elastic EP scale-up requires --dcp-size to divide the fixed " + "attention TP width " + f"(got dcp_size={cfg.dcp_size}, attn_tp_size={attn_tp_size})." + ) decode_backend = cfg.cuda_graph_config.decode.backend assert decode_backend in (Backend.DISABLED, Backend.FULL), ( @@ -673,12 +715,8 @@ def handle_elastic_ep(server_args: Any): ) assert resolved.ep_size == cfg.tp_size, ( "Elastic EP scale-up requires ep_size == tp_size " - f"(got ep_size={resolved.ep_size}, tp_size={cfg.tp_size}); EP, TP " - "and the attention DP group must all coincide with WORLD." - ) - assert cfg.attn_dp_size == cfg.tp_size, ( - "Elastic EP scale-up requires attn_dp_size == tp_size " - f"(got attn_dp_size={cfg.attn_dp_size}, tp_size={cfg.tp_size})." + f"(got ep_size={resolved.ep_size}, tp_size={cfg.tp_size}); " + "physical EP membership must coincide with the launch-time TP span." ) assert resolved.moe_a2a_backend == "nixl", ( "Elastic EP scale-up requires --moe-a2a-backend nixl " diff --git a/python/sglang/srt/elastic_ep/elastic_ep.py b/python/sglang/srt/elastic_ep/elastic_ep.py index fc996713de..40780b113a 100644 --- a/python/sglang/srt/elastic_ep/elastic_ep.py +++ b/python/sglang/srt/elastic_ep/elastic_ep.py @@ -30,6 +30,8 @@ _SCALE_COHORT_KEY_PREFIX = "elastic_ep/scale_cohort" class ScaleCohort(msgspec.Struct, frozen=True, kw_only=True): target_ep_size: int cuda_graph_enabled: bool + attn_tp_size: int + attn_cp_size: int def register_scale_cohort( @@ -38,10 +40,13 @@ def register_scale_cohort( store = get_global_tcp_store() if store is None: raise RuntimeError("Elastic EP scale-up requires the global TCPStore.") + parallel = get_parallel() payload = msgspec.json.encode( ScaleCohort( target_ep_size=target_ep_size, cuda_graph_enabled=cuda_graph_enabled, + attn_tp_size=parallel.attn_tp_size, + attn_cp_size=parallel.attn_cp_size, ) ) store.set(f"{_SCALE_COHORT_KEY_PREFIX}/{rank_offset}", payload) @@ -57,6 +62,39 @@ def get_scale_cohort(rank_offset: int) -> Optional[ScaleCohort]: return msgspec.json.decode(store.get(key), type=ScaleCohort) +def validate_scale_cohort_topology(cohort: ScaleCohort) -> None: + from sglang.srt.elastic_ep.topology import validate_matching_attention_topology + + parallel = get_parallel() + validate_matching_attention_topology( + primary_attn_tp_size=parallel.attn_tp_size, + primary_attn_cp_size=parallel.attn_cp_size, + joining_attn_tp_size=cohort.attn_tp_size, + joining_attn_cp_size=cohort.attn_cp_size, + ) + + +def update_dp_attention_for_elastic_ep( + *, physical_ep_size: int, physical_ep_rank: int +) -> int: + """Apply a physical EP topology to the logical DP-attention runtime.""" + from sglang.srt.elastic_ep.topology import ( + attn_replica_size, + physical_ep_rank_to_dp_rank, + physical_ep_size_to_dp_size, + ) + from sglang.srt.layers.dp_attention import update_dp_attention_post_scale + + replica_size = attn_replica_size() + dp_size = physical_ep_size_to_dp_size(physical_ep_size, replica_size) + update_dp_attention_post_scale( + new_dp_size=dp_size, + new_dp_rank=physical_ep_rank_to_dp_rank(physical_ep_rank, replica_size), + ) + get_parallel().override_permanently(num_dp_ranks=dp_size) + return dp_size + + @dataclass class ElasticEPState: active_ranks: Optional[torch.Tensor] diff --git a/python/sglang/srt/elastic_ep/topology.py b/python/sglang/srt/elastic_ep/topology.py new file mode 100644 index 0000000000..9dd50ab35b --- /dev/null +++ b/python/sglang/srt/elastic_ep/topology.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from typing import Sequence + +from sglang.srt.runtime_context import get_parallel + + +def attn_replica_size() -> int: + """Return the fixed physical width of one attention-DP replica.""" + parallel = get_parallel() + return parallel.attn_tp_size * parallel.attn_cp_size + + +def validate_matching_attention_topology( + *, + primary_attn_tp_size: int, + primary_attn_cp_size: int, + joining_attn_tp_size: int, + joining_attn_cp_size: int, +) -> None: + primary_topology = (primary_attn_tp_size, primary_attn_cp_size) + joining_topology = (joining_attn_tp_size, joining_attn_cp_size) + if joining_topology != primary_topology: + raise ValueError( + "Primary and joining cohort must use the same attention topology " + "for Elastic EP scale-up " + f"(primary attn_tp_size={primary_topology[0]}, " + f"attn_cp_size={primary_topology[1]}; joining " + f"attn_tp_size={joining_topology[0]}, " + f"attn_cp_size={joining_topology[1]})." + ) + + +def physical_ep_size_to_dp_size(ep_size: int, attn_replica_size: int) -> int: + if ep_size <= 0 or attn_replica_size <= 0 or ep_size % attn_replica_size != 0: + raise ValueError( + f"EP size {ep_size} must be divisible by attention replica size " + f"{attn_replica_size}." + ) + return ep_size // attn_replica_size + + +def physical_ep_rank_to_dp_rank(ep_rank: int, attn_replica_size: int) -> int: + if ep_rank < 0 or attn_replica_size <= 0: + raise ValueError( + f"Invalid EP/attention topology: {ep_rank=}, {attn_replica_size=}." + ) + return ep_rank // attn_replica_size + + +def collapse_physical_rank_status( + status: Sequence[bool], attn_replica_size: int +) -> list[bool]: + """Collapse physical-rank health with an all-members replica rule.""" + physical_ep_size_to_dp_size(len(status), attn_replica_size) + return [ + all(bool(value) for value in status[start : start + attn_replica_size]) + for start in range(0, len(status), attn_replica_size) + ] diff --git a/python/sglang/srt/layers/dcp/comm.py b/python/sglang/srt/layers/dcp/comm.py index 37018f0397..a6b921e6ef 100644 --- a/python/sglang/srt/layers/dcp/comm.py +++ b/python/sglang/srt/layers/dcp/comm.py @@ -299,17 +299,19 @@ def all_gather_kv_cache_for_mla_extend( ) dcp_kv_buffer[:dcp_extend_prefix_lens_sum] = gathered_kv - # copy local kv cache into forward_batch.attn_dcp_metadata.dcp_kv_buffer + # Attention TP may append dummy tokens to align an extend batch to its + # group width. The DCP buffer tracks only real sequence tokens. + extend_num_tokens = dcp_kv_buffer.shape[0] - dcp_extend_prefix_lens_sum dcp_kv_buffer[ dcp_extend_prefix_lens_sum:, ..., :kv_lora_rank, - ] = k_nope + ] = k_nope[:extend_num_tokens] dcp_kv_buffer[ dcp_extend_prefix_lens_sum:, ..., kv_lora_rank:, - ] = k_pe + ] = k_pe[:extend_num_tokens] # all gather kv cache and re-org to query orders diff --git a/python/sglang/srt/layers/dp_attention.py b/python/sglang/srt/layers/dp_attention.py index 3a5f46a0a9..2f67a25043 100644 --- a/python/sglang/srt/layers/dp_attention.py +++ b/python/sglang/srt/layers/dp_attention.py @@ -20,6 +20,10 @@ from sglang.srt.distributed import ( from sglang.srt.distributed.device_communicators.pynccl_allocator import ( use_symmetric_memory, ) +from sglang.srt.elastic_ep.topology import ( + attn_replica_size, + physical_ep_rank_to_dp_rank, +) from sglang.srt.environ import envs from sglang.srt.runtime_context import ( derive_attention_ranks, @@ -90,16 +94,20 @@ def dp_slot_in(per_rank) -> int: def dp_gather_slot() -> int: """Return this process's index in the DP gather. - After elastic scale-up, use the TP rank plus the join offset; otherwise - use the attention-DP rank. Inside a draft scope that owns its attention, - it is still the slot in the target's gather (see patch_tensor_parallel_group). + After elastic scale-up, convert the physical EP rank (the TP rank plus the + join offset) to its logical attention-DP replica; otherwise use the local + attention-DP rank. Inside a draft scope that owns its attention, it is still + the slot in the target's gather (see patch_tensor_parallel_group). """ scoped = get_flags().dp.scoped_gather_slot if scoped is not None: return scoped parallel = get_parallel() if world_dp_gather_enabled(): - return parallel.tp_rank + parallel.ep_join_rank_offset + return physical_ep_rank_to_dp_rank( + parallel.tp_rank + parallel.ep_join_rank_offset, + attn_replica_size(), + ) return parallel.attn_dp_rank @@ -355,7 +363,7 @@ def set_dp_buffer_len_from_batch(forward_batch: ForwardBatch) -> None: global_num_tokens = forward_batch.global_num_tokens_padded_cpu if global_num_tokens is None: global_num_tokens = forward_batch.global_num_tokens_cpu - dp_rank = get_parallel().attn_dp_rank if len(global_num_tokens) > 1 else 0 + dp_rank = dp_slot_in(global_num_tokens) set_dp_buffer_len( forward_batch.global_dp_buffer_len, global_num_tokens[dp_rank], diff --git a/python/sglang/srt/layers/vocab_parallel_embedding.py b/python/sglang/srt/layers/vocab_parallel_embedding.py index 4e7a955451..e176d89164 100644 --- a/python/sglang/srt/layers/vocab_parallel_embedding.py +++ b/python/sglang/srt/layers/vocab_parallel_embedding.py @@ -519,7 +519,13 @@ class VocabParallelEmbedding(torch.nn.Module): input_, 0, self.num_embeddings, "VocabParallelEmbedding input id" ) output_parallel = self._embed_local_shard(input_) - if self.tp_size > 1 and not get_attn_tp_context().input_scattered: + # Replicated attention-TP peers have matching shapes, so all of them + # can skip the collective for an empty idle batch. + if ( + output_parallel.numel() > 0 + and self.tp_size > 1 + and not get_attn_tp_context().input_scattered + ): if self.use_attn_tp_group: output_parallel = attn_tp_all_reduce(output_parallel) else: diff --git a/python/sglang/srt/managers/data_parallel_controller.py b/python/sglang/srt/managers/data_parallel_controller.py index dc948a16e8..ee8c971b85 100644 --- a/python/sglang/srt/managers/data_parallel_controller.py +++ b/python/sglang/srt/managers/data_parallel_controller.py @@ -20,6 +20,7 @@ import signal import threading import time from collections.abc import Callable +from dataclasses import dataclass from enum import Enum, auto from multiprocessing.connection import Connection, wait @@ -27,6 +28,12 @@ import psutil import setproctitle import zmq +from sglang.srt.elastic_ep.topology import ( + attn_replica_size, + physical_ep_rank_to_dp_rank, + physical_ep_size_to_dp_size, + validate_matching_attention_topology, +) from sglang.srt.environ import envs from sglang.srt.layers.dp_attention import compute_dp_attention_world_info from sglang.srt.managers.io_struct import ( @@ -87,6 +94,13 @@ logger = logging.getLogger(__name__) SCHEDULER_PIDS_ARG = "scheduler_pids" +@dataclass(frozen=True) +class WorkerPortHandshake: + worker_ports: list[int] + attn_tp_size: int + attn_cp_size: int + + class LoadBalanceMethod(Enum): """Load balance method.""" @@ -183,12 +197,22 @@ class DataParallelController: LoadBalanceMethod.TOTAL_TOKENS, ) - self.launch_dp_size: int = get_parallel().num_dp_ranks - self.max_dp_size: int = ( - get_parallel().max_ep_size or get_parallel().num_dp_ranks - ) + parallel = get_parallel() + self.launch_dp_size: int = parallel.num_dp_ranks + self.attn_replica_size = attn_replica_size() + if ( + get_exec().moe.elastic_ep_backend is not None + and parallel.max_ep_size is not None + and parallel.max_ep_size > parallel.tp_size + ): + self.max_dp_size = physical_ep_size_to_dp_size( + parallel.max_ep_size, + self.attn_replica_size, + ) + else: + self.max_dp_size = parallel.num_dp_ranks assert self.max_dp_size >= self.launch_dp_size, ( - f"--max-ep-size ({self.max_dp_size}) must be >= " + f"maximum DP size ({self.max_dp_size}) must be >= " f"--dp ({self.launch_dp_size})." ) @@ -489,7 +513,16 @@ class DataParallelController: ) else: # Other nodes: Receive worker ports from node 0 - return self._receive_ports_as_client(endpoint) + return self._receive_ports_as_client(endpoint).worker_ports + + @staticmethod + def _make_worker_port_handshake(worker_ports: list[int]) -> WorkerPortHandshake: + parallel = get_parallel() + return WorkerPortHandshake( + worker_ports=worker_ports, + attn_tp_size=parallel.attn_tp_size, + attn_cp_size=parallel.attn_cp_size, + ) def _broadcast_ports_as_server( self, endpoint: str, expected_clients: int, worker_ports: list[int] @@ -497,6 +530,7 @@ class DataParallelController: """Broadcast worker ports to all client nodes.""" logger.debug(f"Broadcasting worker ports to {expected_clients} client nodes") logger.debug(f"Worker ports: {worker_ports}") + handshake = self._make_worker_port_handshake(worker_ports) rep_socket = get_zmq_socket(self.context, zmq.REP, endpoint, True) @@ -508,7 +542,7 @@ class DataParallelController: logger.debug(f"Received handshake from node {client_rank}") # Send worker ports to client - sock_send(rep_socket, wrap_as_pickle(worker_ports)) + sock_send(rep_socket, wrap_as_pickle(handshake)) connected_clients += 1 logger.debug( f"Sent worker ports to {connected_clients}/{expected_clients} nodes" @@ -522,14 +556,14 @@ class DataParallelController: else: threading.Thread( target=self._reply_ports_as_server, - args=(rep_socket, worker_ports), + args=(rep_socket, handshake), daemon=True, ).start() - def _reply_ports_as_server(self, rep_socket: zmq.Socket, worker_ports: list[int]): - """Background thread: serve the pre-bound worker-port list to - late-arriving elastic joiners. Publishes port numbers only; the primary - keeps ownership of every socket.""" + def _reply_ports_as_server( + self, rep_socket: zmq.Socket, handshake: WorkerPortHandshake + ): + """Serve pre-bound worker ports and topology to elastic joiners.""" while True: try: client_rank = sock_recv(rep_socket) @@ -541,10 +575,10 @@ class DataParallelController: logger.debug(f"Received handshake from node {client_rank}") # Send worker ports to client - sock_send(rep_socket, wrap_as_pickle(worker_ports)) + sock_send(rep_socket, wrap_as_pickle(handshake)) logger.debug(f"Sent worker ports to node {client_rank}") - def _receive_ports_as_client(self, endpoint: str) -> list[int]: + def _receive_ports_as_client(self, endpoint: str) -> WorkerPortHandshake: """Receive worker ports from the server node.""" logger.debug("Connecting to node 0 to receive worker ports") node_rank = get_parallel().node_rank @@ -558,9 +592,13 @@ class DataParallelController: sock_send(req_socket, wrap_as_pickle(str(node_rank))) # Receive worker ports - worker_ports = sock_recv(req_socket) - logger.debug(f"Received {len(worker_ports)} worker ports from node 0") - return worker_ports + handshake = sock_recv(req_socket) + if not isinstance(handshake, WorkerPortHandshake): + raise RuntimeError("Invalid worker-port handshake from node 0") + logger.debug( + f"Received {len(handshake.worker_ports)} worker ports from node 0" + ) + return handshake except zmq.Again: logger.error("Timeout waiting for worker ports from node 0") raise RuntimeError( @@ -569,12 +607,6 @@ class DataParallelController: finally: req_socket.close() - def _joiner_local_tp_span(self, server_args: ServerArgs) -> int: - return get_parallel().tp_size - - def _joiner_slot_offset(self, server_args: ServerArgs) -> int: - return get_parallel().ep_join_rank_offset - def launch_dp_attention_schedulers( self, server_args: ServerArgs, port_args: PortArgs ): @@ -590,10 +622,29 @@ class DataParallelController: primary_endpoint = NetworkAddress( primary.host, primary.port + DP_ATTENTION_HANDSHAKE_PORT_DELTA ).to_tcp() - all_ports = self._receive_ports_as_client(primary_endpoint) - offset = self._joiner_slot_offset(server_args) - local_tp_span = self._joiner_local_tp_span(server_args) - broadcasted_ports = all_ports[offset : offset + local_tp_span] + handshake = self._receive_ports_as_client(primary_endpoint) + parallel = get_parallel() + validate_matching_attention_topology( + primary_attn_tp_size=handshake.attn_tp_size, + primary_attn_cp_size=handshake.attn_cp_size, + joining_attn_tp_size=parallel.attn_tp_size, + joining_attn_cp_size=parallel.attn_cp_size, + ) + primary_replica_size = handshake.attn_tp_size * handshake.attn_cp_size + offset = physical_ep_rank_to_dp_rank( + parallel.ep_join_rank_offset, + primary_replica_size, + ) + broadcasted_ports = handshake.worker_ports[ + offset : offset + parallel.num_dp_ranks + ] + if len(broadcasted_ports) != parallel.num_dp_ranks: + raise ValueError( + "Primary has insufficient reserved attention-DP worker ports " + f"for join offset {parallel.ep_join_rank_offset} " + f"(requested {parallel.num_dp_ranks}, got " + f"{len(broadcasted_ports)})." + ) elif get_parallel().node_rank == 0: # Elastic primaries reserve sockets for the maximum DP size. bind_count = ( @@ -709,9 +760,10 @@ class DataParallelController: # Scheduler internals use local ranks; logs use global ranks. offset = get_parallel().ep_join_rank_offset + dp_offset = physical_ep_rank_to_dp_rank(offset, self.attn_replica_size) display_tp_rank = tp_rank + offset display_moe_ep_rank = moe_ep_rank + offset - display_dp_rank = dp_rank + offset if dp_rank is not None else None + display_dp_rank = dp_rank + dp_offset if dp_rank is not None else None with self.env_lock, maybe_reindex_device_id(gpu_id) as gpu_id: scheduler_args = ( diff --git a/python/sglang/srt/managers/io_struct.py b/python/sglang/srt/managers/io_struct.py index d3ee899270..45d6228dab 100644 --- a/python/sglang/srt/managers/io_struct.py +++ b/python/sglang/srt/managers/io_struct.py @@ -2304,10 +2304,12 @@ class ActiveRanksOutput(BaseReq, kw_only=True): class ElasticScaleUpdateReq(BaseReq, kw_only=True): - """Report asynchronous Elastic EP scale completion or failure.""" + """Report scale completion in physical EP and logical DP-slot units.""" success: bool + # Public Elastic EP state remains physical. effective_ep_size: int + # DPC capacity and worker activation use logical DP slots. slot_offset: int = 0 slot_count: int = 0 error: Optional[str] = None diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 509ee26322..5d9f0be785 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -4822,13 +4822,19 @@ class Scheduler( if not (self.attn_dp_enabled and get_exec().moe.elastic_ep_backend is not None): return from sglang.srt.elastic_ep.elastic_ep import ElasticEPStateManager + from sglang.srt.elastic_ep.topology import ( + attn_replica_size, + collapse_physical_rank_status, + ) inst = ElasticEPStateManager.instance() if inst is not None and inst.active_ranks_cpu is not None: + dp_status = collapse_physical_rank_status( + inst.active_ranks_cpu.tolist(), + attn_replica_size(), + ) self.ipc_channels.send_to_tokenizer.send_output( - ActiveRanksOutput( - status=[bool(x) for x in inst.active_ranks_cpu.tolist()] - ) + ActiveRanksOutput(status=dp_status) ) else: logger.debug("[Elastic EP] active rank state is unavailable") @@ -5786,6 +5792,19 @@ class Scheduler( old_ep_size=old_ep_size, new_ep_size=new_ep_size, ) + from sglang.srt.elastic_ep.topology import attn_replica_size + + replica_size = attn_replica_size() + if new_ep_size % replica_size != 0: + return ScaleElasticEPReqOutput( + success=False, + message=( + f"new_ep_size ({new_ep_size}) would admit an incomplete " + f"attention replica with size {replica_size}." + ), + old_ep_size=old_ep_size, + new_ep_size=new_ep_size, + ) if ElasticEPStateManager.is_scaling(): return ScaleElasticEPReqOutput( success=False, diff --git a/python/sglang/srt/managers/scheduler_components/dp_attn.py b/python/sglang/srt/managers/scheduler_components/dp_attn.py index 7101103d5f..765ad679b6 100644 --- a/python/sglang/srt/managers/scheduler_components/dp_attn.py +++ b/python/sglang/srt/managers/scheduler_components/dp_attn.py @@ -89,24 +89,29 @@ def _resolve_elastic_world_num_dp_ranks( live_num_dp_ranks = dp_gather_width() effective_ep_size = ElasticEPStateManager.get_effective_ep_size() # Query live membership because elastic joins can expand WORLD. + from sglang.srt.elastic_ep.topology import attn_replica_size + + replica_size = attn_replica_size() world_size = torch.distributed.get_world_size(group) - if live_num_dp_ranks != effective_ep_size: + if live_num_dp_ranks * replica_size != effective_ep_size: raise RuntimeError( "[Elastic EP] WORLD MLP sync num_dp_ranks is out of sync: " f"rank={torch.distributed.get_rank(group)} " f"live_num_dp_ranks={live_num_dp_ranks} " f"effective_ep_size={effective_ep_size} " + f"attn_replica_size={replica_size} " f"world_size={world_size} configured_num_dp_ranks={num_dp_ranks} " f"local_num_tokens={local_num_tokens} " f"local_forward_mode={local_forward_mode}" ) - if live_num_dp_ranks > world_size: + if effective_ep_size > world_size: raise RuntimeError( - "[Elastic EP] WORLD MLP sync num_dp_ranks exceeds WORLD size: " + "[Elastic EP] WORLD MLP sync EP size exceeds WORLD size: " f"rank={torch.distributed.get_rank(group)} " f"live_num_dp_ranks={live_num_dp_ranks} world_size={world_size} " - f"effective_ep_size={effective_ep_size}" + f"effective_ep_size={effective_ep_size} " + f"attn_replica_size={replica_size}" ) return live_num_dp_ranks diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 7a5eef52f1..4a393cd000 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -63,6 +63,10 @@ from sglang.srt.configs.model_config import ModelConfig from sglang.srt.constants import HEALTH_CHECK_RID_PREFIX from sglang.srt.disaggregation.encoder.receiver import create_mm_receiver from sglang.srt.disaggregation.utils import DisaggregationMode +from sglang.srt.elastic_ep.topology import ( + attn_replica_size, + physical_ep_size_to_dp_size, +) from sglang.srt.environ import envs from sglang.srt.layers.joint_schema_head import ( max_joint_prompt_tokens, @@ -520,7 +524,9 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): self.server_args = server_args assert_published(server_args, role="tokenizer") self.startup_time: Optional[Dict[str, Any]] = None - self.elastic_worker_count = get_parallel().num_dp_ranks + parallel = get_parallel() + self.elastic_worker_count = parallel.num_dp_ranks + self.attn_replica_size = attn_replica_size() self.elastic_pending_ep_size = None self.elastic_scale_phase = "idle" self.elastic_last_error = None @@ -3652,16 +3658,18 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): return self._dispatch_to_scheduler(msg) - self.elastic_worker_count = msg.effective_ep_size + self.elastic_worker_count = physical_ep_size_to_dp_size( + msg.effective_ep_size, self.attn_replica_size + ) self.elastic_pending_ep_size = None self.elastic_scale_phase = "serving_expanded" self.elastic_last_error = None - self.update_control_communicator_fan_out(msg.effective_ep_size) + self.update_control_communicator_fan_out(self.elastic_worker_count) def get_elastic_ep_state(self): return { "is_scaling_elastic_ep": self.elastic_pending_ep_size is not None, - "effective_ep_size": self.elastic_worker_count, + "effective_ep_size": self.elastic_worker_count * self.attn_replica_size, "pending_ep_size": self.elastic_pending_ep_size, "scale_phase": self.elastic_scale_phase, "last_error": self.elastic_last_error, @@ -3678,7 +3686,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): "A previous scale operation has not completed yet. Wait until " "all pending ranks have joined before issuing another scale." ), - old_ep_size=self.elastic_worker_count, + old_ep_size=self.elastic_worker_count * self.attn_replica_size, new_ep_size=obj.new_ep_size, pending_ep_size=self.elastic_pending_ep_size, scale_phase=self.elastic_scale_phase, diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index e3618f516f..c64d7a0974 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -49,8 +49,14 @@ from sglang.srt.elastic_ep.elastic_ep import ( maybe_recover_ep_ranks, register_scale_cohort, try_admit_scale_ranks, + update_dp_attention_for_elastic_ep, + validate_scale_cohort_topology, ) from sglang.srt.elastic_ep.expert_backup_client import ExpertBackupClient +from sglang.srt.elastic_ep.topology import ( + attn_replica_size, + physical_ep_size_to_dp_size, +) from sglang.srt.environ import envs from sglang.srt.eplb.eplb_manager import EPLBManager from sglang.srt.eplb.expert_distribution import ( @@ -545,17 +551,13 @@ class ModelRunner: ) ) - from sglang.srt.layers.dp_attention import ( - enable_joiner_all_gather, - update_dp_attention_post_scale, - ) + from sglang.srt.layers.dp_attention import enable_joiner_all_gather enable_joiner_all_gather() - update_dp_attention_post_scale( - new_dp_size=join_effective_ep_size, - new_dp_rank=global_ep_rank, + update_dp_attention_for_elastic_ep( + physical_ep_size=join_effective_ep_size, + physical_ep_rank=global_ep_rank, ) - get_parallel().override_permanently(num_dp_ranks=join_effective_ep_size) if self.eplb_manager is not None: self.eplb_manager.disable_rebalance( "EPLB rebalance is disabled while elastic EP scale-up " @@ -2326,13 +2328,12 @@ class ModelRunner: "is being finalized" ) - from sglang.srt.layers.dp_attention import update_dp_attention_post_scale - - update_dp_attention_post_scale( - new_dp_size=target_size, - new_dp_rank=self._elastic_global_rank(), + replica_size = attn_replica_size() + effective_dp_size = physical_ep_size_to_dp_size(effective_size, replica_size) + target_dp_size = update_dp_attention_for_elastic_ep( + physical_ep_size=target_size, + physical_ep_rank=self._elastic_global_rank(), ) - get_parallel().override_permanently(num_dp_ranks=target_size) recapture_cuda_graph = self._elastic_cuda_graph_enabled() if recapture_cuda_graph: @@ -2351,8 +2352,8 @@ class ModelRunner: self._pending_elastic_scale_update = ElasticScaleUpdateReq( success=True, effective_ep_size=target_size, - slot_offset=effective_size, - slot_count=target_size - effective_size, + slot_offset=effective_dp_size, + slot_count=target_dp_size - effective_dp_size, ) logger.info( "[Elastic EP] Scale completed: old_ep_size=%d " @@ -2472,6 +2473,19 @@ class ModelRunner: ): logger.error("[Elastic EP] %s", error) return + try: + validate_scale_cohort_topology(cohort) + except ValueError as exc: + error = str(exc) + ElasticEPStateManager.fail_scale(error) + self._reset_eplb_after_elastic_scale_failure() + self._report_elastic_scale_failure(error, effective_size) + if ( + get_parallel().tp_rank == 0 + and not get_exec().moe.is_ep_scale_joiner + ): + logger.error("[Elastic EP] %s", error) + return cuda_graph_enabled = self._elastic_cuda_graph_enabled() if cohort.cuda_graph_enabled != cuda_graph_enabled: error = ( diff --git a/test/manual/ep/test_elastic_scale.py b/test/manual/ep/test_elastic_scale.py index 8cc30557d1..62b575e601 100644 --- a/test/manual/ep/test_elastic_scale.py +++ b/test/manual/ep/test_elastic_scale.py @@ -2,6 +2,7 @@ Test classes: TestElasticScaleUp4To5CudaGraph single-rank decode graph recapture + TestElasticScaleUpDP2TP4ToDP4TP8 two attention-TP2 scale-ups (8 GPUs) TestElasticScaleUp4To6 primary + joiner scale-up (6 GPUs) TestElasticScaleUp4To5To6 two consecutive single-rank scale-ups TestElasticScaleUp4To8 full primary + joiner scale-up (8 GPUs) @@ -123,10 +124,12 @@ PRE_SCALE_JOINER_DELAY_SEC = float( def _scale_up_common_args( dist_init_addr: str, tp_size: int, + dp_size: int, nnodes: int, node_rank: int, cuda_graph_args: list[str], moe_dense_tp_size: int | None, + dcp_size: int, ) -> list[str]: args = [ "--trust-remote-code", @@ -137,7 +140,9 @@ def _scale_up_common_args( "--tp", str(tp_size), "--attn-dp-size", - str(tp_size), + str(dp_size), + "--dcp-size", + str(dcp_size), "--enable-dp-lm-head", "--elastic-ep-backend", "mooncake", @@ -167,14 +172,20 @@ def _scale_up_common_args( class _ElasticScaleUpEndToEndBase(CustomTestCase): - """Shared scale-up E2E plumbing. Subclasses set JOIN_TP/JOIN_NNODES/JOIN_NODE_RANK.""" + """Shared scale-up E2E plumbing for physical EP and logical DP growth.""" + PRIMARY_TP: int = LAUNCH_EP_SIZE + PRIMARY_DP: int = LAUNCH_EP_SIZE JOIN_TP: int + JOIN_DP: int JOIN_NNODES: int JOIN_NODE_RANK: int TARGET_EP_SIZE: int + TARGET_DP_SIZE: int CUDA_GRAPH_ARGS: list[str] MOE_DENSE_TP_SIZE: int | None = 1 + DCP_SIZE: int = 1 + ATTENTION_BACKEND_ARGS: list[str] = [] def setUp(self): if ( @@ -194,20 +205,25 @@ class _ElasticScaleUpEndToEndBase(CustomTestCase): primary_args = _scale_up_common_args( DIST_INIT_ADDR, - tp_size=LAUNCH_EP_SIZE, + tp_size=cls.PRIMARY_TP, + dp_size=cls.PRIMARY_DP, nnodes=1, node_rank=0, cuda_graph_args=cls.CUDA_GRAPH_ARGS, moe_dense_tp_size=cls.MOE_DENSE_TP_SIZE, + dcp_size=cls.DCP_SIZE, ) + primary_args.extend(cls.ATTENTION_BACKEND_ARGS) primary_env = os.environ.copy() visible_devices = _visible_device_ids() - if len(visible_devices) < LAUNCH_EP_SIZE: + if len(visible_devices) < cls.PRIMARY_TP: raise RuntimeError( - f"Scale-up requires {LAUNCH_EP_SIZE} visible GPUs, got " + f"Scale-up requires {cls.PRIMARY_TP} visible GPUs, got " f"{len(visible_devices)}" ) - primary_env["CUDA_VISIBLE_DEVICES"] = ",".join(visible_devices[:LAUNCH_EP_SIZE]) + primary_env["CUDA_VISIBLE_DEVICES"] = ",".join( + visible_devices[: cls.PRIMARY_TP] + ) cls.process = popen_launch_server( cls.model, cls.base_url, @@ -222,6 +238,7 @@ class _ElasticScaleUpEndToEndBase(CustomTestCase): *, rank_offset: int, join_tp: int, + join_dp: int, port: int, ) -> subprocess.Popen: cmd = [ @@ -232,11 +249,14 @@ class _ElasticScaleUpEndToEndBase(CustomTestCase): *_scale_up_common_args( DIST_INIT_ADDR, tp_size=join_tp, + dp_size=join_dp, nnodes=cls.JOIN_NNODES, node_rank=cls.JOIN_NODE_RANK, cuda_graph_args=cls.CUDA_GRAPH_ARGS, moe_dense_tp_size=cls.MOE_DENSE_TP_SIZE, + dcp_size=cls.DCP_SIZE, ), + *cls.ATTENTION_BACKEND_ARGS, "--elastic-ep-join-mode", "scale", "--elastic-ep-join-rank-offset", @@ -336,12 +356,15 @@ class _ElasticScaleUpEndToEndBase(CustomTestCase): *, old_ep_size: int, target_ep_size: int, + target_dp_size: int, join_tp: int, + join_dp: int, port: int, ) -> None: joining_proc = self._launch_joining_group( rank_offset=old_ep_size, join_tp=join_tp, + join_dp=join_dp, port=port, ) self.assertIsNone( @@ -371,7 +394,7 @@ class _ElasticScaleUpEndToEndBase(CustomTestCase): self.assertIsNone(state.get("last_error")) self._generate_ok( "on newest joiner", - routed_dp_rank=target_ep_size - 1, + routed_dp_rank=target_dp_size - 1, ) return try: @@ -412,7 +435,9 @@ class _ElasticScaleUpEndToEndBase(CustomTestCase): self._scale_once( old_ep_size=LAUNCH_EP_SIZE, target_ep_size=self.TARGET_EP_SIZE, + target_dp_size=self.TARGET_DP_SIZE, join_tp=self.JOIN_TP, + join_dp=self.JOIN_DP, port=PORT_B, ) @@ -423,6 +448,48 @@ class _ElasticScaleUpEndToEndBase(CustomTestCase): self._generate_logprob_ok("after post-scale workload") +@unittest.skipUnless( + _count_visible_gpus() >= MAX_EP_SIZE, + f"Repeated attention-TP2 scale-up E2E needs {MAX_EP_SIZE} GPUs.", +) +class TestElasticScaleUpDP2TP4ToDP4TP8(_ElasticScaleUpEndToEndBase): + """Admit two complete attention-TP2/DCP2 replicas.""" + + PRIMARY_DP = 2 + JOIN_TP = 2 + JOIN_DP = 1 + JOIN_NNODES = 2 + JOIN_NODE_RANK = 1 + TARGET_EP_SIZE = 8 + TARGET_DP_SIZE = 4 + CUDA_GRAPH_ARGS = DISABLED_CUDA_GRAPH_ARGS + DCP_SIZE = 2 + ATTENTION_BACKEND_ARGS = [ + "--dcp-comm-backend=ag_rs", + "--attention-backend=flashinfer", + "--flashinfer-mla-disable-ragged", + ] + + def test_scale_up_on_demand(self): + self._generate_ok("pre-scale") + + for old_ep, new_ep, new_dp, port in ( + (4, 6, 3, PORT_B), + (6, 8, 4, PORT_C), + ): + self._scale_once( + old_ep_size=old_ep, + target_ep_size=new_ep, + target_dp_size=new_dp, + join_tp=2, + join_dp=1, + port=port, + ) + self._generate_logprob_ok(f"after scale to EP{new_ep}") + + self._run_post_scale_gsm8k() + + @unittest.skipUnless( _count_visible_gpus() >= 6, "4-to-6 scale-up E2E needs 6 GPUs.", @@ -431,9 +498,11 @@ class TestElasticScaleUp4To6(_ElasticScaleUpEndToEndBase): """Scale from four to six ranks.""" JOIN_TP = 2 + JOIN_DP = 2 JOIN_NNODES = 2 JOIN_NODE_RANK = 1 TARGET_EP_SIZE = 6 + TARGET_DP_SIZE = 6 CUDA_GRAPH_ARGS = DISABLED_CUDA_GRAPH_ARGS MOE_DENSE_TP_SIZE = None @@ -446,9 +515,11 @@ class TestElasticScaleUp4To5To6(_ElasticScaleUpEndToEndBase): """Scale from four to five and then from five to six ranks.""" JOIN_TP = 1 + JOIN_DP = 1 JOIN_NNODES = 2 JOIN_NODE_RANK = 1 TARGET_EP_SIZE = 6 + TARGET_DP_SIZE = 6 CUDA_GRAPH_ARGS = DISABLED_CUDA_GRAPH_ARGS def test_scale_up_on_demand(self): @@ -457,7 +528,9 @@ class TestElasticScaleUp4To5To6(_ElasticScaleUpEndToEndBase): self._scale_once( old_ep_size=4, target_ep_size=5, + target_dp_size=5, join_tp=1, + join_dp=1, port=PORT_B, ) self._generate_ok("after first scale") @@ -466,7 +539,9 @@ class TestElasticScaleUp4To5To6(_ElasticScaleUpEndToEndBase): self._scale_once( old_ep_size=5, target_ep_size=6, + target_dp_size=6, join_tp=1, + join_dp=1, port=PORT_C, ) self._generate_ok("after second scale") @@ -483,9 +558,11 @@ class TestElasticScaleUp4To8(_ElasticScaleUpEndToEndBase): """Scale from four to eight ranks.""" JOIN_TP = LAUNCH_EP_SIZE + JOIN_DP = LAUNCH_EP_SIZE JOIN_NNODES = 2 JOIN_NODE_RANK = 1 TARGET_EP_SIZE = MAX_EP_SIZE + TARGET_DP_SIZE = MAX_EP_SIZE CUDA_GRAPH_ARGS = DISABLED_CUDA_GRAPH_ARGS MOE_DENSE_TP_SIZE = None @@ -498,9 +575,11 @@ class TestElasticScaleUp4To5CudaGraph(_ElasticScaleUpEndToEndBase): """Scale from four to five ranks and recapture decode CUDA graphs.""" JOIN_TP = 1 + JOIN_DP = 1 JOIN_NNODES = 2 JOIN_NODE_RANK = 1 TARGET_EP_SIZE = 5 + TARGET_DP_SIZE = 5 CUDA_GRAPH_ARGS = ELASTIC_CUDA_GRAPH_ARGS @@ -512,9 +591,11 @@ class TestElasticScaleUp4To8CudaGraph(_ElasticScaleUpEndToEndBase): """Scale from four to eight ranks and recapture decode CUDA graphs.""" JOIN_TP = LAUNCH_EP_SIZE + JOIN_DP = LAUNCH_EP_SIZE JOIN_NNODES = 2 JOIN_NODE_RANK = 1 TARGET_EP_SIZE = MAX_EP_SIZE + TARGET_DP_SIZE = MAX_EP_SIZE CUDA_GRAPH_ARGS = ELASTIC_CUDA_GRAPH_ARGS diff --git a/test/registered/kernels/ops/embeddings/test_vocab_parallel_embedding.py b/test/registered/kernels/ops/embeddings/test_vocab_parallel_embedding.py index 6b1de48277..0eef101ec7 100644 --- a/test/registered/kernels/ops/embeddings/test_vocab_parallel_embedding.py +++ b/test/registered/kernels/ops/embeddings/test_vocab_parallel_embedding.py @@ -1,4 +1,5 @@ import types +from unittest.mock import patch import pytest import torch @@ -137,6 +138,37 @@ def test_use_triton_embedding_gate(): assert not gate(_stub_layer(weight=weight), input_ids) +@pytest.mark.parametrize("use_attn_tp_group", [False, True]) +def test_empty_embedding_skips_all_reduce(use_attn_tp_group): + input_ids = torch.empty((0,), dtype=torch.int64, device="cuda") + output = torch.empty((0, 32), dtype=torch.bfloat16, device="cuda") + layer = types.SimpleNamespace( + tp_size=2, + use_attn_tp_group=use_attn_tp_group, + num_embeddings=128, + _embed_local_shard=lambda _: output, + ) + + with ( + patch( + "sglang.srt.layers.vocab_parallel_embedding.get_attn_tp_context", + return_value=types.SimpleNamespace(input_scattered=False), + ), + patch( + "sglang.srt.layers.vocab_parallel_embedding." + "tensor_model_parallel_all_reduce" + ) as tensor_parallel_all_reduce, + patch( + "sglang.srt.layers.vocab_parallel_embedding.attn_tp_all_reduce" + ) as attn_tp_all_reduce, + ): + actual = VocabParallelEmbedding.forward(layer, input_ids) + + assert actual is output + tensor_parallel_all_reduce.assert_not_called() + attn_tp_all_reduce.assert_not_called() + + if __name__ == "__main__": import sys diff --git a/test/registered/unit/elastic_ep/test_topology.py b/test/registered/unit/elastic_ep/test_topology.py new file mode 100644 index 0000000000..d9709136f3 --- /dev/null +++ b/test/registered/unit/elastic_ep/test_topology.py @@ -0,0 +1,147 @@ +import sys +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import msgspec +import pytest + +from sglang.srt.arg_groups.parallel_hook import handle_elastic_ep +from sglang.srt.elastic_ep.elastic_ep import ( + ElasticEPStateManager, + ScaleCohort, + register_scale_cohort, + validate_scale_cohort_topology, +) +from sglang.srt.elastic_ep.topology import ( + attn_replica_size, + collapse_physical_rank_status, + physical_ep_rank_to_dp_rank, + physical_ep_size_to_dp_size, +) +from sglang.srt.managers.io_struct import ScaleElasticEPReqInput +from sglang.srt.managers.scheduler import Scheduler +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=1, suite="base-a-test-cpu") + + +def test_attention_topology_mapping_and_complete_boundaries(): + parallel = SimpleNamespace(attn_tp_size=2, attn_cp_size=1) + with patch("sglang.srt.elastic_ep.topology.get_parallel", return_value=parallel): + replica_size = attn_replica_size() + assert replica_size == 2 + assert physical_ep_size_to_dp_size(6, replica_size) == 3 + mapping = [physical_ep_rank_to_dp_rank(rank, replica_size) for rank in range(6)] + assert mapping == [0, 0, 1, 1, 2, 2] + with pytest.raises(ValueError, match="must be divisible"): + physical_ep_size_to_dp_size(5, replica_size) + assert physical_ep_size_to_dp_size(5, 1) == 5 + + +def test_scale_cohort_records_and_validates_attention_topology(): + store = MagicMock() + joining_parallel = SimpleNamespace(attn_tp_size=2, attn_cp_size=1) + with ( + patch( + "sglang.srt.elastic_ep.elastic_ep.get_global_tcp_store", + return_value=store, + ), + patch( + "sglang.srt.elastic_ep.elastic_ep.get_parallel", + return_value=joining_parallel, + ), + ): + register_scale_cohort( + rank_offset=4, + target_ep_size=6, + cuda_graph_enabled=True, + ) + + key, payload = store.set.call_args.args + assert key == "elastic_ep/scale_cohort/4" + cohort = msgspec.json.decode(payload, type=ScaleCohort) + assert cohort == ScaleCohort( + target_ep_size=6, + cuda_graph_enabled=True, + attn_tp_size=2, + attn_cp_size=1, + ) + + with patch( + "sglang.srt.elastic_ep.elastic_ep.get_parallel", + return_value=joining_parallel, + ): + validate_scale_cohort_topology(cohort) + + primary_parallel = SimpleNamespace(attn_tp_size=1, attn_cp_size=1) + with ( + patch( + "sglang.srt.elastic_ep.elastic_ep.get_parallel", + return_value=primary_parallel, + ), + pytest.raises(ValueError, match="same attention topology"), + ): + validate_scale_cohort_topology(cohort) + + +def test_attention_tp_scale_requires_moe_dense_tp_one(): + cfg = SimpleNamespace( + elastic_ep_backend="mooncake", + enable_eplb=False, + pp_size=1, + mooncake_ib_device=None, + ep_join_mode=None, + ep_join_rank_offset=0, + max_ep_size=8, + tp_size=4, + elastic_ep_initial_size=4, + attn_dp_size=2, + moe_dense_tp_size=None, + ) + resolved = SimpleNamespace(attn_cp_size=1) + with ( + patch("sglang.srt.arg_groups.parallel_hook.resolving_view", return_value=cfg), + patch( + "sglang.srt.arg_groups.parallel_hook.resolved_view", + return_value=resolved, + ), + patch("sglang.srt.arg_groups.parallel_hook.declare_resolution"), + patch( + "sglang.srt.arg_groups.validation_hook.validate_ib_devices", + side_effect=lambda value: value, + ), + pytest.raises(AssertionError, match="moe-dense-tp-size 1"), + ): + handle_elastic_ep(SimpleNamespace()) + + +def test_incomplete_replica_is_rejected_before_scale_request(): + parallel = SimpleNamespace(max_world_size=8, attn_tp_size=2, attn_cp_size=1) + with ( + patch("sglang.srt.managers.scheduler.get_parallel", return_value=parallel), + patch("sglang.srt.elastic_ep.topology.get_parallel", return_value=parallel), + patch.object(ElasticEPStateManager, "get_effective_ep_size", return_value=4), + patch.object(ElasticEPStateManager, "request_scale") as request_scale, + ): + result = Scheduler.handle_scale_elastic_ep( + SimpleNamespace(), ScaleElasticEPReqInput(new_ep_size=5) + ) + + assert not result.success + assert "incomplete attention replica" in result.message + request_scale.assert_not_called() + + +def test_physical_health_uses_all_members_rule(): + assert collapse_physical_rank_status( + [True, True, True, True, True, False], attn_replica_size=2 + ) == [True, True, False] + assert collapse_physical_rank_status([True, False, True], attn_replica_size=1) == [ + True, + False, + True, + ] + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/unit/layers/test_dp_attention.py b/test/registered/unit/layers/test_dp_attention.py index ac51d0de5a..0c00bb77e0 100644 --- a/test/registered/unit/layers/test_dp_attention.py +++ b/test/registered/unit/layers/test_dp_attention.py @@ -15,7 +15,7 @@ from sglang.srt.layers.dp_attention import ( set_dp_buffer_len, set_dp_buffer_len_from_batch, ) -from sglang.srt.runtime_context import get_parallel +from sglang.srt.runtime_context import get_flags, get_parallel from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=5, suite="base-a-test-cpu") @@ -39,7 +39,7 @@ class TestSetDpBufferLenFromBatch(unittest.TestCase): def test_publishes_the_padded_list_and_this_ranks_entry(self): batch = _batch() - with get_parallel().override(attn_dp_rank=1): + with get_parallel().override(attn_dp_rank=1, attn_dp_size=2): set_dp_buffer_len_from_batch(batch) self.assertEqual(get_global_dp_buffer_len(), 8) self.assertEqual(get_local_dp_buffer_len(), 4) @@ -56,7 +56,7 @@ class TestSetDpBufferLenFromBatch(unittest.TestCase): global_num_tokens_padded_cpu=None, dp_padding_mode=DpPaddingMode.SUM_LEN, ) - with get_parallel().override(attn_dp_rank=0): + with get_parallel().override(attn_dp_rank=0, attn_dp_size=2): set_dp_buffer_len_from_batch(batch) self.assertEqual(get_local_dp_buffer_len(), 3) self.assertFalse(is_dp_max_padding()) @@ -74,6 +74,31 @@ class TestSetDpBufferLenFromBatch(unittest.TestCase): set_dp_buffer_len_from_batch(batch) self.assertEqual(get_local_dp_buffer_len(), 3) + def test_elastic_joiner_uses_its_expanded_world_slot(self): + batch = _batch( + global_dp_buffer_len=4, + global_num_tokens_cpu=[3, 1, 0], + global_num_tokens_padded_cpu=None, + global_num_tokens_gpu=torch.tensor([3, 1, 0]), + dp_padding_mode=DpPaddingMode.SUM_LEN, + ) + with ( + get_flags().dp.override( + use_world_group_for_gather=True, + joiner_skip_all_gather=False, + ), + get_parallel().override( + num_dp_ranks=3, + tp_rank=0, + ep_join_rank_offset=4, + attn_tp_size=2, + attn_cp_size=1, + attn_dp_rank=0, + ), + ): + set_dp_buffer_len_from_batch(batch) + self.assertEqual(get_local_dp_buffer_len(), 0) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/managers/test_data_parallel_controller.py b/test/registered/unit/managers/test_data_parallel_controller.py index a513cf872e..e58627442f 100644 --- a/test/registered/unit/managers/test_data_parallel_controller.py +++ b/test/registered/unit/managers/test_data_parallel_controller.py @@ -29,6 +29,7 @@ from sglang.srt.managers.data_parallel_controller import ( DataParallelController, DPBudget, LoadBalanceMethod, + WorkerPortHandshake, ) from sglang.srt.managers.load_snapshot import LoadSnapshot from sglang.srt.runtime_context import get_context @@ -77,6 +78,75 @@ def _req( ) +class TestJoinerWorkerPortAdmission(CustomTestCase): + def _launch_joiner(self, parallel, handshake, launch): + controller = DataParallelController.__new__(DataParallelController) + controller._receive_ports_as_client = MagicMock(return_value=handshake) + controller.launch_tensor_parallel_group = launch + execution = SimpleNamespace(moe=SimpleNamespace(is_ep_scale_joiner=True)) + + with ( + patch( + "sglang.srt.managers.data_parallel_controller.get_parallel", + return_value=parallel, + ), + patch( + "sglang.srt.managers.data_parallel_controller.get_exec", + return_value=execution, + ), + ): + controller.launch_dp_attention_schedulers( + SimpleNamespace(), SimpleNamespace() + ) + + def test_topology_mismatch_is_rejected_before_scheduler_launch(self): + mismatches = [ + # A wider joiner would map physical offset 4 onto active primary slot 2. + ((1, 1), (2, 1), 1), + # A narrower joiner would select beyond the primary's reserved slots. + ((2, 1), (1, 1), 2), + ] + for primary, joining, num_dp_ranks in mismatches: + with self.subTest(primary=primary, joining=joining): + parallel = SimpleNamespace( + dist_init_addr="127.0.0.1:20000", + ep_join_rank_offset=4, + num_dp_ranks=num_dp_ranks, + attn_tp_size=joining[0], + attn_cp_size=joining[1], + ) + handshake = WorkerPortHandshake( + worker_ports=[1000, 1001, 1002, 1003], + attn_tp_size=primary[0], + attn_cp_size=primary[1], + ) + launch = MagicMock() + + with self.assertRaisesRegex(ValueError, "same attention topology"): + self._launch_joiner(parallel, handshake, launch) + + launch.assert_not_called() + + def test_matching_topology_selects_reserved_logical_dp_slot(self): + parallel = SimpleNamespace( + dist_init_addr="127.0.0.1:20000", + ep_join_rank_offset=4, + num_dp_ranks=1, + attn_tp_size=2, + attn_cp_size=1, + ) + handshake = WorkerPortHandshake( + worker_ports=[1000, 1001, 1002, 1003], + attn_tp_size=2, + attn_cp_size=1, + ) + launch = MagicMock() + + self._launch_joiner(parallel, handshake, launch) + + self.assertEqual(launch.call_args.args[-1], [1002]) + + class TestLocalKvEventSources(CustomTestCase): @staticmethod def _source(rank, block_size=64): @@ -124,6 +194,7 @@ class TestLocalKvEventSources(CustomTestCase): controller.scheduler_procs = [] controller.local_kv_event_sources = [] controller.run_scheduler_process_func = MagicMock() + controller.attn_replica_size = 1 controller.prelaunched_schedulers = None # TP8/DP4 over two nodes: this follower's four schedulers own ranks 2,3. readers = [ diff --git a/test/registered/unit/test_runtime_context.py b/test/registered/unit/test_runtime_context.py index d332b8ddfc..e9bb62fe67 100644 --- a/test/registered/unit/test_runtime_context.py +++ b/test/registered/unit/test_runtime_context.py @@ -485,7 +485,7 @@ class TestStampedRanks(_IsolatedOverrides): self.addCleanup(restore) publish( - ServerArgs(model_path="dummy", tp_size=8, attn_dp_size=8), + ServerArgs(model_path="dummy", tp_size=8, attn_dp_size=4), role="test", ranks=SpawnRanks(world_rank=3), ) @@ -493,13 +493,14 @@ class TestStampedRanks(_IsolatedOverrides): dp_flags.use_world_group_for_gather = False self.assertEqual(dp_gather_slot(), parallel.attn_dp_rank) - # After a scale-up the gather spans the expanded WORLD, and the joining - # cohort is numbered from its offset. + # After scale-up the gather spans expanded WORLD, but its slots remain + # logical DP replicas. Physical ranks 8 and 9 therefore share DP slot 4. dp_flags.use_world_group_for_gather = True dp_flags.joiner_skip_all_gather = False parallel.override_permanently(ep_join_rank_offset=8) - self.assertEqual(dp_gather_slot(), 8 + parallel.tp_rank) - self.assertEqual(parallel.attn_dp_size, 8) + self.assertEqual(dp_gather_slot(), (8 + parallel.tp_rank) // 2) + self.assertEqual(parallel.attn_dp_size, 4) + self.assertEqual(parallel.attn_tp_size, 2) self.assertEqual(parallel.tp_size, 8) def test_a_scale_up_writes_no_width(self):