[4/N] elastic-ep: Scale at DP-attention replica granularity (#33728)
This commit is contained in:
@@ -558,6 +558,43 @@ def handle_elastic_ep(server_args: Any):
|
||||
)
|
||||
if scaling_active:
|
||||
resolved = resolved_view(server_args)
|
||||
from sglang.srt.elastic_ep.topology import physical_ep_size_to_dp_size
|
||||
|
||||
try:
|
||||
divisor = cfg.attn_dp_size * resolved.attn_cp_size
|
||||
if (
|
||||
cfg.tp_size <= 0
|
||||
or cfg.attn_dp_size <= 0
|
||||
or resolved.attn_cp_size <= 0
|
||||
or cfg.tp_size % divisor
|
||||
):
|
||||
raise ValueError(
|
||||
"Invalid attention topology: "
|
||||
f"tp_size={cfg.tp_size}, attn_dp_size={cfg.attn_dp_size}, "
|
||||
f"attn_cp_size={resolved.attn_cp_size}."
|
||||
)
|
||||
attn_tp_size = derive_attn_tp_size(
|
||||
tp_size=cfg.tp_size,
|
||||
attn_cp_size=resolved.attn_cp_size,
|
||||
attn_dp_size=cfg.attn_dp_size,
|
||||
)
|
||||
attn_replica_size = attn_tp_size * resolved.attn_cp_size
|
||||
physical_ep_size_to_dp_size(cfg.max_ep_size, attn_replica_size)
|
||||
physical_ep_size_to_dp_size(
|
||||
cfg.elastic_ep_initial_size or cfg.tp_size, attn_replica_size
|
||||
)
|
||||
if cfg.ep_join_mode == "scale":
|
||||
physical_ep_size_to_dp_size(cfg.ep_join_rank_offset, attn_replica_size)
|
||||
except ValueError as exc:
|
||||
raise AssertionError(str(exc)) from exc
|
||||
|
||||
if attn_tp_size > 1:
|
||||
assert cfg.moe_dense_tp_size == 1, (
|
||||
"Elastic EP scale-up with attention TP greater than one requires "
|
||||
"--moe-dense-tp-size 1 "
|
||||
f"(got moe_dense_tp_size={cfg.moe_dense_tp_size})."
|
||||
)
|
||||
|
||||
assert cfg.elastic_ep_scale_timeout > 0, (
|
||||
"--elastic-ep-scale-timeout must be greater than zero."
|
||||
)
|
||||
@@ -622,6 +659,11 @@ def handle_elastic_ep(server_args: Any):
|
||||
"Elastic EP scale-up requires --pp-size 1 "
|
||||
f"(got pp_size={cfg.pp_size}); WORLD must not span PP stages."
|
||||
)
|
||||
assert attn_tp_size % cfg.dcp_size == 0, (
|
||||
"Elastic EP scale-up requires --dcp-size to divide the fixed "
|
||||
"attention TP width "
|
||||
f"(got dcp_size={cfg.dcp_size}, attn_tp_size={attn_tp_size})."
|
||||
)
|
||||
|
||||
decode_backend = cfg.cuda_graph_config.decode.backend
|
||||
assert decode_backend in (Backend.DISABLED, Backend.FULL), (
|
||||
@@ -673,12 +715,8 @@ def handle_elastic_ep(server_args: Any):
|
||||
)
|
||||
assert resolved.ep_size == cfg.tp_size, (
|
||||
"Elastic EP scale-up requires ep_size == tp_size "
|
||||
f"(got ep_size={resolved.ep_size}, tp_size={cfg.tp_size}); EP, TP "
|
||||
"and the attention DP group must all coincide with WORLD."
|
||||
)
|
||||
assert cfg.attn_dp_size == cfg.tp_size, (
|
||||
"Elastic EP scale-up requires attn_dp_size == tp_size "
|
||||
f"(got attn_dp_size={cfg.attn_dp_size}, tp_size={cfg.tp_size})."
|
||||
f"(got ep_size={resolved.ep_size}, tp_size={cfg.tp_size}); "
|
||||
"physical EP membership must coincide with the launch-time TP span."
|
||||
)
|
||||
assert resolved.moe_a2a_backend == "nixl", (
|
||||
"Elastic EP scale-up requires --moe-a2a-backend nixl "
|
||||
|
||||
@@ -30,6 +30,8 @@ _SCALE_COHORT_KEY_PREFIX = "elastic_ep/scale_cohort"
|
||||
class ScaleCohort(msgspec.Struct, frozen=True, kw_only=True):
|
||||
target_ep_size: int
|
||||
cuda_graph_enabled: bool
|
||||
attn_tp_size: int
|
||||
attn_cp_size: int
|
||||
|
||||
|
||||
def register_scale_cohort(
|
||||
@@ -38,10 +40,13 @@ def register_scale_cohort(
|
||||
store = get_global_tcp_store()
|
||||
if store is None:
|
||||
raise RuntimeError("Elastic EP scale-up requires the global TCPStore.")
|
||||
parallel = get_parallel()
|
||||
payload = msgspec.json.encode(
|
||||
ScaleCohort(
|
||||
target_ep_size=target_ep_size,
|
||||
cuda_graph_enabled=cuda_graph_enabled,
|
||||
attn_tp_size=parallel.attn_tp_size,
|
||||
attn_cp_size=parallel.attn_cp_size,
|
||||
)
|
||||
)
|
||||
store.set(f"{_SCALE_COHORT_KEY_PREFIX}/{rank_offset}", payload)
|
||||
@@ -57,6 +62,39 @@ def get_scale_cohort(rank_offset: int) -> Optional[ScaleCohort]:
|
||||
return msgspec.json.decode(store.get(key), type=ScaleCohort)
|
||||
|
||||
|
||||
def validate_scale_cohort_topology(cohort: ScaleCohort) -> None:
|
||||
from sglang.srt.elastic_ep.topology import validate_matching_attention_topology
|
||||
|
||||
parallel = get_parallel()
|
||||
validate_matching_attention_topology(
|
||||
primary_attn_tp_size=parallel.attn_tp_size,
|
||||
primary_attn_cp_size=parallel.attn_cp_size,
|
||||
joining_attn_tp_size=cohort.attn_tp_size,
|
||||
joining_attn_cp_size=cohort.attn_cp_size,
|
||||
)
|
||||
|
||||
|
||||
def update_dp_attention_for_elastic_ep(
|
||||
*, physical_ep_size: int, physical_ep_rank: int
|
||||
) -> int:
|
||||
"""Apply a physical EP topology to the logical DP-attention runtime."""
|
||||
from sglang.srt.elastic_ep.topology import (
|
||||
attn_replica_size,
|
||||
physical_ep_rank_to_dp_rank,
|
||||
physical_ep_size_to_dp_size,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import update_dp_attention_post_scale
|
||||
|
||||
replica_size = attn_replica_size()
|
||||
dp_size = physical_ep_size_to_dp_size(physical_ep_size, replica_size)
|
||||
update_dp_attention_post_scale(
|
||||
new_dp_size=dp_size,
|
||||
new_dp_rank=physical_ep_rank_to_dp_rank(physical_ep_rank, replica_size),
|
||||
)
|
||||
get_parallel().override_permanently(num_dp_ranks=dp_size)
|
||||
return dp_size
|
||||
|
||||
|
||||
@dataclass
|
||||
class ElasticEPState:
|
||||
active_ranks: Optional[torch.Tensor]
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Sequence
|
||||
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
|
||||
def attn_replica_size() -> int:
|
||||
"""Return the fixed physical width of one attention-DP replica."""
|
||||
parallel = get_parallel()
|
||||
return parallel.attn_tp_size * parallel.attn_cp_size
|
||||
|
||||
|
||||
def validate_matching_attention_topology(
|
||||
*,
|
||||
primary_attn_tp_size: int,
|
||||
primary_attn_cp_size: int,
|
||||
joining_attn_tp_size: int,
|
||||
joining_attn_cp_size: int,
|
||||
) -> None:
|
||||
primary_topology = (primary_attn_tp_size, primary_attn_cp_size)
|
||||
joining_topology = (joining_attn_tp_size, joining_attn_cp_size)
|
||||
if joining_topology != primary_topology:
|
||||
raise ValueError(
|
||||
"Primary and joining cohort must use the same attention topology "
|
||||
"for Elastic EP scale-up "
|
||||
f"(primary attn_tp_size={primary_topology[0]}, "
|
||||
f"attn_cp_size={primary_topology[1]}; joining "
|
||||
f"attn_tp_size={joining_topology[0]}, "
|
||||
f"attn_cp_size={joining_topology[1]})."
|
||||
)
|
||||
|
||||
|
||||
def physical_ep_size_to_dp_size(ep_size: int, attn_replica_size: int) -> int:
|
||||
if ep_size <= 0 or attn_replica_size <= 0 or ep_size % attn_replica_size != 0:
|
||||
raise ValueError(
|
||||
f"EP size {ep_size} must be divisible by attention replica size "
|
||||
f"{attn_replica_size}."
|
||||
)
|
||||
return ep_size // attn_replica_size
|
||||
|
||||
|
||||
def physical_ep_rank_to_dp_rank(ep_rank: int, attn_replica_size: int) -> int:
|
||||
if ep_rank < 0 or attn_replica_size <= 0:
|
||||
raise ValueError(
|
||||
f"Invalid EP/attention topology: {ep_rank=}, {attn_replica_size=}."
|
||||
)
|
||||
return ep_rank // attn_replica_size
|
||||
|
||||
|
||||
def collapse_physical_rank_status(
|
||||
status: Sequence[bool], attn_replica_size: int
|
||||
) -> list[bool]:
|
||||
"""Collapse physical-rank health with an all-members replica rule."""
|
||||
physical_ep_size_to_dp_size(len(status), attn_replica_size)
|
||||
return [
|
||||
all(bool(value) for value in status[start : start + attn_replica_size])
|
||||
for start in range(0, len(status), attn_replica_size)
|
||||
]
|
||||
@@ -299,17 +299,19 @@ def all_gather_kv_cache_for_mla_extend(
|
||||
)
|
||||
dcp_kv_buffer[:dcp_extend_prefix_lens_sum] = gathered_kv
|
||||
|
||||
# copy local kv cache into forward_batch.attn_dcp_metadata.dcp_kv_buffer
|
||||
# Attention TP may append dummy tokens to align an extend batch to its
|
||||
# group width. The DCP buffer tracks only real sequence tokens.
|
||||
extend_num_tokens = dcp_kv_buffer.shape[0] - dcp_extend_prefix_lens_sum
|
||||
dcp_kv_buffer[
|
||||
dcp_extend_prefix_lens_sum:,
|
||||
...,
|
||||
:kv_lora_rank,
|
||||
] = k_nope
|
||||
] = k_nope[:extend_num_tokens]
|
||||
dcp_kv_buffer[
|
||||
dcp_extend_prefix_lens_sum:,
|
||||
...,
|
||||
kv_lora_rank:,
|
||||
] = k_pe
|
||||
] = k_pe[:extend_num_tokens]
|
||||
|
||||
|
||||
# all gather kv cache and re-org to query orders
|
||||
|
||||
@@ -20,6 +20,10 @@ from sglang.srt.distributed import (
|
||||
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
use_symmetric_memory,
|
||||
)
|
||||
from sglang.srt.elastic_ep.topology import (
|
||||
attn_replica_size,
|
||||
physical_ep_rank_to_dp_rank,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.runtime_context import (
|
||||
derive_attention_ranks,
|
||||
@@ -90,16 +94,20 @@ def dp_slot_in(per_rank) -> int:
|
||||
def dp_gather_slot() -> int:
|
||||
"""Return this process's index in the DP gather.
|
||||
|
||||
After elastic scale-up, use the TP rank plus the join offset; otherwise
|
||||
use the attention-DP rank. Inside a draft scope that owns its attention,
|
||||
it is still the slot in the target's gather (see patch_tensor_parallel_group).
|
||||
After elastic scale-up, convert the physical EP rank (the TP rank plus the
|
||||
join offset) to its logical attention-DP replica; otherwise use the local
|
||||
attention-DP rank. Inside a draft scope that owns its attention, it is still
|
||||
the slot in the target's gather (see patch_tensor_parallel_group).
|
||||
"""
|
||||
scoped = get_flags().dp.scoped_gather_slot
|
||||
if scoped is not None:
|
||||
return scoped
|
||||
parallel = get_parallel()
|
||||
if world_dp_gather_enabled():
|
||||
return parallel.tp_rank + parallel.ep_join_rank_offset
|
||||
return physical_ep_rank_to_dp_rank(
|
||||
parallel.tp_rank + parallel.ep_join_rank_offset,
|
||||
attn_replica_size(),
|
||||
)
|
||||
return parallel.attn_dp_rank
|
||||
|
||||
|
||||
@@ -355,7 +363,7 @@ def set_dp_buffer_len_from_batch(forward_batch: ForwardBatch) -> None:
|
||||
global_num_tokens = forward_batch.global_num_tokens_padded_cpu
|
||||
if global_num_tokens is None:
|
||||
global_num_tokens = forward_batch.global_num_tokens_cpu
|
||||
dp_rank = get_parallel().attn_dp_rank if len(global_num_tokens) > 1 else 0
|
||||
dp_rank = dp_slot_in(global_num_tokens)
|
||||
set_dp_buffer_len(
|
||||
forward_batch.global_dp_buffer_len,
|
||||
global_num_tokens[dp_rank],
|
||||
|
||||
@@ -519,7 +519,13 @@ class VocabParallelEmbedding(torch.nn.Module):
|
||||
input_, 0, self.num_embeddings, "VocabParallelEmbedding input id"
|
||||
)
|
||||
output_parallel = self._embed_local_shard(input_)
|
||||
if self.tp_size > 1 and not get_attn_tp_context().input_scattered:
|
||||
# Replicated attention-TP peers have matching shapes, so all of them
|
||||
# can skip the collective for an empty idle batch.
|
||||
if (
|
||||
output_parallel.numel() > 0
|
||||
and self.tp_size > 1
|
||||
and not get_attn_tp_context().input_scattered
|
||||
):
|
||||
if self.use_attn_tp_group:
|
||||
output_parallel = attn_tp_all_reduce(output_parallel)
|
||||
else:
|
||||
|
||||
@@ -20,6 +20,7 @@ import signal
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
from multiprocessing.connection import Connection, wait
|
||||
|
||||
@@ -27,6 +28,12 @@ import psutil
|
||||
import setproctitle
|
||||
import zmq
|
||||
|
||||
from sglang.srt.elastic_ep.topology import (
|
||||
attn_replica_size,
|
||||
physical_ep_rank_to_dp_rank,
|
||||
physical_ep_size_to_dp_size,
|
||||
validate_matching_attention_topology,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.dp_attention import compute_dp_attention_world_info
|
||||
from sglang.srt.managers.io_struct import (
|
||||
@@ -87,6 +94,13 @@ logger = logging.getLogger(__name__)
|
||||
SCHEDULER_PIDS_ARG = "scheduler_pids"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WorkerPortHandshake:
|
||||
worker_ports: list[int]
|
||||
attn_tp_size: int
|
||||
attn_cp_size: int
|
||||
|
||||
|
||||
class LoadBalanceMethod(Enum):
|
||||
"""Load balance method."""
|
||||
|
||||
@@ -183,12 +197,22 @@ class DataParallelController:
|
||||
LoadBalanceMethod.TOTAL_TOKENS,
|
||||
)
|
||||
|
||||
self.launch_dp_size: int = get_parallel().num_dp_ranks
|
||||
self.max_dp_size: int = (
|
||||
get_parallel().max_ep_size or get_parallel().num_dp_ranks
|
||||
)
|
||||
parallel = get_parallel()
|
||||
self.launch_dp_size: int = parallel.num_dp_ranks
|
||||
self.attn_replica_size = attn_replica_size()
|
||||
if (
|
||||
get_exec().moe.elastic_ep_backend is not None
|
||||
and parallel.max_ep_size is not None
|
||||
and parallel.max_ep_size > parallel.tp_size
|
||||
):
|
||||
self.max_dp_size = physical_ep_size_to_dp_size(
|
||||
parallel.max_ep_size,
|
||||
self.attn_replica_size,
|
||||
)
|
||||
else:
|
||||
self.max_dp_size = parallel.num_dp_ranks
|
||||
assert self.max_dp_size >= self.launch_dp_size, (
|
||||
f"--max-ep-size ({self.max_dp_size}) must be >= "
|
||||
f"maximum DP size ({self.max_dp_size}) must be >= "
|
||||
f"--dp ({self.launch_dp_size})."
|
||||
)
|
||||
|
||||
@@ -489,7 +513,16 @@ class DataParallelController:
|
||||
)
|
||||
else:
|
||||
# Other nodes: Receive worker ports from node 0
|
||||
return self._receive_ports_as_client(endpoint)
|
||||
return self._receive_ports_as_client(endpoint).worker_ports
|
||||
|
||||
@staticmethod
|
||||
def _make_worker_port_handshake(worker_ports: list[int]) -> WorkerPortHandshake:
|
||||
parallel = get_parallel()
|
||||
return WorkerPortHandshake(
|
||||
worker_ports=worker_ports,
|
||||
attn_tp_size=parallel.attn_tp_size,
|
||||
attn_cp_size=parallel.attn_cp_size,
|
||||
)
|
||||
|
||||
def _broadcast_ports_as_server(
|
||||
self, endpoint: str, expected_clients: int, worker_ports: list[int]
|
||||
@@ -497,6 +530,7 @@ class DataParallelController:
|
||||
"""Broadcast worker ports to all client nodes."""
|
||||
logger.debug(f"Broadcasting worker ports to {expected_clients} client nodes")
|
||||
logger.debug(f"Worker ports: {worker_ports}")
|
||||
handshake = self._make_worker_port_handshake(worker_ports)
|
||||
|
||||
rep_socket = get_zmq_socket(self.context, zmq.REP, endpoint, True)
|
||||
|
||||
@@ -508,7 +542,7 @@ class DataParallelController:
|
||||
logger.debug(f"Received handshake from node {client_rank}")
|
||||
|
||||
# Send worker ports to client
|
||||
sock_send(rep_socket, wrap_as_pickle(worker_ports))
|
||||
sock_send(rep_socket, wrap_as_pickle(handshake))
|
||||
connected_clients += 1
|
||||
logger.debug(
|
||||
f"Sent worker ports to {connected_clients}/{expected_clients} nodes"
|
||||
@@ -522,14 +556,14 @@ class DataParallelController:
|
||||
else:
|
||||
threading.Thread(
|
||||
target=self._reply_ports_as_server,
|
||||
args=(rep_socket, worker_ports),
|
||||
args=(rep_socket, handshake),
|
||||
daemon=True,
|
||||
).start()
|
||||
|
||||
def _reply_ports_as_server(self, rep_socket: zmq.Socket, worker_ports: list[int]):
|
||||
"""Background thread: serve the pre-bound worker-port list to
|
||||
late-arriving elastic joiners. Publishes port numbers only; the primary
|
||||
keeps ownership of every socket."""
|
||||
def _reply_ports_as_server(
|
||||
self, rep_socket: zmq.Socket, handshake: WorkerPortHandshake
|
||||
):
|
||||
"""Serve pre-bound worker ports and topology to elastic joiners."""
|
||||
while True:
|
||||
try:
|
||||
client_rank = sock_recv(rep_socket)
|
||||
@@ -541,10 +575,10 @@ class DataParallelController:
|
||||
logger.debug(f"Received handshake from node {client_rank}")
|
||||
|
||||
# Send worker ports to client
|
||||
sock_send(rep_socket, wrap_as_pickle(worker_ports))
|
||||
sock_send(rep_socket, wrap_as_pickle(handshake))
|
||||
logger.debug(f"Sent worker ports to node {client_rank}")
|
||||
|
||||
def _receive_ports_as_client(self, endpoint: str) -> list[int]:
|
||||
def _receive_ports_as_client(self, endpoint: str) -> WorkerPortHandshake:
|
||||
"""Receive worker ports from the server node."""
|
||||
logger.debug("Connecting to node 0 to receive worker ports")
|
||||
node_rank = get_parallel().node_rank
|
||||
@@ -558,9 +592,13 @@ class DataParallelController:
|
||||
sock_send(req_socket, wrap_as_pickle(str(node_rank)))
|
||||
|
||||
# Receive worker ports
|
||||
worker_ports = sock_recv(req_socket)
|
||||
logger.debug(f"Received {len(worker_ports)} worker ports from node 0")
|
||||
return worker_ports
|
||||
handshake = sock_recv(req_socket)
|
||||
if not isinstance(handshake, WorkerPortHandshake):
|
||||
raise RuntimeError("Invalid worker-port handshake from node 0")
|
||||
logger.debug(
|
||||
f"Received {len(handshake.worker_ports)} worker ports from node 0"
|
||||
)
|
||||
return handshake
|
||||
except zmq.Again:
|
||||
logger.error("Timeout waiting for worker ports from node 0")
|
||||
raise RuntimeError(
|
||||
@@ -569,12 +607,6 @@ class DataParallelController:
|
||||
finally:
|
||||
req_socket.close()
|
||||
|
||||
def _joiner_local_tp_span(self, server_args: ServerArgs) -> int:
|
||||
return get_parallel().tp_size
|
||||
|
||||
def _joiner_slot_offset(self, server_args: ServerArgs) -> int:
|
||||
return get_parallel().ep_join_rank_offset
|
||||
|
||||
def launch_dp_attention_schedulers(
|
||||
self, server_args: ServerArgs, port_args: PortArgs
|
||||
):
|
||||
@@ -590,10 +622,29 @@ class DataParallelController:
|
||||
primary_endpoint = NetworkAddress(
|
||||
primary.host, primary.port + DP_ATTENTION_HANDSHAKE_PORT_DELTA
|
||||
).to_tcp()
|
||||
all_ports = self._receive_ports_as_client(primary_endpoint)
|
||||
offset = self._joiner_slot_offset(server_args)
|
||||
local_tp_span = self._joiner_local_tp_span(server_args)
|
||||
broadcasted_ports = all_ports[offset : offset + local_tp_span]
|
||||
handshake = self._receive_ports_as_client(primary_endpoint)
|
||||
parallel = get_parallel()
|
||||
validate_matching_attention_topology(
|
||||
primary_attn_tp_size=handshake.attn_tp_size,
|
||||
primary_attn_cp_size=handshake.attn_cp_size,
|
||||
joining_attn_tp_size=parallel.attn_tp_size,
|
||||
joining_attn_cp_size=parallel.attn_cp_size,
|
||||
)
|
||||
primary_replica_size = handshake.attn_tp_size * handshake.attn_cp_size
|
||||
offset = physical_ep_rank_to_dp_rank(
|
||||
parallel.ep_join_rank_offset,
|
||||
primary_replica_size,
|
||||
)
|
||||
broadcasted_ports = handshake.worker_ports[
|
||||
offset : offset + parallel.num_dp_ranks
|
||||
]
|
||||
if len(broadcasted_ports) != parallel.num_dp_ranks:
|
||||
raise ValueError(
|
||||
"Primary has insufficient reserved attention-DP worker ports "
|
||||
f"for join offset {parallel.ep_join_rank_offset} "
|
||||
f"(requested {parallel.num_dp_ranks}, got "
|
||||
f"{len(broadcasted_ports)})."
|
||||
)
|
||||
elif get_parallel().node_rank == 0:
|
||||
# Elastic primaries reserve sockets for the maximum DP size.
|
||||
bind_count = (
|
||||
@@ -709,9 +760,10 @@ class DataParallelController:
|
||||
|
||||
# Scheduler internals use local ranks; logs use global ranks.
|
||||
offset = get_parallel().ep_join_rank_offset
|
||||
dp_offset = physical_ep_rank_to_dp_rank(offset, self.attn_replica_size)
|
||||
display_tp_rank = tp_rank + offset
|
||||
display_moe_ep_rank = moe_ep_rank + offset
|
||||
display_dp_rank = dp_rank + offset if dp_rank is not None else None
|
||||
display_dp_rank = dp_rank + dp_offset if dp_rank is not None else None
|
||||
|
||||
with self.env_lock, maybe_reindex_device_id(gpu_id) as gpu_id:
|
||||
scheduler_args = (
|
||||
|
||||
@@ -2304,10 +2304,12 @@ class ActiveRanksOutput(BaseReq, kw_only=True):
|
||||
|
||||
|
||||
class ElasticScaleUpdateReq(BaseReq, kw_only=True):
|
||||
"""Report asynchronous Elastic EP scale completion or failure."""
|
||||
"""Report scale completion in physical EP and logical DP-slot units."""
|
||||
|
||||
success: bool
|
||||
# Public Elastic EP state remains physical.
|
||||
effective_ep_size: int
|
||||
# DPC capacity and worker activation use logical DP slots.
|
||||
slot_offset: int = 0
|
||||
slot_count: int = 0
|
||||
error: Optional[str] = None
|
||||
|
||||
@@ -4822,13 +4822,19 @@ class Scheduler(
|
||||
if not (self.attn_dp_enabled and get_exec().moe.elastic_ep_backend is not None):
|
||||
return
|
||||
from sglang.srt.elastic_ep.elastic_ep import ElasticEPStateManager
|
||||
from sglang.srt.elastic_ep.topology import (
|
||||
attn_replica_size,
|
||||
collapse_physical_rank_status,
|
||||
)
|
||||
|
||||
inst = ElasticEPStateManager.instance()
|
||||
if inst is not None and inst.active_ranks_cpu is not None:
|
||||
dp_status = collapse_physical_rank_status(
|
||||
inst.active_ranks_cpu.tolist(),
|
||||
attn_replica_size(),
|
||||
)
|
||||
self.ipc_channels.send_to_tokenizer.send_output(
|
||||
ActiveRanksOutput(
|
||||
status=[bool(x) for x in inst.active_ranks_cpu.tolist()]
|
||||
)
|
||||
ActiveRanksOutput(status=dp_status)
|
||||
)
|
||||
else:
|
||||
logger.debug("[Elastic EP] active rank state is unavailable")
|
||||
@@ -5786,6 +5792,19 @@ class Scheduler(
|
||||
old_ep_size=old_ep_size,
|
||||
new_ep_size=new_ep_size,
|
||||
)
|
||||
from sglang.srt.elastic_ep.topology import attn_replica_size
|
||||
|
||||
replica_size = attn_replica_size()
|
||||
if new_ep_size % replica_size != 0:
|
||||
return ScaleElasticEPReqOutput(
|
||||
success=False,
|
||||
message=(
|
||||
f"new_ep_size ({new_ep_size}) would admit an incomplete "
|
||||
f"attention replica with size {replica_size}."
|
||||
),
|
||||
old_ep_size=old_ep_size,
|
||||
new_ep_size=new_ep_size,
|
||||
)
|
||||
if ElasticEPStateManager.is_scaling():
|
||||
return ScaleElasticEPReqOutput(
|
||||
success=False,
|
||||
|
||||
@@ -89,24 +89,29 @@ def _resolve_elastic_world_num_dp_ranks(
|
||||
live_num_dp_ranks = dp_gather_width()
|
||||
effective_ep_size = ElasticEPStateManager.get_effective_ep_size()
|
||||
# Query live membership because elastic joins can expand WORLD.
|
||||
from sglang.srt.elastic_ep.topology import attn_replica_size
|
||||
|
||||
replica_size = attn_replica_size()
|
||||
world_size = torch.distributed.get_world_size(group)
|
||||
|
||||
if live_num_dp_ranks != effective_ep_size:
|
||||
if live_num_dp_ranks * replica_size != effective_ep_size:
|
||||
raise RuntimeError(
|
||||
"[Elastic EP] WORLD MLP sync num_dp_ranks is out of sync: "
|
||||
f"rank={torch.distributed.get_rank(group)} "
|
||||
f"live_num_dp_ranks={live_num_dp_ranks} "
|
||||
f"effective_ep_size={effective_ep_size} "
|
||||
f"attn_replica_size={replica_size} "
|
||||
f"world_size={world_size} configured_num_dp_ranks={num_dp_ranks} "
|
||||
f"local_num_tokens={local_num_tokens} "
|
||||
f"local_forward_mode={local_forward_mode}"
|
||||
)
|
||||
if live_num_dp_ranks > world_size:
|
||||
if effective_ep_size > world_size:
|
||||
raise RuntimeError(
|
||||
"[Elastic EP] WORLD MLP sync num_dp_ranks exceeds WORLD size: "
|
||||
"[Elastic EP] WORLD MLP sync EP size exceeds WORLD size: "
|
||||
f"rank={torch.distributed.get_rank(group)} "
|
||||
f"live_num_dp_ranks={live_num_dp_ranks} world_size={world_size} "
|
||||
f"effective_ep_size={effective_ep_size}"
|
||||
f"effective_ep_size={effective_ep_size} "
|
||||
f"attn_replica_size={replica_size}"
|
||||
)
|
||||
|
||||
return live_num_dp_ranks
|
||||
|
||||
@@ -63,6 +63,10 @@ from sglang.srt.configs.model_config import ModelConfig
|
||||
from sglang.srt.constants import HEALTH_CHECK_RID_PREFIX
|
||||
from sglang.srt.disaggregation.encoder.receiver import create_mm_receiver
|
||||
from sglang.srt.disaggregation.utils import DisaggregationMode
|
||||
from sglang.srt.elastic_ep.topology import (
|
||||
attn_replica_size,
|
||||
physical_ep_size_to_dp_size,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.joint_schema_head import (
|
||||
max_joint_prompt_tokens,
|
||||
@@ -520,7 +524,9 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
self.server_args = server_args
|
||||
assert_published(server_args, role="tokenizer")
|
||||
self.startup_time: Optional[Dict[str, Any]] = None
|
||||
self.elastic_worker_count = get_parallel().num_dp_ranks
|
||||
parallel = get_parallel()
|
||||
self.elastic_worker_count = parallel.num_dp_ranks
|
||||
self.attn_replica_size = attn_replica_size()
|
||||
self.elastic_pending_ep_size = None
|
||||
self.elastic_scale_phase = "idle"
|
||||
self.elastic_last_error = None
|
||||
@@ -3652,16 +3658,18 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
return
|
||||
|
||||
self._dispatch_to_scheduler(msg)
|
||||
self.elastic_worker_count = msg.effective_ep_size
|
||||
self.elastic_worker_count = physical_ep_size_to_dp_size(
|
||||
msg.effective_ep_size, self.attn_replica_size
|
||||
)
|
||||
self.elastic_pending_ep_size = None
|
||||
self.elastic_scale_phase = "serving_expanded"
|
||||
self.elastic_last_error = None
|
||||
self.update_control_communicator_fan_out(msg.effective_ep_size)
|
||||
self.update_control_communicator_fan_out(self.elastic_worker_count)
|
||||
|
||||
def get_elastic_ep_state(self):
|
||||
return {
|
||||
"is_scaling_elastic_ep": self.elastic_pending_ep_size is not None,
|
||||
"effective_ep_size": self.elastic_worker_count,
|
||||
"effective_ep_size": self.elastic_worker_count * self.attn_replica_size,
|
||||
"pending_ep_size": self.elastic_pending_ep_size,
|
||||
"scale_phase": self.elastic_scale_phase,
|
||||
"last_error": self.elastic_last_error,
|
||||
@@ -3678,7 +3686,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
"A previous scale operation has not completed yet. Wait until "
|
||||
"all pending ranks have joined before issuing another scale."
|
||||
),
|
||||
old_ep_size=self.elastic_worker_count,
|
||||
old_ep_size=self.elastic_worker_count * self.attn_replica_size,
|
||||
new_ep_size=obj.new_ep_size,
|
||||
pending_ep_size=self.elastic_pending_ep_size,
|
||||
scale_phase=self.elastic_scale_phase,
|
||||
|
||||
@@ -49,8 +49,14 @@ from sglang.srt.elastic_ep.elastic_ep import (
|
||||
maybe_recover_ep_ranks,
|
||||
register_scale_cohort,
|
||||
try_admit_scale_ranks,
|
||||
update_dp_attention_for_elastic_ep,
|
||||
validate_scale_cohort_topology,
|
||||
)
|
||||
from sglang.srt.elastic_ep.expert_backup_client import ExpertBackupClient
|
||||
from sglang.srt.elastic_ep.topology import (
|
||||
attn_replica_size,
|
||||
physical_ep_size_to_dp_size,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.eplb_manager import EPLBManager
|
||||
from sglang.srt.eplb.expert_distribution import (
|
||||
@@ -545,17 +551,13 @@ class ModelRunner:
|
||||
)
|
||||
)
|
||||
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
enable_joiner_all_gather,
|
||||
update_dp_attention_post_scale,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import enable_joiner_all_gather
|
||||
|
||||
enable_joiner_all_gather()
|
||||
update_dp_attention_post_scale(
|
||||
new_dp_size=join_effective_ep_size,
|
||||
new_dp_rank=global_ep_rank,
|
||||
update_dp_attention_for_elastic_ep(
|
||||
physical_ep_size=join_effective_ep_size,
|
||||
physical_ep_rank=global_ep_rank,
|
||||
)
|
||||
get_parallel().override_permanently(num_dp_ranks=join_effective_ep_size)
|
||||
if self.eplb_manager is not None:
|
||||
self.eplb_manager.disable_rebalance(
|
||||
"EPLB rebalance is disabled while elastic EP scale-up "
|
||||
@@ -2326,13 +2328,12 @@ class ModelRunner:
|
||||
"is being finalized"
|
||||
)
|
||||
|
||||
from sglang.srt.layers.dp_attention import update_dp_attention_post_scale
|
||||
|
||||
update_dp_attention_post_scale(
|
||||
new_dp_size=target_size,
|
||||
new_dp_rank=self._elastic_global_rank(),
|
||||
replica_size = attn_replica_size()
|
||||
effective_dp_size = physical_ep_size_to_dp_size(effective_size, replica_size)
|
||||
target_dp_size = update_dp_attention_for_elastic_ep(
|
||||
physical_ep_size=target_size,
|
||||
physical_ep_rank=self._elastic_global_rank(),
|
||||
)
|
||||
get_parallel().override_permanently(num_dp_ranks=target_size)
|
||||
|
||||
recapture_cuda_graph = self._elastic_cuda_graph_enabled()
|
||||
if recapture_cuda_graph:
|
||||
@@ -2351,8 +2352,8 @@ class ModelRunner:
|
||||
self._pending_elastic_scale_update = ElasticScaleUpdateReq(
|
||||
success=True,
|
||||
effective_ep_size=target_size,
|
||||
slot_offset=effective_size,
|
||||
slot_count=target_size - effective_size,
|
||||
slot_offset=effective_dp_size,
|
||||
slot_count=target_dp_size - effective_dp_size,
|
||||
)
|
||||
logger.info(
|
||||
"[Elastic EP] Scale completed: old_ep_size=%d "
|
||||
@@ -2472,6 +2473,19 @@ class ModelRunner:
|
||||
):
|
||||
logger.error("[Elastic EP] %s", error)
|
||||
return
|
||||
try:
|
||||
validate_scale_cohort_topology(cohort)
|
||||
except ValueError as exc:
|
||||
error = str(exc)
|
||||
ElasticEPStateManager.fail_scale(error)
|
||||
self._reset_eplb_after_elastic_scale_failure()
|
||||
self._report_elastic_scale_failure(error, effective_size)
|
||||
if (
|
||||
get_parallel().tp_rank == 0
|
||||
and not get_exec().moe.is_ep_scale_joiner
|
||||
):
|
||||
logger.error("[Elastic EP] %s", error)
|
||||
return
|
||||
cuda_graph_enabled = self._elastic_cuda_graph_enabled()
|
||||
if cohort.cuda_graph_enabled != cuda_graph_enabled:
|
||||
error = (
|
||||
|
||||
Reference in New Issue
Block a user