[4/N] elastic-ep: Scale at DP-attention replica granularity (#33728)

This commit is contained in:
Yoray Zack
2026-10-15 01:17:01 +08:00
committed by GitHub
parent c964d9ba20
commit 87ed1b9526
18 changed files with 695 additions and 87 deletions
+44 -6
View File
@@ -558,6 +558,43 @@ def handle_elastic_ep(server_args: Any):
)
if scaling_active:
resolved = resolved_view(server_args)
from sglang.srt.elastic_ep.topology import physical_ep_size_to_dp_size
try:
divisor = cfg.attn_dp_size * resolved.attn_cp_size
if (
cfg.tp_size <= 0
or cfg.attn_dp_size <= 0
or resolved.attn_cp_size <= 0
or cfg.tp_size % divisor
):
raise ValueError(
"Invalid attention topology: "
f"tp_size={cfg.tp_size}, attn_dp_size={cfg.attn_dp_size}, "
f"attn_cp_size={resolved.attn_cp_size}."
)
attn_tp_size = derive_attn_tp_size(
tp_size=cfg.tp_size,
attn_cp_size=resolved.attn_cp_size,
attn_dp_size=cfg.attn_dp_size,
)
attn_replica_size = attn_tp_size * resolved.attn_cp_size
physical_ep_size_to_dp_size(cfg.max_ep_size, attn_replica_size)
physical_ep_size_to_dp_size(
cfg.elastic_ep_initial_size or cfg.tp_size, attn_replica_size
)
if cfg.ep_join_mode == "scale":
physical_ep_size_to_dp_size(cfg.ep_join_rank_offset, attn_replica_size)
except ValueError as exc:
raise AssertionError(str(exc)) from exc
if attn_tp_size > 1:
assert cfg.moe_dense_tp_size == 1, (
"Elastic EP scale-up with attention TP greater than one requires "
"--moe-dense-tp-size 1 "
f"(got moe_dense_tp_size={cfg.moe_dense_tp_size})."
)
assert cfg.elastic_ep_scale_timeout > 0, (
"--elastic-ep-scale-timeout must be greater than zero."
)
@@ -622,6 +659,11 @@ def handle_elastic_ep(server_args: Any):
"Elastic EP scale-up requires --pp-size 1 "
f"(got pp_size={cfg.pp_size}); WORLD must not span PP stages."
)
assert attn_tp_size % cfg.dcp_size == 0, (
"Elastic EP scale-up requires --dcp-size to divide the fixed "
"attention TP width "
f"(got dcp_size={cfg.dcp_size}, attn_tp_size={attn_tp_size})."
)
decode_backend = cfg.cuda_graph_config.decode.backend
assert decode_backend in (Backend.DISABLED, Backend.FULL), (
@@ -673,12 +715,8 @@ def handle_elastic_ep(server_args: Any):
)
assert resolved.ep_size == cfg.tp_size, (
"Elastic EP scale-up requires ep_size == tp_size "
f"(got ep_size={resolved.ep_size}, tp_size={cfg.tp_size}); EP, TP "
"and the attention DP group must all coincide with WORLD."
)
assert cfg.attn_dp_size == cfg.tp_size, (
"Elastic EP scale-up requires attn_dp_size == tp_size "
f"(got attn_dp_size={cfg.attn_dp_size}, tp_size={cfg.tp_size})."
f"(got ep_size={resolved.ep_size}, tp_size={cfg.tp_size}); "
"physical EP membership must coincide with the launch-time TP span."
)
assert resolved.moe_a2a_backend == "nixl", (
"Elastic EP scale-up requires --moe-a2a-backend nixl "
@@ -30,6 +30,8 @@ _SCALE_COHORT_KEY_PREFIX = "elastic_ep/scale_cohort"
class ScaleCohort(msgspec.Struct, frozen=True, kw_only=True):
target_ep_size: int
cuda_graph_enabled: bool
attn_tp_size: int
attn_cp_size: int
def register_scale_cohort(
@@ -38,10 +40,13 @@ def register_scale_cohort(
store = get_global_tcp_store()
if store is None:
raise RuntimeError("Elastic EP scale-up requires the global TCPStore.")
parallel = get_parallel()
payload = msgspec.json.encode(
ScaleCohort(
target_ep_size=target_ep_size,
cuda_graph_enabled=cuda_graph_enabled,
attn_tp_size=parallel.attn_tp_size,
attn_cp_size=parallel.attn_cp_size,
)
)
store.set(f"{_SCALE_COHORT_KEY_PREFIX}/{rank_offset}", payload)
@@ -57,6 +62,39 @@ def get_scale_cohort(rank_offset: int) -> Optional[ScaleCohort]:
return msgspec.json.decode(store.get(key), type=ScaleCohort)
def validate_scale_cohort_topology(cohort: ScaleCohort) -> None:
from sglang.srt.elastic_ep.topology import validate_matching_attention_topology
parallel = get_parallel()
validate_matching_attention_topology(
primary_attn_tp_size=parallel.attn_tp_size,
primary_attn_cp_size=parallel.attn_cp_size,
joining_attn_tp_size=cohort.attn_tp_size,
joining_attn_cp_size=cohort.attn_cp_size,
)
def update_dp_attention_for_elastic_ep(
*, physical_ep_size: int, physical_ep_rank: int
) -> int:
"""Apply a physical EP topology to the logical DP-attention runtime."""
from sglang.srt.elastic_ep.topology import (
attn_replica_size,
physical_ep_rank_to_dp_rank,
physical_ep_size_to_dp_size,
)
from sglang.srt.layers.dp_attention import update_dp_attention_post_scale
replica_size = attn_replica_size()
dp_size = physical_ep_size_to_dp_size(physical_ep_size, replica_size)
update_dp_attention_post_scale(
new_dp_size=dp_size,
new_dp_rank=physical_ep_rank_to_dp_rank(physical_ep_rank, replica_size),
)
get_parallel().override_permanently(num_dp_ranks=dp_size)
return dp_size
@dataclass
class ElasticEPState:
active_ranks: Optional[torch.Tensor]
+59
View File
@@ -0,0 +1,59 @@
from __future__ import annotations
from typing import Sequence
from sglang.srt.runtime_context import get_parallel
def attn_replica_size() -> int:
"""Return the fixed physical width of one attention-DP replica."""
parallel = get_parallel()
return parallel.attn_tp_size * parallel.attn_cp_size
def validate_matching_attention_topology(
*,
primary_attn_tp_size: int,
primary_attn_cp_size: int,
joining_attn_tp_size: int,
joining_attn_cp_size: int,
) -> None:
primary_topology = (primary_attn_tp_size, primary_attn_cp_size)
joining_topology = (joining_attn_tp_size, joining_attn_cp_size)
if joining_topology != primary_topology:
raise ValueError(
"Primary and joining cohort must use the same attention topology "
"for Elastic EP scale-up "
f"(primary attn_tp_size={primary_topology[0]}, "
f"attn_cp_size={primary_topology[1]}; joining "
f"attn_tp_size={joining_topology[0]}, "
f"attn_cp_size={joining_topology[1]})."
)
def physical_ep_size_to_dp_size(ep_size: int, attn_replica_size: int) -> int:
if ep_size <= 0 or attn_replica_size <= 0 or ep_size % attn_replica_size != 0:
raise ValueError(
f"EP size {ep_size} must be divisible by attention replica size "
f"{attn_replica_size}."
)
return ep_size // attn_replica_size
def physical_ep_rank_to_dp_rank(ep_rank: int, attn_replica_size: int) -> int:
if ep_rank < 0 or attn_replica_size <= 0:
raise ValueError(
f"Invalid EP/attention topology: {ep_rank=}, {attn_replica_size=}."
)
return ep_rank // attn_replica_size
def collapse_physical_rank_status(
status: Sequence[bool], attn_replica_size: int
) -> list[bool]:
"""Collapse physical-rank health with an all-members replica rule."""
physical_ep_size_to_dp_size(len(status), attn_replica_size)
return [
all(bool(value) for value in status[start : start + attn_replica_size])
for start in range(0, len(status), attn_replica_size)
]
+5 -3
View File
@@ -299,17 +299,19 @@ def all_gather_kv_cache_for_mla_extend(
)
dcp_kv_buffer[:dcp_extend_prefix_lens_sum] = gathered_kv
# copy local kv cache into forward_batch.attn_dcp_metadata.dcp_kv_buffer
# Attention TP may append dummy tokens to align an extend batch to its
# group width. The DCP buffer tracks only real sequence tokens.
extend_num_tokens = dcp_kv_buffer.shape[0] - dcp_extend_prefix_lens_sum
dcp_kv_buffer[
dcp_extend_prefix_lens_sum:,
...,
:kv_lora_rank,
] = k_nope
] = k_nope[:extend_num_tokens]
dcp_kv_buffer[
dcp_extend_prefix_lens_sum:,
...,
kv_lora_rank:,
] = k_pe
] = k_pe[:extend_num_tokens]
# all gather kv cache and re-org to query orders
+13 -5
View File
@@ -20,6 +20,10 @@ from sglang.srt.distributed import (
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
use_symmetric_memory,
)
from sglang.srt.elastic_ep.topology import (
attn_replica_size,
physical_ep_rank_to_dp_rank,
)
from sglang.srt.environ import envs
from sglang.srt.runtime_context import (
derive_attention_ranks,
@@ -90,16 +94,20 @@ def dp_slot_in(per_rank) -> int:
def dp_gather_slot() -> int:
"""Return this process's index in the DP gather.
After elastic scale-up, use the TP rank plus the join offset; otherwise
use the attention-DP rank. Inside a draft scope that owns its attention,
it is still the slot in the target's gather (see patch_tensor_parallel_group).
After elastic scale-up, convert the physical EP rank (the TP rank plus the
join offset) to its logical attention-DP replica; otherwise use the local
attention-DP rank. Inside a draft scope that owns its attention, it is still
the slot in the target's gather (see patch_tensor_parallel_group).
"""
scoped = get_flags().dp.scoped_gather_slot
if scoped is not None:
return scoped
parallel = get_parallel()
if world_dp_gather_enabled():
return parallel.tp_rank + parallel.ep_join_rank_offset
return physical_ep_rank_to_dp_rank(
parallel.tp_rank + parallel.ep_join_rank_offset,
attn_replica_size(),
)
return parallel.attn_dp_rank
@@ -355,7 +363,7 @@ def set_dp_buffer_len_from_batch(forward_batch: ForwardBatch) -> None:
global_num_tokens = forward_batch.global_num_tokens_padded_cpu
if global_num_tokens is None:
global_num_tokens = forward_batch.global_num_tokens_cpu
dp_rank = get_parallel().attn_dp_rank if len(global_num_tokens) > 1 else 0
dp_rank = dp_slot_in(global_num_tokens)
set_dp_buffer_len(
forward_batch.global_dp_buffer_len,
global_num_tokens[dp_rank],
@@ -519,7 +519,13 @@ class VocabParallelEmbedding(torch.nn.Module):
input_, 0, self.num_embeddings, "VocabParallelEmbedding input id"
)
output_parallel = self._embed_local_shard(input_)
if self.tp_size > 1 and not get_attn_tp_context().input_scattered:
# Replicated attention-TP peers have matching shapes, so all of them
# can skip the collective for an empty idle batch.
if (
output_parallel.numel() > 0
and self.tp_size > 1
and not get_attn_tp_context().input_scattered
):
if self.use_attn_tp_group:
output_parallel = attn_tp_all_reduce(output_parallel)
else:
@@ -20,6 +20,7 @@ import signal
import threading
import time
from collections.abc import Callable
from dataclasses import dataclass
from enum import Enum, auto
from multiprocessing.connection import Connection, wait
@@ -27,6 +28,12 @@ import psutil
import setproctitle
import zmq
from sglang.srt.elastic_ep.topology import (
attn_replica_size,
physical_ep_rank_to_dp_rank,
physical_ep_size_to_dp_size,
validate_matching_attention_topology,
)
from sglang.srt.environ import envs
from sglang.srt.layers.dp_attention import compute_dp_attention_world_info
from sglang.srt.managers.io_struct import (
@@ -87,6 +94,13 @@ logger = logging.getLogger(__name__)
SCHEDULER_PIDS_ARG = "scheduler_pids"
@dataclass(frozen=True)
class WorkerPortHandshake:
worker_ports: list[int]
attn_tp_size: int
attn_cp_size: int
class LoadBalanceMethod(Enum):
"""Load balance method."""
@@ -183,12 +197,22 @@ class DataParallelController:
LoadBalanceMethod.TOTAL_TOKENS,
)
self.launch_dp_size: int = get_parallel().num_dp_ranks
self.max_dp_size: int = (
get_parallel().max_ep_size or get_parallel().num_dp_ranks
)
parallel = get_parallel()
self.launch_dp_size: int = parallel.num_dp_ranks
self.attn_replica_size = attn_replica_size()
if (
get_exec().moe.elastic_ep_backend is not None
and parallel.max_ep_size is not None
and parallel.max_ep_size > parallel.tp_size
):
self.max_dp_size = physical_ep_size_to_dp_size(
parallel.max_ep_size,
self.attn_replica_size,
)
else:
self.max_dp_size = parallel.num_dp_ranks
assert self.max_dp_size >= self.launch_dp_size, (
f"--max-ep-size ({self.max_dp_size}) must be >= "
f"maximum DP size ({self.max_dp_size}) must be >= "
f"--dp ({self.launch_dp_size})."
)
@@ -489,7 +513,16 @@ class DataParallelController:
)
else:
# Other nodes: Receive worker ports from node 0
return self._receive_ports_as_client(endpoint)
return self._receive_ports_as_client(endpoint).worker_ports
@staticmethod
def _make_worker_port_handshake(worker_ports: list[int]) -> WorkerPortHandshake:
parallel = get_parallel()
return WorkerPortHandshake(
worker_ports=worker_ports,
attn_tp_size=parallel.attn_tp_size,
attn_cp_size=parallel.attn_cp_size,
)
def _broadcast_ports_as_server(
self, endpoint: str, expected_clients: int, worker_ports: list[int]
@@ -497,6 +530,7 @@ class DataParallelController:
"""Broadcast worker ports to all client nodes."""
logger.debug(f"Broadcasting worker ports to {expected_clients} client nodes")
logger.debug(f"Worker ports: {worker_ports}")
handshake = self._make_worker_port_handshake(worker_ports)
rep_socket = get_zmq_socket(self.context, zmq.REP, endpoint, True)
@@ -508,7 +542,7 @@ class DataParallelController:
logger.debug(f"Received handshake from node {client_rank}")
# Send worker ports to client
sock_send(rep_socket, wrap_as_pickle(worker_ports))
sock_send(rep_socket, wrap_as_pickle(handshake))
connected_clients += 1
logger.debug(
f"Sent worker ports to {connected_clients}/{expected_clients} nodes"
@@ -522,14 +556,14 @@ class DataParallelController:
else:
threading.Thread(
target=self._reply_ports_as_server,
args=(rep_socket, worker_ports),
args=(rep_socket, handshake),
daemon=True,
).start()
def _reply_ports_as_server(self, rep_socket: zmq.Socket, worker_ports: list[int]):
"""Background thread: serve the pre-bound worker-port list to
late-arriving elastic joiners. Publishes port numbers only; the primary
keeps ownership of every socket."""
def _reply_ports_as_server(
self, rep_socket: zmq.Socket, handshake: WorkerPortHandshake
):
"""Serve pre-bound worker ports and topology to elastic joiners."""
while True:
try:
client_rank = sock_recv(rep_socket)
@@ -541,10 +575,10 @@ class DataParallelController:
logger.debug(f"Received handshake from node {client_rank}")
# Send worker ports to client
sock_send(rep_socket, wrap_as_pickle(worker_ports))
sock_send(rep_socket, wrap_as_pickle(handshake))
logger.debug(f"Sent worker ports to node {client_rank}")
def _receive_ports_as_client(self, endpoint: str) -> list[int]:
def _receive_ports_as_client(self, endpoint: str) -> WorkerPortHandshake:
"""Receive worker ports from the server node."""
logger.debug("Connecting to node 0 to receive worker ports")
node_rank = get_parallel().node_rank
@@ -558,9 +592,13 @@ class DataParallelController:
sock_send(req_socket, wrap_as_pickle(str(node_rank)))
# Receive worker ports
worker_ports = sock_recv(req_socket)
logger.debug(f"Received {len(worker_ports)} worker ports from node 0")
return worker_ports
handshake = sock_recv(req_socket)
if not isinstance(handshake, WorkerPortHandshake):
raise RuntimeError("Invalid worker-port handshake from node 0")
logger.debug(
f"Received {len(handshake.worker_ports)} worker ports from node 0"
)
return handshake
except zmq.Again:
logger.error("Timeout waiting for worker ports from node 0")
raise RuntimeError(
@@ -569,12 +607,6 @@ class DataParallelController:
finally:
req_socket.close()
def _joiner_local_tp_span(self, server_args: ServerArgs) -> int:
return get_parallel().tp_size
def _joiner_slot_offset(self, server_args: ServerArgs) -> int:
return get_parallel().ep_join_rank_offset
def launch_dp_attention_schedulers(
self, server_args: ServerArgs, port_args: PortArgs
):
@@ -590,10 +622,29 @@ class DataParallelController:
primary_endpoint = NetworkAddress(
primary.host, primary.port + DP_ATTENTION_HANDSHAKE_PORT_DELTA
).to_tcp()
all_ports = self._receive_ports_as_client(primary_endpoint)
offset = self._joiner_slot_offset(server_args)
local_tp_span = self._joiner_local_tp_span(server_args)
broadcasted_ports = all_ports[offset : offset + local_tp_span]
handshake = self._receive_ports_as_client(primary_endpoint)
parallel = get_parallel()
validate_matching_attention_topology(
primary_attn_tp_size=handshake.attn_tp_size,
primary_attn_cp_size=handshake.attn_cp_size,
joining_attn_tp_size=parallel.attn_tp_size,
joining_attn_cp_size=parallel.attn_cp_size,
)
primary_replica_size = handshake.attn_tp_size * handshake.attn_cp_size
offset = physical_ep_rank_to_dp_rank(
parallel.ep_join_rank_offset,
primary_replica_size,
)
broadcasted_ports = handshake.worker_ports[
offset : offset + parallel.num_dp_ranks
]
if len(broadcasted_ports) != parallel.num_dp_ranks:
raise ValueError(
"Primary has insufficient reserved attention-DP worker ports "
f"for join offset {parallel.ep_join_rank_offset} "
f"(requested {parallel.num_dp_ranks}, got "
f"{len(broadcasted_ports)})."
)
elif get_parallel().node_rank == 0:
# Elastic primaries reserve sockets for the maximum DP size.
bind_count = (
@@ -709,9 +760,10 @@ class DataParallelController:
# Scheduler internals use local ranks; logs use global ranks.
offset = get_parallel().ep_join_rank_offset
dp_offset = physical_ep_rank_to_dp_rank(offset, self.attn_replica_size)
display_tp_rank = tp_rank + offset
display_moe_ep_rank = moe_ep_rank + offset
display_dp_rank = dp_rank + offset if dp_rank is not None else None
display_dp_rank = dp_rank + dp_offset if dp_rank is not None else None
with self.env_lock, maybe_reindex_device_id(gpu_id) as gpu_id:
scheduler_args = (
+3 -1
View File
@@ -2304,10 +2304,12 @@ class ActiveRanksOutput(BaseReq, kw_only=True):
class ElasticScaleUpdateReq(BaseReq, kw_only=True):
"""Report asynchronous Elastic EP scale completion or failure."""
"""Report scale completion in physical EP and logical DP-slot units."""
success: bool
# Public Elastic EP state remains physical.
effective_ep_size: int
# DPC capacity and worker activation use logical DP slots.
slot_offset: int = 0
slot_count: int = 0
error: Optional[str] = None
+22 -3
View File
@@ -4822,13 +4822,19 @@ class Scheduler(
if not (self.attn_dp_enabled and get_exec().moe.elastic_ep_backend is not None):
return
from sglang.srt.elastic_ep.elastic_ep import ElasticEPStateManager
from sglang.srt.elastic_ep.topology import (
attn_replica_size,
collapse_physical_rank_status,
)
inst = ElasticEPStateManager.instance()
if inst is not None and inst.active_ranks_cpu is not None:
dp_status = collapse_physical_rank_status(
inst.active_ranks_cpu.tolist(),
attn_replica_size(),
)
self.ipc_channels.send_to_tokenizer.send_output(
ActiveRanksOutput(
status=[bool(x) for x in inst.active_ranks_cpu.tolist()]
)
ActiveRanksOutput(status=dp_status)
)
else:
logger.debug("[Elastic EP] active rank state is unavailable")
@@ -5786,6 +5792,19 @@ class Scheduler(
old_ep_size=old_ep_size,
new_ep_size=new_ep_size,
)
from sglang.srt.elastic_ep.topology import attn_replica_size
replica_size = attn_replica_size()
if new_ep_size % replica_size != 0:
return ScaleElasticEPReqOutput(
success=False,
message=(
f"new_ep_size ({new_ep_size}) would admit an incomplete "
f"attention replica with size {replica_size}."
),
old_ep_size=old_ep_size,
new_ep_size=new_ep_size,
)
if ElasticEPStateManager.is_scaling():
return ScaleElasticEPReqOutput(
success=False,
@@ -89,24 +89,29 @@ def _resolve_elastic_world_num_dp_ranks(
live_num_dp_ranks = dp_gather_width()
effective_ep_size = ElasticEPStateManager.get_effective_ep_size()
# Query live membership because elastic joins can expand WORLD.
from sglang.srt.elastic_ep.topology import attn_replica_size
replica_size = attn_replica_size()
world_size = torch.distributed.get_world_size(group)
if live_num_dp_ranks != effective_ep_size:
if live_num_dp_ranks * replica_size != effective_ep_size:
raise RuntimeError(
"[Elastic EP] WORLD MLP sync num_dp_ranks is out of sync: "
f"rank={torch.distributed.get_rank(group)} "
f"live_num_dp_ranks={live_num_dp_ranks} "
f"effective_ep_size={effective_ep_size} "
f"attn_replica_size={replica_size} "
f"world_size={world_size} configured_num_dp_ranks={num_dp_ranks} "
f"local_num_tokens={local_num_tokens} "
f"local_forward_mode={local_forward_mode}"
)
if live_num_dp_ranks > world_size:
if effective_ep_size > world_size:
raise RuntimeError(
"[Elastic EP] WORLD MLP sync num_dp_ranks exceeds WORLD size: "
"[Elastic EP] WORLD MLP sync EP size exceeds WORLD size: "
f"rank={torch.distributed.get_rank(group)} "
f"live_num_dp_ranks={live_num_dp_ranks} world_size={world_size} "
f"effective_ep_size={effective_ep_size}"
f"effective_ep_size={effective_ep_size} "
f"attn_replica_size={replica_size}"
)
return live_num_dp_ranks
@@ -63,6 +63,10 @@ from sglang.srt.configs.model_config import ModelConfig
from sglang.srt.constants import HEALTH_CHECK_RID_PREFIX
from sglang.srt.disaggregation.encoder.receiver import create_mm_receiver
from sglang.srt.disaggregation.utils import DisaggregationMode
from sglang.srt.elastic_ep.topology import (
attn_replica_size,
physical_ep_size_to_dp_size,
)
from sglang.srt.environ import envs
from sglang.srt.layers.joint_schema_head import (
max_joint_prompt_tokens,
@@ -520,7 +524,9 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
self.server_args = server_args
assert_published(server_args, role="tokenizer")
self.startup_time: Optional[Dict[str, Any]] = None
self.elastic_worker_count = get_parallel().num_dp_ranks
parallel = get_parallel()
self.elastic_worker_count = parallel.num_dp_ranks
self.attn_replica_size = attn_replica_size()
self.elastic_pending_ep_size = None
self.elastic_scale_phase = "idle"
self.elastic_last_error = None
@@ -3652,16 +3658,18 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
return
self._dispatch_to_scheduler(msg)
self.elastic_worker_count = msg.effective_ep_size
self.elastic_worker_count = physical_ep_size_to_dp_size(
msg.effective_ep_size, self.attn_replica_size
)
self.elastic_pending_ep_size = None
self.elastic_scale_phase = "serving_expanded"
self.elastic_last_error = None
self.update_control_communicator_fan_out(msg.effective_ep_size)
self.update_control_communicator_fan_out(self.elastic_worker_count)
def get_elastic_ep_state(self):
return {
"is_scaling_elastic_ep": self.elastic_pending_ep_size is not None,
"effective_ep_size": self.elastic_worker_count,
"effective_ep_size": self.elastic_worker_count * self.attn_replica_size,
"pending_ep_size": self.elastic_pending_ep_size,
"scale_phase": self.elastic_scale_phase,
"last_error": self.elastic_last_error,
@@ -3678,7 +3686,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
"A previous scale operation has not completed yet. Wait until "
"all pending ranks have joined before issuing another scale."
),
old_ep_size=self.elastic_worker_count,
old_ep_size=self.elastic_worker_count * self.attn_replica_size,
new_ep_size=obj.new_ep_size,
pending_ep_size=self.elastic_pending_ep_size,
scale_phase=self.elastic_scale_phase,
@@ -49,8 +49,14 @@ from sglang.srt.elastic_ep.elastic_ep import (
maybe_recover_ep_ranks,
register_scale_cohort,
try_admit_scale_ranks,
update_dp_attention_for_elastic_ep,
validate_scale_cohort_topology,
)
from sglang.srt.elastic_ep.expert_backup_client import ExpertBackupClient
from sglang.srt.elastic_ep.topology import (
attn_replica_size,
physical_ep_size_to_dp_size,
)
from sglang.srt.environ import envs
from sglang.srt.eplb.eplb_manager import EPLBManager
from sglang.srt.eplb.expert_distribution import (
@@ -545,17 +551,13 @@ class ModelRunner:
)
)
from sglang.srt.layers.dp_attention import (
enable_joiner_all_gather,
update_dp_attention_post_scale,
)
from sglang.srt.layers.dp_attention import enable_joiner_all_gather
enable_joiner_all_gather()
update_dp_attention_post_scale(
new_dp_size=join_effective_ep_size,
new_dp_rank=global_ep_rank,
update_dp_attention_for_elastic_ep(
physical_ep_size=join_effective_ep_size,
physical_ep_rank=global_ep_rank,
)
get_parallel().override_permanently(num_dp_ranks=join_effective_ep_size)
if self.eplb_manager is not None:
self.eplb_manager.disable_rebalance(
"EPLB rebalance is disabled while elastic EP scale-up "
@@ -2326,13 +2328,12 @@ class ModelRunner:
"is being finalized"
)
from sglang.srt.layers.dp_attention import update_dp_attention_post_scale
update_dp_attention_post_scale(
new_dp_size=target_size,
new_dp_rank=self._elastic_global_rank(),
replica_size = attn_replica_size()
effective_dp_size = physical_ep_size_to_dp_size(effective_size, replica_size)
target_dp_size = update_dp_attention_for_elastic_ep(
physical_ep_size=target_size,
physical_ep_rank=self._elastic_global_rank(),
)
get_parallel().override_permanently(num_dp_ranks=target_size)
recapture_cuda_graph = self._elastic_cuda_graph_enabled()
if recapture_cuda_graph:
@@ -2351,8 +2352,8 @@ class ModelRunner:
self._pending_elastic_scale_update = ElasticScaleUpdateReq(
success=True,
effective_ep_size=target_size,
slot_offset=effective_size,
slot_count=target_size - effective_size,
slot_offset=effective_dp_size,
slot_count=target_dp_size - effective_dp_size,
)
logger.info(
"[Elastic EP] Scale completed: old_ep_size=%d "
@@ -2472,6 +2473,19 @@ class ModelRunner:
):
logger.error("[Elastic EP] %s", error)
return
try:
validate_scale_cohort_topology(cohort)
except ValueError as exc:
error = str(exc)
ElasticEPStateManager.fail_scale(error)
self._reset_eplb_after_elastic_scale_failure()
self._report_elastic_scale_failure(error, effective_size)
if (
get_parallel().tp_rank == 0
and not get_exec().moe.is_ep_scale_joiner
):
logger.error("[Elastic EP] %s", error)
return
cuda_graph_enabled = self._elastic_cuda_graph_enabled()
if cohort.cuda_graph_enabled != cuda_graph_enabled:
error = (