Signed-off-by: jain-ria <riajain@NVIDIA.com> Co-authored-by: ishandhanani <82981111+ishandhanani@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Ishan Dhanani <ishandhanani@gmail.com>
817 lines
32 KiB
Python
817 lines
32 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
"""Server-argument resolution for context- and decode-context parallelism."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
from typing import Any
|
|
|
|
from sglang.srt.arg_groups.cp_tp_group_sharing_hook import (
|
|
resolve_cp_tp_group_sharing,
|
|
)
|
|
from sglang.srt.arg_groups.overrides import (
|
|
_data_parallelism_defaults,
|
|
_dcp_comm_backend_default,
|
|
_dp_lm_head_validation,
|
|
_tp_lm_head_all_to_all_default,
|
|
declare_resolution,
|
|
model_config_of,
|
|
resolved_view,
|
|
resolving_view,
|
|
run_post_process_pass,
|
|
should_report_expert_balancedness,
|
|
)
|
|
from sglang.srt.arg_groups.resolution_hooks import run_hook
|
|
from sglang.srt.connector import ConnectorType
|
|
from sglang.srt.environ import envs
|
|
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
|
|
from sglang.srt.runtime_context import (
|
|
attn_dp_enabled_of,
|
|
derive_attn_tp_size,
|
|
get_platform,
|
|
)
|
|
from sglang.srt.utils.common import parse_connector_type
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def handle_deprecated_dp_attention(server_args: Any):
|
|
"""Turn the deprecated `--dp-size N --enable-dp-attention` into
|
|
`--attn-dp-size N`.
|
|
|
|
Runs before any handler reads the DP layout. The old spelling counted
|
|
attention-DP groups with `dp_size`; that width moves to `attn_dp_size`, and
|
|
`dp_size` counts replicas again. Next to `--attn-dp-size` the flag adds
|
|
nothing: a serialized config reports it set whenever attention DP runs
|
|
(`ServerArgs.resolved_dict`), so reading one back resolves the same layout.
|
|
"""
|
|
cfg = resolving_view(server_args)
|
|
if not cfg.enable_dp_attention:
|
|
return
|
|
if cfg.attn_dp_size == 1:
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_deprecated_dp_attention",
|
|
attn_dp_size=cfg.dp_size,
|
|
dp_size=1,
|
|
)
|
|
declare_resolution(
|
|
server_args, "_handle_deprecated_dp_attention", enable_dp_attention=False
|
|
)
|
|
|
|
|
|
def _boundary_parallelism_overrides(cfg, model_type: str, architecture: str) -> dict:
|
|
"""Resolve the token-row modes the model's forward actually enters."""
|
|
nemotron = model_type in ("nemotron_h", "nemotron_h_puzzle")
|
|
longcat = model_type == "longcat_flash"
|
|
inkling = architecture in (
|
|
"InklingForConditionalGeneration",
|
|
"InklingForConditionalGenerationMTP",
|
|
)
|
|
kimi_k3 = architecture in (
|
|
"KimiK3ForConditionalGeneration",
|
|
"KimiK3LinearForCausalLM",
|
|
)
|
|
if nemotron and cfg.attn_cp_size > 1:
|
|
raise ValueError("Nemotron-H does not support --attn-cp-size > 1")
|
|
if longcat and cfg.enable_prefill_cp and cfg.attn_cp_size > 1:
|
|
raise ValueError(
|
|
"LongCat-Flash does not support --enable-prefill-cp with --attn-cp-size > 1"
|
|
)
|
|
if (
|
|
nemotron or longcat or kimi_k3 or inkling
|
|
) and cfg.enable_attn_tp_input_scattered:
|
|
# None of these models enters the input-scattered attention scope.
|
|
logger.warning(
|
|
"Disabling input-scattered attention for %s: its forward does not enter that scope",
|
|
architecture,
|
|
)
|
|
return {"enable_attn_tp_input_scattered": False}
|
|
return {}
|
|
|
|
|
|
def handle_context_parallelism(server_args: Any):
|
|
# Through the registry, not a bare call: an out-of-tree replacement of
|
|
# `validate_prefill_cp_platform` registered at its own (earlier) pipeline
|
|
# position must also win here, or a package permitting prefill CP on its
|
|
# own qualified NPU/MUSA build would still hit the original rejection
|
|
# at this later, nested call.
|
|
run_hook(validate_prefill_cp_platform, server_args)
|
|
|
|
cfg = resolving_view(server_args)
|
|
if parse_connector_type(cfg.model_path) != ConnectorType.INSTANCE:
|
|
model_config = model_config_of(server_args)
|
|
hf_config = model_config.hf_config
|
|
model_arch = hf_config.architectures[0]
|
|
declare_resolution(
|
|
server_args,
|
|
"boundary_parallelism",
|
|
**_boundary_parallelism_overrides(
|
|
cfg, model_config.hf_text_config.model_type, model_arch
|
|
),
|
|
)
|
|
if (
|
|
cfg.enable_prefill_cp
|
|
and get_platform().is_hip
|
|
and model_arch != "DeepseekV4ForCausalLM"
|
|
):
|
|
raise ValueError(
|
|
"Prefill CP on HIP is only supported for "
|
|
f"DeepseekV4ForCausalLM, got {model_arch!r}."
|
|
)
|
|
if (
|
|
cfg.enable_prefill_cp
|
|
and model_arch == "DeepseekV32ForCausalLM"
|
|
and cfg.cp_strategy == "zigzag"
|
|
):
|
|
raise ValueError(
|
|
"DeepSeek V3.2 prefill CP does not support --cp-strategy "
|
|
"zigzag; use interleave."
|
|
)
|
|
if cfg.enable_prefill_cp and model_arch in (
|
|
"MiMoV2ForCausalLM",
|
|
"MiMoV2FlashForCausalLM",
|
|
):
|
|
if cfg.cp_strategy != "zigzag":
|
|
raise ValueError(
|
|
"MiMo V2 prefill CP only supports --cp-strategy zigzag."
|
|
)
|
|
if (
|
|
model_config.is_multimodal
|
|
and not cfg.language_only
|
|
and not cfg.language_model_only
|
|
):
|
|
raise ValueError(
|
|
"MiMo V2 prefill CP only supports text inference; add "
|
|
"--language-only."
|
|
)
|
|
resolve_cp_tp_group_sharing(server_args, model_config)
|
|
|
|
if cfg.enable_prefill_cp and cfg.cp_strategy is None:
|
|
raise ValueError(
|
|
"--cp-strategy must be set when --enable-prefill-cp is enabled."
|
|
)
|
|
|
|
view = resolved_view(server_args)
|
|
if view.attn_cp_size > 1:
|
|
# The tp_size is the world size, not the real tensor parallel size
|
|
assert cfg.tp_size % view.attn_cp_size == 0, (
|
|
"tp_size must be divisible by attn_cp_size"
|
|
)
|
|
assert cfg.tp_size % (cfg.attn_dp_size * view.attn_cp_size) == 0, (
|
|
"tp_size must be divisible by attn_dp_size * attn_cp_size"
|
|
)
|
|
|
|
assert not cfg.enable_aiter_allreduce_fusion, (
|
|
"Aiter allreduce fusion is not supported with context parallelism"
|
|
)
|
|
|
|
if cfg.moe_dp_size > 1:
|
|
# The tp_size is the world size, not the real tensor parallel size
|
|
assert cfg.tp_size % cfg.moe_dp_size == 0, (
|
|
"tp_size must be divisible by moe_dp_size"
|
|
)
|
|
assert view.ep_size * cfg.moe_dp_size <= cfg.tp_size, (
|
|
"ep_size * moe_dp_size must be less than or equal to tp_size"
|
|
)
|
|
assert cfg.pp_size == 1, "PP is not supported with context parallelism"
|
|
|
|
if view.ep_size > 1:
|
|
assert view.ep_size * cfg.moe_dp_size == cfg.tp_size, (
|
|
"ep_size * moe_dp_size must be equal to tp_size"
|
|
)
|
|
|
|
assert not cfg.enable_aiter_allreduce_fusion, (
|
|
"Aiter allreduce fusion is not supported with context parallelism"
|
|
)
|
|
|
|
if view.attn_cp_size != cfg.moe_dp_size:
|
|
assert cfg.moe_dp_size == 1, (
|
|
"attn_cp_size != moe_dp_size is only supported when moe_dp_size == 1"
|
|
)
|
|
|
|
from sglang.srt.layers.cp.base import init_cp_strategy
|
|
|
|
init_cp_strategy(
|
|
enable_prefill_cp=bool(cfg.enable_prefill_cp),
|
|
cp_size=cfg.attn_cp_size,
|
|
cp_strategy=cfg.cp_strategy,
|
|
)
|
|
|
|
|
|
def handle_shared_experts_tp(server_args: Any):
|
|
cfg = resolving_view(server_args)
|
|
size = cfg.shared_experts_tp_size
|
|
if size is None:
|
|
return
|
|
|
|
view = resolved_view(server_args)
|
|
attn_tp_size = derive_attn_tp_size(
|
|
tp_size=cfg.tp_size,
|
|
attn_cp_size=view.attn_cp_size,
|
|
attn_dp_size=cfg.attn_dp_size,
|
|
)
|
|
if size < 1 or attn_tp_size % size != 0:
|
|
raise ValueError(
|
|
f"--shared-experts-tp-size ({size}) must be a positive divisor "
|
|
f"of attention TP size ({attn_tp_size})."
|
|
)
|
|
if parse_connector_type(cfg.model_path) == ConnectorType.INSTANCE:
|
|
raise ValueError(
|
|
"--shared-experts-tp-size requires a Kimi-K3 model configuration."
|
|
)
|
|
model_arch = model_config_of(server_args).hf_config.architectures[0]
|
|
if model_arch != "KimiK3ForConditionalGeneration":
|
|
raise ValueError("--shared-experts-tp-size is only supported for Kimi-K3.")
|
|
if cfg.moe_a2a_backend not in (
|
|
"deepep",
|
|
"megamoe",
|
|
"mooncake",
|
|
"ascend_fuseep",
|
|
"mori",
|
|
):
|
|
raise ValueError(
|
|
"--shared-experts-tp-size requires an expert-parallel all-to-all "
|
|
"backend (deepep, megamoe, mooncake, ascend_fuseep or mori)."
|
|
)
|
|
|
|
|
|
def handle_decode_context_parallelism(server_args: Any):
|
|
run_post_process_pass(server_args, _dcp_comm_backend_default)
|
|
cfg = resolving_view(server_args)
|
|
if cfg.dcp_size < 1:
|
|
raise ValueError(
|
|
"Decode context parallel size (--dcp-size / "
|
|
"--decode-context-parallel-size) must be >= 1, but got "
|
|
f"dcp_size={cfg.dcp_size}."
|
|
)
|
|
if cfg.dcp_size > 1 and cfg.enable_prefill_cp:
|
|
raise ValueError("--dcp-size > 1 does not support --enable-prefill-cp.")
|
|
if cfg.dcp_comm_backend in ("a2a", "fi_a2a") and cfg.dcp_size <= 1:
|
|
raise ValueError(
|
|
f"--dcp-comm-backend {cfg.dcp_comm_backend} only affects the "
|
|
"decode context-parallel attention reduction and therefore "
|
|
"requires --dcp-size / --decode-context-parallel-size > 1, but "
|
|
f"got dcp_size={cfg.dcp_size}."
|
|
)
|
|
if cfg.dcp_comm_backend == "fi_a2a" and not get_platform().is_cuda:
|
|
raise ValueError(
|
|
"--dcp-comm-backend fi_a2a delegates the exchange to FlashInfer's "
|
|
"MNNVL All-to-All kernel, which requires Blackwell and a DCP group "
|
|
"within one MNNVL domain. Use 'a2a' or 'ag_rs' elsewhere, or leave "
|
|
"the flag unset to resolve it."
|
|
)
|
|
if cfg.dcp_replicate_q_proj:
|
|
if cfg.dcp_size <= 1:
|
|
raise ValueError("--dcp-replicate-q-proj requires --dcp-size > 1.")
|
|
if cfg.dcp_comm_backend not in ("a2a", "fi_a2a"):
|
|
raise ValueError(
|
|
"--dcp-replicate-q-proj only applies to the a2a/fi_a2a DCP "
|
|
"communication backend (it removes the head-dim Q all-gather); "
|
|
f"got --dcp-comm-backend={cfg.dcp_comm_backend}."
|
|
)
|
|
|
|
|
|
def handle_data_parallelism(server_args: Any):
|
|
# The resets without attention DP live in the resolution pipeline
|
|
# (arg_groups/overrides.py: _data_parallelism_defaults).
|
|
from sglang.srt.arg_groups.cuda_graph_hook import (
|
|
generate_prefill_cuda_graph_batch_sizes,
|
|
)
|
|
|
|
cfg = resolving_view(server_args)
|
|
|
|
if cfg.attn_dp_size < 1:
|
|
raise ValueError(f"--attn-dp-size must be positive (got {cfg.attn_dp_size}).")
|
|
if cfg.attn_dp_size > 1 and cfg.dp_size > 1:
|
|
raise ValueError(
|
|
f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: "
|
|
"data-parallel replicas combined with attention data parallelism "
|
|
"are not supported."
|
|
)
|
|
|
|
run_post_process_pass(server_args, _data_parallelism_defaults)
|
|
|
|
if cfg.mm_enable_dp_encoder:
|
|
if cfg.tp_size == 1:
|
|
logger.warning(
|
|
"--mm-enable-dp-encoder is enabled with TP=1, so the encoder "
|
|
"has no data-parallel work to distribute. Disable it unless "
|
|
"you need to validate this configuration."
|
|
)
|
|
else:
|
|
logger.info(
|
|
"--mm-enable-dp-encoder is enabled across TP=%d. It replicates "
|
|
"the vision encoder and distributes image work across ranks; "
|
|
"this is most useful when high-resolution or multi-image ViT "
|
|
"prefill is a material part of TTFT. Measure against the default "
|
|
"for small-image workloads because replication and aggregation "
|
|
"can increase memory use and overhead.",
|
|
cfg.tp_size,
|
|
)
|
|
|
|
if attn_dp_enabled_of(resolved_view(server_args)):
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_data_parallelism",
|
|
schedule_conservativeness=cfg.schedule_conservativeness * 0.3,
|
|
)
|
|
if envs.SGLANG_RUST_SERVER.get():
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_data_parallelism",
|
|
enable_dp_attention_local_control_broadcast=True,
|
|
)
|
|
assert cfg.tp_size % cfg.attn_dp_size == 0
|
|
original_chunked_prefill_size = cfg.chunked_prefill_size
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_data_parallelism",
|
|
chunked_prefill_size=cfg.chunked_prefill_size // cfg.attn_dp_size,
|
|
)
|
|
logger.warning(
|
|
f"DP attention is enabled. chunked prefill size is adjusted "
|
|
f"from {original_chunked_prefill_size} to {cfg.chunked_prefill_size}."
|
|
)
|
|
|
|
# The prefill CUDA graph max_bs was derived from the pre-DP-division
|
|
# chunked_prefill_size in _handle_gpu_memory_settings (which runs
|
|
# before this handler). Re-clamp it (and the captured shape list) to
|
|
# the per-DP-rank chunked_prefill_size so breakable CUDA graph
|
|
# capture never exceeds the MoE all-to-all's max_num_tokens budget,
|
|
# which is also sized from the DP-adjusted chunked_prefill_size.
|
|
prefill_cfg = cfg.cuda_graph_config.prefill
|
|
if (
|
|
prefill_cfg.backend != Backend.DISABLED
|
|
and prefill_cfg.max_bs is not None
|
|
and prefill_cfg.max_bs > cfg.chunked_prefill_size
|
|
and (Phase.PREFILL, "max_bs") not in server_args._cuda_graph_config_locked
|
|
):
|
|
clamped = {"max_bs": cfg.chunked_prefill_size}
|
|
if (Phase.PREFILL, "bs") not in server_args._cuda_graph_config_locked:
|
|
clamped["bs"] = generate_prefill_cuda_graph_batch_sizes(
|
|
clamped["max_bs"]
|
|
)
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_data_parallelism",
|
|
cuda_graph_config=with_phase(
|
|
cfg.cuda_graph_config, Phase.PREFILL, **clamped
|
|
),
|
|
)
|
|
|
|
# Resolve the phase-aware TP LM-head default before validating the
|
|
# resulting DP/TP LM-head configuration.
|
|
|
|
run_post_process_pass(server_args, _tp_lm_head_all_to_all_default)
|
|
run_post_process_pass(server_args, _dp_lm_head_validation)
|
|
if resolving_view(server_args).enable_tp_lm_head_all_to_all:
|
|
_disable_nccl_graph_buffer_registration(
|
|
"the graph-captured TP LM-head all-to-all can deadlock with "
|
|
"registered buffers"
|
|
)
|
|
if _graph_pool_is_pausable(server_args):
|
|
_disable_nccl_graph_buffer_registration(
|
|
"graph replay can hang once torch_memory_saver resume remaps the "
|
|
"graph pool, leaving the capture-time registrations on released pages"
|
|
)
|
|
|
|
|
|
def _graph_pool_is_pausable(server_args: Any) -> bool:
|
|
"""Whether CUDA graphs are captured into torch_memory_saver memory, which
|
|
release/resume of the `cuda_graph` tag remaps to new physical pages at the
|
|
same virtual addresses."""
|
|
return bool(
|
|
resolving_view(server_args).enable_memory_saver
|
|
and envs.SGLANG_MEMORY_SAVER_CUDA_GRAPH.get()
|
|
)
|
|
|
|
|
|
def _disable_nccl_graph_buffer_registration(reason: str) -> None:
|
|
"""Keep NCCL from registering the buffers of graph-captured collectives.
|
|
|
|
NCCL_GRAPH_REGISTER (default on) registers the send/recv buffers of every
|
|
collective captured in a CUDA graph for the lifetime of the graph, and
|
|
peers then move data through those registrations directly. The TP LM-head
|
|
all-to-all is captured in the decode graphs on graph-pool temporaries,
|
|
whose addresses the pool also hands to other tensors, and the registered
|
|
exchange does not survive that: under a burst of new requests (DP ranks
|
|
ramping at different rates) one rank finishes its step while the others
|
|
spin in ncclDevKernel_SendRecv forever, and every DP rank hangs.
|
|
Reproduced on tp4/dp4/ep4 and on a multi-node tp16/dp16/ep16 PD decode
|
|
deployment; disabling the registration removes the hang while dedicated
|
|
all-to-all buffers alone do not.
|
|
|
|
A pausable graph pool breaks the registrations too: torch_memory_saver
|
|
resume keeps the pool's virtual addresses but maps new physical pages,
|
|
while NCCL's capture-time registrations still point at the released pages,
|
|
so graph replay can hang (fzyzcjy/torch_memory_saver#88).
|
|
|
|
Must run before the schedulers create their NCCL communicators, which
|
|
inherit this environment. An explicit setting wins.
|
|
"""
|
|
if os.environ.setdefault("NCCL_GRAPH_REGISTER", "0") != "0":
|
|
logger.warning(
|
|
"NCCL_GRAPH_REGISTER=%s was set explicitly; %s.",
|
|
os.environ["NCCL_GRAPH_REGISTER"],
|
|
reason,
|
|
)
|
|
|
|
|
|
def handle_dwdp(server_args: Any):
|
|
cfg = resolving_view(server_args)
|
|
if cfg.dwdp_size <= 1:
|
|
return
|
|
|
|
assert cfg.dwdp_size >= 2, (
|
|
f"dwdp_size must be >= 2 when enabled, got {cfg.dwdp_size}"
|
|
)
|
|
assert cfg.dwdp_size == cfg.tp_size, (
|
|
f"dwdp_size ({cfg.dwdp_size}) must equal tp_size ({cfg.tp_size})"
|
|
)
|
|
assert cfg.disaggregation_mode in (
|
|
"null",
|
|
"prefill",
|
|
), "DWDP requires --disaggregation-mode null or prefill"
|
|
assert not cfg.enable_eplb, (
|
|
"EPLB dynamic migration conflicts with static DWDP partitioning"
|
|
)
|
|
assert cfg.speculative_algorithm is None, (
|
|
"DWDP does not support speculative decoding (MTP/draft workers)"
|
|
)
|
|
assert cfg.pp_size == 1, "DWDP requires pp_size == 1"
|
|
assert not cfg.enable_two_batch_overlap, (
|
|
"DWDP's prefetch event protocol does not support two-batch overlap"
|
|
)
|
|
|
|
if cfg.disaggregation_mode == "null":
|
|
logger.warning(
|
|
"DWDP with --disaggregation-mode null: decode steps re-fetch all "
|
|
"remote expert weights every step, which is slow. DWDP is "
|
|
"recommended only with --disaggregation-mode prefill."
|
|
)
|
|
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_dwdp",
|
|
attn_dp_size=cfg.dwdp_size,
|
|
dp_size=1,
|
|
)
|
|
declare_resolution(
|
|
server_args, "_handle_dwdp", enable_dp_attention_local_control_broadcast=True
|
|
)
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_dwdp",
|
|
enable_dp_lm_head=True,
|
|
)
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_dwdp",
|
|
moe_dense_tp_size=1,
|
|
)
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_dwdp",
|
|
ep_size=cfg.dwdp_size,
|
|
)
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_dwdp",
|
|
moe_dp_size=1,
|
|
)
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_dwdp",
|
|
moe_a2a_backend="none",
|
|
)
|
|
|
|
envs.SGLANG_SCHEDULER_SKIP_ALL_GATHER.set(True)
|
|
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_dwdp",
|
|
disable_cuda_graph=True,
|
|
)
|
|
|
|
logger.info(
|
|
f"DWDP enabled: dwdp_size={cfg.dwdp_size}, "
|
|
f"auto-forced attn_dp_size={cfg.attn_dp_size}, ep_size={cfg.dwdp_size}, "
|
|
f"moe_dense_tp_size=1, moe_a2a_backend=none, "
|
|
f"dp_attention_local_control_broadcast=True, "
|
|
f"enable_dp_lm_head=True, SCHEDULER_SKIP_ALL_GATHER=True, "
|
|
f"disable_cuda_graph=True"
|
|
)
|
|
|
|
|
|
def handle_elastic_ep(server_args: Any):
|
|
from sglang.srt.arg_groups.validation_hook import validate_ib_devices
|
|
|
|
cfg = resolving_view(server_args)
|
|
if cfg.elastic_ep_backend is not None:
|
|
if cfg.enable_eplb:
|
|
if cfg.eplb_algorithm == "auto":
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_elastic_ep",
|
|
eplb_algorithm="elasticity_aware",
|
|
)
|
|
assert cfg.eplb_algorithm in [
|
|
"elasticity_aware",
|
|
"elasticity_aware_hierarchical",
|
|
], (
|
|
"Elastic EP requires eplb_algorithm to be set to 'auto' or 'elasticity_aware(_hierarchical)'."
|
|
)
|
|
|
|
assert cfg.pp_size == 1, "PP size should be set to 1 under elastic EP"
|
|
|
|
if cfg.elastic_ep_backend == "mooncake":
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_elastic_ep",
|
|
mooncake_ib_device=validate_ib_devices(cfg.mooncake_ib_device),
|
|
)
|
|
if cfg.ep_join_mode is not None:
|
|
assert cfg.elastic_ep_backend is not None, (
|
|
"--elastic-ep-join-mode requires --elastic-ep-backend to be set."
|
|
)
|
|
if cfg.ep_join_mode == "scale":
|
|
assert cfg.node_rank == 1, (
|
|
"Elastic EP scale-up requires one joining TP group at "
|
|
f"--node-rank 1 (got {cfg.node_rank})."
|
|
)
|
|
assert cfg.ep_join_rank_offset > 0, (
|
|
"Elastic EP scale joiners require "
|
|
"--elastic-ep-join-rank-offset set to the current "
|
|
"effective EP size."
|
|
)
|
|
if cfg.ep_join_rank_offset != 0:
|
|
assert cfg.ep_join_mode == "scale", (
|
|
"--elastic-ep-join-rank-offset is only valid with "
|
|
"--elastic-ep-join-mode scale."
|
|
)
|
|
assert cfg.ep_join_rank_offset >= 0, "elastic EP join rank offset must be >= 0."
|
|
if cfg.max_ep_size is not None:
|
|
assert cfg.elastic_ep_backend is not None, (
|
|
"--max-ep-size requires --elastic-ep-backend to be set."
|
|
)
|
|
assert cfg.max_ep_size > 0, "--max-ep-size must be a positive integer."
|
|
|
|
scaling_active = (
|
|
cfg.elastic_ep_backend is not None
|
|
and cfg.max_ep_size is not None
|
|
and cfg.max_ep_size > cfg.tp_size
|
|
)
|
|
if cfg.elastic_ep_initial_size is not None:
|
|
assert scaling_active, (
|
|
"--elastic-ep-initial-size is only valid for an Elastic EP "
|
|
"deployment with --max-ep-size larger than its local TP size."
|
|
)
|
|
if scaling_active:
|
|
resolved = resolved_view(server_args)
|
|
from sglang.srt.elastic_ep.topology import physical_ep_size_to_dp_size
|
|
|
|
try:
|
|
divisor = cfg.attn_dp_size * resolved.attn_cp_size
|
|
if (
|
|
cfg.tp_size <= 0
|
|
or cfg.attn_dp_size <= 0
|
|
or resolved.attn_cp_size <= 0
|
|
or cfg.tp_size % divisor
|
|
):
|
|
raise ValueError(
|
|
"Invalid attention topology: "
|
|
f"tp_size={cfg.tp_size}, attn_dp_size={cfg.attn_dp_size}, "
|
|
f"attn_cp_size={resolved.attn_cp_size}."
|
|
)
|
|
attn_tp_size = derive_attn_tp_size(
|
|
tp_size=cfg.tp_size,
|
|
attn_cp_size=resolved.attn_cp_size,
|
|
attn_dp_size=cfg.attn_dp_size,
|
|
)
|
|
attn_replica_size = attn_tp_size * resolved.attn_cp_size
|
|
physical_ep_size_to_dp_size(cfg.max_ep_size, attn_replica_size)
|
|
physical_ep_size_to_dp_size(
|
|
cfg.elastic_ep_initial_size or cfg.tp_size, attn_replica_size
|
|
)
|
|
if cfg.ep_join_mode == "scale":
|
|
physical_ep_size_to_dp_size(cfg.ep_join_rank_offset, attn_replica_size)
|
|
except ValueError as exc:
|
|
raise AssertionError(str(exc)) from exc
|
|
|
|
if attn_tp_size > 1:
|
|
assert cfg.moe_dense_tp_size == 1, (
|
|
"Elastic EP scale-up with attention TP greater than one requires "
|
|
"--moe-dense-tp-size 1 "
|
|
f"(got moe_dense_tp_size={cfg.moe_dense_tp_size})."
|
|
)
|
|
|
|
assert cfg.elastic_ep_scale_timeout > 0, (
|
|
"--elastic-ep-scale-timeout must be greater than zero."
|
|
)
|
|
assert cfg.tokenizer_worker_num == 1, (
|
|
"Elastic EP runtime scale-up currently requires --tokenizer-worker-num 1."
|
|
)
|
|
assert not cfg.use_ray, (
|
|
"Elastic EP runtime scale-up does not support --use-ray."
|
|
)
|
|
assert not cfg.enable_elastic_expert_backup, (
|
|
"Elastic EP runtime scale-up does not support "
|
|
"--enable-elastic-expert-backup."
|
|
)
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_elastic_ep",
|
|
enable_dp_attention_local_control_broadcast=True,
|
|
)
|
|
if cfg.ep_join_mode == "scale":
|
|
assert cfg.elastic_ep_initial_size is not None, (
|
|
"Elastic EP scale joiners require --elastic-ep-initial-size "
|
|
"set to the primary deployment's launch-time EP size."
|
|
)
|
|
assert cfg.elastic_ep_initial_size <= cfg.ep_join_rank_offset, (
|
|
"--elastic-ep-initial-size cannot exceed the current EP size "
|
|
f"(initial={cfg.elastic_ep_initial_size}, "
|
|
f"current={cfg.ep_join_rank_offset})."
|
|
)
|
|
join_target = cfg.ep_join_rank_offset + cfg.tp_size
|
|
assert join_target <= cfg.max_ep_size, (
|
|
"Elastic EP joining group exceeds --max-ep-size "
|
|
f"(join_target={join_target}, max_ep_size={cfg.max_ep_size})."
|
|
)
|
|
if cfg.tp_size == 1:
|
|
assert cfg.moe_dense_tp_size == 1, (
|
|
"A single-rank Elastic EP joining group requires "
|
|
"--moe-dense-tp-size 1."
|
|
)
|
|
else:
|
|
if cfg.elastic_ep_initial_size is None:
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_elastic_ep",
|
|
elastic_ep_initial_size=cfg.tp_size,
|
|
)
|
|
assert cfg.elastic_ep_initial_size == cfg.tp_size, (
|
|
"The primary --elastic-ep-initial-size must equal its "
|
|
f"launch-time TP size ({cfg.tp_size})."
|
|
)
|
|
assert cfg.elastic_ep_initial_size > 0
|
|
assert cfg.load_balance_method == "round_robin", (
|
|
"Elastic EP scale-up requires --load-balance-method round_robin; "
|
|
"load-aware methods "
|
|
"require global-rank load snapshots after scale "
|
|
f"(got {cfg.load_balance_method})."
|
|
)
|
|
assert cfg.elastic_ep_backend == "mooncake", (
|
|
"Elastic EP runtime scale-up requires --elastic-ep-backend "
|
|
f"mooncake (got elastic_ep_backend={cfg.elastic_ep_backend})."
|
|
)
|
|
assert cfg.pp_size == 1, (
|
|
"Elastic EP scale-up requires --pp-size 1 "
|
|
f"(got pp_size={cfg.pp_size}); WORLD must not span PP stages."
|
|
)
|
|
assert attn_tp_size % cfg.dcp_size == 0, (
|
|
"Elastic EP scale-up requires --dcp-size to divide the fixed "
|
|
"attention TP width "
|
|
f"(got dcp_size={cfg.dcp_size}, attn_tp_size={attn_tp_size})."
|
|
)
|
|
|
|
decode_backend = cfg.cuda_graph_config.decode.backend
|
|
assert decode_backend in (Backend.DISABLED, Backend.FULL), (
|
|
"Elastic EP runtime scale-up supports decode CUDA graph backend "
|
|
f"'full' or 'disabled' (got {decode_backend!r})."
|
|
)
|
|
assert cfg.cuda_graph_config.prefill.backend == Backend.DISABLED, (
|
|
"Elastic EP runtime scale-up requires prefill CUDA graph to be disabled."
|
|
)
|
|
if decode_backend == Backend.FULL:
|
|
assert cfg.device == "cuda", (
|
|
"Elastic EP CUDA graph recapture requires CUDA "
|
|
f"(got device={cfg.device!r})."
|
|
)
|
|
assert cfg.speculative_algorithm is None, (
|
|
"Elastic EP CUDA graph recapture does not support speculative decoding."
|
|
)
|
|
assert not cfg.is_embedding, (
|
|
"Elastic EP CUDA graph recapture does not support embedding models."
|
|
)
|
|
assert cfg.dllm_algorithm is None, (
|
|
"Elastic EP CUDA graph recapture does not support diffusion models."
|
|
)
|
|
assert not cfg.encoder_only, (
|
|
"Elastic EP CUDA graph recapture does not support encoder-only models."
|
|
)
|
|
assert not cfg.forward_hooks, (
|
|
"Elastic EP CUDA graph recapture does not support forward hooks."
|
|
)
|
|
assert not cfg.enable_pdmux, (
|
|
"Elastic EP CUDA graph recapture does not support PDMux."
|
|
)
|
|
assert attn_dp_enabled_of(resolved), (
|
|
"Elastic EP scale-up requires attention DP (--attn-dp-size); "
|
|
"without it the TP group is not equivalent to WORLD and the "
|
|
"post-scale collective path is invalid."
|
|
)
|
|
assert resolved.enable_dp_lm_head, (
|
|
"Elastic EP scale-up requires --enable-dp-lm-head so output "
|
|
"projection does not depend on the joining group's TP size."
|
|
)
|
|
assert resolved.attn_cp_size == 1, (
|
|
"Elastic EP scale-up requires --attn-cp-size 1 "
|
|
f"(got attn_cp_size={resolved.attn_cp_size})."
|
|
)
|
|
assert cfg.moe_dp_size == 1, (
|
|
"Elastic EP scale-up requires --moe-dp-size 1 "
|
|
f"(got moe_dp_size={cfg.moe_dp_size})."
|
|
)
|
|
assert resolved.ep_size == cfg.tp_size, (
|
|
"Elastic EP scale-up requires ep_size == tp_size "
|
|
f"(got ep_size={resolved.ep_size}, tp_size={cfg.tp_size}); "
|
|
"physical EP membership must coincide with the launch-time TP span."
|
|
)
|
|
assert resolved.moe_a2a_backend == "nixl", (
|
|
"Elastic EP scale-up requires --moe-a2a-backend nixl "
|
|
f"(got moe_a2a_backend={resolved.moe_a2a_backend})."
|
|
)
|
|
|
|
|
|
def handle_eplb_and_dispatch(server_args: Any):
|
|
cfg = resolving_view(server_args)
|
|
if cfg.enable_eplb and (cfg.expert_distribution_recorder_mode is None):
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_eplb_and_dispatch",
|
|
expert_distribution_recorder_mode="stat",
|
|
)
|
|
logger.warning(
|
|
"EPLB is enabled. The expert_distribution_recorder_mode is automatically set."
|
|
)
|
|
|
|
# Without an a2a backend all EP ranks run the MoE over the same tokens and
|
|
# sum their partial outputs, so the pick has to agree across ranks.
|
|
needs_rank_invariant_dispatch = resolved_view(server_args).moe_a2a_backend == "none"
|
|
|
|
if (cfg.enable_eplb or (cfg.init_expert_location != "trivial")) and (
|
|
cfg.ep_dispatch_algorithm is None
|
|
):
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_eplb_and_dispatch",
|
|
ep_dispatch_algorithm=(
|
|
"dynamic" if needs_rank_invariant_dispatch else "static"
|
|
),
|
|
)
|
|
|
|
# `dynamic` / `fake` switch to the row-index pick; `static` reads a
|
|
# per-rank table and `lp` samples inside its kernel.
|
|
if needs_rank_invariant_dispatch and cfg.ep_dispatch_algorithm in (
|
|
"static",
|
|
"lp",
|
|
):
|
|
raise ValueError(
|
|
f"--ep-dispatch-algorithm {cfg.ep_dispatch_algorithm} picks a "
|
|
"different physical replica per rank, which only holds up when an "
|
|
"a2a backend routes each token to a single rank. Use "
|
|
"--ep-dispatch-algorithm dynamic with --moe-a2a-backend none."
|
|
)
|
|
|
|
if cfg.enable_eplb and cfg.ep_join_mode != "scale":
|
|
assert resolved_view(server_args).ep_size > 1
|
|
|
|
|
|
def handle_expert_distribution_metrics(server_args: Any):
|
|
cfg = resolving_view(server_args)
|
|
if should_report_expert_balancedness(server_args) and (
|
|
cfg.expert_distribution_recorder_mode is None
|
|
):
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_expert_distribution_metrics",
|
|
expert_distribution_recorder_mode="stat",
|
|
)
|
|
|
|
if cfg.expert_distribution_recorder_buffer_size is None:
|
|
if (x := cfg.eplb_rebalance_num_iterations) is not None:
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_expert_distribution_metrics",
|
|
expert_distribution_recorder_buffer_size=x,
|
|
)
|
|
elif cfg.expert_distribution_recorder_mode is not None:
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_expert_distribution_metrics",
|
|
expert_distribution_recorder_buffer_size=1000,
|
|
)
|
|
|
|
|
|
def validate_prefill_cp_platform(server_args: Any):
|
|
"""Reject deprecated platform CP before resolving models or CP topology."""
|
|
cfg = resolving_view(server_args)
|
|
platform = get_platform()
|
|
if cfg.enable_prefill_cp and platform.is_musa:
|
|
raise ValueError(
|
|
"Prefill CP on MUSA is deprecated; CP support will be refactored soon."
|
|
)
|