[Feature] Add --attn-dp-size and deprecate --enable-dp-attention (#41818)
This commit is contained in:
@@ -65,7 +65,7 @@ async def warm_up_compile(
|
||||
disaggregation_mode: str, tokenizer_manager: TokenizerManager
|
||||
):
|
||||
print("\nGenerate warm up request for compiling DeepGEMM...\n")
|
||||
dp_size = get_parallel().dp_size
|
||||
dp_size = get_parallel().num_dp_ranks
|
||||
base_ids = [0, 1, 2, 3]
|
||||
sampling_params = {
|
||||
"temperature": 0.0,
|
||||
|
||||
@@ -42,11 +42,11 @@ def print_deprecated_warning(message: str):
|
||||
# * an old valued flag replaced by a renamed valued flag
|
||||
# -> `DeprecatedAliasStoreAction`.
|
||||
#
|
||||
# Only the second has a registration today (`--disable-cuda-graph`). The other
|
||||
# three are kept because the shapes recur -- this package has retired flags of
|
||||
# every one of them -- and the fiddly parts (`nargs=0` on a boolean, where the
|
||||
# const goes, warn-and-continue versus `parser.error`) are what a
|
||||
# reimplementation gets wrong. Pass `new_flag=` so the warning tells the
|
||||
# Only the second has registrations today (`--disable-cuda-graph`,
|
||||
# `--enable-dp-attention`). The other three are kept because the shapes recur
|
||||
# -- this package has retired flags of every one of them -- and the fiddly
|
||||
# parts (`nargs=0` on a boolean, where the const goes, warn-and-continue versus
|
||||
# `parser.error`) are what a reimplementation gets wrong. Pass `new_flag=` so the warning tells the
|
||||
# operator what to switch to; that pointer is the whole point.
|
||||
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ from sglang.srt.model_executor.cuda_graph_config import (
|
||||
with_phase,
|
||||
)
|
||||
from sglang.srt.platforms import current_platform
|
||||
from sglang.srt.runtime_context import get_platform
|
||||
from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform
|
||||
from sglang.srt.utils.common import (
|
||||
is_cpu,
|
||||
is_mps,
|
||||
@@ -186,7 +186,7 @@ def disable_tc_piecewise_cudagraph_if_incompatible(server_args: Any):
|
||||
"model-arch blacklist",
|
||||
lambda: model_config_of(server_args).is_piecewise_cuda_graph_disabled_model,
|
||||
),
|
||||
("DP attention", lambda: resolved_view(server_args).enable_dp_attention),
|
||||
("DP attention", lambda: attn_dp_enabled_of(resolved_view(server_args))),
|
||||
("full torch.compile mode", lambda: cfg.enable_torch_compile),
|
||||
("pipeline parallelism (pp_size > 1)", lambda: cfg.pp_size > 1),
|
||||
(
|
||||
|
||||
@@ -11,7 +11,7 @@ from sglang.srt.arg_groups.overrides import (
|
||||
run_post_process_pass,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.runtime_context import get_platform
|
||||
from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform, num_dp_ranks_of
|
||||
from sglang.srt.utils.common import is_gfx95_supported, is_npu
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -123,10 +123,18 @@ def validate_deepseek_v4_cp(server_args: ServerArgs) -> None:
|
||||
"reindexes with interleave order."
|
||||
)
|
||||
|
||||
# DeepSeek-V4 CP runs data-parallel groups as attention DP.
|
||||
assert not (cfg.attn_dp_size > 1 and cfg.dp_size > 1), (
|
||||
f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: "
|
||||
"data-parallel replicas combined with attention data parallelism "
|
||||
"are not supported."
|
||||
)
|
||||
attn_dp_size = cfg.attn_dp_size * cfg.dp_size
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"validate_deepseek_v4_cp",
|
||||
enable_dp_attention=True,
|
||||
attn_dp_size=attn_dp_size,
|
||||
dp_size=1,
|
||||
)
|
||||
declare_resolution(
|
||||
server_args,
|
||||
@@ -136,10 +144,10 @@ def validate_deepseek_v4_cp(server_args: ServerArgs) -> None:
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"validate_deepseek_v4_cp",
|
||||
attn_cp_size=cfg.tp_size // cfg.dp_size,
|
||||
attn_cp_size=cfg.tp_size // attn_dp_size,
|
||||
)
|
||||
if not is_npu():
|
||||
assert cfg.dp_size == 1, (
|
||||
assert attn_dp_size == 1, (
|
||||
"For round-robin split mode, dp attention is not supported."
|
||||
)
|
||||
assert cfg.tp_size <= 8, (
|
||||
@@ -161,7 +169,7 @@ def validate_deepseek_v4_cp(server_args: ServerArgs) -> None:
|
||||
logger.warning(
|
||||
f"Enable Context Parallel for DeepSeekV4, "
|
||||
f"strategy={cfg.cp_strategy}, "
|
||||
f"dp_size={cfg.dp_size}, moe_dense_tp_size={cfg.moe_dense_tp_size}, "
|
||||
f"attn_dp_size={attn_dp_size}, moe_dense_tp_size={cfg.moe_dense_tp_size}, "
|
||||
f"attn_cp_size={cfg.attn_cp_size}, ep_size={cfg.ep_size}, tp_size={cfg.tp_size}"
|
||||
)
|
||||
|
||||
@@ -193,7 +201,7 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
"prefill CUDA graphs",
|
||||
cfg.cuda_graph_config.prefill.backend != Backend.DISABLED,
|
||||
),
|
||||
("DP attention", cfg.enable_dp_attention),
|
||||
("DP attention", attn_dp_enabled_of(cfg)),
|
||||
("context parallelism", cfg.attn_cp_size > 1),
|
||||
("external cache linker", cfg.enable_unified_cache_external_linker),
|
||||
("unified memory", cfg.enable_unified_memory),
|
||||
@@ -246,8 +254,8 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
if (
|
||||
read_ragged_verify_mode() is not RaggedVerifyMode.STATIC
|
||||
or cfg.disaggregation_transfer_backend != "mooncake"
|
||||
or cfg.dp_size != 1
|
||||
or cfg.enable_dp_attention
|
||||
or num_dp_ranks_of(cfg) != 1
|
||||
or attn_dp_enabled_of(cfg)
|
||||
or cfg.attn_cp_size != 1
|
||||
or cfg.dcp_size != 1
|
||||
):
|
||||
@@ -285,7 +293,7 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
cfg.cuda_graph_config.prefill.backend != Backend.DISABLED,
|
||||
),
|
||||
# input_ids_global is a DP-wide gather, so the tail slice cannot apply.
|
||||
("DP attention", cfg.enable_dp_attention),
|
||||
("DP attention", attn_dp_enabled_of(cfg)),
|
||||
)
|
||||
for feature, enabled in incompatible:
|
||||
if enabled:
|
||||
|
||||
@@ -40,6 +40,7 @@ def handle_expert_pack(server_args: Any) -> None:
|
||||
parallelism = (
|
||||
("tensor", "--tp-size", cfg.tp_size),
|
||||
("data", "--dp-size", cfg.dp_size),
|
||||
("attention data", "--attn-dp-size", cfg.attn_dp_size),
|
||||
("expert", "--ep-size", cfg.ep_size),
|
||||
)
|
||||
for label, option, size in parallelism:
|
||||
|
||||
@@ -79,6 +79,7 @@ POSITIONAL_FIELD_ORDER = (
|
||||
"pp_max_micro_batch_size",
|
||||
"pp_async_batch_depth",
|
||||
"dp_size",
|
||||
"attn_dp_size",
|
||||
"load_balance_method",
|
||||
"attn_cp_size",
|
||||
"moe_dp_size",
|
||||
|
||||
@@ -73,8 +73,19 @@ class Parallel(msgspec.Struct):
|
||||
dp_size: A[
|
||||
int,
|
||||
Arg(
|
||||
help="The data parallelism size.",
|
||||
help="The number of data-parallel replicas of the model.",
|
||||
aliases=["--data-parallel-size"],
|
||||
resolvable=True,
|
||||
),
|
||||
] = 1
|
||||
attn_dp_size: A[
|
||||
int,
|
||||
Arg(
|
||||
help="The attention data parallelism size: the number of "
|
||||
"data-parallel attention groups inside the TP group, one scheduler "
|
||||
"each, while the FFN stays tensor parallel.",
|
||||
aliases=["--attention-data-parallel-size"],
|
||||
resolvable=True,
|
||||
),
|
||||
] = 1
|
||||
load_balance_method: A[
|
||||
@@ -159,14 +170,9 @@ class Parallel(msgspec.Struct):
|
||||
bool,
|
||||
"Enable attention tensor-parallel weight slicing during decode under context parallel (cp_size>1). Slices the replicated attention linears to the local CP partition, eliminating redundant decode GEMMs.",
|
||||
] = False
|
||||
# DP attention
|
||||
enable_dp_attention: A[
|
||||
bool,
|
||||
Arg(
|
||||
help="Enabling data parallelism for attention and tensor parallelism for FFN. The dp size should be equal to the tp size. Currently DeepSeek-V2 and Qwen 2/3 MoE models are supported.",
|
||||
resolvable=True,
|
||||
),
|
||||
] = False
|
||||
# Deprecated spelling of `attn_dp_size`: `--dp-size N --enable-dp-attention`
|
||||
# resolves to `attn_dp_size = N`, `dp_size = 1`.
|
||||
enable_dp_attention: A[bool, Arg(no_cli=True, resolvable=True)] = False
|
||||
enable_dp_attention_local_control_broadcast: A[
|
||||
bool,
|
||||
"With DP-attention, send control messages to every DP group leader and broadcast within attn_tp_group instead of the full tp_group. Eliminates a costly all-ranks gloo sync on every scheduler iteration.",
|
||||
@@ -184,7 +190,7 @@ class Parallel(msgspec.Struct):
|
||||
help="Use all-to-all instead of TP all-gather followed by DP scatter "
|
||||
"for the TP-sharded LM head under DP attention. By default this is "
|
||||
"enabled only on decode-only PD nodes with pure DP attention "
|
||||
"(tp_size == dp_size > 1 and attn_cp_size == 1), and disabled on "
|
||||
"(tp_size == attn_dp_size > 1 and attn_cp_size == 1), and disabled on "
|
||||
"prefill-only and colocated nodes. Pass "
|
||||
"--no-enable-tp-lm-head-all-to-all to opt out. The path is "
|
||||
"incompatible with --enable-dp-lm-head; batches without an equal "
|
||||
@@ -282,10 +288,18 @@ class Parallel(msgspec.Struct):
|
||||
doc="Attention tensor-parallel width: `tp_size` divided by the "
|
||||
"attention-DP and attention-CP dimensions.",
|
||||
)
|
||||
attn_dp_size = Derived(
|
||||
fn="sglang.srt.runtime_context.attn_dp_size_of",
|
||||
doc="Attention data-parallel width: `dp_size` when DP attention is "
|
||||
"on, otherwise one.",
|
||||
attn_dp_enabled = Derived(
|
||||
fn="sglang.srt.runtime_context.attn_dp_enabled_of",
|
||||
doc="Whether attention runs data parallel: `attn_dp_size` is wider "
|
||||
"than one, or this process is an elastic EP scale joiner, which joins "
|
||||
"an attention-DP deployment with a group of its own that may be one "
|
||||
"rank wide.",
|
||||
)
|
||||
num_dp_ranks = Derived(
|
||||
fn="sglang.srt.runtime_context.num_dp_ranks_of",
|
||||
doc="How many data-parallel ranks the deployment serves with, one "
|
||||
"scheduler each: `dp_size * attn_dp_size`. Elastic EP scale-up widens "
|
||||
"it as ranks join.",
|
||||
)
|
||||
attn_dcp_size = Derived(
|
||||
fn="sglang.srt.runtime_context.attn_dcp_size_of",
|
||||
|
||||
@@ -18,7 +18,7 @@ from sglang.srt.arg_groups.overrides import (
|
||||
from sglang.srt.configs.hybrid_arch import mambaish_config
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend
|
||||
from sglang.srt.runtime_context import get_platform
|
||||
from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -450,7 +450,7 @@ def handle_cache_compatibility(server_args: Any) -> None:
|
||||
raise NotImplementedError(
|
||||
"LMCacheUnifiedRadixCache does not yet support speculative decoding"
|
||||
)
|
||||
if cfg.enable_dp_attention:
|
||||
if attn_dp_enabled_of(cfg):
|
||||
raise NotImplementedError(
|
||||
"LMCacheUnifiedRadixCache does not yet support DP attention"
|
||||
)
|
||||
|
||||
@@ -7,6 +7,7 @@ if TYPE_CHECKING:
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
|
||||
from sglang.srt.arg_groups.overrides import model_config_of, resolving_view
|
||||
from sglang.srt.runtime_context import attn_dp_enabled_of
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -24,7 +25,7 @@ def handle_layernorm_sp(server_args: ServerArgs) -> None:
|
||||
validate_layernorm_sp(
|
||||
architecture=architectures[0] if architectures else None,
|
||||
tp_size=cfg.tp_size,
|
||||
enable_dp_attention=cfg.enable_dp_attention,
|
||||
attn_dp_enabled=attn_dp_enabled_of(cfg),
|
||||
speculative_algorithm=cfg.speculative_algorithm,
|
||||
)
|
||||
|
||||
@@ -33,7 +34,7 @@ def validate_layernorm_sp(
|
||||
*,
|
||||
architecture: Optional[str],
|
||||
tp_size: int,
|
||||
enable_dp_attention: bool,
|
||||
attn_dp_enabled: bool,
|
||||
speculative_algorithm: Optional[str],
|
||||
) -> None:
|
||||
"""Fail loud for unsupported / incompatible configs. Callers gate on the flag."""
|
||||
@@ -49,9 +50,9 @@ def validate_layernorm_sp(
|
||||
"--enable-layernorm-sp requires tp_size > 1: there is no sequence to "
|
||||
"shard across a single TP rank."
|
||||
)
|
||||
if enable_dp_attention:
|
||||
if attn_dp_enabled:
|
||||
raise ValueError(
|
||||
"--enable-layernorm-sp is not compatible with --enable-dp-attention: "
|
||||
"--enable-layernorm-sp is not compatible with attention DP: "
|
||||
"SP shards the sequence across the full TP group, which under DP "
|
||||
"attention spans data-parallel groups holding different sequences."
|
||||
)
|
||||
|
||||
@@ -13,7 +13,7 @@ from sglang.srt.arg_groups.overrides import (
|
||||
)
|
||||
from sglang.srt.connector import ConnectorType
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.runtime_context import get_platform
|
||||
from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform
|
||||
from sglang.srt.utils.common import parse_connector_type
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -143,11 +143,11 @@ def validate_mega_moe_token_budget(server_args: ServerArgs, model_label: str) ->
|
||||
local_chunked_prefill_size = (
|
||||
cfg.chunked_prefill_size + token_partition_size - 1
|
||||
) // token_partition_size
|
||||
elif cfg.enable_dp_attention:
|
||||
token_partition_size = cfg.dp_size
|
||||
token_partition_name = "dp_size"
|
||||
elif attn_dp_enabled_of(cfg):
|
||||
token_partition_size = cfg.attn_dp_size
|
||||
token_partition_name = "attn_dp_size"
|
||||
token_alignment = max(
|
||||
cfg.tp_size // cfg.dp_size // cfg.attn_cp_size,
|
||||
cfg.tp_size // cfg.attn_dp_size // cfg.attn_cp_size,
|
||||
1,
|
||||
)
|
||||
local_chunked_prefill_size = cfg.chunked_prefill_size // token_partition_size
|
||||
|
||||
@@ -17,7 +17,7 @@ from sglang.srt.arg_groups.overrides import (
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase
|
||||
from sglang.srt.runtime_context import get_platform
|
||||
from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform
|
||||
from sglang.srt.utils.common import get_device_memory_capacity
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -361,14 +361,14 @@ def reserve_for_graph_mb(server_args: Any) -> float:
|
||||
reserved_mem += decode_cuda_graph_config.max_bs * 2
|
||||
|
||||
if (
|
||||
resolved_view(server_args).enable_dp_attention
|
||||
attn_dp_enabled_of(resolved_view(server_args))
|
||||
and cfg.disaggregation_mode != "prefill"
|
||||
):
|
||||
# DP attention needs more padding for some operations, and much more for large
|
||||
# cuda graph max bs (torch allocator / implementation inefficiencies).
|
||||
reserved_mem += decode_cuda_graph_config.max_bs * cfg.dp_size * 3
|
||||
reserved_mem += decode_cuda_graph_config.max_bs * cfg.attn_dp_size * 3
|
||||
if decode_cuda_graph_config.max_bs > 300:
|
||||
reserved_mem += decode_cuda_graph_config.max_bs * cfg.dp_size * 1.5
|
||||
reserved_mem += decode_cuda_graph_config.max_bs * cfg.attn_dp_size * 1.5
|
||||
|
||||
if (
|
||||
cfg.disaggregation_mode != "decode"
|
||||
|
||||
@@ -37,7 +37,11 @@ from sglang.srt.connector import ConnectorType
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.hardware_backend.mlx.runtime import use_mlx
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
|
||||
from sglang.srt.runtime_context import derive_attention_widths, get_platform
|
||||
from sglang.srt.runtime_context import (
|
||||
attn_dp_enabled_of,
|
||||
derive_attn_tp_size,
|
||||
get_platform,
|
||||
)
|
||||
from sglang.srt.utils.common import (
|
||||
get_quantization_config,
|
||||
is_mps,
|
||||
@@ -290,15 +294,14 @@ def handle_model_specific_adjustments(server_args: Any):
|
||||
)
|
||||
else:
|
||||
# Pure TP and partial DP Attention mode is active for DSA, logging a warning
|
||||
if cfg.dp_size < cfg.tp_size:
|
||||
_, attn_tp_size = derive_attention_widths(
|
||||
tp_size=cfg.tp_size,
|
||||
attn_cp_size=cfg.attn_cp_size,
|
||||
dp_size=cfg.dp_size,
|
||||
enable_dp_attention=cfg.enable_dp_attention,
|
||||
)
|
||||
attn_tp_size = derive_attn_tp_size(
|
||||
tp_size=cfg.tp_size,
|
||||
attn_cp_size=cfg.attn_cp_size,
|
||||
attn_dp_size=cfg.attn_dp_size,
|
||||
)
|
||||
if attn_tp_size > 1:
|
||||
logger.warning(
|
||||
f"DSA with TP mode is active, dp_size={cfg.dp_size}, tp_size={cfg.tp_size}, "
|
||||
f"DSA with TP mode is active, attn_dp_size={cfg.attn_dp_size}, tp_size={cfg.tp_size}, "
|
||||
f"attn_tp_size={attn_tp_size}, attention weights will be sharded across {attn_tp_size} ranks."
|
||||
)
|
||||
|
||||
@@ -410,7 +413,7 @@ def handle_model_specific_adjustments(server_args: Any):
|
||||
if is_deepseek_dsa(hf_config) and not envs.SGLANG_OPT_USE_TOPK_V2.is_set():
|
||||
# Prefer HIP top-k by default while honoring an explicit selection.
|
||||
envs.SGLANG_OPT_USE_TOPK_V2.set(False)
|
||||
if not resolved_view(server_args).enable_dp_attention and cfg.nnodes == 1:
|
||||
if not attn_dp_enabled_of(resolved_view(server_args)) and cfg.nnodes == 1:
|
||||
# TODO (Hubert): Put this back later
|
||||
# server_args.enable_aiter_allreduce_fusion = True
|
||||
|
||||
@@ -514,7 +517,7 @@ def handle_model_specific_adjustments(server_args: Any):
|
||||
quant_method = get_quantization_config(hf_config)
|
||||
is_mxfp4_quant_format = quant_method == "mxfp4"
|
||||
if (
|
||||
not resolved_view(server_args).enable_dp_attention
|
||||
not attn_dp_enabled_of(resolved_view(server_args))
|
||||
and cfg.nnodes == 1
|
||||
and get_platform().is_hip
|
||||
):
|
||||
@@ -543,8 +546,11 @@ def handle_model_specific_adjustments(server_args: Any):
|
||||
if model_arch == "MiMoV2ForCausalLM" and not cfg.encoder_only:
|
||||
expected_attn_tp_size = get_mimo_v2_fused_qkv_expected_tp_size(hf_config)
|
||||
view = resolved_view(server_args)
|
||||
attn_dp_size = cfg.dp_size if view.enable_dp_attention else 1
|
||||
effective_attn_tp_size = cfg.tp_size // attn_dp_size // view.attn_cp_size
|
||||
effective_attn_tp_size = derive_attn_tp_size(
|
||||
tp_size=cfg.tp_size,
|
||||
attn_cp_size=view.attn_cp_size,
|
||||
attn_dp_size=cfg.attn_dp_size,
|
||||
)
|
||||
if (
|
||||
expected_attn_tp_size is not None
|
||||
and expected_attn_tp_size % effective_attn_tp_size != 0
|
||||
@@ -555,10 +561,9 @@ def handle_model_specific_adjustments(server_args: Any):
|
||||
"qkv_proj weights are "
|
||||
f"TP={expected_attn_tp_size}-interleaved; got "
|
||||
f"{effective_attn_tp_size} "
|
||||
f"(tp_size={cfg.tp_size}, dp_size={cfg.dp_size}, "
|
||||
f"enable_dp_attention={view.enable_dp_attention}, "
|
||||
f"(tp_size={cfg.tp_size}, attn_dp_size={cfg.attn_dp_size}, "
|
||||
f"attn_cp_size={view.attn_cp_size}). "
|
||||
"Set --tp, --dp, --enable-dp-attention, and "
|
||||
"Set --tp, --attn-dp-size, and "
|
||||
"--attention-context-parallel-size so the effective "
|
||||
f"attention TP size is {expected_attn_tp_size}."
|
||||
)
|
||||
|
||||
@@ -85,7 +85,15 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
logger.warning(
|
||||
"Context parallel feature is still under experiment. It has only been verified on Hopper platform."
|
||||
)
|
||||
overrides["enable_dp_attention"] = True
|
||||
# DSA CP runs data-parallel groups as attention DP.
|
||||
assert not (cfg.attn_dp_size > 1 and cfg.dp_size > 1), (
|
||||
f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: "
|
||||
"data-parallel replicas combined with attention data parallelism "
|
||||
"are not supported."
|
||||
)
|
||||
attn_dp_size = cfg.attn_dp_size * cfg.dp_size
|
||||
overrides["attn_dp_size"] = attn_dp_size
|
||||
overrides["dp_size"] = 1
|
||||
overrides["moe_dense_tp_size"] = 1
|
||||
if cfg.cp_strategy == "zigzag":
|
||||
overrides["moe_a2a_backend"] = "deepep"
|
||||
@@ -95,7 +103,7 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
"moe_a2a_backend=deepep, ep_size=tp_size, batch_size=1."
|
||||
)
|
||||
else:
|
||||
assert cfg.dp_size == 1, (
|
||||
assert attn_dp_size == 1, (
|
||||
"interleave DSA CP does not support DP attention."
|
||||
)
|
||||
assert cfg.tp_size <= 8, (
|
||||
@@ -104,11 +112,11 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
# Note(kpham-sgl): Keep attn_tp_size == 1 under DSA CP.
|
||||
# The DSA / MLA CP gather and reduce-scatter
|
||||
# (the dsa_cp_* helpers in adapters/context_parallel.py) assume it.
|
||||
attn_cp_size = cfg.tp_size // cfg.dp_size
|
||||
attn_cp_size = cfg.tp_size // attn_dp_size
|
||||
overrides["attn_cp_size"] = attn_cp_size
|
||||
logger.warning(
|
||||
"Enabled DSA context parallel: "
|
||||
f"strategy={cfg.cp_strategy}, dp_size={cfg.dp_size}, "
|
||||
f"strategy={cfg.cp_strategy}, attn_dp_size={attn_dp_size}, "
|
||||
f"moe_dense_tp_size={overrides['moe_dense_tp_size']}, "
|
||||
f"ep_size={overrides.get('ep_size', cfg.ep_size)}, tp_size={cfg.tp_size}, "
|
||||
f"attn_cp_size={attn_cp_size}, "
|
||||
@@ -158,7 +166,15 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
"MLA prefill context parallel is still experimental. "
|
||||
"Verified on Hopper with the fa3 backend."
|
||||
)
|
||||
overrides["enable_dp_attention"] = True
|
||||
# MLA CP runs data-parallel groups as attention DP.
|
||||
assert not (cfg.attn_dp_size > 1 and cfg.dp_size > 1), (
|
||||
f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: "
|
||||
"data-parallel replicas combined with attention data parallelism "
|
||||
"are not supported."
|
||||
)
|
||||
attn_dp_size = cfg.attn_dp_size * cfg.dp_size
|
||||
overrides["attn_dp_size"] = attn_dp_size
|
||||
overrides["dp_size"] = 1
|
||||
# TODO(kpham-sgl) Supports moe_dense_tp_size != 1.
|
||||
overrides["moe_dense_tp_size"] = 1
|
||||
overrides["moe_a2a_backend"] = "deepep"
|
||||
@@ -169,11 +185,11 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
# FIXME(kpham-sgl): Keep attn_tp_size == 1 under MLA CP.
|
||||
# The DSA / MLA CP gather and reduce-scatter
|
||||
# (the dsa_cp_* helpers in adapters/context_parallel.py) assume it.
|
||||
attn_cp_size = cfg.tp_size // cfg.dp_size
|
||||
attn_cp_size = cfg.tp_size // attn_dp_size
|
||||
overrides["attn_cp_size"] = attn_cp_size
|
||||
logger.warning(
|
||||
f"Enable Context Parallel opt for MLA, "
|
||||
f"Setting dp_size == {cfg.dp_size} and "
|
||||
f"Setting attn_dp_size == {attn_dp_size} and "
|
||||
f"attn_cp_size == {attn_cp_size}, "
|
||||
f"moe_dense_tp_size == {overrides['moe_dense_tp_size']}, "
|
||||
f"ep_size == {overrides['ep_size']}, "
|
||||
|
||||
@@ -5,6 +5,7 @@ from sglang.srt.arg_groups.model_override_base import (
|
||||
is_attention_backend_not_set,
|
||||
resolving_view,
|
||||
)
|
||||
from sglang.srt.runtime_context import attn_dp_enabled_of
|
||||
|
||||
|
||||
@_register_for("IQuestQ1ForCausalLM", "IQuestQ1MTP")
|
||||
@@ -33,7 +34,7 @@ def _iquest_q1_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
)
|
||||
if cfg.enable_multi_layer_eagle:
|
||||
raise ValueError("IQuest Q1 MTP drafting does not support multi-layer EAGLE.")
|
||||
if cfg.enable_dp_attention:
|
||||
if attn_dp_enabled_of(cfg):
|
||||
raise ValueError("IQuest Q1 does not support DP attention.")
|
||||
if cfg.pp_size > 1:
|
||||
raise ValueError("IQuest Q1 does not support pipeline parallelism.")
|
||||
|
||||
@@ -8,13 +8,13 @@ from sglang.srt.arg_groups.model_override_base import (
|
||||
resolving_view,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.runtime_context import get_platform
|
||||
from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform
|
||||
|
||||
|
||||
@_register_for("MiniCPMForCausalLM", "MiniCPMSALAForCausalLM")
|
||||
def _minicpm_sala_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
cfg = resolving_view(server_args)
|
||||
if cfg.enable_dp_attention:
|
||||
if attn_dp_enabled_of(cfg):
|
||||
raise ValueError("MiniCPM does not support DP attention")
|
||||
has_sparse_attention = getattr(hf_config, "has_minicpm_sparse_attention", False)
|
||||
has_hybrid_attention = has_sparse_attention or getattr(
|
||||
|
||||
@@ -29,7 +29,7 @@ from sglang.srt.configs.moe_model_registry import (
|
||||
from sglang.srt.connector import ConnectorType
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
|
||||
from sglang.srt.runtime_context import get_platform
|
||||
from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform
|
||||
from sglang.srt.utils.common import is_sm100_supported, parse_connector_type
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -269,8 +269,8 @@ def handle_a2a_moe(server_args: Any):
|
||||
if a2a_backend == "flashinfer_megamoe":
|
||||
validate_flashinfer_megamoe_model(server_args)
|
||||
validate_flashinfer_megamoe_envs()
|
||||
assert cfg.enable_dp_attention and cfg.dp_size == cfg.tp_size, (
|
||||
"FlashInfer MegaMOE is only supported with dp_size == tp_size and --enable-dp-attention"
|
||||
assert attn_dp_enabled_of(cfg) and cfg.attn_dp_size == cfg.tp_size, (
|
||||
"FlashInfer MegaMOE is only supported with --attn-dp-size equal to --tp-size"
|
||||
)
|
||||
if resolved_view(server_args).moe_runner_backend == "auto":
|
||||
declare_resolution(
|
||||
@@ -395,11 +395,8 @@ def handle_a2a_moe(server_args: Any):
|
||||
)
|
||||
|
||||
if a2a_now == "flashinfer":
|
||||
assert (
|
||||
resolved_view(server_args).enable_dp_attention
|
||||
and cfg.dp_size == cfg.tp_size
|
||||
), (
|
||||
"Flashinfer MoE A2A is only supported with dp_size == tp_size and --enable-dp-attention"
|
||||
assert attn_dp_enabled_of(cfg) and cfg.attn_dp_size == cfg.tp_size, (
|
||||
"Flashinfer MoE A2A is only supported with --attn-dp-size equal to --tp-size"
|
||||
)
|
||||
if cfg.deepep_mode != "auto":
|
||||
logger.warning("--deepep-mode is ignored for Flashinfer MoE A2A")
|
||||
@@ -491,12 +488,12 @@ def handle_a2a_moe(server_args: Any):
|
||||
deepep_mode="low_latency",
|
||||
)
|
||||
logger.warning("auto set deepep_mode=`low_latency` for PPLX EP")
|
||||
# pplx-kernels' AllToAll needs numDPGroups (== attention dp_size) > 1;
|
||||
# pplx-kernels' AllToAll needs numDPGroups (== attn_dp_size) > 1;
|
||||
# without DP attention numDPGroups == 1 and construction fails deep in
|
||||
# the kernel. This also implies ep_size >= 2.
|
||||
assert resolved_view(server_args).enable_dp_attention and cfg.dp_size >= 2, (
|
||||
"moe_a2a_backend='pplx' requires --enable-dp-attention with at "
|
||||
"least 2 DP groups (--dp-size >= 2)."
|
||||
assert cfg.attn_dp_size >= 2, (
|
||||
"moe_a2a_backend='pplx' requires attention DP with at least 2 "
|
||||
"groups (--attn-dp-size >= 2)."
|
||||
)
|
||||
# pplx runs the masked DeepGEMM expert path (sm_90a): reject other
|
||||
# runners and resolve auto -> deep_gemm. Unquantized bf16 pplx needs
|
||||
@@ -581,8 +578,7 @@ def validate_deepep_v2_dispatch_token_budget(server_args: Any) -> None:
|
||||
|
||||
graph_bs = decode_config.max_bs or 0
|
||||
if view.max_running_requests is not None:
|
||||
attn_dp_size = view.dp_size if view.enable_dp_attention else 1
|
||||
per_rank_pool_bs = max(1, view.max_running_requests // attn_dp_size)
|
||||
per_rank_pool_bs = max(1, view.max_running_requests // view.attn_dp_size)
|
||||
graph_bs = min(graph_bs, per_rank_pool_bs)
|
||||
tokens_per_req = (
|
||||
max_speculative_num_draft_tokens(server_args) or 1
|
||||
|
||||
@@ -62,8 +62,10 @@ logger = logging.getLogger(__name__)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend
|
||||
from sglang.srt.runtime_context import (
|
||||
attn_dp_enabled_of,
|
||||
get_context,
|
||||
get_platform,
|
||||
num_dp_ranks_of,
|
||||
)
|
||||
from sglang.srt.utils.common import (
|
||||
get_quantization_config,
|
||||
@@ -931,7 +933,7 @@ def _flashinfer_allreduce_fusion_auto_enable(view: Any) -> dict:
|
||||
and not prefer_custom_dsv41
|
||||
and (get_platform().is_sm90 or get_platform().is_sm100)
|
||||
and view.tp_size > 1
|
||||
and not view.enable_dp_attention
|
||||
and not attn_dp_enabled_of(view)
|
||||
and (view.nnodes == 1 or get_platform().is_sm100)
|
||||
and view.moe_a2a_backend == "none"
|
||||
):
|
||||
@@ -1304,8 +1306,8 @@ def _page_size_default(view: Any) -> dict:
|
||||
|
||||
@register_post_process
|
||||
def _data_parallelism_defaults(view: Any) -> dict:
|
||||
if view.dp_size == 1 and view.ep_join_mode != "scale":
|
||||
return {"enable_dp_attention": False, "enable_dp_lm_head": False}
|
||||
if num_dp_ranks_of(view) == 1 and view.ep_join_mode != "scale":
|
||||
return {"enable_dp_lm_head": False}
|
||||
return {}
|
||||
|
||||
|
||||
@@ -1349,9 +1351,8 @@ def _tp_lm_head_all_to_all_default(view: Any) -> dict:
|
||||
|
||||
enable = (
|
||||
view.disaggregation_mode == "decode"
|
||||
and view.enable_dp_attention
|
||||
and view.dp_size > 1
|
||||
and view.tp_size == view.dp_size
|
||||
and view.attn_dp_size > 1
|
||||
and view.tp_size == view.attn_dp_size
|
||||
and view.attn_cp_size == 1
|
||||
and not view.enable_dp_lm_head
|
||||
)
|
||||
@@ -1362,20 +1363,21 @@ def _tp_lm_head_all_to_all_default(view: Any) -> dict:
|
||||
def _dp_lm_head_validation(view: Any) -> dict:
|
||||
"""Require DP attention for DP LM head and TP LM-head all-to-all."""
|
||||
if view.enable_dp_lm_head:
|
||||
assert view.enable_dp_attention, (
|
||||
"Please enable dp attention when setting enable_dp_lm_head. "
|
||||
assert attn_dp_enabled_of(view), (
|
||||
"--enable-dp-lm-head requires attention DP (--attn-dp-size)."
|
||||
)
|
||||
if view.enable_tp_lm_head_all_to_all:
|
||||
assert view.enable_dp_attention, (
|
||||
"Please enable dp attention when setting enable_tp_lm_head_all_to_all."
|
||||
assert attn_dp_enabled_of(view), (
|
||||
"--enable-tp-lm-head-all-to-all requires attention DP (--attn-dp-size)."
|
||||
)
|
||||
assert not view.enable_dp_lm_head, (
|
||||
"--enable-tp-lm-head-all-to-all uses a TP-sharded LM head and is "
|
||||
"incompatible with --enable-dp-lm-head."
|
||||
)
|
||||
assert view.tp_size == view.dp_size, (
|
||||
assert view.tp_size == view.attn_dp_size, (
|
||||
"--enable-tp-lm-head-all-to-all currently requires tp_size == "
|
||||
f"dp_size, got tp_size={view.tp_size}, dp_size={view.dp_size}."
|
||||
f"attn_dp_size, got tp_size={view.tp_size}, "
|
||||
f"attn_dp_size={view.attn_dp_size}."
|
||||
)
|
||||
assert view.attn_cp_size == 1, (
|
||||
"--enable-tp-lm-head-all-to-all currently requires "
|
||||
|
||||
@@ -23,12 +23,40 @@ from sglang.srt.arg_groups.resolution_hooks import run_hook
|
||||
from sglang.srt.connector import ConnectorType
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
|
||||
from sglang.srt.runtime_context import get_platform
|
||||
from sglang.srt.runtime_context import (
|
||||
attn_dp_enabled_of,
|
||||
derive_attn_tp_size,
|
||||
get_platform,
|
||||
)
|
||||
from sglang.srt.utils.common import parse_connector_type
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def handle_deprecated_dp_attention(server_args: Any):
|
||||
"""Turn the deprecated `--dp-size N --enable-dp-attention` into
|
||||
`--attn-dp-size N`.
|
||||
|
||||
Runs before any handler reads the DP layout. The old spelling counted
|
||||
attention-DP groups with `dp_size`; that width moves to `attn_dp_size`, and
|
||||
`dp_size` counts replicas again.
|
||||
"""
|
||||
cfg = resolving_view(server_args)
|
||||
if cfg.enable_dp_attention:
|
||||
if cfg.attn_dp_size != 1:
|
||||
raise ValueError(
|
||||
"--enable-dp-attention is the deprecated spelling of "
|
||||
"--attn-dp-size; pass --attn-dp-size alone."
|
||||
)
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"_handle_deprecated_dp_attention",
|
||||
attn_dp_size=cfg.dp_size,
|
||||
dp_size=1,
|
||||
enable_dp_attention=False,
|
||||
)
|
||||
|
||||
|
||||
def _boundary_parallelism_overrides(cfg, model_type: str) -> dict:
|
||||
"""Resolve the token-row modes the model's forward actually enters."""
|
||||
nemotron = model_type in ("nemotron_h", "nemotron_h_puzzle")
|
||||
@@ -108,8 +136,8 @@ def handle_context_parallelism(server_args: Any):
|
||||
assert cfg.tp_size % view.attn_cp_size == 0, (
|
||||
"tp_size must be divisible by attn_cp_size"
|
||||
)
|
||||
assert cfg.tp_size % (cfg.dp_size * view.attn_cp_size) == 0, (
|
||||
"tp_size must be divisible by dp_size * attn_cp_size"
|
||||
assert cfg.tp_size % (cfg.attn_dp_size * view.attn_cp_size) == 0, (
|
||||
"tp_size must be divisible by attn_dp_size * attn_cp_size"
|
||||
)
|
||||
|
||||
assert not cfg.enable_aiter_allreduce_fusion, (
|
||||
@@ -155,14 +183,11 @@ def handle_shared_experts_tp(server_args: Any):
|
||||
if size is None:
|
||||
return
|
||||
|
||||
from sglang.srt.runtime_context import derive_attention_widths
|
||||
|
||||
view = resolved_view(server_args)
|
||||
_, attn_tp_size = derive_attention_widths(
|
||||
attn_tp_size = derive_attn_tp_size(
|
||||
tp_size=cfg.tp_size,
|
||||
attn_cp_size=view.attn_cp_size,
|
||||
dp_size=cfg.dp_size,
|
||||
enable_dp_attention=view.enable_dp_attention,
|
||||
attn_dp_size=cfg.attn_dp_size,
|
||||
)
|
||||
if size < 1 or attn_tp_size % size != 0:
|
||||
raise ValueError(
|
||||
@@ -224,7 +249,7 @@ def handle_decode_context_parallelism(server_args: Any):
|
||||
|
||||
|
||||
def handle_data_parallelism(server_args: Any):
|
||||
# The dp_size==1 resets moved to the resolution pipeline
|
||||
# The resets without attention DP live in the resolution pipeline
|
||||
# (arg_groups/overrides.py: _data_parallelism_defaults).
|
||||
from sglang.srt.arg_groups.cuda_graph_hook import (
|
||||
generate_prefill_cuda_graph_batch_sizes,
|
||||
@@ -232,6 +257,15 @@ def handle_data_parallelism(server_args: Any):
|
||||
|
||||
cfg = resolving_view(server_args)
|
||||
|
||||
if cfg.attn_dp_size < 1:
|
||||
raise ValueError(f"--attn-dp-size must be positive (got {cfg.attn_dp_size}).")
|
||||
if cfg.attn_dp_size > 1 and cfg.dp_size > 1:
|
||||
raise ValueError(
|
||||
f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: "
|
||||
"data-parallel replicas combined with attention data parallelism "
|
||||
"are not supported."
|
||||
)
|
||||
|
||||
run_post_process_pass(server_args, _data_parallelism_defaults)
|
||||
|
||||
if cfg.mm_enable_dp_encoder:
|
||||
@@ -252,18 +286,18 @@ def handle_data_parallelism(server_args: Any):
|
||||
cfg.tp_size,
|
||||
)
|
||||
|
||||
if resolved_view(server_args).enable_dp_attention:
|
||||
if attn_dp_enabled_of(resolved_view(server_args)):
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"_handle_data_parallelism",
|
||||
schedule_conservativeness=cfg.schedule_conservativeness * 0.3,
|
||||
)
|
||||
assert cfg.tp_size % cfg.dp_size == 0
|
||||
assert cfg.tp_size % cfg.attn_dp_size == 0
|
||||
original_chunked_prefill_size = cfg.chunked_prefill_size
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"_handle_data_parallelism",
|
||||
chunked_prefill_size=cfg.chunked_prefill_size // cfg.dp_size,
|
||||
chunked_prefill_size=cfg.chunked_prefill_size // cfg.attn_dp_size,
|
||||
)
|
||||
logger.warning(
|
||||
f"DP attention is enabled. chunked prefill size is adjusted "
|
||||
@@ -367,12 +401,8 @@ def handle_dwdp(server_args: Any):
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"_handle_dwdp",
|
||||
dp_size=cfg.dwdp_size,
|
||||
)
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"_handle_dwdp",
|
||||
enable_dp_attention=True,
|
||||
attn_dp_size=cfg.dwdp_size,
|
||||
dp_size=1,
|
||||
)
|
||||
declare_resolution(
|
||||
server_args, "_handle_dwdp", enable_dp_attention_local_control_broadcast=True
|
||||
@@ -413,7 +443,7 @@ def handle_dwdp(server_args: Any):
|
||||
|
||||
logger.info(
|
||||
f"DWDP enabled: dwdp_size={cfg.dwdp_size}, "
|
||||
f"auto-forced dp_size={cfg.dp_size}, ep_size={cfg.dwdp_size}, "
|
||||
f"auto-forced attn_dp_size={cfg.attn_dp_size}, ep_size={cfg.dwdp_size}, "
|
||||
f"moe_dense_tp_size=1, moe_a2a_backend=none, "
|
||||
f"dp_attention_local_control_broadcast=True, "
|
||||
f"enable_dp_lm_head=True, SCHEDULER_SKIP_ALL_GATHER=True, "
|
||||
@@ -582,10 +612,10 @@ def handle_elastic_ep(server_args: Any):
|
||||
assert not cfg.enable_pdmux, (
|
||||
"Elastic EP CUDA graph recapture does not support PDMux."
|
||||
)
|
||||
assert resolved.enable_dp_attention, (
|
||||
"Elastic EP scale-up requires --enable-dp-attention; without it "
|
||||
"the TP group is not equivalent to WORLD and the post-scale "
|
||||
"collective path is invalid."
|
||||
assert attn_dp_enabled_of(resolved), (
|
||||
"Elastic EP scale-up requires attention DP (--attn-dp-size); "
|
||||
"without it the TP group is not equivalent to WORLD and the "
|
||||
"post-scale collective path is invalid."
|
||||
)
|
||||
assert resolved.enable_dp_lm_head, (
|
||||
"Elastic EP scale-up requires --enable-dp-lm-head so output "
|
||||
@@ -604,9 +634,9 @@ def handle_elastic_ep(server_args: Any):
|
||||
f"(got ep_size={resolved.ep_size}, tp_size={cfg.tp_size}); EP, TP "
|
||||
"and the attention DP group must all coincide with WORLD."
|
||||
)
|
||||
assert cfg.dp_size == cfg.tp_size, (
|
||||
"Elastic EP scale-up requires dp_size == tp_size "
|
||||
f"(got dp_size={cfg.dp_size}, tp_size={cfg.tp_size})."
|
||||
assert cfg.attn_dp_size == cfg.tp_size, (
|
||||
"Elastic EP scale-up requires attn_dp_size == tp_size "
|
||||
f"(got attn_dp_size={cfg.attn_dp_size}, tp_size={cfg.tp_size})."
|
||||
)
|
||||
assert resolved.moe_a2a_backend == "nixl", (
|
||||
"Elastic EP scale-up requires --moe-a2a-backend nixl "
|
||||
|
||||
@@ -12,6 +12,7 @@ from sglang.srt.arg_groups.overrides import (
|
||||
resolving_view,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.runtime_context import attn_dp_enabled_of, num_dp_ranks_of
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
@@ -102,7 +103,7 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None:
|
||||
f"(--speculative-algorithm {cfg.speculative_algorithm})"
|
||||
)
|
||||
|
||||
if resolved_view(server_args).enable_dp_attention:
|
||||
if attn_dp_enabled_of(resolved_view(server_args)):
|
||||
logger.warning(
|
||||
"EXPERIMENTAL: Decode radix cache with DP attention. "
|
||||
"Requires prefix-aware DP rank routing for optimal cache hits."
|
||||
@@ -128,7 +129,7 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None:
|
||||
if cfg.disaggregation_decode_extra_slots is None:
|
||||
extra_slots = 0
|
||||
if cfg.max_running_requests is not None:
|
||||
per_worker = cfg.max_running_requests // max(1, cfg.dp_size)
|
||||
per_worker = cfg.max_running_requests // num_dp_ranks_of(cfg)
|
||||
if per_worker <= 32:
|
||||
extra_slots = per_worker * 2
|
||||
declare_resolution(
|
||||
@@ -160,8 +161,8 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None:
|
||||
if cfg.enable_pd_role_switch:
|
||||
view = resolved_view(server_args)
|
||||
unsupported = []
|
||||
if view.enable_dp_attention:
|
||||
unsupported.append("DP attention (--enable-dp-attention)")
|
||||
if attn_dp_enabled_of(view):
|
||||
unsupported.append(f"attention DP (--attn-dp-size {view.attn_dp_size})")
|
||||
if view.ep_size > 1:
|
||||
unsupported.append(f"expert parallelism (--ep-size {view.ep_size})")
|
||||
if view.moe_a2a_backend != "none":
|
||||
|
||||
@@ -38,6 +38,11 @@ def run_resolution_pipeline(server_args: Any) -> None:
|
||||
|
||||
cfg = resolving_view(server_args)
|
||||
|
||||
from sglang.srt.arg_groups.parallel_hook import handle_deprecated_dp_attention
|
||||
|
||||
# Before any handler reads the DP layout.
|
||||
run_hook(handle_deprecated_dp_attention, server_args)
|
||||
|
||||
from sglang.srt.arg_groups.mega_moe_hook import handle_mega_moe
|
||||
|
||||
run_hook(handle_mega_moe, server_args)
|
||||
|
||||
@@ -13,6 +13,7 @@ from typing import Any, Callable, Dict, FrozenSet, List
|
||||
# to every invocation of that name, including repeated steps.
|
||||
_OVERRIDABLE_HOOKS: FrozenSet[str] = frozenset(
|
||||
{
|
||||
"handle_deprecated_dp_attention",
|
||||
"handle_mega_moe",
|
||||
"handle_return_hidden_states_mode",
|
||||
"handle_media_url_security",
|
||||
|
||||
@@ -17,7 +17,7 @@ from sglang.srt.arg_groups.overrides import (
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.platforms import current_platform
|
||||
from sglang.srt.runtime_context import get_platform
|
||||
from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
@@ -271,7 +271,7 @@ def _handle_dflash(server_args: ServerArgs) -> None:
|
||||
)
|
||||
|
||||
# DFLASH + dp attention is validated on NPU only.
|
||||
if cfg.enable_dp_attention and not cfg.device == "npu":
|
||||
if attn_dp_enabled_of(cfg) and not cfg.device == "npu":
|
||||
raise ValueError(
|
||||
"Currently DFLASH speculative decoding does not support dp "
|
||||
"attention on non-NPU devices."
|
||||
@@ -538,7 +538,7 @@ def _handle_uno(server_args: ServerArgs) -> None:
|
||||
|
||||
if (cfg.tp_size, cfg.pp_size) != (1, 1):
|
||||
raise ValueError("UNO requires TP=PP=1.")
|
||||
if cfg.enable_dp_attention or cfg.attn_cp_size != 1:
|
||||
if attn_dp_enabled_of(cfg) or cfg.attn_cp_size != 1:
|
||||
raise ValueError("UNO does not support DP attention or context parallelism.")
|
||||
if cfg.enable_lora or cfg.lora_paths:
|
||||
raise ValueError("UNO does not support public Multi-LoRA serving.")
|
||||
@@ -588,8 +588,7 @@ def _handle_dspark(server_args: ServerArgs) -> None:
|
||||
"DSpark speculative decoding only supports CUDA or NPU device."
|
||||
)
|
||||
|
||||
# dp_size==1 with dp_attention is a degenerate flag under DSV4 CP; skip DP-only checks.
|
||||
if cfg.enable_dp_attention and cfg.dp_size > 1:
|
||||
if cfg.attn_dp_size > 1:
|
||||
if not cfg.enable_dp_lm_head:
|
||||
raise ValueError("DSpark with dp attention requires --enable-dp-lm-head.")
|
||||
if not _is_npu and cfg.moe_a2a_backend not in ("none", "megamoe", "mori"):
|
||||
@@ -963,9 +962,8 @@ def _handle_iquest_q1_mtp_draft(server_args: ServerArgs) -> bool:
|
||||
def _handle_eagle_family(server_args: ServerArgs) -> None:
|
||||
cfg = resolving_view(server_args)
|
||||
|
||||
if (
|
||||
cfg.speculative_algorithm == "STANDALONE"
|
||||
and resolved_view(server_args).enable_dp_attention
|
||||
if cfg.speculative_algorithm == "STANDALONE" and attn_dp_enabled_of(
|
||||
resolved_view(server_args)
|
||||
):
|
||||
# TODO: support dp attention for standalone speculative decoding
|
||||
raise ValueError(
|
||||
@@ -1254,7 +1252,7 @@ def _handle_ngram(server_args: ServerArgs) -> None:
|
||||
"and produces incorrect results for paged attention backends. "
|
||||
"This combination is only supported for the 'flashinfer' backend."
|
||||
)
|
||||
if view.enable_dp_attention:
|
||||
if attn_dp_enabled_of(view):
|
||||
# TODO: support dp attention for ngram speculative decoding
|
||||
raise ValueError(
|
||||
"Currently ngram speculative decoding does not support dp attention."
|
||||
|
||||
@@ -18,7 +18,7 @@ from sglang.srt.distributed.device_communicators.mooncake_transfer_engine import
|
||||
parse_ib_device_config,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.runtime_context import get_platform
|
||||
from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform, num_dp_ranks_of
|
||||
from sglang.srt.utils.common import torch_release
|
||||
from sglang.srt.utils.runai_utils import is_runai_obj_uri
|
||||
|
||||
@@ -72,8 +72,9 @@ def check_pipeline_parallel_compat(cfg: Any) -> None:
|
||||
# Every stage rebuilds the same verify input from the relayed
|
||||
# per-request state, so all stages must see the same batch.
|
||||
# DP attention partitions it per DP rank.
|
||||
assert not cfg.enable_dp_attention, (
|
||||
"SGLANG_ENABLE_PP_SPEC is not compatible with --enable-dp-attention"
|
||||
assert not attn_dp_enabled_of(cfg), (
|
||||
"SGLANG_ENABLE_PP_SPEC is not compatible with attention DP "
|
||||
"(--attn-dp-size)"
|
||||
)
|
||||
else:
|
||||
assert cfg.disaggregation_mode == "prefill", (
|
||||
@@ -114,8 +115,9 @@ def check_server_args(server_args: Any):
|
||||
if cfg.pp_size > 1:
|
||||
check_pipeline_parallel_compat(cfg)
|
||||
|
||||
assert not (cfg.dp_size > 1 and cfg.nnodes != 1 and not cfg.enable_dp_attention), (
|
||||
"multi-node data parallel is not supported unless dp attention!"
|
||||
assert not (cfg.dp_size > 1 and cfg.nnodes != 1), (
|
||||
"multi-node data-parallel replicas are not supported; use attention DP "
|
||||
"(--attn-dp-size) across nodes"
|
||||
)
|
||||
|
||||
assert cfg.base_gpu_id >= 0, "base_gpu_id must be non-negative"
|
||||
@@ -375,7 +377,7 @@ def check_load_publish_args(server_args: Any):
|
||||
_, reason = resolve_load_pub_range(
|
||||
kv_endpoint=cfg.endpoint,
|
||||
replay_endpoint=cfg.replay_endpoint,
|
||||
dp_size=server_cfg.dp_size,
|
||||
dp_size=num_dp_ranks_of(server_cfg),
|
||||
load_publish_endpoint=mode,
|
||||
)
|
||||
if reason:
|
||||
@@ -528,10 +530,10 @@ def check_two_batch_overlap(server_args: Any):
|
||||
if (
|
||||
cfg.enable_two_batch_overlap
|
||||
and cfg.moe_a2a_backend == "none"
|
||||
and not cfg.enable_dp_attention
|
||||
and not attn_dp_enabled_of(cfg)
|
||||
):
|
||||
raise ValueError(
|
||||
"When enabling two batch overlap without an EP a2a backend "
|
||||
"(moe_a2a_backend='none'), --enable-dp-attention is required "
|
||||
"(moe_a2a_backend='none'), attention DP (--attn-dp-size) is required "
|
||||
"(DeepSeek-V4 non-EP DP TBO path)."
|
||||
)
|
||||
|
||||
@@ -1744,7 +1744,7 @@ class _SGLangPlugin(_FrameworkPlugin):
|
||||
|
||||
try:
|
||||
parallel = get_parallel()
|
||||
info["enable_dp_attention"] = self._dp_attn.is_dp_attention_enabled()
|
||||
info["attn_dp_enabled"] = self._dp_attn.is_dp_attention_enabled()
|
||||
info["attn_tp_rank"] = parallel.attn_tp_rank
|
||||
info["attn_tp_size"] = parallel.attn_tp_size
|
||||
info["attn_dp_rank"] = parallel.attn_dp_rank
|
||||
|
||||
@@ -200,9 +200,7 @@ class CommonKVManager(BaseKVManager):
|
||||
self.dcp_rank = parallel.attn_dcp_rank
|
||||
self.attn_dp_size = parallel.attn_dp_size
|
||||
self.attn_dp_rank = parallel.attn_dp_rank
|
||||
self.system_dp_size = (
|
||||
1 if get_parallel().enable_dp_attention else get_parallel().dp_size
|
||||
)
|
||||
self.system_dp_size = get_parallel().dp_size
|
||||
self.system_dp_rank = (
|
||||
self.kv_args.system_dp_rank if self.kv_args.system_dp_rank else 0
|
||||
)
|
||||
@@ -1545,11 +1543,12 @@ class CommonKVSender(BaseKVSender):
|
||||
return
|
||||
|
||||
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Bootstrapping)
|
||||
if get_parallel().dp_size > 1 and not req_has_disagg_prefill_dp_rank:
|
||||
if get_parallel().num_dp_ranks > 1 and not req_has_disagg_prefill_dp_rank:
|
||||
if get_parallel().load_balance_method != "follow_bootstrap_room":
|
||||
self._register_prefill_dp_rank()
|
||||
elif (
|
||||
self.kv_mgr.attn_dp_rank != self.bootstrap_room % get_parallel().dp_size
|
||||
self.kv_mgr.attn_dp_rank
|
||||
!= self.bootstrap_room % get_parallel().num_dp_ranks
|
||||
):
|
||||
# follow_bootstrap_room was overridden by external routed_dp_rank
|
||||
if envs.SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK.get():
|
||||
@@ -1560,7 +1559,7 @@ class CommonKVSender(BaseKVSender):
|
||||
f"follow_bootstrap_room conflict: dispatched to dp_rank "
|
||||
f"{self.kv_mgr.attn_dp_rank} but bootstrap_room "
|
||||
f"{self.bootstrap_room} implies dp_rank "
|
||||
f"{self.bootstrap_room % get_parallel().dp_size}. "
|
||||
f"{self.bootstrap_room % get_parallel().num_dp_ranks}. "
|
||||
f"Set SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK=1 "
|
||||
f"to allow mixed routing.",
|
||||
)
|
||||
|
||||
@@ -209,7 +209,7 @@ def launch_server(server_args: ServerArgs):
|
||||
|
||||
configure_logger(server_args, prefix=" encode_server")
|
||||
publish(server_args, role="encoder")
|
||||
if get_parallel().dp_size > 1:
|
||||
if get_parallel().num_dp_ranks > 1:
|
||||
dp_dispatcher = launch_dp_runtime(server_args)
|
||||
# runtime initializes multiprocess metrics before spawning;
|
||||
# HTTP only exposes their endpoint.
|
||||
|
||||
@@ -1744,10 +1744,10 @@ def launch_local_runtime(server_args: ServerArgs) -> EncoderRuntime:
|
||||
This function owns backend construction only. HTTP/gRPC middleware,
|
||||
service registration, and network serving remain Transport concerns.
|
||||
"""
|
||||
if get_parallel().dp_size > 1:
|
||||
if get_parallel().num_dp_ranks > 1:
|
||||
raise ValueError(
|
||||
"launch_local_runtime requires --dp-size 1; got "
|
||||
f"dp_size={get_parallel().dp_size}."
|
||||
"launch_local_runtime requires a single DP rank; got "
|
||||
f"num_dp_ranks={get_parallel().num_dp_ranks}."
|
||||
)
|
||||
|
||||
# Set up prometheus metrics.
|
||||
@@ -1827,8 +1827,8 @@ def launch_dp_runtime(server_args: ServerArgs) -> DPDispatcher:
|
||||
"Encoder DP mode requires --dp-size > 1 and --tp-size 1; got "
|
||||
f"dp_size={get_parallel().dp_size}, tp_size={get_parallel().tp_size}."
|
||||
)
|
||||
dp_size = get_parallel().dp_size
|
||||
logger.info(f"Launching encoder in DP mode: dp_size={dp_size}")
|
||||
num_dp_ranks = get_parallel().num_dp_ranks
|
||||
logger.info(f"Launching encoder in DP mode: num_dp_ranks={num_dp_ranks}")
|
||||
|
||||
# DP mode: workers (subprocesses) write metrics to the shared multiproc dir;
|
||||
# the main process exposes the aggregated /metrics endpoint.
|
||||
@@ -1837,7 +1837,7 @@ def launch_dp_runtime(server_args: ServerArgs) -> DPDispatcher:
|
||||
|
||||
ctx = mp.get_context("spawn")
|
||||
ipc_prefix = random_uuid()
|
||||
async_zmq_ctx = zmq.asyncio.Context(dp_size + 1)
|
||||
async_zmq_ctx = zmq.asyncio.Context(num_dp_ranks + 1)
|
||||
|
||||
result_path = f"ipc:///tmp/{ipc_prefix}_dp_result"
|
||||
result_socket = get_zmq_socket(async_zmq_ctx, zmq.PULL, result_path, True)
|
||||
@@ -1845,13 +1845,13 @@ def launch_dp_runtime(server_args: ServerArgs) -> DPDispatcher:
|
||||
get_zmq_socket(
|
||||
async_zmq_ctx, zmq.PUSH, f"ipc:///tmp/{ipc_prefix}_dp_dispatch_{r}", True
|
||||
)
|
||||
for r in range(dp_size)
|
||||
for r in range(num_dp_ranks)
|
||||
]
|
||||
release_sockets: List[zmq.asyncio.Socket] = [
|
||||
get_zmq_socket(
|
||||
async_zmq_ctx, zmq.PUSH, f"ipc:///tmp/{ipc_prefix}_dp_release_{r}", True
|
||||
)
|
||||
for r in range(dp_size)
|
||||
for r in range(num_dp_ranks)
|
||||
]
|
||||
|
||||
worker_processes: List[mp.Process] = []
|
||||
@@ -1867,7 +1867,7 @@ def launch_dp_runtime(server_args: ServerArgs) -> DPDispatcher:
|
||||
# exception (atexit holds the list ref and reads it at exit time).
|
||||
atexit.register(_kill_workers)
|
||||
|
||||
for dp_rank in range(dp_size):
|
||||
for dp_rank in range(num_dp_ranks):
|
||||
gpu_id = get_device().base_gpu_id + dp_rank
|
||||
# Pin the device parent-side around spawn (same convention as the
|
||||
# scheduler launcher and DP controller) so the child inherits
|
||||
@@ -1894,7 +1894,7 @@ def launch_dp_runtime(server_args: ServerArgs) -> DPDispatcher:
|
||||
if get_observability().extra_metric_labels:
|
||||
labels.update(get_observability().extra_metric_labels)
|
||||
return DPDispatcher(
|
||||
dp_size,
|
||||
num_dp_ranks,
|
||||
dispatch_sockets,
|
||||
release_sockets,
|
||||
result_socket,
|
||||
|
||||
@@ -79,14 +79,11 @@ def _bind_threads_if_cpu(*, device: str) -> "Optional[List[int]]":
|
||||
from sglang.srt.utils import numa_utils
|
||||
|
||||
parallel = get_parallel()
|
||||
# With --enable-dp-attention, dp partitions the existing TP group rather
|
||||
# than spawning additional processes, so dp_size must not be multiplied
|
||||
# into the process count here (unlike regular DP, where dp_size * tp_size *
|
||||
# pp_size is the true worker count).
|
||||
dp_size = 1 if parallel.enable_dp_attention else parallel.dp_size
|
||||
# Attention DP partitions the existing TP group rather than spawning
|
||||
# additional processes, so only the replicas multiply the process count.
|
||||
return numa_utils.init_threads_binding(
|
||||
numa_index=get_device().gpu_id,
|
||||
world_size=dp_size * parallel.tp_size * parallel.pp_size,
|
||||
world_size=parallel.dp_size * parallel.tp_size * parallel.pp_size,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -394,15 +394,15 @@ class Engine(EngineScoreMixin, EngineBase):
|
||||
routed_dp_rank = data_parallel_rank
|
||||
|
||||
if routed_dp_rank is not None:
|
||||
dp_size = get_parallel().dp_size
|
||||
if dp_size <= 1 and routed_dp_rank == 0:
|
||||
num_dp_ranks = get_parallel().num_dp_ranks
|
||||
if num_dp_ranks <= 1 and routed_dp_rank == 0:
|
||||
logger.debug(
|
||||
f"routed_dp_rank={routed_dp_rank} is ignored because dp_size={dp_size}"
|
||||
f"routed_dp_rank={routed_dp_rank} is ignored because num_dp_ranks={num_dp_ranks}"
|
||||
)
|
||||
return None
|
||||
if routed_dp_rank < 0 or routed_dp_rank >= dp_size:
|
||||
if routed_dp_rank < 0 or routed_dp_rank >= num_dp_ranks:
|
||||
raise ValueError(
|
||||
f"routed_dp_rank={routed_dp_rank} out of range [0, {dp_size})"
|
||||
f"routed_dp_rank={routed_dp_rank} out of range [0, {num_dp_ranks})"
|
||||
)
|
||||
|
||||
logger.debug(f"routed_dp_rank: {routed_dp_rank}")
|
||||
@@ -886,7 +886,7 @@ class Engine(EngineScoreMixin, EngineBase):
|
||||
"""
|
||||
scheduler_procs = []
|
||||
use_dp_controller = (
|
||||
get_parallel().dp_size > 1 or get_exec().moe.ep_join_mode == "scale"
|
||||
get_parallel().num_dp_ranks > 1 or get_exec().moe.ep_join_mode == "scale"
|
||||
)
|
||||
|
||||
if not use_dp_controller:
|
||||
|
||||
@@ -2249,7 +2249,7 @@ async def _send_disaggregation_warmup_requests(
|
||||
return await asyncio.gather(
|
||||
*(
|
||||
send_request(session, dp_rank)
|
||||
for dp_rank in range(get_parallel().dp_size)
|
||||
for dp_rank in range(get_parallel().num_dp_ranks)
|
||||
)
|
||||
)
|
||||
|
||||
@@ -2313,9 +2313,11 @@ def _execute_server_warmup(server_args: ServerArgs):
|
||||
},
|
||||
}
|
||||
if get_serving().skip_tokenizer_init:
|
||||
json_data["input_ids"] = [[10, 11, 12] for _ in range(get_parallel().dp_size)]
|
||||
json_data["input_ids"] = [
|
||||
[10, 11, 12] for _ in range(get_parallel().num_dp_ranks)
|
||||
]
|
||||
# TODO Workaround the bug that embedding errors for list of size 1
|
||||
if get_parallel().dp_size == 1:
|
||||
if get_parallel().num_dp_ranks == 1:
|
||||
json_data["input_ids"] = json_data["input_ids"][0]
|
||||
elif (
|
||||
is_vlm
|
||||
@@ -2359,9 +2361,11 @@ def _execute_server_warmup(server_args: ServerArgs):
|
||||
"temperature": 0.0,
|
||||
}
|
||||
else:
|
||||
json_data["text"] = ["The capital city of France is"] * get_parallel().dp_size
|
||||
json_data["text"] = [
|
||||
"The capital city of France is"
|
||||
] * get_parallel().num_dp_ranks
|
||||
# TODO Workaround the bug that embedding errors for list of size 1
|
||||
if get_parallel().dp_size == 1:
|
||||
if get_parallel().num_dp_ranks == 1:
|
||||
json_data["text"] = json_data["text"][0]
|
||||
|
||||
# Config debug dumping
|
||||
@@ -2402,7 +2406,7 @@ def _execute_server_warmup(server_args: ServerArgs):
|
||||
if not failed_status_codes:
|
||||
logger.info(
|
||||
"Disaggregation warmup requests completed for all %s DP ranks",
|
||||
get_parallel().dp_size,
|
||||
get_parallel().num_dp_ranks,
|
||||
)
|
||||
logger.info("End of disaggregation warmup")
|
||||
else:
|
||||
|
||||
@@ -49,8 +49,8 @@ def _num_accelerators_per_dp_rank() -> int:
|
||||
"""
|
||||
parallel = get_parallel()
|
||||
num_accelerators = parallel.tp_size * parallel.pp_size
|
||||
if parallel.enable_dp_attention:
|
||||
num_accelerators //= parallel.dp_size
|
||||
if parallel.attn_dp_enabled:
|
||||
num_accelerators //= parallel.num_dp_ranks
|
||||
return num_accelerators
|
||||
|
||||
|
||||
|
||||
@@ -283,7 +283,7 @@ class NPUGraphRunner(DecodeCudaGraphRunner):
|
||||
# Refresh the static DP token buffers bound by the captured
|
||||
# graph (stale values misalign dp-gather segments across
|
||||
# ranks); mirror the capture-side uniform
|
||||
# [padded_num_tokens] * dp_size.
|
||||
# [padded_num_tokens] * num_dp_ranks.
|
||||
if self.require_mlp_tp_gather:
|
||||
_padded_num_tokens = bs * self.captured_req_width
|
||||
self.buffers.global_num_tokens_gpu.fill_(_padded_num_tokens)
|
||||
|
||||
@@ -525,7 +525,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
16 if _gathered_num_head < 16 else _gathered_num_head
|
||||
)
|
||||
|
||||
self.enable_dp_attention = is_dp_attention_enabled()
|
||||
self.attn_dp_enabled = is_dp_attention_enabled()
|
||||
self.qo_indptr_ = torch.zeros(
|
||||
(max_bs + 1,), dtype=torch.int32, device=model_runner.device
|
||||
)
|
||||
@@ -543,7 +543,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
# need to fall back to non-persist
|
||||
# only use mla_ps_kernel when fp8 kv_cache
|
||||
# for non-fp8 kv_cache on tp8, use non-persist kernel to avoid performance degradation
|
||||
# head_num=16 (tp8 perf issue), head_num=128 (unsupported, like tp1 or --enable-dp-attention with tp8-dp8)
|
||||
# head_num=16 (tp8 perf issue), head_num=128 (unsupported, like tp1 or tp8 with --attn-dp-size 8)
|
||||
# Native 16-head persist is slow on TP8; keep disabled unless zero-pad
|
||||
# (e.g. Kimi K3 h12 -> qh16) where persist ASM is the fast path.
|
||||
if (
|
||||
@@ -628,7 +628,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
nhead = self.mla_kernel_num_head_padded
|
||||
dtype = self.kv_cache_dtype
|
||||
|
||||
if self.enable_dp_attention:
|
||||
if self.attn_dp_enabled:
|
||||
gpu = torch.cuda.current_device()
|
||||
device_properties = torch.cuda.get_device_properties(gpu)
|
||||
cu_num = device_properties.multi_processor_count
|
||||
|
||||
@@ -63,7 +63,7 @@ from sglang.kernels.ops.attention.flash_attention import (
|
||||
|
||||
|
||||
def _should_disable_scheduler_metadata_precompute() -> bool:
|
||||
return bool(get_parallel().enable_prefill_cp or get_parallel().enable_dp_attention)
|
||||
return bool(get_parallel().enable_prefill_cp or get_parallel().attn_dp_enabled)
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -23,7 +23,7 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.runtime_context import (
|
||||
derive_attention_ranks,
|
||||
derive_attention_widths,
|
||||
derive_attn_tp_size,
|
||||
get_device,
|
||||
get_exec,
|
||||
get_flags,
|
||||
@@ -51,13 +51,7 @@ def deployment_attn_dp_size() -> int:
|
||||
the target's replicas.
|
||||
"""
|
||||
parallel = get_parallel()
|
||||
attn_dp_size, _ = derive_attention_widths(
|
||||
tp_size=parallel.tp_size,
|
||||
attn_cp_size=parallel.attn_cp_size,
|
||||
dp_size=parallel.dp_size,
|
||||
enable_dp_attention=parallel.enable_dp_attention,
|
||||
)
|
||||
return attn_dp_size
|
||||
return parallel.num_dp_ranks if parallel.attn_dp_enabled else 1
|
||||
|
||||
|
||||
def dp_gather_width() -> int:
|
||||
@@ -69,7 +63,7 @@ def dp_gather_width() -> int:
|
||||
"""
|
||||
parallel = get_parallel()
|
||||
if world_dp_gather_enabled():
|
||||
return parallel.dp_size
|
||||
return parallel.num_dp_ranks
|
||||
if get_flags().dp.scoped_gather_slot is not None:
|
||||
return deployment_attn_dp_size()
|
||||
return parallel.attn_dp_size
|
||||
@@ -127,7 +121,7 @@ def update_dp_attention_post_scale(new_dp_size: int, new_dp_rank: int):
|
||||
"""
|
||||
get_flags().dp.use_world_group_for_gather = True
|
||||
logger.debug(
|
||||
"[Elastic EP] dp_attention switched to WORLD: dp_size=%d dp_rank=%d",
|
||||
"[Elastic EP] dp_attention switched to WORLD: num_dp_ranks=%d dp_rank=%d",
|
||||
new_dp_size,
|
||||
new_dp_rank,
|
||||
)
|
||||
@@ -154,7 +148,7 @@ class DpPaddingMode(IntEnum):
|
||||
def get_dp_padding_mode(
|
||||
cls, is_extend_in_batch, global_num_tokens: List[int]
|
||||
) -> DpPaddingMode:
|
||||
dp_size = dp_gather_width()
|
||||
gather_width = dp_gather_width()
|
||||
|
||||
# (trangdough) pplx-kernels a2a is a symmetric collective: every EP rank
|
||||
# must dispatch the same number of tokens or the device-side handshake
|
||||
@@ -169,11 +163,11 @@ class DpPaddingMode(IntEnum):
|
||||
if moe_a2a_backend.is_deepep_v2() and envs.SGLANG_DEEPEP_V2_FORCE_MAX_LEN.get():
|
||||
return DpPaddingMode.MAX_LEN
|
||||
|
||||
# When is_extend_in_batch and dp_size > 1, use SUM_LEN to avoid padding
|
||||
# overhead from uneven token distribution.
|
||||
# For dp_size=1, max_len equals sum_len, so prefer MAX_LEN mode
|
||||
# When is_extend_in_batch and the gather spans several DP ranks, use
|
||||
# SUM_LEN to avoid padding overhead from uneven token distribution.
|
||||
# Over one rank, max_len equals sum_len, so prefer MAX_LEN mode
|
||||
# to enable symmetric memory optimization (needed for DSA CP, etc.).
|
||||
if is_extend_in_batch and dp_size > 1:
|
||||
if is_extend_in_batch and gather_width > 1:
|
||||
# Hybrid-SSM models materialize idle ranks via the MAX_LEN
|
||||
# fabricated-row conversion; other models keep mainline SUM_LEN.
|
||||
if get_flags().dp.max_len_with_idle and min(global_num_tokens) == 0:
|
||||
@@ -184,7 +178,7 @@ class DpPaddingMode(IntEnum):
|
||||
# prefer MAX_LEN when communication cost is equal to enable symmetric memory
|
||||
max_len = max(global_num_tokens)
|
||||
sum_len = sum(global_num_tokens)
|
||||
if sum_len * 2 >= max_len * dp_size:
|
||||
if sum_len * 2 >= max_len * gather_width:
|
||||
return cls.MAX_LEN
|
||||
else:
|
||||
return cls.SUM_LEN
|
||||
@@ -405,25 +399,19 @@ def is_dp_max_padding() -> bool:
|
||||
|
||||
|
||||
def compute_dp_attention_world_info(
|
||||
enable_dp_attention, tp_rank, tp_size, dp_size, attn_cp_size: int = 1
|
||||
tp_rank, tp_size, attn_dp_size, attn_cp_size: int = 1
|
||||
):
|
||||
"""This rank's place in the attention topology, plus the widths it sits in.
|
||||
|
||||
The widths come from `derive_attention_widths`; what this adds is the two
|
||||
ranks, which are per-process and so are not among the widths
|
||||
The attention-TP width comes from `derive_attn_tp_size`; what this adds is
|
||||
the two ranks, which are per-process and so are not among the widths
|
||||
`override_permanently` records.
|
||||
"""
|
||||
attn_dp_size, attn_tp_size = derive_attention_widths(
|
||||
tp_size=tp_size,
|
||||
attn_cp_size=attn_cp_size,
|
||||
dp_size=dp_size,
|
||||
enable_dp_attention=enable_dp_attention,
|
||||
attn_tp_size = derive_attn_tp_size(
|
||||
tp_size=tp_size, attn_cp_size=attn_cp_size, attn_dp_size=attn_dp_size
|
||||
)
|
||||
attn_tp_rank, attn_dp_rank = derive_attention_ranks(
|
||||
tp_rank=tp_rank,
|
||||
attn_tp_size=attn_tp_size,
|
||||
attn_cp_size=attn_cp_size,
|
||||
enable_dp_attention=enable_dp_attention,
|
||||
tp_rank=tp_rank, attn_tp_size=attn_tp_size, attn_cp_size=attn_cp_size
|
||||
)
|
||||
return attn_tp_rank, attn_tp_size, attn_dp_rank, attn_dp_size
|
||||
|
||||
@@ -431,7 +419,7 @@ def compute_dp_attention_world_info(
|
||||
def initialize_dp_attention_flags(server_args: ServerArgs):
|
||||
"""Initialize DP runtime flags without changing the worker's placement."""
|
||||
dp = get_flags().dp
|
||||
dp.enabled = get_parallel().enable_dp_attention
|
||||
dp.enabled = get_parallel().attn_dp_enabled
|
||||
|
||||
if get_exec().moe.elastic_ep_backend is not None and get_parallel().max_ep_size:
|
||||
if ep_scale_joiner_of(resolving_view(server_args)):
|
||||
@@ -443,10 +431,9 @@ def initialize_dp_attention(server_args: ServerArgs):
|
||||
initialize_dp_attention_flags(server_args)
|
||||
parallel = get_parallel()
|
||||
_, _, attn_dp_rank, attn_dp_size = compute_dp_attention_world_info(
|
||||
parallel.enable_dp_attention,
|
||||
parallel.tp_rank,
|
||||
parallel.tp_size,
|
||||
parallel.dp_size,
|
||||
parallel.attn_dp_size,
|
||||
parallel.attn_cp_size,
|
||||
)
|
||||
parallel.override_permanently(attn_dp_size=attn_dp_size, attn_dp_rank=attn_dp_rank)
|
||||
@@ -652,7 +639,7 @@ def _dp_gather_via_all_gather(
|
||||
# Variable-length DP-MoE gather (reference https://github.com/ROCm/ATOM/pull/930): instead of padding every
|
||||
# rank to max_len (all_gather) or all-reducing a sum_len zero-buffer (all_reduce),
|
||||
# gather exactly sum(per-rank tokens) via all_gatherv. Env-gated; only the simple
|
||||
# tp_size==dp_size (attn_tp_size==1) case is supported for now (e.g. tp8dp8).
|
||||
# tp_size==attn_dp_size (attn_tp_size==1) case is supported for now (e.g. tp8, attn_dp8).
|
||||
_USE_DP_GATHERV = get_bool_env_var("SGLANG_DP_USE_GATHERV")
|
||||
|
||||
_DP_GATHER_FP8_GROUP = 128
|
||||
@@ -725,7 +712,7 @@ def mask_dp_pad_moe_topk_ids(topk_ids: torch.Tensor) -> None:
|
||||
"""Set MAX_LEN pad rows' (post-translation, local) topk_ids to -1 in place.
|
||||
|
||||
Under dp-attention MAX_LEN padding the gathered MoE buffer is
|
||||
[dp_size * max_len, hidden] with rank r's real rows at
|
||||
[num_dp_ranks * max_len, hidden] with rank r's real rows at
|
||||
[r*max_len, r*max_len + global_num_tokens[r]); the pad rows carry stale
|
||||
hidden values, run the router, and get dispatched into experts whose
|
||||
outputs are then discarded by the post-reorder scatter — pure wasted
|
||||
@@ -794,7 +781,7 @@ def is_dp_gatherv_active() -> bool:
|
||||
"""Variable-length DP-MoE gather/scatter (all_gatherv + reduce_scatterv) is
|
||||
enabled and applicable to the CURRENT forward. Requires:
|
||||
- env SGLANG_DP_USE_GATHERV (default off),
|
||||
- supported layout (attn_tp_size==1, tp_size==dp_size),
|
||||
- supported layout (attn_tp_size==1, tp_size==attn_dp_size),
|
||||
- SUM_LEN padding mode. The gatherv pair (all_gatherv + reduce_scatterv) is
|
||||
only valid under SUM_LEN; under MAX_LEN the buffer is equal-padded and the
|
||||
gather/combine use all_gather / (aiter) reduce_scatter instead. Reading the
|
||||
@@ -1004,7 +991,7 @@ def can_use_dp_reduce_scatter() -> bool:
|
||||
return True
|
||||
|
||||
parallel = get_parallel()
|
||||
return parallel.tp_size == parallel.dp_size * parallel.attn_tp_size
|
||||
return parallel.tp_size == parallel.num_dp_ranks * parallel.attn_tp_size
|
||||
|
||||
|
||||
def dp_reduce_scatter_tensor(output: torch.Tensor, input: torch.Tensor):
|
||||
|
||||
@@ -175,7 +175,7 @@ class StagePlan:
|
||||
self._speculative_algo = SpeculativeAlgorithm.from_string(
|
||||
get_spec().speculative_algorithm
|
||||
)
|
||||
self._publish_lora_layout = get_parallel().enable_dp_attention and bool(
|
||||
self._publish_lora_layout = get_parallel().attn_dp_enabled and bool(
|
||||
get_lora().enable_lora
|
||||
)
|
||||
self._next_input_rows = None
|
||||
|
||||
@@ -84,7 +84,7 @@ _UNQUANTIZED_LM_HEAD_METHODS = {
|
||||
# None outside a FlashInfer autotune pass; inside one, whether that pass runs the
|
||||
# LM head. Not-None means the forward's output is discarded -- attention backends
|
||||
# read that via get_in_autotune_dummy_run() to skip a cross-node exchange.
|
||||
# Skipping the LM head skips its [batch * dp_size, vocab] all-gather, which OOMs
|
||||
# Skipping the LM head skips its [batch * num_dp_ranks, vocab] all-gather, which OOMs
|
||||
# under DP attention with a tight mem_fraction_static.
|
||||
_autotune_run_lm_head: Optional[bool] = None
|
||||
|
||||
@@ -192,7 +192,7 @@ def should_apply_lm_head_quant_method(lm_head, quant_method) -> bool:
|
||||
|
||||
|
||||
# FlashInfer autotune skips the unprofiled LM-head all-gather; its
|
||||
# [batch * dp_size, vocab] output can OOM under tight DP-attention memory.
|
||||
# [batch * num_dp_ranks, vocab] output can OOM under tight DP-attention memory.
|
||||
_in_autotune_dummy_run = False
|
||||
|
||||
|
||||
|
||||
@@ -340,9 +340,9 @@ def ensure_cutedsl_wrapper(layer: torch.nn.Module) -> None:
|
||||
# A2A path: bounded by the dispatcher's own workspace limit.
|
||||
max_num_tokens = dispatcher.max_num_tokens * getattr(dispatcher, "ep_size", 1)
|
||||
else:
|
||||
# Standard allgather path: the MoE sees up to dp_size local forwards
|
||||
# gathered together, so scale the per-rank forward bound by dp_size.
|
||||
max_num_tokens = get_parallel().dp_size * cutedsl_moe_max_num_tokens()
|
||||
# Standard allgather path: the MoE sees up to num_dp_ranks local forwards
|
||||
# gathered together, so scale the per-rank forward bound by num_dp_ranks.
|
||||
max_num_tokens = get_parallel().num_dp_ranks * cutedsl_moe_max_num_tokens()
|
||||
top_k = layer.top_k if layer.top_k is not None else layer.moe_runner_config.top_k
|
||||
# inference_mode(False) ensures the wrapper's pre-allocated CUDA-graph
|
||||
# buffers are normal tensors. This call typically happens inside
|
||||
|
||||
@@ -99,9 +99,9 @@ class LoRAManager:
|
||||
self.enable_lora_overlap_loading: Optional[bool] = (
|
||||
get_lora().enable_lora_overlap_loading
|
||||
)
|
||||
self.enable_dp_attention: bool = get_parallel().enable_dp_attention
|
||||
self.attn_dp_enabled: bool = get_parallel().attn_dp_enabled
|
||||
if (
|
||||
self.enable_dp_attention
|
||||
self.attn_dp_enabled
|
||||
and self.enable_lora_overlap_loading
|
||||
and not LoRAMemoryPool.supports_dp_attention_overlap_loading
|
||||
):
|
||||
@@ -112,11 +112,11 @@ class LoRAManager:
|
||||
)
|
||||
# LoRA routing knows only DP-local and TP-global token layouts. A wider
|
||||
# attention TP/CP group can scatter one DP rank's tokens across ranks.
|
||||
if self.enable_dp_attention and (
|
||||
if self.attn_dp_enabled and (
|
||||
self.attn_tp_size != 1 or get_parallel().attn_cp_size != 1
|
||||
):
|
||||
raise ValueError(
|
||||
"LoRA with DP attention requires --dp-size equal to --tp-size "
|
||||
"LoRA with DP attention requires --attn-dp-size equal to --tp-size "
|
||||
f"(got attention TP size {self.attn_tp_size} and attention CP "
|
||||
f"size {get_parallel().attn_cp_size})"
|
||||
)
|
||||
@@ -138,7 +138,7 @@ class LoRAManager:
|
||||
device=self.device,
|
||||
server_args=server_args,
|
||||
)
|
||||
if self.enable_dp_attention and not self.lora_backend.supports_dp_attention:
|
||||
if self.attn_dp_enabled and not self.lora_backend.supports_dp_attention:
|
||||
raise ValueError(
|
||||
f"LoRA backend {lora_backend!r} does not support DP attention"
|
||||
)
|
||||
@@ -164,9 +164,9 @@ class LoRAManager:
|
||||
num_tokens_per_req=num_tokens_per_req,
|
||||
)
|
||||
max_moe_tokens = max_bs_in_cuda_graph * num_tokens_per_req
|
||||
if self.enable_dp_attention:
|
||||
if self.attn_dp_enabled:
|
||||
self.lora_backend.init_dp_attention_cuda_graph_batch_info(max_moe_tokens)
|
||||
max_moe_tokens *= get_parallel().dp_size
|
||||
max_moe_tokens *= get_parallel().num_dp_ranks
|
||||
if self.lora_backend.resize_cuda_graph_moe_buffers(max_moe_tokens):
|
||||
logger.info(
|
||||
"Right-sized shared MoE LoRA CUDA graph buffers "
|
||||
@@ -211,10 +211,7 @@ class LoRAManager:
|
||||
check_cuda_graph_backend,
|
||||
)
|
||||
|
||||
if (
|
||||
self.enable_dp_attention
|
||||
or not self.lora_backend.supports_prefill_cuda_graph
|
||||
):
|
||||
if self.attn_dp_enabled or not self.lora_backend.supports_prefill_cuda_graph:
|
||||
return False
|
||||
if self.lora_backend.is_moe_lora:
|
||||
return check_cuda_graph_backend(
|
||||
@@ -237,7 +234,7 @@ class LoRAManager:
|
||||
return False
|
||||
# DP attention: per-rank eligibility could diverge across ranks and
|
||||
# desync collectives; keep LoRA prefill eager.
|
||||
if self.enable_dp_attention:
|
||||
if self.attn_dp_enabled:
|
||||
return False
|
||||
# Decode-CUDA-graph extend modes (TARGET_VERIFY, DLLM_EXTEND) are
|
||||
# owned by the decode static batch info path.
|
||||
@@ -333,7 +330,7 @@ class LoRAManager:
|
||||
"""
|
||||
Validate if an adapter can be loaded into the current LoRA memory pool and generate error if it is incompatible.
|
||||
"""
|
||||
assert not self.enable_dp_attention or lora_ref.pinned, (
|
||||
assert not self.attn_dp_enabled or lora_ref.pinned, (
|
||||
"DP-attention LoRA requires pinned adapters"
|
||||
)
|
||||
if lora_config.lora_added_tokens_size > 0:
|
||||
@@ -506,7 +503,7 @@ class LoRAManager:
|
||||
)
|
||||
|
||||
active_lora_ids = set(forward_batch.lora_ids)
|
||||
if self.enable_dp_attention:
|
||||
if self.attn_dp_enabled:
|
||||
get_forward().set("lora_batch_layout", LoRABatchLayout.DP_LOCAL)
|
||||
gathered_lora_ids = get_parallel().tp_group.all_gather_object(
|
||||
forward_batch.lora_ids
|
||||
@@ -551,7 +548,7 @@ class LoRAManager:
|
||||
self.lora_backend.batch_info.has_active_lora = any(
|
||||
lora_ranks[wi] > 0 for wi in weight_indices
|
||||
)
|
||||
if self.enable_dp_attention:
|
||||
if self.attn_dp_enabled:
|
||||
self.lora_backend.prepare_global_lora_batch(forward_batch)
|
||||
if forward_batch.forward_mode.is_idle():
|
||||
# Idle ranks must join the global routing collectives, but
|
||||
@@ -563,9 +560,7 @@ class LoRAManager:
|
||||
hasattr(self, "max_bs_in_cuda_graph")
|
||||
and forward_batch.batch_size <= self.max_bs_in_cuda_graph
|
||||
and forward_batch.forward_mode.is_cuda_graph()
|
||||
and (
|
||||
not self.enable_dp_attention or forward_batch.can_run_decode_cuda_graph
|
||||
)
|
||||
and (not self.attn_dp_enabled or forward_batch.can_run_decode_cuda_graph)
|
||||
)
|
||||
|
||||
def prepare_lora_token_segments(
|
||||
@@ -1194,7 +1189,7 @@ def init_lora_cuda_graph_moe_buffers(
|
||||
# num_draft_tokens per request, and the buffers below are per-token, so
|
||||
# they must be sized in tokens rather than requests.
|
||||
max_tokens = max_bs * (get_spec().speculative_num_draft_tokens or 1)
|
||||
if get_parallel().enable_dp_attention:
|
||||
if get_parallel().attn_dp_enabled:
|
||||
max_tokens *= get_parallel().attn_dp_size
|
||||
max_loras = get_lora().max_loras_per_batch
|
||||
for module in model.modules():
|
||||
|
||||
@@ -180,7 +180,7 @@ class LoRAMemoryPool:
|
||||
)
|
||||
|
||||
# Per-expert MoE weights are sharded by `moe_tp_size`, NOT the outer
|
||||
# `tp_size`: `moe_tp_size = tp_size // ep_size // dp_size`, so under
|
||||
# `tp_size`: `moe_tp_size = tp_size // ep_size // moe_dp_size`, so under
|
||||
# e.g. `--tp 4 --ep 4` each rank holds full-width expert weights
|
||||
# (`moe_tp_size == 1`). Sizing per-expert LoRA buffers by `tp_size`
|
||||
# here would yield a 4x-narrower inner dim than the adapter weight
|
||||
@@ -190,7 +190,7 @@ class LoRAMemoryPool:
|
||||
self.moe_tp_size, self.moe_tp_rank = _get_moe_tp_context()
|
||||
|
||||
# Attention projections shard along the attention TP group, which
|
||||
# under `--enable-dp-attention` is `attn_tp_size = tp_size // dp_size`.
|
||||
# under attention DP is `attn_tp_size = tp_size // attn_dp_size`.
|
||||
# The corresponding LoRA wrappers slice weights by the base layer's
|
||||
# attn_tp-local rank, so the buffer shapes must match that shard.
|
||||
self.attn_tp_size: int = attn_tp_size
|
||||
@@ -273,7 +273,7 @@ class LoRAMemoryPool:
|
||||
"""TP width the module's weights are actually sharded along: routed
|
||||
MoE experts shard by `moe_tp_size` (shared experts by the outer
|
||||
`tp_size` at EP=1), attention projections by `attn_tp_size` (smaller
|
||||
than the outer `tp_size` under `--enable-dp-attention`), everything
|
||||
than the outer `tp_size` under attention DP), everything
|
||||
else by the outer `tp_size`."""
|
||||
if self.is_moe_module(module_name) and not self.is_shared_moe_module(
|
||||
module_name
|
||||
|
||||
@@ -373,7 +373,7 @@ REPLICATED_LINEAR_LORA_NAMES = [
|
||||
*DSA_INDEXER_LORA_NAMES,
|
||||
]
|
||||
# Attention-projection LoRA modules shard on the attention-TP group, which
|
||||
# under `--enable-dp-attention` is `attn_tp_size = tp_size // dp_size` rather
|
||||
# under attention DP is `attn_tp_size = tp_size // attn_dp_size` rather
|
||||
# than the outer TP size. in_proj / in_proj_qkvz (linear-attention hybrids)
|
||||
# belong here too: their layers are built on the attn-TP group (mamba.py and
|
||||
# qwen3_5.py take tp_size/tp_rank from attn_tp when dp attention is enabled).
|
||||
|
||||
@@ -102,11 +102,11 @@ class LoadBalanceMethod(Enum):
|
||||
|
||||
|
||||
class DPBudget:
|
||||
def __init__(self, dp_size: int):
|
||||
self.dp_size = dp_size
|
||||
self.total_requests = [0] * dp_size
|
||||
self.total_tokens = [0] * dp_size
|
||||
self.last_timestamp = [0.0] * dp_size
|
||||
def __init__(self, num_dp_ranks: int):
|
||||
self.num_dp_ranks = num_dp_ranks
|
||||
self.total_requests = [0] * num_dp_ranks
|
||||
self.total_tokens = [0] * num_dp_ranks
|
||||
self.last_timestamp = [0.0] * num_dp_ranks
|
||||
|
||||
def update_budget(self, loads):
|
||||
"""Update budget from shm snapshots, skipping stale reads."""
|
||||
@@ -125,7 +125,7 @@ class DPBudget:
|
||||
elif method == LoadBalanceMethod.TOTAL_TOKENS:
|
||||
# Use total_requests as a tie-breaker when total_tokens are equal
|
||||
target_rank = min(
|
||||
range(self.dp_size),
|
||||
range(self.num_dp_ranks),
|
||||
key=lambda i: (self.total_tokens[i], self.total_requests[i]),
|
||||
)
|
||||
else:
|
||||
@@ -155,7 +155,7 @@ class DataParallelController:
|
||||
self.run_scheduler_process_func = run_scheduler_process_func
|
||||
|
||||
# Init inter-process communication
|
||||
self.context = zmq.Context(1 + get_parallel().dp_size)
|
||||
self.context = zmq.Context(1 + get_parallel().num_dp_ranks)
|
||||
if get_parallel().node_rank == 0:
|
||||
self.recv_from_tokenizer = get_zmq_socket(
|
||||
self.context, zmq.PULL, port_args.scheduler_input_ipc_name, False
|
||||
@@ -175,8 +175,10 @@ class DataParallelController:
|
||||
LoadBalanceMethod.TOTAL_TOKENS,
|
||||
)
|
||||
|
||||
self.launch_dp_size: int = get_parallel().dp_size
|
||||
self.max_dp_size: int = get_parallel().max_ep_size or get_parallel().dp_size
|
||||
self.launch_dp_size: int = get_parallel().num_dp_ranks
|
||||
self.max_dp_size: int = (
|
||||
get_parallel().max_ep_size or get_parallel().num_dp_ranks
|
||||
)
|
||||
assert self.max_dp_size >= self.launch_dp_size, (
|
||||
f"--max-ep-size ({self.max_dp_size}) must be >= "
|
||||
f"--dp ({self.launch_dp_size})."
|
||||
@@ -186,7 +188,7 @@ class DataParallelController:
|
||||
self.max_dp_size - self.launch_dp_size
|
||||
)
|
||||
|
||||
self.dp_budget = DPBudget(get_parallel().dp_size)
|
||||
self.dp_budget = DPBudget(get_parallel().num_dp_ranks)
|
||||
self.load_snapshot_reader = create_load_snapshot_reader(
|
||||
port_args,
|
||||
caller="DataParallelController",
|
||||
@@ -203,7 +205,7 @@ class DataParallelController:
|
||||
self._active_workers: list[int] = list(range(self.launch_dp_size))
|
||||
self._active_count_cache: int = self.launch_dp_size
|
||||
|
||||
if get_parallel().enable_dp_attention:
|
||||
if get_parallel().attn_dp_enabled:
|
||||
self.launch_dp_attention_schedulers(server_args, port_args)
|
||||
# When local control broadcast is enabled, send control messages to
|
||||
# every DP group leader (attn_tp_rank=0) so each leader broadcasts
|
||||
@@ -374,7 +376,7 @@ class DataParallelController:
|
||||
threads = []
|
||||
sockets = []
|
||||
ready_events = []
|
||||
for dp_rank in range(get_parallel().dp_size):
|
||||
for dp_rank in range(get_parallel().num_dp_ranks):
|
||||
tmp_port_args = PortArgs.init_new(server_args)
|
||||
tmp_port_args.tokenizer_ipc_name = port_args.tokenizer_ipc_name
|
||||
tmp_port_args.detokenizer_ipc_name = port_args.detokenizer_ipc_name
|
||||
@@ -577,7 +579,7 @@ class DataParallelController:
|
||||
bind_count = (
|
||||
self.max_dp_size
|
||||
if get_exec().moe.elastic_ep_backend is not None
|
||||
else get_parallel().dp_size
|
||||
else get_parallel().num_dp_ranks
|
||||
)
|
||||
for slot in range(bind_count):
|
||||
worker_port, worker_socket = get_zmq_socket_on_host(
|
||||
@@ -607,7 +609,7 @@ class DataParallelController:
|
||||
dp_rank: int | None,
|
||||
worker_ports: list[int] | None = None,
|
||||
):
|
||||
if not get_parallel().enable_dp_attention:
|
||||
if not get_parallel().attn_dp_enabled:
|
||||
logger.info(f"Launch DP{dp_rank} starting at GPU #{base_gpu_id}.")
|
||||
|
||||
memory_saver_adapter = TorchMemorySaverAdapter.create(
|
||||
@@ -639,13 +641,12 @@ class DataParallelController:
|
||||
for tp_rank in tp_rank_range:
|
||||
rank_port_args = port_args
|
||||
|
||||
if get_parallel().enable_dp_attention:
|
||||
if get_parallel().attn_dp_enabled:
|
||||
# dp attention has different sharding logic
|
||||
_, _, dp_rank, _ = compute_dp_attention_world_info(
|
||||
get_parallel().enable_dp_attention,
|
||||
tp_rank,
|
||||
get_parallel().tp_size,
|
||||
get_parallel().dp_size,
|
||||
get_parallel().attn_dp_size,
|
||||
get_parallel().attn_cp_size,
|
||||
)
|
||||
# compute zmq ports for this dp rank
|
||||
|
||||
@@ -71,7 +71,7 @@ def should_use_zmq() -> bool:
|
||||
``SGLANG_LOAD_SNAPSHOT_USE_ZMQ`` forces zmq mode for testing.
|
||||
"""
|
||||
return (
|
||||
get_parallel().enable_dp_attention and get_parallel().nnodes > 1
|
||||
get_parallel().attn_dp_enabled and get_parallel().nnodes > 1
|
||||
) or envs.SGLANG_LOAD_SNAPSHOT_USE_ZMQ.get()
|
||||
|
||||
|
||||
@@ -120,10 +120,10 @@ def zmq_reader_owner(caller: str) -> bool:
|
||||
return False
|
||||
if caller == "DataParallelController":
|
||||
return (
|
||||
get_parallel().dp_size > 1
|
||||
get_parallel().num_dp_ranks > 1
|
||||
and get_parallel().load_balance_method.lower() in _LOAD_AWARE_METHODS
|
||||
)
|
||||
if get_parallel().dp_size > 1 and (
|
||||
if get_parallel().num_dp_ranks > 1 and (
|
||||
get_parallel().load_balance_method.lower() in _LOAD_AWARE_METHODS
|
||||
):
|
||||
return False
|
||||
@@ -659,9 +659,9 @@ def create_load_snapshot_reader(port_args, caller: str):
|
||||
``"MultiTokenizerRouter"`` -- determines who binds the zmq PULL
|
||||
socket when zmq mode is active.
|
||||
"""
|
||||
dp_size = get_parallel().dp_size
|
||||
num_dp_ranks = get_parallel().num_dp_ranks
|
||||
if zmq_reader_owner(caller):
|
||||
return ZmqShmLoadSnapshotReader(
|
||||
_zmq_addr_for(port_args), shm_path_for(port_args.instance_id), dp_size
|
||||
_zmq_addr_for(port_args), shm_path_for(port_args.instance_id), num_dp_ranks
|
||||
)
|
||||
return ShmLoadSnapshotReader(shm_path_for(port_args.instance_id), dp_size)
|
||||
return ShmLoadSnapshotReader(shm_path_for(port_args.instance_id), num_dp_ranks)
|
||||
|
||||
@@ -106,9 +106,9 @@ class PrefillDelayer:
|
||||
f"queue_trigger_enabled={self._queue_trigger_enabled}"
|
||||
)
|
||||
parallel = get_parallel()
|
||||
self.dp_size = parallel.dp_size
|
||||
self.enable_dp_attention = parallel.enable_dp_attention
|
||||
dp_size_dim = self.dp_size if self.enable_dp_attention else 1
|
||||
self.num_dp_ranks = parallel.num_dp_ranks
|
||||
self.attn_dp_enabled = parallel.attn_dp_enabled
|
||||
dp_size_dim = self.num_dp_ranks if self.attn_dp_enabled else 1
|
||||
|
||||
# Mirror scheduler_dp_attn_mixin's NCCL all-gather path: when the
|
||||
# env flag is on (or overlap scheduling is disabled), ride the NCCL
|
||||
@@ -238,10 +238,10 @@ class PrefillDelayer:
|
||||
**wait_info,
|
||||
)
|
||||
|
||||
if not self.enable_dp_attention:
|
||||
if not self.attn_dp_enabled:
|
||||
max_running_requests = (
|
||||
max_running_requests + self.dp_size - 1
|
||||
) // self.dp_size
|
||||
max_running_requests + self.num_dp_ranks - 1
|
||||
) // self.num_dp_ranks
|
||||
|
||||
global_running_batch_max = int(global_running_batch.max().item())
|
||||
global_max_prefill_bs_max = int(global_max_prefill_bs.max().item())
|
||||
@@ -411,10 +411,10 @@ class PrefillDelayerSinglePassExecutor:
|
||||
waiting_queue_len: int,
|
||||
) -> int:
|
||||
local_max_running_requests = max_running_requests
|
||||
if not self._prefill_delayer.enable_dp_attention:
|
||||
if not self._prefill_delayer.attn_dp_enabled:
|
||||
local_max_running_requests = (
|
||||
max_running_requests + self._prefill_delayer.dp_size - 1
|
||||
) // self._prefill_delayer.dp_size
|
||||
max_running_requests + self._prefill_delayer.num_dp_ranks - 1
|
||||
) // self._prefill_delayer.num_dp_ranks
|
||||
|
||||
# The delayer negotiates before PrefillAdder materializes can_run_list,
|
||||
# so a rejected pass has no exact batch size. This upper bound is exact
|
||||
|
||||
@@ -516,7 +516,7 @@ class Scheduler(
|
||||
self.max_recv_per_poll = envs.SGLANG_SCHEDULER_MAX_RECV_PER_POLL.get()
|
||||
self.max_new_tokens_limit = envs.SGLANG_MAX_NEW_TOKENS_LIMIT.get()
|
||||
self.enable_hisparse = get_memory().enable_hisparse
|
||||
self.enable_dp_attention = get_parallel().enable_dp_attention
|
||||
self.attn_dp_enabled = get_parallel().attn_dp_enabled
|
||||
self.enable_unified_memory = get_memory().enable_unified_memory
|
||||
|
||||
# Init model configs
|
||||
@@ -629,7 +629,7 @@ class Scheduler(
|
||||
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
|
||||
tp_group=(
|
||||
self.attn_tp_cpu_group
|
||||
if self.enable_dp_attention
|
||||
if self.attn_dp_enabled
|
||||
else self.tp_cpu_group
|
||||
),
|
||||
tree_cache=self.tree_cache,
|
||||
@@ -823,7 +823,7 @@ class Scheduler(
|
||||
try:
|
||||
self.load_snapshot_writer = create_load_snapshot_writer(
|
||||
port_args,
|
||||
get_parallel().dp_size,
|
||||
get_parallel().num_dp_ranks,
|
||||
dp_rank,
|
||||
publish_interval=get_observability().load_snapshot_publish_interval,
|
||||
)
|
||||
@@ -1216,9 +1216,7 @@ class Scheduler(
|
||||
# the base TP group. Entry rank is the local rank 0 in that group.
|
||||
# Use the CPU (gloo) group to broadcast VLM Python objects and avoid CUDA
|
||||
# stream/device coupling (#11910).
|
||||
self.dp_tp_group = (
|
||||
self.attn_tp_group if self.enable_dp_attention else self.tp_group
|
||||
)
|
||||
self.dp_tp_group = self.attn_tp_group if self.attn_dp_enabled else self.tp_group
|
||||
self.dp_tp_cpu_group = self.dp_tp_group.cpu_group
|
||||
|
||||
self.pad_input_ids_func = self.tp_worker.get_pad_input_ids_func()
|
||||
@@ -4646,9 +4644,7 @@ class Scheduler(
|
||||
return ret
|
||||
|
||||
def _maybe_report_active_ranks(self) -> None:
|
||||
if not (
|
||||
self.enable_dp_attention and get_exec().moe.elastic_ep_backend is not None
|
||||
):
|
||||
if not (self.attn_dp_enabled and get_exec().moe.elastic_ep_backend is not None):
|
||||
return
|
||||
from sglang.srt.elastic_ep.elastic_ep import ElasticEPStateManager
|
||||
|
||||
@@ -5218,7 +5214,6 @@ class Scheduler(
|
||||
# readback reflects values changed via /set_internal_state, not startup.
|
||||
ret = get_context().resolved_server_args_dict()
|
||||
ret["world_size"] = compute_world_size(
|
||||
enable_dp_attention=get_parallel().enable_dp_attention,
|
||||
dp_size=get_parallel().dp_size,
|
||||
tp_size=get_parallel().tp_size,
|
||||
pp_size=get_parallel().pp_size,
|
||||
|
||||
@@ -65,47 +65,47 @@ def _spec_input_cuda_graph_compatible(
|
||||
return spec_info is None or spec_info.cuda_graph_compatible
|
||||
|
||||
|
||||
def _resolve_elastic_world_dp_size(
|
||||
dp_size: int,
|
||||
def _resolve_elastic_world_num_dp_ranks(
|
||||
num_dp_ranks: int,
|
||||
*,
|
||||
group: torch.distributed.ProcessGroup,
|
||||
local_num_tokens: int,
|
||||
local_forward_mode: int,
|
||||
) -> int:
|
||||
if not world_dp_gather_enabled():
|
||||
return dp_size
|
||||
return num_dp_ranks
|
||||
|
||||
from sglang.srt.elastic_ep.elastic_ep import ElasticEPStateManager
|
||||
|
||||
live_dp_size = dp_gather_width()
|
||||
live_num_dp_ranks = dp_gather_width()
|
||||
effective_ep_size = ElasticEPStateManager.get_effective_ep_size()
|
||||
# Query live membership because elastic joins can expand WORLD.
|
||||
world_size = torch.distributed.get_world_size(group)
|
||||
|
||||
if live_dp_size != effective_ep_size:
|
||||
if live_num_dp_ranks != effective_ep_size:
|
||||
raise RuntimeError(
|
||||
"[Elastic EP] WORLD MLP sync dp_size is out of sync: "
|
||||
"[Elastic EP] WORLD MLP sync num_dp_ranks is out of sync: "
|
||||
f"rank={torch.distributed.get_rank(group)} "
|
||||
f"live_dp_size={live_dp_size} "
|
||||
f"live_num_dp_ranks={live_num_dp_ranks} "
|
||||
f"effective_ep_size={effective_ep_size} "
|
||||
f"world_size={world_size} server_args_dp_size={dp_size} "
|
||||
f"world_size={world_size} configured_num_dp_ranks={num_dp_ranks} "
|
||||
f"local_num_tokens={local_num_tokens} "
|
||||
f"local_forward_mode={local_forward_mode}"
|
||||
)
|
||||
if live_dp_size > world_size:
|
||||
if live_num_dp_ranks > world_size:
|
||||
raise RuntimeError(
|
||||
"[Elastic EP] WORLD MLP sync dp_size exceeds WORLD size: "
|
||||
"[Elastic EP] WORLD MLP sync num_dp_ranks exceeds WORLD size: "
|
||||
f"rank={torch.distributed.get_rank(group)} "
|
||||
f"live_dp_size={live_dp_size} world_size={world_size} "
|
||||
f"live_num_dp_ranks={live_num_dp_ranks} world_size={world_size} "
|
||||
f"effective_ep_size={effective_ep_size}"
|
||||
)
|
||||
|
||||
return live_dp_size
|
||||
return live_num_dp_ranks
|
||||
|
||||
|
||||
@dataclass
|
||||
class MLPSyncBatchInfo:
|
||||
dp_size: int
|
||||
num_dp_ranks: int
|
||||
tp_size: int
|
||||
cp_size: int
|
||||
|
||||
@@ -186,7 +186,7 @@ class MLPSyncBatchInfo:
|
||||
# is a no-op, and the masked fallback writes below would then read and
|
||||
# write the same storage.
|
||||
global_info_tensor = fallback_tensor.repeat(
|
||||
self.dp_size, self.tp_size * self.cp_size, 1
|
||||
self.num_dp_ranks, self.tp_size * self.cp_size, 1
|
||||
)
|
||||
|
||||
if use_all_reduce:
|
||||
@@ -211,7 +211,7 @@ class MLPSyncBatchInfo:
|
||||
)
|
||||
|
||||
tp_info = global_info_tensor.view(
|
||||
self.dp_size * self.tp_size * self.cp_size, info_width
|
||||
self.num_dp_ranks * self.tp_size * self.cp_size, info_width
|
||||
)
|
||||
num_ranks_in_tp_info = tp_info.shape[0]
|
||||
if device == "cpu":
|
||||
@@ -296,7 +296,7 @@ def _update_gather_batch(
|
||||
)
|
||||
|
||||
|
||||
def should_skip_scheduler_all_gather(dp_size: int) -> bool:
|
||||
def should_skip_scheduler_all_gather(num_dp_ranks: int) -> bool:
|
||||
"""Return whether scheduler metadata is already local and rank-invariant.
|
||||
|
||||
With one attention-DP rank there is no cross-DP state to reconcile. The
|
||||
@@ -307,7 +307,7 @@ def should_skip_scheduler_all_gather(dp_size: int) -> bool:
|
||||
DP1.
|
||||
"""
|
||||
|
||||
return dp_size == 1 or envs.SGLANG_SCHEDULER_SKIP_ALL_GATHER.get()
|
||||
return num_dp_ranks == 1 or envs.SGLANG_SCHEDULER_SKIP_ALL_GATHER.get()
|
||||
|
||||
|
||||
def _local_decode_cuda_graph_vote(
|
||||
@@ -421,7 +421,7 @@ def prepare_mlp_sync_batch_raw(
|
||||
dwdp: bool = False,
|
||||
):
|
||||
parallel = get_parallel()
|
||||
dp_size = parallel.dp_size
|
||||
num_dp_ranks = parallel.num_dp_ranks
|
||||
attn_tp_size = parallel.attn_tp_size
|
||||
tp_group = parallel.tp_group
|
||||
# Check if other DP workers have running batches
|
||||
@@ -498,16 +498,16 @@ def prepare_mlp_sync_batch_raw(
|
||||
|
||||
local_can_run_tbo, local_forward_mode = tbo_preparer.prepare_all_gather(local_batch)
|
||||
if use_world_group:
|
||||
dp_size = _resolve_elastic_world_dp_size(
|
||||
dp_size,
|
||||
num_dp_ranks = _resolve_elastic_world_num_dp_ranks(
|
||||
num_dp_ranks,
|
||||
group=group,
|
||||
local_num_tokens=num_tokens,
|
||||
local_forward_mode=local_forward_mode,
|
||||
)
|
||||
skip_all_gather = should_skip_scheduler_all_gather(dp_size)
|
||||
skip_all_gather = should_skip_scheduler_all_gather(num_dp_ranks)
|
||||
|
||||
mlp_sync_info = MLPSyncBatchInfo(
|
||||
dp_size=dp_size,
|
||||
num_dp_ranks=num_dp_ranks,
|
||||
tp_size=attn_tp_size,
|
||||
cp_size=parallel.attn_cp_size,
|
||||
num_tokens=num_tokens,
|
||||
@@ -521,7 +521,7 @@ def prepare_mlp_sync_batch_raw(
|
||||
prefill_cuda_graph_max_prefix_len=prefill_cuda_graph_max_prefix_len,
|
||||
)
|
||||
|
||||
if dp_size == 1:
|
||||
if num_dp_ranks == 1:
|
||||
mlp_sync_info.finalize_local()
|
||||
elif not skip_all_gather:
|
||||
mlp_sync_info.all_gather(
|
||||
@@ -540,7 +540,7 @@ def prepare_mlp_sync_batch_raw(
|
||||
# Decide whether to emit idle batch
|
||||
if skip_all_gather:
|
||||
# Skip idle batch when attn-dp=1 (and always under DWDP: ranks run independently)
|
||||
need_idle_batch = not dwdp and dp_size > 1
|
||||
need_idle_batch = not dwdp and num_dp_ranks > 1
|
||||
else:
|
||||
need_idle_batch = max(mlp_sync_info.global_num_tokens) > 0
|
||||
|
||||
|
||||
@@ -191,8 +191,7 @@ class DynamicChunkSizer:
|
||||
|
||||
if is_dp_attention_enabled():
|
||||
# Profiling runs one request on this rank; other DP ranks report 0.
|
||||
dp_size = get_parallel().attn_dp_size
|
||||
global_num_tokens = [0] * dp_size
|
||||
global_num_tokens = [0] * get_parallel().attn_dp_size
|
||||
dp_rank = get_parallel().attn_dp_rank
|
||||
global_num_tokens[dp_rank] = current_seq_len
|
||||
batch.global_num_tokens = global_num_tokens
|
||||
|
||||
@@ -19,7 +19,7 @@ an explicit address); off by default so an upgrade never reserves a port a
|
||||
co-hosted neighbor's KV publisher binds. The port comes from
|
||||
`resolve_load_pub_range` (the same function `/server_info` advertises with,
|
||||
so the two cannot drift). With `auto`, a worker's ZMQ footprint is
|
||||
`2 * dp_size` ports after its KV base (`2 * dp_size + 1` with the
|
||||
`2 * num_dp_ranks` ports after its KV base (`2 * num_dp_ranks + 1` with the
|
||||
conventional adjacent replay), so co-hosted workers must space their KV
|
||||
bases that far apart or move the range with an explicit address.
|
||||
"""
|
||||
@@ -164,7 +164,7 @@ class SchedulerLoadPublisher:
|
||||
resolved, reason = resolve_load_pub_range(
|
||||
kv_endpoint=cfg.endpoint,
|
||||
replay_endpoint=cfg.replay_endpoint,
|
||||
dp_size=get_parallel().dp_size,
|
||||
dp_size=get_parallel().num_dp_ranks,
|
||||
load_publish_endpoint=load_publish_endpoint,
|
||||
)
|
||||
if resolved is None:
|
||||
|
||||
@@ -285,7 +285,7 @@ class SchedulerProfilerManager:
|
||||
|
||||
if get_parallel().tp_rank != 0:
|
||||
return ""
|
||||
if get_parallel().dp_size > 1 and get_parallel().dp_rank != 0:
|
||||
if get_parallel().num_dp_ranks > 1 and get_parallel().dp_rank != 0:
|
||||
return ""
|
||||
if get_parallel().pp_size > 1 and get_parallel().pp_rank != 0:
|
||||
return ""
|
||||
@@ -340,7 +340,7 @@ class SchedulerProfilerManager:
|
||||
filename_parts = [self.profile_id, f"TP-{get_parallel().tp_rank}"]
|
||||
|
||||
# Only add other ranks if parallelism is enabled (size > 1)
|
||||
if get_parallel().dp_size > 1:
|
||||
if get_parallel().num_dp_ranks > 1:
|
||||
filename_parts.append(f"DP-{get_parallel().dp_rank}")
|
||||
if get_parallel().pp_size > 1:
|
||||
filename_parts.append(f"PP-{get_parallel().pp_rank}")
|
||||
|
||||
@@ -34,7 +34,7 @@ class SchedulerRecvSkipper:
|
||||
return ForwardMode.DECODE
|
||||
|
||||
def __init__(self):
|
||||
self._use_synced_mode = get_parallel().enable_dp_attention
|
||||
self._use_synced_mode = get_parallel().attn_dp_enabled
|
||||
self._counter = 0
|
||||
self._threshold = get_schedule().scheduler_recv_interval
|
||||
# All can be tuned if needed
|
||||
|
||||
@@ -173,7 +173,7 @@ class SchedulerRequestReceiver:
|
||||
rank 0 and would overwrite every DP group's aborts but the first.
|
||||
"""
|
||||
local_reqs = local_reqs or []
|
||||
if get_parallel().enable_dp_attention:
|
||||
if get_parallel().attn_dp_enabled:
|
||||
if get_parallel().attn_tp_rank == 0 and get_parallel().attn_cp_rank == 0:
|
||||
work_reqs, control_reqs = self._split_work_and_control_reqs(recv_reqs)
|
||||
work_reqs.extend(local_reqs)
|
||||
@@ -266,7 +266,7 @@ class SchedulerRequestReceiver:
|
||||
|
||||
# 1. wait until every rank has opened the shared feature segments
|
||||
parallel = get_parallel()
|
||||
if parallel.enable_dp_attention:
|
||||
if parallel.attn_dp_enabled:
|
||||
if parallel.attn_tp_size > 1:
|
||||
barrier(group=self.attn_tp_cpu_group)
|
||||
if parallel.attn_cp_size > 1:
|
||||
@@ -290,7 +290,7 @@ class SchedulerRequestReceiver:
|
||||
failed[index] = 1
|
||||
|
||||
# 3. all ranks reject the same requests before entering model collectives
|
||||
if parallel.enable_dp_attention:
|
||||
if parallel.attn_dp_enabled:
|
||||
if parallel.attn_tp_size > 1:
|
||||
all_reduce(failed, op=ReduceOp.MAX, group=self.attn_tp_cpu_group)
|
||||
if parallel.attn_cp_size > 1:
|
||||
|
||||
@@ -186,7 +186,7 @@ class TokenizerControlMixin:
|
||||
mode = spec[2] if len(spec) > 2 else "queueing"
|
||||
comm = FanOutCommunicator(
|
||||
self._dispatch_to_scheduler,
|
||||
get_parallel().dp_size,
|
||||
get_parallel().num_dp_ranks,
|
||||
mode,
|
||||
)
|
||||
setattr(self, f"{name}_communicator", comm)
|
||||
@@ -195,7 +195,7 @@ class TokenizerControlMixin:
|
||||
|
||||
def update_control_communicator_fan_out(self: TokenizerManager, worker_count: int):
|
||||
primary_group_control = (
|
||||
get_parallel().enable_dp_attention
|
||||
get_parallel().attn_dp_enabled
|
||||
and not get_parallel().enable_dp_attention_local_control_broadcast
|
||||
)
|
||||
if primary_group_control:
|
||||
@@ -444,8 +444,8 @@ class TokenizerControlMixin:
|
||||
request: Optional[fastapi.Request] = None,
|
||||
) -> Tuple[bool, str]:
|
||||
self.auto_create_handle_loop()
|
||||
assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, (
|
||||
"dp_size must be 1 or dp attention must be enabled for update weights from distributed"
|
||||
assert get_parallel().dp_size == 1, (
|
||||
"data-parallel replicas are not supported for update weights from distributed"
|
||||
)
|
||||
|
||||
results = await self.init_weights_update_group_communicator(obj)
|
||||
@@ -457,8 +457,8 @@ class TokenizerControlMixin:
|
||||
request: Optional[fastapi.Request] = None,
|
||||
) -> Tuple[bool, str]:
|
||||
self.auto_create_handle_loop()
|
||||
assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, (
|
||||
"dp_size must be 1 or dp attention must be enabled for destroy parameter update group"
|
||||
assert get_parallel().dp_size == 1, (
|
||||
"data-parallel replicas are not supported for destroy parameter update group"
|
||||
)
|
||||
|
||||
results = await self.destroy_weights_update_group_communicator(obj)
|
||||
@@ -502,8 +502,8 @@ class TokenizerControlMixin:
|
||||
request: Optional[fastapi.Request] = None,
|
||||
) -> Tuple[bool, str]:
|
||||
self.auto_create_handle_loop()
|
||||
assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, (
|
||||
"dp_size must be 1 or dp attention must be enabled for update weights from distributed"
|
||||
assert get_parallel().dp_size == 1, (
|
||||
"data-parallel replicas are not supported for update weights from distributed"
|
||||
)
|
||||
|
||||
if obj.abort_all_requests:
|
||||
@@ -535,8 +535,8 @@ class TokenizerControlMixin:
|
||||
) -> Tuple[bool, str]:
|
||||
self.auto_create_handle_loop()
|
||||
# TODO: support DP
|
||||
assert get_parallel().dp_size == 1, (
|
||||
"dp_size must be 1 for init_weights_send_group_for_remote_instance"
|
||||
assert get_parallel().num_dp_ranks == 1, (
|
||||
"init_weights_send_group_for_remote_instance requires a single DP rank"
|
||||
)
|
||||
result = (
|
||||
await self.init_weights_send_group_for_remote_instance_communicator(obj)
|
||||
@@ -550,8 +550,8 @@ class TokenizerControlMixin:
|
||||
) -> Tuple[bool, str]:
|
||||
self.auto_create_handle_loop()
|
||||
# TODO: support DP
|
||||
assert get_parallel().dp_size == 1, (
|
||||
"dp_size must be 1 for send_weights_to_remote_instance"
|
||||
assert get_parallel().num_dp_ranks == 1, (
|
||||
"send_weights_to_remote_instance requires a single DP rank"
|
||||
)
|
||||
result = (await self.send_weights_to_remote_instance_communicator(obj))[0]
|
||||
return result.success, result.message
|
||||
@@ -562,8 +562,8 @@ class TokenizerControlMixin:
|
||||
request: Optional[fastapi.Request] = None,
|
||||
) -> Tuple[bool, str]:
|
||||
self.auto_create_handle_loop()
|
||||
assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, (
|
||||
"dp_size must be 1 or dp attention must be enabled for update weights from tensor"
|
||||
assert get_parallel().dp_size == 1, (
|
||||
"data-parallel replicas are not supported for update weights from tensor"
|
||||
)
|
||||
|
||||
if obj.abort_all_requests:
|
||||
@@ -600,8 +600,8 @@ class TokenizerControlMixin:
|
||||
self.auto_create_handle_loop()
|
||||
try:
|
||||
# For now, we only support single data parallel instance
|
||||
assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, (
|
||||
"dp_size must be 1 or dp attention must be enabled for update weights from IPC"
|
||||
assert get_parallel().dp_size == 1, (
|
||||
"data-parallel replicas are not supported for update weights from IPC"
|
||||
)
|
||||
logger.info("Starting IPC weight update")
|
||||
|
||||
@@ -665,8 +665,8 @@ class TokenizerControlMixin:
|
||||
"LoRA is not enabled. Please set `--enable-lora` to enable LoRA."
|
||||
)
|
||||
|
||||
assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, (
|
||||
"dp_size must be 1 or dp attention must be enabled for dynamic lora loading"
|
||||
assert get_parallel().dp_size == 1, (
|
||||
"data-parallel replicas are not supported for dynamic lora loading"
|
||||
)
|
||||
logger.info(
|
||||
"Start load Lora adapter. Lora name=%s, path=%s",
|
||||
@@ -751,8 +751,8 @@ class TokenizerControlMixin:
|
||||
"LoRA is not enabled. Please set `--enable-lora` to enable LoRA."
|
||||
)
|
||||
|
||||
assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, (
|
||||
"dp_size must be 1 or dp attention must be enabled for dynamic lora loading"
|
||||
assert get_parallel().dp_size == 1, (
|
||||
"data-parallel replicas are not supported for dynamic lora loading"
|
||||
)
|
||||
logger.info(
|
||||
"Start load Lora adapter from tensors. Lora name=%s",
|
||||
@@ -841,8 +841,8 @@ class TokenizerControlMixin:
|
||||
"lora_name must be provided to unload LoRA adapter"
|
||||
)
|
||||
|
||||
assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, (
|
||||
"dp_size must be 1 or dp attention must be enabled for dynamic lora loading"
|
||||
assert get_parallel().dp_size == 1, (
|
||||
"data-parallel replicas are not supported for dynamic lora loading"
|
||||
)
|
||||
logger.info(
|
||||
"Start unload Lora adapter. Lora name=%s",
|
||||
@@ -862,7 +862,7 @@ class TokenizerControlMixin:
|
||||
self.auto_create_handle_loop()
|
||||
results = await self.get_weights_by_name_communicator(obj)
|
||||
all_parameters = [r.parameter for r in results]
|
||||
if get_parallel().dp_size == 1:
|
||||
if get_parallel().num_dp_ranks == 1:
|
||||
return all_parameters[0]
|
||||
else:
|
||||
return all_parameters
|
||||
|
||||
@@ -488,7 +488,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
self.server_args = server_args
|
||||
assert_published(server_args, role="tokenizer")
|
||||
self.startup_time: Optional[Dict[str, Any]] = None
|
||||
self.elastic_worker_count = get_parallel().dp_size
|
||||
self.elastic_worker_count = get_parallel().num_dp_ranks
|
||||
self.elastic_pending_ep_size = None
|
||||
self.elastic_scale_phase = "idle"
|
||||
self.elastic_last_error = None
|
||||
@@ -872,14 +872,14 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
)
|
||||
|
||||
if isinstance(obj, GenerateReqInput) and obj.routed_dp_rank is not None:
|
||||
dp_size = self.elastic_worker_count
|
||||
if dp_size <= 1 and obj.routed_dp_rank == 0:
|
||||
num_dp_ranks = self.elastic_worker_count
|
||||
if num_dp_ranks <= 1 and obj.routed_dp_rank == 0:
|
||||
logger.debug(
|
||||
f"routed_dp_rank={obj.routed_dp_rank} is ignored because dp_size={dp_size}"
|
||||
f"routed_dp_rank={obj.routed_dp_rank} is ignored because num_dp_ranks={num_dp_ranks}"
|
||||
)
|
||||
elif obj.routed_dp_rank < 0 or obj.routed_dp_rank >= dp_size:
|
||||
elif obj.routed_dp_rank < 0 or obj.routed_dp_rank >= num_dp_ranks:
|
||||
raise ValueError(
|
||||
f"routed_dp_rank={obj.routed_dp_rank} out of range [0, {dp_size})"
|
||||
f"routed_dp_rank={obj.routed_dp_rank} out of range [0, {num_dp_ranks})"
|
||||
)
|
||||
|
||||
self._init_req_state(obj, request)
|
||||
@@ -1670,7 +1670,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
return batch_size > 0 and (
|
||||
get_serving().enable_tokenizer_batch_encode
|
||||
or (
|
||||
(not get_parallel().enable_dp_attention)
|
||||
(not get_parallel().attn_dp_enabled)
|
||||
and (not self._batch_has_text(batch_size, requests))
|
||||
)
|
||||
)
|
||||
|
||||
@@ -359,7 +359,7 @@ def build_kv_cache(
|
||||
),
|
||||
is_eagle=spec_algorithm.is_eagle(),
|
||||
tp_cache_group=(
|
||||
attn_tp_cpu_group if get_parallel().enable_dp_attention else tp_cpu_group
|
||||
attn_tp_cpu_group if get_parallel().attn_dp_enabled else tp_cpu_group
|
||||
),
|
||||
attn_cp_cache_group=attn_cp_cpu_group,
|
||||
attn_tp_cache_group=attn_tp_cpu_group,
|
||||
|
||||
@@ -299,7 +299,7 @@ Supported:
|
||||
`TransferManagerOnRemote` via the same ZMQ channel used for GPU
|
||||
registration.
|
||||
* **CP** (`attn_cp_size > 1`) — sync handled symmetrically with TP.
|
||||
* **DP attention** (`enable_dp_attention=True`) — the inner
|
||||
* **DP attention** (`--attn-dp-size` > 1) — the inner
|
||||
`attn_tp_size` is what FlexKV uses for register-side routing.
|
||||
|
||||
---
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
The public surface is small (see "Public API" below). The class owns:
|
||||
|
||||
* the FlexKV ``KVManager`` (server-client mode when ``dp_size > 1`` or
|
||||
* the FlexKV ``KVManager`` (server-client mode when ``num_dp_ranks > 1`` or
|
||||
multi-instance; in-process otherwise — handled by FlexKV itself);
|
||||
* the per-rank ``KVTPClient`` that registers this rank's GPU KV cache
|
||||
with the FlexKV TransferManager;
|
||||
|
||||
@@ -619,7 +619,7 @@ class CPUGraphRunner:
|
||||
self.speculative_algorithm = get_spec().speculative_algorithm
|
||||
self.enable_profile_cuda_graph = get_exec().graph.enable_profile_cuda_graph
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.dp_size = get_parallel().dp_size
|
||||
self.num_dp_ranks = get_parallel().num_dp_ranks
|
||||
self.pp_size = get_parallel().pp_size
|
||||
|
||||
self.capture_forward_mode = ForwardMode.DECODE
|
||||
@@ -644,7 +644,7 @@ class CPUGraphRunner:
|
||||
"CPUGraphRunner does not support speculative inference yet."
|
||||
)
|
||||
|
||||
assert self.dp_size == 1, "CPUGraphRunner does not support DP yet."
|
||||
assert self.num_dp_ranks == 1, "CPUGraphRunner does not support DP yet."
|
||||
assert self.pp_size == 1, "CPUGraphRunner does not support PP yet."
|
||||
|
||||
# Batch sizes to capture
|
||||
|
||||
@@ -530,7 +530,7 @@ def build_decode_registry(
|
||||
require_mlp_tp_gather: bool = False,
|
||||
# Per-bucket attn-TP sharded (SP) predicate; defaults to replicated.
|
||||
attn_tp_sharded_fn: Callable[[int], bool] = lambda num_tokens: False,
|
||||
dp_size: int = 1,
|
||||
num_dp_ranks: int = 1,
|
||||
register_global_num_tokens: bool = True,
|
||||
share_pool: bool = True,
|
||||
source: Optional[Any] = None,
|
||||
@@ -700,7 +700,7 @@ def build_decode_registry(
|
||||
buf.fill_(ctx.padded_num_tokens)
|
||||
|
||||
_global_shape = (
|
||||
(lambda _bs, _mt: (dp_size,))
|
||||
(lambda _bs, _mt: (num_dp_ranks,))
|
||||
if require_mlp_tp_gather
|
||||
else (lambda _bs, _mt: (1,))
|
||||
)
|
||||
@@ -1048,7 +1048,7 @@ def build_eager_registry(
|
||||
register_global_num_tokens=False,
|
||||
require_gathered_buffer=False,
|
||||
require_mlp_tp_gather=False,
|
||||
dp_size=get_parallel().dp_size,
|
||||
num_dp_ranks=get_parallel().num_dp_ranks,
|
||||
share_pool=True,
|
||||
source=None,
|
||||
)
|
||||
|
||||
@@ -1186,7 +1186,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
# Init lora information
|
||||
if (
|
||||
model_runner.lora_manager is not None
|
||||
and not model_runner.lora_manager.enable_dp_attention
|
||||
and not model_runner.lora_manager.attn_dp_enabled
|
||||
):
|
||||
# In the non-LoRA overlap loading case, we fetch LoRA adapters into the memory pool
|
||||
# as a batch, right before running the batch
|
||||
|
||||
@@ -530,7 +530,7 @@ class ModelRunner:
|
||||
new_dp_size=join_effective_ep_size,
|
||||
new_dp_rank=global_ep_rank,
|
||||
)
|
||||
get_context().override("elastic_ep.scale_join", dp_size=join_effective_ep_size)
|
||||
get_parallel().override_permanently(num_dp_ranks=join_effective_ep_size)
|
||||
if self.eplb_manager is not None:
|
||||
self.eplb_manager.disable_rebalance(
|
||||
"EPLB rebalance is disabled while elastic EP scale-up "
|
||||
@@ -965,7 +965,7 @@ class ModelRunner:
|
||||
device=self.device,
|
||||
tp_group=(
|
||||
get_parallel().attn_tp_group.cpu_group
|
||||
if get_parallel().enable_dp_attention
|
||||
if get_parallel().attn_dp_enabled
|
||||
else get_parallel().tp_group.cpu_group
|
||||
),
|
||||
host_to_device_ratio=hisparse_cfg.host_to_device_ratio,
|
||||
@@ -1628,7 +1628,7 @@ class ModelRunner:
|
||||
forward_batch.prepare_mlp_sync_batch(self)
|
||||
else:
|
||||
forward_batch.prepare_attn_tp_scatter_input(self)
|
||||
if self.lora_manager is not None and self.lora_manager.enable_dp_attention:
|
||||
if self.lora_manager is not None and self.lora_manager.attn_dp_enabled:
|
||||
self.lora_manager.prepare_lora_batch(forward_batch)
|
||||
|
||||
# Derive the LOCAL num_token_non_padded from the GLOBAL scalar. sharded is
|
||||
@@ -2217,7 +2217,7 @@ class ModelRunner:
|
||||
new_dp_size=target_size,
|
||||
new_dp_rank=self._elastic_global_rank(),
|
||||
)
|
||||
get_context().override("elastic_ep.scale", dp_size=target_size)
|
||||
get_parallel().override_permanently(num_dp_ranks=target_size)
|
||||
|
||||
recapture_cuda_graph = self._elastic_cuda_graph_enabled()
|
||||
if recapture_cuda_graph:
|
||||
|
||||
@@ -82,7 +82,7 @@ class StartupWeightLoadOptions:
|
||||
attn_cp_size: int
|
||||
dcp_size: int
|
||||
pp_size: int
|
||||
dp_size: int
|
||||
num_dp_ranks: int
|
||||
ep_size: int
|
||||
cpu_offload_gb: int
|
||||
offload_group_size: int
|
||||
@@ -123,7 +123,7 @@ class StartupWeightLoadOptions:
|
||||
attn_cp_size=get_parallel().attn_cp_size,
|
||||
dcp_size=get_parallel().dcp_size,
|
||||
pp_size=get_parallel().pp_size,
|
||||
dp_size=get_parallel().dp_size,
|
||||
num_dp_ranks=get_parallel().num_dp_ranks,
|
||||
ep_size=get_parallel().ep_size,
|
||||
cpu_offload_gb=get_exec().offload.cpu_offload_gb,
|
||||
offload_group_size=get_exec().offload.offload_group_size,
|
||||
@@ -369,7 +369,7 @@ class StartupWeightLoadManager:
|
||||
"decode context parallelism is not supported",
|
||||
),
|
||||
(options.pp_size != 1, "pipeline parallelism is not supported"),
|
||||
(options.dp_size != 1, "data parallelism is not supported"),
|
||||
(options.num_dp_ranks != 1, "data parallelism is not supported"),
|
||||
(options.ep_size != 1, "expert parallelism is not supported"),
|
||||
(options.cpu_offload_gb > 0, "CPU offload is not supported"),
|
||||
(
|
||||
|
||||
@@ -162,9 +162,11 @@ def _allocate_decode_buffers(
|
||||
encoder_lens = None
|
||||
|
||||
if require_mlp_tp_gather:
|
||||
global_num_tokens_gpu = torch.zeros((parallel.dp_size,), dtype=torch.int32)
|
||||
global_num_tokens_gpu = torch.zeros(
|
||||
(parallel.num_dp_ranks,), dtype=torch.int32
|
||||
)
|
||||
global_num_tokens_for_logprob_gpu = torch.zeros(
|
||||
(parallel.dp_size,), dtype=torch.int32
|
||||
(parallel.num_dp_ranks,), dtype=torch.int32
|
||||
)
|
||||
else:
|
||||
global_num_tokens_gpu = torch.zeros((1,), dtype=torch.int32)
|
||||
@@ -226,8 +228,8 @@ class BaseRunner(ABC):
|
||||
self.model_runner = model_runner
|
||||
self.device = model_runner.device
|
||||
self.device_module = torch.get_device_module(self.device)
|
||||
# elastic-EP scale-up rewrites dp_size on the published config
|
||||
self.dp_size = get_parallel().dp_size
|
||||
# elastic-EP scale-up widens num_dp_ranks on the published config
|
||||
self.num_dp_ranks = get_parallel().num_dp_ranks
|
||||
self.pp_size = get_parallel().pp_size
|
||||
self.enable_pdmux = get_disagg().enable_pdmux
|
||||
self.return_hidden_states_mode = (
|
||||
@@ -613,7 +615,7 @@ class BaseRunner(ABC):
|
||||
assert require_mlp_tp_gather_ or require_attn_tp_gather_
|
||||
|
||||
if require_mlp_tp_gather_:
|
||||
global_num_tokens_cpu = [num_tokens] * get_parallel().dp_size
|
||||
global_num_tokens_cpu = [num_tokens] * get_parallel().num_dp_ranks
|
||||
elif require_attn_tp_gather_:
|
||||
global_num_tokens_cpu = [num_tokens]
|
||||
else:
|
||||
|
||||
@@ -260,7 +260,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
self.require_mlp_tp_gather or self.require_attn_tp_gather
|
||||
)
|
||||
self.require_mlp_sync = (
|
||||
get_parallel().enable_dp_attention or self.require_gathered_buffer
|
||||
get_parallel().attn_dp_enabled or self.require_gathered_buffer
|
||||
)
|
||||
self.enable_two_batch_overlap = get_exec().overlap.enable_two_batch_overlap
|
||||
self.use_ngram_embedding = model_runner.ngram_embedding_manager.enabled
|
||||
@@ -414,7 +414,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
self.model_runner.model_config.vocab_size, rows=logits_buffer_rows
|
||||
),
|
||||
dtype=self.model_runner.model_config.dtype,
|
||||
dp_size=self.dp_size,
|
||||
num_dp_ranks=self.num_dp_ranks,
|
||||
pp_size=self.pp_size,
|
||||
is_encoder_decoder=self.is_encoder_decoder,
|
||||
require_mlp_tp_gather=self.require_mlp_tp_gather,
|
||||
@@ -458,7 +458,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
enable_prefill_cp=self.enable_prefill_cp,
|
||||
require_mlp_tp_gather=self.require_mlp_tp_gather,
|
||||
attn_tp_sharded_fn=self.model_runner.attn_tp_sequence_sharded,
|
||||
dp_size=self.dp_size,
|
||||
num_dp_ranks=self.num_dp_ranks,
|
||||
source=self.buffers,
|
||||
)
|
||||
|
||||
@@ -584,7 +584,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
|
||||
def _global_num_tokens_for_graph(self, num_tokens: int) -> Optional[list[int]]:
|
||||
if self.require_mlp_tp_gather:
|
||||
return [num_tokens] * self.dp_size
|
||||
return [num_tokens] * self.num_dp_ranks
|
||||
if self.require_attn_tp_gather:
|
||||
return [num_tokens]
|
||||
return None
|
||||
@@ -1427,7 +1427,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
)
|
||||
if (
|
||||
self.model_runner.lora_manager is not None
|
||||
and self.model_runner.lora_manager.enable_dp_attention
|
||||
and self.model_runner.lora_manager.attn_dp_enabled
|
||||
):
|
||||
self.model_runner.lora_manager.prepare_lora_batch(
|
||||
cast(ForwardBatch, fb_view)
|
||||
|
||||
@@ -311,7 +311,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
self.prefill_backend_name = prefill_config.backend
|
||||
self.prefer_eager_mixed_prefill = (
|
||||
self.prefill_backend_name == Backend.BREAKABLE
|
||||
and get_parallel().enable_dp_attention
|
||||
and get_parallel().attn_dp_enabled
|
||||
and getattr(
|
||||
model_runner.attn_backend,
|
||||
"prefer_eager_mixed_prefill_under_dp_attention",
|
||||
@@ -1432,7 +1432,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
return registry.get_slot(name).slice_for(bs, num_tokens)
|
||||
|
||||
if self.require_mlp_tp_gather:
|
||||
global_num_tokens_cpu = [num_tokens] * self.dp_size
|
||||
global_num_tokens_cpu = [num_tokens] * self.num_dp_ranks
|
||||
elif self.require_attn_tp_gather:
|
||||
global_num_tokens_cpu = [num_tokens]
|
||||
else:
|
||||
|
||||
@@ -136,7 +136,7 @@ class DecodeInputBuffers(ForwardInputBuffers):
|
||||
hidden_size: int,
|
||||
next_token_logits_buffer: torch.Tensor,
|
||||
dtype: torch.dtype,
|
||||
dp_size: int,
|
||||
num_dp_ranks: int,
|
||||
pp_size: int,
|
||||
is_encoder_decoder: bool,
|
||||
require_mlp_tp_gather: bool,
|
||||
@@ -207,9 +207,9 @@ class DecodeInputBuffers(ForwardInputBuffers):
|
||||
encoder_lens = None
|
||||
|
||||
if require_mlp_tp_gather:
|
||||
global_num_tokens_gpu = torch.zeros((dp_size,), dtype=torch.int32)
|
||||
global_num_tokens_gpu = torch.zeros((num_dp_ranks,), dtype=torch.int32)
|
||||
global_num_tokens_for_logprob_gpu = torch.zeros(
|
||||
(dp_size,), dtype=torch.int32
|
||||
(num_dp_ranks,), dtype=torch.int32
|
||||
)
|
||||
else:
|
||||
global_num_tokens_gpu = torch.zeros((1,), dtype=torch.int32)
|
||||
|
||||
@@ -3896,9 +3896,9 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
# Applies to BOTH prefill and decode: the shared expert is a per-token MLP,
|
||||
# so computing it on this rank's local tokens (M_local rows) is identical to
|
||||
# computing it on the gathered global buffer (M_global rows) and keeping the
|
||||
# local slice -- but costs 1/dp_size the rows. With a replicated (TP1) shared
|
||||
# local slice -- but costs 1/attn_dp_size the rows. With a replicated (TP1) shared
|
||||
# expert this cancels the TP1 "full-dim" cost in decode (M_local * dim ==
|
||||
# M_global * dim/tp), so decode no longer pays the ~dp_size x penalty.
|
||||
# M_global * dim/tp), so decode no longer pays the ~attn_dp_size x penalty.
|
||||
_shared_local = None
|
||||
_do_shared_local = (
|
||||
_SHARED_EXPERT_LOCAL
|
||||
|
||||
@@ -842,7 +842,7 @@ class ZayaAttention(nn.Module):
|
||||
assert attn_tp_size == self.tp_size, (
|
||||
f"ZAYA1 head-parallel attention requires the attention TP group "
|
||||
f"({attn_tp_size}) to equal the global TP group ({self.tp_size}); "
|
||||
"DP attention (enable_dp_attention) is not supported for ZAYA1."
|
||||
"attention DP is not supported for ZAYA1."
|
||||
)
|
||||
assert self.num_q_heads_full % self.tp_size == 0, (
|
||||
f"num_attention_heads ({self.num_q_heads_full}) must be divisible "
|
||||
|
||||
@@ -77,7 +77,7 @@ class RayDataParallelController(DataParallelController):
|
||||
sockets = []
|
||||
dp_port_args_list = []
|
||||
|
||||
for dp_rank in range(get_parallel().dp_size):
|
||||
for dp_rank in range(get_parallel().num_dp_ranks):
|
||||
tmp_port_args = PortArgs.init_new(server_args)
|
||||
tmp_port_args.tokenizer_ipc_name = port_args.tokenizer_ipc_name
|
||||
tmp_port_args.detokenizer_ipc_name = port_args.detokenizer_ipc_name
|
||||
@@ -101,7 +101,7 @@ class RayDataParallelController(DataParallelController):
|
||||
sock.close()
|
||||
|
||||
# Create actors for each DP rank sequentially
|
||||
for dp_rank in range(get_parallel().dp_size):
|
||||
for dp_rank in range(get_parallel().num_dp_ranks):
|
||||
self._launch_ray_tp_group(server_args, dp_port_args_list[dp_rank], dp_rank)
|
||||
|
||||
def launch_dp_attention_schedulers(
|
||||
@@ -112,7 +112,7 @@ class RayDataParallelController(DataParallelController):
|
||||
# rank-0 node IP instead of tcp://* to avoid exposing unauthenticated
|
||||
# ZMQ sockets (CVE-2026-3060).
|
||||
worker_ports = []
|
||||
for dp_rank in range(get_parallel().dp_size):
|
||||
for dp_rank in range(get_parallel().num_dp_ranks):
|
||||
worker_port, worker_socket = get_zmq_socket_on_host(
|
||||
self.context, zmq.PUSH, host=self.rank0_node_ip
|
||||
)
|
||||
@@ -158,12 +158,11 @@ class RayDataParallelController(DataParallelController):
|
||||
)
|
||||
|
||||
parallel = get_parallel()
|
||||
if parallel.enable_dp_attention:
|
||||
if parallel.attn_dp_enabled:
|
||||
_, _, actual_dp_rank, _ = compute_dp_attention_world_info(
|
||||
parallel.enable_dp_attention,
|
||||
tp_rank,
|
||||
parallel.tp_size,
|
||||
parallel.dp_size,
|
||||
parallel.attn_dp_size,
|
||||
parallel.attn_cp_size,
|
||||
)
|
||||
rank_port_args = PortArgs.init_new(
|
||||
@@ -230,12 +229,11 @@ class RayDataParallelController(DataParallelController):
|
||||
|
||||
bundle_idx = bundle_indices[global_rank]
|
||||
|
||||
if get_parallel().enable_dp_attention:
|
||||
if get_parallel().attn_dp_enabled:
|
||||
_, _, actual_dp_rank, _ = compute_dp_attention_world_info(
|
||||
get_parallel().enable_dp_attention,
|
||||
tp_rank,
|
||||
get_parallel().tp_size,
|
||||
get_parallel().dp_size,
|
||||
get_parallel().attn_dp_size,
|
||||
get_parallel().attn_cp_size,
|
||||
)
|
||||
rank_port_args = PortArgs.init_new(
|
||||
|
||||
@@ -112,7 +112,6 @@ def _compute_world_size() -> int:
|
||||
will hold the process groups, so there is nothing live to ask.
|
||||
"""
|
||||
return compute_world_size(
|
||||
enable_dp_attention=get_parallel().enable_dp_attention,
|
||||
dp_size=get_parallel().dp_size,
|
||||
tp_size=get_parallel().tp_size,
|
||||
pp_size=get_parallel().pp_size,
|
||||
@@ -274,10 +273,10 @@ class RayEngine(Engine):
|
||||
)
|
||||
|
||||
parallel = get_parallel()
|
||||
if parallel.enable_dp_attention:
|
||||
if parallel.attn_dp_enabled:
|
||||
total_gpus = parallel.tp_size * parallel.pp_size
|
||||
else:
|
||||
total_gpus = parallel.dp_size * parallel.tp_size * parallel.pp_size
|
||||
total_gpus = parallel.num_dp_ranks * parallel.tp_size * parallel.pp_size
|
||||
|
||||
nnodes = parallel.nnodes
|
||||
gpus_per_node = total_gpus // nnodes
|
||||
@@ -318,7 +317,7 @@ class RayEngine(Engine):
|
||||
rank0_bundle_idx = int(indices_str.split(",")[0]) if indices_str else 0
|
||||
rank0_node_ip = _get_bundle_node_ip(pg, rank0_bundle_idx)
|
||||
|
||||
if get_parallel().dp_size == 1:
|
||||
if get_parallel().num_dp_ranks == 1:
|
||||
dist_init_addr = f"{rank0_node_ip}:{port_args.nccl_port}"
|
||||
logger.info(f"dist_init_addr: {dist_init_addr}")
|
||||
|
||||
@@ -448,17 +447,17 @@ class RayEngine(Engine):
|
||||
)
|
||||
|
||||
parallel = get_parallel()
|
||||
if parallel.enable_dp_attention:
|
||||
if parallel.attn_dp_enabled:
|
||||
# DP attention folds DP into TP — total GPUs = tp_size * pp_size
|
||||
total_gpus = parallel.tp_size * parallel.pp_size
|
||||
else:
|
||||
total_gpus = parallel.dp_size * parallel.tp_size * parallel.pp_size
|
||||
total_gpus = parallel.num_dp_ranks * parallel.tp_size * parallel.pp_size
|
||||
gpus_per_node = total_gpus // parallel.nnodes
|
||||
logger.info(
|
||||
f"Ray DP cluster: {parallel.nnodes} nodes, "
|
||||
f"{gpus_per_node} GPUs/node, dp_size={parallel.dp_size}, "
|
||||
f"tp_size={parallel.tp_size}, pp_size={parallel.pp_size}, "
|
||||
f"enable_dp_attention={parallel.enable_dp_attention}"
|
||||
f"attn_dp_size={parallel.attn_dp_size}, "
|
||||
f"tp_size={parallel.tp_size}, pp_size={parallel.pp_size}"
|
||||
)
|
||||
|
||||
# Declared on the record itself so `PortArgs.init_new()` can compute
|
||||
|
||||
@@ -70,16 +70,14 @@ def _parallel_fields() -> frozenset:
|
||||
return frozenset(_parallel_config_leaves() | derived)
|
||||
|
||||
|
||||
def derive_attention_widths(
|
||||
*, tp_size: int, attn_cp_size: int, dp_size: int, enable_dp_attention: bool
|
||||
) -> tuple:
|
||||
"""Return (attn_dp_size, attn_tp_size) from the configured widths."""
|
||||
attn_dp_size = dp_size if enable_dp_attention else 1
|
||||
return attn_dp_size, tp_size // attn_dp_size // attn_cp_size
|
||||
def derive_attn_tp_size(*, tp_size: int, attn_cp_size: int, attn_dp_size: int) -> int:
|
||||
"""Return the attention-TP width: what is left of the TP group after the
|
||||
attention-DP and attention-CP dimensions."""
|
||||
return tp_size // attn_dp_size // attn_cp_size
|
||||
|
||||
|
||||
def derive_attention_ranks(
|
||||
*, tp_rank: int, attn_tp_size: int, attn_cp_size: int, enable_dp_attention: bool
|
||||
*, tp_rank: int, attn_tp_size: int, attn_cp_size: int
|
||||
) -> tuple:
|
||||
"""Return (attn_tp_rank, attn_dp_rank) for a process at ``tp_rank``.
|
||||
|
||||
@@ -87,11 +85,11 @@ def derive_attention_ranks(
|
||||
|
||||
tp_rank = (attn_dp_rank * attn_cp_size + attn_cp_rank) * attn_tp_size
|
||||
+ attn_tp_rank
|
||||
|
||||
Without attention DP the TP group is one attention-DP group, so
|
||||
``attn_dp_rank`` is zero.
|
||||
"""
|
||||
attn_tp_rank = tp_rank % attn_tp_size
|
||||
if not enable_dp_attention:
|
||||
return attn_tp_rank, 0
|
||||
return attn_tp_rank, tp_rank // (attn_tp_size * attn_cp_size)
|
||||
return tp_rank % attn_tp_size, tp_rank // (attn_tp_size * attn_cp_size)
|
||||
|
||||
|
||||
def spawn_world_rank(server_args, *, tp_rank: int, pp_rank: int) -> int:
|
||||
@@ -149,13 +147,9 @@ def derive_parallel_widths(
|
||||
"""Derive attention and MoE widths and DCP settings from configuration."""
|
||||
return {
|
||||
"attn_dp_size": attn_dp_size,
|
||||
# `attn_dp_size` already accounts for disabled DP attention.
|
||||
"attn_tp_size": derive_attention_widths(
|
||||
tp_size=tp_size,
|
||||
attn_cp_size=attn_cp_size,
|
||||
dp_size=attn_dp_size,
|
||||
enable_dp_attention=True,
|
||||
)[1],
|
||||
"attn_tp_size": derive_attn_tp_size(
|
||||
tp_size=tp_size, attn_cp_size=attn_cp_size, attn_dp_size=attn_dp_size
|
||||
),
|
||||
"moe_ep_size": moe_ep_size,
|
||||
"moe_tp_size": tp_size // moe_ep_size // moe_dp_size,
|
||||
"dcp_enabled": dcp_enabled,
|
||||
@@ -165,16 +159,10 @@ def derive_parallel_widths(
|
||||
|
||||
def parallel_widths_of(cfg: Any) -> dict:
|
||||
"""Return derived parallel settings from resolved configuration."""
|
||||
attn_dp_size, _ = derive_attention_widths(
|
||||
tp_size=cfg.tp_size,
|
||||
attn_cp_size=cfg.attn_cp_size,
|
||||
dp_size=cfg.dp_size,
|
||||
enable_dp_attention=cfg.enable_dp_attention,
|
||||
)
|
||||
return derive_parallel_widths(
|
||||
tp_size=cfg.tp_size,
|
||||
attn_cp_size=cfg.attn_cp_size,
|
||||
attn_dp_size=attn_dp_size,
|
||||
attn_dp_size=cfg.attn_dp_size,
|
||||
moe_ep_size=cfg.ep_size,
|
||||
moe_dp_size=cfg.moe_dp_size,
|
||||
dcp_size=cfg.dcp_size,
|
||||
@@ -200,9 +188,23 @@ def attn_tp_size_of(cfg: Any):
|
||||
return parallel_widths_of(cfg)["attn_tp_size"]
|
||||
|
||||
|
||||
def attn_dp_size_of(cfg: Any):
|
||||
"""`attn_dp_size`, computed at publish. See `parallel_widths_of`."""
|
||||
return parallel_widths_of(cfg)["attn_dp_size"]
|
||||
def attn_dp_enabled_of(cfg: Any) -> bool:
|
||||
"""`attn_dp_enabled`, computed at publish.
|
||||
|
||||
A scale joiner's own group can be one rank wide, but it joins an
|
||||
attention-DP deployment and runs its DP-attention paths.
|
||||
"""
|
||||
from sglang.srt.arg_groups.model_override_base import ep_scale_joiner_of
|
||||
|
||||
return cfg.attn_dp_size > 1 or ep_scale_joiner_of(cfg)
|
||||
|
||||
|
||||
def num_dp_ranks_of(cfg: Any) -> int:
|
||||
"""`num_dp_ranks`, computed at publish: replicas times attention-DP groups.
|
||||
|
||||
DP attention does not run inside replicas, so one of the two is one.
|
||||
"""
|
||||
return cfg.dp_size * cfg.attn_dp_size
|
||||
|
||||
|
||||
def attn_dcp_size_of(cfg: Any):
|
||||
@@ -1560,7 +1562,6 @@ def _attention_ranks(parallel, tp_rank: int) -> dict:
|
||||
tp_rank=tp_rank,
|
||||
attn_tp_size=parallel.attn_tp_size,
|
||||
attn_cp_size=parallel.attn_cp_size,
|
||||
enable_dp_attention=parallel.enable_dp_attention,
|
||||
)
|
||||
return {"attn_tp_rank": attn_tp_rank, "attn_dp_rank": attn_dp_rank}
|
||||
|
||||
@@ -2119,7 +2120,8 @@ def describe_kv_events_publisher(server_args: Any) -> Optional[dict]:
|
||||
"endpoint_port_base": port,
|
||||
"topic": cfg.topic,
|
||||
"block_size": kv_event_block_size_of(resolved),
|
||||
"dp_size": resolved.dp_size,
|
||||
# The wire key predates attention DP; it counts DP ranks.
|
||||
"dp_size": num_dp_ranks_of(resolved),
|
||||
}
|
||||
# Load range, from the same resolver SchedulerLoadPublisher binds
|
||||
# with (so the two can't drift). The decline reason is logged once at
|
||||
@@ -2127,7 +2129,7 @@ def describe_kv_events_publisher(server_args: Any) -> Optional[dict]:
|
||||
resolved_range, _reason = resolve_load_pub_range(
|
||||
kv_endpoint=cfg.endpoint,
|
||||
replay_endpoint=cfg.replay_endpoint,
|
||||
dp_size=resolved.dp_size,
|
||||
dp_size=num_dp_ranks_of(resolved),
|
||||
load_publish_endpoint=resolved.load_publish_endpoint,
|
||||
)
|
||||
if resolved_range is not None:
|
||||
|
||||
@@ -120,7 +120,9 @@ class RustServer:
|
||||
os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
|
||||
|
||||
# Preserve the DP startup log; ports use node-local offsets.
|
||||
dp_rank = get_parallel().attn_dp_rank if get_parallel().dp_size > 1 else None
|
||||
dp_rank = (
|
||||
get_parallel().attn_dp_rank if get_parallel().num_dp_ranks > 1 else None
|
||||
)
|
||||
if get_exec().moe.is_ep_scale_joiner:
|
||||
# The joining TP group is entirely local to this node.
|
||||
tp_size_per_node = get_parallel().tp_size
|
||||
@@ -180,7 +182,9 @@ class RustServer:
|
||||
# Under DP every rank runs its own server on its own port, so the rank is
|
||||
# what tells two otherwise identical startup lines apart.
|
||||
dp_note = (
|
||||
"" if dp_rank is None else f" (DP rank {dp_rank}/{get_parallel().dp_size})"
|
||||
""
|
||||
if dp_rank is None
|
||||
else f" (DP rank {dp_rank}/{get_parallel().num_dp_ranks})"
|
||||
)
|
||||
logger.info(
|
||||
"SGLANG_RUST_SERVER enabled, Rust server listen on %s%s",
|
||||
|
||||
@@ -43,7 +43,12 @@ from sglang.srt.arg_groups.overrides import (
|
||||
resolving_view,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.runtime_context import get_platform, publish
|
||||
from sglang.srt.runtime_context import (
|
||||
attn_dp_enabled_of,
|
||||
get_platform,
|
||||
num_dp_ranks_of,
|
||||
publish,
|
||||
)
|
||||
from sglang.srt.speculative.decoupled_spec_io import DecoupledSpecIpcConfig
|
||||
from sglang.srt.utils.network import NetworkAddress, get_free_port, wait_port_available
|
||||
|
||||
@@ -345,6 +350,15 @@ class ServerArgs:
|
||||
new_flag="--cuda-graph-backend-{decode,prefill}=disabled",
|
||||
help="Deprecated. Use --cuda-graph-backend-{decode,prefill}=disabled instead.",
|
||||
)
|
||||
# `enable_dp_attention` is `no_cli=True` too; resolution turns it into
|
||||
# `attn_dp_size`.
|
||||
parser.add_argument(
|
||||
"--enable-dp-attention",
|
||||
action=DeprecatedStoreTrueAction,
|
||||
new_flag="--attn-dp-size <dp-size>",
|
||||
help="Deprecated. Use --attn-dp-size <dp-size> instead of "
|
||||
"--dp-size <dp-size> --enable-dp-attention.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--enable-flashinfer-allreduce-fusion",
|
||||
action="store_true",
|
||||
@@ -488,16 +502,15 @@ def resolve_encoder_transfer_backend(
|
||||
return "zmq_to_scheduler"
|
||||
|
||||
|
||||
def compute_world_size(
|
||||
*, enable_dp_attention: bool, dp_size: int, tp_size: int, pp_size: int
|
||||
) -> int:
|
||||
def compute_world_size(*, dp_size: int, tp_size: int, pp_size: int) -> int:
|
||||
"""Total GPU count across all data-parallel replicas.
|
||||
|
||||
Takes the values rather than a config object: the two sizes are the widths
|
||||
Takes the values rather than a config object: the sizes are the widths
|
||||
the launch asked for, which the Ray driver needs before any process group
|
||||
exists, and passing a context would hand it the live groups instead.
|
||||
Attention DP runs inside the TP group, so it adds no GPUs.
|
||||
"""
|
||||
return (1 if enable_dp_attention else dp_size) * tp_size * pp_size
|
||||
return dp_size * tp_size * pp_size
|
||||
|
||||
|
||||
def m3_fp8_attn_gemm_enabled(args) -> bool:
|
||||
@@ -706,7 +719,7 @@ class PortArgs:
|
||||
rank=int(server_args.decoupled_spec_rank),
|
||||
)
|
||||
|
||||
if not cfg.enable_dp_attention:
|
||||
if not attn_dp_enabled_of(cfg):
|
||||
# Normal case, use IPC within a single node
|
||||
return PortArgs(
|
||||
tokenizer_ipc_name=f"ipc://{tempfile.NamedTemporaryFile(delete=False).name}",
|
||||
@@ -732,12 +745,12 @@ class PortArgs:
|
||||
dist_init_host = na.host
|
||||
dist_init_port = na.port
|
||||
|
||||
# Reserve port_base+0..NUM_DERIVED_PORTS-1 (6 fixed ports + dp_size
|
||||
# rust-path slots); derive from server_args only (never dp_rank) so
|
||||
# Reserve port_base+0..NUM_DERIVED_PORTS-1 (6 fixed ports + one
|
||||
# rust-path slot per DP rank); derive from server_args only (never dp_rank) so
|
||||
# every init_new call agrees, decrementing below dist_init_port on
|
||||
# overflow.
|
||||
is_rust_server = envs.SGLANG_RUST_SERVER.get()
|
||||
NUM_DERIVED_PORTS = 6 if not is_rust_server else 6 + cfg.dp_size
|
||||
NUM_DERIVED_PORTS = 6 if not is_rust_server else 6 + num_dp_ranks_of(cfg)
|
||||
if ep_scale_joiner_of(resolving_view(server_args)):
|
||||
port_base = server_args.port + ZMQ_TCP_PORT_DELTA
|
||||
if port_base + NUM_DERIVED_PORTS > 65535:
|
||||
|
||||
@@ -16,6 +16,7 @@ from sglang.srt.arg_groups.overrides import (
|
||||
resolved_view,
|
||||
resolving_view,
|
||||
)
|
||||
from sglang.srt.runtime_context import attn_dp_enabled_of
|
||||
from sglang.srt.utils import log_info_on_rank0
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -66,9 +67,9 @@ def adaptive_unsupported_reason(server_args: ServerArgs) -> str | None:
|
||||
f"speculative_eagle_topk={cfg.speculative_eagle_topk} "
|
||||
"(only topk=1 is supported)"
|
||||
)
|
||||
if resolved_view(server_args).enable_dp_attention:
|
||||
if attn_dp_enabled_of(resolved_view(server_args)):
|
||||
return (
|
||||
"enable_dp_attention=True is not supported "
|
||||
"attention DP is not supported "
|
||||
"(adaptive tier decisions are not synchronized across DP ranks)"
|
||||
)
|
||||
if resolved_view(server_args).enable_multi_layer_eagle:
|
||||
|
||||
@@ -403,14 +403,14 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
# spec broadcasts must stay within the attn-TP group.
|
||||
self._tp_sync = SpecTpSync(
|
||||
get_parallel().attn_tp_group
|
||||
if get_parallel().enable_dp_attention
|
||||
if get_parallel().attn_dp_enabled
|
||||
else get_parallel().tp_group
|
||||
)
|
||||
|
||||
# Under dp attention, the draft worker runs on the per-DP attn-TP
|
||||
# group, independent of idle peer DP ranks; it is built and run under
|
||||
# the same placement.
|
||||
self.draft_owns_attention = get_parallel().enable_dp_attention
|
||||
self.draft_owns_attention = get_parallel().attn_dp_enabled
|
||||
# Inside the draft scope the context answers the draft's narrowed rank.
|
||||
self._target_tp_rank = get_parallel().tp_rank
|
||||
with draft_pp_context(), draft_tp_context(self.draft_owns_attention):
|
||||
@@ -645,7 +645,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
"device graph capture.",
|
||||
type(current_platform).__name__,
|
||||
)
|
||||
if get_parallel().enable_dp_attention and capture_decode_cuda_graph:
|
||||
if get_parallel().attn_dp_enabled and capture_decode_cuda_graph:
|
||||
# Idle DP ranks skip the draft step, so they cannot join a
|
||||
# shared graph capture/replay; keep the draft eager under dp
|
||||
# attention.
|
||||
@@ -1295,7 +1295,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
the full embedding once during init and keep it replicated, making
|
||||
the draft block-id lookup a collective-free on-device F.embedding.
|
||||
"""
|
||||
if not get_parallel().enable_dp_attention:
|
||||
if not get_parallel().attn_dp_enabled:
|
||||
return
|
||||
|
||||
tp_group = get_parallel().tp_group
|
||||
@@ -2335,7 +2335,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
# verify forward (IDLE mode) so its cross-DP collectives stay in
|
||||
# lockstep with the active DP group; the draft block's
|
||||
# collectives are within-rank and skipped.
|
||||
if get_parallel().enable_dp_attention:
|
||||
if get_parallel().attn_dp_enabled:
|
||||
idle_verify_input = DFlashVerifyInput(
|
||||
draft_token=torch.empty((0,), dtype=torch.long, device=self.device),
|
||||
positions=torch.empty((0,), dtype=torch.int64, device=self.device),
|
||||
@@ -2726,7 +2726,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
# their DP-gather segment offsets disagree. Fall back to eager verify
|
||||
# whenever any DP rank is idle this round.
|
||||
if (
|
||||
get_parallel().enable_dp_attention
|
||||
get_parallel().attn_dp_enabled
|
||||
and verify_forward_batch.original_global_num_tokens_cpu is not None
|
||||
and min(verify_forward_batch.original_global_num_tokens_cpu) == 0
|
||||
):
|
||||
@@ -2735,7 +2735,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
# Mixed-round guard: an extend rank's raw token counts disagree with
|
||||
# the verify batch's spec-scaled counts on the DP-gather layout; run
|
||||
# eager, symmetric with the idle guard above.
|
||||
if get_parallel().enable_dp_attention and batch.is_extend_in_batch:
|
||||
if get_parallel().attn_dp_enabled and batch.is_extend_in_batch:
|
||||
verify_forward_batch.can_run_decode_cuda_graph = False
|
||||
|
||||
target_out = self.target_worker.forward_batch_generation(
|
||||
|
||||
@@ -165,7 +165,7 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
|
||||
self._draft_is_moe = draft_is_deepseek_v4()
|
||||
self._draft_dp_context_enabled = (
|
||||
get_parallel().enable_dp_attention and not self._draft_is_moe
|
||||
get_parallel().attn_dp_enabled and not self._draft_is_moe
|
||||
)
|
||||
self._is_pd_prefill = get_disagg().disaggregation_mode == "prefill"
|
||||
self._decode_graph_allowed = (
|
||||
@@ -173,7 +173,7 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
and not self._is_pd_prefill
|
||||
)
|
||||
if (
|
||||
get_parallel().enable_dp_attention
|
||||
get_parallel().attn_dp_enabled
|
||||
and self._draft_is_moe
|
||||
and get_parallel().attn_tp_size > 1
|
||||
):
|
||||
@@ -232,9 +232,7 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
|
||||
parallel = get_parallel()
|
||||
self._tp_sync = SpecTpSync(
|
||||
parallel.attn_tp_group
|
||||
if parallel.enable_dp_attention
|
||||
else parallel.tp_group
|
||||
parallel.attn_tp_group if parallel.attn_dp_enabled else parallel.tp_group
|
||||
)
|
||||
self._draft_graph_group = (
|
||||
parallel.attn_tp_group
|
||||
@@ -294,13 +292,13 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
tp_sync=self._tp_sync,
|
||||
)
|
||||
if (
|
||||
get_parallel().enable_dp_attention
|
||||
get_parallel().attn_dp_enabled
|
||||
and not self._draft_is_moe
|
||||
and self._verify_planner.is_compact_mode
|
||||
and self._decode_graph_allowed
|
||||
):
|
||||
raise ValueError(
|
||||
"DSpark dense-draft compact verify under --enable-dp-attention does not "
|
||||
"DSpark dense-draft compact verify under attention DP does not "
|
||||
"yet support cuda graph (idle DP groups cannot join the token-keyed "
|
||||
"compact graph). Re-run with --disable-cuda-graph (eager is lossless), "
|
||||
"or use SGLANG_RAGGED_VERIFY_MODE=static. The dsv4 (MoE) draft supports "
|
||||
@@ -321,7 +319,7 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
mask_token_id=self._mask_token_id,
|
||||
draft_block_spec_info=self._draft_block_spec_info,
|
||||
tp_sync=self._tp_sync,
|
||||
dp_moe_sync=self._draft_is_moe and get_parallel().enable_dp_attention,
|
||||
dp_moe_sync=self._draft_is_moe and get_parallel().attn_dp_enabled,
|
||||
)
|
||||
self._verify_epilogue = None
|
||||
target_is_dsv41 = (
|
||||
@@ -336,7 +334,7 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
target_is_dsv41
|
||||
and self._verify_planner.mode_value == "static"
|
||||
and self._draft_is_moe
|
||||
and not get_parallel().enable_dp_attention
|
||||
and not get_parallel().attn_dp_enabled
|
||||
and get_parallel().pp_size == 1
|
||||
)
|
||||
# ROCm (V4.1 target only): inside a HIP graph the accept-site TP broadcasts
|
||||
@@ -628,7 +626,7 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
pp_proxy_tensors: Optional[PPProxyTensors] = None,
|
||||
) -> GenerationBatchResult:
|
||||
if batch.forward_mode.is_idle():
|
||||
if get_parallel().enable_dp_attention:
|
||||
if get_parallel().attn_dp_enabled:
|
||||
self.target_worker.forward_batch_generation(
|
||||
batch, capture_hidden_mode=CaptureHiddenMode.FULL
|
||||
)
|
||||
@@ -744,7 +742,7 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
def _dp_verify_tier_num_tokens(self, batch: ScheduleBatch) -> Optional[int]:
|
||||
if not (
|
||||
self._draft_is_moe
|
||||
and get_parallel().enable_dp_attention
|
||||
and get_parallel().attn_dp_enabled
|
||||
and batch.global_num_tokens is not None
|
||||
and self._verify_planner.is_compact_mode
|
||||
):
|
||||
@@ -793,7 +791,7 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
global_num_reqs = (
|
||||
max(batch.global_num_tokens)
|
||||
if self._draft_is_moe
|
||||
and get_parallel().enable_dp_attention
|
||||
and get_parallel().attn_dp_enabled
|
||||
and batch.global_num_tokens is not None
|
||||
else None
|
||||
)
|
||||
@@ -809,7 +807,7 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
|
||||
if batch.forward_mode.is_idle():
|
||||
self._observers.note_idle_decode_step()
|
||||
if get_parallel().enable_dp_attention:
|
||||
if get_parallel().attn_dp_enabled:
|
||||
if self._draft_is_moe:
|
||||
self._proposer.run_idle_participation(batch)
|
||||
if coordination_plan is not None:
|
||||
|
||||
@@ -186,8 +186,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
|
||||
# Use the same attention topology during draft construction and execution.
|
||||
self.draft_owns_attention = (
|
||||
get_parallel().enable_dp_attention
|
||||
and self.speculative_algorithm.is_eagle3()
|
||||
get_parallel().attn_dp_enabled and self.speculative_algorithm.is_eagle3()
|
||||
)
|
||||
with (
|
||||
draft_tp_context(self.draft_owns_attention),
|
||||
|
||||
@@ -155,7 +155,7 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
# Fields the parent's capture() reads:
|
||||
self.device = model_runner.device
|
||||
self.device_module = torch.get_device_module(self.device)
|
||||
self.dp_size = get_parallel().dp_size
|
||||
self.num_dp_ranks = get_parallel().num_dp_ranks
|
||||
self.pp_size = get_parallel().pp_size
|
||||
self.enable_torch_compile = get_flags().capture.enable_torch_compile
|
||||
self.disable_padding = get_exec().graph.disable_cuda_graph_padding
|
||||
@@ -283,8 +283,8 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
next_token_logits_buffer = buffers.next_token_logits_buffer[:num_tokens]
|
||||
|
||||
if self.require_mlp_tp_gather:
|
||||
global_num_tokens_cpu = [num_tokens] * self.dp_size
|
||||
global_num_tokens_for_logprob_cpu = [num_tokens] * self.dp_size
|
||||
global_num_tokens_cpu = [num_tokens] * self.num_dp_ranks
|
||||
global_num_tokens_for_logprob_cpu = [num_tokens] * self.num_dp_ranks
|
||||
elif self.require_attn_tp_gather:
|
||||
global_num_tokens_cpu = [num_tokens]
|
||||
# DRAFT_EXTEND_V2 produces logits for all tokens, not bs (see mlp branch above)
|
||||
@@ -678,10 +678,12 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
|
||||
|
||||
if self.require_gathered_buffer:
|
||||
if self.require_mlp_tp_gather:
|
||||
dp_size = runner.dp_size
|
||||
global_num_tokens_gpu = torch.zeros((dp_size,), dtype=torch.int32)
|
||||
num_dp_ranks = runner.num_dp_ranks
|
||||
global_num_tokens_gpu = torch.zeros(
|
||||
(num_dp_ranks,), dtype=torch.int32
|
||||
)
|
||||
global_num_tokens_for_logprob_gpu = torch.zeros(
|
||||
(dp_size,), dtype=torch.int32
|
||||
(num_dp_ranks,), dtype=torch.int32
|
||||
)
|
||||
else:
|
||||
global_num_tokens_gpu = torch.zeros((1,), dtype=torch.int32)
|
||||
|
||||
@@ -71,14 +71,14 @@ class RoutedExpertsCapturer(BaseTopkCapturer):
|
||||
topk_size = model_config.hf_text_config.num_experts_per_tok
|
||||
num_layers = model_config.hf_text_config.num_hidden_layers
|
||||
|
||||
# Scale by dp_size so the buffer covers the full DP-concatenated batch.
|
||||
# Scale by num_dp_ranks so the buffer covers the full DP-concatenated batch.
|
||||
# _get_local_slice indexes into [attention_dp_rank * cuda_graph_batch, ...)
|
||||
# and otherwise overflows on dp_rank > 0 when max_running_requests >
|
||||
# chunked_prefill_size.
|
||||
# FIXME: spec decoding's num_verify_tokens is still not accounted for.
|
||||
max_batch_size = max(
|
||||
get_schedule().chunked_prefill_size * get_parallel().dp_size,
|
||||
max_running_requests * get_parallel().dp_size,
|
||||
get_schedule().chunked_prefill_size * get_parallel().num_dp_ranks,
|
||||
max_running_requests * get_parallel().num_dp_ranks,
|
||||
)
|
||||
|
||||
super().__init__(
|
||||
|
||||
@@ -3985,9 +3985,11 @@ def require_mlp_tp_gather(*, moe_a2a_backend=None):
|
||||
elif not isinstance(moe_a2a_backend, MoeA2ABackend):
|
||||
moe_a2a_backend = MoeA2ABackend(moe_a2a_backend)
|
||||
|
||||
# elastic-EP scale-up rewrites dp_size on the published config
|
||||
if get_parallel().enable_dp_attention:
|
||||
assert get_parallel().dp_size > 1, "dp_size must be greater than 1"
|
||||
# elastic-EP scale-up widens num_dp_ranks on the published config
|
||||
if get_parallel().attn_dp_enabled:
|
||||
assert get_parallel().num_dp_ranks > 1, (
|
||||
"attention DP needs more than one DP rank"
|
||||
)
|
||||
if get_exec().moe.elastic_ep_backend is not None:
|
||||
from sglang.srt.elastic_ep.elastic_ep import (
|
||||
elastic_expanded_world_enabled,
|
||||
@@ -4029,7 +4031,7 @@ def require_mlp_tp_gather(*, moe_a2a_backend=None):
|
||||
else:
|
||||
return (
|
||||
get_parallel().moe_dense_tp_size
|
||||
> get_parallel().tp_size // get_parallel().dp_size
|
||||
> get_parallel().tp_size // get_parallel().num_dp_ranks
|
||||
)
|
||||
else:
|
||||
return False
|
||||
@@ -4053,8 +4055,8 @@ def require_attn_tp_gather():
|
||||
not get_moe_a2a_backend().is_none()
|
||||
or get_parallel().moe_dense_tp_size is not None
|
||||
):
|
||||
if get_parallel().enable_dp_attention:
|
||||
return get_parallel().dp_size < get_parallel().tp_size
|
||||
if get_parallel().attn_dp_enabled:
|
||||
return get_parallel().num_dp_ranks < get_parallel().tp_size
|
||||
else:
|
||||
return True
|
||||
else:
|
||||
@@ -4067,7 +4069,7 @@ def require_gathered_buffer():
|
||||
|
||||
def require_mlp_sync():
|
||||
|
||||
return get_parallel().enable_dp_attention or require_gathered_buffer()
|
||||
return get_parallel().attn_dp_enabled or require_gathered_buffer()
|
||||
|
||||
|
||||
def get_cuda_graph_batch_size_alignment() -> int:
|
||||
|
||||
@@ -192,8 +192,8 @@ def _contains_tensor_container(value) -> bool:
|
||||
|
||||
|
||||
def get_vmm_feature_consumer_count() -> int:
|
||||
if get_parallel().enable_dp_attention:
|
||||
return get_parallel().tp_size // get_parallel().dp_size
|
||||
if get_parallel().attn_dp_enabled:
|
||||
return get_parallel().tp_size // get_parallel().num_dp_ranks
|
||||
return get_parallel().tp_size
|
||||
|
||||
|
||||
|
||||
@@ -210,7 +210,7 @@ class OffloaderV2(BaseOffloader):
|
||||
mode: str,
|
||||
):
|
||||
parallel = get_parallel()
|
||||
dp_rank, dp_size = parallel.dp_rank, parallel.dp_size
|
||||
dp_rank, num_dp_ranks = parallel.dp_rank, parallel.num_dp_ranks
|
||||
self.group_size = group_size
|
||||
self.num_in_group = num_in_group
|
||||
self.prefetch_step = prefetch_step
|
||||
@@ -224,7 +224,7 @@ class OffloaderV2(BaseOffloader):
|
||||
set_naive_distributed(
|
||||
NaiveDistributed(
|
||||
rank=dp_rank,
|
||||
world_size=dp_size,
|
||||
world_size=num_dp_ranks,
|
||||
rendezvous=f"/tmp/{run_id}",
|
||||
)
|
||||
)
|
||||
|
||||
@@ -357,7 +357,7 @@ class _ProfilerTorch(_ProfilerConcreteBase):
|
||||
filename_parts = [self.profile_id, f"TP-{get_parallel().tp_rank}"]
|
||||
|
||||
# Only add other ranks if parallelism is enabled (size > 1)
|
||||
if get_parallel().dp_size > 1:
|
||||
if get_parallel().num_dp_ranks > 1:
|
||||
filename_parts.append(f"DP-{get_parallel().dp_rank}")
|
||||
if get_parallel().pp_size > 1:
|
||||
filename_parts.append(f"PP-{get_parallel().pp_rank}")
|
||||
|
||||
@@ -162,10 +162,8 @@ class WeightCacheDaemon:
|
||||
self.tp_rank = tp_rank
|
||||
self.pp_size = cfg.pp_size
|
||||
self.pp_rank = pp_rank
|
||||
self.dp_size = cfg.dp_size
|
||||
self.ep_size = cfg.ep_size
|
||||
self.moe_dp_size = cfg.moe_dp_size
|
||||
self.enable_dp_attention = cfg.enable_dp_attention
|
||||
self.enable_dp_lm_head = cfg.enable_dp_lm_head
|
||||
self.attn_cp_size = cfg.attn_cp_size
|
||||
self.moe_dense_tp_size = cfg.moe_dense_tp_size
|
||||
@@ -338,12 +336,12 @@ class WeightCacheDaemon:
|
||||
tp_rank=self.tp_rank,
|
||||
pp_size=self.pp_size,
|
||||
pp_rank=self.pp_rank,
|
||||
dp_size=self.dp_size,
|
||||
dp_size=get_parallel().dp_size,
|
||||
ep_size=self.ep_size,
|
||||
moe_dp_size=self.moe_dp_size,
|
||||
moe_dp_rank=moe_dp_rank,
|
||||
moe_ep_rank=moe_ep_rank,
|
||||
enable_dp_attention=self.enable_dp_attention,
|
||||
attn_dp_size=get_parallel().attn_dp_size,
|
||||
enable_dp_lm_head=self.enable_dp_lm_head,
|
||||
attn_cp_size=self.attn_cp_size,
|
||||
moe_dense_tp_size=self.moe_dense_tp_size,
|
||||
|
||||
@@ -515,8 +515,6 @@ class IpcModelLoader(BaseModelLoader):
|
||||
moe_dp_rank = parallel.moe_dp_rank
|
||||
moe_ep_rank = parallel.moe_ep_rank
|
||||
|
||||
dp_size = get_parallel().dp_size
|
||||
|
||||
quant_method, quant_config = self._resolve_engine_quant(model_config)
|
||||
|
||||
engine_config = CacheConfig(
|
||||
@@ -530,12 +528,12 @@ class IpcModelLoader(BaseModelLoader):
|
||||
tp_rank=tp_rank,
|
||||
pp_size=pp_size,
|
||||
pp_rank=pp_rank,
|
||||
dp_size=dp_size,
|
||||
dp_size=parallel.dp_size,
|
||||
ep_size=ep_size,
|
||||
moe_dp_size=moe_dp_size,
|
||||
moe_dp_rank=moe_dp_rank,
|
||||
moe_ep_rank=moe_ep_rank,
|
||||
enable_dp_attention=parallel.enable_dp_attention,
|
||||
attn_dp_size=parallel.attn_dp_size,
|
||||
enable_dp_lm_head=parallel.enable_dp_lm_head,
|
||||
attn_cp_size=parallel.attn_cp_size,
|
||||
moe_dense_tp_size=parallel.moe_dense_tp_size,
|
||||
|
||||
@@ -39,7 +39,7 @@ class CacheConfig(msgspec.Struct):
|
||||
moe_dp_size: int
|
||||
moe_dp_rank: int
|
||||
moe_ep_rank: int
|
||||
enable_dp_attention: bool
|
||||
attn_dp_size: int
|
||||
enable_dp_lm_head: bool
|
||||
attn_cp_size: int
|
||||
moe_dense_tp_size: Optional[int]
|
||||
|
||||
@@ -190,7 +190,7 @@ def get_jitter_engine(
|
||||
engine_kwargs.setdefault("schedule_conservativeness", 0.05)
|
||||
|
||||
if (
|
||||
engine_kwargs.get("enable_dp_attention")
|
||||
engine_kwargs.get("attn_dp_size", 1) > 1
|
||||
and "dist_init_addr" not in engine_kwargs
|
||||
):
|
||||
import portpicker
|
||||
|
||||
@@ -367,7 +367,6 @@ class MockModelRunner(ModelRunner):
|
||||
dllm_algorithm=None,
|
||||
dllm_algorithm_config=None,
|
||||
dp_size=1,
|
||||
enable_dp_attention=False,
|
||||
enable_deterministic_inference=False,
|
||||
enable_mis=False,
|
||||
is_embedding=False,
|
||||
|
||||
@@ -341,7 +341,6 @@ class DSAMockModelRunner(ModelRunner):
|
||||
dsa_prefill_backend=dsa_prefill_backend,
|
||||
device=device,
|
||||
enable_deterministic_inference=False,
|
||||
enable_dp_attention=False,
|
||||
enable_prefill_cp=False,
|
||||
enable_mis=False,
|
||||
is_embedding=False,
|
||||
|
||||
@@ -364,7 +364,6 @@ class MockDSV4ModelRunner:
|
||||
disaggregation_mode=None,
|
||||
dp_size=1,
|
||||
enable_deterministic_inference=False,
|
||||
enable_dp_attention=False,
|
||||
enable_mis=False,
|
||||
is_embedding=False,
|
||||
kv_cache_dtype="auto",
|
||||
|
||||
@@ -271,7 +271,6 @@ class MockMLAModelRunner(ModelRunner):
|
||||
dllm_algorithm=None,
|
||||
dllm_algorithm_config=None,
|
||||
dp_size=1,
|
||||
enable_dp_attention=False,
|
||||
enable_deterministic_inference=False,
|
||||
enable_mis=False,
|
||||
flashinfer_mla_disable_ragged=True,
|
||||
|
||||
@@ -23,9 +23,8 @@ DSV4_POOL_SERVER_ARGS: Final[tuple[str, ...]] = (
|
||||
"--trust-remote-code",
|
||||
"--tp",
|
||||
"4",
|
||||
"--dp",
|
||||
"--attn-dp-size",
|
||||
"4",
|
||||
"--enable-dp-attention",
|
||||
"--moe-a2a-backend",
|
||||
"deepep",
|
||||
"--cuda-graph-max-bs-decode",
|
||||
|
||||
@@ -581,7 +581,7 @@ class SRTRunner:
|
||||
context_length: Optional[int] = None,
|
||||
max_total_tokens: Optional[int] = None,
|
||||
page_size: Optional[int] = None,
|
||||
dp_size: int = 1,
|
||||
attn_dp_size: int = 1,
|
||||
tokenizer_path: Optional[str] = None,
|
||||
mem_fraction_static: float = 0.65,
|
||||
trust_remote_code: bool = False,
|
||||
@@ -607,7 +607,6 @@ class SRTRunner:
|
||||
):
|
||||
self.model_type = model_type
|
||||
self.is_generation = model_type == "generation"
|
||||
enable_dp_attention = dp_size > 1
|
||||
|
||||
spec_kwargs = {}
|
||||
if speculative_draft_model_path:
|
||||
@@ -645,8 +644,7 @@ class SRTRunner:
|
||||
context_length=context_length,
|
||||
max_total_tokens=max_total_tokens,
|
||||
page_size=page_size,
|
||||
enable_dp_attention=enable_dp_attention,
|
||||
dp_size=dp_size,
|
||||
attn_dp_size=attn_dp_size,
|
||||
tokenizer_path=tokenizer_path,
|
||||
disable_overlap_schedule=disable_overlap_schedule,
|
||||
cuda_graph_max_bs_decode=cuda_graph_max_bs_decode,
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
Variants combine `DsaMtpServerBase` (server lifecycle) with
|
||||
`DsaMtpEvalConfigDefaults` (shared eval thresholds/params),
|
||||
`GSM8KMixin` and `SpecDecodingMixin`, then set `model` and per-variant
|
||||
overrides (`enable_dp_attention`, `mem_fraction_static`, `bs_1_speed_thres`).
|
||||
overrides (`attn_dp_size`, `mem_fraction_static`, `bs_1_speed_thres`).
|
||||
|
||||
Example:
|
||||
class TestDsv32DP(
|
||||
@@ -13,7 +13,7 @@ Example:
|
||||
SpecDecodingMixin,
|
||||
):
|
||||
model = "deepseek-ai/DeepSeek-V3.2"
|
||||
enable_dp_attention = True
|
||||
attn_dp_size = 8
|
||||
bs_1_speed_thres = 90
|
||||
|
||||
The base itself is NOT a runnable test (no `test_*` methods until a subclass
|
||||
@@ -50,9 +50,8 @@ class DsaMtpServerBase(CustomTestCase):
|
||||
# Subclasses must set `model`; the others have sensible defaults.
|
||||
model: str = ""
|
||||
tp_size: int = 8
|
||||
dp_size: int = 8
|
||||
attn_dp_size: int = 1
|
||||
mem_fraction_static: float = 0.7
|
||||
enable_dp_attention: bool = False
|
||||
extra_server_args = ()
|
||||
|
||||
# EAGLE MTP config (fixed across DSA-MTP variants).
|
||||
@@ -65,8 +64,8 @@ class DsaMtpServerBase(CustomTestCase):
|
||||
def get_server_args(cls):
|
||||
assert cls.model, f"{cls.__name__} must set `model`"
|
||||
args = ["--trust-remote-code", "--tp", str(cls.tp_size)]
|
||||
if cls.enable_dp_attention:
|
||||
args += ["--dp", str(cls.dp_size), "--enable-dp-attention"]
|
||||
if cls.attn_dp_size > 1:
|
||||
args += ["--attn-dp-size", str(cls.attn_dp_size)]
|
||||
args += [
|
||||
"--speculative-algorithm",
|
||||
cls.speculative_algorithm,
|
||||
|
||||
Reference in New Issue
Block a user