[Feature] Add --attn-dp-size and deprecate --enable-dp-attention (#41818)

This commit is contained in:
Cheng Wan
2026-09-30 13:34:18 -07:00
committed by GitHub
parent 201c4b9b2a
commit a9c97c9f69
366 changed files with 1697 additions and 1900 deletions
+1 -1
View File
@@ -65,7 +65,7 @@ async def warm_up_compile(
disaggregation_mode: str, tokenizer_manager: TokenizerManager
):
print("\nGenerate warm up request for compiling DeepGEMM...\n")
dp_size = get_parallel().dp_size
dp_size = get_parallel().num_dp_ranks
base_ids = [0, 1, 2, 3]
sampling_params = {
"temperature": 0.0,
@@ -42,11 +42,11 @@ def print_deprecated_warning(message: str):
# * an old valued flag replaced by a renamed valued flag
# -> `DeprecatedAliasStoreAction`.
#
# Only the second has a registration today (`--disable-cuda-graph`). The other
# three are kept because the shapes recur -- this package has retired flags of
# every one of them -- and the fiddly parts (`nargs=0` on a boolean, where the
# const goes, warn-and-continue versus `parser.error`) are what a
# reimplementation gets wrong. Pass `new_flag=` so the warning tells the
# Only the second has registrations today (`--disable-cuda-graph`,
# `--enable-dp-attention`). The other three are kept because the shapes recur
# -- this package has retired flags of every one of them -- and the fiddly
# parts (`nargs=0` on a boolean, where the const goes, warn-and-continue versus
# `parser.error`) are what a reimplementation gets wrong. Pass `new_flag=` so the warning tells the
# operator what to switch to; that pointer is the whole point.
@@ -23,7 +23,7 @@ from sglang.srt.model_executor.cuda_graph_config import (
with_phase,
)
from sglang.srt.platforms import current_platform
from sglang.srt.runtime_context import get_platform
from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform
from sglang.srt.utils.common import (
is_cpu,
is_mps,
@@ -186,7 +186,7 @@ def disable_tc_piecewise_cudagraph_if_incompatible(server_args: Any):
"model-arch blacklist",
lambda: model_config_of(server_args).is_piecewise_cuda_graph_disabled_model,
),
("DP attention", lambda: resolved_view(server_args).enable_dp_attention),
("DP attention", lambda: attn_dp_enabled_of(resolved_view(server_args))),
("full torch.compile mode", lambda: cfg.enable_torch_compile),
("pipeline parallelism (pp_size > 1)", lambda: cfg.pp_size > 1),
(
@@ -11,7 +11,7 @@ from sglang.srt.arg_groups.overrides import (
run_post_process_pass,
)
from sglang.srt.environ import envs
from sglang.srt.runtime_context import get_platform
from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform, num_dp_ranks_of
from sglang.srt.utils.common import is_gfx95_supported, is_npu
if TYPE_CHECKING:
@@ -123,10 +123,18 @@ def validate_deepseek_v4_cp(server_args: ServerArgs) -> None:
"reindexes with interleave order."
)
# DeepSeek-V4 CP runs data-parallel groups as attention DP.
assert not (cfg.attn_dp_size > 1 and cfg.dp_size > 1), (
f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: "
"data-parallel replicas combined with attention data parallelism "
"are not supported."
)
attn_dp_size = cfg.attn_dp_size * cfg.dp_size
declare_resolution(
server_args,
"validate_deepseek_v4_cp",
enable_dp_attention=True,
attn_dp_size=attn_dp_size,
dp_size=1,
)
declare_resolution(
server_args,
@@ -136,10 +144,10 @@ def validate_deepseek_v4_cp(server_args: ServerArgs) -> None:
declare_resolution(
server_args,
"validate_deepseek_v4_cp",
attn_cp_size=cfg.tp_size // cfg.dp_size,
attn_cp_size=cfg.tp_size // attn_dp_size,
)
if not is_npu():
assert cfg.dp_size == 1, (
assert attn_dp_size == 1, (
"For round-robin split mode, dp attention is not supported."
)
assert cfg.tp_size <= 8, (
@@ -161,7 +169,7 @@ def validate_deepseek_v4_cp(server_args: ServerArgs) -> None:
logger.warning(
f"Enable Context Parallel for DeepSeekV4, "
f"strategy={cfg.cp_strategy}, "
f"dp_size={cfg.dp_size}, moe_dense_tp_size={cfg.moe_dense_tp_size}, "
f"attn_dp_size={attn_dp_size}, moe_dense_tp_size={cfg.moe_dense_tp_size}, "
f"attn_cp_size={cfg.attn_cp_size}, ep_size={cfg.ep_size}, tp_size={cfg.tp_size}"
)
@@ -193,7 +201,7 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
"prefill CUDA graphs",
cfg.cuda_graph_config.prefill.backend != Backend.DISABLED,
),
("DP attention", cfg.enable_dp_attention),
("DP attention", attn_dp_enabled_of(cfg)),
("context parallelism", cfg.attn_cp_size > 1),
("external cache linker", cfg.enable_unified_cache_external_linker),
("unified memory", cfg.enable_unified_memory),
@@ -246,8 +254,8 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
if (
read_ragged_verify_mode() is not RaggedVerifyMode.STATIC
or cfg.disaggregation_transfer_backend != "mooncake"
or cfg.dp_size != 1
or cfg.enable_dp_attention
or num_dp_ranks_of(cfg) != 1
or attn_dp_enabled_of(cfg)
or cfg.attn_cp_size != 1
or cfg.dcp_size != 1
):
@@ -285,7 +293,7 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
cfg.cuda_graph_config.prefill.backend != Backend.DISABLED,
),
# input_ids_global is a DP-wide gather, so the tail slice cannot apply.
("DP attention", cfg.enable_dp_attention),
("DP attention", attn_dp_enabled_of(cfg)),
)
for feature, enabled in incompatible:
if enabled:
@@ -40,6 +40,7 @@ def handle_expert_pack(server_args: Any) -> None:
parallelism = (
("tensor", "--tp-size", cfg.tp_size),
("data", "--dp-size", cfg.dp_size),
("attention data", "--attn-dp-size", cfg.attn_dp_size),
("expert", "--ep-size", cfg.ep_size),
)
for label, option, size in parallelism:
@@ -79,6 +79,7 @@ POSITIONAL_FIELD_ORDER = (
"pp_max_micro_batch_size",
"pp_async_batch_depth",
"dp_size",
"attn_dp_size",
"load_balance_method",
"attn_cp_size",
"moe_dp_size",
+28 -14
View File
@@ -73,8 +73,19 @@ class Parallel(msgspec.Struct):
dp_size: A[
int,
Arg(
help="The data parallelism size.",
help="The number of data-parallel replicas of the model.",
aliases=["--data-parallel-size"],
resolvable=True,
),
] = 1
attn_dp_size: A[
int,
Arg(
help="The attention data parallelism size: the number of "
"data-parallel attention groups inside the TP group, one scheduler "
"each, while the FFN stays tensor parallel.",
aliases=["--attention-data-parallel-size"],
resolvable=True,
),
] = 1
load_balance_method: A[
@@ -159,14 +170,9 @@ class Parallel(msgspec.Struct):
bool,
"Enable attention tensor-parallel weight slicing during decode under context parallel (cp_size>1). Slices the replicated attention linears to the local CP partition, eliminating redundant decode GEMMs.",
] = False
# DP attention
enable_dp_attention: A[
bool,
Arg(
help="Enabling data parallelism for attention and tensor parallelism for FFN. The dp size should be equal to the tp size. Currently DeepSeek-V2 and Qwen 2/3 MoE models are supported.",
resolvable=True,
),
] = False
# Deprecated spelling of `attn_dp_size`: `--dp-size N --enable-dp-attention`
# resolves to `attn_dp_size = N`, `dp_size = 1`.
enable_dp_attention: A[bool, Arg(no_cli=True, resolvable=True)] = False
enable_dp_attention_local_control_broadcast: A[
bool,
"With DP-attention, send control messages to every DP group leader and broadcast within attn_tp_group instead of the full tp_group. Eliminates a costly all-ranks gloo sync on every scheduler iteration.",
@@ -184,7 +190,7 @@ class Parallel(msgspec.Struct):
help="Use all-to-all instead of TP all-gather followed by DP scatter "
"for the TP-sharded LM head under DP attention. By default this is "
"enabled only on decode-only PD nodes with pure DP attention "
"(tp_size == dp_size > 1 and attn_cp_size == 1), and disabled on "
"(tp_size == attn_dp_size > 1 and attn_cp_size == 1), and disabled on "
"prefill-only and colocated nodes. Pass "
"--no-enable-tp-lm-head-all-to-all to opt out. The path is "
"incompatible with --enable-dp-lm-head; batches without an equal "
@@ -282,10 +288,18 @@ class Parallel(msgspec.Struct):
doc="Attention tensor-parallel width: `tp_size` divided by the "
"attention-DP and attention-CP dimensions.",
)
attn_dp_size = Derived(
fn="sglang.srt.runtime_context.attn_dp_size_of",
doc="Attention data-parallel width: `dp_size` when DP attention is "
"on, otherwise one.",
attn_dp_enabled = Derived(
fn="sglang.srt.runtime_context.attn_dp_enabled_of",
doc="Whether attention runs data parallel: `attn_dp_size` is wider "
"than one, or this process is an elastic EP scale joiner, which joins "
"an attention-DP deployment with a group of its own that may be one "
"rank wide.",
)
num_dp_ranks = Derived(
fn="sglang.srt.runtime_context.num_dp_ranks_of",
doc="How many data-parallel ranks the deployment serves with, one "
"scheduler each: `dp_size * attn_dp_size`. Elastic EP scale-up widens "
"it as ranks join.",
)
attn_dcp_size = Derived(
fn="sglang.srt.runtime_context.attn_dcp_size_of",
@@ -18,7 +18,7 @@ from sglang.srt.arg_groups.overrides import (
from sglang.srt.configs.hybrid_arch import mambaish_config
from sglang.srt.environ import envs
from sglang.srt.model_executor.cuda_graph_config import Backend
from sglang.srt.runtime_context import get_platform
from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform
logger = logging.getLogger(__name__)
@@ -450,7 +450,7 @@ def handle_cache_compatibility(server_args: Any) -> None:
raise NotImplementedError(
"LMCacheUnifiedRadixCache does not yet support speculative decoding"
)
if cfg.enable_dp_attention:
if attn_dp_enabled_of(cfg):
raise NotImplementedError(
"LMCacheUnifiedRadixCache does not yet support DP attention"
)
@@ -7,6 +7,7 @@ if TYPE_CHECKING:
from sglang.srt.server_args import ServerArgs
from sglang.srt.arg_groups.overrides import model_config_of, resolving_view
from sglang.srt.runtime_context import attn_dp_enabled_of
logger = logging.getLogger(__name__)
@@ -24,7 +25,7 @@ def handle_layernorm_sp(server_args: ServerArgs) -> None:
validate_layernorm_sp(
architecture=architectures[0] if architectures else None,
tp_size=cfg.tp_size,
enable_dp_attention=cfg.enable_dp_attention,
attn_dp_enabled=attn_dp_enabled_of(cfg),
speculative_algorithm=cfg.speculative_algorithm,
)
@@ -33,7 +34,7 @@ def validate_layernorm_sp(
*,
architecture: Optional[str],
tp_size: int,
enable_dp_attention: bool,
attn_dp_enabled: bool,
speculative_algorithm: Optional[str],
) -> None:
"""Fail loud for unsupported / incompatible configs. Callers gate on the flag."""
@@ -49,9 +50,9 @@ def validate_layernorm_sp(
"--enable-layernorm-sp requires tp_size > 1: there is no sequence to "
"shard across a single TP rank."
)
if enable_dp_attention:
if attn_dp_enabled:
raise ValueError(
"--enable-layernorm-sp is not compatible with --enable-dp-attention: "
"--enable-layernorm-sp is not compatible with attention DP: "
"SP shards the sequence across the full TP group, which under DP "
"attention spans data-parallel groups holding different sequences."
)
@@ -13,7 +13,7 @@ from sglang.srt.arg_groups.overrides import (
)
from sglang.srt.connector import ConnectorType
from sglang.srt.environ import envs
from sglang.srt.runtime_context import get_platform
from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform
from sglang.srt.utils.common import parse_connector_type
logger = logging.getLogger(__name__)
@@ -143,11 +143,11 @@ def validate_mega_moe_token_budget(server_args: ServerArgs, model_label: str) ->
local_chunked_prefill_size = (
cfg.chunked_prefill_size + token_partition_size - 1
) // token_partition_size
elif cfg.enable_dp_attention:
token_partition_size = cfg.dp_size
token_partition_name = "dp_size"
elif attn_dp_enabled_of(cfg):
token_partition_size = cfg.attn_dp_size
token_partition_name = "attn_dp_size"
token_alignment = max(
cfg.tp_size // cfg.dp_size // cfg.attn_cp_size,
cfg.tp_size // cfg.attn_dp_size // cfg.attn_cp_size,
1,
)
local_chunked_prefill_size = cfg.chunked_prefill_size // token_partition_size
+4 -4
View File
@@ -17,7 +17,7 @@ from sglang.srt.arg_groups.overrides import (
)
from sglang.srt.environ import envs
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase
from sglang.srt.runtime_context import get_platform
from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform
from sglang.srt.utils.common import get_device_memory_capacity
logger = logging.getLogger(__name__)
@@ -361,14 +361,14 @@ def reserve_for_graph_mb(server_args: Any) -> float:
reserved_mem += decode_cuda_graph_config.max_bs * 2
if (
resolved_view(server_args).enable_dp_attention
attn_dp_enabled_of(resolved_view(server_args))
and cfg.disaggregation_mode != "prefill"
):
# DP attention needs more padding for some operations, and much more for large
# cuda graph max bs (torch allocator / implementation inefficiencies).
reserved_mem += decode_cuda_graph_config.max_bs * cfg.dp_size * 3
reserved_mem += decode_cuda_graph_config.max_bs * cfg.attn_dp_size * 3
if decode_cuda_graph_config.max_bs > 300:
reserved_mem += decode_cuda_graph_config.max_bs * cfg.dp_size * 1.5
reserved_mem += decode_cuda_graph_config.max_bs * cfg.attn_dp_size * 1.5
if (
cfg.disaggregation_mode != "decode"
+21 -16
View File
@@ -37,7 +37,11 @@ from sglang.srt.connector import ConnectorType
from sglang.srt.environ import envs
from sglang.srt.hardware_backend.mlx.runtime import use_mlx
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
from sglang.srt.runtime_context import derive_attention_widths, get_platform
from sglang.srt.runtime_context import (
attn_dp_enabled_of,
derive_attn_tp_size,
get_platform,
)
from sglang.srt.utils.common import (
get_quantization_config,
is_mps,
@@ -290,15 +294,14 @@ def handle_model_specific_adjustments(server_args: Any):
)
else:
# Pure TP and partial DP Attention mode is active for DSA, logging a warning
if cfg.dp_size < cfg.tp_size:
_, attn_tp_size = derive_attention_widths(
tp_size=cfg.tp_size,
attn_cp_size=cfg.attn_cp_size,
dp_size=cfg.dp_size,
enable_dp_attention=cfg.enable_dp_attention,
)
attn_tp_size = derive_attn_tp_size(
tp_size=cfg.tp_size,
attn_cp_size=cfg.attn_cp_size,
attn_dp_size=cfg.attn_dp_size,
)
if attn_tp_size > 1:
logger.warning(
f"DSA with TP mode is active, dp_size={cfg.dp_size}, tp_size={cfg.tp_size}, "
f"DSA with TP mode is active, attn_dp_size={cfg.attn_dp_size}, tp_size={cfg.tp_size}, "
f"attn_tp_size={attn_tp_size}, attention weights will be sharded across {attn_tp_size} ranks."
)
@@ -410,7 +413,7 @@ def handle_model_specific_adjustments(server_args: Any):
if is_deepseek_dsa(hf_config) and not envs.SGLANG_OPT_USE_TOPK_V2.is_set():
# Prefer HIP top-k by default while honoring an explicit selection.
envs.SGLANG_OPT_USE_TOPK_V2.set(False)
if not resolved_view(server_args).enable_dp_attention and cfg.nnodes == 1:
if not attn_dp_enabled_of(resolved_view(server_args)) and cfg.nnodes == 1:
# TODO (Hubert): Put this back later
# server_args.enable_aiter_allreduce_fusion = True
@@ -514,7 +517,7 @@ def handle_model_specific_adjustments(server_args: Any):
quant_method = get_quantization_config(hf_config)
is_mxfp4_quant_format = quant_method == "mxfp4"
if (
not resolved_view(server_args).enable_dp_attention
not attn_dp_enabled_of(resolved_view(server_args))
and cfg.nnodes == 1
and get_platform().is_hip
):
@@ -543,8 +546,11 @@ def handle_model_specific_adjustments(server_args: Any):
if model_arch == "MiMoV2ForCausalLM" and not cfg.encoder_only:
expected_attn_tp_size = get_mimo_v2_fused_qkv_expected_tp_size(hf_config)
view = resolved_view(server_args)
attn_dp_size = cfg.dp_size if view.enable_dp_attention else 1
effective_attn_tp_size = cfg.tp_size // attn_dp_size // view.attn_cp_size
effective_attn_tp_size = derive_attn_tp_size(
tp_size=cfg.tp_size,
attn_cp_size=view.attn_cp_size,
attn_dp_size=cfg.attn_dp_size,
)
if (
expected_attn_tp_size is not None
and expected_attn_tp_size % effective_attn_tp_size != 0
@@ -555,10 +561,9 @@ def handle_model_specific_adjustments(server_args: Any):
"qkv_proj weights are "
f"TP={expected_attn_tp_size}-interleaved; got "
f"{effective_attn_tp_size} "
f"(tp_size={cfg.tp_size}, dp_size={cfg.dp_size}, "
f"enable_dp_attention={view.enable_dp_attention}, "
f"(tp_size={cfg.tp_size}, attn_dp_size={cfg.attn_dp_size}, "
f"attn_cp_size={view.attn_cp_size}). "
"Set --tp, --dp, --enable-dp-attention, and "
"Set --tp, --attn-dp-size, and "
"--attention-context-parallel-size so the effective "
f"attention TP size is {expected_attn_tp_size}."
)
@@ -85,7 +85,15 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
logger.warning(
"Context parallel feature is still under experiment. It has only been verified on Hopper platform."
)
overrides["enable_dp_attention"] = True
# DSA CP runs data-parallel groups as attention DP.
assert not (cfg.attn_dp_size > 1 and cfg.dp_size > 1), (
f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: "
"data-parallel replicas combined with attention data parallelism "
"are not supported."
)
attn_dp_size = cfg.attn_dp_size * cfg.dp_size
overrides["attn_dp_size"] = attn_dp_size
overrides["dp_size"] = 1
overrides["moe_dense_tp_size"] = 1
if cfg.cp_strategy == "zigzag":
overrides["moe_a2a_backend"] = "deepep"
@@ -95,7 +103,7 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
"moe_a2a_backend=deepep, ep_size=tp_size, batch_size=1."
)
else:
assert cfg.dp_size == 1, (
assert attn_dp_size == 1, (
"interleave DSA CP does not support DP attention."
)
assert cfg.tp_size <= 8, (
@@ -104,11 +112,11 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
# Note(kpham-sgl): Keep attn_tp_size == 1 under DSA CP.
# The DSA / MLA CP gather and reduce-scatter
# (the dsa_cp_* helpers in adapters/context_parallel.py) assume it.
attn_cp_size = cfg.tp_size // cfg.dp_size
attn_cp_size = cfg.tp_size // attn_dp_size
overrides["attn_cp_size"] = attn_cp_size
logger.warning(
"Enabled DSA context parallel: "
f"strategy={cfg.cp_strategy}, dp_size={cfg.dp_size}, "
f"strategy={cfg.cp_strategy}, attn_dp_size={attn_dp_size}, "
f"moe_dense_tp_size={overrides['moe_dense_tp_size']}, "
f"ep_size={overrides.get('ep_size', cfg.ep_size)}, tp_size={cfg.tp_size}, "
f"attn_cp_size={attn_cp_size}, "
@@ -158,7 +166,15 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
"MLA prefill context parallel is still experimental. "
"Verified on Hopper with the fa3 backend."
)
overrides["enable_dp_attention"] = True
# MLA CP runs data-parallel groups as attention DP.
assert not (cfg.attn_dp_size > 1 and cfg.dp_size > 1), (
f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: "
"data-parallel replicas combined with attention data parallelism "
"are not supported."
)
attn_dp_size = cfg.attn_dp_size * cfg.dp_size
overrides["attn_dp_size"] = attn_dp_size
overrides["dp_size"] = 1
# TODO(kpham-sgl) Supports moe_dense_tp_size != 1.
overrides["moe_dense_tp_size"] = 1
overrides["moe_a2a_backend"] = "deepep"
@@ -169,11 +185,11 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
# FIXME(kpham-sgl): Keep attn_tp_size == 1 under MLA CP.
# The DSA / MLA CP gather and reduce-scatter
# (the dsa_cp_* helpers in adapters/context_parallel.py) assume it.
attn_cp_size = cfg.tp_size // cfg.dp_size
attn_cp_size = cfg.tp_size // attn_dp_size
overrides["attn_cp_size"] = attn_cp_size
logger.warning(
f"Enable Context Parallel opt for MLA, "
f"Setting dp_size == {cfg.dp_size} and "
f"Setting attn_dp_size == {attn_dp_size} and "
f"attn_cp_size == {attn_cp_size}, "
f"moe_dense_tp_size == {overrides['moe_dense_tp_size']}, "
f"ep_size == {overrides['ep_size']}, "
@@ -5,6 +5,7 @@ from sglang.srt.arg_groups.model_override_base import (
is_attention_backend_not_set,
resolving_view,
)
from sglang.srt.runtime_context import attn_dp_enabled_of
@_register_for("IQuestQ1ForCausalLM", "IQuestQ1MTP")
@@ -33,7 +34,7 @@ def _iquest_q1_overrides(server_args: Any, hf_config: Any) -> dict:
)
if cfg.enable_multi_layer_eagle:
raise ValueError("IQuest Q1 MTP drafting does not support multi-layer EAGLE.")
if cfg.enable_dp_attention:
if attn_dp_enabled_of(cfg):
raise ValueError("IQuest Q1 does not support DP attention.")
if cfg.pp_size > 1:
raise ValueError("IQuest Q1 does not support pipeline parallelism.")
@@ -8,13 +8,13 @@ from sglang.srt.arg_groups.model_override_base import (
resolving_view,
)
from sglang.srt.environ import envs
from sglang.srt.runtime_context import get_platform
from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform
@_register_for("MiniCPMForCausalLM", "MiniCPMSALAForCausalLM")
def _minicpm_sala_overrides(server_args: Any, hf_config: Any) -> dict:
cfg = resolving_view(server_args)
if cfg.enable_dp_attention:
if attn_dp_enabled_of(cfg):
raise ValueError("MiniCPM does not support DP attention")
has_sparse_attention = getattr(hf_config, "has_minicpm_sparse_attention", False)
has_hybrid_attention = has_sparse_attention or getattr(
+10 -14
View File
@@ -29,7 +29,7 @@ from sglang.srt.configs.moe_model_registry import (
from sglang.srt.connector import ConnectorType
from sglang.srt.environ import envs
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
from sglang.srt.runtime_context import get_platform
from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform
from sglang.srt.utils.common import is_sm100_supported, parse_connector_type
logger = logging.getLogger(__name__)
@@ -269,8 +269,8 @@ def handle_a2a_moe(server_args: Any):
if a2a_backend == "flashinfer_megamoe":
validate_flashinfer_megamoe_model(server_args)
validate_flashinfer_megamoe_envs()
assert cfg.enable_dp_attention and cfg.dp_size == cfg.tp_size, (
"FlashInfer MegaMOE is only supported with dp_size == tp_size and --enable-dp-attention"
assert attn_dp_enabled_of(cfg) and cfg.attn_dp_size == cfg.tp_size, (
"FlashInfer MegaMOE is only supported with --attn-dp-size equal to --tp-size"
)
if resolved_view(server_args).moe_runner_backend == "auto":
declare_resolution(
@@ -395,11 +395,8 @@ def handle_a2a_moe(server_args: Any):
)
if a2a_now == "flashinfer":
assert (
resolved_view(server_args).enable_dp_attention
and cfg.dp_size == cfg.tp_size
), (
"Flashinfer MoE A2A is only supported with dp_size == tp_size and --enable-dp-attention"
assert attn_dp_enabled_of(cfg) and cfg.attn_dp_size == cfg.tp_size, (
"Flashinfer MoE A2A is only supported with --attn-dp-size equal to --tp-size"
)
if cfg.deepep_mode != "auto":
logger.warning("--deepep-mode is ignored for Flashinfer MoE A2A")
@@ -491,12 +488,12 @@ def handle_a2a_moe(server_args: Any):
deepep_mode="low_latency",
)
logger.warning("auto set deepep_mode=`low_latency` for PPLX EP")
# pplx-kernels' AllToAll needs numDPGroups (== attention dp_size) > 1;
# pplx-kernels' AllToAll needs numDPGroups (== attn_dp_size) > 1;
# without DP attention numDPGroups == 1 and construction fails deep in
# the kernel. This also implies ep_size >= 2.
assert resolved_view(server_args).enable_dp_attention and cfg.dp_size >= 2, (
"moe_a2a_backend='pplx' requires --enable-dp-attention with at "
"least 2 DP groups (--dp-size >= 2)."
assert cfg.attn_dp_size >= 2, (
"moe_a2a_backend='pplx' requires attention DP with at least 2 "
"groups (--attn-dp-size >= 2)."
)
# pplx runs the masked DeepGEMM expert path (sm_90a): reject other
# runners and resolve auto -> deep_gemm. Unquantized bf16 pplx needs
@@ -581,8 +578,7 @@ def validate_deepep_v2_dispatch_token_budget(server_args: Any) -> None:
graph_bs = decode_config.max_bs or 0
if view.max_running_requests is not None:
attn_dp_size = view.dp_size if view.enable_dp_attention else 1
per_rank_pool_bs = max(1, view.max_running_requests // attn_dp_size)
per_rank_pool_bs = max(1, view.max_running_requests // view.attn_dp_size)
graph_bs = min(graph_bs, per_rank_pool_bs)
tokens_per_req = (
max_speculative_num_draft_tokens(server_args) or 1
+14 -12
View File
@@ -62,8 +62,10 @@ logger = logging.getLogger(__name__)
from sglang.srt.environ import envs
from sglang.srt.model_executor.cuda_graph_config import Backend
from sglang.srt.runtime_context import (
attn_dp_enabled_of,
get_context,
get_platform,
num_dp_ranks_of,
)
from sglang.srt.utils.common import (
get_quantization_config,
@@ -931,7 +933,7 @@ def _flashinfer_allreduce_fusion_auto_enable(view: Any) -> dict:
and not prefer_custom_dsv41
and (get_platform().is_sm90 or get_platform().is_sm100)
and view.tp_size > 1
and not view.enable_dp_attention
and not attn_dp_enabled_of(view)
and (view.nnodes == 1 or get_platform().is_sm100)
and view.moe_a2a_backend == "none"
):
@@ -1304,8 +1306,8 @@ def _page_size_default(view: Any) -> dict:
@register_post_process
def _data_parallelism_defaults(view: Any) -> dict:
if view.dp_size == 1 and view.ep_join_mode != "scale":
return {"enable_dp_attention": False, "enable_dp_lm_head": False}
if num_dp_ranks_of(view) == 1 and view.ep_join_mode != "scale":
return {"enable_dp_lm_head": False}
return {}
@@ -1349,9 +1351,8 @@ def _tp_lm_head_all_to_all_default(view: Any) -> dict:
enable = (
view.disaggregation_mode == "decode"
and view.enable_dp_attention
and view.dp_size > 1
and view.tp_size == view.dp_size
and view.attn_dp_size > 1
and view.tp_size == view.attn_dp_size
and view.attn_cp_size == 1
and not view.enable_dp_lm_head
)
@@ -1362,20 +1363,21 @@ def _tp_lm_head_all_to_all_default(view: Any) -> dict:
def _dp_lm_head_validation(view: Any) -> dict:
"""Require DP attention for DP LM head and TP LM-head all-to-all."""
if view.enable_dp_lm_head:
assert view.enable_dp_attention, (
"Please enable dp attention when setting enable_dp_lm_head. "
assert attn_dp_enabled_of(view), (
"--enable-dp-lm-head requires attention DP (--attn-dp-size)."
)
if view.enable_tp_lm_head_all_to_all:
assert view.enable_dp_attention, (
"Please enable dp attention when setting enable_tp_lm_head_all_to_all."
assert attn_dp_enabled_of(view), (
"--enable-tp-lm-head-all-to-all requires attention DP (--attn-dp-size)."
)
assert not view.enable_dp_lm_head, (
"--enable-tp-lm-head-all-to-all uses a TP-sharded LM head and is "
"incompatible with --enable-dp-lm-head."
)
assert view.tp_size == view.dp_size, (
assert view.tp_size == view.attn_dp_size, (
"--enable-tp-lm-head-all-to-all currently requires tp_size == "
f"dp_size, got tp_size={view.tp_size}, dp_size={view.dp_size}."
f"attn_dp_size, got tp_size={view.tp_size}, "
f"attn_dp_size={view.attn_dp_size}."
)
assert view.attn_cp_size == 1, (
"--enable-tp-lm-head-all-to-all currently requires "
+56 -26
View File
@@ -23,12 +23,40 @@ from sglang.srt.arg_groups.resolution_hooks import run_hook
from sglang.srt.connector import ConnectorType
from sglang.srt.environ import envs
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
from sglang.srt.runtime_context import get_platform
from sglang.srt.runtime_context import (
attn_dp_enabled_of,
derive_attn_tp_size,
get_platform,
)
from sglang.srt.utils.common import parse_connector_type
logger = logging.getLogger(__name__)
def handle_deprecated_dp_attention(server_args: Any):
"""Turn the deprecated `--dp-size N --enable-dp-attention` into
`--attn-dp-size N`.
Runs before any handler reads the DP layout. The old spelling counted
attention-DP groups with `dp_size`; that width moves to `attn_dp_size`, and
`dp_size` counts replicas again.
"""
cfg = resolving_view(server_args)
if cfg.enable_dp_attention:
if cfg.attn_dp_size != 1:
raise ValueError(
"--enable-dp-attention is the deprecated spelling of "
"--attn-dp-size; pass --attn-dp-size alone."
)
declare_resolution(
server_args,
"_handle_deprecated_dp_attention",
attn_dp_size=cfg.dp_size,
dp_size=1,
enable_dp_attention=False,
)
def _boundary_parallelism_overrides(cfg, model_type: str) -> dict:
"""Resolve the token-row modes the model's forward actually enters."""
nemotron = model_type in ("nemotron_h", "nemotron_h_puzzle")
@@ -108,8 +136,8 @@ def handle_context_parallelism(server_args: Any):
assert cfg.tp_size % view.attn_cp_size == 0, (
"tp_size must be divisible by attn_cp_size"
)
assert cfg.tp_size % (cfg.dp_size * view.attn_cp_size) == 0, (
"tp_size must be divisible by dp_size * attn_cp_size"
assert cfg.tp_size % (cfg.attn_dp_size * view.attn_cp_size) == 0, (
"tp_size must be divisible by attn_dp_size * attn_cp_size"
)
assert not cfg.enable_aiter_allreduce_fusion, (
@@ -155,14 +183,11 @@ def handle_shared_experts_tp(server_args: Any):
if size is None:
return
from sglang.srt.runtime_context import derive_attention_widths
view = resolved_view(server_args)
_, attn_tp_size = derive_attention_widths(
attn_tp_size = derive_attn_tp_size(
tp_size=cfg.tp_size,
attn_cp_size=view.attn_cp_size,
dp_size=cfg.dp_size,
enable_dp_attention=view.enable_dp_attention,
attn_dp_size=cfg.attn_dp_size,
)
if size < 1 or attn_tp_size % size != 0:
raise ValueError(
@@ -224,7 +249,7 @@ def handle_decode_context_parallelism(server_args: Any):
def handle_data_parallelism(server_args: Any):
# The dp_size==1 resets moved to the resolution pipeline
# The resets without attention DP live in the resolution pipeline
# (arg_groups/overrides.py: _data_parallelism_defaults).
from sglang.srt.arg_groups.cuda_graph_hook import (
generate_prefill_cuda_graph_batch_sizes,
@@ -232,6 +257,15 @@ def handle_data_parallelism(server_args: Any):
cfg = resolving_view(server_args)
if cfg.attn_dp_size < 1:
raise ValueError(f"--attn-dp-size must be positive (got {cfg.attn_dp_size}).")
if cfg.attn_dp_size > 1 and cfg.dp_size > 1:
raise ValueError(
f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: "
"data-parallel replicas combined with attention data parallelism "
"are not supported."
)
run_post_process_pass(server_args, _data_parallelism_defaults)
if cfg.mm_enable_dp_encoder:
@@ -252,18 +286,18 @@ def handle_data_parallelism(server_args: Any):
cfg.tp_size,
)
if resolved_view(server_args).enable_dp_attention:
if attn_dp_enabled_of(resolved_view(server_args)):
declare_resolution(
server_args,
"_handle_data_parallelism",
schedule_conservativeness=cfg.schedule_conservativeness * 0.3,
)
assert cfg.tp_size % cfg.dp_size == 0
assert cfg.tp_size % cfg.attn_dp_size == 0
original_chunked_prefill_size = cfg.chunked_prefill_size
declare_resolution(
server_args,
"_handle_data_parallelism",
chunked_prefill_size=cfg.chunked_prefill_size // cfg.dp_size,
chunked_prefill_size=cfg.chunked_prefill_size // cfg.attn_dp_size,
)
logger.warning(
f"DP attention is enabled. chunked prefill size is adjusted "
@@ -367,12 +401,8 @@ def handle_dwdp(server_args: Any):
declare_resolution(
server_args,
"_handle_dwdp",
dp_size=cfg.dwdp_size,
)
declare_resolution(
server_args,
"_handle_dwdp",
enable_dp_attention=True,
attn_dp_size=cfg.dwdp_size,
dp_size=1,
)
declare_resolution(
server_args, "_handle_dwdp", enable_dp_attention_local_control_broadcast=True
@@ -413,7 +443,7 @@ def handle_dwdp(server_args: Any):
logger.info(
f"DWDP enabled: dwdp_size={cfg.dwdp_size}, "
f"auto-forced dp_size={cfg.dp_size}, ep_size={cfg.dwdp_size}, "
f"auto-forced attn_dp_size={cfg.attn_dp_size}, ep_size={cfg.dwdp_size}, "
f"moe_dense_tp_size=1, moe_a2a_backend=none, "
f"dp_attention_local_control_broadcast=True, "
f"enable_dp_lm_head=True, SCHEDULER_SKIP_ALL_GATHER=True, "
@@ -582,10 +612,10 @@ def handle_elastic_ep(server_args: Any):
assert not cfg.enable_pdmux, (
"Elastic EP CUDA graph recapture does not support PDMux."
)
assert resolved.enable_dp_attention, (
"Elastic EP scale-up requires --enable-dp-attention; without it "
"the TP group is not equivalent to WORLD and the post-scale "
"collective path is invalid."
assert attn_dp_enabled_of(resolved), (
"Elastic EP scale-up requires attention DP (--attn-dp-size); "
"without it the TP group is not equivalent to WORLD and the "
"post-scale collective path is invalid."
)
assert resolved.enable_dp_lm_head, (
"Elastic EP scale-up requires --enable-dp-lm-head so output "
@@ -604,9 +634,9 @@ def handle_elastic_ep(server_args: Any):
f"(got ep_size={resolved.ep_size}, tp_size={cfg.tp_size}); EP, TP "
"and the attention DP group must all coincide with WORLD."
)
assert cfg.dp_size == cfg.tp_size, (
"Elastic EP scale-up requires dp_size == tp_size "
f"(got dp_size={cfg.dp_size}, tp_size={cfg.tp_size})."
assert cfg.attn_dp_size == cfg.tp_size, (
"Elastic EP scale-up requires attn_dp_size == tp_size "
f"(got attn_dp_size={cfg.attn_dp_size}, tp_size={cfg.tp_size})."
)
assert resolved.moe_a2a_backend == "nixl", (
"Elastic EP scale-up requires --moe-a2a-backend nixl "
@@ -12,6 +12,7 @@ from sglang.srt.arg_groups.overrides import (
resolving_view,
)
from sglang.srt.environ import envs
from sglang.srt.runtime_context import attn_dp_enabled_of, num_dp_ranks_of
if TYPE_CHECKING:
from sglang.srt.server_args import ServerArgs
@@ -102,7 +103,7 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None:
f"(--speculative-algorithm {cfg.speculative_algorithm})"
)
if resolved_view(server_args).enable_dp_attention:
if attn_dp_enabled_of(resolved_view(server_args)):
logger.warning(
"EXPERIMENTAL: Decode radix cache with DP attention. "
"Requires prefix-aware DP rank routing for optimal cache hits."
@@ -128,7 +129,7 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None:
if cfg.disaggregation_decode_extra_slots is None:
extra_slots = 0
if cfg.max_running_requests is not None:
per_worker = cfg.max_running_requests // max(1, cfg.dp_size)
per_worker = cfg.max_running_requests // num_dp_ranks_of(cfg)
if per_worker <= 32:
extra_slots = per_worker * 2
declare_resolution(
@@ -160,8 +161,8 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None:
if cfg.enable_pd_role_switch:
view = resolved_view(server_args)
unsupported = []
if view.enable_dp_attention:
unsupported.append("DP attention (--enable-dp-attention)")
if attn_dp_enabled_of(view):
unsupported.append(f"attention DP (--attn-dp-size {view.attn_dp_size})")
if view.ep_size > 1:
unsupported.append(f"expert parallelism (--ep-size {view.ep_size})")
if view.moe_a2a_backend != "none":
+5
View File
@@ -38,6 +38,11 @@ def run_resolution_pipeline(server_args: Any) -> None:
cfg = resolving_view(server_args)
from sglang.srt.arg_groups.parallel_hook import handle_deprecated_dp_attention
# Before any handler reads the DP layout.
run_hook(handle_deprecated_dp_attention, server_args)
from sglang.srt.arg_groups.mega_moe_hook import handle_mega_moe
run_hook(handle_mega_moe, server_args)
@@ -13,6 +13,7 @@ from typing import Any, Callable, Dict, FrozenSet, List
# to every invocation of that name, including repeated steps.
_OVERRIDABLE_HOOKS: FrozenSet[str] = frozenset(
{
"handle_deprecated_dp_attention",
"handle_mega_moe",
"handle_return_hidden_states_mode",
"handle_media_url_security",
@@ -17,7 +17,7 @@ from sglang.srt.arg_groups.overrides import (
)
from sglang.srt.environ import envs
from sglang.srt.platforms import current_platform
from sglang.srt.runtime_context import get_platform
from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform
if TYPE_CHECKING:
from sglang.srt.server_args import ServerArgs
@@ -271,7 +271,7 @@ def _handle_dflash(server_args: ServerArgs) -> None:
)
# DFLASH + dp attention is validated on NPU only.
if cfg.enable_dp_attention and not cfg.device == "npu":
if attn_dp_enabled_of(cfg) and not cfg.device == "npu":
raise ValueError(
"Currently DFLASH speculative decoding does not support dp "
"attention on non-NPU devices."
@@ -538,7 +538,7 @@ def _handle_uno(server_args: ServerArgs) -> None:
if (cfg.tp_size, cfg.pp_size) != (1, 1):
raise ValueError("UNO requires TP=PP=1.")
if cfg.enable_dp_attention or cfg.attn_cp_size != 1:
if attn_dp_enabled_of(cfg) or cfg.attn_cp_size != 1:
raise ValueError("UNO does not support DP attention or context parallelism.")
if cfg.enable_lora or cfg.lora_paths:
raise ValueError("UNO does not support public Multi-LoRA serving.")
@@ -588,8 +588,7 @@ def _handle_dspark(server_args: ServerArgs) -> None:
"DSpark speculative decoding only supports CUDA or NPU device."
)
# dp_size==1 with dp_attention is a degenerate flag under DSV4 CP; skip DP-only checks.
if cfg.enable_dp_attention and cfg.dp_size > 1:
if cfg.attn_dp_size > 1:
if not cfg.enable_dp_lm_head:
raise ValueError("DSpark with dp attention requires --enable-dp-lm-head.")
if not _is_npu and cfg.moe_a2a_backend not in ("none", "megamoe", "mori"):
@@ -963,9 +962,8 @@ def _handle_iquest_q1_mtp_draft(server_args: ServerArgs) -> bool:
def _handle_eagle_family(server_args: ServerArgs) -> None:
cfg = resolving_view(server_args)
if (
cfg.speculative_algorithm == "STANDALONE"
and resolved_view(server_args).enable_dp_attention
if cfg.speculative_algorithm == "STANDALONE" and attn_dp_enabled_of(
resolved_view(server_args)
):
# TODO: support dp attention for standalone speculative decoding
raise ValueError(
@@ -1254,7 +1252,7 @@ def _handle_ngram(server_args: ServerArgs) -> None:
"and produces incorrect results for paged attention backends. "
"This combination is only supported for the 'flashinfer' backend."
)
if view.enable_dp_attention:
if attn_dp_enabled_of(view):
# TODO: support dp attention for ngram speculative decoding
raise ValueError(
"Currently ngram speculative decoding does not support dp attention."
@@ -18,7 +18,7 @@ from sglang.srt.distributed.device_communicators.mooncake_transfer_engine import
parse_ib_device_config,
)
from sglang.srt.environ import envs
from sglang.srt.runtime_context import get_platform
from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform, num_dp_ranks_of
from sglang.srt.utils.common import torch_release
from sglang.srt.utils.runai_utils import is_runai_obj_uri
@@ -72,8 +72,9 @@ def check_pipeline_parallel_compat(cfg: Any) -> None:
# Every stage rebuilds the same verify input from the relayed
# per-request state, so all stages must see the same batch.
# DP attention partitions it per DP rank.
assert not cfg.enable_dp_attention, (
"SGLANG_ENABLE_PP_SPEC is not compatible with --enable-dp-attention"
assert not attn_dp_enabled_of(cfg), (
"SGLANG_ENABLE_PP_SPEC is not compatible with attention DP "
"(--attn-dp-size)"
)
else:
assert cfg.disaggregation_mode == "prefill", (
@@ -114,8 +115,9 @@ def check_server_args(server_args: Any):
if cfg.pp_size > 1:
check_pipeline_parallel_compat(cfg)
assert not (cfg.dp_size > 1 and cfg.nnodes != 1 and not cfg.enable_dp_attention), (
"multi-node data parallel is not supported unless dp attention!"
assert not (cfg.dp_size > 1 and cfg.nnodes != 1), (
"multi-node data-parallel replicas are not supported; use attention DP "
"(--attn-dp-size) across nodes"
)
assert cfg.base_gpu_id >= 0, "base_gpu_id must be non-negative"
@@ -375,7 +377,7 @@ def check_load_publish_args(server_args: Any):
_, reason = resolve_load_pub_range(
kv_endpoint=cfg.endpoint,
replay_endpoint=cfg.replay_endpoint,
dp_size=server_cfg.dp_size,
dp_size=num_dp_ranks_of(server_cfg),
load_publish_endpoint=mode,
)
if reason:
@@ -528,10 +530,10 @@ def check_two_batch_overlap(server_args: Any):
if (
cfg.enable_two_batch_overlap
and cfg.moe_a2a_backend == "none"
and not cfg.enable_dp_attention
and not attn_dp_enabled_of(cfg)
):
raise ValueError(
"When enabling two batch overlap without an EP a2a backend "
"(moe_a2a_backend='none'), --enable-dp-attention is required "
"(moe_a2a_backend='none'), attention DP (--attn-dp-size) is required "
"(DeepSeek-V4 non-EP DP TBO path)."
)
+1 -1
View File
@@ -1744,7 +1744,7 @@ class _SGLangPlugin(_FrameworkPlugin):
try:
parallel = get_parallel()
info["enable_dp_attention"] = self._dp_attn.is_dp_attention_enabled()
info["attn_dp_enabled"] = self._dp_attn.is_dp_attention_enabled()
info["attn_tp_rank"] = parallel.attn_tp_rank
info["attn_tp_size"] = parallel.attn_tp_size
info["attn_dp_rank"] = parallel.attn_dp_rank
@@ -200,9 +200,7 @@ class CommonKVManager(BaseKVManager):
self.dcp_rank = parallel.attn_dcp_rank
self.attn_dp_size = parallel.attn_dp_size
self.attn_dp_rank = parallel.attn_dp_rank
self.system_dp_size = (
1 if get_parallel().enable_dp_attention else get_parallel().dp_size
)
self.system_dp_size = get_parallel().dp_size
self.system_dp_rank = (
self.kv_args.system_dp_rank if self.kv_args.system_dp_rank else 0
)
@@ -1545,11 +1543,12 @@ class CommonKVSender(BaseKVSender):
return
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Bootstrapping)
if get_parallel().dp_size > 1 and not req_has_disagg_prefill_dp_rank:
if get_parallel().num_dp_ranks > 1 and not req_has_disagg_prefill_dp_rank:
if get_parallel().load_balance_method != "follow_bootstrap_room":
self._register_prefill_dp_rank()
elif (
self.kv_mgr.attn_dp_rank != self.bootstrap_room % get_parallel().dp_size
self.kv_mgr.attn_dp_rank
!= self.bootstrap_room % get_parallel().num_dp_ranks
):
# follow_bootstrap_room was overridden by external routed_dp_rank
if envs.SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK.get():
@@ -1560,7 +1559,7 @@ class CommonKVSender(BaseKVSender):
f"follow_bootstrap_room conflict: dispatched to dp_rank "
f"{self.kv_mgr.attn_dp_rank} but bootstrap_room "
f"{self.bootstrap_room} implies dp_rank "
f"{self.bootstrap_room % get_parallel().dp_size}. "
f"{self.bootstrap_room % get_parallel().num_dp_ranks}. "
f"Set SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK=1 "
f"to allow mixed routing.",
)
@@ -209,7 +209,7 @@ def launch_server(server_args: ServerArgs):
configure_logger(server_args, prefix=" encode_server")
publish(server_args, role="encoder")
if get_parallel().dp_size > 1:
if get_parallel().num_dp_ranks > 1:
dp_dispatcher = launch_dp_runtime(server_args)
# runtime initializes multiprocess metrics before spawning;
# HTTP only exposes their endpoint.
@@ -1744,10 +1744,10 @@ def launch_local_runtime(server_args: ServerArgs) -> EncoderRuntime:
This function owns backend construction only. HTTP/gRPC middleware,
service registration, and network serving remain Transport concerns.
"""
if get_parallel().dp_size > 1:
if get_parallel().num_dp_ranks > 1:
raise ValueError(
"launch_local_runtime requires --dp-size 1; got "
f"dp_size={get_parallel().dp_size}."
"launch_local_runtime requires a single DP rank; got "
f"num_dp_ranks={get_parallel().num_dp_ranks}."
)
# Set up prometheus metrics.
@@ -1827,8 +1827,8 @@ def launch_dp_runtime(server_args: ServerArgs) -> DPDispatcher:
"Encoder DP mode requires --dp-size > 1 and --tp-size 1; got "
f"dp_size={get_parallel().dp_size}, tp_size={get_parallel().tp_size}."
)
dp_size = get_parallel().dp_size
logger.info(f"Launching encoder in DP mode: dp_size={dp_size}")
num_dp_ranks = get_parallel().num_dp_ranks
logger.info(f"Launching encoder in DP mode: num_dp_ranks={num_dp_ranks}")
# DP mode: workers (subprocesses) write metrics to the shared multiproc dir;
# the main process exposes the aggregated /metrics endpoint.
@@ -1837,7 +1837,7 @@ def launch_dp_runtime(server_args: ServerArgs) -> DPDispatcher:
ctx = mp.get_context("spawn")
ipc_prefix = random_uuid()
async_zmq_ctx = zmq.asyncio.Context(dp_size + 1)
async_zmq_ctx = zmq.asyncio.Context(num_dp_ranks + 1)
result_path = f"ipc:///tmp/{ipc_prefix}_dp_result"
result_socket = get_zmq_socket(async_zmq_ctx, zmq.PULL, result_path, True)
@@ -1845,13 +1845,13 @@ def launch_dp_runtime(server_args: ServerArgs) -> DPDispatcher:
get_zmq_socket(
async_zmq_ctx, zmq.PUSH, f"ipc:///tmp/{ipc_prefix}_dp_dispatch_{r}", True
)
for r in range(dp_size)
for r in range(num_dp_ranks)
]
release_sockets: List[zmq.asyncio.Socket] = [
get_zmq_socket(
async_zmq_ctx, zmq.PUSH, f"ipc:///tmp/{ipc_prefix}_dp_release_{r}", True
)
for r in range(dp_size)
for r in range(num_dp_ranks)
]
worker_processes: List[mp.Process] = []
@@ -1867,7 +1867,7 @@ def launch_dp_runtime(server_args: ServerArgs) -> DPDispatcher:
# exception (atexit holds the list ref and reads it at exit time).
atexit.register(_kill_workers)
for dp_rank in range(dp_size):
for dp_rank in range(num_dp_ranks):
gpu_id = get_device().base_gpu_id + dp_rank
# Pin the device parent-side around spawn (same convention as the
# scheduler launcher and DP controller) so the child inherits
@@ -1894,7 +1894,7 @@ def launch_dp_runtime(server_args: ServerArgs) -> DPDispatcher:
if get_observability().extra_metric_labels:
labels.update(get_observability().extra_metric_labels)
return DPDispatcher(
dp_size,
num_dp_ranks,
dispatch_sockets,
release_sockets,
result_socket,
+3 -6
View File
@@ -79,14 +79,11 @@ def _bind_threads_if_cpu(*, device: str) -> "Optional[List[int]]":
from sglang.srt.utils import numa_utils
parallel = get_parallel()
# With --enable-dp-attention, dp partitions the existing TP group rather
# than spawning additional processes, so dp_size must not be multiplied
# into the process count here (unlike regular DP, where dp_size * tp_size *
# pp_size is the true worker count).
dp_size = 1 if parallel.enable_dp_attention else parallel.dp_size
# Attention DP partitions the existing TP group rather than spawning
# additional processes, so only the replicas multiply the process count.
return numa_utils.init_threads_binding(
numa_index=get_device().gpu_id,
world_size=dp_size * parallel.tp_size * parallel.pp_size,
world_size=parallel.dp_size * parallel.tp_size * parallel.pp_size,
)
+6 -6
View File
@@ -394,15 +394,15 @@ class Engine(EngineScoreMixin, EngineBase):
routed_dp_rank = data_parallel_rank
if routed_dp_rank is not None:
dp_size = get_parallel().dp_size
if dp_size <= 1 and routed_dp_rank == 0:
num_dp_ranks = get_parallel().num_dp_ranks
if num_dp_ranks <= 1 and routed_dp_rank == 0:
logger.debug(
f"routed_dp_rank={routed_dp_rank} is ignored because dp_size={dp_size}"
f"routed_dp_rank={routed_dp_rank} is ignored because num_dp_ranks={num_dp_ranks}"
)
return None
if routed_dp_rank < 0 or routed_dp_rank >= dp_size:
if routed_dp_rank < 0 or routed_dp_rank >= num_dp_ranks:
raise ValueError(
f"routed_dp_rank={routed_dp_rank} out of range [0, {dp_size})"
f"routed_dp_rank={routed_dp_rank} out of range [0, {num_dp_ranks})"
)
logger.debug(f"routed_dp_rank: {routed_dp_rank}")
@@ -886,7 +886,7 @@ class Engine(EngineScoreMixin, EngineBase):
"""
scheduler_procs = []
use_dp_controller = (
get_parallel().dp_size > 1 or get_exec().moe.ep_join_mode == "scale"
get_parallel().num_dp_ranks > 1 or get_exec().moe.ep_join_mode == "scale"
)
if not use_dp_controller:
+10 -6
View File
@@ -2249,7 +2249,7 @@ async def _send_disaggregation_warmup_requests(
return await asyncio.gather(
*(
send_request(session, dp_rank)
for dp_rank in range(get_parallel().dp_size)
for dp_rank in range(get_parallel().num_dp_ranks)
)
)
@@ -2313,9 +2313,11 @@ def _execute_server_warmup(server_args: ServerArgs):
},
}
if get_serving().skip_tokenizer_init:
json_data["input_ids"] = [[10, 11, 12] for _ in range(get_parallel().dp_size)]
json_data["input_ids"] = [
[10, 11, 12] for _ in range(get_parallel().num_dp_ranks)
]
# TODO Workaround the bug that embedding errors for list of size 1
if get_parallel().dp_size == 1:
if get_parallel().num_dp_ranks == 1:
json_data["input_ids"] = json_data["input_ids"][0]
elif (
is_vlm
@@ -2359,9 +2361,11 @@ def _execute_server_warmup(server_args: ServerArgs):
"temperature": 0.0,
}
else:
json_data["text"] = ["The capital city of France is"] * get_parallel().dp_size
json_data["text"] = [
"The capital city of France is"
] * get_parallel().num_dp_ranks
# TODO Workaround the bug that embedding errors for list of size 1
if get_parallel().dp_size == 1:
if get_parallel().num_dp_ranks == 1:
json_data["text"] = json_data["text"][0]
# Config debug dumping
@@ -2402,7 +2406,7 @@ def _execute_server_warmup(server_args: ServerArgs):
if not failed_status_codes:
logger.info(
"Disaggregation warmup requests completed for all %s DP ranks",
get_parallel().dp_size,
get_parallel().num_dp_ranks,
)
logger.info("End of disaggregation warmup")
else:
+2 -2
View File
@@ -49,8 +49,8 @@ def _num_accelerators_per_dp_rank() -> int:
"""
parallel = get_parallel()
num_accelerators = parallel.tp_size * parallel.pp_size
if parallel.enable_dp_attention:
num_accelerators //= parallel.dp_size
if parallel.attn_dp_enabled:
num_accelerators //= parallel.num_dp_ranks
return num_accelerators
@@ -283,7 +283,7 @@ class NPUGraphRunner(DecodeCudaGraphRunner):
# Refresh the static DP token buffers bound by the captured
# graph (stale values misalign dp-gather segments across
# ranks); mirror the capture-side uniform
# [padded_num_tokens] * dp_size.
# [padded_num_tokens] * num_dp_ranks.
if self.require_mlp_tp_gather:
_padded_num_tokens = bs * self.captured_req_width
self.buffers.global_num_tokens_gpu.fill_(_padded_num_tokens)
@@ -525,7 +525,7 @@ class AiterAttnBackend(AttentionBackend):
16 if _gathered_num_head < 16 else _gathered_num_head
)
self.enable_dp_attention = is_dp_attention_enabled()
self.attn_dp_enabled = is_dp_attention_enabled()
self.qo_indptr_ = torch.zeros(
(max_bs + 1,), dtype=torch.int32, device=model_runner.device
)
@@ -543,7 +543,7 @@ class AiterAttnBackend(AttentionBackend):
# need to fall back to non-persist
# only use mla_ps_kernel when fp8 kv_cache
# for non-fp8 kv_cache on tp8, use non-persist kernel to avoid performance degradation
# head_num=16 (tp8 perf issue), head_num=128 (unsupported, like tp1 or --enable-dp-attention with tp8-dp8)
# head_num=16 (tp8 perf issue), head_num=128 (unsupported, like tp1 or tp8 with --attn-dp-size 8)
# Native 16-head persist is slow on TP8; keep disabled unless zero-pad
# (e.g. Kimi K3 h12 -> qh16) where persist ASM is the fast path.
if (
@@ -628,7 +628,7 @@ class AiterAttnBackend(AttentionBackend):
nhead = self.mla_kernel_num_head_padded
dtype = self.kv_cache_dtype
if self.enable_dp_attention:
if self.attn_dp_enabled:
gpu = torch.cuda.current_device()
device_properties = torch.cuda.get_device_properties(gpu)
cu_num = device_properties.multi_processor_count
@@ -63,7 +63,7 @@ from sglang.kernels.ops.attention.flash_attention import (
def _should_disable_scheduler_metadata_precompute() -> bool:
return bool(get_parallel().enable_prefill_cp or get_parallel().enable_dp_attention)
return bool(get_parallel().enable_prefill_cp or get_parallel().attn_dp_enabled)
@dataclass
+22 -35
View File
@@ -23,7 +23,7 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
from sglang.srt.environ import envs
from sglang.srt.runtime_context import (
derive_attention_ranks,
derive_attention_widths,
derive_attn_tp_size,
get_device,
get_exec,
get_flags,
@@ -51,13 +51,7 @@ def deployment_attn_dp_size() -> int:
the target's replicas.
"""
parallel = get_parallel()
attn_dp_size, _ = derive_attention_widths(
tp_size=parallel.tp_size,
attn_cp_size=parallel.attn_cp_size,
dp_size=parallel.dp_size,
enable_dp_attention=parallel.enable_dp_attention,
)
return attn_dp_size
return parallel.num_dp_ranks if parallel.attn_dp_enabled else 1
def dp_gather_width() -> int:
@@ -69,7 +63,7 @@ def dp_gather_width() -> int:
"""
parallel = get_parallel()
if world_dp_gather_enabled():
return parallel.dp_size
return parallel.num_dp_ranks
if get_flags().dp.scoped_gather_slot is not None:
return deployment_attn_dp_size()
return parallel.attn_dp_size
@@ -127,7 +121,7 @@ def update_dp_attention_post_scale(new_dp_size: int, new_dp_rank: int):
"""
get_flags().dp.use_world_group_for_gather = True
logger.debug(
"[Elastic EP] dp_attention switched to WORLD: dp_size=%d dp_rank=%d",
"[Elastic EP] dp_attention switched to WORLD: num_dp_ranks=%d dp_rank=%d",
new_dp_size,
new_dp_rank,
)
@@ -154,7 +148,7 @@ class DpPaddingMode(IntEnum):
def get_dp_padding_mode(
cls, is_extend_in_batch, global_num_tokens: List[int]
) -> DpPaddingMode:
dp_size = dp_gather_width()
gather_width = dp_gather_width()
# (trangdough) pplx-kernels a2a is a symmetric collective: every EP rank
# must dispatch the same number of tokens or the device-side handshake
@@ -169,11 +163,11 @@ class DpPaddingMode(IntEnum):
if moe_a2a_backend.is_deepep_v2() and envs.SGLANG_DEEPEP_V2_FORCE_MAX_LEN.get():
return DpPaddingMode.MAX_LEN
# When is_extend_in_batch and dp_size > 1, use SUM_LEN to avoid padding
# overhead from uneven token distribution.
# For dp_size=1, max_len equals sum_len, so prefer MAX_LEN mode
# When is_extend_in_batch and the gather spans several DP ranks, use
# SUM_LEN to avoid padding overhead from uneven token distribution.
# Over one rank, max_len equals sum_len, so prefer MAX_LEN mode
# to enable symmetric memory optimization (needed for DSA CP, etc.).
if is_extend_in_batch and dp_size > 1:
if is_extend_in_batch and gather_width > 1:
# Hybrid-SSM models materialize idle ranks via the MAX_LEN
# fabricated-row conversion; other models keep mainline SUM_LEN.
if get_flags().dp.max_len_with_idle and min(global_num_tokens) == 0:
@@ -184,7 +178,7 @@ class DpPaddingMode(IntEnum):
# prefer MAX_LEN when communication cost is equal to enable symmetric memory
max_len = max(global_num_tokens)
sum_len = sum(global_num_tokens)
if sum_len * 2 >= max_len * dp_size:
if sum_len * 2 >= max_len * gather_width:
return cls.MAX_LEN
else:
return cls.SUM_LEN
@@ -405,25 +399,19 @@ def is_dp_max_padding() -> bool:
def compute_dp_attention_world_info(
enable_dp_attention, tp_rank, tp_size, dp_size, attn_cp_size: int = 1
tp_rank, tp_size, attn_dp_size, attn_cp_size: int = 1
):
"""This rank's place in the attention topology, plus the widths it sits in.
The widths come from `derive_attention_widths`; what this adds is the two
ranks, which are per-process and so are not among the widths
The attention-TP width comes from `derive_attn_tp_size`; what this adds is
the two ranks, which are per-process and so are not among the widths
`override_permanently` records.
"""
attn_dp_size, attn_tp_size = derive_attention_widths(
tp_size=tp_size,
attn_cp_size=attn_cp_size,
dp_size=dp_size,
enable_dp_attention=enable_dp_attention,
attn_tp_size = derive_attn_tp_size(
tp_size=tp_size, attn_cp_size=attn_cp_size, attn_dp_size=attn_dp_size
)
attn_tp_rank, attn_dp_rank = derive_attention_ranks(
tp_rank=tp_rank,
attn_tp_size=attn_tp_size,
attn_cp_size=attn_cp_size,
enable_dp_attention=enable_dp_attention,
tp_rank=tp_rank, attn_tp_size=attn_tp_size, attn_cp_size=attn_cp_size
)
return attn_tp_rank, attn_tp_size, attn_dp_rank, attn_dp_size
@@ -431,7 +419,7 @@ def compute_dp_attention_world_info(
def initialize_dp_attention_flags(server_args: ServerArgs):
"""Initialize DP runtime flags without changing the worker's placement."""
dp = get_flags().dp
dp.enabled = get_parallel().enable_dp_attention
dp.enabled = get_parallel().attn_dp_enabled
if get_exec().moe.elastic_ep_backend is not None and get_parallel().max_ep_size:
if ep_scale_joiner_of(resolving_view(server_args)):
@@ -443,10 +431,9 @@ def initialize_dp_attention(server_args: ServerArgs):
initialize_dp_attention_flags(server_args)
parallel = get_parallel()
_, _, attn_dp_rank, attn_dp_size = compute_dp_attention_world_info(
parallel.enable_dp_attention,
parallel.tp_rank,
parallel.tp_size,
parallel.dp_size,
parallel.attn_dp_size,
parallel.attn_cp_size,
)
parallel.override_permanently(attn_dp_size=attn_dp_size, attn_dp_rank=attn_dp_rank)
@@ -652,7 +639,7 @@ def _dp_gather_via_all_gather(
# Variable-length DP-MoE gather (reference https://github.com/ROCm/ATOM/pull/930): instead of padding every
# rank to max_len (all_gather) or all-reducing a sum_len zero-buffer (all_reduce),
# gather exactly sum(per-rank tokens) via all_gatherv. Env-gated; only the simple
# tp_size==dp_size (attn_tp_size==1) case is supported for now (e.g. tp8dp8).
# tp_size==attn_dp_size (attn_tp_size==1) case is supported for now (e.g. tp8, attn_dp8).
_USE_DP_GATHERV = get_bool_env_var("SGLANG_DP_USE_GATHERV")
_DP_GATHER_FP8_GROUP = 128
@@ -725,7 +712,7 @@ def mask_dp_pad_moe_topk_ids(topk_ids: torch.Tensor) -> None:
"""Set MAX_LEN pad rows' (post-translation, local) topk_ids to -1 in place.
Under dp-attention MAX_LEN padding the gathered MoE buffer is
[dp_size * max_len, hidden] with rank r's real rows at
[num_dp_ranks * max_len, hidden] with rank r's real rows at
[r*max_len, r*max_len + global_num_tokens[r]); the pad rows carry stale
hidden values, run the router, and get dispatched into experts whose
outputs are then discarded by the post-reorder scatter — pure wasted
@@ -794,7 +781,7 @@ def is_dp_gatherv_active() -> bool:
"""Variable-length DP-MoE gather/scatter (all_gatherv + reduce_scatterv) is
enabled and applicable to the CURRENT forward. Requires:
- env SGLANG_DP_USE_GATHERV (default off),
- supported layout (attn_tp_size==1, tp_size==dp_size),
- supported layout (attn_tp_size==1, tp_size==attn_dp_size),
- SUM_LEN padding mode. The gatherv pair (all_gatherv + reduce_scatterv) is
only valid under SUM_LEN; under MAX_LEN the buffer is equal-padded and the
gather/combine use all_gather / (aiter) reduce_scatter instead. Reading the
@@ -1004,7 +991,7 @@ def can_use_dp_reduce_scatter() -> bool:
return True
parallel = get_parallel()
return parallel.tp_size == parallel.dp_size * parallel.attn_tp_size
return parallel.tp_size == parallel.num_dp_ranks * parallel.attn_tp_size
def dp_reduce_scatter_tensor(output: torch.Tensor, input: torch.Tensor):
@@ -175,7 +175,7 @@ class StagePlan:
self._speculative_algo = SpeculativeAlgorithm.from_string(
get_spec().speculative_algorithm
)
self._publish_lora_layout = get_parallel().enable_dp_attention and bool(
self._publish_lora_layout = get_parallel().attn_dp_enabled and bool(
get_lora().enable_lora
)
self._next_input_rows = None
+2 -2
View File
@@ -84,7 +84,7 @@ _UNQUANTIZED_LM_HEAD_METHODS = {
# None outside a FlashInfer autotune pass; inside one, whether that pass runs the
# LM head. Not-None means the forward's output is discarded -- attention backends
# read that via get_in_autotune_dummy_run() to skip a cross-node exchange.
# Skipping the LM head skips its [batch * dp_size, vocab] all-gather, which OOMs
# Skipping the LM head skips its [batch * num_dp_ranks, vocab] all-gather, which OOMs
# under DP attention with a tight mem_fraction_static.
_autotune_run_lm_head: Optional[bool] = None
@@ -192,7 +192,7 @@ def should_apply_lm_head_quant_method(lm_head, quant_method) -> bool:
# FlashInfer autotune skips the unprofiled LM-head all-gather; its
# [batch * dp_size, vocab] output can OOM under tight DP-attention memory.
# [batch * num_dp_ranks, vocab] output can OOM under tight DP-attention memory.
_in_autotune_dummy_run = False
@@ -340,9 +340,9 @@ def ensure_cutedsl_wrapper(layer: torch.nn.Module) -> None:
# A2A path: bounded by the dispatcher's own workspace limit.
max_num_tokens = dispatcher.max_num_tokens * getattr(dispatcher, "ep_size", 1)
else:
# Standard allgather path: the MoE sees up to dp_size local forwards
# gathered together, so scale the per-rank forward bound by dp_size.
max_num_tokens = get_parallel().dp_size * cutedsl_moe_max_num_tokens()
# Standard allgather path: the MoE sees up to num_dp_ranks local forwards
# gathered together, so scale the per-rank forward bound by num_dp_ranks.
max_num_tokens = get_parallel().num_dp_ranks * cutedsl_moe_max_num_tokens()
top_k = layer.top_k if layer.top_k is not None else layer.moe_runner_config.top_k
# inference_mode(False) ensures the wrapper's pre-allocated CUDA-graph
# buffers are normal tensors. This call typically happens inside
+14 -19
View File
@@ -99,9 +99,9 @@ class LoRAManager:
self.enable_lora_overlap_loading: Optional[bool] = (
get_lora().enable_lora_overlap_loading
)
self.enable_dp_attention: bool = get_parallel().enable_dp_attention
self.attn_dp_enabled: bool = get_parallel().attn_dp_enabled
if (
self.enable_dp_attention
self.attn_dp_enabled
and self.enable_lora_overlap_loading
and not LoRAMemoryPool.supports_dp_attention_overlap_loading
):
@@ -112,11 +112,11 @@ class LoRAManager:
)
# LoRA routing knows only DP-local and TP-global token layouts. A wider
# attention TP/CP group can scatter one DP rank's tokens across ranks.
if self.enable_dp_attention and (
if self.attn_dp_enabled and (
self.attn_tp_size != 1 or get_parallel().attn_cp_size != 1
):
raise ValueError(
"LoRA with DP attention requires --dp-size equal to --tp-size "
"LoRA with DP attention requires --attn-dp-size equal to --tp-size "
f"(got attention TP size {self.attn_tp_size} and attention CP "
f"size {get_parallel().attn_cp_size})"
)
@@ -138,7 +138,7 @@ class LoRAManager:
device=self.device,
server_args=server_args,
)
if self.enable_dp_attention and not self.lora_backend.supports_dp_attention:
if self.attn_dp_enabled and not self.lora_backend.supports_dp_attention:
raise ValueError(
f"LoRA backend {lora_backend!r} does not support DP attention"
)
@@ -164,9 +164,9 @@ class LoRAManager:
num_tokens_per_req=num_tokens_per_req,
)
max_moe_tokens = max_bs_in_cuda_graph * num_tokens_per_req
if self.enable_dp_attention:
if self.attn_dp_enabled:
self.lora_backend.init_dp_attention_cuda_graph_batch_info(max_moe_tokens)
max_moe_tokens *= get_parallel().dp_size
max_moe_tokens *= get_parallel().num_dp_ranks
if self.lora_backend.resize_cuda_graph_moe_buffers(max_moe_tokens):
logger.info(
"Right-sized shared MoE LoRA CUDA graph buffers "
@@ -211,10 +211,7 @@ class LoRAManager:
check_cuda_graph_backend,
)
if (
self.enable_dp_attention
or not self.lora_backend.supports_prefill_cuda_graph
):
if self.attn_dp_enabled or not self.lora_backend.supports_prefill_cuda_graph:
return False
if self.lora_backend.is_moe_lora:
return check_cuda_graph_backend(
@@ -237,7 +234,7 @@ class LoRAManager:
return False
# DP attention: per-rank eligibility could diverge across ranks and
# desync collectives; keep LoRA prefill eager.
if self.enable_dp_attention:
if self.attn_dp_enabled:
return False
# Decode-CUDA-graph extend modes (TARGET_VERIFY, DLLM_EXTEND) are
# owned by the decode static batch info path.
@@ -333,7 +330,7 @@ class LoRAManager:
"""
Validate if an adapter can be loaded into the current LoRA memory pool and generate error if it is incompatible.
"""
assert not self.enable_dp_attention or lora_ref.pinned, (
assert not self.attn_dp_enabled or lora_ref.pinned, (
"DP-attention LoRA requires pinned adapters"
)
if lora_config.lora_added_tokens_size > 0:
@@ -506,7 +503,7 @@ class LoRAManager:
)
active_lora_ids = set(forward_batch.lora_ids)
if self.enable_dp_attention:
if self.attn_dp_enabled:
get_forward().set("lora_batch_layout", LoRABatchLayout.DP_LOCAL)
gathered_lora_ids = get_parallel().tp_group.all_gather_object(
forward_batch.lora_ids
@@ -551,7 +548,7 @@ class LoRAManager:
self.lora_backend.batch_info.has_active_lora = any(
lora_ranks[wi] > 0 for wi in weight_indices
)
if self.enable_dp_attention:
if self.attn_dp_enabled:
self.lora_backend.prepare_global_lora_batch(forward_batch)
if forward_batch.forward_mode.is_idle():
# Idle ranks must join the global routing collectives, but
@@ -563,9 +560,7 @@ class LoRAManager:
hasattr(self, "max_bs_in_cuda_graph")
and forward_batch.batch_size <= self.max_bs_in_cuda_graph
and forward_batch.forward_mode.is_cuda_graph()
and (
not self.enable_dp_attention or forward_batch.can_run_decode_cuda_graph
)
and (not self.attn_dp_enabled or forward_batch.can_run_decode_cuda_graph)
)
def prepare_lora_token_segments(
@@ -1194,7 +1189,7 @@ def init_lora_cuda_graph_moe_buffers(
# num_draft_tokens per request, and the buffers below are per-token, so
# they must be sized in tokens rather than requests.
max_tokens = max_bs * (get_spec().speculative_num_draft_tokens or 1)
if get_parallel().enable_dp_attention:
if get_parallel().attn_dp_enabled:
max_tokens *= get_parallel().attn_dp_size
max_loras = get_lora().max_loras_per_batch
for module in model.modules():
+3 -3
View File
@@ -180,7 +180,7 @@ class LoRAMemoryPool:
)
# Per-expert MoE weights are sharded by `moe_tp_size`, NOT the outer
# `tp_size`: `moe_tp_size = tp_size // ep_size // dp_size`, so under
# `tp_size`: `moe_tp_size = tp_size // ep_size // moe_dp_size`, so under
# e.g. `--tp 4 --ep 4` each rank holds full-width expert weights
# (`moe_tp_size == 1`). Sizing per-expert LoRA buffers by `tp_size`
# here would yield a 4x-narrower inner dim than the adapter weight
@@ -190,7 +190,7 @@ class LoRAMemoryPool:
self.moe_tp_size, self.moe_tp_rank = _get_moe_tp_context()
# Attention projections shard along the attention TP group, which
# under `--enable-dp-attention` is `attn_tp_size = tp_size // dp_size`.
# under attention DP is `attn_tp_size = tp_size // attn_dp_size`.
# The corresponding LoRA wrappers slice weights by the base layer's
# attn_tp-local rank, so the buffer shapes must match that shard.
self.attn_tp_size: int = attn_tp_size
@@ -273,7 +273,7 @@ class LoRAMemoryPool:
"""TP width the module's weights are actually sharded along: routed
MoE experts shard by `moe_tp_size` (shared experts by the outer
`tp_size` at EP=1), attention projections by `attn_tp_size` (smaller
than the outer `tp_size` under `--enable-dp-attention`), everything
than the outer `tp_size` under attention DP), everything
else by the outer `tp_size`."""
if self.is_moe_module(module_name) and not self.is_shared_moe_module(
module_name
+1 -1
View File
@@ -373,7 +373,7 @@ REPLICATED_LINEAR_LORA_NAMES = [
*DSA_INDEXER_LORA_NAMES,
]
# Attention-projection LoRA modules shard on the attention-TP group, which
# under `--enable-dp-attention` is `attn_tp_size = tp_size // dp_size` rather
# under attention DP is `attn_tp_size = tp_size // attn_dp_size` rather
# than the outer TP size. in_proj / in_proj_qkvz (linear-attention hybrids)
# belong here too: their layers are built on the attn-TP group (mamba.py and
# qwen3_5.py take tp_size/tp_rank from attn_tp when dp attention is enabled).
@@ -102,11 +102,11 @@ class LoadBalanceMethod(Enum):
class DPBudget:
def __init__(self, dp_size: int):
self.dp_size = dp_size
self.total_requests = [0] * dp_size
self.total_tokens = [0] * dp_size
self.last_timestamp = [0.0] * dp_size
def __init__(self, num_dp_ranks: int):
self.num_dp_ranks = num_dp_ranks
self.total_requests = [0] * num_dp_ranks
self.total_tokens = [0] * num_dp_ranks
self.last_timestamp = [0.0] * num_dp_ranks
def update_budget(self, loads):
"""Update budget from shm snapshots, skipping stale reads."""
@@ -125,7 +125,7 @@ class DPBudget:
elif method == LoadBalanceMethod.TOTAL_TOKENS:
# Use total_requests as a tie-breaker when total_tokens are equal
target_rank = min(
range(self.dp_size),
range(self.num_dp_ranks),
key=lambda i: (self.total_tokens[i], self.total_requests[i]),
)
else:
@@ -155,7 +155,7 @@ class DataParallelController:
self.run_scheduler_process_func = run_scheduler_process_func
# Init inter-process communication
self.context = zmq.Context(1 + get_parallel().dp_size)
self.context = zmq.Context(1 + get_parallel().num_dp_ranks)
if get_parallel().node_rank == 0:
self.recv_from_tokenizer = get_zmq_socket(
self.context, zmq.PULL, port_args.scheduler_input_ipc_name, False
@@ -175,8 +175,10 @@ class DataParallelController:
LoadBalanceMethod.TOTAL_TOKENS,
)
self.launch_dp_size: int = get_parallel().dp_size
self.max_dp_size: int = get_parallel().max_ep_size or get_parallel().dp_size
self.launch_dp_size: int = get_parallel().num_dp_ranks
self.max_dp_size: int = (
get_parallel().max_ep_size or get_parallel().num_dp_ranks
)
assert self.max_dp_size >= self.launch_dp_size, (
f"--max-ep-size ({self.max_dp_size}) must be >= "
f"--dp ({self.launch_dp_size})."
@@ -186,7 +188,7 @@ class DataParallelController:
self.max_dp_size - self.launch_dp_size
)
self.dp_budget = DPBudget(get_parallel().dp_size)
self.dp_budget = DPBudget(get_parallel().num_dp_ranks)
self.load_snapshot_reader = create_load_snapshot_reader(
port_args,
caller="DataParallelController",
@@ -203,7 +205,7 @@ class DataParallelController:
self._active_workers: list[int] = list(range(self.launch_dp_size))
self._active_count_cache: int = self.launch_dp_size
if get_parallel().enable_dp_attention:
if get_parallel().attn_dp_enabled:
self.launch_dp_attention_schedulers(server_args, port_args)
# When local control broadcast is enabled, send control messages to
# every DP group leader (attn_tp_rank=0) so each leader broadcasts
@@ -374,7 +376,7 @@ class DataParallelController:
threads = []
sockets = []
ready_events = []
for dp_rank in range(get_parallel().dp_size):
for dp_rank in range(get_parallel().num_dp_ranks):
tmp_port_args = PortArgs.init_new(server_args)
tmp_port_args.tokenizer_ipc_name = port_args.tokenizer_ipc_name
tmp_port_args.detokenizer_ipc_name = port_args.detokenizer_ipc_name
@@ -577,7 +579,7 @@ class DataParallelController:
bind_count = (
self.max_dp_size
if get_exec().moe.elastic_ep_backend is not None
else get_parallel().dp_size
else get_parallel().num_dp_ranks
)
for slot in range(bind_count):
worker_port, worker_socket = get_zmq_socket_on_host(
@@ -607,7 +609,7 @@ class DataParallelController:
dp_rank: int | None,
worker_ports: list[int] | None = None,
):
if not get_parallel().enable_dp_attention:
if not get_parallel().attn_dp_enabled:
logger.info(f"Launch DP{dp_rank} starting at GPU #{base_gpu_id}.")
memory_saver_adapter = TorchMemorySaverAdapter.create(
@@ -639,13 +641,12 @@ class DataParallelController:
for tp_rank in tp_rank_range:
rank_port_args = port_args
if get_parallel().enable_dp_attention:
if get_parallel().attn_dp_enabled:
# dp attention has different sharding logic
_, _, dp_rank, _ = compute_dp_attention_world_info(
get_parallel().enable_dp_attention,
tp_rank,
get_parallel().tp_size,
get_parallel().dp_size,
get_parallel().attn_dp_size,
get_parallel().attn_cp_size,
)
# compute zmq ports for this dp rank
+6 -6
View File
@@ -71,7 +71,7 @@ def should_use_zmq() -> bool:
``SGLANG_LOAD_SNAPSHOT_USE_ZMQ`` forces zmq mode for testing.
"""
return (
get_parallel().enable_dp_attention and get_parallel().nnodes > 1
get_parallel().attn_dp_enabled and get_parallel().nnodes > 1
) or envs.SGLANG_LOAD_SNAPSHOT_USE_ZMQ.get()
@@ -120,10 +120,10 @@ def zmq_reader_owner(caller: str) -> bool:
return False
if caller == "DataParallelController":
return (
get_parallel().dp_size > 1
get_parallel().num_dp_ranks > 1
and get_parallel().load_balance_method.lower() in _LOAD_AWARE_METHODS
)
if get_parallel().dp_size > 1 and (
if get_parallel().num_dp_ranks > 1 and (
get_parallel().load_balance_method.lower() in _LOAD_AWARE_METHODS
):
return False
@@ -659,9 +659,9 @@ def create_load_snapshot_reader(port_args, caller: str):
``"MultiTokenizerRouter"`` -- determines who binds the zmq PULL
socket when zmq mode is active.
"""
dp_size = get_parallel().dp_size
num_dp_ranks = get_parallel().num_dp_ranks
if zmq_reader_owner(caller):
return ZmqShmLoadSnapshotReader(
_zmq_addr_for(port_args), shm_path_for(port_args.instance_id), dp_size
_zmq_addr_for(port_args), shm_path_for(port_args.instance_id), num_dp_ranks
)
return ShmLoadSnapshotReader(shm_path_for(port_args.instance_id), dp_size)
return ShmLoadSnapshotReader(shm_path_for(port_args.instance_id), num_dp_ranks)
@@ -106,9 +106,9 @@ class PrefillDelayer:
f"queue_trigger_enabled={self._queue_trigger_enabled}"
)
parallel = get_parallel()
self.dp_size = parallel.dp_size
self.enable_dp_attention = parallel.enable_dp_attention
dp_size_dim = self.dp_size if self.enable_dp_attention else 1
self.num_dp_ranks = parallel.num_dp_ranks
self.attn_dp_enabled = parallel.attn_dp_enabled
dp_size_dim = self.num_dp_ranks if self.attn_dp_enabled else 1
# Mirror scheduler_dp_attn_mixin's NCCL all-gather path: when the
# env flag is on (or overlap scheduling is disabled), ride the NCCL
@@ -238,10 +238,10 @@ class PrefillDelayer:
**wait_info,
)
if not self.enable_dp_attention:
if not self.attn_dp_enabled:
max_running_requests = (
max_running_requests + self.dp_size - 1
) // self.dp_size
max_running_requests + self.num_dp_ranks - 1
) // self.num_dp_ranks
global_running_batch_max = int(global_running_batch.max().item())
global_max_prefill_bs_max = int(global_max_prefill_bs.max().item())
@@ -411,10 +411,10 @@ class PrefillDelayerSinglePassExecutor:
waiting_queue_len: int,
) -> int:
local_max_running_requests = max_running_requests
if not self._prefill_delayer.enable_dp_attention:
if not self._prefill_delayer.attn_dp_enabled:
local_max_running_requests = (
max_running_requests + self._prefill_delayer.dp_size - 1
) // self._prefill_delayer.dp_size
max_running_requests + self._prefill_delayer.num_dp_ranks - 1
) // self._prefill_delayer.num_dp_ranks
# The delayer negotiates before PrefillAdder materializes can_run_list,
# so a rejected pass has no exact batch size. This upper bound is exact
+5 -10
View File
@@ -516,7 +516,7 @@ class Scheduler(
self.max_recv_per_poll = envs.SGLANG_SCHEDULER_MAX_RECV_PER_POLL.get()
self.max_new_tokens_limit = envs.SGLANG_MAX_NEW_TOKENS_LIMIT.get()
self.enable_hisparse = get_memory().enable_hisparse
self.enable_dp_attention = get_parallel().enable_dp_attention
self.attn_dp_enabled = get_parallel().attn_dp_enabled
self.enable_unified_memory = get_memory().enable_unified_memory
# Init model configs
@@ -629,7 +629,7 @@ class Scheduler(
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
tp_group=(
self.attn_tp_cpu_group
if self.enable_dp_attention
if self.attn_dp_enabled
else self.tp_cpu_group
),
tree_cache=self.tree_cache,
@@ -823,7 +823,7 @@ class Scheduler(
try:
self.load_snapshot_writer = create_load_snapshot_writer(
port_args,
get_parallel().dp_size,
get_parallel().num_dp_ranks,
dp_rank,
publish_interval=get_observability().load_snapshot_publish_interval,
)
@@ -1216,9 +1216,7 @@ class Scheduler(
# the base TP group. Entry rank is the local rank 0 in that group.
# Use the CPU (gloo) group to broadcast VLM Python objects and avoid CUDA
# stream/device coupling (#11910).
self.dp_tp_group = (
self.attn_tp_group if self.enable_dp_attention else self.tp_group
)
self.dp_tp_group = self.attn_tp_group if self.attn_dp_enabled else self.tp_group
self.dp_tp_cpu_group = self.dp_tp_group.cpu_group
self.pad_input_ids_func = self.tp_worker.get_pad_input_ids_func()
@@ -4646,9 +4644,7 @@ class Scheduler(
return ret
def _maybe_report_active_ranks(self) -> None:
if not (
self.enable_dp_attention and get_exec().moe.elastic_ep_backend is not None
):
if not (self.attn_dp_enabled and get_exec().moe.elastic_ep_backend is not None):
return
from sglang.srt.elastic_ep.elastic_ep import ElasticEPStateManager
@@ -5218,7 +5214,6 @@ class Scheduler(
# readback reflects values changed via /set_internal_state, not startup.
ret = get_context().resolved_server_args_dict()
ret["world_size"] = compute_world_size(
enable_dp_attention=get_parallel().enable_dp_attention,
dp_size=get_parallel().dp_size,
tp_size=get_parallel().tp_size,
pp_size=get_parallel().pp_size,
@@ -65,47 +65,47 @@ def _spec_input_cuda_graph_compatible(
return spec_info is None or spec_info.cuda_graph_compatible
def _resolve_elastic_world_dp_size(
dp_size: int,
def _resolve_elastic_world_num_dp_ranks(
num_dp_ranks: int,
*,
group: torch.distributed.ProcessGroup,
local_num_tokens: int,
local_forward_mode: int,
) -> int:
if not world_dp_gather_enabled():
return dp_size
return num_dp_ranks
from sglang.srt.elastic_ep.elastic_ep import ElasticEPStateManager
live_dp_size = dp_gather_width()
live_num_dp_ranks = dp_gather_width()
effective_ep_size = ElasticEPStateManager.get_effective_ep_size()
# Query live membership because elastic joins can expand WORLD.
world_size = torch.distributed.get_world_size(group)
if live_dp_size != effective_ep_size:
if live_num_dp_ranks != effective_ep_size:
raise RuntimeError(
"[Elastic EP] WORLD MLP sync dp_size is out of sync: "
"[Elastic EP] WORLD MLP sync num_dp_ranks is out of sync: "
f"rank={torch.distributed.get_rank(group)} "
f"live_dp_size={live_dp_size} "
f"live_num_dp_ranks={live_num_dp_ranks} "
f"effective_ep_size={effective_ep_size} "
f"world_size={world_size} server_args_dp_size={dp_size} "
f"world_size={world_size} configured_num_dp_ranks={num_dp_ranks} "
f"local_num_tokens={local_num_tokens} "
f"local_forward_mode={local_forward_mode}"
)
if live_dp_size > world_size:
if live_num_dp_ranks > world_size:
raise RuntimeError(
"[Elastic EP] WORLD MLP sync dp_size exceeds WORLD size: "
"[Elastic EP] WORLD MLP sync num_dp_ranks exceeds WORLD size: "
f"rank={torch.distributed.get_rank(group)} "
f"live_dp_size={live_dp_size} world_size={world_size} "
f"live_num_dp_ranks={live_num_dp_ranks} world_size={world_size} "
f"effective_ep_size={effective_ep_size}"
)
return live_dp_size
return live_num_dp_ranks
@dataclass
class MLPSyncBatchInfo:
dp_size: int
num_dp_ranks: int
tp_size: int
cp_size: int
@@ -186,7 +186,7 @@ class MLPSyncBatchInfo:
# is a no-op, and the masked fallback writes below would then read and
# write the same storage.
global_info_tensor = fallback_tensor.repeat(
self.dp_size, self.tp_size * self.cp_size, 1
self.num_dp_ranks, self.tp_size * self.cp_size, 1
)
if use_all_reduce:
@@ -211,7 +211,7 @@ class MLPSyncBatchInfo:
)
tp_info = global_info_tensor.view(
self.dp_size * self.tp_size * self.cp_size, info_width
self.num_dp_ranks * self.tp_size * self.cp_size, info_width
)
num_ranks_in_tp_info = tp_info.shape[0]
if device == "cpu":
@@ -296,7 +296,7 @@ def _update_gather_batch(
)
def should_skip_scheduler_all_gather(dp_size: int) -> bool:
def should_skip_scheduler_all_gather(num_dp_ranks: int) -> bool:
"""Return whether scheduler metadata is already local and rank-invariant.
With one attention-DP rank there is no cross-DP state to reconcile. The
@@ -307,7 +307,7 @@ def should_skip_scheduler_all_gather(dp_size: int) -> bool:
DP1.
"""
return dp_size == 1 or envs.SGLANG_SCHEDULER_SKIP_ALL_GATHER.get()
return num_dp_ranks == 1 or envs.SGLANG_SCHEDULER_SKIP_ALL_GATHER.get()
def _local_decode_cuda_graph_vote(
@@ -421,7 +421,7 @@ def prepare_mlp_sync_batch_raw(
dwdp: bool = False,
):
parallel = get_parallel()
dp_size = parallel.dp_size
num_dp_ranks = parallel.num_dp_ranks
attn_tp_size = parallel.attn_tp_size
tp_group = parallel.tp_group
# Check if other DP workers have running batches
@@ -498,16 +498,16 @@ def prepare_mlp_sync_batch_raw(
local_can_run_tbo, local_forward_mode = tbo_preparer.prepare_all_gather(local_batch)
if use_world_group:
dp_size = _resolve_elastic_world_dp_size(
dp_size,
num_dp_ranks = _resolve_elastic_world_num_dp_ranks(
num_dp_ranks,
group=group,
local_num_tokens=num_tokens,
local_forward_mode=local_forward_mode,
)
skip_all_gather = should_skip_scheduler_all_gather(dp_size)
skip_all_gather = should_skip_scheduler_all_gather(num_dp_ranks)
mlp_sync_info = MLPSyncBatchInfo(
dp_size=dp_size,
num_dp_ranks=num_dp_ranks,
tp_size=attn_tp_size,
cp_size=parallel.attn_cp_size,
num_tokens=num_tokens,
@@ -521,7 +521,7 @@ def prepare_mlp_sync_batch_raw(
prefill_cuda_graph_max_prefix_len=prefill_cuda_graph_max_prefix_len,
)
if dp_size == 1:
if num_dp_ranks == 1:
mlp_sync_info.finalize_local()
elif not skip_all_gather:
mlp_sync_info.all_gather(
@@ -540,7 +540,7 @@ def prepare_mlp_sync_batch_raw(
# Decide whether to emit idle batch
if skip_all_gather:
# Skip idle batch when attn-dp=1 (and always under DWDP: ranks run independently)
need_idle_batch = not dwdp and dp_size > 1
need_idle_batch = not dwdp and num_dp_ranks > 1
else:
need_idle_batch = max(mlp_sync_info.global_num_tokens) > 0
@@ -191,8 +191,7 @@ class DynamicChunkSizer:
if is_dp_attention_enabled():
# Profiling runs one request on this rank; other DP ranks report 0.
dp_size = get_parallel().attn_dp_size
global_num_tokens = [0] * dp_size
global_num_tokens = [0] * get_parallel().attn_dp_size
dp_rank = get_parallel().attn_dp_rank
global_num_tokens[dp_rank] = current_seq_len
batch.global_num_tokens = global_num_tokens
@@ -19,7 +19,7 @@ an explicit address); off by default so an upgrade never reserves a port a
co-hosted neighbor's KV publisher binds. The port comes from
`resolve_load_pub_range` (the same function `/server_info` advertises with,
so the two cannot drift). With `auto`, a worker's ZMQ footprint is
`2 * dp_size` ports after its KV base (`2 * dp_size + 1` with the
`2 * num_dp_ranks` ports after its KV base (`2 * num_dp_ranks + 1` with the
conventional adjacent replay), so co-hosted workers must space their KV
bases that far apart or move the range with an explicit address.
"""
@@ -164,7 +164,7 @@ class SchedulerLoadPublisher:
resolved, reason = resolve_load_pub_range(
kv_endpoint=cfg.endpoint,
replay_endpoint=cfg.replay_endpoint,
dp_size=get_parallel().dp_size,
dp_size=get_parallel().num_dp_ranks,
load_publish_endpoint=load_publish_endpoint,
)
if resolved is None:
@@ -285,7 +285,7 @@ class SchedulerProfilerManager:
if get_parallel().tp_rank != 0:
return ""
if get_parallel().dp_size > 1 and get_parallel().dp_rank != 0:
if get_parallel().num_dp_ranks > 1 and get_parallel().dp_rank != 0:
return ""
if get_parallel().pp_size > 1 and get_parallel().pp_rank != 0:
return ""
@@ -340,7 +340,7 @@ class SchedulerProfilerManager:
filename_parts = [self.profile_id, f"TP-{get_parallel().tp_rank}"]
# Only add other ranks if parallelism is enabled (size > 1)
if get_parallel().dp_size > 1:
if get_parallel().num_dp_ranks > 1:
filename_parts.append(f"DP-{get_parallel().dp_rank}")
if get_parallel().pp_size > 1:
filename_parts.append(f"PP-{get_parallel().pp_rank}")
@@ -34,7 +34,7 @@ class SchedulerRecvSkipper:
return ForwardMode.DECODE
def __init__(self):
self._use_synced_mode = get_parallel().enable_dp_attention
self._use_synced_mode = get_parallel().attn_dp_enabled
self._counter = 0
self._threshold = get_schedule().scheduler_recv_interval
# All can be tuned if needed
@@ -173,7 +173,7 @@ class SchedulerRequestReceiver:
rank 0 and would overwrite every DP group's aborts but the first.
"""
local_reqs = local_reqs or []
if get_parallel().enable_dp_attention:
if get_parallel().attn_dp_enabled:
if get_parallel().attn_tp_rank == 0 and get_parallel().attn_cp_rank == 0:
work_reqs, control_reqs = self._split_work_and_control_reqs(recv_reqs)
work_reqs.extend(local_reqs)
@@ -266,7 +266,7 @@ class SchedulerRequestReceiver:
# 1. wait until every rank has opened the shared feature segments
parallel = get_parallel()
if parallel.enable_dp_attention:
if parallel.attn_dp_enabled:
if parallel.attn_tp_size > 1:
barrier(group=self.attn_tp_cpu_group)
if parallel.attn_cp_size > 1:
@@ -290,7 +290,7 @@ class SchedulerRequestReceiver:
failed[index] = 1
# 3. all ranks reject the same requests before entering model collectives
if parallel.enable_dp_attention:
if parallel.attn_dp_enabled:
if parallel.attn_tp_size > 1:
all_reduce(failed, op=ReduceOp.MAX, group=self.attn_tp_cpu_group)
if parallel.attn_cp_size > 1:
@@ -186,7 +186,7 @@ class TokenizerControlMixin:
mode = spec[2] if len(spec) > 2 else "queueing"
comm = FanOutCommunicator(
self._dispatch_to_scheduler,
get_parallel().dp_size,
get_parallel().num_dp_ranks,
mode,
)
setattr(self, f"{name}_communicator", comm)
@@ -195,7 +195,7 @@ class TokenizerControlMixin:
def update_control_communicator_fan_out(self: TokenizerManager, worker_count: int):
primary_group_control = (
get_parallel().enable_dp_attention
get_parallel().attn_dp_enabled
and not get_parallel().enable_dp_attention_local_control_broadcast
)
if primary_group_control:
@@ -444,8 +444,8 @@ class TokenizerControlMixin:
request: Optional[fastapi.Request] = None,
) -> Tuple[bool, str]:
self.auto_create_handle_loop()
assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, (
"dp_size must be 1 or dp attention must be enabled for update weights from distributed"
assert get_parallel().dp_size == 1, (
"data-parallel replicas are not supported for update weights from distributed"
)
results = await self.init_weights_update_group_communicator(obj)
@@ -457,8 +457,8 @@ class TokenizerControlMixin:
request: Optional[fastapi.Request] = None,
) -> Tuple[bool, str]:
self.auto_create_handle_loop()
assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, (
"dp_size must be 1 or dp attention must be enabled for destroy parameter update group"
assert get_parallel().dp_size == 1, (
"data-parallel replicas are not supported for destroy parameter update group"
)
results = await self.destroy_weights_update_group_communicator(obj)
@@ -502,8 +502,8 @@ class TokenizerControlMixin:
request: Optional[fastapi.Request] = None,
) -> Tuple[bool, str]:
self.auto_create_handle_loop()
assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, (
"dp_size must be 1 or dp attention must be enabled for update weights from distributed"
assert get_parallel().dp_size == 1, (
"data-parallel replicas are not supported for update weights from distributed"
)
if obj.abort_all_requests:
@@ -535,8 +535,8 @@ class TokenizerControlMixin:
) -> Tuple[bool, str]:
self.auto_create_handle_loop()
# TODO: support DP
assert get_parallel().dp_size == 1, (
"dp_size must be 1 for init_weights_send_group_for_remote_instance"
assert get_parallel().num_dp_ranks == 1, (
"init_weights_send_group_for_remote_instance requires a single DP rank"
)
result = (
await self.init_weights_send_group_for_remote_instance_communicator(obj)
@@ -550,8 +550,8 @@ class TokenizerControlMixin:
) -> Tuple[bool, str]:
self.auto_create_handle_loop()
# TODO: support DP
assert get_parallel().dp_size == 1, (
"dp_size must be 1 for send_weights_to_remote_instance"
assert get_parallel().num_dp_ranks == 1, (
"send_weights_to_remote_instance requires a single DP rank"
)
result = (await self.send_weights_to_remote_instance_communicator(obj))[0]
return result.success, result.message
@@ -562,8 +562,8 @@ class TokenizerControlMixin:
request: Optional[fastapi.Request] = None,
) -> Tuple[bool, str]:
self.auto_create_handle_loop()
assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, (
"dp_size must be 1 or dp attention must be enabled for update weights from tensor"
assert get_parallel().dp_size == 1, (
"data-parallel replicas are not supported for update weights from tensor"
)
if obj.abort_all_requests:
@@ -600,8 +600,8 @@ class TokenizerControlMixin:
self.auto_create_handle_loop()
try:
# For now, we only support single data parallel instance
assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, (
"dp_size must be 1 or dp attention must be enabled for update weights from IPC"
assert get_parallel().dp_size == 1, (
"data-parallel replicas are not supported for update weights from IPC"
)
logger.info("Starting IPC weight update")
@@ -665,8 +665,8 @@ class TokenizerControlMixin:
"LoRA is not enabled. Please set `--enable-lora` to enable LoRA."
)
assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, (
"dp_size must be 1 or dp attention must be enabled for dynamic lora loading"
assert get_parallel().dp_size == 1, (
"data-parallel replicas are not supported for dynamic lora loading"
)
logger.info(
"Start load Lora adapter. Lora name=%s, path=%s",
@@ -751,8 +751,8 @@ class TokenizerControlMixin:
"LoRA is not enabled. Please set `--enable-lora` to enable LoRA."
)
assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, (
"dp_size must be 1 or dp attention must be enabled for dynamic lora loading"
assert get_parallel().dp_size == 1, (
"data-parallel replicas are not supported for dynamic lora loading"
)
logger.info(
"Start load Lora adapter from tensors. Lora name=%s",
@@ -841,8 +841,8 @@ class TokenizerControlMixin:
"lora_name must be provided to unload LoRA adapter"
)
assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, (
"dp_size must be 1 or dp attention must be enabled for dynamic lora loading"
assert get_parallel().dp_size == 1, (
"data-parallel replicas are not supported for dynamic lora loading"
)
logger.info(
"Start unload Lora adapter. Lora name=%s",
@@ -862,7 +862,7 @@ class TokenizerControlMixin:
self.auto_create_handle_loop()
results = await self.get_weights_by_name_communicator(obj)
all_parameters = [r.parameter for r in results]
if get_parallel().dp_size == 1:
if get_parallel().num_dp_ranks == 1:
return all_parameters[0]
else:
return all_parameters
@@ -488,7 +488,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
self.server_args = server_args
assert_published(server_args, role="tokenizer")
self.startup_time: Optional[Dict[str, Any]] = None
self.elastic_worker_count = get_parallel().dp_size
self.elastic_worker_count = get_parallel().num_dp_ranks
self.elastic_pending_ep_size = None
self.elastic_scale_phase = "idle"
self.elastic_last_error = None
@@ -872,14 +872,14 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
)
if isinstance(obj, GenerateReqInput) and obj.routed_dp_rank is not None:
dp_size = self.elastic_worker_count
if dp_size <= 1 and obj.routed_dp_rank == 0:
num_dp_ranks = self.elastic_worker_count
if num_dp_ranks <= 1 and obj.routed_dp_rank == 0:
logger.debug(
f"routed_dp_rank={obj.routed_dp_rank} is ignored because dp_size={dp_size}"
f"routed_dp_rank={obj.routed_dp_rank} is ignored because num_dp_ranks={num_dp_ranks}"
)
elif obj.routed_dp_rank < 0 or obj.routed_dp_rank >= dp_size:
elif obj.routed_dp_rank < 0 or obj.routed_dp_rank >= num_dp_ranks:
raise ValueError(
f"routed_dp_rank={obj.routed_dp_rank} out of range [0, {dp_size})"
f"routed_dp_rank={obj.routed_dp_rank} out of range [0, {num_dp_ranks})"
)
self._init_req_state(obj, request)
@@ -1670,7 +1670,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
return batch_size > 0 and (
get_serving().enable_tokenizer_batch_encode
or (
(not get_parallel().enable_dp_attention)
(not get_parallel().attn_dp_enabled)
and (not self._batch_has_text(batch_size, requests))
)
)
@@ -359,7 +359,7 @@ def build_kv_cache(
),
is_eagle=spec_algorithm.is_eagle(),
tp_cache_group=(
attn_tp_cpu_group if get_parallel().enable_dp_attention else tp_cpu_group
attn_tp_cpu_group if get_parallel().attn_dp_enabled else tp_cpu_group
),
attn_cp_cache_group=attn_cp_cpu_group,
attn_tp_cache_group=attn_tp_cpu_group,
@@ -299,7 +299,7 @@ Supported:
`TransferManagerOnRemote` via the same ZMQ channel used for GPU
registration.
* **CP** (`attn_cp_size > 1`) — sync handled symmetrically with TP.
* **DP attention** (`enable_dp_attention=True`) — the inner
* **DP attention** (`--attn-dp-size` > 1) — the inner
`attn_tp_size` is what FlexKV uses for register-side routing.
---
@@ -2,7 +2,7 @@
The public surface is small (see "Public API" below). The class owns:
* the FlexKV ``KVManager`` (server-client mode when ``dp_size > 1`` or
* the FlexKV ``KVManager`` (server-client mode when ``num_dp_ranks > 1`` or
multi-instance; in-process otherwise — handled by FlexKV itself);
* the per-rank ``KVTPClient`` that registers this rank's GPU KV cache
with the FlexKV TransferManager;
@@ -619,7 +619,7 @@ class CPUGraphRunner:
self.speculative_algorithm = get_spec().speculative_algorithm
self.enable_profile_cuda_graph = get_exec().graph.enable_profile_cuda_graph
self.tp_size = get_parallel().tp_size
self.dp_size = get_parallel().dp_size
self.num_dp_ranks = get_parallel().num_dp_ranks
self.pp_size = get_parallel().pp_size
self.capture_forward_mode = ForwardMode.DECODE
@@ -644,7 +644,7 @@ class CPUGraphRunner:
"CPUGraphRunner does not support speculative inference yet."
)
assert self.dp_size == 1, "CPUGraphRunner does not support DP yet."
assert self.num_dp_ranks == 1, "CPUGraphRunner does not support DP yet."
assert self.pp_size == 1, "CPUGraphRunner does not support PP yet."
# Batch sizes to capture
@@ -530,7 +530,7 @@ def build_decode_registry(
require_mlp_tp_gather: bool = False,
# Per-bucket attn-TP sharded (SP) predicate; defaults to replicated.
attn_tp_sharded_fn: Callable[[int], bool] = lambda num_tokens: False,
dp_size: int = 1,
num_dp_ranks: int = 1,
register_global_num_tokens: bool = True,
share_pool: bool = True,
source: Optional[Any] = None,
@@ -700,7 +700,7 @@ def build_decode_registry(
buf.fill_(ctx.padded_num_tokens)
_global_shape = (
(lambda _bs, _mt: (dp_size,))
(lambda _bs, _mt: (num_dp_ranks,))
if require_mlp_tp_gather
else (lambda _bs, _mt: (1,))
)
@@ -1048,7 +1048,7 @@ def build_eager_registry(
register_global_num_tokens=False,
require_gathered_buffer=False,
require_mlp_tp_gather=False,
dp_size=get_parallel().dp_size,
num_dp_ranks=get_parallel().num_dp_ranks,
share_pool=True,
source=None,
)
@@ -1186,7 +1186,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
# Init lora information
if (
model_runner.lora_manager is not None
and not model_runner.lora_manager.enable_dp_attention
and not model_runner.lora_manager.attn_dp_enabled
):
# In the non-LoRA overlap loading case, we fetch LoRA adapters into the memory pool
# as a batch, right before running the batch
@@ -530,7 +530,7 @@ class ModelRunner:
new_dp_size=join_effective_ep_size,
new_dp_rank=global_ep_rank,
)
get_context().override("elastic_ep.scale_join", dp_size=join_effective_ep_size)
get_parallel().override_permanently(num_dp_ranks=join_effective_ep_size)
if self.eplb_manager is not None:
self.eplb_manager.disable_rebalance(
"EPLB rebalance is disabled while elastic EP scale-up "
@@ -965,7 +965,7 @@ class ModelRunner:
device=self.device,
tp_group=(
get_parallel().attn_tp_group.cpu_group
if get_parallel().enable_dp_attention
if get_parallel().attn_dp_enabled
else get_parallel().tp_group.cpu_group
),
host_to_device_ratio=hisparse_cfg.host_to_device_ratio,
@@ -1628,7 +1628,7 @@ class ModelRunner:
forward_batch.prepare_mlp_sync_batch(self)
else:
forward_batch.prepare_attn_tp_scatter_input(self)
if self.lora_manager is not None and self.lora_manager.enable_dp_attention:
if self.lora_manager is not None and self.lora_manager.attn_dp_enabled:
self.lora_manager.prepare_lora_batch(forward_batch)
# Derive the LOCAL num_token_non_padded from the GLOBAL scalar. sharded is
@@ -2217,7 +2217,7 @@ class ModelRunner:
new_dp_size=target_size,
new_dp_rank=self._elastic_global_rank(),
)
get_context().override("elastic_ep.scale", dp_size=target_size)
get_parallel().override_permanently(num_dp_ranks=target_size)
recapture_cuda_graph = self._elastic_cuda_graph_enabled()
if recapture_cuda_graph:
@@ -82,7 +82,7 @@ class StartupWeightLoadOptions:
attn_cp_size: int
dcp_size: int
pp_size: int
dp_size: int
num_dp_ranks: int
ep_size: int
cpu_offload_gb: int
offload_group_size: int
@@ -123,7 +123,7 @@ class StartupWeightLoadOptions:
attn_cp_size=get_parallel().attn_cp_size,
dcp_size=get_parallel().dcp_size,
pp_size=get_parallel().pp_size,
dp_size=get_parallel().dp_size,
num_dp_ranks=get_parallel().num_dp_ranks,
ep_size=get_parallel().ep_size,
cpu_offload_gb=get_exec().offload.cpu_offload_gb,
offload_group_size=get_exec().offload.offload_group_size,
@@ -369,7 +369,7 @@ class StartupWeightLoadManager:
"decode context parallelism is not supported",
),
(options.pp_size != 1, "pipeline parallelism is not supported"),
(options.dp_size != 1, "data parallelism is not supported"),
(options.num_dp_ranks != 1, "data parallelism is not supported"),
(options.ep_size != 1, "expert parallelism is not supported"),
(options.cpu_offload_gb > 0, "CPU offload is not supported"),
(
@@ -162,9 +162,11 @@ def _allocate_decode_buffers(
encoder_lens = None
if require_mlp_tp_gather:
global_num_tokens_gpu = torch.zeros((parallel.dp_size,), dtype=torch.int32)
global_num_tokens_gpu = torch.zeros(
(parallel.num_dp_ranks,), dtype=torch.int32
)
global_num_tokens_for_logprob_gpu = torch.zeros(
(parallel.dp_size,), dtype=torch.int32
(parallel.num_dp_ranks,), dtype=torch.int32
)
else:
global_num_tokens_gpu = torch.zeros((1,), dtype=torch.int32)
@@ -226,8 +228,8 @@ class BaseRunner(ABC):
self.model_runner = model_runner
self.device = model_runner.device
self.device_module = torch.get_device_module(self.device)
# elastic-EP scale-up rewrites dp_size on the published config
self.dp_size = get_parallel().dp_size
# elastic-EP scale-up widens num_dp_ranks on the published config
self.num_dp_ranks = get_parallel().num_dp_ranks
self.pp_size = get_parallel().pp_size
self.enable_pdmux = get_disagg().enable_pdmux
self.return_hidden_states_mode = (
@@ -613,7 +615,7 @@ class BaseRunner(ABC):
assert require_mlp_tp_gather_ or require_attn_tp_gather_
if require_mlp_tp_gather_:
global_num_tokens_cpu = [num_tokens] * get_parallel().dp_size
global_num_tokens_cpu = [num_tokens] * get_parallel().num_dp_ranks
elif require_attn_tp_gather_:
global_num_tokens_cpu = [num_tokens]
else:
@@ -260,7 +260,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
self.require_mlp_tp_gather or self.require_attn_tp_gather
)
self.require_mlp_sync = (
get_parallel().enable_dp_attention or self.require_gathered_buffer
get_parallel().attn_dp_enabled or self.require_gathered_buffer
)
self.enable_two_batch_overlap = get_exec().overlap.enable_two_batch_overlap
self.use_ngram_embedding = model_runner.ngram_embedding_manager.enabled
@@ -414,7 +414,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
self.model_runner.model_config.vocab_size, rows=logits_buffer_rows
),
dtype=self.model_runner.model_config.dtype,
dp_size=self.dp_size,
num_dp_ranks=self.num_dp_ranks,
pp_size=self.pp_size,
is_encoder_decoder=self.is_encoder_decoder,
require_mlp_tp_gather=self.require_mlp_tp_gather,
@@ -458,7 +458,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
enable_prefill_cp=self.enable_prefill_cp,
require_mlp_tp_gather=self.require_mlp_tp_gather,
attn_tp_sharded_fn=self.model_runner.attn_tp_sequence_sharded,
dp_size=self.dp_size,
num_dp_ranks=self.num_dp_ranks,
source=self.buffers,
)
@@ -584,7 +584,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
def _global_num_tokens_for_graph(self, num_tokens: int) -> Optional[list[int]]:
if self.require_mlp_tp_gather:
return [num_tokens] * self.dp_size
return [num_tokens] * self.num_dp_ranks
if self.require_attn_tp_gather:
return [num_tokens]
return None
@@ -1427,7 +1427,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
)
if (
self.model_runner.lora_manager is not None
and self.model_runner.lora_manager.enable_dp_attention
and self.model_runner.lora_manager.attn_dp_enabled
):
self.model_runner.lora_manager.prepare_lora_batch(
cast(ForwardBatch, fb_view)
@@ -311,7 +311,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
self.prefill_backend_name = prefill_config.backend
self.prefer_eager_mixed_prefill = (
self.prefill_backend_name == Backend.BREAKABLE
and get_parallel().enable_dp_attention
and get_parallel().attn_dp_enabled
and getattr(
model_runner.attn_backend,
"prefer_eager_mixed_prefill_under_dp_attention",
@@ -1432,7 +1432,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
return registry.get_slot(name).slice_for(bs, num_tokens)
if self.require_mlp_tp_gather:
global_num_tokens_cpu = [num_tokens] * self.dp_size
global_num_tokens_cpu = [num_tokens] * self.num_dp_ranks
elif self.require_attn_tp_gather:
global_num_tokens_cpu = [num_tokens]
else:
@@ -136,7 +136,7 @@ class DecodeInputBuffers(ForwardInputBuffers):
hidden_size: int,
next_token_logits_buffer: torch.Tensor,
dtype: torch.dtype,
dp_size: int,
num_dp_ranks: int,
pp_size: int,
is_encoder_decoder: bool,
require_mlp_tp_gather: bool,
@@ -207,9 +207,9 @@ class DecodeInputBuffers(ForwardInputBuffers):
encoder_lens = None
if require_mlp_tp_gather:
global_num_tokens_gpu = torch.zeros((dp_size,), dtype=torch.int32)
global_num_tokens_gpu = torch.zeros((num_dp_ranks,), dtype=torch.int32)
global_num_tokens_for_logprob_gpu = torch.zeros(
(dp_size,), dtype=torch.int32
(num_dp_ranks,), dtype=torch.int32
)
else:
global_num_tokens_gpu = torch.zeros((1,), dtype=torch.int32)
+2 -2
View File
@@ -3896,9 +3896,9 @@ class DeepseekV4DecoderLayer(nn.Module):
# Applies to BOTH prefill and decode: the shared expert is a per-token MLP,
# so computing it on this rank's local tokens (M_local rows) is identical to
# computing it on the gathered global buffer (M_global rows) and keeping the
# local slice -- but costs 1/dp_size the rows. With a replicated (TP1) shared
# local slice -- but costs 1/attn_dp_size the rows. With a replicated (TP1) shared
# expert this cancels the TP1 "full-dim" cost in decode (M_local * dim ==
# M_global * dim/tp), so decode no longer pays the ~dp_size x penalty.
# M_global * dim/tp), so decode no longer pays the ~attn_dp_size x penalty.
_shared_local = None
_do_shared_local = (
_SHARED_EXPERT_LOCAL
+1 -1
View File
@@ -842,7 +842,7 @@ class ZayaAttention(nn.Module):
assert attn_tp_size == self.tp_size, (
f"ZAYA1 head-parallel attention requires the attention TP group "
f"({attn_tp_size}) to equal the global TP group ({self.tp_size}); "
"DP attention (enable_dp_attention) is not supported for ZAYA1."
"attention DP is not supported for ZAYA1."
)
assert self.num_q_heads_full % self.tp_size == 0, (
f"num_attention_heads ({self.num_q_heads_full}) must be divisible "
@@ -77,7 +77,7 @@ class RayDataParallelController(DataParallelController):
sockets = []
dp_port_args_list = []
for dp_rank in range(get_parallel().dp_size):
for dp_rank in range(get_parallel().num_dp_ranks):
tmp_port_args = PortArgs.init_new(server_args)
tmp_port_args.tokenizer_ipc_name = port_args.tokenizer_ipc_name
tmp_port_args.detokenizer_ipc_name = port_args.detokenizer_ipc_name
@@ -101,7 +101,7 @@ class RayDataParallelController(DataParallelController):
sock.close()
# Create actors for each DP rank sequentially
for dp_rank in range(get_parallel().dp_size):
for dp_rank in range(get_parallel().num_dp_ranks):
self._launch_ray_tp_group(server_args, dp_port_args_list[dp_rank], dp_rank)
def launch_dp_attention_schedulers(
@@ -112,7 +112,7 @@ class RayDataParallelController(DataParallelController):
# rank-0 node IP instead of tcp://* to avoid exposing unauthenticated
# ZMQ sockets (CVE-2026-3060).
worker_ports = []
for dp_rank in range(get_parallel().dp_size):
for dp_rank in range(get_parallel().num_dp_ranks):
worker_port, worker_socket = get_zmq_socket_on_host(
self.context, zmq.PUSH, host=self.rank0_node_ip
)
@@ -158,12 +158,11 @@ class RayDataParallelController(DataParallelController):
)
parallel = get_parallel()
if parallel.enable_dp_attention:
if parallel.attn_dp_enabled:
_, _, actual_dp_rank, _ = compute_dp_attention_world_info(
parallel.enable_dp_attention,
tp_rank,
parallel.tp_size,
parallel.dp_size,
parallel.attn_dp_size,
parallel.attn_cp_size,
)
rank_port_args = PortArgs.init_new(
@@ -230,12 +229,11 @@ class RayDataParallelController(DataParallelController):
bundle_idx = bundle_indices[global_rank]
if get_parallel().enable_dp_attention:
if get_parallel().attn_dp_enabled:
_, _, actual_dp_rank, _ = compute_dp_attention_world_info(
get_parallel().enable_dp_attention,
tp_rank,
get_parallel().tp_size,
get_parallel().dp_size,
get_parallel().attn_dp_size,
get_parallel().attn_cp_size,
)
rank_port_args = PortArgs.init_new(
+7 -8
View File
@@ -112,7 +112,6 @@ def _compute_world_size() -> int:
will hold the process groups, so there is nothing live to ask.
"""
return compute_world_size(
enable_dp_attention=get_parallel().enable_dp_attention,
dp_size=get_parallel().dp_size,
tp_size=get_parallel().tp_size,
pp_size=get_parallel().pp_size,
@@ -274,10 +273,10 @@ class RayEngine(Engine):
)
parallel = get_parallel()
if parallel.enable_dp_attention:
if parallel.attn_dp_enabled:
total_gpus = parallel.tp_size * parallel.pp_size
else:
total_gpus = parallel.dp_size * parallel.tp_size * parallel.pp_size
total_gpus = parallel.num_dp_ranks * parallel.tp_size * parallel.pp_size
nnodes = parallel.nnodes
gpus_per_node = total_gpus // nnodes
@@ -318,7 +317,7 @@ class RayEngine(Engine):
rank0_bundle_idx = int(indices_str.split(",")[0]) if indices_str else 0
rank0_node_ip = _get_bundle_node_ip(pg, rank0_bundle_idx)
if get_parallel().dp_size == 1:
if get_parallel().num_dp_ranks == 1:
dist_init_addr = f"{rank0_node_ip}:{port_args.nccl_port}"
logger.info(f"dist_init_addr: {dist_init_addr}")
@@ -448,17 +447,17 @@ class RayEngine(Engine):
)
parallel = get_parallel()
if parallel.enable_dp_attention:
if parallel.attn_dp_enabled:
# DP attention folds DP into TP — total GPUs = tp_size * pp_size
total_gpus = parallel.tp_size * parallel.pp_size
else:
total_gpus = parallel.dp_size * parallel.tp_size * parallel.pp_size
total_gpus = parallel.num_dp_ranks * parallel.tp_size * parallel.pp_size
gpus_per_node = total_gpus // parallel.nnodes
logger.info(
f"Ray DP cluster: {parallel.nnodes} nodes, "
f"{gpus_per_node} GPUs/node, dp_size={parallel.dp_size}, "
f"tp_size={parallel.tp_size}, pp_size={parallel.pp_size}, "
f"enable_dp_attention={parallel.enable_dp_attention}"
f"attn_dp_size={parallel.attn_dp_size}, "
f"tp_size={parallel.tp_size}, pp_size={parallel.pp_size}"
)
# Declared on the record itself so `PortArgs.init_new()` can compute
+33 -31
View File
@@ -70,16 +70,14 @@ def _parallel_fields() -> frozenset:
return frozenset(_parallel_config_leaves() | derived)
def derive_attention_widths(
*, tp_size: int, attn_cp_size: int, dp_size: int, enable_dp_attention: bool
) -> tuple:
"""Return (attn_dp_size, attn_tp_size) from the configured widths."""
attn_dp_size = dp_size if enable_dp_attention else 1
return attn_dp_size, tp_size // attn_dp_size // attn_cp_size
def derive_attn_tp_size(*, tp_size: int, attn_cp_size: int, attn_dp_size: int) -> int:
"""Return the attention-TP width: what is left of the TP group after the
attention-DP and attention-CP dimensions."""
return tp_size // attn_dp_size // attn_cp_size
def derive_attention_ranks(
*, tp_rank: int, attn_tp_size: int, attn_cp_size: int, enable_dp_attention: bool
*, tp_rank: int, attn_tp_size: int, attn_cp_size: int
) -> tuple:
"""Return (attn_tp_rank, attn_dp_rank) for a process at ``tp_rank``.
@@ -87,11 +85,11 @@ def derive_attention_ranks(
tp_rank = (attn_dp_rank * attn_cp_size + attn_cp_rank) * attn_tp_size
+ attn_tp_rank
Without attention DP the TP group is one attention-DP group, so
``attn_dp_rank`` is zero.
"""
attn_tp_rank = tp_rank % attn_tp_size
if not enable_dp_attention:
return attn_tp_rank, 0
return attn_tp_rank, tp_rank // (attn_tp_size * attn_cp_size)
return tp_rank % attn_tp_size, tp_rank // (attn_tp_size * attn_cp_size)
def spawn_world_rank(server_args, *, tp_rank: int, pp_rank: int) -> int:
@@ -149,13 +147,9 @@ def derive_parallel_widths(
"""Derive attention and MoE widths and DCP settings from configuration."""
return {
"attn_dp_size": attn_dp_size,
# `attn_dp_size` already accounts for disabled DP attention.
"attn_tp_size": derive_attention_widths(
tp_size=tp_size,
attn_cp_size=attn_cp_size,
dp_size=attn_dp_size,
enable_dp_attention=True,
)[1],
"attn_tp_size": derive_attn_tp_size(
tp_size=tp_size, attn_cp_size=attn_cp_size, attn_dp_size=attn_dp_size
),
"moe_ep_size": moe_ep_size,
"moe_tp_size": tp_size // moe_ep_size // moe_dp_size,
"dcp_enabled": dcp_enabled,
@@ -165,16 +159,10 @@ def derive_parallel_widths(
def parallel_widths_of(cfg: Any) -> dict:
"""Return derived parallel settings from resolved configuration."""
attn_dp_size, _ = derive_attention_widths(
tp_size=cfg.tp_size,
attn_cp_size=cfg.attn_cp_size,
dp_size=cfg.dp_size,
enable_dp_attention=cfg.enable_dp_attention,
)
return derive_parallel_widths(
tp_size=cfg.tp_size,
attn_cp_size=cfg.attn_cp_size,
attn_dp_size=attn_dp_size,
attn_dp_size=cfg.attn_dp_size,
moe_ep_size=cfg.ep_size,
moe_dp_size=cfg.moe_dp_size,
dcp_size=cfg.dcp_size,
@@ -200,9 +188,23 @@ def attn_tp_size_of(cfg: Any):
return parallel_widths_of(cfg)["attn_tp_size"]
def attn_dp_size_of(cfg: Any):
"""`attn_dp_size`, computed at publish. See `parallel_widths_of`."""
return parallel_widths_of(cfg)["attn_dp_size"]
def attn_dp_enabled_of(cfg: Any) -> bool:
"""`attn_dp_enabled`, computed at publish.
A scale joiner's own group can be one rank wide, but it joins an
attention-DP deployment and runs its DP-attention paths.
"""
from sglang.srt.arg_groups.model_override_base import ep_scale_joiner_of
return cfg.attn_dp_size > 1 or ep_scale_joiner_of(cfg)
def num_dp_ranks_of(cfg: Any) -> int:
"""`num_dp_ranks`, computed at publish: replicas times attention-DP groups.
DP attention does not run inside replicas, so one of the two is one.
"""
return cfg.dp_size * cfg.attn_dp_size
def attn_dcp_size_of(cfg: Any):
@@ -1560,7 +1562,6 @@ def _attention_ranks(parallel, tp_rank: int) -> dict:
tp_rank=tp_rank,
attn_tp_size=parallel.attn_tp_size,
attn_cp_size=parallel.attn_cp_size,
enable_dp_attention=parallel.enable_dp_attention,
)
return {"attn_tp_rank": attn_tp_rank, "attn_dp_rank": attn_dp_rank}
@@ -2119,7 +2120,8 @@ def describe_kv_events_publisher(server_args: Any) -> Optional[dict]:
"endpoint_port_base": port,
"topic": cfg.topic,
"block_size": kv_event_block_size_of(resolved),
"dp_size": resolved.dp_size,
# The wire key predates attention DP; it counts DP ranks.
"dp_size": num_dp_ranks_of(resolved),
}
# Load range, from the same resolver SchedulerLoadPublisher binds
# with (so the two can't drift). The decline reason is logged once at
@@ -2127,7 +2129,7 @@ def describe_kv_events_publisher(server_args: Any) -> Optional[dict]:
resolved_range, _reason = resolve_load_pub_range(
kv_endpoint=cfg.endpoint,
replay_endpoint=cfg.replay_endpoint,
dp_size=resolved.dp_size,
dp_size=num_dp_ranks_of(resolved),
load_publish_endpoint=resolved.load_publish_endpoint,
)
if resolved_range is not None:
+6 -2
View File
@@ -120,7 +120,9 @@ class RustServer:
os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
# Preserve the DP startup log; ports use node-local offsets.
dp_rank = get_parallel().attn_dp_rank if get_parallel().dp_size > 1 else None
dp_rank = (
get_parallel().attn_dp_rank if get_parallel().num_dp_ranks > 1 else None
)
if get_exec().moe.is_ep_scale_joiner:
# The joining TP group is entirely local to this node.
tp_size_per_node = get_parallel().tp_size
@@ -180,7 +182,9 @@ class RustServer:
# Under DP every rank runs its own server on its own port, so the rank is
# what tells two otherwise identical startup lines apart.
dp_note = (
"" if dp_rank is None else f" (DP rank {dp_rank}/{get_parallel().dp_size})"
""
if dp_rank is None
else f" (DP rank {dp_rank}/{get_parallel().num_dp_ranks})"
)
logger.info(
"SGLANG_RUST_SERVER enabled, Rust server listen on %s%s",
+23 -10
View File
@@ -43,7 +43,12 @@ from sglang.srt.arg_groups.overrides import (
resolving_view,
)
from sglang.srt.environ import envs
from sglang.srt.runtime_context import get_platform, publish
from sglang.srt.runtime_context import (
attn_dp_enabled_of,
get_platform,
num_dp_ranks_of,
publish,
)
from sglang.srt.speculative.decoupled_spec_io import DecoupledSpecIpcConfig
from sglang.srt.utils.network import NetworkAddress, get_free_port, wait_port_available
@@ -345,6 +350,15 @@ class ServerArgs:
new_flag="--cuda-graph-backend-{decode,prefill}=disabled",
help="Deprecated. Use --cuda-graph-backend-{decode,prefill}=disabled instead.",
)
# `enable_dp_attention` is `no_cli=True` too; resolution turns it into
# `attn_dp_size`.
parser.add_argument(
"--enable-dp-attention",
action=DeprecatedStoreTrueAction,
new_flag="--attn-dp-size <dp-size>",
help="Deprecated. Use --attn-dp-size <dp-size> instead of "
"--dp-size <dp-size> --enable-dp-attention.",
)
parser.add_argument(
"--enable-flashinfer-allreduce-fusion",
action="store_true",
@@ -488,16 +502,15 @@ def resolve_encoder_transfer_backend(
return "zmq_to_scheduler"
def compute_world_size(
*, enable_dp_attention: bool, dp_size: int, tp_size: int, pp_size: int
) -> int:
def compute_world_size(*, dp_size: int, tp_size: int, pp_size: int) -> int:
"""Total GPU count across all data-parallel replicas.
Takes the values rather than a config object: the two sizes are the widths
Takes the values rather than a config object: the sizes are the widths
the launch asked for, which the Ray driver needs before any process group
exists, and passing a context would hand it the live groups instead.
Attention DP runs inside the TP group, so it adds no GPUs.
"""
return (1 if enable_dp_attention else dp_size) * tp_size * pp_size
return dp_size * tp_size * pp_size
def m3_fp8_attn_gemm_enabled(args) -> bool:
@@ -706,7 +719,7 @@ class PortArgs:
rank=int(server_args.decoupled_spec_rank),
)
if not cfg.enable_dp_attention:
if not attn_dp_enabled_of(cfg):
# Normal case, use IPC within a single node
return PortArgs(
tokenizer_ipc_name=f"ipc://{tempfile.NamedTemporaryFile(delete=False).name}",
@@ -732,12 +745,12 @@ class PortArgs:
dist_init_host = na.host
dist_init_port = na.port
# Reserve port_base+0..NUM_DERIVED_PORTS-1 (6 fixed ports + dp_size
# rust-path slots); derive from server_args only (never dp_rank) so
# Reserve port_base+0..NUM_DERIVED_PORTS-1 (6 fixed ports + one
# rust-path slot per DP rank); derive from server_args only (never dp_rank) so
# every init_new call agrees, decrementing below dist_init_port on
# overflow.
is_rust_server = envs.SGLANG_RUST_SERVER.get()
NUM_DERIVED_PORTS = 6 if not is_rust_server else 6 + cfg.dp_size
NUM_DERIVED_PORTS = 6 if not is_rust_server else 6 + num_dp_ranks_of(cfg)
if ep_scale_joiner_of(resolving_view(server_args)):
port_base = server_args.port + ZMQ_TCP_PORT_DELTA
if port_base + NUM_DERIVED_PORTS > 65535:
@@ -16,6 +16,7 @@ from sglang.srt.arg_groups.overrides import (
resolved_view,
resolving_view,
)
from sglang.srt.runtime_context import attn_dp_enabled_of
from sglang.srt.utils import log_info_on_rank0
if TYPE_CHECKING:
@@ -66,9 +67,9 @@ def adaptive_unsupported_reason(server_args: ServerArgs) -> str | None:
f"speculative_eagle_topk={cfg.speculative_eagle_topk} "
"(only topk=1 is supported)"
)
if resolved_view(server_args).enable_dp_attention:
if attn_dp_enabled_of(resolved_view(server_args)):
return (
"enable_dp_attention=True is not supported "
"attention DP is not supported "
"(adaptive tier decisions are not synchronized across DP ranks)"
)
if resolved_view(server_args).enable_multi_layer_eagle:
@@ -403,14 +403,14 @@ class DFlashWorkerV2(BaseSpecWorker):
# spec broadcasts must stay within the attn-TP group.
self._tp_sync = SpecTpSync(
get_parallel().attn_tp_group
if get_parallel().enable_dp_attention
if get_parallel().attn_dp_enabled
else get_parallel().tp_group
)
# Under dp attention, the draft worker runs on the per-DP attn-TP
# group, independent of idle peer DP ranks; it is built and run under
# the same placement.
self.draft_owns_attention = get_parallel().enable_dp_attention
self.draft_owns_attention = get_parallel().attn_dp_enabled
# Inside the draft scope the context answers the draft's narrowed rank.
self._target_tp_rank = get_parallel().tp_rank
with draft_pp_context(), draft_tp_context(self.draft_owns_attention):
@@ -645,7 +645,7 @@ class DFlashWorkerV2(BaseSpecWorker):
"device graph capture.",
type(current_platform).__name__,
)
if get_parallel().enable_dp_attention and capture_decode_cuda_graph:
if get_parallel().attn_dp_enabled and capture_decode_cuda_graph:
# Idle DP ranks skip the draft step, so they cannot join a
# shared graph capture/replay; keep the draft eager under dp
# attention.
@@ -1295,7 +1295,7 @@ class DFlashWorkerV2(BaseSpecWorker):
the full embedding once during init and keep it replicated, making
the draft block-id lookup a collective-free on-device F.embedding.
"""
if not get_parallel().enable_dp_attention:
if not get_parallel().attn_dp_enabled:
return
tp_group = get_parallel().tp_group
@@ -2335,7 +2335,7 @@ class DFlashWorkerV2(BaseSpecWorker):
# verify forward (IDLE mode) so its cross-DP collectives stay in
# lockstep with the active DP group; the draft block's
# collectives are within-rank and skipped.
if get_parallel().enable_dp_attention:
if get_parallel().attn_dp_enabled:
idle_verify_input = DFlashVerifyInput(
draft_token=torch.empty((0,), dtype=torch.long, device=self.device),
positions=torch.empty((0,), dtype=torch.int64, device=self.device),
@@ -2726,7 +2726,7 @@ class DFlashWorkerV2(BaseSpecWorker):
# their DP-gather segment offsets disagree. Fall back to eager verify
# whenever any DP rank is idle this round.
if (
get_parallel().enable_dp_attention
get_parallel().attn_dp_enabled
and verify_forward_batch.original_global_num_tokens_cpu is not None
and min(verify_forward_batch.original_global_num_tokens_cpu) == 0
):
@@ -2735,7 +2735,7 @@ class DFlashWorkerV2(BaseSpecWorker):
# Mixed-round guard: an extend rank's raw token counts disagree with
# the verify batch's spec-scaled counts on the DP-gather layout; run
# eager, symmetric with the idle guard above.
if get_parallel().enable_dp_attention and batch.is_extend_in_batch:
if get_parallel().attn_dp_enabled and batch.is_extend_in_batch:
verify_forward_batch.can_run_decode_cuda_graph = False
target_out = self.target_worker.forward_batch_generation(
@@ -165,7 +165,7 @@ class DSparkWorkerV2(BaseSpecWorker):
self._draft_is_moe = draft_is_deepseek_v4()
self._draft_dp_context_enabled = (
get_parallel().enable_dp_attention and not self._draft_is_moe
get_parallel().attn_dp_enabled and not self._draft_is_moe
)
self._is_pd_prefill = get_disagg().disaggregation_mode == "prefill"
self._decode_graph_allowed = (
@@ -173,7 +173,7 @@ class DSparkWorkerV2(BaseSpecWorker):
and not self._is_pd_prefill
)
if (
get_parallel().enable_dp_attention
get_parallel().attn_dp_enabled
and self._draft_is_moe
and get_parallel().attn_tp_size > 1
):
@@ -232,9 +232,7 @@ class DSparkWorkerV2(BaseSpecWorker):
parallel = get_parallel()
self._tp_sync = SpecTpSync(
parallel.attn_tp_group
if parallel.enable_dp_attention
else parallel.tp_group
parallel.attn_tp_group if parallel.attn_dp_enabled else parallel.tp_group
)
self._draft_graph_group = (
parallel.attn_tp_group
@@ -294,13 +292,13 @@ class DSparkWorkerV2(BaseSpecWorker):
tp_sync=self._tp_sync,
)
if (
get_parallel().enable_dp_attention
get_parallel().attn_dp_enabled
and not self._draft_is_moe
and self._verify_planner.is_compact_mode
and self._decode_graph_allowed
):
raise ValueError(
"DSpark dense-draft compact verify under --enable-dp-attention does not "
"DSpark dense-draft compact verify under attention DP does not "
"yet support cuda graph (idle DP groups cannot join the token-keyed "
"compact graph). Re-run with --disable-cuda-graph (eager is lossless), "
"or use SGLANG_RAGGED_VERIFY_MODE=static. The dsv4 (MoE) draft supports "
@@ -321,7 +319,7 @@ class DSparkWorkerV2(BaseSpecWorker):
mask_token_id=self._mask_token_id,
draft_block_spec_info=self._draft_block_spec_info,
tp_sync=self._tp_sync,
dp_moe_sync=self._draft_is_moe and get_parallel().enable_dp_attention,
dp_moe_sync=self._draft_is_moe and get_parallel().attn_dp_enabled,
)
self._verify_epilogue = None
target_is_dsv41 = (
@@ -336,7 +334,7 @@ class DSparkWorkerV2(BaseSpecWorker):
target_is_dsv41
and self._verify_planner.mode_value == "static"
and self._draft_is_moe
and not get_parallel().enable_dp_attention
and not get_parallel().attn_dp_enabled
and get_parallel().pp_size == 1
)
# ROCm (V4.1 target only): inside a HIP graph the accept-site TP broadcasts
@@ -628,7 +626,7 @@ class DSparkWorkerV2(BaseSpecWorker):
pp_proxy_tensors: Optional[PPProxyTensors] = None,
) -> GenerationBatchResult:
if batch.forward_mode.is_idle():
if get_parallel().enable_dp_attention:
if get_parallel().attn_dp_enabled:
self.target_worker.forward_batch_generation(
batch, capture_hidden_mode=CaptureHiddenMode.FULL
)
@@ -744,7 +742,7 @@ class DSparkWorkerV2(BaseSpecWorker):
def _dp_verify_tier_num_tokens(self, batch: ScheduleBatch) -> Optional[int]:
if not (
self._draft_is_moe
and get_parallel().enable_dp_attention
and get_parallel().attn_dp_enabled
and batch.global_num_tokens is not None
and self._verify_planner.is_compact_mode
):
@@ -793,7 +791,7 @@ class DSparkWorkerV2(BaseSpecWorker):
global_num_reqs = (
max(batch.global_num_tokens)
if self._draft_is_moe
and get_parallel().enable_dp_attention
and get_parallel().attn_dp_enabled
and batch.global_num_tokens is not None
else None
)
@@ -809,7 +807,7 @@ class DSparkWorkerV2(BaseSpecWorker):
if batch.forward_mode.is_idle():
self._observers.note_idle_decode_step()
if get_parallel().enable_dp_attention:
if get_parallel().attn_dp_enabled:
if self._draft_is_moe:
self._proposer.run_idle_participation(batch)
if coordination_plan is not None:
@@ -186,8 +186,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
# Use the same attention topology during draft construction and execution.
self.draft_owns_attention = (
get_parallel().enable_dp_attention
and self.speculative_algorithm.is_eagle3()
get_parallel().attn_dp_enabled and self.speculative_algorithm.is_eagle3()
)
with (
draft_tp_context(self.draft_owns_attention),
@@ -155,7 +155,7 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
# Fields the parent's capture() reads:
self.device = model_runner.device
self.device_module = torch.get_device_module(self.device)
self.dp_size = get_parallel().dp_size
self.num_dp_ranks = get_parallel().num_dp_ranks
self.pp_size = get_parallel().pp_size
self.enable_torch_compile = get_flags().capture.enable_torch_compile
self.disable_padding = get_exec().graph.disable_cuda_graph_padding
@@ -283,8 +283,8 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
next_token_logits_buffer = buffers.next_token_logits_buffer[:num_tokens]
if self.require_mlp_tp_gather:
global_num_tokens_cpu = [num_tokens] * self.dp_size
global_num_tokens_for_logprob_cpu = [num_tokens] * self.dp_size
global_num_tokens_cpu = [num_tokens] * self.num_dp_ranks
global_num_tokens_for_logprob_cpu = [num_tokens] * self.num_dp_ranks
elif self.require_attn_tp_gather:
global_num_tokens_cpu = [num_tokens]
# DRAFT_EXTEND_V2 produces logits for all tokens, not bs (see mlp branch above)
@@ -678,10 +678,12 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
if self.require_gathered_buffer:
if self.require_mlp_tp_gather:
dp_size = runner.dp_size
global_num_tokens_gpu = torch.zeros((dp_size,), dtype=torch.int32)
num_dp_ranks = runner.num_dp_ranks
global_num_tokens_gpu = torch.zeros(
(num_dp_ranks,), dtype=torch.int32
)
global_num_tokens_for_logprob_gpu = torch.zeros(
(dp_size,), dtype=torch.int32
(num_dp_ranks,), dtype=torch.int32
)
else:
global_num_tokens_gpu = torch.zeros((1,), dtype=torch.int32)
@@ -71,14 +71,14 @@ class RoutedExpertsCapturer(BaseTopkCapturer):
topk_size = model_config.hf_text_config.num_experts_per_tok
num_layers = model_config.hf_text_config.num_hidden_layers
# Scale by dp_size so the buffer covers the full DP-concatenated batch.
# Scale by num_dp_ranks so the buffer covers the full DP-concatenated batch.
# _get_local_slice indexes into [attention_dp_rank * cuda_graph_batch, ...)
# and otherwise overflows on dp_rank > 0 when max_running_requests >
# chunked_prefill_size.
# FIXME: spec decoding's num_verify_tokens is still not accounted for.
max_batch_size = max(
get_schedule().chunked_prefill_size * get_parallel().dp_size,
max_running_requests * get_parallel().dp_size,
get_schedule().chunked_prefill_size * get_parallel().num_dp_ranks,
max_running_requests * get_parallel().num_dp_ranks,
)
super().__init__(
+9 -7
View File
@@ -3985,9 +3985,11 @@ def require_mlp_tp_gather(*, moe_a2a_backend=None):
elif not isinstance(moe_a2a_backend, MoeA2ABackend):
moe_a2a_backend = MoeA2ABackend(moe_a2a_backend)
# elastic-EP scale-up rewrites dp_size on the published config
if get_parallel().enable_dp_attention:
assert get_parallel().dp_size > 1, "dp_size must be greater than 1"
# elastic-EP scale-up widens num_dp_ranks on the published config
if get_parallel().attn_dp_enabled:
assert get_parallel().num_dp_ranks > 1, (
"attention DP needs more than one DP rank"
)
if get_exec().moe.elastic_ep_backend is not None:
from sglang.srt.elastic_ep.elastic_ep import (
elastic_expanded_world_enabled,
@@ -4029,7 +4031,7 @@ def require_mlp_tp_gather(*, moe_a2a_backend=None):
else:
return (
get_parallel().moe_dense_tp_size
> get_parallel().tp_size // get_parallel().dp_size
> get_parallel().tp_size // get_parallel().num_dp_ranks
)
else:
return False
@@ -4053,8 +4055,8 @@ def require_attn_tp_gather():
not get_moe_a2a_backend().is_none()
or get_parallel().moe_dense_tp_size is not None
):
if get_parallel().enable_dp_attention:
return get_parallel().dp_size < get_parallel().tp_size
if get_parallel().attn_dp_enabled:
return get_parallel().num_dp_ranks < get_parallel().tp_size
else:
return True
else:
@@ -4067,7 +4069,7 @@ def require_gathered_buffer():
def require_mlp_sync():
return get_parallel().enable_dp_attention or require_gathered_buffer()
return get_parallel().attn_dp_enabled or require_gathered_buffer()
def get_cuda_graph_batch_size_alignment() -> int:
@@ -192,8 +192,8 @@ def _contains_tensor_container(value) -> bool:
def get_vmm_feature_consumer_count() -> int:
if get_parallel().enable_dp_attention:
return get_parallel().tp_size // get_parallel().dp_size
if get_parallel().attn_dp_enabled:
return get_parallel().tp_size // get_parallel().num_dp_ranks
return get_parallel().tp_size
+2 -2
View File
@@ -210,7 +210,7 @@ class OffloaderV2(BaseOffloader):
mode: str,
):
parallel = get_parallel()
dp_rank, dp_size = parallel.dp_rank, parallel.dp_size
dp_rank, num_dp_ranks = parallel.dp_rank, parallel.num_dp_ranks
self.group_size = group_size
self.num_in_group = num_in_group
self.prefetch_step = prefetch_step
@@ -224,7 +224,7 @@ class OffloaderV2(BaseOffloader):
set_naive_distributed(
NaiveDistributed(
rank=dp_rank,
world_size=dp_size,
world_size=num_dp_ranks,
rendezvous=f"/tmp/{run_id}",
)
)
+1 -1
View File
@@ -357,7 +357,7 @@ class _ProfilerTorch(_ProfilerConcreteBase):
filename_parts = [self.profile_id, f"TP-{get_parallel().tp_rank}"]
# Only add other ranks if parallelism is enabled (size > 1)
if get_parallel().dp_size > 1:
if get_parallel().num_dp_ranks > 1:
filename_parts.append(f"DP-{get_parallel().dp_rank}")
if get_parallel().pp_size > 1:
filename_parts.append(f"PP-{get_parallel().pp_rank}")
+2 -4
View File
@@ -162,10 +162,8 @@ class WeightCacheDaemon:
self.tp_rank = tp_rank
self.pp_size = cfg.pp_size
self.pp_rank = pp_rank
self.dp_size = cfg.dp_size
self.ep_size = cfg.ep_size
self.moe_dp_size = cfg.moe_dp_size
self.enable_dp_attention = cfg.enable_dp_attention
self.enable_dp_lm_head = cfg.enable_dp_lm_head
self.attn_cp_size = cfg.attn_cp_size
self.moe_dense_tp_size = cfg.moe_dense_tp_size
@@ -338,12 +336,12 @@ class WeightCacheDaemon:
tp_rank=self.tp_rank,
pp_size=self.pp_size,
pp_rank=self.pp_rank,
dp_size=self.dp_size,
dp_size=get_parallel().dp_size,
ep_size=self.ep_size,
moe_dp_size=self.moe_dp_size,
moe_dp_rank=moe_dp_rank,
moe_ep_rank=moe_ep_rank,
enable_dp_attention=self.enable_dp_attention,
attn_dp_size=get_parallel().attn_dp_size,
enable_dp_lm_head=self.enable_dp_lm_head,
attn_cp_size=self.attn_cp_size,
moe_dense_tp_size=self.moe_dense_tp_size,
+2 -4
View File
@@ -515,8 +515,6 @@ class IpcModelLoader(BaseModelLoader):
moe_dp_rank = parallel.moe_dp_rank
moe_ep_rank = parallel.moe_ep_rank
dp_size = get_parallel().dp_size
quant_method, quant_config = self._resolve_engine_quant(model_config)
engine_config = CacheConfig(
@@ -530,12 +528,12 @@ class IpcModelLoader(BaseModelLoader):
tp_rank=tp_rank,
pp_size=pp_size,
pp_rank=pp_rank,
dp_size=dp_size,
dp_size=parallel.dp_size,
ep_size=ep_size,
moe_dp_size=moe_dp_size,
moe_dp_rank=moe_dp_rank,
moe_ep_rank=moe_ep_rank,
enable_dp_attention=parallel.enable_dp_attention,
attn_dp_size=parallel.attn_dp_size,
enable_dp_lm_head=parallel.enable_dp_lm_head,
attn_cp_size=parallel.attn_cp_size,
moe_dense_tp_size=parallel.moe_dense_tp_size,
+1 -1
View File
@@ -39,7 +39,7 @@ class CacheConfig(msgspec.Struct):
moe_dp_size: int
moe_dp_rank: int
moe_ep_rank: int
enable_dp_attention: bool
attn_dp_size: int
enable_dp_lm_head: bool
attn_cp_size: int
moe_dense_tp_size: Optional[int]
@@ -190,7 +190,7 @@ def get_jitter_engine(
engine_kwargs.setdefault("schedule_conservativeness", 0.05)
if (
engine_kwargs.get("enable_dp_attention")
engine_kwargs.get("attn_dp_size", 1) > 1
and "dist_init_addr" not in engine_kwargs
):
import portpicker
@@ -367,7 +367,6 @@ class MockModelRunner(ModelRunner):
dllm_algorithm=None,
dllm_algorithm_config=None,
dp_size=1,
enable_dp_attention=False,
enable_deterministic_inference=False,
enable_mis=False,
is_embedding=False,
@@ -341,7 +341,6 @@ class DSAMockModelRunner(ModelRunner):
dsa_prefill_backend=dsa_prefill_backend,
device=device,
enable_deterministic_inference=False,
enable_dp_attention=False,
enable_prefill_cp=False,
enable_mis=False,
is_embedding=False,
@@ -364,7 +364,6 @@ class MockDSV4ModelRunner:
disaggregation_mode=None,
dp_size=1,
enable_deterministic_inference=False,
enable_dp_attention=False,
enable_mis=False,
is_embedding=False,
kv_cache_dtype="auto",
@@ -271,7 +271,6 @@ class MockMLAModelRunner(ModelRunner):
dllm_algorithm=None,
dllm_algorithm_config=None,
dp_size=1,
enable_dp_attention=False,
enable_deterministic_inference=False,
enable_mis=False,
flashinfer_mla_disable_ragged=True,
+1 -2
View File
@@ -23,9 +23,8 @@ DSV4_POOL_SERVER_ARGS: Final[tuple[str, ...]] = (
"--trust-remote-code",
"--tp",
"4",
"--dp",
"--attn-dp-size",
"4",
"--enable-dp-attention",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs-decode",
+2 -4
View File
@@ -581,7 +581,7 @@ class SRTRunner:
context_length: Optional[int] = None,
max_total_tokens: Optional[int] = None,
page_size: Optional[int] = None,
dp_size: int = 1,
attn_dp_size: int = 1,
tokenizer_path: Optional[str] = None,
mem_fraction_static: float = 0.65,
trust_remote_code: bool = False,
@@ -607,7 +607,6 @@ class SRTRunner:
):
self.model_type = model_type
self.is_generation = model_type == "generation"
enable_dp_attention = dp_size > 1
spec_kwargs = {}
if speculative_draft_model_path:
@@ -645,8 +644,7 @@ class SRTRunner:
context_length=context_length,
max_total_tokens=max_total_tokens,
page_size=page_size,
enable_dp_attention=enable_dp_attention,
dp_size=dp_size,
attn_dp_size=attn_dp_size,
tokenizer_path=tokenizer_path,
disable_overlap_schedule=disable_overlap_schedule,
cuda_graph_max_bs_decode=cuda_graph_max_bs_decode,
@@ -3,7 +3,7 @@
Variants combine `DsaMtpServerBase` (server lifecycle) with
`DsaMtpEvalConfigDefaults` (shared eval thresholds/params),
`GSM8KMixin` and `SpecDecodingMixin`, then set `model` and per-variant
overrides (`enable_dp_attention`, `mem_fraction_static`, `bs_1_speed_thres`).
overrides (`attn_dp_size`, `mem_fraction_static`, `bs_1_speed_thres`).
Example:
class TestDsv32DP(
@@ -13,7 +13,7 @@ Example:
SpecDecodingMixin,
):
model = "deepseek-ai/DeepSeek-V3.2"
enable_dp_attention = True
attn_dp_size = 8
bs_1_speed_thres = 90
The base itself is NOT a runnable test (no `test_*` methods until a subclass
@@ -50,9 +50,8 @@ class DsaMtpServerBase(CustomTestCase):
# Subclasses must set `model`; the others have sensible defaults.
model: str = ""
tp_size: int = 8
dp_size: int = 8
attn_dp_size: int = 1
mem_fraction_static: float = 0.7
enable_dp_attention: bool = False
extra_server_args = ()
# EAGLE MTP config (fixed across DSA-MTP variants).
@@ -65,8 +64,8 @@ class DsaMtpServerBase(CustomTestCase):
def get_server_args(cls):
assert cls.model, f"{cls.__name__} must set `model`"
args = ["--trust-remote-code", "--tp", str(cls.tp_size)]
if cls.enable_dp_attention:
args += ["--dp", str(cls.dp_size), "--enable-dp-attention"]
if cls.attn_dp_size > 1:
args += ["--attn-dp-size", str(cls.attn_dp_size)]
args += [
"--speculative-algorithm",
cls.speculative_algorithm,