[CP V1 Deprecation 2/5] Make strategy prefill CP canonical (#36223)
This commit is contained in:
@@ -168,6 +168,9 @@ def validate_deepseek_v4_cp(server_args: ServerArgs) -> None:
|
|||||||
f"DeepSeekV4 only supports interleave CP strategy, got {cfg.cp_strategy}"
|
f"DeepSeekV4 only supports interleave CP strategy, got {cfg.cp_strategy}"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if get_platform().is_hip or get_platform().is_npu:
|
||||||
|
# Protected platform implementations still consume the legacy runtime
|
||||||
|
# fields. Generic backends use enable_prefill_cp/cp_strategy directly.
|
||||||
declare_resolution(
|
declare_resolution(
|
||||||
server_args,
|
server_args,
|
||||||
"validate_deepseek_v4_cp",
|
"validate_deepseek_v4_cp",
|
||||||
|
|||||||
@@ -31,35 +31,36 @@ def handle_context_parallelism(server_args: Any):
|
|||||||
|
|
||||||
cfg = resolving_view(server_args)
|
cfg = resolving_view(server_args)
|
||||||
if parse_connector_type(cfg.model_path) != ConnectorType.INSTANCE:
|
if parse_connector_type(cfg.model_path) != ConnectorType.INSTANCE:
|
||||||
from sglang.srt.configs.model_config import is_deepseek_dsa
|
|
||||||
from sglang.srt.layers.cp.utils import CP_V2_DEFAULT_MODEL_CLASSES
|
|
||||||
|
|
||||||
model_config = model_config_of(server_args)
|
model_config = model_config_of(server_args)
|
||||||
hf_config = model_config.hf_config
|
hf_config = model_config.hf_config
|
||||||
model_arch = hf_config.architectures[0]
|
model_arch = hf_config.architectures[0]
|
||||||
if model_arch in CP_V2_DEFAULT_MODEL_CLASSES:
|
platform = get_platform()
|
||||||
is_dsa_default_model = is_deepseek_dsa(hf_config)
|
|
||||||
# DSA CP-v2 currently supports only the interleave strategy.
|
|
||||||
enable_default_cp_v2 = not is_dsa_default_model or (
|
|
||||||
cfg.enable_prefill_cp and cfg.cp_strategy == "interleave"
|
|
||||||
)
|
|
||||||
if enable_default_cp_v2 and not envs.SGLANG_ENABLE_CP_V2.is_set():
|
|
||||||
envs.SGLANG_ENABLE_CP_V2.set(True)
|
|
||||||
|
|
||||||
if (
|
if (
|
||||||
cfg.enable_prefill_cp
|
cfg.enable_prefill_cp
|
||||||
and model_arch in ("MiMoV2ForCausalLM", "MiMoV2FlashForCausalLM")
|
and model_arch == "DeepseekV32ForCausalLM"
|
||||||
and envs.SGLANG_ENABLE_CP_V2.get()
|
and cfg.cp_strategy == "zigzag"
|
||||||
|
and not (platform.is_hip or platform.is_npu)
|
||||||
|
):
|
||||||
|
raise ValueError(
|
||||||
|
"DeepSeek V3.2 prefill CP does not support --cp-strategy "
|
||||||
|
"zigzag; use interleave."
|
||||||
|
)
|
||||||
|
if cfg.enable_prefill_cp and model_arch in (
|
||||||
|
"MiMoV2ForCausalLM",
|
||||||
|
"MiMoV2FlashForCausalLM",
|
||||||
):
|
):
|
||||||
if cfg.cp_strategy != "zigzag":
|
if cfg.cp_strategy != "zigzag":
|
||||||
raise ValueError("MiMo V2 CP-v2 only supports --cp-strategy zigzag.")
|
raise ValueError(
|
||||||
|
"MiMo V2 prefill CP only supports --cp-strategy zigzag."
|
||||||
|
)
|
||||||
if (
|
if (
|
||||||
model_config.is_multimodal
|
model_config.is_multimodal
|
||||||
and not cfg.language_only
|
and not cfg.language_only
|
||||||
and not cfg.language_model_only
|
and not cfg.language_model_only
|
||||||
):
|
):
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
"MiMo V2 CP-v2 only supports text inference; add --language-only."
|
"MiMo V2 prefill CP only supports text inference; add "
|
||||||
|
"--language-only."
|
||||||
)
|
)
|
||||||
|
|
||||||
if cfg.enable_prefill_cp and cfg.cp_strategy is None:
|
if cfg.enable_prefill_cp and cfg.cp_strategy is None:
|
||||||
@@ -559,43 +560,62 @@ def handle_eplb_and_dispatch(server_args: Any):
|
|||||||
assert resolved_view(server_args).ep_size > 1
|
assert resolved_view(server_args).ep_size > 1
|
||||||
|
|
||||||
|
|
||||||
def handle_legacy_cp_arguments(server_args: Any):
|
def handle_platform_cp_compatibility(server_args: Any):
|
||||||
cfg = resolving_view(server_args)
|
cfg = resolving_view(server_args)
|
||||||
|
platform = get_platform()
|
||||||
|
is_protected_platform = platform.is_hip or platform.is_npu
|
||||||
|
if not is_protected_platform:
|
||||||
|
if (
|
||||||
|
server_args.enable_prefill_context_parallel
|
||||||
|
or server_args.enable_dsa_prefill_context_parallel
|
||||||
|
):
|
||||||
|
raise ValueError(
|
||||||
|
"Legacy prefill context-parallel options are supported only "
|
||||||
|
"by protected HIP or Ascend NPU paths. Use "
|
||||||
|
"--enable-prefill-cp with --cp-strategy."
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
legacy_mode_to_strategy = {
|
legacy_mode_to_strategy = {
|
||||||
"in-seq-split": "zigzag",
|
"in-seq-split": "zigzag",
|
||||||
"round-robin-split": "interleave",
|
"round-robin-split": "interleave",
|
||||||
}
|
}
|
||||||
strategy_to_legacy_mode = {
|
|
||||||
"zigzag": "in-seq-split",
|
|
||||||
"interleave": "round-robin-split",
|
|
||||||
}
|
|
||||||
|
|
||||||
if cfg.enable_prefill_context_parallel or cfg.enable_dsa_prefill_context_parallel:
|
if cfg.enable_prefill_context_parallel or cfg.enable_dsa_prefill_context_parallel:
|
||||||
declare_resolution(
|
declare_resolution(
|
||||||
server_args,
|
server_args,
|
||||||
"_handle_legacy_cp_arguments",
|
"_handle_platform_cp_compatibility",
|
||||||
enable_prefill_cp=True,
|
enable_prefill_cp=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
if cfg.enable_prefill_context_parallel and cfg.cp_strategy is None:
|
if cfg.enable_prefill_context_parallel and cfg.cp_strategy is None:
|
||||||
declare_resolution(
|
declare_resolution(
|
||||||
server_args,
|
server_args,
|
||||||
"_handle_legacy_cp_arguments",
|
"_handle_platform_cp_compatibility",
|
||||||
cp_strategy=legacy_mode_to_strategy[cfg.prefill_cp_mode],
|
cp_strategy=legacy_mode_to_strategy[cfg.prefill_cp_mode],
|
||||||
)
|
)
|
||||||
if cfg.enable_dsa_prefill_context_parallel and cfg.cp_strategy is None:
|
if cfg.enable_dsa_prefill_context_parallel and cfg.cp_strategy is None:
|
||||||
declare_resolution(
|
declare_resolution(
|
||||||
server_args,
|
server_args,
|
||||||
"_handle_legacy_cp_arguments",
|
"_handle_platform_cp_compatibility",
|
||||||
cp_strategy=legacy_mode_to_strategy[cfg.dsa_prefill_cp_mode],
|
cp_strategy=legacy_mode_to_strategy[cfg.dsa_prefill_cp_mode],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def handle_legacy_cp_runtime_compatibility(server_args: Any):
|
||||||
|
"""Project canonical CP settings for runtime consumers removed by PR3."""
|
||||||
|
cfg = resolving_view(server_args)
|
||||||
|
|
||||||
if cfg.enable_prefill_context_parallel and cfg.enable_dsa_prefill_context_parallel:
|
if cfg.enable_prefill_context_parallel and cfg.enable_dsa_prefill_context_parallel:
|
||||||
return
|
return
|
||||||
|
|
||||||
if not cfg.enable_prefill_cp or cfg.cp_strategy is None:
|
if not cfg.enable_prefill_cp or cfg.cp_strategy is None:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
strategy_to_legacy_mode = {
|
||||||
|
"zigzag": "in-seq-split",
|
||||||
|
"interleave": "round-robin-split",
|
||||||
|
}
|
||||||
mode = strategy_to_legacy_mode[cfg.cp_strategy]
|
mode = strategy_to_legacy_mode[cfg.cp_strategy]
|
||||||
use_dsa_legacy_aliases = cfg.enable_dsa_prefill_context_parallel or getattr(
|
use_dsa_legacy_aliases = cfg.enable_dsa_prefill_context_parallel or getattr(
|
||||||
resolved_view(server_args), "attention_backend", None
|
resolved_view(server_args), "attention_backend", None
|
||||||
@@ -603,28 +623,28 @@ def handle_legacy_cp_arguments(server_args: Any):
|
|||||||
if use_dsa_legacy_aliases:
|
if use_dsa_legacy_aliases:
|
||||||
declare_resolution(
|
declare_resolution(
|
||||||
server_args,
|
server_args,
|
||||||
"_handle_legacy_cp_arguments",
|
"_handle_legacy_cp_runtime_compatibility",
|
||||||
enable_dsa_prefill_context_parallel=True,
|
enable_dsa_prefill_context_parallel=True,
|
||||||
)
|
)
|
||||||
declare_resolution(
|
declare_resolution(
|
||||||
server_args,
|
server_args,
|
||||||
"_handle_legacy_cp_arguments",
|
"_handle_legacy_cp_runtime_compatibility",
|
||||||
enable_prefill_context_parallel=False,
|
enable_prefill_context_parallel=False,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
declare_resolution(
|
declare_resolution(
|
||||||
server_args,
|
server_args,
|
||||||
"_handle_legacy_cp_arguments",
|
"_handle_legacy_cp_runtime_compatibility",
|
||||||
enable_prefill_context_parallel=True,
|
enable_prefill_context_parallel=True,
|
||||||
)
|
)
|
||||||
declare_resolution(
|
declare_resolution(
|
||||||
server_args,
|
server_args,
|
||||||
"_handle_legacy_cp_arguments",
|
"_handle_legacy_cp_runtime_compatibility",
|
||||||
dsa_prefill_cp_mode=mode,
|
dsa_prefill_cp_mode=mode,
|
||||||
)
|
)
|
||||||
declare_resolution(
|
declare_resolution(
|
||||||
server_args,
|
server_args,
|
||||||
"_handle_legacy_cp_arguments",
|
"_handle_legacy_cp_runtime_compatibility",
|
||||||
prefill_cp_mode=mode,
|
prefill_cp_mode=mode,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -146,8 +146,8 @@ def run_resolution_pipeline(server_args: Any) -> None:
|
|||||||
|
|
||||||
handle_pd_disaggregation(server_args)
|
handle_pd_disaggregation(server_args)
|
||||||
|
|
||||||
# Normalize deprecated CP aliases before validations or model-specific
|
# Normalize protected-platform CP aliases before validations or
|
||||||
# defaults inspect enable_prefill_cp/cp_strategy.
|
# model-specific defaults inspect enable_prefill_cp/cp_strategy.
|
||||||
from sglang.srt.arg_groups.parallel_hook import (
|
from sglang.srt.arg_groups.parallel_hook import (
|
||||||
handle_context_parallelism,
|
handle_context_parallelism,
|
||||||
handle_data_parallelism,
|
handle_data_parallelism,
|
||||||
@@ -156,10 +156,11 @@ def run_resolution_pipeline(server_args: Any) -> None:
|
|||||||
handle_elastic_ep,
|
handle_elastic_ep,
|
||||||
handle_eplb_and_dispatch,
|
handle_eplb_and_dispatch,
|
||||||
handle_expert_distribution_metrics,
|
handle_expert_distribution_metrics,
|
||||||
handle_legacy_cp_arguments,
|
handle_legacy_cp_runtime_compatibility,
|
||||||
|
handle_platform_cp_compatibility,
|
||||||
)
|
)
|
||||||
|
|
||||||
handle_legacy_cp_arguments(server_args)
|
handle_platform_cp_compatibility(server_args)
|
||||||
from sglang.srt.arg_groups.kv_cache_hook import (
|
from sglang.srt.arg_groups.kv_cache_hook import (
|
||||||
handle_cache_compatibility,
|
handle_cache_compatibility,
|
||||||
handle_kv4_compatibility,
|
handle_kv4_compatibility,
|
||||||
@@ -286,9 +287,9 @@ def run_resolution_pipeline(server_args: Any) -> None:
|
|||||||
# Normalize load balancing defaults.
|
# Normalize load balancing defaults.
|
||||||
handle_load_balance_method(server_args)
|
handle_load_balance_method(server_args)
|
||||||
|
|
||||||
# Re-apply after model-specific defaults resolve attention_backend so
|
# The old runtime distinguishes DSA from other CP paths through legacy
|
||||||
# canonical CP mirrors to the right legacy runtime aliases.
|
# fields, so project only after attention_backend has been resolved.
|
||||||
handle_legacy_cp_arguments(server_args)
|
handle_legacy_cp_runtime_compatibility(server_args)
|
||||||
|
|
||||||
# Handle context parallelism.
|
# Handle context parallelism.
|
||||||
handle_context_parallelism(server_args)
|
handle_context_parallelism(server_args)
|
||||||
|
|||||||
@@ -677,7 +677,6 @@ class Envs:
|
|||||||
# ===================================================================
|
# ===================================================================
|
||||||
# Distributed and model-parallel runtime
|
# Distributed and model-parallel runtime
|
||||||
# ===================================================================
|
# ===================================================================
|
||||||
SGLANG_ENABLE_CP_V2 = EnvBool(False)
|
|
||||||
SGLANG_ONE_VISIBLE_DEVICE_PER_PROCESS = EnvBool(False)
|
SGLANG_ONE_VISIBLE_DEVICE_PER_PROCESS = EnvBool(False)
|
||||||
# Comma-separated bundle indices for Ray Custom PG mode (e.g., "0,1,2,7").
|
# Comma-separated bundle indices for Ray Custom PG mode (e.g., "0,1,2,7").
|
||||||
SGLANG_RAY_BUNDLE_INDICES = EnvStr("")
|
SGLANG_RAY_BUNDLE_INDICES = EnvStr("")
|
||||||
@@ -1753,6 +1752,9 @@ _DEPRECATED_ENVS: Dict[str, _DeprecatedEnv] = {
|
|||||||
note="Note the unit change: milliseconds -> seconds.",
|
note="Note the unit change: milliseconds -> seconds.",
|
||||||
),
|
),
|
||||||
# Removed without replacement.
|
# Removed without replacement.
|
||||||
|
"SGLANG_ENABLE_CP_V2": _DeprecatedEnv(
|
||||||
|
note="Strategy-based prefill context parallelism is now the only generic implementation."
|
||||||
|
),
|
||||||
"SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2": _DeprecatedEnv(),
|
"SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2": _DeprecatedEnv(),
|
||||||
# Superseded by the unified JIT per_token_group_quant, the default CUDA path.
|
# Superseded by the unified JIT per_token_group_quant, the default CUDA path.
|
||||||
"SGLANG_OPT_USE_JIT_PER_TOKEN_GROUP_QUANT": _DeprecatedEnv(),
|
"SGLANG_OPT_USE_JIT_PER_TOKEN_GROUP_QUANT": _DeprecatedEnv(),
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ from sglang.srt.runtime_context import (
|
|||||||
get_parallel,
|
get_parallel,
|
||||||
process_model_config,
|
process_model_config,
|
||||||
)
|
)
|
||||||
from sglang.srt.utils import get_bool_env_var, is_cuda, is_hip
|
from sglang.srt.utils import get_bool_env_var, is_cuda, is_hip, is_npu
|
||||||
from sglang.srt.utils.common import ceil_align, ceil_div
|
from sglang.srt.utils.common import ceil_align, ceil_div
|
||||||
|
|
||||||
|
|
||||||
@@ -105,12 +105,11 @@ def should_use_dsa_fused_topk(seed_dsa_topk_from_draft_extend: bool) -> bool:
|
|||||||
|
|
||||||
|
|
||||||
def is_dsa_enable_prefill_cp():
|
def is_dsa_enable_prefill_cp():
|
||||||
if not envs.SGLANG_ENABLE_CP_V2.get():
|
if is_hip() or is_npu():
|
||||||
return get_parallel().enable_dsa_prefill_context_parallel
|
return get_parallel().enable_dsa_prefill_context_parallel
|
||||||
|
|
||||||
# Derive from the runtime CP topology + model arch rather than the legacy
|
# Generic prefill CP derives activation from the runtime topology and model
|
||||||
# flag under CP-v2: DSA prefill CP is active when the CP group is on for a
|
# architecture. Protected HIP/NPU paths continue to use their legacy field.
|
||||||
# DeepSeek Sparse Attention model.
|
|
||||||
if get_parallel().attn_cp_size <= 1:
|
if get_parallel().attn_cp_size <= 1:
|
||||||
return False
|
return False
|
||||||
from sglang.srt.configs.model_config import is_deepseek_dsa, is_deepseek_v4
|
from sglang.srt.configs.model_config import is_deepseek_dsa, is_deepseek_v4
|
||||||
|
|||||||
@@ -40,18 +40,6 @@ from sglang.srt.runtime_context import get_parallel
|
|||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||||
|
|
||||||
CP_V2_DEFAULT_MODEL_CLASSES = frozenset(
|
|
||||||
{
|
|
||||||
"DeepseekV32ForCausalLM",
|
|
||||||
"GlmMoeDsaForCausalLM",
|
|
||||||
"GptOssForCausalLM",
|
|
||||||
"MiMoV2FlashForCausalLM",
|
|
||||||
"MiMoV2ForCausalLM",
|
|
||||||
"Qwen3MoeForCausalLM",
|
|
||||||
"DeepseekV3ForCausalLM",
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def is_glm_dsa_cache_layer_split_enabled(model_runner: "ModelRunner") -> bool:
|
def is_glm_dsa_cache_layer_split_enabled(model_runner: "ModelRunner") -> bool:
|
||||||
"""Whether DSA GPU KV/indexer cache layers are sharded across CP ranks.
|
"""Whether DSA GPU KV/indexer cache layers are sharded across CP ranks.
|
||||||
@@ -130,10 +118,10 @@ def get_layer_owner(local_layer_idx: int, shard_size: int, total_layers: int) ->
|
|||||||
|
|
||||||
|
|
||||||
def enable_cp_v2() -> bool:
|
def enable_cp_v2() -> bool:
|
||||||
"""Return whether the CP-v2 path is enabled for this process."""
|
"""Return whether the strategy-based generic prefill CP path is available."""
|
||||||
from sglang.srt.environ import envs
|
from sglang.srt.utils import is_hip, is_npu
|
||||||
|
|
||||||
return bool(envs.SGLANG_ENABLE_CP_V2.get())
|
return not (is_hip() or is_npu())
|
||||||
|
|
||||||
|
|
||||||
def is_cp_v2_active(forward_batch) -> bool:
|
def is_cp_v2_active(forward_batch) -> bool:
|
||||||
@@ -322,7 +310,6 @@ __all__ = [
|
|||||||
"InterleaveContextParallelMetadata",
|
"InterleaveContextParallelMetadata",
|
||||||
"ZigzagCPStrategy",
|
"ZigzagCPStrategy",
|
||||||
"ZigzagContextParallelMetadata",
|
"ZigzagContextParallelMetadata",
|
||||||
"CP_V2_DEFAULT_MODEL_CLASSES",
|
|
||||||
"enable_cp_v2",
|
"enable_cp_v2",
|
||||||
"get_cp_strategy",
|
"get_cp_strategy",
|
||||||
"is_cp_v2_active",
|
"is_cp_v2_active",
|
||||||
|
|||||||
@@ -4019,13 +4019,6 @@ class ServerArgs:
|
|||||||
dest="cuda_graph_max_bs_prefill",
|
dest="cuda_graph_max_bs_prefill",
|
||||||
help="Deprecated alias for --cuda-graph-max-bs-prefill.",
|
help="Deprecated alias for --cuda-graph-max-bs-prefill.",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
|
||||||
"--enable-dsa-prefill-context-parallel",
|
|
||||||
dest="enable_dsa_prefill_context_parallel",
|
|
||||||
action=DeprecatedStoreTrueAction,
|
|
||||||
new_flag="--enable-prefill-cp",
|
|
||||||
help="[Deprecated] Use --enable-prefill-cp instead.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--enable-nsa-prefill-context-parallel",
|
"--enable-nsa-prefill-context-parallel",
|
||||||
dest="enable_dsa_prefill_context_parallel",
|
dest="enable_dsa_prefill_context_parallel",
|
||||||
@@ -4047,20 +4040,6 @@ class ServerArgs:
|
|||||||
new_flag="--enable-prefill-cp",
|
new_flag="--enable-prefill-cp",
|
||||||
help="[Deprecated] Use --enable-prefill-cp instead.",
|
help="[Deprecated] Use --enable-prefill-cp instead.",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
|
||||||
"--dsa-prefill-cp-mode",
|
|
||||||
dest="dsa_prefill_cp_mode",
|
|
||||||
action=DeprecatedAliasStoreAction,
|
|
||||||
new_flag="--cp-strategy",
|
|
||||||
type=str,
|
|
||||||
default=ServerArgs.dsa_prefill_cp_mode,
|
|
||||||
choices=["in-seq-split", "round-robin-split"],
|
|
||||||
help=(
|
|
||||||
"[Deprecated] Use --cp-strategy {zigzag,interleave} instead. "
|
|
||||||
"'in-seq-split' maps to 'zigzag'; 'round-robin-split' maps to "
|
|
||||||
"'interleave'."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--nsa-prefill-cp-mode",
|
"--nsa-prefill-cp-mode",
|
||||||
dest="dsa_prefill_cp_mode",
|
dest="dsa_prefill_cp_mode",
|
||||||
@@ -4071,19 +4050,6 @@ class ServerArgs:
|
|||||||
choices=["in-seq-split", "round-robin-split"],
|
choices=["in-seq-split", "round-robin-split"],
|
||||||
help="[Deprecated] Use --cp-strategy instead.",
|
help="[Deprecated] Use --cp-strategy instead.",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
|
||||||
"--prefill-cp-mode",
|
|
||||||
dest="prefill_cp_mode",
|
|
||||||
action=DeprecatedAliasStoreAction,
|
|
||||||
new_flag="--cp-strategy",
|
|
||||||
type=str,
|
|
||||||
default=ServerArgs.prefill_cp_mode,
|
|
||||||
choices=["in-seq-split"],
|
|
||||||
help=(
|
|
||||||
"[Deprecated] Use --cp-strategy {zigzag,interleave} instead. "
|
|
||||||
"'in-seq-split' maps to 'zigzag'."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--enable-flashinfer-allreduce-fusion",
|
"--enable-flashinfer-allreduce-fusion",
|
||||||
action="store_true",
|
action="store_true",
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ from unittest.mock import patch
|
|||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
|
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
|
||||||
from sglang.srt.layers.cp.base import (
|
from sglang.srt.layers.cp.base import (
|
||||||
ContextParallelStrategyKind,
|
ContextParallelStrategyKind,
|
||||||
get_cp_strategy,
|
get_cp_strategy,
|
||||||
@@ -94,13 +95,44 @@ class TestCPStrategyUnit(CustomTestCase):
|
|||||||
cp_strategy="interleave",
|
cp_strategy="interleave",
|
||||||
)
|
)
|
||||||
|
|
||||||
with patch(
|
|
||||||
"sglang.srt.environ.envs.SGLANG_ENABLE_CP_V2.get", return_value=True
|
|
||||||
):
|
|
||||||
self.assertIsNotNone(get_cp_strategy())
|
self.assertIsNotNone(get_cp_strategy())
|
||||||
self.assertTrue(is_cp_enabled())
|
self.assertTrue(is_cp_enabled())
|
||||||
self.assertTrue(is_interleave())
|
self.assertTrue(is_interleave())
|
||||||
|
|
||||||
|
def test_hip_dsa_cp_uses_protected_legacy_runtime_flag(self):
|
||||||
|
parallel = SimpleNamespace(
|
||||||
|
enable_dsa_prefill_context_parallel=False,
|
||||||
|
attn_cp_size=2,
|
||||||
|
)
|
||||||
|
model_config = SimpleNamespace(hf_config=SimpleNamespace())
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch(
|
||||||
|
"sglang.srt.layers.attention.dsa.utils.get_parallel",
|
||||||
|
return_value=parallel,
|
||||||
|
),
|
||||||
|
patch(
|
||||||
|
"sglang.srt.layers.attention.dsa.utils.process_model_config",
|
||||||
|
return_value=model_config,
|
||||||
|
),
|
||||||
|
patch("sglang.srt.layers.attention.dsa.utils.is_hip", return_value=True),
|
||||||
|
patch(
|
||||||
|
"sglang.srt.configs.model_config.is_deepseek_dsa",
|
||||||
|
return_value=True,
|
||||||
|
),
|
||||||
|
):
|
||||||
|
self.assertFalse(is_dsa_enable_prefill_cp())
|
||||||
|
|
||||||
|
@patch("sglang.srt.utils.is_npu", return_value=False)
|
||||||
|
@patch("sglang.srt.utils.is_hip", return_value=True)
|
||||||
|
def test_hip_keeps_strategy_cp_disabled(self, _mock_is_hip, _mock_is_npu):
|
||||||
|
self.assertFalse(enable_cp_v2())
|
||||||
|
|
||||||
|
@patch("sglang.srt.utils.is_npu", return_value=True)
|
||||||
|
@patch("sglang.srt.utils.is_hip", return_value=False)
|
||||||
|
def test_npu_keeps_strategy_cp_disabled(self, _mock_is_hip, _mock_is_npu):
|
||||||
|
self.assertFalse(enable_cp_v2())
|
||||||
|
|
||||||
|
|
||||||
class TestPrefillCPBCGReplay(CustomTestCase):
|
class TestPrefillCPBCGReplay(CustomTestCase):
|
||||||
def tearDown(self):
|
def tearDown(self):
|
||||||
@@ -155,10 +187,6 @@ class TestPrefillCPBCGReplay(CustomTestCase):
|
|||||||
self._enable_zigzag()
|
self._enable_zigzag()
|
||||||
|
|
||||||
with (
|
with (
|
||||||
patch(
|
|
||||||
"sglang.srt.environ.envs.SGLANG_ENABLE_CP_V2.get",
|
|
||||||
return_value=True,
|
|
||||||
),
|
|
||||||
patch(
|
patch(
|
||||||
"sglang.srt.layers.cp.bcg.get_cp_padding_align_size",
|
"sglang.srt.layers.cp.bcg.get_cp_padding_align_size",
|
||||||
return_value=8,
|
return_value=8,
|
||||||
@@ -209,10 +237,6 @@ class TestPrefillCPBCGReplay(CustomTestCase):
|
|||||||
|
|
||||||
with (
|
with (
|
||||||
get_parallel().override(attn_cp_rank=0, attn_cp_size=4),
|
get_parallel().override(attn_cp_rank=0, attn_cp_size=4),
|
||||||
patch(
|
|
||||||
"sglang.srt.environ.envs.SGLANG_ENABLE_CP_V2.get",
|
|
||||||
return_value=True,
|
|
||||||
),
|
|
||||||
patch(
|
patch(
|
||||||
"sglang.srt.layers.cp.bcg.get_cp_padding_align_size",
|
"sglang.srt.layers.cp.bcg.get_cp_padding_align_size",
|
||||||
return_value=8,
|
return_value=8,
|
||||||
@@ -243,10 +267,6 @@ class TestPrefillCPBCGReplay(CustomTestCase):
|
|||||||
|
|
||||||
with (
|
with (
|
||||||
get_parallel().override(attn_cp_rank=0, attn_cp_size=4),
|
get_parallel().override(attn_cp_rank=0, attn_cp_size=4),
|
||||||
patch(
|
|
||||||
"sglang.srt.environ.envs.SGLANG_ENABLE_CP_V2.get",
|
|
||||||
return_value=True,
|
|
||||||
),
|
|
||||||
patch(
|
patch(
|
||||||
"sglang.srt.layers.cp.bcg.get_cp_padding_align_size",
|
"sglang.srt.layers.cp.bcg.get_cp_padding_align_size",
|
||||||
return_value=8,
|
return_value=8,
|
||||||
@@ -298,9 +318,7 @@ class TestCPZigzagStrategy(CustomTestCase):
|
|||||||
extend_seq_lens_cpu=[7],
|
extend_seq_lens_cpu=[7],
|
||||||
)
|
)
|
||||||
|
|
||||||
with patch(
|
with patch.dict("os.environ", {"SGLANG_ENABLE_CP_V2": "0"}):
|
||||||
"sglang.srt.environ.envs.SGLANG_ENABLE_CP_V2.get", return_value=True
|
|
||||||
):
|
|
||||||
self.assertTrue(enable_cp_v2())
|
self.assertTrue(enable_cp_v2())
|
||||||
self.assertTrue(is_cp_v2_active(active_batch))
|
self.assertTrue(is_cp_v2_active(active_batch))
|
||||||
self.assertFalse(is_cp_v2_active(inactive_batch))
|
self.assertFalse(is_cp_v2_active(inactive_batch))
|
||||||
@@ -494,9 +512,6 @@ class TestCPZigzagStrategy(CustomTestCase):
|
|||||||
|
|
||||||
local_x = strategy.shard_hidden_states(x, fb)
|
local_x = strategy.shard_hidden_states(x, fb)
|
||||||
local_positions = strategy.shard_position_ids(positions, fb)
|
local_positions = strategy.shard_position_ids(positions, fb)
|
||||||
with patch(
|
|
||||||
"sglang.srt.environ.envs.SGLANG_ENABLE_CP_V2.get", return_value=True
|
|
||||||
):
|
|
||||||
helper_x, helper_positions = cp_split_before_forward(
|
helper_x, helper_positions = cp_split_before_forward(
|
||||||
x,
|
x,
|
||||||
positions,
|
positions,
|
||||||
@@ -916,10 +931,6 @@ class TestCPInterleaveStrategy(CustomTestCase):
|
|||||||
|
|
||||||
with (
|
with (
|
||||||
get_parallel().override(attn_cp_rank=2, attn_cp_size=4),
|
get_parallel().override(attn_cp_rank=2, attn_cp_size=4),
|
||||||
patch(
|
|
||||||
"sglang.srt.environ.envs.SGLANG_ENABLE_CP_V2.get",
|
|
||||||
return_value=True,
|
|
||||||
),
|
|
||||||
patch(
|
patch(
|
||||||
"sglang.srt.layers.cp.padding.get_cp_padding_align_size",
|
"sglang.srt.layers.cp.padding.get_cp_padding_align_size",
|
||||||
return_value=4,
|
return_value=4,
|
||||||
@@ -965,10 +976,6 @@ class TestCPInterleaveStrategy(CustomTestCase):
|
|||||||
local_x = strategy.shard_hidden_states(x, fb)
|
local_x = strategy.shard_hidden_states(x, fb)
|
||||||
local_positions = strategy.shard_position_ids(positions, fb)
|
local_positions = strategy.shard_position_ids(positions, fb)
|
||||||
|
|
||||||
with patch(
|
|
||||||
"sglang.srt.environ.envs.SGLANG_ENABLE_CP_V2.get",
|
|
||||||
return_value=True,
|
|
||||||
):
|
|
||||||
helper_x, helper_positions = cp_split_before_forward(
|
helper_x, helper_positions = cp_split_before_forward(
|
||||||
x,
|
x,
|
||||||
positions,
|
positions,
|
||||||
|
|||||||
@@ -47,7 +47,8 @@ from sglang.srt.arg_groups.overrides import (
|
|||||||
from sglang.srt.arg_groups.parallel_hook import (
|
from sglang.srt.arg_groups.parallel_hook import (
|
||||||
handle_context_parallelism,
|
handle_context_parallelism,
|
||||||
handle_data_parallelism,
|
handle_data_parallelism,
|
||||||
handle_legacy_cp_arguments,
|
handle_legacy_cp_runtime_compatibility,
|
||||||
|
handle_platform_cp_compatibility,
|
||||||
)
|
)
|
||||||
from sglang.srt.arg_groups.pd_disaggregation_hook import handle_pd_disaggregation
|
from sglang.srt.arg_groups.pd_disaggregation_hook import handle_pd_disaggregation
|
||||||
from sglang.srt.arg_groups.serving_hook import (
|
from sglang.srt.arg_groups.serving_hook import (
|
||||||
@@ -1056,52 +1057,116 @@ class TestContextParallelServerArgs(CustomTestCase):
|
|||||||
with self.assertRaisesRegex(ValueError, "--cp-strategy"):
|
with self.assertRaisesRegex(ValueError, "--cp-strategy"):
|
||||||
handle_context_parallelism(server_args)
|
handle_context_parallelism(server_args)
|
||||||
|
|
||||||
def test_deprecated_dsa_cp_mode_maps_to_unified_strategy(self):
|
@override_platform(is_hip=False, is_npu=False)
|
||||||
|
def test_deepseek_v32_prefill_cp_rejects_zigzag(self):
|
||||||
|
server_args = self._new_cp_args(
|
||||||
|
model_path="deepseek-ai/DeepSeek-V3.2",
|
||||||
|
enable_prefill_cp=True,
|
||||||
|
cp_strategy="zigzag",
|
||||||
|
)
|
||||||
|
server_args._model_config = SimpleNamespace(
|
||||||
|
hf_config=SimpleNamespace(architectures=["DeepseekV32ForCausalLM"]),
|
||||||
|
is_multimodal=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
with self.assertRaisesRegex(ValueError, "DeepSeek V3.2.*interleave"):
|
||||||
|
handle_context_parallelism(server_args)
|
||||||
|
|
||||||
|
@override_platform(is_hip=False, is_npu=False)
|
||||||
|
def test_generic_canonical_cp_mirrors_to_transitional_runtime_fields(self):
|
||||||
|
cases = (
|
||||||
|
(
|
||||||
|
"zigzag_mla_or_gqa",
|
||||||
|
"zigzag",
|
||||||
|
"fa3",
|
||||||
|
True,
|
||||||
|
False,
|
||||||
|
"in-seq-split",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"interleave_dsa",
|
||||||
|
"interleave",
|
||||||
|
"dsa",
|
||||||
|
False,
|
||||||
|
True,
|
||||||
|
"round-robin-split",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
for name, strategy, backend, expect_generic, expect_dsa, mode in cases:
|
||||||
|
with self.subTest(name=name):
|
||||||
|
server_args = self._new_cp_args(
|
||||||
|
enable_prefill_cp=True,
|
||||||
|
cp_strategy=strategy,
|
||||||
|
attention_backend=backend,
|
||||||
|
)
|
||||||
|
|
||||||
|
handle_platform_cp_compatibility(server_args)
|
||||||
|
|
||||||
|
self.assertFalse(
|
||||||
|
resolution_result(server_args, "enable_prefill_context_parallel")
|
||||||
|
)
|
||||||
|
self.assertFalse(
|
||||||
|
resolution_result(
|
||||||
|
server_args, "enable_dsa_prefill_context_parallel"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
handle_legacy_cp_runtime_compatibility(server_args)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
resolution_result(server_args, "enable_prefill_context_parallel"),
|
||||||
|
expect_generic,
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
resolution_result(
|
||||||
|
server_args, "enable_dsa_prefill_context_parallel"
|
||||||
|
),
|
||||||
|
expect_dsa,
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
resolution_result(server_args, "dsa_prefill_cp_mode"), mode
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
resolution_result(server_args, "prefill_cp_mode"), mode
|
||||||
|
)
|
||||||
|
|
||||||
|
@override_platform(is_hip=False, is_npu=False)
|
||||||
|
def test_non_platform_legacy_prefill_cp_is_rejected(self):
|
||||||
|
server_args = ServerArgs(
|
||||||
|
model_path="instance://127.0.0.1:8000/dummy",
|
||||||
|
enable_prefill_context_parallel=True,
|
||||||
|
)
|
||||||
|
with self.assertRaisesRegex(ValueError, "HIP or Ascend NPU"):
|
||||||
|
handle_platform_cp_compatibility(server_args)
|
||||||
|
|
||||||
|
def test_generic_v1_cp_options_are_not_public_cli(self):
|
||||||
|
removed_options = (
|
||||||
|
("--enable-dsa-prefill-context-parallel", []),
|
||||||
|
("--dsa-prefill-cp-mode", ["round-robin-split"]),
|
||||||
|
("--prefill-cp-mode", ["in-seq-split"]),
|
||||||
|
)
|
||||||
|
|
||||||
|
for option, values in removed_options:
|
||||||
|
with self.subTest(option=option), self.assertRaises(SystemExit):
|
||||||
|
self.parser.parse_args(["--model", "dummy", option, *values])
|
||||||
|
|
||||||
|
def test_npu_cp_compatibility_options_remain_public_cli(self):
|
||||||
args = self.parser.parse_args(
|
args = self.parser.parse_args(
|
||||||
[
|
[
|
||||||
"--model",
|
"--model",
|
||||||
"dummy",
|
"dummy",
|
||||||
"--enable-dsa-prefill-context-parallel",
|
"--enable-prefill-context-parallel",
|
||||||
"--dsa-prefill-cp-mode",
|
"--enable-nsa-prefill-context-parallel",
|
||||||
|
"--nsa-prefill-cp-mode",
|
||||||
"round-robin-split",
|
"round-robin-split",
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
server_args = self._new_cp_args(
|
|
||||||
enable_dsa_prefill_context_parallel=(
|
|
||||||
resolution_result(args, "enable_dsa_prefill_context_parallel")
|
|
||||||
),
|
|
||||||
dsa_prefill_cp_mode=resolution_result(args, "dsa_prefill_cp_mode"),
|
|
||||||
)
|
|
||||||
|
|
||||||
handle_legacy_cp_arguments(server_args)
|
self.assertTrue(resolution_result(args, "enable_prefill_context_parallel"))
|
||||||
|
self.assertTrue(resolution_result(args, "enable_dsa_prefill_context_parallel"))
|
||||||
self.assertTrue(resolution_result(server_args, "enable_prefill_cp"))
|
|
||||||
self.assertEqual(resolution_result(server_args, "cp_strategy"), "interleave")
|
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
resolution_result(server_args, "dsa_prefill_cp_mode"), "round-robin-split"
|
resolution_result(args, "dsa_prefill_cp_mode"), "round-robin-split"
|
||||||
)
|
|
||||||
|
|
||||||
def test_canonical_interleave_cp_mirrors_to_dsa_runtime_aliases(self):
|
|
||||||
server_args = self._new_cp_args(
|
|
||||||
enable_prefill_cp=True,
|
|
||||||
cp_strategy="interleave",
|
|
||||||
attention_backend="dsa",
|
|
||||||
)
|
|
||||||
|
|
||||||
handle_legacy_cp_arguments(server_args)
|
|
||||||
handle_context_parallelism(server_args)
|
|
||||||
|
|
||||||
self.assertTrue(
|
|
||||||
resolution_result(server_args, "enable_dsa_prefill_context_parallel")
|
|
||||||
)
|
|
||||||
self.assertFalse(
|
|
||||||
resolution_result(server_args, "enable_prefill_context_parallel")
|
|
||||||
)
|
|
||||||
self.assertEqual(
|
|
||||||
resolution_result(server_args, "dsa_prefill_cp_mode"), "round-robin-split"
|
|
||||||
)
|
|
||||||
self.assertEqual(
|
|
||||||
resolution_result(server_args, "prefill_cp_mode"), "round-robin-split"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_context_parallel_handler_initializes_cp_strategy(self):
|
def test_context_parallel_handler_initializes_cp_strategy(self):
|
||||||
@@ -1117,97 +1182,6 @@ class TestContextParallelServerArgs(CustomTestCase):
|
|||||||
self.assertTrue(is_cp_enabled())
|
self.assertTrue(is_cp_enabled())
|
||||||
self.assertTrue(is_interleave())
|
self.assertTrue(is_interleave())
|
||||||
|
|
||||||
def test_registered_cp_legacy_args_map_to_unified_strategy(self):
|
|
||||||
cases = [
|
|
||||||
(
|
|
||||||
"deepseek_v3_mla_cp",
|
|
||||||
dict(enable_prefill_context_parallel=True),
|
|
||||||
"zigzag",
|
|
||||||
"in-seq-split",
|
|
||||||
False,
|
|
||||||
True,
|
|
||||||
),
|
|
||||||
(
|
|
||||||
"qwen3_gqa_cp",
|
|
||||||
dict(
|
|
||||||
enable_prefill_context_parallel=True,
|
|
||||||
tp_size=4,
|
|
||||||
attn_cp_size=2,
|
|
||||||
),
|
|
||||||
"zigzag",
|
|
||||||
"in-seq-split",
|
|
||||||
False,
|
|
||||||
True,
|
|
||||||
),
|
|
||||||
(
|
|
||||||
"deepseek_v32_dsa_in_seq_split",
|
|
||||||
dict(
|
|
||||||
enable_dsa_prefill_context_parallel=True,
|
|
||||||
dsa_prefill_cp_mode="in-seq-split",
|
|
||||||
tp_size=8,
|
|
||||||
dp_size=2,
|
|
||||||
attn_cp_size=4,
|
|
||||||
),
|
|
||||||
"zigzag",
|
|
||||||
"in-seq-split",
|
|
||||||
True,
|
|
||||||
False,
|
|
||||||
),
|
|
||||||
(
|
|
||||||
"deepseek_v32_dsa_round_robin_split",
|
|
||||||
dict(
|
|
||||||
enable_dsa_prefill_context_parallel=True,
|
|
||||||
tp_size=8,
|
|
||||||
attn_cp_size=8,
|
|
||||||
),
|
|
||||||
"interleave",
|
|
||||||
"round-robin-split",
|
|
||||||
True,
|
|
||||||
False,
|
|
||||||
),
|
|
||||||
(
|
|
||||||
"deepseek_v4_flash_fp4_b200_dsa_round_robin_split",
|
|
||||||
dict(
|
|
||||||
enable_dsa_prefill_context_parallel=True,
|
|
||||||
dsa_prefill_cp_mode="round-robin-split",
|
|
||||||
tp_size=4,
|
|
||||||
attn_cp_size=4,
|
|
||||||
),
|
|
||||||
"interleave",
|
|
||||||
"round-robin-split",
|
|
||||||
True,
|
|
||||||
False,
|
|
||||||
),
|
|
||||||
]
|
|
||||||
|
|
||||||
for name, overrides, strategy, mode, expect_dsa, expect_generic in cases:
|
|
||||||
with self.subTest(name=name):
|
|
||||||
server_args = self._new_cp_args(**overrides)
|
|
||||||
|
|
||||||
handle_legacy_cp_arguments(server_args)
|
|
||||||
handle_context_parallelism(server_args)
|
|
||||||
|
|
||||||
self.assertTrue(resolution_result(server_args, "enable_prefill_cp"))
|
|
||||||
self.assertEqual(
|
|
||||||
resolution_result(server_args, "cp_strategy"), strategy
|
|
||||||
)
|
|
||||||
self.assertEqual(
|
|
||||||
resolution_result(server_args, "dsa_prefill_cp_mode"), mode
|
|
||||||
)
|
|
||||||
self.assertEqual(
|
|
||||||
resolution_result(server_args, "prefill_cp_mode"), mode
|
|
||||||
)
|
|
||||||
self.assertEqual(
|
|
||||||
resolution_result(
|
|
||||||
server_args, "enable_dsa_prefill_context_parallel"
|
|
||||||
),
|
|
||||||
expect_dsa,
|
|
||||||
)
|
|
||||||
self.assertEqual(
|
|
||||||
resolution_result(server_args, "enable_prefill_context_parallel"),
|
|
||||||
expect_generic,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class TestPortArgs(unittest.TestCase):
|
class TestPortArgs(unittest.TestCase):
|
||||||
@patch("sglang.srt.server_args.tempfile.NamedTemporaryFile")
|
@patch("sglang.srt.server_args.tempfile.NamedTemporaryFile")
|
||||||
@@ -1804,7 +1778,6 @@ class TestPrefillOnlyDisableKvCache(unittest.TestCase):
|
|||||||
|
|
||||||
def _validate_prefill_only_args(self, **overrides):
|
def _validate_prefill_only_args(self, **overrides):
|
||||||
sa = ServerArgs(**self._base_kwargs(**overrides))
|
sa = ServerArgs(**self._base_kwargs(**overrides))
|
||||||
handle_legacy_cp_arguments(sa)
|
|
||||||
validate_prefill_only_disable_kv_cache_args(sa)
|
validate_prefill_only_disable_kv_cache_args(sa)
|
||||||
return sa
|
return sa
|
||||||
|
|
||||||
@@ -1830,7 +1803,10 @@ class TestPrefillOnlyDisableKvCache(unittest.TestCase):
|
|||||||
|
|
||||||
def test_rejects_prefill_context_parallel(self):
|
def test_rejects_prefill_context_parallel(self):
|
||||||
with self.assertRaisesRegex(ValueError, "--enable-prefill-cp"):
|
with self.assertRaisesRegex(ValueError, "--enable-prefill-cp"):
|
||||||
self._validate_prefill_only_args(enable_prefill_context_parallel=True)
|
self._validate_prefill_only_args(
|
||||||
|
enable_prefill_cp=True,
|
||||||
|
cp_strategy="zigzag",
|
||||||
|
)
|
||||||
|
|
||||||
def test_rejects_hisparse(self):
|
def test_rejects_hisparse(self):
|
||||||
with self.assertRaisesRegex(ValueError, "--enable-hisparse"):
|
with self.assertRaisesRegex(ValueError, "--enable-hisparse"):
|
||||||
|
|||||||
Reference in New Issue
Block a user