config: resolution reads the declarations, not the fields (#36253)

This commit is contained in:
Cheng Wan
2026-08-26 05:05:28 -07:00
committed by GitHub
parent ae5feb4b9c
commit 5b7fc61306
34 changed files with 1865 additions and 1495 deletions
@@ -3,7 +3,10 @@ from __future__ import annotations
import logging import logging
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from sglang.srt.arg_groups.overrides import declare_resolution from sglang.srt.arg_groups.overrides import (
declare_resolution,
resolving_view,
)
from sglang.srt.environ import envs from sglang.srt.environ import envs
if TYPE_CHECKING: if TYPE_CHECKING:
@@ -16,18 +19,16 @@ def validate_deepseek_v4_mega_moe_token_budget(
server_args: ServerArgs, server_args: ServerArgs,
) -> None: ) -> None:
"""Ensure the DSV4 prefill budget fits MegaMoE's per-rank buffer.""" """Ensure the DSV4 prefill budget fits MegaMoE's per-rank buffer."""
mega_moe_enabled = server_args.moe_a2a_backend == "megamoe" cfg = resolving_view(server_args)
if not mega_moe_enabled or server_args.disaggregation_mode == "decode": mega_moe_enabled = cfg.moe_a2a_backend == "megamoe"
if not mega_moe_enabled or cfg.disaggregation_mode == "decode":
# decode node will skip the check because decode bs is not relevant with --chunk-prefill-size # decode node will skip the check because decode bs is not relevant with --chunk-prefill-size
return return
if server_args.pp_size > 1 and server_args.enable_dynamic_chunking: if cfg.pp_size > 1 and cfg.enable_dynamic_chunking:
return return
if ( if cfg.chunked_prefill_size is None or cfg.chunked_prefill_size <= 0:
server_args.chunked_prefill_size is None
or server_args.chunked_prefill_size <= 0
):
raise ValueError( raise ValueError(
"DeepSeekV4 with MegaMoE requires chunked prefill to be enabled. " "DeepSeekV4 with MegaMoE requires chunked prefill to be enabled. "
"Set --chunked-prefill-size to a positive value; " "Set --chunked-prefill-size to a positive value; "
@@ -35,40 +36,38 @@ def validate_deepseek_v4_mega_moe_token_budget(
"token requirement would not have a strict prefill-forward bound." "token requirement would not have a strict prefill-forward bound."
) )
if server_args.enable_prefill_cp: if cfg.enable_prefill_cp:
token_partition_size = server_args.attn_cp_size token_partition_size = cfg.attn_cp_size
token_partition_name = "attn_cp_size" token_partition_name = "attn_cp_size"
token_alignment = 1 token_alignment = 1
local_chunked_prefill_size = ( local_chunked_prefill_size = (
server_args.chunked_prefill_size + token_partition_size - 1 cfg.chunked_prefill_size + token_partition_size - 1
) // token_partition_size ) // token_partition_size
elif server_args.enable_dp_attention: elif cfg.enable_dp_attention:
token_partition_size = server_args.dp_size token_partition_size = cfg.dp_size
token_partition_name = "dp_size" token_partition_name = "dp_size"
token_alignment = max( token_alignment = max(
server_args.tp_size // server_args.dp_size // server_args.attn_cp_size, cfg.tp_size // cfg.dp_size // cfg.attn_cp_size,
1, 1,
) )
local_chunked_prefill_size = ( local_chunked_prefill_size = cfg.chunked_prefill_size // token_partition_size
server_args.chunked_prefill_size // token_partition_size
)
else: else:
# Pure TP and PP with static chunking are handled here. # Pure TP and PP with static chunking are handled here.
token_partition_size = 1 token_partition_size = 1
token_partition_name = "none" token_partition_name = "none"
# global_num_tokens will ceil_align to attn_tp_size so the validation needs to do alignment as well # global_num_tokens will ceil_align to attn_tp_size so the validation needs to do alignment as well
token_alignment = max( token_alignment = max(
server_args.tp_size // token_partition_size // server_args.attn_cp_size, cfg.tp_size // token_partition_size // cfg.attn_cp_size,
1, 1,
) )
local_chunked_prefill_size = server_args.chunked_prefill_size local_chunked_prefill_size = cfg.chunked_prefill_size
if local_chunked_prefill_size <= 0: if local_chunked_prefill_size <= 0:
raise ValueError( raise ValueError(
"DeepSeekV4 with MegaMoE requires a positive effective per-rank " "DeepSeekV4 with MegaMoE requires a positive effective per-rank "
"chunked prefill size. " "chunked prefill size. "
f"Current values: chunked_prefill_size=" f"Current values: chunked_prefill_size="
f"{server_args.chunked_prefill_size}, " f"{cfg.chunked_prefill_size}, "
f"token_partition={token_partition_name}, " f"token_partition={token_partition_name}, "
f"token_partition_size={token_partition_size}." f"token_partition_size={token_partition_size}."
) )
@@ -87,7 +86,7 @@ def validate_deepseek_v4_mega_moe_token_budget(
"SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK to " "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK to "
"cover each rank's effective prefill token budget. " "cover each rank's effective prefill token budget. "
f"Current values: chunked_prefill_size=" f"Current values: chunked_prefill_size="
f"{server_args.chunked_prefill_size}, " f"{cfg.chunked_prefill_size}, "
f"token_partition={token_partition_name}, " f"token_partition={token_partition_name}, "
f"token_partition_size={token_partition_size}, " f"token_partition_size={token_partition_size}, "
f"token_alignment={token_alignment}, " f"token_alignment={token_alignment}, "
@@ -112,6 +111,7 @@ def apply_deepseek_v4_defaults(server_args: ServerArgs, model_arch: str) -> None
max_running_requests fill (the speculative hook is a later writer of max_running_requests fill (the speculative hook is a later writer of
that field) and the validations. that field) and the validations.
""" """
cfg = resolving_view(server_args)
from sglang.srt.utils import is_hip from sglang.srt.utils import is_hip
# FlashMLA sparse prefill (SGLANG_OPT_FLASHMLA_SPARSE_PREFILL, default on) # FlashMLA sparse prefill (SGLANG_OPT_FLASHMLA_SPARSE_PREFILL, default on)
@@ -136,36 +136,36 @@ def apply_deepseek_v4_defaults(server_args: ServerArgs, model_arch: str) -> None
run_post_process_pass(server_args, _deepseek_v4_kv_cache_dtype) run_post_process_pass(server_args, _deepseek_v4_kv_cache_dtype)
if server_args.max_running_requests is None: if cfg.max_running_requests is None:
declare_resolution( declare_resolution(
server_args, server_args,
"apply_deepseek_v4_defaults", "apply_deepseek_v4_defaults",
max_running_requests=256, max_running_requests=256,
) )
logger.warning( logger.warning(
f"Setting max_running_requests to {server_args.max_running_requests} for {model_arch}." f"Setting max_running_requests to {cfg.max_running_requests} for {model_arch}."
) )
if server_args.speculative_algorithm is not None: if cfg.speculative_algorithm is not None:
assert server_args.speculative_algorithm in ( assert cfg.speculative_algorithm in (
"EAGLE", "EAGLE",
"DSPARK", "DSPARK",
), f"Only EAGLE and DSPARK speculative algorithms are supported for {model_arch}" ), f"Only EAGLE and DSPARK speculative algorithms are supported for {model_arch}"
if server_args.speculative_algorithm == "EAGLE": if cfg.speculative_algorithm == "EAGLE":
assert ( assert (
server_args.speculative_eagle_topk == 1 cfg.speculative_eagle_topk == 1
), f"Only EAGLE speculative algorithm with topk == 1 is supported for {model_arch}" ), f"Only EAGLE speculative algorithm with topk == 1 is supported for {model_arch}"
def validate_deepseek_v4_cp(server_args: ServerArgs) -> None: def validate_deepseek_v4_cp(server_args: ServerArgs) -> None:
"""Validate DeepSeek V4 context-parallel configuration.""" """Validate DeepSeek V4 context-parallel configuration."""
if not server_args.enable_prefill_cp: cfg = resolving_view(server_args)
if not cfg.enable_prefill_cp:
return return
if server_args.cp_strategy != "interleave": if cfg.cp_strategy != "interleave":
raise ValueError( raise ValueError(
"DeepSeekV4 only supports interleave CP strategy, " "DeepSeekV4 only supports interleave CP strategy, " f"got {cfg.cp_strategy}"
f"got {server_args.cp_strategy}"
) )
declare_resolution( declare_resolution(
@@ -196,19 +196,19 @@ def validate_deepseek_v4_cp(server_args: ServerArgs) -> None:
declare_resolution( declare_resolution(
server_args, server_args,
"validate_deepseek_v4_cp", "validate_deepseek_v4_cp",
attn_cp_size=server_args.tp_size // server_args.dp_size, attn_cp_size=cfg.tp_size // cfg.dp_size,
) )
assert ( assert (
server_args.dp_size == 1 cfg.dp_size == 1
), "For round-robin split mode, dp attention is not supported." ), "For round-robin split mode, dp attention is not supported."
assert ( assert (
server_args.tp_size <= 8 cfg.tp_size <= 8
), "Context parallel only supports single machine (tp_size <= 8). Cross-machine CP has precision issues." ), "Context parallel only supports single machine (tp_size <= 8). Cross-machine CP has precision issues."
if server_args.moe_a2a_backend not in ("none", "deepep", "megamoe"): if cfg.moe_a2a_backend not in ("none", "deepep", "megamoe"):
raise ValueError( raise ValueError(
"DeepSeekV4 CP supports moe_a2a_backend in " "DeepSeekV4 CP supports moe_a2a_backend in "
"('none', 'deepep', 'megamoe'), " "('none', 'deepep', 'megamoe'), "
f"got {server_args.moe_a2a_backend!r}." f"got {cfg.moe_a2a_backend!r}."
) )
logger.warning( logger.warning(
"Disabling SGLANG_OPT_FLASHMLA_SPARSE_PREFILL because DeepSeekV4 " "Disabling SGLANG_OPT_FLASHMLA_SPARSE_PREFILL because DeepSeekV4 "
@@ -217,6 +217,6 @@ def validate_deepseek_v4_cp(server_args: ServerArgs) -> None:
envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.set(False) envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.set(False)
logger.warning( logger.warning(
f"Enable Context Parallel for DeepSeekV4, " f"Enable Context Parallel for DeepSeekV4, "
f"dp_size={server_args.dp_size}, moe_dense_tp_size={server_args.moe_dense_tp_size}, " f"dp_size={cfg.dp_size}, moe_dense_tp_size={cfg.moe_dense_tp_size}, "
f"attn_cp_size={server_args.attn_cp_size}, ep_size={server_args.ep_size}, tp_size={server_args.tp_size}" f"attn_cp_size={cfg.attn_cp_size}, ep_size={cfg.ep_size}, tp_size={cfg.tp_size}"
) )
@@ -9,7 +9,7 @@ import os
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
from sglang.srt.arg_groups.overrides import declare_resolution from sglang.srt.arg_groups.overrides import declare_resolution, resolving_view
from sglang.srt.environ import envs from sglang.srt.environ import envs
from sglang.srt.model_executor.cuda_graph_config import ( from sglang.srt.model_executor.cuda_graph_config import (
Backend, Backend,
@@ -27,31 +27,32 @@ logger = logging.getLogger(__name__)
def handle_expert_pack(server_args: Any) -> None: def handle_expert_pack(server_args: Any) -> None:
"""Normalize expert-pack settings and report all startup errors together.""" """Normalize expert-pack settings and report all startup errors together."""
if server_args.load_format != "expert_pack": cfg = resolving_view(server_args)
if cfg.load_format != "expert_pack":
return return
errors = [] errors = []
parallelism = ( parallelism = (
("tensor", "--tp-size", server_args.tp_size), ("tensor", "--tp-size", cfg.tp_size),
("data", "--dp-size", server_args.dp_size), ("data", "--dp-size", cfg.dp_size),
("expert", "--ep-size", server_args.ep_size), ("expert", "--ep-size", cfg.ep_size),
) )
for label, option, size in parallelism: for label, option, size in parallelism:
if size != 1: if size != 1:
errors.append(f"{label} parallelism ({option}) must be 1, got {size}") errors.append(f"{label} parallelism ({option}) must be 1, got {size}")
if server_args.enforce_shared_experts_fusion: if cfg.enforce_shared_experts_fusion:
errors.append( errors.append(
"--enforce-shared-experts-fusion is incompatible with expert_pack" "--enforce-shared-experts-fusion is incompatible with expert_pack"
) )
if server_args.enable_waterfill: if cfg.enable_waterfill:
errors.append("--enable-waterfill is incompatible with expert_pack") errors.append("--enable-waterfill is incompatible with expert_pack")
explicit_cuda_graph_backends = { explicit_cuda_graph_backends = {
Phase.DECODE: server_args.cuda_graph_backend_decode, Phase.DECODE: cfg.cuda_graph_backend_decode,
Phase.PREFILL: server_args.cuda_graph_backend_prefill, Phase.PREFILL: cfg.cuda_graph_backend_prefill,
} }
raw_cuda_graph_config = server_args.cuda_graph_config raw_cuda_graph_config = cfg.cuda_graph_config
if isinstance(raw_cuda_graph_config, CudaGraphConfig): if isinstance(raw_cuda_graph_config, CudaGraphConfig):
raw_cuda_graph_config = raw_cuda_graph_config.to_dict() raw_cuda_graph_config = raw_cuda_graph_config.to_dict()
for phase in Phase.ALL: for phase in Phase.ALL:
@@ -69,7 +70,7 @@ def handle_expert_pack(server_args: Any) -> None:
f"disabled, got {explicit_backend!r}" f"disabled, got {explicit_backend!r}"
) )
loader_config = server_args.model_loader_extra_config or {} loader_config = cfg.model_loader_extra_config or {}
if isinstance(loader_config, str): if isinstance(loader_config, str):
try: try:
loader_config = json.loads(loader_config) loader_config = json.loads(loader_config)
@@ -82,7 +83,7 @@ def handle_expert_pack(server_args: Any) -> None:
# A raw GGUF path is the public input form. Preparation is performed once # A raw GGUF path is the public input form. Preparation is performed once
# here, before model-config parsing and before the loader is constructed. # here, before model-config parsing and before the loader is constructed.
raw_model_path = Path(server_args.model_path).expanduser() raw_model_path = Path(cfg.model_path).expanduser()
raw_preparation_failed = False raw_preparation_failed = False
if not errors and raw_model_path.is_file(): if not errors and raw_model_path.is_file():
try: try:
@@ -136,12 +137,12 @@ def handle_expert_pack(server_args: Any) -> None:
errors.append(f"expert-pack file does not exist: {pack_path}") errors.append(f"expert-pack file does not exist: {pack_path}")
model_kind = None model_kind = None
model_path = parse_path("--model-path", server_args.model_path) model_path = parse_path("--model-path", cfg.model_path)
if not raw_preparation_failed: if not raw_preparation_failed:
if model_path is None or not model_path.is_dir(): if model_path is None or not model_path.is_dir():
errors.append( errors.append(
"--model-path must be a local GGUF shard or tokenizer/config " "--model-path must be a local GGUF shard or tokenizer/config "
f"directory for expert_pack, got {server_args.model_path!r}" f"directory for expert_pack, got {cfg.model_path!r}"
) )
else: else:
try: try:
@@ -3,6 +3,8 @@ from __future__ import annotations
import logging import logging
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from sglang.srt.arg_groups.overrides import resolving_view
if TYPE_CHECKING: if TYPE_CHECKING:
from sglang.srt.server_args import ServerArgs from sglang.srt.server_args import ServerArgs
@@ -79,7 +81,8 @@ def validate_hisparse_kv_cache_dtype(server_args: ServerArgs) -> None:
def validate_hisparse(server_args: ServerArgs) -> None: def validate_hisparse(server_args: ServerArgs) -> None:
"""Validate --enable-hisparse constraints (model class, radix cache, DSA backend).""" """Validate --enable-hisparse constraints (model class, radix cache, DSA backend)."""
if not server_args.enable_hisparse: cfg = resolving_view(server_args)
if not cfg.enable_hisparse:
return return
from sglang.srt.configs.model_config import ( from sglang.srt.configs.model_config import (
@@ -96,7 +99,7 @@ def validate_hisparse(server_args: ServerArgs) -> None:
) )
assert ( assert (
server_args.disable_radix_cache cfg.disable_radix_cache
), "Hierarchical sparse attention currently requires --disable-radix-cache." ), "Hierarchical sparse attention currently requires --disable-radix-cache."
# DSv4 hisparse handles its own dtype/backend pairing elsewhere; the dtype- # DSv4 hisparse handles its own dtype/backend pairing elsewhere; the dtype-
+16 -10
View File
@@ -3,7 +3,10 @@ from __future__ import annotations
import logging import logging
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from sglang.srt.arg_groups.overrides import declare_resolution from sglang.srt.arg_groups.overrides import (
declare_resolution,
resolving_view,
)
if TYPE_CHECKING: if TYPE_CHECKING:
from sglang.srt.server_args import ServerArgs from sglang.srt.server_args import ServerArgs
@@ -13,15 +16,16 @@ logger = logging.getLogger(__name__)
def apply_kimi_k3_spec_backend_defaults(server_args: ServerArgs) -> None: def apply_kimi_k3_spec_backend_defaults(server_args: ServerArgs) -> None:
"""Apply speculative backend defaults for Kimi hybrid models.""" """Apply speculative backend defaults for Kimi hybrid models."""
cfg = resolving_view(server_args)
from sglang.srt.utils import is_sm100_supported from sglang.srt.utils import is_sm100_supported
if server_args.speculative_algorithm is None: if cfg.speculative_algorithm is None:
return return
# Use the fused Kimi-K3/DSPARK CuTeDSL kernel for KDA target verification. # Use the fused Kimi-K3/DSPARK CuTeDSL kernel for KDA target verification.
# Decode is left free (its bf16-ssm SM100+ flashinfer default is fine -- the # Decode is left free (its bf16-ssm SM100+ flashinfer default is fine -- the
# target only verifies under spec); the verify backend is pinned directly. # target only verifies under spec); the verify backend is pinned directly.
if server_args.linear_attn_verify_backend is None: if cfg.linear_attn_verify_backend is None:
declare_resolution( declare_resolution(
server_args, server_args,
"apply_kimi_k3_spec_backend_defaults", "apply_kimi_k3_spec_backend_defaults",
@@ -36,8 +40,8 @@ def apply_kimi_k3_spec_backend_defaults(server_args: ServerArgs) -> None:
# dspark's draft is dense MQA; trtllm_mha avoids flashinfer's blocking # dspark's draft is dense MQA; trtllm_mha avoids flashinfer's blocking
# per-step host plan. DSPARK-only: other spec algos use MLA-family drafts. # per-step host plan. DSPARK-only: other spec algos use MLA-family drafts.
if ( if (
server_args.speculative_algorithm == "DSPARK" cfg.speculative_algorithm == "DSPARK"
and server_args.speculative_draft_attention_backend is None and cfg.speculative_draft_attention_backend is None
and is_sm100_supported() and is_sm100_supported()
): ):
declare_resolution( declare_resolution(
@@ -63,20 +67,21 @@ def disable_kimi_k3_symm_mem(server_args: ServerArgs) -> None:
Gates on the arch itself: this runs from cuda-graph resolution, which is earlier Gates on the arch itself: this runs from cuda-graph resolution, which is earlier
than the model-specific hook block. than the model-specific hook block.
""" """
cfg = resolving_view(server_args)
from sglang.srt.connector import ConnectorType from sglang.srt.connector import ConnectorType
from sglang.srt.model_executor.cuda_graph_config import Backend from sglang.srt.model_executor.cuda_graph_config import Backend
from sglang.srt.utils import parse_connector_type from sglang.srt.utils import parse_connector_type
if not server_args.enable_symm_mem: if not cfg.enable_symm_mem:
return return
if parse_connector_type(server_args.model_path) == ConnectorType.INSTANCE: if parse_connector_type(cfg.model_path) == ConnectorType.INSTANCE:
return return
if server_args.get_model_config().hf_config.architectures[0] not in ( if server_args.get_model_config().hf_config.architectures[0] not in (
"KimiLinearForCausalLM", "KimiLinearForCausalLM",
"KimiK3ForConditionalGeneration", "KimiK3ForConditionalGeneration",
): ):
return return
graph = server_args.cuda_graph_config graph = cfg.cuda_graph_config
if ( if (
graph.decode.backend == Backend.DISABLED graph.decode.backend == Backend.DISABLED
and graph.prefill.backend == Backend.DISABLED and graph.prefill.backend == Backend.DISABLED
@@ -100,14 +105,15 @@ def disable_kimi_k3_symm_mem(server_args: ServerArgs) -> None:
def apply_kimi_k3_linear_attn_defaults(server_args: ServerArgs) -> None: def apply_kimi_k3_linear_attn_defaults(server_args: ServerArgs) -> None:
"""KDA decode-fallback default for Kimi hybrid models (spec-independent).""" """KDA decode-fallback default for Kimi hybrid models (spec-independent)."""
cfg = resolving_view(server_args)
from sglang.srt.utils import is_sm100_supported from sglang.srt.utils import is_sm100_supported
# Preempts the generic SM100+bf16 flashinfer switch (a GDN default): on # Preempts the generic SM100+bf16 flashinfer switch (a GDN default): on
# KDA shapes the triton packed decode measures ~35% faster than # KDA shapes the triton packed decode measures ~35% faster than
# recurrent_kda across bs 1-256, and ReplaySSM requires triton. # recurrent_kda across bs 1-256, and ReplaySSM requires triton.
if ( if (
server_args.linear_attn_decode_backend is None cfg.linear_attn_decode_backend is None
and server_args.mamba_ssm_dtype == "bfloat16" and cfg.mamba_ssm_dtype == "bfloat16"
and is_sm100_supported() and is_sm100_supported()
): ):
declare_resolution( declare_resolution(
+10 -5
View File
@@ -7,7 +7,10 @@ from typing import TYPE_CHECKING
if TYPE_CHECKING: if TYPE_CHECKING:
from sglang.srt.server_args import ServerArgs from sglang.srt.server_args import ServerArgs
from sglang.srt.arg_groups.overrides import declare_resolution from sglang.srt.arg_groups.overrides import (
declare_resolution,
resolving_view,
)
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -18,15 +21,16 @@ def handle_mega_moe(server_args: ServerArgs) -> None:
def handle_moe_runner_backend_alias(server_args: ServerArgs) -> None: def handle_moe_runner_backend_alias(server_args: ServerArgs) -> None:
if server_args.moe_runner_backend != "megamoe": cfg = resolving_view(server_args)
if cfg.moe_runner_backend != "megamoe":
return return
if server_args.moe_a2a_backend not in ("none", "megamoe"): if cfg.moe_a2a_backend not in ("none", "megamoe"):
logger.warning( logger.warning(
"--moe-runner-backend megamoe is an alias for " "--moe-runner-backend megamoe is an alias for "
"--moe-a2a-backend megamoe; overriding " "--moe-a2a-backend megamoe; overriding "
"--moe-a2a-backend %s.", "--moe-a2a-backend %s.",
server_args.moe_a2a_backend, cfg.moe_a2a_backend,
) )
declare_resolution( declare_resolution(
server_args, server_args,
@@ -37,7 +41,8 @@ def handle_moe_runner_backend_alias(server_args: ServerArgs) -> None:
def handle_w4a4_mxfp4_megamoe_env(server_args: ServerArgs) -> None: def handle_w4a4_mxfp4_megamoe_env(server_args: ServerArgs) -> None:
if not server_args.enable_w4a4_mxfp4_megamoe: cfg = resolving_view(server_args)
if not cfg.enable_w4a4_mxfp4_megamoe:
return return
os.environ["DG_USE_FP4_ACTS"] = "1" os.environ["DG_USE_FP4_ACTS"] = "1"
+179 -135
View File
@@ -150,6 +150,42 @@ class ResolvedView:
) )
class ResolvingConfig:
"""Live read view of the resolution result: the declaration stash over the
record's fields, looked up per read.
``ResolvedView`` snapshots the overlay when it is built, which is what a
post-process pass wants -- it reads the state at its slot. A resolver that
reads *after* declaring, or after calling something that declares, needs the
current answer instead, so this one walks the stash on every read. It falls
through to the field, which is where the raw input lives.
"""
__slots__ = ("_server_args",)
def __init__(self, server_args: Any):
object.__setattr__(self, "_server_args", server_args)
def __getattr__(self, name: str) -> Any:
server_args = object.__getattribute__(self, "_server_args")
for _source, declared in reversed(
getattr(server_args, "_resolved_overrides", None) or ()
):
if name in declared:
return declared[name]
return getattr(server_args, name)
def __setattr__(self, name: str, value: Any) -> None:
raise AttributeError(
"ResolvingConfig is read-only; resolution writes through declarations"
)
def resolving_view(server_args: Any) -> ResolvingConfig:
"""A live read view of what resolution has decided so far."""
return ResolvingConfig(server_args)
# Ordered post-process passes (the normalization stage). List order is the # Ordered post-process passes (the normalization stage). List order is the
# end-state execution order and mirrors today's handler call sequence in # end-state execution order and mirrors today's handler call sequence in
# __post_init__; during the transition each pass is invoked from its legacy # __post_init__; during the transition each pass is invoked from its legacy
@@ -581,16 +617,17 @@ def _require_kimi_k3_cutedsl_dcp_support() -> None:
@_register_for("KimiK3ForConditionalGeneration") @_register_for("KimiK3ForConditionalGeneration")
def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict: def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict:
if server_args.dcp_size > 1: cfg = resolving_view(server_args)
if cfg.dcp_size > 1:
overrides = {} overrides = {}
if server_args.enable_symm_mem: if cfg.enable_symm_mem:
logger.warning( logger.warning(
"Kimi-K3 DCP disables --enable-symm-mem due to decode CUDA " "Kimi-K3 DCP disables --enable-symm-mem due to decode CUDA "
"graph correctness issues." "graph correctness issues."
) )
overrides["enable_symm_mem"] = False overrides["enable_symm_mem"] = False
if server_args.speculative_algorithm == "DSPARK": if cfg.speculative_algorithm == "DSPARK":
from sglang.srt.speculative.ragged_verify import ( from sglang.srt.speculative.ragged_verify import (
RaggedVerifyMode, RaggedVerifyMode,
read_ragged_verify_mode, read_ragged_verify_mode,
@@ -611,7 +648,7 @@ def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict:
# lacks that DCP path (TypeError: unexpected kwarg 'causal_seqs'). # lacks that DCP path (TypeError: unexpected kwarg 'causal_seqs').
overrides["speculative_attention_mode"] = "decode" overrides["speculative_attention_mode"] = "decode"
prefill_backend, decode_backend = attention_backends_of(server_args) prefill_backend, decode_backend = attention_backends_of(cfg)
if decode_backend == "cutedsl_mla" or decode_backend is None: if decode_backend == "cutedsl_mla" or decode_backend is None:
_require_kimi_k3_cutedsl_dcp_support() _require_kimi_k3_cutedsl_dcp_support()
logger.info( logger.info(
@@ -630,7 +667,7 @@ def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict:
) )
logger.info( logger.info(
"Kimi-K3 DCP with tokenspeed mla backend overrides KV cache dtype: " "Kimi-K3 DCP with tokenspeed mla backend overrides KV cache dtype: "
f"{server_args.kv_cache_dtype!r} -> 'fp8_e4m3'." f"{cfg.kv_cache_dtype!r} -> 'fp8_e4m3'."
) )
overrides.update( overrides.update(
prefill_attention_backend="tokenspeed_mla", prefill_attention_backend="tokenspeed_mla",
@@ -642,7 +679,7 @@ def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict:
f"Decode attention backend for Kimi-K3 DCP must be 'cutedsl_mla' or 'tokenspeed_mla', got {decode_backend!r}." f"Decode attention backend for Kimi-K3 DCP must be 'cutedsl_mla' or 'tokenspeed_mla', got {decode_backend!r}."
) )
if server_args.dcp_replicate_q_proj is None: if cfg.dcp_replicate_q_proj is None:
logger.info("Kimi-K3 DCP enables replicated Q projection by default.") logger.info("Kimi-K3 DCP enables replicated Q projection by default.")
overrides["dcp_replicate_q_proj"] = True overrides["dcp_replicate_q_proj"] = True
@@ -650,7 +687,7 @@ def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict:
dcp_comm_backend = "fi_a2a" if is_mnnvl_fabric_device() else "a2a" dcp_comm_backend = "fi_a2a" if is_mnnvl_fabric_device() else "a2a"
logger.info( logger.info(
"Kimi-K3 DCP selects communication backend on " "Kimi-K3 DCP selects communication backend on "
f"{device_name!r}: {server_args.dcp_comm_backend!r} -> " f"{device_name!r}: {cfg.dcp_comm_backend!r} -> "
f"{dcp_comm_backend!r}." f"{dcp_comm_backend!r}."
) )
overrides["dcp_comm_backend"] = dcp_comm_backend overrides["dcp_comm_backend"] = dcp_comm_backend
@@ -659,7 +696,7 @@ def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict:
if not (is_sm100_supported() and get_device_sm() in (100, 103)): if not (is_sm100_supported() and get_device_sm() in (100, 103)):
return {} return {}
backends_unset = server_args.is_attention_backend_not_set() backends_unset = server_args.is_attention_backend_not_set()
if server_args.speculative_algorithm != "DSPARK": if cfg.speculative_algorithm != "DSPARK":
if not backends_unset: if not backends_unset:
return {} return {}
logger.info( logger.info(
@@ -673,9 +710,9 @@ def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict:
# DSPARK: verify runs on the decode backend (mode=decode below), so this # DSPARK: verify runs on the decode backend (mode=decode below), so this
# picks the verify kernel -- mode=prefill routes it to flashinfer, which is # picks the verify kernel -- mode=prefill routes it to flashinfer, which is
# slow and syncs, while plain decode is cold under dspark. # slow and syncs, while plain decode is cold under dspark.
q_len = server_args.speculative_num_draft_tokens or ( q_len = cfg.speculative_num_draft_tokens or (
server_args.speculative_dspark_block_size + 1 cfg.speculative_dspark_block_size + 1
if server_args.speculative_dspark_block_size is not None if cfg.speculative_dspark_block_size is not None
# Checkpoint auto-infer happens after overrides; K3 draft uses block 7. # Checkpoint auto-infer happens after overrides; K3 draft uses block 7.
else 8 else 8
) )
@@ -688,8 +725,8 @@ def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict:
# Explicit backend knobs keep priority, but the mode is a separate knob # Explicit backend knobs keep priority, but the mode is a separate knob
# that still needs declaring -- else verify stays on the prefill backend, # that still needs declaring -- else verify stays on the prefill backend,
# whose host-side plan (flashinfer by default) forces a per-step D2H. # whose host-side plan (flashinfer by default) forces a per-step D2H.
_, backend = attention_backends_of(server_args) _, backend = attention_backends_of(cfg)
if _dspark_verify_on_decode_backend(backend, q_len, server_args.kv_cache_dtype): if _dspark_verify_on_decode_backend(backend, q_len, cfg.kv_cache_dtype):
overrides["speculative_attention_mode"] = "decode" overrides["speculative_attention_mode"] = "decode"
logger.info( logger.info(
"Kimi-K3 DSPARK on SM100/SM103: decode/verify attention backend " "Kimi-K3 DSPARK on SM100/SM103: decode/verify attention backend "
@@ -727,7 +764,8 @@ def _kimi_k3_moe_runner_overrides(server_args: Any, hf_config: Any) -> dict:
# (M=bs) and the target-verify (M=bs*(gamma+1)) regimes on SM100/SM103. # (M=bs) and the target-verify (M=bs*(gamma+1)) regimes on SM100/SM103.
# SM107 uses the same packed-MXFP4 runner; leaving auto unresolved falls # SM107 uses the same packed-MXFP4 runner; leaving auto unresolved falls
# back to BF16 weight materialization during model loading. # back to BF16 weight materialization during model loading.
if server_args.moe_runner_backend != "auto": cfg = resolving_view(server_args)
if cfg.moe_runner_backend != "auto":
return {} return {}
if not (is_sm100_supported() and get_device_sm() in (100, 103, 107)): if not (is_sm100_supported() and get_device_sm() in (100, 103, 107)):
return {} return {}
@@ -757,6 +795,7 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
writers), the kv-cache/split-backend defaults, the quant/moe block (read writers), the kv-cache/split-backend defaults, the quant/moe block (read
before it by _set_default_dsa_kv_cache_dtype) and the env writes stay in before it by _set_default_dsa_kv_cache_dtype) and the env writes stay in
the branch.""" the branch."""
cfg = resolving_view(server_args)
from sglang.srt.configs.model_config import is_deepseek_dsa from sglang.srt.configs.model_config import is_deepseek_dsa
overrides: Dict[str, Any] = {} overrides: Dict[str, Any] = {}
@@ -767,39 +806,39 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
overrides["attention_backend"] = "dsa" overrides["attention_backend"] = "dsa"
logger.info("Use dsa attention backend for DeepSeek with DSA.") logger.info("Use dsa attention backend for DeepSeek with DSA.")
if not is_npu() and not is_xpu(): # CUDA or ROCm GPU if not is_npu() and not is_xpu(): # CUDA or ROCm GPU
if server_args.enable_prefill_cp: if cfg.enable_prefill_cp:
logger.warning( logger.warning(
"Context parallel feature is still under experiment. It has only been verified on Hopper platform." "Context parallel feature is still under experiment. It has only been verified on Hopper platform."
) )
overrides["enable_dp_attention"] = True overrides["enable_dp_attention"] = True
overrides["moe_dense_tp_size"] = 1 overrides["moe_dense_tp_size"] = 1
if server_args.cp_strategy == "zigzag": if cfg.cp_strategy == "zigzag":
overrides["moe_a2a_backend"] = "deepep" overrides["moe_a2a_backend"] = "deepep"
overrides["ep_size"] = server_args.tp_size overrides["ep_size"] = cfg.tp_size
logger.warning( logger.warning(
"zigzag DSA CP requires moe_dense_tp_size=1, " "zigzag DSA CP requires moe_dense_tp_size=1, "
"moe_a2a_backend=deepep, ep_size=tp_size, batch_size=1." "moe_a2a_backend=deepep, ep_size=tp_size, batch_size=1."
) )
else: else:
assert ( assert (
server_args.dp_size == 1 cfg.dp_size == 1
), "interleave DSA CP does not support DP attention." ), "interleave DSA CP does not support DP attention."
assert ( assert (
server_args.tp_size <= 8 cfg.tp_size <= 8
), "Context parallel only supports single machine (tp_size <= 8). Cross-machine CP has precision issues." ), "Context parallel only supports single machine (tp_size <= 8). Cross-machine CP has precision issues."
# Note(kpham-sgl): Keep attn_tp_size == 1 under DSA CP. # Note(kpham-sgl): Keep attn_tp_size == 1 under DSA CP.
# DSACPLayerCommunicator does not all-reduce attention-TP # DSACPLayerCommunicator does not all-reduce attention-TP
# partial o_proj outputs before replicated dense FFNs. # partial o_proj outputs before replicated dense FFNs.
attn_cp_size = server_args.tp_size // server_args.dp_size attn_cp_size = cfg.tp_size // cfg.dp_size
overrides["attn_cp_size"] = attn_cp_size overrides["attn_cp_size"] = attn_cp_size
logger.warning( logger.warning(
"Enabled DSA context parallel: " "Enabled DSA context parallel: "
f"strategy={server_args.cp_strategy}, dp_size={server_args.dp_size}, " f"strategy={cfg.cp_strategy}, dp_size={cfg.dp_size}, "
f"moe_dense_tp_size={overrides['moe_dense_tp_size']}, " f"moe_dense_tp_size={overrides['moe_dense_tp_size']}, "
f"ep_size={overrides.get('ep_size', server_args.ep_size)}, tp_size={server_args.tp_size}, " f"ep_size={overrides.get('ep_size', cfg.ep_size)}, tp_size={cfg.tp_size}, "
f"attn_cp_size={attn_cp_size}, " f"attn_cp_size={attn_cp_size}, "
f"kv_cache_dtype={server_args.kv_cache_dtype}, " f"kv_cache_dtype={cfg.kv_cache_dtype}, "
f"moe_a2a_backend={overrides.get('moe_a2a_backend', server_args.moe_a2a_backend)}, " f"moe_a2a_backend={overrides.get('moe_a2a_backend', cfg.moe_a2a_backend)}, "
f"cuda_graph_config[prefill].backend=disabled" f"cuda_graph_config[prefill].backend=disabled"
) )
@@ -826,9 +865,9 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
# DeepSeek V3/R1/V3.1 # DeepSeek V3/R1/V3.1
if is_sm100_supported(): if is_sm100_supported():
if ( if (
server_args.attention_backend is None cfg.attention_backend is None
and server_args.prefill_attention_backend is None and cfg.prefill_attention_backend is None
and server_args.decode_attention_backend is None and cfg.decode_attention_backend is None
): ):
overrides["attention_backend"] = "trtllm_mla" overrides["attention_backend"] = "trtllm_mla"
logger.info( logger.info(
@@ -836,7 +875,7 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
) )
# MLA prefill CP auto-config. Mirrors the NSA CP block above # MLA prefill CP auto-config. Mirrors the NSA CP block above
# (minus the in-seq/round-robin mode split, which MLA CP does not support) # (minus the in-seq/round-robin mode split, which MLA CP does not support)
if server_args.enable_prefill_cp and server_args.use_mla_backend(): if cfg.enable_prefill_cp and server_args.use_mla_backend():
logger.warning( logger.warning(
"MLA prefill context parallel is still experimental. " "MLA prefill context parallel is still experimental. "
"Verified on Hopper with the fa3 backend." "Verified on Hopper with the fa3 backend."
@@ -845,22 +884,22 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
# TODO(kpham-sgl) Supports moe_dense_tp_size != 1. # TODO(kpham-sgl) Supports moe_dense_tp_size != 1.
overrides["moe_dense_tp_size"] = 1 overrides["moe_dense_tp_size"] = 1
overrides["moe_a2a_backend"] = "deepep" overrides["moe_a2a_backend"] = "deepep"
overrides["ep_size"] = server_args.tp_size overrides["ep_size"] = cfg.tp_size
logger.warning( logger.warning(
"For MLA CP, we have the following restrictions: moe_dense_tp_size == 1, moe_a2a_backend == deepep, ep_size == tp_size, batch_size == 1" "For MLA CP, we have the following restrictions: moe_dense_tp_size == 1, moe_a2a_backend == deepep, ep_size == tp_size, batch_size == 1"
) )
# FIXME(kpham-sgl): Keep attn_tp_size == 1 under MLA CP. # FIXME(kpham-sgl): Keep attn_tp_size == 1 under MLA CP.
# DSACPLayerCommunicator does not all-reduce attention-TP # DSACPLayerCommunicator does not all-reduce attention-TP
# partial o_proj outputs before replicated dense FFNs. # partial o_proj outputs before replicated dense FFNs.
attn_cp_size = server_args.tp_size // server_args.dp_size attn_cp_size = cfg.tp_size // cfg.dp_size
overrides["attn_cp_size"] = attn_cp_size overrides["attn_cp_size"] = attn_cp_size
logger.warning( logger.warning(
f"Enable Context Parallel opt for MLA, " f"Enable Context Parallel opt for MLA, "
f"Setting dp_size == {server_args.dp_size} and " f"Setting dp_size == {cfg.dp_size} and "
f"attn_cp_size == {attn_cp_size}, " f"attn_cp_size == {attn_cp_size}, "
f"moe_dense_tp_size == {overrides['moe_dense_tp_size']}, " f"moe_dense_tp_size == {overrides['moe_dense_tp_size']}, "
f"ep_size == {overrides['ep_size']}, " f"ep_size == {overrides['ep_size']}, "
f"tp_size == {server_args.tp_size}, " f"tp_size == {cfg.tp_size}, "
f"moe_a2a_backend {overrides['moe_a2a_backend']}, " f"moe_a2a_backend {overrides['moe_a2a_backend']}, "
f"cuda_graph_config[prefill].backend=disabled" f"cuda_graph_config[prefill].backend=disabled"
) )
@@ -870,8 +909,9 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
# Keep in sync with MIMO_V2_MODEL_ARCHS (server_args.py / configs/hf_config.py). # Keep in sync with MIMO_V2_MODEL_ARCHS (server_args.py / configs/hf_config.py).
@_register_for("MiMoV2ForCausalLM", "MiMoV2FlashForCausalLM") @_register_for("MiMoV2ForCausalLM", "MiMoV2FlashForCausalLM")
def _mimo_v2_overrides(server_args: Any, hf_config: Any) -> dict: def _mimo_v2_overrides(server_args: Any, hf_config: Any) -> dict:
cfg = resolving_view(server_args)
overrides: Dict[str, Any] = {} overrides: Dict[str, Any] = {}
if server_args.speculative_algorithm == "EAGLE": if cfg.speculative_algorithm == "EAGLE":
logger.info("Enable multi-layer EAGLE speculative decoding for MiMoV2 model.") logger.info("Enable multi-layer EAGLE speculative decoding for MiMoV2 model.")
overrides["enable_multi_layer_eagle"] = True overrides["enable_multi_layer_eagle"] = True
@@ -879,7 +919,7 @@ def _mimo_v2_overrides(server_args: Any, hf_config: Any) -> dict:
# slower at bs=1 decode. FP4 checkpoints use flashinfer_mxfp4 instead. # slower at bs=1 decode. FP4 checkpoints use flashinfer_mxfp4 instead.
if ( if (
is_sm100_supported() is_sm100_supported()
and server_args.moe_runner_backend == "auto" and cfg.moe_runner_backend == "auto"
and get_quantization_config(hf_config) == "fp8" and get_quantization_config(hf_config) == "fp8"
): ):
overrides["moe_runner_backend"] = "flashinfer_trtllm" overrides["moe_runner_backend"] = "flashinfer_trtllm"
@@ -889,13 +929,14 @@ def _mimo_v2_overrides(server_args: Any, hf_config: Any) -> dict:
@_register_for("MiniMaxM2ForCausalLM") @_register_for("MiniMaxM2ForCausalLM")
def _minimax_m2_overrides(server_args: Any, hf_config: Any) -> dict: def _minimax_m2_overrides(server_args: Any, hf_config: Any) -> dict:
cfg = resolving_view(server_args)
overrides = {"enable_tf32_matmul": True} overrides = {"enable_tf32_matmul": True}
logger.info( logger.info(
"Enable TF32 matmul for MiniMaxM2ForCausalLM model to improve gate gemm performance." "Enable TF32 matmul for MiniMaxM2ForCausalLM model to improve gate gemm performance."
) )
if ( if (
is_sm100_supported() is_sm100_supported()
and server_args.moe_runner_backend == "auto" and cfg.moe_runner_backend == "auto"
and server_args.get_model_config().quantization == "modelopt_fp4" and server_args.get_model_config().quantization == "modelopt_fp4"
): ):
overrides["moe_runner_backend"] = "flashinfer_trtllm_routed" overrides["moe_runner_backend"] = "flashinfer_trtllm_routed"
@@ -909,10 +950,11 @@ def _minimax_m2_overrides(server_args: Any, hf_config: Any) -> dict:
@_register_for("MiniMaxM3SparseForCausalLM", "MiniMaxM3SparseForConditionalGeneration") @_register_for("MiniMaxM3SparseForCausalLM", "MiniMaxM3SparseForConditionalGeneration")
def _minimax_m3_overrides(server_args: Any, hf_config: Any) -> dict: def _minimax_m3_overrides(server_args: Any, hf_config: Any) -> dict:
cfg = resolving_view(server_args)
overrides: Dict[str, Any] = {} overrides: Dict[str, Any] = {}
quant_method = get_quantization_config(hf_config) quant_method = get_quantization_config(hf_config)
quant_resolved = server_args.quantization quant_resolved = cfg.quantization
if ( if (
quant_resolved is None quant_resolved is None
and not server_args._quantization_explicitly_unset and not server_args._quantization_explicitly_unset
@@ -924,16 +966,12 @@ def _minimax_m3_overrides(server_args: Any, hf_config: Any) -> dict:
if is_hip(): if is_hip():
if server_args.is_attention_backend_not_set(): if server_args.is_attention_backend_not_set():
overrides["attention_backend"] = "triton" overrides["attention_backend"] = "triton"
if server_args.moe_runner_backend == "auto" and quant_resolved == "mxfp8": if cfg.moe_runner_backend == "auto" and quant_resolved == "mxfp8":
overrides["moe_runner_backend"] = "triton" overrides["moe_runner_backend"] = "triton"
if not envs.USE_ROCM_AITER_ROPE_BACKEND.is_set(): if not envs.USE_ROCM_AITER_ROPE_BACKEND.is_set():
envs.USE_ROCM_AITER_ROPE_BACKEND.set("0") envs.USE_ROCM_AITER_ROPE_BACKEND.set("0")
aiter_fusion_resolved = server_args.enable_aiter_allreduce_fusion aiter_fusion_resolved = cfg.enable_aiter_allreduce_fusion
if ( if cfg.ep_size > 1 and cfg.moe_a2a_backend == "none" and aiter_fusion_resolved:
server_args.ep_size > 1
and server_args.moe_a2a_backend == "none"
and aiter_fusion_resolved
):
logger.warning( logger.warning(
"Disable --enable-aiter-allreduce-fusion for MiniMax-M3 " "Disable --enable-aiter-allreduce-fusion for MiniMax-M3 "
"standard EP on ROCm because the deferred fused all-reduce " "standard EP on ROCm because the deferred fused all-reduce "
@@ -951,7 +989,7 @@ def _minimax_m3_overrides(server_args: Any, hf_config: Any) -> dict:
elif is_sm100_supported(): elif is_sm100_supported():
if server_args.is_attention_backend_not_set(): if server_args.is_attention_backend_not_set():
if ( if (
server_args.kv_cache_dtype == "fp8_e4m3" cfg.kv_cache_dtype == "fp8_e4m3"
and not envs.SGLANG_DISABLE_M3_FP8_ATTN_GEMM.get() and not envs.SGLANG_DISABLE_M3_FP8_ATTN_GEMM.get()
): ):
# fp8 attention GEMMs activate whenever possible # fp8 attention GEMMs activate whenever possible
@@ -962,42 +1000,36 @@ def _minimax_m3_overrides(server_args: Any, hf_config: Any) -> dict:
overrides["attention_backend"] = "trtllm_mha" overrides["attention_backend"] = "trtllm_mha"
else: else:
overrides["attention_backend"] = "fa4" overrides["attention_backend"] = "fa4"
backend_resolved = overrides.get( backend_resolved = overrides.get("attention_backend", cfg.attention_backend)
"attention_backend", server_args.attention_backend page_resolved = cfg.page_size
)
page_resolved = server_args.page_size
# fa4 (fmha_sm100) and trtllm_mha both allow the page_size == 128 # fa4 (fmha_sm100) and trtllm_mha both allow the page_size == 128
# sparse block MSA needs (trtllm_mha via trtllm-gen's dynamic # sparse block MSA needs (trtllm_mha via trtllm-gen's dynamic
# tokens-per-page kernels). # tokens-per-page kernels).
if page_resolved is None and backend_resolved in ("fa4", "trtllm_mha"): if page_resolved is None and backend_resolved in ("fa4", "trtllm_mha"):
overrides["page_size"] = 128 overrides["page_size"] = 128
page_resolved = 128 page_resolved = 128
if server_args.moe_runner_backend == "auto" and quant_resolved == "mxfp8": if cfg.moe_runner_backend == "auto" and quant_resolved == "mxfp8":
overrides["moe_runner_backend"] = "deep_gemm" overrides["moe_runner_backend"] = "deep_gemm"
elif ( elif cfg.moe_runner_backend == "auto" and quant_resolved == "modelopt_mixed":
server_args.moe_runner_backend == "auto"
and quant_resolved == "modelopt_mixed"
):
overrides["moe_runner_backend"] = "flashinfer_trtllm_routed" overrides["moe_runner_backend"] = "flashinfer_trtllm_routed"
logger.info( logger.info(
"MiniMax-M3 on SM100: attention_backend=" "MiniMax-M3 on SM100: attention_backend="
f"{overrides.get('attention_backend', server_args.attention_backend)}, page_size={page_resolved}, " f"{overrides.get('attention_backend', cfg.attention_backend)}, page_size={page_resolved}, "
f"moe_runner_backend={overrides.get('moe_runner_backend', server_args.moe_runner_backend)}." f"moe_runner_backend={overrides.get('moe_runner_backend', cfg.moe_runner_backend)}."
) )
elif is_sm90_supported(): elif is_sm90_supported():
if server_args.is_attention_backend_not_set(): if server_args.is_attention_backend_not_set():
overrides["attention_backend"] = "fa3" overrides["attention_backend"] = "fa3"
page_resolved = server_args.page_size page_resolved = cfg.page_size
if ( if (
page_resolved is None page_resolved is None
and overrides.get("attention_backend", server_args.attention_backend) and overrides.get("attention_backend", cfg.attention_backend) == "fa3"
== "fa3"
): ):
overrides["page_size"] = 128 overrides["page_size"] = 128
page_resolved = 128 page_resolved = 128
logger.info( logger.info(
"MiniMax-M3 on Hopper: attention_backend=" "MiniMax-M3 on Hopper: attention_backend="
f"{overrides.get('attention_backend', server_args.attention_backend)}, page_size={page_resolved} " f"{overrides.get('attention_backend', cfg.attention_backend)}, page_size={page_resolved} "
"(MSA is SM100-only; sparse attention runs on the Triton path)." "(MSA is SM100-only; sparse attention runs on the Triton path)."
) )
@@ -1008,7 +1040,7 @@ def _minimax_m3_overrides(server_args: Any, hf_config: Any) -> dict:
# silently dispatch the e4m3 kernel, so e5m2 stays on the widening Triton # silently dispatch the e4m3 kernel, so e5m2 stays on the widening Triton
# path), log when the fp8 GEMM mode is active, and log when the # path), log when the fp8 GEMM mode is active, and log when the
# SGLANG_DISABLE_M3_FP8_ATTN_GEMM kill switch suppresses it. # SGLANG_DISABLE_M3_FP8_ATTN_GEMM kill switch suppresses it.
if server_args.kv_cache_dtype == "fp8_e5m2": if cfg.kv_cache_dtype == "fp8_e5m2":
logger.warning( logger.warning(
"MiniMax-M3 with kv_cache_dtype fp8_e5m2: fp8 attention GEMMs stay " "MiniMax-M3 with kv_cache_dtype fp8_e5m2: fp8 attention GEMMs stay "
"DISABLED (fmha_sm100's variant lookup would silently dispatch the " "DISABLED (fmha_sm100's variant lookup would silently dispatch the "
@@ -1016,9 +1048,8 @@ def _minimax_m3_overrides(server_args: Any, hf_config: Any) -> dict:
"Triton path. Use --kv-cache-dtype fp8_e4m3 for fp8 attention GEMMs." "Triton path. Use --kv-cache-dtype fp8_e4m3 for fp8 attention GEMMs."
) )
elif ( elif (
server_args.kv_cache_dtype == "fp8_e4m3" cfg.kv_cache_dtype == "fp8_e4m3"
and overrides.get("attention_backend", server_args.attention_backend) and overrides.get("attention_backend", cfg.attention_backend) == "trtllm_mha"
== "trtllm_mha"
and is_sm100_supported() and is_sm100_supported()
): ):
if envs.SGLANG_DISABLE_M3_FP8_ATTN_GEMM.get(): if envs.SGLANG_DISABLE_M3_FP8_ATTN_GEMM.get():
@@ -1036,9 +1067,7 @@ def _minimax_m3_overrides(server_args: Any, hf_config: Any) -> dict:
"force the pre-fp8 numerics." "force the pre-fp8 numerics."
) )
moe_runner_resolved = overrides.get( moe_runner_resolved = overrides.get("moe_runner_backend", cfg.moe_runner_backend)
"moe_runner_backend", server_args.moe_runner_backend
)
if quant_resolved is None and moe_runner_resolved in ("auto", "deep_gemm"): if quant_resolved is None and moe_runner_resolved in ("auto", "deep_gemm"):
if moe_runner_resolved == "deep_gemm": if moe_runner_resolved == "deep_gemm":
logger.warning( logger.warning(
@@ -1078,6 +1107,7 @@ def _exaone_overrides(server_args: Any, hf_config: Any) -> dict:
@_register_for("GptOssForCausalLM") @_register_for("GptOssForCausalLM")
def _gpt_oss_overrides(server_args: Any, hf_config: Any) -> dict: def _gpt_oss_overrides(server_args: Any, hf_config: Any) -> dict:
cfg = resolving_view(server_args)
overrides: Dict[str, Any] = {} overrides: Dict[str, Any] = {}
# Set attention backend for GPT-OSS # Set attention backend for GPT-OSS
if server_args.is_attention_backend_not_set(): if server_args.is_attention_backend_not_set():
@@ -1100,14 +1130,14 @@ def _gpt_oss_overrides(server_args: Any, hf_config: Any) -> dict:
# Check for bf16 dtype on Intel XPU. Reads the pristine dtype request, # Check for bf16 dtype on Intel XPU. Reads the pristine dtype request,
# which equals the legacy mid-branch read: dtype had no earlier writer # which equals the legacy mid-branch read: dtype had no earlier writer
# for this arch. # for this arch.
if server_args.dtype == "auto": if cfg.dtype == "auto":
logger.warning( logger.warning(
"GptOssForCausalLM on Intel XPU currently supports bfloat16 dtype only" "GptOssForCausalLM on Intel XPU currently supports bfloat16 dtype only"
) )
elif server_args.dtype not in ["bfloat16"]: elif cfg.dtype not in ["bfloat16"]:
raise NotImplementedError( raise NotImplementedError(
f"GptOssForCausalLM on Intel XPU only supports bfloat16 dtype, " f"GptOssForCausalLM on Intel XPU only supports bfloat16 dtype, "
f"but got '{server_args.dtype}'. Please use --dtype bfloat16 or remove --dtype to use auto." f"but got '{cfg.dtype}'. Please use --dtype bfloat16 or remove --dtype to use auto."
) )
quantization_config = getattr(hf_config, "quantization_config", None) quantization_config = getattr(hf_config, "quantization_config", None)
is_mxfp4_quant_format = ( is_mxfp4_quant_format = (
@@ -1117,7 +1147,7 @@ def _gpt_oss_overrides(server_args: Any, hf_config: Any) -> dict:
if is_mxfp4_quant_format: if is_mxfp4_quant_format:
# use bf16 for mxfp4 triton kernels # use bf16 for mxfp4 triton kernels
overrides["dtype"] = "bfloat16" overrides["dtype"] = "bfloat16"
if server_args.moe_runner_backend == "auto": if cfg.moe_runner_backend == "auto":
if is_sm100_supported() and is_mxfp4_quant_format: if is_sm100_supported() and is_mxfp4_quant_format:
overrides["moe_runner_backend"] = "flashinfer_mxfp4" overrides["moe_runner_backend"] = "flashinfer_mxfp4"
@@ -1155,9 +1185,9 @@ def _gpt_oss_overrides(server_args: Any, hf_config: Any) -> dict:
"Detected MUSA with SGLANG_DEEPEP_BF16_DISPATCH for bf16 model, using deep_gemm kernel." "Detected MUSA with SGLANG_DEEPEP_BF16_DISPATCH for bf16 model, using deep_gemm kernel."
) )
elif ( elif (
server_args.ep_size == 1 cfg.ep_size == 1
and is_triton_kernels_available() and is_triton_kernels_available()
and server_args.quantization is None and cfg.quantization is None
and not (is_cpu() and cpu_has_amx_support()) and not (is_cpu() and cpu_has_amx_support())
): ):
# The triton_kernels package segfaults on Blackwell (B200) # The triton_kernels package segfaults on Blackwell (B200)
@@ -1179,18 +1209,19 @@ def _gpt_oss_overrides(server_args: Any, hf_config: Any) -> dict:
# Keep in sync with LLAMA4_MODEL_ARCHS (server_args.py). # Keep in sync with LLAMA4_MODEL_ARCHS (server_args.py).
@_register_for("Llama4ForConditionalGeneration", "Llama4ForCausalLM") @_register_for("Llama4ForConditionalGeneration", "Llama4ForCausalLM")
def _llama4_overrides(server_args: Any, hf_config: Any) -> dict: def _llama4_overrides(server_args: Any, hf_config: Any) -> dict:
if server_args.device == "cpu": cfg = resolving_view(server_args)
if cfg.device == "cpu":
return {} return {}
overrides: Dict[str, Any] = {} overrides: Dict[str, Any] = {}
# Auto-select attention backend for Llama4 if not specified # Auto-select attention backend for Llama4 if not specified
if server_args.attention_backend is None: if cfg.attention_backend is None:
if is_sm100_supported(): if is_sm100_supported():
backend, platform = "trtllm_mha", "sm100" backend, platform = "trtllm_mha", "sm100"
elif is_sm90_supported(): elif is_sm90_supported():
backend, platform = "fa3", "sm90" backend, platform = "fa3", "sm90"
elif is_hip(): elif is_hip():
backend, platform = "aiter", "hip" backend, platform = "aiter", "hip"
elif server_args.device == "xpu": elif cfg.device == "xpu":
backend, platform = "intel_xpu", "xpu" backend, platform = "intel_xpu", "xpu"
else: else:
backend, platform = "triton", "other platforms" backend, platform = "triton", "other platforms"
@@ -1198,8 +1229,8 @@ def _llama4_overrides(server_args: Any, hf_config: Any) -> dict:
f"Use {backend} as attention backend on {platform} for Llama4 model" f"Use {backend} as attention backend on {platform} for Llama4 model"
) )
overrides["attention_backend"] = backend overrides["attention_backend"] = backend
if is_sm100_supported() and server_args.moe_runner_backend == "auto": if is_sm100_supported() and cfg.moe_runner_backend == "auto":
if server_args.quantization in {"fp8", "modelopt_fp8"}: if cfg.quantization in {"fp8", "modelopt_fp8"}:
overrides["moe_runner_backend"] = "flashinfer_trtllm" overrides["moe_runner_backend"] = "flashinfer_trtllm"
logger.info( logger.info(
"Use flashinfer_trtllm as MoE runner backend on SM100 for Llama4" "Use flashinfer_trtllm as MoE runner backend on SM100 for Llama4"
@@ -1213,6 +1244,7 @@ def _llama4_overrides(server_args: Any, hf_config: Any) -> dict:
"Gemma4UnifiedForConditionalGeneration", "Gemma4UnifiedForConditionalGeneration",
) )
def _gemma4_overrides(server_args: Any, hf_config: Any) -> dict: def _gemma4_overrides(server_args: Any, hf_config: Any) -> dict:
cfg = resolving_view(server_args)
overrides: Dict[str, Any] = {} overrides: Dict[str, Any] = {}
default_attention_backend = "trtllm_mha" if is_sm100_supported() else "triton" default_attention_backend = "trtllm_mha" if is_sm100_supported() else "triton"
if server_args.is_attention_backend_not_set(): if server_args.is_attention_backend_not_set():
@@ -1223,9 +1255,9 @@ def _gemma4_overrides(server_args: Any, hf_config: Any) -> dict:
# If only one split backend is set, keep the other side on a # If only one split backend is set, keep the other side on a
# Gemma4-compatible fallback instead of letting generic backend selection # Gemma4-compatible fallback instead of letting generic backend selection
# choose an unsupported backend later. # choose an unsupported backend later.
elif server_args.attention_backend is None: elif cfg.attention_backend is None:
overrides["attention_backend"] = default_attention_backend overrides["attention_backend"] = default_attention_backend
if is_sm100_supported() and server_args.moe_runner_backend == "auto": if is_sm100_supported() and cfg.moe_runner_backend == "auto":
if server_args.get_model_config().quantization == "modelopt_fp4": if server_args.get_model_config().quantization == "modelopt_fp4":
overrides["quantization"] = "modelopt_fp4" overrides["quantization"] = "modelopt_fp4"
overrides["moe_runner_backend"] = "flashinfer_trtllm" overrides["moe_runner_backend"] = "flashinfer_trtllm"
@@ -1255,7 +1287,8 @@ def _moss_vl_overrides(server_args: Any, hf_config: Any) -> dict:
@_register_for("MiniCPMForCausalLM", "MiniCPMSALAForCausalLM") @_register_for("MiniCPMForCausalLM", "MiniCPMSALAForCausalLM")
def _minicpm_sala_overrides(server_args: Any, hf_config: Any) -> dict: def _minicpm_sala_overrides(server_args: Any, hf_config: Any) -> dict:
if server_args.enable_dp_attention: cfg = resolving_view(server_args)
if cfg.enable_dp_attention:
raise ValueError("MiniCPM does not support DP attention") raise ValueError("MiniCPM does not support DP attention")
has_sparse_attention = getattr(hf_config, "has_minicpm_sparse_attention", False) has_sparse_attention = getattr(hf_config, "has_minicpm_sparse_attention", False)
has_hybrid_attention = has_sparse_attention or getattr( has_hybrid_attention = has_sparse_attention or getattr(
@@ -1263,7 +1296,7 @@ def _minicpm_sala_overrides(server_args: Any, hf_config: Any) -> dict:
) )
overrides: Dict[str, Any] = {} overrides: Dict[str, Any] = {}
if has_hybrid_attention: if has_hybrid_attention:
if server_args.enable_hierarchical_cache: if cfg.enable_hierarchical_cache:
raise ValueError("MiniCPM SALA does not support hierarchical cache") raise ValueError("MiniCPM SALA does not support hierarchical cache")
overrides["disable_radix_cache"] = True overrides["disable_radix_cache"] = True
if envs.SGLANG_MINICPM_FORCE_DENSE.get(): if envs.SGLANG_MINICPM_FORCE_DENSE.get():
@@ -1273,29 +1306,29 @@ def _minicpm_sala_overrides(server_args: Any, hf_config: Any) -> dict:
} }
# Literal keys keep the written-field set statically derivable; a loop # Literal keys keep the written-field set statically derivable; a loop
# variable hides it from the census in test_chain_read_ratchet.py. # variable hides it from the census in test_chain_read_ratchet.py.
dense_attention = dense_backends.get(server_args.attention_backend) dense_attention = dense_backends.get(cfg.attention_backend)
if dense_attention is not None: if dense_attention is not None:
overrides["attention_backend"] = dense_attention overrides["attention_backend"] = dense_attention
dense_prefill = dense_backends.get(server_args.prefill_attention_backend) dense_prefill = dense_backends.get(cfg.prefill_attention_backend)
if dense_prefill is not None: if dense_prefill is not None:
overrides["prefill_attention_backend"] = dense_prefill overrides["prefill_attention_backend"] = dense_prefill
dense_decode = dense_backends.get(server_args.decode_attention_backend) dense_decode = dense_backends.get(cfg.decode_attention_backend)
if dense_decode is not None: if dense_decode is not None:
overrides["decode_attention_backend"] = dense_decode overrides["decode_attention_backend"] = dense_decode
elif has_sparse_attention: elif has_sparse_attention:
uses_sparse_backend = server_args.is_attention_backend_not_set() or any( uses_sparse_backend = cfg.is_attention_backend_not_set() or any(
backend in ("minicpm_flashattn", "minicpm_flashinfer") backend in ("minicpm_flashattn", "minicpm_flashinfer")
for backend in ( for backend in (
server_args.attention_backend, cfg.attention_backend,
server_args.prefill_attention_backend, cfg.prefill_attention_backend,
server_args.decode_attention_backend, cfg.decode_attention_backend,
) )
) )
if uses_sparse_backend and server_args.disaggregation_mode != "null": if uses_sparse_backend and cfg.disaggregation_mode != "null":
raise ValueError( raise ValueError(
"MiniCPM sparse attention does not support PD disaggregation" "MiniCPM sparse attention does not support PD disaggregation"
) )
if server_args.is_attention_backend_not_set(): if cfg.is_attention_backend_not_set():
overrides["attention_backend"] = ( overrides["attention_backend"] = (
"minicpm_flashinfer" "minicpm_flashinfer"
if is_blackwell_supported() if is_blackwell_supported()
@@ -1306,7 +1339,8 @@ def _minicpm_sala_overrides(server_args: Any, hf_config: Any) -> dict:
@_register_for("MiniCPMV4_6ForConditionalGeneration") @_register_for("MiniCPMV4_6ForConditionalGeneration")
def _minicpm_v4_6_overrides(server_args: Any, hf_config: Any) -> dict: def _minicpm_v4_6_overrides(server_args: Any, hf_config: Any) -> dict:
if is_sm100_supported() and server_args.attention_backend is None: cfg = resolving_view(server_args)
if is_sm100_supported() and cfg.attention_backend is None:
return {"attention_backend": "triton"} return {"attention_backend": "triton"}
return {} return {}
@@ -1315,24 +1349,27 @@ def _minicpm_v4_6_overrides(server_args: Any, hf_config: Any) -> dict:
"FalconH1ForCausalLM", "JetNemotronForCausalLM", "JetVLMForConditionalGeneration" "FalconH1ForCausalLM", "JetNemotronForCausalLM", "JetVLMForConditionalGeneration"
) )
def _falcon_h1_jet_overrides(server_args: Any, hf_config: Any) -> dict: def _falcon_h1_jet_overrides(server_args: Any, hf_config: Any) -> dict:
if is_sm100_supported() and server_args.attention_backend is None: cfg = resolving_view(server_args)
if is_sm100_supported() and cfg.attention_backend is None:
return {"attention_backend": "triton"} return {"attention_backend": "triton"}
return {} return {}
@_register_for("GraniteMoeHybridForCausalLM") @_register_for("GraniteMoeHybridForCausalLM")
def _granite_moe_hybrid_overrides(server_args: Any, hf_config: Any) -> dict: def _granite_moe_hybrid_overrides(server_args: Any, hf_config: Any) -> dict:
cfg = resolving_view(server_args)
has_mamba = any( has_mamba = any(
layer_type == "mamba" for layer_type in getattr(hf_config, "layer_types", []) layer_type == "mamba" for layer_type in getattr(hf_config, "layer_types", [])
) )
if has_mamba and is_sm100_supported() and server_args.attention_backend is None: if has_mamba and is_sm100_supported() and cfg.attention_backend is None:
return {"attention_backend": "flashinfer"} return {"attention_backend": "flashinfer"}
return {} return {}
@_register_for("Lfm2ForCausalLM", "Lfm2MoeForCausalLM") @_register_for("Lfm2ForCausalLM", "Lfm2MoeForCausalLM")
def _lfm2_overrides(server_args: Any, hf_config: Any) -> dict: def _lfm2_overrides(server_args: Any, hf_config: Any) -> dict:
if is_sm100_supported() and server_args.attention_backend is None: cfg = resolving_view(server_args)
if is_sm100_supported() and cfg.attention_backend is None:
return {"attention_backend": "flashinfer"} return {"attention_backend": "flashinfer"}
return {} return {}
@@ -1343,13 +1380,14 @@ def _deepseek_v4_overrides(server_args: Any, hf_config: Any) -> dict:
arg_groups/deepseek_v4_hook.py). The kv-cache dtype and NPU split-backend arg_groups/deepseek_v4_hook.py). The kv-cache dtype and NPU split-backend
writes, the max_running_requests fill and the validations stay in the writes, the max_running_requests fill and the validations stay in the
hook at its legacy slot.""" hook at its legacy slot."""
cfg = resolving_view(server_args)
from sglang.srt.server_args import ServerArgs from sglang.srt.server_args import ServerArgs
model_arch = hf_config.architectures[0] model_arch = hf_config.architectures[0]
overrides: Dict[str, Any] = {"attention_backend": "dsv4"} overrides: Dict[str, Any] = {"attention_backend": "dsv4"}
page_size = 256 page_size = 256
if server_args.device == "npu": if cfg.device == "npu":
# NPU keeps the device-aware "dsv4" backend (the registry routes it to # NPU keeps the device-aware "dsv4" backend (the registry routes it to
# the Ascend V4 subclass); only the pool geometry / dtype differ. # the Ascend V4 subclass); only the pool geometry / dtype differ.
# set_default_server_args() pins all three backends to "ascend" for # set_default_server_args() pins all three backends to "ascend" for
@@ -1363,11 +1401,11 @@ def _deepseek_v4_overrides(server_args: Any, hf_config: Any) -> dict:
f"Use dsv4 attention backend for {model_arch}, setting page_size to {page_size}." f"Use dsv4 attention backend for {model_arch}, setting page_size to {page_size}."
) )
if server_args.swa_full_tokens_ratio == ServerArgs.swa_full_tokens_ratio: if cfg.swa_full_tokens_ratio == ServerArgs.swa_full_tokens_ratio:
overrides["swa_full_tokens_ratio"] = 0.1 overrides["swa_full_tokens_ratio"] = 0.1
logger.info(f"Setting swa_full_tokens_ratio to 0.1 for {model_arch}.") logger.info(f"Setting swa_full_tokens_ratio to 0.1 for {model_arch}.")
if server_args.moe_runner_backend == "auto": if cfg.moe_runner_backend == "auto":
model_config = server_args.get_model_config() model_config = server_args.get_model_config()
# nvidia/DeepSeek-V4-Pro-NVFP4 uses the routed TRT-LLM runner. # nvidia/DeepSeek-V4-Pro-NVFP4 uses the routed TRT-LLM runner.
if model_config.nvfp4_moe_meta is not None: if model_config.nvfp4_moe_meta is not None:
@@ -1377,9 +1415,9 @@ def _deepseek_v4_overrides(server_args: Any, hf_config: Any) -> dict:
f"{model_arch} hybrid FP8+NVFP4 checkpoint." f"{model_arch} hybrid FP8+NVFP4 checkpoint."
) )
elif ( elif (
server_args.device == "cuda" cfg.device == "cuda"
and not is_hip() and not is_hip()
and server_args.moe_a2a_backend == "none" and cfg.moe_a2a_backend == "none"
and not envs.SGLANG_DSV4_FP4_DEQUANT.get() and not envs.SGLANG_DSV4_FP4_DEQUANT.get()
and model_config.is_fp4_experts and model_config.is_fp4_experts
and (is_sm90_supported() or is_sm100_supported() or is_sm120_supported()) and (is_sm90_supported() or is_sm100_supported() or is_sm120_supported())
@@ -1407,6 +1445,7 @@ def _inkling_overrides(server_args: Any, hf_config: Any) -> dict:
prefill.backend, and an explicit --cuda-graph-backend-prefill / prefill.backend, and an explicit --cuda-graph-backend-prefill /
--disable-prefill-cuda-graph still wins. The unified-radix env write follows --disable-prefill-cuda-graph still wins. The unified-radix env write follows
the MiniMax-M3 handler precedent (env is not a resolvable server-arg).""" the MiniMax-M3 handler precedent (env is not a resolvable server-arg)."""
cfg = resolving_view(server_args)
from sglang.srt.server_args import ServerArgs from sglang.srt.server_args import ServerArgs
overrides: Dict[str, Any] = {} overrides: Dict[str, Any] = {}
@@ -1415,14 +1454,14 @@ def _inkling_overrides(server_args: Any, hf_config: Any) -> dict:
# cuda_graph_backend_prefill declared here lands too late (the breakable # cuda_graph_backend_prefill declared here lands too late (the breakable
# default would already have been auto-disabled for this multimodal arch). # default would already have been auto-disabled for this multimodal arch).
# It is set inline before _handle_cuda_graph_config instead. # It is set inline before _handle_cuda_graph_config instead.
if server_args.swa_full_tokens_ratio == ServerArgs.swa_full_tokens_ratio: if cfg.swa_full_tokens_ratio == ServerArgs.swa_full_tokens_ratio:
overrides["swa_full_tokens_ratio"] = 0.1 overrides["swa_full_tokens_ratio"] = 0.1
if server_args.mamba_full_memory_ratio == ServerArgs.mamba_full_memory_ratio: if cfg.mamba_full_memory_ratio == ServerArgs.mamba_full_memory_ratio:
overrides["mamba_full_memory_ratio"] = 0.1 overrides["mamba_full_memory_ratio"] = 0.1
# Inkling requires the extra-buffer mamba strategy (inkling.py asserts # Inkling requires the extra-buffer mamba strategy (inkling.py asserts
# enable_mamba_extra_buffer()); the generic "auto" resolution does not cover # enable_mamba_extra_buffer()); the generic "auto" resolution does not cover
# Inkling, so pin it here. Yields to an explicit --mamba-scheduler-strategy. # Inkling, so pin it here. Yields to an explicit --mamba-scheduler-strategy.
if server_args.mamba_radix_cache_strategy == ServerArgs.mamba_radix_cache_strategy: if cfg.mamba_radix_cache_strategy == ServerArgs.mamba_radix_cache_strategy:
overrides["mamba_radix_cache_strategy"] = "extra_buffer" overrides["mamba_radix_cache_strategy"] = "extra_buffer"
# Inkling attention runs only on the fa4 (Blackwell) or triton backends -- # Inkling attention runs only on the fa4 (Blackwell) or triton backends --
# models/inkling_common/attn.py asserts attention_backend in {fa4, triton}. # models/inkling_common/attn.py asserts attention_backend in {fa4, triton}.
@@ -1447,6 +1486,7 @@ def _nemotron_h_overrides(server_args: Any, hf_config: Any) -> dict:
"""NemotronH quantization / MoE runner / attention backend defaults """NemotronH quantization / MoE runner / attention backend defaults
(absorbed from the retired arg_groups/nemotron_h_hook.py; the mamba radix (absorbed from the retired arg_groups/nemotron_h_hook.py; the mamba radix
cache handling and the triton-backend assert stay in the arch branch).""" cache handling and the triton-backend assert stay in the arch branch)."""
cfg = resolving_view(server_args)
model_arch = hf_config.architectures[0] model_arch = hf_config.architectures[0]
model_config = server_args.get_model_config() model_config = server_args.get_model_config()
overrides: Dict[str, Any] = {} overrides: Dict[str, Any] = {}
@@ -1457,7 +1497,7 @@ def _nemotron_h_overrides(server_args: Any, hf_config: Any) -> dict:
"modelopt_fp4", "modelopt_fp4",
"modelopt_mixed", "modelopt_mixed",
] ]
quantization = server_args.quantization quantization = cfg.quantization
if is_modelopt: if is_modelopt:
assert model_config.hf_config.mlp_hidden_act == "relu2" assert model_config.hf_config.mlp_hidden_act == "relu2"
if model_config.quantization == "modelopt": if model_config.quantization == "modelopt":
@@ -1482,22 +1522,22 @@ def _nemotron_h_overrides(server_args: Any, hf_config: Any) -> dict:
) )
if has_w4a16_moe_layers: if has_w4a16_moe_layers:
if server_args.moe_a2a_backend != "none": if cfg.moe_a2a_backend != "none":
raise ValueError("W4A16_NVFP4 MoE layers require --moe-a2a-backend=none.") raise ValueError("W4A16_NVFP4 MoE layers require --moe-a2a-backend=none.")
if server_args.moe_runner_backend not in ("auto", "marlin"): if cfg.moe_runner_backend not in ("auto", "marlin"):
raise ValueError( raise ValueError(
"W4A16_NVFP4 MoE layers require --moe-runner-backend=marlin." "W4A16_NVFP4 MoE layers require --moe-runner-backend=marlin."
) )
if server_args.moe_runner_backend == "auto": if cfg.moe_runner_backend == "auto":
overrides["moe_runner_backend"] = "marlin" overrides["moe_runner_backend"] = "marlin"
logger.info( logger.info(
"Use marlin as MoE runner backend for " "Use marlin as MoE runner backend for "
f"{model_arch} with W4A16_NVFP4 MoE layers" f"{model_arch} with W4A16_NVFP4 MoE layers"
) )
elif (is_modelopt or model_config.quantization is None) and ( elif (is_modelopt or model_config.quantization is None) and (
server_args.moe_runner_backend == "auto" cfg.moe_runner_backend == "auto"
): ):
if is_sm100_supported() and server_args.moe_a2a_backend == "none": if is_sm100_supported() and cfg.moe_a2a_backend == "none":
overrides["moe_runner_backend"] = "flashinfer_trtllm" overrides["moe_runner_backend"] = "flashinfer_trtllm"
logger.info( logger.info(
f"Use flashinfer_trtllm as MoE runner backend on sm100 for {model_arch}" f"Use flashinfer_trtllm as MoE runner backend on sm100 for {model_arch}"
@@ -1518,27 +1558,27 @@ def _nemotron_h_overrides(server_args: Any, hf_config: Any) -> dict:
else: else:
overrides["moe_runner_backend"] = "flashinfer_cutlass" overrides["moe_runner_backend"] = "flashinfer_cutlass"
if is_blackwell_supported() and server_args.is_attention_backend_not_set(): if is_blackwell_supported() and cfg.is_attention_backend_not_set():
if server_args.speculative_algorithm is not None: if cfg.speculative_algorithm is not None:
speculative_algorithm = server_args.speculative_algorithm.upper() speculative_algorithm = cfg.speculative_algorithm.upper()
if is_sm100_supported() and server_args.speculative_eagle_topk in ( if is_sm100_supported() and cfg.speculative_eagle_topk in (
None, None,
1, 1,
): ):
overrides["attention_backend"] = "trtllm_mha" overrides["attention_backend"] = "trtllm_mha"
if server_args.page_size is None: if cfg.page_size is None:
overrides["page_size"] = 64 overrides["page_size"] = 64
if server_args.mamba_radix_cache_strategy == "auto": if cfg.mamba_radix_cache_strategy == "auto":
overrides["mamba_radix_cache_strategy"] = "extra_buffer" overrides["mamba_radix_cache_strategy"] = "extra_buffer"
if ( if (
server_args.speculative_draft_attention_backend is None cfg.speculative_draft_attention_backend is None
and speculative_algorithm in ("EAGLE", "NEXTN", "DSPARK") and speculative_algorithm in ("EAGLE", "NEXTN", "DSPARK")
): ):
overrides["speculative_draft_attention_backend"] = "trtllm_mha" overrides["speculative_draft_attention_backend"] = "trtllm_mha"
else: else:
overrides["attention_backend"] = "triton" overrides["attention_backend"] = "triton"
if ( if (
server_args.speculative_draft_attention_backend is None cfg.speculative_draft_attention_backend is None
and speculative_algorithm in ("EAGLE", "NEXTN", "DFLASH", "DSPARK") and speculative_algorithm in ("EAGLE", "NEXTN", "DFLASH", "DSPARK")
): ):
overrides["speculative_draft_attention_backend"] = "flashinfer" overrides["speculative_draft_attention_backend"] = "flashinfer"
@@ -1555,7 +1595,8 @@ def _nemotron_h_overrides(server_args: Any, hf_config: Any) -> dict:
"Qwen3_5ForConditionalGeneration", "Qwen3_5ForConditionalGeneration",
) )
def _qwen3_5_hybrid_overrides(server_args: Any, hf_config: Any) -> dict: def _qwen3_5_hybrid_overrides(server_args: Any, hf_config: Any) -> dict:
if not is_sm100_supported() or server_args.attention_backend is not None: cfg = resolving_view(server_args)
if not is_sm100_supported() or cfg.attention_backend is not None:
return {} return {}
sm100_default_attn_backend = "triton" sm100_default_attn_backend = "triton"
# trtllm_mha requires speculative_eagle_topk == 1 and page_size > 1. # trtllm_mha requires speculative_eagle_topk == 1 and page_size > 1.
@@ -1572,8 +1613,8 @@ def _qwen3_5_hybrid_overrides(server_args: Any, hf_config: Any) -> dict:
# already-written field here). # already-written field here).
if default_attn_backend == "trtllm_mha" and not ( if default_attn_backend == "trtllm_mha" and not (
not mamba_extra_buffer_of(resolved_view(server_args)) not mamba_extra_buffer_of(resolved_view(server_args))
and not server_args.disable_radix_cache and not cfg.disable_radix_cache
and server_args.speculative_algorithm is None and cfg.speculative_algorithm is None
): ):
sm100_default_attn_backend = "trtllm_mha" sm100_default_attn_backend = "trtllm_mha"
return { return {
@@ -1585,7 +1626,8 @@ def _qwen3_5_hybrid_overrides(server_args: Any, hf_config: Any) -> dict:
@_register_for("InternS2MobiusForConditionalGeneration") @_register_for("InternS2MobiusForConditionalGeneration")
def _interns2_mobius_baseline_overrides(server_args: Any, hf_config: Any) -> dict: def _interns2_mobius_baseline_overrides(server_args: Any, hf_config: Any) -> dict:
"""Select the only MoE runner validated for the 2,560-expert baseline.""" """Select the only MoE runner validated for the 2,560-expert baseline."""
if server_args.moe_runner_backend == "auto": cfg = resolving_view(server_args)
if cfg.moe_runner_backend == "auto":
return {"moe_runner_backend": "triton_kernel"} return {"moe_runner_backend": "triton_kernel"}
return {} return {}
@@ -1593,11 +1635,8 @@ def _interns2_mobius_baseline_overrides(server_args: Any, hf_config: Any) -> dic
@_register_for("Qwen3VLForConditionalGeneration") @_register_for("Qwen3VLForConditionalGeneration")
def _qwen3vl_overrides(server_args: Any, hf_config: Any) -> dict: def _qwen3vl_overrides(server_args: Any, hf_config: Any) -> dict:
if ( cfg = resolving_view(server_args)
is_hip() if is_hip() and envs.SGLANG_USE_AITER_UNIFIED_ATTN.get() and cfg.page_size is None:
and envs.SGLANG_USE_AITER_UNIFIED_ATTN.get()
and server_args.page_size is None
):
logger.info( logger.info(
"Setting page_size=16 for aiter unified attention on Qwen3VLForConditionalGeneration." "Setting page_size=16 for aiter unified attention on Qwen3VLForConditionalGeneration."
) )
@@ -1614,10 +1653,11 @@ def _qwen3vl_overrides(server_args: Any, hf_config: Any) -> dict:
"Qwen3_5ForConditionalGeneration", "Qwen3_5ForConditionalGeneration",
) )
def _qwen3_moe_family_overrides(server_args: Any, hf_config: Any) -> dict: def _qwen3_moe_family_overrides(server_args: Any, hf_config: Any) -> dict:
cfg = resolving_view(server_args)
overrides: Dict[str, Any] = {} overrides: Dict[str, Any] = {}
if is_sm100_supported(): if is_sm100_supported():
quant_method = get_quantization_config(hf_config) quant_method = get_quantization_config(hf_config)
quantization = server_args.quantization quantization = cfg.quantization
if ( if (
quantization is None quantization is None
and not server_args._quantization_explicitly_unset and not server_args._quantization_explicitly_unset
@@ -1627,8 +1667,8 @@ def _qwen3_moe_family_overrides(server_args: Any, hf_config: Any) -> dict:
quantization = quant_method quantization = quant_method
if ( if (
(quantization in ("fp8", "modelopt_fp4") or quantization is None) (quantization in ("fp8", "modelopt_fp4") or quantization is None)
and server_args.moe_a2a_backend == "none" and cfg.moe_a2a_backend == "none"
and server_args.moe_runner_backend == "auto" and cfg.moe_runner_backend == "auto"
): ):
overrides["moe_runner_backend"] = "flashinfer_trtllm" overrides["moe_runner_backend"] = "flashinfer_trtllm"
logger.info( logger.info(
@@ -1640,6 +1680,7 @@ def _qwen3_moe_family_overrides(server_args: Any, hf_config: Any) -> dict:
@_register_for("Glm4MoeForCausalLM") @_register_for("Glm4MoeForCausalLM")
def _glm4_moe_overrides(server_args: Any, hf_config: Any) -> dict: def _glm4_moe_overrides(server_args: Any, hf_config: Any) -> dict:
cfg = resolving_view(server_args)
overrides: Dict[str, Any] = {} overrides: Dict[str, Any] = {}
if is_sm100_supported(): if is_sm100_supported():
quantization_config = getattr(hf_config, "quantization_config", None) quantization_config = getattr(hf_config, "quantization_config", None)
@@ -1648,7 +1689,7 @@ def _glm4_moe_overrides(server_args: Any, hf_config: Any) -> dict:
if quantization_config is not None if quantization_config is not None
else None else None
) )
quantization = server_args.quantization quantization = cfg.quantization
if ( if (
quantization is None quantization is None
and not server_args._quantization_explicitly_unset and not server_args._quantization_explicitly_unset
@@ -1658,8 +1699,8 @@ def _glm4_moe_overrides(server_args: Any, hf_config: Any) -> dict:
quantization = quant_method quantization = quant_method
if ( if (
quantization in {"modelopt_fp4", None} quantization in {"modelopt_fp4", None}
and server_args.moe_a2a_backend == "none" and cfg.moe_a2a_backend == "none"
and server_args.moe_runner_backend == "auto" and cfg.moe_runner_backend == "auto"
): ):
overrides["moe_runner_backend"] = "flashinfer_trtllm" overrides["moe_runner_backend"] = "flashinfer_trtllm"
logger.info( logger.info(
@@ -1674,13 +1715,14 @@ def _glm4_moe_overrides(server_args: Any, hf_config: Any) -> dict:
@_register_for("Olmo2ForCausalLM") @_register_for("Olmo2ForCausalLM")
def _olmo2_overrides(server_args: Any, hf_config: Any) -> dict: def _olmo2_overrides(server_args: Any, hf_config: Any) -> dict:
cfg = resolving_view(server_args)
overrides: Dict[str, Any] = {} overrides: Dict[str, Any] = {}
# FIXME: https://github.com/sgl-project/sglang/pull/7367 is not compatible with Olmo3 model. # FIXME: https://github.com/sgl-project/sglang/pull/7367 is not compatible with Olmo3 model.
logger.warning( logger.warning(
f"Disabling hybrid SWA memory for {hf_config.architectures[0]} as it is not yet supported." f"Disabling hybrid SWA memory for {hf_config.architectures[0]} as it is not yet supported."
) )
overrides["disable_hybrid_swa_memory"] = True overrides["disable_hybrid_swa_memory"] = True
if server_args.attention_backend is None: if cfg.attention_backend is None:
if is_cuda() and is_sm100_supported(): if is_cuda() and is_sm100_supported():
overrides["attention_backend"] = "trtllm_mha" overrides["attention_backend"] = "trtllm_mha"
elif is_cuda() and get_device_sm() >= 80: elif is_cuda() and get_device_sm() >= 80:
@@ -1695,6 +1737,7 @@ def _olmo2_overrides(server_args: Any, hf_config: Any) -> dict:
or "Step3p7ForConditionalGeneration" in arch or "Step3p7ForConditionalGeneration" in arch
) )
def _step3p_overrides(server_args: Any, hf_config: Any) -> dict: def _step3p_overrides(server_args: Any, hf_config: Any) -> dict:
cfg = resolving_view(server_args)
overrides: Dict[str, Any] = {} overrides: Dict[str, Any] = {}
if server_args.is_attention_backend_not_set(): if server_args.is_attention_backend_not_set():
if is_blackwell_supported(): if is_blackwell_supported():
@@ -1703,12 +1746,12 @@ def _step3p_overrides(server_args: Any, hf_config: Any) -> dict:
elif is_sm90_supported(): elif is_sm90_supported():
logger.info("Auto-select fa3 attention backend for Step3p7 on Hopper.") logger.info("Auto-select fa3 attention backend for Step3p7 on Hopper.")
overrides["attention_backend"] = "fa3" overrides["attention_backend"] = "fa3"
if server_args.speculative_algorithm == "EAGLE": if cfg.speculative_algorithm == "EAGLE":
logger.info( logger.info(
"Enable multi-layer EAGLE speculative decoding for Step3p5ForCausalLM model." "Enable multi-layer EAGLE speculative decoding for Step3p5ForCausalLM model."
) )
overrides["enable_multi_layer_eagle"] = True overrides["enable_multi_layer_eagle"] = True
if server_args.enable_hierarchical_cache: if cfg.enable_hierarchical_cache:
logger.warning( logger.warning(
"Reset swa_full_tokens_ratio to 1.0 for Step3p5ForCausalLM model with hierarchical cache" "Reset swa_full_tokens_ratio to 1.0 for Step3p5ForCausalLM model with hierarchical cache"
) )
@@ -2170,7 +2213,8 @@ def _deepseek_v4_kv_cache_dtype(view: Any) -> dict:
@_register_for("MuseGlimmerForConditionalGeneration", "MuseGlimmerForCausalLM") @_register_for("MuseGlimmerForConditionalGeneration", "MuseGlimmerForCausalLM")
def _muse_glimmer_fp4_gemm_runner_overrides(server_args: Any, hf_config: Any) -> dict: def _muse_glimmer_fp4_gemm_runner_overrides(server_args: Any, hf_config: Any) -> dict:
if is_sm120_supported() and server_args.fp4_gemm_runner_backend == "auto": cfg = resolving_view(server_args)
if is_sm120_supported() and cfg.fp4_gemm_runner_backend == "auto":
logger.info("Use marlin as FP4 GEMM runner backend on SM120 for Muse Glimmer") logger.info("Use marlin as FP4 GEMM runner backend on SM120 for Muse Glimmer")
return {"fp4_gemm_runner_backend": "marlin"} return {"fp4_gemm_runner_backend": "marlin"}
return {} return {}
@@ -5,7 +5,10 @@ import logging
import os import os
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from sglang.srt.arg_groups.overrides import declare_resolution from sglang.srt.arg_groups.overrides import (
declare_resolution,
resolving_view,
)
from sglang.srt.environ import envs from sglang.srt.environ import envs
if TYPE_CHECKING: if TYPE_CHECKING:
@@ -16,10 +19,11 @@ logger = logging.getLogger(__name__)
def handle_pd_disaggregation(server_args: ServerArgs) -> None: def handle_pd_disaggregation(server_args: ServerArgs) -> None:
"""Validate and normalize PD-disaggregation server args.""" """Validate and normalize PD-disaggregation server args."""
cfg = resolving_view(server_args)
# "mooncake_tcp" is mooncake with the TCP transport forced: set MC_FORCE_TCP # "mooncake_tcp" is mooncake with the TCP transport forced: set MC_FORCE_TCP
# so mooncake installs TcpTransport instead of RDMA, rewrite the backend to # so mooncake installs TcpTransport instead of RDMA, rewrite the backend to
# mooncake, and skip RDMA HCA selection. Must run before backend-name checks. # mooncake, and skip RDMA HCA selection. Must run before backend-name checks.
if server_args.disaggregation_transfer_backend == "mooncake_tcp": if cfg.disaggregation_transfer_backend == "mooncake_tcp":
os.environ.setdefault("MC_FORCE_TCP", "1") os.environ.setdefault("MC_FORCE_TCP", "1")
declare_resolution( declare_resolution(
server_args, server_args,
@@ -36,17 +40,17 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None:
"with MC_FORCE_TCP=1 (TCP transport, no RDMA)" "with MC_FORCE_TCP=1 (TCP transport, no RDMA)"
) )
if server_args.disaggregation_mode == "prefill" and server_args.dcp_size > 1: if cfg.disaggregation_mode == "prefill" and cfg.dcp_size > 1:
logger.warning( logger.warning(
"DCP on a PD prefill server is supported when prefill and decode " "DCP on a PD prefill server is supported when prefill and decode "
"use the same DCP layout, but it usually adds communication " "use the same DCP layout, but it usually adds communication "
"overhead without improving prefill performance." "overhead without improving prefill performance."
) )
if server_args.disaggregation_mode == "decode" and server_args.dcp_size > 1: if cfg.disaggregation_mode == "decode" and cfg.dcp_size > 1:
# Fake transfer moves no KV and is only used for synthetic decode # Fake transfer moves no KV and is only used for synthetic decode
# benchmarks, so it does not need the DCP relayout from Mooncake/NIXL. # benchmarks, so it does not need the DCP relayout from Mooncake/NIXL.
if server_args.disaggregation_transfer_backend not in ( if cfg.disaggregation_transfer_backend not in (
"mooncake", "mooncake",
"nixl", "nixl",
"fake", "fake",
@@ -54,36 +58,36 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None:
raise ValueError( raise ValueError(
"PD decode DCP requires --disaggregation-transfer-backend " "PD decode DCP requires --disaggregation-transfer-backend "
"mooncake, nixl, or fake for synthetic benchmarking, got " "mooncake, nixl, or fake for synthetic benchmarking, got "
f"{server_args.disaggregation_transfer_backend!r}." f"{cfg.disaggregation_transfer_backend!r}."
) )
if server_args.disaggregation_decode_enable_radix_cache: if cfg.disaggregation_decode_enable_radix_cache:
raise ValueError( raise ValueError(
"PD decode DCP currently requires chunk cache; " "PD decode DCP currently requires chunk cache; "
"--disaggregation-decode-enable-radix-cache is not supported." "--disaggregation-decode-enable-radix-cache is not supported."
) )
if server_args.enable_hierarchical_cache: if cfg.enable_hierarchical_cache:
raise ValueError( raise ValueError(
"PD decode DCP currently requires chunk cache; " "PD decode DCP currently requires chunk cache; "
"--enable-hierarchical-cache is not supported." "--enable-hierarchical-cache is not supported."
) )
if server_args.disaggregation_mode == "decode": if cfg.disaggregation_mode == "decode":
if server_args.disaggregation_decode_enable_radix_cache: if cfg.disaggregation_decode_enable_radix_cache:
if server_args.enable_hisparse: if cfg.enable_hisparse:
raise ValueError( raise ValueError(
"--disaggregation-decode-enable-radix-cache is incompatible " "--disaggregation-decode-enable-radix-cache is incompatible "
"with --enable-hisparse" "with --enable-hisparse"
) )
if server_args.disaggregation_transfer_backend == "fake": if cfg.disaggregation_transfer_backend == "fake":
raise ValueError( raise ValueError(
"--disaggregation-decode-enable-radix-cache is incompatible " "--disaggregation-decode-enable-radix-cache is incompatible "
"with --disaggregation-transfer-backend fake" "with --disaggregation-transfer-backend fake"
) )
if server_args.speculative_algorithm is not None: if cfg.speculative_algorithm is not None:
raise ValueError( raise ValueError(
"--disaggregation-decode-enable-radix-cache is incompatible " "--disaggregation-decode-enable-radix-cache is incompatible "
"with speculative decoding " "with speculative decoding "
f"(--speculative-algorithm {server_args.speculative_algorithm})" f"(--speculative-algorithm {cfg.speculative_algorithm})"
) )
from sglang.srt.arg_groups.overrides import resolved_view from sglang.srt.arg_groups.overrides import resolved_view
@@ -110,12 +114,10 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None:
# in-transfer (being-received-from-prefill) requests, on top of the # in-transfer (being-received-from-prefill) requests, on top of the
# max_running_requests-derived pool. Large batches get none; small # max_running_requests-derived pool. Large batches get none; small
# per-worker batches reserve 2x the batch as cheap overlap headroom. # per-worker batches reserve 2x the batch as cheap overlap headroom.
if server_args.disaggregation_decode_extra_slots is None: if cfg.disaggregation_decode_extra_slots is None:
extra_slots = 0 extra_slots = 0
if server_args.max_running_requests is not None: if cfg.max_running_requests is not None:
per_worker = server_args.max_running_requests // max( per_worker = cfg.max_running_requests // max(1, cfg.dp_size)
1, server_args.dp_size
)
if per_worker <= 32: if per_worker <= 32:
extra_slots = per_worker * 2 extra_slots = per_worker * 2
declare_resolution( declare_resolution(
@@ -124,23 +126,23 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None:
disaggregation_decode_extra_slots=extra_slots, disaggregation_decode_extra_slots=extra_slots,
) )
elif server_args.disaggregation_mode == "prefill": elif cfg.disaggregation_mode == "prefill":
assert ( assert (
server_args.disaggregation_transfer_backend != "fake" cfg.disaggregation_transfer_backend != "fake"
), "Prefill server does not support 'fake' as the transfer backend" ), "Prefill server does not support 'fake' as the transfer backend"
if envs.SGLANG_RUST_SERVER.get(): if envs.SGLANG_RUST_SERVER.get():
_alias_bootstrap_port_to_api_port(server_args) _alias_bootstrap_port_to_api_port(server_args)
if server_args.disaggregation_mode in ("prefill", "decode"): if cfg.disaggregation_mode in ("prefill", "decode"):
if ( if (
envs.SGLANG_DISAGG_STAGING_BUFFER.get() envs.SGLANG_DISAGG_STAGING_BUFFER.get()
and server_args.disaggregation_transfer_backend not in ("mooncake", "nixl") and cfg.disaggregation_transfer_backend not in ("mooncake", "nixl")
): ):
raise ValueError( raise ValueError(
f"SGLANG_DISAGG_STAGING_BUFFER requires " f"SGLANG_DISAGG_STAGING_BUFFER requires "
f"disaggregation_transfer_backend='mooncake' or 'nixl', " f"disaggregation_transfer_backend='mooncake' or 'nixl', "
f"got '{server_args.disaggregation_transfer_backend}'." f"got '{cfg.disaggregation_transfer_backend}'."
) )
@@ -151,31 +153,32 @@ def _alias_bootstrap_port_to_api_port(server_args: ServerArgs) -> None:
field and agrees automatically. Decode is untouched: there the field names field and agrees automatically. Decode is untouched: there the field names
the PREFILL side's bootstrap port and must stay as the operator set it. the PREFILL side's bootstrap port and must stay as the operator set it.
""" """
cfg = resolving_view(server_args)
default_port = next( default_port = next(
f.default f.default
for f in dataclasses.fields(server_args) for f in dataclasses.fields(server_args)
if f.name == "disaggregation_bootstrap_port" if f.name == "disaggregation_bootstrap_port"
) )
if server_args.disaggregation_bootstrap_port not in ( if cfg.disaggregation_bootstrap_port not in (
default_port, default_port,
server_args.port, cfg.port,
): ):
raise ValueError( raise ValueError(
"SGLANG_RUST_SERVER serves the PD KV bootstrap registry on the api " "SGLANG_RUST_SERVER serves the PD KV bootstrap registry on the api "
"port itself; --disaggregation-bootstrap-port " "port itself; --disaggregation-bootstrap-port "
f"{server_args.disaggregation_bootstrap_port} conflicts with --port " f"{cfg.disaggregation_bootstrap_port} conflicts with --port "
f"{server_args.port}. Drop --disaggregation-bootstrap-port (decode " f"{cfg.port}. Drop --disaggregation-bootstrap-port (decode "
"nodes and the PD router must then target the prefill api port)." "nodes and the PD router must then target the prefill api port)."
) )
if server_args.disaggregation_bootstrap_port != server_args.port: if cfg.disaggregation_bootstrap_port != cfg.port:
logger.info( logger.info(
"SGLANG_RUST_SERVER: KV bootstrap registry is served on the api " "SGLANG_RUST_SERVER: KV bootstrap registry is served on the api "
"port; disaggregation_bootstrap_port %d -> %d", "port; disaggregation_bootstrap_port %d -> %d",
server_args.disaggregation_bootstrap_port, cfg.disaggregation_bootstrap_port,
server_args.port, cfg.port,
) )
declare_resolution( declare_resolution(
server_args, server_args,
"_alias_bootstrap_port_to_api_port", "_alias_bootstrap_port_to_api_port",
disaggregation_bootstrap_port=server_args.port, disaggregation_bootstrap_port=cfg.port,
) )
+151 -147
View File
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Optional
from sglang.srt.arg_groups.overrides import ( from sglang.srt.arg_groups.overrides import (
declare_direct_writes, declare_direct_writes,
declare_resolution, declare_resolution,
resolving_view,
) )
if TYPE_CHECKING: if TYPE_CHECKING:
@@ -17,7 +18,8 @@ logger = logging.getLogger(__name__)
def _disable_overlap_schedule_for_cpu(server_args: ServerArgs) -> None: def _disable_overlap_schedule_for_cpu(server_args: ServerArgs) -> None:
if server_args.device != "cpu" or server_args.disable_overlap_schedule: cfg = resolving_view(server_args)
if cfg.device != "cpu" or cfg.disable_overlap_schedule:
return return
declare_resolution( declare_resolution(
@@ -71,9 +73,10 @@ def _resolve_speculative_algorithm_alias(
def handle_speculative_decoding(server_args: ServerArgs) -> None: def handle_speculative_decoding(server_args: ServerArgs) -> None:
cfg = resolving_view(server_args)
if ( if (
server_args.speculative_draft_model_path is not None cfg.speculative_draft_model_path is not None
and server_args.speculative_draft_model_revision is None and cfg.speculative_draft_model_revision is None
): ):
declare_resolution( declare_resolution(
server_args, server_args,
@@ -90,11 +93,11 @@ def handle_speculative_decoding(server_args: ServerArgs) -> None:
run_post_process_pass(server_args, _speculative_moe_runner_default) run_post_process_pass(server_args, _speculative_moe_runner_default)
if server_args.speculative_algorithm is not None: if cfg.speculative_algorithm is not None:
declare_resolution( declare_resolution(
server_args, server_args,
"handle_speculative_decoding", "handle_speculative_decoding",
speculative_algorithm=server_args.speculative_algorithm.upper(), speculative_algorithm=cfg.speculative_algorithm.upper(),
) )
# Removal notice for the retired env var; raw os.getenv on purpose -- the # Removal notice for the retired env var; raw os.getenv on purpose -- the
@@ -108,7 +111,7 @@ def handle_speculative_decoding(server_args: ServerArgs) -> None:
kwargs = {} kwargs = {}
override_config_file = server_args.decrypted_draft_config_file override_config_file = cfg.decrypted_draft_config_file
if override_config_file and override_config_file.strip(): if override_config_file and override_config_file.strip():
kwargs["_configuration_file"] = override_config_file.strip() kwargs["_configuration_file"] = override_config_file.strip()
@@ -116,17 +119,17 @@ def handle_speculative_decoding(server_args: ServerArgs) -> None:
server_args, server_args,
"handle_speculative_decoding", "handle_speculative_decoding",
speculative_algorithm=_resolve_speculative_algorithm_alias( speculative_algorithm=_resolve_speculative_algorithm_alias(
server_args.speculative_algorithm, cfg.speculative_algorithm,
server_args.speculative_draft_model_path, cfg.speculative_draft_model_path,
trust_remote_code=server_args.trust_remote_code, trust_remote_code=cfg.trust_remote_code,
kwargs=kwargs, kwargs=kwargs,
), ),
) )
# Validate --speculative-draft-window-size once, regardless of algorithm. # Validate --speculative-draft-window-size once, regardless of algorithm.
# Consumed by DFLASH (compact draft KV cache) and Llama EAGLE-3 (drafter attention SWA). # Consumed by DFLASH (compact draft KV cache) and Llama EAGLE-3 (drafter attention SWA).
if server_args.speculative_draft_window_size is not None: if cfg.speculative_draft_window_size is not None:
window_size = int(server_args.speculative_draft_window_size) window_size = int(cfg.speculative_draft_window_size)
if window_size <= 0: if window_size <= 0:
raise ValueError( raise ValueError(
f"--speculative-draft-window-size must be positive, got {window_size}." f"--speculative-draft-window-size must be positive, got {window_size}."
@@ -136,19 +139,19 @@ def handle_speculative_decoding(server_args: ServerArgs) -> None:
"handle_speculative_decoding", "handle_speculative_decoding",
speculative_draft_window_size=window_size, speculative_draft_window_size=window_size,
) )
if server_args.speculative_algorithm not in ("EAGLE3", "DFLASH"): if cfg.speculative_algorithm not in ("EAGLE3", "DFLASH"):
logger.warning( logger.warning(
"--speculative-draft-window-size has no effect with " "--speculative-draft-window-size has no effect with "
"speculative_algorithm=%s (honored by Llama EAGLE-3 and DFLASH only).", "speculative_algorithm=%s (honored by Llama EAGLE-3 and DFLASH only).",
server_args.speculative_algorithm, cfg.speculative_algorithm,
) )
algo = None algo = None
if server_args.speculative_algorithm is not None: if cfg.speculative_algorithm is not None:
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.srt.speculative.spec_registry import CustomSpecAlgo from sglang.srt.speculative.spec_registry import CustomSpecAlgo
algo = SpeculativeAlgorithm.from_string(server_args.speculative_algorithm) algo = SpeculativeAlgorithm.from_string(cfg.speculative_algorithm)
# TODO: move the per-algorithm validation below into spec module hooks. # TODO: move the per-algorithm validation below into spec module hooks.
if isinstance(algo, CustomSpecAlgo) and algo.validate_server_args is not None: if isinstance(algo, CustomSpecAlgo) and algo.validate_server_args is not None:
@@ -158,15 +161,15 @@ def handle_speculative_decoding(server_args: ServerArgs) -> None:
algo.validate_server_args, algo.validate_server_args,
) )
if server_args.speculative_skip_dp_mlp_sync: if cfg.speculative_skip_dp_mlp_sync:
assert server_args.speculative_algorithm == "EAGLE", ( assert cfg.speculative_algorithm == "EAGLE", (
"--speculative-skip-dp-mlp-sync is only supported with " "--speculative-skip-dp-mlp-sync is only supported with "
f"speculative_algorithm == EAGLE, got {server_args.speculative_algorithm}." f"speculative_algorithm == EAGLE, got {cfg.speculative_algorithm}."
) )
if server_args.speculative_adaptive: if cfg.speculative_adaptive:
_maybe_disable_adaptive(server_args) _maybe_disable_adaptive(server_args)
if server_args.speculative_adaptive: if cfg.speculative_adaptive:
_init_adaptive_speculative_params(server_args) _init_adaptive_speculative_params(server_args)
if algo is not None: if algo is not None:
@@ -180,9 +183,10 @@ def handle_speculative_decoding(server_args: ServerArgs) -> None:
def _handle_dflash(server_args: ServerArgs) -> None: def _handle_dflash(server_args: ServerArgs) -> None:
cfg = resolving_view(server_args)
from sglang.srt.arg_groups.overrides import resolved_view from sglang.srt.arg_groups.overrides import resolved_view
if not (server_args.device.startswith("cuda") or server_args.device == "npu"): if not (cfg.device.startswith("cuda") or cfg.device == "npu"):
raise ValueError( raise ValueError(
"DFLASH speculative decoding only supports CUDA and NPU devices." "DFLASH speculative decoding only supports CUDA and NPU devices."
) )
@@ -192,12 +196,12 @@ def _handle_dflash(server_args: ServerArgs) -> None:
"Currently DFLASH speculative decoding does not support dp attention." "Currently DFLASH speculative decoding does not support dp attention."
) )
if server_args.pp_size != 1: if cfg.pp_size != 1:
raise ValueError( raise ValueError(
"Currently DFLASH speculative decoding only supports pp_size == 1." "Currently DFLASH speculative decoding only supports pp_size == 1."
) )
if server_args.speculative_draft_model_path is None: if cfg.speculative_draft_model_path is None:
raise ValueError( raise ValueError(
"DFLASH speculative decoding requires setting --speculative-draft-model-path." "DFLASH speculative decoding requires setting --speculative-draft-model-path."
) )
@@ -207,16 +211,16 @@ def _handle_dflash(server_args: ServerArgs) -> None:
# RoPE reservation). Force them to 1 to avoid surprising memory behavior. # RoPE reservation). Force them to 1 to avoid surprising memory behavior.
# #
# For DFlash, the natural unit is `block_size` (verify window length). # For DFlash, the natural unit is `block_size` (verify window length).
if server_args.speculative_num_steps is None: if cfg.speculative_num_steps is None:
declare_resolution( declare_resolution(
server_args, server_args,
"_handle_dflash", "_handle_dflash",
speculative_num_steps=1, speculative_num_steps=1,
) )
elif int(server_args.speculative_num_steps) != 1: elif int(cfg.speculative_num_steps) != 1:
logger.warning( logger.warning(
"DFLASH only supports speculative_num_steps == 1; overriding speculative_num_steps=%s to 1.", "DFLASH only supports speculative_num_steps == 1; overriding speculative_num_steps=%s to 1.",
server_args.speculative_num_steps, cfg.speculative_num_steps,
) )
declare_resolution( declare_resolution(
server_args, server_args,
@@ -224,16 +228,16 @@ def _handle_dflash(server_args: ServerArgs) -> None:
speculative_num_steps=1, speculative_num_steps=1,
) )
if server_args.speculative_eagle_topk is None: if cfg.speculative_eagle_topk is None:
declare_resolution( declare_resolution(
server_args, server_args,
"_handle_dflash", "_handle_dflash",
speculative_eagle_topk=1, speculative_eagle_topk=1,
) )
elif int(server_args.speculative_eagle_topk) != 1: elif int(cfg.speculative_eagle_topk) != 1:
logger.warning( logger.warning(
"DFLASH only supports speculative_eagle_topk == 1; overriding speculative_eagle_topk=%s to 1.", "DFLASH only supports speculative_eagle_topk == 1; overriding speculative_eagle_topk=%s to 1.",
server_args.speculative_eagle_topk, cfg.speculative_eagle_topk,
) )
declare_resolution( declare_resolution(
server_args, server_args,
@@ -241,41 +245,41 @@ def _handle_dflash(server_args: ServerArgs) -> None:
speculative_eagle_topk=1, speculative_eagle_topk=1,
) )
if server_args.speculative_dflash_block_size is not None: if cfg.speculative_dflash_block_size is not None:
if int(server_args.speculative_dflash_block_size) <= 0: if int(cfg.speculative_dflash_block_size) <= 0:
raise ValueError( raise ValueError(
"DFLASH requires --speculative-dflash-block-size to be positive, " "DFLASH requires --speculative-dflash-block-size to be positive, "
f"got {server_args.speculative_dflash_block_size}." f"got {cfg.speculative_dflash_block_size}."
) )
if server_args.speculative_num_draft_tokens is not None and int( if cfg.speculative_num_draft_tokens is not None and int(
server_args.speculative_num_draft_tokens cfg.speculative_num_draft_tokens
) != int(server_args.speculative_dflash_block_size): ) != int(cfg.speculative_dflash_block_size):
raise ValueError( raise ValueError(
"Both --speculative-num-draft-tokens and --speculative-dflash-block-size are set " "Both --speculative-num-draft-tokens and --speculative-dflash-block-size are set "
"but they differ. For DFLASH they must match. " "but they differ. For DFLASH they must match. "
f"speculative_num_draft_tokens={server_args.speculative_num_draft_tokens}, " f"speculative_num_draft_tokens={cfg.speculative_num_draft_tokens}, "
f"speculative_dflash_block_size={server_args.speculative_dflash_block_size}." f"speculative_dflash_block_size={cfg.speculative_dflash_block_size}."
) )
declare_resolution( declare_resolution(
server_args, server_args,
"_handle_dflash", "_handle_dflash",
speculative_num_draft_tokens=int(server_args.speculative_dflash_block_size), speculative_num_draft_tokens=int(cfg.speculative_dflash_block_size),
) )
if server_args.speculative_num_draft_tokens is None: if cfg.speculative_num_draft_tokens is None:
from sglang.srt.speculative.dflash_utils import ( from sglang.srt.speculative.dflash_utils import (
parse_dflash_draft_config, parse_dflash_draft_config,
) )
model_override_args = json.loads(server_args.json_model_override_args) model_override_args = json.loads(cfg.json_model_override_args)
inferred_block_size = None inferred_block_size = None
try: try:
from sglang.srt.utils.hf_transformers_utils import get_config from sglang.srt.utils.hf_transformers_utils import get_config
draft_hf_config = get_config( draft_hf_config = get_config(
server_args.speculative_draft_model_path, cfg.speculative_draft_model_path,
trust_remote_code=server_args.trust_remote_code, trust_remote_code=cfg.trust_remote_code,
revision=server_args.speculative_draft_model_revision, revision=cfg.speculative_draft_model_revision,
model_override_args=model_override_args, model_override_args=model_override_args,
) )
inferred_block_size = parse_dflash_draft_config( inferred_block_size = parse_dflash_draft_config(
@@ -300,18 +304,18 @@ def _handle_dflash(server_args: ServerArgs) -> None:
speculative_num_draft_tokens=inferred_block_size, speculative_num_draft_tokens=inferred_block_size,
) )
if server_args.speculative_draft_window_size is not None: if cfg.speculative_draft_window_size is not None:
draft_tokens = int(server_args.speculative_num_draft_tokens) draft_tokens = int(cfg.speculative_num_draft_tokens)
if server_args.speculative_draft_window_size < draft_tokens: if cfg.speculative_draft_window_size < draft_tokens:
raise ValueError( raise ValueError(
"--speculative-draft-window-size must be >= " "--speculative-draft-window-size must be >= "
"--speculative-num-draft-tokens (block_size). " "--speculative-num-draft-tokens (block_size). "
f"window_size={server_args.speculative_draft_window_size}, block_size={draft_tokens}." f"window_size={cfg.speculative_draft_window_size}, block_size={draft_tokens}."
) )
_resolve_dflash_draft_attention_backend(server_args) _resolve_dflash_draft_attention_backend(server_args)
if server_args.max_running_requests is None: if cfg.max_running_requests is None:
declare_resolution( declare_resolution(
server_args, server_args,
"_handle_dflash", "_handle_dflash",
@@ -321,7 +325,7 @@ def _handle_dflash(server_args: ServerArgs) -> None:
"Max running requests is reset to 48 for speculative decoding. You can override this by explicitly setting --max-running-requests." "Max running requests is reset to 48 for speculative decoding. You can override this by explicitly setting --max-running-requests."
) )
if server_args.enable_mixed_chunk: if cfg.enable_mixed_chunk:
declare_resolution( declare_resolution(
server_args, server_args,
"_handle_dflash", "_handle_dflash",
@@ -341,23 +345,24 @@ def _target_checkpoint_bundles_dspark_draft(server_args: ServerArgs) -> bool:
def _handle_dspark(server_args: ServerArgs) -> None: def _handle_dspark(server_args: ServerArgs) -> None:
_is_npu = server_args.device.startswith("npu") cfg = resolving_view(server_args)
if not server_args.device.startswith(("cuda", "npu")): _is_npu = cfg.device.startswith("npu")
if not cfg.device.startswith(("cuda", "npu")):
raise ValueError( raise ValueError(
"DSpark speculative decoding only supports CUDA or NPU device." "DSpark speculative decoding only supports CUDA or NPU device."
) )
# dp_size==1 with dp_attention is a degenerate flag under DSV4 CP; skip DP-only checks. # dp_size==1 with dp_attention is a degenerate flag under DSV4 CP; skip DP-only checks.
if server_args.enable_dp_attention and server_args.dp_size > 1: if cfg.enable_dp_attention and cfg.dp_size > 1:
if not server_args.enable_dp_lm_head: if not cfg.enable_dp_lm_head:
raise ValueError("DSpark with dp attention requires --enable-dp-lm-head.") raise ValueError("DSpark with dp attention requires --enable-dp-lm-head.")
if not _is_npu and server_args.moe_a2a_backend not in ("none", "megamoe"): if not _is_npu and cfg.moe_a2a_backend not in ("none", "megamoe"):
raise ValueError( raise ValueError(
"DSpark with dp attention supports moe_a2a_backend 'none' " "DSpark with dp attention supports moe_a2a_backend 'none' "
"(built-in TP MoE) or 'megamoe', got " "(built-in TP MoE) or 'megamoe', got "
f"{server_args.moe_a2a_backend!r}." f"{cfg.moe_a2a_backend!r}."
) )
if not _is_npu and server_args.moe_a2a_backend != "none": if not _is_npu and cfg.moe_a2a_backend != "none":
from sglang.srt.speculative.ragged_verify import ( from sglang.srt.speculative.ragged_verify import (
RaggedVerifyMode, RaggedVerifyMode,
read_ragged_verify_mode, read_ragged_verify_mode,
@@ -366,46 +371,46 @@ def _handle_dspark(server_args: ServerArgs) -> None:
if read_ragged_verify_mode() is not RaggedVerifyMode.STATIC: if read_ragged_verify_mode() is not RaggedVerifyMode.STATIC:
raise ValueError( raise ValueError(
"DSpark with dp attention + " "DSpark with dp attention + "
f"moe_a2a_backend={server_args.moe_a2a_backend!r} requires " f"moe_a2a_backend={cfg.moe_a2a_backend!r} requires "
"SGLANG_RAGGED_VERIFY_MODE=static." "SGLANG_RAGGED_VERIFY_MODE=static."
) )
if server_args.attn_cp_size > 1: if cfg.attn_cp_size > 1:
raise ValueError( raise ValueError(
"DSpark with dp attention does not support context parallel " "DSpark with dp attention does not support context parallel "
f"(attn_cp_size={server_args.attn_cp_size})." f"(attn_cp_size={cfg.attn_cp_size})."
) )
if ( if (
not _is_npu not _is_npu
and server_args.speculative_moe_a2a_backend is not None and cfg.speculative_moe_a2a_backend is not None
and server_args.speculative_moe_a2a_backend != server_args.moe_a2a_backend and cfg.speculative_moe_a2a_backend != cfg.moe_a2a_backend
): ):
raise ValueError( raise ValueError(
"DSpark ignores --speculative-moe-a2a-backend; with dp attention it " "DSpark ignores --speculative-moe-a2a-backend; with dp attention it "
f"must match the target moe_a2a_backend={server_args.moe_a2a_backend!r} " f"must match the target moe_a2a_backend={cfg.moe_a2a_backend!r} "
f"(got {server_args.speculative_moe_a2a_backend!r})." f"(got {cfg.speculative_moe_a2a_backend!r})."
) )
if server_args.pp_size != 1: if cfg.pp_size != 1:
raise ValueError( raise ValueError(
"Currently DSpark speculative decoding only supports pp_size == 1." "Currently DSpark speculative decoding only supports pp_size == 1."
) )
if server_args.speculative_draft_model_path is None: if cfg.speculative_draft_model_path is None:
if _target_checkpoint_bundles_dspark_draft(server_args): if _target_checkpoint_bundles_dspark_draft(server_args):
declare_resolution( declare_resolution(
server_args, server_args,
"_handle_dspark", "_handle_dspark",
speculative_draft_model_path=server_args.model_path, speculative_draft_model_path=cfg.model_path,
) )
declare_resolution( declare_resolution(
server_args, server_args,
"_handle_dspark", "_handle_dspark",
speculative_draft_model_revision=server_args.revision, speculative_draft_model_revision=cfg.revision,
) )
logger.info( logger.info(
"DSpark draft weights are bundled in the target checkpoint; " "DSpark draft weights are bundled in the target checkpoint; "
"defaulting --speculative-draft-model-path to --model-path (%s).", "defaulting --speculative-draft-model-path to --model-path (%s).",
server_args.model_path, cfg.model_path,
) )
else: else:
raise ValueError( raise ValueError(
@@ -413,16 +418,16 @@ def _handle_dspark(server_args: ServerArgs) -> None:
"--speculative-draft-model-path." "--speculative-draft-model-path."
) )
if server_args.speculative_num_steps is None: if cfg.speculative_num_steps is None:
declare_resolution( declare_resolution(
server_args, server_args,
"_handle_dspark", "_handle_dspark",
speculative_num_steps=1, speculative_num_steps=1,
) )
elif int(server_args.speculative_num_steps) != 1: elif int(cfg.speculative_num_steps) != 1:
logger.warning( logger.warning(
"DSpark only supports speculative_num_steps == 1; overriding speculative_num_steps=%s to 1.", "DSpark only supports speculative_num_steps == 1; overriding speculative_num_steps=%s to 1.",
server_args.speculative_num_steps, cfg.speculative_num_steps,
) )
declare_resolution( declare_resolution(
server_args, server_args,
@@ -430,16 +435,16 @@ def _handle_dspark(server_args: ServerArgs) -> None:
speculative_num_steps=1, speculative_num_steps=1,
) )
if server_args.speculative_eagle_topk is None: if cfg.speculative_eagle_topk is None:
declare_resolution( declare_resolution(
server_args, server_args,
"_handle_dspark", "_handle_dspark",
speculative_eagle_topk=1, speculative_eagle_topk=1,
) )
elif int(server_args.speculative_eagle_topk) != 1: elif int(cfg.speculative_eagle_topk) != 1:
logger.warning( logger.warning(
"DSpark only supports speculative_eagle_topk == 1; overriding speculative_eagle_topk=%s to 1.", "DSpark only supports speculative_eagle_topk == 1; overriding speculative_eagle_topk=%s to 1.",
server_args.speculative_eagle_topk, cfg.speculative_eagle_topk,
) )
declare_resolution( declare_resolution(
server_args, server_args,
@@ -463,17 +468,17 @@ def _handle_dspark(server_args: ServerArgs) -> None:
) )
gamma: Optional[int] = None gamma: Optional[int] = None
if server_args.speculative_dspark_block_size is not None: if cfg.speculative_dspark_block_size is not None:
if int(server_args.speculative_dspark_block_size) <= 0: if int(cfg.speculative_dspark_block_size) <= 0:
raise ValueError( raise ValueError(
"DSpark requires --speculative-dspark-block-size to be positive, " "DSpark requires --speculative-dspark-block-size to be positive, "
f"got {server_args.speculative_dspark_block_size}." f"got {cfg.speculative_dspark_block_size}."
) )
gamma = int(server_args.speculative_dspark_block_size) gamma = int(cfg.speculative_dspark_block_size)
else: else:
if draft_config is not None: if draft_config is not None:
gamma = draft_config.resolve_gamma(default=None) gamma = draft_config.resolve_gamma(default=None)
if gamma is None and server_args.speculative_num_draft_tokens is None: if gamma is None and cfg.speculative_num_draft_tokens is None:
gamma = DEFAULT_DSPARK_GAMMA gamma = DEFAULT_DSPARK_GAMMA
logger.warning( logger.warning(
"DSpark gamma is not set; defaulting to %d.", "DSpark gamma is not set; defaulting to %d.",
@@ -483,13 +488,13 @@ def _handle_dspark(server_args: ServerArgs) -> None:
if gamma is not None: if gamma is not None:
verify_window = int(gamma) + 1 verify_window = int(gamma) + 1
if ( if (
server_args.speculative_num_draft_tokens is not None cfg.speculative_num_draft_tokens is not None
and int(server_args.speculative_num_draft_tokens) != verify_window and int(cfg.speculative_num_draft_tokens) != verify_window
): ):
raise ValueError( raise ValueError(
"DSpark speculative_num_draft_tokens must equal gamma + 1 " "DSpark speculative_num_draft_tokens must equal gamma + 1 "
f"(= {verify_window} for gamma={gamma}), but got " f"(= {verify_window} for gamma={gamma}), but got "
f"speculative_num_draft_tokens={server_args.speculative_num_draft_tokens}." f"speculative_num_draft_tokens={cfg.speculative_num_draft_tokens}."
) )
declare_resolution( declare_resolution(
server_args, server_args,
@@ -497,18 +502,18 @@ def _handle_dspark(server_args: ServerArgs) -> None:
speculative_num_draft_tokens=verify_window, speculative_num_draft_tokens=verify_window,
) )
if server_args.speculative_num_draft_tokens is None: if cfg.speculative_num_draft_tokens is None:
raise ValueError( raise ValueError(
"DSpark could not resolve speculative_num_draft_tokens; set " "DSpark could not resolve speculative_num_draft_tokens; set "
"--speculative-dspark-block-size (= gamma)." "--speculative-dspark-block-size (= gamma)."
) )
if int(server_args.speculative_num_draft_tokens) < 2: if int(cfg.speculative_num_draft_tokens) < 2:
raise ValueError( raise ValueError(
"DSpark speculative_num_draft_tokens must be >= 2 (= gamma + 1), " "DSpark speculative_num_draft_tokens must be >= 2 (= gamma + 1), "
f"got {server_args.speculative_num_draft_tokens}." f"got {cfg.speculative_num_draft_tokens}."
) )
if server_args.max_running_requests is None: if cfg.max_running_requests is None:
declare_resolution( declare_resolution(
server_args, server_args,
"_handle_dspark", "_handle_dspark",
@@ -518,7 +523,7 @@ def _handle_dspark(server_args: ServerArgs) -> None:
"Max running requests is reset to 48 for speculative decoding. You can override this by explicitly setting --max-running-requests." "Max running requests is reset to 48 for speculative decoding. You can override this by explicitly setting --max-running-requests."
) )
if server_args.enable_mixed_chunk: if cfg.enable_mixed_chunk:
declare_resolution( declare_resolution(
server_args, server_args,
"_handle_dspark", "_handle_dspark",
@@ -535,7 +540,7 @@ def _handle_dspark(server_args: ServerArgs) -> None:
ragged_mode = read_ragged_verify_mode() ragged_mode = read_ragged_verify_mode()
if ( if (
server_args.speculative_dspark_align_verify_tokens_to_graph_tier cfg.speculative_dspark_align_verify_tokens_to_graph_tier
and ragged_mode is not RaggedVerifyMode.COMPACT and ragged_mode is not RaggedVerifyMode.COMPACT
): ):
logger.warning( logger.warning(
@@ -544,10 +549,7 @@ def _handle_dspark(server_args: ServerArgs) -> None:
"a no-op.", "a no-op.",
ragged_mode.value, ragged_mode.value,
) )
if ( if cfg.speculative_dspark_sps_table_path and ragged_mode is RaggedVerifyMode.STATIC:
server_args.speculative_dspark_sps_table_path
and ragged_mode is RaggedVerifyMode.STATIC
):
logger.warning( logger.warning(
"--speculative-dspark-sps-table-path feeds the ragged-verify budget " "--speculative-dspark-sps-table-path feeds the ragged-verify budget "
"scheduler, which is off under SGLANG_RAGGED_VERIFY_MODE=static; it " "scheduler, which is off under SGLANG_RAGGED_VERIFY_MODE=static; it "
@@ -561,6 +563,7 @@ def _resolve_dflash_draft_attention_backend(server_args: ServerArgs) -> None:
Consumed by ModelRunner's `is_draft_worker` override (one backend for all Consumed by ModelRunner's `is_draft_worker` override (one backend for all
draft modes). draft modes).
""" """
cfg = resolving_view(server_args)
from sglang.srt.utils import is_hip from sglang.srt.utils import is_hip
supported_draft_backends = ( supported_draft_backends = (
@@ -574,7 +577,7 @@ def _resolve_dflash_draft_attention_backend(server_args: ServerArgs) -> None:
# Use triton on ROCm (no FlashInfer), flashinfer on CUDA. # Use triton on ROCm (no FlashInfer), flashinfer on CUDA.
fallback_backend = "triton" if is_hip() else "flashinfer" fallback_backend = "triton" if is_hip() else "flashinfer"
draft_backend = server_args.speculative_draft_attention_backend draft_backend = cfg.speculative_draft_attention_backend
if draft_backend is None: if draft_backend is None:
from sglang.srt.arg_groups.overrides import ( from sglang.srt.arg_groups.overrides import (
attention_backends_of, attention_backends_of,
@@ -589,10 +592,10 @@ def _resolve_dflash_draft_attention_backend(server_args: ServerArgs) -> None:
from sglang.srt.utils.hf_transformers_utils import get_config from sglang.srt.utils.hf_transformers_utils import get_config
draft_hf_config = get_config( draft_hf_config = get_config(
server_args.speculative_draft_model_path, cfg.speculative_draft_model_path,
trust_remote_code=server_args.trust_remote_code, trust_remote_code=cfg.trust_remote_code,
revision=server_args.speculative_draft_model_revision, revision=cfg.speculative_draft_model_revision,
model_override_args=json.loads(server_args.json_model_override_args), model_override_args=json.loads(cfg.json_model_override_args),
) )
draft_text_config = ( draft_text_config = (
getattr(draft_hf_config, "text_config", None) or draft_hf_config getattr(draft_hf_config, "text_config", None) or draft_hf_config
@@ -635,7 +638,8 @@ def _resolve_dflash_draft_attention_backend(server_args: ServerArgs) -> None:
def _handle_frozen_kv_mtp(server_args: ServerArgs) -> None: def _handle_frozen_kv_mtp(server_args: ServerArgs) -> None:
if server_args.max_running_requests is None: cfg = resolving_view(server_args)
if cfg.max_running_requests is None:
declare_resolution( declare_resolution(
server_args, server_args,
"_handle_frozen_kv_mtp", "_handle_frozen_kv_mtp",
@@ -645,7 +649,7 @@ def _handle_frozen_kv_mtp(server_args: ServerArgs) -> None:
"Max running requests is reset to 48 for speculative decoding. You can override this by explicitly setting --max-running-requests." "Max running requests is reset to 48 for speculative decoding. You can override this by explicitly setting --max-running-requests."
) )
if server_args.enable_mixed_chunk: if cfg.enable_mixed_chunk:
declare_resolution( declare_resolution(
server_args, server_args,
"_handle_frozen_kv_mtp", "_handle_frozen_kv_mtp",
@@ -658,13 +662,14 @@ def _handle_frozen_kv_mtp(server_args: ServerArgs) -> None:
def _handle_eagle_family(server_args: ServerArgs) -> None: def _handle_eagle_family(server_args: ServerArgs) -> None:
cfg = resolving_view(server_args)
from sglang.srt.arg_groups.overrides import ( from sglang.srt.arg_groups.overrides import (
attention_backends_of, attention_backends_of,
resolved_view, resolved_view,
) )
if ( if (
server_args.speculative_algorithm == "STANDALONE" cfg.speculative_algorithm == "STANDALONE"
and resolved_view(server_args).enable_dp_attention and resolved_view(server_args).enable_dp_attention
): ):
# TODO: support dp attention for standalone speculative decoding # TODO: support dp attention for standalone speculative decoding
@@ -672,7 +677,7 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
"Currently standalone speculative decoding does not support dp attention." "Currently standalone speculative decoding does not support dp attention."
) )
if server_args.max_running_requests is None: if cfg.max_running_requests is None:
declare_resolution( declare_resolution(
server_args, server_args,
"_handle_eagle_family", "_handle_eagle_family",
@@ -690,7 +695,7 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
"speculative decoding." "speculative decoding."
) )
if server_args.enable_mixed_chunk: if cfg.enable_mixed_chunk:
declare_resolution( declare_resolution(
server_args, server_args,
"_handle_eagle_family", "_handle_eagle_family",
@@ -716,16 +721,16 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
"PixtralForConditionalGeneration", "PixtralForConditionalGeneration",
"HYV3ForCausalLM", "HYV3ForCausalLM",
]: ]:
if server_args.speculative_draft_model_path is None: if cfg.speculative_draft_model_path is None:
declare_resolution( declare_resolution(
server_args, server_args,
"_handle_eagle_family", "_handle_eagle_family",
speculative_draft_model_path=server_args.model_path, speculative_draft_model_path=cfg.model_path,
) )
declare_resolution( declare_resolution(
server_args, server_args,
"_handle_eagle_family", "_handle_eagle_family",
speculative_draft_model_revision=server_args.revision, speculative_draft_model_revision=cfg.revision,
) )
else: else:
if model_arch not in [ if model_arch not in [
@@ -736,13 +741,10 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
"DeepSeek MTP does not require setting speculative_draft_model_path." "DeepSeek MTP does not require setting speculative_draft_model_path."
) )
if ( if not cfg.speculative_adaptive and cfg.speculative_num_steps is None:
not server_args.speculative_adaptive
and server_args.speculative_num_steps is None
):
assert ( assert (
server_args.speculative_eagle_topk is None cfg.speculative_eagle_topk is None
and server_args.speculative_num_draft_tokens is None and cfg.speculative_num_draft_tokens is None
) )
steps, topk, draft_tokens = _auto_choose_speculative_params( steps, topk, draft_tokens = _auto_choose_speculative_params(
@@ -757,29 +759,29 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
) )
if "trtllm_mha" in attention_backends_of(resolved_view(server_args)): if "trtllm_mha" in attention_backends_of(resolved_view(server_args)):
if server_args.speculative_eagle_topk > 1: if cfg.speculative_eagle_topk > 1:
raise ValueError( raise ValueError(
"trtllm_mha backend only supports topk = 1 for speculative decoding." "trtllm_mha backend only supports topk = 1 for speculative decoding."
) )
if server_args.speculative_use_rejection_sampling: if cfg.speculative_use_rejection_sampling:
# Resolved alias by now: NEXTN -> EAGLE, Gemma4 draft -> FROZEN_KV_MTP. # Resolved alias by now: NEXTN -> EAGLE, Gemma4 draft -> FROZEN_KV_MTP.
# Only the EAGLE/EAGLE3 draft workers emit a target-vocab proposal that # Only the EAGLE/EAGLE3 draft workers emit a target-vocab proposal that
# the rejection-sampling kernel consumes; everything else (STANDALONE, # the rejection-sampling kernel consumes; everything else (STANDALONE,
# FROZEN_KV_MTP, NGRAM, DFLASH) is unsupported. # FROZEN_KV_MTP, NGRAM, DFLASH) is unsupported.
if server_args.speculative_algorithm not in ("EAGLE", "EAGLE3"): if cfg.speculative_algorithm not in ("EAGLE", "EAGLE3"):
raise NotImplementedError( raise NotImplementedError(
"--speculative-use-rejection-sampling is only supported for " "--speculative-use-rejection-sampling is only supported for "
"EAGLE / EAGLE3 / NEXTN, not " "EAGLE / EAGLE3 / NEXTN, not "
f"speculative_algorithm={server_args.speculative_algorithm}." f"speculative_algorithm={cfg.speculative_algorithm}."
) )
if server_args.speculative_eagle_topk != 1: if cfg.speculative_eagle_topk != 1:
raise ValueError( raise ValueError(
"--speculative-use-rejection-sampling requires --speculative-eagle-topk=1." "--speculative-use-rejection-sampling requires --speculative-eagle-topk=1."
) )
if ( if (
server_args.speculative_accept_threshold_single != 1.0 cfg.speculative_accept_threshold_single != 1.0
or server_args.speculative_accept_threshold_acc != 1.0 or cfg.speculative_accept_threshold_acc != 1.0
): ):
raise ValueError( raise ValueError(
"--speculative-use-rejection-sampling is incompatible with " "--speculative-use-rejection-sampling is incompatible with "
@@ -787,7 +789,7 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
"--speculative-accept-threshold-acc; rejection sampling ignores " "--speculative-accept-threshold-acc; rejection sampling ignores "
"the accept thresholds." "the accept thresholds."
) )
if server_args.enable_deterministic_inference: if cfg.enable_deterministic_inference:
raise ValueError( raise ValueError(
"--speculative-use-rejection-sampling is incompatible with " "--speculative-use-rejection-sampling is incompatible with "
"--enable-deterministic-inference; the sampling kernel draws " "--enable-deterministic-inference; the sampling kernel draws "
@@ -798,7 +800,7 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
if ( if (
resolved_view(server_args).enable_multi_layer_eagle resolved_view(server_args).enable_multi_layer_eagle
and server_args.speculative_eagle_topk != 1 and cfg.speculative_eagle_topk != 1
): ):
raise ValueError( raise ValueError(
"--speculative-use-rejection-sampling with multi-layer EAGLE " "--speculative-use-rejection-sampling with multi-layer EAGLE "
@@ -811,9 +813,8 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
) )
if ( if (
server_args.speculative_eagle_topk == 1 cfg.speculative_eagle_topk == 1
and server_args.speculative_num_draft_tokens and cfg.speculative_num_draft_tokens != cfg.speculative_num_steps + 1
!= server_args.speculative_num_steps + 1
): ):
logger.warning( logger.warning(
"speculative_num_draft_tokens is adjusted to speculative_num_steps + 1 when speculative_eagle_topk == 1" "speculative_num_draft_tokens is adjusted to speculative_num_steps + 1 when speculative_eagle_topk == 1"
@@ -821,7 +822,7 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
declare_resolution( declare_resolution(
server_args, server_args,
"_handle_eagle_family", "_handle_eagle_family",
speculative_num_draft_tokens=server_args.speculative_num_steps + 1, speculative_num_draft_tokens=cfg.speculative_num_steps + 1,
) )
# topk > 1 + page_size > 1 needs the two-pass cascade draft-decode (shared prefix # topk > 1 + page_size > 1 needs the two-pass cascade draft-decode (shared prefix
@@ -830,7 +831,7 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
_PAGE_TREE_SPEC_BACKENDS = ("flashinfer", "fa3", "triton") _PAGE_TREE_SPEC_BACKENDS = ("flashinfer", "fa3", "triton")
view = resolved_view(server_args) view = resolved_view(server_args)
if ( if (
server_args.speculative_eagle_topk > 1 cfg.speculative_eagle_topk > 1
and view.page_size > 1 and view.page_size > 1
and view.attention_backend not in _PAGE_TREE_SPEC_BACKENDS and view.attention_backend not in _PAGE_TREE_SPEC_BACKENDS
): ):
@@ -842,14 +843,15 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
def _handle_ngram(server_args: ServerArgs) -> None: def _handle_ngram(server_args: ServerArgs) -> None:
if server_args.device not in ("cuda", "cpu"): cfg = resolving_view(server_args)
if cfg.device not in ("cuda", "cpu"):
raise ValueError( raise ValueError(
"Ngram speculative decoding only supports CUDA or CPU devices." "Ngram speculative decoding only supports CUDA or CPU devices."
) )
_disable_overlap_schedule_for_cpu(server_args) _disable_overlap_schedule_for_cpu(server_args)
if server_args.max_running_requests is None: if cfg.max_running_requests is None:
declare_resolution( declare_resolution(
server_args, server_args,
"_handle_ngram", "_handle_ngram",
@@ -867,9 +869,9 @@ def _handle_ngram(server_args: ServerArgs) -> None:
declare_resolution( declare_resolution(
server_args, server_args,
"_handle_ngram", "_handle_ngram",
speculative_eagle_topk=server_args.speculative_ngram_max_bfs_breadth, speculative_eagle_topk=cfg.speculative_ngram_max_bfs_breadth,
) )
if server_args.speculative_num_draft_tokens is None: if cfg.speculative_num_draft_tokens is None:
declare_resolution( declare_resolution(
server_args, server_args,
"_handle_ngram", "_handle_ngram",
@@ -879,31 +881,31 @@ def _handle_ngram(server_args: ServerArgs) -> None:
"speculative_num_draft_tokens is set to 12 by default for ngram speculative decoding. " "speculative_num_draft_tokens is set to 12 by default for ngram speculative decoding. "
"You can override this by explicitly setting --speculative-num-draft-tokens." "You can override this by explicitly setting --speculative-num-draft-tokens."
) )
if server_args.speculative_num_steps is None: if cfg.speculative_num_steps is None:
declare_resolution( declare_resolution(
server_args, server_args,
"_handle_ngram", "_handle_ngram",
speculative_num_steps=server_args.speculative_num_draft_tokens speculative_num_steps=cfg.speculative_num_draft_tokens
// server_args.speculative_eagle_topk, // cfg.speculative_eagle_topk,
) )
if server_args.speculative_ngram_external_corpus_path is not None: if cfg.speculative_ngram_external_corpus_path is not None:
if server_args.speculative_ngram_external_sam_budget <= 0: if cfg.speculative_ngram_external_sam_budget <= 0:
raise ValueError( raise ValueError(
"--speculative-ngram-external-sam-budget must be positive when " "--speculative-ngram-external-sam-budget must be positive when "
"--speculative-ngram-external-corpus-path is set." "--speculative-ngram-external-corpus-path is set."
) )
if server_args.speculative_ngram_external_corpus_max_tokens <= 0: if cfg.speculative_ngram_external_corpus_max_tokens <= 0:
raise ValueError( raise ValueError(
"--speculative-ngram-external-corpus-max-tokens must be positive when " "--speculative-ngram-external-corpus-max-tokens must be positive when "
"--speculative-ngram-external-corpus-path is set." "--speculative-ngram-external-corpus-path is set."
) )
if ( if (
server_args.speculative_ngram_external_sam_budget cfg.speculative_ngram_external_sam_budget
> server_args.speculative_num_draft_tokens - 1 > cfg.speculative_num_draft_tokens - 1
): ):
raise ValueError( raise ValueError(
"speculative_ngram_external_sam_budget must be less than or equal to " "speculative_ngram_external_sam_budget must be less than or equal to "
f"speculative_num_draft_tokens - 1 ({server_args.speculative_num_draft_tokens - 1})." f"speculative_num_draft_tokens - 1 ({cfg.speculative_num_draft_tokens - 1})."
) )
logger.warning( logger.warning(
"The mixed chunked prefill are disabled because of " "The mixed chunked prefill are disabled because of "
@@ -914,12 +916,12 @@ def _handle_ngram(server_args: ServerArgs) -> None:
view = resolved_view(server_args) view = resolved_view(server_args)
if ( if (
server_args.speculative_eagle_topk > 1 cfg.speculative_eagle_topk > 1
and view.page_size > 1 and view.page_size > 1
and view.attention_backend != "flashinfer" and view.attention_backend != "flashinfer"
): ):
raise ValueError( raise ValueError(
f"speculative_eagle_topk({server_args.speculative_eagle_topk}) > 1 " f"speculative_eagle_topk({cfg.speculative_eagle_topk}) > 1 "
f"with page_size({view.page_size}) > 1 is unstable " f"with page_size({view.page_size}) > 1 is unstable "
"and produces incorrect results for paged attention backends. " "and produces incorrect results for paged attention backends. "
"This combination is only supported for the 'flashinfer' backend." "This combination is only supported for the 'flashinfer' backend."
@@ -950,31 +952,32 @@ def _maybe_disable_adaptive(server_args: ServerArgs) -> None:
def _init_adaptive_speculative_params(server_args: ServerArgs) -> None: def _init_adaptive_speculative_params(server_args: ServerArgs) -> None:
cfg = resolving_view(server_args)
from sglang.srt.speculative.adaptive_spec_params import ( from sglang.srt.speculative.adaptive_spec_params import (
resolve_candidate_steps_from_config, resolve_candidate_steps_from_config,
) )
candidate_steps = resolve_candidate_steps_from_config( candidate_steps = resolve_candidate_steps_from_config(
cfg_path=server_args.speculative_adaptive_config, cfg_path=cfg.speculative_adaptive_config,
) )
if server_args.speculative_eagle_topk is None: if cfg.speculative_eagle_topk is None:
declare_resolution( declare_resolution(
server_args, server_args,
"_init_adaptive_speculative_params", "_init_adaptive_speculative_params",
speculative_eagle_topk=1, speculative_eagle_topk=1,
) )
if server_args.speculative_num_steps is None: if cfg.speculative_num_steps is None:
declare_resolution( declare_resolution(
server_args, server_args,
"_init_adaptive_speculative_params", "_init_adaptive_speculative_params",
speculative_num_steps=candidate_steps[len(candidate_steps) // 2], speculative_num_steps=candidate_steps[len(candidate_steps) // 2],
) )
if server_args.speculative_num_steps not in candidate_steps: if cfg.speculative_num_steps not in candidate_steps:
raise ValueError( raise ValueError(
f"--speculative-num-steps={server_args.speculative_num_steps} " f"--speculative-num-steps={cfg.speculative_num_steps} "
f"is not in the adaptive config candidate_steps {candidate_steps}. " f"is not in the adaptive config candidate_steps {candidate_steps}. "
"Pass one of those values." "Pass one of those values."
) )
@@ -982,7 +985,7 @@ def _init_adaptive_speculative_params(server_args: ServerArgs) -> None:
declare_resolution( declare_resolution(
server_args, server_args,
"_init_adaptive_speculative_params", "_init_adaptive_speculative_params",
speculative_num_draft_tokens=server_args.speculative_num_steps + 1, speculative_num_draft_tokens=cfg.speculative_num_steps + 1,
) )
@@ -992,7 +995,8 @@ def _auto_choose_speculative_params(server_args: ServerArgs, model_arch: str) ->
You can tune them on your own models and prompts with scripts/playground/bench_speculative.py You can tune them on your own models and prompts with scripts/playground/bench_speculative.py
""" """
if server_args.speculative_algorithm == "STANDALONE": cfg = resolving_view(server_args)
if cfg.speculative_algorithm == "STANDALONE":
return (3, 1, 4) return (3, 1, 4)
if model_arch in ["LlamaForCausalLM"]: if model_arch in ["LlamaForCausalLM"]:
return (5, 4, 8) return (5, 4, 8)
+26 -26
View File
@@ -589,46 +589,46 @@ class ModelConfig:
context_length: Optional[int] = None, context_length: Optional[int] = None,
**kwargs, **kwargs,
): ):
from sglang.srt.arg_groups.overrides import resolving_view
cfg = resolving_view(server_args)
quantization = ( quantization = (
server_args.speculative_draft_model_quantization cfg.speculative_draft_model_quantization
if is_draft_model if is_draft_model
else server_args.quantization else cfg.quantization
) )
override_config_file = ( override_config_file = (
server_args.decrypted_draft_config_file cfg.decrypted_draft_config_file
if is_draft_model if is_draft_model
else server_args.decrypted_config_file else cfg.decrypted_config_file
) )
return ModelConfig( return ModelConfig(
model_path=model_path or server_args.model_path, model_path=model_path or cfg.model_path,
trust_remote_code=server_args.trust_remote_code, trust_remote_code=cfg.trust_remote_code,
revision=model_revision or server_args.revision, revision=model_revision or cfg.revision,
context_length=( context_length=(
context_length context_length if context_length is not None else cfg.context_length
if context_length is not None
else server_args.context_length
), ),
model_override_args=server_args.json_model_override_args, model_override_args=cfg.json_model_override_args,
is_embedding=server_args.is_embedding, is_embedding=cfg.is_embedding,
enable_multimodal=server_args.enable_multimodal, enable_multimodal=cfg.enable_multimodal,
dtype=server_args.dtype, dtype=cfg.dtype,
quantization=quantization, quantization=quantization,
model_impl=server_args.model_impl, model_impl=cfg.model_impl,
sampling_defaults=server_args.sampling_defaults, sampling_defaults=cfg.sampling_defaults,
quantize_and_serve=server_args.quantize_and_serve, quantize_and_serve=cfg.quantize_and_serve,
override_config_file=override_config_file, override_config_file=override_config_file,
is_multi_layer_eagle=server_args.enable_multi_layer_eagle, is_multi_layer_eagle=cfg.enable_multi_layer_eagle,
language_only=server_args.language_only, language_only=cfg.language_only,
language_model_only=server_args.language_model_only, language_model_only=cfg.language_model_only,
encoder_only=server_args.encoder_only, encoder_only=cfg.encoder_only,
is_draft_model=is_draft_model, is_draft_model=is_draft_model,
is_draft_quantization_explicit=( is_draft_quantization_explicit=(
is_draft_model is_draft_model and cfg._speculative_draft_quantization_explicitly_set
and server_args._speculative_draft_quantization_explicitly_set
), ),
disable_hybrid_swa_memory=server_args.disable_hybrid_swa_memory, disable_hybrid_swa_memory=cfg.disable_hybrid_swa_memory,
model_config_parser=server_args.model_config_parser, model_config_parser=cfg.model_config_parser,
speculative_algorithm=server_args.speculative_algorithm, speculative_algorithm=cfg.speculative_algorithm,
**kwargs, **kwargs,
) )
+11 -10
View File
@@ -25,13 +25,16 @@ class DllmConfig:
def from_server_args( def from_server_args(
server_args: ServerArgs, server_args: ServerArgs,
): ):
if server_args.dllm_algorithm is None: from sglang.srt.arg_groups.overrides import resolving_view
cfg = resolving_view(server_args)
if cfg.dllm_algorithm is None:
return None return None
model_config = ModelConfig.from_server_args( model_config = ModelConfig.from_server_args(
server_args, server_args,
model_path=server_args.model_path, model_path=cfg.model_path,
model_revision=server_args.revision, model_revision=cfg.revision,
) )
DLLM_PARAMS = { DLLM_PARAMS = {
"LLaDA2MoeModelLM": {"block_size": 32, "mask_id": 156895}, "LLaDA2MoeModelLM": {"block_size": 32, "mask_id": 156895},
@@ -48,13 +51,11 @@ class DllmConfig:
raise RuntimeError(f"Unknown diffusion LLM: {arch}") raise RuntimeError(f"Unknown diffusion LLM: {arch}")
max_running_requests = ( max_running_requests = (
1 1 if cfg.max_running_requests is None else cfg.max_running_requests
if server_args.max_running_requests is None
else server_args.max_running_requests
) )
algorithm_config = {} algorithm_config = {}
if server_args.dllm_algorithm_config is not None: if cfg.dllm_algorithm_config is not None:
try: try:
import yaml import yaml
except ImportError: except ImportError:
@@ -62,17 +63,17 @@ class DllmConfig:
"Please install PyYAML to use YAML config files. " "Please install PyYAML to use YAML config files. "
"`pip install pyyaml`" "`pip install pyyaml`"
) )
with open(server_args.dllm_algorithm_config, "r") as f: with open(cfg.dllm_algorithm_config, "r") as f:
algorithm_config = yaml.safe_load(f) algorithm_config = yaml.safe_load(f)
# Parse common algorithm configurations # Parse common algorithm configurations
block_size = algorithm_config.get("block_size", block_size) block_size = algorithm_config.get("block_size", block_size)
return DllmConfig( return DllmConfig(
algorithm=server_args.dllm_algorithm, algorithm=cfg.dllm_algorithm,
algorithm_config=algorithm_config, algorithm_config=algorithm_config,
block_size=block_size, block_size=block_size,
mask_id=mask_id, mask_id=mask_id,
max_running_requests=max_running_requests, max_running_requests=max_running_requests,
first_done_first_out_mode=server_args.dllm_fdfo, first_done_first_out_mode=cfg.dllm_fdfo,
) )
@@ -43,6 +43,9 @@ def set_default_server_args(args: "ServerArgs"):
""" """
Set default server arguments for NPU backend. Set default server arguments for NPU backend.
""" """
from sglang.srt.arg_groups.overrides import resolving_view
cfg = resolving_view(args)
# NPU only works with "ascend" attention backend for now # NPU only works with "ascend" attention backend for now
declare_resolution( declare_resolution(
@@ -60,7 +63,7 @@ def set_default_server_args(args: "ServerArgs"):
"set_default_server_args", "set_default_server_args",
decode_attention_backend="ascend", decode_attention_backend="ascend",
) )
if args.page_size is None: if cfg.page_size is None:
declare_resolution( declare_resolution(
args, args,
"set_default_server_args", "set_default_server_args",
@@ -68,33 +71,33 @@ def set_default_server_args(args: "ServerArgs"):
) )
# NPU memory settings # NPU memory settings
decode = args.cuda_graph_config.decode decode = cfg.cuda_graph_config.decode
npu_mem = get_npu_memory_capacity() npu_mem = get_npu_memory_capacity()
if npu_mem <= 32 * 1024: if npu_mem <= 32 * 1024:
# Ascend 910B4,910B4_1 # Ascend 910B4,910B4_1
# (chunked_prefill_size 4k, max_bs 16 if tp < 4 else 64) # (chunked_prefill_size 4k, max_bs 16 if tp < 4 else 64)
if args.chunked_prefill_size is None: if cfg.chunked_prefill_size is None:
declare_resolution( declare_resolution(
args, args,
"set_default_server_args", "set_default_server_args",
chunked_prefill_size=4 * 1024, chunked_prefill_size=4 * 1024,
) )
if decode.max_bs is None: if decode.max_bs is None:
if args.tp_size < 4: if cfg.tp_size < 4:
decode.max_bs = 16 decode.max_bs = 16
else: else:
decode.max_bs = 64 decode.max_bs = 64
elif npu_mem <= 64 * 1024: elif npu_mem <= 64 * 1024:
# Ascend 910B1,910B2,910B2C,910B3,910_9391,910_9392,910_9381,910_9382,910_9372,910_9362 # Ascend 910B1,910B2,910B2C,910B3,910_9391,910_9392,910_9381,910_9382,910_9372,910_9362
# (chunked_prefill_size 8k, max_bs 64 if tp < 4 else 256) # (chunked_prefill_size 8k, max_bs 64 if tp < 4 else 256)
if args.chunked_prefill_size is None: if cfg.chunked_prefill_size is None:
declare_resolution( declare_resolution(
args, args,
"set_default_server_args", "set_default_server_args",
chunked_prefill_size=8 * 1024, chunked_prefill_size=8 * 1024,
) )
if decode.max_bs is None: if decode.max_bs is None:
if args.tp_size < 4: if cfg.tp_size < 4:
decode.max_bs = 64 decode.max_bs = 64
else: else:
decode.max_bs = 256 decode.max_bs = 256
@@ -107,7 +110,7 @@ def set_default_server_args(args: "ServerArgs"):
) )
# handles hierarchical cache configs # handles hierarchical cache configs
if args.enable_hierarchical_cache: if cfg.enable_hierarchical_cache:
declare_resolution( declare_resolution(
args, args,
"set_default_server_args", "set_default_server_args",
+7 -4
View File
@@ -239,18 +239,21 @@ _STRATEGY: Optional[ContextParallelStrategy] = None
def init_cp_strategy(server_args: ServerArgs) -> None: def init_cp_strategy(server_args: ServerArgs) -> None:
"""Bind the configured CP strategy for this process.""" """Bind the configured CP strategy for this process."""
from sglang.srt.arg_groups.overrides import resolving_view
cfg = resolving_view(server_args)
global _STRATEGY global _STRATEGY
if not getattr(server_args, "enable_prefill_cp", False): if not cfg.enable_prefill_cp:
_STRATEGY = None _STRATEGY = None
return return
cp_size = getattr(server_args, "attn_cp_size", 1) cp_size = cfg.attn_cp_size
if cp_size <= 1: if cp_size <= 1:
_STRATEGY = None _STRATEGY = None
return return
kind = ContextParallelStrategyKind.from_string(server_args.cp_strategy) kind = ContextParallelStrategyKind.from_string(cfg.cp_strategy)
if kind == ContextParallelStrategyKind.ZIGZAG: if kind == ContextParallelStrategyKind.ZIGZAG:
from sglang.srt.layers.cp.zigzag import ZigzagCPStrategy from sglang.srt.layers.cp.zigzag import ZigzagCPStrategy
@@ -262,7 +265,7 @@ def init_cp_strategy(server_args: ServerArgs) -> None:
else: else:
raise ValueError( raise ValueError(
f"Unsupported cp_strategy kind {kind} for " f"Unsupported cp_strategy kind {kind} for "
f"cp_strategy={server_args.cp_strategy!r}" f"cp_strategy={cfg.cp_strategy!r}"
) )
+6 -3
View File
@@ -42,12 +42,15 @@ if TYPE_CHECKING:
def supports_prefill_cp_bcg(server_args: ServerArgs) -> bool: def supports_prefill_cp_bcg(server_args: ServerArgs) -> bool:
"""Return whether the selected prefill-CP configuration supports BCG.""" """Return whether the selected prefill-CP configuration supports BCG."""
from sglang.srt.arg_groups.overrides import resolving_view
cfg = resolving_view(server_args)
resolved = server_args._resolved() resolved = server_args._resolved()
prefill_attention_backend, _ = server_args._resolved_attention_backends() prefill_attention_backend, _ = server_args._resolved_attention_backends()
return ( return (
server_args.enable_prefill_cp cfg.enable_prefill_cp
and resolved.attn_cp_size == server_args.tp_size and resolved.attn_cp_size == cfg.tp_size
and server_args.cp_strategy == "zigzag" and cfg.cp_strategy == "zigzag"
and prefill_attention_backend == "trtllm_mha" and prefill_attention_backend == "trtllm_mha"
) )
@@ -14,6 +14,9 @@ def validate_experimental_sgl_marlin_server_args(
server_args: Any, resolved_args: Any server_args: Any, resolved_args: Any
) -> None: ) -> None:
"""Validate startup options before the experimental runner is constructed.""" """Validate startup options before the experimental runner is constructed."""
from sglang.srt.arg_groups.overrides import resolving_view
cfg = resolving_view(server_args)
if resolved_args.ep_size > 1 and resolved_args.moe_a2a_backend != "none": if resolved_args.ep_size > 1 and resolved_args.moe_a2a_backend != "none":
raise ValueError("experimental_sgl_marlin EP requires --moe-a2a-backend none") raise ValueError("experimental_sgl_marlin EP requires --moe-a2a-backend none")
@@ -21,16 +24,16 @@ def validate_experimental_sgl_marlin_server_args(
# A provided adapter path implicitly enables LoRA later unless it was # A provided adapter path implicitly enables LoRA later unless it was
# explicitly disabled. No-LoRA delegates to the stock Marlin fused path. # explicitly disabled. No-LoRA delegates to the stock Marlin fused path.
lora_enabled = bool(resolved_args.enable_lora) or ( lora_enabled = bool(resolved_args.enable_lora) or (
resolved_args.enable_lora is None and bool(server_args.lora_paths) resolved_args.enable_lora is None and bool(cfg.lora_paths)
) )
if not lora_enabled: if not lora_enabled:
return return
if not server_args.lora_use_virtual_experts: if not cfg.lora_use_virtual_experts:
raise ValueError( raise ValueError(
"experimental_sgl_marlin LoRA requires --lora-use-virtual-experts" "experimental_sgl_marlin LoRA requires --lora-use-virtual-experts"
) )
if server_args.lora_backend != "triton": if cfg.lora_backend != "triton":
# The temporary dense/sink kernels consume Triton SGEMM batch metadata # The temporary dense/sink kernels consume Triton SGEMM batch metadata
# directly; other global backends are not adapted in this tree. # directly; other global backends are not adapted in this tree.
raise ValueError("experimental_sgl_marlin LoRA requires --lora-backend triton") raise ValueError("experimental_sgl_marlin LoRA requires --lora-backend triton")
@@ -38,12 +41,12 @@ def validate_experimental_sgl_marlin_server_args(
return return
if ( if (
server_args.init_expert_location != "trivial" cfg.init_expert_location != "trivial"
or server_args.ep_num_redundant_experts != 0 or cfg.ep_num_redundant_experts != 0
or server_args.enable_eplb or cfg.enable_eplb
or server_args.elastic_ep_backend is not None or cfg.elastic_ep_backend is not None
or server_args.enable_elastic_expert_backup or cfg.enable_elastic_expert_backup
or server_args.elastic_ep_rejoin or cfg.elastic_ep_rejoin
): ):
raise ValueError( raise ValueError(
"experimental_sgl_marlin EP requires trivial expert placement " "experimental_sgl_marlin EP requires trivial expert placement "
@@ -196,10 +196,14 @@ def prepare_raw_kimi_server_args(
server_args: Any, loader_config: dict[str, Any] server_args: Any, loader_config: dict[str, Any]
) -> None: ) -> None:
"""Resolve a raw GGUF model path into the normal loader inputs.""" """Resolve a raw GGUF model path into the normal loader inputs."""
model_path = Path(server_args.model_path).expanduser()
from sglang.srt.arg_groups.overrides import resolving_view
cfg = resolving_view(server_args)
model_path = Path(cfg.model_path).expanduser()
if not model_path.is_file() or model_path.suffix.lower() != ".gguf": if not model_path.is_file() or model_path.suffix.lower() != ".gguf":
return return
tokenizer_path = server_args.tokenizer_path tokenizer_path = cfg.tokenizer_path
if tokenizer_path and Path(tokenizer_path).expanduser() == model_path: if tokenizer_path and Path(tokenizer_path).expanduser() == model_path:
tokenizer_path = None tokenizer_path = None
assets = ensure_kimi_assets( assets = ensure_kimi_assets(
@@ -503,7 +507,11 @@ def prepare_raw_deepseek_server_args(
server_args: Any, loader_config: dict[str, Any] server_args: Any, loader_config: dict[str, Any]
) -> None: ) -> None:
"""Resolve a raw DeepSeek V4 GGUF into metadata and Expert Pack inputs.""" """Resolve a raw DeepSeek V4 GGUF into metadata and Expert Pack inputs."""
source = Path(server_args.model_path).expanduser().resolve(strict=True)
from sglang.srt.arg_groups.overrides import resolving_view
cfg = resolving_view(server_args)
source = Path(cfg.model_path).expanduser().resolve(strict=True)
if not source.is_file(): if not source.is_file():
return return
repo = _repo_root() repo = _repo_root()
@@ -546,7 +554,11 @@ def prepare_raw_expert_pack_server_args(
server_args: Any, loader_config: dict[str, Any] server_args: Any, loader_config: dict[str, Any]
) -> None: ) -> None:
"""Dispatch a raw GGUF to the model-specific expert-pack preparation path.""" """Dispatch a raw GGUF to the model-specific expert-pack preparation path."""
source = Path(server_args.model_path).expanduser()
from sglang.srt.arg_groups.overrides import resolving_view
cfg = resolving_view(server_args)
source = Path(cfg.model_path).expanduser()
if not source.is_file(): if not source.is_file():
return return
name = source.name.upper() name = source.name.upper()
+11 -9
View File
@@ -29,7 +29,7 @@ import jinja2.ext
import jinja2.nodes import jinja2.nodes
import jinja2.sandbox import jinja2.sandbox
from sglang.srt.arg_groups.overrides import declare_late_resolution from sglang.srt.arg_groups.overrides import declare_late_resolution, resolving_view
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -666,11 +666,12 @@ def _architecture_auto_parsers(server_args, needs: Tuple[str, ...]) -> Dict[str,
"""The parsers the model architecture implies, for the fields still on auto.""" """The parsers the model architecture implies, for the fields still on auto."""
from sglang.srt.utils.hf_transformers_utils import get_config from sglang.srt.utils.hf_transformers_utils import get_config
cfg = resolving_view(server_args)
config = get_config( config = get_config(
server_args.model_path, cfg.model_path,
trust_remote_code=server_args.trust_remote_code, trust_remote_code=cfg.trust_remote_code,
revision=getattr(server_args, "revision", None), revision=getattr(cfg, "revision", None),
model_config_parser=getattr(server_args, "model_config_parser", "auto"), model_config_parser=getattr(cfg, "model_config_parser", "auto"),
) )
architectures = getattr(config, "architectures", None) or [] architectures = getattr(config, "architectures", None) or []
arch = architectures[0] if architectures else "" arch = architectures[0] if architectures else ""
@@ -708,17 +709,18 @@ def resolve_auto_parsers(server_args) -> None:
the schedulers it forks, the HTTP server, and the tokenizer workers it is the schedulers it forks, the HTTP server, and the tokenizer workers it is
serialized for. serialized for.
""" """
cfg = resolving_view(server_args)
needs = tuple( needs = tuple(
attr attr
for attr in ("reasoning_parser", "tool_call_parser") for attr in ("reasoning_parser", "tool_call_parser")
if getattr(server_args, attr) == "auto" if getattr(cfg, attr) == "auto"
) )
if not needs: if not needs:
return return
from sglang.srt.utils.hf_transformers_utils import get_tokenizer from sglang.srt.utils.hf_transformers_utils import get_tokenizer
chat_template_arg = getattr(server_args, "chat_template", None) chat_template_arg = getattr(cfg, "chat_template", None)
try: try:
explicit_jinja_template = _load_explicit_jinja_template(chat_template_arg) explicit_jinja_template = _load_explicit_jinja_template(chat_template_arg)
except Exception as e: except Exception as e:
@@ -731,8 +733,8 @@ def resolve_auto_parsers(server_args) -> None:
tokenizer = None tokenizer = None
try: try:
tokenizer = get_tokenizer( tokenizer = get_tokenizer(
server_args.model_path, cfg.model_path,
trust_remote_code=server_args.trust_remote_code, trust_remote_code=cfg.trust_remote_code,
) )
except Exception as e: except Exception as e:
logger.warning(f"Failed to load tokenizer for auto-detection: {e}") logger.warning(f"Failed to load tokenizer for auto-detection: {e}")
File diff suppressed because it is too large Load Diff
@@ -49,19 +49,19 @@ DEFAULT_ADAPTIVE_CONFIG: dict[str, dict] = {
def adaptive_unsupported_reason(server_args: ServerArgs) -> str | None: def adaptive_unsupported_reason(server_args: ServerArgs) -> str | None:
"""Return why adaptive spec cannot run under the given server args, or None if supported.""" """Return why adaptive spec cannot run under the given server args, or None if supported."""
from sglang.srt.arg_groups.overrides import resolving_view
cfg = resolving_view(server_args)
from sglang.srt.arg_groups.overrides import resolved_view from sglang.srt.arg_groups.overrides import resolved_view
if server_args.speculative_algorithm not in ("EAGLE", "EAGLE3"): if cfg.speculative_algorithm not in ("EAGLE", "EAGLE3"):
return ( return (
f"speculative_algorithm={server_args.speculative_algorithm} " f"speculative_algorithm={cfg.speculative_algorithm} "
"(only EAGLE/EAGLE3 are supported)" "(only EAGLE/EAGLE3 are supported)"
) )
if ( if cfg.speculative_eagle_topk is not None and cfg.speculative_eagle_topk != 1:
server_args.speculative_eagle_topk is not None
and server_args.speculative_eagle_topk != 1
):
return ( return (
f"speculative_eagle_topk={server_args.speculative_eagle_topk} " f"speculative_eagle_topk={cfg.speculative_eagle_topk} "
"(only topk=1 is supported)" "(only topk=1 is supported)"
) )
if resolved_view(server_args).enable_dp_attention: if resolved_view(server_args).enable_dp_attention:
@@ -74,12 +74,12 @@ def adaptive_unsupported_reason(server_args: ServerArgs) -> str | None:
"enable_multi_layer_eagle=True is not supported " "enable_multi_layer_eagle=True is not supported "
"(MultiLayerEagleWorkerV2 does not implement adaptive)" "(MultiLayerEagleWorkerV2 does not implement adaptive)"
) )
if server_args.enable_two_batch_overlap: if cfg.enable_two_batch_overlap:
return ( return (
"enable_two_batch_overlap=True is not supported " "enable_two_batch_overlap=True is not supported "
"(adaptive state swap would discard the TboAttnBackend wrapper)" "(adaptive state swap would discard the TboAttnBackend wrapper)"
) )
if server_args.enable_pdmux: if cfg.enable_pdmux:
return ( return (
"enable_pdmux=True is not supported " "enable_pdmux=True is not supported "
"(adaptive state swap does not update decode_attn_backend_group)" "(adaptive state swap does not update decode_attn_backend_group)"
+4 -1
View File
@@ -254,6 +254,9 @@ class SpeculativeAlgorithm(Enum):
def create_worker( def create_worker(
self, server_args: ServerArgs self, server_args: ServerArgs
) -> Optional[Union[Type[BaseSpecWorker], Type[TpModelWorker], Type[NGRAMWorker]]]: ) -> Optional[Union[Type[BaseSpecWorker], Type[TpModelWorker], Type[NGRAMWorker]]]:
from sglang.srt.arg_groups.overrides import resolving_view
cfg = resolving_view(server_args)
assert ( assert (
not self.is_none() not self.is_none()
), "Cannot create worker for NONE speculative algorithm." ), "Cannot create worker for NONE speculative algorithm."
@@ -283,7 +286,7 @@ class SpeculativeAlgorithm(Enum):
# EAGLE / EAGLE3 / STANDALONE / MULTI_LAYER always use the V2 worker, # EAGLE / EAGLE3 / STANDALONE / MULTI_LAYER always use the V2 worker,
# even with overlap disabled (scheduler drives it synchronously). # even with overlap disabled (scheduler drives it synchronously).
if self.is_eagle() and server_args.enable_multi_layer_eagle: if self.is_eagle() and cfg.enable_multi_layer_eagle:
from sglang.srt.speculative.multi_layer_eagle_worker_v2 import ( from sglang.srt.speculative.multi_layer_eagle_worker_v2 import (
MultiLayerEagleWorkerV2, MultiLayerEagleWorkerV2,
) )
@@ -108,7 +108,10 @@ class CustomSpecAlgo:
pass pass
def create_worker(self, server_args: ServerArgs) -> Type: def create_worker(self, server_args: ServerArgs) -> Type:
if not server_args.disable_overlap_schedule and not self.supports_overlap: from sglang.srt.arg_groups.overrides import resolving_view
cfg = resolving_view(server_args)
if not cfg.disable_overlap_schedule and not self.supports_overlap:
raise ValueError( raise ValueError(
f"Speculative algorithm {self.name} does not support overlap scheduling." f"Speculative algorithm {self.name} does not support overlap scheduling."
) )
@@ -1,6 +1,7 @@
import unittest import unittest
from types import SimpleNamespace from types import SimpleNamespace
from sglang.srt.arg_groups.overrides import resolution_result
from sglang.srt.arg_groups.speculative_hook import ( from sglang.srt.arg_groups.speculative_hook import (
_handle_dspark, _handle_dspark,
_target_checkpoint_bundles_dspark_draft, _target_checkpoint_bundles_dspark_draft,
@@ -63,8 +64,13 @@ class TestDsparkDraftPathDefaulting(CustomTestCase):
model_path=_BUNDLED_MODEL_PATH, hf_config=_bundled_hf_config() model_path=_BUNDLED_MODEL_PATH, hf_config=_bundled_hf_config()
) )
_handle_dspark(server_args) _handle_dspark(server_args)
self.assertEqual(server_args.speculative_draft_model_path, _BUNDLED_MODEL_PATH) self.assertEqual(
self.assertEqual(server_args.speculative_num_draft_tokens, 6) resolution_result(server_args, "speculative_draft_model_path"),
_BUNDLED_MODEL_PATH,
)
self.assertEqual(
resolution_result(server_args, "speculative_num_draft_tokens"), 6
)
def test_plain_target_without_draft_path_raises(self): def test_plain_target_without_draft_path_raises(self):
server_args = _make_dspark_server_args( server_args = _make_dspark_server_args(
@@ -80,7 +86,7 @@ class TestDsparkDraftPathDefaulting(CustomTestCase):
server_args.speculative_draft_model_path = "deepseek-ai/some-other-dspark-draft" server_args.speculative_draft_model_path = "deepseek-ai/some-other-dspark-draft"
_handle_dspark(server_args) _handle_dspark(server_args)
self.assertEqual( self.assertEqual(
server_args.speculative_draft_model_path, resolution_result(server_args, "speculative_draft_model_path"),
"deepseek-ai/some-other-dspark-draft", "deepseek-ai/some-other-dspark-draft",
) )
@@ -4,6 +4,7 @@ import unittest
from types import SimpleNamespace from types import SimpleNamespace
from unittest.mock import patch from unittest.mock import patch
from sglang.srt.arg_groups.overrides import resolution_result
from sglang.srt.configs.embedding_model_spec import resolve_embedding_model_spec from sglang.srt.configs.embedding_model_spec import resolve_embedding_model_spec
from sglang.srt.configs.model_config import ( from sglang.srt.configs.model_config import (
is_multimodal_piecewise_cuda_graph_supported, is_multimodal_piecewise_cuda_graph_supported,
@@ -90,7 +91,10 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
): ):
args._apply_cuda_graph_compatibility() args._apply_cuda_graph_compatibility()
self.assertEqual(args.cuda_graph_config.prefill.backend, Backend.TC_PIECEWISE) self.assertEqual(
resolution_result(args, "cuda_graph_config").prefill.backend,
Backend.TC_PIECEWISE,
)
disable_if_incompatible.assert_called_once() disable_if_incompatible.assert_called_once()
def test_trtllm_mla_stays_on_breakable(self): def test_trtllm_mla_stays_on_breakable(self):
@@ -118,7 +122,10 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
): ):
args._apply_cuda_graph_compatibility() args._apply_cuda_graph_compatibility()
self.assertEqual(args.cuda_graph_config.prefill.backend, Backend.BREAKABLE) self.assertEqual(
resolution_result(args, "cuda_graph_config").prefill.backend,
Backend.BREAKABLE,
)
def test_explicit_tc_piecewise_overrides_trtllm_mla_default(self): def test_explicit_tc_piecewise_overrides_trtllm_mla_default(self):
args = ServerArgs(model_path="dummy") args = ServerArgs(model_path="dummy")
@@ -134,7 +141,10 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
): ):
args._apply_cuda_graph_compatibility() args._apply_cuda_graph_compatibility()
self.assertEqual(args.cuda_graph_config.prefill.backend, Backend.TC_PIECEWISE) self.assertEqual(
resolution_result(args, "cuda_graph_config").prefill.backend,
Backend.TC_PIECEWISE,
)
def test_multimodal_inputs_keep_tc_piecewise_prefill_enabled(self): def test_multimodal_inputs_keep_tc_piecewise_prefill_enabled(self):
runner = self._make_prefill_runner(Backend.TC_PIECEWISE) runner = self._make_prefill_runner(Backend.TC_PIECEWISE)
@@ -178,10 +188,16 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
): ):
args._handle_model_capability_adjustments() args._handle_model_capability_adjustments()
self.assertTrue(args.disable_radix_cache) self.assertTrue(resolution_result(args, "disable_radix_cache"))
self.assertEqual(args.chunked_prefill_size, -1) self.assertEqual(resolution_result(args, "chunked_prefill_size"), -1)
self.assertEqual(args.cuda_graph_config.decode.backend, Backend.DISABLED) self.assertEqual(
self.assertEqual(args.cuda_graph_config.prefill.backend, Backend.BREAKABLE) resolution_result(args, "cuda_graph_config").decode.backend,
Backend.DISABLED,
)
self.assertEqual(
resolution_result(args, "cuda_graph_config").prefill.backend,
Backend.BREAKABLE,
)
def test_encoder_embedding_model_enables_embedding_mode_without_flag(self): def test_encoder_embedding_model_enables_embedding_mode_without_flag(self):
args = ServerArgs(model_path="dummy") args = ServerArgs(model_path="dummy")
@@ -199,7 +215,7 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
with patch.object(args, "get_model_config", return_value=args.model_config): with patch.object(args, "get_model_config", return_value=args.model_config):
args._handle_model_capability_adjustments() args._handle_model_capability_adjustments()
self.assertTrue(args.is_embedding) self.assertTrue(resolution_result(args, "is_embedding"))
if __name__ == "__main__": if __name__ == "__main__":
@@ -15,6 +15,7 @@ import zmq.asyncio
from fastapi import HTTPException from fastapi import HTTPException
from PIL import Image from PIL import Image
from sglang.srt.arg_groups.overrides import resolution_result
from sglang.srt.disaggregation.encoder.preprocessor import ( from sglang.srt.disaggregation.encoder.preprocessor import (
EncoderPreprocessor, EncoderPreprocessor,
EncoderPreprocessResult, EncoderPreprocessResult,
@@ -195,7 +196,7 @@ def test_epd_rejection_reads_the_resolved_transfer_backend():
finally: finally:
shutil.rmtree(config_dir, ignore_errors=True) shutil.rmtree(config_dir, ignore_errors=True)
assert resolved.encoder_transfer_backend == "zmq_to_tokenizer" assert resolution_result(resolved, "encoder_transfer_backend") == "zmq_to_tokenizer"
# Publish that record: the guard reads the resolved value out of the bags, # Publish that record: the guard reads the resolved value out of the bags,
# so a raw record does not silently disable the rejection. # so a raw record does not silently disable the rejection.
publish(resolved, role="tokenizer") publish(resolved, role="tokenizer")
@@ -6,6 +6,7 @@ from unittest.mock import MagicMock, patch
import torch import torch
from sglang.srt.arg_groups.overrides import resolution_result
from sglang.srt.environ import envs from sglang.srt.environ import envs
from sglang.srt.runtime_context import get_context from sglang.srt.runtime_context import get_context
from sglang.srt.server_args import ServerArgs from sglang.srt.server_args import ServerArgs
@@ -25,17 +26,20 @@ class TestMmProcessConfigValidation(CustomTestCase):
def test_valid_config_accepted(self): def test_valid_config_accepted(self):
args = self._validate_config({"image": {"max_pixels": 5000000}}) args = self._validate_config({"image": {"max_pixels": 5000000}})
self.assertEqual(args.mm_process_config, {"image": {"max_pixels": 5000000}}) self.assertEqual(
resolution_result(args, "mm_process_config"),
{"image": {"max_pixels": 5000000}},
)
def test_empty_config_accepted(self): def test_empty_config_accepted(self):
args = self._validate_config({}) args = self._validate_config({})
self.assertEqual(args.mm_process_config, {}) self.assertEqual(resolution_result(args, "mm_process_config"), {})
def test_none_config_defaults_to_empty_dict(self): def test_none_config_defaults_to_empty_dict(self):
args = self._validate_config(None) args = self._validate_config(None)
# None is kept as-is for dummy models (default happens after early return) # None is kept as-is for dummy models (default happens after early return)
# but for real models it would be set to {} # but for real models it would be set to {}
self.assertIsNone(args.mm_process_config) self.assertIsNone(resolution_result(args, "mm_process_config"))
def test_top_level_non_dict_rejected(self): def test_top_level_non_dict_rejected(self):
with self.assertRaises(TypeError) as ctx: with self.assertRaises(TypeError) as ctx:
@@ -64,7 +68,7 @@ class TestMmProcessConfigValidation(CustomTestCase):
"audio": {"sample_rate": 16000}, "audio": {"sample_rate": 16000},
} }
args = self._validate_config(config) args = self._validate_config(config)
self.assertEqual(args.mm_process_config, config) self.assertEqual(resolution_result(args, "mm_process_config"), config)
class TestBaseProcessorConfigExtraction(CustomTestCase): class TestBaseProcessorConfigExtraction(CustomTestCase):
@@ -125,6 +125,13 @@ def _registry_collection_is_after_the_build():
def _server_args_names(tree, path): def _server_args_names(tree, path):
"""Every local that names the record, including the read views over it.
A resolution-time reader reads through `resolving_view(server_args)` (the
declaration stash over the fields): declaration-only resolvers write no
field, so a field read there answers with the raw input. `cfg.dtype` after `cfg = resolving_view(sa)` is
the same read this scan is looking for, so the local it binds counts.
"""
names = {"self"} if path.name == "server_args.py" else {"server_args"} names = {"self"} if path.name == "server_args.py" else {"server_args"}
for node in ast.walk(tree): for node in ast.walk(tree):
if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
@@ -142,6 +149,32 @@ def _server_args_names(tree, path):
continue continue
if text == "ServerArgs": if text == "ServerArgs":
names.add(arg.arg) names.add(arg.arg)
# `cfg = resolving_view(server_args)` / `resolved_view(server_args)`
for _ in range(2): # a view over a view-holding local is still one
for node in ast.walk(tree):
if not isinstance(node, ast.Assign):
continue
value = node.value
bare = (
isinstance(value, ast.Call)
and isinstance(value.func, ast.Name)
and value.func.id in ("resolving_view", "resolved_view")
and value.args
and isinstance(value.args[0], ast.Name)
and value.args[0].id in names
)
# `resolved = self._resolved()` is the same view, spelled as the
# record's own member.
member = (
isinstance(value, ast.Call)
and isinstance(value.func, ast.Attribute)
and value.func.attr == "_resolved"
and isinstance(value.func.value, ast.Name)
and value.func.value.id in names
)
if not (bare or member):
continue
names |= {t.id for t in node.targets if isinstance(t, ast.Name)}
return names return names
@@ -191,11 +224,12 @@ def _late_resolution_fields():
for name in ( for name in (
"server_args.py", "server_args.py",
"arg_groups/overrides.py", "arg_groups/overrides.py",
"utils/template_detection.py", "parser/template_detection.py",
): ):
path = _SRT / name path = _SRT / name
if not path.exists(): # A named file that moved away has to be loud; skipping it silently
continue # leaves the scan believing it read a module it never opened.
assert path.exists(), f"{name} is not where this scan looks for it"
tree = _parsed(path) tree = _parsed(path)
for node in ast.walk(tree): for node in ast.walk(tree):
if not isinstance(node, ast.Call): if not isinstance(node, ast.Call):
@@ -764,7 +764,7 @@ class TestResolutionDeclarations(CustomTestCase):
# Snapshot before publishing: the bag serves the very object the record # Snapshot before publishing: the bag serves the very object the record
# holds, so comparing them after the fact compares an object with # holds, so comparing them after the fact compares an object with
# itself and passes however the projection behaves. # itself and passes however the projection behaves.
expected = copy.deepcopy(server_args.cuda_graph_config) expected = copy.deepcopy(resolution_result(server_args, "cuda_graph_config"))
publish(server_args, role="scheduler") publish(server_args, role="scheduler")
published = get_exec().graph.cuda_graph_config published = get_exec().graph.cuda_graph_config
resolved = expected resolved = expected
@@ -36,6 +36,7 @@ import unittest.mock
import torch import torch
import sglang import sglang
from sglang.srt.arg_groups.overrides import resolution_result
from sglang.srt.environ import EnvField, envs from sglang.srt.environ import EnvField, envs
from sglang.srt.server_args import ServerArgs from sglang.srt.server_args import ServerArgs
from sglang.srt.utils import is_cuda from sglang.srt.utils import is_cuda
@@ -270,7 +271,10 @@ class TestResolutionIsReproducible(_RestoresProcessState, CustomTestCase):
for field in dataclasses.fields(server_args): for field in dataclasses.fields(server_args):
if field.name in _NOT_COMPARABLE: if field.name in _NOT_COMPARABLE:
continue continue
value = getattr(server_args, field.name) # The resolution result, not the field: a declaration-only resolver
# never writes the field, so comparing fields would miss exactly
# the decisions a leak would shift.
value = resolution_result(server_args, field.name)
# Nested dataclasses (cuda_graph_config) compare structurally, and # Nested dataclasses (cuda_graph_config) compare structurally, and
# everything else is deep-copied: a snapshot that stored the live # everything else is deep-copied: a snapshot that stored the live
# list/dict would follow an in-place mutation, which is exactly the # list/dict would follow an in-place mutation, which is exactly the
@@ -382,10 +386,13 @@ class TestResolutionIsReproducible(_RestoresProcessState, CustomTestCase):
# differs from the cpu that `default_before` resolved to. # differs from the cpu that `default_before` resolved to.
expected = ( expected = (
"cuda_ipc" "cuda_ipc"
if intermediate.mm_feature_transport == "cuda_ipc" if resolution_result(intermediate, "mm_feature_transport")
== "cuda_ipc"
else "cpu" else "cpu"
) )
self.assertEqual(after.mm_feature_transport, expected) self.assertEqual(
resolution_result(after, "mm_feature_transport"), expected
)
def test_resolving_a_sibling_leaves_the_first_alone(self): def test_resolving_a_sibling_leaves_the_first_alone(self):
for label, config, kwargs in _SHAPES: for label, config, kwargs in _SHAPES:
@@ -847,9 +854,9 @@ class TestACopyStaysResolved(_RestoresProcessState, CustomTestCase):
self.assertEqual(get_parallel().config.dist_init_addr, "1.2.3.4:5000") self.assertEqual(get_parallel().config.dist_init_addr, "1.2.3.4:5000")
self.assertEqual( self.assertEqual(
get_schedule().chunked_prefill_size, get_schedule().chunked_prefill_size,
parent.chunked_prefill_size, resolution_result(parent, "chunked_prefill_size"),
"publishing the copy re-ran resolution; the bag disagrees with the " "publishing the copy re-ran resolution; the bag disagrees with what "
"record the parent resolved", "the parent's resolution decided",
) )
def test_no_bare_replace_of_a_record_outside_the_helper(self): def test_no_bare_replace_of_a_record_outside_the_helper(self):
@@ -10,6 +10,7 @@ from unittest.mock import MagicMock, patch
import sglang.srt.server_args as server_args_module import sglang.srt.server_args as server_args_module
from sglang.srt.arg_groups import pd_disaggregation_hook from sglang.srt.arg_groups import pd_disaggregation_hook
from sglang.srt.arg_groups.overrides import resolution_result
from sglang.srt.arg_groups.speculative_hook import handle_speculative_decoding from sglang.srt.arg_groups.speculative_hook import handle_speculative_decoding
from sglang.srt.entrypoints.sidecar import ( from sglang.srt.entrypoints.sidecar import (
SGLANG_GRPC_ENDPOINT_ENV, SGLANG_GRPC_ENDPOINT_ENV,
@@ -61,7 +62,7 @@ class TestPrepareServerArgs(CustomTestCase):
args.resolve_once() args.resolve_once()
self.assertTrue(args.enable_w4a4_mxfp4_megamoe) self.assertTrue(resolution_result(args, "enable_w4a4_mxfp4_megamoe"))
self.assertEqual(os.environ["DG_USE_FP4_ACTS"], "1") self.assertEqual(os.environ["DG_USE_FP4_ACTS"], "1")
self.assertEqual(os.environ["DG_USE_MXF4_KIND"], "1") self.assertEqual(os.environ["DG_USE_MXF4_KIND"], "1")
@@ -76,14 +77,14 @@ class TestPrepareServerArgs(CustomTestCase):
# nothing to be untouched by. # nothing to be untouched by.
args.resolve_once() args.resolve_once()
self.assertFalse(args.enable_w4a4_mxfp4_megamoe) self.assertFalse(resolution_result(args, "enable_w4a4_mxfp4_megamoe"))
self.assertEqual(os.environ["DG_USE_FP4_ACTS"], "0") self.assertEqual(os.environ["DG_USE_FP4_ACTS"], "0")
self.assertEqual(os.environ["DG_USE_MXF4_KIND"], "0") self.assertEqual(os.environ["DG_USE_MXF4_KIND"], "0")
def test_prefill_decode_interval(self): def test_prefill_decode_interval(self):
args = ServerArgs(model_path="dummy", prefill_decode_interval=16) args = ServerArgs(model_path="dummy", prefill_decode_interval=16)
args.resolve_once() args.resolve_once()
self.assertEqual(args.prefill_decode_interval, 16) self.assertEqual(resolution_result(args, "prefill_decode_interval"), 16)
with self.assertRaisesRegex( with self.assertRaisesRegex(
ValueError, "--prefill-decode-interval must be non-negative" ValueError, "--prefill-decode-interval must be non-negative"
@@ -114,22 +115,24 @@ class TestPrepareServerArgs(CustomTestCase):
return server_args return server_args
disabled = _resolved(model_path="dummy") disabled = _resolved(model_path="dummy")
self.assertFalse(disabled.enable_return_hidden_states) self.assertFalse(resolution_result(disabled, "enable_return_hidden_states"))
self.assertIsNone(disabled.return_hidden_states_mode) self.assertIsNone(resolution_result(disabled, "return_hidden_states_mode"))
last = _resolved( last = _resolved(
model_path="dummy", model_path="dummy",
return_hidden_states_mode="last", return_hidden_states_mode="last",
) )
self.assertTrue(last.enable_return_hidden_states) self.assertTrue(resolution_result(last, "enable_return_hidden_states"))
self.assertEqual(last.return_hidden_states_mode, "last") self.assertEqual(resolution_result(last, "return_hidden_states_mode"), "last")
legacy_full = _resolved( legacy_full = _resolved(
model_path="dummy", model_path="dummy",
enable_return_hidden_states=True, enable_return_hidden_states=True,
) )
self.assertTrue(legacy_full.enable_return_hidden_states) self.assertTrue(resolution_result(legacy_full, "enable_return_hidden_states"))
self.assertEqual(legacy_full.return_hidden_states_mode, "full") self.assertEqual(
resolution_result(legacy_full, "return_hidden_states_mode"), "full"
)
parsed_last = prepare_server_args( parsed_last = prepare_server_args(
[ [
@@ -140,8 +143,10 @@ class TestPrepareServerArgs(CustomTestCase):
] ]
) )
parsed_last.resolve_once() parsed_last.resolve_once()
self.assertTrue(parsed_last.enable_return_hidden_states) self.assertTrue(resolution_result(parsed_last, "enable_return_hidden_states"))
self.assertEqual(parsed_last.return_hidden_states_mode, "last") self.assertEqual(
resolution_result(parsed_last, "return_hidden_states_mode"), "last"
)
# The rejection is resolution's, not the constructor's. # The rejection is resolution's, not the constructor's.
with self.assertRaisesRegex( with self.assertRaisesRegex(
@@ -156,13 +161,24 @@ class TestPrepareServerArgs(CustomTestCase):
def test_draft_quantization_explicitness_survives_asdict_round_trip(self): def test_draft_quantization_explicitness_survives_asdict_round_trip(self):
inherited = ServerArgs(model_path="dummy", quantization="modelopt_fp4") inherited = ServerArgs(model_path="dummy", quantization="modelopt_fp4")
inherited._handle_missing_default_values() inherited._handle_missing_default_values()
self.assertEqual(inherited.speculative_draft_model_quantization, "modelopt_fp4") self.assertEqual(
self.assertFalse(inherited._speculative_draft_quantization_explicitly_set) resolution_result(inherited, "speculative_draft_model_quantization"),
"modelopt_fp4",
)
self.assertFalse(
resolution_result(
inherited, "_speculative_draft_quantization_explicitly_set"
)
)
reconstructed = ServerArgs(**dataclasses.asdict(inherited)) reconstructed = ServerArgs(**dataclasses.asdict(inherited))
reconstructed._handle_missing_default_values() reconstructed._handle_missing_default_values()
self.assertFalse(reconstructed._speculative_draft_quantization_explicitly_set) self.assertFalse(
resolution_result(
reconstructed, "_speculative_draft_quantization_explicitly_set"
)
)
def test_config_nested_dict_args_are_json(self): def test_config_nested_dict_args_are_json(self):
with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f:
@@ -219,8 +235,10 @@ class TestImageProcessorBackend(CustomTestCase):
server_args._handle_deprecated_args() server_args._handle_deprecated_args()
self.assertEqual(server_args.image_processor_backend, "pil") self.assertEqual(
self.assertFalse(server_args.disable_fast_image_processor) resolution_result(server_args, "image_processor_backend"), "pil"
)
self.assertFalse(resolution_result(server_args, "disable_fast_image_processor"))
def test_legacy_flag_maps_to_pil_with_one_warning(self): def test_legacy_flag_maps_to_pil_with_one_warning(self):
server_args = ServerArgs(model_path="dummy", disable_fast_image_processor=True) server_args = ServerArgs(model_path="dummy", disable_fast_image_processor=True)
@@ -228,8 +246,10 @@ class TestImageProcessorBackend(CustomTestCase):
with self.assertLogs(server_args_module.logger, level="WARNING") as logs: with self.assertLogs(server_args_module.logger, level="WARNING") as logs:
server_args._handle_deprecated_args() server_args._handle_deprecated_args()
self.assertEqual(server_args.image_processor_backend, "pil") self.assertEqual(
self.assertTrue(server_args.disable_fast_image_processor) resolution_result(server_args, "image_processor_backend"), "pil"
)
self.assertTrue(resolution_result(server_args, "disable_fast_image_processor"))
self.assertEqual( self.assertEqual(
sum( sum(
"--disable-fast-image-processor is deprecated" in x for x in logs.output "--disable-fast-image-processor is deprecated" in x for x in logs.output
@@ -266,7 +286,9 @@ class TestMultimodalFeatureTransport(CustomTestCase):
with self.assertLogs(server_args_module.logger, level="INFO") as logs: with self.assertLogs(server_args_module.logger, level="INFO") as logs:
server_args._handle_multimodal_feature_transport() server_args._handle_multimodal_feature_transport()
self.assertEqual(server_args.mm_feature_transport, "cuda_ipc") self.assertEqual(
resolution_result(server_args, "mm_feature_transport"), "cuda_ipc"
)
self.assertTrue(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get()) self.assertTrue(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get())
output = "\n".join(logs.output) output = "\n".join(logs.output)
@@ -281,8 +303,12 @@ class TestMultimodalFeatureTransport(CustomTestCase):
with self.assertLogs(server_args_module.logger, level="WARNING") as logs: with self.assertLogs(server_args_module.logger, level="WARNING") as logs:
server_args._handle_multimodal_feature_transport() server_args._handle_multimodal_feature_transport()
self.assertEqual(server_args.mm_feature_transport, "cuda_ipc") self.assertEqual(
self.assertFalse(server_args.keep_mm_feature_on_device) resolution_result(server_args, "mm_feature_transport"), "cuda_ipc"
)
self.assertFalse(
resolution_result(server_args, "keep_mm_feature_on_device")
)
self.assertTrue(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get()) self.assertTrue(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get())
self.assertIn("deprecated", logs.output[0]) self.assertIn("deprecated", logs.output[0])
@@ -305,7 +331,9 @@ class TestMultimodalFeatureTransport(CustomTestCase):
with self.assertLogs(server_args_module.logger, level="WARNING") as logs: with self.assertLogs(server_args_module.logger, level="WARNING") as logs:
server_args._handle_multimodal_feature_transport() server_args._handle_multimodal_feature_transport()
self.assertEqual(server_args.mm_feature_transport, "cpu") self.assertEqual(
resolution_result(server_args, "mm_feature_transport"), "cpu"
)
self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get()) self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get())
self.assertIn("overrides", logs.output[0]) self.assertIn("overrides", logs.output[0])
@@ -316,7 +344,9 @@ class TestMultimodalFeatureTransport(CustomTestCase):
with patch.dict(os.environ, {"SGLANG_USE_CUDA_IPC_TRANSPORT": "0"}): with patch.dict(os.environ, {"SGLANG_USE_CUDA_IPC_TRANSPORT": "0"}):
server_args._handle_multimodal_feature_transport() server_args._handle_multimodal_feature_transport()
self.assertEqual(server_args.mm_feature_transport, "cpu") self.assertEqual(
resolution_result(server_args, "mm_feature_transport"), "cpu"
)
self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get()) self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get())
@patch("sglang.srt.server_args.is_cuda", return_value=True) @patch("sglang.srt.server_args.is_cuda", return_value=True)
@@ -329,7 +359,9 @@ class TestMultimodalFeatureTransport(CustomTestCase):
with self.assertNoLogs(server_args_module.logger, level="INFO"): with self.assertNoLogs(server_args_module.logger, level="INFO"):
server_args._handle_multimodal_feature_transport() server_args._handle_multimodal_feature_transport()
self.assertEqual(server_args.mm_feature_transport, "cpu") self.assertEqual(
resolution_result(server_args, "mm_feature_transport"), "cpu"
)
self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get()) self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get())
@patch("sglang.srt.server_args.is_cuda", return_value=True) @patch("sglang.srt.server_args.is_cuda", return_value=True)
@@ -342,7 +374,9 @@ class TestMultimodalFeatureTransport(CustomTestCase):
with self.assertNoLogs(server_args_module.logger, level="INFO"): with self.assertNoLogs(server_args_module.logger, level="INFO"):
server_args._handle_multimodal_feature_transport() server_args._handle_multimodal_feature_transport()
self.assertEqual(server_args.mm_feature_transport, "cpu") self.assertEqual(
resolution_result(server_args, "mm_feature_transport"), "cpu"
)
self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get()) self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get())
@patch("sglang.srt.server_args.os.path.exists", return_value=True) @patch("sglang.srt.server_args.os.path.exists", return_value=True)
@@ -367,7 +401,9 @@ class TestMultimodalFeatureTransport(CustomTestCase):
with self.assertLogs(server_args_module.logger, level="INFO") as logs: with self.assertLogs(server_args_module.logger, level="INFO") as logs:
server_args._handle_multimodal_feature_transport() server_args._handle_multimodal_feature_transport()
self.assertEqual(server_args.mm_feature_transport, "cuda_vmm") self.assertEqual(
resolution_result(server_args, "mm_feature_transport"), "cuda_vmm"
)
self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get()) self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get())
output = "\n".join(logs.output) output = "\n".join(logs.output)
@@ -394,7 +430,7 @@ class TestMultimodalFeatureTransport(CustomTestCase):
with self.assertLogs(server_args_module.logger, level="INFO") as logs: with self.assertLogs(server_args_module.logger, level="INFO") as logs:
server_args._handle_multimodal_feature_transport() server_args._handle_multimodal_feature_transport()
self.assertEqual(server_args.mm_feature_transport, "cpu") self.assertEqual(resolution_result(server_args, "mm_feature_transport"), "cpu")
self.assertIn("has not opted into CUDA VMM", "\n".join(logs.output)) self.assertIn("has not opted into CUDA VMM", "\n".join(logs.output))
@patch("sglang.srt.server_args.os.path.exists", return_value=False) @patch("sglang.srt.server_args.os.path.exists", return_value=False)
@@ -411,7 +447,9 @@ class TestMultimodalFeatureTransport(CustomTestCase):
with self.assertLogs(server_args_module.logger, level="INFO") as logs: with self.assertLogs(server_args_module.logger, level="INFO") as logs:
server_args._handle_multimodal_feature_transport() server_args._handle_multimodal_feature_transport()
self.assertEqual(server_args.mm_feature_transport, "cpu") self.assertEqual(
resolution_result(server_args, "mm_feature_transport"), "cpu"
)
self.assertIn("no IMEX channel", "\n".join(logs.output)) self.assertIn("no IMEX channel", "\n".join(logs.output))
@@ -427,7 +465,9 @@ class TestMultimodalFeatureTransport(CustomTestCase):
envs.SGLANG_USE_CUDA_IPC_TRANSPORT.clear() envs.SGLANG_USE_CUDA_IPC_TRANSPORT.clear()
server_args._handle_multimodal_feature_transport() server_args._handle_multimodal_feature_transport()
self.assertEqual(server_args.mm_feature_transport, "cpu") self.assertEqual(
resolution_result(server_args, "mm_feature_transport"), "cpu"
)
self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get()) self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get())
@patch("sglang.srt.server_args.is_cuda", return_value=True) @patch("sglang.srt.server_args.is_cuda", return_value=True)
@@ -439,7 +479,9 @@ class TestMultimodalFeatureTransport(CustomTestCase):
envs.SGLANG_USE_CUDA_IPC_TRANSPORT.clear() envs.SGLANG_USE_CUDA_IPC_TRANSPORT.clear()
server_args._handle_multimodal_feature_transport() server_args._handle_multimodal_feature_transport()
self.assertEqual(server_args.mm_feature_transport, "cpu") self.assertEqual(
resolution_result(server_args, "mm_feature_transport"), "cpu"
)
self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get()) self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get())
@patch("sglang.srt.server_args.is_cuda", return_value=False) @patch("sglang.srt.server_args.is_cuda", return_value=False)
@@ -474,7 +516,9 @@ class TestMultimodalFeatureTransport(CustomTestCase):
with self.assertLogs(server_args_module.logger, level="INFO") as logs: with self.assertLogs(server_args_module.logger, level="INFO") as logs:
server_args._handle_multimodal_feature_transport() server_args._handle_multimodal_feature_transport()
self.assertEqual(server_args.mm_feature_transport, "cuda_vmm") self.assertEqual(
resolution_result(server_args, "mm_feature_transport"), "cuda_vmm"
)
self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get()) self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get())
output = "\n".join(logs.output) output = "\n".join(logs.output)
@@ -555,15 +599,22 @@ class TestLoadBalanceMethod(unittest.TestCase):
def test_non_pd_defaults_to_round_robin(self): def test_non_pd_defaults_to_round_robin(self):
server_args = self._load_balance_args(disaggregation_mode="null") server_args = self._load_balance_args(disaggregation_mode="null")
self.assertEqual(server_args.load_balance_method, "round_robin") self.assertEqual(
resolution_result(server_args, "load_balance_method"), "round_robin"
)
def test_pd_prefill_defaults_to_follow_bootstrap_room(self): def test_pd_prefill_defaults_to_follow_bootstrap_room(self):
server_args = self._load_balance_args(disaggregation_mode="prefill") server_args = self._load_balance_args(disaggregation_mode="prefill")
self.assertEqual(server_args.load_balance_method, "follow_bootstrap_room") self.assertEqual(
resolution_result(server_args, "load_balance_method"),
"follow_bootstrap_room",
)
def test_pd_decode_defaults_to_round_robin(self): def test_pd_decode_defaults_to_round_robin(self):
server_args = self._load_balance_args(disaggregation_mode="decode") server_args = self._load_balance_args(disaggregation_mode="decode")
self.assertEqual(server_args.load_balance_method, "round_robin") self.assertEqual(
resolution_result(server_args, "load_balance_method"), "round_robin"
)
def test_pd_prefill_dcp_warns_about_performance(self): def test_pd_prefill_dcp_warns_about_performance(self):
server_args = ServerArgs( server_args = ServerArgs(
@@ -581,7 +632,7 @@ class TestLoadBalanceMethod(unittest.TestCase):
disaggregation_transfer_backend="mooncake", disaggregation_transfer_backend="mooncake",
dcp_size=4, dcp_size=4,
) )
self.assertTrue(server_args.disable_radix_cache) self.assertTrue(resolution_result(server_args, "disable_radix_cache"))
def test_pd_decode_dcp_rejects_unsupported_transfer_backend(self): def test_pd_decode_dcp_rejects_unsupported_transfer_backend(self):
server_args = ServerArgs( server_args = ServerArgs(
@@ -601,7 +652,7 @@ class TestLoadBalanceMethod(unittest.TestCase):
disaggregation_transfer_backend="fake", disaggregation_transfer_backend="fake",
dcp_size=4, dcp_size=4,
) )
self.assertTrue(server_args.disable_radix_cache) self.assertTrue(resolution_result(server_args, "disable_radix_cache"))
def test_pd_decode_dcp_rejects_radix_cache(self): def test_pd_decode_dcp_rejects_radix_cache(self):
server_args = ServerArgs( server_args = ServerArgs(
@@ -665,8 +716,11 @@ class TestLoadBalanceMethod(unittest.TestCase):
disaggregation_transfer_backend="mooncake_tcp", disaggregation_transfer_backend="mooncake_tcp",
) )
self.assertFalse(server_args.disable_radix_cache) self.assertFalse(resolution_result(server_args, "disable_radix_cache"))
self.assertEqual(server_args.disaggregation_transfer_backend, "mooncake") self.assertEqual(
resolution_result(server_args, "disaggregation_transfer_backend"),
"mooncake",
)
class TestSkipTokenizerInit(unittest.TestCase): class TestSkipTokenizerInit(unittest.TestCase):
@@ -681,8 +735,8 @@ class TestSkipTokenizerInit(unittest.TestCase):
server_args._handle_tokenizer_batching() server_args._handle_tokenizer_batching()
# Tokenizer fanout preserved; detokenizer coerced to 1 (no decode work). # Tokenizer fanout preserved; detokenizer coerced to 1 (no decode work).
self.assertEqual(server_args.tokenizer_worker_num, 4) self.assertEqual(resolution_result(server_args, "tokenizer_worker_num"), 4)
self.assertEqual(server_args.detokenizer_worker_num, 1) self.assertEqual(resolution_result(server_args, "detokenizer_worker_num"), 1)
class TestHiSparseDsaBackendPolicy(unittest.TestCase): class TestHiSparseDsaBackendPolicy(unittest.TestCase):
@@ -870,7 +924,7 @@ class TestFa4PageSizeAutoForce(CustomTestCase):
from sglang.srt.arg_groups.overrides import resolved_view from sglang.srt.arg_groups.overrides import resolved_view
self.assertEqual(args.page_size, 1) # dual-apply retired: pristine self.assertEqual(args.page_size, 1) # the field stays pristine
self.assertEqual(resolved_view(args).page_size, 128) self.assertEqual(resolved_view(args).page_size, 128)
@patch("sglang.srt.arg_groups.overrides.is_sm100_supported", return_value=True) @patch("sglang.srt.arg_groups.overrides.is_sm100_supported", return_value=True)
@@ -883,7 +937,7 @@ class TestFa4PageSizeAutoForce(CustomTestCase):
from sglang.srt.arg_groups.overrides import resolved_view from sglang.srt.arg_groups.overrides import resolved_view
self.assertEqual(args.page_size, 1) # dual-apply retired: pristine self.assertEqual(args.page_size, 1) # the field stays pristine
self.assertEqual(resolved_view(args).page_size, 128) self.assertEqual(resolved_view(args).page_size, 128)
@@ -918,12 +972,12 @@ class TestContextParallelServerArgs(CustomTestCase):
def test_canonical_prefill_cp_requires_strategy(self): def test_canonical_prefill_cp_requires_strategy(self):
args = self.parser.parse_args(["--model", "dummy", "--enable-prefill-cp"]) args = self.parser.parse_args(["--model", "dummy", "--enable-prefill-cp"])
self.assertTrue(args.enable_prefill_cp) self.assertTrue(resolution_result(args, "enable_prefill_cp"))
self.assertIsNone(args.cp_strategy) self.assertIsNone(resolution_result(args, "cp_strategy"))
server_args = self._new_cp_args( server_args = self._new_cp_args(
enable_prefill_cp=args.enable_prefill_cp, enable_prefill_cp=resolution_result(args, "enable_prefill_cp"),
cp_strategy=args.cp_strategy, cp_strategy=resolution_result(args, "cp_strategy"),
) )
with self.assertRaisesRegex(ValueError, "--cp-strategy"): with self.assertRaisesRegex(ValueError, "--cp-strategy"):
server_args._handle_context_parallelism() server_args._handle_context_parallelism()
@@ -940,16 +994,18 @@ class TestContextParallelServerArgs(CustomTestCase):
) )
server_args = self._new_cp_args( server_args = self._new_cp_args(
enable_dsa_prefill_context_parallel=( enable_dsa_prefill_context_parallel=(
args.enable_dsa_prefill_context_parallel resolution_result(args, "enable_dsa_prefill_context_parallel")
), ),
dsa_prefill_cp_mode=args.dsa_prefill_cp_mode, dsa_prefill_cp_mode=resolution_result(args, "dsa_prefill_cp_mode"),
) )
server_args._handle_legacy_cp_arguments() server_args._handle_legacy_cp_arguments()
self.assertTrue(server_args.enable_prefill_cp) self.assertTrue(resolution_result(server_args, "enable_prefill_cp"))
self.assertEqual(server_args.cp_strategy, "interleave") self.assertEqual(resolution_result(server_args, "cp_strategy"), "interleave")
self.assertEqual(server_args.dsa_prefill_cp_mode, "round-robin-split") self.assertEqual(
resolution_result(server_args, "dsa_prefill_cp_mode"), "round-robin-split"
)
def test_canonical_interleave_cp_mirrors_to_dsa_runtime_aliases(self): def test_canonical_interleave_cp_mirrors_to_dsa_runtime_aliases(self):
server_args = self._new_cp_args( server_args = self._new_cp_args(
@@ -961,10 +1017,18 @@ class TestContextParallelServerArgs(CustomTestCase):
server_args._handle_legacy_cp_arguments() server_args._handle_legacy_cp_arguments()
server_args._handle_context_parallelism() server_args._handle_context_parallelism()
self.assertTrue(server_args.enable_dsa_prefill_context_parallel) self.assertTrue(
self.assertFalse(server_args.enable_prefill_context_parallel) resolution_result(server_args, "enable_dsa_prefill_context_parallel")
self.assertEqual(server_args.dsa_prefill_cp_mode, "round-robin-split") )
self.assertEqual(server_args.prefill_cp_mode, "round-robin-split") self.assertFalse(
resolution_result(server_args, "enable_prefill_context_parallel")
)
self.assertEqual(
resolution_result(server_args, "dsa_prefill_cp_mode"), "round-robin-split"
)
self.assertEqual(
resolution_result(server_args, "prefill_cp_mode"), "round-robin-split"
)
def test_context_parallel_handler_initializes_cp_strategy(self): def test_context_parallel_handler_initializes_cp_strategy(self):
server_args = self._new_cp_args( server_args = self._new_cp_args(
@@ -1049,15 +1113,25 @@ class TestContextParallelServerArgs(CustomTestCase):
server_args._handle_legacy_cp_arguments() server_args._handle_legacy_cp_arguments()
server_args._handle_context_parallelism() server_args._handle_context_parallelism()
self.assertTrue(server_args.enable_prefill_cp) self.assertTrue(resolution_result(server_args, "enable_prefill_cp"))
self.assertEqual(server_args.cp_strategy, strategy)
self.assertEqual(server_args.dsa_prefill_cp_mode, mode)
self.assertEqual(server_args.prefill_cp_mode, mode)
self.assertEqual( self.assertEqual(
server_args.enable_dsa_prefill_context_parallel, expect_dsa resolution_result(server_args, "cp_strategy"), strategy
) )
self.assertEqual( self.assertEqual(
server_args.enable_prefill_context_parallel, expect_generic resolution_result(server_args, "dsa_prefill_cp_mode"), mode
)
self.assertEqual(
resolution_result(server_args, "prefill_cp_mode"), mode
)
self.assertEqual(
resolution_result(
server_args, "enable_dsa_prefill_context_parallel"
),
expect_dsa,
)
self.assertEqual(
resolution_result(server_args, "enable_prefill_context_parallel"),
expect_generic,
) )
@@ -1308,7 +1382,7 @@ class TestSSLArgs(unittest.TestCase):
ssl_certfile="cert.pem", ssl_certfile="cert.pem",
enable_ssl_refresh=True, enable_ssl_refresh=True,
) )
self.assertTrue(server_args.enable_ssl_refresh) self.assertTrue(resolution_result(server_args, "enable_ssl_refresh"))
class TestHiCacheArgs(unittest.TestCase): class TestHiCacheArgs(unittest.TestCase):
@@ -1328,10 +1402,17 @@ class TestHiCacheArgs(unittest.TestCase):
expected_mem_layout: str, expected_mem_layout: str,
expected_decode_backend: str | None = None, expected_decode_backend: str | None = None,
): ):
self.assertEqual(args.hicache_io_backend, expected_io_backend) self.assertEqual(
self.assertEqual(args.hicache_mem_layout, expected_mem_layout) resolution_result(args, "hicache_io_backend"), expected_io_backend
)
self.assertEqual(
resolution_result(args, "hicache_mem_layout"), expected_mem_layout
)
if expected_decode_backend is not None: if expected_decode_backend is not None:
self.assertEqual(args.decode_attention_backend, expected_decode_backend) self.assertEqual(
resolution_result(args, "decode_attention_backend"),
expected_decode_backend,
)
def test_hicache_io_backend_and_mem_layout_compatibility(self): def test_hicache_io_backend_and_mem_layout_compatibility(self):
cases = [ cases = [
@@ -1409,9 +1490,9 @@ class TestHiCacheArgs(unittest.TestCase):
) )
args._handle_hicache() args._handle_hicache()
self.assertEqual(args.hicache_io_backend, "kernel") self.assertEqual(resolution_result(args, "hicache_io_backend"), "kernel")
self.assertEqual(args.hicache_mem_layout, "page_first") self.assertEqual(resolution_result(args, "hicache_mem_layout"), "page_first")
self.assertIsNone(args.decode_attention_backend) self.assertIsNone(resolution_result(args, "decode_attention_backend"))
def test_decode_offload_rejects_host_pool_retraction(self): def test_decode_offload_rejects_host_pool_retraction(self):
args = self._make_args( args = self._make_args(
@@ -1494,11 +1575,19 @@ class TestDecoupledSpecArgs(CustomTestCase):
"/tmp/tr", "/tmp/tr",
] ]
) )
self.assertEqual(server_args.decoupled_spec_role, "verifier") self.assertEqual(
self.assertEqual(server_args.decoupled_spec_bind_endpoint, "ipc:///tmp/v") resolution_result(server_args, "decoupled_spec_role"), "verifier"
self.assertEqual(server_args.decoupled_spec_connect_endpoints, ["ipc:///tmp/d"]) )
self.assertEqual(server_args.decoupled_spec_rank, 0) self.assertEqual(
self.assertEqual(server_args.spec_trace_dir, "/tmp/tr") resolution_result(server_args, "decoupled_spec_bind_endpoint"),
"ipc:///tmp/v",
)
self.assertEqual(
resolution_result(server_args, "decoupled_spec_connect_endpoints"),
["ipc:///tmp/d"],
)
self.assertEqual(resolution_result(server_args, "decoupled_spec_rank"), 0)
self.assertEqual(resolution_result(server_args, "spec_trace_dir"), "/tmp/tr")
def test_decoupled_spec_role_rejects_invalid_choice(self): def test_decoupled_spec_role_rejects_invalid_choice(self):
with self.assertRaises(SystemExit): with self.assertRaises(SystemExit):
@@ -1533,10 +1622,10 @@ class TestAdaptiveSpecArgs(CustomTestCase):
handle_speculative_decoding(args) handle_speculative_decoding(args)
self.assertTrue(args.speculative_adaptive) self.assertTrue(resolution_result(args, "speculative_adaptive"))
self.assertEqual(args.speculative_eagle_topk, 1) self.assertEqual(resolution_result(args, "speculative_eagle_topk"), 1)
self.assertEqual(args.speculative_num_steps, 3) self.assertEqual(resolution_result(args, "speculative_num_steps"), 3)
self.assertEqual(args.speculative_num_draft_tokens, 4) self.assertEqual(resolution_result(args, "speculative_num_draft_tokens"), 4)
class TestWaterfillArgs(CustomTestCase): class TestWaterfillArgs(CustomTestCase):
@@ -1552,10 +1641,9 @@ class TestWaterfillArgs(CustomTestCase):
from sglang.srt.arg_groups.overrides import resolved_view from sglang.srt.arg_groups.overrides import resolved_view
# dual-apply retired: the fields stay pristine, the declarations win
self.assertTrue(server_args.disable_shared_experts_fusion) self.assertTrue(server_args.disable_shared_experts_fusion)
self.assertFalse(resolved_view(server_args).disable_shared_experts_fusion) self.assertFalse(resolved_view(server_args).disable_shared_experts_fusion)
self.assertTrue(server_args.enforce_shared_experts_fusion) self.assertTrue(resolution_result(server_args, "enforce_shared_experts_fusion"))
def test_waterfill_overrides_moe_a2a_backend_to_deepep(self): def test_waterfill_overrides_moe_a2a_backend_to_deepep(self):
server_args = ServerArgs( server_args = ServerArgs(
@@ -1570,7 +1658,7 @@ class TestWaterfillArgs(CustomTestCase):
self.assertEqual(server_args.moe_a2a_backend, "none") # pristine self.assertEqual(server_args.moe_a2a_backend, "none") # pristine
self.assertEqual(resolved_view(server_args).moe_a2a_backend, "deepep") self.assertEqual(resolved_view(server_args).moe_a2a_backend, "deepep")
self.assertTrue(server_args.enforce_shared_experts_fusion) self.assertTrue(resolution_result(server_args, "enforce_shared_experts_fusion"))
def test_waterfill_keeps_megamoe_backend(self): def test_waterfill_keeps_megamoe_backend(self):
server_args = ServerArgs( server_args = ServerArgs(
@@ -1586,7 +1674,7 @@ class TestWaterfillArgs(CustomTestCase):
self.assertEqual(resolved_view(server_args).moe_a2a_backend, "megamoe") self.assertEqual(resolved_view(server_args).moe_a2a_backend, "megamoe")
self.assertFalse(resolved_view(server_args).disable_shared_experts_fusion) self.assertFalse(resolved_view(server_args).disable_shared_experts_fusion)
self.assertTrue(server_args.enforce_shared_experts_fusion) self.assertTrue(resolution_result(server_args, "enforce_shared_experts_fusion"))
def test_waterfill_supports_deepep_low_latency_mode(self): def test_waterfill_supports_deepep_low_latency_mode(self):
server_args = ServerArgs( server_args = ServerArgs(
@@ -1598,9 +1686,9 @@ class TestWaterfillArgs(CustomTestCase):
# dummy-model path short-circuits __post_init__; invoke the handler directly. # dummy-model path short-circuits __post_init__; invoke the handler directly.
server_args._handle_a2a_moe() server_args._handle_a2a_moe()
self.assertEqual(server_args.deepep_mode, "low_latency") self.assertEqual(resolution_result(server_args, "deepep_mode"), "low_latency")
self.assertFalse(server_args.disable_cuda_graph) self.assertFalse(resolution_result(server_args, "disable_cuda_graph"))
self.assertTrue(server_args.enforce_shared_experts_fusion) self.assertTrue(resolution_result(server_args, "enforce_shared_experts_fusion"))
class TestPrefillOnlyDisableKvCache(unittest.TestCase): class TestPrefillOnlyDisableKvCache(unittest.TestCase):
@@ -1635,7 +1723,7 @@ class TestPrefillOnlyDisableKvCache(unittest.TestCase):
def test_valid_minimal_config_constructs(self): def test_valid_minimal_config_constructs(self):
sa = self._validate_prefill_only_args() sa = self._validate_prefill_only_args()
self.assertTrue(sa.prefill_only_disable_kv_cache) self.assertTrue(resolution_result(sa, "prefill_only_disable_kv_cache"))
def test_rejects_when_not_embedding(self): def test_rejects_when_not_embedding(self):
with self.assertRaisesRegex(ValueError, "requires --is-embedding"): with self.assertRaisesRegex(ValueError, "requires --is-embedding"):
@@ -1725,15 +1813,27 @@ class TestCudaGraphDisaggregationRoles(CustomTestCase):
def test_cuda_graph_prefill_role_defaults_disable_decode_graph(self): def test_cuda_graph_prefill_role_defaults_disable_decode_graph(self):
args = self._handled_args(disaggregation_mode="prefill") args = self._handled_args(disaggregation_mode="prefill")
self.assertFalse(args.disable_cuda_graph) self.assertFalse(resolution_result(args, "disable_cuda_graph"))
self.assertEqual(args.cuda_graph_config.decode.backend, Backend.DISABLED) self.assertEqual(
self.assertEqual(args.cuda_graph_config.prefill.backend, Backend.BREAKABLE) resolution_result(args, "cuda_graph_config").decode.backend,
Backend.DISABLED,
)
self.assertEqual(
resolution_result(args, "cuda_graph_config").prefill.backend,
Backend.BREAKABLE,
)
def test_cuda_graph_decode_role_defaults_disable_prefill_graph(self): def test_cuda_graph_decode_role_defaults_disable_prefill_graph(self):
args = self._handled_args(disaggregation_mode="decode") args = self._handled_args(disaggregation_mode="decode")
self.assertEqual(args.cuda_graph_config.prefill.backend, Backend.DISABLED) self.assertEqual(
self.assertNotEqual(args.cuda_graph_config.decode.backend, Backend.DISABLED) resolution_result(args, "cuda_graph_config").prefill.backend,
Backend.DISABLED,
)
self.assertNotEqual(
resolution_result(args, "cuda_graph_config").decode.backend,
Backend.DISABLED,
)
def test_cuda_graph_global_disable_still_disables_both_phases_for_all_roles(self): def test_cuda_graph_global_disable_still_disables_both_phases_for_all_roles(self):
for disaggregation_mode in ("prefill", "decode", "null"): for disaggregation_mode in ("prefill", "decode", "null"):
@@ -1744,10 +1844,12 @@ class TestCudaGraphDisaggregationRoles(CustomTestCase):
) )
self.assertEqual( self.assertEqual(
args.cuda_graph_config.decode.backend, Backend.DISABLED resolution_result(args, "cuda_graph_config").decode.backend,
Backend.DISABLED,
) )
self.assertEqual( self.assertEqual(
args.cuda_graph_config.prefill.backend, Backend.DISABLED resolution_result(args, "cuda_graph_config").prefill.backend,
Backend.DISABLED,
) )
def test_cuda_graph_explicit_decode_backend_survives_prefill_role(self): def test_cuda_graph_explicit_decode_backend_survives_prefill_role(self):
@@ -1756,7 +1858,9 @@ class TestCudaGraphDisaggregationRoles(CustomTestCase):
cuda_graph_backend_decode=Backend.FULL, cuda_graph_backend_decode=Backend.FULL,
) )
self.assertEqual(args.cuda_graph_config.decode.backend, Backend.FULL) self.assertEqual(
resolution_result(args, "cuda_graph_config").decode.backend, Backend.FULL
)
self.assertIn((Phase.DECODE, "backend"), args._cuda_graph_config_locked) self.assertIn((Phase.DECODE, "backend"), args._cuda_graph_config_locked)
@@ -1782,12 +1886,18 @@ class TestPrefillCudaGraphLoRACompatibility(CustomTestCase):
def test_enable_lora_keeps_breakable_prefill_graph(self): def test_enable_lora_keeps_breakable_prefill_graph(self):
args = self._handled_args(enable_lora=True) args = self._handled_args(enable_lora=True)
self.assertEqual(args.cuda_graph_config.prefill.backend, Backend.BREAKABLE) self.assertEqual(
resolution_result(args, "cuda_graph_config").prefill.backend,
Backend.BREAKABLE,
)
def test_lora_paths_keep_breakable_prefill_graph(self): def test_lora_paths_keep_breakable_prefill_graph(self):
args = self._handled_args(lora_paths=["dummy/lora-adapter"]) args = self._handled_args(lora_paths=["dummy/lora-adapter"])
self.assertEqual(args.cuda_graph_config.prefill.backend, Backend.BREAKABLE) self.assertEqual(
resolution_result(args, "cuda_graph_config").prefill.backend,
Backend.BREAKABLE,
)
def test_lora_still_disables_tc_piecewise_prefill_graph(self): def test_lora_still_disables_tc_piecewise_prefill_graph(self):
# Pin the tc_piecewise LoRA rule itself, with the hardware rule # Pin the tc_piecewise LoRA rule itself, with the hardware rule
@@ -1811,7 +1921,10 @@ class TestPrefillCudaGraphLoRACompatibility(CustomTestCase):
): ):
args._disable_tc_piecewise_cudagraph_if_incompatible() args._disable_tc_piecewise_cudagraph_if_incompatible()
self.assertEqual(args.cuda_graph_config.prefill.backend, Backend.DISABLED) self.assertEqual(
resolution_result(args, "cuda_graph_config").prefill.backend,
Backend.DISABLED,
)
class TestBreakableCudaGraphMultimodalAllowlist(CustomTestCase): class TestBreakableCudaGraphMultimodalAllowlist(CustomTestCase):
@@ -1840,7 +1953,10 @@ class TestBreakableCudaGraphMultimodalAllowlist(CustomTestCase):
is_multimodal=True, is_multimodal=True,
allowlisted=False, allowlisted=False,
) )
self.assertEqual(args.cuda_graph_config.prefill.backend, Backend.DISABLED) self.assertEqual(
resolution_result(args, "cuda_graph_config").prefill.backend,
Backend.DISABLED,
)
def test_allowlisted_multimodal_arch_keeps_prefill_breakable(self): def test_allowlisted_multimodal_arch_keeps_prefill_breakable(self):
args = self._handled_args( args = self._handled_args(
@@ -1848,7 +1964,10 @@ class TestBreakableCudaGraphMultimodalAllowlist(CustomTestCase):
is_multimodal=True, is_multimodal=True,
allowlisted=True, allowlisted=True,
) )
self.assertEqual(args.cuda_graph_config.prefill.backend, Backend.BREAKABLE) self.assertEqual(
resolution_result(args, "cuda_graph_config").prefill.backend,
Backend.BREAKABLE,
)
def test_allowlist_membership(self): def test_allowlist_membership(self):
from sglang.srt.configs.model_config import ( from sglang.srt.configs.model_config import (
@@ -2046,20 +2165,20 @@ class TestGrpcServerArgs(CustomTestCase):
def test_http_only_high_port_does_not_derive_grpc_port(self): def test_http_only_high_port_does_not_derive_grpc_port(self):
sa = self._args(port=56000) sa = self._args(port=56000)
sa._handle_deprecated_args() sa._handle_deprecated_args()
self.assertIsNone(sa.grpc_port) self.assertIsNone(resolution_result(sa, "grpc_port"))
def test_grpc_port_enables_native_and_env_knobs(self): def test_grpc_port_enables_native_and_env_knobs(self):
sa = self._args(grpc_port=50051) sa = self._args(grpc_port=50051)
with envs.SGLANG_GRPC_WORKER_THREADS.override(8): with envs.SGLANG_GRPC_WORKER_THREADS.override(8):
sa._handle_deprecated_args() sa._handle_deprecated_args()
self.assertEqual(sa.grpc_port, 50051) self.assertEqual(resolution_result(sa, "grpc_port"), 50051)
self.assertEqual(sa.grpc_worker_threads, 8) self.assertEqual(sa.grpc_worker_threads, 8)
def test_env_grpc_port_enables_native(self): def test_env_grpc_port_enables_native(self):
sa = self._args(port=30000) sa = self._args(port=30000)
with envs.SGLANG_GRPC_PORT.override(45000): with envs.SGLANG_GRPC_PORT.override(45000):
sa._handle_deprecated_args() sa._handle_deprecated_args()
self.assertEqual(sa.grpc_port, 45000) self.assertEqual(resolution_result(sa, "grpc_port"), 45000)
@staticmethod @staticmethod
def _sidecar_parser(): def _sidecar_parser():
@@ -2196,20 +2315,20 @@ class TestGrpcServerArgs(CustomTestCase):
def test_legacy_smg_derives_grpc_port_from_http_port(self): def test_legacy_smg_derives_grpc_port_from_http_port(self):
sa = self._args(port=30000, smg_grpc_mode=True) sa = self._args(port=30000, smg_grpc_mode=True)
sa._handle_deprecated_args() sa._handle_deprecated_args()
self.assertEqual(sa.grpc_port, 40000) self.assertEqual(resolution_result(sa, "grpc_port"), 40000)
def test_grpc_mode_is_deprecated_alias_for_smg_grpc_mode(self): def test_grpc_mode_is_deprecated_alias_for_smg_grpc_mode(self):
sa = self._args(grpc_mode=True) sa = self._args(grpc_mode=True)
with self.assertLogs(server_args_module.logger, level="WARNING") as cm: with self.assertLogs(server_args_module.logger, level="WARNING") as cm:
sa._handle_deprecated_args() sa._handle_deprecated_args()
self.assertTrue(sa.smg_grpc_mode) self.assertTrue(resolution_result(sa, "smg_grpc_mode"))
self.assertTrue(any("--grpc-mode is deprecated" in line for line in cm.output)) self.assertTrue(any("--grpc-mode is deprecated" in line for line in cm.output))
def test_legacy_smg_takes_precedence_over_grpc_port(self): def test_legacy_smg_takes_precedence_over_grpc_port(self):
sa = self._args(grpc_port=50051, smg_grpc_mode=True) sa = self._args(grpc_port=50051, smg_grpc_mode=True)
sa._handle_deprecated_args() sa._handle_deprecated_args()
self.assertTrue(sa.smg_grpc_mode) self.assertTrue(resolution_result(sa, "smg_grpc_mode"))
self.assertEqual(sa.grpc_port, 50051) self.assertEqual(resolution_result(sa, "grpc_port"), 50051)
def test_native_grpc_rejects_multi_tokenizer(self): def test_native_grpc_rejects_multi_tokenizer(self):
sa = self._args(grpc_port=40000, tokenizer_worker_num=2) sa = self._args(grpc_port=40000, tokenizer_worker_num=2)
@@ -2254,7 +2373,7 @@ class TestGrpcServerArgs(CustomTestCase):
tokenizer_manager=MagicMock(), tokenizer_manager=MagicMock(),
template_manager=MagicMock(), template_manager=MagicMock(),
scheduler_info={}, scheduler_info={},
grpc_port=server_args.grpc_port, grpc_port=resolution_result(server_args, "grpc_port"),
) )
self.assertEqual(handle, "handle") self.assertEqual(handle, "handle")
@@ -1,6 +1,7 @@
import unittest import unittest
from types import SimpleNamespace from types import SimpleNamespace
from sglang.srt.arg_groups.overrides import resolution_result
from sglang.srt.arg_groups.speculative_hook import handle_speculative_decoding from sglang.srt.arg_groups.speculative_hook import handle_speculative_decoding
from sglang.srt.server_args import ServerArgs from sglang.srt.server_args import ServerArgs
from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.ci.ci_register import register_cpu_ci
@@ -33,18 +34,18 @@ def _make_spec_args(device: str, algorithm: str = "EAGLE", **overrides) -> Serve
class TestSpecCPUOverlapConstraint(CustomTestCase): class TestSpecCPUOverlapConstraint(CustomTestCase):
def test_cpu_eagle_forces_disable_overlap_schedule(self): def test_cpu_eagle_forces_disable_overlap_schedule(self):
args = _make_spec_args(device="cpu") args = _make_spec_args(device="cpu")
self.assertFalse(args.disable_overlap_schedule) self.assertFalse(resolution_result(args, "disable_overlap_schedule"))
handle_speculative_decoding(args) handle_speculative_decoding(args)
self.assertTrue(args.disable_overlap_schedule) self.assertTrue(resolution_result(args, "disable_overlap_schedule"))
def test_cpu_eagle3_forces_disable_overlap_schedule(self): def test_cpu_eagle3_forces_disable_overlap_schedule(self):
args = _make_spec_args(device="cpu", algorithm="EAGLE3") args = _make_spec_args(device="cpu", algorithm="EAGLE3")
handle_speculative_decoding(args) handle_speculative_decoding(args)
self.assertTrue(args.disable_overlap_schedule) self.assertTrue(resolution_result(args, "disable_overlap_schedule"))
def test_cpu_explicit_disable_overlap_is_preserved(self): def test_cpu_explicit_disable_overlap_is_preserved(self):
args = _make_spec_args(device="cpu", disable_overlap_schedule=True) args = _make_spec_args(device="cpu", disable_overlap_schedule=True)
@@ -56,7 +57,7 @@ class TestSpecCPUOverlapConstraint(CustomTestCase):
) as logs: ) as logs:
handle_speculative_decoding(args) handle_speculative_decoding(args)
self.assertTrue(args.disable_overlap_schedule) self.assertTrue(resolution_result(args, "disable_overlap_schedule"))
self.assertFalse( self.assertFalse(
any("Overlap schedule" in message for message in logs.output), any("Overlap schedule" in message for message in logs.output),
f"hook warned about overriding an already-disabled overlap: {logs.output}", f"hook warned about overriding an already-disabled overlap: {logs.output}",
@@ -68,7 +69,7 @@ class TestSpecCPUOverlapConstraint(CustomTestCase):
handle_speculative_decoding(args) handle_speculative_decoding(args)
self.assertFalse(args.disable_overlap_schedule) self.assertFalse(resolution_result(args, "disable_overlap_schedule"))
if __name__ == "__main__": if __name__ == "__main__":
@@ -4,6 +4,7 @@ import unittest
from types import SimpleNamespace from types import SimpleNamespace
from unittest.mock import MagicMock from unittest.mock import MagicMock
from sglang.srt.arg_groups.overrides import resolution_result
from sglang.srt.arg_groups.speculative_hook import handle_speculative_decoding from sglang.srt.arg_groups.speculative_hook import handle_speculative_decoding
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.srt.speculative.spec_registry import ( from sglang.srt.speculative.spec_registry import (
@@ -237,7 +238,9 @@ class TestServerArgsHook(_RegistryIsolated):
handle_speculative_decoding(server_args) handle_speculative_decoding(server_args)
self.assertEqual(server_args.speculative_algorithm, "MY_HANDLE_ARGS") self.assertEqual(
resolution_result(server_args, "speculative_algorithm"), "MY_HANDLE_ARGS"
)
self.assertEqual(server_args.custom_spec_handle_seen, "MY_HANDLE_ARGS") self.assertEqual(server_args.custom_spec_handle_seen, "MY_HANDLE_ARGS")
self.assertEqual(server_args.speculative_num_draft_tokens, 7) self.assertEqual(server_args.speculative_num_draft_tokens, 7)
+8 -2
View File
@@ -280,10 +280,16 @@ class TestPublishInstallsSlot(_IsolatedPublish):
set_global_server_args_for_scheduler(sa) set_global_server_args_for_scheduler(sa)
self.assertIs(get_server_args(), sa) self.assertIs(get_server_args(), sa)
# Publishing is what resolved it; the handlers ahead of the dummy # Publishing is what resolved it; the handlers ahead of the dummy
# short-circuit still declare. # short-circuit still declare. What they decided is the projection --
# the fields keep what the caller passed.
from sglang.srt.arg_groups.overrides import resolution_result
self.assertTrue(sa._resolved_overrides, "publishing declared nothing")
for source, declared in sa._resolved_overrides: for source, declared in sa._resolved_overrides:
for field, value in declared.items(): for field, value in declared.items():
self.assertEqual(getattr(sa, field), value, f"{source}: {field}") self.assertEqual(
resolution_result(sa, field), value, f"{source}: {field}"
)
class TestGoldenModelOverrides(_IsolatedPublish): class TestGoldenModelOverrides(_IsolatedPublish):
@@ -31,11 +31,15 @@ class TestContextOverride(CustomTestCase):
def test_override_writes_bag_not_server_args(self): def test_override_writes_bag_not_server_args(self):
sa = self._publish() sa = self._publish()
before = sa.hicache_ratio # The published leaf, not the field: `hicache_ratio` is resolved by
# declaration, so the field still holds what the caller passed.
before = rc.get_memory().hicache_ratio
pristine = sa.hicache_ratio
rc.get_context().override("test", hicache_ratio=before + 1.0) rc.get_context().override("test", hicache_ratio=before + 1.0)
self.assertEqual(rc.get_memory().hicache_ratio, before + 1.0) self.assertEqual(rc.get_memory().hicache_ratio, before + 1.0)
# server_args stays the pristine startup record. # server_args stays the pristine startup record: the override does not
self.assertEqual(sa.hicache_ratio, before) # touch it, and neither did resolution.
self.assertEqual(sa.hicache_ratio, pristine)
def test_override_routes_across_namespaces(self): def test_override_routes_across_namespaces(self):
self._publish() self._publish()
@@ -7,6 +7,7 @@ translates field annotations into argparse arguments.
import argparse import argparse
import unittest import unittest
from sglang.srt.arg_groups.overrides import resolution_result
from sglang.srt.server_args import ServerArgs from sglang.srt.server_args import ServerArgs
from sglang.srt.utils.common import configure_media_url_security from sglang.srt.utils.common import configure_media_url_security
from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.ci.ci_register import register_cpu_ci
@@ -95,8 +96,10 @@ class TestServerArgsAnnotatedCli(CustomTestCase):
"32", "32",
] ]
) )
# The normalization is a declaration.
self.assertEqual( self.assertEqual(
sa.allowed_media_domains, ["127.0.0.1", "media.example.com"] resolution_result(sa, "allowed_media_domains"),
["127.0.0.1", "media.example.com"],
) )
self.assertEqual(sa.media_url_max_file_size_mb, 32) self.assertEqual(sa.media_url_max_file_size_mb, 32)
finally: finally:
@@ -134,10 +134,10 @@ _ENV_MATRIX = (({}, {"SGLANG_IS_IN_CI": "true"}),)
# are step-12 exposure like any other pair. # are step-12 exposure like any other pair.
_PASSED = frozenset({"model_path", "device", "random_seed"}) _PASSED = frozenset({"model_path", "device", "random_seed"})
# The reads that still take a value off the supplied instance. `initialize_moe_config`
# is handed the record until the replay goes away; the rest are pre-publish launcher
# reads.
_EXPOSED = { _EXPOSED = {
("dllm/config.py", "max_running_requests"),
("dllm/config.py", "model_path"),
("speculative/spec_registry.py", "disable_overlap_schedule"),
("disaggregation/encoder/server.py", "model_loader_extra_config"), ("disaggregation/encoder/server.py", "model_loader_extra_config"),
("layers/moe/utils.py", "deepep_mode"), ("layers/moe/utils.py", "deepep_mode"),
("layers/moe/utils.py", "disable_shared_experts_fusion"), ("layers/moe/utils.py", "disable_shared_experts_fusion"),
@@ -150,40 +150,16 @@ _EXPOSED = {
("configs/embedding_model_spec.py", "disable_radix_cache"), ("configs/embedding_model_spec.py", "disable_radix_cache"),
("configs/embedding_model_spec.py", "is_embedding"), ("configs/embedding_model_spec.py", "is_embedding"),
("configs/embedding_model_spec.py", "prefill_only_disable_kv_cache"), ("configs/embedding_model_spec.py", "prefill_only_disable_kv_cache"),
("configs/model_config.py", "_speculative_draft_quantization_explicitly_set"),
("configs/model_config.py", "disable_hybrid_swa_memory"),
("configs/model_config.py", "dtype"),
("configs/model_config.py", "enable_multi_layer_eagle"),
("configs/model_config.py", "is_embedding"),
("configs/model_config.py", "model_path"),
("configs/model_config.py", "quantization"),
("configs/model_config.py", "speculative_algorithm"),
("configs/model_config.py", "speculative_draft_model_quantization"),
("dllm/config.py", "max_running_requests"),
("dllm/config.py", "model_path"),
("entrypoints/engine.py", "enable_symm_mem"), ("entrypoints/engine.py", "enable_symm_mem"),
("entrypoints/engine.py", "reasoning_parser"), ("entrypoints/engine.py", "reasoning_parser"),
("entrypoints/engine.py", "tool_call_parser"), ("entrypoints/engine.py", "tool_call_parser"),
("layers/cp/base.py", "attn_cp_size"),
("layers/cp/base.py", "cp_strategy"),
("layers/cp/base.py", "enable_prefill_cp"),
("layers/cp/bcg.py", "cp_strategy"),
("layers/cp/bcg.py", "enable_prefill_cp"),
("layers/flashinfer_comm_fusion.py", "flashinfer_allreduce_fusion_backend"), ("layers/flashinfer_comm_fusion.py", "flashinfer_allreduce_fusion_backend"),
("layers/moe/utils.py", "deepep_mode"), ("layers/moe/utils.py", "deepep_mode"),
("layers/moe/utils.py", "moe_a2a_backend"), ("layers/moe/utils.py", "moe_a2a_backend"),
("layers/moe/utils.py", "moe_runner_backend"), ("layers/moe/utils.py", "moe_runner_backend"),
("layers/moe/utils.py", "quantization"), ("layers/moe/utils.py", "quantization"),
("layers/moe/utils.py", "speculative_moe_runner_backend"), ("layers/moe/utils.py", "speculative_moe_runner_backend"),
("lora/marlin_lora_temp/policy.py", "lora_paths"),
("model_loader/expert_pack_runtime.py", "model_path"),
("model_loader/expert_pack_runtime.py", "tokenizer_path"),
("parser/template_detection.py", "model_path"),
("speculative/adaptive_spec_params.py", "speculative_algorithm"),
("speculative/adaptive_spec_params.py", "speculative_eagle_topk"),
("speculative/draft_worker_common.py", "speculative_draft_attention_backend"), ("speculative/draft_worker_common.py", "speculative_draft_attention_backend"),
("speculative/spec_info.py", "enable_multi_layer_eagle"),
("speculative/spec_registry.py", "disable_overlap_schedule"),
("utils/common.py", "speculative_num_draft_tokens"), ("utils/common.py", "speculative_num_draft_tokens"),
("utils/common.py", "speculative_num_steps"), ("utils/common.py", "speculative_num_steps"),
("utils/hf_transformers/processor.py", "image_processor_backend"), ("utils/hf_transformers/processor.py", "image_processor_backend"),
@@ -216,26 +192,19 @@ _EXPOSED_CUDA_ONLY: frozenset = frozenset()
# some code overrides post-publish. Each needs an ordering judgment, not a blanket # some code overrides post-publish. Each needs an ordering judgment, not a blanket
# conversion; the list exists so a new one is a decision made when it is written. # conversion; the list exists so a new one is a decision made when it is written.
_OVERRIDDEN_AND_READ = { _OVERRIDDEN_AND_READ = {
("configs/model_config.py", "dtype"),
("configs/model_config.py", "model_path"),
("dllm/config.py", "model_path"),
("entrypoints/engine.py", "reasoning_parser"), ("entrypoints/engine.py", "reasoning_parser"),
("entrypoints/engine.py", "tool_call_parser"), ("entrypoints/engine.py", "tool_call_parser"),
("model_loader/expert_pack_runtime.py", "model_path"),
("weight_cache/daemon.py", "dp_size"), ("weight_cache/daemon.py", "dp_size"),
("weight_cache/daemon.py", "dtype"), ("weight_cache/daemon.py", "dtype"),
("weight_cache/daemon.py", "ep_size"), ("weight_cache/daemon.py", "ep_size"),
("weight_cache/daemon.py", "load_format"), ("weight_cache/daemon.py", "load_format"),
("weight_cache/daemon.py", "model_path"), ("weight_cache/daemon.py", "model_path"),
("configs/model_config.py", "dtype"),
("configs/model_config.py", "model_path"),
("mem_cache/pool_host/common.py", "hicache_storage_backend"), ("mem_cache/pool_host/common.py", "hicache_storage_backend"),
("mem_cache/pool_host/common.py", "hicache_storage_backend_extra_config"), ("mem_cache/pool_host/common.py", "hicache_storage_backend_extra_config"),
("mem_cache/unified_radix_cache.py", "hicache_storage_backend"), ("mem_cache/unified_radix_cache.py", "hicache_storage_backend"),
("mem_cache/unified_radix_cache.py", "hicache_storage_backend_extra_config"), ("mem_cache/unified_radix_cache.py", "hicache_storage_backend_extra_config"),
("mem_cache/unified_radix_cache.py", "hicache_storage_prefetch_policy"), ("mem_cache/unified_radix_cache.py", "hicache_storage_prefetch_policy"),
("mem_cache/unified_radix_cache.py", "hicache_write_policy"), ("mem_cache/unified_radix_cache.py", "hicache_write_policy"),
("parser/template_detection.py", "model_path"),
("utils/common.py", "speculative_num_draft_tokens"), ("utils/common.py", "speculative_num_draft_tokens"),
("utils/common.py", "speculative_num_steps"), ("utils/common.py", "speculative_num_steps"),
("weight_cache/daemon.py", "dp_size"), ("weight_cache/daemon.py", "dp_size"),